feat: Vendor pezkuwi-subxt and pezkuwi-zombienet-sdk into monorepo
- Add pezkuwi-subxt crates to vendor/pezkuwi-subxt - Add pezkuwi-zombienet-sdk crates to vendor/pezkuwi-zombienet-sdk - Convert git dependencies to path dependencies - Add vendor crates to workspace members - Remove test/example crates from vendor (not needed for SDK) - Fix feature propagation issues detected by zepter - Fix workspace inheritance for internal dependencies - All 606 crates now in workspace - All 6919 internal dependency links verified correct - No git dependencies remaining
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
/target
|
||||
/Cargo.lock
|
||||
@@ -0,0 +1,47 @@
|
||||
[package]
|
||||
name = "zombienet-provider"
|
||||
version.workspace = true
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
publish = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
description = "Zombienet provider, implement the logic to run the nodes in the native provider"
|
||||
keywords = ["zombienet", "provider", "native"]
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
async-trait = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive", "rc"] }
|
||||
serde_json = { workspace = true }
|
||||
serde_yaml = { workspace = true }
|
||||
tokio = { workspace = true, features = [
|
||||
"process",
|
||||
"macros",
|
||||
"fs",
|
||||
"time",
|
||||
"rt",
|
||||
] }
|
||||
tokio-util = { workspace = true, features = ["compat"] }
|
||||
thiserror = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
uuid = { workspace = true, features = ["v4"] }
|
||||
nix = { workspace = true, features = ["signal"] }
|
||||
kube = { workspace = true, features = ["ws", "runtime"] }
|
||||
k8s-openapi = { workspace = true, features = ["v1_27"] }
|
||||
tar = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
hex = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
regex = { workspace = true }
|
||||
url = { workspace = true }
|
||||
flate2 = { workspace = true }
|
||||
erased-serde = { workspace = true }
|
||||
|
||||
# Zomebienet deps
|
||||
support = { workspace = true }
|
||||
configuration = { workspace = true }
|
||||
@@ -0,0 +1,6 @@
|
||||
mod client;
|
||||
mod namespace;
|
||||
mod node;
|
||||
mod provider;
|
||||
|
||||
pub use provider::DockerProvider;
|
||||
@@ -0,0 +1,596 @@
|
||||
use std::{collections::HashMap, path::Path, process::Stdio};
|
||||
|
||||
use anyhow::anyhow;
|
||||
use futures::future::try_join_all;
|
||||
use serde::{Deserialize, Deserializer};
|
||||
use tokio::process::Command;
|
||||
use tracing::{info, trace};
|
||||
|
||||
use crate::types::{ExecutionResult, Port};
|
||||
|
||||
#[derive(thiserror::Error, Debug)]
|
||||
#[error(transparent)]
|
||||
pub struct Error(#[from] anyhow::Error);
|
||||
|
||||
pub type Result<T> = core::result::Result<T, Error>;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct DockerClient {
|
||||
using_podman: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ContainerRunOptions {
|
||||
image: String,
|
||||
command: Vec<String>,
|
||||
env: Option<Vec<(String, String)>>,
|
||||
volume_mounts: Option<HashMap<String, String>>,
|
||||
name: Option<String>,
|
||||
entrypoint: Option<String>,
|
||||
port_mapping: HashMap<Port, Port>,
|
||||
rm: bool,
|
||||
detach: bool,
|
||||
}
|
||||
|
||||
enum Container {
|
||||
Docker(DockerContainer),
|
||||
Podman(PodmanContainer),
|
||||
}
|
||||
|
||||
// TODO: we may don't need this
|
||||
#[allow(dead_code)]
|
||||
#[derive(Deserialize, Debug)]
|
||||
struct DockerContainer {
|
||||
#[serde(alias = "Names", deserialize_with = "deserialize_list")]
|
||||
names: Vec<String>,
|
||||
#[serde(alias = "Ports", deserialize_with = "deserialize_list")]
|
||||
ports: Vec<String>,
|
||||
#[serde(alias = "State")]
|
||||
state: String,
|
||||
}
|
||||
|
||||
// TODO: we may don't need this
|
||||
#[allow(dead_code)]
|
||||
#[derive(Deserialize, Debug)]
|
||||
struct PodmanPort {
|
||||
host_ip: String,
|
||||
container_port: u16,
|
||||
host_port: u16,
|
||||
range: u16,
|
||||
protocol: String,
|
||||
}
|
||||
|
||||
// TODO: we may don't need this
|
||||
#[allow(dead_code)]
|
||||
#[derive(Deserialize, Debug)]
|
||||
struct PodmanContainer {
|
||||
#[serde(alias = "Id")]
|
||||
id: String,
|
||||
#[serde(alias = "Image")]
|
||||
image: String,
|
||||
#[serde(alias = "Mounts")]
|
||||
mounts: Vec<String>,
|
||||
#[serde(alias = "Names")]
|
||||
names: Vec<String>,
|
||||
#[serde(alias = "Ports", deserialize_with = "deserialize_null_as_default")]
|
||||
ports: Vec<PodmanPort>,
|
||||
#[serde(alias = "State")]
|
||||
state: String,
|
||||
}
|
||||
|
||||
fn deserialize_list<'de, D>(deserializer: D) -> std::result::Result<Vec<String>, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let str_sequence = String::deserialize(deserializer)?;
|
||||
Ok(str_sequence
|
||||
.split(',')
|
||||
.filter(|item| !item.is_empty())
|
||||
.map(|item| item.to_owned())
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn deserialize_null_as_default<'de, D, T>(deserializer: D) -> std::result::Result<T, D::Error>
|
||||
where
|
||||
T: Default + Deserialize<'de>,
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let opt = Option::deserialize(deserializer)?;
|
||||
Ok(opt.unwrap_or_default())
|
||||
}
|
||||
|
||||
impl ContainerRunOptions {
|
||||
pub fn new<S>(image: &str, command: Vec<S>) -> Self
|
||||
where
|
||||
S: Into<String> + std::fmt::Debug + Send + Clone,
|
||||
{
|
||||
ContainerRunOptions {
|
||||
image: image.to_string(),
|
||||
command: command
|
||||
.clone()
|
||||
.into_iter()
|
||||
.map(|s| s.into())
|
||||
.collect::<Vec<_>>(),
|
||||
env: None,
|
||||
volume_mounts: None,
|
||||
name: None,
|
||||
entrypoint: None,
|
||||
port_mapping: HashMap::default(),
|
||||
rm: false,
|
||||
detach: true, // add -d flag by default
|
||||
}
|
||||
}
|
||||
|
||||
pub fn env<S>(mut self, env: Vec<(S, S)>) -> Self
|
||||
where
|
||||
S: Into<String> + std::fmt::Debug + Send + Clone,
|
||||
{
|
||||
self.env = Some(
|
||||
env.into_iter()
|
||||
.map(|(name, value)| (name.into(), value.into()))
|
||||
.collect(),
|
||||
);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn volume_mounts<S>(mut self, volume_mounts: HashMap<S, S>) -> Self
|
||||
where
|
||||
S: Into<String> + std::fmt::Debug + Send + Clone,
|
||||
{
|
||||
self.volume_mounts = Some(
|
||||
volume_mounts
|
||||
.into_iter()
|
||||
.map(|(source, target)| (source.into(), target.into()))
|
||||
.collect(),
|
||||
);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn name<S>(mut self, name: S) -> Self
|
||||
where
|
||||
S: Into<String> + std::fmt::Debug + Send + Clone,
|
||||
{
|
||||
self.name = Some(name.into());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn entrypoint<S>(mut self, entrypoint: S) -> Self
|
||||
where
|
||||
S: Into<String> + std::fmt::Debug + Send + Clone,
|
||||
{
|
||||
self.entrypoint = Some(entrypoint.into());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn port_mapping(mut self, port_mapping: &HashMap<Port, Port>) -> Self {
|
||||
self.port_mapping.clone_from(port_mapping);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn rm(mut self) -> Self {
|
||||
self.rm = true;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn detach(mut self, choice: bool) -> Self {
|
||||
self.detach = choice;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl DockerClient {
|
||||
pub async fn new() -> Result<Self> {
|
||||
let using_podman = Self::is_using_podman().await?;
|
||||
|
||||
Ok(DockerClient { using_podman })
|
||||
}
|
||||
|
||||
pub fn client_binary(&self) -> String {
|
||||
String::from(if self.using_podman {
|
||||
"podman"
|
||||
} else {
|
||||
"docker"
|
||||
})
|
||||
}
|
||||
|
||||
async fn is_using_podman() -> Result<bool> {
|
||||
if let Ok(output) = tokio::process::Command::new("docker")
|
||||
.arg("version")
|
||||
.output()
|
||||
.await
|
||||
{
|
||||
// detect whether we're actually running podman with docker emulation
|
||||
return Ok(String::from_utf8_lossy(&output.stdout)
|
||||
.to_lowercase()
|
||||
.contains("podman"));
|
||||
}
|
||||
|
||||
tokio::process::Command::new("podman")
|
||||
.arg("--version")
|
||||
.output()
|
||||
.await
|
||||
.map_err(|err| anyhow!("Failed to detect container engine: {err}"))?;
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
impl DockerClient {
|
||||
fn client_command(&self) -> tokio::process::Command {
|
||||
tokio::process::Command::new(self.client_binary())
|
||||
}
|
||||
|
||||
pub async fn create_volume(&self, name: &str) -> Result<()> {
|
||||
let result = self
|
||||
.client_command()
|
||||
.args(["volume", "create", name])
|
||||
.output()
|
||||
.await
|
||||
.map_err(|err| anyhow!("Failed to create volume '{name}': {err}"))?;
|
||||
|
||||
if !result.status.success() {
|
||||
return Err(anyhow!(
|
||||
"Failed to create volume '{name}': {}",
|
||||
String::from_utf8_lossy(&result.stderr)
|
||||
)
|
||||
.into());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn container_run(&self, options: ContainerRunOptions) -> Result<String> {
|
||||
let mut cmd = self.client_command();
|
||||
cmd.args(["run", "--platform", "linux/amd64"]);
|
||||
|
||||
if options.detach {
|
||||
cmd.arg("-d");
|
||||
}
|
||||
|
||||
Self::apply_cmd_options(&mut cmd, &options);
|
||||
|
||||
trace!("cmd: {:?}", cmd);
|
||||
|
||||
let result = cmd.output().await.map_err(|err| {
|
||||
anyhow!(
|
||||
"Failed to run container with image '{image}' and command '{command}': {err}",
|
||||
image = options.image,
|
||||
command = options.command.join(" "),
|
||||
)
|
||||
})?;
|
||||
|
||||
if !result.status.success() {
|
||||
return Err(anyhow!(
|
||||
"Failed to run container with image '{image}' and command '{command}': {err}",
|
||||
image = options.image,
|
||||
command = options.command.join(" "),
|
||||
err = String::from_utf8_lossy(&result.stderr)
|
||||
)
|
||||
.into());
|
||||
}
|
||||
|
||||
Ok(String::from_utf8_lossy(&result.stdout).to_string())
|
||||
}
|
||||
|
||||
pub async fn container_create(&self, options: ContainerRunOptions) -> Result<String> {
|
||||
let mut cmd = self.client_command();
|
||||
cmd.args(["container", "create"]);
|
||||
|
||||
Self::apply_cmd_options(&mut cmd, &options);
|
||||
|
||||
trace!("cmd: {:?}", cmd);
|
||||
|
||||
let result = cmd.output().await.map_err(|err| {
|
||||
anyhow!(
|
||||
"Failed to run container with image '{image}' and command '{command}': {err}",
|
||||
image = options.image,
|
||||
command = options.command.join(" "),
|
||||
)
|
||||
})?;
|
||||
|
||||
if !result.status.success() {
|
||||
return Err(anyhow!(
|
||||
"Failed to run container with image '{image}' and command '{command}': {err}",
|
||||
image = options.image,
|
||||
command = options.command.join(" "),
|
||||
err = String::from_utf8_lossy(&result.stderr)
|
||||
)
|
||||
.into());
|
||||
}
|
||||
|
||||
Ok(String::from_utf8_lossy(&result.stdout).to_string())
|
||||
}
|
||||
|
||||
pub async fn container_exec<S>(
|
||||
&self,
|
||||
name: &str,
|
||||
command: Vec<S>,
|
||||
env: Option<Vec<(S, S)>>,
|
||||
as_user: Option<S>,
|
||||
) -> Result<ExecutionResult>
|
||||
where
|
||||
S: Into<String> + std::fmt::Debug + Send + Clone,
|
||||
{
|
||||
let mut cmd = self.client_command();
|
||||
cmd.arg("exec");
|
||||
|
||||
if let Some(env) = env {
|
||||
for env_var in env {
|
||||
cmd.args(["-e", &format!("{}={}", env_var.0.into(), env_var.1.into())]);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(user) = as_user {
|
||||
cmd.args(["-u", user.into().as_ref()]);
|
||||
}
|
||||
|
||||
cmd.arg(name);
|
||||
|
||||
cmd.args(
|
||||
command
|
||||
.clone()
|
||||
.into_iter()
|
||||
.map(|s| <S as Into<String>>::into(s)),
|
||||
);
|
||||
|
||||
trace!("cmd is : {:?}", cmd);
|
||||
|
||||
let result = cmd.output().await.map_err(|err| {
|
||||
anyhow!(
|
||||
"Failed to exec '{}' on '{}': {err}",
|
||||
command
|
||||
.into_iter()
|
||||
.map(|s| <S as Into<String>>::into(s))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" "),
|
||||
name,
|
||||
)
|
||||
})?;
|
||||
|
||||
if !result.status.success() {
|
||||
return Ok(Err((
|
||||
result.status,
|
||||
String::from_utf8_lossy(&result.stderr).to_string(),
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(Ok(String::from_utf8_lossy(&result.stdout).to_string()))
|
||||
}
|
||||
|
||||
pub async fn container_cp(
|
||||
&self,
|
||||
name: &str,
|
||||
local_path: &Path,
|
||||
remote_path: &Path,
|
||||
) -> Result<()> {
|
||||
let result = self
|
||||
.client_command()
|
||||
.args([
|
||||
"cp",
|
||||
local_path.to_string_lossy().as_ref(),
|
||||
&format!("{name}:{}", remote_path.to_string_lossy().as_ref()),
|
||||
])
|
||||
.output()
|
||||
.await
|
||||
.map_err(|err| {
|
||||
anyhow!(
|
||||
"Failed copy file '{file}' to container '{name}': {err}",
|
||||
file = local_path.to_string_lossy(),
|
||||
)
|
||||
})?;
|
||||
|
||||
if !result.status.success() {
|
||||
return Err(anyhow!(
|
||||
"Failed to copy file '{file}' to container '{name}': {err}",
|
||||
file = local_path.to_string_lossy(),
|
||||
err = String::from_utf8_lossy(&result.stderr)
|
||||
)
|
||||
.into());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn container_rm(&self, name: &str) -> Result<()> {
|
||||
let result = self
|
||||
.client_command()
|
||||
.args(["rm", "--force", "--volumes", name])
|
||||
.output()
|
||||
.await
|
||||
.map_err(|err| anyhow!("Failed do remove container '{name}: {err}"))?;
|
||||
|
||||
if !result.status.success() {
|
||||
return Err(anyhow!(
|
||||
"Failed to remove container '{name}': {err}",
|
||||
err = String::from_utf8_lossy(&result.stderr)
|
||||
)
|
||||
.into());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn namespaced_containers_rm(&self, namespace: &str) -> Result<()> {
|
||||
let container_names: Vec<String> = self
|
||||
.get_containers()
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter_map(|container| match container {
|
||||
Container::Docker(container) => {
|
||||
if let Some(name) = container.names.first() {
|
||||
if name.starts_with(namespace) {
|
||||
return Some(name.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
},
|
||||
Container::Podman(container) => {
|
||||
if let Some(name) = container.names.first() {
|
||||
if name.starts_with(namespace) {
|
||||
return Some(name.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
},
|
||||
})
|
||||
.collect();
|
||||
|
||||
info!("{:?}", container_names);
|
||||
let futures = container_names
|
||||
.iter()
|
||||
.map(|name| self.container_rm(name))
|
||||
.collect::<Vec<_>>();
|
||||
try_join_all(futures).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn container_ip(&self, container_name: &str) -> Result<String> {
|
||||
let ip = if self.using_podman {
|
||||
"127.0.0.1".into()
|
||||
} else {
|
||||
let mut cmd = tokio::process::Command::new("docker");
|
||||
cmd.args(vec![
|
||||
"inspect",
|
||||
"-f",
|
||||
"{{ .NetworkSettings.IPAddress }}",
|
||||
container_name,
|
||||
]);
|
||||
|
||||
trace!("CMD: {cmd:?}");
|
||||
|
||||
let res = cmd
|
||||
.output()
|
||||
.await
|
||||
.map_err(|err| anyhow!("Failed to get docker container ip, output: {err}"))?;
|
||||
|
||||
String::from_utf8(res.stdout)
|
||||
.map_err(|err| anyhow!("Failed to get docker container ip, output: {err}"))?
|
||||
.trim()
|
||||
.into()
|
||||
};
|
||||
|
||||
trace!("IP: {ip}");
|
||||
Ok(ip)
|
||||
}
|
||||
|
||||
async fn get_containers(&self) -> Result<Vec<Container>> {
|
||||
let containers = if self.using_podman {
|
||||
self.get_podman_containers()
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(Container::Podman)
|
||||
.collect()
|
||||
} else {
|
||||
self.get_docker_containers()
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(Container::Docker)
|
||||
.collect()
|
||||
};
|
||||
|
||||
Ok(containers)
|
||||
}
|
||||
|
||||
async fn get_podman_containers(&self) -> Result<Vec<PodmanContainer>> {
|
||||
let res = tokio::process::Command::new("podman")
|
||||
.args(vec!["ps", "--all", "--no-trunc", "--format", "json"])
|
||||
.output()
|
||||
.await
|
||||
.map_err(|err| anyhow!("Failed to get podman containers output: {err}"))?;
|
||||
|
||||
let stdout = String::from_utf8_lossy(&res.stdout);
|
||||
|
||||
let containers = serde_json::from_str(&stdout)
|
||||
.map_err(|err| anyhow!("Failed to parse podman containers output: {err}"))?;
|
||||
|
||||
Ok(containers)
|
||||
}
|
||||
|
||||
async fn get_docker_containers(&self) -> Result<Vec<DockerContainer>> {
|
||||
let res = tokio::process::Command::new("docker")
|
||||
.args(vec!["ps", "--all", "--no-trunc", "--format", "json"])
|
||||
.output()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let stdout = String::from_utf8_lossy(&res.stdout);
|
||||
|
||||
let mut containers = vec![];
|
||||
for line in stdout.lines() {
|
||||
containers.push(
|
||||
serde_json::from_str::<DockerContainer>(line)
|
||||
.map_err(|err| anyhow!("Failed to parse docker container output: {err}"))?,
|
||||
);
|
||||
}
|
||||
|
||||
Ok(containers)
|
||||
}
|
||||
|
||||
pub(crate) async fn container_logs(&self, container_name: &str) -> Result<String> {
|
||||
let output = Command::new("sh")
|
||||
.arg("-c")
|
||||
.arg(format!("docker logs -t '{container_name}' 2>&1"))
|
||||
.stdout(Stdio::piped())
|
||||
.output()
|
||||
.await
|
||||
.map_err(|err| {
|
||||
anyhow!(
|
||||
"Failed to spawn docker logs command for container '{container_name}': {err}"
|
||||
)
|
||||
})?;
|
||||
|
||||
let logs = String::from_utf8_lossy(&output.stdout).to_string();
|
||||
|
||||
if !output.status.success() {
|
||||
// stderr was redirected to stdout, so logs should contain the error message if any
|
||||
return Err(anyhow!(
|
||||
"Failed to get logs for container '{name}': {logs}",
|
||||
name = container_name,
|
||||
logs = &logs
|
||||
)
|
||||
.into());
|
||||
}
|
||||
|
||||
Ok(logs)
|
||||
}
|
||||
|
||||
fn apply_cmd_options(cmd: &mut Command, options: &ContainerRunOptions) {
|
||||
if options.rm {
|
||||
cmd.arg("--rm");
|
||||
}
|
||||
|
||||
if let Some(entrypoint) = options.entrypoint.as_ref() {
|
||||
cmd.args(["--entrypoint", entrypoint]);
|
||||
}
|
||||
|
||||
if let Some(volume_mounts) = options.volume_mounts.as_ref() {
|
||||
for (source, target) in volume_mounts {
|
||||
cmd.args(["-v", &format!("{source}:{target}")]);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(env) = options.env.as_ref() {
|
||||
for env_var in env {
|
||||
cmd.args(["-e", &format!("{}={}", env_var.0, env_var.1)]);
|
||||
}
|
||||
}
|
||||
|
||||
// add published ports
|
||||
for (container_port, host_port) in options.port_mapping.iter() {
|
||||
cmd.args(["-p", &format!("{host_port}:{container_port}")]);
|
||||
}
|
||||
|
||||
if let Some(name) = options.name.as_ref() {
|
||||
cmd.args(["--name", name]);
|
||||
}
|
||||
|
||||
cmd.arg(&options.image);
|
||||
|
||||
for arg in &options.command {
|
||||
cmd.arg(arg);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,494 @@
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, Weak},
|
||||
thread,
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use support::{constants::THIS_IS_A_BUG, fs::FileSystem};
|
||||
use tokio::sync::{Mutex, RwLock};
|
||||
use tracing::{debug, trace, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::{
|
||||
client::{ContainerRunOptions, DockerClient},
|
||||
node::DockerNode,
|
||||
DockerProvider,
|
||||
};
|
||||
use crate::{
|
||||
constants::NAMESPACE_PREFIX,
|
||||
docker::{
|
||||
node::{DeserializableDockerNodeOptions, DockerNodeOptions},
|
||||
provider,
|
||||
},
|
||||
shared::helpers::extract_execution_result,
|
||||
types::{
|
||||
GenerateFileCommand, GenerateFilesOptions, ProviderCapabilities, RunCommandOptions,
|
||||
SpawnNodeOptions,
|
||||
},
|
||||
DynNode, ProviderError, ProviderNamespace, ProviderNode,
|
||||
};
|
||||
|
||||
pub struct DockerNamespace<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone,
|
||||
{
|
||||
weak: Weak<DockerNamespace<FS>>,
|
||||
#[allow(dead_code)]
|
||||
provider: Weak<DockerProvider<FS>>,
|
||||
name: String,
|
||||
base_dir: PathBuf,
|
||||
capabilities: ProviderCapabilities,
|
||||
docker_client: DockerClient,
|
||||
filesystem: FS,
|
||||
delete_on_drop: Arc<Mutex<bool>>,
|
||||
pub(super) nodes: RwLock<HashMap<String, Arc<DockerNode<FS>>>>,
|
||||
}
|
||||
|
||||
impl<FS> DockerNamespace<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
pub(super) async fn new(
|
||||
provider: &Weak<DockerProvider<FS>>,
|
||||
tmp_dir: &PathBuf,
|
||||
capabilities: &ProviderCapabilities,
|
||||
docker_client: &DockerClient,
|
||||
filesystem: &FS,
|
||||
custom_base_dir: Option<&Path>,
|
||||
) -> Result<Arc<Self>, ProviderError> {
|
||||
let name = format!("{}{}", NAMESPACE_PREFIX, Uuid::new_v4());
|
||||
let base_dir = if let Some(custom_base_dir) = custom_base_dir {
|
||||
if !filesystem.exists(custom_base_dir).await {
|
||||
filesystem.create_dir(custom_base_dir).await?;
|
||||
} else {
|
||||
warn!(
|
||||
"⚠️ Using and existing directory {} as base dir",
|
||||
custom_base_dir.to_string_lossy()
|
||||
);
|
||||
}
|
||||
PathBuf::from(custom_base_dir)
|
||||
} else {
|
||||
let base_dir = PathBuf::from_iter([tmp_dir, &PathBuf::from(&name)]);
|
||||
filesystem.create_dir(&base_dir).await?;
|
||||
base_dir
|
||||
};
|
||||
|
||||
let namespace = Arc::new_cyclic(|weak| DockerNamespace {
|
||||
weak: weak.clone(),
|
||||
provider: provider.clone(),
|
||||
name,
|
||||
base_dir,
|
||||
capabilities: capabilities.clone(),
|
||||
filesystem: filesystem.clone(),
|
||||
docker_client: docker_client.clone(),
|
||||
nodes: RwLock::new(HashMap::new()),
|
||||
delete_on_drop: Arc::new(Mutex::new(true)),
|
||||
});
|
||||
|
||||
namespace.initialize().await?;
|
||||
|
||||
Ok(namespace)
|
||||
}
|
||||
|
||||
pub(super) async fn attach_to_live(
|
||||
provider: &Weak<DockerProvider<FS>>,
|
||||
capabilities: &ProviderCapabilities,
|
||||
docker_client: &DockerClient,
|
||||
filesystem: &FS,
|
||||
custom_base_dir: &Path,
|
||||
name: &str,
|
||||
) -> Result<Arc<Self>, ProviderError> {
|
||||
let base_dir = custom_base_dir.to_path_buf();
|
||||
|
||||
let namespace = Arc::new_cyclic(|weak| DockerNamespace {
|
||||
weak: weak.clone(),
|
||||
provider: provider.clone(),
|
||||
name: name.to_owned(),
|
||||
base_dir,
|
||||
capabilities: capabilities.clone(),
|
||||
filesystem: filesystem.clone(),
|
||||
docker_client: docker_client.clone(),
|
||||
nodes: RwLock::new(HashMap::new()),
|
||||
delete_on_drop: Arc::new(Mutex::new(false)),
|
||||
});
|
||||
|
||||
Ok(namespace)
|
||||
}
|
||||
|
||||
async fn initialize(&self) -> Result<(), ProviderError> {
|
||||
// let ns_scripts_shared = PathBuf::from_iter([&self.base_dir, &PathBuf::from("shared-scripts")]);
|
||||
// self.filesystem.create_dir(&ns_scripts_shared).await?;
|
||||
self.initialize_zombie_scripts_volume().await?;
|
||||
self.initialize_helper_binaries_volume().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn initialize_zombie_scripts_volume(&self) -> Result<(), ProviderError> {
|
||||
let local_zombie_wrapper_path =
|
||||
PathBuf::from_iter([&self.base_dir, &PathBuf::from("zombie-wrapper.sh")]);
|
||||
|
||||
self.filesystem
|
||||
.write(
|
||||
&local_zombie_wrapper_path,
|
||||
include_str!("../shared/scripts/zombie-wrapper.sh"),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let local_helper_binaries_downloader_path = PathBuf::from_iter([
|
||||
&self.base_dir,
|
||||
&PathBuf::from("helper-binaries-downloader.sh"),
|
||||
]);
|
||||
|
||||
self.filesystem
|
||||
.write(
|
||||
&local_helper_binaries_downloader_path,
|
||||
include_str!("../shared/scripts/helper-binaries-downloader.sh"),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let zombie_wrapper_volume_name = format!("{}-zombie-wrapper", self.name);
|
||||
let zombie_wrapper_container_name = format!("{}-scripts", self.name);
|
||||
|
||||
self.docker_client
|
||||
.create_volume(&zombie_wrapper_volume_name)
|
||||
.await
|
||||
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
|
||||
|
||||
self.docker_client
|
||||
.container_create(
|
||||
ContainerRunOptions::new("alpine:latest", vec!["tail", "-f", "/dev/null"])
|
||||
.volume_mounts(HashMap::from([(
|
||||
zombie_wrapper_volume_name.as_str(),
|
||||
"/scripts",
|
||||
)]))
|
||||
.name(&zombie_wrapper_container_name)
|
||||
.rm(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
|
||||
|
||||
// copy the scripts
|
||||
self.docker_client
|
||||
.container_cp(
|
||||
&zombie_wrapper_container_name,
|
||||
&local_zombie_wrapper_path,
|
||||
&PathBuf::from("/scripts/zombie-wrapper.sh"),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
|
||||
|
||||
self.docker_client
|
||||
.container_cp(
|
||||
&zombie_wrapper_container_name,
|
||||
&local_helper_binaries_downloader_path,
|
||||
&PathBuf::from("/scripts/helper-binaries-downloader.sh"),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
|
||||
|
||||
// set permissions for rwx on whole volume recursively
|
||||
self.docker_client
|
||||
.container_run(
|
||||
ContainerRunOptions::new("alpine:latest", vec!["chmod", "-R", "777", "/scripts"])
|
||||
.volume_mounts(HashMap::from([(
|
||||
zombie_wrapper_volume_name.as_ref(),
|
||||
"/scripts",
|
||||
)]))
|
||||
.rm(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn initialize_helper_binaries_volume(&self) -> Result<(), ProviderError> {
|
||||
let helper_binaries_volume_name = format!("{}-helper-binaries", self.name);
|
||||
let zombie_wrapper_volume_name = format!("{}-zombie-wrapper", self.name);
|
||||
|
||||
self.docker_client
|
||||
.create_volume(&helper_binaries_volume_name)
|
||||
.await
|
||||
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
|
||||
|
||||
// download binaries to volume
|
||||
self.docker_client
|
||||
.container_run(
|
||||
ContainerRunOptions::new(
|
||||
"alpine:latest",
|
||||
vec!["ash", "/scripts/helper-binaries-downloader.sh"],
|
||||
)
|
||||
.volume_mounts(HashMap::from([
|
||||
(
|
||||
helper_binaries_volume_name.as_str(),
|
||||
"/helpers",
|
||||
),
|
||||
(
|
||||
zombie_wrapper_volume_name.as_ref(),
|
||||
"/scripts",
|
||||
)
|
||||
]))
|
||||
// wait until complete
|
||||
.detach(false)
|
||||
.rm(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
|
||||
|
||||
// set permissions for rwx on whole volume recursively
|
||||
self.docker_client
|
||||
.container_run(
|
||||
ContainerRunOptions::new("alpine:latest", vec!["chmod", "-R", "777", "/helpers"])
|
||||
.volume_mounts(HashMap::from([(
|
||||
helper_binaries_volume_name.as_ref(),
|
||||
"/helpers",
|
||||
)]))
|
||||
.rm(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn set_delete_on_drop(&self, delete_on_drop: bool) {
|
||||
*self.delete_on_drop.lock().await = delete_on_drop;
|
||||
}
|
||||
|
||||
pub async fn delete_on_drop(&self) -> bool {
|
||||
if let Ok(delete_on_drop) = self.delete_on_drop.try_lock() {
|
||||
*delete_on_drop
|
||||
} else {
|
||||
// if we can't lock just remove the ns
|
||||
true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<FS> ProviderNamespace for DockerNamespace<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
fn name(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
|
||||
fn base_dir(&self) -> &PathBuf {
|
||||
&self.base_dir
|
||||
}
|
||||
|
||||
fn capabilities(&self) -> &ProviderCapabilities {
|
||||
&self.capabilities
|
||||
}
|
||||
|
||||
fn provider_name(&self) -> &str {
|
||||
provider::PROVIDER_NAME
|
||||
}
|
||||
|
||||
async fn detach(&self) {
|
||||
self.set_delete_on_drop(false).await;
|
||||
}
|
||||
|
||||
async fn is_detached(&self) -> bool {
|
||||
self.delete_on_drop().await
|
||||
}
|
||||
|
||||
async fn nodes(&self) -> HashMap<String, DynNode> {
|
||||
self.nodes
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.map(|(name, node)| (name.clone(), node.clone() as DynNode))
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn get_node_available_args(
|
||||
&self,
|
||||
(command, image): (String, Option<String>),
|
||||
) -> Result<String, ProviderError> {
|
||||
let node_image = image.expect(&format!("image should be present when getting node available args with docker provider {THIS_IS_A_BUG}"));
|
||||
|
||||
let temp_node = self
|
||||
.spawn_node(
|
||||
&SpawnNodeOptions::new(format!("temp-{}", Uuid::new_v4()), "cat".to_string())
|
||||
.image(node_image.clone()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let available_args_output = temp_node
|
||||
.run_command(RunCommandOptions::new(command.clone()).args(vec!["--help"]))
|
||||
.await?
|
||||
.map_err(|(_exit, status)| {
|
||||
ProviderError::NodeAvailableArgsError(node_image, command, status)
|
||||
})?;
|
||||
|
||||
temp_node.destroy().await?;
|
||||
|
||||
Ok(available_args_output)
|
||||
}
|
||||
|
||||
async fn spawn_node(&self, options: &SpawnNodeOptions) -> Result<DynNode, ProviderError> {
|
||||
debug!("spawn option {:?}", options);
|
||||
|
||||
let node = DockerNode::new(DockerNodeOptions {
|
||||
namespace: &self.weak,
|
||||
namespace_base_dir: &self.base_dir,
|
||||
name: &options.name,
|
||||
image: options.image.as_ref(),
|
||||
program: &options.program,
|
||||
args: &options.args,
|
||||
env: &options.env,
|
||||
startup_files: &options.injected_files,
|
||||
db_snapshot: options.db_snapshot.as_ref(),
|
||||
docker_client: &self.docker_client,
|
||||
container_name: format!("{}-{}", self.name, options.name),
|
||||
filesystem: &self.filesystem,
|
||||
port_mapping: options.port_mapping.as_ref().unwrap_or(&HashMap::default()),
|
||||
})
|
||||
.await?;
|
||||
|
||||
self.nodes
|
||||
.write()
|
||||
.await
|
||||
.insert(node.name().to_string(), node.clone());
|
||||
|
||||
Ok(node)
|
||||
}
|
||||
|
||||
async fn spawn_node_from_json(
|
||||
&self,
|
||||
json_value: &serde_json::Value,
|
||||
) -> Result<DynNode, ProviderError> {
|
||||
let deserializable: DeserializableDockerNodeOptions =
|
||||
serde_json::from_value(json_value.clone())?;
|
||||
let options = DockerNodeOptions::from_deserializable(
|
||||
&deserializable,
|
||||
&self.weak,
|
||||
&self.base_dir,
|
||||
&self.docker_client,
|
||||
&self.filesystem,
|
||||
);
|
||||
|
||||
let node = DockerNode::attach_to_live(options).await?;
|
||||
|
||||
self.nodes
|
||||
.write()
|
||||
.await
|
||||
.insert(node.name().to_string(), node.clone());
|
||||
|
||||
Ok(node)
|
||||
}
|
||||
|
||||
async fn generate_files(&self, options: GenerateFilesOptions) -> Result<(), ProviderError> {
|
||||
debug!("generate files options {options:#?}");
|
||||
|
||||
let node_name = options
|
||||
.temp_name
|
||||
.unwrap_or_else(|| format!("temp-{}", Uuid::new_v4()));
|
||||
let node_image = options.image.expect(&format!(
|
||||
"image should be present when generating files with docker provider {THIS_IS_A_BUG}"
|
||||
));
|
||||
|
||||
// run dummy command in a new container
|
||||
let temp_node = self
|
||||
.spawn_node(
|
||||
&SpawnNodeOptions::new(node_name, "cat".to_string())
|
||||
.injected_files(options.injected_files)
|
||||
.image(node_image),
|
||||
)
|
||||
.await?;
|
||||
|
||||
for GenerateFileCommand {
|
||||
program,
|
||||
args,
|
||||
env,
|
||||
local_output_path,
|
||||
} in options.commands
|
||||
{
|
||||
let local_output_full_path = format!(
|
||||
"{}{}{}",
|
||||
self.base_dir.to_string_lossy(),
|
||||
if local_output_path.starts_with("/") {
|
||||
""
|
||||
} else {
|
||||
"/"
|
||||
},
|
||||
local_output_path.to_string_lossy()
|
||||
);
|
||||
|
||||
let contents = extract_execution_result(
|
||||
&temp_node,
|
||||
RunCommandOptions { program, args, env },
|
||||
options.expected_path.as_ref(),
|
||||
)
|
||||
.await?;
|
||||
self.filesystem
|
||||
.write(local_output_full_path, contents)
|
||||
.await
|
||||
.map_err(|err| ProviderError::FileGenerationFailed(err.into()))?;
|
||||
}
|
||||
|
||||
temp_node.destroy().await
|
||||
}
|
||||
|
||||
async fn static_setup(&self) -> Result<(), ProviderError> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
async fn destroy(&self) -> Result<(), ProviderError> {
|
||||
let _ = self
|
||||
.docker_client
|
||||
.namespaced_containers_rm(&self.name)
|
||||
.await
|
||||
.map_err(|err| ProviderError::DeleteNamespaceFailed(self.name.clone(), err.into()))?;
|
||||
|
||||
if let Some(provider) = self.provider.upgrade() {
|
||||
provider.namespaces.write().await.remove(&self.name);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<FS> Drop for DockerNamespace<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone,
|
||||
{
|
||||
fn drop(&mut self) {
|
||||
let ns_name = self.name.clone();
|
||||
if let Ok(delete_on_drop) = self.delete_on_drop.try_lock() {
|
||||
if *delete_on_drop {
|
||||
let client = self.docker_client.clone();
|
||||
let provider = self.provider.upgrade();
|
||||
|
||||
let handler = thread::spawn(move || {
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
rt.block_on(async move {
|
||||
trace!("🧟 deleting ns {ns_name} from cluster");
|
||||
let _ = client.namespaced_containers_rm(&ns_name).await;
|
||||
trace!("✅ deleted");
|
||||
});
|
||||
});
|
||||
|
||||
if handler.join().is_ok() {
|
||||
if let Some(provider) = provider {
|
||||
if let Ok(mut p) = provider.namespaces.try_write() {
|
||||
p.remove(&self.name);
|
||||
} else {
|
||||
warn!(
|
||||
"⚠️ Can not acquire write lock to the provider, ns {} not removed",
|
||||
self.name
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
trace!("⚠️ leaking ns {ns_name} in cluster");
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,659 @@
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
net::IpAddr,
|
||||
path::{Component, Path, PathBuf},
|
||||
sync::{Arc, Weak},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use anyhow::anyhow;
|
||||
use async_trait::async_trait;
|
||||
use configuration::types::AssetLocation;
|
||||
use futures::future::try_join_all;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use support::{constants::THIS_IS_A_BUG, fs::FileSystem};
|
||||
use tokio::{time::sleep, try_join};
|
||||
use tracing::debug;
|
||||
|
||||
use super::{
|
||||
client::{ContainerRunOptions, DockerClient},
|
||||
namespace::DockerNamespace,
|
||||
};
|
||||
use crate::{
|
||||
constants::{NODE_CONFIG_DIR, NODE_DATA_DIR, NODE_RELAY_DATA_DIR, NODE_SCRIPTS_DIR},
|
||||
docker,
|
||||
types::{ExecutionResult, Port, RunCommandOptions, RunScriptOptions, TransferedFile},
|
||||
ProviderError, ProviderNamespace, ProviderNode,
|
||||
};
|
||||
|
||||
pub(super) struct DockerNodeOptions<'a, FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
pub(super) namespace: &'a Weak<DockerNamespace<FS>>,
|
||||
pub(super) namespace_base_dir: &'a PathBuf,
|
||||
pub(super) name: &'a str,
|
||||
pub(super) image: Option<&'a String>,
|
||||
pub(super) program: &'a str,
|
||||
pub(super) args: &'a [String],
|
||||
pub(super) env: &'a [(String, String)],
|
||||
pub(super) startup_files: &'a [TransferedFile],
|
||||
pub(super) db_snapshot: Option<&'a AssetLocation>,
|
||||
pub(super) docker_client: &'a DockerClient,
|
||||
pub(super) container_name: String,
|
||||
pub(super) filesystem: &'a FS,
|
||||
pub(super) port_mapping: &'a HashMap<Port, Port>,
|
||||
}
|
||||
|
||||
impl<'a, FS> DockerNodeOptions<'a, FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
pub fn from_deserializable(
|
||||
deserializable: &'a DeserializableDockerNodeOptions,
|
||||
namespace: &'a Weak<DockerNamespace<FS>>,
|
||||
namespace_base_dir: &'a PathBuf,
|
||||
docker_client: &'a DockerClient,
|
||||
filesystem: &'a FS,
|
||||
) -> Self {
|
||||
DockerNodeOptions {
|
||||
namespace,
|
||||
namespace_base_dir,
|
||||
name: &deserializable.name,
|
||||
image: deserializable.image.as_ref(),
|
||||
program: &deserializable.program,
|
||||
args: &deserializable.args,
|
||||
env: &deserializable.env,
|
||||
startup_files: &[],
|
||||
db_snapshot: None,
|
||||
docker_client,
|
||||
container_name: deserializable.container_name.clone(),
|
||||
filesystem,
|
||||
port_mapping: &deserializable.port_mapping,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub(super) struct DeserializableDockerNodeOptions {
|
||||
pub(super) name: String,
|
||||
pub(super) image: Option<String>,
|
||||
pub(super) program: String,
|
||||
pub(super) args: Vec<String>,
|
||||
pub(super) env: Vec<(String, String)>,
|
||||
pub(super) container_name: String,
|
||||
pub(super) port_mapping: HashMap<Port, Port>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct DockerNode<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone,
|
||||
{
|
||||
#[serde(skip)]
|
||||
namespace: Weak<DockerNamespace<FS>>,
|
||||
name: String,
|
||||
image: String,
|
||||
program: String,
|
||||
args: Vec<String>,
|
||||
env: Vec<(String, String)>,
|
||||
base_dir: PathBuf,
|
||||
config_dir: PathBuf,
|
||||
data_dir: PathBuf,
|
||||
relay_data_dir: PathBuf,
|
||||
scripts_dir: PathBuf,
|
||||
log_path: PathBuf,
|
||||
#[serde(skip)]
|
||||
docker_client: DockerClient,
|
||||
container_name: String,
|
||||
port_mapping: HashMap<Port, Port>,
|
||||
#[allow(dead_code)]
|
||||
#[serde(skip)]
|
||||
filesystem: FS,
|
||||
provider_tag: String,
|
||||
}
|
||||
|
||||
impl<FS> DockerNode<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
pub(super) async fn new(
|
||||
options: DockerNodeOptions<'_, FS>,
|
||||
) -> Result<Arc<Self>, ProviderError> {
|
||||
let image = options.image.ok_or_else(|| {
|
||||
ProviderError::MissingNodeInfo(options.name.to_string(), "missing image".to_string())
|
||||
})?;
|
||||
|
||||
let filesystem = options.filesystem.clone();
|
||||
|
||||
let base_dir =
|
||||
PathBuf::from_iter([options.namespace_base_dir, &PathBuf::from(options.name)]);
|
||||
filesystem.create_dir_all(&base_dir).await?;
|
||||
|
||||
let base_dir_raw = base_dir.to_string_lossy();
|
||||
let config_dir = PathBuf::from(format!("{base_dir_raw}{NODE_CONFIG_DIR}"));
|
||||
let data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_DATA_DIR}"));
|
||||
let relay_data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_RELAY_DATA_DIR}"));
|
||||
let scripts_dir = PathBuf::from(format!("{base_dir_raw}{NODE_SCRIPTS_DIR}"));
|
||||
let log_path = base_dir.join("node.log");
|
||||
|
||||
try_join!(
|
||||
filesystem.create_dir_all(&config_dir),
|
||||
filesystem.create_dir_all(&data_dir),
|
||||
filesystem.create_dir_all(&relay_data_dir),
|
||||
filesystem.create_dir_all(&scripts_dir),
|
||||
)?;
|
||||
|
||||
let node = Arc::new(DockerNode {
|
||||
namespace: options.namespace.clone(),
|
||||
name: options.name.to_string(),
|
||||
image: image.to_string(),
|
||||
program: options.program.to_string(),
|
||||
args: options.args.to_vec(),
|
||||
env: options.env.to_vec(),
|
||||
base_dir,
|
||||
config_dir,
|
||||
data_dir,
|
||||
relay_data_dir,
|
||||
scripts_dir,
|
||||
log_path,
|
||||
filesystem: filesystem.clone(),
|
||||
docker_client: options.docker_client.clone(),
|
||||
container_name: options.container_name,
|
||||
port_mapping: options.port_mapping.clone(),
|
||||
provider_tag: docker::provider::PROVIDER_NAME.to_string(),
|
||||
});
|
||||
|
||||
node.initialize_docker().await?;
|
||||
|
||||
if let Some(db_snap) = options.db_snapshot {
|
||||
node.initialize_db_snapshot(db_snap).await?;
|
||||
}
|
||||
|
||||
node.initialize_startup_files(options.startup_files).await?;
|
||||
|
||||
node.start().await?;
|
||||
|
||||
Ok(node)
|
||||
}
|
||||
|
||||
pub(super) async fn attach_to_live(
|
||||
options: DockerNodeOptions<'_, FS>,
|
||||
) -> Result<Arc<Self>, ProviderError> {
|
||||
let image = options.image.ok_or_else(|| {
|
||||
ProviderError::MissingNodeInfo(options.name.to_string(), "missing image".to_string())
|
||||
})?;
|
||||
|
||||
let filesystem = options.filesystem.clone();
|
||||
|
||||
let base_dir =
|
||||
PathBuf::from_iter([options.namespace_base_dir, &PathBuf::from(options.name)]);
|
||||
filesystem.create_dir_all(&base_dir).await?;
|
||||
|
||||
let base_dir_raw = base_dir.to_string_lossy();
|
||||
let config_dir = PathBuf::from(format!("{base_dir_raw}{NODE_CONFIG_DIR}"));
|
||||
let data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_DATA_DIR}"));
|
||||
let relay_data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_RELAY_DATA_DIR}"));
|
||||
let scripts_dir = PathBuf::from(format!("{base_dir_raw}{NODE_SCRIPTS_DIR}"));
|
||||
let log_path = base_dir.join("node.log");
|
||||
|
||||
let node = Arc::new(DockerNode {
|
||||
namespace: options.namespace.clone(),
|
||||
name: options.name.to_string(),
|
||||
image: image.to_string(),
|
||||
program: options.program.to_string(),
|
||||
args: options.args.to_vec(),
|
||||
env: options.env.to_vec(),
|
||||
base_dir,
|
||||
config_dir,
|
||||
data_dir,
|
||||
relay_data_dir,
|
||||
scripts_dir,
|
||||
log_path,
|
||||
filesystem: filesystem.clone(),
|
||||
docker_client: options.docker_client.clone(),
|
||||
container_name: options.container_name,
|
||||
port_mapping: options.port_mapping.clone(),
|
||||
provider_tag: docker::provider::PROVIDER_NAME.to_string(),
|
||||
});
|
||||
|
||||
Ok(node)
|
||||
}
|
||||
|
||||
async fn initialize_docker(&self) -> Result<(), ProviderError> {
|
||||
let command = [vec![self.program.to_string()], self.args.to_vec()].concat();
|
||||
|
||||
self.docker_client
|
||||
.container_run(
|
||||
ContainerRunOptions::new(&self.image, command)
|
||||
.name(&self.container_name)
|
||||
.env(self.env.clone())
|
||||
.volume_mounts(HashMap::from([
|
||||
(
|
||||
format!("{}-zombie-wrapper", self.namespace_name(),),
|
||||
"/scripts".to_string(),
|
||||
),
|
||||
(
|
||||
format!("{}-helper-binaries", self.namespace_name()),
|
||||
"/helpers".to_string(),
|
||||
),
|
||||
(
|
||||
self.config_dir.to_string_lossy().into_owned(),
|
||||
"/cfg".to_string(),
|
||||
),
|
||||
(
|
||||
self.data_dir.to_string_lossy().into_owned(),
|
||||
"/data".to_string(),
|
||||
),
|
||||
(
|
||||
self.relay_data_dir.to_string_lossy().into_owned(),
|
||||
"/relay-data".to_string(),
|
||||
),
|
||||
]))
|
||||
.entrypoint("/scripts/zombie-wrapper.sh")
|
||||
.port_mapping(&self.port_mapping),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ProviderError::NodeSpawningFailed(self.name.clone(), err.into()))?;
|
||||
|
||||
// change dirs permission
|
||||
let _ = self
|
||||
.docker_client
|
||||
.container_exec(
|
||||
&self.container_name,
|
||||
["chmod", "777", "/cfg", "/data", "/relay-data"].into(),
|
||||
None,
|
||||
Some("root"),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ProviderError::NodeSpawningFailed(self.name.clone(), err.into()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn initialize_db_snapshot(
|
||||
&self,
|
||||
_db_snapshot: &AssetLocation,
|
||||
) -> Result<(), ProviderError> {
|
||||
todo!()
|
||||
// trace!("snap: {db_snapshot}");
|
||||
// let url_of_snap = match db_snapshot {
|
||||
// AssetLocation::Url(location) => location.clone(),
|
||||
// AssetLocation::FilePath(filepath) => self.upload_to_fileserver(filepath).await?,
|
||||
// };
|
||||
|
||||
// // we need to get the snapshot from a public access
|
||||
// // and extract to /data
|
||||
// let opts = RunCommandOptions::new("mkdir").args([
|
||||
// "-p",
|
||||
// "/data/",
|
||||
// "&&",
|
||||
// "mkdir",
|
||||
// "-p",
|
||||
// "/relay-data/",
|
||||
// "&&",
|
||||
// // Use our version of curl
|
||||
// "/cfg/curl",
|
||||
// url_of_snap.as_ref(),
|
||||
// "--output",
|
||||
// "/data/db.tgz",
|
||||
// "&&",
|
||||
// "cd",
|
||||
// "/",
|
||||
// "&&",
|
||||
// "tar",
|
||||
// "--skip-old-files",
|
||||
// "-xzvf",
|
||||
// "/data/db.tgz",
|
||||
// ]);
|
||||
|
||||
// trace!("cmd opts: {:#?}", opts);
|
||||
// let _ = self.run_command(opts).await?;
|
||||
|
||||
// Ok(())
|
||||
}
|
||||
|
||||
async fn initialize_startup_files(
|
||||
&self,
|
||||
startup_files: &[TransferedFile],
|
||||
) -> Result<(), ProviderError> {
|
||||
try_join_all(
|
||||
startup_files
|
||||
.iter()
|
||||
.map(|file| self.send_file(&file.local_path, &file.remote_path, &file.mode)),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn start(&self) -> Result<(), ProviderError> {
|
||||
self.docker_client
|
||||
.container_exec(
|
||||
&self.container_name,
|
||||
vec!["sh", "-c", "echo start > /tmp/zombiepipe"],
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
ProviderError::NodeSpawningFailed(
|
||||
format!("failed to start pod {} after spawning", self.name),
|
||||
err.into(),
|
||||
)
|
||||
})?
|
||||
.map_err(|err| {
|
||||
ProviderError::NodeSpawningFailed(
|
||||
format!("failed to start pod {} after spawning", self.name,),
|
||||
anyhow!("command failed in container: status {}: {}", err.0, err.1),
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_remote_parent_dir(&self, remote_file_path: &Path) -> Option<PathBuf> {
|
||||
if let Some(remote_parent_dir) = remote_file_path.parent() {
|
||||
if matches!(
|
||||
remote_parent_dir.components().rev().peekable().peek(),
|
||||
Some(Component::Normal(_))
|
||||
) {
|
||||
return Some(remote_parent_dir.to_path_buf());
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
async fn create_remote_dir(&self, remote_dir: &Path) -> Result<(), ProviderError> {
|
||||
let _ = self
|
||||
.docker_client
|
||||
.container_exec(
|
||||
&self.container_name,
|
||||
vec!["mkdir", "-p", &remote_dir.to_string_lossy()],
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
ProviderError::NodeSpawningFailed(
|
||||
format!(
|
||||
"failed to create dir {} for container {}",
|
||||
remote_dir.to_string_lossy(),
|
||||
&self.name
|
||||
),
|
||||
err.into(),
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn namespace_name(&self) -> String {
|
||||
self.namespace
|
||||
.upgrade()
|
||||
.map(|namespace| namespace.name().to_string())
|
||||
.unwrap_or_else(|| panic!("namespace shouldn't be dropped, {THIS_IS_A_BUG}"))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<FS> ProviderNode for DockerNode<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
fn name(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
|
||||
fn args(&self) -> Vec<&str> {
|
||||
self.args.iter().map(|arg| arg.as_str()).collect()
|
||||
}
|
||||
|
||||
fn base_dir(&self) -> &PathBuf {
|
||||
&self.base_dir
|
||||
}
|
||||
|
||||
fn config_dir(&self) -> &PathBuf {
|
||||
&self.config_dir
|
||||
}
|
||||
|
||||
fn data_dir(&self) -> &PathBuf {
|
||||
&self.data_dir
|
||||
}
|
||||
|
||||
fn relay_data_dir(&self) -> &PathBuf {
|
||||
&self.relay_data_dir
|
||||
}
|
||||
|
||||
fn scripts_dir(&self) -> &PathBuf {
|
||||
&self.scripts_dir
|
||||
}
|
||||
|
||||
fn log_path(&self) -> &PathBuf {
|
||||
&self.log_path
|
||||
}
|
||||
|
||||
fn log_cmd(&self) -> String {
|
||||
format!(
|
||||
"{} logs -f {}",
|
||||
self.docker_client.client_binary(),
|
||||
self.container_name
|
||||
)
|
||||
}
|
||||
|
||||
fn path_in_node(&self, file: &Path) -> PathBuf {
|
||||
// here is just a noop op since we will receive the path
|
||||
// for the file inside the pod
|
||||
PathBuf::from(file)
|
||||
}
|
||||
|
||||
async fn logs(&self) -> Result<String, ProviderError> {
|
||||
self.docker_client
|
||||
.container_logs(&self.container_name)
|
||||
.await
|
||||
.map_err(|err| ProviderError::GetLogsFailed(self.name.to_string(), err.into()))
|
||||
}
|
||||
|
||||
async fn dump_logs(&self, local_dest: PathBuf) -> Result<(), ProviderError> {
|
||||
let logs = self.logs().await?;
|
||||
|
||||
self.filesystem
|
||||
.write(local_dest, logs)
|
||||
.await
|
||||
.map_err(|err| ProviderError::DumpLogsFailed(self.name.to_string(), err.into()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_command(
|
||||
&self,
|
||||
options: RunCommandOptions,
|
||||
) -> Result<ExecutionResult, ProviderError> {
|
||||
debug!(
|
||||
"running command for {} with options {:?}",
|
||||
self.name, options
|
||||
);
|
||||
let command = [vec![options.program], options.args].concat();
|
||||
|
||||
self.docker_client
|
||||
.container_exec(
|
||||
&self.container_name,
|
||||
vec!["sh", "-c", &command.join(" ")],
|
||||
Some(
|
||||
options
|
||||
.env
|
||||
.iter()
|
||||
.map(|(k, v)| (k.as_ref(), v.as_ref()))
|
||||
.collect(),
|
||||
),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
ProviderError::RunCommandError(
|
||||
format!("sh -c {}", &command.join(" ")),
|
||||
format!("in pod {}", self.name),
|
||||
err.into(),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
async fn run_script(
|
||||
&self,
|
||||
_options: RunScriptOptions,
|
||||
) -> Result<ExecutionResult, ProviderError> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
async fn send_file(
|
||||
&self,
|
||||
local_file_path: &Path,
|
||||
remote_file_path: &Path,
|
||||
mode: &str,
|
||||
) -> Result<(), ProviderError> {
|
||||
if let Some(remote_parent_dir) = self.get_remote_parent_dir(remote_file_path) {
|
||||
self.create_remote_dir(&remote_parent_dir).await?;
|
||||
}
|
||||
|
||||
debug!(
|
||||
"starting sending file for {}: {} to {} with mode {}",
|
||||
self.name,
|
||||
local_file_path.to_string_lossy(),
|
||||
remote_file_path.to_string_lossy(),
|
||||
mode
|
||||
);
|
||||
|
||||
let _ = self
|
||||
.docker_client
|
||||
.container_cp(&self.container_name, local_file_path, remote_file_path)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
ProviderError::SendFile(
|
||||
local_file_path.to_string_lossy().to_string(),
|
||||
self.name.clone(),
|
||||
err.into(),
|
||||
)
|
||||
});
|
||||
|
||||
let _ = self
|
||||
.docker_client
|
||||
.container_exec(
|
||||
&self.container_name,
|
||||
vec!["chmod", mode, &remote_file_path.to_string_lossy()],
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
ProviderError::SendFile(
|
||||
self.name.clone(),
|
||||
local_file_path.to_string_lossy().to_string(),
|
||||
err.into(),
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn receive_file(
|
||||
&self,
|
||||
_remote_src: &Path,
|
||||
_local_dest: &Path,
|
||||
) -> Result<(), ProviderError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn ip(&self) -> Result<IpAddr, ProviderError> {
|
||||
let ip = self
|
||||
.docker_client
|
||||
.container_ip(&self.container_name)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
ProviderError::InvalidConfig(format!("Error getting container ip, err: {err}"))
|
||||
})?;
|
||||
|
||||
Ok(ip.parse::<IpAddr>().map_err(|err| {
|
||||
ProviderError::InvalidConfig(format!(
|
||||
"Can not parse the container ip: {ip}, err: {err}"
|
||||
))
|
||||
})?)
|
||||
}
|
||||
|
||||
async fn pause(&self) -> Result<(), ProviderError> {
|
||||
self.docker_client
|
||||
.container_exec(
|
||||
&self.container_name,
|
||||
vec!["sh", "-c", "echo pause > /tmp/zombiepipe"],
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ProviderError::PauseNodeFailed(self.name.to_string(), err.into()))?
|
||||
.map_err(|err| {
|
||||
ProviderError::PauseNodeFailed(
|
||||
self.name.to_string(),
|
||||
anyhow!("error when pausing node: status {}: {}", err.0, err.1),
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn resume(&self) -> Result<(), ProviderError> {
|
||||
self.docker_client
|
||||
.container_exec(
|
||||
&self.container_name,
|
||||
vec!["sh", "-c", "echo resume > /tmp/zombiepipe"],
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ProviderError::PauseNodeFailed(self.name.to_string(), err.into()))?
|
||||
.map_err(|err| {
|
||||
ProviderError::PauseNodeFailed(
|
||||
self.name.to_string(),
|
||||
anyhow!("error when pausing node: status {}: {}", err.0, err.1),
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn restart(&self, after: Option<Duration>) -> Result<(), ProviderError> {
|
||||
if let Some(duration) = after {
|
||||
sleep(duration).await;
|
||||
}
|
||||
|
||||
self.docker_client
|
||||
.container_exec(
|
||||
&self.container_name,
|
||||
vec!["sh", "-c", "echo restart > /tmp/zombiepipe"],
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ProviderError::PauseNodeFailed(self.name.to_string(), err.into()))?
|
||||
.map_err(|err| {
|
||||
ProviderError::PauseNodeFailed(
|
||||
self.name.to_string(),
|
||||
anyhow!("error when pausing node: status {}: {}", err.0, err.1),
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn destroy(&self) -> Result<(), ProviderError> {
|
||||
self.docker_client
|
||||
.container_rm(&self.container_name)
|
||||
.await
|
||||
.map_err(|err| ProviderError::KillNodeFailed(self.name.to_string(), err.into()))?;
|
||||
|
||||
if let Some(namespace) = self.namespace.upgrade() {
|
||||
namespace.nodes.write().await.remove(&self.name);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, Weak},
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use support::fs::FileSystem;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use super::{client::DockerClient, namespace::DockerNamespace};
|
||||
use crate::{
|
||||
shared::helpers::extract_namespace_info, types::ProviderCapabilities, DynNamespace, Provider,
|
||||
ProviderError, ProviderNamespace,
|
||||
};
|
||||
|
||||
pub const PROVIDER_NAME: &str = "docker";
|
||||
|
||||
pub struct DockerProvider<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone,
|
||||
{
|
||||
weak: Weak<DockerProvider<FS>>,
|
||||
capabilities: ProviderCapabilities,
|
||||
tmp_dir: PathBuf,
|
||||
docker_client: DockerClient,
|
||||
filesystem: FS,
|
||||
pub(super) namespaces: RwLock<HashMap<String, Arc<DockerNamespace<FS>>>>,
|
||||
}
|
||||
|
||||
impl<FS> DockerProvider<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
pub async fn new(filesystem: FS) -> Arc<Self> {
|
||||
let docker_client = DockerClient::new().await.unwrap();
|
||||
|
||||
let provider = Arc::new_cyclic(|weak| DockerProvider {
|
||||
weak: weak.clone(),
|
||||
capabilities: ProviderCapabilities {
|
||||
requires_image: true,
|
||||
has_resources: false,
|
||||
prefix_with_full_path: false,
|
||||
use_default_ports_in_cmd: true,
|
||||
},
|
||||
tmp_dir: std::env::temp_dir(),
|
||||
docker_client,
|
||||
filesystem,
|
||||
namespaces: RwLock::new(HashMap::new()),
|
||||
});
|
||||
|
||||
let cloned_provider = provider.clone();
|
||||
tokio::spawn(async move {
|
||||
tokio::signal::ctrl_c().await.unwrap();
|
||||
for (_, ns) in cloned_provider.namespaces().await {
|
||||
if ns.is_detached().await {
|
||||
// best effort
|
||||
let _ = ns.destroy().await;
|
||||
}
|
||||
}
|
||||
|
||||
// exit the process (130, SIGINT)
|
||||
std::process::exit(130)
|
||||
});
|
||||
|
||||
provider
|
||||
}
|
||||
|
||||
pub fn tmp_dir(mut self, tmp_dir: impl Into<PathBuf>) -> Self {
|
||||
self.tmp_dir = tmp_dir.into();
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<FS> Provider for DockerProvider<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
fn name(&self) -> &str {
|
||||
PROVIDER_NAME
|
||||
}
|
||||
|
||||
fn capabilities(&self) -> &ProviderCapabilities {
|
||||
&self.capabilities
|
||||
}
|
||||
|
||||
async fn namespaces(&self) -> HashMap<String, DynNamespace> {
|
||||
self.namespaces
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.map(|(name, namespace)| (name.clone(), namespace.clone() as DynNamespace))
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn create_namespace(&self) -> Result<DynNamespace, ProviderError> {
|
||||
let namespace = DockerNamespace::new(
|
||||
&self.weak,
|
||||
&self.tmp_dir,
|
||||
&self.capabilities,
|
||||
&self.docker_client,
|
||||
&self.filesystem,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
self.namespaces
|
||||
.write()
|
||||
.await
|
||||
.insert(namespace.name().to_string(), namespace.clone());
|
||||
|
||||
Ok(namespace)
|
||||
}
|
||||
|
||||
async fn create_namespace_with_base_dir(
|
||||
&self,
|
||||
base_dir: &Path,
|
||||
) -> Result<DynNamespace, ProviderError> {
|
||||
let namespace = DockerNamespace::new(
|
||||
&self.weak,
|
||||
&self.tmp_dir,
|
||||
&self.capabilities,
|
||||
&self.docker_client,
|
||||
&self.filesystem,
|
||||
Some(base_dir),
|
||||
)
|
||||
.await?;
|
||||
|
||||
self.namespaces
|
||||
.write()
|
||||
.await
|
||||
.insert(namespace.name().to_string(), namespace.clone());
|
||||
|
||||
Ok(namespace)
|
||||
}
|
||||
|
||||
async fn create_namespace_from_json(
|
||||
&self,
|
||||
json_value: &serde_json::Value,
|
||||
) -> Result<DynNamespace, ProviderError> {
|
||||
let (base_dir, name) = extract_namespace_info(json_value)?;
|
||||
|
||||
let namespace = DockerNamespace::attach_to_live(
|
||||
&self.weak,
|
||||
&self.capabilities,
|
||||
&self.docker_client,
|
||||
&self.filesystem,
|
||||
&base_dir,
|
||||
&name,
|
||||
)
|
||||
.await?;
|
||||
|
||||
self.namespaces
|
||||
.write()
|
||||
.await
|
||||
.insert(namespace.name().to_string(), namespace.clone());
|
||||
|
||||
Ok(namespace)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
mod client;
|
||||
mod namespace;
|
||||
mod node;
|
||||
mod pod_spec_builder;
|
||||
mod provider;
|
||||
|
||||
pub use provider::KubernetesProvider;
|
||||
@@ -0,0 +1,602 @@
|
||||
use std::{
|
||||
collections::BTreeMap, fmt::Debug, os::unix::process::ExitStatusExt, process::ExitStatus,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use anyhow::anyhow;
|
||||
use futures::{StreamExt, TryStreamExt};
|
||||
use k8s_openapi::api::core::v1::{
|
||||
ConfigMap, Namespace, Pod, PodSpec, PodStatus, Service, ServiceSpec,
|
||||
};
|
||||
use kube::{
|
||||
api::{AttachParams, DeleteParams, ListParams, LogParams, PostParams, WatchParams},
|
||||
core::{DynamicObject, GroupVersionKind, ObjectMeta, TypeMeta, WatchEvent},
|
||||
discovery::ApiResource,
|
||||
runtime::{conditions, wait::await_condition},
|
||||
Api, Resource,
|
||||
};
|
||||
use serde::de::DeserializeOwned;
|
||||
use support::constants::THIS_IS_A_BUG;
|
||||
use tokio::{
|
||||
io::{AsyncRead, ErrorKind},
|
||||
net::TcpListener,
|
||||
task::JoinHandle,
|
||||
};
|
||||
use tokio_util::compat::FuturesAsyncReadCompatExt;
|
||||
use tracing::{debug, trace};
|
||||
|
||||
use crate::{constants::LOCALHOST, types::ExecutionResult};
|
||||
|
||||
#[derive(thiserror::Error, Debug)]
|
||||
#[error(transparent)]
|
||||
pub struct Error(#[from] anyhow::Error);
|
||||
|
||||
pub type Result<T> = core::result::Result<T, Error>;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct KubernetesClient {
|
||||
inner: kube::Client,
|
||||
}
|
||||
|
||||
impl KubernetesClient {
|
||||
pub(super) async fn new() -> Result<Self> {
|
||||
Ok(Self {
|
||||
// TODO: make it more flexible with path to kube config
|
||||
inner: kube::Client::try_default()
|
||||
.await
|
||||
.map_err(|err| Error::from(anyhow!("error initializing kubers client: {err}")))?,
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub(super) async fn get_namespace(&self, name: &str) -> Result<Option<Namespace>> {
|
||||
Api::<Namespace>::all(self.inner.clone())
|
||||
.get_opt(name.as_ref())
|
||||
.await
|
||||
.map_err(|err| Error::from(anyhow!("error while getting namespace {name}: {err}")))
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub(super) async fn get_namespaces(&self) -> Result<Vec<Namespace>> {
|
||||
Ok(Api::<Namespace>::all(self.inner.clone())
|
||||
.list(&ListParams::default())
|
||||
.await
|
||||
.map_err(|err| Error::from(anyhow!("error while getting all namespaces: {err}")))?
|
||||
.into_iter()
|
||||
.filter(|ns| matches!(&ns.meta().name, Some(name) if name.starts_with("zombienet")))
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub(super) async fn create_namespace(
|
||||
&self,
|
||||
name: &str,
|
||||
labels: BTreeMap<String, String>,
|
||||
) -> Result<Namespace> {
|
||||
let namespaces = Api::<Namespace>::all(self.inner.clone());
|
||||
|
||||
let namespace = Namespace {
|
||||
metadata: ObjectMeta {
|
||||
name: Some(name.to_string()),
|
||||
labels: Some(labels),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
namespaces
|
||||
.create(&PostParams::default(), &namespace)
|
||||
.await
|
||||
.map_err(|err| Error::from(anyhow!("error while created namespace {name}: {err}")))?;
|
||||
|
||||
self.wait_created(namespaces, name).await?;
|
||||
|
||||
Ok(namespace)
|
||||
}
|
||||
|
||||
pub(super) async fn delete_namespace(&self, name: &str) -> Result<()> {
|
||||
let namespaces = Api::<Namespace>::all(self.inner.clone());
|
||||
|
||||
namespaces
|
||||
.delete(name, &DeleteParams::default())
|
||||
.await
|
||||
.map_err(|err| Error::from(anyhow!("error while deleting namespace {name}: {err}")))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn create_config_map_from_file(
|
||||
&self,
|
||||
namespace: &str,
|
||||
name: &str,
|
||||
file_name: &str,
|
||||
file_contents: &str,
|
||||
labels: BTreeMap<String, String>,
|
||||
) -> Result<ConfigMap> {
|
||||
let config_maps = Api::<ConfigMap>::namespaced(self.inner.clone(), namespace);
|
||||
|
||||
let config_map = ConfigMap {
|
||||
metadata: ObjectMeta {
|
||||
name: Some(name.to_string()),
|
||||
namespace: Some(namespace.to_string()),
|
||||
labels: Some(labels),
|
||||
..Default::default()
|
||||
},
|
||||
data: Some(BTreeMap::from([(
|
||||
file_name.to_string(),
|
||||
file_contents.to_string(),
|
||||
)])),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
config_maps
|
||||
.create(&PostParams::default(), &config_map)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
Error::from(anyhow!(
|
||||
"error while creating config map {name} for {file_name}: {err}"
|
||||
))
|
||||
})?;
|
||||
|
||||
self.wait_created(config_maps, name).await?;
|
||||
|
||||
Ok(config_map)
|
||||
}
|
||||
|
||||
pub(super) async fn create_pod(
|
||||
&self,
|
||||
namespace: &str,
|
||||
name: &str,
|
||||
spec: PodSpec,
|
||||
labels: BTreeMap<String, String>,
|
||||
) -> Result<Pod> {
|
||||
let pods = Api::<Pod>::namespaced(self.inner.clone(), namespace);
|
||||
|
||||
let pod = Pod {
|
||||
metadata: ObjectMeta {
|
||||
name: Some(name.to_string()),
|
||||
namespace: Some(namespace.to_string()),
|
||||
labels: Some(labels),
|
||||
..Default::default()
|
||||
},
|
||||
spec: Some(spec),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
pods.create(&PostParams::default(), &pod)
|
||||
.await
|
||||
.map_err(|err| Error::from(anyhow!("error while creating pod {name}: {err}")))?;
|
||||
|
||||
trace!("Pod {name} checking for ready state!");
|
||||
let wait_ready = await_condition(pods, name, helpers::is_pod_ready());
|
||||
// TODO: we should use the `node_spawn_timeout` from global settings here.
|
||||
let _ = tokio::time::timeout(Duration::from_secs(600), wait_ready)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
Error::from(anyhow!("error while awaiting pod {name} running: {err}"))
|
||||
})?;
|
||||
|
||||
debug!("Pod {name} is Ready!");
|
||||
Ok(pod)
|
||||
}
|
||||
|
||||
pub(super) async fn pod_logs(&self, namespace: &str, name: &str) -> Result<String> {
|
||||
Api::<Pod>::namespaced(self.inner.clone(), namespace)
|
||||
.logs(
|
||||
name,
|
||||
&LogParams {
|
||||
pretty: true,
|
||||
timestamps: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|err| Error::from(anyhow!("error while getting logs for pod {name}: {err}")))
|
||||
}
|
||||
|
||||
pub(super) async fn pod_status(&self, namespace: &str, name: &str) -> Result<PodStatus> {
|
||||
let pod = Api::<Pod>::namespaced(self.inner.clone(), namespace)
|
||||
.get(name)
|
||||
.await
|
||||
.map_err(|err| Error::from(anyhow!("error while getting pod {name}: {err}")))?;
|
||||
|
||||
let status = pod.status.ok_or(Error::from(anyhow!(
|
||||
"error while getting status for pod {name}"
|
||||
)))?;
|
||||
Ok(status)
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub(super) async fn create_pod_logs_stream(
|
||||
&self,
|
||||
namespace: &str,
|
||||
name: &str,
|
||||
) -> Result<Box<dyn AsyncRead + Send + Unpin>> {
|
||||
Ok(Box::new(
|
||||
Api::<Pod>::namespaced(self.inner.clone(), namespace)
|
||||
.log_stream(
|
||||
name,
|
||||
&LogParams {
|
||||
follow: true,
|
||||
pretty: true,
|
||||
timestamps: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
Error::from(anyhow!(
|
||||
"error while getting a log stream for {name}: {err}"
|
||||
))
|
||||
})?
|
||||
.compat(),
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) async fn pod_exec<S>(
|
||||
&self,
|
||||
namespace: &str,
|
||||
name: &str,
|
||||
command: Vec<S>,
|
||||
) -> Result<ExecutionResult>
|
||||
where
|
||||
S: Into<String> + std::fmt::Debug + Send,
|
||||
{
|
||||
trace!("running command: {command:?} on pod {name} for ns {namespace}");
|
||||
let mut process = Api::<Pod>::namespaced(self.inner.clone(), namespace)
|
||||
.exec(
|
||||
name,
|
||||
command,
|
||||
&AttachParams::default().stdout(true).stderr(true),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| Error::from(anyhow!("error while exec in the pod {name}: {err}")))?;
|
||||
|
||||
let stdout_stream = process.stdout().expect(&format!(
|
||||
"stdout shouldn't be None when true passed to exec {THIS_IS_A_BUG}"
|
||||
));
|
||||
let stdout = tokio_util::io::ReaderStream::new(stdout_stream)
|
||||
.filter_map(|r| async { r.ok().and_then(|v| String::from_utf8(v.to_vec()).ok()) })
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.join("");
|
||||
let stderr_stream = process.stderr().expect(&format!(
|
||||
"stderr shouldn't be None when true passed to exec {THIS_IS_A_BUG}"
|
||||
));
|
||||
let stderr = tokio_util::io::ReaderStream::new(stderr_stream)
|
||||
.filter_map(|r| async { r.ok().and_then(|v| String::from_utf8(v.to_vec()).ok()) })
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.join("");
|
||||
|
||||
let status = process
|
||||
.take_status()
|
||||
.expect(&format!(
|
||||
"first call to status shouldn't fail {THIS_IS_A_BUG}"
|
||||
))
|
||||
.await;
|
||||
|
||||
// await process to finish
|
||||
process.join().await.map_err(|err| {
|
||||
Error::from(anyhow!(
|
||||
"error while joining process during exec for {name}: {err}"
|
||||
))
|
||||
})?;
|
||||
|
||||
match status {
|
||||
// command succeeded with stdout
|
||||
Some(status) if status.status.as_ref().is_some_and(|s| s == "Success") => {
|
||||
Ok(Ok(stdout))
|
||||
},
|
||||
// command failed
|
||||
Some(status) if status.status.as_ref().is_some_and(|s| s == "Failure") => {
|
||||
match status.reason {
|
||||
// due to exit code
|
||||
Some(reason) if reason == "NonZeroExitCode" => {
|
||||
let exit_status = status
|
||||
.details
|
||||
.and_then(|details| {
|
||||
details.causes.and_then(|causes| {
|
||||
causes.first().and_then(|cause| {
|
||||
cause.message.as_deref().and_then(|message| {
|
||||
message.parse::<i32>().ok().map(ExitStatus::from_raw)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
.expect(
|
||||
&format!("command with non-zero exit code should have exit code present {THIS_IS_A_BUG}")
|
||||
);
|
||||
|
||||
Ok(Err((exit_status, stderr)))
|
||||
},
|
||||
// due to other unknown reason
|
||||
Some(ref reason) => Err(Error::from(anyhow!(
|
||||
format!("unhandled reason while exec for {name}: {reason}, stderr: {stderr}, status: {status:?}")
|
||||
))),
|
||||
None => {
|
||||
panic!("command had status but no reason was present, this is a bug")
|
||||
},
|
||||
}
|
||||
},
|
||||
Some(_) => {
|
||||
unreachable!("command had status but it didn't matches either Success or Failure, this is a bug from the kube.rs library itself");
|
||||
},
|
||||
None => {
|
||||
panic!("command has no status following its execution, this is a bug");
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn delete_pod(&self, namespace: &str, name: &str) -> Result<()> {
|
||||
let pods = Api::<Pod>::namespaced(self.inner.clone(), namespace);
|
||||
|
||||
pods.delete(name, &DeleteParams::default())
|
||||
.await
|
||||
.map_err(|err| Error::from(anyhow!("error when deleting pod {name}: {err}")))?;
|
||||
|
||||
await_condition(pods, name, conditions::is_deleted(name))
|
||||
.await
|
||||
.map_err(|err| {
|
||||
Error::from(anyhow!(
|
||||
"error when waiting for pod {name} to be deleted: {err}"
|
||||
))
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn create_service(
|
||||
&self,
|
||||
namespace: &str,
|
||||
name: &str,
|
||||
spec: ServiceSpec,
|
||||
labels: BTreeMap<String, String>,
|
||||
) -> Result<Service> {
|
||||
let services = Api::<Service>::namespaced(self.inner.clone(), namespace);
|
||||
|
||||
let service = Service {
|
||||
metadata: ObjectMeta {
|
||||
name: Some(name.to_string()),
|
||||
namespace: Some(namespace.to_string()),
|
||||
labels: Some(labels),
|
||||
..Default::default()
|
||||
},
|
||||
spec: Some(spec),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
services
|
||||
.create(&PostParams::default(), &service)
|
||||
.await
|
||||
.map_err(|err| Error::from(anyhow!("error while creating service {name}: {err}")))?;
|
||||
|
||||
Ok(service)
|
||||
}
|
||||
|
||||
pub(super) async fn create_pod_port_forward(
|
||||
&self,
|
||||
namespace: &str,
|
||||
name: &str,
|
||||
local_port: u16,
|
||||
remote_port: u16,
|
||||
) -> Result<(u16, JoinHandle<()>)> {
|
||||
let pods = Api::<Pod>::namespaced(self.inner.clone(), namespace);
|
||||
let bind = TcpListener::bind((LOCALHOST, local_port))
|
||||
.await
|
||||
.map_err(|err| {
|
||||
Error::from(anyhow!(
|
||||
"error binding port {local_port} for pod {name}: {err}"
|
||||
))
|
||||
})?;
|
||||
let local_port = bind.local_addr().map_err(|err| Error(err.into()))?.port();
|
||||
let name = name.to_string();
|
||||
|
||||
const MAX_FAILURES: usize = 5;
|
||||
let monitor_handle = tokio::spawn(async move {
|
||||
let mut consecutive_failures = 0;
|
||||
loop {
|
||||
let (mut client_conn, _) = match bind.accept().await {
|
||||
Ok(conn) => {
|
||||
consecutive_failures = 0;
|
||||
conn
|
||||
},
|
||||
Err(e) => {
|
||||
if consecutive_failures < MAX_FAILURES {
|
||||
trace!("Port-forward accept error: {e:?}, retrying in 1s");
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
consecutive_failures += 1;
|
||||
continue;
|
||||
} else {
|
||||
trace!("Port-forward accept failed too many times, giving up");
|
||||
break;
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
let peer = match client_conn.peer_addr() {
|
||||
Ok(addr) => addr,
|
||||
Err(e) => {
|
||||
trace!("Failed to get peer address: {e:?}");
|
||||
break;
|
||||
},
|
||||
};
|
||||
|
||||
trace!("new connection on local_port: {local_port}, peer: {peer}");
|
||||
let (name, pods) = (name.clone(), pods.clone());
|
||||
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
// Try to establish port-forward
|
||||
let mut forwarder = match pods.portforward(&name, &[remote_port]).await {
|
||||
Ok(f) => {
|
||||
consecutive_failures = 0;
|
||||
f
|
||||
},
|
||||
Err(e) => {
|
||||
consecutive_failures += 1;
|
||||
if consecutive_failures < MAX_FAILURES {
|
||||
trace!("portforward failed to establish ({}/{}): {e:?}, retrying in 1s",
|
||||
consecutive_failures, MAX_FAILURES);
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
continue;
|
||||
} else {
|
||||
trace!("portforward failed to establish after {} attempts: {e:?}, closing connection",
|
||||
consecutive_failures);
|
||||
break;
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
let mut upstream_conn = match forwarder.take_stream(remote_port) {
|
||||
Some(s) => s,
|
||||
None => {
|
||||
trace!("Failed to take stream for remote_port: {remote_port}, retrying in 1s");
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
continue;
|
||||
},
|
||||
};
|
||||
|
||||
match tokio::io::copy_bidirectional(&mut client_conn, &mut upstream_conn)
|
||||
.await
|
||||
{
|
||||
Ok((_n1, _n2)) => {
|
||||
// EOF reached, close connection
|
||||
trace!("copy_bidirectional finished (EOF), closing connection");
|
||||
|
||||
drop(upstream_conn);
|
||||
let _ = forwarder.join().await;
|
||||
|
||||
break;
|
||||
},
|
||||
Err(e) => {
|
||||
let kind = e.kind();
|
||||
match kind {
|
||||
ErrorKind::ConnectionReset
|
||||
| ErrorKind::ConnectionAborted
|
||||
| ErrorKind::ConnectionRefused
|
||||
| ErrorKind::TimedOut => {
|
||||
consecutive_failures += 1;
|
||||
if consecutive_failures < MAX_FAILURES {
|
||||
trace!("Network error ({kind:?}): {e:?}, retrying port-forward for this connection");
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
continue;
|
||||
} else {
|
||||
trace!("portforward failed to establish after {} attempts: {e:?}, closing connection",
|
||||
consecutive_failures);
|
||||
break;
|
||||
}
|
||||
},
|
||||
_ => {
|
||||
trace!("Non-network error ({kind:?}): {e:?}, closing connection");
|
||||
break;
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
trace!("finished forwarder process for local port: {local_port}, peer: {peer}");
|
||||
}
|
||||
});
|
||||
|
||||
Ok((local_port, monitor_handle))
|
||||
}
|
||||
|
||||
/// Create resources from yamls in `static-configs` directory
|
||||
pub(super) async fn create_static_resource(
|
||||
&self,
|
||||
namespace: &str,
|
||||
raw_manifest: &str,
|
||||
) -> Result<()> {
|
||||
let tm: TypeMeta = serde_yaml::from_str(raw_manifest).map_err(|err| {
|
||||
Error::from(anyhow!(
|
||||
"error while extracting TypeMeta from manifest: {raw_manifest}: {err}"
|
||||
))
|
||||
})?;
|
||||
let gvk = GroupVersionKind::try_from(&tm).map_err(|err| {
|
||||
Error::from(anyhow!(
|
||||
"error while extracting GroupVersionKind from manifest: {raw_manifest}: {err}"
|
||||
))
|
||||
})?;
|
||||
|
||||
let ar = ApiResource::from_gvk(&gvk);
|
||||
let api: Api<DynamicObject> = Api::namespaced_with(self.inner.clone(), namespace, &ar);
|
||||
|
||||
api.create(
|
||||
&PostParams::default(),
|
||||
&serde_yaml::from_str(raw_manifest).unwrap(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
Error::from(anyhow!(
|
||||
"error while creating static-config {raw_manifest}: {err}"
|
||||
))
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_created<K>(&self, api: Api<K>, name: &str) -> Result<()>
|
||||
where
|
||||
K: Clone + DeserializeOwned + Debug,
|
||||
{
|
||||
let params = &WatchParams::default().fields(&format!("metadata.name={name}"));
|
||||
let mut stream = api
|
||||
.watch(params, "0")
|
||||
.await
|
||||
.map_err(|err| {
|
||||
Error::from(anyhow!(
|
||||
"error while awaiting first response when resource {name} is created: {err}"
|
||||
))
|
||||
})?
|
||||
.boxed();
|
||||
|
||||
while let Some(status) = stream.try_next().await.map_err(|err| {
|
||||
Error::from(anyhow!(
|
||||
"error while awaiting next change after resource {name} is created: {err}"
|
||||
))
|
||||
})? {
|
||||
match status {
|
||||
WatchEvent::Added(_) => break,
|
||||
WatchEvent::Error(err) => Err(Error::from(anyhow!(
|
||||
"error while awaiting resource {name} is created: {err}"
|
||||
)))?,
|
||||
WatchEvent::Bookmark(_) => {
|
||||
// bookmark events are periodically sent as keep-alive/checkpoint, we should continue waiting
|
||||
}
|
||||
any_other_event => panic!("Unexpected event happened while creating '{name}' {THIS_IS_A_BUG}. Event: {any_other_event:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
mod helpers {
|
||||
use k8s_openapi::api::core::v1::Pod;
|
||||
use kube::runtime::wait::Condition;
|
||||
use tracing::trace;
|
||||
|
||||
/// An await condition for `Pod` that returns `true` once it is ready
|
||||
/// based on [`kube::runtime::wait::conditions::is_pod_running`]
|
||||
pub fn is_pod_ready() -> impl Condition<Pod> {
|
||||
|obj: Option<&Pod>| {
|
||||
if let Some(pod) = &obj {
|
||||
if let Some(status) = &pod.status {
|
||||
if let Some(conditions) = &status.conditions {
|
||||
let ready = conditions
|
||||
.iter()
|
||||
.any(|cond| cond.status == "True" && cond.type_ == "Ready");
|
||||
|
||||
if ready {
|
||||
trace!("{:#?}", status);
|
||||
return ready;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,600 @@
|
||||
use std::{
|
||||
collections::{BTreeMap, HashMap},
|
||||
env,
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, Weak},
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use k8s_openapi::{
|
||||
api::core::v1::{
|
||||
Container, ContainerPort, HTTPGetAction, PodSpec, Probe, ServicePort, ServiceSpec,
|
||||
},
|
||||
apimachinery::pkg::util::intstr::IntOrString,
|
||||
};
|
||||
use support::{constants::THIS_IS_A_BUG, fs::FileSystem, replacer::apply_replacements};
|
||||
use tokio::sync::{Mutex, RwLock};
|
||||
use tracing::{debug, trace, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::{client::KubernetesClient, node::KubernetesNode};
|
||||
use crate::{
|
||||
constants::NAMESPACE_PREFIX,
|
||||
kubernetes::{
|
||||
node::{DeserializableKubernetesNodeOptions, KubernetesNodeOptions},
|
||||
provider,
|
||||
},
|
||||
shared::helpers::{extract_execution_result, running_in_ci},
|
||||
types::{
|
||||
GenerateFileCommand, GenerateFilesOptions, ProviderCapabilities, RunCommandOptions,
|
||||
SpawnNodeOptions,
|
||||
},
|
||||
DynNode, KubernetesProvider, ProviderError, ProviderNamespace, ProviderNode,
|
||||
};
|
||||
|
||||
const FILE_SERVER_IMAGE: &str = "europe-west3-docker.pkg.dev/parity-zombienet/zombienet-public-images/zombienet-file-server:latest";
|
||||
|
||||
// env var used by our internal CI to pass the namespace created and ready to use
|
||||
const ZOMBIE_K8S_CI_NAMESPACE: &str = "ZOMBIE_K8S_CI_NAMESPACE";
|
||||
|
||||
pub(super) struct KubernetesNamespace<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone,
|
||||
{
|
||||
weak: Weak<KubernetesNamespace<FS>>,
|
||||
provider: Weak<KubernetesProvider<FS>>,
|
||||
name: String,
|
||||
base_dir: PathBuf,
|
||||
capabilities: ProviderCapabilities,
|
||||
k8s_client: KubernetesClient,
|
||||
filesystem: FS,
|
||||
file_server_fw_task: RwLock<Option<tokio::task::JoinHandle<()>>>,
|
||||
delete_on_drop: Arc<Mutex<bool>>,
|
||||
pub(super) file_server_port: RwLock<Option<u16>>,
|
||||
pub(super) nodes: RwLock<HashMap<String, Arc<KubernetesNode<FS>>>>,
|
||||
}
|
||||
|
||||
impl<FS> KubernetesNamespace<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
pub(super) async fn new(
|
||||
provider: &Weak<KubernetesProvider<FS>>,
|
||||
tmp_dir: &PathBuf,
|
||||
capabilities: &ProviderCapabilities,
|
||||
k8s_client: &KubernetesClient,
|
||||
filesystem: &FS,
|
||||
custom_base_dir: Option<&Path>,
|
||||
) -> Result<Arc<Self>, ProviderError> {
|
||||
// If the namespace is already provided
|
||||
let name = if let Ok(name) = env::var(ZOMBIE_K8S_CI_NAMESPACE) {
|
||||
name
|
||||
} else {
|
||||
format!("{}{}", NAMESPACE_PREFIX, Uuid::new_v4())
|
||||
};
|
||||
|
||||
let base_dir = if let Some(custom_base_dir) = custom_base_dir {
|
||||
if !filesystem.exists(custom_base_dir).await {
|
||||
filesystem.create_dir(custom_base_dir).await?;
|
||||
} else {
|
||||
warn!(
|
||||
"⚠️ Using and existing directory {} as base dir",
|
||||
custom_base_dir.to_string_lossy()
|
||||
);
|
||||
}
|
||||
PathBuf::from(custom_base_dir)
|
||||
} else {
|
||||
let base_dir = PathBuf::from_iter([tmp_dir, &PathBuf::from(&name)]);
|
||||
filesystem.create_dir(&base_dir).await?;
|
||||
base_dir
|
||||
};
|
||||
|
||||
let namespace = Arc::new_cyclic(|weak| KubernetesNamespace {
|
||||
weak: weak.clone(),
|
||||
provider: provider.clone(),
|
||||
name,
|
||||
base_dir,
|
||||
capabilities: capabilities.clone(),
|
||||
filesystem: filesystem.clone(),
|
||||
k8s_client: k8s_client.clone(),
|
||||
file_server_port: RwLock::new(None),
|
||||
file_server_fw_task: RwLock::new(None),
|
||||
nodes: RwLock::new(HashMap::new()),
|
||||
delete_on_drop: Arc::new(Mutex::new(true)),
|
||||
});
|
||||
|
||||
namespace.initialize().await?;
|
||||
|
||||
Ok(namespace)
|
||||
}
|
||||
|
||||
pub(super) async fn attach_to_live(
|
||||
provider: &Weak<KubernetesProvider<FS>>,
|
||||
capabilities: &ProviderCapabilities,
|
||||
k8s_client: &KubernetesClient,
|
||||
filesystem: &FS,
|
||||
custom_base_dir: &Path,
|
||||
name: &str,
|
||||
) -> Result<Arc<Self>, ProviderError> {
|
||||
let base_dir = custom_base_dir.to_path_buf();
|
||||
|
||||
let namespace = Arc::new_cyclic(|weak| KubernetesNamespace {
|
||||
weak: weak.clone(),
|
||||
provider: provider.clone(),
|
||||
name: name.to_owned(),
|
||||
base_dir,
|
||||
capabilities: capabilities.clone(),
|
||||
filesystem: filesystem.clone(),
|
||||
k8s_client: k8s_client.clone(),
|
||||
file_server_port: RwLock::new(None),
|
||||
file_server_fw_task: RwLock::new(None),
|
||||
nodes: RwLock::new(HashMap::new()),
|
||||
delete_on_drop: Arc::new(Mutex::new(false)),
|
||||
});
|
||||
|
||||
namespace.setup_file_server_port_fwd("fileserver").await?;
|
||||
|
||||
Ok(namespace)
|
||||
}
|
||||
|
||||
async fn initialize(&self) -> Result<(), ProviderError> {
|
||||
// Initialize the namespace IFF
|
||||
// we are not in CI or we don't have the env `ZOMBIE_NAMESPACE` set
|
||||
if env::var(ZOMBIE_K8S_CI_NAMESPACE).is_err() || !running_in_ci() {
|
||||
self.initialize_k8s().await?;
|
||||
}
|
||||
|
||||
// Ensure namespace isolation and minimal resources IFF we are running in CI
|
||||
if running_in_ci() {
|
||||
self.initialize_static_resources().await?
|
||||
}
|
||||
|
||||
self.initialize_file_server().await?;
|
||||
|
||||
self.setup_script_config_map(
|
||||
"zombie-wrapper",
|
||||
include_str!("../shared/scripts/zombie-wrapper.sh"),
|
||||
"zombie_wrapper_config_map_manifest.yaml",
|
||||
// TODO: add correct labels
|
||||
BTreeMap::new(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
self.setup_script_config_map(
|
||||
"helper-binaries-downloader",
|
||||
include_str!("../shared/scripts/helper-binaries-downloader.sh"),
|
||||
"helper_binaries_downloader_config_map_manifest.yaml",
|
||||
// TODO: add correct labels
|
||||
BTreeMap::new(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn initialize_k8s(&self) -> Result<(), ProviderError> {
|
||||
// TODO (javier): check with Hamid if we are using this labels in any scheduling logic.
|
||||
let labels = BTreeMap::from([
|
||||
(
|
||||
"jobId".to_string(),
|
||||
env::var("CI_JOB_ID").unwrap_or("".to_string()),
|
||||
),
|
||||
(
|
||||
"projectName".to_string(),
|
||||
env::var("CI_PROJECT_NAME").unwrap_or("".to_string()),
|
||||
),
|
||||
(
|
||||
"projectId".to_string(),
|
||||
env::var("CI_PROJECT_ID").unwrap_or("".to_string()),
|
||||
),
|
||||
]);
|
||||
|
||||
let manifest = self
|
||||
.k8s_client
|
||||
.create_namespace(&self.name, labels)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
ProviderError::CreateNamespaceFailed(self.name.to_string(), err.into())
|
||||
})?;
|
||||
|
||||
let serialized_manifest = serde_yaml::to_string(&manifest).map_err(|err| {
|
||||
ProviderError::CreateNamespaceFailed(self.name.to_string(), err.into())
|
||||
})?;
|
||||
|
||||
let dest_path =
|
||||
PathBuf::from_iter([&self.base_dir, &PathBuf::from("namespace_manifest.yaml")]);
|
||||
|
||||
self.filesystem
|
||||
.write(dest_path, serialized_manifest)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn initialize_static_resources(&self) -> Result<(), ProviderError> {
|
||||
let np_manifest = apply_replacements(
|
||||
include_str!("./static-configs/namespace-network-policy.yaml"),
|
||||
&HashMap::from([("namespace", self.name())]),
|
||||
);
|
||||
|
||||
// Apply NetworkPolicy manifest
|
||||
self.k8s_client
|
||||
.create_static_resource(&self.name, &np_manifest)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
ProviderError::CreateNamespaceFailed(self.name.to_string(), err.into())
|
||||
})?;
|
||||
|
||||
// Apply LimitRange manifest
|
||||
self.k8s_client
|
||||
.create_static_resource(
|
||||
&self.name,
|
||||
include_str!("./static-configs/baseline-resources.yaml"),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
ProviderError::CreateNamespaceFailed(self.name.to_string(), err.into())
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn initialize_file_server(&self) -> Result<(), ProviderError> {
|
||||
let name = "fileserver".to_string();
|
||||
let labels = BTreeMap::from([
|
||||
("app.kubernetes.io/name".to_string(), name.clone()),
|
||||
(
|
||||
"x-infra-instance".to_string(),
|
||||
env::var("X_INFRA_INSTANCE").unwrap_or("ondemand".to_string()),
|
||||
),
|
||||
]);
|
||||
|
||||
let pod_spec = PodSpec {
|
||||
hostname: Some(name.clone()),
|
||||
containers: vec![Container {
|
||||
name: name.clone(),
|
||||
image: Some(FILE_SERVER_IMAGE.to_string()),
|
||||
image_pull_policy: Some("Always".to_string()),
|
||||
ports: Some(vec![ContainerPort {
|
||||
container_port: 80,
|
||||
..Default::default()
|
||||
}]),
|
||||
startup_probe: Some(Probe {
|
||||
http_get: Some(HTTPGetAction {
|
||||
path: Some("/".to_string()),
|
||||
port: IntOrString::Int(80),
|
||||
..Default::default()
|
||||
}),
|
||||
initial_delay_seconds: Some(1),
|
||||
period_seconds: Some(2),
|
||||
failure_threshold: Some(3),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
}],
|
||||
restart_policy: Some("OnFailure".into()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let pod_manifest = self
|
||||
.k8s_client
|
||||
.create_pod(&self.name, &name, pod_spec, labels.clone())
|
||||
.await
|
||||
.map_err(|err| ProviderError::FileServerSetupError(err.into()))?;
|
||||
|
||||
// TODO: remove duplication across methods
|
||||
let pod_serialized_manifest = serde_yaml::to_string(&pod_manifest)
|
||||
.map_err(|err| ProviderError::FileServerSetupError(err.into()))?;
|
||||
|
||||
let pod_dest_path = PathBuf::from_iter([
|
||||
&self.base_dir,
|
||||
&PathBuf::from("file_server_pod_manifest.yaml"),
|
||||
]);
|
||||
|
||||
self.filesystem
|
||||
.write(pod_dest_path, pod_serialized_manifest)
|
||||
.await?;
|
||||
|
||||
let service_spec = ServiceSpec {
|
||||
selector: Some(labels.clone()),
|
||||
ports: Some(vec![ServicePort {
|
||||
port: 80,
|
||||
..Default::default()
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let service_manifest = self
|
||||
.k8s_client
|
||||
.create_service(&self.name, &name, service_spec, labels)
|
||||
.await
|
||||
.map_err(|err| ProviderError::FileServerSetupError(err.into()))?;
|
||||
|
||||
let serialized_service_manifest = serde_yaml::to_string(&service_manifest)
|
||||
.map_err(|err| ProviderError::FileServerSetupError(err.into()))?;
|
||||
|
||||
let service_dest_path = PathBuf::from_iter([
|
||||
&self.base_dir,
|
||||
&PathBuf::from("file_server_service_manifest.yaml"),
|
||||
]);
|
||||
|
||||
self.filesystem
|
||||
.write(service_dest_path, serialized_service_manifest)
|
||||
.await?;
|
||||
|
||||
self.setup_file_server_port_fwd(&name).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn setup_file_server_port_fwd(&self, name: &str) -> Result<(), ProviderError> {
|
||||
let (port, task) = self
|
||||
.k8s_client
|
||||
.create_pod_port_forward(&self.name, name, 0, 80)
|
||||
.await
|
||||
.map_err(|err| ProviderError::FileServerSetupError(err.into()))?;
|
||||
|
||||
*self.file_server_port.write().await = Some(port);
|
||||
*self.file_server_fw_task.write().await = Some(task);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn setup_script_config_map(
|
||||
&self,
|
||||
name: &str,
|
||||
script_contents: &str,
|
||||
local_manifest_name: &str,
|
||||
labels: BTreeMap<String, String>,
|
||||
) -> Result<(), ProviderError> {
|
||||
let manifest = self
|
||||
.k8s_client
|
||||
.create_config_map_from_file(
|
||||
&self.name,
|
||||
name,
|
||||
&format!("{name}.sh"),
|
||||
script_contents,
|
||||
labels,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
ProviderError::CreateNamespaceFailed(self.name.to_string(), err.into())
|
||||
})?;
|
||||
|
||||
let serializer_manifest = serde_yaml::to_string(&manifest).map_err(|err| {
|
||||
ProviderError::CreateNamespaceFailed(self.name.to_string(), err.into())
|
||||
})?;
|
||||
|
||||
let dest_path = PathBuf::from_iter([&self.base_dir, &PathBuf::from(local_manifest_name)]);
|
||||
|
||||
self.filesystem
|
||||
.write(dest_path, serializer_manifest)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn set_delete_on_drop(&self, delete_on_drop: bool) {
|
||||
*self.delete_on_drop.lock().await = delete_on_drop;
|
||||
}
|
||||
|
||||
pub async fn delete_on_drop(&self) -> bool {
|
||||
if let Ok(delete_on_drop) = self.delete_on_drop.try_lock() {
|
||||
*delete_on_drop
|
||||
} else {
|
||||
// if we can't lock just remove the ns
|
||||
true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<FS> Drop for KubernetesNamespace<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone,
|
||||
{
|
||||
fn drop(&mut self) {
|
||||
let ns_name = self.name.clone();
|
||||
if let Ok(delete_on_drop) = self.delete_on_drop.try_lock() {
|
||||
if *delete_on_drop {
|
||||
let client = self.k8s_client.clone();
|
||||
let provider = self.provider.upgrade();
|
||||
futures::executor::block_on(async move {
|
||||
trace!("🧟 deleting ns {ns_name} from cluster");
|
||||
let _ = client.delete_namespace(&ns_name).await;
|
||||
if let Some(provider) = provider {
|
||||
provider.namespaces.write().await.remove(&ns_name);
|
||||
}
|
||||
|
||||
trace!("✅ deleted");
|
||||
});
|
||||
} else {
|
||||
trace!("⚠️ leaking ns {ns_name} in cluster");
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<FS> ProviderNamespace for KubernetesNamespace<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
fn name(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
|
||||
fn base_dir(&self) -> &PathBuf {
|
||||
&self.base_dir
|
||||
}
|
||||
|
||||
fn capabilities(&self) -> &ProviderCapabilities {
|
||||
&self.capabilities
|
||||
}
|
||||
|
||||
fn provider_name(&self) -> &str {
|
||||
provider::PROVIDER_NAME
|
||||
}
|
||||
|
||||
async fn detach(&self) {
|
||||
self.set_delete_on_drop(false).await;
|
||||
}
|
||||
|
||||
async fn is_detached(&self) -> bool {
|
||||
self.delete_on_drop().await
|
||||
}
|
||||
|
||||
async fn nodes(&self) -> HashMap<String, DynNode> {
|
||||
self.nodes
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.map(|(name, node)| (name.clone(), node.clone() as DynNode))
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn get_node_available_args(
|
||||
&self,
|
||||
(command, image): (String, Option<String>),
|
||||
) -> Result<String, ProviderError> {
|
||||
let node_image = image.expect(&format!("image should be present when getting node available args with kubernetes provider {THIS_IS_A_BUG}"));
|
||||
|
||||
// run dummy command in new pod
|
||||
let temp_node = self
|
||||
.spawn_node(
|
||||
&SpawnNodeOptions::new(format!("temp-{}", Uuid::new_v4()), "cat".to_string())
|
||||
.image(node_image.clone()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let available_args_output = temp_node
|
||||
.run_command(RunCommandOptions::new(command.clone()).args(vec!["--help"]))
|
||||
.await?
|
||||
.map_err(|(_exit, status)| {
|
||||
ProviderError::NodeAvailableArgsError(node_image, command, status)
|
||||
})?;
|
||||
|
||||
temp_node.destroy().await?;
|
||||
|
||||
Ok(available_args_output)
|
||||
}
|
||||
|
||||
async fn spawn_node(&self, options: &SpawnNodeOptions) -> Result<DynNode, ProviderError> {
|
||||
trace!("spawn node options {options:?}");
|
||||
|
||||
let node = KubernetesNode::new(KubernetesNodeOptions {
|
||||
namespace: &self.weak,
|
||||
namespace_base_dir: &self.base_dir,
|
||||
name: &options.name,
|
||||
image: options.image.as_ref(),
|
||||
program: &options.program,
|
||||
args: &options.args,
|
||||
env: &options.env,
|
||||
startup_files: &options.injected_files,
|
||||
resources: options.resources.as_ref(),
|
||||
db_snapshot: options.db_snapshot.as_ref(),
|
||||
k8s_client: &self.k8s_client,
|
||||
filesystem: &self.filesystem,
|
||||
})
|
||||
.await?;
|
||||
|
||||
self.nodes
|
||||
.write()
|
||||
.await
|
||||
.insert(node.name().to_string(), node.clone());
|
||||
|
||||
Ok(node)
|
||||
}
|
||||
|
||||
async fn spawn_node_from_json(
|
||||
&self,
|
||||
json_value: &serde_json::Value,
|
||||
) -> Result<DynNode, ProviderError> {
|
||||
let deserializable: DeserializableKubernetesNodeOptions =
|
||||
serde_json::from_value(json_value.clone())?;
|
||||
let options = KubernetesNodeOptions::from_deserializable(
|
||||
&deserializable,
|
||||
&self.weak,
|
||||
&self.base_dir,
|
||||
&self.k8s_client,
|
||||
&self.filesystem,
|
||||
);
|
||||
|
||||
let node = KubernetesNode::attach_to_live(options).await?;
|
||||
|
||||
self.nodes
|
||||
.write()
|
||||
.await
|
||||
.insert(node.name().to_string(), node.clone());
|
||||
|
||||
Ok(node)
|
||||
}
|
||||
|
||||
async fn generate_files(&self, options: GenerateFilesOptions) -> Result<(), ProviderError> {
|
||||
debug!("generate files options {options:#?}");
|
||||
|
||||
let node_name = options
|
||||
.temp_name
|
||||
.unwrap_or_else(|| format!("temp-{}", Uuid::new_v4()));
|
||||
let node_image = options
|
||||
.image
|
||||
.expect(&format!("image should be present when generating files with kubernetes provider {THIS_IS_A_BUG}"));
|
||||
|
||||
// run dummy command in new pod
|
||||
let temp_node = self
|
||||
.spawn_node(
|
||||
&SpawnNodeOptions::new(node_name, "cat".to_string())
|
||||
.injected_files(options.injected_files)
|
||||
.image(node_image),
|
||||
)
|
||||
.await?;
|
||||
|
||||
for GenerateFileCommand {
|
||||
program,
|
||||
args,
|
||||
env,
|
||||
local_output_path,
|
||||
} in options.commands
|
||||
{
|
||||
let local_output_full_path = format!(
|
||||
"{}{}{}",
|
||||
self.base_dir.to_string_lossy(),
|
||||
if local_output_path.starts_with("/") {
|
||||
""
|
||||
} else {
|
||||
"/"
|
||||
},
|
||||
local_output_path.to_string_lossy()
|
||||
);
|
||||
|
||||
let contents = extract_execution_result(
|
||||
&temp_node,
|
||||
RunCommandOptions { program, args, env },
|
||||
options.expected_path.as_ref(),
|
||||
)
|
||||
.await?;
|
||||
self.filesystem
|
||||
.write(local_output_full_path, contents)
|
||||
.await
|
||||
.map_err(|err| ProviderError::FileGenerationFailed(err.into()))?;
|
||||
}
|
||||
|
||||
temp_node.destroy().await
|
||||
}
|
||||
|
||||
async fn static_setup(&self) -> Result<(), ProviderError> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
async fn destroy(&self) -> Result<(), ProviderError> {
|
||||
let _ = self
|
||||
.k8s_client
|
||||
.delete_namespace(&self.name)
|
||||
.await
|
||||
.map_err(|err| ProviderError::DeleteNamespaceFailed(self.name.clone(), err.into()))?;
|
||||
|
||||
if let Some(provider) = self.provider.upgrade() {
|
||||
provider.namespaces.write().await.remove(&self.name);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,886 @@
|
||||
use std::{
|
||||
collections::{BTreeMap, HashMap},
|
||||
env,
|
||||
net::IpAddr,
|
||||
path::{Component, Path, PathBuf},
|
||||
sync::{Arc, Weak},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use anyhow::anyhow;
|
||||
use async_trait::async_trait;
|
||||
use configuration::{shared::resources::Resources, types::AssetLocation};
|
||||
use futures::future::try_join_all;
|
||||
use k8s_openapi::api::core::v1::{ServicePort, ServiceSpec};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::Digest;
|
||||
use support::{constants::THIS_IS_A_BUG, fs::FileSystem};
|
||||
use tokio::{sync::RwLock, task::JoinHandle, time::sleep, try_join};
|
||||
use tracing::{debug, trace, warn};
|
||||
use url::Url;
|
||||
|
||||
use super::{
|
||||
client::KubernetesClient, namespace::KubernetesNamespace, pod_spec_builder::PodSpecBuilder,
|
||||
};
|
||||
use crate::{
|
||||
constants::{
|
||||
NODE_CONFIG_DIR, NODE_DATA_DIR, NODE_RELAY_DATA_DIR, NODE_SCRIPTS_DIR, P2P_PORT,
|
||||
PROMETHEUS_PORT, RPC_HTTP_PORT, RPC_WS_PORT,
|
||||
},
|
||||
kubernetes,
|
||||
types::{ExecutionResult, RunCommandOptions, RunScriptOptions, TransferedFile},
|
||||
ProviderError, ProviderNamespace, ProviderNode,
|
||||
};
|
||||
|
||||
pub(super) struct KubernetesNodeOptions<'a, FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
pub(super) namespace: &'a Weak<KubernetesNamespace<FS>>,
|
||||
pub(super) namespace_base_dir: &'a PathBuf,
|
||||
pub(super) name: &'a str,
|
||||
pub(super) image: Option<&'a String>,
|
||||
pub(super) program: &'a str,
|
||||
pub(super) args: &'a [String],
|
||||
pub(super) env: &'a [(String, String)],
|
||||
pub(super) startup_files: &'a [TransferedFile],
|
||||
pub(super) resources: Option<&'a Resources>,
|
||||
pub(super) db_snapshot: Option<&'a AssetLocation>,
|
||||
pub(super) k8s_client: &'a KubernetesClient,
|
||||
pub(super) filesystem: &'a FS,
|
||||
}
|
||||
|
||||
impl<'a, FS> KubernetesNodeOptions<'a, FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
pub(super) fn from_deserializable(
|
||||
deserializable: &'a DeserializableKubernetesNodeOptions,
|
||||
namespace: &'a Weak<KubernetesNamespace<FS>>,
|
||||
namespace_base_dir: &'a PathBuf,
|
||||
k8s_client: &'a KubernetesClient,
|
||||
filesystem: &'a FS,
|
||||
) -> KubernetesNodeOptions<'a, FS> {
|
||||
KubernetesNodeOptions {
|
||||
namespace,
|
||||
namespace_base_dir,
|
||||
name: &deserializable.name,
|
||||
image: deserializable.image.as_ref(),
|
||||
program: &deserializable.program,
|
||||
args: &deserializable.args,
|
||||
env: &deserializable.env,
|
||||
startup_files: &[],
|
||||
resources: deserializable.resources.as_ref(),
|
||||
db_snapshot: None,
|
||||
k8s_client,
|
||||
filesystem,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub(super) struct DeserializableKubernetesNodeOptions {
|
||||
pub(super) name: String,
|
||||
pub(super) image: Option<String>,
|
||||
pub(super) program: String,
|
||||
pub(super) args: Vec<String>,
|
||||
pub(super) env: Vec<(String, String)>,
|
||||
pub(super) resources: Option<Resources>,
|
||||
}
|
||||
|
||||
type FwdInfo = (u16, JoinHandle<()>);
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct KubernetesNode<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone,
|
||||
{
|
||||
#[serde(skip)]
|
||||
namespace: Weak<KubernetesNamespace<FS>>,
|
||||
name: String,
|
||||
image: String,
|
||||
program: String,
|
||||
args: Vec<String>,
|
||||
env: Vec<(String, String)>,
|
||||
resources: Option<Resources>,
|
||||
base_dir: PathBuf,
|
||||
config_dir: PathBuf,
|
||||
data_dir: PathBuf,
|
||||
relay_data_dir: PathBuf,
|
||||
scripts_dir: PathBuf,
|
||||
log_path: PathBuf,
|
||||
#[serde(skip)]
|
||||
k8s_client: KubernetesClient,
|
||||
#[serde(skip)]
|
||||
http_client: reqwest::Client,
|
||||
#[serde(skip)]
|
||||
filesystem: FS,
|
||||
#[serde(skip)]
|
||||
port_fwds: RwLock<HashMap<u16, FwdInfo>>,
|
||||
provider_tag: String,
|
||||
}
|
||||
|
||||
impl<FS> KubernetesNode<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
pub(super) async fn new(
|
||||
options: KubernetesNodeOptions<'_, FS>,
|
||||
) -> Result<Arc<Self>, ProviderError> {
|
||||
let image = options.image.ok_or_else(|| {
|
||||
ProviderError::MissingNodeInfo(options.name.to_string(), "missing image".to_string())
|
||||
})?;
|
||||
|
||||
let filesystem = options.filesystem.clone();
|
||||
|
||||
let base_dir =
|
||||
PathBuf::from_iter([options.namespace_base_dir, &PathBuf::from(options.name)]);
|
||||
filesystem.create_dir_all(&base_dir).await?;
|
||||
|
||||
let base_dir_raw = base_dir.to_string_lossy();
|
||||
let config_dir = PathBuf::from(format!("{base_dir_raw}{NODE_CONFIG_DIR}"));
|
||||
let data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_DATA_DIR}"));
|
||||
let relay_data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_RELAY_DATA_DIR}"));
|
||||
let scripts_dir = PathBuf::from(format!("{base_dir_raw}{NODE_SCRIPTS_DIR}"));
|
||||
let log_path = base_dir.join("node.log");
|
||||
|
||||
try_join!(
|
||||
filesystem.create_dir(&config_dir),
|
||||
filesystem.create_dir(&data_dir),
|
||||
filesystem.create_dir(&relay_data_dir),
|
||||
filesystem.create_dir(&scripts_dir),
|
||||
)?;
|
||||
|
||||
let node = Arc::new(KubernetesNode {
|
||||
namespace: options.namespace.clone(),
|
||||
name: options.name.to_string(),
|
||||
image: image.to_string(),
|
||||
program: options.program.to_string(),
|
||||
args: options.args.to_vec(),
|
||||
env: options.env.to_vec(),
|
||||
resources: options.resources.cloned(),
|
||||
base_dir,
|
||||
config_dir,
|
||||
data_dir,
|
||||
relay_data_dir,
|
||||
scripts_dir,
|
||||
log_path,
|
||||
filesystem: filesystem.clone(),
|
||||
k8s_client: options.k8s_client.clone(),
|
||||
http_client: reqwest::Client::new(),
|
||||
port_fwds: Default::default(),
|
||||
provider_tag: kubernetes::provider::PROVIDER_NAME.to_string(),
|
||||
});
|
||||
|
||||
node.initialize_k8s().await?;
|
||||
|
||||
if let Some(db_snap) = options.db_snapshot {
|
||||
node.initialize_db_snapshot(db_snap).await?;
|
||||
}
|
||||
|
||||
node.initialize_startup_files(options.startup_files).await?;
|
||||
|
||||
node.start().await?;
|
||||
|
||||
Ok(node)
|
||||
}
|
||||
|
||||
pub(super) async fn attach_to_live(
|
||||
options: KubernetesNodeOptions<'_, FS>,
|
||||
) -> Result<Arc<Self>, ProviderError> {
|
||||
let image = options.image.ok_or_else(|| {
|
||||
ProviderError::MissingNodeInfo(options.name.to_string(), "missing image".to_string())
|
||||
})?;
|
||||
|
||||
let filesystem = options.filesystem.clone();
|
||||
|
||||
let base_dir =
|
||||
PathBuf::from_iter([options.namespace_base_dir, &PathBuf::from(options.name)]);
|
||||
filesystem.create_dir_all(&base_dir).await?;
|
||||
|
||||
let base_dir_raw = base_dir.to_string_lossy();
|
||||
let config_dir = PathBuf::from(format!("{base_dir_raw}{NODE_CONFIG_DIR}"));
|
||||
let data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_DATA_DIR}"));
|
||||
let relay_data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_RELAY_DATA_DIR}"));
|
||||
let scripts_dir = PathBuf::from(format!("{base_dir_raw}{NODE_SCRIPTS_DIR}"));
|
||||
let log_path = base_dir.join("node.log");
|
||||
|
||||
let node = Arc::new(KubernetesNode {
|
||||
namespace: options.namespace.clone(),
|
||||
name: options.name.to_string(),
|
||||
image: image.to_string(),
|
||||
program: options.program.to_string(),
|
||||
args: options.args.to_vec(),
|
||||
env: options.env.to_vec(),
|
||||
resources: options.resources.cloned(),
|
||||
base_dir,
|
||||
config_dir,
|
||||
data_dir,
|
||||
relay_data_dir,
|
||||
scripts_dir,
|
||||
log_path,
|
||||
filesystem: filesystem.clone(),
|
||||
k8s_client: options.k8s_client.clone(),
|
||||
http_client: reqwest::Client::new(),
|
||||
port_fwds: Default::default(),
|
||||
provider_tag: kubernetes::provider::PROVIDER_NAME.to_string(),
|
||||
});
|
||||
|
||||
Ok(node)
|
||||
}
|
||||
|
||||
async fn initialize_k8s(&self) -> Result<(), ProviderError> {
|
||||
let labels = BTreeMap::from([
|
||||
(
|
||||
"app.kubernetes.io/name".to_string(),
|
||||
self.name().to_string(),
|
||||
),
|
||||
(
|
||||
"x-infra-instance".to_string(),
|
||||
env::var("X_INFRA_INSTANCE").unwrap_or("ondemand".to_string()),
|
||||
),
|
||||
]);
|
||||
|
||||
// Create pod
|
||||
let pod_spec = PodSpecBuilder::build(
|
||||
&self.name,
|
||||
&self.image,
|
||||
self.resources.as_ref(),
|
||||
&self.program,
|
||||
&self.args,
|
||||
&self.env,
|
||||
);
|
||||
|
||||
let manifest = self
|
||||
.k8s_client
|
||||
.create_pod(&self.namespace_name(), &self.name, pod_spec, labels.clone())
|
||||
.await
|
||||
.map_err(|err| ProviderError::NodeSpawningFailed(self.name.clone(), err.into()))?;
|
||||
|
||||
let serialized_manifest = serde_yaml::to_string(&manifest)
|
||||
.map_err(|err| ProviderError::NodeSpawningFailed(self.name.to_string(), err.into()))?;
|
||||
|
||||
let dest_path = PathBuf::from_iter([
|
||||
&self.base_dir,
|
||||
&PathBuf::from(format!("{}_manifest.yaml", &self.name)),
|
||||
]);
|
||||
|
||||
self.filesystem
|
||||
.write(dest_path, serialized_manifest)
|
||||
.await
|
||||
.map_err(|err| ProviderError::NodeSpawningFailed(self.name.to_string(), err.into()))?;
|
||||
|
||||
// Create service for pod
|
||||
let service_spec = ServiceSpec {
|
||||
selector: Some(labels.clone()),
|
||||
ports: Some(vec![
|
||||
ServicePort {
|
||||
port: P2P_PORT.into(),
|
||||
name: Some("p2p".into()),
|
||||
..Default::default()
|
||||
},
|
||||
ServicePort {
|
||||
port: RPC_WS_PORT.into(),
|
||||
name: Some("rpc".into()),
|
||||
..Default::default()
|
||||
},
|
||||
ServicePort {
|
||||
port: RPC_HTTP_PORT.into(),
|
||||
name: Some("rpc-http".into()),
|
||||
..Default::default()
|
||||
},
|
||||
ServicePort {
|
||||
port: PROMETHEUS_PORT.into(),
|
||||
name: Some("prom".into()),
|
||||
..Default::default()
|
||||
},
|
||||
]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let service_manifest = self
|
||||
.k8s_client
|
||||
.create_service(&self.namespace_name(), &self.name, service_spec, labels)
|
||||
.await
|
||||
.map_err(|err| ProviderError::FileServerSetupError(err.into()))?;
|
||||
|
||||
let serialized_service_manifest = serde_yaml::to_string(&service_manifest)
|
||||
.map_err(|err| ProviderError::FileServerSetupError(err.into()))?;
|
||||
|
||||
let service_dest_path = PathBuf::from_iter([
|
||||
&self.base_dir,
|
||||
&PathBuf::from(format!("{}_svc_manifest.yaml", &self.name)),
|
||||
]);
|
||||
|
||||
self.filesystem
|
||||
.write(service_dest_path, serialized_service_manifest)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn initialize_db_snapshot(
|
||||
&self,
|
||||
db_snapshot: &AssetLocation,
|
||||
) -> Result<(), ProviderError> {
|
||||
trace!("snap: {db_snapshot}");
|
||||
let url_of_snap = match db_snapshot {
|
||||
AssetLocation::Url(location) => location.clone(),
|
||||
AssetLocation::FilePath(filepath) => {
|
||||
let (url, _) = self.upload_to_fileserver(filepath).await?;
|
||||
url
|
||||
},
|
||||
};
|
||||
|
||||
// we need to get the snapshot from a public access
|
||||
// and extract to /data
|
||||
let opts = RunCommandOptions::new("mkdir").args([
|
||||
"-p",
|
||||
"/data/",
|
||||
"&&",
|
||||
"mkdir",
|
||||
"-p",
|
||||
"/relay-data/",
|
||||
"&&",
|
||||
// Use our version of curl
|
||||
"/cfg/curl",
|
||||
url_of_snap.as_ref(),
|
||||
"--output",
|
||||
"/data/db.tgz",
|
||||
"&&",
|
||||
"cd",
|
||||
"/",
|
||||
"&&",
|
||||
"tar",
|
||||
"--skip-old-files",
|
||||
"-xzvf",
|
||||
"/data/db.tgz",
|
||||
]);
|
||||
|
||||
trace!("cmd opts: {:#?}", opts);
|
||||
let _ = self.run_command(opts).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn initialize_startup_files(
|
||||
&self,
|
||||
startup_files: &[TransferedFile],
|
||||
) -> Result<(), ProviderError> {
|
||||
try_join_all(
|
||||
startup_files
|
||||
.iter()
|
||||
.map(|file| self.send_file(&file.local_path, &file.remote_path, &file.mode)),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn start(&self) -> Result<(), ProviderError> {
|
||||
self.k8s_client
|
||||
.pod_exec(
|
||||
&self.namespace_name(),
|
||||
&self.name,
|
||||
vec!["sh", "-c", "echo start > /tmp/zombiepipe"],
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
ProviderError::NodeSpawningFailed(
|
||||
format!("failed to start pod {} after spawning", self.name),
|
||||
err.into(),
|
||||
)
|
||||
})?
|
||||
.map_err(|err| {
|
||||
ProviderError::NodeSpawningFailed(
|
||||
format!("failed to start pod {} after spawning", self.name,),
|
||||
anyhow!("command failed in container: status {}: {}", err.0, err.1),
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_remote_parent_dir(&self, remote_file_path: &Path) -> Option<PathBuf> {
|
||||
if let Some(remote_parent_dir) = remote_file_path.parent() {
|
||||
if matches!(
|
||||
remote_parent_dir.components().rev().peekable().peek(),
|
||||
Some(Component::Normal(_))
|
||||
) {
|
||||
return Some(remote_parent_dir.to_path_buf());
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
async fn create_remote_dir(&self, remote_dir: &Path) -> Result<(), ProviderError> {
|
||||
let _ = self
|
||||
.k8s_client
|
||||
.pod_exec(
|
||||
&self.namespace_name(),
|
||||
&self.name,
|
||||
vec!["mkdir", "-p", &remote_dir.to_string_lossy()],
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
ProviderError::NodeSpawningFailed(
|
||||
format!(
|
||||
"failed to create dir {} for pod {}",
|
||||
remote_dir.to_string_lossy(),
|
||||
&self.name
|
||||
),
|
||||
err.into(),
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn namespace_name(&self) -> String {
|
||||
self.namespace
|
||||
.upgrade()
|
||||
.map(|namespace| namespace.name().to_string())
|
||||
.unwrap_or_else(|| panic!("namespace shouldn't be dropped, {THIS_IS_A_BUG}"))
|
||||
}
|
||||
|
||||
async fn upload_to_fileserver(&self, location: &Path) -> Result<(Url, String), ProviderError> {
|
||||
let file_name = if let Some(name) = location.file_name() {
|
||||
name.to_string_lossy()
|
||||
} else {
|
||||
"unnamed".into()
|
||||
};
|
||||
|
||||
let data = self.filesystem.read(location).await?;
|
||||
let content_hashed = hex::encode(sha2::Sha256::digest(&data));
|
||||
let req = self
|
||||
.http_client
|
||||
.head(format!(
|
||||
"http://{}/{content_hashed}__{file_name}",
|
||||
self.file_server_local_host().await?
|
||||
))
|
||||
.build()
|
||||
.map_err(|err| {
|
||||
ProviderError::UploadFile(location.to_string_lossy().to_string(), err.into())
|
||||
})?;
|
||||
|
||||
let url = req.url().clone();
|
||||
let res = self.http_client.execute(req).await.map_err(|err| {
|
||||
ProviderError::UploadFile(location.to_string_lossy().to_string(), err.into())
|
||||
})?;
|
||||
|
||||
if res.status() != reqwest::StatusCode::OK {
|
||||
// we need to upload the file
|
||||
self.http_client
|
||||
.post(url.as_ref())
|
||||
.body(data)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| {
|
||||
ProviderError::UploadFile(location.to_string_lossy().to_string(), err.into())
|
||||
})?;
|
||||
}
|
||||
|
||||
Ok((url, content_hashed))
|
||||
}
|
||||
|
||||
async fn file_server_local_host(&self) -> Result<String, ProviderError> {
|
||||
if let Some(namespace) = self.namespace.upgrade() {
|
||||
if let Some(port) = *namespace.file_server_port.read().await {
|
||||
return Ok(format!("localhost:{port}"));
|
||||
}
|
||||
}
|
||||
|
||||
Err(ProviderError::FileServerSetupError(anyhow!(
|
||||
"file server port not bound locally"
|
||||
)))
|
||||
}
|
||||
|
||||
async fn download_file(
|
||||
&self,
|
||||
url: &str,
|
||||
remote_file_path: &Path,
|
||||
hash: Option<&str>,
|
||||
) -> Result<(), ProviderError> {
|
||||
let r = self
|
||||
.k8s_client
|
||||
.pod_exec(
|
||||
&self.namespace_name(),
|
||||
&self.name,
|
||||
vec![
|
||||
"/cfg/curl",
|
||||
url,
|
||||
"--output",
|
||||
&remote_file_path.to_string_lossy(),
|
||||
],
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
ProviderError::DownloadFile(
|
||||
remote_file_path.to_string_lossy().to_string(),
|
||||
anyhow!(format!("node: {}, err: {}", self.name(), err)),
|
||||
)
|
||||
})?;
|
||||
|
||||
trace!("download url {} result: {:?}", url, r);
|
||||
|
||||
if r.is_err() {
|
||||
return Err(ProviderError::DownloadFile(
|
||||
remote_file_path.to_string_lossy().to_string(),
|
||||
anyhow!(format!("node: {}, err downloading file", self.name())),
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(hash) = hash {
|
||||
// check if the hash of the file is correct
|
||||
let res = self
|
||||
.k8s_client
|
||||
.pod_exec(
|
||||
&self.namespace_name(),
|
||||
&self.name,
|
||||
vec![
|
||||
"/cfg/coreutils",
|
||||
"sha256sum",
|
||||
&remote_file_path.to_string_lossy(),
|
||||
],
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
ProviderError::DownloadFile(
|
||||
remote_file_path.to_string_lossy().to_string(),
|
||||
anyhow!(format!("node: {}, err: {}", self.name(), err)),
|
||||
)
|
||||
})?;
|
||||
|
||||
if let Ok(output) = res {
|
||||
if !output.contains(hash) {
|
||||
return Err(ProviderError::DownloadFile(
|
||||
remote_file_path.to_string_lossy().to_string(),
|
||||
anyhow!(format!("node: {}, invalid sha256sum hash: {hash} for file, output was {output}", self.name())),
|
||||
));
|
||||
}
|
||||
} else {
|
||||
return Err(ProviderError::DownloadFile(
|
||||
remote_file_path.to_string_lossy().to_string(),
|
||||
anyhow!(format!(
|
||||
"node: {}, err calculating sha256sum for file {:?}",
|
||||
self.name(),
|
||||
res
|
||||
)),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<FS> ProviderNode for KubernetesNode<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
fn name(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
|
||||
fn args(&self) -> Vec<&str> {
|
||||
self.args.iter().map(|arg| arg.as_str()).collect()
|
||||
}
|
||||
|
||||
fn base_dir(&self) -> &PathBuf {
|
||||
&self.base_dir
|
||||
}
|
||||
|
||||
fn config_dir(&self) -> &PathBuf {
|
||||
&self.config_dir
|
||||
}
|
||||
|
||||
fn data_dir(&self) -> &PathBuf {
|
||||
&self.data_dir
|
||||
}
|
||||
|
||||
fn relay_data_dir(&self) -> &PathBuf {
|
||||
&self.relay_data_dir
|
||||
}
|
||||
|
||||
fn scripts_dir(&self) -> &PathBuf {
|
||||
&self.scripts_dir
|
||||
}
|
||||
|
||||
fn log_path(&self) -> &PathBuf {
|
||||
&self.log_path
|
||||
}
|
||||
|
||||
fn log_cmd(&self) -> String {
|
||||
format!("kubectl -n {} logs {}", self.namespace_name(), self.name)
|
||||
}
|
||||
|
||||
fn path_in_node(&self, file: &Path) -> PathBuf {
|
||||
// here is just a noop op since we will receive the path
|
||||
// for the file inside the pod
|
||||
PathBuf::from(file)
|
||||
}
|
||||
|
||||
// TODO: handle log rotation as we do in v1
|
||||
async fn logs(&self) -> Result<String, ProviderError> {
|
||||
self.k8s_client
|
||||
.pod_logs(&self.namespace_name(), &self.name)
|
||||
.await
|
||||
.map_err(|err| ProviderError::GetLogsFailed(self.name.to_string(), err.into()))
|
||||
}
|
||||
|
||||
async fn dump_logs(&self, local_dest: PathBuf) -> Result<(), ProviderError> {
|
||||
let logs = self.logs().await?;
|
||||
|
||||
self.filesystem
|
||||
.write(local_dest, logs)
|
||||
.await
|
||||
.map_err(|err| ProviderError::DumpLogsFailed(self.name.to_string(), err.into()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn create_port_forward(
|
||||
&self,
|
||||
local_port: u16,
|
||||
remote_port: u16,
|
||||
) -> Result<Option<u16>, ProviderError> {
|
||||
// If the fwd exist just return the local port
|
||||
if let Some(fwd_info) = self.port_fwds.read().await.get(&remote_port) {
|
||||
return Ok(Some(fwd_info.0));
|
||||
};
|
||||
|
||||
let (port, task) = self
|
||||
.k8s_client
|
||||
.create_pod_port_forward(&self.namespace_name(), &self.name, local_port, remote_port)
|
||||
.await
|
||||
.map_err(|err| ProviderError::PortForwardError(local_port, remote_port, err.into()))?;
|
||||
|
||||
self.port_fwds
|
||||
.write()
|
||||
.await
|
||||
.insert(remote_port, (port, task));
|
||||
|
||||
Ok(Some(port))
|
||||
}
|
||||
|
||||
async fn run_command(
|
||||
&self,
|
||||
options: RunCommandOptions,
|
||||
) -> Result<ExecutionResult, ProviderError> {
|
||||
let mut command = vec![];
|
||||
|
||||
for (name, value) in options.env {
|
||||
command.push(format!("export {name}={value};"));
|
||||
}
|
||||
|
||||
command.push(options.program);
|
||||
|
||||
for arg in options.args {
|
||||
command.push(arg);
|
||||
}
|
||||
|
||||
self.k8s_client
|
||||
.pod_exec(
|
||||
&self.namespace_name(),
|
||||
&self.name,
|
||||
vec!["sh", "-c", &command.join(" ")],
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
ProviderError::RunCommandError(
|
||||
format!("sh -c {}", &command.join(" ")),
|
||||
format!("in pod {}", self.name),
|
||||
err.into(),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
async fn run_script(
|
||||
&self,
|
||||
options: RunScriptOptions,
|
||||
) -> Result<ExecutionResult, ProviderError> {
|
||||
let file_name = options
|
||||
.local_script_path
|
||||
.file_name()
|
||||
.expect(&format!(
|
||||
"file name should be present at this point {THIS_IS_A_BUG}"
|
||||
))
|
||||
.to_string_lossy();
|
||||
|
||||
self.run_command(RunCommandOptions {
|
||||
program: format!("/tmp/{file_name}"),
|
||||
args: options.args,
|
||||
env: options.env,
|
||||
})
|
||||
.await
|
||||
.map_err(|err| ProviderError::RunScriptError(self.name.to_string(), err.into()))
|
||||
}
|
||||
|
||||
async fn send_file(
|
||||
&self,
|
||||
local_file_path: &Path,
|
||||
remote_file_path: &Path,
|
||||
mode: &str,
|
||||
) -> Result<(), ProviderError> {
|
||||
if let Some(remote_parent_dir) = self.get_remote_parent_dir(remote_file_path) {
|
||||
self.create_remote_dir(&remote_parent_dir).await?;
|
||||
}
|
||||
|
||||
debug!(
|
||||
"Uploading file: {} IFF not present in the fileserver",
|
||||
local_file_path.to_string_lossy()
|
||||
);
|
||||
|
||||
// we need to override the url to use inside the pod
|
||||
let (mut url, hash) = self.upload_to_fileserver(local_file_path).await?;
|
||||
let _ = url.set_host(Some("fileserver"));
|
||||
let _ = url.set_port(Some(80));
|
||||
|
||||
// Sometimes downloading the file fails (the file is corrupted)
|
||||
// Add at most 5 retries
|
||||
let mut last_err = None;
|
||||
for i in 0..5 {
|
||||
if i > 0 {
|
||||
warn!("retrying number {i} download file {:?}", remote_file_path);
|
||||
tokio::time::sleep(Duration::from_secs(i)).await;
|
||||
}
|
||||
|
||||
let res = self
|
||||
.download_file(url.as_ref(), remote_file_path, Some(&hash))
|
||||
.await;
|
||||
|
||||
last_err = res.err();
|
||||
|
||||
if last_err.is_none() {
|
||||
// ready to continue
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(last_err) = last_err {
|
||||
return Err(last_err);
|
||||
}
|
||||
|
||||
let _ = self
|
||||
.k8s_client
|
||||
.pod_exec(
|
||||
&self.namespace_name(),
|
||||
&self.name,
|
||||
vec!["chmod", mode, &remote_file_path.to_string_lossy()],
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
ProviderError::SendFile(
|
||||
self.name.clone(),
|
||||
local_file_path.to_string_lossy().to_string(),
|
||||
err.into(),
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn receive_file(
|
||||
&self,
|
||||
_remote_src: &Path,
|
||||
_local_dest: &Path,
|
||||
) -> Result<(), ProviderError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn ip(&self) -> Result<IpAddr, ProviderError> {
|
||||
let status = self
|
||||
.k8s_client
|
||||
.pod_status(&self.namespace_name(), &self.name)
|
||||
.await
|
||||
.map_err(|_| ProviderError::MissingNode(self.name.clone()))?;
|
||||
|
||||
if let Some(ip) = status.pod_ip {
|
||||
// Pod ip should be parseable
|
||||
Ok(ip.parse::<IpAddr>().map_err(|err| {
|
||||
ProviderError::InvalidConfig(format!("Can not parse the pod ip: {ip}, err: {err}"))
|
||||
})?)
|
||||
} else {
|
||||
Err(ProviderError::InvalidConfig(format!(
|
||||
"Can not find ip of pod: {}",
|
||||
self.name()
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
async fn pause(&self) -> Result<(), ProviderError> {
|
||||
self.k8s_client
|
||||
.pod_exec(
|
||||
&self.namespace_name(),
|
||||
&self.name,
|
||||
vec!["sh", "-c", "echo pause > /tmp/zombiepipe"],
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ProviderError::PauseNodeFailed(self.name.to_string(), err.into()))?
|
||||
.map_err(|err| {
|
||||
ProviderError::PauseNodeFailed(
|
||||
self.name.to_string(),
|
||||
anyhow!("error when pausing node: status {}: {}", err.0, err.1),
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn resume(&self) -> Result<(), ProviderError> {
|
||||
self.k8s_client
|
||||
.pod_exec(
|
||||
&self.namespace_name(),
|
||||
&self.name,
|
||||
vec!["sh", "-c", "echo resume > /tmp/zombiepipe"],
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ProviderError::ResumeNodeFailed(self.name.to_string(), err.into()))?
|
||||
.map_err(|err| {
|
||||
ProviderError::ResumeNodeFailed(
|
||||
self.name.to_string(),
|
||||
anyhow!("error when pausing node: status {}: {}", err.0, err.1),
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn restart(&self, after: Option<Duration>) -> Result<(), ProviderError> {
|
||||
if let Some(duration) = after {
|
||||
sleep(duration).await;
|
||||
}
|
||||
|
||||
self.k8s_client
|
||||
.pod_exec(
|
||||
&self.namespace_name(),
|
||||
&self.name,
|
||||
vec!["sh", "-c", "echo restart > /tmp/zombiepipe"],
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ProviderError::RestartNodeFailed(self.name.to_string(), err.into()))?
|
||||
.map_err(|err| {
|
||||
ProviderError::RestartNodeFailed(
|
||||
self.name.to_string(),
|
||||
anyhow!("error when restarting node: status {}: {}", err.0, err.1),
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn destroy(&self) -> Result<(), ProviderError> {
|
||||
self.k8s_client
|
||||
.delete_pod(&self.namespace_name(), &self.name)
|
||||
.await
|
||||
.map_err(|err| ProviderError::KillNodeFailed(self.name.to_string(), err.into()))?;
|
||||
|
||||
if let Some(namespace) = self.namespace.upgrade() {
|
||||
namespace.nodes.write().await.remove(&self.name);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
+188
@@ -0,0 +1,188 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use configuration::shared::resources::{ResourceQuantity, Resources};
|
||||
use k8s_openapi::{
|
||||
api::core::v1::{
|
||||
ConfigMapVolumeSource, Container, EnvVar, PodSpec, ResourceRequirements, Volume,
|
||||
VolumeMount,
|
||||
},
|
||||
apimachinery::pkg::api::resource::Quantity,
|
||||
};
|
||||
|
||||
pub(super) struct PodSpecBuilder;
|
||||
|
||||
impl PodSpecBuilder {
|
||||
pub(super) fn build(
|
||||
name: &str,
|
||||
image: &str,
|
||||
resources: Option<&Resources>,
|
||||
program: &str,
|
||||
args: &[String],
|
||||
env: &[(String, String)],
|
||||
) -> PodSpec {
|
||||
PodSpec {
|
||||
hostname: Some(name.to_string()),
|
||||
init_containers: Some(vec![Self::build_helper_binaries_setup_container()]),
|
||||
containers: vec![Self::build_main_container(
|
||||
name, image, resources, program, args, env,
|
||||
)],
|
||||
volumes: Some(Self::build_volumes()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn build_main_container(
|
||||
name: &str,
|
||||
image: &str,
|
||||
resources: Option<&Resources>,
|
||||
program: &str,
|
||||
args: &[String],
|
||||
env: &[(String, String)],
|
||||
) -> Container {
|
||||
Container {
|
||||
name: name.to_string(),
|
||||
image: Some(image.to_string()),
|
||||
image_pull_policy: Some("Always".to_string()),
|
||||
command: Some(
|
||||
[
|
||||
vec!["/zombie-wrapper.sh".to_string(), program.to_string()],
|
||||
args.to_vec(),
|
||||
]
|
||||
.concat(),
|
||||
),
|
||||
env: Some(
|
||||
env.iter()
|
||||
.map(|(name, value)| EnvVar {
|
||||
name: name.clone(),
|
||||
value: Some(value.clone()),
|
||||
value_from: None,
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
volume_mounts: Some(Self::build_volume_mounts(vec![VolumeMount {
|
||||
name: "zombie-wrapper-volume".to_string(),
|
||||
mount_path: "/zombie-wrapper.sh".to_string(),
|
||||
sub_path: Some("zombie-wrapper.sh".to_string()),
|
||||
..Default::default()
|
||||
}])),
|
||||
resources: Self::build_resources_requirements(resources),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn build_helper_binaries_setup_container() -> Container {
|
||||
Container {
|
||||
name: "helper-binaries-setup".to_string(),
|
||||
image: Some("europe-west3-docker.pkg.dev/parity-zombienet/zombienet-public-images/alpine:latest".to_string()),
|
||||
image_pull_policy: Some("IfNotPresent".to_string()),
|
||||
volume_mounts: Some(Self::build_volume_mounts(vec![VolumeMount {
|
||||
name: "helper-binaries-downloader-volume".to_string(),
|
||||
mount_path: "/helper-binaries-downloader.sh".to_string(),
|
||||
sub_path: Some("helper-binaries-downloader.sh".to_string()),
|
||||
..Default::default()
|
||||
}])),
|
||||
command: Some(vec![
|
||||
"ash".to_string(),
|
||||
"/helper-binaries-downloader.sh".to_string(),
|
||||
]),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn build_volumes() -> Vec<Volume> {
|
||||
vec![
|
||||
Volume {
|
||||
name: "cfg".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
Volume {
|
||||
name: "data".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
Volume {
|
||||
name: "relay-data".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
Volume {
|
||||
name: "zombie-wrapper-volume".to_string(),
|
||||
config_map: Some(ConfigMapVolumeSource {
|
||||
name: Some("zombie-wrapper".to_string()),
|
||||
default_mode: Some(0o755),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
Volume {
|
||||
name: "helper-binaries-downloader-volume".to_string(),
|
||||
config_map: Some(ConfigMapVolumeSource {
|
||||
name: Some("helper-binaries-downloader".to_string()),
|
||||
default_mode: Some(0o755),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
fn build_volume_mounts(non_default_mounts: Vec<VolumeMount>) -> Vec<VolumeMount> {
|
||||
[
|
||||
vec![
|
||||
VolumeMount {
|
||||
name: "cfg".to_string(),
|
||||
mount_path: "/cfg".to_string(),
|
||||
read_only: Some(false),
|
||||
..Default::default()
|
||||
},
|
||||
VolumeMount {
|
||||
name: "data".to_string(),
|
||||
mount_path: "/data".to_string(),
|
||||
read_only: Some(false),
|
||||
..Default::default()
|
||||
},
|
||||
VolumeMount {
|
||||
name: "relay-data".to_string(),
|
||||
mount_path: "/relay-data".to_string(),
|
||||
read_only: Some(false),
|
||||
..Default::default()
|
||||
},
|
||||
],
|
||||
non_default_mounts,
|
||||
]
|
||||
.concat()
|
||||
}
|
||||
|
||||
fn build_resources_requirements(resources: Option<&Resources>) -> Option<ResourceRequirements> {
|
||||
resources.map(|resources| ResourceRequirements {
|
||||
limits: Self::build_resources_requirements_quantities(
|
||||
resources.limit_cpu(),
|
||||
resources.limit_memory(),
|
||||
),
|
||||
requests: Self::build_resources_requirements_quantities(
|
||||
resources.request_cpu(),
|
||||
resources.request_memory(),
|
||||
),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
fn build_resources_requirements_quantities(
|
||||
cpu: Option<&ResourceQuantity>,
|
||||
memory: Option<&ResourceQuantity>,
|
||||
) -> Option<BTreeMap<String, Quantity>> {
|
||||
let mut quantities = BTreeMap::new();
|
||||
|
||||
if let Some(cpu) = cpu {
|
||||
quantities.insert("cpu".to_string(), Quantity(cpu.as_str().to_string()));
|
||||
}
|
||||
|
||||
if let Some(memory) = memory {
|
||||
quantities.insert("memory".to_string(), Quantity(memory.as_str().to_string()));
|
||||
}
|
||||
|
||||
if !quantities.is_empty() {
|
||||
Some(quantities)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, Weak},
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use support::fs::FileSystem;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use super::{client::KubernetesClient, namespace::KubernetesNamespace};
|
||||
use crate::{
|
||||
shared::helpers::extract_namespace_info, types::ProviderCapabilities, DynNamespace, Provider,
|
||||
ProviderError, ProviderNamespace,
|
||||
};
|
||||
|
||||
pub const PROVIDER_NAME: &str = "k8s";
|
||||
|
||||
pub struct KubernetesProvider<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone,
|
||||
{
|
||||
weak: Weak<KubernetesProvider<FS>>,
|
||||
capabilities: ProviderCapabilities,
|
||||
tmp_dir: PathBuf,
|
||||
k8s_client: KubernetesClient,
|
||||
filesystem: FS,
|
||||
pub(super) namespaces: RwLock<HashMap<String, Arc<KubernetesNamespace<FS>>>>,
|
||||
}
|
||||
|
||||
impl<FS> KubernetesProvider<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone,
|
||||
{
|
||||
pub async fn new(filesystem: FS) -> Arc<Self> {
|
||||
let k8s_client = KubernetesClient::new().await.unwrap();
|
||||
|
||||
Arc::new_cyclic(|weak| KubernetesProvider {
|
||||
weak: weak.clone(),
|
||||
capabilities: ProviderCapabilities {
|
||||
requires_image: true,
|
||||
has_resources: true,
|
||||
prefix_with_full_path: false,
|
||||
use_default_ports_in_cmd: true,
|
||||
},
|
||||
tmp_dir: std::env::temp_dir(),
|
||||
k8s_client,
|
||||
filesystem,
|
||||
namespaces: RwLock::new(HashMap::new()),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn tmp_dir(mut self, tmp_dir: impl Into<PathBuf>) -> Self {
|
||||
self.tmp_dir = tmp_dir.into();
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<FS> Provider for KubernetesProvider<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
fn name(&self) -> &str {
|
||||
PROVIDER_NAME
|
||||
}
|
||||
|
||||
fn capabilities(&self) -> &ProviderCapabilities {
|
||||
&self.capabilities
|
||||
}
|
||||
|
||||
async fn namespaces(&self) -> HashMap<String, DynNamespace> {
|
||||
self.namespaces
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.map(|(name, namespace)| (name.clone(), namespace.clone() as DynNamespace))
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn create_namespace(&self) -> Result<DynNamespace, ProviderError> {
|
||||
let namespace = KubernetesNamespace::new(
|
||||
&self.weak,
|
||||
&self.tmp_dir,
|
||||
&self.capabilities,
|
||||
&self.k8s_client,
|
||||
&self.filesystem,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
self.namespaces
|
||||
.write()
|
||||
.await
|
||||
.insert(namespace.name().to_string(), namespace.clone());
|
||||
|
||||
Ok(namespace)
|
||||
}
|
||||
|
||||
async fn create_namespace_with_base_dir(
|
||||
&self,
|
||||
base_dir: &Path,
|
||||
) -> Result<DynNamespace, ProviderError> {
|
||||
let namespace = KubernetesNamespace::new(
|
||||
&self.weak,
|
||||
&self.tmp_dir,
|
||||
&self.capabilities,
|
||||
&self.k8s_client,
|
||||
&self.filesystem,
|
||||
Some(base_dir),
|
||||
)
|
||||
.await?;
|
||||
|
||||
self.namespaces
|
||||
.write()
|
||||
.await
|
||||
.insert(namespace.name().to_string(), namespace.clone());
|
||||
|
||||
Ok(namespace)
|
||||
}
|
||||
|
||||
async fn create_namespace_from_json(
|
||||
&self,
|
||||
json_value: &serde_json::Value,
|
||||
) -> Result<DynNamespace, ProviderError> {
|
||||
let (base_dir, name) = extract_namespace_info(json_value)?;
|
||||
|
||||
let namespace = KubernetesNamespace::attach_to_live(
|
||||
&self.weak,
|
||||
&self.capabilities,
|
||||
&self.k8s_client,
|
||||
&self.filesystem,
|
||||
&base_dir,
|
||||
&name,
|
||||
)
|
||||
.await?;
|
||||
|
||||
self.namespaces
|
||||
.write()
|
||||
.await
|
||||
.insert(namespace.name().to_string(), namespace.clone());
|
||||
|
||||
Ok(namespace)
|
||||
}
|
||||
}
|
||||
Vendored
+10
@@ -0,0 +1,10 @@
|
||||
apiVersion: v1
|
||||
kind: LimitRange
|
||||
metadata:
|
||||
name: mem-limit-range
|
||||
spec:
|
||||
limits:
|
||||
- defaultRequest:
|
||||
memory: 1G
|
||||
cpu: 0.5
|
||||
type: Container
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: internal-access
|
||||
spec:
|
||||
podSelector: {}
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchExpressions:
|
||||
- key: kubernetes.io/metadata.name
|
||||
operator: In
|
||||
values:
|
||||
- {{namespace}}
|
||||
- gitlab
|
||||
- arc-runner
|
||||
- loki
|
||||
- tempo
|
||||
- monitoring
|
||||
- parachain-exporter
|
||||
- default
|
||||
policyTypes:
|
||||
- Ingress
|
||||
@@ -0,0 +1,264 @@
|
||||
#![allow(clippy::expect_fun_call)]
|
||||
mod docker;
|
||||
mod kubernetes;
|
||||
mod native;
|
||||
pub mod shared;
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
net::IpAddr,
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use shared::{
|
||||
constants::LOCALHOST,
|
||||
types::{
|
||||
ExecutionResult, GenerateFilesOptions, ProviderCapabilities, RunCommandOptions,
|
||||
RunScriptOptions, SpawnNodeOptions,
|
||||
},
|
||||
};
|
||||
use support::fs::FileSystemError;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[allow(missing_docs)]
|
||||
pub enum ProviderError {
|
||||
#[error("Failed to create client '{0}': {1}")]
|
||||
CreateClientFailed(String, anyhow::Error),
|
||||
|
||||
#[error("Failed to create namespace '{0}': {1}")]
|
||||
CreateNamespaceFailed(String, anyhow::Error),
|
||||
|
||||
#[error("Failed to spawn node '{0}': {1}")]
|
||||
NodeSpawningFailed(String, anyhow::Error),
|
||||
|
||||
#[error("Error running command '{0}' {1}: {2}")]
|
||||
RunCommandError(String, String, anyhow::Error),
|
||||
|
||||
#[error("Error running script'{0}': {1}")]
|
||||
RunScriptError(String, anyhow::Error),
|
||||
|
||||
#[error("Invalid network configuration field {0}")]
|
||||
InvalidConfig(String),
|
||||
|
||||
#[error("Failed to retrieve node available args using image {0} and command {1}: {2}")]
|
||||
NodeAvailableArgsError(String, String, String),
|
||||
|
||||
#[error("Can not recover node: {0}")]
|
||||
MissingNode(String),
|
||||
|
||||
#[error("Can not recover node: {0} info, field: {1}")]
|
||||
MissingNodeInfo(String, String),
|
||||
|
||||
#[error("File generation failed: {0}")]
|
||||
FileGenerationFailed(anyhow::Error),
|
||||
|
||||
#[error(transparent)]
|
||||
FileSystemError(#[from] FileSystemError),
|
||||
|
||||
#[error("Invalid script path for {0}")]
|
||||
InvalidScriptPath(anyhow::Error),
|
||||
|
||||
#[error("Script with path {0} not found")]
|
||||
ScriptNotFound(PathBuf),
|
||||
|
||||
#[error("Failed to retrieve process ID for node '{0}'")]
|
||||
ProcessIdRetrievalFailed(String),
|
||||
|
||||
#[error("Failed to pause node '{0}': {1}")]
|
||||
PauseNodeFailed(String, anyhow::Error),
|
||||
|
||||
#[error("Failed to resume node '{0}': {1}")]
|
||||
ResumeNodeFailed(String, anyhow::Error),
|
||||
|
||||
#[error("Failed to kill node '{0}': {1}")]
|
||||
KillNodeFailed(String, anyhow::Error),
|
||||
|
||||
#[error("Failed to restart node '{0}': {1}")]
|
||||
RestartNodeFailed(String, anyhow::Error),
|
||||
|
||||
#[error("Failed to destroy node '{0}': {1}")]
|
||||
DestroyNodeFailed(String, anyhow::Error),
|
||||
|
||||
#[error("Failed to get logs for node '{0}': {1}")]
|
||||
GetLogsFailed(String, anyhow::Error),
|
||||
|
||||
#[error("Failed to dump logs for node '{0}': {1}")]
|
||||
DumpLogsFailed(String, anyhow::Error),
|
||||
|
||||
#[error("Failed to copy file from node '{0}': {1}")]
|
||||
CopyFileFromNodeError(String, anyhow::Error),
|
||||
|
||||
#[error("Failed to setup fileserver: {0}")]
|
||||
FileServerSetupError(anyhow::Error),
|
||||
|
||||
#[error("Error uploading file: '{0}': {1}")]
|
||||
UploadFile(String, anyhow::Error),
|
||||
|
||||
#[error("Error downloading file: '{0}': {1}")]
|
||||
DownloadFile(String, anyhow::Error),
|
||||
|
||||
#[error("Error sending file '{0}' to {1}: {2}")]
|
||||
SendFile(String, String, anyhow::Error),
|
||||
|
||||
#[error("Error creating port-forward '{0}:{1}': {2}")]
|
||||
PortForwardError(u16, u16, anyhow::Error),
|
||||
|
||||
#[error("Failed to delete namespace '{0}': {1}")]
|
||||
DeleteNamespaceFailed(String, anyhow::Error),
|
||||
|
||||
#[error("Serialization error")]
|
||||
SerializationError(#[from] serde_json::Error),
|
||||
|
||||
#[error("Failed to acquire lock: {0}")]
|
||||
FailedToAcquireLock(String),
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait Provider {
|
||||
fn name(&self) -> &str;
|
||||
|
||||
fn capabilities(&self) -> &ProviderCapabilities;
|
||||
|
||||
async fn namespaces(&self) -> HashMap<String, DynNamespace>;
|
||||
|
||||
async fn create_namespace(&self) -> Result<DynNamespace, ProviderError>;
|
||||
|
||||
async fn create_namespace_with_base_dir(
|
||||
&self,
|
||||
base_dir: &Path,
|
||||
) -> Result<DynNamespace, ProviderError>;
|
||||
|
||||
async fn create_namespace_from_json(
|
||||
&self,
|
||||
json_value: &serde_json::Value,
|
||||
) -> Result<DynNamespace, ProviderError>;
|
||||
}
|
||||
|
||||
pub type DynProvider = Arc<dyn Provider + Send + Sync>;
|
||||
|
||||
#[async_trait]
|
||||
pub trait ProviderNamespace {
|
||||
fn name(&self) -> &str;
|
||||
|
||||
fn base_dir(&self) -> &PathBuf;
|
||||
|
||||
fn capabilities(&self) -> &ProviderCapabilities;
|
||||
|
||||
fn provider_name(&self) -> &str;
|
||||
|
||||
async fn detach(&self) {
|
||||
// noop by default
|
||||
warn!("Detach is not implemented for {}", self.name());
|
||||
}
|
||||
|
||||
async fn is_detached(&self) -> bool {
|
||||
// false by default
|
||||
false
|
||||
}
|
||||
|
||||
async fn nodes(&self) -> HashMap<String, DynNode>;
|
||||
|
||||
async fn get_node_available_args(
|
||||
&self,
|
||||
options: (String, Option<String>),
|
||||
) -> Result<String, ProviderError>;
|
||||
|
||||
async fn spawn_node(&self, options: &SpawnNodeOptions) -> Result<DynNode, ProviderError>;
|
||||
|
||||
async fn spawn_node_from_json(
|
||||
&self,
|
||||
json_value: &serde_json::Value,
|
||||
) -> Result<DynNode, ProviderError>;
|
||||
|
||||
async fn generate_files(&self, options: GenerateFilesOptions) -> Result<(), ProviderError>;
|
||||
|
||||
async fn destroy(&self) -> Result<(), ProviderError>;
|
||||
|
||||
async fn static_setup(&self) -> Result<(), ProviderError>;
|
||||
}
|
||||
|
||||
pub type DynNamespace = Arc<dyn ProviderNamespace + Send + Sync>;
|
||||
|
||||
#[async_trait]
|
||||
pub trait ProviderNode: erased_serde::Serialize {
|
||||
fn name(&self) -> &str;
|
||||
|
||||
fn args(&self) -> Vec<&str>;
|
||||
|
||||
fn base_dir(&self) -> &PathBuf;
|
||||
|
||||
fn config_dir(&self) -> &PathBuf;
|
||||
|
||||
fn data_dir(&self) -> &PathBuf;
|
||||
|
||||
fn relay_data_dir(&self) -> &PathBuf;
|
||||
|
||||
fn scripts_dir(&self) -> &PathBuf;
|
||||
|
||||
fn log_path(&self) -> &PathBuf;
|
||||
|
||||
fn log_cmd(&self) -> String;
|
||||
|
||||
// Return the absolute path to the file in the `node` perspective
|
||||
// TODO: purpose?
|
||||
fn path_in_node(&self, file: &Path) -> PathBuf;
|
||||
|
||||
async fn logs(&self) -> Result<String, ProviderError>;
|
||||
|
||||
async fn dump_logs(&self, local_dest: PathBuf) -> Result<(), ProviderError>;
|
||||
|
||||
// By default return localhost, should be overrided for k8s
|
||||
async fn ip(&self) -> Result<IpAddr, ProviderError> {
|
||||
Ok(LOCALHOST)
|
||||
}
|
||||
|
||||
// Noop by default (native/docker provider)
|
||||
async fn create_port_forward(
|
||||
&self,
|
||||
_local_port: u16,
|
||||
_remote_port: u16,
|
||||
) -> Result<Option<u16>, ProviderError> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn run_command(
|
||||
&self,
|
||||
options: RunCommandOptions,
|
||||
) -> Result<ExecutionResult, ProviderError>;
|
||||
|
||||
async fn run_script(&self, options: RunScriptOptions)
|
||||
-> Result<ExecutionResult, ProviderError>;
|
||||
|
||||
async fn send_file(
|
||||
&self,
|
||||
local_file_path: &Path,
|
||||
remote_file_path: &Path,
|
||||
mode: &str,
|
||||
) -> Result<(), ProviderError>;
|
||||
|
||||
async fn receive_file(
|
||||
&self,
|
||||
remote_file_path: &Path,
|
||||
local_file_path: &Path,
|
||||
) -> Result<(), ProviderError>;
|
||||
|
||||
async fn pause(&self) -> Result<(), ProviderError>;
|
||||
|
||||
async fn resume(&self) -> Result<(), ProviderError>;
|
||||
|
||||
async fn restart(&self, after: Option<Duration>) -> Result<(), ProviderError>;
|
||||
|
||||
async fn destroy(&self) -> Result<(), ProviderError>;
|
||||
}
|
||||
|
||||
pub type DynNode = Arc<dyn ProviderNode + Send + Sync>;
|
||||
|
||||
// re-export
|
||||
pub use docker::*;
|
||||
pub use kubernetes::*;
|
||||
pub use native::*;
|
||||
pub use shared::{constants, types};
|
||||
use tracing::warn;
|
||||
@@ -0,0 +1,5 @@
|
||||
mod namespace;
|
||||
mod node;
|
||||
mod provider;
|
||||
|
||||
pub use provider::NativeProvider;
|
||||
@@ -0,0 +1,374 @@
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, Weak},
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use support::fs::FileSystem;
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{trace, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::node::{NativeNode, NativeNodeOptions};
|
||||
use crate::{
|
||||
constants::NAMESPACE_PREFIX,
|
||||
native::{node::DeserializableNativeNodeOptions, provider},
|
||||
shared::helpers::extract_execution_result,
|
||||
types::{
|
||||
GenerateFileCommand, GenerateFilesOptions, ProviderCapabilities, RunCommandOptions,
|
||||
SpawnNodeOptions,
|
||||
},
|
||||
DynNode, NativeProvider, ProviderError, ProviderNamespace, ProviderNode,
|
||||
};
|
||||
|
||||
pub(super) struct NativeNamespace<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone,
|
||||
{
|
||||
weak: Weak<NativeNamespace<FS>>,
|
||||
name: String,
|
||||
provider: Weak<NativeProvider<FS>>,
|
||||
base_dir: PathBuf,
|
||||
capabilities: ProviderCapabilities,
|
||||
filesystem: FS,
|
||||
pub(super) nodes: RwLock<HashMap<String, Arc<NativeNode<FS>>>>,
|
||||
}
|
||||
|
||||
impl<FS> NativeNamespace<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
pub(super) async fn new(
|
||||
provider: &Weak<NativeProvider<FS>>,
|
||||
tmp_dir: &PathBuf,
|
||||
capabilities: &ProviderCapabilities,
|
||||
filesystem: &FS,
|
||||
custom_base_dir: Option<&Path>,
|
||||
) -> Result<Arc<Self>, ProviderError> {
|
||||
let name = format!("{}{}", NAMESPACE_PREFIX, Uuid::new_v4());
|
||||
let base_dir = if let Some(custom_base_dir) = custom_base_dir {
|
||||
if !filesystem.exists(custom_base_dir).await {
|
||||
filesystem.create_dir_all(custom_base_dir).await?;
|
||||
} else {
|
||||
warn!(
|
||||
"⚠️ Using and existing directory {} as base dir",
|
||||
custom_base_dir.to_string_lossy()
|
||||
);
|
||||
}
|
||||
PathBuf::from(custom_base_dir)
|
||||
} else {
|
||||
let base_dir = PathBuf::from_iter([tmp_dir, &PathBuf::from(&name)]);
|
||||
filesystem.create_dir(&base_dir).await?;
|
||||
base_dir
|
||||
};
|
||||
|
||||
Ok(Arc::new_cyclic(|weak| NativeNamespace {
|
||||
weak: weak.clone(),
|
||||
provider: provider.clone(),
|
||||
name,
|
||||
base_dir,
|
||||
capabilities: capabilities.clone(),
|
||||
filesystem: filesystem.clone(),
|
||||
nodes: RwLock::new(HashMap::new()),
|
||||
}))
|
||||
}
|
||||
|
||||
pub(super) async fn attach_to_live(
|
||||
provider: &Weak<NativeProvider<FS>>,
|
||||
capabilities: &ProviderCapabilities,
|
||||
filesystem: &FS,
|
||||
custom_base_dir: &Path,
|
||||
name: &str,
|
||||
) -> Result<Arc<Self>, ProviderError> {
|
||||
let base_dir = custom_base_dir.to_path_buf();
|
||||
|
||||
Ok(Arc::new_cyclic(|weak| NativeNamespace {
|
||||
weak: weak.clone(),
|
||||
provider: provider.clone(),
|
||||
name: name.to_string(),
|
||||
base_dir,
|
||||
capabilities: capabilities.clone(),
|
||||
filesystem: filesystem.clone(),
|
||||
nodes: RwLock::new(HashMap::new()),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<FS> ProviderNamespace for NativeNamespace<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
fn name(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
|
||||
fn base_dir(&self) -> &PathBuf {
|
||||
&self.base_dir
|
||||
}
|
||||
|
||||
fn capabilities(&self) -> &ProviderCapabilities {
|
||||
&self.capabilities
|
||||
}
|
||||
|
||||
fn provider_name(&self) -> &str {
|
||||
provider::PROVIDER_NAME
|
||||
}
|
||||
|
||||
async fn nodes(&self) -> HashMap<String, DynNode> {
|
||||
self.nodes
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.map(|(name, node)| (name.clone(), node.clone() as DynNode))
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn get_node_available_args(
|
||||
&self,
|
||||
(command, _image): (String, Option<String>),
|
||||
) -> Result<String, ProviderError> {
|
||||
let temp_node = self
|
||||
.spawn_node(
|
||||
&SpawnNodeOptions::new(format!("temp-{}", Uuid::new_v4()), "bash".to_string())
|
||||
.args(vec!["-c", "while :; do sleep 1; done"]),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let available_args_output = temp_node
|
||||
.run_command(RunCommandOptions::new(command.clone()).args(vec!["--help"]))
|
||||
.await?
|
||||
.map_err(|(_exit, status)| {
|
||||
ProviderError::NodeAvailableArgsError("".to_string(), command, status)
|
||||
})?;
|
||||
|
||||
temp_node.destroy().await?;
|
||||
|
||||
Ok(available_args_output)
|
||||
}
|
||||
|
||||
async fn spawn_node(&self, options: &SpawnNodeOptions) -> Result<DynNode, ProviderError> {
|
||||
trace!("spawn node options {options:?}");
|
||||
|
||||
let node = NativeNode::new(NativeNodeOptions {
|
||||
namespace: &self.weak,
|
||||
namespace_base_dir: &self.base_dir,
|
||||
name: &options.name,
|
||||
program: &options.program,
|
||||
args: &options.args,
|
||||
env: &options.env,
|
||||
startup_files: &options.injected_files,
|
||||
created_paths: &options.created_paths,
|
||||
db_snapshot: options.db_snapshot.as_ref(),
|
||||
filesystem: &self.filesystem,
|
||||
node_log_path: options.node_log_path.as_ref(),
|
||||
})
|
||||
.await?;
|
||||
|
||||
self.nodes
|
||||
.write()
|
||||
.await
|
||||
.insert(options.name.clone(), node.clone());
|
||||
|
||||
Ok(node)
|
||||
}
|
||||
|
||||
async fn spawn_node_from_json(
|
||||
&self,
|
||||
json_value: &serde_json::Value,
|
||||
) -> Result<DynNode, ProviderError> {
|
||||
let deserializable: DeserializableNativeNodeOptions =
|
||||
serde_json::from_value(json_value.clone())?;
|
||||
let options = NativeNodeOptions::from_deserializable(
|
||||
&deserializable,
|
||||
&self.weak,
|
||||
&self.base_dir,
|
||||
&self.filesystem,
|
||||
);
|
||||
|
||||
let pid = json_value
|
||||
.get("process_handle")
|
||||
.and_then(|v| v.as_i64())
|
||||
.ok_or_else(|| ProviderError::InvalidConfig("Missing pid field".to_string()))?
|
||||
as i32;
|
||||
let node = NativeNode::attach_to_live(options, pid).await?;
|
||||
|
||||
self.nodes
|
||||
.write()
|
||||
.await
|
||||
.insert(node.name().to_string(), node.clone());
|
||||
|
||||
Ok(node)
|
||||
}
|
||||
|
||||
async fn generate_files(&self, options: GenerateFilesOptions) -> Result<(), ProviderError> {
|
||||
let node_name = if let Some(name) = options.temp_name {
|
||||
name
|
||||
} else {
|
||||
format!("temp-{}", Uuid::new_v4())
|
||||
};
|
||||
|
||||
// we spawn a node doing nothing but looping so we can execute our commands
|
||||
let temp_node = self
|
||||
.spawn_node(
|
||||
&SpawnNodeOptions::new(node_name, "bash".to_string())
|
||||
.args(vec!["-c", "while :; do sleep 1; done"])
|
||||
.injected_files(options.injected_files),
|
||||
)
|
||||
.await?;
|
||||
|
||||
for GenerateFileCommand {
|
||||
program,
|
||||
args,
|
||||
env,
|
||||
local_output_path,
|
||||
} in options.commands
|
||||
{
|
||||
trace!(
|
||||
"🏗 building file {:?} in path {} with command {} {}",
|
||||
local_output_path.as_os_str(),
|
||||
self.base_dir.to_string_lossy(),
|
||||
program,
|
||||
args.join(" ")
|
||||
);
|
||||
let local_output_full_path = format!(
|
||||
"{}{}{}",
|
||||
self.base_dir.to_string_lossy(),
|
||||
if local_output_path.starts_with("/") {
|
||||
""
|
||||
} else {
|
||||
"/"
|
||||
},
|
||||
local_output_path.to_string_lossy()
|
||||
);
|
||||
|
||||
let contents = extract_execution_result(
|
||||
&temp_node,
|
||||
RunCommandOptions { program, args, env },
|
||||
options.expected_path.as_ref(),
|
||||
)
|
||||
.await?;
|
||||
self.filesystem
|
||||
.write(local_output_full_path, contents)
|
||||
.await
|
||||
.map_err(|err| ProviderError::FileGenerationFailed(err.into()))?;
|
||||
}
|
||||
|
||||
temp_node.destroy().await
|
||||
}
|
||||
|
||||
async fn static_setup(&self) -> Result<(), ProviderError> {
|
||||
// no static setup exists for native provider
|
||||
todo!()
|
||||
}
|
||||
|
||||
async fn destroy(&self) -> Result<(), ProviderError> {
|
||||
let mut names = vec![];
|
||||
|
||||
for node in self.nodes.read().await.values() {
|
||||
node.abort()
|
||||
.await
|
||||
.map_err(|err| ProviderError::DestroyNodeFailed(node.name().to_string(), err))?;
|
||||
names.push(node.name().to_string());
|
||||
}
|
||||
|
||||
let mut nodes = self.nodes.write().await;
|
||||
for name in names {
|
||||
nodes.remove(&name);
|
||||
}
|
||||
|
||||
if let Some(provider) = self.provider.upgrade() {
|
||||
provider.namespaces.write().await.remove(&self.name);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use support::fs::local::LocalFileSystem;
|
||||
|
||||
use super::*;
|
||||
use crate::{
|
||||
types::{GenerateFileCommand, GenerateFilesOptions},
|
||||
NativeProvider, Provider,
|
||||
};
|
||||
|
||||
fn unique_temp_dir() -> PathBuf {
|
||||
let mut base = std::env::temp_dir();
|
||||
base.push(format!("znet_native_ns_test_{}", uuid::Uuid::new_v4()));
|
||||
base
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generate_files_uses_expected_path_when_provided() {
|
||||
let fs = LocalFileSystem;
|
||||
let provider = NativeProvider::new(fs.clone());
|
||||
let base_dir = unique_temp_dir();
|
||||
// Namespace builder will create directory if needed
|
||||
let ns = provider
|
||||
.create_namespace_with_base_dir(&base_dir)
|
||||
.await
|
||||
.expect("namespace should be created");
|
||||
|
||||
// Create a unique on-host path that the native node will write to
|
||||
let expected_path =
|
||||
std::env::temp_dir().join(format!("znet_expected_{}.json", uuid::Uuid::new_v4()));
|
||||
|
||||
// Command will write JSON into expected_path; stdout will be something else to ensure we don't read it
|
||||
let program = "bash".to_string();
|
||||
let script = format!(
|
||||
"echo -n '{{\"hello\":\"world\"}}' > {} && echo should_not_be_used",
|
||||
expected_path.to_string_lossy()
|
||||
);
|
||||
let args: Vec<String> = vec!["-lc".into(), script];
|
||||
|
||||
let out_name = PathBuf::from("result_expected.json");
|
||||
let cmd = GenerateFileCommand::new(program, out_name.clone()).args(args);
|
||||
let options = GenerateFilesOptions::new(vec![cmd], None, Some(expected_path.clone()));
|
||||
|
||||
ns.generate_files(options)
|
||||
.await
|
||||
.expect("generation should succeed");
|
||||
|
||||
// Read produced file from namespace base_dir
|
||||
let produced_path = base_dir.join(out_name);
|
||||
let produced = fs
|
||||
.read_to_string(&produced_path)
|
||||
.await
|
||||
.expect("should read produced file");
|
||||
assert_eq!(produced, "{\"hello\":\"world\"}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generate_files_uses_stdout_when_expected_path_absent() {
|
||||
let fs = LocalFileSystem;
|
||||
let provider = NativeProvider::new(fs.clone());
|
||||
let base_dir = unique_temp_dir();
|
||||
let ns = provider
|
||||
.create_namespace_with_base_dir(&base_dir)
|
||||
.await
|
||||
.expect("namespace should be created");
|
||||
|
||||
// Command prints to stdout only
|
||||
let program = "bash".to_string();
|
||||
let args: Vec<String> = vec!["-lc".into(), "echo -n 42".into()];
|
||||
|
||||
let out_name = PathBuf::from("result_stdout.txt");
|
||||
let cmd = GenerateFileCommand::new(program, out_name.clone()).args(args);
|
||||
let options = GenerateFilesOptions::new(vec![cmd], None, None);
|
||||
|
||||
ns.generate_files(options)
|
||||
.await
|
||||
.expect("generation should succeed");
|
||||
|
||||
let produced_path = base_dir.join(out_name);
|
||||
let produced = fs
|
||||
.read_to_string(&produced_path)
|
||||
.await
|
||||
.expect("should read produced file");
|
||||
assert_eq!(produced, "42");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,734 @@
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
env,
|
||||
path::{Path, PathBuf},
|
||||
process::Stdio,
|
||||
sync::{Arc, Weak},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use anyhow::anyhow;
|
||||
use async_trait::async_trait;
|
||||
use configuration::types::AssetLocation;
|
||||
use flate2::read::GzDecoder;
|
||||
use futures::future::try_join_all;
|
||||
use nix::{
|
||||
sys::signal::{kill, Signal},
|
||||
unistd::Pid,
|
||||
};
|
||||
use serde::{ser::Error, Deserialize, Serialize, Serializer};
|
||||
use sha2::Digest;
|
||||
use support::{constants::THIS_IS_A_BUG, fs::FileSystem};
|
||||
use tar::Archive;
|
||||
use tokio::{
|
||||
fs,
|
||||
io::{AsyncRead, AsyncReadExt, BufReader},
|
||||
process::{Child, ChildStderr, ChildStdout, Command},
|
||||
sync::{
|
||||
mpsc::{self, Sender},
|
||||
RwLock,
|
||||
},
|
||||
task::JoinHandle,
|
||||
time::sleep,
|
||||
try_join,
|
||||
};
|
||||
use tracing::trace;
|
||||
|
||||
use super::namespace::NativeNamespace;
|
||||
use crate::{
|
||||
constants::{NODE_CONFIG_DIR, NODE_DATA_DIR, NODE_RELAY_DATA_DIR, NODE_SCRIPTS_DIR},
|
||||
native,
|
||||
types::{ExecutionResult, RunCommandOptions, RunScriptOptions, TransferedFile},
|
||||
ProviderError, ProviderNamespace, ProviderNode,
|
||||
};
|
||||
|
||||
pub(super) struct NativeNodeOptions<'a, FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
pub(super) namespace: &'a Weak<NativeNamespace<FS>>,
|
||||
pub(super) namespace_base_dir: &'a PathBuf,
|
||||
pub(super) name: &'a str,
|
||||
pub(super) program: &'a str,
|
||||
pub(super) args: &'a [String],
|
||||
pub(super) env: &'a [(String, String)],
|
||||
pub(super) startup_files: &'a [TransferedFile],
|
||||
pub(super) created_paths: &'a [PathBuf],
|
||||
pub(super) db_snapshot: Option<&'a AssetLocation>,
|
||||
pub(super) filesystem: &'a FS,
|
||||
pub(super) node_log_path: Option<&'a PathBuf>,
|
||||
}
|
||||
|
||||
impl<'a, FS> NativeNodeOptions<'a, FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
pub(super) fn from_deserializable(
|
||||
deserializable: &'a DeserializableNativeNodeOptions,
|
||||
namespace: &'a Weak<NativeNamespace<FS>>,
|
||||
namespace_base_dir: &'a PathBuf,
|
||||
filesystem: &'a FS,
|
||||
) -> NativeNodeOptions<'a, FS> {
|
||||
NativeNodeOptions {
|
||||
namespace,
|
||||
namespace_base_dir,
|
||||
name: &deserializable.name,
|
||||
program: &deserializable.program,
|
||||
args: &deserializable.args,
|
||||
env: &deserializable.env,
|
||||
startup_files: &[],
|
||||
created_paths: &[],
|
||||
db_snapshot: None,
|
||||
filesystem,
|
||||
node_log_path: deserializable.node_log_path.as_ref(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub(super) struct DeserializableNativeNodeOptions {
|
||||
pub name: String,
|
||||
pub program: String,
|
||||
pub args: Vec<String>,
|
||||
pub env: Vec<(String, String)>,
|
||||
pub node_log_path: Option<PathBuf>,
|
||||
}
|
||||
|
||||
enum ProcessHandle {
|
||||
Spawned(Child, Pid),
|
||||
Attached(Pid),
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct NativeNode<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone,
|
||||
{
|
||||
#[serde(skip)]
|
||||
namespace: Weak<NativeNamespace<FS>>,
|
||||
name: String,
|
||||
program: String,
|
||||
args: Vec<String>,
|
||||
env: Vec<(String, String)>,
|
||||
base_dir: PathBuf,
|
||||
config_dir: PathBuf,
|
||||
data_dir: PathBuf,
|
||||
relay_data_dir: PathBuf,
|
||||
scripts_dir: PathBuf,
|
||||
log_path: PathBuf,
|
||||
#[serde(serialize_with = "serialize_process_handle")]
|
||||
// using RwLock from std to serialize properly, generally using sync locks is ok in async code as long as they
|
||||
// are not held across await points
|
||||
process_handle: std::sync::RwLock<Option<ProcessHandle>>,
|
||||
#[serde(skip)]
|
||||
stdout_reading_task: RwLock<Option<JoinHandle<()>>>,
|
||||
#[serde(skip)]
|
||||
stderr_reading_task: RwLock<Option<JoinHandle<()>>>,
|
||||
#[serde(skip)]
|
||||
log_writing_task: RwLock<Option<JoinHandle<()>>>,
|
||||
#[serde(skip)]
|
||||
filesystem: FS,
|
||||
provider_tag: String,
|
||||
}
|
||||
|
||||
impl<FS> NativeNode<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
pub(super) async fn new(
|
||||
options: NativeNodeOptions<'_, FS>,
|
||||
) -> Result<Arc<Self>, ProviderError> {
|
||||
let filesystem = options.filesystem.clone();
|
||||
|
||||
let base_dir =
|
||||
PathBuf::from_iter([options.namespace_base_dir, &PathBuf::from(options.name)]);
|
||||
trace!("creating base_dir {:?}", base_dir);
|
||||
options.filesystem.create_dir_all(&base_dir).await?;
|
||||
trace!("created base_dir {:?}", base_dir);
|
||||
|
||||
let base_dir_raw = base_dir.to_string_lossy();
|
||||
let config_dir = PathBuf::from(format!("{base_dir_raw}{NODE_CONFIG_DIR}"));
|
||||
let data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_DATA_DIR}"));
|
||||
let relay_data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_RELAY_DATA_DIR}"));
|
||||
let scripts_dir = PathBuf::from(format!("{base_dir_raw}{NODE_SCRIPTS_DIR}"));
|
||||
let log_path = options
|
||||
.node_log_path
|
||||
.cloned()
|
||||
.unwrap_or_else(|| base_dir.join(format!("{}.log", options.name)));
|
||||
|
||||
trace!("creating dirs {:?}", config_dir);
|
||||
try_join!(
|
||||
filesystem.create_dir_all(&config_dir),
|
||||
filesystem.create_dir_all(&data_dir),
|
||||
filesystem.create_dir_all(&relay_data_dir),
|
||||
filesystem.create_dir_all(&scripts_dir),
|
||||
)?;
|
||||
trace!("created!");
|
||||
|
||||
let node = Arc::new(NativeNode {
|
||||
namespace: options.namespace.clone(),
|
||||
name: options.name.to_string(),
|
||||
program: options.program.to_string(),
|
||||
args: options.args.to_vec(),
|
||||
env: options.env.to_vec(),
|
||||
base_dir,
|
||||
config_dir,
|
||||
data_dir,
|
||||
relay_data_dir,
|
||||
scripts_dir,
|
||||
log_path,
|
||||
process_handle: std::sync::RwLock::new(None),
|
||||
stdout_reading_task: RwLock::new(None),
|
||||
stderr_reading_task: RwLock::new(None),
|
||||
log_writing_task: RwLock::new(None),
|
||||
filesystem: filesystem.clone(),
|
||||
provider_tag: native::provider::PROVIDER_NAME.to_string(),
|
||||
});
|
||||
|
||||
node.initialize_startup_paths(options.created_paths).await?;
|
||||
node.initialize_startup_files(options.startup_files).await?;
|
||||
|
||||
if let Some(db_snap) = options.db_snapshot {
|
||||
node.initialize_db_snapshot(db_snap).await?;
|
||||
}
|
||||
|
||||
let (stdout, stderr) = node.initialize_process().await?;
|
||||
|
||||
node.initialize_log_writing(stdout, stderr).await;
|
||||
|
||||
Ok(node)
|
||||
}
|
||||
|
||||
pub(super) async fn attach_to_live(
|
||||
options: NativeNodeOptions<'_, FS>,
|
||||
pid: i32,
|
||||
) -> Result<Arc<Self>, ProviderError> {
|
||||
let filesystem = options.filesystem.clone();
|
||||
|
||||
let base_dir =
|
||||
PathBuf::from_iter([options.namespace_base_dir, &PathBuf::from(options.name)]);
|
||||
trace!("creating base_dir {:?}", base_dir);
|
||||
options.filesystem.create_dir_all(&base_dir).await?;
|
||||
trace!("created base_dir {:?}", base_dir);
|
||||
|
||||
let base_dir_raw = base_dir.to_string_lossy();
|
||||
let config_dir = PathBuf::from(format!("{base_dir_raw}{NODE_CONFIG_DIR}"));
|
||||
let data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_DATA_DIR}"));
|
||||
let relay_data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_RELAY_DATA_DIR}"));
|
||||
let scripts_dir = PathBuf::from(format!("{base_dir_raw}{NODE_SCRIPTS_DIR}"));
|
||||
let log_path = options
|
||||
.node_log_path
|
||||
.cloned()
|
||||
.unwrap_or_else(|| base_dir.join(format!("{}.log", options.name)));
|
||||
|
||||
let pid = Pid::from_raw(pid);
|
||||
|
||||
let node = Arc::new(NativeNode {
|
||||
namespace: options.namespace.clone(),
|
||||
name: options.name.to_string(),
|
||||
program: options.program.to_string(),
|
||||
args: options.args.to_vec(),
|
||||
env: options.env.to_vec(),
|
||||
base_dir,
|
||||
config_dir,
|
||||
data_dir,
|
||||
relay_data_dir,
|
||||
scripts_dir,
|
||||
log_path,
|
||||
process_handle: std::sync::RwLock::new(Some(ProcessHandle::Attached(pid))),
|
||||
stdout_reading_task: RwLock::new(None),
|
||||
stderr_reading_task: RwLock::new(None),
|
||||
log_writing_task: RwLock::new(None),
|
||||
filesystem: filesystem.clone(),
|
||||
provider_tag: native::provider::PROVIDER_NAME.to_string(),
|
||||
});
|
||||
|
||||
Ok(node)
|
||||
}
|
||||
|
||||
async fn initialize_startup_paths(&self, paths: &[PathBuf]) -> Result<(), ProviderError> {
|
||||
trace!("creating paths {:?}", paths);
|
||||
let base_dir_raw = self.base_dir.to_string_lossy();
|
||||
try_join_all(paths.iter().map(|file| {
|
||||
let full_path = format!("{base_dir_raw}{}", file.to_string_lossy());
|
||||
self.filesystem.create_dir_all(full_path)
|
||||
}))
|
||||
.await?;
|
||||
trace!("paths created!");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn initialize_startup_files(
|
||||
&self,
|
||||
startup_files: &[TransferedFile],
|
||||
) -> Result<(), ProviderError> {
|
||||
trace!("creating files {:?}", startup_files);
|
||||
try_join_all(
|
||||
startup_files
|
||||
.iter()
|
||||
.map(|file| self.send_file(&file.local_path, &file.remote_path, &file.mode)),
|
||||
)
|
||||
.await?;
|
||||
trace!("files created!");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn initialize_db_snapshot(
|
||||
&self,
|
||||
db_snapshot: &AssetLocation,
|
||||
) -> Result<(), ProviderError> {
|
||||
trace!("snap: {db_snapshot}");
|
||||
|
||||
// check if we need to get the db or is already in the ns
|
||||
let ns_base_dir = self.namespace_base_dir();
|
||||
let hashed_location = match db_snapshot {
|
||||
AssetLocation::Url(location) => hex::encode(sha2::Sha256::digest(location.to_string())),
|
||||
AssetLocation::FilePath(filepath) => {
|
||||
hex::encode(sha2::Sha256::digest(filepath.to_string_lossy().to_string()))
|
||||
},
|
||||
};
|
||||
|
||||
let full_path = format!("{ns_base_dir}/{hashed_location}.tgz");
|
||||
trace!("db_snap fullpath in ns: {full_path}");
|
||||
if !self.filesystem.exists(&full_path).await {
|
||||
// needs to download/copy
|
||||
self.get_db_snapshot(db_snapshot, &full_path).await?;
|
||||
}
|
||||
|
||||
let contents = self.filesystem.read(&full_path).await.unwrap();
|
||||
let gz = GzDecoder::new(&contents[..]);
|
||||
let mut archive = Archive::new(gz);
|
||||
archive
|
||||
.unpack(self.base_dir.to_string_lossy().as_ref())
|
||||
.unwrap();
|
||||
|
||||
if std::env::var("ZOMBIE_RM_TGZ_AFTER_EXTRACT").is_ok() {
|
||||
let res = fs::remove_file(&full_path).await;
|
||||
trace!("removing {}, result {:?}", full_path, res);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn get_db_snapshot(
|
||||
&self,
|
||||
location: &AssetLocation,
|
||||
full_path: &str,
|
||||
) -> Result<(), ProviderError> {
|
||||
trace!("getting db_snapshot from: {:?} to: {full_path}", location);
|
||||
match location {
|
||||
AssetLocation::Url(location) => {
|
||||
let res = reqwest::get(location.as_ref())
|
||||
.await
|
||||
.map_err(|err| ProviderError::DownloadFile(location.to_string(), err.into()))?;
|
||||
|
||||
let contents: &[u8] = &res.bytes().await.unwrap();
|
||||
trace!("writing: {full_path}");
|
||||
self.filesystem.write(full_path, contents).await?;
|
||||
},
|
||||
AssetLocation::FilePath(filepath) => {
|
||||
self.filesystem.copy(filepath, full_path).await?;
|
||||
},
|
||||
};
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn initialize_process(&self) -> Result<(ChildStdout, ChildStderr), ProviderError> {
|
||||
let filtered_env: HashMap<String, String> = env::vars()
|
||||
.filter(|(k, _)| k == "TZ" || k == "LANG" || k == "PATH")
|
||||
.collect();
|
||||
|
||||
let mut process = Command::new(&self.program)
|
||||
.args(&self.args)
|
||||
.env_clear()
|
||||
.envs(&filtered_env) // minimal environment
|
||||
.envs(self.env.to_vec())
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.kill_on_drop(true)
|
||||
.current_dir(&self.base_dir)
|
||||
.spawn()
|
||||
.map_err(|err| ProviderError::NodeSpawningFailed(self.name.to_string(), err.into()))?;
|
||||
let stdout = process
|
||||
.stdout
|
||||
.take()
|
||||
.expect(&format!("infaillible, stdout is piped {THIS_IS_A_BUG}"));
|
||||
let stderr = process
|
||||
.stderr
|
||||
.take()
|
||||
.expect(&format!("infaillible, stderr is piped {THIS_IS_A_BUG}"));
|
||||
|
||||
let pid = Pid::from_raw(
|
||||
process
|
||||
.id()
|
||||
.ok_or_else(|| ProviderError::ProcessIdRetrievalFailed(self.name.to_string()))?
|
||||
as i32,
|
||||
);
|
||||
self.process_handle
|
||||
.write()
|
||||
.map_err(|_e| ProviderError::FailedToAcquireLock(self.name.clone()))?
|
||||
.replace(ProcessHandle::Spawned(process, pid));
|
||||
|
||||
Ok((stdout, stderr))
|
||||
}
|
||||
|
||||
async fn initialize_log_writing(&self, stdout: ChildStdout, stderr: ChildStderr) {
|
||||
let (stdout_tx, mut rx) = mpsc::channel(10);
|
||||
let stderr_tx = stdout_tx.clone();
|
||||
|
||||
self.stdout_reading_task
|
||||
.write()
|
||||
.await
|
||||
.replace(self.create_stream_polling_task(stdout, stdout_tx));
|
||||
self.stderr_reading_task
|
||||
.write()
|
||||
.await
|
||||
.replace(self.create_stream_polling_task(stderr, stderr_tx));
|
||||
|
||||
let filesystem = self.filesystem.clone();
|
||||
let log_path = self.log_path.clone();
|
||||
|
||||
self.log_writing_task
|
||||
.write()
|
||||
.await
|
||||
.replace(tokio::spawn(async move {
|
||||
loop {
|
||||
while let Some(Ok(data)) = rx.recv().await {
|
||||
// TODO: find a better way instead of ignoring error ?
|
||||
let _ = filesystem.append(&log_path, data).await;
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
fn create_stream_polling_task(
|
||||
&self,
|
||||
stream: impl AsyncRead + Unpin + Send + 'static,
|
||||
tx: Sender<Result<Vec<u8>, std::io::Error>>,
|
||||
) -> JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
let mut reader = BufReader::new(stream);
|
||||
let mut buffer = vec![0u8; 1024];
|
||||
|
||||
loop {
|
||||
match reader.read(&mut buffer).await {
|
||||
Ok(0) => {
|
||||
let _ = tx.send(Ok(Vec::new())).await;
|
||||
break;
|
||||
},
|
||||
Ok(n) => {
|
||||
let _ = tx.send(Ok(buffer[..n].to_vec())).await;
|
||||
},
|
||||
Err(e) => {
|
||||
let _ = tx.send(Err(e)).await;
|
||||
break;
|
||||
},
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn process_id(&self) -> Result<Pid, ProviderError> {
|
||||
let pid = self
|
||||
.process_handle
|
||||
.read()
|
||||
.map_err(|_e| ProviderError::FailedToAcquireLock(self.name.clone()))?
|
||||
.as_ref()
|
||||
.map(|handle| match handle {
|
||||
ProcessHandle::Spawned(_, pid) => *pid,
|
||||
ProcessHandle::Attached(pid) => *pid,
|
||||
})
|
||||
.ok_or_else(|| ProviderError::ProcessIdRetrievalFailed(self.name.to_string()))?;
|
||||
|
||||
Ok(pid)
|
||||
}
|
||||
|
||||
pub(crate) async fn abort(&self) -> anyhow::Result<()> {
|
||||
if let Some(task) = self.log_writing_task.write().await.take() {
|
||||
task.abort();
|
||||
}
|
||||
if let Some(task) = self.stdout_reading_task.write().await.take() {
|
||||
task.abort();
|
||||
}
|
||||
if let Some(task) = self.stderr_reading_task.write().await.take() {
|
||||
task.abort();
|
||||
}
|
||||
|
||||
let process_handle = {
|
||||
let mut guard = self
|
||||
.process_handle
|
||||
.write()
|
||||
.map_err(|_e| ProviderError::FailedToAcquireLock(self.name.clone()))?;
|
||||
guard
|
||||
.take()
|
||||
.ok_or_else(|| anyhow!("no process was attached for the node"))?
|
||||
};
|
||||
|
||||
match process_handle {
|
||||
ProcessHandle::Spawned(mut child, _pid) => {
|
||||
child.kill().await?;
|
||||
},
|
||||
ProcessHandle::Attached(pid) => {
|
||||
kill(pid, Signal::SIGKILL)
|
||||
.map_err(|err| anyhow!("Failed to kill attached process {pid}: {err}"))?;
|
||||
},
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn namespace_base_dir(&self) -> String {
|
||||
self.namespace
|
||||
.upgrade()
|
||||
.map(|namespace| namespace.base_dir().to_string_lossy().to_string())
|
||||
.unwrap_or_else(|| panic!("namespace shouldn't be dropped, {THIS_IS_A_BUG}"))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<FS> ProviderNode for NativeNode<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
fn name(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
|
||||
fn args(&self) -> Vec<&str> {
|
||||
self.args.iter().map(|arg| arg.as_str()).collect()
|
||||
}
|
||||
|
||||
fn base_dir(&self) -> &PathBuf {
|
||||
&self.base_dir
|
||||
}
|
||||
|
||||
fn config_dir(&self) -> &PathBuf {
|
||||
&self.config_dir
|
||||
}
|
||||
|
||||
fn data_dir(&self) -> &PathBuf {
|
||||
&self.data_dir
|
||||
}
|
||||
|
||||
fn relay_data_dir(&self) -> &PathBuf {
|
||||
&self.relay_data_dir
|
||||
}
|
||||
|
||||
fn scripts_dir(&self) -> &PathBuf {
|
||||
&self.scripts_dir
|
||||
}
|
||||
|
||||
fn log_path(&self) -> &PathBuf {
|
||||
&self.log_path
|
||||
}
|
||||
|
||||
fn log_cmd(&self) -> String {
|
||||
format!("tail -f {}", self.log_path().to_string_lossy())
|
||||
}
|
||||
|
||||
fn path_in_node(&self, file: &Path) -> PathBuf {
|
||||
let full_path = format!(
|
||||
"{}/{}",
|
||||
self.base_dir.to_string_lossy(),
|
||||
file.to_string_lossy()
|
||||
);
|
||||
PathBuf::from(full_path)
|
||||
}
|
||||
|
||||
async fn logs(&self) -> Result<String, ProviderError> {
|
||||
Ok(self.filesystem.read_to_string(&self.log_path).await?)
|
||||
}
|
||||
|
||||
async fn dump_logs(&self, local_dest: PathBuf) -> Result<(), ProviderError> {
|
||||
Ok(self.filesystem.copy(&self.log_path, local_dest).await?)
|
||||
}
|
||||
|
||||
async fn run_command(
|
||||
&self,
|
||||
options: RunCommandOptions,
|
||||
) -> Result<ExecutionResult, ProviderError> {
|
||||
let result = Command::new(options.program.clone())
|
||||
.args(options.args.clone())
|
||||
.envs(options.env)
|
||||
.current_dir(&self.base_dir)
|
||||
.output()
|
||||
.await
|
||||
.map_err(|err| {
|
||||
ProviderError::RunCommandError(
|
||||
format!("{} {}", &options.program, &options.args.join(" ")),
|
||||
"locally".to_string(),
|
||||
err.into(),
|
||||
)
|
||||
})?;
|
||||
|
||||
if result.status.success() {
|
||||
Ok(Ok(String::from_utf8_lossy(&result.stdout).to_string()))
|
||||
} else {
|
||||
Ok(Err((
|
||||
result.status,
|
||||
String::from_utf8_lossy(&result.stderr).to_string(),
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_script(
|
||||
&self,
|
||||
options: RunScriptOptions,
|
||||
) -> Result<ExecutionResult, ProviderError> {
|
||||
let local_script_path = PathBuf::from(&options.local_script_path);
|
||||
|
||||
if !self.filesystem.exists(&local_script_path).await {
|
||||
return Err(ProviderError::ScriptNotFound(local_script_path));
|
||||
}
|
||||
|
||||
// extract file name and build remote file path
|
||||
let script_file_name = local_script_path
|
||||
.file_name()
|
||||
.map(|file_name| file_name.to_string_lossy().to_string())
|
||||
.ok_or(ProviderError::InvalidScriptPath(anyhow!(
|
||||
"Can't retrieve filename from script with path: {:?}",
|
||||
options.local_script_path
|
||||
)))?;
|
||||
let remote_script_path = format!(
|
||||
"{}/{}",
|
||||
self.scripts_dir.to_string_lossy(),
|
||||
script_file_name
|
||||
);
|
||||
|
||||
// copy and set script's execute permission
|
||||
self.filesystem
|
||||
.copy(local_script_path, &remote_script_path)
|
||||
.await?;
|
||||
self.filesystem.set_mode(&remote_script_path, 0o744).await?;
|
||||
|
||||
// execute script
|
||||
self.run_command(RunCommandOptions {
|
||||
program: remote_script_path,
|
||||
args: options.args,
|
||||
env: options.env,
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn send_file(
|
||||
&self,
|
||||
local_file_path: &Path,
|
||||
remote_file_path: &Path,
|
||||
mode: &str,
|
||||
) -> Result<(), ProviderError> {
|
||||
let namespaced_remote_file_path = PathBuf::from(format!(
|
||||
"{}{}",
|
||||
&self.base_dir.to_string_lossy(),
|
||||
remote_file_path.to_string_lossy()
|
||||
));
|
||||
|
||||
self.filesystem
|
||||
.copy(local_file_path, &namespaced_remote_file_path)
|
||||
.await?;
|
||||
|
||||
self.run_command(
|
||||
RunCommandOptions::new("chmod")
|
||||
.args(vec![mode, &namespaced_remote_file_path.to_string_lossy()]),
|
||||
)
|
||||
.await?
|
||||
.map_err(|(_, err)| {
|
||||
ProviderError::SendFile(
|
||||
self.name.clone(),
|
||||
local_file_path.to_string_lossy().to_string(),
|
||||
anyhow!("{err}"),
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn receive_file(
|
||||
&self,
|
||||
remote_file_path: &Path,
|
||||
local_file_path: &Path,
|
||||
) -> Result<(), ProviderError> {
|
||||
let namespaced_remote_file_path = PathBuf::from(format!(
|
||||
"{}{}",
|
||||
&self.base_dir.to_string_lossy(),
|
||||
remote_file_path.to_string_lossy()
|
||||
));
|
||||
|
||||
self.filesystem
|
||||
.copy(namespaced_remote_file_path, local_file_path)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn pause(&self) -> Result<(), ProviderError> {
|
||||
let process_id = self.process_id()?;
|
||||
|
||||
kill(process_id, Signal::SIGSTOP)
|
||||
.map_err(|err| ProviderError::PauseNodeFailed(self.name.clone(), err.into()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn resume(&self) -> Result<(), ProviderError> {
|
||||
let process_id = self.process_id()?;
|
||||
|
||||
nix::sys::signal::kill(process_id, Signal::SIGCONT)
|
||||
.map_err(|err| ProviderError::ResumeNodeFailed(self.name.clone(), err.into()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn restart(&self, after: Option<Duration>) -> Result<(), ProviderError> {
|
||||
if let Some(duration) = after {
|
||||
sleep(duration).await;
|
||||
}
|
||||
|
||||
self.abort()
|
||||
.await
|
||||
.map_err(|err| ProviderError::RestartNodeFailed(self.name.clone(), err))?;
|
||||
|
||||
let (stdout, stderr) = self
|
||||
.initialize_process()
|
||||
.await
|
||||
.map_err(|err| ProviderError::RestartNodeFailed(self.name.clone(), err.into()))?;
|
||||
|
||||
self.initialize_log_writing(stdout, stderr).await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn destroy(&self) -> Result<(), ProviderError> {
|
||||
self.abort()
|
||||
.await
|
||||
.map_err(|err| ProviderError::DestroyNodeFailed(self.name.clone(), err))?;
|
||||
|
||||
if let Some(namespace) = self.namespace.upgrade() {
|
||||
namespace.nodes.write().await.remove(&self.name);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn serialize_process_handle<S>(
|
||||
process_handle: &std::sync::RwLock<Option<ProcessHandle>>,
|
||||
serializer: S,
|
||||
) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
let pid = process_handle
|
||||
.read()
|
||||
.map_err(|_e| S::Error::custom("failed to acquire read lock"))?
|
||||
.as_ref()
|
||||
.map(|handle| match handle {
|
||||
ProcessHandle::Spawned(_, pid) => pid.as_raw(),
|
||||
ProcessHandle::Attached(pid) => pid.as_raw(),
|
||||
});
|
||||
pid.serialize(serializer)
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, Weak},
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use support::fs::FileSystem;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use super::namespace::NativeNamespace;
|
||||
use crate::{
|
||||
shared::helpers::extract_namespace_info, types::ProviderCapabilities, DynNamespace, Provider,
|
||||
ProviderError, ProviderNamespace,
|
||||
};
|
||||
|
||||
pub const PROVIDER_NAME: &str = "native";
|
||||
|
||||
pub struct NativeProvider<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone,
|
||||
{
|
||||
weak: Weak<NativeProvider<FS>>,
|
||||
capabilities: ProviderCapabilities,
|
||||
tmp_dir: PathBuf,
|
||||
filesystem: FS,
|
||||
pub(super) namespaces: RwLock<HashMap<String, Arc<NativeNamespace<FS>>>>,
|
||||
}
|
||||
|
||||
impl<FS> NativeProvider<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone,
|
||||
{
|
||||
pub fn new(filesystem: FS) -> Arc<Self> {
|
||||
Arc::new_cyclic(|weak| NativeProvider {
|
||||
weak: weak.clone(),
|
||||
capabilities: ProviderCapabilities {
|
||||
has_resources: false,
|
||||
requires_image: false,
|
||||
prefix_with_full_path: true,
|
||||
use_default_ports_in_cmd: false,
|
||||
},
|
||||
// NOTE: temp_dir in linux return `/tmp` but on mac something like
|
||||
// `/var/folders/rz/1cyx7hfj31qgb98d8_cg7jwh0000gn/T/`, having
|
||||
// one `trailing slash` and the other no can cause issues if
|
||||
// you try to build a fullpath by concatenate. Use Pathbuf to prevent the issue.
|
||||
tmp_dir: std::env::temp_dir(),
|
||||
filesystem,
|
||||
namespaces: RwLock::new(HashMap::new()),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn tmp_dir(mut self, tmp_dir: impl Into<PathBuf>) -> Self {
|
||||
self.tmp_dir = tmp_dir.into();
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<FS> Provider for NativeProvider<FS>
|
||||
where
|
||||
FS: FileSystem + Send + Sync + Clone + 'static,
|
||||
{
|
||||
fn name(&self) -> &str {
|
||||
PROVIDER_NAME
|
||||
}
|
||||
|
||||
fn capabilities(&self) -> &ProviderCapabilities {
|
||||
&self.capabilities
|
||||
}
|
||||
|
||||
async fn namespaces(&self) -> HashMap<String, DynNamespace> {
|
||||
self.namespaces
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.map(|(name, namespace)| (name.clone(), namespace.clone() as DynNamespace))
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn create_namespace(&self) -> Result<DynNamespace, ProviderError> {
|
||||
let namespace = NativeNamespace::new(
|
||||
&self.weak,
|
||||
&self.tmp_dir,
|
||||
&self.capabilities,
|
||||
&self.filesystem,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
self.namespaces
|
||||
.write()
|
||||
.await
|
||||
.insert(namespace.name().to_string(), namespace.clone());
|
||||
|
||||
Ok(namespace)
|
||||
}
|
||||
|
||||
async fn create_namespace_with_base_dir(
|
||||
&self,
|
||||
base_dir: &Path,
|
||||
) -> Result<DynNamespace, ProviderError> {
|
||||
let namespace = NativeNamespace::new(
|
||||
&self.weak,
|
||||
&self.tmp_dir,
|
||||
&self.capabilities,
|
||||
&self.filesystem,
|
||||
Some(base_dir),
|
||||
)
|
||||
.await?;
|
||||
|
||||
self.namespaces
|
||||
.write()
|
||||
.await
|
||||
.insert(namespace.name().to_string(), namespace.clone());
|
||||
|
||||
Ok(namespace)
|
||||
}
|
||||
|
||||
async fn create_namespace_from_json(
|
||||
&self,
|
||||
json_value: &serde_json::Value,
|
||||
) -> Result<DynNamespace, ProviderError> {
|
||||
let (base_dir, name) = extract_namespace_info(json_value)?;
|
||||
|
||||
let namespace = NativeNamespace::attach_to_live(
|
||||
&self.weak,
|
||||
&self.capabilities,
|
||||
&self.filesystem,
|
||||
&base_dir,
|
||||
&name,
|
||||
)
|
||||
.await?;
|
||||
|
||||
self.namespaces
|
||||
.write()
|
||||
.await
|
||||
.insert(namespace.name().to_string(), namespace.clone());
|
||||
|
||||
Ok(namespace)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
pub mod constants;
|
||||
pub mod helpers;
|
||||
pub mod types;
|
||||
@@ -0,0 +1,22 @@
|
||||
use std::net::{IpAddr, Ipv4Addr};
|
||||
|
||||
/// Namespace prefix
|
||||
pub const NAMESPACE_PREFIX: &str = "zombie-";
|
||||
/// Directory for node configuration
|
||||
pub const NODE_CONFIG_DIR: &str = "/cfg";
|
||||
/// Directory for node data dir
|
||||
pub const NODE_DATA_DIR: &str = "/data";
|
||||
/// Directory for node relay data dir
|
||||
pub const NODE_RELAY_DATA_DIR: &str = "/relay-data";
|
||||
/// Directory for node scripts
|
||||
pub const NODE_SCRIPTS_DIR: &str = "/scripts";
|
||||
/// Localhost ip
|
||||
pub const LOCALHOST: IpAddr = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1));
|
||||
/// The port substrate listens for p2p connections on
|
||||
pub const P2P_PORT: u16 = 30333;
|
||||
/// The remote port Prometheus can be accessed with
|
||||
pub const PROMETHEUS_PORT: u16 = 9615;
|
||||
/// The remote port websocket to access the RPC
|
||||
pub const RPC_WS_PORT: u16 = 9944;
|
||||
/// The remote port HTTP to access the RPC
|
||||
pub const RPC_HTTP_PORT: u16 = 9933;
|
||||
@@ -0,0 +1,79 @@
|
||||
use std::{env, path::PathBuf};
|
||||
|
||||
use anyhow::anyhow;
|
||||
|
||||
use crate::{types::RunCommandOptions, DynNode, ProviderError};
|
||||
|
||||
/// Check if we are running in `CI` by checking the 'RUN_IN_CI' env var
|
||||
pub fn running_in_ci() -> bool {
|
||||
env::var("RUN_IN_CI").unwrap_or_default() == "1"
|
||||
}
|
||||
|
||||
/// Executes a command on a temporary node and extracts the execution result either from the
|
||||
/// standard output or a file.
|
||||
pub async fn extract_execution_result(
|
||||
temp_node: &DynNode,
|
||||
options: RunCommandOptions,
|
||||
expected_path: Option<&PathBuf>,
|
||||
) -> Result<String, ProviderError> {
|
||||
let output_contents = temp_node
|
||||
.run_command(options)
|
||||
.await?
|
||||
.map_err(|(_, msg)| ProviderError::FileGenerationFailed(anyhow!("{msg}")))?;
|
||||
|
||||
// If an expected_path is provided, read the file contents from inside the container
|
||||
if let Some(expected_path) = expected_path.as_ref() {
|
||||
Ok(temp_node
|
||||
.run_command(
|
||||
RunCommandOptions::new("cat")
|
||||
.args(vec![expected_path.to_string_lossy().to_string()]),
|
||||
)
|
||||
.await?
|
||||
.map_err(|(_, msg)| {
|
||||
ProviderError::FileGenerationFailed(anyhow!(format!(
|
||||
"failed reading expected_path {}: {}",
|
||||
expected_path.display(),
|
||||
msg
|
||||
)))
|
||||
})?)
|
||||
} else {
|
||||
Ok(output_contents)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn extract_namespace_info(
|
||||
json_value: &serde_json::Value,
|
||||
) -> Result<(PathBuf, String), ProviderError> {
|
||||
let base_dir = json_value
|
||||
.get("local_base_dir")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(PathBuf::from)
|
||||
.ok_or(ProviderError::InvalidConfig(
|
||||
"`field local_base_dir` is missing from zombie.json".to_string(),
|
||||
))?;
|
||||
|
||||
let name =
|
||||
json_value
|
||||
.get("ns")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or(ProviderError::InvalidConfig(
|
||||
"field `ns` is missing from zombie.json".to_string(),
|
||||
))?;
|
||||
|
||||
Ok((base_dir, name.to_string()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn check_runing_in_ci_env_var() {
|
||||
assert!(!running_in_ci());
|
||||
// now set the env var
|
||||
env::set_var("RUN_IN_CI", "1");
|
||||
assert!(running_in_ci());
|
||||
// reset
|
||||
env::set_var("RUN_IN_CI", "");
|
||||
}
|
||||
}
|
||||
Vendored
+34
@@ -0,0 +1,34 @@
|
||||
#!/bin/ash
|
||||
|
||||
log() {
|
||||
echo "$(date +"%F %T") $1"
|
||||
}
|
||||
|
||||
# used to handle the distinction where /cfg is used for k8s and /helpers for docker/podman
|
||||
# to share a volume across nodes containing helper binaries and independent from /cfg
|
||||
# where some node files are stored
|
||||
OUTDIR=$([ -d /helpers ] && echo "/helpers" || echo "/cfg")
|
||||
|
||||
# Allow to use our image and just cp'd the binaries.
|
||||
|
||||
if [ -f /tmp/curl ]; then
|
||||
cp /tmp/curl $OUTDIR/curl
|
||||
log "curl copied"
|
||||
else
|
||||
wget github.com/moparisthebest/static-curl/releases/download/v7.83.1/curl-amd64 -O "$OUTDIR/curl"
|
||||
log "curl downloaded"
|
||||
fi;
|
||||
|
||||
chmod +x "$OUTDIR/curl"
|
||||
log "curl chmoded"
|
||||
|
||||
if [ -f /tmp/coreutils ]; then
|
||||
cp /tmp/coreutils $OUTDIR/coreutils
|
||||
log "coreutils copied"
|
||||
else
|
||||
wget -qO- github.com/uutils/coreutils/releases/download/0.0.17/coreutils-0.0.17-x86_64-unknown-linux-musl.tar.gz | tar -xz -C $OUTDIR --strip-components=1 coreutils-0.0.17-x86_64-unknown-linux-musl/coreutils
|
||||
log "coreutils downloaded"
|
||||
fi;
|
||||
|
||||
chmod +x "$OUTDIR/coreutils"
|
||||
log "coreutils chmoded"
|
||||
Vendored
Executable
+178
@@ -0,0 +1,178 @@
|
||||
#!/bin/bash
|
||||
set -uxo pipefail
|
||||
|
||||
if [ -f /cfg/coreutils ]; then
|
||||
RM="/cfg/coreutils rm"
|
||||
MKFIFO="/cfg/coreutils mkfifo"
|
||||
MKNOD="/cfg/coreutils mknod"
|
||||
LS="/cfg/coreutils ls"
|
||||
KILL="/cfg/coreutils kill"
|
||||
SLEEP="/cfg/coreutils sleep"
|
||||
ECHO="/cfg/coreutils echo"
|
||||
elif [ -f /helpers/coreutils ]; then
|
||||
# used for docker/podman to have a single volume sharing helper binaries
|
||||
# across nodes independent from the /cfg where some files are stored
|
||||
# by the node itself
|
||||
RM="/helpers/coreutils rm"
|
||||
MKFIFO="/helpers/coreutils mkfifo"
|
||||
MKNOD="/helpers/coreutils mknod"
|
||||
LS="/helpers/coreutils ls"
|
||||
KILL="/helpers/coreutils kill"
|
||||
SLEEP="/helpers/coreutils sleep"
|
||||
ECHO="/helpers/coreutils echo"
|
||||
else
|
||||
RM="rm"
|
||||
MKFIFO="mkfifo"
|
||||
MKNOD="mknod"
|
||||
LS="ls"
|
||||
KILL="kill"
|
||||
SLEEP="sleep"
|
||||
ECHO="echo"
|
||||
fi
|
||||
|
||||
echo "COMMANDS DEFINED"
|
||||
|
||||
# add /cfg as first `looking dir` to allow to overrides commands.
|
||||
export PATH="/cfg":$PATH
|
||||
|
||||
echo "EXPORT PATH"
|
||||
|
||||
# setup pipe
|
||||
pipe=/tmp/zombiepipe
|
||||
trap "$RM -f $pipe" EXIT
|
||||
|
||||
# try mkfifo first and allow to fail
|
||||
if [[ ! -p $pipe ]]; then
|
||||
$MKFIFO $pipe
|
||||
fi
|
||||
|
||||
# set immediately exit on any non 0 exit code
|
||||
set -e
|
||||
|
||||
# if fails try mknod
|
||||
if [[ ! -p $pipe ]]; then
|
||||
$MKNOD $pipe p
|
||||
fi
|
||||
|
||||
echo "PIPE CREATED"
|
||||
|
||||
# init empty
|
||||
child_pid=""
|
||||
|
||||
# get the command to exec
|
||||
CMD=($@)
|
||||
|
||||
# File to store CMD (and update from there)
|
||||
ZOMBIE_CMD_FILE=/tmp/zombie.cmd
|
||||
ZOMBIE_CMD_PID=/tmp/zombie.pid
|
||||
|
||||
# Store the cmd and make it available to later usage
|
||||
# NOTE: echo without new line to allow to customize the cmd later
|
||||
$ECHO -n "${CMD[@]}" > $ZOMBIE_CMD_FILE
|
||||
|
||||
echo "COMMAND TO RUN IS: $CMD"
|
||||
|
||||
start() {
|
||||
# redirect the output to be expored to loki
|
||||
"${CMD[@]}" >> /proc/1/fd/1 2>> /proc/1/fd/2 &
|
||||
if [[ "$CMD" != "cat" ]]; then
|
||||
child_pid="$!"
|
||||
|
||||
$ECHO $(cat $ZOMBIE_CMD_FILE)
|
||||
# store pid
|
||||
$ECHO ${child_pid} > $ZOMBIE_CMD_PID
|
||||
|
||||
# sleep a few secs to detect errors bootstraping the node
|
||||
sleep 3
|
||||
|
||||
# check if the process is running
|
||||
if ! $LS /proc/$child_pid > /dev/null 2>&1 ; then
|
||||
echo "child process doesn't exist, quiting...";
|
||||
exit 1;
|
||||
else
|
||||
echo "PID: $child_pid alive";
|
||||
fi;
|
||||
else
|
||||
echo "Process not started, PID not stored, since was 'cat'";
|
||||
fi;
|
||||
|
||||
}
|
||||
|
||||
restart() {
|
||||
if [ ! -z "${child_pid}" ]; then
|
||||
$KILL -9 "$child_pid"
|
||||
fi
|
||||
|
||||
# check if we have timeout
|
||||
if [[ "$1" ]]; then
|
||||
$SLEEP "$1"
|
||||
fi
|
||||
|
||||
start
|
||||
}
|
||||
|
||||
pause() {
|
||||
if [ ! -z "${child_pid}" ]; then
|
||||
echo "send -STOP to process $child_pid"
|
||||
$KILL -STOP "$child_pid"
|
||||
echo "result $?"
|
||||
|
||||
# Wait until the process is actually stopped (state 'T')
|
||||
for i in {1..10}; do
|
||||
local state
|
||||
state=$(awk '{print $3}' /proc/$child_pid/stat 2>/dev/null)
|
||||
if [ "$state" = "T" ]; then
|
||||
echo "Process $child_pid is paused (state: $state)"
|
||||
return
|
||||
fi
|
||||
$SLEEP 0.2
|
||||
done
|
||||
|
||||
echo "Warning: Process $child_pid not paused after SIGSTOP"
|
||||
fi
|
||||
}
|
||||
|
||||
resume() {
|
||||
if [ ! -z "${child_pid}" ]; then
|
||||
echo "send -CONT to process $child_pid"
|
||||
$KILL -CONT "$child_pid"
|
||||
echo "result $?"
|
||||
|
||||
# Wait until the process is actually resumed (state not 'T')
|
||||
for i in {1..10}; do
|
||||
local state
|
||||
state=$(awk '{print $3}' /proc/$child_pid/stat 2>/dev/null)
|
||||
if [ "$state" != "T" ] && [ -n "$state" ]; then
|
||||
echo "Process $child_pid is resumed (state: $state)"
|
||||
return
|
||||
fi
|
||||
$SLEEP 0.2
|
||||
done
|
||||
|
||||
echo "Warning: Process $child_pid not resumed after SIGCONT"
|
||||
fi
|
||||
}
|
||||
|
||||
# keep listening from the pipe
|
||||
while read line <$pipe
|
||||
echo "read line: ${line}"
|
||||
do
|
||||
if [[ "$line" == "start" ]]; then
|
||||
start
|
||||
elif [[ "$line" == "quit" ]]; then
|
||||
break
|
||||
elif [[ "$line" =~ "restart" ]]; then
|
||||
# check if we have timeout between restart
|
||||
if [[ $line =~ [^0-9]+([0-9]+) ]]; then
|
||||
restart "${BASH_REMATCH[1]}"
|
||||
else
|
||||
restart 0
|
||||
fi;
|
||||
elif [[ "$line" == "pause" ]]; then
|
||||
pause
|
||||
elif [[ "$line" == "resume" ]]; then
|
||||
resume
|
||||
fi
|
||||
done
|
||||
|
||||
exit 0
|
||||
@@ -0,0 +1,375 @@
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
path::{Path, PathBuf},
|
||||
process::ExitStatus,
|
||||
};
|
||||
|
||||
use configuration::{shared::resources::Resources, types::AssetLocation};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub type Port = u16;
|
||||
|
||||
pub type ExecutionResult = Result<String, (ExitStatus, String)>;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct ProviderCapabilities {
|
||||
// default ports internal
|
||||
/// Ensure that we have an image for each node (k8s/podman/docker)
|
||||
pub requires_image: bool,
|
||||
/// Allow to customize the resources through manifest (k8s).
|
||||
pub has_resources: bool,
|
||||
/// Used in native to prefix filepath with fullpath
|
||||
pub prefix_with_full_path: bool,
|
||||
/// Use default ports in node cmd/args.
|
||||
/// NOTE: generally used in k8s/dockers since the images expose those ports.
|
||||
pub use_default_ports_in_cmd: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SpawnNodeOptions {
|
||||
/// Name of the node
|
||||
pub name: String,
|
||||
/// Image of the node (IFF is supported by the provider)
|
||||
pub image: Option<String>,
|
||||
/// Resources to apply to the node (IFF is supported by the provider)
|
||||
pub resources: Option<Resources>,
|
||||
/// Main command to execute
|
||||
pub program: String,
|
||||
/// Arguments to pass to the main command
|
||||
pub args: Vec<String>,
|
||||
/// Environment to set when running the `program`
|
||||
pub env: Vec<(String, String)>,
|
||||
// TODO: rename startup_files
|
||||
/// Files to inject at startup
|
||||
pub injected_files: Vec<TransferedFile>,
|
||||
/// Paths to create before start the node (e.g keystore)
|
||||
/// should be created with `create_dir_all` in order
|
||||
/// to create the full path even when we have missing parts
|
||||
pub created_paths: Vec<PathBuf>,
|
||||
/// Database snapshot to be injected (should be a tgz file)
|
||||
/// Could be a local or remote asset
|
||||
pub db_snapshot: Option<AssetLocation>,
|
||||
pub port_mapping: Option<HashMap<Port, Port>>,
|
||||
/// Optionally specify a log path for the node
|
||||
pub node_log_path: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl SpawnNodeOptions {
|
||||
pub fn new<S>(name: S, program: S) -> Self
|
||||
where
|
||||
S: AsRef<str>,
|
||||
{
|
||||
Self {
|
||||
name: name.as_ref().to_string(),
|
||||
image: None,
|
||||
resources: None,
|
||||
program: program.as_ref().to_string(),
|
||||
args: vec![],
|
||||
env: vec![],
|
||||
injected_files: vec![],
|
||||
created_paths: vec![],
|
||||
db_snapshot: None,
|
||||
port_mapping: None,
|
||||
node_log_path: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn image<S>(mut self, image: S) -> Self
|
||||
where
|
||||
S: AsRef<str>,
|
||||
{
|
||||
self.image = Some(image.as_ref().to_string());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn resources(mut self, resources: Resources) -> Self {
|
||||
self.resources = Some(resources);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn db_snapshot(mut self, db_snap: Option<AssetLocation>) -> Self {
|
||||
self.db_snapshot = db_snap;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn args<S, I>(mut self, args: I) -> Self
|
||||
where
|
||||
S: AsRef<str>,
|
||||
I: IntoIterator<Item = S>,
|
||||
{
|
||||
self.args = args.into_iter().map(|s| s.as_ref().to_string()).collect();
|
||||
self
|
||||
}
|
||||
|
||||
pub fn env<S, I>(mut self, env: I) -> Self
|
||||
where
|
||||
S: AsRef<str>,
|
||||
I: IntoIterator<Item = (S, S)>,
|
||||
{
|
||||
self.env = env
|
||||
.into_iter()
|
||||
.map(|(name, value)| (name.as_ref().to_string(), value.as_ref().to_string()))
|
||||
.collect();
|
||||
self
|
||||
}
|
||||
|
||||
pub fn injected_files<I>(mut self, injected_files: I) -> Self
|
||||
where
|
||||
I: IntoIterator<Item = TransferedFile>,
|
||||
{
|
||||
self.injected_files = injected_files.into_iter().collect();
|
||||
self
|
||||
}
|
||||
|
||||
pub fn created_paths<P, I>(mut self, created_paths: I) -> Self
|
||||
where
|
||||
P: AsRef<Path>,
|
||||
I: IntoIterator<Item = P>,
|
||||
{
|
||||
self.created_paths = created_paths
|
||||
.into_iter()
|
||||
.map(|path| path.as_ref().into())
|
||||
.collect();
|
||||
self
|
||||
}
|
||||
|
||||
pub fn port_mapping(mut self, ports: HashMap<Port, Port>) -> Self {
|
||||
self.port_mapping = Some(ports);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn node_log_path(mut self, path: Option<PathBuf>) -> Self {
|
||||
self.node_log_path = path;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct GenerateFileCommand {
|
||||
pub program: String,
|
||||
pub args: Vec<String>,
|
||||
pub env: Vec<(String, String)>,
|
||||
pub local_output_path: PathBuf,
|
||||
}
|
||||
|
||||
impl GenerateFileCommand {
|
||||
pub fn new<S, P>(program: S, local_output_path: P) -> Self
|
||||
where
|
||||
S: AsRef<str>,
|
||||
P: AsRef<Path>,
|
||||
{
|
||||
Self {
|
||||
program: program.as_ref().to_string(),
|
||||
args: vec![],
|
||||
env: vec![],
|
||||
local_output_path: local_output_path.as_ref().into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn args<S, I>(mut self, args: I) -> Self
|
||||
where
|
||||
S: AsRef<str>,
|
||||
I: IntoIterator<Item = S>,
|
||||
{
|
||||
self.args = args.into_iter().map(|s| s.as_ref().to_string()).collect();
|
||||
self
|
||||
}
|
||||
|
||||
pub fn env<S, I>(mut self, env: I) -> Self
|
||||
where
|
||||
S: AsRef<str>,
|
||||
I: IntoIterator<Item = (S, S)>,
|
||||
{
|
||||
self.env = env
|
||||
.into_iter()
|
||||
.map(|(name, value)| (name.as_ref().to_string(), value.as_ref().to_string()))
|
||||
.collect();
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct GenerateFilesOptions {
|
||||
pub commands: Vec<GenerateFileCommand>,
|
||||
pub image: Option<String>,
|
||||
pub injected_files: Vec<TransferedFile>,
|
||||
// Allow to control the name of the node used to create the files.
|
||||
pub temp_name: Option<String>,
|
||||
pub expected_path: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl GenerateFilesOptions {
|
||||
pub fn new<I>(commands: I, image: Option<String>, expected_path: Option<PathBuf>) -> Self
|
||||
where
|
||||
I: IntoIterator<Item = GenerateFileCommand>,
|
||||
{
|
||||
Self {
|
||||
commands: commands.into_iter().collect(),
|
||||
injected_files: vec![],
|
||||
image,
|
||||
temp_name: None,
|
||||
expected_path,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_files<I>(
|
||||
commands: I,
|
||||
image: Option<String>,
|
||||
injected_files: &[TransferedFile],
|
||||
expected_path: Option<PathBuf>,
|
||||
) -> Self
|
||||
where
|
||||
I: IntoIterator<Item = GenerateFileCommand>,
|
||||
{
|
||||
Self {
|
||||
commands: commands.into_iter().collect(),
|
||||
injected_files: injected_files.into(),
|
||||
image,
|
||||
temp_name: None,
|
||||
expected_path,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn image<S>(mut self, image: S) -> Self
|
||||
where
|
||||
S: AsRef<str>,
|
||||
{
|
||||
self.image = Some(image.as_ref().to_string());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn injected_files<I>(mut self, injected_files: I) -> Self
|
||||
where
|
||||
I: IntoIterator<Item = TransferedFile>,
|
||||
{
|
||||
self.injected_files = injected_files.into_iter().collect();
|
||||
self
|
||||
}
|
||||
|
||||
pub fn temp_name(mut self, name: impl Into<String>) -> Self {
|
||||
self.temp_name = Some(name.into());
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct RunCommandOptions {
|
||||
pub program: String,
|
||||
pub args: Vec<String>,
|
||||
pub env: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
impl RunCommandOptions {
|
||||
pub fn new<S>(program: S) -> Self
|
||||
where
|
||||
S: AsRef<str>,
|
||||
{
|
||||
Self {
|
||||
program: program.as_ref().to_string(),
|
||||
args: vec![],
|
||||
env: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
pub fn args<S, I>(mut self, args: I) -> Self
|
||||
where
|
||||
S: AsRef<str>,
|
||||
I: IntoIterator<Item = S>,
|
||||
{
|
||||
self.args = args.into_iter().map(|s| s.as_ref().to_string()).collect();
|
||||
self
|
||||
}
|
||||
|
||||
pub fn env<S, I>(mut self, env: I) -> Self
|
||||
where
|
||||
S: AsRef<str>,
|
||||
I: IntoIterator<Item = (S, S)>,
|
||||
{
|
||||
self.env = env
|
||||
.into_iter()
|
||||
.map(|(name, value)| (name.as_ref().to_string(), value.as_ref().to_string()))
|
||||
.collect();
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
pub struct RunScriptOptions {
|
||||
pub local_script_path: PathBuf,
|
||||
pub args: Vec<String>,
|
||||
pub env: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
impl RunScriptOptions {
|
||||
pub fn new<P>(local_script_path: P) -> Self
|
||||
where
|
||||
P: AsRef<Path>,
|
||||
{
|
||||
Self {
|
||||
local_script_path: local_script_path.as_ref().into(),
|
||||
args: vec![],
|
||||
env: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
pub fn args<S, I>(mut self, args: I) -> Self
|
||||
where
|
||||
S: AsRef<str>,
|
||||
I: IntoIterator<Item = S>,
|
||||
{
|
||||
self.args = args.into_iter().map(|s| s.as_ref().to_string()).collect();
|
||||
self
|
||||
}
|
||||
|
||||
pub fn env<S, I>(mut self, env: I) -> Self
|
||||
where
|
||||
S: AsRef<str>,
|
||||
I: IntoIterator<Item = (S, S)>,
|
||||
{
|
||||
self.env = env
|
||||
.into_iter()
|
||||
.map(|(name, value)| (name.as_ref().to_string(), value.as_ref().to_string()))
|
||||
.collect();
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
// TODO(team): I think we can rename it to FileMap?
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TransferedFile {
|
||||
pub local_path: PathBuf,
|
||||
pub remote_path: PathBuf,
|
||||
// TODO: Can be narrowed to have strict typing on this?
|
||||
pub mode: String,
|
||||
}
|
||||
|
||||
impl TransferedFile {
|
||||
pub fn new<P>(local_path: P, remote_path: P) -> Self
|
||||
where
|
||||
P: AsRef<Path>,
|
||||
{
|
||||
Self {
|
||||
local_path: local_path.as_ref().into(),
|
||||
remote_path: remote_path.as_ref().into(),
|
||||
mode: "0644".to_string(), // default to rw-r--r--
|
||||
}
|
||||
}
|
||||
|
||||
pub fn mode<S>(mut self, mode: S) -> Self
|
||||
where
|
||||
S: AsRef<str>,
|
||||
{
|
||||
self.mode = mode.as_ref().to_string();
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for TransferedFile {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"File to transfer (local: {}, remote: {})",
|
||||
self.local_path.display(),
|
||||
self.remote_path.display()
|
||||
)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user