Retire puppet workers (#1449)

Closes #583

After the separation of PVF worker binaries, dedicated puppet workers
are not needed for tests anymore. The production workers can be used
instead, avoiding some code duplication and decreasing complexity.

The changes also make it possible to further refactor the code to
isolate workers completely.
This commit is contained in:
s0me0ne-unkn0wn
2023-09-11 19:14:07 +02:00
committed by GitHub
parent 4b8bd9060e
commit 2c8021f998
20 changed files with 71 additions and 376 deletions
Generated
-12
View File
@@ -3899,14 +3899,6 @@ dependencies = [
"sp-trie",
]
[[package]]
name = "cumulus-test-relay-validation-worker-provider"
version = "0.1.0"
dependencies = [
"polkadot-node-core-pvf",
"toml 0.7.6",
]
[[package]]
name = "cumulus-test-runtime"
version = "0.1.0"
@@ -3958,7 +3950,6 @@ dependencies = [
"cumulus-relay-chain-minimal-node",
"cumulus-test-client",
"cumulus-test-relay-sproof-builder",
"cumulus-test-relay-validation-worker-provider",
"cumulus-test-runtime",
"frame-system",
"frame-system-rpc-runtime-api",
@@ -12018,7 +12009,6 @@ dependencies = [
"slotmap",
"sp-core",
"sp-maybe-compressed-blob",
"sp-tracing",
"sp-wasm-interface",
"substrate-build-script-utils",
"tempfile",
@@ -18462,7 +18452,6 @@ dependencies = [
"sp-keyring",
"substrate-test-utils",
"test-parachain-adder",
"test-parachain-adder-collator",
"tokio",
]
@@ -18511,7 +18500,6 @@ dependencies = [
"sp-keyring",
"substrate-test-utils",
"test-parachain-undying",
"test-parachain-undying-collator",
"tokio",
]
-1
View File
@@ -92,7 +92,6 @@ members = [
"cumulus/primitives/utility",
"cumulus/test/client",
"cumulus/test/relay-sproof-builder",
"cumulus/test/relay-validation-worker-provider",
"cumulus/test/runtime",
"cumulus/test/service",
"cumulus/xcm/xcm-emulator",
@@ -1,15 +0,0 @@
[package]
name = "cumulus-test-relay-validation-worker-provider"
version = "0.1.0"
authors.workspace = true
edition.workspace = true
build = "build.rs"
publish = false
[dependencies]
# Polkadot
polkadot-node-core-pvf = { path = "../../../polkadot/node/core/pvf", features = ["test-utils"] }
[build-dependencies]
toml = "0.7.6"
@@ -1,169 +0,0 @@
// Copyright (C) Parity Technologies (UK) Ltd.
// This file is part of Cumulus.
// Cumulus is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Cumulus is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Cumulus. If not, see <http://www.gnu.org/licenses/>.
use std::{
env, fs,
path::{Path, PathBuf},
process::{self, Command},
};
use toml::value::Table;
/// The name of the project we will building.
const PROJECT_NAME: &str = "validation-worker";
/// The env variable that instructs us to skip the build.
const SKIP_ENV: &str = "SKIP_BUILD";
fn main() {
if env::var(SKIP_ENV).is_ok() {
return
}
let out_dir = PathBuf::from(env::var("OUT_DIR").expect("`OUT_DIR` is set by cargo"));
let project = create_project(&out_dir);
build_project(&project.join("Cargo.toml"));
fs::copy(project.join("target/release").join(PROJECT_NAME), out_dir.join(PROJECT_NAME))
.expect("Copies validation worker");
}
fn find_cargo_lock() -> PathBuf {
let mut path = PathBuf::from(
env::var("CARGO_MANIFEST_DIR").expect("`CARGO_MANIFEST_DIR` is set by cargo"),
);
loop {
if path.join("Cargo.lock").exists() {
return path.join("Cargo.lock")
}
if !path.pop() {
panic!("Could not find `Cargo.lock`")
}
}
}
fn create_project(out_dir: &Path) -> PathBuf {
let project_dir = out_dir.join(format!("{}-project", PROJECT_NAME));
fs::create_dir_all(project_dir.join("src")).expect("Creates project dir and project src dir");
let mut project_toml = Table::new();
let mut package = Table::new();
package.insert("name".into(), PROJECT_NAME.into());
package.insert("version".into(), "1.0.0".into());
package.insert("edition".into(), "2021".into());
project_toml.insert("package".into(), package.into());
project_toml.insert("workspace".into(), Table::new().into());
let mut dependencies = Table::new();
let mut dependency_project = Table::new();
dependency_project.insert(
"path".into(),
env::var("CARGO_MANIFEST_DIR")
.expect("`CARGO_MANIFEST_DIR` is set by cargo")
.into(),
);
dependencies
.insert("cumulus-test-relay-validation-worker-provider".into(), dependency_project.into());
project_toml.insert("dependencies".into(), dependencies.into());
add_patches(&mut project_toml);
fs::write(
project_dir.join("Cargo.toml"),
toml::to_string_pretty(&project_toml).expect("Wasm workspace toml is valid; qed"),
)
.expect("Writes project `Cargo.toml`");
fs::write(
project_dir.join("src").join("main.rs"),
r#"
cumulus_test_relay_validation_worker_provider::polkadot_node_core_pvf::decl_puppet_worker_main!();
"#,
)
.expect("Writes `main.rs`");
let cargo_lock = find_cargo_lock();
fs::copy(&cargo_lock, project_dir.join("Cargo.lock")).expect("Copies `Cargo.lock`");
println!("cargo:rerun-if-changed={}", cargo_lock.display());
project_dir
}
fn add_patches(project_toml: &mut Table) {
let workspace_toml_path = PathBuf::from(
env::var("CARGO_MANIFEST_DIR").expect("`CARGO_MANIFEST_DIR` is set by cargo"),
)
.join("../../../Cargo.toml");
let mut workspace_toml: Table = toml::from_str(
&fs::read_to_string(&workspace_toml_path).expect("Workspace root `Cargo.toml` exists; qed"),
)
.expect("Workspace root `Cargo.toml` is a valid toml file; qed");
let mut workspace_path = workspace_toml_path;
workspace_path.pop();
while let Some(mut patch) =
workspace_toml.remove("patch").and_then(|p| p.try_into::<Table>().ok())
{
// Iterate over all patches and make the patch path absolute from the workspace root path.
patch
.iter_mut()
.filter_map(|p| {
p.1.as_table_mut().map(|t| t.iter_mut().filter_map(|t| t.1.as_table_mut()))
})
.flatten()
.for_each(|p| {
p.iter_mut().filter(|(k, _)| k == &"path").for_each(|(_, v)| {
if let Some(path) = v.as_str().map(PathBuf::from) {
if path.is_relative() {
*v = workspace_path.join(path).display().to_string().into();
}
}
})
});
project_toml.insert("patch".into(), patch.into());
}
}
fn build_project(cargo_toml: &Path) {
let cargo = env::var("CARGO").expect("`CARGO` env variable is always set by cargo");
let status = Command::new(cargo)
.arg("build")
.arg("--release")
.arg(format!("--manifest-path={}", cargo_toml.display()))
// Unset the `CARGO_TARGET_DIR` to prevent a cargo deadlock (cargo locks a target dir
// exclusive).
.env_remove("CARGO_TARGET_DIR")
// Do not call us recursively.
.env(SKIP_ENV, "1")
.status();
match status.map(|s| s.success()) {
Ok(true) => {},
// Use `process.exit(1)` to have a clean error output.
_ => process::exit(1),
}
}
@@ -1,27 +0,0 @@
// Copyright (C) Parity Technologies (UK) Ltd.
// This file is part of Cumulus.
// Cumulus is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Cumulus is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Cumulus. If not, see <http://www.gnu.org/licenses/>.
//! Provides the [`VALIDATION_WORKER`] for integration tests in Cumulus.
//!
//! The validation worker is used by the relay chain to validate parachains. This worker is placed
//! in an extra process to provide better security and to ensure that a worker can be killed etc.
//!
//! !!This should only be used for tests!!
pub use polkadot_node_core_pvf;
/// The path to the validation worker.
pub const VALIDATION_WORKER: &str = concat!(env!("OUT_DIR"), "/validation-worker");
-1
View File
@@ -72,7 +72,6 @@ cumulus-primitives-core = { path = "../../primitives/core" }
cumulus-primitives-parachain-inherent = { path = "../../primitives/parachain-inherent" }
cumulus-relay-chain-inprocess-interface = { path = "../../client/relay-chain-inprocess-interface" }
cumulus-relay-chain-interface = { path = "../../client/relay-chain-interface" }
cumulus-test-relay-validation-worker-provider = { path = "../relay-validation-worker-provider" }
cumulus-test-runtime = { path = "../runtime" }
cumulus-relay-chain-minimal-node = { path = "../../client/relay-chain-minimal-node" }
cumulus-client-pov-recovery = { path = "../../client/pov-recovery" }
+5 -4
View File
@@ -903,8 +903,9 @@ pub fn run_relay_chain_validator_node(
config.rpc_addr = Some(SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), port));
}
polkadot_test_service::run_validator_node(
config,
Some(cumulus_test_relay_validation_worker_provider::VALIDATION_WORKER.into()),
)
let mut workers_path = std::env::current_exe().unwrap();
workers_path.pop();
workers_path.pop();
polkadot_test_service::run_validator_node(config, Some(workers_path))
}
-8
View File
@@ -6,11 +6,6 @@ authors.workspace = true
edition.workspace = true
license.workspace = true
[[bin]]
name = "puppet_worker"
path = "bin/puppet_worker.rs"
required-features = ["test-utils"]
[dependencies]
always-assert = "0.1"
futures = "0.3.21"
@@ -35,7 +30,6 @@ polkadot-primitives = { path = "../../../primitives" }
sp-core = { path = "../../../../substrate/primitives/core" }
sp-wasm-interface = { path = "../../../../substrate/primitives/wasm-interface" }
sp-maybe-compressed-blob = { path = "../../../../substrate/primitives/maybe-compressed-blob" }
sp-tracing = { path = "../../../../substrate/primitives/tracing", optional = true }
polkadot-node-core-pvf-prepare-worker = { path = "prepare-worker", optional = true }
polkadot-node-core-pvf-execute-worker = { path = "execute-worker", optional = true }
@@ -56,9 +50,7 @@ halt = { package = "test-parachain-halt", path = "../../../parachain/test-parach
ci-only-tests = []
jemalloc-allocator = [ "polkadot-node-core-pvf-common/jemalloc-allocator" ]
# This feature is used to export test code to other crates without putting it in the production build.
# This is also used by the `puppet_worker` binary.
test-utils = [
"polkadot-node-core-pvf-execute-worker",
"polkadot-node-core-pvf-prepare-worker",
"sp-tracing",
]
@@ -60,6 +60,10 @@ macro_rules! decl_worker_main {
println!("{}", $worker_version);
return
},
"test-sleep" => {
std::thread::sleep(std::time::Duration::from_secs(5));
return
},
subcommand => {
// Must be passed for compatibility with the single-binary test workers.
if subcommand != $expected_command {
-10
View File
@@ -100,10 +100,6 @@ mod worker_intf;
#[cfg(feature = "test-utils")]
pub mod testing;
// Used by `decl_puppet_worker_main!`.
#[cfg(feature = "test-utils")]
pub use sp_tracing;
pub use error::{InvalidCandidate, ValidationError};
pub use host::{start, Config, ValidationHost, EXECUTE_BINARY_NAME, PREPARE_BINARY_NAME};
pub use metrics::Metrics;
@@ -117,11 +113,5 @@ pub use polkadot_node_core_pvf_common::{
pvf::PvfPrepData,
};
// Re-export worker entrypoints.
#[cfg(feature = "test-utils")]
pub use polkadot_node_core_pvf_execute_worker::worker_entrypoint as execute_worker_entrypoint;
#[cfg(feature = "test-utils")]
pub use polkadot_node_core_pvf_prepare_worker::worker_entrypoint as prepare_worker_entrypoint;
/// The log target for this crate.
pub const LOG_TARGET: &str = "parachain::pvf";
-42
View File
@@ -47,45 +47,3 @@ pub fn validate_candidate(
Ok(result)
}
/// Use this macro to declare a `fn main() {}` that will check the arguments and dispatch them to
/// the appropriate worker, making the executable that can be used for spawning workers.
#[macro_export]
macro_rules! decl_puppet_worker_main {
() => {
fn main() {
$crate::sp_tracing::try_init_simple();
let args = std::env::args().collect::<Vec<_>>();
if args.len() == 1 {
panic!("wrong number of arguments");
}
let entrypoint = match args[1].as_ref() {
"exit" => {
std::process::exit(1);
},
"sleep" => {
std::thread::sleep(std::time::Duration::from_secs(5));
return
},
"prepare-worker" => $crate::prepare_worker_entrypoint,
"execute-worker" => $crate::execute_worker_entrypoint,
other => panic!("unknown subcommand: {}", other),
};
let mut node_version = None;
let mut socket_path: &str = "";
for i in (2..args.len()).step_by(2) {
match args[i].as_ref() {
"--socket-path" => socket_path = args[i + 1].as_str(),
"--node-impl-version" => node_version = Some(args[i + 1].as_str()),
arg => panic!("Unexpected argument found: {}", arg),
}
}
entrypoint(&socket_path, node_version, None);
}
};
}
+9
View File
@@ -0,0 +1,9 @@
# PVF host integration tests
## Testing
Before running these tests, make sure the worker binaries are built first. This can be done with:
```sh
cargo build --bin polkadot-execute-worker --bin polkadot-prepare-worker
```
+13 -4
View File
@@ -33,7 +33,6 @@ use tokio::sync::Mutex;
mod adder;
mod worker_common;
const PUPPET_EXE: &str = env!("CARGO_BIN_EXE_puppet_worker");
const TEST_EXECUTION_TIMEOUT: Duration = Duration::from_secs(3);
const TEST_PREPARATION_TIMEOUT: Duration = Duration::from_secs(3);
@@ -51,10 +50,20 @@ impl TestHost {
where
F: FnOnce(&mut Config),
{
let mut workers_path = std::env::current_exe().unwrap();
workers_path.pop();
workers_path.pop();
let mut prepare_worker_path = workers_path.clone();
prepare_worker_path.push("polkadot-prepare-worker");
let mut execute_worker_path = workers_path.clone();
execute_worker_path.push("polkadot-execute-worker");
let cache_dir = tempfile::tempdir().unwrap();
let program_path = std::path::PathBuf::from(PUPPET_EXE);
let mut config =
Config::new(cache_dir.path().to_owned(), None, program_path.clone(), program_path);
let mut config = Config::new(
cache_dir.path().to_owned(),
None,
prepare_worker_path,
execute_worker_path,
);
f(&mut config);
let (host, task) = start(config, Metrics::default());
let _ = tokio::task::spawn(task);
@@ -14,26 +14,41 @@
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
use polkadot_node_core_pvf::testing::{spawn_with_program_path, SpawnErr};
use std::time::Duration;
use polkadot_node_core_pvf::testing::{spawn_with_program_path, SpawnErr};
use crate::PUPPET_EXE;
fn worker_path(name: &str) -> std::path::PathBuf {
let mut worker_path = std::env::current_exe().unwrap();
worker_path.pop();
worker_path.pop();
worker_path.push(name);
worker_path
}
// Test spawning a program that immediately exits with a failure code.
#[tokio::test]
async fn spawn_immediate_exit() {
let result =
spawn_with_program_path("integration-test", PUPPET_EXE, &["exit"], Duration::from_secs(2))
.await;
// There's no explicit `exit` subcommand in the worker; it will panic on an unknown
// subcommand anyway
let result = spawn_with_program_path(
"integration-test",
worker_path("polkadot-prepare-worker"),
&["exit"],
Duration::from_secs(2),
)
.await;
assert!(matches!(result, Err(SpawnErr::AcceptTimeout)));
}
#[tokio::test]
async fn spawn_timeout() {
let result =
spawn_with_program_path("integration-test", PUPPET_EXE, &["sleep"], Duration::from_secs(2))
.await;
let result = spawn_with_program_path(
"integration-test",
worker_path("polkadot-execute-worker"),
&["test-sleep"],
Duration::from_secs(2),
)
.await;
assert!(matches!(result, Err(SpawnErr::AcceptTimeout)));
}
@@ -41,7 +56,7 @@ async fn spawn_timeout() {
async fn should_connect() {
let _ = spawn_with_program_path(
"integration-test",
PUPPET_EXE,
worker_path("polkadot-prepare-worker"),
&["prepare-worker"],
Duration::from_secs(2),
)
@@ -11,11 +11,6 @@ license.workspace = true
name = "adder-collator"
path = "src/main.rs"
[[bin]]
name = "adder_collator_puppet_worker"
path = "bin/puppet_worker.rs"
required-features = ["test-utils"]
[dependencies]
parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive"] }
clap = { version = "4.4.2", features = ["derive"] }
@@ -33,25 +28,17 @@ polkadot-node-subsystem = { path = "../../../../node/subsystem" }
sc-cli = { path = "../../../../../substrate/client/cli" }
sp-core = { path = "../../../../../substrate/primitives/core" }
sc-service = { path = "../../../../../substrate/client/service" }
# This one is tricky. Even though it is not used directly by the collator, we still need it for the
# `puppet_worker` binary, which is required for the integration test. However, this shouldn't be
# a big problem since it is used transitively anyway.
polkadot-node-core-pvf = { path = "../../../../node/core/pvf", features = ["test-utils"], optional = true }
[dev-dependencies]
polkadot-parachain-primitives = { path = "../../.." }
polkadot-test-service = { path = "../../../../node/test/service" }
polkadot-node-core-pvf = { path = "../../../../node/core/pvf", features = ["test-utils"] }
substrate-test-utils = { path = "../../../../../substrate/test-utils" }
sc-service = { path = "../../../../../substrate/client/service" }
sp-keyring = { path = "../../../../../substrate/primitives/keyring" }
# For the puppet worker, depend on ourselves with the test-utils feature.
test-parachain-adder-collator = { path = "", features = ["test-utils"] }
tokio = { version = "1.24.2", features = ["macros"] }
[features]
network-protocol-staging = [ "polkadot-cli/network-protocol-staging" ]
# This feature is used to export test code to other crates without putting it in the production build.
# This is also used by the `puppet_worker` binary.
test-utils = [ "polkadot-node-core-pvf/test-utils" ]
@@ -1,17 +0,0 @@
// Copyright (C) Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
polkadot_node_core_pvf::decl_puppet_worker_main!();
@@ -17,8 +17,6 @@
//! Integration test that ensures that we can build and include parachain
//! blocks of the adder parachain.
const PUPPET_EXE: &str = env!("CARGO_BIN_EXE_adder_collator_puppet_worker");
// If this test is failing, make sure to run all tests with the `real-overseer` feature being
// enabled.
@@ -41,8 +39,12 @@ async fn collating_using_adder_collator() {
true,
);
let mut workers_path = std::env::current_exe().unwrap();
workers_path.pop();
workers_path.pop();
// start alice
let alice = polkadot_test_service::run_validator_node(alice_config, Some(PUPPET_EXE.into()));
let alice = polkadot_test_service::run_validator_node(alice_config, Some(workers_path.clone()));
let bob_config = polkadot_test_service::node_config(
|| {},
@@ -53,7 +55,7 @@ async fn collating_using_adder_collator() {
);
// start bob
let bob = polkadot_test_service::run_validator_node(bob_config, Some(PUPPET_EXE.into()));
let bob = polkadot_test_service::run_validator_node(bob_config, Some(workers_path));
let collator = test_parachain_adder_collator::Collator::new();
@@ -11,15 +11,10 @@ publish = false
name = "undying-collator"
path = "src/main.rs"
[[bin]]
name = "undying_collator_puppet_worker"
path = "bin/puppet_worker.rs"
required-features = ["test-utils"]
[dependencies]
parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive"] }
clap = { version = "4.4.2", features = ["derive"] }
futures = "0.3.19"
futures = "0.3.21"
futures-timer = "3.0.2"
log = "0.4.17"
@@ -33,24 +28,14 @@ polkadot-node-subsystem = { path = "../../../../node/subsystem" }
sc-cli = { path = "../../../../../substrate/client/cli" }
sp-core = { path = "../../../../../substrate/primitives/core" }
sc-service = { path = "../../../../../substrate/client/service" }
# This one is tricky. Even though it is not used directly by the collator, we still need it for the
# `puppet_worker` binary, which is required for the integration test. However, this shouldn't be
# a big problem since it is used transitively anyway.
polkadot-node-core-pvf = { path = "../../../../node/core/pvf", features = ["test-utils"], optional = true }
[dev-dependencies]
polkadot-parachain-primitives = { path = "../../.." }
polkadot-test-service = { path = "../../../../node/test/service" }
# For the puppet worker, depend on ourselves with the test-utils feature.
test-parachain-undying-collator = { path = "", features = ["test-utils"] }
polkadot-node-core-pvf = { path = "../../../../node/core/pvf", features = ["test-utils"] }
substrate-test-utils = { path = "../../../../../substrate/test-utils" }
sc-service = { path = "../../../../../substrate/client/service" }
sp-keyring = { path = "../../../../../substrate/primitives/keyring" }
tokio = { version = "1.24.2", features = ["macros"] }
[features]
# This feature is used to export test code to other crates without putting it in the production build.
# This is also used by the `puppet_worker` binary.
test-utils = [ "polkadot-node-core-pvf/test-utils" ]
@@ -1,17 +0,0 @@
// Copyright (C) Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
polkadot_node_core_pvf::decl_puppet_worker_main!();
@@ -17,8 +17,6 @@
//! Integration test that ensures that we can build and include parachain
//! blocks of the `Undying` parachain.
const PUPPET_EXE: &str = env!("CARGO_BIN_EXE_undying_collator_puppet_worker");
// If this test is failing, make sure to run all tests with the `real-overseer` feature being
// enabled.
#[tokio::test(flavor = "multi_thread")]
@@ -40,8 +38,12 @@ async fn collating_using_undying_collator() {
true,
);
let mut workers_path = std::env::current_exe().unwrap();
workers_path.pop();
workers_path.pop();
// start alice
let alice = polkadot_test_service::run_validator_node(alice_config, Some(PUPPET_EXE.into()));
let alice = polkadot_test_service::run_validator_node(alice_config, Some(workers_path.clone()));
let bob_config = polkadot_test_service::node_config(
|| {},
@@ -52,7 +54,7 @@ async fn collating_using_undying_collator() {
);
// start bob
let bob = polkadot_test_service::run_validator_node(bob_config, Some(PUPPET_EXE.into()));
let bob = polkadot_test_service::run_validator_node(bob_config, Some(workers_path));
let collator = test_parachain_undying_collator::Collator::new(1_000, 1);