pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-05-30 09:21:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

New PVF validation host (#2710)

Browse Source 
* Implement PVF validation host

* WIP: Diener

* Increase the alloted compilation time

* Add more comments

* Minor clean up

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Fix pruning artifact removal

* Fix formatting and newlines

* Fix the thread pool

* Update node/core/pvf/src/executor_intf.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Remove redundant test declaration

* Don't convert the path into an intermediate string

* Try to workaround the test failure

* Use the puppet_worker trick again

* Fix a blip

* Move `ensure_wasmtime_version` under the tests mod

* Add a macro for puppet_workers

* fix build for not real-overseer

* Rename the puppet worker for adder collator

* play it safe with the name of adder puppet worker

* Typo: triggered

* Add more comments

* Do not kill exec worker on every error

* Plumb Duration for timeouts

* typo: critical

* Add proofs

* Clean unused imports

* Revert "WIP: Diener"

This reverts commit b9f54e513366c7a6dfdd117ac19fbdc46b900b4d.

* Sync version of wasmtime

* Update cargo.lock

* Update Substrate

* Merge fixes still

* Update wasmtime version in test

* bastifmt

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Squash spaces

* Trailing new line for testing.rs

* Remove controversial code

* comment about biasing

* Fix suggestion

* Add comments

* make it more clear why unwrap_err

* tmpfile retry

* proper proofs for claim_idle

* Remove mutex from ValidationHost

* Add some more logging

* Extract exec timeout into a constant

* Add some clarifying logging

* Use blake2_256

* Clean up the merge

Specifically the leftovers after removing real-overseer

* Update parachain/test-parachains/adder/collator/Cargo.toml

Co-authored-by: Andronik Ordian <write@reusable.software>

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
Co-authored-by: Andronik Ordian <write@reusable.software>
This commit is contained in:
Sergei Shulepov
2021-04-09 01:09:56 +03:00
committed by GitHub
parent 896ec8dbc3
commit 59b4d6511f
43 changed files with 5108 additions and 1991 deletions
Download Patch File Download Diff File Expand all files Collapse all files
polkadot/parachain/test-parachains/adder/collator/Cargo.toml
+9
View File
@@ -9,6 +9,10 @@ edition = "2018"
name = "adder-collator"
path = "src/main.rs"
[[bin]]
name = "adder_collator_puppet_worker"
path = "bin/puppet_worker.rs"
[dependencies]
parity-scale-codec = { version = "2.0.0", default-features = false, features = ["derive"] }
futures = "0.3.12"
@@ -28,6 +32,11 @@ sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
sc-authority-discovery = { git = "https://github.com/paritytech/substrate", branch = "master" }
sc-service = { git = "https://github.com/paritytech/substrate", branch = "master" }
# This one is tricky. Even though it is not used directly by the collator, we still need it for the
# `puppet_worker` binary, which is required for the integration test. However, this shouldn't be
# a big problem since it is used transitively anyway.
polkadot-node-core-pvf = { path = "../../../../node/core/pvf" }
[dev-dependencies]
polkadot-parachain = { path = "../../.." }
polkadot-test-service = { path = "../../../../node/test/service" }
polkadot/parachain/test-parachains/tests/lib.rs → polkadot/parachain/test-parachains/adder/collator/bin/puppet_worker.rs
+2 -15
View File
@@ -1,4 +1,4 @@
// Copyright 2019-2020 Parity Technologies (UK) Ltd.
// Copyright 2021 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
@@ -14,17 +14,4 @@
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
mod adder;
mod wasm_executor;
use parachain::wasm_executor::run_worker;
// This is not an actual test, but rather an entry point for out-of process WASM executor.
// When executing tests the executor spawns currently executing binary, which happens to be test binary.
// It then passes "validation_worker" on CLI effectivly making rust test executor to run this single test.
#[test]
fn validation_worker() {
if let Some(id) = std::env::args().find(|a| a.starts_with("/shmem_")) {
run_worker(&id, None).unwrap()
}
}
polkadot_node_core_pvf::decl_puppet_worker_main!();
polkadot/parachain/test-parachains/adder/collator/src/lib.rs
+7 -6
View File
@@ -233,7 +233,7 @@ mod tests {
use super::*;
use futures::executor::block_on;
use polkadot_parachain::{primitives::ValidationParams, wasm_executor::IsolationStrategy};
use polkadot_parachain::{primitives::{ValidationParams, ValidationResult}};
use polkadot_primitives::v1::PersistedValidationData;
#[test]
@@ -268,18 +268,19 @@ mod tests {
parent_head: HeadData,
collation: Collation,
) {
let ret = polkadot_parachain::wasm_executor::validate_candidate(
use polkadot_node_core_pvf::testing::validate_candidate;
let ret_buf = validate_candidate(
collator.validation_code(),
ValidationParams {
&ValidationParams {
parent_head: parent_head.encode().into(),
block_data: collation.proof_of_validity.block_data,
relay_parent_number: 1,
relay_parent_storage_root: Default::default(),
},
&IsolationStrategy::InProcess,
sp_core::testing::TaskExecutor::new(),
}.encode(),
)
.unwrap();
let ret = ValidationResult::decode(&mut &ret_buf[..]).unwrap();
let new_head = HeadData::decode(&mut &ret.head_data.0[..]).unwrap();
assert_eq!(
polkadot/parachain/test-parachains/adder/collator/tests/integration.rs
+10 -1
View File
@@ -17,6 +17,9 @@
//! Integration test that ensures that we can build and include parachain
//! blocks of the adder parachain.
const PUPPET_EXE: &str = env!("CARGO_BIN_EXE_adder_collator_puppet_worker");
// If this test is failing, make sure to run all tests with the `real-overseer` feature being enabled.
#[substrate_test_utils::test]
async fn collating_using_adder_collator(task_executor: sc_service::TaskExecutor) {
use sp_keyring::AccountKeyring::*;
@@ -30,7 +33,12 @@ async fn collating_using_adder_collator(task_executor: sc_service::TaskExecutor)
let para_id = ParaId::from(100);
// start alice
let alice = polkadot_test_service::run_validator_node(task_executor.clone(), Alice, || {}, vec![]);
let alice = polkadot_test_service::run_validator_node(
task_executor.clone(),
Alice, || {},
vec![],
Some(PUPPET_EXE.into()),
);
// start bob
let bob = polkadot_test_service::run_validator_node(
@@ -38,6 +46,7 @@ async fn collating_using_adder_collator(task_executor: sc_service::TaskExecutor)
Bob,
|| {},
vec![alice.addr.clone()],
Some(PUPPET_EXE.into()),
);
let collator = test_parachain_adder_collator::Collator::new();
polkadot/parachain/test-parachains/tests/adder/mod.rs
-153
View File
@@ -1,153 +0,0 @@
// Copyright 2017-2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Basic parachain that adds a number as part of its state.
const WORKER_ARGS_TEST: &[&'static str] = &["--nocapture", "validation_worker"];
use parachain::{
primitives::{
RelayChainBlockNumber,
BlockData as GenericBlockData,
HeadData as GenericHeadData,
ValidationParams,
},
wasm_executor::{ValidationPool, IsolationStrategy}
};
use parity_scale_codec::{Decode, Encode};
use adder::{HeadData, BlockData, hash_state};
fn isolation_strategy() -> IsolationStrategy {
IsolationStrategy::ExternalProcessCustomHost {
pool: ValidationPool::new(),
binary: std::env::current_exe().unwrap(),
args: WORKER_ARGS_TEST.iter().map(|x| x.to_string()).collect(),
}
}
#[test]
fn execute_good_on_parent_with_inprocess_validation() {
let isolation_strategy = IsolationStrategy::InProcess;
execute_good_on_parent(isolation_strategy);
}
#[test]
pub fn execute_good_on_parent_with_external_process_validation() {
let isolation_strategy = isolation_strategy();
execute_good_on_parent(isolation_strategy);
}
fn execute_good_on_parent(isolation_strategy: IsolationStrategy) {
let parent_head = HeadData {
number: 0,
parent_hash: [0; 32],
post_state: hash_state(0),
};
let block_data = BlockData {
state: 0,
add: 512,
};
let ret = parachain::wasm_executor::validate_candidate(
adder::wasm_binary_unwrap(),
ValidationParams {
parent_head: GenericHeadData(parent_head.encode()),
block_data: GenericBlockData(block_data.encode()),
relay_parent_number: 1,
relay_parent_storage_root: Default::default(),
},
&isolation_strategy,
sp_core::testing::TaskExecutor::new(),
).unwrap();
let new_head = HeadData::decode(&mut &ret.head_data.0[..]).unwrap();
assert_eq!(new_head.number, 1);
assert_eq!(new_head.parent_hash, parent_head.hash());
assert_eq!(new_head.post_state, hash_state(512));
}
#[test]
fn execute_good_chain_on_parent() {
let mut number = 0;
let mut parent_hash = [0; 32];
let mut last_state = 0;
let isolation_strategy = isolation_strategy();
for add in 0..10 {
let parent_head = HeadData {
number,
parent_hash,
post_state: hash_state(last_state),
};
let block_data = BlockData {
state: last_state,
add,
};
let ret = parachain::wasm_executor::validate_candidate(
adder::wasm_binary_unwrap(),
ValidationParams {
parent_head: GenericHeadData(parent_head.encode()),
block_data: GenericBlockData(block_data.encode()),
relay_parent_number: number as RelayChainBlockNumber + 1,
relay_parent_storage_root: Default::default(),
},
&isolation_strategy,
sp_core::testing::TaskExecutor::new(),
).unwrap();
let new_head = HeadData::decode(&mut &ret.head_data.0[..]).unwrap();
assert_eq!(new_head.number, number + 1);
assert_eq!(new_head.parent_hash, parent_head.hash());
assert_eq!(new_head.post_state, hash_state(last_state + add));
number += 1;
parent_hash = new_head.hash();
last_state += add;
}
}
#[test]
fn execute_bad_on_parent() {
let isolation_strategy = isolation_strategy();
let parent_head = HeadData {
number: 0,
parent_hash: [0; 32],
post_state: hash_state(0),
};
let block_data = BlockData {
state: 256, // start state is wrong.
add: 256,
};
let _ret = parachain::wasm_executor::validate_candidate(
adder::wasm_binary_unwrap(),
ValidationParams {
parent_head: GenericHeadData(parent_head.encode()),
block_data: GenericBlockData(block_data.encode()),
relay_parent_number: 1,
relay_parent_storage_root: Default::default(),
},
&isolation_strategy,
sp_core::testing::TaskExecutor::new(),
).unwrap_err();
}
polkadot/parachain/test-parachains/tests/wasm_executor/mod.rs
-94
View File
@@ -1,94 +0,0 @@
// Copyright 2019-2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Basic parachain that adds a number as part of its state.
const WORKER_ARGS_TEST: &[&'static str] = &["--nocapture", "validation_worker"];
use crate::adder;
use parachain::{
primitives::{BlockData, ValidationParams},
wasm_executor::{ValidationError, InvalidCandidate, EXECUTION_TIMEOUT_SEC, IsolationStrategy, ValidationPool},
};
fn isolation_strategy() -> IsolationStrategy {
IsolationStrategy::ExternalProcessCustomHost {
pool: ValidationPool::new(),
binary: std::env::current_exe().unwrap(),
args: WORKER_ARGS_TEST.iter().map(|x| x.to_string()).collect(),
}
}
#[test]
fn terminates_on_timeout() {
let isolation_strategy = isolation_strategy();
let result = parachain::wasm_executor::validate_candidate(
halt::wasm_binary_unwrap(),
ValidationParams {
block_data: BlockData(Vec::new()),
parent_head: Default::default(),
relay_parent_number: 1,
relay_parent_storage_root: Default::default(),
},
&isolation_strategy,
sp_core::testing::TaskExecutor::new(),
);
match result {
Err(ValidationError::InvalidCandidate(InvalidCandidate::Timeout)) => {},
r => panic!("{:?}", r),
}
// check that another parachain can validate normaly
adder::execute_good_on_parent_with_external_process_validation();
}
#[test]
fn parallel_execution() {
let isolation_strategy = isolation_strategy();
let isolation_strategy_clone = isolation_strategy.clone();
let start = std::time::Instant::now();
let thread = std::thread::spawn(move ||
parachain::wasm_executor::validate_candidate(
halt::wasm_binary_unwrap(),
ValidationParams {
block_data: BlockData(Vec::new()),
parent_head: Default::default(),
relay_parent_number: 1,
relay_parent_storage_root: Default::default(),
},
&isolation_strategy,
sp_core::testing::TaskExecutor::new(),
).ok());
let _ = parachain::wasm_executor::validate_candidate(
halt::wasm_binary_unwrap(),
ValidationParams {
block_data: BlockData(Vec::new()),
parent_head: Default::default(),
relay_parent_storage_root: Default::default(),
relay_parent_number: 1,
},
&isolation_strategy_clone,
sp_core::testing::TaskExecutor::new(),
);
thread.join().unwrap();
// total time should be < 2 x EXECUTION_TIMEOUT_SEC
assert!(
std::time::Instant::now().duration_since(start)
< std::time::Duration::from_secs(EXECUTION_TIMEOUT_SEC * 2)
);
}