Fix integration test again (#201)

* Initial commit

Forked at: 10533db948
Parent branch: origin/master

* WIP

Forked at: 10533db948
Parent branch: origin/master

* WIP

Forked at: 10533db948
Parent branch: origin/master

* WIP

Forked at: 10533db948
Parent branch: origin/master

* CLEANUP

Forked at: 10533db948
Parent branch: origin/master

* CLEANUP

Forked at: 10533db948
Parent branch: origin/master

* WIP

Forked at: 10533db948
Parent branch: origin/master

* Pushed branch for polkadot & substrate

* WIP

Forked at: 10533db948
Parent branch: origin/master

* using rococo and tick

* Revert "using rococo and tick"

This reverts commit d81db9246ff7061478649ffea3e49e417fcb6959.

* debug

* WIP

Forked at: 10533db948
Parent branch: origin/master

* Revert "Revert "using rococo and tick""

This reverts commit 45ec2be89f2b8af82da8dcb9d19d900571598766.

* WIP

Forked at: 10533db948
Parent branch: origin/master

* Update rococo-parachains/src/service.rs

* WIP

Forked at: 10533db948
Parent branch: origin/master

* WIP

Forked at: 10533db948
Parent branch: origin/master

* WIP

Forked at: 10533db948
Parent branch: origin/master

* Revert "WIP"

This reverts commit d3f63ed0a314ffe12c0066124076736017981b80.

* WIP

Forked at: 10533db948
Parent branch: origin/master

* CLEANUP

Forked at: 10533db948
Parent branch: origin/master

* CLEANUP

Forked at: 10533db948
Parent branch: origin/master

* CLEANUP

Forked at: 10533db948
Parent branch: origin/master

* Use inprocess validation

* CLEANUP

Forked at: 10533db948
Parent branch: origin/master

* CLEANUP

Forked at: 10533db948
Parent branch: origin/master

* Fix failing test

* CLEANUP

Forked at: 10533db948
Parent branch: origin/master

* CLEANUP

Forked at: 10533db948
Parent branch: origin/master

* increase logs

* Removed a bit of logs

* Revert branch change

* CLEANUP

Forked at: 10533db948
Parent branch: origin/master

* Test without STDIN close detection

* Bypass validation pool

* Switch to rococo-branch

* Move start_test_collator to rococo-collator

* CLEANUP

Forked at: 10533db948
Parent branch: origin/master

* CLEANUP

Forked at: 10533db948
Parent branch: origin/master
This commit is contained in:
Cecile Tonglet
2020-09-16 08:54:34 +02:00
committed by GitHub
parent 9c4519f0f4
commit badf52b1e1
7 changed files with 881 additions and 691 deletions
+688 -511
View File
File diff suppressed because it is too large Load Diff
+27 -90
View File
@@ -29,8 +29,7 @@ use cumulus_primitives::{
};
use cumulus_runtime::ParachainBlockData;
use sc_client_api::{BlockBackend, Finalizer, StateBackend, UsageProvider};
use sp_api::ApiExt;
use sc_client_api::{Backend as BackendT, BlockBackend, Finalizer, StateBackend, UsageProvider};
use sp_blockchain::HeaderBackend;
use sp_consensus::{
BlockImport, BlockImportParams, BlockOrigin, BlockStatus, Environment, Error as ConsensusError,
@@ -426,14 +425,19 @@ where
{
type ParachainContext = Collator<Block, PF, BI, BS>;
fn build<Spawner>(
fn build<Spawner, PClient, PBackend, PNetwork>(
self,
polkadot_client: polkadot_collator::Client,
polkadot_client: Arc<PClient>,
spawner: Spawner,
polkadot_network: impl CollatorNetwork + SyncOracle + Clone + 'static,
polkadot_network: PNetwork,
) -> Result<Self::ParachainContext, ()>
where
Spawner: SpawnNamed + Clone + Send + Sync + 'static,
PBackend: BackendT<PBlock>,
PBackend::State: StateBackend<BlakeTwo256>,
PClient: polkadot_service::AbstractClient<PBlock, PBackend> + 'static,
PClient::Api: RuntimeApiCollection<StateBackend = PBackend::State>,
PNetwork: CollatorNetwork + SyncOracle + Clone + 'static,
{
let CollatorBuilder {
proposer_factory,
@@ -446,83 +450,17 @@ where
delayed_block_announce_validator,
_marker,
} = self;
polkadot_client.execute_with(CollatorBuilderWithClient {
spawner,
polkadot_network,
proposer_factory,
inherent_data_providers,
block_import,
block_status,
delayed_block_announce_validator.set(Box::new(JustifiedBlockAnnounceValidator::new(
polkadot_client.clone(),
para_id,
client,
announce_block,
delayed_block_announce_validator,
_marker,
})
}
}
pub struct CollatorBuilderWithClient<Block: BlockT, PF, BI, Backend, Client, BS, Spawner, Network> {
proposer_factory: PF,
inherent_data_providers: InherentDataProviders,
block_import: BI,
block_status: Arc<BS>,
para_id: ParaId,
client: Arc<Client>,
announce_block: Arc<dyn Fn(Block::Hash, Vec<u8>) + Send + Sync>,
delayed_block_announce_validator: DelayedBlockAnnounceValidator<Block>,
_marker: PhantomData<(Block, Backend)>,
spawner: Spawner,
polkadot_network: Network,
}
impl<Block: BlockT, PF, BI, Backend, Client, BS, Spawner, Network>
polkadot_service::ExecuteWithClient
for CollatorBuilderWithClient<Block, PF, BI, Backend, Client, BS, Spawner, Network>
where
PF: Environment<Block> + Send + 'static,
BI: BlockImport<Block, Error = sp_consensus::Error, Transaction = TransactionFor<PF, Block>>
+ Send
+ Sync
+ 'static,
Backend: sc_client_api::Backend<Block> + 'static,
Client: Finalizer<Block, Backend>
+ UsageProvider<Block>
+ HeaderBackend<Block>
+ Send
+ Sync
+ BlockBackend<Block>
+ 'static,
for<'a> &'a Client: BlockImport<Block>,
BS: BlockBackend<Block>,
Spawner: SpawnNamed + Clone + Send + Sync + 'static,
Network: CollatorNetwork + SyncOracle + Clone + 'static,
{
type Output = Result<Collator<Block, PF, BI, BS>, ()>;
fn execute_with_client<PClient, Api, PBackend>(
self,
polkadot_client: Arc<PClient>,
) -> Self::Output
where
<Api as ApiExt<PBlock>>::StateBackend: sp_api::StateBackend<BlakeTwo256>,
PBackend: sc_client_api::Backend<PBlock>,
PBackend::State: StateBackend<BlakeTwo256>,
Api: RuntimeApiCollection<StateBackend = PBackend::State>,
PClient: polkadot_service::AbstractClient<PBlock, PBackend, Api = Api> + 'static,
{
self.delayed_block_announce_validator
.set(Box::new(JustifiedBlockAnnounceValidator::new(
polkadot_client.clone(),
self.para_id,
Box::new(self.polkadot_network.clone()),
)));
Box::new(polkadot_network.clone()),
)));
let follow = match cumulus_consensus::follow_polkadot(
self.para_id,
self.client,
para_id,
client,
polkadot_client,
self.announce_block.clone(),
announce_block.clone(),
) {
Ok(follow) => follow,
Err(e) => {
@@ -530,17 +468,16 @@ where
}
};
self.spawner
.spawn("cumulus-follow-polkadot", follow.map(|_| ()).boxed());
spawner.spawn("cumulus-follow-polkadot", follow.map(|_| ()).boxed());
Ok(Collator::new(
self.proposer_factory,
self.inherent_data_providers,
self.polkadot_network,
self.block_import,
self.block_status,
Arc::new(self.spawner),
self.announce_block,
proposer_factory,
inherent_data_providers,
polkadot_network,
block_import,
block_status,
Arc::new(spawner),
announce_block,
))
}
}
@@ -685,8 +622,8 @@ mod tests {
block_announce_validator,
);
let context = builder
.build(
polkadot_service::Client::Polkadot(Arc::new(
.build::<_, _, polkadot_service::FullBackend, _>(
Arc::new(
substrate_test_client::TestClientBuilder::<_, _, _, ()>::default()
.build_with_native_executor::<polkadot_service::polkadot_runtime::RuntimeApi, _>(
Some(NativeExecutor::<polkadot_service::PolkadotExecutor>::new(
@@ -696,7 +633,7 @@ mod tests {
)),
)
.0,
)),
),
spawner,
DummyCollatorNetwork,
)
+2 -1
View File
@@ -80,7 +80,7 @@ substrate-build-script-utils = { git = "https://github.com/paritytech/substrate"
assert_cmd = "0.12"
nix = "0.17"
rand = "0.7.3"
tokio = { version = "0.2.13", features = ["macros"] }
tokio = { version = "0.2.21", features = ["macros"] }
# Polkadot dependencies
polkadot-runtime-common = { git = "https://github.com/paritytech/polkadot", branch = "rococo-branch" }
@@ -92,3 +92,4 @@ polkadot-test-service = { git = "https://github.com/paritytech/polkadot", branch
pallet-sudo = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }
substrate-test-client = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }
substrate-test-runtime-client = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }
substrate-test-utils = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }
+2
View File
@@ -292,6 +292,7 @@ pub fn run() -> Result<()> {
polkadot_config,
id,
cli.run.base.validator,
false,
)
} else {
crate::service::start_node(
@@ -300,6 +301,7 @@ pub fn run() -> Result<()> {
polkadot_config,
id,
cli.run.base.validator,
false,
)
.map(|r| r.0)
}
@@ -15,122 +15,94 @@
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
use codec::Encode;
use futures::{
future::{self, FutureExt},
pin_mut, select,
};
use futures::future;
use polkadot_primitives::v0::{Id as ParaId, Info, Scheduling};
use polkadot_runtime_common::registrar;
use polkadot_test_runtime_client::Sr25519Keyring;
use sc_chain_spec::ChainSpec;
use sc_client_api::execution_extensions::ExecutionStrategies;
use sc_informant::OutputFormat;
use sc_network::{config::TransportConfig, multiaddr};
use sc_service::{
config::{
DatabaseConfig, KeystoreConfig, MultiaddrWithPeerId, NetworkConfiguration,
WasmExecutionMethod,
OffchainWorkerConfig, PruningMode, WasmExecutionMethod,
},
BasePath, Configuration, Error as ServiceError, Role, TaskExecutor,
};
use std::{sync::Arc, time::Duration};
use sp_api::BlockT;
use std::sync::Arc;
use substrate_test_client::BlockchainEventsExt;
use substrate_test_runtime_client::AccountKeyring::*;
use tokio::{spawn, time::delay_for as sleep};
static INTEGRATION_TEST_ALLOWED_TIME: Option<&str> = option_env!("INTEGRATION_TEST_ALLOWED_TIME");
#[tokio::test]
#[substrate_test_utils::test]
#[ignore]
async fn integration_test() {
let task_executor: TaskExecutor = (|fut, _| spawn(fut).map(|_| ())).into();
async fn integration_test(task_executor: TaskExecutor) {
let para_id = ParaId::from(100);
// generate parachain spec
let spec = Box::new(crate::chain_spec::get_chain_spec(para_id));
// start alice
let mut alice =
polkadot_test_service::run_test_node(task_executor.clone(), Alice, || {}, vec![]);
let alice = polkadot_test_service::run_test_node(task_executor.clone(), Alice, || {}, vec![]);
// start bob
let mut bob = polkadot_test_service::run_test_node(
let bob = polkadot_test_service::run_test_node(
task_executor.clone(),
Bob,
|| {},
vec![alice.addr.clone()],
);
let t1 = sleep(Duration::from_secs(
INTEGRATION_TEST_ALLOWED_TIME
.and_then(|x| x.parse().ok())
.unwrap_or(600),
))
.fuse();
// ensure alice and bob can produce blocks
future::join(alice.wait_for_blocks(2), bob.wait_for_blocks(2)).await;
let t2 = async {
let para_id = ParaId::from(100);
// export genesis state
let block = crate::command::generate_genesis_state(&(spec.clone() as Box<_>)).unwrap();
let genesis_state = block.header().encode();
future::join(alice.wait_for_blocks(2), bob.wait_for_blocks(2)).await;
// create and sign transaction to register parachain
let function = polkadot_test_runtime::Call::Sudo(pallet_sudo::Call::sudo(Box::new(
polkadot_test_runtime::Call::Registrar(registrar::Call::register_para(
para_id,
Info {
scheduling: Scheduling::Always,
},
parachain_runtime::WASM_BINARY
.expect("You need to build the WASM binary to run this test!")
.to_vec()
.into(),
genesis_state.into(),
)),
)));
// export genesis state
let spec = crate::chain_spec::get_chain_spec(para_id);
let genesis_state = crate::command::generate_genesis_state(&(Box::new(spec) as Box<_>))
.unwrap()
.encode();
// register parachain
let _ = alice.call_function(function, Alice).await.unwrap();
// create and sign transaction
let function = polkadot_test_runtime::Call::Sudo(pallet_sudo::Call::sudo(Box::new(
polkadot_test_runtime::Call::Registrar(registrar::Call::register_para(
para_id,
Info {
scheduling: Scheduling::Always,
},
parachain_runtime::WASM_BINARY
.expect("You need to build the WASM binary to run this test!")
.to_vec()
.into(),
genesis_state.into(),
)),
)));
// run cumulus charlie
let key = Arc::new(sp_core::Pair::generate().0);
let polkadot_config = polkadot_test_service::node_config(
|| {},
task_executor.clone(),
Charlie,
vec![alice.addr.clone(), bob.addr.clone()],
);
let parachain_config = parachain_config(task_executor.clone(), Charlie, vec![], spec).unwrap();
let (charlie_task_manager, charlie_client) =
crate::service::start_node(parachain_config, key, polkadot_config, para_id, true, true)
.unwrap();
charlie_client.wait_for_blocks(4).await;
// register parachain
let _ = alice.call_function(function, Alice).await.unwrap();
// run cumulus charlie
let key = Arc::new(sp_core::Pair::from_seed(&[10; 32]));
let mut polkadot_config = polkadot_test_service::node_config(
|| {},
task_executor.clone(),
Charlie,
vec![alice.addr.clone(), bob.addr.clone()],
);
use std::net::{Ipv4Addr, SocketAddr};
polkadot_config.rpc_http = Some(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 27016));
polkadot_config.rpc_methods = sc_service::config::RpcMethods::Unsafe;
let parachain_config =
parachain_config(task_executor.clone(), Charlie, vec![], para_id).unwrap();
let (_service, charlie_client) =
crate::service::start_node(parachain_config, key, polkadot_config, para_id, true)
.unwrap();
sleep(Duration::from_secs(3)).await;
charlie_client.wait_for_blocks(4).await;
alice.task_manager.terminate();
bob.task_manager.terminate();
}
.fuse();
pin_mut!(t1, t2);
select! {
_ = t1 => {
panic!("the test took too long, maybe no parachain blocks have been produced");
},
_ = t2 => {},
}
alice.task_manager.clean_shutdown();
bob.task_manager.clean_shutdown();
charlie_task_manager.clean_shutdown();
}
pub fn parachain_config(
task_executor: TaskExecutor,
key: Sr25519Keyring,
boot_nodes: Vec<MultiaddrWithPeerId>,
para_id: ParaId,
spec: Box<dyn ChainSpec>,
) -> Result<Configuration, ServiceError> {
let base_path = BasePath::new_temp_dir()?;
let root = base_path.path().to_path_buf();
@@ -138,7 +110,6 @@ pub fn parachain_config(
sentry_nodes: Vec::new(),
};
let key_seed = key.to_seed();
let spec = crate::chain_spec::get_chain_spec(para_id);
let mut network_config = NetworkConfiguration::new(
format!("Cumulus Test Node for: {}", key_seed),
@@ -153,7 +124,7 @@ pub fn parachain_config(
network_config.boot_nodes = boot_nodes;
network_config.allow_non_globals_in_dht = true;
network_config.allow_non_globals_in_dht = false;
network_config
.listen_addresses
@@ -176,10 +147,10 @@ pub fn parachain_config(
path: root.join("db"),
cache_size: 128,
},
state_cache_size: 16777216,
state_cache_size: 67108864,
state_cache_child_ratio: None,
pruning: Default::default(),
chain_spec: Box::new(spec),
pruning: PruningMode::ArchiveAll,
chain_spec: spec,
wasm_method: WasmExecutionMethod::Interpreted,
// NOTE: we enforce the use of the native runtime to make the errors more debuggable
execution_strategies: ExecutionStrategies {
@@ -199,7 +170,10 @@ pub fn parachain_config(
telemetry_endpoints: None,
telemetry_external_transport: None,
default_heap_pages: None,
offchain_worker: Default::default(),
offchain_worker: OffchainWorkerConfig {
enabled: true,
indexing_enabled: false,
},
force_authoring: false,
disable_grandpa: false,
dev_key_seed: Some(key_seed),
+101 -2
View File
@@ -15,18 +15,23 @@
// along with Cumulus. If not, see <http://www.gnu.org/licenses/>.
use ansi_term::Color;
use cumulus_collator::CollatorBuilder;
use cumulus_network::DelayedBlockAnnounceValidator;
use cumulus_service::{
prepare_node_config, start_collator, start_full_node, StartCollatorParams, StartFullNodeParams,
};
use polkadot_primitives::v0::CollatorPair;
use rococo_parachain_primitives::Block;
use sc_client_api::{Backend as BackendT, BlockBackend, Finalizer, UsageProvider};
use sc_executor::native_executor_instance;
pub use sc_executor::NativeExecutor;
use sc_informant::OutputFormat;
use sc_service::{Configuration, PartialComponents, Role, TFullBackend, TFullClient, TaskManager};
use sp_api::ConstructRuntimeApi;
use sp_runtime::traits::BlakeTwo256;
use sp_blockchain::HeaderBackend;
use sp_consensus::{BlockImport, Environment, Error as ConsensusError, Proposer};
use sp_core::crypto::Pair;
use sp_runtime::traits::{BlakeTwo256, Block as BlockT};
use sp_trie::PrefixedMemoryDB;
use std::sync::Arc;
@@ -116,6 +121,91 @@ where
Ok(params)
}
/// Start a test collator node for a parachain.
///
/// A collator is similar to a validator in a normal blockchain.
/// It is responsible for producing blocks and sending the blocks to a
/// parachain validator for validation and inclusion into the relay chain.
pub fn start_test_collator<'a, Block, PF, BI, BS, Client, Backend>(
StartCollatorParams {
para_id,
proposer_factory,
inherent_data_providers,
block_import,
block_status,
announce_block,
client,
block_announce_validator,
task_manager,
polkadot_config,
collator_key,
}: StartCollatorParams<'a, Block, PF, BI, BS, Client>,
) -> sc_service::error::Result<()>
where
Block: BlockT,
PF: Environment<Block> + Send + 'static,
BI: BlockImport<
Block,
Error = ConsensusError,
Transaction = <PF::Proposer as Proposer<Block>>::Transaction,
> + Send
+ Sync
+ 'static,
BS: BlockBackend<Block> + Send + Sync + 'static,
Client: Finalizer<Block, Backend>
+ UsageProvider<Block>
+ HeaderBackend<Block>
+ Send
+ Sync
+ BlockBackend<Block>
+ 'static,
for<'b> &'b Client: BlockImport<Block>,
Backend: BackendT<Block> + 'static,
{
let builder = CollatorBuilder::new(
proposer_factory,
inherent_data_providers,
block_import,
block_status,
para_id,
client,
announce_block,
block_announce_validator,
);
let (polkadot_future, polkadot_task_manager) = {
let (task_manager, client, handles, _network, _rpc_handlers) =
polkadot_test_service::polkadot_test_new_full(
polkadot_config,
Some((collator_key.public(), para_id)),
None,
false,
6000,
)?;
let test_client = polkadot_test_service::TestClient(client);
let future = polkadot_collator::build_collator_service(
task_manager.spawn_handle(),
handles,
test_client,
para_id,
collator_key,
builder,
)?;
(future, task_manager)
};
task_manager
.spawn_essential_handle()
.spawn("polkadot", polkadot_future);
task_manager.add_child(polkadot_task_manager);
Ok(())
}
/// Start a node with the given parachain `Configuration` and relay chain `Configuration`.
///
/// This is the actual implementation that is abstract over the executor and the runtime api.
@@ -126,6 +216,7 @@ fn start_node_impl<RuntimeApi, Executor, RB>(
id: polkadot_primitives::v0::Id,
validator: bool,
rpc_ext_builder: RB,
test: bool,
) -> sc_service::error::Result<(TaskManager, Arc<TFullClient<Block, RuntimeApi, Executor>>)>
where
RuntimeApi: ConstructRuntimeApi<Block, TFullClient<Block, RuntimeApi, Executor>>
@@ -240,7 +331,11 @@ where
collator_key,
};
start_collator(params)?;
if test {
start_test_collator(params)?;
} else {
start_collator(params)?;
}
} else {
let params = StartFullNodeParams {
client: client.clone(),
@@ -267,6 +362,7 @@ pub fn start_node(
polkadot_config: polkadot_collator::Configuration,
id: polkadot_primitives::v0::Id,
validator: bool,
test: bool,
) -> sc_service::error::Result<(
TaskManager,
Arc<TFullClient<Block, parachain_runtime::RuntimeApi, RuntimeExecutor>>,
@@ -278,6 +374,7 @@ pub fn start_node(
id,
validator,
|_| Default::default(),
test,
)
}
@@ -288,6 +385,7 @@ pub fn start_contracts_node(
polkadot_config: polkadot_collator::Configuration,
id: polkadot_primitives::v0::Id,
validator: bool,
test: bool,
) -> sc_service::error::Result<TaskManager> {
start_node_impl::<parachain_contracts_runtime::RuntimeApi, ContractsRuntimeExecutor, _>(
parachain_config,
@@ -302,6 +400,7 @@ pub fn start_contracts_node(
io.extend_with(ContractsApi::to_delegate(Contracts::new(client)));
io
},
test,
)
.map(|r| r.0)
}
+2 -2
View File
@@ -22,7 +22,7 @@ use cumulus_collator::CollatorBuilder;
use cumulus_network::{DelayedBlockAnnounceValidator, JustifiedBlockAnnounceValidator};
use cumulus_primitives::ParaId;
use polkadot_primitives::v0::{Block as PBlock, CollatorPair};
use polkadot_service::{AbstractClient, RuntimeApiCollection};
use polkadot_service::{AbstractClient, ClientHandle, RuntimeApiCollection};
use sc_client_api::{Backend as BackendT, BlockBackend, Finalizer, UsageProvider};
use sc_service::{Configuration, Role, TaskManager};
use sp_blockchain::{HeaderBackend, Result as ClientResult};
@@ -50,7 +50,7 @@ pub struct StartCollatorParams<'a, Block: BlockT, PF, BI, BS, Client> {
/// Start a collator node for a parachain.
///
/// A collator is similar to a validator in a normal blockchain.
/// It is reponsible for producing blocks and sending the blocks to a
/// It is responsible for producing blocks and sending the blocks to a
/// parachain validator for validation and inclusion into the relay chain.
pub fn start_collator<'a, Block, PF, BI, BS, Client, Backend>(
StartCollatorParams {