XCM revamp & Ping pallet (#391)

* Add spambot

* Fixes

* Add some extra functions to spambot, bump version

* Lock..

* Aggregate HRMP (XCMP/HMP) messages. Payloads for spambot.

* Fix tests, bump Polkadot.

* Fix HMP tests

* Rename Hrmp -> Xcmp for handler/sender

* Use master branch

* Test Xcm message passing & rename away from HMP

* Docs

* Introduce fee payment mechanics into XCM.

* Rename spambot -> ping

* Lock

* XCMP message dispatch system reimagining

- Moved most of the logic into xcm-handler pallet
- Altered the outgoing XCMP API from push to pull
- Changed underlying outgoing queue data structures to avoid multi-page read/writes
- Introduced queuing for incoming messages
- Introduced signal messages as a flow-control sub-stream
- Introduced flow-control with basic threshold back-pressure
- Introduced overall weight limitation on messages executed
- Additonal alterations to XCM APIs for the new system

* Should process any remaining XCM messages when we're not doing anything else.

* Update API usage and preparation for the big build.

* Some build fixes

* Build fixes

* xcm-handler builds

* Fix warnings

* Docs

* Parachains system builds

* Parachain runtime building

* Fix build

* Introduce transfer_asset specialisation.

* Fixes

* Two-stage upgrade for parachains.

* Fixes

* Fixes

* Updates for message sending.

* Repotting/renaming. Add primitives/utility.

* Remove real-overseer and bump refs

* Configure & document Rococo XCM runtime.

* Add shell runtime, some companion changes for #8589

* Bumps & fixes

* Fix test

* Build fix

* Update pallets/xcmp-queue/src/lib.rs

Co-authored-by: Amar Singh <asinghchrony@protonmail.com>

* Make tests compile

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* remove unused

* remove unused event stuff

* Adds proper validation-worker to make integration tests work

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* import saturating

* remove panic test

Co-authored-by: Robert Habermeier <rphmeier@gmail.com>
Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
Co-authored-by: Amar Singh <asinghchrony@protonmail.com>
Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>
Co-authored-by: Bastian Köcher <info@kchr.de>

client/collator/src/lib.rs

@@ -400,10 +400,15 @@ mod tests {
 	use cumulus_test_runtime::{Block, Header};
 	use futures::{channel::mpsc, executor::block_on, StreamExt};
 	use polkadot_node_subsystem_test_helpers::ForwardSubsystem;
-	use polkadot_overseer::{AllSubsystems, Overseer};
+	use polkadot_overseer::{AllSubsystems, Overseer, HeadSupportsParachains};
 	use sp_consensus::BlockOrigin;
 	use sp_core::{testing::TaskExecutor, Pair};
 
+	struct AlwaysSupportsParachains;
+	impl HeadSupportsParachains for AlwaysSupportsParachains {
+		fn head_supports_parachains(&self, _head: &PHash) -> bool { true }
+	}
+
 	#[derive(Clone)]
 	struct DummyParachainConsensus {
 		client: Arc<Client>,
@@ -454,8 +459,13 @@ mod tests {
 
 		let all_subsystems =
 			AllSubsystems::<()>::dummy().replace_collation_generation(ForwardSubsystem(sub_tx));
-		let (overseer, handler) = Overseer::new(Vec::new(), all_subsystems, None, spawner.clone())
-			.expect("Creates overseer");
+		let (overseer, handler) = Overseer::new(
+			Vec::new(),
+			all_subsystems,
+			None,
+			AlwaysSupportsParachains,
+			spawner.clone(),
+		).expect("Creates overseer");
 
 		spawner.spawn("overseer", overseer.run().then(|_| async { () }).boxed());
 

client/consensus/relay-chain/Cargo.toml

@@ -18,7 +18,7 @@ sc-client-api = { git = "https://github.com/paritytech/substrate", branch = "mas
 substrate-prometheus-endpoint = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
 # Polkadot dependencies
-polkadot-service = { git = "https://github.com/paritytech/polkadot", branch = "master", features = [ "real-overseer" ] }
+polkadot-service = { git = "https://github.com/paritytech/polkadot", branch = "master" }
 
 # Cumulus dependencies
 cumulus-client-consensus-common = { path = "../common" }

client/network/Cargo.toml

@@ -38,7 +38,6 @@ cumulus-primitives-core = { path = "../../primitives/core" }
 
 # Polkadot deps
 polkadot-test-client = { git = "https://github.com/paritytech/polkadot", branch = "master" }
-polkadot-test-service = { git = "https://github.com/paritytech/polkadot", branch = "master" }
 
 # substrate deps
 sc-client-api = { git = "https://github.com/paritytech/substrate", branch = "master" }

client/network/tests/sync.rs

@@ -15,7 +15,7 @@
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
 use cumulus_primitives_core::ParaId;
-use cumulus_test_service::{initial_head_data, Keyring::*};
+use cumulus_test_service::{initial_head_data, run_relay_chain_validator_node, Keyring::*};
 use futures::join;
 use sc_service::TaskExecutor;
 
@@ -28,16 +28,11 @@ async fn sync_blocks_from_tip_without_being_connected_to_a_collator(task_executo
 	let para_id = ParaId::from(100);
 
 	// start alice
-	let alice =
-		polkadot_test_service::run_validator_node(task_executor.clone(), Alice, || {}, vec![]);
+	let alice = run_relay_chain_validator_node(task_executor.clone(), Alice, || {}, vec![]);
 
 	// start bob
-	let bob = polkadot_test_service::run_validator_node(
-		task_executor.clone(),
-		Bob,
-		|| {},
-		vec![alice.addr.clone()],
-	);
+	let bob =
+		run_relay_chain_validator_node(task_executor.clone(), Bob, || {}, vec![alice.addr.clone()]);
 
 	// register parachain
 	alice