XCM revamp & Ping pallet (#391)

* Add spambot

* Fixes

* Add some extra functions to spambot, bump version

* Lock..

* Aggregate HRMP (XCMP/HMP) messages. Payloads for spambot.

* Fix tests, bump Polkadot.

* Fix HMP tests

* Rename Hrmp -> Xcmp for handler/sender

* Use master branch

* Test Xcm message passing & rename away from HMP

* Docs

* Introduce fee payment mechanics into XCM.

* Rename spambot -> ping

* Lock

* XCMP message dispatch system reimagining

- Moved most of the logic into xcm-handler pallet
- Altered the outgoing XCMP API from push to pull
- Changed underlying outgoing queue data structures to avoid multi-page read/writes
- Introduced queuing for incoming messages
- Introduced signal messages as a flow-control sub-stream
- Introduced flow-control with basic threshold back-pressure
- Introduced overall weight limitation on messages executed
- Additonal alterations to XCM APIs for the new system

* Should process any remaining XCM messages when we're not doing anything else.

* Update API usage and preparation for the big build.

* Some build fixes

* Build fixes

* xcm-handler builds

* Fix warnings

* Docs

* Parachains system builds

* Parachain runtime building

* Fix build

* Introduce transfer_asset specialisation.

* Fixes

* Two-stage upgrade for parachains.

* Fixes

* Fixes

* Updates for message sending.

* Repotting/renaming. Add primitives/utility.

* Remove real-overseer and bump refs

* Configure & document Rococo XCM runtime.

* Add shell runtime, some companion changes for #8589

* Bumps & fixes

* Fix test

* Build fix

* Update pallets/xcmp-queue/src/lib.rs

Co-authored-by: Amar Singh <asinghchrony@protonmail.com>

* Make tests compile

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* remove unused

* remove unused event stuff

* Adds proper validation-worker to make integration tests work

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* import saturating

* remove panic test

Co-authored-by: Robert Habermeier <rphmeier@gmail.com>
Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
Co-authored-by: Amar Singh <asinghchrony@protonmail.com>
Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>
Co-authored-by: Bastian Köcher <info@kchr.de>

primitives/core/Cargo.toml

@@ -9,16 +9,19 @@ edition = "2018"
 sp-std = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" }
 sp-runtime = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" }
 sp-trie = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" }
+frame-support = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" }
 
 # Polkadot dependencies
 polkadot-parachain = { git = "https://github.com/paritytech/polkadot", default-features = false, branch = "master" }
 polkadot-primitives = { git = "https://github.com/paritytech/polkadot", default-features = false, branch = "master" }
 polkadot-core-primitives = { git = "https://github.com/paritytech/polkadot", default-features = false, branch = "master" }
+xcm = { git = "https://github.com/paritytech/polkadot", default-features = false, branch = "master" }
 
 # Other dependencies
 impl-trait-for-tuples = "0.2.1"
 codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = [ "derive" ] }
 
+
 [features]
 default = [ "std" ]
 std = [
@@ -29,4 +32,5 @@ std = [
 	"polkadot-core-primitives/std",
 	"sp-runtime/std",
 	"sp-trie/std",
+	"frame-support/std",
 ]

primitives/core/src/lib.rs

@@ -18,7 +18,10 @@
 
 #![cfg_attr(not(feature = "std"), no_std)]
 
-use sp_runtime::traits::Block as BlockT;
+use sp_std::prelude::*;
+use codec::{Encode, Decode};
+use sp_runtime::{RuntimeDebug, traits::Block as BlockT};
+use frame_support::weights::Weight;
 
 pub use polkadot_core_primitives::InboundDownwardMessage;
 pub use polkadot_parachain::primitives::{Id as ParaId, UpwardMessage, ValidationParams};
@@ -32,6 +35,7 @@ pub mod relay_chain {
 	pub use polkadot_primitives::v1;
 	pub use polkadot_primitives::v1::well_known_keys;
 }
+use relay_chain::BlockNumber as RelayBlockNumber;
 
 /// An inbound HRMP message.
 pub type InboundHrmpMessage = polkadot_primitives::v1::InboundHrmpMessage<relay_chain::BlockNumber>;
@@ -39,6 +43,52 @@ pub type InboundHrmpMessage = polkadot_primitives::v1::InboundHrmpMessage<relay_
 /// And outbound HRMP message
 pub type OutboundHrmpMessage = polkadot_primitives::v1::OutboundHrmpMessage<ParaId>;
 
+/// Error description of a message send failure.
+#[derive(Eq, PartialEq, Copy, Clone, RuntimeDebug, Encode, Decode)]
+pub enum MessageSendError {
+	/// The dispatch queue is full.
+	QueueFull,
+	/// There does not exist a channel for sending the message.
+	NoChannel,
+	/// The message is too big to ever fit in a channel.
+	TooBig,
+	/// Some other error.
+	Other,
+}
+
+impl From<MessageSendError> for &'static str {
+	fn from(e: MessageSendError) -> Self {
+		use MessageSendError::*;
+		match e {
+			QueueFull => "QueueFull",
+			NoChannel => "NoChannel",
+			TooBig => "TooBig",
+			Other => "Other",
+		}
+	}
+}
+
+/// Information about an XCMP channel.
+pub struct ChannelInfo {
+	/// The maximum number of messages that can be pending in the channel at once.
+	pub max_capacity: u32,
+	/// The maximum total size of the messages that can be pending in the channel at once.
+	pub max_total_size: u32,
+	/// The maximum message size that could be put into the channel.
+	pub max_message_size: u32,
+	/// The current number of messages pending in the channel.
+	/// Invariant: should be less or equal to `max_capacity`.s`.
+	pub msg_count: u32,
+	/// The total size in bytes of all message payloads in the channel.
+	/// Invariant: should be less or equal to `max_total_size`.
+	pub total_size: u32,
+}
+
+pub trait GetChannelInfo {
+	fn get_channel_status(id: ParaId) -> ChannelStatus;
+	fn get_channel_max(id: ParaId) -> Option<usize>;
+}
+
 /// Well known keys for values in the storage.
 pub mod well_known_keys {
 	/// The storage key for the upward messages.
@@ -68,29 +118,81 @@ pub mod well_known_keys {
 }
 
 /// Something that should be called when a downward message is received.
-#[impl_trait_for_tuples::impl_for_tuples(30)]
 pub trait DownwardMessageHandler {
 	/// Handle the given downward message.
-	fn handle_downward_message(msg: InboundDownwardMessage);
+	fn handle_downward_message(msg: InboundDownwardMessage) -> Weight;
+}
+impl DownwardMessageHandler for () {
+	fn handle_downward_message(_msg: InboundDownwardMessage) -> Weight { 0 }
 }
 
-/// Something that should be called when an HRMP message is received.
-#[impl_trait_for_tuples::impl_for_tuples(30)]
-pub trait HrmpMessageHandler {
-	/// Handle the given HRMP message.
-	fn handle_hrmp_message(sender: ParaId, msg: InboundHrmpMessage);
+/// Something that should be called for each batch of messages received over XCMP.
+pub trait XcmpMessageHandler {
+	/// Handle some incoming XCMP messages (note these are the big one-per-block aggregate
+	/// messages).
+	///
+	/// Also, process messages up to some `max_weight`.
+	fn handle_xcmp_messages<'a, I: Iterator<Item=(ParaId, RelayBlockNumber, &'a [u8])>>(
+		iter: I,
+		max_weight: Weight,
+	) -> Weight;
+}
+impl XcmpMessageHandler for () {
+	fn handle_xcmp_messages<'a, I: Iterator<Item=(ParaId, RelayBlockNumber, &'a [u8])>>(
+		iter: I,
+		_max_weight: Weight,
+	) -> Weight { for _ in iter {} 0 }
 }
 
 /// Something that should be called when sending an upward message.
 pub trait UpwardMessageSender {
-	/// Send the given upward message.
-	fn send_upward_message(msg: UpwardMessage) -> Result<(), ()>;
+	/// Send the given UMP message; return the expected number of blocks before the message will
+	/// be dispatched or an error if the message cannot be sent.
+	fn send_upward_message(msg: UpwardMessage) -> Result<u32, MessageSendError>;
+}
+impl UpwardMessageSender for () {
+	fn send_upward_message(_msg: UpwardMessage) -> Result<u32, MessageSendError> {
+		Err(MessageSendError::NoChannel)
+	}
 }
 
-/// Something that should be called when sending an HRMP message.
-pub trait HrmpMessageSender {
-	/// Send the given HRMP message.
-	fn send_hrmp_message(msg: OutboundHrmpMessage) -> Result<(), ()>;
+/// The status of a channel.
+pub enum ChannelStatus {
+	/// Channel doesn't exist/has been closed.
+	Closed,
+	/// Channel is completely full right now.
+	Full,
+	/// Channel is ready for sending; the two parameters are the maximum size a valid message may
+	/// have right now, and the maximum size a message may ever have (this will generally have been
+	/// available during message construction, but it's possible the channel parameters changed in
+	/// the meantime).
+	Ready(usize, usize),
+}
+
+/// A means of figuring out what outbound XCMP messages should be being sent.
+pub trait XcmpMessageSource {
+	/// Take a single XCMP message from the queue for the given `dest`, if one exists.
+	fn take_outbound_messages(
+		maximum_channels: usize,
+	) -> Vec<(ParaId, Vec<u8>)>;
+}
+
+impl XcmpMessageSource for () {
+	fn take_outbound_messages(
+		_maximum_channels: usize,
+	) -> Vec<(ParaId, Vec<u8>)> { vec![] }
+}
+
+/// The "quality of service" considerations for message sending.
+#[derive(Eq, PartialEq, Clone, Copy, Encode, Decode, RuntimeDebug)]
+pub enum ServiceQuality {
+	/// Ensure that this message is dispatched in the same relative order as any other messages that
+	/// were also sent with `Ordered`. This only guarantees message ordering on the dispatch side,
+	/// and not necessarily on the execution side.
+	Ordered,
+	/// Ensure that the message is dispatched as soon as possible, which could result in it being
+	/// dispatched before other messages which are larger and/or rely on relative ordering.
+	Fast,
 }
 
 /// A trait which is called when the validation data is set.

primitives/utility/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "cumulus-primitives-utility"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+# Substrate dependencies
+sp-std = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" }
+sp-runtime = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" }
+sp-trie = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" }
+frame-support = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" }
+
+# Polkadot dependencies
+polkadot-parachain = { git = "https://github.com/paritytech/polkadot", default-features = false, branch = "master" }
+polkadot-primitives = { git = "https://github.com/paritytech/polkadot", default-features = false, branch = "master" }
+polkadot-core-primitives = { git = "https://github.com/paritytech/polkadot", default-features = false, branch = "master" }
+xcm = { git = "https://github.com/paritytech/polkadot", default-features = false, branch = "master" }
+
+cumulus-primitives-core = { path = "../core", default-features = false }
+
+# Other dependencies
+impl-trait-for-tuples = "0.2.1"
+codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = [ "derive" ] }
+
+
+[features]
+default = [ "std" ]
+std = [
+	"codec/std",
+	"sp-std/std",
+	"polkadot-primitives/std",
+	"polkadot-parachain/std",
+	"polkadot-core-primitives/std",
+	"sp-runtime/std",
+	"sp-trie/std",
+	"frame-support/std",
+	"cumulus-primitives-core/std",
+]

primitives/utility/src/lib.rs

@@ -0,0 +1,77 @@
+// Copyright 2020-2021 Parity Technologies (UK) Ltd.
+// This file is part of Cumulus.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Cumulus.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Helper datatypes for cumulus. This includes the [`ParentAsUmp`] routing type which will route
+//! messages into an [`UpwardMessageSender`] if the destination is `Parent`.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+
+use sp_std::{marker::PhantomData, convert::TryFrom};
+use codec::{Encode, Decode};
+use cumulus_primitives_core::{UpwardMessageSender, DownwardMessageHandler, InboundDownwardMessage};
+use xcm::{VersionedXcm, v0::{Xcm, MultiLocation, Junction, SendXcm, Error as XcmError, ExecuteXcm}};
+use frame_support::{traits::Get, dispatch::Weight};
+
+/// Xcm router which recognises the `Parent` destination and handles it by sending the message into
+/// the given UMP `UpwardMessageSender` implementation. Thus this essentially adapts an
+/// `UpwardMessageSender` trait impl into a `SendXcm` trait impl.
+///
+/// NOTE: This is a pretty dumb "just send it" router; we will probably want to introduce queuing
+/// to UMP eventually and when we do, the pallet which implements the queuing will be responsible
+/// for the `SendXcm` implementation.
+pub struct ParentAsUmp<T>(PhantomData<T>);
+impl<T: UpwardMessageSender> SendXcm for ParentAsUmp<T> {
+	fn send_xcm(dest: MultiLocation, msg: Xcm<()>) -> Result<(), XcmError> {
+		match &dest {
+			// An upward message for the relay chain.
+			MultiLocation::X1(Junction::Parent) => {
+				let data = VersionedXcm::<()>::from(msg).encode();
+
+				T::send_upward_message(data)
+					.map_err(|e| XcmError::SendFailed(e.into()))?;
+
+				Ok(())
+			}
+			// Anything else is unhandled. This includes a message this is meant for us.
+			_ => Err(XcmError::CannotReachDestination(dest, msg)),
+		}
+	}
+}
+
+/// For an incoming downward message, this just adapts an XCM executor and executes DMP messages
+/// immediately up until some `MaxWeight` at which point it errors. Their origin is asserted to be
+/// the Parent location.
+pub struct UnqueuedDmpAsParent<MaxWeight, XcmExecutor, Call>(
+	PhantomData<(MaxWeight, XcmExecutor, Call)>
+);
+impl<
+	MaxWeight: Get<Weight>,
+	XcmExecutor: ExecuteXcm<Call>,
+	Call,
+> DownwardMessageHandler for UnqueuedDmpAsParent<MaxWeight, XcmExecutor, Call> {
+	fn handle_downward_message(msg: InboundDownwardMessage) -> Weight {
+		let msg = VersionedXcm::<Call>::decode(&mut &msg.msg[..])
+			.map(Xcm::<Call>::try_from);
+		match msg {
+			Ok(Ok(x)) => {
+				let weight_limit = MaxWeight::get();
+				XcmExecutor::execute_xcm(Junction::Parent.into(), x, weight_limit).weight_used()
+			}
+			Ok(Err(..)) => 0,
+			Err(..) => 0,
+		}
+	}
+}