Mock XCM (#876)

* sketch downward messages

* bring in attempt to mock mqc-head from moonbeam

* just patch individual crates

* fing comma

* add some logs

* Holy shit, we actually imported a block!

* Actually mock the message queue chain

* use relay parent number for `sent_at`

* finish moving MQC to primitives

* more complete mock and better config type

* change name

* fix export

* better map types

* fix dependencies after rebase

* try-rejigging branches because this is an override

* try to re-jig for hrmp mcqs

* fix branches

* actually fix branches better

* even better

* Removestray log lines

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Nicer handling of default `ParachainSystem` name

* better docs

* Default MockXcm for people who only who don't care to mock xcm.

* cargo fmt

* trailing commas

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* use the variable for hrmp to

* fix deref

* deduplicate MessageQueueChain

* better docs for MessageQueueChain

* Use `Vec<u8>` instead of `&'static [u8]`

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* cargo fmt

* associated changes for using Vec<u8>

* Unused import

* Fix compilation

Co-authored-by: Joshy Orndorff <admin@joshyorndorff.com>
Co-authored-by: Joshy Orndorff <JoshOrndorff@users.noreply.github.com>

Cargo.lock

@@ -1795,6 +1795,7 @@ dependencies = [
  "sp-runtime",
  "sp-state-machine",
  "sp-std",
+ "sp-storage",
  "sp-trie",
  "tracing",
 ]

pallets/parachain-system/src/lib.rs

@@ -33,7 +33,7 @@ use cumulus_primitives_core::{
 	OutboundHrmpMessage, ParaId, PersistedValidationData, UpwardMessage, UpwardMessageSender,
 	XcmpMessageHandler, XcmpMessageSource,
 };
-use cumulus_primitives_parachain_inherent::ParachainInherentData;
+use cumulus_primitives_parachain_inherent::{MessageQueueChain, ParachainInherentData};
 use frame_support::{
 	dispatch::{DispatchError, DispatchResult},
 	ensure,
@@ -46,7 +46,7 @@ use frame_system::{ensure_none, ensure_root};
 use polkadot_parachain::primitives::RelayChainBlockNumber;
 use relay_state_snapshot::MessagingStateSnapshot;
 use sp_runtime::{
-	traits::{BlakeTwo256, Block as BlockT, BlockNumberProvider, Hash},
+	traits::{Block as BlockT, BlockNumberProvider, Hash},
 	transaction_validity::{
 		InvalidTransaction, TransactionLongevity, TransactionSource, TransactionValidity,
 		ValidTransaction,
@@ -750,7 +750,7 @@ impl<T: Config> Pallet<T> {
 			weight_used += T::DmpMessageHandler::handle_dmp_messages(message_iter, max_weight);
 			<LastDmqMqcHead<T>>::put(&dmq_head);
 
-			Self::deposit_event(Event::DownwardMessagesProcessed(weight_used, dmq_head.0));
+			Self::deposit_event(Event::DownwardMessagesProcessed(weight_used, dmq_head.head()));
 		}
 
 		// After hashing each message in the message queue chain submitted by the collator, we
@@ -758,7 +758,7 @@ impl<T: Config> Pallet<T> {
 		//
 		// A mismatch means that at least some of the submitted messages were altered, omitted or
 		// added improperly.
-		assert_eq!(dmq_head.0, expected_dmq_mqc_head);
+		assert_eq!(dmq_head.head(), expected_dmq_mqc_head);
 
 		ProcessedDownwardMessages::<T>::put(dm_count);
 
@@ -945,43 +945,6 @@ impl<T: Config> frame_system::SetCode<T> for ParachainSetCode<T> {
 	}
 }
 
-/// This struct provides ability to extend a message queue chain (MQC) and compute a new head.
-///
-/// MQC is an instance of a [hash chain] applied to a message queue. Using a hash chain it's
-/// possible to represent a sequence of messages using only a single hash.
-///
-/// A head for an empty chain is agreed to be a zero hash.
-///
-/// [hash chain]: https://en.wikipedia.org/wiki/Hash_chain
-#[derive(Default, Clone, codec::Encode, codec::Decode, scale_info::TypeInfo)]
-struct MessageQueueChain(relay_chain::Hash);
-
-impl MessageQueueChain {
-	fn extend_hrmp(&mut self, horizontal_message: &InboundHrmpMessage) -> &mut Self {
-		let prev_head = self.0;
-		self.0 = BlakeTwo256::hash_of(&(
-			prev_head,
-			horizontal_message.sent_at,
-			BlakeTwo256::hash_of(&horizontal_message.data),
-		));
-		self
-	}
-
-	fn extend_downward(&mut self, downward_message: &InboundDownwardMessage) -> &mut Self {
-		let prev_head = self.0;
-		self.0 = BlakeTwo256::hash_of(&(
-			prev_head,
-			downward_message.sent_at,
-			BlakeTwo256::hash_of(&downward_message.msg),
-		));
-		self
-	}
-
-	fn head(&self) -> relay_chain::Hash {
-		self.0
-	}
-}
-
 impl<T: Config> Pallet<T> {
 	pub fn send_upward_message(message: UpwardMessage) -> Result<u32, MessageSendError> {
 		// Check if the message fits into the relay-chain constraints.

pallets/parachain-system/src/tests.rs

@@ -33,7 +33,10 @@ use frame_system::{InitKind, RawOrigin};
 use hex_literal::hex;
 use relay_chain::v1::HrmpChannelId;
 use sp_core::H256;
-use sp_runtime::{testing::Header, traits::IdentityLookup};
+use sp_runtime::{
+	testing::Header,
+	traits::{BlakeTwo256, IdentityLookup},
+};
 use sp_version::RuntimeVersion;
 use std::cell::RefCell;
 

primitives/parachain-inherent/Cargo.toml

@@ -14,6 +14,7 @@ sp-std = { git = "https://github.com/paritytech/substrate", default-features = f
 sp-state-machine = { git = "https://github.com/paritytech/substrate", optional = true , branch = "master" }
 sp-trie = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" }
 sp-api = { git = "https://github.com/paritytech/substrate", optional = true , branch = "master" }
+sp-storage = { git = "https://github.com/paritytech/substrate", optional = true , branch = "master" }
 
 # Cumulus dependencies
 cumulus-primitives-core = { path = "../core", default-features = false }
@@ -42,6 +43,8 @@ std = [
 	"sp-runtime",
 	"sc-client-api",
 	"sp-api",
+	"sp-storage",
+	"cumulus-test-relay-sproof-builder",
 	"cumulus-relay-chain-interface",
 	"cumulus-test-relay-sproof-builder"
 ]

primitives/parachain-inherent/src/lib.rs

@@ -28,6 +28,7 @@
 #![cfg_attr(not(feature = "std"), no_std)]
 
 use cumulus_primitives_core::{
+	relay_chain::{BlakeTwo256, Hash as RelayHash, HashT as _},
 	InboundDownwardMessage, InboundHrmpMessage, ParaId, PersistedValidationData,
 };
 
@@ -42,7 +43,7 @@ pub use client_side::*;
 #[cfg(feature = "std")]
 mod mock;
 #[cfg(feature = "std")]
-pub use mock::MockValidationDataInherentDataProvider;
+pub use mock::{MockValidationDataInherentDataProvider, MockXcmConfig};
 
 /// The identifier for the parachain inherent.
 pub const INHERENT_IDENTIFIER: InherentIdentifier = *b"sysi1337";
@@ -68,3 +69,50 @@ pub struct ParachainInherentData {
 	/// this means `sent_at` is **strictly** greater than the previous one (if any).
 	pub horizontal_messages: BTreeMap<ParaId, Vec<InboundHrmpMessage>>,
 }
+
+/// This struct provides ability to extend a message queue chain (MQC) and compute a new head.
+///
+/// MQC is an instance of a [hash chain] applied to a message queue. Using a hash chain it's
+/// possible to represent a sequence of messages using only a single hash.
+///
+/// A head for an empty chain is agreed to be a zero hash.
+///
+/// An instance is used to track either DMP from the relay chain or HRMP across a channel.
+/// But a given instance is never used to track both. Therefore, you should call either
+/// `extend_downward` or `extend_hrmp`, but not both methods on a single instance.
+///
+/// [hash chain]: https://en.wikipedia.org/wiki/Hash_chain
+#[derive(Default, Clone, codec::Encode, codec::Decode, scale_info::TypeInfo)]
+pub struct MessageQueueChain(RelayHash);
+
+impl MessageQueueChain {
+	/// Extend the hash chain with an HRMP message. This method should be used only when
+	/// this chain is tracking HRMP.
+	pub fn extend_hrmp(&mut self, horizontal_message: &InboundHrmpMessage) -> &mut Self {
+		let prev_head = self.0;
+		self.0 = BlakeTwo256::hash_of(&(
+			prev_head,
+			horizontal_message.sent_at,
+			BlakeTwo256::hash_of(&horizontal_message.data),
+		));
+		self
+	}
+
+	/// Extend the hash chain with a downward message. This method should be used only when
+	/// this chain is tracking DMP.
+	pub fn extend_downward(&mut self, downward_message: &InboundDownwardMessage) -> &mut Self {
+		let prev_head = self.0;
+		self.0 = BlakeTwo256::hash_of(&(
+			prev_head,
+			downward_message.sent_at,
+			BlakeTwo256::hash_of(&downward_message.msg),
+		));
+		self
+	}
+
+	/// Return the current mead of the message queue hash chain.
+	/// This is agreed to be the zero hash for an empty chain.
+	pub fn head(&self) -> RelayHash {
+		self.0
+	}
+}

primitives/parachain-inherent/src/mock.rs

@@ -15,8 +15,16 @@
 // along with Cumulus.  If not, see <http://www.gnu.org/licenses/>.
 
 use crate::{ParachainInherentData, INHERENT_IDENTIFIER};
-use cumulus_primitives_core::PersistedValidationData;
+use codec::Decode;
+use cumulus_primitives_core::{
+	relay_chain, InboundDownwardMessage, InboundHrmpMessage, ParaId, PersistedValidationData,
+};
+use sc_client_api::{Backend, StorageProvider};
+use sp_api::BlockId;
+use sp_core::twox_128;
 use sp_inherents::{InherentData, InherentDataProvider};
+use sp_runtime::traits::Block;
+use std::collections::BTreeMap;
 
 use cumulus_test_relay_sproof_builder::RelayStateSproofBuilder;
 
@@ -29,6 +37,11 @@ use cumulus_test_relay_sproof_builder::RelayStateSproofBuilder;
 /// relay_block_number = offset + relay_blocks_per_para_block * current_para_block
 /// To simulate a parachain that starts in relay block 1000 and gets a block in every other relay
 /// block, use 1000 and 2
+///
+/// Optionally, mock XCM messages can be injected into the runtime. When mocking XCM,
+/// in addition to the messages themselves, you must provide some information about
+/// your parachain's configuration in order to mock the MQC heads properly.
+/// See [`MockXcmConfig`] for more information
 pub struct MockValidationDataInherentDataProvider {
 	/// The current block number of the local block chain (the parachain)
 	pub current_para_block: u32,
@@ -38,6 +51,82 @@ pub struct MockValidationDataInherentDataProvider {
 	/// The number of relay blocks that elapses between each parablock. Probably set this to 1 or 2
 	/// to simulate optimistic or realistic relay chain behavior.
 	pub relay_blocks_per_para_block: u32,
+	/// XCM messages and associated configuration information.
+	pub xcm_config: MockXcmConfig,
+	/// Inbound downward XCM messages to be injected into the block.
+	pub raw_downward_messages: Vec<Vec<u8>>,
+	// Inbound Horizontal messages sorted by channel
+	pub raw_horizontal_messages: Vec<(ParaId, Vec<u8>)>,
+}
+
+/// Parameters for how the Mock inherent data provider should inject XCM messages.
+/// In addition to the messages themselves, some information about the parachain's
+/// configuration is also required so that the MQC heads can be read out of the
+/// parachain's storage, and the corresponding relay data mocked.
+#[derive(Default)]
+pub struct MockXcmConfig {
+	/// The parachain id of the parachain being mocked.
+	pub para_id: ParaId,
+	/// The starting state of the dmq_mqc_head.
+	pub starting_dmq_mqc_head: relay_chain::Hash,
+	/// The starting state of each parachain's mqc head
+	pub starting_hrmp_mqc_heads: BTreeMap<ParaId, relay_chain::Hash>,
+}
+
+/// The name of the parachain system in the runtime.
+///
+/// This name is used by frame to prefix storage items and will be required to read data from the storage.
+///
+/// The `Default` implementation sets the name to `ParachainSystem`.
+pub struct ParachainSystemName(pub Vec<u8>);
+
+impl Default for ParachainSystemName {
+	fn default() -> Self {
+		Self(b"ParachainSystem".to_vec())
+	}
+}
+
+impl MockXcmConfig {
+	/// Create a MockXcmConfig by reading the mqc_heads directly
+	/// from the storage of a previous block.
+	pub fn new<B: Block, BE: Backend<B>, C: StorageProvider<B, BE>>(
+		client: &C,
+		parent_block: B::Hash,
+		para_id: ParaId,
+		parachain_system_name: ParachainSystemName,
+	) -> Self {
+		let starting_dmq_mqc_head = client
+			.storage(
+				&BlockId::Hash(parent_block),
+				&sp_storage::StorageKey(
+					[twox_128(&parachain_system_name.0), twox_128(b"LastDmqMqcHead")]
+						.concat()
+						.to_vec(),
+				),
+			)
+			.expect("We should be able to read storage from the parent block.")
+			.map(|ref mut raw_data| {
+				Decode::decode(&mut &raw_data.0[..]).expect("Stored data should decode correctly")
+			})
+			.unwrap_or_default();
+
+		let starting_hrmp_mqc_heads = client
+			.storage(
+				&BlockId::Hash(parent_block),
+				&sp_storage::StorageKey(
+					[twox_128(&parachain_system_name.0), twox_128(b"LastHrmpMqcHeads")]
+						.concat()
+						.to_vec(),
+				),
+			)
+			.expect("We should be able to read storage from the parent block.")
+			.map(|ref mut raw_data| {
+				Decode::decode(&mut &raw_data.0[..]).expect("Stored data should decode correctly")
+			})
+			.unwrap_or_default();
+
+		Self { para_id, starting_dmq_mqc_head, starting_hrmp_mqc_heads }
+	}
 }
 
 #[async_trait::async_trait]
@@ -46,27 +135,65 @@ impl InherentDataProvider for MockValidationDataInherentDataProvider {
 		&self,
 		inherent_data: &mut InherentData,
 	) -> Result<(), sp_inherents::Error> {
-		// Use the "sproof" (spoof proof) builder to build valid mock state root and proof.
-		let (relay_storage_root, proof) =
-			RelayStateSproofBuilder::default().into_state_root_and_proof();
-
 		// Calculate the mocked relay block based on the current para block
 		let relay_parent_number =
 			self.relay_offset + self.relay_blocks_per_para_block * self.current_para_block;
 
-		let data = ParachainInherentData {
-			validation_data: PersistedValidationData {
-				parent_head: Default::default(),
-				relay_parent_storage_root: relay_storage_root,
-				relay_parent_number,
-				max_pov_size: Default::default(),
-			},
-			downward_messages: Default::default(),
-			horizontal_messages: Default::default(),
-			relay_chain_state: proof,
-		};
+		// Use the "sproof" (spoof proof) builder to build valid mock state root and proof.
+		let mut sproof_builder = RelayStateSproofBuilder::default();
+		sproof_builder.para_id = self.xcm_config.para_id;
 
-		inherent_data.put_data(INHERENT_IDENTIFIER, &data)
+		// Process the downward messages and set up the correct head
+		let mut downward_messages = Vec::new();
+		let mut dmq_mqc = crate::MessageQueueChain(self.xcm_config.starting_dmq_mqc_head);
+		for msg in &self.raw_downward_messages {
+			let wrapped = InboundDownwardMessage { sent_at: relay_parent_number, msg: msg.clone() };
+
+			dmq_mqc.extend_downward(&wrapped);
+			downward_messages.push(wrapped);
+		}
+		sproof_builder.dmq_mqc_head = Some(dmq_mqc.head());
+
+		// Process the hrmp messages and set up the correct heads
+		// Begin by collecting them into a Map
+		let mut horizontal_messages = BTreeMap::<ParaId, Vec<InboundHrmpMessage>>::new();
+		for (para_id, msg) in &self.raw_horizontal_messages {
+			let wrapped = InboundHrmpMessage { sent_at: relay_parent_number, data: msg.clone() };
+
+			horizontal_messages.entry(*para_id).or_default().push(wrapped);
+		}
+
+		// Now iterate again, updating the heads as we go
+		for (para_id, messages) in &horizontal_messages {
+			let mut channel_mqc = crate::MessageQueueChain(
+				*self
+					.xcm_config
+					.starting_hrmp_mqc_heads
+					.get(para_id)
+					.unwrap_or(&relay_chain::Hash::default()),
+			);
+			for message in messages {
+				channel_mqc.extend_hrmp(message);
+			}
+			sproof_builder.upsert_inbound_channel(*para_id).mqc_head = Some(channel_mqc.head());
+		}
+
+		let (relay_parent_storage_root, proof) = sproof_builder.into_state_root_and_proof();
+
+		inherent_data.put_data(
+			INHERENT_IDENTIFIER,
+			&ParachainInherentData {
+				validation_data: PersistedValidationData {
+					parent_head: Default::default(),
+					relay_parent_storage_root,
+					relay_parent_number,
+					max_pov_size: Default::default(),
+				},
+				downward_messages,
+				horizontal_messages,
+				relay_chain_state: proof,
+			},
+		)
 	}
 
 	// Copied from the real implementation