Downward & Upward messages (#1266)

* Downward messages, the front-end.

* Move types around to make them accessible from Parachains

* Fix compilation

* Fix branch

* Make it compile for Cumulus

* Update the branch names

* Add default generic parameter

* Implement `Partialeq`

* Move upward messages into the `ValidationResult`

* Support disabling of the runtime api

* Update branch

* Adds support for handling downward messages

* Implement sending XCMP messages as up/downward messages

* service: update to latest ServiceBuilder changes

* Make it compile

* Initial commit

Forked at: ef2aa428d7
Parent branch: origin/master

* Update substrate branch to cecton-update-polkadot-substrate

* Update substrate & polkadot to cumulus-branch

* Reset branch

* Update primitives/src/parachain.rs

Co-authored-by: Robert Habermeier <rphmeier@gmail.com>

* Update runtime/common/src/parachains.rs

Co-authored-by: Robert Habermeier <rphmeier@gmail.com>

* Update runtime/common/src/parachains.rs

Co-authored-by: Robert Habermeier <rphmeier@gmail.com>

* Minor fixes

* Fix wasm build

Co-authored-by: Gav Wood <gavin@parity.io>
Co-authored-by: André Silva <andre.beat@gmail.com>
Co-authored-by: Cecile Tonglet <cecile.tonglet@cecton.com>
Co-authored-by: Robert Habermeier <rphmeier@gmail.com>

polkadot/parachain/src/primitives.rs

@@ -28,9 +28,8 @@ use serde::{Serialize, Deserialize};
 #[cfg(feature = "std")]
 use sp_core::bytes;
 
-/// The block number of the relay chain.
-/// 32-bits will allow for 136 years of blocks assuming 1 block per second.
-pub type RelayChainBlockNumber = u32;
+/// Block number type used by the relay chain.
+pub use polkadot_core_primitives::BlockNumber as RelayChainBlockNumber;
 
 /// Parachain head data included in the chain.
 #[derive(PartialEq, Eq, Clone, PartialOrd, Ord, Encode, Decode, RuntimeDebug)]
@@ -186,10 +185,6 @@ impl sp_std::convert::TryFrom<u8> for ParachainDispatchOrigin {
 
 /// A message from a parachain to its Relay Chain.
 #[derive(Clone, PartialEq, Eq, Encode, Decode)]
-#[cfg_attr(
-	any(feature = "std", feature = "wasm-api"),
-	derive(sp_runtime_interface::pass_by::PassByCodec,
-))]
 #[cfg_attr(feature = "std", derive(Debug))]
 pub struct UpwardMessage {
 	/// The origin for the message to be sent from.
@@ -212,13 +207,13 @@ pub struct ValidationParams {
 	/// The maximum head-data size permitted, in bytes.
 	pub max_head_data_size: u32,
 	/// The current relay-chain block number.
-	pub relay_chain_height: RelayChainBlockNumber,
+	pub relay_chain_height: polkadot_core_primitives::BlockNumber,
 	/// Whether a code upgrade is allowed or not, and at which height the upgrade
 	/// would be applied after, if so. The parachain logic should apply any upgrade
 	/// issued in this block after the first block
 	/// with `relay_chain_height` at least this value, if `Some`. if `None`, issue
 	/// no upgrade.
-	pub code_upgrade_allowed: Option<RelayChainBlockNumber>,
+	pub code_upgrade_allowed: Option<polkadot_core_primitives::BlockNumber>,
 }
 
 /// The result of parachain validation.
@@ -230,4 +225,10 @@ pub struct ValidationResult {
 	pub head_data: HeadData,
 	/// An update to the validation code that should be scheduled in the relay chain.
 	pub new_validation_code: Option<ValidationCode>,
+	/// Upward messages send by the Parachain.
+	pub upward_messages: Vec<UpwardMessage>,
+	/// Number of downward messages that were processed by the Parachain.
+	///
+	/// It is expected that the Parachain processes them from first to last.
+	pub processed_downward_messages: u32,
 }

polkadot/parachain/src/wasm_api.rs

@@ -16,29 +16,6 @@
 
 //! Utilities for writing parachain WASM.
 
-#[cfg(any(feature = "std", all(not(feature = "std"), feature = "wasm-api")))]
-use crate::primitives::UpwardMessage;
-#[cfg(any(feature = "std", all(not(feature = "std"), feature = "wasm-api")))]
-use sp_runtime_interface::runtime_interface;
-#[cfg(feature = "std")]
-use sp_externalities::ExternalitiesExt;
-
-/// The parachain api for posting messages.
-// Either activate on `std` to get access to the `HostFunctions` or when `wasm-api` is given and on
-// `no_std`.
-#[cfg(any(feature = "std", all(not(feature = "std"), feature = "wasm-api")))]
-#[runtime_interface]
-pub trait Parachain {
-	/// Post a message to this parachain's relay chain.
-	#[allow(dead_code)]
-	fn post_upward_message(&mut self, msg: UpwardMessage) {
-		self.extension::<crate::wasm_executor::ParachainExt>()
-			.expect("No `ParachainExt` associated with the current context.")
-			.post_upward_message(msg)
-			.expect("Failed to post upward message")
-	}
-}
-
 /// Load the validation params from memory when implementing a Rust parachain.
 ///
 /// Offset and length must have been provided by the validation

polkadot/parachain/src/wasm_executor/mod.rs

@@ -21,12 +21,12 @@
 //! a WASM VM for re-execution of a parachain candidate.
 
 use std::any::{TypeId, Any};
-use crate::primitives::{ValidationParams, ValidationResult, UpwardMessage};
+use crate::primitives::{ValidationParams, ValidationResult};
 use codec::{Decode, Encode};
 use sp_core::storage::ChildInfo;
 use sp_core::traits::CallInWasm;
-use sp_wasm_interface::HostFunctions as _;
 use sp_externalities::Extensions;
+use sp_wasm_interface::HostFunctions as _;
 
 #[cfg(not(any(target_os = "android", target_os = "unknown")))]
 pub use validation_host::{run_worker, ValidationPool, EXECUTION_TIMEOUT_SEC};
@@ -37,18 +37,6 @@ mod validation_host;
 const MAX_RUNTIME_MEM: usize = 1024 * 1024 * 1024; // 1 GiB
 const MAX_CODE_MEM: usize = 16 * 1024 * 1024; // 16 MiB
 
-sp_externalities::decl_extension! {
-	/// The extension that is registered at the `Externalities` when validating a parachain state
-	/// transition.
-	pub(crate) struct ParachainExt(Box<dyn Externalities>);
-}
-
-impl ParachainExt {
-	pub fn new<T: Externalities + 'static>(ext: T) -> Self {
-		Self(Box::new(ext))
-	}
-}
-
 /// A stub validation-pool defined when compiling for Android or WASM.
 #[cfg(any(target_os = "android", target_os = "unknown"))]
 #[derive(Clone)]
@@ -124,32 +112,25 @@ impl std::error::Error for Error {
 	}
 }
 
-/// Externalities for parachain validation.
-pub trait Externalities: Send {
-	/// Called when a message is to be posted to the parachain's relay chain.
-	fn post_upward_message(&mut self, message: UpwardMessage) -> Result<(), String>;
-}
-
 /// Validate a candidate under the given validation code.
 ///
 /// This will fail if the validation code is not a proper parachain validation module.
-pub fn validate_candidate<E: Externalities + 'static>(
+pub fn validate_candidate(
 	validation_code: &[u8],
 	params: ValidationParams,
-	ext: E,
 	options: ExecutionMode<'_>,
 ) -> Result<ValidationResult, Error> {
 	match options {
 		ExecutionMode::Local => {
-			validate_candidate_internal(validation_code, &params.encode(), ext)
+			validate_candidate_internal(validation_code, &params.encode())
 		},
 		#[cfg(not(any(target_os = "android", target_os = "unknown")))]
 		ExecutionMode::Remote(pool) => {
-			pool.validate_candidate(validation_code, params, ext, false)
+			pool.validate_candidate(validation_code, params, false)
 		},
 		#[cfg(not(any(target_os = "android", target_os = "unknown")))]
 		ExecutionMode::RemoteTest(pool) => {
-			pool.validate_candidate(validation_code, params, ext, true)
+			pool.validate_candidate(validation_code, params, true)
 		},
 		#[cfg(any(target_os = "android", target_os = "unknown"))]
 		ExecutionMode::Remote(pool) =>
@@ -165,21 +146,16 @@ pub fn validate_candidate<E: Externalities + 'static>(
 }
 
 /// The host functions provided by the wasm executor to the parachain wasm blob.
-type HostFunctions = (
-	sp_io::SubstrateHostFunctions,
-	crate::wasm_api::parachain::HostFunctions,
-);
+type HostFunctions = sp_io::SubstrateHostFunctions;
 
 /// Validate a candidate under the given validation code.
 ///
 /// This will fail if the validation code is not a proper parachain validation module.
-pub fn validate_candidate_internal<E: Externalities + 'static>(
+pub fn validate_candidate_internal(
 	validation_code: &[u8],
 	encoded_call_data: &[u8],
-	externalities: E,
 ) -> Result<ValidationResult, Error> {
 	let mut extensions = Extensions::new();
-	extensions.register(ParachainExt::new(externalities));
 	extensions.register(sp_core::traits::TaskExecutorExt(sp_core::tasks::executor()));
 
 	let mut ext = ValidationExternalities(extensions);

polkadot/parachain/src/wasm_executor/validation_host.rs

@@ -16,18 +16,15 @@
 
 #![cfg(not(any(target_os = "android", target_os = "unknown")))]
 
-use std::{process, env, sync::Arc, sync::atomic, mem};
-use codec::{Decode, Encode, EncodeAppend};
-use crate::primitives::{ValidationParams, ValidationResult, UpwardMessage};
-use super::{validate_candidate_internal, Error, Externalities};
+use std::{process, env, sync::Arc, sync::atomic};
+use codec::{Decode, Encode};
+use crate::primitives::{ValidationParams, ValidationResult};
+use super::{validate_candidate_internal, Error};
 use super::{MAX_CODE_MEM, MAX_RUNTIME_MEM};
 use shared_memory::{SharedMem, SharedMemConf, EventState, WriteLockable, EventWait, EventSet};
 use parking_lot::Mutex;
 use log::{debug, trace};
 
-// Message data limit
-const MAX_MESSAGE_MEM: usize = 16 * 1024 * 1024; // 16 MiB
-
 const WORKER_ARGS_TEST: &[&'static str] = &["--nocapture", "validation_worker"];
 /// CLI Argument to start in validation worker mode.
 const WORKER_ARG: &'static str = "validation-worker";
@@ -40,27 +37,6 @@ pub const EXECUTION_TIMEOUT_SEC: u64 =  30;
 #[cfg(not(debug_assertions))]
 pub const EXECUTION_TIMEOUT_SEC: u64 =  5;
 
-#[derive(Default)]
-struct WorkerExternalitiesInner {
-	up_data: Vec<u8>,
-}
-
-#[derive(Default, Clone)]
-struct WorkerExternalities {
-	inner: Arc<Mutex<WorkerExternalitiesInner>>,
-}
-
-impl Externalities for WorkerExternalities {
-	fn post_upward_message(&mut self, message: UpwardMessage) -> Result<(), String> {
-		let mut inner = self.inner.lock();
-		inner.up_data = <Vec::<UpwardMessage> as EncodeAppend>::append_or_new(
-			mem::replace(&mut inner.up_data, Vec::new()),
-			std::iter::once(message),
-		).map_err(|e| e.what())?;
-		Ok(())
-	}
-}
-
 enum Event {
 	CandidateReady = 0,
 	ResultReady = 1,
@@ -87,21 +63,20 @@ impl ValidationPool {
 	/// free validation host.
 	///
 	/// This will fail if the validation code is not a proper parachain validation module.
-	pub fn validate_candidate<E: Externalities>(
+	pub fn validate_candidate(
 		&self,
 		validation_code: &[u8],
 		params: ValidationParams,
-		externalities: E,
 		test_mode: bool,
 	) -> Result<ValidationResult, Error> {
 		for host in self.hosts.iter() {
 			if let Some(mut host) = host.try_lock() {
-				return host.validate_candidate(validation_code, params, externalities, test_mode);
+				return host.validate_candidate(validation_code, params, test_mode);
 			}
 		}
 
 		// all workers are busy, just wait for the first one
-		self.hosts[0].lock().validate_candidate(validation_code, params, externalities, test_mode)
+		self.hosts[0].lock().validate_candidate(validation_code, params, test_mode)
 	}
 }
 
@@ -116,8 +91,6 @@ pub fn run_worker(mem_id: &str) -> Result<(), String> {
 		}
 	};
 
-	let worker_ext = WorkerExternalities::default();
-
 	let exit = Arc::new(atomic::AtomicBool::new(false));
 	// spawn parent monitor thread
 	let watch_exit = exit.clone();
@@ -166,21 +139,11 @@ pub fn run_worker(mem_id: &str) -> Result<(), String> {
 				let (call_data, _) = rest.split_at_mut(MAX_RUNTIME_MEM);
 				let (call_data, _) = call_data.split_at_mut(header.params_size as usize);
 
-				let result = validate_candidate_internal(code, call_data, worker_ext.clone());
+				let result = validate_candidate_internal(code, call_data);
 				debug!("{} Candidate validated: {:?}", process::id(), result);
 
 				match result {
-					Ok(r) => {
-						let inner = worker_ext.inner.lock();
-						let up_data = &inner.up_data;
-						let up_len = up_data.len();
-
-						if up_len > MAX_MESSAGE_MEM {
-							ValidationResultHeader::Error("Message data is too large".into())
-						} else {
-							ValidationResultHeader::Ok(r)
-						}
-					},
+					Ok(r) => ValidationResultHeader::Ok(r),
 					Err(e) => ValidationResultHeader::Error(e.to_string()),
 				}
 			};
@@ -226,7 +189,7 @@ impl Drop for ValidationHost {
 
 impl ValidationHost {
 	fn create_memory() -> Result<SharedMem, Error> {
-		let mem_size = MAX_RUNTIME_MEM + MAX_CODE_MEM + MAX_MESSAGE_MEM + 1024;
+		let mem_size = MAX_RUNTIME_MEM + MAX_CODE_MEM + 1024;
 		let mem_config = SharedMemConf::default()
 			.set_size(mem_size)
 			.add_lock(shared_memory::LockType::Mutex, 0, mem_size)?
@@ -268,11 +231,10 @@ impl ValidationHost {
 	/// Validate a candidate under the given validation code.
 	///
 	/// This will fail if the validation code is not a proper parachain validation module.
-	pub fn validate_candidate<E: Externalities>(
+	pub fn validate_candidate(
 		&mut self,
 		validation_code: &[u8],
 		params: ValidationParams,
-		mut externalities: E,
 		test_mode: bool,
 	) -> Result<ValidationResult, Error> {
 		if validation_code.len() > MAX_CODE_MEM {
@@ -322,24 +284,11 @@ impl ValidationHost {
 		{
 			debug!("{} Reading results", self.id);
 			let data: &[u8] = &**memory.wlock_as_slice(0)?;
-			let (header_buf, rest) = data.split_at(1024);
-			let (_, rest) = rest.split_at(MAX_CODE_MEM);
-			let (_, message_data) = rest.split_at(MAX_RUNTIME_MEM);
+			let (header_buf, _) = data.split_at(1024);
 			let mut header_buf: &[u8] = header_buf;
-			let mut message_data: &[u8] = message_data;
 			let header = ValidationResultHeader::decode(&mut header_buf).unwrap();
 			match header {
-				ValidationResultHeader::Ok(result) => {
-					let upwards = Vec::<UpwardMessage>::decode(&mut message_data)
-						.map_err(|e|
-							Error::External(
-								format!("Could not decode upward messages: {}", e.what())
-							)
-						)?;
-					upwards.into_iter().try_for_each(|msg| externalities.post_upward_message(msg))?;
-
-					Ok(result)
-				}
+				ValidationResultHeader::Ok(result) => Ok(result),
 				ValidationResultHeader::Error(message) => {
 					debug!("{} Validation error: {}", self.id, message);
 					Err(Error::External(message).into())