Upgradeable validation functions (#918)

* upgrade primitives to allow changing validation function

* set up storage schema for old parachains code

* fix compilation errors

* fix test compilation

* add some tests for past code meta

* most of the runtime logic for code upgrades

* implement old-code pruning

* add a couple tests

* clean up remaining TODOs

* add a whole bunch of tests for runtime functionality

* remove unused function

* fix runtime compilation

* extract some primitives to parachain crate

* add validation-code upgrades to validation params and result

* extend validation params with code upgrade fields

* provide maximums to validation params

* port test-parachains

* add a code-upgrader test-parachain and tests

* fix collator tests

* move test-parachains to own folder to work around compilation errors

* fix test compilation

* update the Cargo.lock

* fix parachains tests

* remove dbg! invocation

* use new pool in code-upgrader

* bump lockfile

* link TODO to issue

polkadot/parachain/src/lib.rs

@@ -45,182 +45,9 @@
 
 #[cfg(feature = "std")]
 pub mod wasm_executor;
+pub mod primitives;
 
 mod wasm_api;
 
-use sp_std::vec::Vec;
-
-use codec::{Encode, Decode, CompactAs};
-use sp_core::{RuntimeDebug, TypeId};
-
 #[cfg(all(not(feature = "std"), feature = "wasm-api"))]
 pub use wasm_api::*;
-
-/// Validation parameters for evaluating the parachain validity function.
-// TODO: balance downloads (https://github.com/paritytech/polkadot/issues/220)
-#[derive(PartialEq, Eq, Decode)]
-#[cfg_attr(feature = "std", derive(Debug, Encode))]
-pub struct ValidationParams {
-	/// The collation body.
-	pub block_data: Vec<u8>,
-	/// Previous head-data.
-	pub parent_head: Vec<u8>,
-}
-
-/// The result of parachain validation.
-// TODO: egress and balance uploads (https://github.com/paritytech/polkadot/issues/220)
-#[derive(PartialEq, Eq, Encode)]
-#[cfg_attr(feature = "std", derive(Debug, Decode))]
-pub struct ValidationResult {
-	/// New head data that should be included in the relay chain state.
-	pub head_data: Vec<u8>,
-}
-
-/// Unique identifier of a parachain.
-#[derive(
-	Clone, CompactAs, Copy, Decode, Default, Encode, Eq,
-	Hash, Ord, PartialEq, PartialOrd, RuntimeDebug,
-)]
-#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize, derive_more::Display))]
-pub struct Id(u32);
-
-impl TypeId for Id {
-	const TYPE_ID: [u8; 4] = *b"para";
-}
-
-/// Type for determining the active set of parachains.
-pub trait ActiveThreads {
-	/// Return the current ordered set of `Id`s of active parathreads.
-	fn active_threads() -> Vec<Id>;
-}
-
-impl From<Id> for u32 {
-	fn from(x: Id) -> Self { x.0 }
-}
-
-impl From<u32> for Id {
-	fn from(x: u32) -> Self { Id(x) }
-}
-
-const USER_INDEX_START: u32 = 1000;
-
-/// The ID of the first user (non-system) parachain.
-pub const LOWEST_USER_ID: Id = Id(USER_INDEX_START);
-
-impl Id {
-	/// Create an `Id`.
-	pub const fn new(id: u32) -> Self {
-		Self(id)
-	}
-
-	/// Returns `true` if this parachain runs with system-level privileges.
-	pub fn is_system(&self) -> bool { self.0 < USER_INDEX_START }
-}
-
-impl sp_std::ops::Add<u32> for Id {
-	type Output = Self;
-
-	fn add(self, other: u32) -> Self {
-		Self(self.0 + other)
-	}
-}
-
-// TODO: Remove all of this, move sp-runtime::AccountIdConversion to own crate and and use that.
-// #360
-struct TrailingZeroInput<'a>(&'a [u8]);
-impl<'a> codec::Input for TrailingZeroInput<'a> {
-	fn remaining_len(&mut self) -> Result<Option<usize>, codec::Error> {
-		Ok(None)
-	}
-
-	fn read(&mut self, into: &mut [u8]) -> Result<(), codec::Error> {
-		let len = into.len().min(self.0.len());
-		into[..len].copy_from_slice(&self.0[..len]);
-		for i in &mut into[len..] {
-			*i = 0;
-		}
-		self.0 = &self.0[len..];
-		Ok(())
-	}
-}
-
-/// This type can be converted into and possibly from an AccountId (which itself is generic).
-pub trait AccountIdConversion<AccountId>: Sized {
-	/// Convert into an account ID. This is infallible.
-	fn into_account(&self) -> AccountId;
-
- 	/// Try to convert an account ID into this type. Might not succeed.
-	fn try_from_account(a: &AccountId) -> Option<Self>;
-}
-
-/// Format is b"para" ++ encode(parachain ID) ++ 00.... where 00... is indefinite trailing
-/// zeroes to fill AccountId.
-impl<T: Encode + Decode + Default> AccountIdConversion<T> for Id {
-	fn into_account(&self) -> T {
-		(b"para", self).using_encoded(|b|
-			T::decode(&mut TrailingZeroInput(b))
-		).unwrap_or_default()
-	}
-
- 	fn try_from_account(x: &T) -> Option<Self> {
-		x.using_encoded(|d| {
-			if &d[0..4] != b"para" { return None }
-			let mut cursor = &d[4..];
-			let result = Decode::decode(&mut cursor).ok()?;
-			if cursor.iter().all(|x| *x == 0) {
-				Some(result)
-			} else {
-				None
-			}
-		})
-	}
-}
-
-/// Which origin a parachain's message to the relay chain should be dispatched from.
-#[derive(Clone, PartialEq, Eq, Encode, Decode)]
-#[cfg_attr(feature = "std", derive(Debug))]
-#[repr(u8)]
-pub enum ParachainDispatchOrigin {
-	/// As a simple `Origin::Signed`, using `ParaId::account_id` as its value. This is good when
-	/// interacting with standard modules such as `balances`.
-	Signed,
-	/// As the special `Origin::Parachain(ParaId)`. This is good when interacting with parachain-
-	/// aware modules which need to succinctly verify that the origin is a parachain.
-	Parachain,
-	/// As the simple, superuser `Origin::Root`. This can only be done on specially permissioned
-	/// parachains.
-	Root,
-}
-
-impl sp_std::convert::TryFrom<u8> for ParachainDispatchOrigin {
-	type Error = ();
-	fn try_from(x: u8) -> core::result::Result<ParachainDispatchOrigin, ()> {
-		const SIGNED: u8 = ParachainDispatchOrigin::Signed as u8;
-		const PARACHAIN: u8 = ParachainDispatchOrigin::Parachain as u8;
-		Ok(match x {
-			SIGNED => ParachainDispatchOrigin::Signed,
-			PARACHAIN => ParachainDispatchOrigin::Parachain,
-			_ => return Err(()),
-		})
-	}
-}
-
-/// A message from a parachain to its Relay Chain.
-#[derive(Clone, PartialEq, Eq, Encode, Decode, sp_runtime_interface::pass_by::PassByCodec)]
-#[cfg_attr(feature = "std", derive(Debug))]
-pub struct UpwardMessage {
-	/// The origin for the message to be sent from.
-	pub origin: ParachainDispatchOrigin,
-	/// The message data.
-	pub data: Vec<u8>,
-}
-
-/// An incoming message.
-#[derive(PartialEq, Eq, Decode)]
-#[cfg_attr(feature = "std", derive(Debug, Encode))]
-pub struct IncomingMessage {
-	/// The source parachain.
-	pub source: Id,
-	/// The data of the message.
-	pub data: Vec<u8>,
-}

polkadot/parachain/src/primitives.rs

@@ -0,0 +1,221 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Primitive types which are strictly necessary from a parachain-execution point
+//! of view.
+
+use sp_std::vec::Vec;
+
+use codec::{Encode, Decode, CompactAs};
+use sp_core::{RuntimeDebug, TypeId};
+
+#[cfg(feature = "std")]
+use serde::{Serialize, Deserialize};
+
+#[cfg(feature = "std")]
+use sp_core::bytes;
+
+/// The block number of the relay chain.
+/// 32-bits will allow for 136 years of blocks assuming 1 block per second.
+pub type RelayChainBlockNumber = u32;
+
+/// Parachain head data included in the chain.
+#[derive(PartialEq, Eq, Clone, PartialOrd, Ord, Encode, Decode)]
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug, Default))]
+pub struct HeadData(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>);
+
+/// Parachain validation code.
+#[derive(Default, PartialEq, Eq, Clone, Encode, Decode)]
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))]
+pub struct ValidationCode(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>);
+
+/// Parachain block data.
+///
+/// Contains everything required to validate para-block, may contain block and witness data.
+#[derive(PartialEq, Eq, Clone, Encode, Decode)]
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))]
+pub struct BlockData(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>);
+
+/// Unique identifier of a parachain.
+#[derive(
+	Clone, CompactAs, Copy, Decode, Default, Encode, Eq,
+	Hash, Ord, PartialEq, PartialOrd, RuntimeDebug,
+)]
+#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize, derive_more::Display))]
+pub struct Id(u32);
+
+impl TypeId for Id {
+	const TYPE_ID: [u8; 4] = *b"para";
+}
+
+impl From<Id> for u32 {
+	fn from(x: Id) -> Self { x.0 }
+}
+
+impl From<u32> for Id {
+	fn from(x: u32) -> Self { Id(x) }
+}
+
+const USER_INDEX_START: u32 = 1000;
+
+/// The ID of the first user (non-system) parachain.
+pub const LOWEST_USER_ID: Id = Id(USER_INDEX_START);
+
+impl Id {
+	/// Create an `Id`.
+	pub const fn new(id: u32) -> Self {
+		Self(id)
+	}
+
+	/// Returns `true` if this parachain runs with system-level privileges.
+	pub fn is_system(&self) -> bool { self.0 < USER_INDEX_START }
+}
+
+impl sp_std::ops::Add<u32> for Id {
+	type Output = Self;
+
+	fn add(self, other: u32) -> Self {
+		Self(self.0 + other)
+	}
+}
+
+/// This type can be converted into and possibly from an AccountId (which itself is generic).
+pub trait AccountIdConversion<AccountId>: Sized {
+	/// Convert into an account ID. This is infallible.
+	fn into_account(&self) -> AccountId;
+
+ 	/// Try to convert an account ID into this type. Might not succeed.
+	fn try_from_account(a: &AccountId) -> Option<Self>;
+}
+
+// TODO: Remove all of this, move sp-runtime::AccountIdConversion to own crate and and use that.
+// #360
+struct TrailingZeroInput<'a>(&'a [u8]);
+impl<'a> codec::Input for TrailingZeroInput<'a> {
+	fn remaining_len(&mut self) -> Result<Option<usize>, codec::Error> {
+		Ok(None)
+	}
+
+	fn read(&mut self, into: &mut [u8]) -> Result<(), codec::Error> {
+		let len = into.len().min(self.0.len());
+		into[..len].copy_from_slice(&self.0[..len]);
+		for i in &mut into[len..] {
+			*i = 0;
+		}
+		self.0 = &self.0[len..];
+		Ok(())
+	}
+}
+
+/// Format is b"para" ++ encode(parachain ID) ++ 00.... where 00... is indefinite trailing
+/// zeroes to fill AccountId.
+impl<T: Encode + Decode + Default> AccountIdConversion<T> for Id {
+	fn into_account(&self) -> T {
+		(b"para", self).using_encoded(|b|
+			T::decode(&mut TrailingZeroInput(b))
+		).unwrap_or_default()
+	}
+
+ 	fn try_from_account(x: &T) -> Option<Self> {
+		x.using_encoded(|d| {
+			if &d[0..4] != b"para" { return None }
+			let mut cursor = &d[4..];
+			let result = Decode::decode(&mut cursor).ok()?;
+			if cursor.iter().all(|x| *x == 0) {
+				Some(result)
+			} else {
+				None
+			}
+		})
+	}
+}
+
+/// Which origin a parachain's message to the relay chain should be dispatched from.
+#[derive(Clone, PartialEq, Eq, Encode, Decode)]
+#[cfg_attr(feature = "std", derive(Debug))]
+#[repr(u8)]
+pub enum ParachainDispatchOrigin {
+	/// As a simple `Origin::Signed`, using `ParaId::account_id` as its value. This is good when
+	/// interacting with standard modules such as `balances`.
+	Signed,
+	/// As the special `Origin::Parachain(ParaId)`. This is good when interacting with parachain-
+	/// aware modules which need to succinctly verify that the origin is a parachain.
+	Parachain,
+	/// As the simple, superuser `Origin::Root`. This can only be done on specially permissioned
+	/// parachains.
+	Root,
+}
+
+impl sp_std::convert::TryFrom<u8> for ParachainDispatchOrigin {
+	type Error = ();
+	fn try_from(x: u8) -> core::result::Result<ParachainDispatchOrigin, ()> {
+		const SIGNED: u8 = ParachainDispatchOrigin::Signed as u8;
+		const PARACHAIN: u8 = ParachainDispatchOrigin::Parachain as u8;
+		Ok(match x {
+			SIGNED => ParachainDispatchOrigin::Signed,
+			PARACHAIN => ParachainDispatchOrigin::Parachain,
+			_ => return Err(()),
+		})
+	}
+}
+
+/// A message from a parachain to its Relay Chain.
+#[derive(Clone, PartialEq, Eq, Encode, Decode)]
+#[cfg_attr(
+	any(feature = "std", feature = "wasm-api"),
+	derive(sp_runtime_interface::pass_by::PassByCodec,
+))]
+#[cfg_attr(feature = "std", derive(Debug))]
+pub struct UpwardMessage {
+	/// The origin for the message to be sent from.
+	pub origin: ParachainDispatchOrigin,
+	/// The message data.
+	pub data: Vec<u8>,
+}
+
+/// Validation parameters for evaluating the parachain validity function.
+// TODO: balance downloads (https://github.com/paritytech/polkadot/issues/220)
+#[derive(PartialEq, Eq, Decode)]
+#[cfg_attr(feature = "std", derive(Debug, Encode))]
+pub struct ValidationParams {
+	/// The collation body.
+	pub block_data: BlockData,
+	/// Previous head-data.
+	pub parent_head: HeadData,
+	/// The maximum code size permitted, in bytes.
+	pub max_code_size: u32,
+	/// The maximum head-data size permitted, in bytes.
+	pub max_head_data_size: u32,
+	/// The current relay-chain block number.
+	pub relay_chain_height: RelayChainBlockNumber,
+	/// Whether a code upgrade is allowed or not, and at which height the upgrade
+	/// would be applied after, if so. The parachain logic should apply any upgrade
+	/// issued in this block after the first block
+	/// with `relay_chain_height` at least this value, if `Some`. if `None`, issue
+	/// no upgrade.
+	pub code_upgrade_allowed: Option<RelayChainBlockNumber>,
+}
+
+/// The result of parachain validation.
+// TODO: egress and balance uploads (https://github.com/paritytech/polkadot/issues/220)
+#[derive(PartialEq, Eq, Encode)]
+#[cfg_attr(feature = "std", derive(Debug, Decode))]
+pub struct ValidationResult {
+	/// New head data that should be included in the relay chain state.
+	pub head_data: HeadData,
+	/// An update to the validation code that should be scheduled in the relay chain.
+	pub new_validation_code: Option<ValidationCode>,
+}

polkadot/parachain/src/wasm_api.rs

@@ -16,7 +16,9 @@
 
 //! Utilities for writing parachain WASM.
 
-use crate::UpwardMessage;
+#[cfg(any(feature = "std", all(not(feature = "std"), feature = "wasm-api")))]
+use crate::primitives::UpwardMessage;
+#[cfg(any(feature = "std", all(not(feature = "std"), feature = "wasm-api")))]
 use sp_runtime_interface::runtime_interface;
 #[cfg(feature = "std")]
 use sp_externalities::ExternalitiesExt;
@@ -42,7 +44,9 @@ pub trait Parachain {
 /// Offset and length must have been provided by the validation
 /// function's entry point.
 #[cfg(not(feature = "std"))]
-pub unsafe fn load_params(params: *const u8, len: usize) -> crate::ValidationParams {
+pub unsafe fn load_params(params: *const u8, len: usize)
+	-> crate::primitives::ValidationParams
+{
 	let mut slice = sp_std::slice::from_raw_parts(params, len);
 
 	codec::Decode::decode(&mut slice).expect("Invalid input data")
@@ -53,6 +57,6 @@ pub unsafe fn load_params(params: *const u8, len: usize) -> crate::ValidationPar
 /// As described in the crate docs, this is a pointer to the appended length
 /// of the vector.
 #[cfg(not(feature = "std"))]
-pub fn write_result(result: &crate::ValidationResult) -> u64 {
+pub fn write_result(result: &crate::primitives::ValidationResult) -> u64 {
 	sp_core::to_substrate_wasm_fn_return_value(&result)
 }

polkadot/parachain/src/wasm_executor/mod.rs

@@ -21,7 +21,7 @@
 //! a WASM VM for re-execution of a parachain candidate.
 
 use std::any::{TypeId, Any};
-use crate::{ValidationParams, ValidationResult, UpwardMessage};
+use crate::primitives::{ValidationParams, ValidationResult, UpwardMessage};
 use codec::{Decode, Encode};
 use sp_core::storage::{ChildStorageKey, ChildInfo};
 use sp_core::traits::CallInWasm;

polkadot/parachain/src/wasm_executor/validation_host.rs

@@ -18,7 +18,7 @@
 
 use std::{process, env, sync::Arc, sync::atomic, mem};
 use codec::{Decode, Encode, EncodeAppend};
-use crate::{ValidationParams, ValidationResult, UpwardMessage};
+use crate::primitives::{ValidationParams, ValidationResult, UpwardMessage};
 use super::{validate_candidate_internal, Error, Externalities};
 use super::{MAX_CODE_MEM, MAX_RUNTIME_MEM};
 use shared_memory::{SharedMem, SharedMemConf, EventState, WriteLockable, EventWait, EventSet};