Upgradeable validation functions (#918)

* upgrade primitives to allow changing validation function

* set up storage schema for old parachains code

* fix compilation errors

* fix test compilation

* add some tests for past code meta

* most of the runtime logic for code upgrades

* implement old-code pruning

* add a couple tests

* clean up remaining TODOs

* add a whole bunch of tests for runtime functionality

* remove unused function

* fix runtime compilation

* extract some primitives to parachain crate

* add validation-code upgrades to validation params and result

* extend validation params with code upgrade fields

* provide maximums to validation params

* port test-parachains

* add a code-upgrader test-parachain and tests

* fix collator tests

* move test-parachains to own folder to work around compilation errors

* fix test compilation

* update the Cargo.lock

* fix parachains tests

* remove dbg! invocation

* use new pool in code-upgrader

* bump lockfile

* link TODO to issue

polkadot/parachain/test-parachains/.gitignore

@@ -0,0 +1,2 @@
+target/
+Cargo.lock

polkadot/parachain/test-parachains/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "test-parachains"
+version = "0.7.22"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "Integration tests using the test-parachains"
+edition = "2018"
+
+[dependencies]
+tiny-keccak = "1.5.0"
+codec = { package = "parity-scale-codec", version = "1.1.0", default-features = false, features = ["derive"] }
+
+parachain = { package = "polkadot-parachain", path = ".." }
+adder = { package = "test-parachain-adder", path = "adder" }
+halt = { package = "test-parachain-halt", path = "halt" }
+code-upgrader = { package = "test-parachain-code-upgrader", path = "code-upgrader" }
+
+[features]
+default = [ "std" ]
+std = [
+	"adder/std",
+	"halt/std",
+	"code-upgrader/std",
+]

polkadot/parachain/test-parachains/README.md

@@ -0,0 +1,3 @@
+# Test Parachains
+
+Each parachain consists of three parts: a `#![no_std]` library with the main execution logic, a WASM crate which wraps this logic, and a collator node.

polkadot/parachain/test-parachains/adder/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "test-parachain-adder"
+version = "0.7.29-pre1"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "Test parachain which adds to a number as its state transition"
+edition = "2018"
+build = "build.rs"
+
+[dependencies]
+parachain = { package = "polkadot-parachain", path = "../../", default-features = false, features = [ "wasm-api" ] }
+codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] }
+tiny-keccak = "1.5.0"
+dlmalloc = { version = "0.1.3", features = [ "global" ] }
+
+# We need to make sure the global allocator is disabled until we have support of full substrate externalities
+runtime-io = { package = "sp-io", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, features = [ "disable_allocator" ] }
+
+[build-dependencies]
+wasm-builder-runner = { package = "substrate-wasm-builder-runner", version = "1.0.5" }
+
+[features]
+default = [ "std" ]
+std = ["parachain/std"]

polkadot/parachain/test-parachains/adder/build.rs

@@ -0,0 +1,25 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use wasm_builder_runner::WasmBuilder;
+
+fn main() {
+	WasmBuilder::new()
+		.with_current_project()
+		.with_wasm_builder_from_git("https://github.com/paritytech/substrate.git", "8c672e107789ed10973d937ba8cac245404377e2")
+		.export_heap_base()
+		.build()
+}

polkadot/parachain/test-parachains/adder/collator/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "test-parachain-adder-collator"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+adder = { package = "test-parachain-adder", path = ".." }
+parachain = { package = "polkadot-parachain", path = "../../.." }
+collator = { package = "polkadot-collator", path = "../../../../collator" }
+primitives = { package = "polkadot-primitives", path = "../../../../primitives" }
+sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
+client = { package = "sc-client", git = "https://github.com/paritytech/substrate", branch = "master" }
+client-api = { package = "sc-client-api", git = "https://github.com/paritytech/substrate", branch = "master" }
+parking_lot = "0.10.0"
+codec = { package = "parity-scale-codec", version = "1.2.0" }
+futures = "0.3.4"

polkadot/parachain/test-parachains/adder/collator/src/main.rs

@@ -0,0 +1,149 @@
+// Copyright 2018-2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Collator for polkadot
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use adder::{HeadData as AdderHead, BlockData as AdderBody};
+use sp_core::Pair;
+use codec::{Encode, Decode};
+use primitives::{
+	Hash,
+	parachain::{HeadData, BlockData, Id as ParaId, LocalValidationData, GlobalValidationSchedule},
+};
+use collator::{
+	InvalidHead, ParachainContext, Network, BuildParachainContext, load_spec, Configuration,
+};
+use parking_lot::Mutex;
+use futures::future::{Ready, ok, err};
+
+const GENESIS: AdderHead = AdderHead {
+	number: 0,
+	parent_hash: [0; 32],
+	post_state: [
+		1, 27, 77, 3, 221, 140, 1, 241, 4, 145, 67, 207, 156, 76, 129, 126, 75,
+		22, 127, 29, 27, 131, 229, 198, 240, 241, 13, 137, 186, 30, 123, 206
+	],
+};
+
+const GENESIS_BODY: AdderBody = AdderBody {
+	state: 0,
+	add: 0,
+};
+
+#[derive(Clone)]
+struct AdderContext {
+	db: Arc<Mutex<HashMap<AdderHead, AdderBody>>>,
+	/// We store it here to make sure that our interfaces require the correct bounds.
+	_network: Option<Arc<dyn Network>>,
+}
+
+/// The parachain context.
+impl ParachainContext for AdderContext {
+	type ProduceCandidate = Ready<Result<(BlockData, HeadData), InvalidHead>>;
+
+	fn produce_candidate(
+		&mut self,
+		_relay_parent: Hash,
+		_global_validation: GlobalValidationSchedule,
+		local_validation: LocalValidationData,
+	) -> Self::ProduceCandidate
+	{
+		let adder_head = match AdderHead::decode(&mut &local_validation.parent_head.0[..]) {
+			Ok(adder_head) => adder_head,
+			Err(_) => return err(InvalidHead)
+		};
+
+		let mut db = self.db.lock();
+
+		let last_body = if adder_head == GENESIS {
+			GENESIS_BODY
+		} else {
+			db.get(&adder_head)
+				.expect("All past bodies stored since this is the only collator")
+				.clone()
+		};
+
+		let next_body = AdderBody {
+			state: last_body.state.overflowing_add(last_body.add).0,
+			add: adder_head.number % 100,
+		};
+
+		let next_head = adder::execute(adder_head.hash(), adder_head, &next_body)
+			.expect("good execution params; qed");
+
+		let encoded_head = HeadData(next_head.encode());
+		let encoded_body = BlockData(next_body.encode());
+
+		println!("Created collation for #{}, post-state={}",
+			next_head.number, next_body.state.overflowing_add(next_body.add).0);
+
+		db.insert(next_head.clone(), next_body);
+		ok((encoded_body, encoded_head))
+	}
+}
+
+impl BuildParachainContext for AdderContext {
+	type ParachainContext = Self;
+
+	fn build<B, E, R, SP, Extrinsic>(
+		self,
+		_: Arc<collator::PolkadotClient<B, E, R>>,
+		_: SP,
+		network: impl Network + Clone + 'static,
+	) -> Result<Self::ParachainContext, ()> {
+		Ok(Self { _network: Some(Arc::new(network)), ..self })
+	}
+}
+
+fn main() {
+	let key = Arc::new(Pair::from_seed(&[1; 32]));
+	let id: ParaId = 100.into();
+
+	println!("Starting adder collator with genesis: ");
+
+	{
+		let encoded = GENESIS.encode();
+		println!("Dec: {:?}", encoded);
+		print!("Hex: 0x");
+		for byte in encoded {
+			print!("{:02x}", byte);
+		}
+
+		println!();
+	}
+
+	let context = AdderContext {
+		db: Arc::new(Mutex::new(HashMap::new())),
+		_network: None,
+	};
+
+	let mut config = Configuration::default();
+	config.chain_spec = Some(load_spec("dev", false).unwrap());
+
+	let res = collator::run_collator(
+		context,
+		id,
+		key,
+		config,
+	);
+
+	if let Err(e) = res {
+		println!("{}", e);
+	}
+}

polkadot/parachain/test-parachains/adder/src/lib.rs

@@ -0,0 +1,90 @@
+// Copyright 2017-2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Basic parachain that adds a number as part of its state.
+
+#![no_std]
+
+#![cfg_attr(not(feature = "std"), feature(core_intrinsics, lang_items, core_panic_info, alloc_error_handler))]
+
+use codec::{Encode, Decode};
+
+#[cfg(not(feature = "std"))]
+mod wasm_validation;
+
+#[cfg(not(feature = "std"))]
+#[global_allocator]
+static ALLOC: dlmalloc::GlobalDlmalloc = dlmalloc::GlobalDlmalloc;
+
+// Make the WASM binary available.
+#[cfg(feature = "std")]
+include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs"));
+
+/// Head data for this parachain.
+#[derive(Default, Clone, Hash, Eq, PartialEq, Encode, Decode)]
+pub struct HeadData {
+	/// Block number
+	pub number: u64,
+	/// parent block keccak256
+	pub parent_hash: [u8; 32],
+	/// hash of post-execution state.
+	pub post_state: [u8; 32],
+}
+
+impl HeadData {
+	pub fn hash(&self) -> [u8; 32] {
+		tiny_keccak::keccak256(&self.encode())
+	}
+}
+
+/// Block data for this parachain.
+#[derive(Default, Clone, Encode, Decode)]
+pub struct BlockData {
+	/// State to begin from.
+	pub state: u64,
+	/// Amount to add (overflowing)
+	pub add: u64,
+}
+
+pub fn hash_state(state: u64) -> [u8; 32] {
+	tiny_keccak::keccak256(state.encode().as_slice())
+}
+
+/// Start state mismatched with parent header's state hash.
+#[derive(Debug)]
+pub struct StateMismatch;
+
+/// Execute a block body on top of given parent head, producing new parent head
+/// if valid.
+pub fn execute(
+	parent_hash: [u8; 32],
+	parent_head: HeadData,
+	block_data: &BlockData,
+) -> Result<HeadData, StateMismatch> {
+	debug_assert_eq!(parent_hash, parent_head.hash());
+
+	if hash_state(block_data.state) != parent_head.post_state {
+		return Err(StateMismatch);
+	}
+
+	let new_state = block_data.state.overflowing_add(block_data.add).0;
+
+	Ok(HeadData {
+		number: parent_head.number + 1,
+		parent_hash,
+		post_state: hash_state(new_state),
+	})
+}

polkadot/parachain/test-parachains/adder/src/wasm_validation.rs

@@ -0,0 +1,60 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! WASM validation for adder parachain.
+
+use crate::{HeadData, BlockData};
+use core::{intrinsics, panic};
+use parachain::primitives::{ValidationResult, HeadData as GenericHeadData};
+use codec::{Encode, Decode};
+
+#[panic_handler]
+#[no_mangle]
+pub fn panic(_info: &panic::PanicInfo) -> ! {
+	unsafe {
+		intrinsics::abort()
+	}
+}
+
+#[alloc_error_handler]
+#[no_mangle]
+pub fn oom(_: core::alloc::Layout) -> ! {
+	unsafe {
+		intrinsics::abort();
+	}
+}
+
+#[no_mangle]
+pub extern fn validate_block(params: *const u8, len: usize) -> u64 {
+	let params = unsafe { parachain::load_params(params, len) };
+	let parent_head = HeadData::decode(&mut &params.parent_head.0[..])
+		.expect("invalid parent head format.");
+
+	let block_data = BlockData::decode(&mut &params.block_data.0[..])
+		.expect("invalid block data format.");
+
+	let parent_hash = tiny_keccak::keccak256(&params.parent_head.0[..]);
+
+	match crate::execute(parent_hash, parent_head, &block_data) {
+		Ok(new_head) => parachain::write_result(
+			&ValidationResult {
+				head_data: GenericHeadData(new_head.encode()),
+				new_validation_code: None,
+			}
+		),
+		Err(_) => panic!("execution failure"),
+	}
+}

polkadot/parachain/test-parachains/code-upgrader/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "test-parachain-code-upgrader"
+version = "0.7.22"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "Test parachain which can upgrade code"
+edition = "2018"
+build = "build.rs"
+
+[dependencies]
+parachain = { package = "polkadot-parachain", path = "../../", default-features = false, features = [ "wasm-api" ] }
+codec = { package = "parity-scale-codec", version = "1.1.0", default-features = false, features = ["derive"] }
+tiny-keccak = "1.5.0"
+dlmalloc = { version = "0.1.3", features = [ "global" ] }
+
+# We need to make sure the global allocator is disabled until we have support of full substrate externalities
+runtime-io = { package = "sp-io", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, features = [ "disable_allocator" ] }
+
+[build-dependencies]
+wasm-builder-runner = { package = "substrate-wasm-builder-runner", version = "1.0.5" }
+
+[features]
+default = [ "std" ]
+std = ["parachain/std"]

polkadot/parachain/test-parachains/code-upgrader/build.rs

@@ -0,0 +1,25 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use wasm_builder_runner::WasmBuilder;
+
+fn main() {
+	WasmBuilder::new()
+		.with_current_project()
+		.with_wasm_builder_from_git("https://github.com/paritytech/substrate.git", "8c672e107789ed10973d937ba8cac245404377e2")
+		.export_heap_base()
+		.build()
+}

polkadot/parachain/test-parachains/code-upgrader/src/lib.rs

@@ -0,0 +1,156 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Test parachain WASM which implements code ugprades.
+
+#![no_std]
+
+#![cfg_attr(not(feature = "std"), feature(core_intrinsics, lang_items, core_panic_info, alloc_error_handler))]
+
+use codec::{Encode, Decode};
+use parachain::primitives::{RelayChainBlockNumber, ValidationCode};
+
+#[cfg(not(feature = "std"))]
+mod wasm_validation;
+
+#[cfg(not(feature = "std"))]
+#[global_allocator]
+static ALLOC: dlmalloc::GlobalDlmalloc = dlmalloc::GlobalDlmalloc;
+
+// Make the WASM binary available.
+#[cfg(feature = "std")]
+include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs"));
+
+#[derive(Encode, Decode, Clone, Default)]
+pub struct State {
+	/// The current code that is "active" in this chain.
+	pub code: ValidationCode,
+	/// Code upgrade that is pending.
+	pub pending_code: Option<(ValidationCode, RelayChainBlockNumber)>,
+}
+
+/// Head data for this parachain.
+#[derive(Default, Clone, Hash, Eq, PartialEq, Encode, Decode)]
+pub struct HeadData {
+	/// Block number
+	pub number: u64,
+	/// parent block keccak256
+	pub parent_hash: [u8; 32],
+	/// hash of post-execution state.
+	pub post_state: [u8; 32],
+}
+
+impl HeadData {
+	pub fn hash(&self) -> [u8; 32] {
+		tiny_keccak::keccak256(&self.encode())
+	}
+}
+
+/// Block data for this parachain.
+#[derive(Default, Clone, Encode, Decode)]
+pub struct BlockData {
+	/// State to begin from.
+	pub state: State,
+	/// Code to upgrade to.
+	pub new_validation_code: Option<ValidationCode>,
+}
+
+pub fn hash_state(state: &State) -> [u8; 32] {
+	tiny_keccak::keccak256(state.encode().as_slice())
+}
+
+#[derive(Debug)]
+pub enum Error {
+	/// Start state mismatched with parent header's state hash.
+	StateMismatch,
+	/// New validation code too large.
+	NewCodeTooLarge,
+	/// Code upgrades not allowed at this time.
+	CodeUpgradeDisallowed,
+}
+
+pub struct ValidationResult {
+	/// The new head data.
+	pub head_data: HeadData,
+	/// The new validation code.
+	pub new_validation_code: Option<ValidationCode>,
+}
+
+pub struct RelayChainParams {
+	/// Whether a code upgrade is allowed and at what relay-chain block number
+	/// to process it after.
+	pub code_upgrade_allowed: Option<RelayChainBlockNumber>,
+	/// The maximum code size allowed for an upgrade.
+	pub max_code_size: u32,
+	/// The relay-chain block number.
+	pub relay_chain_block_number: RelayChainBlockNumber,
+}
+
+/// Execute a block body on top of given parent head, producing new parent head
+/// if valid.
+pub fn execute(
+	parent_hash: [u8; 32],
+	parent_head: HeadData,
+	block_data: BlockData,
+	relay_params: &RelayChainParams,
+) -> Result<ValidationResult, Error> {
+	debug_assert_eq!(parent_hash, parent_head.hash());
+
+	if hash_state(&block_data.state) != parent_head.post_state {
+		return Err(Error::StateMismatch);
+	}
+
+	let mut new_state = block_data.state;
+
+	if let Some((pending_code, after)) = new_state.pending_code.take() {
+		if after <= relay_params.relay_chain_block_number {
+			// code applied.
+			new_state.code = pending_code;
+		} else {
+			// reinstate.
+			new_state.pending_code = Some((pending_code, after));
+		}
+	}
+
+	let new_validation_code = if let Some(ref new_validation_code) = block_data.new_validation_code {
+		if new_validation_code.0.len() as u32 > relay_params.max_code_size {
+			return Err(Error::NewCodeTooLarge);
+		}
+
+		// replace the code if allowed and we don't have an upgrade pending.
+		match (new_state.pending_code.is_some(), relay_params.code_upgrade_allowed) {
+			(_, None) => return Err(Error::CodeUpgradeDisallowed),
+			(false, Some(after)) => {
+				new_state.pending_code = Some((new_validation_code.clone(), after));
+				Some(new_validation_code.clone())
+			}
+			(true, Some(_)) => None,
+		}
+	} else {
+		None
+	};
+
+	let head_data = HeadData {
+		number: parent_head.number + 1,
+		parent_hash,
+		post_state: hash_state(&new_state),
+	};
+
+	Ok(ValidationResult {
+		head_data,
+		new_validation_code: new_validation_code,
+	})
+}

polkadot/parachain/test-parachains/code-upgrader/src/wasm_validation.rs

@@ -0,0 +1,71 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! WASM validation for adder parachain.
+
+use crate::{HeadData, BlockData, RelayChainParams};
+use core::{intrinsics, panic};
+use parachain::primitives::{ValidationResult, HeadData as GenericHeadData};
+use codec::{Encode, Decode};
+
+#[panic_handler]
+#[no_mangle]
+pub fn panic(_info: &panic::PanicInfo) -> ! {
+	unsafe {
+		intrinsics::abort()
+	}
+}
+
+#[alloc_error_handler]
+#[no_mangle]
+pub fn oom(_: core::alloc::Layout) -> ! {
+	unsafe {
+		intrinsics::abort();
+	}
+}
+
+#[no_mangle]
+pub extern fn validate_block(params: *const u8, len: usize) -> u64 {
+	let params = unsafe { parachain::load_params(params, len) };
+	let parent_head = HeadData::decode(&mut &params.parent_head.0[..])
+		.expect("invalid parent head format.");
+
+	let block_data = BlockData::decode(&mut &params.block_data.0[..])
+		.expect("invalid block data format.");
+
+	let parent_hash = tiny_keccak::keccak256(&params.parent_head.0[..]);
+
+	let res = crate::execute(
+		parent_hash,
+		parent_head,
+		block_data,
+		&RelayChainParams {
+			code_upgrade_allowed: params.code_upgrade_allowed,
+			max_code_size: params.max_code_size,
+			relay_chain_block_number: params.relay_chain_height,
+		},
+	);
+
+	match res {
+		Ok(output) => parachain::write_result(
+			&ValidationResult {
+				head_data: GenericHeadData(output.head_data.encode()),
+				new_validation_code: output.new_validation_code,
+			}
+		),
+		Err(_) => panic!("execution failure"),
+	}
+}

polkadot/parachain/test-parachains/halt/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "test-parachain-halt"
+version = "0.7.29-pre1"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "Test parachain which executes forever"
+edition = "2018"
+build = "build.rs"
+
+[dependencies]
+
+[build-dependencies]
+wasm-builder-runner = { package = "substrate-wasm-builder-runner", version = "1.0.5" }
+
+[features]
+default = [ "std" ]
+std = []

polkadot/parachain/test-parachains/halt/build.rs

@@ -0,0 +1,25 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use wasm_builder_runner::WasmBuilder;
+
+fn main() {
+	WasmBuilder::new()
+		.with_current_project()
+		.with_wasm_builder_from_git("https://github.com/paritytech/substrate.git", "8c672e107789ed10973d937ba8cac245404377e2")
+		.export_heap_base()
+		.build()
+}

polkadot/parachain/test-parachains/halt/src/lib.rs

@@ -0,0 +1,49 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Basic parachain that executes forever.
+
+#![no_std]
+#![cfg_attr(not(feature = "std"), feature(core_intrinsics, lang_items, core_panic_info, alloc_error_handler))]
+
+// Make the WASM binary available.
+#[cfg(feature = "std")]
+include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs"));
+
+#[cfg(not(feature = "std"))]
+#[panic_handler]
+#[no_mangle]
+pub fn panic(_info: &core::panic::PanicInfo) -> ! {
+	unsafe {
+		core::intrinsics::abort()
+	}
+}
+
+#[cfg(not(feature = "std"))]
+#[alloc_error_handler]
+#[no_mangle]
+pub fn oom(_: core::alloc::Layout) -> ! {
+	unsafe {
+		core::intrinsics::abort();
+	}
+}
+
+#[cfg(not(feature = "std"))]
+#[no_mangle]
+pub extern fn validate_block(params: *const u8, len: usize) -> usize {
+	loop {}
+}
+

polkadot/parachain/test-parachains/src/lib.rs

@@ -0,0 +1,17 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Stub - the fundamental logic of this crate is the integration tests.

polkadot/parachain/test-parachains/tests/adder/mod.rs

@@ -0,0 +1,170 @@
+// Copyright 2017-2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Basic parachain that adds a number as part of its state.
+
+use crate::{
+	DummyExt,
+	parachain,
+	parachain::primitives::{
+		RelayChainBlockNumber,
+		BlockData as GenericBlockData,
+		HeadData as GenericHeadData,
+		ValidationParams,
+	},
+};
+use codec::{Decode, Encode};
+
+/// Head data for this parachain.
+#[derive(Default, Clone, Encode, Decode)]
+struct HeadData {
+	/// Block number
+	number: u64,
+	/// parent block keccak256
+	parent_hash: [u8; 32],
+	/// hash of post-execution state.
+	post_state: [u8; 32],
+}
+
+/// Block data for this parachain.
+#[derive(Default, Clone, Encode, Decode)]
+struct BlockData {
+	/// State to begin from.
+	state: u64,
+	/// Amount to add (overflowing)
+	add: u64,
+}
+
+const TEST_CODE: &[u8] = adder::WASM_BINARY;
+
+fn hash_state(state: u64) -> [u8; 32] {
+	tiny_keccak::keccak256(state.encode().as_slice())
+}
+
+fn hash_head(head: &HeadData) -> [u8; 32] {
+	tiny_keccak::keccak256(head.encode().as_slice())
+}
+
+#[test]
+pub fn execute_good_on_parent() {
+	let parent_head = HeadData {
+		number: 0,
+		parent_hash: [0; 32],
+		post_state: hash_state(0),
+	};
+
+	let block_data = BlockData {
+		state: 0,
+		add: 512,
+	};
+
+	let pool = parachain::wasm_executor::ValidationPool::new();
+
+	let ret = parachain::wasm_executor::validate_candidate(
+		TEST_CODE,
+		ValidationParams {
+			parent_head: GenericHeadData(parent_head.encode()),
+			block_data: GenericBlockData(block_data.encode()),
+			max_code_size: 1024,
+			max_head_data_size: 1024,
+			relay_chain_height: 1,
+			code_upgrade_allowed: None,
+		},
+		DummyExt,
+		parachain::wasm_executor::ExecutionMode::RemoteTest(&pool),
+	).unwrap();
+
+	let new_head = HeadData::decode(&mut &ret.head_data.0[..]).unwrap();
+
+	assert_eq!(new_head.number, 1);
+	assert_eq!(new_head.parent_hash, hash_head(&parent_head));
+	assert_eq!(new_head.post_state, hash_state(512));
+}
+
+#[test]
+fn execute_good_chain_on_parent() {
+	let mut number = 0;
+	let mut parent_hash = [0; 32];
+	let mut last_state = 0;
+	let pool = parachain::wasm_executor::ValidationPool::new();
+
+	for add in 0..10 {
+		let parent_head = HeadData {
+			number,
+			parent_hash,
+			post_state: hash_state(last_state),
+		};
+
+		let block_data = BlockData {
+			state: last_state,
+			add,
+		};
+
+		let ret = parachain::wasm_executor::validate_candidate(
+			TEST_CODE,
+			ValidationParams {
+				parent_head: GenericHeadData(parent_head.encode()),
+				block_data: GenericBlockData(block_data.encode()),
+				max_code_size: 1024,
+				max_head_data_size: 1024,
+				relay_chain_height: number as RelayChainBlockNumber + 1,
+				code_upgrade_allowed: None,
+			},
+			DummyExt,
+			parachain::wasm_executor::ExecutionMode::RemoteTest(&pool),
+		).unwrap();
+
+		let new_head = HeadData::decode(&mut &ret.head_data.0[..]).unwrap();
+
+		assert_eq!(new_head.number, number + 1);
+		assert_eq!(new_head.parent_hash, hash_head(&parent_head));
+		assert_eq!(new_head.post_state, hash_state(last_state + add));
+
+		number += 1;
+		parent_hash = hash_head(&new_head);
+		last_state += add;
+	}
+}
+
+#[test]
+fn execute_bad_on_parent() {
+	let pool = parachain::wasm_executor::ValidationPool::new();
+
+	let parent_head = HeadData {
+		number: 0,
+		parent_hash: [0; 32],
+		post_state: hash_state(0),
+	};
+
+	let block_data = BlockData {
+		state: 256, // start state is wrong.
+		add: 256,
+	};
+
+	let _ret = parachain::wasm_executor::validate_candidate(
+		TEST_CODE,
+		ValidationParams {
+			parent_head: GenericHeadData(parent_head.encode()),
+			block_data: GenericBlockData(block_data.encode()),
+			max_code_size: 1024,
+			max_head_data_size: 1024,
+			relay_chain_height: 1,
+			code_upgrade_allowed: None,
+		},
+		DummyExt,
+		parachain::wasm_executor::ExecutionMode::RemoteTest(&pool),
+	).unwrap_err();
+}

polkadot/parachain/test-parachains/tests/code_upgrader/mod.rs

@@ -0,0 +1,224 @@
+// Copyright 2017-2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Basic parachain that adds a number as part of its state.
+
+use parachain;
+
+use crate::{
+	DummyExt,
+	parachain::primitives::{
+		BlockData as GenericBlockData,
+		HeadData as GenericHeadData,
+		ValidationParams, ValidationCode,
+	},
+};
+use codec::{Decode, Encode};
+use code_upgrader::{hash_state, HeadData, BlockData, State};
+
+const TEST_CODE: &[u8] = code_upgrader::WASM_BINARY;
+
+#[test]
+pub fn execute_good_no_upgrade() {
+	let pool = parachain::wasm_executor::ValidationPool::new();
+
+	let parent_head = HeadData {
+		number: 0,
+		parent_hash: [0; 32],
+		post_state: hash_state(&State::default()),
+	};
+
+	let block_data = BlockData {
+		state: State::default(),
+		new_validation_code: None,
+	};
+
+	let ret = parachain::wasm_executor::validate_candidate(
+		TEST_CODE,
+		ValidationParams {
+			parent_head: GenericHeadData(parent_head.encode()),
+			block_data: GenericBlockData(block_data.encode()),
+			max_code_size: 1024,
+			max_head_data_size: 1024,
+			relay_chain_height: 1,
+			code_upgrade_allowed: None,
+		},
+		DummyExt,
+		parachain::wasm_executor::ExecutionMode::RemoteTest(&pool),
+	).unwrap();
+
+	let new_head = HeadData::decode(&mut &ret.head_data.0[..]).unwrap();
+
+	assert!(ret.new_validation_code.is_none());
+	assert_eq!(new_head.number, 1);
+	assert_eq!(new_head.parent_hash, parent_head.hash());
+	assert_eq!(new_head.post_state, hash_state(&State::default()));
+}
+
+#[test]
+pub fn execute_good_with_upgrade() {
+	let pool = parachain::wasm_executor::ValidationPool::new();
+
+	let parent_head = HeadData {
+		number: 0,
+		parent_hash: [0; 32],
+		post_state: hash_state(&State::default()),
+	};
+
+	let block_data = BlockData {
+		state: State::default(),
+		new_validation_code: Some(ValidationCode(vec![1, 2, 3])),
+	};
+
+	let ret = parachain::wasm_executor::validate_candidate(
+		TEST_CODE,
+		ValidationParams {
+			parent_head: GenericHeadData(parent_head.encode()),
+			block_data: GenericBlockData(block_data.encode()),
+			max_code_size: 1024,
+			max_head_data_size: 1024,
+			relay_chain_height: 1,
+			code_upgrade_allowed: Some(20),
+		},
+		DummyExt,
+		parachain::wasm_executor::ExecutionMode::RemoteTest(&pool),
+	).unwrap();
+
+	let new_head = HeadData::decode(&mut &ret.head_data.0[..]).unwrap();
+
+	assert_eq!(ret.new_validation_code.unwrap(), ValidationCode(vec![1, 2, 3]));
+	assert_eq!(new_head.number, 1);
+	assert_eq!(new_head.parent_hash, parent_head.hash());
+	assert_eq!(
+		new_head.post_state,
+		hash_state(&State {
+			code: ValidationCode::default(),
+			pending_code: Some((ValidationCode(vec![1, 2, 3]), 20)),
+		}),
+	);
+}
+
+#[test]
+#[should_panic]
+pub fn code_upgrade_not_allowed() {
+	let pool = parachain::wasm_executor::ValidationPool::new();
+
+	let parent_head = HeadData {
+		number: 0,
+		parent_hash: [0; 32],
+		post_state: hash_state(&State::default()),
+	};
+
+	let block_data = BlockData {
+		state: State::default(),
+		new_validation_code: Some(ValidationCode(vec![1, 2, 3])),
+	};
+
+	parachain::wasm_executor::validate_candidate(
+		TEST_CODE,
+		ValidationParams {
+			parent_head: GenericHeadData(parent_head.encode()),
+			block_data: GenericBlockData(block_data.encode()),
+			max_code_size: 1024,
+			max_head_data_size: 1024,
+			relay_chain_height: 1,
+			code_upgrade_allowed: None,
+		},
+		DummyExt,
+		parachain::wasm_executor::ExecutionMode::RemoteTest(&pool),
+	).unwrap();
+}
+
+#[test]
+pub fn applies_code_upgrade_after_delay() {
+	let pool = parachain::wasm_executor::ValidationPool::new();
+
+	let (new_head, state) = {
+		let parent_head = HeadData {
+			number: 0,
+			parent_hash: [0; 32],
+			post_state: hash_state(&State::default()),
+		};
+
+		let block_data = BlockData {
+			state: State::default(),
+			new_validation_code: Some(ValidationCode(vec![1, 2, 3])),
+		};
+
+		let ret = parachain::wasm_executor::validate_candidate(
+			TEST_CODE,
+			ValidationParams {
+				parent_head: GenericHeadData(parent_head.encode()),
+				block_data: GenericBlockData(block_data.encode()),
+				max_code_size: 1024,
+				max_head_data_size: 1024,
+				relay_chain_height: 1,
+				code_upgrade_allowed: Some(2),
+			},
+			DummyExt,
+			parachain::wasm_executor::ExecutionMode::RemoteTest(&pool),
+		).unwrap();
+
+		let new_head = HeadData::decode(&mut &ret.head_data.0[..]).unwrap();
+
+		let parent_hash = parent_head.hash();
+		let state = State {
+			code: ValidationCode::default(),
+			pending_code: Some((ValidationCode(vec![1, 2, 3]), 2)),
+		};
+		assert_eq!(ret.new_validation_code.unwrap(), ValidationCode(vec![1, 2, 3]));
+		assert_eq!(new_head.number, 1);
+		assert_eq!(new_head.parent_hash, parent_hash);
+		assert_eq!(new_head.post_state, hash_state(&state));
+
+		(new_head, state)
+	};
+
+	{
+		let parent_head = new_head;
+		let block_data = BlockData {
+			state,
+			new_validation_code: None,
+		};
+
+		let ret = parachain::wasm_executor::validate_candidate(
+			TEST_CODE,
+			ValidationParams {
+				parent_head: GenericHeadData(parent_head.encode()),
+				block_data: GenericBlockData(block_data.encode()),
+				max_code_size: 1024,
+				max_head_data_size: 1024,
+				relay_chain_height: 2,
+				code_upgrade_allowed: None,
+			},
+			DummyExt,
+			parachain::wasm_executor::ExecutionMode::RemoteTest(&pool),
+		).unwrap();
+
+		let new_head = HeadData::decode(&mut &ret.head_data.0[..]).unwrap();
+
+		assert!(ret.new_validation_code.is_none());
+		assert_eq!(new_head.number, 2);
+		assert_eq!(new_head.parent_hash, parent_head.hash());
+		assert_eq!(
+			new_head.post_state,
+			hash_state(&State {
+				code: ValidationCode(vec![1, 2, 3]),
+				pending_code: None,
+			}),
+		);
+	}
+}

polkadot/parachain/test-parachains/tests/lib.rs

@@ -0,0 +1,40 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+mod adder;
+mod code_upgrader;
+mod wasm_executor;
+
+use parachain::{
+	self, primitives::UpwardMessage, wasm_executor::{Externalities, run_worker},
+};
+
+struct DummyExt;
+impl Externalities for DummyExt {
+	fn post_upward_message(&mut self, _: UpwardMessage) -> Result<(), String> {
+		Ok(())
+	}
+}
+
+// This is not an actual test, but rather an entry point for out-of process WASM executor.
+// When executing tests the executor spawns currently executing binary, which happens to be test binary.
+// It then passes "validation_worker" on CLI effectivly making rust test executor to run this single test.
+#[test]
+fn validation_worker() {
+	if let Some(id) = std::env::args().find(|a| a.starts_with("/shmem_rs_")) {
+		run_worker(&id).unwrap()
+	}
+}

polkadot/parachain/test-parachains/tests/wasm_executor/mod.rs

@@ -0,0 +1,95 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Basic parachain that adds a number as part of its state.
+
+use parachain;
+use crate::{adder, DummyExt};
+use crate::parachain::{
+	primitives::{BlockData, ValidationParams},
+	wasm_executor::EXECUTION_TIMEOUT_SEC,
+};
+
+// Code that exposes `validate_block` and loops infinitely
+const INFINITE_LOOP_CODE: &[u8] = halt::WASM_BINARY;
+
+#[test]
+fn terminates_on_timeout() {
+	let pool = parachain::wasm_executor::ValidationPool::new();
+
+	let result = parachain::wasm_executor::validate_candidate(
+		INFINITE_LOOP_CODE,
+		ValidationParams {
+			block_data: BlockData(Vec::new()),
+			parent_head: Default::default(),
+			max_code_size: 1024,
+			max_head_data_size: 1024,
+			relay_chain_height: 1,
+			code_upgrade_allowed: None,
+		},
+		DummyExt,
+		parachain::wasm_executor::ExecutionMode::RemoteTest(&pool),
+	);
+	match result {
+		Err(parachain::wasm_executor::Error::Timeout) => {},
+		r => panic!("{:?}", r),
+	}
+
+	// check that another parachain can validate normaly
+	adder::execute_good_on_parent();
+}
+
+#[test]
+fn parallel_execution() {
+	let pool = parachain::wasm_executor::ValidationPool::new();
+
+	let start = std::time::Instant::now();
+
+	let pool2 = pool.clone();
+	let thread = std::thread::spawn(move ||
+		parachain::wasm_executor::validate_candidate(
+		INFINITE_LOOP_CODE,
+		ValidationParams {
+			block_data: BlockData(Vec::new()),
+			parent_head: Default::default(),
+			max_code_size: 1024,
+			max_head_data_size: 1024,
+			relay_chain_height: 1,
+			code_upgrade_allowed: None,
+		},
+		DummyExt,
+		parachain::wasm_executor::ExecutionMode::RemoteTest(&pool2),
+	).ok());
+	let _ = parachain::wasm_executor::validate_candidate(
+		INFINITE_LOOP_CODE,
+		ValidationParams {
+			block_data: BlockData(Vec::new()),
+			parent_head: Default::default(),
+			max_code_size: 1024,
+			max_head_data_size: 1024,
+			relay_chain_height: 1,
+			code_upgrade_allowed: None,
+		},
+		DummyExt,
+		parachain::wasm_executor::ExecutionMode::RemoteTest(&pool),
+	);
+	thread.join().unwrap();
+	// total time should be < 2 x EXECUTION_TIMEOUT_SEC
+	assert!(
+		std::time::Instant::now().duration_since(start)
+		< std::time::Duration::from_secs(EXECUTION_TIMEOUT_SEC * 2)
+	);
+}