Upgradeable validation functions (#918)

* upgrade primitives to allow changing validation function

* set up storage schema for old parachains code

* fix compilation errors

* fix test compilation

* add some tests for past code meta

* most of the runtime logic for code upgrades

* implement old-code pruning

* add a couple tests

* clean up remaining TODOs

* add a whole bunch of tests for runtime functionality

* remove unused function

* fix runtime compilation

* extract some primitives to parachain crate

* add validation-code upgrades to validation params and result

* extend validation params with code upgrade fields

* provide maximums to validation params

* port test-parachains

* add a code-upgrader test-parachain and tests

* fix collator tests

* move test-parachains to own folder to work around compilation errors

* fix test compilation

* update the Cargo.lock

* fix parachains tests

* remove dbg! invocation

* use new pool in code-upgrader

* bump lockfile

* link TODO to issue

polkadot/parachain/test-parachains/tests/adder/mod.rs

@@ -0,0 +1,170 @@
+// Copyright 2017-2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Basic parachain that adds a number as part of its state.
+
+use crate::{
+	DummyExt,
+	parachain,
+	parachain::primitives::{
+		RelayChainBlockNumber,
+		BlockData as GenericBlockData,
+		HeadData as GenericHeadData,
+		ValidationParams,
+	},
+};
+use codec::{Decode, Encode};
+
+/// Head data for this parachain.
+#[derive(Default, Clone, Encode, Decode)]
+struct HeadData {
+	/// Block number
+	number: u64,
+	/// parent block keccak256
+	parent_hash: [u8; 32],
+	/// hash of post-execution state.
+	post_state: [u8; 32],
+}
+
+/// Block data for this parachain.
+#[derive(Default, Clone, Encode, Decode)]
+struct BlockData {
+	/// State to begin from.
+	state: u64,
+	/// Amount to add (overflowing)
+	add: u64,
+}
+
+const TEST_CODE: &[u8] = adder::WASM_BINARY;
+
+fn hash_state(state: u64) -> [u8; 32] {
+	tiny_keccak::keccak256(state.encode().as_slice())
+}
+
+fn hash_head(head: &HeadData) -> [u8; 32] {
+	tiny_keccak::keccak256(head.encode().as_slice())
+}
+
+#[test]
+pub fn execute_good_on_parent() {
+	let parent_head = HeadData {
+		number: 0,
+		parent_hash: [0; 32],
+		post_state: hash_state(0),
+	};
+
+	let block_data = BlockData {
+		state: 0,
+		add: 512,
+	};
+
+	let pool = parachain::wasm_executor::ValidationPool::new();
+
+	let ret = parachain::wasm_executor::validate_candidate(
+		TEST_CODE,
+		ValidationParams {
+			parent_head: GenericHeadData(parent_head.encode()),
+			block_data: GenericBlockData(block_data.encode()),
+			max_code_size: 1024,
+			max_head_data_size: 1024,
+			relay_chain_height: 1,
+			code_upgrade_allowed: None,
+		},
+		DummyExt,
+		parachain::wasm_executor::ExecutionMode::RemoteTest(&pool),
+	).unwrap();
+
+	let new_head = HeadData::decode(&mut &ret.head_data.0[..]).unwrap();
+
+	assert_eq!(new_head.number, 1);
+	assert_eq!(new_head.parent_hash, hash_head(&parent_head));
+	assert_eq!(new_head.post_state, hash_state(512));
+}
+
+#[test]
+fn execute_good_chain_on_parent() {
+	let mut number = 0;
+	let mut parent_hash = [0; 32];
+	let mut last_state = 0;
+	let pool = parachain::wasm_executor::ValidationPool::new();
+
+	for add in 0..10 {
+		let parent_head = HeadData {
+			number,
+			parent_hash,
+			post_state: hash_state(last_state),
+		};
+
+		let block_data = BlockData {
+			state: last_state,
+			add,
+		};
+
+		let ret = parachain::wasm_executor::validate_candidate(
+			TEST_CODE,
+			ValidationParams {
+				parent_head: GenericHeadData(parent_head.encode()),
+				block_data: GenericBlockData(block_data.encode()),
+				max_code_size: 1024,
+				max_head_data_size: 1024,
+				relay_chain_height: number as RelayChainBlockNumber + 1,
+				code_upgrade_allowed: None,
+			},
+			DummyExt,
+			parachain::wasm_executor::ExecutionMode::RemoteTest(&pool),
+		).unwrap();
+
+		let new_head = HeadData::decode(&mut &ret.head_data.0[..]).unwrap();
+
+		assert_eq!(new_head.number, number + 1);
+		assert_eq!(new_head.parent_hash, hash_head(&parent_head));
+		assert_eq!(new_head.post_state, hash_state(last_state + add));
+
+		number += 1;
+		parent_hash = hash_head(&new_head);
+		last_state += add;
+	}
+}
+
+#[test]
+fn execute_bad_on_parent() {
+	let pool = parachain::wasm_executor::ValidationPool::new();
+
+	let parent_head = HeadData {
+		number: 0,
+		parent_hash: [0; 32],
+		post_state: hash_state(0),
+	};
+
+	let block_data = BlockData {
+		state: 256, // start state is wrong.
+		add: 256,
+	};
+
+	let _ret = parachain::wasm_executor::validate_candidate(
+		TEST_CODE,
+		ValidationParams {
+			parent_head: GenericHeadData(parent_head.encode()),
+			block_data: GenericBlockData(block_data.encode()),
+			max_code_size: 1024,
+			max_head_data_size: 1024,
+			relay_chain_height: 1,
+			code_upgrade_allowed: None,
+		},
+		DummyExt,
+		parachain::wasm_executor::ExecutionMode::RemoteTest(&pool),
+	).unwrap_err();
+}

polkadot/parachain/test-parachains/tests/code_upgrader/mod.rs

@@ -0,0 +1,224 @@
+// Copyright 2017-2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Basic parachain that adds a number as part of its state.
+
+use parachain;
+
+use crate::{
+	DummyExt,
+	parachain::primitives::{
+		BlockData as GenericBlockData,
+		HeadData as GenericHeadData,
+		ValidationParams, ValidationCode,
+	},
+};
+use codec::{Decode, Encode};
+use code_upgrader::{hash_state, HeadData, BlockData, State};
+
+const TEST_CODE: &[u8] = code_upgrader::WASM_BINARY;
+
+#[test]
+pub fn execute_good_no_upgrade() {
+	let pool = parachain::wasm_executor::ValidationPool::new();
+
+	let parent_head = HeadData {
+		number: 0,
+		parent_hash: [0; 32],
+		post_state: hash_state(&State::default()),
+	};
+
+	let block_data = BlockData {
+		state: State::default(),
+		new_validation_code: None,
+	};
+
+	let ret = parachain::wasm_executor::validate_candidate(
+		TEST_CODE,
+		ValidationParams {
+			parent_head: GenericHeadData(parent_head.encode()),
+			block_data: GenericBlockData(block_data.encode()),
+			max_code_size: 1024,
+			max_head_data_size: 1024,
+			relay_chain_height: 1,
+			code_upgrade_allowed: None,
+		},
+		DummyExt,
+		parachain::wasm_executor::ExecutionMode::RemoteTest(&pool),
+	).unwrap();
+
+	let new_head = HeadData::decode(&mut &ret.head_data.0[..]).unwrap();
+
+	assert!(ret.new_validation_code.is_none());
+	assert_eq!(new_head.number, 1);
+	assert_eq!(new_head.parent_hash, parent_head.hash());
+	assert_eq!(new_head.post_state, hash_state(&State::default()));
+}
+
+#[test]
+pub fn execute_good_with_upgrade() {
+	let pool = parachain::wasm_executor::ValidationPool::new();
+
+	let parent_head = HeadData {
+		number: 0,
+		parent_hash: [0; 32],
+		post_state: hash_state(&State::default()),
+	};
+
+	let block_data = BlockData {
+		state: State::default(),
+		new_validation_code: Some(ValidationCode(vec![1, 2, 3])),
+	};
+
+	let ret = parachain::wasm_executor::validate_candidate(
+		TEST_CODE,
+		ValidationParams {
+			parent_head: GenericHeadData(parent_head.encode()),
+			block_data: GenericBlockData(block_data.encode()),
+			max_code_size: 1024,
+			max_head_data_size: 1024,
+			relay_chain_height: 1,
+			code_upgrade_allowed: Some(20),
+		},
+		DummyExt,
+		parachain::wasm_executor::ExecutionMode::RemoteTest(&pool),
+	).unwrap();
+
+	let new_head = HeadData::decode(&mut &ret.head_data.0[..]).unwrap();
+
+	assert_eq!(ret.new_validation_code.unwrap(), ValidationCode(vec![1, 2, 3]));
+	assert_eq!(new_head.number, 1);
+	assert_eq!(new_head.parent_hash, parent_head.hash());
+	assert_eq!(
+		new_head.post_state,
+		hash_state(&State {
+			code: ValidationCode::default(),
+			pending_code: Some((ValidationCode(vec![1, 2, 3]), 20)),
+		}),
+	);
+}
+
+#[test]
+#[should_panic]
+pub fn code_upgrade_not_allowed() {
+	let pool = parachain::wasm_executor::ValidationPool::new();
+
+	let parent_head = HeadData {
+		number: 0,
+		parent_hash: [0; 32],
+		post_state: hash_state(&State::default()),
+	};
+
+	let block_data = BlockData {
+		state: State::default(),
+		new_validation_code: Some(ValidationCode(vec![1, 2, 3])),
+	};
+
+	parachain::wasm_executor::validate_candidate(
+		TEST_CODE,
+		ValidationParams {
+			parent_head: GenericHeadData(parent_head.encode()),
+			block_data: GenericBlockData(block_data.encode()),
+			max_code_size: 1024,
+			max_head_data_size: 1024,
+			relay_chain_height: 1,
+			code_upgrade_allowed: None,
+		},
+		DummyExt,
+		parachain::wasm_executor::ExecutionMode::RemoteTest(&pool),
+	).unwrap();
+}
+
+#[test]
+pub fn applies_code_upgrade_after_delay() {
+	let pool = parachain::wasm_executor::ValidationPool::new();
+
+	let (new_head, state) = {
+		let parent_head = HeadData {
+			number: 0,
+			parent_hash: [0; 32],
+			post_state: hash_state(&State::default()),
+		};
+
+		let block_data = BlockData {
+			state: State::default(),
+			new_validation_code: Some(ValidationCode(vec![1, 2, 3])),
+		};
+
+		let ret = parachain::wasm_executor::validate_candidate(
+			TEST_CODE,
+			ValidationParams {
+				parent_head: GenericHeadData(parent_head.encode()),
+				block_data: GenericBlockData(block_data.encode()),
+				max_code_size: 1024,
+				max_head_data_size: 1024,
+				relay_chain_height: 1,
+				code_upgrade_allowed: Some(2),
+			},
+			DummyExt,
+			parachain::wasm_executor::ExecutionMode::RemoteTest(&pool),
+		).unwrap();
+
+		let new_head = HeadData::decode(&mut &ret.head_data.0[..]).unwrap();
+
+		let parent_hash = parent_head.hash();
+		let state = State {
+			code: ValidationCode::default(),
+			pending_code: Some((ValidationCode(vec![1, 2, 3]), 2)),
+		};
+		assert_eq!(ret.new_validation_code.unwrap(), ValidationCode(vec![1, 2, 3]));
+		assert_eq!(new_head.number, 1);
+		assert_eq!(new_head.parent_hash, parent_hash);
+		assert_eq!(new_head.post_state, hash_state(&state));
+
+		(new_head, state)
+	};
+
+	{
+		let parent_head = new_head;
+		let block_data = BlockData {
+			state,
+			new_validation_code: None,
+		};
+
+		let ret = parachain::wasm_executor::validate_candidate(
+			TEST_CODE,
+			ValidationParams {
+				parent_head: GenericHeadData(parent_head.encode()),
+				block_data: GenericBlockData(block_data.encode()),
+				max_code_size: 1024,
+				max_head_data_size: 1024,
+				relay_chain_height: 2,
+				code_upgrade_allowed: None,
+			},
+			DummyExt,
+			parachain::wasm_executor::ExecutionMode::RemoteTest(&pool),
+		).unwrap();
+
+		let new_head = HeadData::decode(&mut &ret.head_data.0[..]).unwrap();
+
+		assert!(ret.new_validation_code.is_none());
+		assert_eq!(new_head.number, 2);
+		assert_eq!(new_head.parent_hash, parent_head.hash());
+		assert_eq!(
+			new_head.post_state,
+			hash_state(&State {
+				code: ValidationCode(vec![1, 2, 3]),
+				pending_code: None,
+			}),
+		);
+	}
+}

polkadot/parachain/test-parachains/tests/lib.rs

@@ -0,0 +1,40 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+mod adder;
+mod code_upgrader;
+mod wasm_executor;
+
+use parachain::{
+	self, primitives::UpwardMessage, wasm_executor::{Externalities, run_worker},
+};
+
+struct DummyExt;
+impl Externalities for DummyExt {
+	fn post_upward_message(&mut self, _: UpwardMessage) -> Result<(), String> {
+		Ok(())
+	}
+}
+
+// This is not an actual test, but rather an entry point for out-of process WASM executor.
+// When executing tests the executor spawns currently executing binary, which happens to be test binary.
+// It then passes "validation_worker" on CLI effectivly making rust test executor to run this single test.
+#[test]
+fn validation_worker() {
+	if let Some(id) = std::env::args().find(|a| a.starts_with("/shmem_rs_")) {
+		run_worker(&id).unwrap()
+	}
+}

polkadot/parachain/test-parachains/tests/wasm_executor/mod.rs

@@ -0,0 +1,95 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Basic parachain that adds a number as part of its state.
+
+use parachain;
+use crate::{adder, DummyExt};
+use crate::parachain::{
+	primitives::{BlockData, ValidationParams},
+	wasm_executor::EXECUTION_TIMEOUT_SEC,
+};
+
+// Code that exposes `validate_block` and loops infinitely
+const INFINITE_LOOP_CODE: &[u8] = halt::WASM_BINARY;
+
+#[test]
+fn terminates_on_timeout() {
+	let pool = parachain::wasm_executor::ValidationPool::new();
+
+	let result = parachain::wasm_executor::validate_candidate(
+		INFINITE_LOOP_CODE,
+		ValidationParams {
+			block_data: BlockData(Vec::new()),
+			parent_head: Default::default(),
+			max_code_size: 1024,
+			max_head_data_size: 1024,
+			relay_chain_height: 1,
+			code_upgrade_allowed: None,
+		},
+		DummyExt,
+		parachain::wasm_executor::ExecutionMode::RemoteTest(&pool),
+	);
+	match result {
+		Err(parachain::wasm_executor::Error::Timeout) => {},
+		r => panic!("{:?}", r),
+	}
+
+	// check that another parachain can validate normaly
+	adder::execute_good_on_parent();
+}
+
+#[test]
+fn parallel_execution() {
+	let pool = parachain::wasm_executor::ValidationPool::new();
+
+	let start = std::time::Instant::now();
+
+	let pool2 = pool.clone();
+	let thread = std::thread::spawn(move ||
+		parachain::wasm_executor::validate_candidate(
+		INFINITE_LOOP_CODE,
+		ValidationParams {
+			block_data: BlockData(Vec::new()),
+			parent_head: Default::default(),
+			max_code_size: 1024,
+			max_head_data_size: 1024,
+			relay_chain_height: 1,
+			code_upgrade_allowed: None,
+		},
+		DummyExt,
+		parachain::wasm_executor::ExecutionMode::RemoteTest(&pool2),
+	).ok());
+	let _ = parachain::wasm_executor::validate_candidate(
+		INFINITE_LOOP_CODE,
+		ValidationParams {
+			block_data: BlockData(Vec::new()),
+			parent_head: Default::default(),
+			max_code_size: 1024,
+			max_head_data_size: 1024,
+			relay_chain_height: 1,
+			code_upgrade_allowed: None,
+		},
+		DummyExt,
+		parachain::wasm_executor::ExecutionMode::RemoteTest(&pool),
+	);
+	thread.join().unwrap();
+	// total time should be < 2 x EXECUTION_TIMEOUT_SEC
+	assert!(
+		std::time::Instant::now().duration_since(start)
+		< std::time::Duration::from_secs(EXECUTION_TIMEOUT_SEC * 2)
+	);
+}