Upgradeable validation functions (#918)

* upgrade primitives to allow changing validation function

* set up storage schema for old parachains code

* fix compilation errors

* fix test compilation

* add some tests for past code meta

* most of the runtime logic for code upgrades

* implement old-code pruning

* add a couple tests

* clean up remaining TODOs

* add a whole bunch of tests for runtime functionality

* remove unused function

* fix runtime compilation

* extract some primitives to parachain crate

* add validation-code upgrades to validation params and result

* extend validation params with code upgrade fields

* provide maximums to validation params

* port test-parachains

* add a code-upgrader test-parachain and tests

* fix collator tests

* move test-parachains to own folder to work around compilation errors

* fix test compilation

* update the Cargo.lock

* fix parachains tests

* remove dbg! invocation

* use new pool in code-upgrader

* bump lockfile

* link TODO to issue

polkadot/parachain/test-parachains/adder/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "test-parachain-adder"
+version = "0.7.29-pre1"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "Test parachain which adds to a number as its state transition"
+edition = "2018"
+build = "build.rs"
+
+[dependencies]
+parachain = { package = "polkadot-parachain", path = "../../", default-features = false, features = [ "wasm-api" ] }
+codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] }
+tiny-keccak = "1.5.0"
+dlmalloc = { version = "0.1.3", features = [ "global" ] }
+
+# We need to make sure the global allocator is disabled until we have support of full substrate externalities
+runtime-io = { package = "sp-io", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, features = [ "disable_allocator" ] }
+
+[build-dependencies]
+wasm-builder-runner = { package = "substrate-wasm-builder-runner", version = "1.0.5" }
+
+[features]
+default = [ "std" ]
+std = ["parachain/std"]

polkadot/parachain/test-parachains/adder/build.rs

@@ -0,0 +1,25 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use wasm_builder_runner::WasmBuilder;
+
+fn main() {
+	WasmBuilder::new()
+		.with_current_project()
+		.with_wasm_builder_from_git("https://github.com/paritytech/substrate.git", "8c672e107789ed10973d937ba8cac245404377e2")
+		.export_heap_base()
+		.build()
+}

polkadot/parachain/test-parachains/adder/collator/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "test-parachain-adder-collator"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+adder = { package = "test-parachain-adder", path = ".." }
+parachain = { package = "polkadot-parachain", path = "../../.." }
+collator = { package = "polkadot-collator", path = "../../../../collator" }
+primitives = { package = "polkadot-primitives", path = "../../../../primitives" }
+sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
+client = { package = "sc-client", git = "https://github.com/paritytech/substrate", branch = "master" }
+client-api = { package = "sc-client-api", git = "https://github.com/paritytech/substrate", branch = "master" }
+parking_lot = "0.10.0"
+codec = { package = "parity-scale-codec", version = "1.2.0" }
+futures = "0.3.4"

polkadot/parachain/test-parachains/adder/collator/src/main.rs

@@ -0,0 +1,149 @@
+// Copyright 2018-2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Collator for polkadot
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use adder::{HeadData as AdderHead, BlockData as AdderBody};
+use sp_core::Pair;
+use codec::{Encode, Decode};
+use primitives::{
+	Hash,
+	parachain::{HeadData, BlockData, Id as ParaId, LocalValidationData, GlobalValidationSchedule},
+};
+use collator::{
+	InvalidHead, ParachainContext, Network, BuildParachainContext, load_spec, Configuration,
+};
+use parking_lot::Mutex;
+use futures::future::{Ready, ok, err};
+
+const GENESIS: AdderHead = AdderHead {
+	number: 0,
+	parent_hash: [0; 32],
+	post_state: [
+		1, 27, 77, 3, 221, 140, 1, 241, 4, 145, 67, 207, 156, 76, 129, 126, 75,
+		22, 127, 29, 27, 131, 229, 198, 240, 241, 13, 137, 186, 30, 123, 206
+	],
+};
+
+const GENESIS_BODY: AdderBody = AdderBody {
+	state: 0,
+	add: 0,
+};
+
+#[derive(Clone)]
+struct AdderContext {
+	db: Arc<Mutex<HashMap<AdderHead, AdderBody>>>,
+	/// We store it here to make sure that our interfaces require the correct bounds.
+	_network: Option<Arc<dyn Network>>,
+}
+
+/// The parachain context.
+impl ParachainContext for AdderContext {
+	type ProduceCandidate = Ready<Result<(BlockData, HeadData), InvalidHead>>;
+
+	fn produce_candidate(
+		&mut self,
+		_relay_parent: Hash,
+		_global_validation: GlobalValidationSchedule,
+		local_validation: LocalValidationData,
+	) -> Self::ProduceCandidate
+	{
+		let adder_head = match AdderHead::decode(&mut &local_validation.parent_head.0[..]) {
+			Ok(adder_head) => adder_head,
+			Err(_) => return err(InvalidHead)
+		};
+
+		let mut db = self.db.lock();
+
+		let last_body = if adder_head == GENESIS {
+			GENESIS_BODY
+		} else {
+			db.get(&adder_head)
+				.expect("All past bodies stored since this is the only collator")
+				.clone()
+		};
+
+		let next_body = AdderBody {
+			state: last_body.state.overflowing_add(last_body.add).0,
+			add: adder_head.number % 100,
+		};
+
+		let next_head = adder::execute(adder_head.hash(), adder_head, &next_body)
+			.expect("good execution params; qed");
+
+		let encoded_head = HeadData(next_head.encode());
+		let encoded_body = BlockData(next_body.encode());
+
+		println!("Created collation for #{}, post-state={}",
+			next_head.number, next_body.state.overflowing_add(next_body.add).0);
+
+		db.insert(next_head.clone(), next_body);
+		ok((encoded_body, encoded_head))
+	}
+}
+
+impl BuildParachainContext for AdderContext {
+	type ParachainContext = Self;
+
+	fn build<B, E, R, SP, Extrinsic>(
+		self,
+		_: Arc<collator::PolkadotClient<B, E, R>>,
+		_: SP,
+		network: impl Network + Clone + 'static,
+	) -> Result<Self::ParachainContext, ()> {
+		Ok(Self { _network: Some(Arc::new(network)), ..self })
+	}
+}
+
+fn main() {
+	let key = Arc::new(Pair::from_seed(&[1; 32]));
+	let id: ParaId = 100.into();
+
+	println!("Starting adder collator with genesis: ");
+
+	{
+		let encoded = GENESIS.encode();
+		println!("Dec: {:?}", encoded);
+		print!("Hex: 0x");
+		for byte in encoded {
+			print!("{:02x}", byte);
+		}
+
+		println!();
+	}
+
+	let context = AdderContext {
+		db: Arc::new(Mutex::new(HashMap::new())),
+		_network: None,
+	};
+
+	let mut config = Configuration::default();
+	config.chain_spec = Some(load_spec("dev", false).unwrap());
+
+	let res = collator::run_collator(
+		context,
+		id,
+		key,
+		config,
+	);
+
+	if let Err(e) = res {
+		println!("{}", e);
+	}
+}

polkadot/parachain/test-parachains/adder/src/lib.rs

@@ -0,0 +1,90 @@
+// Copyright 2017-2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Basic parachain that adds a number as part of its state.
+
+#![no_std]
+
+#![cfg_attr(not(feature = "std"), feature(core_intrinsics, lang_items, core_panic_info, alloc_error_handler))]
+
+use codec::{Encode, Decode};
+
+#[cfg(not(feature = "std"))]
+mod wasm_validation;
+
+#[cfg(not(feature = "std"))]
+#[global_allocator]
+static ALLOC: dlmalloc::GlobalDlmalloc = dlmalloc::GlobalDlmalloc;
+
+// Make the WASM binary available.
+#[cfg(feature = "std")]
+include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs"));
+
+/// Head data for this parachain.
+#[derive(Default, Clone, Hash, Eq, PartialEq, Encode, Decode)]
+pub struct HeadData {
+	/// Block number
+	pub number: u64,
+	/// parent block keccak256
+	pub parent_hash: [u8; 32],
+	/// hash of post-execution state.
+	pub post_state: [u8; 32],
+}
+
+impl HeadData {
+	pub fn hash(&self) -> [u8; 32] {
+		tiny_keccak::keccak256(&self.encode())
+	}
+}
+
+/// Block data for this parachain.
+#[derive(Default, Clone, Encode, Decode)]
+pub struct BlockData {
+	/// State to begin from.
+	pub state: u64,
+	/// Amount to add (overflowing)
+	pub add: u64,
+}
+
+pub fn hash_state(state: u64) -> [u8; 32] {
+	tiny_keccak::keccak256(state.encode().as_slice())
+}
+
+/// Start state mismatched with parent header's state hash.
+#[derive(Debug)]
+pub struct StateMismatch;
+
+/// Execute a block body on top of given parent head, producing new parent head
+/// if valid.
+pub fn execute(
+	parent_hash: [u8; 32],
+	parent_head: HeadData,
+	block_data: &BlockData,
+) -> Result<HeadData, StateMismatch> {
+	debug_assert_eq!(parent_hash, parent_head.hash());
+
+	if hash_state(block_data.state) != parent_head.post_state {
+		return Err(StateMismatch);
+	}
+
+	let new_state = block_data.state.overflowing_add(block_data.add).0;
+
+	Ok(HeadData {
+		number: parent_head.number + 1,
+		parent_hash,
+		post_state: hash_state(new_state),
+	})
+}

polkadot/parachain/test-parachains/adder/src/wasm_validation.rs

@@ -0,0 +1,60 @@
+// Copyright 2019-2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! WASM validation for adder parachain.
+
+use crate::{HeadData, BlockData};
+use core::{intrinsics, panic};
+use parachain::primitives::{ValidationResult, HeadData as GenericHeadData};
+use codec::{Encode, Decode};
+
+#[panic_handler]
+#[no_mangle]
+pub fn panic(_info: &panic::PanicInfo) -> ! {
+	unsafe {
+		intrinsics::abort()
+	}
+}
+
+#[alloc_error_handler]
+#[no_mangle]
+pub fn oom(_: core::alloc::Layout) -> ! {
+	unsafe {
+		intrinsics::abort();
+	}
+}
+
+#[no_mangle]
+pub extern fn validate_block(params: *const u8, len: usize) -> u64 {
+	let params = unsafe { parachain::load_params(params, len) };
+	let parent_head = HeadData::decode(&mut &params.parent_head.0[..])
+		.expect("invalid parent head format.");
+
+	let block_data = BlockData::decode(&mut &params.block_data.0[..])
+		.expect("invalid block data format.");
+
+	let parent_hash = tiny_keccak::keccak256(&params.parent_head.0[..]);
+
+	match crate::execute(parent_hash, parent_head, &block_data) {
+		Ok(new_head) => parachain::write_result(
+			&ValidationResult {
+				head_data: GenericHeadData(new_head.encode()),
+				new_validation_code: None,
+			}
+		),
+		Err(_) => panic!("execution failure"),
+	}
+}