State cache and other performance optimizations (#1345)

* State caching

* Better code caching

* Execution optimizaton

* More optimizations

* Updated wasmi

* Caching test

* Style

* Style

* Reverted some minor changes

* Style and typos

* Style and typos

* Removed panics on missing memory

substrate/core/executor/src/lib.rs

@@ -44,9 +44,6 @@ extern crate parking_lot;
 #[macro_use]
 extern crate log;
 
-#[macro_use]
-extern crate lazy_static;
-
 #[macro_use]
 extern crate error_chain;
 
@@ -84,7 +81,5 @@ pub trait RuntimeInfo {
 	fn runtime_version<E: Externalities<Blake2Hasher>> (
 		&self,
 		ext: &mut E,
-		heap_pages: usize,
-		code: &[u8]
 	) -> Option<RuntimeVersion>;
 }

substrate/core/executor/src/native_executor.rs

@@ -14,38 +14,34 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
+use std::borrow::BorrowMut;
+use std::cell::{RefMut, RefCell};
 use error::{Error, ErrorKind, Result};
 use state_machine::{CodeExecutor, Externalities};
 use wasm_executor::WasmExecutor;
-use wasmi::Module as WasmModule;
+use wasmi::{Module as WasmModule, ModuleRef as WasmModuleInstanceRef};
 use runtime_version::{NativeVersion, RuntimeVersion};
 use std::collections::HashMap;
 use codec::Decode;
-use primitives::hashing::blake2_256;
-use parking_lot::{Mutex, MutexGuard};
 use RuntimeInfo;
 use primitives::Blake2Hasher;
+use primitives::storage::well_known_keys;
+
+/// Default num of pages for the heap
+const DEFAULT_HEAP_PAGES :u64 = 1024;
 
 // For the internal Runtime Cache:
 // Is it compatible enough to run this natively or do we need to fall back on the WasmModule
 
 enum RuntimePreproc {
 	InvalidCode,
-	ValidCode(WasmModule, Option<RuntimeVersion>),
+	ValidCode(WasmModuleInstanceRef, Option<RuntimeVersion>),
 }
 
 type CacheType = HashMap<[u8; 32], RuntimePreproc>;
 
-lazy_static! {
-	static ref RUNTIMES_CACHE: Mutex<CacheType> = Mutex::new(HashMap::new());
-}
-
-// helper function to generate low-over-head caching_keys
-// it is asserted that part of the audit process that any potential on-chain code change
-// will have done is to ensure that the two-x hash is different to that of any other
-// :code value from the same chain
-fn gen_cache_key(code: &[u8]) -> [u8; 32] {
-	blake2_256(code)
+thread_local! {
+	static RUNTIMES_CACHE: RefCell<CacheType> = RefCell::new(HashMap::new());
 }
 
 /// fetch a runtime version from the cache or if there is no cached version yet, create
@@ -53,27 +49,48 @@ fn gen_cache_key(code: &[u8]) -> [u8; 32] {
 /// can be used by comparing returned RuntimeVersion to `ref_version`
 fn fetch_cached_runtime_version<'a, E: Externalities<Blake2Hasher>>(
 	wasm_executor: &WasmExecutor,
-	cache: &'a mut MutexGuard<CacheType>,
+	cache: &'a mut RefMut<CacheType>,
 	ext: &mut E,
-	heap_pages: usize,
-	code: &[u8]
-) -> Result<(&'a WasmModule, &'a Option<RuntimeVersion>)> {
-	let maybe_runtime_preproc = cache.entry(gen_cache_key(code))
-		.or_insert_with(|| match WasmModule::from_buffer(code) {
-			Ok(module) => {
-				let version = wasm_executor.call_in_wasm_module(ext, heap_pages, &module, "Core_version", &[])
-					.ok()
-					.and_then(|v| RuntimeVersion::decode(&mut v.as_slice()));
-				RuntimePreproc::ValidCode(module, version)
-			}
-			Err(e) => {
-				trace!(target: "executor", "Invalid code presented to executor ({:?})", e);
-				RuntimePreproc::InvalidCode
+) -> Result<(&'a WasmModuleInstanceRef, &'a Option<RuntimeVersion>)> {
+
+	let code_hash = match ext.storage_hash(well_known_keys::CODE) {
+		Some(code_hash) => code_hash,
+		None => return Err(ErrorKind::InvalidCode(vec![]).into()),
+	};
+	let maybe_runtime_preproc = cache.borrow_mut().entry(code_hash.into())
+		.or_insert_with(|| {
+			let code = match ext.storage(well_known_keys::CODE) {
+				Some(code) => code,
+				None => return RuntimePreproc::InvalidCode,
+			};
+			let heap_pages = match ext.storage(well_known_keys::HEAP_PAGES) {
+				Some(pages) => u64::decode(&mut &pages[..]).unwrap_or(DEFAULT_HEAP_PAGES),
+				None => DEFAULT_HEAP_PAGES,
+			};
+			match WasmModule::from_buffer(code)
+				.map_err(|_| ErrorKind::InvalidCode(vec![]).into())
+				.and_then(|module| wasm_executor.prepare_module(ext, heap_pages as usize, &module))
+			{
+				Ok(module) => {
+					let version = wasm_executor.call_in_wasm_module(ext, &module, "Core_version", &[])
+						.ok()
+						.and_then(|v| RuntimeVersion::decode(&mut v.as_slice()));
+					RuntimePreproc::ValidCode(module, version)
+				}
+				Err(e) => {
+					trace!(target: "executor", "Invalid code presented to executor ({:?})", e);
+					RuntimePreproc::InvalidCode
+				}
 			}
 		});
 	match maybe_runtime_preproc {
-		RuntimePreproc::InvalidCode => Err(ErrorKind::InvalidCode(code.into()).into()),
-		RuntimePreproc::ValidCode(m, v) => Ok((m, v)),
+		RuntimePreproc::InvalidCode => {
+			let code = ext.storage(well_known_keys::CODE).unwrap_or(vec![]);
+			Err(ErrorKind::InvalidCode(code).into())
+		},
+		RuntimePreproc::ValidCode(m, v) => {
+			Ok((m, v))
+		}
 	}
 }
 
@@ -154,10 +171,10 @@ impl<D: NativeExecutionDispatch> RuntimeInfo for NativeExecutor<D> {
 	fn runtime_version<E: Externalities<Blake2Hasher>>(
 		&self,
 		ext: &mut E,
-		heap_pages: usize,
-		code: &[u8],
 	) -> Option<RuntimeVersion> {
-		fetch_cached_runtime_version(&self.fallback, &mut RUNTIMES_CACHE.lock(), ext, heap_pages, code).ok()?.1.clone()
+		RUNTIMES_CACHE.with(|c|
+			fetch_cached_runtime_version(&self.fallback, &mut c.borrow_mut(), ext).ok()?.1.clone()
+		)
 	}
 }
 
@@ -167,30 +184,30 @@ impl<D: NativeExecutionDispatch> CodeExecutor<Blake2Hasher> for NativeExecutor<D
 	fn call<E: Externalities<Blake2Hasher>>(
 		&self,
 		ext: &mut E,
-		heap_pages: usize,
-		code: &[u8],
 		method: &str,
 		data: &[u8],
 		use_native: bool,
 	) -> (Result<Vec<u8>>, bool) {
-		let mut c = RUNTIMES_CACHE.lock();
-		let (module, onchain_version) = match fetch_cached_runtime_version(&self.fallback, &mut c, ext, heap_pages, code) {
-			Ok((module, onchain_version)) => (module, onchain_version),
-			Err(_) => return (Err(ErrorKind::InvalidCode(code.into()).into()), false),
-		};
-		match (use_native, onchain_version.as_ref().map_or(false, |v| v.can_call_with(&self.native_version.runtime_version))) {
-			(_, false) => {
-				trace!(target: "executor", "Request for native execution failed (native: {}, chain: {})", self.native_version.runtime_version, onchain_version.as_ref().map_or_else(||"<None>".into(), |v| format!("{}", v)));
-				(self.fallback.call_in_wasm_module(ext, heap_pages, module, method, data), false)
+		RUNTIMES_CACHE.with(|c| {
+			let mut c = c.borrow_mut();
+			let (module, onchain_version) = match fetch_cached_runtime_version(&self.fallback, &mut c, ext) {
+				Ok((module, onchain_version)) => (module, onchain_version),
+				Err(e) => return (Err(e), false),
+			};
+			match (use_native, onchain_version.as_ref().map_or(false, |v| v.can_call_with(&self.native_version.runtime_version))) {
+				(_, false) => {
+					trace!(target: "executor", "Request for native execution failed (native: {}, chain: {})", self.native_version.runtime_version, onchain_version.as_ref().map_or_else(||"<None>".into(), |v| format!("{}", v)));
+					(self.fallback.call_in_wasm_module(ext, module, method, data), false)
+				}
+				(false, _) => {
+					(self.fallback.call_in_wasm_module(ext, module, method, data), false)
+				}
+				_ => {
+					trace!(target: "executor", "Request for native execution succeeded (native: {}, chain: {})", self.native_version.runtime_version, onchain_version.as_ref().map_or_else(||"<None>".into(), |v| format!("{}", v)));
+					(D::dispatch(ext, method, data), true)
+				}
 			}
-			(false, _) => {
-				(self.fallback.call_in_wasm_module(ext, heap_pages, module, method, data), false)
-			}
-			_ => {
-				trace!(target: "executor", "Request for native execution succeeded (native: {}, chain: {})", self.native_version.runtime_version, onchain_version.as_ref().map_or_else(||"<None>".into(), |v| format!("{}", v)));
-				(D::dispatch(ext, method, data), true)
-			}
-		}
+		})
 	}
 }
 

substrate/core/executor/src/wasm_executor.rs

@@ -19,10 +19,10 @@
 use std::collections::HashMap;
 
 use wasmi::{
-	Module, ModuleInstance, MemoryInstance, MemoryRef, TableRef, ImportsBuilder
+	Module, ModuleInstance, MemoryInstance, MemoryRef, TableRef, ImportsBuilder, ModuleRef,
 };
 use wasmi::RuntimeValue::{I32, I64};
-use wasmi::memory_units::{Pages, Bytes};
+use wasmi::memory_units::Pages;
 use state_machine::Externalities;
 use error::{Error, ErrorKind, Result};
 use wasm_utils::UserError;
@@ -39,19 +39,17 @@ struct Heap {
 }
 
 impl Heap {
-	/// Construct new `Heap` struct with a given number of pages.
+	/// Construct new `Heap` struct.
 	///
 	/// Returns `Err` if the heap couldn't allocate required
 	/// number of pages.
 	///
 	/// This could mean that wasm binary specifies memory
 	/// limit and we are trying to allocate beyond that limit.
-	fn new(memory: &MemoryRef, pages: usize) -> Result<Self> {
-		let prev_page_count = memory.initial();
-		memory.grow(Pages(pages)).map_err(|_| Error::from(ErrorKind::Runtime))?;
-		Ok(Heap {
-			end: Bytes::from(prev_page_count).0 as u32,
-		})
+	fn new(memory: &MemoryRef) -> Self {
+		Heap {
+			end: memory.used_size().0 as u32,
+		}
 	}
 
 	fn allocate(&mut self, size: u32) -> u32 {
@@ -83,10 +81,10 @@ struct FunctionExecutor<'e, E: Externalities<Blake2Hasher> + 'e> {
 }
 
 impl<'e, E: Externalities<Blake2Hasher>> FunctionExecutor<'e, E> {
-	fn new(m: MemoryRef, heap_pages: usize, t: Option<TableRef>, e: &'e mut E) -> Result<Self> {
+	fn new(m: MemoryRef, t: Option<TableRef>, e: &'e mut E) -> Result<Self> {
 		Ok(FunctionExecutor {
 			sandbox_store: sandbox::Store::new(),
-			heap: Heap::new(&m, heap_pages)?,
+			heap: Heap::new(&m),
 			memory: m,
 			table: t,
 			ext: e,
@@ -638,51 +636,43 @@ impl WasmExecutor {
 		method: &str,
 		data: &[u8],
 		) -> Result<Vec<u8>> {
-		let module = ::wasmi::Module::from_buffer(code).expect("all modules compiled with rustc are valid wasm code; qed");
-		self.call_in_wasm_module(ext, heap_pages, &module, method, data)
+		let module = ::wasmi::Module::from_buffer(code)?;
+		let module = self.prepare_module(ext, heap_pages, &module)?;
+		self.call_in_wasm_module(ext, &module, method, data)
+	}
+
+	fn get_mem_instance(module: &ModuleRef) -> Result<MemoryRef> {
+		Ok(module
+			.export_by_name("memory")
+			.ok_or_else(|| Error::from(ErrorKind::InvalidMemoryReference))?
+			.as_memory()
+			.ok_or_else(|| Error::from(ErrorKind::InvalidMemoryReference))?
+			.clone())
 	}
 
 	/// Call a given method in the given wasm-module runtime.
 	pub fn call_in_wasm_module<E: Externalities<Blake2Hasher>>(
 		&self,
 		ext: &mut E,
-		heap_pages: usize,
-		module: &Module,
+		module_instance: &ModuleRef,
 		method: &str,
 		data: &[u8],
 	) -> Result<Vec<u8>> {
-		// start module instantiation. Don't run 'start' function yet.
-		let intermediate_instance = ModuleInstance::new(
-			module,
-			&ImportsBuilder::new()
-				.with_resolver("env", FunctionExecutor::<E>::resolver())
-		)?;
-
 		// extract a reference to a linear memory, optional reference to a table
 		// and then initialize FunctionExecutor.
-		let memory = intermediate_instance
-			.not_started_instance()
-			.export_by_name("memory")
-			// TODO: with code coming from the blockchain it isn't strictly been compiled with rustc anymore.
-			// these assumptions are probably not true anymore
-			.expect("all modules compiled with rustc should have an export named 'memory'; qed")
-			.as_memory()
-			.expect("in module generated by rustc export named 'memory' should be a memory; qed")
-			.clone();
-		let table: Option<TableRef> = intermediate_instance
-			.not_started_instance()
+		let memory = Self::get_mem_instance(module_instance)?;
+		let table: Option<TableRef> = module_instance
 			.export_by_name("__indirect_function_table")
 			.and_then(|e| e.as_table().cloned());
 
-		let mut fec = FunctionExecutor::new(memory.clone(), heap_pages, table, ext)?;
-
-		// finish instantiation by running 'start' function (if any).
-		let instance = intermediate_instance.run_start(&mut fec)?;
+		let low = memory.lowest_used();
+		let used_mem = memory.used_size();
+		let mut fec = FunctionExecutor::new(memory.clone(), table, ext)?;
 		let size = data.len() as u32;
 		let offset = fec.heap.allocate(size);
 		memory.set(offset, &data)?;
 
-		let result = instance.invoke_export(
+		let result = module_instance.invoke_export(
 			method,
 			&[
 				I32(offset as i32),
@@ -690,22 +680,57 @@ impl WasmExecutor {
 			],
 			&mut fec
 		);
-		let returned = match result {
-			Ok(x) => x,
+		let result = match result {
+			Ok(Some(I64(r))) => {
+				let offset = r as u32;
+				let length = (r >> 32) as u32 as usize;
+				memory.get(offset, length)
+					.map_err(|_| ErrorKind::Runtime.into())
+			},
+			Ok(_) => Err(ErrorKind::InvalidReturn.into()),
 			Err(e) => {
-				trace!(target: "wasm-executor", "Failed to execute code with {} pages", heap_pages);
-				return Err(e.into())
+				trace!(target: "wasm-executor", "Failed to execute code with {} pages", memory.current_size().0);
+				Err(e.into())
 			},
 		};
 
-		if let Some(I64(r)) = returned {
-			let offset = r as u32;
-			let length = (r >> 32) as u32 as usize;
-			memory.get(offset, length)
-				.map_err(|_| ErrorKind::Runtime.into())
-		} else {
-			Err(ErrorKind::InvalidReturn.into())
+		// cleanup module instance for next use
+		let new_low = memory.lowest_used();
+		if new_low < low {
+			memory.zero(new_low as usize, (low - new_low) as usize)?;
+			memory.reset_lowest_used(low);
 		}
+		memory.with_direct_access_mut(|buf| buf.resize(used_mem.0, 0));
+		result
+	}
+
+	/// Prepare module instance
+	pub fn prepare_module<E: Externalities<Blake2Hasher>>(
+		&self,
+		ext: &mut E,
+		heap_pages: usize,
+		module: &Module,
+		) -> Result<ModuleRef>
+	{
+		// start module instantiation. Don't run 'start' function yet.
+		let intermediate_instance = ModuleInstance::new(
+			module,
+			&ImportsBuilder::new()
+			.with_resolver("env", FunctionExecutor::<E>::resolver())
+			)?;
+
+		// extract a reference to a linear memory, optional reference to a table
+		// and then initialize FunctionExecutor.
+		let memory = Self::get_mem_instance(intermediate_instance.not_started_instance())?;
+		memory.grow(Pages(heap_pages)).map_err(|_| Error::from(ErrorKind::Runtime))?;
+		let table: Option<TableRef> = intermediate_instance
+			.not_started_instance()
+			.export_by_name("__indirect_function_table")
+			.and_then(|e| e.as_table().cloned());
+		let mut fec = FunctionExecutor::new(memory.clone(), table, ext)?;
+
+		// finish instantiation by running 'start' function (if any).
+		Ok(intermediate_instance.run_start(&mut fec)?)
 	}
 }