Enable wasmtime caching for PVF (companion for #8057) (#2387)

* Wasm caching

* Fix compilation errors

* Rename `cache_path` to `cache_base_path`

* "Update Substrate"

* Fix check-web-wasm build

Co-authored-by: parity-processbot <>

polkadot/cli/src/cli.rs

@@ -60,6 +60,9 @@ pub enum Subcommand {
 #[allow(missing_docs)]
 #[derive(Debug, StructOpt)]
 pub struct ValidationWorkerCommand {
+	/// The path that the executor can use for it's caching purposes.
+	pub cache_base_path: Option<std::path::PathBuf>,
+
 	#[allow(missing_docs)]
 	pub mem_id: String,
 }

polkadot/cli/src/command.rs

@@ -257,7 +257,10 @@ pub fn run() -> Result<()> {
 				Err(sc_cli::Error::Input("Cannot run validation worker in browser".into()).into())
 			} else {
 				#[cfg(not(any(target_os = "android", feature = "browser")))]
-				polkadot_parachain::wasm_executor::run_worker(&cmd.mem_id)?;
+				polkadot_parachain::wasm_executor::run_worker(
+					&cmd.mem_id,
+					cmd.cache_base_path.clone(),
+				)?;
 				Ok(())
 			}
 		},

polkadot/node/service/src/lib.rs

@@ -1005,13 +1005,26 @@ pub fn build_full(
 	grandpa_pause: Option<(u32, u32)>,
 	jaeger_agent: Option<std::net::SocketAddr>,
 ) -> Result<NewFull<Client>, Error> {
+	let isolation_strategy = {
+		#[cfg(not(any(target_os = "android", target_os = "unknown")))]
+		{
+			let cache_base_path = config.database.path();
+			IsolationStrategy::external_process_with_caching(cache_base_path)
+		}
+
+		#[cfg(any(target_os = "android", target_os = "unknown"))]
+		{
+			IsolationStrategy::InProcess
+		}
+	};
+
 	if config.chain_spec.is_rococo() {
 		new_full::<rococo_runtime::RuntimeApi, RococoExecutor>(
 			config,
 			is_collator,
 			grandpa_pause,
 			jaeger_agent,
-			Default::default(),
+			isolation_strategy,
 		).map(|full| full.with_client(Client::Rococo))
 	} else if config.chain_spec.is_kusama() {
 		new_full::<kusama_runtime::RuntimeApi, KusamaExecutor>(
@@ -1019,7 +1032,7 @@ pub fn build_full(
 			is_collator,
 			grandpa_pause,
 			jaeger_agent,
-			Default::default(),
+			isolation_strategy,
 		).map(|full| full.with_client(Client::Kusama))
 	} else if config.chain_spec.is_westend() {
 		new_full::<westend_runtime::RuntimeApi, WestendExecutor>(
@@ -1027,7 +1040,7 @@ pub fn build_full(
 			is_collator,
 			grandpa_pause,
 			jaeger_agent,
-			Default::default(),
+			isolation_strategy,
 		).map(|full| full.with_client(Client::Westend))
 	} else {
 		new_full::<polkadot_runtime::RuntimeApi, PolkadotExecutor>(
@@ -1035,7 +1048,7 @@ pub fn build_full(
 			is_collator,
 			grandpa_pause,
 			jaeger_agent,
-			Default::default(),
+			isolation_strategy,
 		).map(|full| full.with_client(Client::Polkadot))
 	}
 }

polkadot/parachain/src/wasm_executor/mod.rs

@@ -20,7 +20,7 @@
 //! Assuming the parameters are correct, this module provides a wrapper around
 //! a WASM VM for re-execution of a parachain candidate.
 
-use std::{any::{TypeId, Any}, path::PathBuf};
+use std::{any::{TypeId, Any}, path::{Path, PathBuf}};
 use crate::primitives::{ValidationParams, ValidationResult};
 use parity_scale_codec::{Decode, Encode};
 use sp_core::{storage::{ChildInfo, TrackedStorageKey}, traits::{CallInWasm, SpawnNamed}};
@@ -76,7 +76,10 @@ pub enum IsolationStrategy {
 	/// The validation worker is ran using the process' executable and the subcommand `validation-worker` is passed
 	/// following by the address of the shared memory.
 	#[cfg(not(any(target_os = "android", target_os = "unknown")))]
-	ExternalProcessSelfHost(ValidationPool),
+	ExternalProcessSelfHost {
+		pool: ValidationPool,
+		cache_base_path: Option<String>,
+	},
 	/// The validation worker is ran using the command provided and the argument provided. The address of the shared
 	/// memory is added at the end of the arguments.
 	#[cfg(not(any(target_os = "android", target_os = "unknown")))]
@@ -91,16 +94,16 @@ pub enum IsolationStrategy {
 	},
 }
 
-impl Default for IsolationStrategy {
-	fn default() -> Self {
-		#[cfg(not(any(target_os = "android", target_os = "unknown")))]
-		{
-			Self::ExternalProcessSelfHost(ValidationPool::new())
-		}
+impl IsolationStrategy {
+	#[cfg(not(any(target_os = "android", target_os = "unknown")))]
+	pub fn external_process_with_caching(cache_base_path: Option<&Path>) -> Self {
+		// Convert cache path to string here so that we don't have to do that each time we launch
+		// validation worker.
+		let cache_base_path = cache_base_path.map(|path| path.display().to_string());
 
-		#[cfg(any(target_os = "android", target_os = "unknown"))]
-		{
-			Self::InProcess
+		Self::ExternalProcessSelfHost {
+			pool: ValidationPool::new(),
+			cache_base_path,
 		}
 	}
 }
@@ -165,8 +168,12 @@ pub enum InternalError {
 /// This should be reused across candidate validation instances.
 pub struct ExecutorCache(sc_executor::WasmExecutor);
 
-impl Default for ExecutorCache {
-	fn default() -> Self {
+impl ExecutorCache {
+	/// Returns a new instance of an executor cache.
+	///
+	/// `cache_base_path` allows to specify a directory where the executor is allowed to store files
+	/// for caching, e.g. compilation artifacts.
+	pub fn new(cache_base_path: Option<PathBuf>) -> ExecutorCache {
 		ExecutorCache(sc_executor::WasmExecutor::new(
 			#[cfg(all(feature = "wasmtime", not(any(target_os = "android", target_os = "unknown"))))]
 			sc_executor::WasmExecutionMethod::Compiled,
@@ -175,7 +182,8 @@ impl Default for ExecutorCache {
 			// TODO: Make sure we don't use more than 1GB: https://github.com/paritytech/polkadot/issues/699
 			Some(1024),
 			HostFunctions::host_functions(),
-			8
+			8,
+			cache_base_path,
 		))
 	}
 }
@@ -192,15 +200,15 @@ pub fn validate_candidate(
 	match isolation_strategy {
 		IsolationStrategy::InProcess => {
 			validate_candidate_internal(
-				&ExecutorCache::default(),
+				&ExecutorCache::new(None),
 				validation_code,
 				&params.encode(),
 				spawner,
 			)
 		},
 		#[cfg(not(any(target_os = "android", target_os = "unknown")))]
-		IsolationStrategy::ExternalProcessSelfHost(pool) => {
-			pool.validate_candidate(validation_code, params)
+		IsolationStrategy::ExternalProcessSelfHost { pool, cache_base_path } => {
+			pool.validate_candidate(validation_code, params, cache_base_path.as_deref())
 		},
 		#[cfg(not(any(target_os = "android", target_os = "unknown")))]
 		IsolationStrategy::ExternalProcessCustomHost { pool, binary, args } => {

polkadot/parachain/src/wasm_executor/validation_host.rs

@@ -87,17 +87,35 @@ impl ValidationPool {
 	///
 	/// This will fail if the validation code is not a proper parachain validation module.
 	///
-	/// This function will use `std::env::current_exe()` with the default arguments [`WORKER_ARGS`] to run the worker.
+	/// This function will use `std::env::current_exe()` with the arguments that consist of [`WORKER_ARGS`]
+	/// with appended `cache_base_path` (if any).
 	pub fn validate_candidate(
 		&self,
 		validation_code: &[u8],
 		params: ValidationParams,
+		cache_base_path: Option<&str>,
 	) -> Result<ValidationResult, ValidationError> {
+		use std::{iter, borrow::Cow};
+
+		let worker_cli_args = match cache_base_path {
+			Some(cache_base_path) => {
+				let worker_cli_args: Vec<&str> =
+					WORKER_ARGS.into_iter()
+						.cloned()
+						.chain(iter::once(cache_base_path))
+						.collect();
+				Cow::from(worker_cli_args)
+			}
+			None => {
+				Cow::from(WORKER_ARGS)
+			},
+		};
+
 		self.validate_candidate_custom(
 			validation_code,
 			params,
 			&env::current_exe().map_err(|err| ValidationError::Internal(err.into()))?,
-			WORKER_ARGS,
+			&worker_cli_args,
 		)
 	}
 
@@ -126,7 +144,7 @@ impl ValidationPool {
 
 /// Validation worker process entry point. Runs a loop waiting for candidates to validate
 /// and sends back results via shared memory.
-pub fn run_worker(mem_id: &str) -> Result<(), String> {
+pub fn run_worker(mem_id: &str, cache_base_path: Option<PathBuf>) -> Result<(), String> {
 	let mut memory = match SharedMem::open(mem_id) {
 		Ok(memory) => memory,
 		Err(e) => {
@@ -151,7 +169,7 @@ pub fn run_worker(mem_id: &str) -> Result<(), String> {
 	memory.set(Event::WorkerReady as usize, EventState::Signaled)
 		.map_err(|e| format!("{} Error setting shared event: {:?}", process::id(), e))?;
 
-	let executor = super::ExecutorCache::default();
+	let executor = super::ExecutorCache::new(cache_base_path);
 
 	loop {
 		if watch_exit.load(atomic::Ordering::Relaxed) {

polkadot/parachain/test-parachains/tests/lib.rs

@@ -25,6 +25,6 @@ use parachain::wasm_executor::run_worker;
 #[test]
 fn validation_worker() {
 	if let Some(id) = std::env::args().find(|a| a.starts_with("/shmem_rs_")) {
-		run_worker(&id).unwrap()
+		run_worker(&id, None).unwrap()
 	}
 }