Forwardport: Validation: don't detect STDIN closing when running in process (#1695) (#1703)

* Initial commit

Forked at: 8b59544660
Parent branch: origin/master

* Validation: don't detect STDIN closing when running in process  (#1695)

polkadot/parachain/src/wasm_executor/mod.rs

@@ -20,7 +20,7 @@
 //! Assuming the parameters are correct, this module provides a wrapper around
 //! a WASM VM for re-execution of a parachain candidate.
 
-use std::any::{TypeId, Any};
+use std::{any::{TypeId, Any}, path::PathBuf};
 use crate::primitives::{ValidationParams, ValidationResult};
 use codec::{Decode, Encode};
 use sp_core::{storage::{ChildInfo, TrackedStorageKey}, traits::{CallInWasm, SpawnNamed}};
@@ -28,7 +28,7 @@ use sp_externalities::Extensions;
 use sp_wasm_interface::HostFunctions as _;
 
 #[cfg(not(any(target_os = "android", target_os = "unknown")))]
-pub use validation_host::{run_worker, ValidationPool, EXECUTION_TIMEOUT_SEC, ValidationExecutionMode};
+pub use validation_host::{run_worker, ValidationPool, EXECUTION_TIMEOUT_SEC, WORKER_ARGS};
 
 mod validation_host;
 
@@ -58,16 +58,29 @@ pub fn run_worker(_: &str) -> Result<(), String> {
 	Err("Cannot run validation worker on this platform".to_string())
 }
 
-/// WASM code execution mode.
-///
-/// > Note: When compiling for WASM, the `Remote` variants are not available.
-pub enum ExecutionMode<'a> {
-	/// Execute in-process. The execution can not be interrupted or aborted.
-	Local,
-	/// Remote execution in a spawned process.
-	Remote(&'a ValidationPool),
+/// The execution mode for the `ValidationPool`.
+#[derive(Clone)]
+#[cfg_attr(not(any(target_os = "android", target_os = "unknown")), derive(Debug))]
+pub enum ExecutionMode {
+	/// The validation worker is ran in a thread inside the same process.
+	InProcess,
+	/// The validation worker is ran using the process' executable and the subcommand `validation-worker` is passed
+	/// following by the address of the shared memory.
+	ExternalProcessSelfHost(ValidationPool),
+	/// The validation worker is ran using the command provided and the argument provided. The address of the shared
+	/// memory is added at the end of the arguments.
+	ExternalProcessCustomHost {
+		/// Validation pool.
+		pool: ValidationPool,
+		/// Path to the validation worker. The file must exists and be executable.
+		binary: PathBuf,
+		/// List of arguments passed to the validation worker. The address of the shared memory will be automatically
+		/// added after the arguments.
+		args: Vec<String>,
+	},
 }
 
+
 #[derive(Debug, derive_more::Display, derive_more::From)]
 /// Candidate validation error.
 pub enum ValidationError {
@@ -132,19 +145,24 @@ impl std::error::Error for ValidationError {
 pub fn validate_candidate(
 	validation_code: &[u8],
 	params: ValidationParams,
-	options: ExecutionMode<'_>,
+	execution_mode: &ExecutionMode,
 	spawner: impl SpawnNamed + 'static,
 ) -> Result<ValidationResult, ValidationError> {
-	match options {
-		ExecutionMode::Local => {
+	match execution_mode {
+		ExecutionMode::InProcess => {
 			validate_candidate_internal(validation_code, &params.encode(), spawner)
 		},
 		#[cfg(not(any(target_os = "android", target_os = "unknown")))]
-		ExecutionMode::Remote(pool) => {
+		ExecutionMode::ExternalProcessSelfHost(pool) => {
 			pool.validate_candidate(validation_code, params)
 		},
+		#[cfg(not(any(target_os = "android", target_os = "unknown")))]
+		ExecutionMode::ExternalProcessCustomHost { pool, binary, args } => {
+			let args: Vec<&str> = args.iter().map(|x| x.as_str()).collect();
+			pool.validate_candidate_custom(validation_code, params, binary, &args)
+		},
 		#[cfg(any(target_os = "android", target_os = "unknown"))]
-		ExecutionMode::Remote(_pool) =>
+		ExecutionMode::ExternalProcessSelfHost(_) | ExecutionMode::ExternalProcessCustomHost { .. } =>
 			Err(ValidationError::Internal(InternalError::System(
 				Box::<dyn std::error::Error + Send + Sync>::from(
 					"Remote validator not available".to_string()

polkadot/parachain/src/wasm_executor/validation_host.rs

@@ -29,9 +29,9 @@ use log::{debug, trace};
 use futures::executor::ThreadPool;
 use sp_core::traits::SpawnNamed;
 
-/// CLI Argument to start in validation worker mode.
 const WORKER_ARG: &'static str = "validation-worker";
-const WORKER_ARGS: &[&'static str] = &[WORKER_ARG];
+/// CLI Argument to start in validation worker mode.
+pub const WORKER_ARGS: &[&'static str] = &[WORKER_ARG];
 
 /// Execution timeout in seconds;
 #[cfg(debug_assertions)]
@@ -65,60 +65,60 @@ impl SpawnNamed for TaskExecutor {
 	}
 }
 
-/// The execution mode for the `ValidationPool`.
-#[derive(Debug, Clone)]
-pub enum ValidationExecutionMode {
-	/// The validation worker is ran in a thread inside the same process.
-	InProcess,
-	/// The validation worker is ran using the process' executable and the subcommand `validation-worker` is passed
-	/// following by the address of the shared memory.
-	ExternalProcessSelfHost,
-	/// The validation worker is ran using the command provided and the argument provided. The address of the shared
-	/// memory is added at the end of the arguments.
-	ExternalProcessCustomHost {
-		/// Path to the validation worker. The file must exists and be executable.
-		binary: PathBuf,
-		/// List of arguments passed to the validation worker. The address of the shared memory will be automatically
-		/// added after the arguments.
-		args: Vec<String>,
-	},
-}
-
 /// A pool of hosts.
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct ValidationPool {
 	hosts: Arc<Vec<Mutex<ValidationHost>>>,
-	execution_mode: ValidationExecutionMode,
 }
 
 const DEFAULT_NUM_HOSTS: usize = 8;
 
 impl ValidationPool {
 	/// Creates a validation pool with the default configuration.
-	pub fn new(execution_mode: ValidationExecutionMode) -> ValidationPool {
+	pub fn new() -> ValidationPool {
 		ValidationPool {
 			hosts: Arc::new((0..DEFAULT_NUM_HOSTS).map(|_| Default::default()).collect()),
-			execution_mode,
 		}
 	}
 
-	/// Validate a candidate under the given validation code using the next
-	/// free validation host.
+	/// Validate a candidate under the given validation code using the next free validation host.
 	///
 	/// This will fail if the validation code is not a proper parachain validation module.
+	///
+	/// This function will use `std::env::current_exe()` with the default arguments [`WORKER_ARGS`] to run the worker.
 	pub fn validate_candidate(
 		&self,
 		validation_code: &[u8],
 		params: ValidationParams,
+	) -> Result<ValidationResult, ValidationError> {
+		self.validate_candidate_custom(
+			validation_code,
+			params,
+			&env::current_exe().map_err(|err| ValidationError::Internal(err.into()))?,
+			WORKER_ARGS,
+		)
+	}
+
+	/// Validate a candidate under the given validation code using the next free validation host.
+	///
+	/// This will fail if the validation code is not a proper parachain validation module.
+	///
+	/// This function will use the command and the arguments provided in the function's arguments to run the worker.
+	pub fn validate_candidate_custom(
+		&self,
+		validation_code: &[u8],
+		params: ValidationParams,
+		command: &PathBuf,
+		args: &[&str],
 	) -> Result<ValidationResult, ValidationError> {
 		for host in self.hosts.iter() {
 			if let Some(mut host) = host.try_lock() {
-				return host.validate_candidate(validation_code, params, self.execution_mode.clone());
+				return host.validate_candidate(validation_code, params, command, args)
 			}
 		}
 
 		// all workers are busy, just wait for the first one
-		self.hosts[0].lock().validate_candidate(validation_code, params, self.execution_mode.clone())
+		self.hosts[0].lock().validate_candidate(validation_code, params, command, args)
 	}
 }
 
@@ -224,11 +224,32 @@ enum ValidationResultHeader {
 
 unsafe impl Send for ValidationHost {}
 
-#[derive(Default)]
+struct ValidationHostMemory(SharedMem);
+
+impl std::fmt::Debug for ValidationHostMemory {
+	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+		write!(f, "ValidationHostMemory")
+	}
+}
+
+impl std::ops::Deref for ValidationHostMemory {
+	type Target = SharedMem;
+
+	fn deref(&self) -> &Self::Target {
+		&self.0
+	}
+}
+
+impl std::ops::DerefMut for ValidationHostMemory {
+	fn deref_mut(&mut self) -> &mut Self::Target {
+		&mut self.0
+	}
+}
+
+#[derive(Default, Debug)]
 struct ValidationHost {
 	worker: Option<process::Child>,
-	worker_thread: Option<std::thread::JoinHandle<Result<(), String>>>,
-	memory: Option<SharedMem>,
+	memory: Option<ValidationHostMemory>,
 	id: u32,
 }
 
@@ -253,7 +274,7 @@ impl ValidationHost {
 		Ok(mem_config.create()?)
 	}
 
-	fn start_worker(&mut self, execution_mode: ValidationExecutionMode) -> Result<(), InternalError> {
+	fn start_worker(&mut self, cmd: &PathBuf, args: &[&str]) -> Result<(), InternalError> {
 		if let Some(ref mut worker) = self.worker {
 			// Check if still alive
 			if let Ok(None) = worker.try_wait() {
@@ -261,44 +282,23 @@ impl ValidationHost {
 				return Ok(());
 			}
 		}
-		if self.worker_thread.is_some() {
-			return Ok(());
-		}
 
 		let memory = Self::create_memory()?;
 
-		let mut run_worker_process = |cmd: PathBuf, args: Vec<String>| -> Result<(), std::io::Error> {
-			debug!("Starting worker at {:?} with arguments: {:?} and {:?}", cmd, args, memory.get_os_path());
-			let worker = process::Command::new(cmd)
-				.args(args)
-				.arg(memory.get_os_path())
-				.stdin(process::Stdio::piped())
-				.spawn()?;
-			self.id = worker.id();
-			self.worker = Some(worker);
-			Ok(())
-		};
-
-		match execution_mode {
-			ValidationExecutionMode::InProcess => {
-				let mem_id = memory.get_os_path().to_string();
-				self.worker_thread = Some(std::thread::spawn(move || run_worker(mem_id.as_str())));
-			},
-			ValidationExecutionMode::ExternalProcessSelfHost => run_worker_process(
-				env::current_exe()?,
-				WORKER_ARGS.iter().map(|x| x.to_string()).collect(),
-			)?,
-			ValidationExecutionMode::ExternalProcessCustomHost { binary, args } => run_worker_process(
-				binary,
-				args,
-			)?,
-		};
+		debug!("Starting worker at {:?} with arguments: {:?} and {:?}", cmd, args, memory.get_os_path());
+		let worker = process::Command::new(cmd)
+			.args(args)
+			.arg(memory.get_os_path())
+			.stdin(process::Stdio::piped())
+			.spawn()?;
+		self.id = worker.id();
+		self.worker = Some(worker);
 
 		memory.wait(
 			Event::WorkerReady as usize,
 			shared_memory::Timeout::Sec(EXECUTION_TIMEOUT_SEC as usize),
 		)?;
-		self.memory = Some(memory);
+		self.memory = Some(ValidationHostMemory(memory));
 		Ok(())
 	}
 
@@ -309,13 +309,14 @@ impl ValidationHost {
 		&mut self,
 		validation_code: &[u8],
 		params: ValidationParams,
-		execution_mode: ValidationExecutionMode,
+		binary: &PathBuf,
+		args: &[&str],
 	) -> Result<ValidationResult, ValidationError> {
 		if validation_code.len() > MAX_CODE_MEM {
 			return Err(ValidationError::InvalidCandidate(InvalidCandidate::CodeTooLarge(validation_code.len())));
 		}
 		// First, check if need to spawn the child process
-		self.start_worker(execution_mode)?;
+		self.start_worker(binary, args)?;
 		let memory = self.memory.as_mut()
 			.expect("memory is always `Some` after `start_worker` completes successfully");
 		{