PVF: Refactor workers into separate crates, remove host dependency (#7253)

* PVF: Refactor workers into separate crates, remove host dependency

* Fix compile error

* Remove some leftover code

* Fix compile errors

* Update Cargo.lock

* Remove worker main.rs files

I accidentally copied these from the other PR. This PR isn't intended to
introduce standalone workers yet.

* Address review comments

* cargo fmt

* Update a couple of comments

* Update log targets

polkadot/node/core/pvf/src/artifacts.rs

@@ -55,8 +55,9 @@
 //!    older by a predefined parameter. This process is run very rarely (say, once a day). Once the
 //!    artifact is expired it is removed from disk eagerly atomically.
 
-use crate::{error::PrepareError, host::PrepareResultSender, prepare::PrepareStats};
+use crate::host::PrepareResultSender;
 use always_assert::always;
+use polkadot_node_core_pvf_common::{error::PrepareError, prepare::PrepareStats, pvf::PvfPrepData};
 use polkadot_parachain::primitives::ValidationCodeHash;
 use polkadot_primitives::ExecutorParamsHash;
 use std::{
@@ -65,22 +66,6 @@ use std::{
 	time::{Duration, SystemTime},
 };
 
-/// Contains the bytes for a successfully compiled artifact.
-pub struct CompiledArtifact(Vec<u8>);
-
-impl CompiledArtifact {
-	/// Creates a `CompiledArtifact`.
-	pub fn new(code: Vec<u8>) -> Self {
-		Self(code)
-	}
-}
-
-impl AsRef<[u8]> for CompiledArtifact {
-	fn as_ref(&self) -> &[u8] {
-		self.0.as_slice()
-	}
-}
-
 /// Identifier of an artifact. Encodes a code hash of the PVF and a hash of executor parameter set.
 #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct ArtifactId {
@@ -96,6 +81,11 @@ impl ArtifactId {
 		Self { code_hash, executor_params_hash }
 	}
 
+	/// Returns an artifact ID that corresponds to the PVF with given executor params.
+	pub fn from_pvf_prep_data(pvf: &PvfPrepData) -> Self {
+		Self::new(pvf.code_hash(), pvf.executor_params().hash())
+	}
+
 	/// Tries to recover the artifact id from the given file name.
 	#[cfg(test)]
 	pub fn from_file_name(file_name: &str) -> Option<Self> {
@@ -304,7 +294,7 @@ mod tests {
 
 	#[tokio::test]
 	async fn artifacts_removes_cache_on_startup() {
-		let fake_cache_path = crate::worker_common::tmpfile("test-cache").await.unwrap();
+		let fake_cache_path = crate::worker_intf::tmpfile("test-cache").await.unwrap();
 		let fake_artifact_path = {
 			let mut p = fake_cache_path.clone();
 			p.push("wasmtime_0x1234567890123456789012345678901234567890123456789012345678901234");

polkadot/node/core/pvf/src/error.rs

@@ -14,65 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-use crate::prepare::PrepareStats;
-use parity_scale_codec::{Decode, Encode};
-use std::fmt;
-
-/// Result of PVF preparation performed by the validation host. Contains stats about the preparation if
-/// successful
-pub type PrepareResult = Result<PrepareStats, PrepareError>;
-
-/// An error that occurred during the prepare part of the PVF pipeline.
-#[derive(Debug, Clone, Encode, Decode)]
-pub enum PrepareError {
-	/// During the prevalidation stage of preparation an issue was found with the PVF.
-	Prevalidation(String),
-	/// Compilation failed for the given PVF.
-	Preparation(String),
-	/// An unexpected panic has occurred in the preparation worker.
-	Panic(String),
-	/// Failed to prepare the PVF due to the time limit.
-	TimedOut,
-	/// An IO error occurred. This state is reported by either the validation host or by the worker.
-	IoErr(String),
-	/// The temporary file for the artifact could not be created at the given cache path. This state is reported by the
-	/// validation host (not by the worker).
-	CreateTmpFileErr(String),
-	/// The response from the worker is received, but the file cannot be renamed (moved) to the final destination
-	/// location. This state is reported by the validation host (not by the worker).
-	RenameTmpFileErr(String),
-}
-
-impl PrepareError {
-	/// Returns whether this is a deterministic error, i.e. one that should trigger reliably. Those
-	/// errors depend on the PVF itself and the sc-executor/wasmtime logic.
-	///
-	/// Non-deterministic errors can happen spuriously. Typically, they occur due to resource
-	/// starvation, e.g. under heavy load or memory pressure. Those errors are typically transient
-	/// but may persist e.g. if the node is run by overwhelmingly underpowered machine.
-	pub fn is_deterministic(&self) -> bool {
-		use PrepareError::*;
-		match self {
-			Prevalidation(_) | Preparation(_) | Panic(_) => true,
-			TimedOut | IoErr(_) | CreateTmpFileErr(_) | RenameTmpFileErr(_) => false,
-		}
-	}
-}
-
-impl fmt::Display for PrepareError {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		use PrepareError::*;
-		match self {
-			Prevalidation(err) => write!(f, "prevalidation: {}", err),
-			Preparation(err) => write!(f, "preparation: {}", err),
-			Panic(err) => write!(f, "panic: {}", err),
-			TimedOut => write!(f, "prepare: timeout"),
-			IoErr(err) => write!(f, "prepare: io error while receiving response: {}", err),
-			CreateTmpFileErr(err) => write!(f, "prepare: error creating tmp file: {}", err),
-			RenameTmpFileErr(err) => write!(f, "prepare: error renaming tmp file: {}", err),
-		}
-	}
-}
+use polkadot_node_core_pvf_common::error::{InternalValidationError, PrepareError};
 
 /// A error raised during validation of the candidate.
 #[derive(Debug, Clone)]
@@ -122,37 +64,6 @@ pub enum InvalidCandidate {
 	Panic(String),
 }
 
-/// Some internal error occurred.
-///
-/// Should only ever be used for validation errors independent of the candidate and PVF, or for errors we ruled out
-/// during pre-checking (so preparation errors are fine).
-#[derive(Debug, Clone, Encode, Decode)]
-pub enum InternalValidationError {
-	/// Some communication error occurred with the host.
-	HostCommunication(String),
-	/// Could not find or open compiled artifact file.
-	CouldNotOpenFile(String),
-	/// An error occurred in the CPU time monitor thread. Should be totally unrelated to validation.
-	CpuTimeMonitorThread(String),
-	/// Some non-deterministic preparation error occurred.
-	NonDeterministicPrepareError(PrepareError),
-}
-
-impl fmt::Display for InternalValidationError {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		use InternalValidationError::*;
-		match self {
-			HostCommunication(err) =>
-				write!(f, "validation: some communication error occurred with the host: {}", err),
-			CouldNotOpenFile(err) =>
-				write!(f, "validation: could not find or open compiled artifact file: {}", err),
-			CpuTimeMonitorThread(err) =>
-				write!(f, "validation: an error occurred in the CPU time monitor thread: {}", err),
-			NonDeterministicPrepareError(err) => write!(f, "validation: prepare: {}", err),
-		}
-	}
-}
-
 impl From<InternalValidationError> for ValidationError {
 	fn from(error: InternalValidationError) -> Self {
 		Self::InternalError(error)

polkadot/node/core/pvf/src/execute/mod.rs

@@ -24,4 +24,3 @@ mod queue;
 mod worker_intf;
 
 pub use queue::{start, PendingExecutionRequest, ToQueue};
-pub use worker_intf::{Handshake as ExecuteHandshake, Response as ExecuteResponse};

polkadot/node/core/pvf/src/execute/queue.rs

@@ -21,7 +21,7 @@ use crate::{
 	artifacts::{ArtifactId, ArtifactPathId},
 	host::ResultSender,
 	metrics::Metrics,
-	worker_common::{IdleWorker, WorkerHandle},
+	worker_intf::{IdleWorker, WorkerHandle},
 	InvalidCandidate, ValidationError, LOG_TARGET,
 };
 use futures::{

polkadot/node/core/pvf/src/execute/worker_intf.rs

@@ -18,17 +18,20 @@
 
 use crate::{
 	artifacts::ArtifactPathId,
-	error::InternalValidationError,
-	worker_common::{
-		framed_recv, framed_send, path_to_bytes, spawn_with_program_path, IdleWorker, SpawnErr,
-		WorkerHandle, JOB_TIMEOUT_WALL_CLOCK_FACTOR,
+	worker_intf::{
+		path_to_bytes, spawn_with_program_path, IdleWorker, SpawnErr, WorkerHandle,
+		JOB_TIMEOUT_WALL_CLOCK_FACTOR,
 	},
 	LOG_TARGET,
 };
 use futures::FutureExt;
 use futures_timer::Delay;
 use parity_scale_codec::{Decode, Encode};
-
+use polkadot_node_core_pvf_common::{
+	error::InternalValidationError,
+	execute::{Handshake, Response},
+	framed_recv, framed_send,
+};
 use polkadot_parachain::primitives::ValidationResult;
 use polkadot_primitives::ExecutorParams;
 use std::{path::Path, time::Duration};
@@ -208,42 +211,3 @@ async fn recv_response(stream: &mut UnixStream) -> io::Result<Response> {
 		)
 	})
 }
-
-/// The payload of the one-time handshake that is done when a worker process is created. Carries
-/// data from the host to the worker.
-#[derive(Encode, Decode)]
-pub struct Handshake {
-	/// The executor parameters.
-	pub executor_params: ExecutorParams,
-}
-
-/// The response from an execution job on the worker.
-#[derive(Encode, Decode)]
-pub enum Response {
-	/// The job completed successfully.
-	Ok {
-		/// The result of parachain validation.
-		result_descriptor: ValidationResult,
-		/// The amount of CPU time taken by the job.
-		duration: Duration,
-	},
-	/// The candidate is invalid.
-	InvalidCandidate(String),
-	/// The job timed out.
-	TimedOut,
-	/// An unexpected panic has occurred in the execution worker.
-	Panic(String),
-	/// Some internal error occurred.
-	InternalError(InternalValidationError),
-}
-
-impl Response {
-	/// Creates an invalid response from a context `ctx` and a message `msg` (which can be empty).
-	pub fn format_invalid(ctx: &'static str, msg: &str) -> Self {
-		if msg.is_empty() {
-			Self::InvalidCandidate(ctx.to_string())
-		} else {
-			Self::InvalidCandidate(format!("{}: {}", ctx, msg))
-		}
-	}
-}

polkadot/node/core/pvf/src/host.rs

@@ -22,16 +22,19 @@
 
 use crate::{
 	artifacts::{ArtifactId, ArtifactPathId, ArtifactState, Artifacts},
-	error::PrepareError,
 	execute::{self, PendingExecutionRequest},
 	metrics::Metrics,
-	prepare, PrepareResult, Priority, PvfPrepData, ValidationError, LOG_TARGET,
+	prepare, Priority, ValidationError, LOG_TARGET,
 };
 use always_assert::never;
 use futures::{
 	channel::{mpsc, oneshot},
 	Future, FutureExt, SinkExt, StreamExt,
 };
+use polkadot_node_core_pvf_common::{
+	error::{PrepareError, PrepareResult},
+	pvf::PvfPrepData,
+};
 use polkadot_parachain::primitives::ValidationResult;
 use std::{
 	collections::HashMap,
@@ -423,7 +426,7 @@ async fn handle_precheck_pvf(
 	pvf: PvfPrepData,
 	result_sender: PrepareResultSender,
 ) -> Result<(), Fatal> {
-	let artifact_id = pvf.as_artifact_id();
+	let artifact_id = ArtifactId::from_pvf_prep_data(&pvf);
 
 	if let Some(state) = artifacts.artifact_state_mut(&artifact_id) {
 		match state {
@@ -467,7 +470,7 @@ async fn handle_execute_pvf(
 	inputs: ExecutePvfInputs,
 ) -> Result<(), Fatal> {
 	let ExecutePvfInputs { pvf, exec_timeout, params, priority, result_tx } = inputs;
-	let artifact_id = pvf.as_artifact_id();
+	let artifact_id = ArtifactId::from_pvf_prep_data(&pvf);
 	let executor_params = (*pvf.executor_params()).clone();
 
 	if let Some(state) = artifacts.artifact_state_mut(&artifact_id) {
@@ -590,7 +593,7 @@ async fn handle_heads_up(
 	let now = SystemTime::now();
 
 	for active_pvf in active_pvfs {
-		let artifact_id = active_pvf.as_artifact_id();
+		let artifact_id = ArtifactId::from_pvf_prep_data(&active_pvf);
 		if let Some(state) = artifacts.artifact_state_mut(&artifact_id) {
 			match state {
 				ArtifactState::Prepared { last_time_needed, .. } => {
@@ -854,9 +857,10 @@ fn pulse_every(interval: std::time::Duration) -> impl futures::Stream<Item = ()>
 #[cfg(test)]
 pub(crate) mod tests {
 	use super::*;
-	use crate::{prepare::PrepareStats, InvalidCandidate, PrepareError};
+	use crate::InvalidCandidate;
 	use assert_matches::assert_matches;
 	use futures::future::BoxFuture;
+	use polkadot_node_core_pvf_common::{error::PrepareError, prepare::PrepareStats};
 
 	const TEST_EXECUTION_TIMEOUT: Duration = Duration::from_secs(3);
 	pub(crate) const TEST_PREPARATION_TIMEOUT: Duration = Duration::from_secs(30);
@@ -877,7 +881,7 @@ pub(crate) mod tests {
 
 	/// Creates a new PVF which artifact id can be uniquely identified by the given number.
 	fn artifact_id(descriminator: u32) -> ArtifactId {
-		PvfPrepData::from_discriminator(descriminator).as_artifact_id()
+		ArtifactId::from_pvf_prep_data(&PvfPrepData::from_discriminator(descriminator))
 	}
 
 	fn artifact_path(descriminator: u32) -> PathBuf {

polkadot/node/core/pvf/src/lib.rs

@@ -95,27 +95,31 @@ mod host;
 mod metrics;
 mod prepare;
 mod priority;
-mod pvf;
-mod worker_common;
-
-pub use artifacts::CompiledArtifact;
-pub use error::{
-	InternalValidationError, InvalidCandidate, PrepareError, PrepareResult, ValidationError,
-};
-pub use execute::{ExecuteHandshake, ExecuteResponse};
-#[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))]
-pub use prepare::MemoryAllocationStats;
-pub use prepare::{MemoryStats, PrepareStats};
-pub use priority::Priority;
-pub use pvf::PvfPrepData;
-
-pub use host::{start, Config, ValidationHost};
-pub use metrics::Metrics;
-pub use worker_common::{framed_recv, framed_send, JOB_TIMEOUT_WALL_CLOCK_FACTOR};
-
-const LOG_TARGET: &str = "parachain::pvf";
+mod worker_intf;
 
 #[doc(hidden)]
-pub mod testing {
-	pub use crate::worker_common::{spawn_with_program_path, SpawnErr};
-}
+pub mod testing;
+
+// Used by `decl_puppet_worker_main!`.
+#[doc(hidden)]
+pub use sp_tracing;
+
+pub use error::{InvalidCandidate, ValidationError};
+pub use host::{start, Config, ValidationHost};
+pub use metrics::Metrics;
+pub use priority::Priority;
+pub use worker_intf::{framed_recv, framed_send, JOB_TIMEOUT_WALL_CLOCK_FACTOR};
+
+// Re-export some common types.
+pub use polkadot_node_core_pvf_common::{
+	error::{InternalValidationError, PrepareError},
+	prepare::PrepareStats,
+	pvf::PvfPrepData,
+};
+
+// Re-export worker entrypoints.
+pub use polkadot_node_core_pvf_execute_worker::worker_entrypoint as execute_worker_entrypoint;
+pub use polkadot_node_core_pvf_prepare_worker::worker_entrypoint as prepare_worker_entrypoint;
+
+/// The log target for this crate.
+pub const LOG_TARGET: &str = "parachain::pvf";

polkadot/node/core/pvf/src/metrics.rs

@@ -16,7 +16,7 @@
 
 //! Prometheus metrics related to the validation host.
 
-use crate::prepare::MemoryStats;
+use polkadot_node_core_pvf_common::prepare::MemoryStats;
 use polkadot_node_metrics::metrics::{self, prometheus};
 
 /// Validation host metrics.

polkadot/node/core/pvf/src/prepare/mod.rs

@@ -28,36 +28,3 @@ mod worker_intf;
 
 pub use pool::start as start_pool;
 pub use queue::{start as start_queue, FromQueue, ToQueue};
-
-use parity_scale_codec::{Decode, Encode};
-
-/// Preparation statistics, including the CPU time and memory taken.
-#[derive(Debug, Clone, Default, Encode, Decode)]
-pub struct PrepareStats {
-	/// The CPU time that elapsed for the preparation job.
-	pub cpu_time_elapsed: std::time::Duration,
-	/// The observed memory statistics for the preparation job.
-	pub memory_stats: MemoryStats,
-}
-
-/// Helper struct to contain all the memory stats, including `MemoryAllocationStats` and, if
-/// supported by the OS, `ru_maxrss`.
-#[derive(Clone, Debug, Default, Encode, Decode)]
-pub struct MemoryStats {
-	/// Memory stats from `tikv_jemalloc_ctl`.
-	#[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))]
-	pub memory_tracker_stats: Option<MemoryAllocationStats>,
-	/// `ru_maxrss` from `getrusage`. `None` if an error occurred.
-	#[cfg(target_os = "linux")]
-	pub max_rss: Option<i64>,
-}
-
-/// Statistics of collected memory metrics.
-#[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))]
-#[derive(Clone, Debug, Default, Encode, Decode)]
-pub struct MemoryAllocationStats {
-	/// Total resident memory, in bytes.
-	pub resident: u64,
-	/// Total allocated memory, in bytes.
-	pub allocated: u64,
-}

polkadot/node/core/pvf/src/prepare/pool.rs

@@ -16,16 +16,18 @@
 
 use super::worker_intf::{self, Outcome};
 use crate::{
-	error::{PrepareError, PrepareResult},
 	metrics::Metrics,
-	pvf::PvfPrepData,
-	worker_common::{IdleWorker, WorkerHandle},
+	worker_intf::{IdleWorker, WorkerHandle},
 	LOG_TARGET,
 };
 use always_assert::never;
 use futures::{
 	channel::mpsc, future::BoxFuture, stream::FuturesUnordered, Future, FutureExt, StreamExt,
 };
+use polkadot_node_core_pvf_common::{
+	error::{PrepareError, PrepareResult},
+	pvf::PvfPrepData,
+};
 use slotmap::HopSlotMap;
 use std::{
 	fmt,

polkadot/node/core/pvf/src/prepare/queue.rs

@@ -17,11 +17,10 @@
 //! A queue that handles requests for PVF preparation.
 
 use super::pool::{self, Worker};
-use crate::{
-	artifacts::ArtifactId, metrics::Metrics, PrepareResult, Priority, PvfPrepData, LOG_TARGET,
-};
+use crate::{artifacts::ArtifactId, metrics::Metrics, Priority, LOG_TARGET};
 use always_assert::{always, never};
 use futures::{channel::mpsc, stream::StreamExt as _, Future, SinkExt};
+use polkadot_node_core_pvf_common::{error::PrepareResult, pvf::PvfPrepData};
 use std::{
 	collections::{HashMap, VecDeque},
 	path::PathBuf,
@@ -231,7 +230,7 @@ async fn handle_enqueue(
 	);
 	queue.metrics.prepare_enqueued();
 
-	let artifact_id = pvf.as_artifact_id();
+	let artifact_id = ArtifactId::from_pvf_prep_data(&pvf);
 	if never!(
 		queue.artifact_id_to_job.contains_key(&artifact_id),
 		"second Enqueue sent for a known artifact"
@@ -339,7 +338,7 @@ async fn handle_worker_concluded(
 	// this can't be None;
 	// qed.
 	let job_data = never_none!(queue.jobs.remove(job));
-	let artifact_id = job_data.pvf.as_artifact_id();
+	let artifact_id = ArtifactId::from_pvf_prep_data(&job_data.pvf);
 
 	queue.artifact_id_to_job.remove(&artifact_id);
 
@@ -425,7 +424,7 @@ async fn spawn_extra_worker(queue: &mut Queue, critical: bool) -> Result<(), Fat
 async fn assign(queue: &mut Queue, worker: Worker, job: Job) -> Result<(), Fatal> {
 	let job_data = &mut queue.jobs[job];
 
-	let artifact_id = job_data.pvf.as_artifact_id();
+	let artifact_id = ArtifactId::from_pvf_prep_data(&job_data.pvf);
 	let artifact_path = artifact_id.path(&queue.cache_path);
 
 	job_data.worker = Some(worker);
@@ -488,11 +487,10 @@ pub fn start(
 #[cfg(test)]
 mod tests {
 	use super::*;
-	use crate::{
-		error::PrepareError, host::tests::TEST_PREPARATION_TIMEOUT, prepare::PrepareStats,
-	};
+	use crate::host::tests::TEST_PREPARATION_TIMEOUT;
 	use assert_matches::assert_matches;
 	use futures::{future::BoxFuture, FutureExt};
+	use polkadot_node_core_pvf_common::{error::PrepareError, prepare::PrepareStats};
 	use slotmap::SlotMap;
 	use std::task::Poll;
 
@@ -616,7 +614,10 @@ mod tests {
 			result: Ok(PrepareStats::default()),
 		});
 
-		assert_eq!(test.poll_and_recv_from_queue().await.artifact_id, pvf(1).as_artifact_id());
+		assert_eq!(
+			test.poll_and_recv_from_queue().await.artifact_id,
+			ArtifactId::from_pvf_prep_data(&pvf(1))
+		);
 	}
 
 	#[tokio::test]
@@ -735,7 +736,10 @@ mod tests {
 		// Since there is still work, the queue requested one extra worker to spawn to handle the
 		// remaining enqueued work items.
 		assert_eq!(test.poll_and_recv_to_pool().await, pool::ToPool::Spawn);
-		assert_eq!(test.poll_and_recv_from_queue().await.artifact_id, pvf(1).as_artifact_id());
+		assert_eq!(
+			test.poll_and_recv_from_queue().await.artifact_id,
+			ArtifactId::from_pvf_prep_data(&pvf(1))
+		);
 	}
 
 	#[tokio::test]

polkadot/node/core/pvf/src/prepare/worker_intf.rs

@@ -17,17 +17,20 @@
 //! Host interface to the prepare worker.
 
 use crate::{
-	error::{PrepareError, PrepareResult},
 	metrics::Metrics,
-	prepare::PrepareStats,
-	pvf::PvfPrepData,
-	worker_common::{
-		framed_recv, framed_send, path_to_bytes, spawn_with_program_path, tmpfile_in, IdleWorker,
-		SpawnErr, WorkerHandle, JOB_TIMEOUT_WALL_CLOCK_FACTOR,
+	worker_intf::{
+		path_to_bytes, spawn_with_program_path, tmpfile_in, IdleWorker, SpawnErr, WorkerHandle,
+		JOB_TIMEOUT_WALL_CLOCK_FACTOR,
 	},
 	LOG_TARGET,
 };
 use parity_scale_codec::{Decode, Encode};
+use polkadot_node_core_pvf_common::{
+	error::{PrepareError, PrepareResult},
+	framed_recv, framed_send,
+	prepare::PrepareStats,
+	pvf::PvfPrepData,
+};
 
 use sp_core::hexdisplay::HexDisplay;
 use std::{

polkadot/node/core/pvf/src/pvf.rs

@@ -1,116 +0,0 @@
-// Copyright (C) Parity Technologies (UK) Ltd.
-// This file is part of Polkadot.
-
-// Polkadot is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Polkadot is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
-
-use crate::artifacts::ArtifactId;
-use parity_scale_codec::{Decode, Encode};
-use polkadot_parachain::primitives::ValidationCodeHash;
-use polkadot_primitives::ExecutorParams;
-use sp_core::blake2_256;
-use std::{
-	cmp::{Eq, PartialEq},
-	fmt,
-	sync::Arc,
-	time::Duration,
-};
-
-#[cfg(test)]
-use crate::host::tests::TEST_PREPARATION_TIMEOUT;
-
-/// A struct that carries the exhaustive set of data to prepare an artifact out of plain
-/// Wasm binary
-///
-/// Should be cheap to clone.
-#[derive(Clone, Encode, Decode)]
-pub struct PvfPrepData {
-	/// Wasm code (uncompressed)
-	code: Arc<Vec<u8>>,
-	/// Wasm code hash
-	code_hash: ValidationCodeHash,
-	/// Executor environment parameters for the session for which artifact is prepared
-	executor_params: Arc<ExecutorParams>,
-	/// Preparation timeout
-	prep_timeout: Duration,
-}
-
-impl PvfPrepData {
-	/// Returns an instance of the PVF out of the given PVF code and executor params.
-	pub fn from_code(
-		code: Vec<u8>,
-		executor_params: ExecutorParams,
-		prep_timeout: Duration,
-	) -> Self {
-		let code = Arc::new(code);
-		let code_hash = blake2_256(&code).into();
-		let executor_params = Arc::new(executor_params);
-		Self { code, code_hash, executor_params, prep_timeout }
-	}
-
-	/// Returns artifact ID that corresponds to the PVF with given executor params
-	pub(crate) fn as_artifact_id(&self) -> ArtifactId {
-		ArtifactId::new(self.code_hash, self.executor_params.hash())
-	}
-
-	/// Returns validation code hash for the PVF
-	pub(crate) fn code_hash(&self) -> ValidationCodeHash {
-		self.code_hash
-	}
-
-	/// Returns PVF code
-	pub fn code(&self) -> Arc<Vec<u8>> {
-		self.code.clone()
-	}
-
-	/// Returns executor params
-	pub fn executor_params(&self) -> Arc<ExecutorParams> {
-		self.executor_params.clone()
-	}
-
-	/// Returns preparation timeout.
-	pub fn prep_timeout(&self) -> Duration {
-		self.prep_timeout
-	}
-
-	/// Creates a structure for tests
-	#[cfg(test)]
-	pub(crate) fn from_discriminator_and_timeout(num: u32, timeout: Duration) -> Self {
-		let descriminator_buf = num.to_le_bytes().to_vec();
-		Self::from_code(descriminator_buf, ExecutorParams::default(), timeout)
-	}
-
-	#[cfg(test)]
-	pub(crate) fn from_discriminator(num: u32) -> Self {
-		Self::from_discriminator_and_timeout(num, TEST_PREPARATION_TIMEOUT)
-	}
-}
-
-impl fmt::Debug for PvfPrepData {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		write!(
-			f,
-			"Pvf {{ code, code_hash: {:?}, executor_params: {:?}, prep_timeout: {:?} }}",
-			self.code_hash, self.executor_params, self.prep_timeout
-		)
-	}
-}
-
-impl PartialEq for PvfPrepData {
-	fn eq(&self, other: &Self) -> bool {
-		self.code_hash == other.code_hash &&
-			self.executor_params.hash() == other.executor_params.hash()
-	}
-}
-
-impl Eq for PvfPrepData {}

polkadot/node/core/pvf/src/testing.rs

@@ -0,0 +1,94 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Various things for testing other crates.
+//!
+//! N.B. This is not guarded with some feature flag. Overexposing items here may affect the final
+//!      artifact even for production builds.
+
+#[doc(hidden)]
+pub use crate::worker_intf::{spawn_with_program_path, SpawnErr};
+
+use polkadot_primitives::ExecutorParams;
+
+/// A function that emulates the stitches together behaviors of the preparation and the execution
+/// worker in a single synchronous function.
+pub fn validate_candidate(
+	code: &[u8],
+	params: &[u8],
+) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
+	use polkadot_node_core_pvf_execute_worker::Executor;
+	use polkadot_node_core_pvf_prepare_worker::{prepare, prevalidate};
+
+	let code = sp_maybe_compressed_blob::decompress(code, 10 * 1024 * 1024)
+		.expect("Decompressing code failed");
+
+	let blob = prevalidate(&code)?;
+	let compiled_artifact_blob = prepare(blob, &ExecutorParams::default())?;
+
+	let executor = Executor::new(ExecutorParams::default())?;
+	let result = unsafe {
+		// SAFETY: This is trivially safe since the artifact is obtained by calling `prepare`
+		//         and is written into a temporary directory in an unmodified state.
+		executor.execute(&compiled_artifact_blob, params)?
+	};
+
+	Ok(result)
+}
+
+/// Use this macro to declare a `fn main() {}` that will check the arguments and dispatch them to
+/// the appropriate worker, making the executable that can be used for spawning workers.
+#[macro_export]
+macro_rules! decl_puppet_worker_main {
+	() => {
+		fn main() {
+			$crate::sp_tracing::try_init_simple();
+
+			let args = std::env::args().collect::<Vec<_>>();
+			if args.len() < 3 {
+				panic!("wrong number of arguments");
+			}
+
+			let mut version = None;
+			let mut socket_path: &str = "";
+
+			for i in 2..args.len() {
+				match args[i].as_ref() {
+					"--socket-path" => socket_path = args[i + 1].as_str(),
+					"--node-version" => version = Some(args[i + 1].as_str()),
+					_ => (),
+				}
+			}
+
+			let subcommand = &args[1];
+			match subcommand.as_ref() {
+				"exit" => {
+					std::process::exit(1);
+				},
+				"sleep" => {
+					std::thread::sleep(std::time::Duration::from_secs(5));
+				},
+				"prepare-worker" => {
+					$crate::prepare_worker_entrypoint(&socket_path, version);
+				},
+				"execute-worker" => {
+					$crate::execute_worker_entrypoint(&socket_path, version);
+				},
+				other => panic!("unknown subcommand: {}", other),
+			}
+		}
+	};
+}