feat: initialize Kurdistan SDK - independent fork of Polkadot SDK

pezkuwi/node/core/pvf-checker/src/interest_view.rs

@@ -0,0 +1,156 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use pezkuwi_primitives::{Hash, ValidationCodeHash};
+use std::collections::{
+	btree_map::{self, BTreeMap},
+	HashSet,
+};
+
+/// Whether the PVF passed pre-checking or not.
+#[derive(Copy, Clone, Debug)]
+pub enum Judgement {
+	Valid,
+	Invalid,
+}
+
+impl Judgement {
+	/// Whether the PVF is valid or not.
+	pub fn is_valid(&self) -> bool {
+		match self {
+			Judgement::Valid => true,
+			Judgement::Invalid => false,
+		}
+	}
+}
+
+/// Data about a particular validation code.
+#[derive(Default, Debug)]
+struct PvfData {
+	/// If `Some` then the PVF pre-checking was run for this PVF. If `None` we are either waiting
+	/// for the judgement to come in or the PVF pre-checking failed.
+	judgement: Option<Judgement>,
+
+	/// The set of block hashes where this PVF was seen.
+	seen_in: HashSet<Hash>,
+}
+
+impl PvfData {
+	/// Initialize a new `PvfData` which is awaiting for the initial judgement.
+	fn pending(origin: Hash) -> Self {
+		// Preallocate the hashset with 5 items. This is the anticipated maximum leaves we can
+		// deal at the same time. In the vast majority of the cases it will have length of 1.
+		let mut seen_in = HashSet::with_capacity(5);
+		seen_in.insert(origin);
+		Self { judgement: None, seen_in }
+	}
+
+	/// Mark the `PvfData` as seen in the provided relay-chain block referenced by `relay_hash`.
+	pub fn seen_in(&mut self, relay_hash: Hash) {
+		self.seen_in.insert(relay_hash);
+	}
+
+	/// Removes the given `relay_hash` from the set of seen in, and returns if the set is now empty.
+	pub fn remove_origin(&mut self, relay_hash: &Hash) -> bool {
+		self.seen_in.remove(relay_hash);
+		self.seen_in.is_empty()
+	}
+}
+
+/// The result of [`InterestView::on_leaves_update`].
+pub struct OnLeavesUpdateOutcome {
+	/// The list of PVFs that we first seen in the activated block.
+	pub newcomers: Vec<ValidationCodeHash>,
+	/// The number of PVFs that were removed from the view.
+	pub left_num: usize,
+}
+
+/// A structure that keeps track of relevant PVFs and judgements about them. A relevant PVF is one
+/// that resides in at least a single active leaf.
+#[derive(Debug)]
+pub struct InterestView {
+	active_leaves: BTreeMap<Hash, HashSet<ValidationCodeHash>>,
+	pvfs: BTreeMap<ValidationCodeHash, PvfData>,
+}
+
+impl InterestView {
+	pub fn new() -> Self {
+		Self { active_leaves: BTreeMap::new(), pvfs: BTreeMap::new() }
+	}
+
+	pub fn on_leaves_update(
+		&mut self,
+		activated: Option<(Hash, Vec<ValidationCodeHash>)>,
+		deactivated: &[Hash],
+	) -> OnLeavesUpdateOutcome {
+		let mut newcomers = Vec::new();
+
+		if let Some((leaf, pending_pvfs)) = activated {
+			for pvf in &pending_pvfs {
+				match self.pvfs.entry(*pvf) {
+					btree_map::Entry::Vacant(v) => {
+						v.insert(PvfData::pending(leaf));
+						newcomers.push(*pvf);
+					},
+					btree_map::Entry::Occupied(mut o) => {
+						o.get_mut().seen_in(leaf);
+					},
+				}
+			}
+			self.active_leaves.entry(leaf).or_default().extend(pending_pvfs);
+		}
+
+		let mut left_num = 0;
+		for leaf in deactivated {
+			let pvfs = self.active_leaves.remove(leaf);
+			for pvf in pvfs.into_iter().flatten() {
+				if let btree_map::Entry::Occupied(mut o) = self.pvfs.entry(pvf) {
+					let now_empty = o.get_mut().remove_origin(leaf);
+					if now_empty {
+						left_num += 1;
+						o.remove();
+					}
+				}
+			}
+		}
+
+		OnLeavesUpdateOutcome { newcomers, left_num }
+	}
+
+	/// Handles a new judgement for the given `pvf`.
+	///
+	/// Returns `Err` if the given PVF hash is not known.
+	pub fn on_judgement(
+		&mut self,
+		subject: ValidationCodeHash,
+		judgement: Judgement,
+	) -> Result<(), ()> {
+		match self.pvfs.get_mut(&subject) {
+			Some(data) => {
+				data.judgement = Some(judgement);
+				Ok(())
+			},
+			None => Err(()),
+		}
+	}
+
+	/// Returns all PVFs that previously received a judgement.
+	pub fn judgements(&self) -> impl Iterator<Item = (ValidationCodeHash, Judgement)> + '_ {
+		self.pvfs
+			.iter()
+			.filter_map(|(code_hash, data)| data.judgement.map(|judgement| (*code_hash, judgement)))
+	}
+}

pezkuwi/node/core/pvf-checker/src/lib.rs

@@ -0,0 +1,559 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Implements the PVF pre-checking subsystem.
+//!
+//! This subsystem is responsible for scanning the chain for PVFs that are pending for the approval
+//! as well as submitting statements regarding them passing or not the PVF pre-checking.
+
+use futures::{channel::oneshot, future::BoxFuture, prelude::*, stream::FuturesUnordered};
+
+use pezkuwi_node_subsystem::{
+	messages::{CandidateValidationMessage, PreCheckOutcome, PvfCheckerMessage, RuntimeApiMessage},
+	overseer, ActiveLeavesUpdate, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError,
+	SubsystemResult, SubsystemSender,
+};
+use pezkuwi_primitives::{
+	BlockNumber, Hash, PvfCheckStatement, SessionIndex, ValidationCodeHash, ValidatorId,
+	ValidatorIndex,
+};
+use sp_keystore::KeystorePtr;
+use std::collections::HashSet;
+
+const LOG_TARGET: &str = "teyrchain::pvf-checker";
+
+mod interest_view;
+mod metrics;
+mod runtime_api;
+
+#[cfg(test)]
+mod tests;
+
+use self::{
+	interest_view::{InterestView, Judgement},
+	metrics::Metrics,
+};
+
+/// PVF pre-checking subsystem.
+pub struct PvfCheckerSubsystem {
+	keystore: KeystorePtr,
+	metrics: Metrics,
+}
+
+impl PvfCheckerSubsystem {
+	pub fn new(keystore: KeystorePtr, metrics: Metrics) -> Self {
+		PvfCheckerSubsystem { keystore, metrics }
+	}
+}
+
+#[overseer::subsystem(PvfChecker, error=SubsystemError, prefix = self::overseer)]
+impl<Context> PvfCheckerSubsystem {
+	fn start(self, ctx: Context) -> SpawnedSubsystem {
+		let future = run(ctx, self.keystore, self.metrics)
+			.map_err(|e| SubsystemError::with_origin("pvf-checker", e))
+			.boxed();
+
+		SpawnedSubsystem { name: "pvf-checker-subsystem", future }
+	}
+}
+
+/// A struct that holds the credentials required to sign the PVF check statements. These credentials
+/// are implicitly to pinned to a session where our node acts as a validator.
+struct SigningCredentials {
+	/// The validator public key.
+	validator_key: ValidatorId,
+	/// The validator index in the current session.
+	validator_index: ValidatorIndex,
+}
+
+struct State {
+	/// If `Some` then our node is in the active validator set during the current session.
+	///
+	/// Updated when a new session index is detected in one of the heads.
+	credentials: Option<SigningCredentials>,
+
+	/// The number and the hash of the most recent block that we have seen.
+	///
+	/// This is only updated when the PVF pre-checking API is detected in a new leaf block.
+	recent_block: Option<(BlockNumber, Hash)>,
+
+	/// The session index of the most recent session that we have seen.
+	///
+	/// This is only updated when the PVF pre-checking API is detected in a new leaf block.
+	latest_session: Option<SessionIndex>,
+
+	/// The set of PVF hashes that we cast a vote for within the current session.
+	voted: HashSet<ValidationCodeHash>,
+
+	/// The collection of PVFs that are observed throughout the active heads.
+	view: InterestView,
+
+	/// The container for the futures that are waiting for the outcome of the pre-checking.
+	///
+	/// Here are some fun facts about these futures:
+	///
+	/// - Pre-checking can take quite some time, in the matter of tens of seconds, so the futures
+	///   here can soak for quite some time.
+	/// - Pre-checking of one PVF can take drastically more time than pre-checking of another PVF.
+	///   This leads to results coming out of order.
+	///
+	/// Resolving to `None` means that the request was dropped before replying.
+	currently_checking:
+		FuturesUnordered<BoxFuture<'static, Option<(PreCheckOutcome, ValidationCodeHash)>>>,
+}
+
+#[overseer::contextbounds(PvfChecker, prefix = self::overseer)]
+async fn run<Context>(
+	mut ctx: Context,
+	keystore: KeystorePtr,
+	metrics: Metrics,
+) -> SubsystemResult<()> {
+	let mut state = State {
+		credentials: None,
+		recent_block: None,
+		latest_session: None,
+		voted: HashSet::with_capacity(16),
+		view: InterestView::new(),
+		currently_checking: FuturesUnordered::new(),
+	};
+
+	loop {
+		let mut sender = ctx.sender().clone();
+		futures::select! {
+			precheck_response = state.currently_checking.select_next_some() => {
+				if let Some((outcome, validation_code_hash)) = precheck_response {
+					handle_pvf_check(
+						&mut state,
+						&mut sender,
+						&keystore,
+						&metrics,
+						outcome,
+						validation_code_hash,
+					).await;
+				} else {
+					// See note in `initiate_precheck` for why this is possible and why we do not
+					// care here.
+				}
+			}
+			from_overseer = ctx.recv().fuse() => {
+				let outcome = handle_from_overseer(
+					&mut state,
+					&mut sender,
+					&keystore,
+					&metrics,
+					from_overseer?,
+				)
+				.await;
+				if let Some(Conclude) = outcome {
+					return Ok(());
+				}
+			}
+		}
+	}
+}
+
+/// Handle an incoming PVF pre-check result from the candidate-validation subsystem.
+async fn handle_pvf_check(
+	state: &mut State,
+	sender: &mut impl overseer::PvfCheckerSenderTrait,
+	keystore: &KeystorePtr,
+	metrics: &Metrics,
+	outcome: PreCheckOutcome,
+	validation_code_hash: ValidationCodeHash,
+) {
+	gum::debug!(
+		target: LOG_TARGET,
+		?validation_code_hash,
+		"Received pre-check result: {:?}",
+		outcome,
+	);
+
+	let judgement = match outcome {
+		PreCheckOutcome::Valid => Judgement::Valid,
+		PreCheckOutcome::Invalid => Judgement::Invalid,
+		PreCheckOutcome::Failed => {
+			// Always vote against in case of failures. Voting against a PVF when encountering a
+			// timeout (or an unlikely node-specific issue) can be considered safe, since
+			// there is no slashing for being on the wrong side on a pre-check vote.
+			//
+			// Also, by being more strict here, we can safely be more lenient during preparation and
+			// avoid the risk of getting slashed there.
+			gum::info!(
+				target: LOG_TARGET,
+				?validation_code_hash,
+				"Pre-check failed, voting against",
+			);
+			Judgement::Invalid
+		},
+	};
+
+	match state.view.on_judgement(validation_code_hash, judgement) {
+		Ok(()) => (),
+		Err(()) => {
+			gum::debug!(
+				target: LOG_TARGET,
+				?validation_code_hash,
+				"received judgement for an unknown (or removed) PVF hash",
+			);
+			return;
+		},
+	}
+
+	match (state.credentials.as_ref(), state.recent_block, state.latest_session) {
+		// Note, the availability of credentials implies the availability of the recent block and
+		// the session index.
+		(Some(credentials), Some(recent_block), Some(session_index)) => {
+			sign_and_submit_pvf_check_statement(
+				sender,
+				keystore,
+				&mut state.voted,
+				credentials,
+				metrics,
+				recent_block.1,
+				session_index,
+				judgement,
+				validation_code_hash,
+			)
+			.await;
+		},
+		_ => (),
+	}
+}
+
+/// A marker for the outer loop that the subsystem should stop.
+struct Conclude;
+
+async fn handle_from_overseer(
+	state: &mut State,
+	sender: &mut impl overseer::PvfCheckerSenderTrait,
+	keystore: &KeystorePtr,
+	metrics: &Metrics,
+	from_overseer: FromOrchestra<PvfCheckerMessage>,
+) -> Option<Conclude> {
+	match from_overseer {
+		FromOrchestra::Signal(OverseerSignal::Conclude) => {
+			gum::info!(target: LOG_TARGET, "Received `Conclude` signal, exiting");
+			Some(Conclude)
+		},
+		FromOrchestra::Signal(OverseerSignal::BlockFinalized(_, _)) => {
+			// ignore
+			None
+		},
+		FromOrchestra::Signal(OverseerSignal::ActiveLeaves(update)) => {
+			handle_leaves_update(state, sender, keystore, metrics, update).await;
+			None
+		},
+		FromOrchestra::Communication { msg } => match msg {
+				// uninhabited type, thus statically unreachable.
+			},
+	}
+}
+
+async fn handle_leaves_update(
+	state: &mut State,
+	sender: &mut impl overseer::PvfCheckerSenderTrait,
+	keystore: &KeystorePtr,
+	metrics: &Metrics,
+	update: ActiveLeavesUpdate,
+) {
+	if let Some(activated) = update.activated {
+		let ActivationEffect { new_session_index, recent_block, pending_pvfs } =
+			match examine_activation(state, sender, keystore, activated.hash, activated.number)
+				.await
+			{
+				None => {
+					// None indicates that the pre-checking runtime API is not supported.
+					return;
+				},
+				Some(e) => e,
+			};
+
+		// Note that this is not necessarily the newly activated leaf.
+		let recent_block_hash = recent_block.1;
+		state.recent_block = Some(recent_block);
+
+		// Update the PVF view and get the previously unseen PVFs and start working on them.
+		let outcome = state
+			.view
+			.on_leaves_update(Some((activated.hash, pending_pvfs)), &update.deactivated);
+		metrics.on_pvf_observed(outcome.newcomers.len());
+		metrics.on_pvf_left(outcome.left_num);
+		for newcomer in outcome.newcomers {
+			initiate_precheck(state, sender, activated.hash, newcomer, metrics).await;
+		}
+
+		if let Some((new_session_index, credentials)) = new_session_index {
+			// New session change:
+			// - update the session index
+			// - reset the set of all PVFs we voted.
+			// - set (or reset) the credentials.
+			state.latest_session = Some(new_session_index);
+			state.voted.clear();
+			state.credentials = credentials;
+
+			// If our node is a validator in the new session, we need to re-sign and submit all
+			// previously obtained judgements.
+			if let Some(ref credentials) = state.credentials {
+				for (code_hash, judgement) in state.view.judgements() {
+					sign_and_submit_pvf_check_statement(
+						sender,
+						keystore,
+						&mut state.voted,
+						credentials,
+						metrics,
+						recent_block_hash,
+						new_session_index,
+						judgement,
+						code_hash,
+					)
+					.await;
+				}
+			}
+		}
+	} else {
+		state.view.on_leaves_update(None, &update.deactivated);
+	}
+}
+
+struct ActivationEffect {
+	/// If the activated leaf is in a new session, the index of the new session. If the new session
+	/// has a validator in the set our node happened to have private key for, the signing
+	new_session_index: Option<(SessionIndex, Option<SigningCredentials>)>,
+	/// This is the block hash and number of the newly activated block if it's "better" than the
+	/// last one we've seen. The block is better if it's number is higher or if there are no blocks
+	/// observed whatsoever. If the leaf is not better then this holds the existing recent block.
+	recent_block: (BlockNumber, Hash),
+	/// The full list of PVFs that are pending pre-checking according to the runtime API. In case
+	/// the API returned an error this list is empty.
+	pending_pvfs: Vec<ValidationCodeHash>,
+}
+
+/// Examines the new leaf and returns the effects of the examination.
+///
+/// Returns `None` if the PVF pre-checking runtime API is not supported for the given leaf hash.
+async fn examine_activation(
+	state: &mut State,
+	sender: &mut impl overseer::PvfCheckerSenderTrait,
+	keystore: &KeystorePtr,
+	leaf_hash: Hash,
+	leaf_number: BlockNumber,
+) -> Option<ActivationEffect> {
+	gum::debug!(
+		target: LOG_TARGET,
+		"Examining activation of leaf {:?} ({})",
+		leaf_hash,
+		leaf_number,
+	);
+
+	let pending_pvfs = match runtime_api::pvfs_require_precheck(sender, leaf_hash).await {
+		Err(runtime_api::RuntimeRequestError::NotSupported) => return None,
+		Err(_) => {
+			gum::debug!(
+				target: LOG_TARGET,
+				relay_parent = ?leaf_hash,
+				"cannot fetch PVFs that require pre-checking from runtime API",
+			);
+			Vec::new()
+		},
+		Ok(v) => v,
+	};
+
+	let recent_block = match state.recent_block {
+		Some((recent_block_num, recent_block_hash)) if leaf_number < recent_block_num => {
+			// the existing recent block is not worse than the new activation, so leave it.
+			(recent_block_num, recent_block_hash)
+		},
+		_ => (leaf_number, leaf_hash),
+	};
+
+	let new_session_index = match runtime_api::session_index_for_child(sender, leaf_hash).await {
+		Ok(session_index) =>
+			if state.latest_session.map_or(true, |l| l < session_index) {
+				let signing_credentials =
+					check_signing_credentials(sender, keystore, leaf_hash).await;
+				Some((session_index, signing_credentials))
+			} else {
+				None
+			},
+		Err(e) => {
+			gum::warn!(
+				target: LOG_TARGET,
+				relay_parent = ?leaf_hash,
+				"cannot fetch session index from runtime API: {:?}",
+				e,
+			);
+			None
+		},
+	};
+
+	Some(ActivationEffect { new_session_index, recent_block, pending_pvfs })
+}
+
+/// Checks the active validators for the given leaf. If we have a signing key for one of them,
+/// returns the [`SigningCredentials`].
+async fn check_signing_credentials(
+	sender: &mut impl SubsystemSender<RuntimeApiMessage>,
+	keystore: &KeystorePtr,
+	leaf: Hash,
+) -> Option<SigningCredentials> {
+	let validators = match runtime_api::validators(sender, leaf).await {
+		Ok(v) => v,
+		Err(e) => {
+			gum::warn!(
+				target: LOG_TARGET,
+				relay_parent = ?leaf,
+				"error occurred during requesting validators: {:?}",
+				e
+			);
+			return None;
+		},
+	};
+
+	pezkuwi_node_subsystem_util::signing_key_and_index(&validators, keystore).map(
+		|(validator_key, validator_index)| SigningCredentials { validator_key, validator_index },
+	)
+}
+
+/// Signs and submits a vote for or against a given validation code.
+///
+/// If the validator already voted for the given code, this function does nothing.
+async fn sign_and_submit_pvf_check_statement(
+	sender: &mut impl overseer::PvfCheckerSenderTrait,
+	keystore: &KeystorePtr,
+	voted: &mut HashSet<ValidationCodeHash>,
+	credentials: &SigningCredentials,
+	metrics: &Metrics,
+	relay_parent: Hash,
+	session_index: SessionIndex,
+	judgement: Judgement,
+	validation_code_hash: ValidationCodeHash,
+) {
+	gum::debug!(
+		target: LOG_TARGET,
+		?validation_code_hash,
+		?relay_parent,
+		"submitting a PVF check statement for validation code = {:?}",
+		judgement,
+	);
+
+	metrics.on_vote_submission_started();
+
+	if voted.contains(&validation_code_hash) {
+		gum::trace!(
+			target: LOG_TARGET,
+			relay_parent = ?relay_parent,
+			?validation_code_hash,
+			"already voted for this validation code",
+		);
+		metrics.on_vote_duplicate();
+		return;
+	}
+
+	voted.insert(validation_code_hash);
+
+	let stmt = PvfCheckStatement {
+		accept: judgement.is_valid(),
+		session_index,
+		subject: validation_code_hash,
+		validator_index: credentials.validator_index,
+	};
+	let signature = match pezkuwi_node_subsystem_util::sign(
+		keystore,
+		&credentials.validator_key,
+		&stmt.signing_payload(),
+	) {
+		Ok(Some(signature)) => signature,
+		Ok(None) => {
+			gum::warn!(
+				target: LOG_TARGET,
+				?relay_parent,
+				validator_index = ?credentials.validator_index,
+				?validation_code_hash,
+				"private key for signing is not available",
+			);
+			return;
+		},
+		Err(e) => {
+			gum::warn!(
+				target: LOG_TARGET,
+				?relay_parent,
+				validator_index = ?credentials.validator_index,
+				?validation_code_hash,
+				"error signing the statement: {:?}",
+				e,
+			);
+			return;
+		},
+	};
+
+	match runtime_api::submit_pvf_check_statement(sender, relay_parent, stmt, signature).await {
+		Ok(()) => {
+			metrics.on_vote_submitted();
+		},
+		Err(e) => {
+			gum::warn!(
+				target: LOG_TARGET,
+				?relay_parent,
+				?validation_code_hash,
+				"error occurred during submitting a vote: {:?}",
+				e,
+			);
+		},
+	}
+}
+
+/// Sends a request to the candidate-validation subsystem to validate the given PVF.
+///
+/// The relay-parent is used as an anchor from where to fetch the PVF code. The request will be put
+/// into the `currently_checking` set.
+async fn initiate_precheck(
+	state: &mut State,
+	sender: &mut impl overseer::PvfCheckerSenderTrait,
+	relay_parent: Hash,
+	validation_code_hash: ValidationCodeHash,
+	metrics: &Metrics,
+) {
+	gum::debug!(target: LOG_TARGET, ?validation_code_hash, ?relay_parent, "initiating a precheck",);
+
+	let (tx, rx) = oneshot::channel();
+	sender
+		.send_message(CandidateValidationMessage::PreCheck {
+			relay_parent,
+			validation_code_hash,
+			response_sender: tx,
+		})
+		.await;
+
+	let timer = metrics.time_pre_check_judgement();
+	state.currently_checking.push(Box::pin(async move {
+		let _timer = timer;
+		match rx.await {
+			Ok(accept) => Some((accept, validation_code_hash)),
+			Err(oneshot::Canceled) => {
+				// Pre-checking request dropped before replying. That can happen in case the
+				// overseer is shutting down. Our part of shutdown will be handled by the
+				// overseer conclude signal. Log it here just in case.
+				gum::debug!(
+					target: LOG_TARGET,
+					?validation_code_hash,
+					?relay_parent,
+					"precheck request was canceled",
+				);
+				None
+			},
+		}
+	}));
+}

pezkuwi/node/core/pvf-checker/src/metrics.rs

@@ -0,0 +1,130 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Metrics definitions for the PVF pre-checking subsystem.
+
+use pezkuwi_node_subsystem_util::metrics::{self, prometheus};
+
+#[derive(Clone)]
+struct MetricsInner {
+	pre_check_judgement: prometheus::Histogram,
+	votes_total: prometheus::Counter<prometheus::U64>,
+	votes_started: prometheus::Counter<prometheus::U64>,
+	votes_duplicate: prometheus::Counter<prometheus::U64>,
+	pvfs_observed: prometheus::Counter<prometheus::U64>,
+	pvfs_left: prometheus::Counter<prometheus::U64>,
+}
+
+#[derive(Default, Clone)]
+pub struct Metrics(Option<MetricsInner>);
+
+impl Metrics {
+	/// Time between sending the pre-check request to receiving the response.
+	pub(crate) fn time_pre_check_judgement(
+		&self,
+	) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.pre_check_judgement.start_timer())
+	}
+
+	/// Called when a PVF vote/statement is submitted.
+	pub(crate) fn on_vote_submitted(&self) {
+		if let Some(metrics) = &self.0 {
+			metrics.votes_total.inc();
+		}
+	}
+
+	/// Called when a PVF vote/statement is started submission.
+	pub(crate) fn on_vote_submission_started(&self) {
+		if let Some(metrics) = &self.0 {
+			metrics.votes_started.inc();
+		}
+	}
+
+	/// Called when the vote is a duplicate.
+	pub(crate) fn on_vote_duplicate(&self) {
+		if let Some(metrics) = &self.0 {
+			metrics.votes_duplicate.inc();
+		}
+	}
+
+	/// Called when a new PVF is observed.
+	pub(crate) fn on_pvf_observed(&self, num: usize) {
+		if let Some(metrics) = &self.0 {
+			metrics.pvfs_observed.inc_by(num as u64);
+		}
+	}
+
+	/// Called when a PVF left the view.
+	pub(crate) fn on_pvf_left(&self, num: usize) {
+		if let Some(metrics) = &self.0 {
+			metrics.pvfs_left.inc_by(num as u64);
+		}
+	}
+}
+
+impl metrics::Metrics for Metrics {
+	fn try_register(registry: &prometheus::Registry) -> Result<Self, prometheus::PrometheusError> {
+		let metrics = MetricsInner {
+			pre_check_judgement: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"pezkuwi_pvf_precheck_judgement",
+						"Time between sending the pre-check request to receiving the response.",
+					)
+					.buckets(vec![0.1, 0.5, 1.0, 10.0, 20.0, 30.0, 40.0, 50.0, 60.0]),
+				)?,
+				registry,
+			)?,
+			votes_total: prometheus::register(
+				prometheus::Counter::new(
+					"pezkuwi_pvf_precheck_votes_total",
+					"The total number of votes submitted.",
+				)?,
+				registry,
+			)?,
+			votes_started: prometheus::register(
+				prometheus::Counter::new(
+					"pezkuwi_pvf_precheck_votes_started",
+					"The number of votes that are pending submission",
+				)?,
+				registry,
+			)?,
+			votes_duplicate: prometheus::register(
+				prometheus::Counter::new(
+					"pezkuwi_pvf_precheck_votes_duplicate",
+					"The number of votes that are submitted more than once for the same code within\
+the same session.",
+				)?,
+				registry,
+			)?,
+			pvfs_observed: prometheus::register(
+				prometheus::Counter::new(
+					"pezkuwi_pvf_precheck_pvfs_observed",
+					"The number of new PVFs observed.",
+				)?,
+				registry,
+			)?,
+			pvfs_left: prometheus::register(
+				prometheus::Counter::new(
+					"pezkuwi_pvf_precheck_pvfs_left",
+					"The number of PVFs removed from the view.",
+				)?,
+				registry,
+			)?,
+		};
+		Ok(Self(Some(metrics)))
+	}
+}

pezkuwi/node/core/pvf-checker/src/runtime_api.rs

@@ -0,0 +1,108 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::LOG_TARGET;
+use futures::channel::oneshot;
+use pezkuwi_node_subsystem::{
+	errors::RuntimeApiError as RuntimeApiSubsystemError,
+	messages::{RuntimeApiMessage, RuntimeApiRequest},
+	SubsystemSender,
+};
+use pezkuwi_primitives::{
+	Hash, PvfCheckStatement, SessionIndex, ValidationCodeHash, ValidatorId, ValidatorSignature,
+};
+
+pub(crate) async fn session_index_for_child(
+	sender: &mut impl SubsystemSender<RuntimeApiMessage>,
+	relay_parent: Hash,
+) -> Result<SessionIndex, RuntimeRequestError> {
+	let (tx, rx) = oneshot::channel();
+	runtime_api_request(sender, relay_parent, RuntimeApiRequest::SessionIndexForChild(tx), rx).await
+}
+
+pub(crate) async fn validators(
+	sender: &mut impl SubsystemSender<RuntimeApiMessage>,
+	relay_parent: Hash,
+) -> Result<Vec<ValidatorId>, RuntimeRequestError> {
+	let (tx, rx) = oneshot::channel();
+	runtime_api_request(sender, relay_parent, RuntimeApiRequest::Validators(tx), rx).await
+}
+
+pub(crate) async fn submit_pvf_check_statement(
+	sender: &mut impl SubsystemSender<RuntimeApiMessage>,
+	relay_parent: Hash,
+	stmt: PvfCheckStatement,
+	signature: ValidatorSignature,
+) -> Result<(), RuntimeRequestError> {
+	let (tx, rx) = oneshot::channel();
+	runtime_api_request(
+		sender,
+		relay_parent,
+		RuntimeApiRequest::SubmitPvfCheckStatement(stmt, signature, tx),
+		rx,
+	)
+	.await
+}
+
+pub(crate) async fn pvfs_require_precheck(
+	sender: &mut impl SubsystemSender<RuntimeApiMessage>,
+	relay_parent: Hash,
+) -> Result<Vec<ValidationCodeHash>, RuntimeRequestError> {
+	let (tx, rx) = oneshot::channel();
+	runtime_api_request(sender, relay_parent, RuntimeApiRequest::PvfsRequirePrecheck(tx), rx).await
+}
+
+#[derive(Debug)]
+pub(crate) enum RuntimeRequestError {
+	NotSupported,
+	ApiError,
+	CommunicationError,
+}
+
+pub(crate) async fn runtime_api_request<T>(
+	sender: &mut impl SubsystemSender<RuntimeApiMessage>,
+	relay_parent: Hash,
+	request: RuntimeApiRequest,
+	receiver: oneshot::Receiver<Result<T, RuntimeApiSubsystemError>>,
+) -> Result<T, RuntimeRequestError> {
+	sender
+		.send_message(RuntimeApiMessage::Request(relay_parent, request).into())
+		.await;
+
+	receiver
+		.await
+		.map_err(|_| {
+			gum::debug!(target: LOG_TARGET, ?relay_parent, "Runtime API request dropped");
+			RuntimeRequestError::CommunicationError
+		})
+		.and_then(|res| {
+			res.map_err(|e| {
+				use RuntimeApiSubsystemError::*;
+				match e {
+					Execution { .. } => {
+						gum::debug!(
+							target: LOG_TARGET,
+							?relay_parent,
+							err = ?e,
+							"Runtime API request internal error"
+						);
+						RuntimeRequestError::ApiError
+					},
+					NotSupported { .. } => RuntimeRequestError::NotSupported,
+				}
+			})
+		})
+}

pezkuwi/node/core/pvf-checker/src/tests.rs

@@ -0,0 +1,930 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use futures::{channel::oneshot, future::BoxFuture, prelude::*};
+use pezkuwi_node_subsystem::{
+	messages::{
+		AllMessages, CandidateValidationMessage, PreCheckOutcome, PvfCheckerMessage,
+		RuntimeApiMessage, RuntimeApiRequest,
+	},
+	ActiveLeavesUpdate, FromOrchestra, OverseerSignal, RuntimeApiError,
+};
+use pezkuwi_node_subsystem_test_helpers::{
+	make_subsystem_context, mock::new_leaf, TestSubsystemContextHandle,
+};
+use pezkuwi_primitives::{
+	BlockNumber, Hash, Header, PvfCheckStatement, SessionIndex, ValidationCode, ValidationCodeHash,
+	ValidatorId,
+};
+use pezkuwi_primitives_test_helpers::{dummy_digest, dummy_hash, validator_pubkeys};
+use sp_application_crypto::AppCrypto;
+use sp_core::testing::TaskExecutor;
+use sp_keyring::Sr25519Keyring;
+use sp_keystore::Keystore;
+use sp_runtime::traits::AppVerify;
+use std::{collections::HashMap, sync::Arc, time::Duration};
+
+type VirtualOverseer = TestSubsystemContextHandle<PvfCheckerMessage>;
+
+fn dummy_validation_code_hash(discriminator: u8) -> ValidationCodeHash {
+	ValidationCode(vec![discriminator]).hash()
+}
+
+struct StartsNewSession {
+	session_index: SessionIndex,
+	validators: Vec<Sr25519Keyring>,
+}
+
+#[derive(Debug, Clone)]
+struct FakeLeaf {
+	block_hash: Hash,
+	block_number: BlockNumber,
+	pvfs: Vec<ValidationCodeHash>,
+}
+
+impl FakeLeaf {
+	fn new(parent_hash: Hash, block_number: BlockNumber, pvfs: Vec<ValidationCodeHash>) -> Self {
+		let block_header = Header {
+			parent_hash,
+			number: block_number,
+			digest: dummy_digest(),
+			state_root: dummy_hash(),
+			extrinsics_root: dummy_hash(),
+		};
+		let block_hash = block_header.hash();
+		Self { block_hash, block_number, pvfs }
+	}
+
+	fn descendant(&self, pvfs: Vec<ValidationCodeHash>) -> FakeLeaf {
+		FakeLeaf::new(self.block_hash, self.block_number + 1, pvfs)
+	}
+}
+
+struct LeafState {
+	/// The session index at which this leaf was activated.
+	session_index: SessionIndex,
+
+	/// The list of PVFs that are pending in this leaf.
+	pvfs: Vec<ValidationCodeHash>,
+}
+
+/// The state we model about a session.
+struct SessionState {
+	validators: Vec<ValidatorId>,
+}
+
+struct TestState {
+	leaves: HashMap<Hash, LeafState>,
+	sessions: HashMap<SessionIndex, SessionState>,
+	last_session_index: SessionIndex,
+}
+
+const OUR_VALIDATOR: Sr25519Keyring = Sr25519Keyring::Alice;
+
+impl TestState {
+	fn new() -> Self {
+		// Initialize the default session 1. No validators are present there.
+		let last_session_index = 1;
+		let mut sessions = HashMap::new();
+		sessions.insert(last_session_index, SessionState { validators: vec![] });
+
+		let mut leaves = HashMap::new();
+		leaves.insert(dummy_hash(), LeafState { session_index: last_session_index, pvfs: vec![] });
+
+		Self { leaves, sessions, last_session_index }
+	}
+
+	/// A convenience function to receive a message from the overseer and returning `None` if
+	/// nothing was received within a reasonable (for local tests anyway) timeout.
+	async fn recv_timeout(&mut self, handle: &mut VirtualOverseer) -> Option<AllMessages> {
+		futures::select! {
+			msg = handle.recv().fuse() => {
+				Some(msg)
+			}
+			_ = futures_timer::Delay::new(Duration::from_millis(500)).fuse() => {
+				None
+			}
+		}
+	}
+
+	async fn send_conclude(&mut self, handle: &mut VirtualOverseer) {
+		// To ensure that no messages are left in the queue there is no better way to just wait.
+		match self.recv_timeout(handle).await {
+			Some(msg) => {
+				panic!("we supposed to conclude, but received a message: {:#?}", msg);
+			},
+			None => {
+				// No messages are received. We are good.
+			},
+		}
+
+		handle.send(FromOrchestra::Signal(OverseerSignal::Conclude)).await;
+	}
+
+	/// Convenience function to invoke [`active_leaves_update`] with the new leaf that starts a new
+	/// session and there are no deactivated leaves.
+	///
+	/// Returns the block hash of the newly activated leaf.
+	async fn activate_leaf_with_session(
+		&mut self,
+		handle: &mut VirtualOverseer,
+		leaf: FakeLeaf,
+		starts_new_session: StartsNewSession,
+	) {
+		self.active_leaves_update(handle, Some(leaf), Some(starts_new_session), &[])
+			.await
+	}
+
+	/// Convenience function to invoke [`active_leaves_update`] with a new leaf. The leaf does not
+	/// start a new session and there are no deactivated leaves.
+	async fn activate_leaf(&mut self, handle: &mut VirtualOverseer, leaf: FakeLeaf) {
+		self.active_leaves_update(handle, Some(leaf), None, &[]).await
+	}
+
+	async fn deactivate_leaves(
+		&mut self,
+		handle: &mut VirtualOverseer,
+		deactivated: impl IntoIterator<Item = &Hash>,
+	) {
+		self.active_leaves_update(handle, None, None, deactivated).await
+	}
+
+	/// Sends an `ActiveLeavesUpdate` message to the overseer and also updates the test state to
+	/// record leaves and session changes.
+	///
+	/// NOTE: This function may stall if there is an unhandled message for the overseer.
+	async fn active_leaves_update(
+		&mut self,
+		handle: &mut VirtualOverseer,
+		fake_leaf: Option<FakeLeaf>,
+		starts_new_session: Option<StartsNewSession>,
+		deactivated: impl IntoIterator<Item = &Hash>,
+	) {
+		if let Some(new_session) = starts_new_session {
+			assert!(fake_leaf.is_some(), "Session can be started only with an activated leaf");
+			self.last_session_index = new_session.session_index;
+			let prev = self.sessions.insert(
+				new_session.session_index,
+				SessionState { validators: validator_pubkeys(&new_session.validators) },
+			);
+			assert!(prev.is_none(), "Session {} already exists", new_session.session_index);
+		}
+
+		let activated = if let Some(activated_leaf) = fake_leaf {
+			self.leaves.insert(
+				activated_leaf.block_hash,
+				LeafState {
+					session_index: self.last_session_index,
+					pvfs: activated_leaf.pvfs.clone(),
+				},
+			);
+
+			Some(new_leaf(activated_leaf.block_hash, activated_leaf.block_number))
+		} else {
+			None
+		};
+
+		handle
+			.send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate {
+				activated,
+				deactivated: deactivated.into_iter().cloned().collect(),
+			})))
+			.await;
+	}
+
+	/// Expects that the subsystem has sent a `Validators` Runtime API request. Answers with the
+	/// mocked validators for the requested leaf.
+	async fn expect_validators(&mut self, handle: &mut VirtualOverseer) {
+		match self.recv_timeout(handle).await.expect("timeout waiting for a message") {
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+				relay_parent,
+				RuntimeApiRequest::Validators(tx),
+			)) => match self.leaves.get(&relay_parent) {
+				Some(leaf) => {
+					let session_index = leaf.session_index;
+					let session = self.sessions.get(&session_index).unwrap();
+					tx.send(Ok(session.validators.clone())).unwrap();
+				},
+				None => {
+					panic!("a request to an unknown relay parent has been made");
+				},
+			},
+			msg => panic!("Unexpected message was received: {:#?}", msg),
+		}
+	}
+
+	/// Expects that the subsystem has sent a `SessionIndexForChild` Runtime API request. Answers
+	/// with the mocked session index for the requested leaf.
+	async fn expect_session_for_child(&mut self, handle: &mut VirtualOverseer) {
+		match self.recv_timeout(handle).await.expect("timeout waiting for a message") {
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+				relay_parent,
+				RuntimeApiRequest::SessionIndexForChild(tx),
+			)) => match self.leaves.get(&relay_parent) {
+				Some(leaf) => {
+					tx.send(Ok(leaf.session_index)).unwrap();
+				},
+				None => {
+					panic!("a request to an unknown relay parent has been made");
+				},
+			},
+			msg => panic!("Unexpected message was received: {:#?}", msg),
+		}
+	}
+
+	/// Expects that the subsystem has sent a `PvfsRequirePrecheck` Runtime API request. Answers
+	/// with the mocked PVF set for the requested leaf.
+	async fn expect_pvfs_require_precheck(
+		&mut self,
+		handle: &mut VirtualOverseer,
+	) -> ExpectPvfsRequirePrecheck<'_> {
+		match self.recv_timeout(handle).await.expect("timeout waiting for a message") {
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+				relay_parent,
+				RuntimeApiRequest::PvfsRequirePrecheck(tx),
+			)) => ExpectPvfsRequirePrecheck { test_state: self, relay_parent, tx },
+			msg => panic!("Unexpected message was received: {:#?}", msg),
+		}
+	}
+
+	/// Expects that the subsystem has sent a pre-checking request to candidate-validation. Returns
+	/// a mocked handle for the request.
+	async fn expect_candidate_precheck(
+		&mut self,
+		handle: &mut VirtualOverseer,
+	) -> ExpectCandidatePrecheck {
+		match self.recv_timeout(handle).await.expect("timeout waiting for a message") {
+			AllMessages::CandidateValidation(CandidateValidationMessage::PreCheck {
+				relay_parent,
+				validation_code_hash,
+				response_sender,
+				..
+			}) => ExpectCandidatePrecheck { relay_parent, validation_code_hash, tx: response_sender },
+			msg => panic!("Unexpected message was received: {:#?}", msg),
+		}
+	}
+
+	/// Expects that the subsystem has sent a `SubmitPvfCheckStatement` runtime API request. Returns
+	/// a mocked handle for the request.
+	async fn expect_submit_vote(&mut self, handle: &mut VirtualOverseer) -> ExpectSubmitVote {
+		match self.recv_timeout(handle).await.expect("timeout waiting for a message") {
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+				relay_parent,
+				RuntimeApiRequest::SubmitPvfCheckStatement(stmt, signature, tx),
+			)) => {
+				let signing_payload = stmt.signing_payload();
+				assert!(signature.verify(&signing_payload[..], &OUR_VALIDATOR.public().into()));
+
+				ExpectSubmitVote { relay_parent, stmt, tx }
+			},
+			msg => panic!("Unexpected message was received: {:#?}", msg),
+		}
+	}
+}
+
+#[must_use]
+struct ExpectPvfsRequirePrecheck<'a> {
+	test_state: &'a mut TestState,
+	relay_parent: Hash,
+	tx: oneshot::Sender<Result<Vec<ValidationCodeHash>, RuntimeApiError>>,
+}
+
+impl<'a> ExpectPvfsRequirePrecheck<'a> {
+	fn reply_mock(self) {
+		match self.test_state.leaves.get(&self.relay_parent) {
+			Some(leaf) => {
+				self.tx.send(Ok(leaf.pvfs.clone())).unwrap();
+			},
+			None => {
+				panic!(
+					"a request to an unknown relay parent has been made: {:#?}",
+					self.relay_parent
+				);
+			},
+		}
+	}
+
+	fn reply_not_supported(self) {
+		self.tx
+			.send(Err(RuntimeApiError::NotSupported { runtime_api_name: "pvfs_require_precheck" }))
+			.unwrap();
+	}
+}
+
+#[must_use]
+struct ExpectCandidatePrecheck {
+	relay_parent: Hash,
+	validation_code_hash: ValidationCodeHash,
+	tx: oneshot::Sender<PreCheckOutcome>,
+}
+
+impl ExpectCandidatePrecheck {
+	fn reply(self, outcome: PreCheckOutcome) {
+		self.tx.send(outcome).unwrap();
+	}
+}
+
+#[must_use]
+struct ExpectSubmitVote {
+	relay_parent: Hash,
+	stmt: PvfCheckStatement,
+	tx: oneshot::Sender<Result<(), RuntimeApiError>>,
+}
+
+impl ExpectSubmitVote {
+	fn reply_ok(self) {
+		self.tx.send(Ok(())).unwrap();
+	}
+}
+
+fn test_harness(test: impl FnOnce(TestState, VirtualOverseer) -> BoxFuture<'static, ()>) {
+	let pool = TaskExecutor::new();
+	let (ctx, handle) = make_subsystem_context::<PvfCheckerMessage, _>(pool.clone());
+	let keystore = Arc::new(sc_keystore::LocalKeystore::in_memory());
+
+	// Add OUR_VALIDATOR (which is Alice) to the keystore.
+	Keystore::sr25519_generate_new(&*keystore, ValidatorId::ID, Some(&OUR_VALIDATOR.to_seed()))
+		.expect("Generating keys for our node failed");
+
+	let subsystem_task = crate::run(ctx, keystore, crate::Metrics::default()).map(|x| x.unwrap());
+
+	let test_state = TestState::new();
+	let test_task = test(test_state, handle);
+
+	futures::executor::block_on(future::join(subsystem_task, test_task));
+}
+
+#[test]
+fn concludes_correctly() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn reacts_to_new_pvfs_in_heads() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			let block = FakeLeaf::new(dummy_hash(), 1, vec![dummy_validation_code_hash(1)]);
+
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					block.clone(),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			let pre_check = test_state.expect_candidate_precheck(&mut handle).await;
+			assert_eq!(pre_check.relay_parent, block.block_hash);
+			pre_check.reply(PreCheckOutcome::Valid);
+
+			let vote = test_state.expect_submit_vote(&mut handle).await;
+			assert_eq!(vote.relay_parent, block.block_hash);
+			assert_eq!(vote.stmt.accept, true);
+			assert_eq!(vote.stmt.session_index, 2);
+			assert_eq!(vote.stmt.validator_index, 0.into());
+			assert_eq!(vote.stmt.subject, dummy_validation_code_hash(1));
+			vote.reply_ok();
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn no_new_session_no_validators_request() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					FakeLeaf::new(dummy_hash(), 1, vec![]),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			test_state
+				.activate_leaf(&mut handle, FakeLeaf::new(dummy_hash(), 2, vec![]))
+				.await;
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn activation_of_descendant_leaves_pvfs_in_view() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			let block_1 = FakeLeaf::new(dummy_hash(), 1, vec![dummy_validation_code_hash(1)]);
+			let block_2 = block_1.descendant(vec![dummy_validation_code_hash(1)]);
+
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					block_1.clone(),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			test_state
+				.expect_candidate_precheck(&mut handle)
+				.await
+				.reply(PreCheckOutcome::Valid);
+			test_state.expect_submit_vote(&mut handle).await.reply_ok();
+
+			// Now we deactivate the first block and activate it's descendant.
+			test_state
+				.active_leaves_update(
+					&mut handle,
+					Some(block_2),
+					None, // no new session started
+					&[block_1.block_hash],
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn reactivating_pvf_leads_to_second_check() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			let pvf = dummy_validation_code_hash(1);
+			let block_1 = FakeLeaf::new(dummy_hash(), 1, vec![pvf]);
+			let block_2 = block_1.descendant(vec![]);
+			let block_3 = block_2.descendant(vec![pvf]);
+
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					block_1.clone(),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+			test_state
+				.expect_candidate_precheck(&mut handle)
+				.await
+				.reply(PreCheckOutcome::Valid);
+			test_state.expect_submit_vote(&mut handle).await.reply_ok();
+
+			// Now activate a descendant leaf, where the PVF is not present.
+			test_state
+				.active_leaves_update(
+					&mut handle,
+					Some(block_2.clone()),
+					None,
+					&[block_1.block_hash],
+				)
+				.await;
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+
+			// Now the third block is activated, where the PVF is present.
+			test_state.activate_leaf(&mut handle, block_3).await;
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state
+				.expect_candidate_precheck(&mut handle)
+				.await
+				.reply(PreCheckOutcome::Valid);
+
+			// We do not vote here, because the PVF was already voted on within this session.
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn dont_double_vote_for_pvfs_in_view() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			let pvf = dummy_validation_code_hash(1);
+			let block_1_1 = FakeLeaf::new([1; 32].into(), 1, vec![pvf]);
+			let block_2_1 = FakeLeaf::new([2; 32].into(), 1, vec![pvf]);
+			let block_1_2 = block_1_1.descendant(vec![pvf]);
+
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					block_1_1.clone(),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			// Pre-checking will take quite some time.
+			let pre_check = test_state.expect_candidate_precheck(&mut handle).await;
+
+			// Activate a sibiling leaf, has the same PVF.
+			test_state.activate_leaf(&mut handle, block_2_1).await;
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+
+			// Now activate a descendant leaf with the same PVF.
+			test_state
+				.active_leaves_update(
+					&mut handle,
+					Some(block_1_2.clone()),
+					None,
+					&[block_1_1.block_hash],
+				)
+				.await;
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+
+			// Now finish the pre-checking request.
+			pre_check.reply(PreCheckOutcome::Valid);
+			test_state.expect_submit_vote(&mut handle).await.reply_ok();
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn judgements_come_out_of_order() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			let pvf_1 = dummy_validation_code_hash(1);
+			let pvf_2 = dummy_validation_code_hash(2);
+
+			let block_1 = FakeLeaf::new([1; 32].into(), 1, vec![pvf_1]);
+			let block_2 = FakeLeaf::new([2; 32].into(), 1, vec![pvf_2]);
+
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					block_1.clone(),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			let pre_check_1 = test_state.expect_candidate_precheck(&mut handle).await;
+
+			// Activate a sibiling leaf, has the second PVF.
+			test_state.activate_leaf(&mut handle, block_2).await;
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+
+			let pre_check_2 = test_state.expect_candidate_precheck(&mut handle).await;
+
+			// Resolve the PVF pre-checks out of order.
+			pre_check_2.reply(PreCheckOutcome::Valid);
+			pre_check_1.reply(PreCheckOutcome::Invalid);
+
+			// Catch the vote for the second PVF.
+			let vote_2 = test_state.expect_submit_vote(&mut handle).await;
+			assert_eq!(vote_2.stmt.accept, true);
+			assert_eq!(vote_2.stmt.subject, pvf_2.clone());
+			vote_2.reply_ok();
+
+			// Catch the vote for the first PVF.
+			let vote_1 = test_state.expect_submit_vote(&mut handle).await;
+			assert_eq!(vote_1.stmt.accept, false);
+			assert_eq!(vote_1.stmt.subject, pvf_1.clone());
+			vote_1.reply_ok();
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn dont_vote_until_a_validator() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					FakeLeaf::new(dummy_hash(), 1, vec![dummy_validation_code_hash(1)]),
+					StartsNewSession { session_index: 2, validators: vec![Sr25519Keyring::Bob] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			test_state
+				.expect_candidate_precheck(&mut handle)
+				.await
+				.reply(PreCheckOutcome::Invalid);
+
+			// Now a leaf brings a new session. In this session our validator comes into the active
+			// set. That means it will cast a vote for each judgement it has.
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					FakeLeaf::new(dummy_hash(), 2, vec![dummy_validation_code_hash(1)]),
+					StartsNewSession {
+						session_index: 3,
+						validators: vec![Sr25519Keyring::Bob, OUR_VALIDATOR],
+					},
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+			let vote = test_state.expect_submit_vote(&mut handle).await;
+			assert_eq!(vote.stmt.accept, false);
+			assert_eq!(vote.stmt.session_index, 3);
+			assert_eq!(vote.stmt.validator_index, 1.into());
+			assert_eq!(vote.stmt.subject, dummy_validation_code_hash(1));
+			vote.reply_ok();
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn resign_on_session_change() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			let pvf_1 = dummy_validation_code_hash(1);
+			let pvf_2 = dummy_validation_code_hash(2);
+
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					FakeLeaf::new(dummy_hash(), 1, vec![pvf_1, pvf_2]),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			let pre_check_1 = test_state.expect_candidate_precheck(&mut handle).await;
+			assert_eq!(pre_check_1.validation_code_hash, pvf_1);
+			pre_check_1.reply(PreCheckOutcome::Valid);
+			let pre_check_2 = test_state.expect_candidate_precheck(&mut handle).await;
+			assert_eq!(pre_check_2.validation_code_hash, pvf_2);
+			pre_check_2.reply(PreCheckOutcome::Invalid);
+
+			test_state.expect_submit_vote(&mut handle).await.reply_ok();
+			test_state.expect_submit_vote(&mut handle).await.reply_ok();
+
+			// So far so good. Now we change the session.
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					FakeLeaf::new(dummy_hash(), 2, vec![pvf_1, pvf_2]),
+					StartsNewSession { session_index: 3, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			// The votes should be re-signed and re-submitted.
+			let mut statements = Vec::new();
+			let vote_1 = test_state.expect_submit_vote(&mut handle).await;
+			statements.push(vote_1.stmt.clone());
+			vote_1.reply_ok();
+			let vote_2 = test_state.expect_submit_vote(&mut handle).await;
+			statements.push(vote_2.stmt.clone());
+			vote_2.reply_ok();
+
+			// Find and check the votes.
+			// Unfortunately, the order of revoting is not deterministic so we have to resort to
+			// a bit of trickery.
+			assert_eq!(statements.iter().find(|s| s.subject == pvf_1).unwrap().accept, true);
+			assert_eq!(statements.iter().find(|s| s.subject == pvf_2).unwrap().accept, false);
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn dont_resign_if_not_us() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			let pvf_1 = dummy_validation_code_hash(1);
+			let pvf_2 = dummy_validation_code_hash(2);
+
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					FakeLeaf::new(dummy_hash(), 1, vec![pvf_1, pvf_2]),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			let pre_check_1 = test_state.expect_candidate_precheck(&mut handle).await;
+			assert_eq!(pre_check_1.validation_code_hash, pvf_1);
+			pre_check_1.reply(PreCheckOutcome::Valid);
+			let pre_check_2 = test_state.expect_candidate_precheck(&mut handle).await;
+			assert_eq!(pre_check_2.validation_code_hash, pvf_2);
+			pre_check_2.reply(PreCheckOutcome::Invalid);
+
+			test_state.expect_submit_vote(&mut handle).await.reply_ok();
+			test_state.expect_submit_vote(&mut handle).await.reply_ok();
+
+			// So far so good. Now we change the session.
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					FakeLeaf::new(dummy_hash(), 2, vec![pvf_1, pvf_2]),
+					StartsNewSession {
+						session_index: 3,
+						// not us
+						validators: vec![Sr25519Keyring::Bob],
+					},
+				)
+				.await;
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			// We do not expect any votes to be re-signed.
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn api_not_supported() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					FakeLeaf::new(dummy_hash(), 1, vec![dummy_validation_code_hash(1)]),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_not_supported();
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn not_supported_api_becomes_supported() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					FakeLeaf::new(dummy_hash(), 1, vec![dummy_validation_code_hash(1)]),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_not_supported();
+
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					FakeLeaf::new(dummy_hash(), 1, vec![dummy_validation_code_hash(1)]),
+					StartsNewSession { session_index: 3, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+			test_state
+				.expect_candidate_precheck(&mut handle)
+				.await
+				.reply(PreCheckOutcome::Valid);
+			test_state.expect_submit_vote(&mut handle).await.reply_ok();
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn unexpected_pvf_check_judgement() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			let block_1 = FakeLeaf::new(dummy_hash(), 1, vec![dummy_validation_code_hash(1)]);
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					block_1.clone(),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			// Catch the pre-check request, but don't reply just yet.
+			let pre_check = test_state.expect_candidate_precheck(&mut handle).await;
+
+			// Now deactivate the leaf and reply to the precheck request.
+			test_state.deactivate_leaves(&mut handle, &[block_1.block_hash]).await;
+			pre_check.reply(PreCheckOutcome::Invalid);
+
+			// the subsystem must remain silent.
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+// Check that we do not abstain for a nondeterministic failure. Currently, this means the behavior
+// is the same as if the pre-check returned `PreCheckOutcome::Invalid`.
+#[test]
+fn dont_abstain_for_nondeterministic_pvfcheck_failure() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			let block_1 = FakeLeaf::new(dummy_hash(), 1, vec![dummy_validation_code_hash(1)]);
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					block_1.clone(),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			// Catch the pre-check request, but don't reply just yet.
+			let pre_check = test_state.expect_candidate_precheck(&mut handle).await;
+
+			// Now deactivate the leaf and reply to the precheck request.
+			test_state.deactivate_leaves(&mut handle, &[block_1.block_hash]).await;
+			pre_check.reply(PreCheckOutcome::Failed);
+
+			// the subsystem must remain silent.
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}