feat: initialize Kurdistan SDK - independent fork of Polkadot SDK

pezkuwi/node/collation-generation/Cargo.toml

@@ -0,0 +1,46 @@
+[package]
+name = "pezkuwi-node-collation-generation"
+version = "7.0.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+description = "Collator-side subsystem that handles incoming candidate submissions from the teyrchain."
+homepage.workspace = true
+repository.workspace = true
+
+[lints]
+workspace = true
+
+[dependencies]
+codec = { features = ["bit-vec", "derive"], workspace = true }
+futures = { workspace = true }
+gum = { workspace = true, default-features = true }
+pezkuwi-erasure-coding = { workspace = true, default-features = true }
+pezkuwi-node-primitives = { workspace = true, default-features = true }
+pezkuwi-node-subsystem = { workspace = true, default-features = true }
+pezkuwi-node-subsystem-util = { workspace = true, default-features = true }
+pezkuwi-primitives = { workspace = true, default-features = true }
+schnellru = { workspace = true }
+sp-core = { workspace = true, default-features = true }
+thiserror = { workspace = true }
+
+[dev-dependencies]
+assert_matches = { workspace = true }
+pezkuwi-node-subsystem-test-helpers = { workspace = true }
+pezkuwi-primitives = { workspace = true, features = ["test"] }
+pezkuwi-primitives-test-helpers = { workspace = true }
+rstest = { workspace = true }
+sp-keyring = { workspace = true, default-features = true }
+
+[features]
+runtime-benchmarks = [
+	"gum/runtime-benchmarks",
+	"pezkuwi-erasure-coding/runtime-benchmarks",
+	"pezkuwi-node-primitives/runtime-benchmarks",
+	"pezkuwi-node-subsystem-test-helpers/runtime-benchmarks",
+	"pezkuwi-node-subsystem-util/runtime-benchmarks",
+	"pezkuwi-node-subsystem/runtime-benchmarks",
+	"pezkuwi-primitives-test-helpers/runtime-benchmarks",
+	"pezkuwi-primitives/runtime-benchmarks",
+	"sp-keyring/runtime-benchmarks",
+]

pezkuwi/node/collation-generation/src/error.rs

@@ -0,0 +1,42 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use pezkuwi_primitives::CommittedCandidateReceiptError;
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+pub enum Error {
+	#[error(transparent)]
+	Subsystem(#[from] pezkuwi_node_subsystem::SubsystemError),
+	#[error(transparent)]
+	OneshotRecv(#[from] futures::channel::oneshot::Canceled),
+	#[error(transparent)]
+	Runtime(#[from] pezkuwi_node_subsystem::errors::RuntimeApiError),
+	#[error(transparent)]
+	Util(#[from] pezkuwi_node_subsystem_util::Error),
+	#[error(transparent)]
+	UtilRuntime(#[from] pezkuwi_node_subsystem_util::runtime::Error),
+	#[error(transparent)]
+	Erasure(#[from] pezkuwi_erasure_coding::Error),
+	#[error("Collation submitted before initialization")]
+	SubmittedBeforeInit,
+	#[error("V2 core index check failed: {0}")]
+	CandidateReceiptCheck(CommittedCandidateReceiptError),
+	#[error("PoV size {0} exceeded maximum size of {1}")]
+	POVSizeExceeded(usize, usize),
+}
+
+pub type Result<T> = std::result::Result<T, Error>;

pezkuwi/node/collation-generation/src/lib.rs

@@ -0,0 +1,637 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! The collation generation subsystem is the interface between pezkuwi and the collators.
+//!
+//! # Protocol
+//!
+//! On every `ActiveLeavesUpdate`:
+//!
+//! * If there is no collation generation config, ignore.
+//! * Otherwise, for each `activated` head in the update:
+//!   * Determine if the para is scheduled on any core by fetching the `availability_cores` Runtime
+//!     API.
+//!   * Use the Runtime API subsystem to fetch the full validation data.
+//!   * Invoke the `collator`, and use its outputs to produce a
+//!     [`pezkuwi_primitives::CandidateReceiptV2`], signed with the configuration's `key`.
+//!   * Dispatch a [`CollatorProtocolMessage::DistributeCollation`]`(receipt, pov)`.
+
+#![deny(missing_docs)]
+
+use codec::Encode;
+use error::{Error, Result};
+use futures::{channel::oneshot, future::FutureExt, select};
+use pezkuwi_node_primitives::{
+	AvailableData, Collation, CollationGenerationConfig, CollationSecondedSignal, PoV,
+	SubmitCollationParams,
+};
+use pezkuwi_node_subsystem::{
+	messages::{CollationGenerationMessage, CollatorProtocolMessage, RuntimeApiMessage},
+	overseer, ActiveLeavesUpdate, FromOrchestra, OverseerSignal, SpawnedSubsystem,
+	SubsystemContext, SubsystemError, SubsystemResult, SubsystemSender,
+};
+use pezkuwi_node_subsystem_util::{
+	request_claim_queue, request_persisted_validation_data, request_session_index_for_child,
+	request_validation_code_hash, request_validators, runtime::ClaimQueueSnapshot,
+};
+use pezkuwi_primitives::{
+	transpose_claim_queue, CandidateCommitments, CandidateDescriptorV2,
+	CommittedCandidateReceiptV2, CoreIndex, Hash, Id as ParaId, OccupiedCoreAssumption,
+	PersistedValidationData, SessionIndex, TransposedClaimQueue, ValidationCodeHash,
+};
+use schnellru::{ByLength, LruMap};
+use std::{collections::HashSet, sync::Arc};
+
+mod error;
+
+#[cfg(test)]
+mod tests;
+
+mod metrics;
+use self::metrics::Metrics;
+
+const LOG_TARGET: &'static str = "teyrchain::collation-generation";
+
+/// Collation Generation Subsystem
+pub struct CollationGenerationSubsystem {
+	config: Option<Arc<CollationGenerationConfig>>,
+	session_info_cache: SessionInfoCache,
+	metrics: Metrics,
+}
+
+#[overseer::contextbounds(CollationGeneration, prefix = self::overseer)]
+impl CollationGenerationSubsystem {
+	/// Create a new instance of the `CollationGenerationSubsystem`.
+	pub fn new(metrics: Metrics) -> Self {
+		Self { config: None, metrics, session_info_cache: SessionInfoCache::new() }
+	}
+
+	/// Run this subsystem
+	///
+	/// Conceptually, this is very simple: it just loops forever.
+	///
+	/// - On incoming overseer messages, it starts or stops jobs as appropriate.
+	/// - On other incoming messages, if they can be converted into `Job::ToJob` and include a hash,
+	///   then they're forwarded to the appropriate individual job.
+	/// - On outgoing messages from the jobs, it forwards them to the overseer.
+	///
+	/// If `err_tx` is not `None`, errors are forwarded onto that channel as they occur.
+	/// Otherwise, most are logged and then discarded.
+	async fn run<Context>(mut self, mut ctx: Context) {
+		loop {
+			select! {
+				incoming = ctx.recv().fuse() => {
+					if self.handle_incoming::<Context>(incoming, &mut ctx).await {
+						break;
+					}
+				},
+			}
+		}
+	}
+
+	// handle an incoming message. return true if we should break afterwards.
+	// note: this doesn't strictly need to be a separate function; it's more an administrative
+	// function so that we don't clutter the run loop. It could in principle be inlined directly
+	// into there. it should hopefully therefore be ok that it's an async function mutably borrowing
+	// self.
+	async fn handle_incoming<Context>(
+		&mut self,
+		incoming: SubsystemResult<FromOrchestra<<Context as SubsystemContext>::Message>>,
+		ctx: &mut Context,
+	) -> bool {
+		match incoming {
+			Ok(FromOrchestra::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate {
+				activated,
+				..
+			}))) => {
+				if let Err(err) = self.handle_new_activation(activated.map(|v| v.hash), ctx).await {
+					gum::warn!(target: LOG_TARGET, err = ?err, "failed to handle new activation");
+				}
+
+				false
+			},
+			Ok(FromOrchestra::Signal(OverseerSignal::Conclude)) => true,
+			Ok(FromOrchestra::Communication {
+				msg: CollationGenerationMessage::Initialize(config),
+			}) => {
+				if self.config.is_some() {
+					gum::error!(target: LOG_TARGET, "double initialization");
+				} else {
+					self.config = Some(Arc::new(config));
+				}
+				false
+			},
+			Ok(FromOrchestra::Communication {
+				msg: CollationGenerationMessage::Reinitialize(config),
+			}) => {
+				self.config = Some(Arc::new(config));
+				false
+			},
+			Ok(FromOrchestra::Communication {
+				msg: CollationGenerationMessage::SubmitCollation(params),
+			}) => {
+				if let Err(err) = self.handle_submit_collation(params, ctx).await {
+					gum::error!(target: LOG_TARGET, ?err, "Failed to submit collation");
+				}
+
+				false
+			},
+			Ok(FromOrchestra::Signal(OverseerSignal::BlockFinalized(..))) => false,
+			Err(err) => {
+				gum::error!(
+					target: LOG_TARGET,
+					err = ?err,
+					"error receiving message from subsystem context: {:?}",
+					err
+				);
+				true
+			},
+		}
+	}
+
+	async fn handle_submit_collation<Context>(
+		&mut self,
+		params: SubmitCollationParams,
+		ctx: &mut Context,
+	) -> Result<()> {
+		let Some(config) = &self.config else {
+			return Err(Error::SubmittedBeforeInit);
+		};
+		let _timer = self.metrics.time_submit_collation();
+
+		let SubmitCollationParams {
+			relay_parent,
+			collation,
+			parent_head,
+			validation_code_hash,
+			result_sender,
+			core_index,
+		} = params;
+
+		let mut validation_data = match request_persisted_validation_data(
+			relay_parent,
+			config.para_id,
+			OccupiedCoreAssumption::TimedOut,
+			ctx.sender(),
+		)
+		.await
+		.await??
+		{
+			Some(v) => v,
+			None => {
+				gum::debug!(
+					target: LOG_TARGET,
+					relay_parent = ?relay_parent,
+					our_para = %config.para_id,
+					"No validation data for para - does it exist at this relay-parent?",
+				);
+				return Ok(());
+			},
+		};
+
+		// We need to swap the parent-head data, but all other fields here will be correct.
+		validation_data.parent_head = parent_head;
+
+		let claim_queue = request_claim_queue(relay_parent, ctx.sender()).await.await??;
+
+		let session_index =
+			request_session_index_for_child(relay_parent, ctx.sender()).await.await??;
+
+		let session_info =
+			self.session_info_cache.get(relay_parent, session_index, ctx.sender()).await?;
+		let collation = PreparedCollation {
+			collation,
+			relay_parent,
+			para_id: config.para_id,
+			validation_data,
+			validation_code_hash,
+			n_validators: session_info.n_validators,
+			core_index,
+			session_index,
+		};
+
+		construct_and_distribute_receipt(
+			collation,
+			ctx.sender(),
+			result_sender,
+			&mut self.metrics,
+			&transpose_claim_queue(claim_queue),
+		)
+		.await?;
+
+		Ok(())
+	}
+
+	async fn handle_new_activation<Context>(
+		&mut self,
+		maybe_activated: Option<Hash>,
+		ctx: &mut Context,
+	) -> Result<()> {
+		let Some(config) = &self.config else {
+			return Ok(());
+		};
+
+		let Some(relay_parent) = maybe_activated else { return Ok(()) };
+
+		// If there is no collation function provided, bail out early.
+		// Important: Lookahead collator and slot based collator do not use `CollatorFn`.
+		if config.collator.is_none() {
+			return Ok(());
+		}
+
+		let para_id = config.para_id;
+
+		let _timer = self.metrics.time_new_activation();
+
+		let session_index =
+			request_session_index_for_child(relay_parent, ctx.sender()).await.await??;
+
+		let session_info =
+			self.session_info_cache.get(relay_parent, session_index, ctx.sender()).await?;
+		let n_validators = session_info.n_validators;
+
+		let claim_queue =
+			ClaimQueueSnapshot::from(request_claim_queue(relay_parent, ctx.sender()).await.await??);
+
+		let assigned_cores = claim_queue
+			.iter_all_claims()
+			.filter_map(|(core_idx, para_ids)| {
+				para_ids.iter().any(|&para_id| para_id == config.para_id).then_some(*core_idx)
+			})
+			.collect::<Vec<_>>();
+
+		// Nothing to do if no core is assigned to us at any depth.
+		if assigned_cores.is_empty() {
+			return Ok(());
+		}
+
+		// We are being very optimistic here, but one of the cores could be pending availability
+		// for some more blocks, or even time out. We assume all cores are being freed.
+
+		let mut validation_data = match request_persisted_validation_data(
+			relay_parent,
+			para_id,
+			// Just use included assumption always. If there are no pending candidates it's a
+			// no-op.
+			OccupiedCoreAssumption::Included,
+			ctx.sender(),
+		)
+		.await
+		.await??
+		{
+			Some(v) => v,
+			None => {
+				gum::debug!(
+					target: LOG_TARGET,
+					relay_parent = ?relay_parent,
+					our_para = %para_id,
+					"validation data is not available",
+				);
+				return Ok(());
+			},
+		};
+
+		let validation_code_hash = match request_validation_code_hash(
+			relay_parent,
+			para_id,
+			// Just use included assumption always. If there are no pending candidates it's a
+			// no-op.
+			OccupiedCoreAssumption::Included,
+			ctx.sender(),
+		)
+		.await
+		.await??
+		{
+			Some(v) => v,
+			None => {
+				gum::debug!(
+					target: LOG_TARGET,
+					relay_parent = ?relay_parent,
+					our_para = %para_id,
+					"validation code hash is not found.",
+				);
+				return Ok(());
+			},
+		};
+
+		let task_config = config.clone();
+		let metrics = self.metrics.clone();
+		let mut task_sender = ctx.sender().clone();
+
+		ctx.spawn(
+			"chained-collation-builder",
+			Box::pin(async move {
+				let transposed_claim_queue = transpose_claim_queue(claim_queue.0.clone());
+
+				// Track used core indexes not to submit collations on the same core.
+				let mut used_cores = HashSet::new();
+
+				for i in 0..assigned_cores.len() {
+					// Get the collation.
+					let collator_fn = match task_config.collator.as_ref() {
+						Some(x) => x,
+						None => return,
+					};
+
+					let (collation, result_sender) =
+						match collator_fn(relay_parent, &validation_data).await {
+							Some(collation) => collation.into_inner(),
+							None => {
+								gum::debug!(
+									target: LOG_TARGET,
+									?para_id,
+									"collator returned no collation on collate",
+								);
+								return;
+							},
+						};
+
+					// Use the core_selector method from CandidateCommitments to extract
+					// CoreSelector and ClaimQueueOffset.
+					let mut commitments = CandidateCommitments::default();
+					commitments.upward_messages = collation.upward_messages.clone();
+
+					let ump_signals = match commitments.ump_signals() {
+						Ok(signals) => signals,
+						Err(err) => {
+							gum::debug!(
+								target: LOG_TARGET,
+								?para_id,
+								"error processing UMP signals: {}",
+								err
+							);
+							return;
+						},
+					};
+
+					let (cs_index, cq_offset) = ump_signals
+						.core_selector()
+						.map(|(cs_index, cq_offset)| (cs_index.0 as usize, cq_offset.0 as usize))
+						.unwrap_or((i, 0));
+
+					// Identify the cores to build collations on using the given claim queue offset.
+					let cores_to_build_on = claim_queue
+						.iter_claims_at_depth(cq_offset)
+						.filter_map(|(core_idx, para_id)| {
+							(para_id == task_config.para_id).then_some(core_idx)
+						})
+						.collect::<Vec<_>>();
+
+					if cores_to_build_on.is_empty() {
+						gum::debug!(
+							target: LOG_TARGET,
+							?para_id,
+							"no core is assigned to para at depth {}",
+							cq_offset,
+						);
+						return;
+					}
+
+					let descriptor_core_index =
+						cores_to_build_on[cs_index % cores_to_build_on.len()];
+
+					// Ensure the core index has not been used before.
+					if used_cores.contains(&descriptor_core_index.0) {
+						gum::warn!(
+							target: LOG_TARGET,
+							?para_id,
+							"teyrchain repeatedly selected the same core index: {}",
+							descriptor_core_index.0,
+						);
+						return;
+					}
+
+					used_cores.insert(descriptor_core_index.0);
+					gum::trace!(
+						target: LOG_TARGET,
+						?para_id,
+						"selected core index: {}",
+						descriptor_core_index.0,
+					);
+
+					// Distribute the collation.
+					let parent_head = collation.head_data.clone();
+					if let Err(err) = construct_and_distribute_receipt(
+						PreparedCollation {
+							collation,
+							para_id,
+							relay_parent,
+							validation_data: validation_data.clone(),
+							validation_code_hash,
+							n_validators,
+							core_index: descriptor_core_index,
+							session_index,
+						},
+						&mut task_sender,
+						result_sender,
+						&metrics,
+						&transposed_claim_queue,
+					)
+					.await
+					{
+						gum::error!(
+							target: LOG_TARGET,
+							"Failed to construct and distribute collation: {}",
+							err
+						);
+						return;
+					}
+
+					// Chain the collations. All else stays the same as we build the chained
+					// collation on same relay parent.
+					validation_data.parent_head = parent_head;
+				}
+			}),
+		)?;
+
+		Ok(())
+	}
+}
+
+#[overseer::subsystem(CollationGeneration, error=SubsystemError, prefix=self::overseer)]
+impl<Context> CollationGenerationSubsystem {
+	fn start(self, ctx: Context) -> SpawnedSubsystem {
+		let future = async move {
+			self.run(ctx).await;
+			Ok(())
+		}
+		.boxed();
+
+		SpawnedSubsystem { name: "collation-generation-subsystem", future }
+	}
+}
+
+#[derive(Clone)]
+struct PerSessionInfo {
+	n_validators: usize,
+}
+
+struct SessionInfoCache(LruMap<SessionIndex, PerSessionInfo>);
+
+impl SessionInfoCache {
+	fn new() -> Self {
+		Self(LruMap::new(ByLength::new(2)))
+	}
+
+	async fn get<Sender: SubsystemSender<RuntimeApiMessage>>(
+		&mut self,
+		relay_parent: Hash,
+		session_index: SessionIndex,
+		sender: &mut Sender,
+	) -> Result<PerSessionInfo> {
+		if let Some(info) = self.0.get(&session_index) {
+			return Ok(info.clone());
+		}
+
+		let n_validators =
+			request_validators(relay_parent, &mut sender.clone()).await.await??.len();
+
+		let info = PerSessionInfo { n_validators };
+		self.0.insert(session_index, info);
+		Ok(self.0.get(&session_index).expect("Just inserted").clone())
+	}
+}
+
+struct PreparedCollation {
+	collation: Collation,
+	para_id: ParaId,
+	relay_parent: Hash,
+	validation_data: PersistedValidationData,
+	validation_code_hash: ValidationCodeHash,
+	n_validators: usize,
+	core_index: CoreIndex,
+	session_index: SessionIndex,
+}
+
+/// Takes a prepared collation, along with its context, and produces a candidate receipt
+/// which is distributed to validators.
+async fn construct_and_distribute_receipt(
+	collation: PreparedCollation,
+	sender: &mut impl overseer::CollationGenerationSenderTrait,
+	result_sender: Option<oneshot::Sender<CollationSecondedSignal>>,
+	metrics: &Metrics,
+	transposed_claim_queue: &TransposedClaimQueue,
+) -> Result<()> {
+	let PreparedCollation {
+		collation,
+		para_id,
+		relay_parent,
+		validation_data,
+		validation_code_hash,
+		n_validators,
+		core_index,
+		session_index,
+	} = collation;
+
+	let persisted_validation_data_hash = validation_data.hash();
+	let parent_head_data = validation_data.parent_head.clone();
+	let parent_head_data_hash = validation_data.parent_head.hash();
+
+	// Apply compression to the block data.
+	let pov = {
+		let pov = collation.proof_of_validity.into_compressed();
+		let encoded_size = pov.encoded_size();
+
+		// As long as `POV_BOMB_LIMIT` is at least `max_pov_size`, this ensures
+		// that honest collators never produce a PoV which is uncompressed.
+		//
+		// As such, honest collators never produce an uncompressed PoV which starts with
+		// a compression magic number, which would lead validators to reject the collation.
+		if encoded_size > validation_data.max_pov_size as usize {
+			return Err(Error::POVSizeExceeded(encoded_size, validation_data.max_pov_size as usize));
+		}
+
+		pov
+	};
+
+	let pov_hash = pov.hash();
+
+	let erasure_root = erasure_root(n_validators, validation_data, pov.clone())?;
+
+	let commitments = CandidateCommitments {
+		upward_messages: collation.upward_messages,
+		horizontal_messages: collation.horizontal_messages,
+		new_validation_code: collation.new_validation_code,
+		head_data: collation.head_data,
+		processed_downward_messages: collation.processed_downward_messages,
+		hrmp_watermark: collation.hrmp_watermark,
+	};
+
+	let receipt = {
+		let ccr = CommittedCandidateReceiptV2 {
+			descriptor: CandidateDescriptorV2::new(
+				para_id,
+				relay_parent,
+				core_index,
+				session_index,
+				persisted_validation_data_hash,
+				pov_hash,
+				erasure_root,
+				commitments.head_data.hash(),
+				validation_code_hash,
+			),
+			commitments: commitments.clone(),
+		};
+
+		ccr.parse_ump_signals(&transposed_claim_queue)
+			.map_err(Error::CandidateReceiptCheck)?;
+
+		ccr.to_plain()
+	};
+
+	gum::debug!(
+		target: LOG_TARGET,
+		candidate_hash = ?receipt.hash(),
+		?pov_hash,
+		?relay_parent,
+		para_id = %para_id,
+		?core_index,
+		"Candidate generated",
+	);
+	gum::trace!(
+		target: LOG_TARGET,
+		?commitments,
+		candidate_hash = ?receipt.hash(),
+		"Candidate commitments",
+	);
+
+	metrics.on_collation_generated();
+
+	sender
+		.send_message(CollatorProtocolMessage::DistributeCollation {
+			candidate_receipt: receipt,
+			parent_head_data_hash,
+			pov,
+			parent_head_data,
+			result_sender,
+			core_index,
+		})
+		.await;
+
+	Ok(())
+}
+
+fn erasure_root(
+	n_validators: usize,
+	persisted_validation: PersistedValidationData,
+	pov: PoV,
+) -> Result<Hash> {
+	let available_data =
+		AvailableData { validation_data: persisted_validation, pov: Arc::new(pov) };
+
+	let chunks = pezkuwi_erasure_coding::obtain_chunks_v1(n_validators, &available_data)?;
+	Ok(pezkuwi_erasure_coding::branches(&chunks).root())
+}

pezkuwi/node/collation-generation/src/metrics.rs

@@ -0,0 +1,75 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use pezkuwi_node_subsystem_util::metrics::{self, prometheus};
+
+#[derive(Clone)]
+pub(crate) struct MetricsInner {
+	pub(crate) collations_generated_total: prometheus::Counter<prometheus::U64>,
+	pub(crate) new_activation: prometheus::Histogram,
+	pub(crate) submit_collation: prometheus::Histogram,
+}
+
+/// `CollationGenerationSubsystem` metrics.
+#[derive(Default, Clone)]
+pub struct Metrics(pub(crate) Option<MetricsInner>);
+
+impl Metrics {
+	pub fn on_collation_generated(&self) {
+		if let Some(metrics) = &self.0 {
+			metrics.collations_generated_total.inc();
+		}
+	}
+
+	/// Provide a timer for new activations which updates on drop.
+	pub fn time_new_activation(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.new_activation.start_timer())
+	}
+
+	/// Provide a timer for submitting a collation which updates on drop.
+	pub fn time_submit_collation(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.submit_collation.start_timer())
+	}
+}
+
+impl metrics::Metrics for Metrics {
+	fn try_register(registry: &prometheus::Registry) -> Result<Self, prometheus::PrometheusError> {
+		let metrics = MetricsInner {
+			collations_generated_total: prometheus::register(
+				prometheus::Counter::new(
+					"pezkuwi_teyrchain_collations_generated_total",
+					"Number of collations generated.",
+				)?,
+				registry,
+			)?,
+			new_activation: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_collation_generation_new_activations",
+					"Time spent within fn handle_new_activation",
+				))?,
+				registry,
+			)?,
+			submit_collation: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_collation_generation_submit_collation",
+					"Time spent preparing and submitting a collation to the network protocol",
+				))?,
+				registry,
+			)?,
+		};
+		Ok(Metrics(Some(metrics)))
+	}
+}

pezkuwi/node/collation-generation/src/tests.rs

@@ -0,0 +1,748 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use super::*;
+use assert_matches::assert_matches;
+use futures::{self, Future, StreamExt};
+use pezkuwi_node_primitives::{
+	BlockData, Collation, CollationResult, CollatorFn, MaybeCompressedPoV, PoV,
+};
+use pezkuwi_node_subsystem::{
+	messages::{AllMessages, RuntimeApiMessage, RuntimeApiRequest},
+	ActivatedLeaf,
+};
+use pezkuwi_node_subsystem_test_helpers::TestSubsystemContextHandle;
+use pezkuwi_node_subsystem_util::TimeoutExt;
+use pezkuwi_primitives::{
+	CandidateDescriptorVersion, CandidateReceiptV2, ClaimQueueOffset, CollatorPair, CoreSelector,
+	PersistedValidationData, UMPSignal, UMP_SEPARATOR,
+};
+use pezkuwi_primitives_test_helpers::dummy_head_data;
+use rstest::rstest;
+use sp_core::Pair;
+use sp_keyring::sr25519::Keyring as Sr25519Keyring;
+use std::{
+	collections::{BTreeMap, VecDeque},
+	sync::Mutex,
+};
+
+type VirtualOverseer = TestSubsystemContextHandle<CollationGenerationMessage>;
+
+fn test_harness<T: Future<Output = VirtualOverseer>>(test: impl FnOnce(VirtualOverseer) -> T) {
+	let pool = sp_core::testing::TaskExecutor::new();
+	let (context, virtual_overseer) =
+		pezkuwi_node_subsystem_test_helpers::make_subsystem_context(pool);
+	let subsystem = async move {
+		let subsystem = crate::CollationGenerationSubsystem::new(Metrics::default());
+
+		subsystem.run(context).await;
+	};
+
+	let test_fut = test(virtual_overseer);
+
+	futures::pin_mut!(test_fut);
+	futures::executor::block_on(futures::future::join(
+		async move {
+			let mut virtual_overseer = test_fut.await;
+			// Ensure we have handled all responses.
+			if let Some(msg) = virtual_overseer.rx.next().timeout(TIMEOUT).await {
+				panic!("Did not handle all responses: {:?}", msg);
+			}
+			// Conclude.
+			virtual_overseer.send(FromOrchestra::Signal(OverseerSignal::Conclude)).await;
+		},
+		subsystem,
+	));
+}
+
+fn test_collation() -> Collation {
+	Collation {
+		upward_messages: Default::default(),
+		horizontal_messages: Default::default(),
+		new_validation_code: None,
+		head_data: dummy_head_data(),
+		proof_of_validity: MaybeCompressedPoV::Raw(PoV { block_data: BlockData(Vec::new()) }),
+		processed_downward_messages: 0_u32,
+		hrmp_watermark: 0_u32.into(),
+	}
+}
+
+struct CoreSelectorData {
+	// The core selector index.
+	index: u8,
+	// The increment value for the core selector index. Normally 1, but can be set to 0 or another
+	// value for testing scenarios where a teyrchain repeatedly selects the same core index.
+	increment_index_by: u8,
+	// The claim queue offset.
+	cq_offset: u8,
+}
+
+impl CoreSelectorData {
+	fn new(index: u8, increment_index_by: u8, cq_offset: u8) -> Self {
+		Self { index, increment_index_by, cq_offset }
+	}
+}
+
+struct State {
+	core_selector_data: Option<CoreSelectorData>,
+}
+
+impl State {
+	fn new(core_selector_data: Option<CoreSelectorData>) -> Self {
+		Self { core_selector_data }
+	}
+}
+
+struct TestCollator {
+	state: Arc<Mutex<State>>,
+}
+
+impl TestCollator {
+	fn new(core_selector_data: Option<CoreSelectorData>) -> Self {
+		Self { state: Arc::new(Mutex::new(State::new(core_selector_data))) }
+	}
+
+	pub fn create_collation_function(&self) -> CollatorFn {
+		let state = Arc::clone(&self.state);
+
+		Box::new(move |_relay_parent: Hash, _validation_data: &PersistedValidationData| {
+			let mut collation = test_collation();
+			let mut state_guard = state.lock().unwrap();
+
+			if let Some(core_selector_data) = &mut state_guard.core_selector_data {
+				collation.upward_messages.force_push(UMP_SEPARATOR);
+				collation.upward_messages.force_push(
+					UMPSignal::SelectCore(
+						CoreSelector(core_selector_data.index),
+						ClaimQueueOffset(core_selector_data.cq_offset),
+					)
+					.encode(),
+				);
+				core_selector_data.index += core_selector_data.increment_index_by;
+			}
+
+			async move { Some(CollationResult { collation, result_sender: None }) }.boxed()
+		})
+	}
+}
+
+const TIMEOUT: std::time::Duration = std::time::Duration::from_millis(2000);
+
+async fn overseer_recv(overseer: &mut VirtualOverseer) -> AllMessages {
+	overseer
+		.recv()
+		.timeout(TIMEOUT)
+		.await
+		.expect(&format!("{:?} is long enough to receive messages", TIMEOUT))
+}
+
+fn test_config<Id: Into<ParaId>>(
+	para_id: Id,
+	core_selector_data: Option<CoreSelectorData>,
+) -> CollationGenerationConfig {
+	let test_collator = TestCollator::new(core_selector_data);
+	CollationGenerationConfig {
+		key: CollatorPair::generate().0,
+		collator: Some(test_collator.create_collation_function()),
+		para_id: para_id.into(),
+	}
+}
+
+fn test_config_no_collator<Id: Into<ParaId>>(para_id: Id) -> CollationGenerationConfig {
+	CollationGenerationConfig {
+		key: CollatorPair::generate().0,
+		collator: None,
+		para_id: para_id.into(),
+	}
+}
+
+#[test]
+fn submit_collation_is_no_op_before_initialization() {
+	test_harness(|mut virtual_overseer| async move {
+		virtual_overseer
+			.send(FromOrchestra::Communication {
+				msg: CollationGenerationMessage::SubmitCollation(SubmitCollationParams {
+					relay_parent: Hash::repeat_byte(0),
+					collation: test_collation(),
+					parent_head: vec![1, 2, 3].into(),
+					validation_code_hash: Hash::repeat_byte(1).into(),
+					result_sender: None,
+					core_index: CoreIndex(0),
+				}),
+			})
+			.await;
+
+		virtual_overseer
+	});
+}
+
+#[test]
+fn submit_collation_leads_to_distribution() {
+	let relay_parent = Hash::repeat_byte(0);
+	let validation_code_hash = ValidationCodeHash::from(Hash::repeat_byte(42));
+	let parent_head = dummy_head_data();
+	let para_id = ParaId::from(5);
+	let expected_pvd = PersistedValidationData {
+		parent_head: parent_head.clone(),
+		relay_parent_number: 10,
+		relay_parent_storage_root: Hash::repeat_byte(1),
+		max_pov_size: 1024,
+	};
+
+	test_harness(|mut virtual_overseer| async move {
+		virtual_overseer
+			.send(FromOrchestra::Communication {
+				msg: CollationGenerationMessage::Initialize(test_config_no_collator(para_id)),
+			})
+			.await;
+
+		virtual_overseer
+			.send(FromOrchestra::Communication {
+				msg: CollationGenerationMessage::SubmitCollation(SubmitCollationParams {
+					relay_parent,
+					collation: test_collation(),
+					parent_head: dummy_head_data(),
+					validation_code_hash,
+					result_sender: None,
+					core_index: CoreIndex(0),
+				}),
+			})
+			.await;
+
+		helpers::handle_runtime_calls_on_submit_collation(
+			&mut virtual_overseer,
+			relay_parent,
+			para_id,
+			expected_pvd.clone(),
+			[(CoreIndex(0), VecDeque::from([para_id]))].into(),
+		)
+		.await;
+
+		assert_matches!(
+			overseer_recv(&mut virtual_overseer).await,
+			AllMessages::CollatorProtocol(CollatorProtocolMessage::DistributeCollation {
+				candidate_receipt,
+				parent_head_data_hash,
+				..
+			}) => {
+				let CandidateReceiptV2 { descriptor, .. } = candidate_receipt;
+				assert_eq!(parent_head_data_hash, parent_head.hash());
+				assert_eq!(descriptor.persisted_validation_data_hash(), expected_pvd.hash());
+				assert_eq!(descriptor.para_head(), dummy_head_data().hash());
+				assert_eq!(descriptor.validation_code_hash(), validation_code_hash);
+			}
+		);
+
+		virtual_overseer
+	});
+}
+
+#[test]
+fn distribute_collation_only_for_assigned_para_id_at_offset_0() {
+	let activated_hash: Hash = [1; 32].into();
+	let para_id = ParaId::from(5);
+
+	let claim_queue = (0..=5)
+		.into_iter()
+		// Set all cores assigned to para_id 5 at the second and third depths. This shouldn't
+		// matter.
+		.map(|idx| (CoreIndex(idx), VecDeque::from([ParaId::from(idx), para_id, para_id])))
+		.collect::<BTreeMap<_, _>>();
+
+	test_harness(|mut virtual_overseer| async move {
+		helpers::initialize_collator(&mut virtual_overseer, para_id, None).await;
+		helpers::activate_new_head(&mut virtual_overseer, activated_hash).await;
+		helpers::handle_runtime_calls_on_new_head_activation(
+			&mut virtual_overseer,
+			activated_hash,
+			claim_queue,
+		)
+		.await;
+
+		helpers::handle_cores_processing_for_a_leaf(
+			&mut virtual_overseer,
+			activated_hash,
+			para_id,
+			vec![5], // Only core 5 is assigned to paraid 5.
+		)
+		.await;
+
+		virtual_overseer
+	});
+}
+
+// There are variable number of cores assigned to the paraid.
+// On new head activation `CollationGeneration` should produce and distribute the right number of
+// new collations with proper assumption about the para candidate chain availability at next block.
+#[rstest]
+#[case(0)]
+#[case(1)]
+#[case(2)]
+#[case(3)]
+fn distribute_collation_with_elastic_scaling(#[case] total_cores: u32) {
+	let activated_hash: Hash = [1; 32].into();
+	let para_id = ParaId::from(5);
+
+	let claim_queue = (0..total_cores)
+		.into_iter()
+		.map(|idx| (CoreIndex(idx), VecDeque::from([para_id])))
+		.collect::<BTreeMap<_, _>>();
+
+	test_harness(|mut virtual_overseer| async move {
+		helpers::initialize_collator(&mut virtual_overseer, para_id, None).await;
+		helpers::activate_new_head(&mut virtual_overseer, activated_hash).await;
+		helpers::handle_runtime_calls_on_new_head_activation(
+			&mut virtual_overseer,
+			activated_hash,
+			claim_queue,
+		)
+		.await;
+
+		helpers::handle_cores_processing_for_a_leaf(
+			&mut virtual_overseer,
+			activated_hash,
+			para_id,
+			(0..total_cores).collect(),
+		)
+		.await;
+
+		virtual_overseer
+	});
+}
+
+// Tests when submission core indexes need to be selected using the core selectors provided in the
+// UMP signals. The core selector index is an increasing number that can start with a non-negative
+// value (even greater than the core index), but the collation generation protocol uses the
+// remainder to select the core. UMP signals may also contain a claim queue offset, based on which
+// we need to select the assigned core indexes for the para from that offset in the claim queue.
+#[rstest]
+#[case(1, 0, 0)]
+#[case(2, 0, 1)]
+fn distribute_collation_with_core_selectors(
+	#[case] total_cores: u32,
+	// The core selector index that will be obtained from the first collation.
+	#[case] init_cs_index: u8,
+	// Claim queue offset where the assigned cores will be stored.
+	#[case] cq_offset: u8,
+) {
+	let activated_hash: Hash = [1; 32].into();
+	let para_id = ParaId::from(5);
+	let other_para_id = ParaId::from(10);
+
+	let claim_queue = (0..total_cores)
+		.into_iter()
+		.map(|idx| {
+			// Set all cores assigned to para_id 5 at the cq_offset depth.
+			let mut vec = VecDeque::from(vec![other_para_id; cq_offset as usize]);
+			vec.push_back(para_id);
+			(CoreIndex(idx), vec)
+		})
+		.collect::<BTreeMap<_, _>>();
+
+	test_harness(|mut virtual_overseer| async move {
+		helpers::initialize_collator(
+			&mut virtual_overseer,
+			para_id,
+			Some(CoreSelectorData::new(init_cs_index, 1, cq_offset)),
+		)
+		.await;
+		helpers::activate_new_head(&mut virtual_overseer, activated_hash).await;
+		helpers::handle_runtime_calls_on_new_head_activation(
+			&mut virtual_overseer,
+			activated_hash,
+			claim_queue,
+		)
+		.await;
+
+		let mut cores_assigned = (0..total_cores).collect::<Vec<_>>();
+		if total_cores > 1 && init_cs_index > 0 {
+			// We need to rotate the list of cores because the first core selector index was
+			// non-zero, which should change the sequence of submissions. However, collations should
+			// still be submitted on all cores.
+			cores_assigned.rotate_left((init_cs_index as u32 % total_cores) as usize);
+		}
+		helpers::handle_cores_processing_for_a_leaf(
+			&mut virtual_overseer,
+			activated_hash,
+			para_id,
+			cores_assigned,
+		)
+		.await;
+
+		virtual_overseer
+	});
+}
+
+// Tests the behavior when a teyrchain repeatedly selects the same core index.
+// Ensures that the system handles this behavior correctly while maintaining expected functionality.
+#[rstest]
+#[case(3, 0, vec![0])]
+#[case(3, 1, vec![0, 1, 2])]
+#[case(3, 2, vec![0, 2, 1])]
+#[case(3, 3, vec![0])]
+#[case(3, 4, vec![0, 1, 2])]
+fn distribute_collation_with_repeated_core_selector_index(
+	#[case] total_cores: u32,
+	#[case] increment_cs_index_by: u8,
+	#[case] expected_selected_cores: Vec<u32>,
+) {
+	let activated_hash: Hash = [1; 32].into();
+	let para_id = ParaId::from(5);
+
+	let claim_queue = (0..total_cores)
+		.into_iter()
+		.map(|idx| (CoreIndex(idx), VecDeque::from([para_id])))
+		.collect::<BTreeMap<_, _>>();
+
+	test_harness(|mut virtual_overseer| async move {
+		helpers::initialize_collator(
+			&mut virtual_overseer,
+			para_id,
+			Some(CoreSelectorData::new(0, increment_cs_index_by, 0)),
+		)
+		.await;
+		helpers::activate_new_head(&mut virtual_overseer, activated_hash).await;
+		helpers::handle_runtime_calls_on_new_head_activation(
+			&mut virtual_overseer,
+			activated_hash,
+			claim_queue,
+		)
+		.await;
+
+		helpers::handle_cores_processing_for_a_leaf(
+			&mut virtual_overseer,
+			activated_hash,
+			para_id,
+			expected_selected_cores,
+		)
+		.await;
+
+		virtual_overseer
+	});
+}
+
+#[test]
+fn v2_receipts_failed_core_index_check() {
+	let relay_parent = Hash::repeat_byte(0);
+	let validation_code_hash = ValidationCodeHash::from(Hash::repeat_byte(42));
+	let parent_head = dummy_head_data();
+	let para_id = ParaId::from(5);
+	let expected_pvd = PersistedValidationData {
+		parent_head: parent_head.clone(),
+		relay_parent_number: 10,
+		relay_parent_storage_root: Hash::repeat_byte(1),
+		max_pov_size: 1024,
+	};
+
+	test_harness(|mut virtual_overseer| async move {
+		virtual_overseer
+			.send(FromOrchestra::Communication {
+				msg: CollationGenerationMessage::Initialize(test_config_no_collator(para_id)),
+			})
+			.await;
+
+		virtual_overseer
+			.send(FromOrchestra::Communication {
+				msg: CollationGenerationMessage::SubmitCollation(SubmitCollationParams {
+					relay_parent,
+					collation: test_collation(),
+					parent_head: dummy_head_data(),
+					validation_code_hash,
+					result_sender: None,
+					core_index: CoreIndex(0),
+				}),
+			})
+			.await;
+
+		helpers::handle_runtime_calls_on_submit_collation(
+			&mut virtual_overseer,
+			relay_parent,
+			para_id,
+			expected_pvd.clone(),
+			// Core index commitment is on core 0 but don't add any assignment for core 0.
+			[(CoreIndex(1), [para_id].into_iter().collect())].into_iter().collect(),
+		)
+		.await;
+
+		// No collation is distributed.
+
+		virtual_overseer
+	});
+}
+
+#[test]
+// Verify that an ApprovedPeer UMP signal does not break the subsystem (DistributeCollation is
+// sent), assuming CandidateReceiptV2 node feature is enabled.
+fn approved_peer_signal() {
+	let relay_parent = Hash::repeat_byte(0);
+	let validation_code_hash = ValidationCodeHash::from(Hash::repeat_byte(42));
+	let parent_head = dummy_head_data();
+	let para_id = ParaId::from(5);
+	let expected_pvd = PersistedValidationData {
+		parent_head: parent_head.clone(),
+		relay_parent_number: 10,
+		relay_parent_storage_root: Hash::repeat_byte(1),
+		max_pov_size: 1024,
+	};
+
+	test_harness(|mut virtual_overseer| async move {
+		virtual_overseer
+			.send(FromOrchestra::Communication {
+				msg: CollationGenerationMessage::Initialize(test_config_no_collator(para_id)),
+			})
+			.await;
+
+		let mut collation = test_collation();
+		collation.upward_messages.force_push(UMP_SEPARATOR);
+		collation
+			.upward_messages
+			.force_push(UMPSignal::ApprovedPeer(vec![1, 2, 3, 4, 5].try_into().unwrap()).encode());
+
+		virtual_overseer
+			.send(FromOrchestra::Communication {
+				msg: CollationGenerationMessage::SubmitCollation(SubmitCollationParams {
+					relay_parent,
+					collation,
+					parent_head: dummy_head_data(),
+					validation_code_hash,
+					result_sender: None,
+					core_index: CoreIndex(0),
+				}),
+			})
+			.await;
+
+		helpers::handle_runtime_calls_on_submit_collation(
+			&mut virtual_overseer,
+			relay_parent,
+			para_id,
+			expected_pvd.clone(),
+			[(CoreIndex(0), [para_id].into_iter().collect())].into_iter().collect(),
+		)
+		.await;
+
+		assert_matches!(
+			overseer_recv(&mut virtual_overseer).await,
+			AllMessages::CollatorProtocol(CollatorProtocolMessage::DistributeCollation {
+				candidate_receipt,
+				parent_head_data_hash,
+				..
+			}) => {
+				let CandidateReceiptV2 { descriptor, .. } = candidate_receipt;
+				assert_eq!(parent_head_data_hash, parent_head.hash());
+				assert_eq!(descriptor.persisted_validation_data_hash(), expected_pvd.hash());
+				assert_eq!(descriptor.para_head(), dummy_head_data().hash());
+				assert_eq!(descriptor.validation_code_hash(), validation_code_hash);
+				assert_eq!(descriptor.version(), CandidateDescriptorVersion::V2);
+			}
+		);
+
+		virtual_overseer
+	});
+}
+
+mod helpers {
+	use super::*;
+	use std::collections::{BTreeMap, VecDeque};
+
+	// Sends `Initialize` with a collator config
+	pub async fn initialize_collator(
+		virtual_overseer: &mut VirtualOverseer,
+		para_id: ParaId,
+		core_selector_data: Option<CoreSelectorData>,
+	) {
+		virtual_overseer
+			.send(FromOrchestra::Communication {
+				msg: CollationGenerationMessage::Initialize(test_config(
+					para_id,
+					core_selector_data,
+				)),
+			})
+			.await;
+	}
+
+	// Sends `ActiveLeaves` for a single leaf with the specified hash. Block number is hardcoded.
+	pub async fn activate_new_head(virtual_overseer: &mut VirtualOverseer, activated_hash: Hash) {
+		virtual_overseer
+			.send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate {
+				activated: Some(ActivatedLeaf {
+					hash: activated_hash,
+					number: 10,
+					unpin_handle: pezkuwi_node_subsystem_test_helpers::mock::dummy_unpin_handle(
+						activated_hash,
+					),
+				}),
+				..Default::default()
+			})))
+			.await;
+	}
+
+	// Handle all runtime calls performed in `handle_new_activation`.
+	pub async fn handle_runtime_calls_on_new_head_activation(
+		virtual_overseer: &mut VirtualOverseer,
+		activated_hash: Hash,
+		claim_queue: BTreeMap<CoreIndex, VecDeque<ParaId>>,
+	) {
+		assert_matches!(
+			overseer_recv(virtual_overseer).await,
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(hash, RuntimeApiRequest::SessionIndexForChild(tx))) => {
+				assert_eq!(hash, activated_hash);
+				tx.send(Ok(1)).unwrap();
+			}
+		);
+
+		assert_matches!(
+			overseer_recv(virtual_overseer).await,
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(hash, RuntimeApiRequest::Validators(tx))) => {
+				assert_eq!(hash, activated_hash);
+				tx.send(Ok(vec![
+					Sr25519Keyring::Alice.public().into(),
+					Sr25519Keyring::Bob.public().into(),
+					Sr25519Keyring::Charlie.public().into(),
+				])).unwrap();
+			}
+		);
+
+		assert_matches!(
+			overseer_recv(virtual_overseer).await,
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(hash, RuntimeApiRequest::ClaimQueue(tx))) => {
+				assert_eq!(hash, activated_hash);
+				tx.send(Ok(claim_queue)).unwrap();
+			}
+		);
+	}
+
+	// Handles all runtime requests performed in `handle_new_activation` for the case when a
+	// collation should be prepared for the new leaf
+	pub async fn handle_cores_processing_for_a_leaf(
+		virtual_overseer: &mut VirtualOverseer,
+		activated_hash: Hash,
+		para_id: ParaId,
+		cores_assigned: Vec<u32>,
+	) {
+		// Expect no messages if no cores is assigned to the para
+		if cores_assigned.is_empty() {
+			return;
+		}
+
+		// Some hardcoded data - if needed, extract to parameters
+		let validation_code_hash = ValidationCodeHash::from(Hash::repeat_byte(42));
+		let parent_head = dummy_head_data();
+		let pvd = PersistedValidationData {
+			parent_head: parent_head.clone(),
+			relay_parent_number: 10,
+			relay_parent_storage_root: Hash::repeat_byte(1),
+			max_pov_size: 1024,
+		};
+
+		assert_matches!(
+			overseer_recv(virtual_overseer).await,
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(hash, RuntimeApiRequest::PersistedValidationData(id, a, tx))) => {
+				assert_eq!(hash, activated_hash);
+				assert_eq!(id, para_id);
+				assert_eq!(a, OccupiedCoreAssumption::Included);
+
+				let _ = tx.send(Ok(Some(pvd.clone())));
+			}
+		);
+
+		assert_matches!(
+			overseer_recv(virtual_overseer).await,
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+				hash,
+				RuntimeApiRequest::ValidationCodeHash(
+					id,
+					assumption,
+					tx,
+				),
+			)) => {
+				assert_eq!(hash, activated_hash);
+				assert_eq!(id, para_id);
+				assert_eq!(assumption, OccupiedCoreAssumption::Included);
+
+				let _ = tx.send(Ok(Some(validation_code_hash)));
+			}
+		);
+
+		for core in cores_assigned {
+			assert_matches!(
+				overseer_recv(virtual_overseer).await,
+				AllMessages::CollatorProtocol(CollatorProtocolMessage::DistributeCollation{
+					candidate_receipt,
+					parent_head_data_hash,
+					core_index,
+					..
+				}) => {
+					assert_eq!(CoreIndex(core), core_index);
+					assert_eq!(parent_head_data_hash, parent_head.hash());
+					assert_eq!(candidate_receipt.descriptor().persisted_validation_data_hash(), pvd.hash());
+					assert_eq!(candidate_receipt.descriptor().para_head(), dummy_head_data().hash());
+					assert_eq!(candidate_receipt.descriptor().validation_code_hash(), validation_code_hash);
+				}
+			);
+		}
+	}
+
+	// Handles all runtime requests performed in `handle_submit_collation`
+	pub async fn handle_runtime_calls_on_submit_collation(
+		virtual_overseer: &mut VirtualOverseer,
+		relay_parent: Hash,
+		para_id: ParaId,
+		expected_pvd: PersistedValidationData,
+		claim_queue: BTreeMap<CoreIndex, VecDeque<ParaId>>,
+	) {
+		assert_matches!(
+			overseer_recv(virtual_overseer).await,
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(rp, RuntimeApiRequest::PersistedValidationData(id, a, tx))) => {
+				assert_eq!(rp, relay_parent);
+				assert_eq!(id, para_id);
+				assert_eq!(a, OccupiedCoreAssumption::TimedOut);
+
+				tx.send(Ok(Some(expected_pvd))).unwrap();
+			}
+		);
+
+		assert_matches!(
+			overseer_recv(virtual_overseer).await,
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+				rp,
+				RuntimeApiRequest::ClaimQueue(tx),
+			)) => {
+				assert_eq!(rp, relay_parent);
+				tx.send(Ok(claim_queue)).unwrap();
+			}
+		);
+
+		assert_matches!(
+			overseer_recv(virtual_overseer).await,
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(rp, RuntimeApiRequest::SessionIndexForChild(tx))) => {
+				assert_eq!(rp, relay_parent);
+				tx.send(Ok(1)).unwrap();
+			}
+		);
+
+		assert_matches!(
+			overseer_recv(virtual_overseer).await,
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(rp, RuntimeApiRequest::Validators(tx))) => {
+				assert_eq!(rp, relay_parent);
+				tx.send(Ok(vec![
+					Sr25519Keyring::Alice.public().into(),
+					Sr25519Keyring::Bob.public().into(),
+					Sr25519Keyring::Charlie.public().into(),
+				])).unwrap();
+			}
+		);
+	}
+}

pezkuwi/node/core/README.md

@@ -0,0 +1 @@
+This folder contains core subsystems, each with their own crate.

pezkuwi/node/core/approval-voting-parallel/Cargo.toml

@@ -0,0 +1,62 @@
+[package]
+name = "pezkuwi-node-core-approval-voting-parallel"
+version = "7.0.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+description = "Approval Voting Subsystem running approval work in parallel"
+homepage.workspace = true
+repository.workspace = true
+
+[lints]
+workspace = true
+
+[dependencies]
+async-trait = { workspace = true }
+futures = { workspace = true }
+gum = { workspace = true }
+itertools = { workspace = true }
+
+pezkuwi-approval-distribution = { workspace = true, default-features = true }
+pezkuwi-node-core-approval-voting = { workspace = true, default-features = true }
+pezkuwi-node-metrics = { workspace = true, default-features = true }
+pezkuwi-node-network-protocol = { workspace = true, default-features = true }
+pezkuwi-node-primitives = { workspace = true, default-features = true }
+pezkuwi-node-subsystem = { workspace = true, default-features = true }
+pezkuwi-node-subsystem-util = { workspace = true, default-features = true }
+pezkuwi-overseer = { workspace = true, default-features = true }
+pezkuwi-primitives = { workspace = true, default-features = true }
+
+sc-keystore = { workspace = true, default-features = false }
+sp-consensus = { workspace = true, default-features = false }
+
+rand = { workspace = true }
+rand_core = { workspace = true }
+
+[dev-dependencies]
+assert_matches = { workspace = true }
+kvdb-memorydb = { workspace = true }
+pezkuwi-node-subsystem-test-helpers = { workspace = true, default-features = true }
+schnorrkel = { workspace = true, default-features = true }
+sp-consensus-babe = { workspace = true, default-features = true }
+sp-core = { workspace = true, default-features = true }
+sp-keyring = { workspace = true, default-features = true }
+sp-tracing = { workspace = true }
+
+[features]
+runtime-benchmarks = [
+	"gum/runtime-benchmarks",
+	"pezkuwi-approval-distribution/runtime-benchmarks",
+	"pezkuwi-node-core-approval-voting/runtime-benchmarks",
+	"pezkuwi-node-metrics/runtime-benchmarks",
+	"pezkuwi-node-network-protocol/runtime-benchmarks",
+	"pezkuwi-node-primitives/runtime-benchmarks",
+	"pezkuwi-node-subsystem-test-helpers/runtime-benchmarks",
+	"pezkuwi-node-subsystem-util/runtime-benchmarks",
+	"pezkuwi-node-subsystem/runtime-benchmarks",
+	"pezkuwi-overseer/runtime-benchmarks",
+	"pezkuwi-primitives/runtime-benchmarks",
+	"sp-consensus-babe/runtime-benchmarks",
+	"sp-consensus/runtime-benchmarks",
+	"sp-keyring/runtime-benchmarks",
+]

pezkuwi/node/core/approval-voting-parallel/src/lib.rs

@@ -0,0 +1,878 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! The Approval Voting Parallel Subsystem.
+//!
+//! This subsystem is responsible for orchestrating the work done by
+//! approval-voting and approval-distribution subsystem, so they can
+//! do their work in parallel, rather than serially, when they are run
+//! as independent subsystems.
+use itertools::Itertools;
+use metrics::{Meters, MetricsWatcher};
+use pezkuwi_node_core_approval_voting::{Config, RealAssignmentCriteria};
+use pezkuwi_node_metrics::metered::{
+	self, channel, unbounded, MeteredReceiver, MeteredSender, UnboundedMeteredReceiver,
+	UnboundedMeteredSender,
+};
+
+use pezkuwi_node_primitives::{
+	approval::time::{Clock, SystemClock},
+	DISPUTE_WINDOW,
+};
+use pezkuwi_node_subsystem::{
+	messages::{ApprovalDistributionMessage, ApprovalVotingMessage, ApprovalVotingParallelMessage},
+	overseer, FromOrchestra, SpawnedSubsystem, SubsystemError, SubsystemResult,
+};
+use pezkuwi_node_subsystem_util::{
+	self,
+	database::Database,
+	runtime::{Config as RuntimeInfoConfig, RuntimeInfo},
+};
+use pezkuwi_overseer::{OverseerSignal, Priority, SubsystemSender, TimeoutExt};
+use pezkuwi_primitives::{CandidateIndex, Hash, ValidatorIndex, ValidatorSignature};
+use rand::SeedableRng;
+
+use sc_keystore::LocalKeystore;
+use sp_consensus::SyncOracle;
+
+use futures::{channel::oneshot, prelude::*, StreamExt};
+pub use metrics::Metrics;
+use pezkuwi_node_core_approval_voting::{
+	approval_db::common::Config as DatabaseConfig, ApprovalVotingWorkProvider,
+};
+use std::{
+	collections::{HashMap, HashSet},
+	fmt::Debug,
+	sync::Arc,
+	time::Duration,
+};
+use stream::{select_with_strategy, PollNext, SelectWithStrategy};
+pub mod metrics;
+
+#[cfg(test)]
+mod tests;
+
+pub(crate) const LOG_TARGET: &str = "teyrchain::approval-voting-parallel";
+// Value rather arbitrarily: Should not be hit in practice, it exists to more easily diagnose dead
+// lock issues for example.
+const WAIT_FOR_SIGS_GATHER_TIMEOUT: Duration = Duration::from_millis(2000);
+
+/// The number of workers used for running the approval-distribution logic.
+pub const APPROVAL_DISTRIBUTION_WORKER_COUNT: usize = 4;
+
+/// The default channel size for the workers, can be overridden by the user through
+/// `overseer_channel_capacity_override`
+pub const DEFAULT_WORKERS_CHANNEL_SIZE: usize = 64000 / APPROVAL_DISTRIBUTION_WORKER_COUNT;
+
+fn prio_right<'a>(_val: &'a mut ()) -> PollNext {
+	PollNext::Right
+}
+
+/// The approval voting parallel subsystem.
+pub struct ApprovalVotingParallelSubsystem {
+	/// `LocalKeystore` is needed for assignment keys, but not necessarily approval keys.
+	///
+	/// We do a lot of VRF signing and need the keys to have low latency.
+	keystore: Arc<LocalKeystore>,
+	db_config: DatabaseConfig,
+	slot_duration_millis: u64,
+	db: Arc<dyn Database>,
+	sync_oracle: Box<dyn SyncOracle + Send>,
+	metrics: Metrics,
+	spawner: Arc<dyn overseer::gen::Spawner + 'static>,
+	clock: Arc<dyn Clock + Send + Sync>,
+	overseer_message_channel_capacity_override: Option<usize>,
+}
+
+impl ApprovalVotingParallelSubsystem {
+	/// Create a new approval voting subsystem with the given keystore, config, and database.
+	pub fn with_config(
+		config: Config,
+		db: Arc<dyn Database>,
+		keystore: Arc<LocalKeystore>,
+		sync_oracle: Box<dyn SyncOracle + Send>,
+		metrics: Metrics,
+		spawner: impl overseer::gen::Spawner + 'static + Clone,
+		overseer_message_channel_capacity_override: Option<usize>,
+	) -> Self {
+		ApprovalVotingParallelSubsystem::with_config_and_clock(
+			config,
+			db,
+			keystore,
+			sync_oracle,
+			metrics,
+			Arc::new(SystemClock {}),
+			spawner,
+			overseer_message_channel_capacity_override,
+		)
+	}
+
+	/// Create a new approval voting subsystem with the given keystore, config, clock, and database.
+	pub fn with_config_and_clock(
+		config: Config,
+		db: Arc<dyn Database>,
+		keystore: Arc<LocalKeystore>,
+		sync_oracle: Box<dyn SyncOracle + Send>,
+		metrics: Metrics,
+		clock: Arc<dyn Clock + Send + Sync>,
+		spawner: impl overseer::gen::Spawner + 'static,
+		overseer_message_channel_capacity_override: Option<usize>,
+	) -> Self {
+		ApprovalVotingParallelSubsystem {
+			keystore,
+			slot_duration_millis: config.slot_duration_millis,
+			db,
+			db_config: DatabaseConfig { col_approval_data: config.col_approval_data },
+			sync_oracle,
+			metrics,
+			spawner: Arc::new(spawner),
+			clock,
+			overseer_message_channel_capacity_override,
+		}
+	}
+
+	/// The size of the channel used for the workers.
+	fn workers_channel_size(&self) -> usize {
+		self.overseer_message_channel_capacity_override
+			.unwrap_or(DEFAULT_WORKERS_CHANNEL_SIZE)
+	}
+}
+
+#[overseer::subsystem(ApprovalVotingParallel, error = SubsystemError, prefix = self::overseer)]
+impl<Context: Send> ApprovalVotingParallelSubsystem {
+	fn start(self, ctx: Context) -> SpawnedSubsystem {
+		let future = run::<Context>(ctx, self)
+			.map_err(|e| SubsystemError::with_origin("approval-voting-parallel", e))
+			.boxed();
+
+		SpawnedSubsystem { name: "approval-voting-parallel-subsystem", future }
+	}
+}
+
+// It starts worker for the approval voting subsystem and the `APPROVAL_DISTRIBUTION_WORKER_COUNT`
+// workers for the approval distribution subsystem.
+//
+// It returns handles that can be used to send messages to the workers.
+#[overseer::contextbounds(ApprovalVotingParallel, prefix = self::overseer)]
+async fn start_workers<Context>(
+	ctx: &mut Context,
+	subsystem: ApprovalVotingParallelSubsystem,
+	metrics_watcher: &mut MetricsWatcher,
+) -> SubsystemResult<(ToWorker<ApprovalVotingMessage>, Vec<ToWorker<ApprovalDistributionMessage>>)>
+where
+{
+	gum::info!(target: LOG_TARGET, "Starting approval distribution workers");
+
+	// Build approval voting handles.
+	let (to_approval_voting_worker, approval_voting_work_provider) = build_worker_handles(
+		"approval-voting-parallel-db".into(),
+		subsystem.workers_channel_size(),
+		metrics_watcher,
+		prio_right,
+	);
+	let mut to_approval_distribution_workers = Vec::new();
+	let slot_duration_millis = subsystem.slot_duration_millis;
+
+	for i in 0..APPROVAL_DISTRIBUTION_WORKER_COUNT {
+		let mut network_sender = ctx.sender().clone();
+		let mut runtime_api_sender = ctx.sender().clone();
+		let mut approval_distribution_to_approval_voting = to_approval_voting_worker.clone();
+
+		let approval_distr_instance =
+			pezkuwi_approval_distribution::ApprovalDistribution::new_with_clock(
+				subsystem.metrics.approval_distribution_metrics(),
+				subsystem.slot_duration_millis,
+				subsystem.clock.clone(),
+				Arc::new(RealAssignmentCriteria {}),
+			);
+		let task_name = format!("approval-voting-parallel-{}", i);
+		let (to_approval_distribution_worker, mut approval_distribution_work_provider) =
+			build_worker_handles(
+				task_name.clone(),
+				subsystem.workers_channel_size(),
+				metrics_watcher,
+				prio_right,
+			);
+
+		metrics_watcher.watch(task_name.clone(), to_approval_distribution_worker.meter());
+
+		subsystem.spawner.spawn_blocking(
+			task_name.leak(),
+			Some("approval-voting-parallel"),
+			Box::pin(async move {
+				let mut state =
+					pezkuwi_approval_distribution::State::with_config(slot_duration_millis);
+				let mut rng = rand::rngs::StdRng::from_entropy();
+				let mut session_info_provider = RuntimeInfo::new_with_config(RuntimeInfoConfig {
+					keystore: None,
+					session_cache_lru_size: DISPUTE_WINDOW.get(),
+				});
+
+				loop {
+					let message = match approval_distribution_work_provider.next().await {
+						Some(message) => message,
+						None => {
+							gum::info!(
+								target: LOG_TARGET,
+								"Approval distribution stream finished, most likely shutting down",
+							);
+							break;
+						},
+					};
+					if approval_distr_instance
+						.handle_from_orchestra(
+							message,
+							&mut approval_distribution_to_approval_voting,
+							&mut network_sender,
+							&mut runtime_api_sender,
+							&mut state,
+							&mut rng,
+							&mut session_info_provider,
+						)
+						.await
+					{
+						gum::info!(
+							target: LOG_TARGET,
+							"Approval distribution worker {}, exiting because of shutdown", i
+						);
+					};
+				}
+			}),
+		);
+		to_approval_distribution_workers.push(to_approval_distribution_worker);
+	}
+
+	gum::info!(target: LOG_TARGET, "Starting approval voting workers");
+
+	let sender = ctx.sender().clone();
+	let to_approval_distribution = ApprovalVotingToApprovalDistribution(sender.clone());
+	pezkuwi_node_core_approval_voting::start_approval_worker(
+		approval_voting_work_provider,
+		sender.clone(),
+		to_approval_distribution,
+		pezkuwi_node_core_approval_voting::Config {
+			slot_duration_millis: subsystem.slot_duration_millis,
+			col_approval_data: subsystem.db_config.col_approval_data,
+		},
+		subsystem.db.clone(),
+		subsystem.keystore.clone(),
+		subsystem.sync_oracle,
+		subsystem.metrics.approval_voting_metrics(),
+		subsystem.spawner.clone(),
+		"approval-voting-parallel-db",
+		"approval-voting-parallel",
+		subsystem.clock.clone(),
+	)
+	.await?;
+
+	Ok((to_approval_voting_worker, to_approval_distribution_workers))
+}
+
+// The main run function of the approval parallel voting subsystem.
+#[overseer::contextbounds(ApprovalVotingParallel, prefix = self::overseer)]
+async fn run<Context>(
+	mut ctx: Context,
+	subsystem: ApprovalVotingParallelSubsystem,
+) -> SubsystemResult<()> {
+	let mut metrics_watcher = MetricsWatcher::new(subsystem.metrics.clone());
+	gum::info!(
+		target: LOG_TARGET,
+		"Starting workers"
+	);
+
+	let (to_approval_voting_worker, to_approval_distribution_workers) =
+		start_workers(&mut ctx, subsystem, &mut metrics_watcher).await?;
+
+	gum::info!(
+		target: LOG_TARGET,
+		"Starting main subsystem loop"
+	);
+
+	run_main_loop(ctx, to_approval_voting_worker, to_approval_distribution_workers, metrics_watcher)
+		.await
+}
+
+// Main loop of the subsystem, it shouldn't include any logic just dispatching of messages to
+// the workers.
+//
+// It listens for messages from the overseer and dispatches them to the workers.
+#[overseer::contextbounds(ApprovalVotingParallel, prefix = self::overseer)]
+async fn run_main_loop<Context>(
+	mut ctx: Context,
+	mut to_approval_voting_worker: ToWorker<ApprovalVotingMessage>,
+	mut to_approval_distribution_workers: Vec<ToWorker<ApprovalDistributionMessage>>,
+	metrics_watcher: MetricsWatcher,
+) -> SubsystemResult<()> {
+	loop {
+		futures::select! {
+			next_msg = ctx.recv().fuse() => {
+				let next_msg = match next_msg {
+					Ok(msg) => msg,
+					Err(err) => {
+						gum::info!(target: LOG_TARGET, ?err, "Approval voting parallel subsystem received an error");
+						return Err(err);
+					}
+				};
+
+				match next_msg {
+					FromOrchestra::Signal(msg) => {
+						if matches!(msg, OverseerSignal::ActiveLeaves(_)) {
+							metrics_watcher.collect_metrics();
+						}
+
+						for worker in to_approval_distribution_workers.iter_mut() {
+							worker
+								.send_signal(msg.clone()).await?;
+						}
+
+						to_approval_voting_worker.send_signal(msg.clone()).await?;
+						if matches!(msg, OverseerSignal::Conclude) {
+							break;
+						}
+					},
+					FromOrchestra::Communication { msg } => match msg {
+						// The message the approval voting subsystem would've handled.
+						ApprovalVotingParallelMessage::ApprovedAncestor(_, _,_) |
+						ApprovalVotingParallelMessage::GetApprovalSignaturesForCandidate(_, _)  => {
+							to_approval_voting_worker.send_message_with_priority::<overseer::HighPriority>(
+								msg.try_into().expect(
+									"Message is one of ApprovedAncestor, GetApprovalSignaturesForCandidate
+									 and that can be safely converted to ApprovalVotingMessage; qed"
+								)
+							).await;
+						},
+						// Now the message the approval distribution subsystem would've handled and need to
+						// be forwarded to the workers.
+						ApprovalVotingParallelMessage::NewBlocks(msg) => {
+							for worker in to_approval_distribution_workers.iter_mut() {
+								worker
+									.send_message(
+										ApprovalDistributionMessage::NewBlocks(msg.clone()),
+									)
+									.await;
+							}
+						},
+						ApprovalVotingParallelMessage::DistributeAssignment(assignment, claimed) => {
+							let worker = assigned_worker_for_validator(assignment.validator, &mut to_approval_distribution_workers);
+							worker
+								.send_message(
+									ApprovalDistributionMessage::DistributeAssignment(assignment, claimed)
+								)
+								.await;
+
+						},
+						ApprovalVotingParallelMessage::DistributeApproval(vote) => {
+							let worker = assigned_worker_for_validator(vote.validator, &mut to_approval_distribution_workers);
+							worker
+								.send_message(
+									ApprovalDistributionMessage::DistributeApproval(vote)
+								).await;
+
+						},
+						ApprovalVotingParallelMessage::NetworkBridgeUpdate(msg) => {
+							if let pezkuwi_node_subsystem::messages::NetworkBridgeEvent::PeerMessage(
+								peer_id,
+								msg,
+							) = msg
+							{
+								let (all_msgs_from_same_validator, messages_split_by_validator) = validator_index_for_msg(msg);
+
+								for (validator_index, msg) in all_msgs_from_same_validator.into_iter().chain(messages_split_by_validator.into_iter().flatten()) {
+									let worker = assigned_worker_for_validator(validator_index, &mut to_approval_distribution_workers);
+
+									worker
+										.send_message(
+											ApprovalDistributionMessage::NetworkBridgeUpdate(
+												pezkuwi_node_subsystem::messages::NetworkBridgeEvent::PeerMessage(
+													peer_id, msg,
+												),
+											),
+										).await;
+								}
+							} else {
+								for worker in to_approval_distribution_workers.iter_mut() {
+									worker
+										.send_message_with_priority::<overseer::HighPriority>(
+											ApprovalDistributionMessage::NetworkBridgeUpdate(msg.clone()),
+										).await;
+								}
+							}
+						},
+						ApprovalVotingParallelMessage::GetApprovalSignatures(indices, tx) => {
+							handle_get_approval_signatures(&mut ctx, &mut to_approval_distribution_workers, indices, tx).await;
+						},
+						ApprovalVotingParallelMessage::ApprovalCheckingLagUpdate(lag) => {
+							for worker in to_approval_distribution_workers.iter_mut() {
+								worker
+									.send_message(
+										ApprovalDistributionMessage::ApprovalCheckingLagUpdate(lag)
+									).await;
+							}
+						},
+					},
+				};
+
+			},
+		};
+	}
+	Ok(())
+}
+
+// It sends a message to all approval workers to get the approval signatures for the requested
+// candidates and then merges them all together and sends them back to the requester.
+#[overseer::contextbounds(ApprovalVotingParallel, prefix = self::overseer)]
+async fn handle_get_approval_signatures<Context>(
+	ctx: &mut Context,
+	to_approval_distribution_workers: &mut Vec<ToWorker<ApprovalDistributionMessage>>,
+	requested_candidates: HashSet<(Hash, CandidateIndex)>,
+	result_channel: oneshot::Sender<
+		HashMap<ValidatorIndex, (Hash, Vec<CandidateIndex>, ValidatorSignature)>,
+	>,
+) {
+	let mut sigs = HashMap::new();
+	let mut signatures_channels = Vec::new();
+	for worker in to_approval_distribution_workers.iter_mut() {
+		let (tx, rx) = oneshot::channel();
+		worker.send_unbounded_message(ApprovalDistributionMessage::GetApprovalSignatures(
+			requested_candidates.clone(),
+			tx,
+		));
+		signatures_channels.push(rx);
+	}
+
+	let gather_signatures = async move {
+		let Some(results) = futures::future::join_all(signatures_channels)
+			.timeout(WAIT_FOR_SIGS_GATHER_TIMEOUT)
+			.await
+		else {
+			gum::warn!(
+				target: LOG_TARGET,
+				"Waiting for approval signatures timed out - dead lock?"
+			);
+			return;
+		};
+
+		for result in results {
+			let worker_sigs = match result {
+				Ok(sigs) => sigs,
+				Err(_) => {
+					gum::error!(
+						target: LOG_TARGET,
+						"Getting approval signatures failed, oneshot got closed"
+					);
+					continue;
+				},
+			};
+			sigs.extend(worker_sigs);
+		}
+
+		if let Err(_) = result_channel.send(sigs) {
+			gum::debug!(
+					target: LOG_TARGET,
+					"Sending back approval signatures failed, oneshot got closed"
+			);
+		}
+	};
+
+	if let Err(err) = ctx.spawn("approval-voting-gather-signatures", Box::pin(gather_signatures)) {
+		gum::warn!(target: LOG_TARGET, "Failed to spawn gather signatures task: {:?}", err);
+	}
+}
+
+// Returns the worker that should receive the message for the given validator.
+fn assigned_worker_for_validator(
+	validator: ValidatorIndex,
+	to_approval_distribution_workers: &mut Vec<ToWorker<ApprovalDistributionMessage>>,
+) -> &mut ToWorker<ApprovalDistributionMessage> {
+	let worker_index = validator.0 as usize % to_approval_distribution_workers.len();
+	to_approval_distribution_workers
+		.get_mut(worker_index)
+		.expect("Worker index is obtained modulo len; qed")
+}
+
+// Returns the validators that initially created this assignments/votes, the validator index
+// is later used to decide which approval-distribution worker should receive the message.
+//
+// Because this is on the hot path and we don't want to be unnecessarily slow, it contains two logic
+// paths. The ultra fast path where all messages have the same validator index and we don't do
+// any cloning or allocation and the path where we need to split the messages into multiple
+// messages, because they have different validator indices, where we do need to clone and allocate.
+// In practice most of the message will fall on the ultra fast path.
+fn validator_index_for_msg(
+	msg: pezkuwi_node_network_protocol::ApprovalDistributionMessage,
+) -> (
+	Option<(ValidatorIndex, pezkuwi_node_network_protocol::ApprovalDistributionMessage)>,
+	Option<Vec<(ValidatorIndex, pezkuwi_node_network_protocol::ApprovalDistributionMessage)>>,
+) {
+	match msg {
+		pezkuwi_node_network_protocol::ValidationProtocols::V3(ref message) => match message {
+			pezkuwi_node_network_protocol::v3::ApprovalDistributionMessage::Assignments(msgs) =>
+				if let Ok(validator) = msgs.iter().map(|(msg, _)| msg.validator).all_equal_value() {
+					(Some((validator, msg)), None)
+				} else {
+					let split = msgs
+						.iter()
+						.map(|(msg, claimed_candidates)| {
+							(
+								msg.validator,
+								pezkuwi_node_network_protocol::ValidationProtocols::V3(
+									pezkuwi_node_network_protocol::v3::ApprovalDistributionMessage::Assignments(
+										vec![(msg.clone(), claimed_candidates.clone())]
+									),
+								),
+							)
+						})
+						.collect_vec();
+					(None, Some(split))
+				},
+			pezkuwi_node_network_protocol::v3::ApprovalDistributionMessage::Approvals(msgs) =>
+				if let Ok(validator) = msgs.iter().map(|msg| msg.validator).all_equal_value() {
+					(Some((validator, msg)), None)
+				} else {
+					let split = msgs
+						.iter()
+						.map(|vote| {
+							(
+								vote.validator,
+								pezkuwi_node_network_protocol::ValidationProtocols::V3(
+									pezkuwi_node_network_protocol::v3::ApprovalDistributionMessage::Approvals(
+										vec![vote.clone()]
+									),
+								),
+							)
+						})
+						.collect_vec();
+					(None, Some(split))
+				},
+		},
+	}
+}
+
+/// A handler object that both type of workers use for receiving work.
+///
+/// In practive this is just a wrapper over two channels Receiver, that is injected into
+/// approval-voting worker and approval-distribution workers.
+type WorkProvider<M, Clos, State> = WorkProviderImpl<
+	SelectWithStrategy<
+		MeteredReceiver<FromOrchestra<M>>,
+		UnboundedMeteredReceiver<FromOrchestra<M>>,
+		Clos,
+		State,
+	>,
+>;
+
+pub struct WorkProviderImpl<T>(T);
+
+impl<T, M> Stream for WorkProviderImpl<T>
+where
+	T: Stream<Item = FromOrchestra<M>> + Unpin + Send,
+{
+	type Item = FromOrchestra<M>;
+
+	fn poll_next(
+		mut self: std::pin::Pin<&mut Self>,
+		cx: &mut std::task::Context<'_>,
+	) -> std::task::Poll<Option<Self::Item>> {
+		self.0.poll_next_unpin(cx)
+	}
+}
+
+#[async_trait::async_trait]
+impl<T> ApprovalVotingWorkProvider for WorkProviderImpl<T>
+where
+	T: Stream<Item = FromOrchestra<ApprovalVotingMessage>> + Unpin + Send,
+{
+	async fn recv(&mut self) -> SubsystemResult<FromOrchestra<ApprovalVotingMessage>> {
+		self.0.next().await.ok_or(SubsystemError::Context(
+			"ApprovalVotingWorkProviderImpl: Channel closed".to_string(),
+		))
+	}
+}
+
+impl<M, Clos, State> WorkProvider<M, Clos, State>
+where
+	M: Send + Sync + 'static,
+	Clos: FnMut(&mut State) -> PollNext,
+	State: Default,
+{
+	// Constructs a work providers from the channels handles.
+	fn from_rx_worker(rx: RxWorker<M>, prio: Clos) -> Self {
+		let prioritised = select_with_strategy(rx.0, rx.1, prio);
+		WorkProviderImpl(prioritised)
+	}
+}
+
+/// Just a wrapper for implementing `overseer::SubsystemSender<ApprovalVotingMessage>` and
+/// `overseer::SubsystemSender<ApprovalDistributionMessage>`.
+///
+/// The instance of this struct can be injected into the workers, so they can talk
+/// directly with each other without intermediating in this subsystem loop.
+pub struct ToWorker<T: Send + Sync + 'static>(
+	MeteredSender<FromOrchestra<T>>,
+	UnboundedMeteredSender<FromOrchestra<T>>,
+);
+
+impl<T: Send + Sync + 'static> Clone for ToWorker<T> {
+	fn clone(&self) -> Self {
+		Self(self.0.clone(), self.1.clone())
+	}
+}
+
+impl<T: Send + Sync + 'static> ToWorker<T> {
+	async fn send_signal(&mut self, signal: OverseerSignal) -> Result<(), SubsystemError> {
+		self.1
+			.unbounded_send(FromOrchestra::Signal(signal))
+			.map_err(|err| SubsystemError::QueueError(err.into_send_error()))
+	}
+
+	fn meter(&self) -> Meters {
+		Meters::new(self.0.meter(), self.1.meter())
+	}
+}
+
+impl<T: Send + Sync + 'static + Debug> overseer::SubsystemSender<T> for ToWorker<T> {
+	fn send_message<'life0, 'async_trait>(
+		&'life0 mut self,
+		msg: T,
+	) -> ::core::pin::Pin<
+		Box<dyn ::core::future::Future<Output = ()> + ::core::marker::Send + 'async_trait>,
+	>
+	where
+		'life0: 'async_trait,
+		Self: 'async_trait,
+	{
+		async {
+			if let Err(err) =
+				self.0.send(pezkuwi_overseer::FromOrchestra::Communication { msg }).await
+			{
+				gum::error!(
+					target: LOG_TARGET,
+					"Failed to send message to approval voting worker: {:?}, subsystem is probably shutting down.",
+					err
+				);
+			}
+		}
+		.boxed()
+	}
+
+	fn try_send_message(&mut self, msg: T) -> Result<(), metered::TrySendError<T>> {
+		self.0
+			.try_send(pezkuwi_overseer::FromOrchestra::Communication { msg })
+			.map_err(|result| {
+				let is_full = result.is_full();
+				let msg = match result.into_inner() {
+					pezkuwi_overseer::FromOrchestra::Signal(_) => {
+						panic!("Cannot happen variant is never built")
+					},
+					pezkuwi_overseer::FromOrchestra::Communication { msg } => msg,
+				};
+				if is_full {
+					metered::TrySendError::Full(msg)
+				} else {
+					metered::TrySendError::Closed(msg)
+				}
+			})
+	}
+
+	fn send_messages<'life0, 'async_trait, I>(
+		&'life0 mut self,
+		msgs: I,
+	) -> ::core::pin::Pin<
+		Box<dyn ::core::future::Future<Output = ()> + ::core::marker::Send + 'async_trait>,
+	>
+	where
+		I: IntoIterator<Item = T> + Send,
+		I::IntoIter: Send,
+		I: 'async_trait,
+		'life0: 'async_trait,
+		Self: 'async_trait,
+	{
+		async {
+			for msg in msgs {
+				self.send_message(msg).await;
+			}
+		}
+		.boxed()
+	}
+
+	fn send_unbounded_message(&mut self, msg: T) {
+		if let Err(err) =
+			self.1.unbounded_send(pezkuwi_overseer::FromOrchestra::Communication { msg })
+		{
+			gum::error!(
+				target: LOG_TARGET,
+				"Failed to send unbounded message to approval voting worker: {:?}, subsystem is probably shutting down.",
+				err
+			);
+		}
+	}
+
+	fn send_message_with_priority<'life0, 'async_trait, P>(
+		&'life0 mut self,
+		msg: T,
+	) -> ::core::pin::Pin<
+		Box<dyn ::core::future::Future<Output = ()> + ::core::marker::Send + 'async_trait>,
+	>
+	where
+		P: 'async_trait + Priority,
+		'life0: 'async_trait,
+		Self: 'async_trait,
+	{
+		match P::priority() {
+			pezkuwi_overseer::PriorityLevel::Normal => self.send_message(msg),
+			pezkuwi_overseer::PriorityLevel::High =>
+				async { self.send_unbounded_message(msg) }.boxed(),
+		}
+	}
+
+	fn try_send_message_with_priority<P: Priority>(
+		&mut self,
+		msg: T,
+	) -> Result<(), metered::TrySendError<T>> {
+		match P::priority() {
+			pezkuwi_overseer::PriorityLevel::Normal => self.try_send_message(msg),
+			pezkuwi_overseer::PriorityLevel::High => Ok(self.send_unbounded_message(msg)),
+		}
+	}
+}
+
+/// Handles that are used by an worker to receive work.
+pub struct RxWorker<T: Send + Sync + 'static>(
+	MeteredReceiver<FromOrchestra<T>>,
+	UnboundedMeteredReceiver<FromOrchestra<T>>,
+);
+
+// Build all the necessary channels for sending messages to an worker
+// and for the worker to receive them.
+fn build_channels<T: Send + Sync + 'static>(
+	channel_name: String,
+	channel_size: usize,
+	metrics_watcher: &mut MetricsWatcher,
+) -> (ToWorker<T>, RxWorker<T>) {
+	let (tx_work, rx_work) = channel::<FromOrchestra<T>>(channel_size);
+	let (tx_work_unbounded, rx_work_unbounded) = unbounded::<FromOrchestra<T>>();
+	let to_worker = ToWorker(tx_work, tx_work_unbounded);
+
+	metrics_watcher.watch(channel_name, to_worker.meter());
+
+	(to_worker, RxWorker(rx_work, rx_work_unbounded))
+}
+
+/// Build the worker handles used for interacting with the workers.
+///
+/// `ToWorker` is used for sending messages to the workers.
+/// `WorkProvider` is used by the workers for receiving the messages.
+fn build_worker_handles<M, Clos, State>(
+	channel_name: String,
+	channel_size: usize,
+	metrics_watcher: &mut MetricsWatcher,
+	prio_right: Clos,
+) -> (ToWorker<M>, WorkProvider<M, Clos, State>)
+where
+	M: Send + Sync + 'static,
+	Clos: FnMut(&mut State) -> PollNext,
+	State: Default,
+{
+	let (to_worker, rx_worker) = build_channels(channel_name, channel_size, metrics_watcher);
+	(to_worker, WorkProviderImpl::from_rx_worker(rx_worker, prio_right))
+}
+
+/// Just a wrapper for implementing `overseer::SubsystemSender<ApprovalDistributionMessage>`, so
+/// that we can inject into the approval voting subsystem.
+#[derive(Clone)]
+pub struct ApprovalVotingToApprovalDistribution<S: SubsystemSender<ApprovalVotingParallelMessage>>(
+	S,
+);
+
+impl<S: SubsystemSender<ApprovalVotingParallelMessage>>
+	overseer::SubsystemSender<ApprovalDistributionMessage>
+	for ApprovalVotingToApprovalDistribution<S>
+{
+	#[allow(clippy::type_complexity, clippy::type_repetition_in_bounds)]
+	fn send_message<'life0, 'async_trait>(
+		&'life0 mut self,
+		msg: ApprovalDistributionMessage,
+	) -> ::core::pin::Pin<
+		Box<dyn ::core::future::Future<Output = ()> + ::core::marker::Send + 'async_trait>,
+	>
+	where
+		'life0: 'async_trait,
+		Self: 'async_trait,
+	{
+		self.0.send_message(msg.into())
+	}
+
+	fn try_send_message(
+		&mut self,
+		msg: ApprovalDistributionMessage,
+	) -> Result<(), metered::TrySendError<ApprovalDistributionMessage>> {
+		self.0.try_send_message(msg.into()).map_err(|err| match err {
+			// Safe to unwrap because it was built from the same type.
+			metered::TrySendError::Closed(msg) =>
+				metered::TrySendError::Closed(msg.try_into().unwrap()),
+			metered::TrySendError::Full(msg) =>
+				metered::TrySendError::Full(msg.try_into().unwrap()),
+		})
+	}
+
+	#[allow(clippy::type_complexity, clippy::type_repetition_in_bounds)]
+	fn send_messages<'life0, 'async_trait, I>(
+		&'life0 mut self,
+		msgs: I,
+	) -> ::core::pin::Pin<
+		Box<dyn ::core::future::Future<Output = ()> + ::core::marker::Send + 'async_trait>,
+	>
+	where
+		I: IntoIterator<Item = ApprovalDistributionMessage> + Send,
+		I::IntoIter: Send,
+		I: 'async_trait,
+		'life0: 'async_trait,
+		Self: 'async_trait,
+	{
+		self.0.send_messages(msgs.into_iter().map(|msg| msg.into()))
+	}
+
+	fn send_unbounded_message(&mut self, msg: ApprovalDistributionMessage) {
+		self.0.send_unbounded_message(msg.into())
+	}
+
+	fn send_message_with_priority<'life0, 'async_trait, P>(
+		&'life0 mut self,
+		msg: ApprovalDistributionMessage,
+	) -> ::core::pin::Pin<
+		Box<dyn ::core::future::Future<Output = ()> + ::core::marker::Send + 'async_trait>,
+	>
+	where
+		P: 'async_trait + Priority,
+		'life0: 'async_trait,
+		Self: 'async_trait,
+	{
+		self.0.send_message_with_priority::<P>(msg.into())
+	}
+
+	fn try_send_message_with_priority<P: Priority>(
+		&mut self,
+		msg: ApprovalDistributionMessage,
+	) -> Result<(), metered::TrySendError<ApprovalDistributionMessage>> {
+		self.0.try_send_message_with_priority::<P>(msg.into()).map_err(|err| match err {
+			// Safe to unwrap because it was built from the same type.
+			metered::TrySendError::Closed(msg) =>
+				metered::TrySendError::Closed(msg.try_into().unwrap()),
+			metered::TrySendError::Full(msg) =>
+				metered::TrySendError::Full(msg.try_into().unwrap()),
+		})
+	}
+}

pezkuwi/node/core/approval-voting-parallel/src/metrics.rs

@@ -0,0 +1,234 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! The Metrics for Approval Voting Parallel Subsystem.
+
+use std::collections::HashMap;
+
+use pezkuwi_node_metrics::{metered::Meter, metrics};
+use pezkuwi_overseer::prometheus;
+
+#[derive(Default, Clone)]
+pub struct Metrics(Option<MetricsInner>);
+
+/// Approval Voting parallel metrics.
+#[derive(Clone)]
+pub struct MetricsInner {
+	// The inner metrics of the approval distribution workers.
+	approval_distribution: pezkuwi_approval_distribution::metrics::Metrics,
+	// The inner metrics of the approval voting workers.
+	approval_voting: pezkuwi_node_core_approval_voting::Metrics,
+
+	// Time of flight metrics for bounded channels.
+	to_worker_bounded_tof: prometheus::HistogramVec,
+	// Number of elements sent to the worker's bounded queue.
+	to_worker_bounded_sent: prometheus::GaugeVec<prometheus::U64>,
+	// Number of elements received by the worker's bounded queue.
+	to_worker_bounded_received: prometheus::GaugeVec<prometheus::U64>,
+	// Number of times senders blocked while sending messages to the worker.
+	to_worker_bounded_blocked: prometheus::GaugeVec<prometheus::U64>,
+	// Time of flight metrics for unbounded channels.
+	to_worker_unbounded_tof: prometheus::HistogramVec,
+	// Number of elements sent to the worker's unbounded queue.
+	to_worker_unbounded_sent: prometheus::GaugeVec<prometheus::U64>,
+	// Number of elements received by the worker's unbounded queue.
+	to_worker_unbounded_received: prometheus::GaugeVec<prometheus::U64>,
+}
+
+impl Metrics {
+	/// Get the approval distribution metrics.
+	pub fn approval_distribution_metrics(&self) -> pezkuwi_approval_distribution::metrics::Metrics {
+		self.0
+			.as_ref()
+			.map(|metrics_inner| metrics_inner.approval_distribution.clone())
+			.unwrap_or_default()
+	}
+
+	/// Get the approval voting metrics.
+	pub fn approval_voting_metrics(&self) -> pezkuwi_node_core_approval_voting::Metrics {
+		self.0
+			.as_ref()
+			.map(|metrics_inner| metrics_inner.approval_voting.clone())
+			.unwrap_or_default()
+	}
+}
+
+impl metrics::Metrics for Metrics {
+	/// Try to register the metrics.
+	fn try_register(
+		registry: &prometheus::Registry,
+	) -> std::result::Result<Self, prometheus::PrometheusError> {
+		Ok(Metrics(Some(MetricsInner {
+			approval_distribution: pezkuwi_approval_distribution::metrics::Metrics::try_register(
+				registry,
+			)?,
+			approval_voting: pezkuwi_node_core_approval_voting::Metrics::try_register(registry)?,
+			to_worker_bounded_tof: prometheus::register(
+				prometheus::HistogramVec::new(
+					prometheus::HistogramOpts::new(
+						"pezkuwi_approval_voting_parallel_worker_bounded_tof",
+						"Duration spent in a particular approval voting worker channel from entrance to removal",
+					)
+					.buckets(vec![
+						0.0001, 0.0004, 0.0016, 0.0064, 0.0256, 0.1024, 0.4096, 1.6384, 3.2768,
+						4.9152, 6.5536,
+					]),
+					&["worker_name"],
+				)?,
+				registry,
+			)?,
+			to_worker_bounded_sent: prometheus::register(
+				prometheus::GaugeVec::<prometheus::U64>::new(
+					prometheus::Opts::new(
+						"pezkuwi_approval_voting_parallel_worker_bounded_sent",
+						"Number of elements sent to approval voting workers' bounded queues",
+					),
+					&["worker_name"],
+				)?,
+				registry,
+			)?,
+			to_worker_bounded_received: prometheus::register(
+				prometheus::GaugeVec::<prometheus::U64>::new(
+					prometheus::Opts::new(
+						"pezkuwi_approval_voting_parallel_worker_bounded_received",
+						"Number of elements received by approval voting workers' bounded queues",
+					),
+					&["worker_name"],
+				)?,
+				registry,
+			)?,
+			to_worker_bounded_blocked: prometheus::register(
+				prometheus::GaugeVec::<prometheus::U64>::new(
+					prometheus::Opts::new(
+						"pezkuwi_approval_voting_parallel_worker_bounded_blocked",
+						"Number of times approval voting workers blocked while sending messages to a subsystem",
+					),
+					&["worker_name"],
+				)?,
+				registry,
+			)?,
+			to_worker_unbounded_tof: prometheus::register(
+				prometheus::HistogramVec::new(
+					prometheus::HistogramOpts::new(
+						"pezkuwi_approval_voting_parallel_worker_unbounded_tof",
+						"Duration spent in a particular approval voting worker channel from entrance to removal",
+					)
+					.buckets(vec![
+						0.0001, 0.0004, 0.0016, 0.0064, 0.0256, 0.1024, 0.4096, 1.6384, 3.2768,
+						4.9152, 6.5536,
+					]),
+					&["worker_name"],
+				)?,
+				registry,
+			)?,
+			to_worker_unbounded_sent: prometheus::register(
+				prometheus::GaugeVec::<prometheus::U64>::new(
+					prometheus::Opts::new(
+						"pezkuwi_approval_voting_parallel_worker_unbounded_sent",
+						"Number of elements sent to approval voting workers' unbounded queues",
+					),
+					&["worker_name"],
+				)?,
+				registry,
+			)?,
+			to_worker_unbounded_received: prometheus::register(
+				prometheus::GaugeVec::<prometheus::U64>::new(
+					prometheus::Opts::new(
+						"pezkuwi_approval_voting_parallel_worker_unbounded_received",
+						"Number of elements received by approval voting workers' unbounded queues",
+					),
+					&["worker_name"],
+				)?,
+				registry,
+			)?,
+		})))
+	}
+}
+
+/// The meters to watch.
+#[derive(Clone)]
+pub struct Meters {
+	bounded: Meter,
+	unbounded: Meter,
+}
+
+impl Meters {
+	pub fn new(bounded: &Meter, unbounded: &Meter) -> Self {
+		Self { bounded: bounded.clone(), unbounded: unbounded.clone() }
+	}
+}
+
+/// A metrics watcher that watches the meters and updates the metrics.
+pub struct MetricsWatcher {
+	to_watch: HashMap<String, Meters>,
+	metrics: Metrics,
+}
+
+impl MetricsWatcher {
+	/// Create a new metrics watcher.
+	pub fn new(metrics: Metrics) -> Self {
+		Self { to_watch: HashMap::new(), metrics }
+	}
+
+	/// Watch the meters of a worker with this name.
+	pub fn watch(&mut self, worker_name: String, meters: Meters) {
+		self.to_watch.insert(worker_name, meters);
+	}
+
+	/// Collect all the metrics.
+	pub fn collect_metrics(&self) {
+		for (name, meter) in &self.to_watch {
+			let bounded_readouts = meter.bounded.read();
+			let unbounded_readouts = meter.unbounded.read();
+			if let Some(metrics) = self.metrics.0.as_ref() {
+				metrics
+					.to_worker_bounded_sent
+					.with_label_values(&[name])
+					.set(bounded_readouts.sent as u64);
+
+				metrics
+					.to_worker_bounded_received
+					.with_label_values(&[name])
+					.set(bounded_readouts.received as u64);
+
+				metrics
+					.to_worker_bounded_blocked
+					.with_label_values(&[name])
+					.set(bounded_readouts.blocked as u64);
+
+				metrics
+					.to_worker_unbounded_sent
+					.with_label_values(&[name])
+					.set(unbounded_readouts.sent as u64);
+
+				metrics
+					.to_worker_unbounded_received
+					.with_label_values(&[name])
+					.set(unbounded_readouts.received as u64);
+
+				let hist_bounded = metrics.to_worker_bounded_tof.with_label_values(&[name]);
+				for tof in bounded_readouts.tof {
+					hist_bounded.observe(tof.as_f64());
+				}
+
+				let hist_unbounded = metrics.to_worker_unbounded_tof.with_label_values(&[name]);
+				for tof in unbounded_readouts.tof {
+					hist_unbounded.observe(tof.as_f64());
+				}
+			}
+		}
+	}
+}

pezkuwi/node/core/approval-voting-parallel/src/tests.rs

@@ -0,0 +1,982 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! The tests for Approval Voting Parallel Subsystem.
+
+use std::{
+	collections::{HashMap, HashSet},
+	future::Future,
+	sync::Arc,
+	time::Duration,
+};
+
+use crate::{
+	build_worker_handles, metrics::MetricsWatcher, prio_right, run_main_loop, start_workers,
+	validator_index_for_msg, ApprovalVotingParallelSubsystem, Metrics, WorkProvider,
+};
+use assert_matches::assert_matches;
+use futures::{channel::oneshot, future, stream::PollNext, StreamExt};
+use itertools::Itertools;
+use pezkuwi_node_core_approval_voting::{ApprovalVotingWorkProvider, Config};
+use pezkuwi_node_network_protocol::{peer_set::ValidationVersion, ObservedRole, PeerId, View};
+use pezkuwi_node_primitives::approval::{
+	time::SystemClock,
+	v1::RELAY_VRF_MODULO_CONTEXT,
+	v2::{
+		AssignmentCertKindV2, AssignmentCertV2, CoreBitfield, IndirectAssignmentCertV2,
+		IndirectSignedApprovalVoteV2,
+	},
+};
+use pezkuwi_node_subsystem::{
+	messages::{ApprovalDistributionMessage, ApprovalVotingMessage, ApprovalVotingParallelMessage},
+	FromOrchestra,
+};
+use pezkuwi_node_subsystem_test_helpers::{mock::new_leaf, TestSubsystemContext};
+use pezkuwi_overseer::{ActiveLeavesUpdate, OverseerSignal, SpawnGlue, TimeoutExt};
+use pezkuwi_primitives::{CandidateHash, CoreIndex, Hash, ValidatorIndex};
+use sc_keystore::{Keystore, LocalKeystore};
+use sp_consensus::SyncOracle;
+use sp_consensus_babe::{VrfPreOutput, VrfProof, VrfSignature};
+use sp_core::{testing::TaskExecutor, H256};
+use sp_keyring::Sr25519Keyring;
+type VirtualOverseer =
+	pezkuwi_node_subsystem_test_helpers::TestSubsystemContextHandle<ApprovalVotingParallelMessage>;
+
+const SLOT_DURATION_MILLIS: u64 = 6000;
+
+pub mod test_constants {
+	pub(crate) const DATA_COL: u32 = 0;
+	pub(crate) const NUM_COLUMNS: u32 = 1;
+}
+
+fn fake_assignment_cert_v2(
+	block_hash: Hash,
+	validator: ValidatorIndex,
+	core_bitfield: CoreBitfield,
+) -> IndirectAssignmentCertV2 {
+	let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT);
+	let msg = b"WhenTeyrchains?";
+	let mut prng = rand_core::OsRng;
+	let keypair = schnorrkel::Keypair::generate_with(&mut prng);
+	let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg));
+	let preout = inout.to_preout();
+
+	IndirectAssignmentCertV2 {
+		block_hash,
+		validator,
+		cert: AssignmentCertV2 {
+			kind: AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield },
+			vrf: VrfSignature { pre_output: VrfPreOutput(preout), proof: VrfProof(proof) },
+		},
+	}
+}
+
+/// Creates a meaningless signature
+pub fn dummy_signature() -> pezkuwi_primitives::ValidatorSignature {
+	sp_core::crypto::UncheckedFrom::unchecked_from([1u8; 64])
+}
+
+fn build_subsystem(
+	sync_oracle: Box<dyn SyncOracle + Send>,
+) -> (
+	ApprovalVotingParallelSubsystem,
+	TestSubsystemContext<ApprovalVotingParallelMessage, SpawnGlue<TaskExecutor>>,
+	VirtualOverseer,
+) {
+	sp_tracing::init_for_tests();
+
+	let pool = sp_core::testing::TaskExecutor::new();
+	let (context, virtual_overseer) = pezkuwi_node_subsystem_test_helpers::make_subsystem_context::<
+		ApprovalVotingParallelMessage,
+		_,
+	>(pool.clone());
+
+	let keystore = LocalKeystore::in_memory();
+	let _ = keystore.sr25519_generate_new(
+		pezkuwi_primitives::TEYRCHAIN_KEY_TYPE_ID,
+		Some(&Sr25519Keyring::Alice.to_seed()),
+	);
+
+	let clock = Arc::new(SystemClock {});
+	let db = kvdb_memorydb::create(test_constants::NUM_COLUMNS);
+	let db = pezkuwi_node_subsystem_util::database::kvdb_impl::DbAdapter::new(db, &[]);
+
+	(
+		ApprovalVotingParallelSubsystem::with_config_and_clock(
+			Config {
+				col_approval_data: test_constants::DATA_COL,
+				slot_duration_millis: SLOT_DURATION_MILLIS,
+			},
+			Arc::new(db),
+			Arc::new(keystore),
+			sync_oracle,
+			Metrics::default(),
+			clock.clone(),
+			SpawnGlue(pool),
+			None,
+		),
+		context,
+		virtual_overseer,
+	)
+}
+
+#[derive(Clone)]
+struct TestSyncOracle {}
+
+impl SyncOracle for TestSyncOracle {
+	fn is_major_syncing(&self) -> bool {
+		false
+	}
+
+	fn is_offline(&self) -> bool {
+		unimplemented!("not used in network bridge")
+	}
+}
+
+fn test_harness<T, Clos, State>(
+	num_approval_distro_workers: usize,
+	prio_right: Clos,
+	subsystem_gracefully_exits: bool,
+	test_fn: impl FnOnce(
+		VirtualOverseer,
+		WorkProvider<ApprovalVotingMessage, Clos, State>,
+		Vec<WorkProvider<ApprovalDistributionMessage, Clos, State>>,
+	) -> T,
+) where
+	T: Future<Output = VirtualOverseer>,
+	Clos: Clone + FnMut(&mut State) -> PollNext,
+	State: Default,
+{
+	let (subsystem, context, virtual_overseer) = build_subsystem(Box::new(TestSyncOracle {}));
+	let mut metrics_watcher = MetricsWatcher::new(subsystem.metrics.clone());
+	let channel_size = 5;
+
+	let (to_approval_voting_worker, approval_voting_work_provider) =
+		build_worker_handles::<ApprovalVotingMessage, _, _>(
+			"to_approval_voting_worker".into(),
+			channel_size,
+			&mut metrics_watcher,
+			prio_right.clone(),
+		);
+
+	let approval_distribution_channels = { 0..num_approval_distro_workers }
+		.into_iter()
+		.map(|worker_index| {
+			build_worker_handles::<ApprovalDistributionMessage, _, _>(
+				format!("to_approval_distro/{}", worker_index),
+				channel_size,
+				&mut metrics_watcher,
+				prio_right.clone(),
+			)
+		})
+		.collect_vec();
+
+	let to_approval_distribution_workers =
+		approval_distribution_channels.iter().map(|(tx, _)| tx.clone()).collect_vec();
+	let approval_distribution_work_providers =
+		approval_distribution_channels.into_iter().map(|(_, rx)| rx).collect_vec();
+
+	let subsystem = async move {
+		let result = run_main_loop(
+			context,
+			to_approval_voting_worker,
+			to_approval_distribution_workers,
+			metrics_watcher,
+		)
+		.await;
+
+		if subsystem_gracefully_exits && result.is_err() {
+			result
+		} else {
+			Ok(())
+		}
+	};
+
+	let test_fut = test_fn(
+		virtual_overseer,
+		approval_voting_work_provider,
+		approval_distribution_work_providers,
+	);
+
+	futures::pin_mut!(test_fut);
+	futures::pin_mut!(subsystem);
+
+	futures::executor::block_on(future::join(
+		async move {
+			let _overseer = test_fut.await;
+		},
+		subsystem,
+	))
+	.1
+	.unwrap();
+}
+
+const TIMEOUT: Duration = Duration::from_millis(2000);
+
+async fn overseer_signal(overseer: &mut VirtualOverseer, signal: OverseerSignal) {
+	overseer
+		.send(FromOrchestra::Signal(signal))
+		.timeout(TIMEOUT)
+		.await
+		.expect(&format!("{:?} is more than enough for sending signals.", TIMEOUT));
+}
+
+async fn overseer_message(overseer: &mut VirtualOverseer, msg: ApprovalVotingParallelMessage) {
+	overseer
+		.send(FromOrchestra::Communication { msg })
+		.timeout(TIMEOUT)
+		.await
+		.expect(&format!("{:?} is more than enough for sending signals.", TIMEOUT));
+}
+
+async fn run_start_workers() {
+	let (subsystem, mut context, _) = build_subsystem(Box::new(TestSyncOracle {}));
+	let mut metrics_watcher = MetricsWatcher::new(subsystem.metrics.clone());
+	let _workers = start_workers(&mut context, subsystem, &mut metrics_watcher).await.unwrap();
+}
+
+// Test starting the workers succeeds.
+#[test]
+fn start_workers_succeeds() {
+	futures::executor::block_on(run_start_workers());
+}
+
+// Test main loop forwards messages to the correct worker for all type of messages.
+#[test]
+fn test_main_loop_forwards_correctly() {
+	let num_approval_distro_workers = 4;
+	test_harness(
+		num_approval_distro_workers,
+		prio_right,
+		true,
+		|mut overseer, mut approval_voting_work_provider, mut rx_approval_distribution_workers| async move {
+			// 1. Check Signals are correctly forwarded to the workers.
+			let signal = OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf(
+				Hash::random(),
+				1,
+			)));
+			overseer_signal(&mut overseer, signal.clone()).await;
+			let approval_voting_receives = approval_voting_work_provider.recv().await.unwrap();
+			assert_matches!(approval_voting_receives, FromOrchestra::Signal(_));
+			for rx_approval_distribution_worker in rx_approval_distribution_workers.iter_mut() {
+				let approval_distribution_receives =
+					rx_approval_distribution_worker.next().await.unwrap();
+				assert_matches!(approval_distribution_receives, FromOrchestra::Signal(_));
+			}
+
+			let (test_tx, _rx) = oneshot::channel();
+			let test_hash = Hash::random();
+			let test_block_nr = 2;
+			overseer_message(
+				&mut overseer,
+				ApprovalVotingParallelMessage::ApprovedAncestor(test_hash, test_block_nr, test_tx),
+			)
+			.await;
+			assert_matches!(
+				approval_voting_work_provider.recv().await.unwrap(),
+				FromOrchestra::Communication {
+					msg: ApprovalVotingMessage::ApprovedAncestor(hash, block_nr, _)
+				} => {
+					assert_eq!(hash, test_hash);
+					assert_eq!(block_nr, test_block_nr);
+				}
+			);
+			for rx_approval_distribution_worker in rx_approval_distribution_workers.iter_mut() {
+				assert!(rx_approval_distribution_worker
+					.next()
+					.timeout(Duration::from_millis(200))
+					.await
+					.is_none());
+			}
+
+			// 2. Check GetApprovalSignaturesForCandidate is correctly forwarded to the workers.
+			let (test_tx, _rx) = oneshot::channel();
+			let test_hash = CandidateHash(Hash::random());
+			overseer_message(
+				&mut overseer,
+				ApprovalVotingParallelMessage::GetApprovalSignaturesForCandidate(
+					test_hash, test_tx,
+				),
+			)
+			.await;
+
+			assert_matches!(
+				approval_voting_work_provider.recv().await.unwrap(),
+				FromOrchestra::Communication {
+					msg: ApprovalVotingMessage::GetApprovalSignaturesForCandidate(hash, _)
+				} => {
+					assert_eq!(hash, test_hash);
+				}
+			);
+
+			for rx_approval_distribution_worker in rx_approval_distribution_workers.iter_mut() {
+				assert!(rx_approval_distribution_worker
+					.next()
+					.timeout(Duration::from_millis(200))
+					.await
+					.is_none());
+			}
+
+			// 3. Check NewBlocks is correctly forwarded to the workers.
+			overseer_message(&mut overseer, ApprovalVotingParallelMessage::NewBlocks(vec![])).await;
+			for rx_approval_distribution_worker in rx_approval_distribution_workers.iter_mut() {
+				assert_matches!(rx_approval_distribution_worker.next().await.unwrap(),
+					FromOrchestra::Communication {
+						msg: ApprovalDistributionMessage::NewBlocks(blocks)
+					} => {
+						assert!(blocks.is_empty());
+					}
+				);
+			}
+			assert!(approval_voting_work_provider
+				.recv()
+				.timeout(Duration::from_millis(200))
+				.await
+				.is_none());
+
+			// 4. Check DistributeAssignment is correctly forwarded to the workers.
+			let validator_index = ValidatorIndex(17);
+			let assignment =
+				fake_assignment_cert_v2(Hash::random(), validator_index, CoreIndex(1).into());
+			overseer_message(
+				&mut overseer,
+				ApprovalVotingParallelMessage::DistributeAssignment(assignment.clone(), 1.into()),
+			)
+			.await;
+
+			for (index, rx_approval_distribution_worker) in
+				rx_approval_distribution_workers.iter_mut().enumerate()
+			{
+				if index == validator_index.0 as usize % num_approval_distro_workers {
+					assert_matches!(rx_approval_distribution_worker.next().await.unwrap(),
+						FromOrchestra::Communication {
+							msg: ApprovalDistributionMessage::DistributeAssignment(cert, bitfield)
+						} => {
+							assert_eq!(cert, assignment);
+							assert_eq!(bitfield, 1.into());
+						}
+					);
+				} else {
+					assert!(rx_approval_distribution_worker
+						.next()
+						.timeout(Duration::from_millis(200))
+						.await
+						.is_none());
+				}
+			}
+			assert!(approval_voting_work_provider
+				.recv()
+				.timeout(Duration::from_millis(200))
+				.await
+				.is_none());
+
+			// 5. Check DistributeApproval is correctly forwarded to the workers.
+			let validator_index = ValidatorIndex(26);
+			let expected_vote = IndirectSignedApprovalVoteV2 {
+				block_hash: H256::random(),
+				candidate_indices: 1.into(),
+				validator: validator_index,
+				signature: dummy_signature(),
+			};
+
+			overseer_message(
+				&mut overseer,
+				ApprovalVotingParallelMessage::DistributeApproval(expected_vote.clone()),
+			)
+			.await;
+
+			for (index, rx_approval_distribution_worker) in
+				rx_approval_distribution_workers.iter_mut().enumerate()
+			{
+				if index == validator_index.0 as usize % num_approval_distro_workers {
+					assert_matches!(rx_approval_distribution_worker.next().await.unwrap(),
+						FromOrchestra::Communication {
+							msg: ApprovalDistributionMessage::DistributeApproval(vote)
+						} => {
+							assert_eq!(vote, expected_vote);
+						}
+					);
+				} else {
+					assert!(rx_approval_distribution_worker
+						.next()
+						.timeout(Duration::from_millis(200))
+						.await
+						.is_none());
+				}
+			}
+
+			// 6. Check NetworkBridgeUpdate::PeerMessage is correctly forwarded just to one of the
+			//    workers.
+			let approvals = vec![
+				IndirectSignedApprovalVoteV2 {
+					block_hash: H256::random(),
+					candidate_indices: 1.into(),
+					validator: validator_index,
+					signature: dummy_signature(),
+				},
+				IndirectSignedApprovalVoteV2 {
+					block_hash: H256::random(),
+					candidate_indices: 2.into(),
+					validator: validator_index,
+					signature: dummy_signature(),
+				},
+			];
+			let expected_msg = pezkuwi_node_network_protocol::ValidationProtocols::V3(
+				pezkuwi_node_network_protocol::v3::ApprovalDistributionMessage::Approvals(
+					approvals.clone(),
+				),
+			);
+			overseer_message(
+				&mut overseer,
+				ApprovalVotingParallelMessage::NetworkBridgeUpdate(
+					pezkuwi_node_subsystem::messages::NetworkBridgeEvent::PeerMessage(
+						PeerId::random(),
+						expected_msg.clone(),
+					),
+				),
+			)
+			.await;
+
+			for (index, rx_approval_distribution_worker) in
+				rx_approval_distribution_workers.iter_mut().enumerate()
+			{
+				if index == validator_index.0 as usize % num_approval_distro_workers {
+					assert_matches!(rx_approval_distribution_worker.next().await.unwrap(),
+						FromOrchestra::Communication {
+							msg: ApprovalDistributionMessage::NetworkBridgeUpdate(
+								pezkuwi_node_subsystem::messages::NetworkBridgeEvent::PeerMessage(
+									_,
+									msg,
+								),
+							)
+						} => {
+							assert_eq!(msg, expected_msg);
+						}
+					);
+				} else {
+					assert!(rx_approval_distribution_worker
+						.next()
+						.timeout(Duration::from_millis(200))
+						.await
+						.is_none());
+				}
+			}
+			assert!(approval_voting_work_provider
+				.recv()
+				.timeout(Duration::from_millis(200))
+				.await
+				.is_none());
+
+			assert!(approval_voting_work_provider
+				.recv()
+				.timeout(Duration::from_millis(200))
+				.await
+				.is_none());
+
+			// 7. Check NetworkBridgeUpdate::PeerConnected is correctly forwarded to all workers.
+			let expected_peer_id = PeerId::random();
+			overseer_message(
+				&mut overseer,
+				ApprovalVotingParallelMessage::NetworkBridgeUpdate(
+					pezkuwi_node_subsystem::messages::NetworkBridgeEvent::PeerConnected(
+						expected_peer_id,
+						ObservedRole::Authority,
+						ValidationVersion::V3.into(),
+						None,
+					),
+				),
+			)
+			.await;
+
+			for rx_approval_distribution_worker in rx_approval_distribution_workers.iter_mut() {
+				assert_matches!(rx_approval_distribution_worker.next().await.unwrap(),
+					FromOrchestra::Communication {
+						msg: ApprovalDistributionMessage::NetworkBridgeUpdate(
+							pezkuwi_node_subsystem::messages::NetworkBridgeEvent::PeerConnected(
+								peer_id,
+								role,
+								version,
+								authority_id,
+							),
+						)
+					} => {
+						assert_eq!(peer_id, expected_peer_id);
+						assert_eq!(role, ObservedRole::Authority);
+						assert_eq!(version, ValidationVersion::V3.into());
+						assert_eq!(authority_id, None);
+					}
+				);
+			}
+			assert!(approval_voting_work_provider
+				.recv()
+				.timeout(Duration::from_millis(200))
+				.await
+				.is_none());
+
+			// 8. Check ApprovalCheckingLagUpdate is correctly forwarded to all workers.
+			overseer_message(
+				&mut overseer,
+				ApprovalVotingParallelMessage::ApprovalCheckingLagUpdate(7),
+			)
+			.await;
+
+			for rx_approval_distribution_worker in rx_approval_distribution_workers.iter_mut() {
+				assert_matches!(rx_approval_distribution_worker.next().await.unwrap(),
+					FromOrchestra::Communication {
+						msg: ApprovalDistributionMessage::ApprovalCheckingLagUpdate(
+							lag
+						)
+					} => {
+						assert_eq!(lag, 7);
+					}
+				);
+			}
+			assert!(approval_voting_work_provider
+				.recv()
+				.timeout(Duration::from_millis(200))
+				.await
+				.is_none());
+
+			overseer_signal(&mut overseer, OverseerSignal::Conclude).await;
+
+			overseer
+		},
+	);
+}
+
+/// Test GetApprovalSignatures correctly gatheres the signatures from all workers.
+#[test]
+fn test_handle_get_approval_signatures() {
+	let num_approval_distro_workers = 4;
+
+	test_harness(
+		num_approval_distro_workers,
+		prio_right,
+		true,
+		|mut overseer, mut approval_voting_work_provider, mut rx_approval_distribution_workers| async move {
+			let (tx, rx) = oneshot::channel();
+			let first_block = Hash::random();
+			let second_block = Hash::random();
+			let expected_candidates: HashSet<_> =
+				vec![(first_block, 2), (second_block, 3)].into_iter().collect();
+
+			overseer_message(
+				&mut overseer,
+				ApprovalVotingParallelMessage::GetApprovalSignatures(
+					expected_candidates.clone(),
+					tx,
+				),
+			)
+			.await;
+
+			assert!(approval_voting_work_provider
+				.recv()
+				.timeout(Duration::from_millis(200))
+				.await
+				.is_none());
+			let mut all_votes = HashMap::new();
+			for (index, rx_approval_distribution_worker) in
+				rx_approval_distribution_workers.iter_mut().enumerate()
+			{
+				assert_matches!(rx_approval_distribution_worker.next().await.unwrap(),
+					FromOrchestra::Communication {
+						msg: ApprovalDistributionMessage::GetApprovalSignatures(
+							candidates, tx
+						)
+					} => {
+						assert_eq!(candidates, expected_candidates);
+						let to_send: HashMap<_, _> = {0..10}.into_iter().map(|validator| {
+							let validator_index = ValidatorIndex(validator as u32 * num_approval_distro_workers as u32 + index as u32);
+							(validator_index, (first_block, vec![2, 4], dummy_signature()))
+						}).collect();
+						tx.send(to_send.clone()).unwrap();
+						all_votes.extend(to_send.clone());
+
+					}
+				);
+			}
+
+			let received_votes = rx.await.unwrap();
+			assert_eq!(received_votes, all_votes);
+			overseer_signal(&mut overseer, OverseerSignal::Conclude).await;
+
+			overseer
+		},
+	)
+}
+
+/// Test subsystem exits with error when approval_voting_work_provider exits.
+#[test]
+fn test_subsystem_exits_with_error_if_approval_voting_worker_errors() {
+	let num_approval_distro_workers = 4;
+
+	test_harness(
+		num_approval_distro_workers,
+		prio_right,
+		false,
+		|overseer, approval_voting_work_provider, _rx_approval_distribution_workers| async move {
+			// Drop the approval_voting_work_provider to simulate an error.
+			std::mem::drop(approval_voting_work_provider);
+
+			overseer
+		},
+	)
+}
+
+/// Test subsystem exits with error when approval_distribution_workers exits.
+#[test]
+fn test_subsystem_exits_with_error_if_approval_distribution_worker_errors() {
+	let num_approval_distro_workers = 4;
+
+	test_harness(
+		num_approval_distro_workers,
+		prio_right,
+		false,
+		|overseer, _approval_voting_work_provider, rx_approval_distribution_workers| async move {
+			// Drop the approval_distribution_workers to simulate an error.
+			std::mem::drop(rx_approval_distribution_workers.into_iter().next().unwrap());
+			overseer
+		},
+	)
+}
+
+/// Test signals sent before messages are processed in order.
+#[test]
+fn test_signal_before_message_keeps_receive_order() {
+	let num_approval_distro_workers = 4;
+
+	test_harness(
+		num_approval_distro_workers,
+		prio_right,
+		true,
+		|mut overseer, mut approval_voting_work_provider, mut rx_approval_distribution_workers| async move {
+			let signal = OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf(
+				Hash::random(),
+				1,
+			)));
+			overseer_signal(&mut overseer, signal.clone()).await;
+
+			let validator_index = ValidatorIndex(17);
+			let assignment =
+				fake_assignment_cert_v2(Hash::random(), validator_index, CoreIndex(1).into());
+			overseer_message(
+				&mut overseer,
+				ApprovalVotingParallelMessage::DistributeAssignment(assignment.clone(), 1.into()),
+			)
+			.await;
+
+			let approval_voting_receives = approval_voting_work_provider.recv().await.unwrap();
+			assert_matches!(approval_voting_receives, FromOrchestra::Signal(_));
+			let rx_approval_distribution_worker = rx_approval_distribution_workers
+				.get_mut(validator_index.0 as usize % num_approval_distro_workers)
+				.unwrap();
+			let approval_distribution_receives =
+				rx_approval_distribution_worker.next().await.unwrap();
+			assert_matches!(approval_distribution_receives, FromOrchestra::Signal(_));
+			assert_matches!(
+				rx_approval_distribution_worker.next().await.unwrap(),
+				FromOrchestra::Communication {
+					msg: ApprovalDistributionMessage::DistributeAssignment(_, _)
+				}
+			);
+
+			overseer_signal(&mut overseer, OverseerSignal::Conclude).await;
+			overseer
+		},
+	)
+}
+
+/// Test signals sent after messages are processed with the highest priority.
+#[test]
+fn test_signal_is_prioritized_when_unread_messages_in_the_queue() {
+	let num_approval_distro_workers = 4;
+
+	test_harness(
+		num_approval_distro_workers,
+		prio_right,
+		true,
+		|mut overseer, mut approval_voting_work_provider, mut rx_approval_distribution_workers| async move {
+			let validator_index = ValidatorIndex(17);
+			let assignment =
+				fake_assignment_cert_v2(Hash::random(), validator_index, CoreIndex(1).into());
+			overseer_message(
+				&mut overseer,
+				ApprovalVotingParallelMessage::DistributeAssignment(assignment.clone(), 1.into()),
+			)
+			.await;
+
+			let signal = OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf(
+				Hash::random(),
+				1,
+			)));
+			overseer_signal(&mut overseer, signal.clone()).await;
+
+			let approval_voting_receives = approval_voting_work_provider.recv().await.unwrap();
+			assert_matches!(approval_voting_receives, FromOrchestra::Signal(_));
+			let rx_approval_distribution_worker = rx_approval_distribution_workers
+				.get_mut(validator_index.0 as usize % num_approval_distro_workers)
+				.unwrap();
+			let approval_distribution_receives =
+				rx_approval_distribution_worker.next().await.unwrap();
+			assert_matches!(approval_distribution_receives, FromOrchestra::Signal(_));
+			assert_matches!(
+				rx_approval_distribution_worker.next().await.unwrap(),
+				FromOrchestra::Communication {
+					msg: ApprovalDistributionMessage::DistributeAssignment(_, _)
+				}
+			);
+
+			overseer_signal(&mut overseer, OverseerSignal::Conclude).await;
+			overseer
+		},
+	)
+}
+
+/// Test peer view updates have higher priority than normal messages.
+#[test]
+fn test_peer_view_is_prioritized_when_unread_messages_in_the_queue() {
+	let num_approval_distro_workers = 4;
+
+	test_harness(
+		num_approval_distro_workers,
+		prio_right,
+		true,
+		|mut overseer, mut approval_voting_work_provider, mut rx_approval_distribution_workers| async move {
+			let validator_index = ValidatorIndex(17);
+			let approvals = vec![
+				IndirectSignedApprovalVoteV2 {
+					block_hash: H256::random(),
+					candidate_indices: 1.into(),
+					validator: validator_index,
+					signature: dummy_signature(),
+				},
+				IndirectSignedApprovalVoteV2 {
+					block_hash: H256::random(),
+					candidate_indices: 2.into(),
+					validator: validator_index,
+					signature: dummy_signature(),
+				},
+			];
+			let expected_msg = pezkuwi_node_network_protocol::ValidationProtocols::V3(
+				pezkuwi_node_network_protocol::v3::ApprovalDistributionMessage::Approvals(
+					approvals.clone(),
+				),
+			);
+			overseer_message(
+				&mut overseer,
+				ApprovalVotingParallelMessage::NetworkBridgeUpdate(
+					pezkuwi_node_subsystem::messages::NetworkBridgeEvent::PeerMessage(
+						PeerId::random(),
+						expected_msg.clone(),
+					),
+				),
+			)
+			.await;
+
+			overseer_message(
+				&mut overseer,
+				ApprovalVotingParallelMessage::NetworkBridgeUpdate(
+					pezkuwi_node_subsystem::messages::NetworkBridgeEvent::PeerViewChange(
+						PeerId::random(),
+						View::default(),
+					),
+				),
+			)
+			.await;
+
+			for (index, rx_approval_distribution_worker) in
+				rx_approval_distribution_workers.iter_mut().enumerate()
+			{
+				assert_matches!(rx_approval_distribution_worker.next().await.unwrap(),
+					FromOrchestra::Communication {
+						msg: ApprovalDistributionMessage::NetworkBridgeUpdate(
+							pezkuwi_node_subsystem::messages::NetworkBridgeEvent::PeerViewChange(
+								_,
+								_,
+							),
+						)
+					} => {
+					}
+				);
+				if index == validator_index.0 as usize % num_approval_distro_workers {
+					assert_matches!(rx_approval_distribution_worker.next().await.unwrap(),
+						FromOrchestra::Communication {
+							msg: ApprovalDistributionMessage::NetworkBridgeUpdate(
+								pezkuwi_node_subsystem::messages::NetworkBridgeEvent::PeerMessage(
+									_,
+									msg,
+								),
+							)
+						} => {
+							assert_eq!(msg, expected_msg);
+						}
+					);
+				} else {
+					assert!(rx_approval_distribution_worker
+						.next()
+						.timeout(Duration::from_millis(200))
+						.await
+						.is_none());
+				}
+			}
+
+			assert!(approval_voting_work_provider
+				.recv()
+				.timeout(Duration::from_millis(200))
+				.await
+				.is_none());
+
+			overseer_signal(&mut overseer, OverseerSignal::Conclude).await;
+			overseer
+		},
+	)
+}
+
+// Test validator_index_for_msg with empty messages.
+#[test]
+fn test_validator_index_with_empty_message() {
+	let result = validator_index_for_msg(pezkuwi_node_network_protocol::ValidationProtocols::V3(
+		pezkuwi_node_network_protocol::v3::ApprovalDistributionMessage::Assignments(vec![]),
+	));
+
+	assert_eq!(result, (None, Some(vec![])));
+
+	let result = validator_index_for_msg(pezkuwi_node_network_protocol::ValidationProtocols::V3(
+		pezkuwi_node_network_protocol::v3::ApprovalDistributionMessage::Approvals(vec![]),
+	));
+
+	assert_eq!(result, (None, Some(vec![])));
+}
+
+// Test validator_index_for_msg when all the messages are originating from the same validator.
+#[test]
+fn test_validator_index_with_all_messages_from_the_same_validator() {
+	let validator_index = ValidatorIndex(3);
+	let v3_assignment = pezkuwi_node_network_protocol::ValidationProtocols::V3(
+		pezkuwi_node_network_protocol::v3::ApprovalDistributionMessage::Assignments(vec![
+			(
+				fake_assignment_cert_v2(H256::random(), validator_index, CoreIndex(1).into()),
+				1.into(),
+			),
+			(
+				fake_assignment_cert_v2(H256::random(), validator_index, CoreIndex(3).into()),
+				3.into(),
+			),
+		]),
+	);
+	let result = validator_index_for_msg(v3_assignment.clone());
+
+	assert_eq!(result, (Some((validator_index, v3_assignment)), None));
+
+	let v3_approval = pezkuwi_node_network_protocol::ValidationProtocols::V3(
+		pezkuwi_node_network_protocol::v3::ApprovalDistributionMessage::Approvals(vec![
+			IndirectSignedApprovalVoteV2 {
+				block_hash: H256::random(),
+				candidate_indices: 1.into(),
+				validator: validator_index,
+				signature: dummy_signature(),
+			},
+			IndirectSignedApprovalVoteV2 {
+				block_hash: H256::random(),
+				candidate_indices: 1.into(),
+				validator: validator_index,
+				signature: dummy_signature(),
+			},
+		]),
+	);
+	let result = validator_index_for_msg(v3_approval.clone());
+
+	assert_eq!(result, (Some((validator_index, v3_approval)), None));
+}
+
+// Test validator_index_for_msg when all the messages are originating from different validators,
+// so the function should split them by validator index, so we can forward them separately to the
+// worker they are assigned to.
+#[test]
+fn test_validator_index_with_messages_from_different_validators() {
+	let first_validator_index = ValidatorIndex(3);
+	let second_validator_index = ValidatorIndex(4);
+	let assignments = vec![
+		(
+			fake_assignment_cert_v2(H256::random(), first_validator_index, CoreIndex(1).into()),
+			1.into(),
+		),
+		(
+			fake_assignment_cert_v2(H256::random(), second_validator_index, CoreIndex(3).into()),
+			3.into(),
+		),
+	];
+
+	let v3_assignment = pezkuwi_node_network_protocol::ValidationProtocols::V3(
+		pezkuwi_node_network_protocol::v3::ApprovalDistributionMessage::Assignments(
+			assignments.clone(),
+		),
+	);
+	let result = validator_index_for_msg(v3_assignment.clone());
+
+	assert_matches!(result, (None, Some(_)));
+	let messsages_split_by_validator = result.1.unwrap();
+	assert_eq!(messsages_split_by_validator.len(), assignments.len());
+	for (index, (validator_index, message)) in messsages_split_by_validator.into_iter().enumerate()
+	{
+		assert_eq!(validator_index, assignments[index].0.validator);
+		assert_eq!(
+			message,
+			pezkuwi_node_network_protocol::ValidationProtocols::V3(
+				pezkuwi_node_network_protocol::v3::ApprovalDistributionMessage::Assignments(
+					assignments.get(index).into_iter().cloned().collect(),
+				),
+			)
+		);
+	}
+
+	let approvals = vec![
+		IndirectSignedApprovalVoteV2 {
+			block_hash: H256::random(),
+			candidate_indices: 1.into(),
+			validator: first_validator_index,
+			signature: dummy_signature(),
+		},
+		IndirectSignedApprovalVoteV2 {
+			block_hash: H256::random(),
+			candidate_indices: 2.into(),
+			validator: second_validator_index,
+			signature: dummy_signature(),
+		},
+	];
+	let v3_approvals = pezkuwi_node_network_protocol::ValidationProtocols::V3(
+		pezkuwi_node_network_protocol::v3::ApprovalDistributionMessage::Approvals(
+			approvals.clone(),
+		),
+	);
+	let result = validator_index_for_msg(v3_approvals.clone());
+
+	assert_matches!(result, (None, Some(_)));
+	let messsages_split_by_validator = result.1.unwrap();
+	assert_eq!(messsages_split_by_validator.len(), approvals.len());
+	for (index, (validator_index, message)) in messsages_split_by_validator.into_iter().enumerate()
+	{
+		assert_eq!(validator_index, approvals[index].validator);
+		assert_eq!(
+			message,
+			pezkuwi_node_network_protocol::ValidationProtocols::V3(
+				pezkuwi_node_network_protocol::v3::ApprovalDistributionMessage::Approvals(
+					approvals.get(index).into_iter().cloned().collect(),
+				),
+			)
+		);
+	}
+}

pezkuwi/node/core/approval-voting/Cargo.toml

@@ -0,0 +1,82 @@
+[package]
+name = "pezkuwi-node-core-approval-voting"
+version = "7.0.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+description = "Approval Voting Subsystem of the Pezkuwi node"
+homepage.workspace = true
+repository.workspace = true
+
+[lints]
+workspace = true
+
+[[bench]]
+name = "approval-voting-regression-bench"
+path = "benches/approval-voting-regression-bench.rs"
+harness = false
+required-features = ["subsystem-benchmarks"]
+
+[dependencies]
+async-trait = { workspace = true }
+bitvec = { features = ["alloc"], workspace = true }
+codec = { features = ["bit-vec", "derive"], workspace = true }
+derive_more = { workspace = true, default-features = true }
+futures = { workspace = true }
+futures-timer = { workspace = true }
+gum = { workspace = true, default-features = true }
+itertools = { workspace = true }
+merlin = { workspace = true, default-features = true }
+schnellru = { workspace = true }
+schnorrkel = { workspace = true, default-features = true }
+thiserror = { workspace = true }
+
+pezkuwi-node-primitives = { workspace = true, default-features = true }
+pezkuwi-node-subsystem = { workspace = true, default-features = true }
+pezkuwi-node-subsystem-util = { workspace = true, default-features = true }
+pezkuwi-overseer = { workspace = true, default-features = true }
+pezkuwi-primitives = { workspace = true, default-features = true }
+
+rand = { workspace = true, default-features = true }
+rand_chacha = { workspace = true, default-features = true }
+# rand_core should match schnorrkel
+rand_core = { workspace = true }
+sc-keystore = { workspace = true }
+sp-application-crypto = { features = ["full_crypto"], workspace = true }
+sp-consensus = { workspace = true }
+sp-consensus-slots = { workspace = true }
+sp-runtime = { workspace = true }
+
+[dev-dependencies]
+assert_matches = { workspace = true }
+kvdb-memorydb = { workspace = true }
+parking_lot = { workspace = true, default-features = true }
+pezkuwi-node-subsystem-test-helpers = { workspace = true }
+pezkuwi-primitives = { workspace = true, features = ["test"] }
+pezkuwi-primitives-test-helpers = { workspace = true }
+sp-consensus-babe = { workspace = true, default-features = true }
+sp-core = { workspace = true, default-features = true }
+sp-keyring = { workspace = true, default-features = true }
+sp-keystore = { workspace = true, default-features = true }
+sp-tracing = { workspace = true }
+
+pezkuwi-subsystem-bench = { workspace = true }
+
+[features]
+subsystem-benchmarks = []
+runtime-benchmarks = [
+	"gum/runtime-benchmarks",
+	"pezkuwi-node-primitives/runtime-benchmarks",
+	"pezkuwi-node-subsystem-test-helpers/runtime-benchmarks",
+	"pezkuwi-node-subsystem-util/runtime-benchmarks",
+	"pezkuwi-node-subsystem/runtime-benchmarks",
+	"pezkuwi-overseer/runtime-benchmarks",
+	"pezkuwi-primitives-test-helpers/runtime-benchmarks",
+	"pezkuwi-primitives/runtime-benchmarks",
+	"pezkuwi-subsystem-bench/runtime-benchmarks",
+	"sp-consensus-babe/runtime-benchmarks",
+	"sp-consensus-slots/runtime-benchmarks",
+	"sp-consensus/runtime-benchmarks",
+	"sp-keyring/runtime-benchmarks",
+	"sp-runtime/runtime-benchmarks",
+]

pezkuwi/node/core/approval-voting/benches/approval-voting-regression-bench.rs

@@ -0,0 +1,93 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! approval-voting throughput test
+//!
+//! Approval Voting benchmark based on Kusama parameters and scale.
+//!
+//! Subsystems involved:
+//! - approval-distribution
+//! - approval-voting
+
+use pezkuwi_subsystem_bench::{
+	self,
+	approval::{bench_approvals, prepare_test, ApprovalsOptions},
+	configuration::TestConfiguration,
+	usage::BenchmarkUsage,
+	utils::save_to_file,
+};
+use std::io::Write;
+
+const BENCH_COUNT: usize = 10;
+
+fn main() -> Result<(), String> {
+	let mut messages = vec![];
+	let mut config = TestConfiguration::default();
+	config.n_cores = 100;
+	config.n_validators = 500;
+	config.num_blocks = 10;
+	config.peer_bandwidth = 524288000000;
+	config.bandwidth = 524288000000;
+	config.latency = None;
+	config.connectivity = 100;
+	config.generate_pov_sizes();
+	let options = ApprovalsOptions {
+		last_considered_tranche: 89,
+		coalesce_mean: 3.0,
+		coalesce_std_dev: 1.0,
+		coalesce_tranche_diff: 12,
+		enable_assignments_v2: true,
+		stop_when_approved: false,
+		workdir_prefix: "/tmp".to_string(),
+		num_no_shows_per_candidate: 0,
+		approval_voting_parallel_enabled: true,
+	};
+
+	println!("Benchmarking...");
+	let usages: Vec<BenchmarkUsage> = (0..BENCH_COUNT)
+		.map(|n| {
+			print!("\r[{}{}]", "#".repeat(n), "_".repeat(BENCH_COUNT - n));
+			std::io::stdout().flush().unwrap();
+			let (mut env, state) = prepare_test(config.clone(), options.clone(), false);
+			env.runtime().block_on(bench_approvals(&mut env, state))
+		})
+		.collect();
+	println!("\rDone!{}", " ".repeat(BENCH_COUNT));
+
+	let average_usage = BenchmarkUsage::average(&usages);
+	save_to_file(
+		"charts/approval-voting-regression-bench.json",
+		average_usage.to_chart_json().map_err(|e| e.to_string())?,
+	)
+	.map_err(|e| e.to_string())?;
+	println!("{}", average_usage);
+
+	// We expect some small variance for received and sent because the
+	// test messages are generated at every benchmark run and they contain
+	// random data so use 0.01 as the accepted variance.
+	messages.extend(average_usage.check_network_usage(&[
+		("Received from peers", 52941.6071, 0.01),
+		("Sent to peers", 63995.2200, 0.01),
+	]));
+	messages.extend(average_usage.check_cpu_usage(&[("approval-voting-parallel", 12.3817, 0.1)]));
+
+	if messages.is_empty() {
+		Ok(())
+	} else {
+		eprintln!("{}", messages.join("\n"));
+		Err("Regressions found".to_string())
+	}
+}

pezkuwi/node/core/approval-voting/src/approval_db/common/migration_helpers.rs

@@ -0,0 +1,40 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec};
+
+use pezkuwi_node_primitives::approval::{
+	v1::{AssignmentCert, AssignmentCertKind, VrfProof, VrfSignature, RELAY_VRF_MODULO_CONTEXT},
+	v2::VrfPreOutput,
+};
+
+pub fn make_bitvec(len: usize) -> BitVec<u8, BitOrderLsb0> {
+	bitvec::bitvec![u8, BitOrderLsb0; 0; len]
+}
+
+pub fn dummy_assignment_cert(kind: AssignmentCertKind) -> AssignmentCert {
+	let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT);
+	let msg = b"test-garbage";
+	let mut prng = rand_core::OsRng;
+	let keypair = schnorrkel::Keypair::generate_with(&mut prng);
+	let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg));
+	let preout = inout.to_preout();
+
+	AssignmentCert {
+		kind,
+		vrf: VrfSignature { pre_output: VrfPreOutput(preout), proof: VrfProof(proof) },
+	}
+}

pezkuwi/node/core/approval-voting/src/approval_db/common/mod.rs

@@ -0,0 +1,293 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Common helper functions for all versions of approval-voting database.
+use std::sync::Arc;
+
+use codec::{Decode, Encode};
+use pezkuwi_node_subsystem::{SubsystemError, SubsystemResult};
+use pezkuwi_node_subsystem_util::database::{DBTransaction, Database};
+use pezkuwi_primitives::{BlockNumber, CandidateHash, CandidateIndex, Hash};
+
+use crate::{
+	backend::{Backend, BackendWriteOp, V1ReadBackend, V2ReadBackend},
+	persisted_entries,
+};
+
+use super::{
+	v2::{load_block_entry_v1, load_candidate_entry_v1},
+	v3::{load_block_entry_v2, load_candidate_entry_v2, BlockEntry, CandidateEntry},
+};
+
+pub mod migration_helpers;
+
+const STORED_BLOCKS_KEY: &[u8] = b"Approvals_StoredBlocks";
+
+/// A range from earliest..last block number stored within the DB.
+#[derive(Encode, Decode, Debug, Clone, PartialEq)]
+pub struct StoredBlockRange(pub BlockNumber, pub BlockNumber);
+/// The database config.
+#[derive(Debug, Clone, Copy)]
+pub struct Config {
+	/// The column family in the database where data is stored.
+	pub col_approval_data: u32,
+}
+
+/// `DbBackend` is a concrete implementation of the higher-level Backend trait
+pub struct DbBackend {
+	inner: Arc<dyn Database>,
+	config: Config,
+}
+
+impl DbBackend {
+	/// Create a new [`DbBackend`] with the supplied key-value store and
+	/// config.
+	pub fn new(db: Arc<dyn Database>, config: Config) -> Self {
+		DbBackend { inner: db, config }
+	}
+}
+
+/// Errors while accessing things from the DB.
+#[derive(Debug, derive_more::From, derive_more::Display)]
+pub enum Error {
+	Io(std::io::Error),
+	InvalidDecoding(codec::Error),
+	InternalError(SubsystemError),
+}
+
+impl std::error::Error for Error {}
+
+/// Result alias for DB errors.
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl Backend for DbBackend {
+	fn load_block_entry(
+		&self,
+		block_hash: &Hash,
+	) -> SubsystemResult<Option<persisted_entries::BlockEntry>> {
+		load_block_entry(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into))
+	}
+
+	fn load_candidate_entry(
+		&self,
+		candidate_hash: &CandidateHash,
+	) -> SubsystemResult<Option<persisted_entries::CandidateEntry>> {
+		load_candidate_entry(&*self.inner, &self.config, candidate_hash).map(|e| e.map(Into::into))
+	}
+
+	fn load_blocks_at_height(&self, block_height: &BlockNumber) -> SubsystemResult<Vec<Hash>> {
+		load_blocks_at_height(&*self.inner, &self.config, block_height)
+	}
+
+	fn load_all_blocks(&self) -> SubsystemResult<Vec<Hash>> {
+		load_all_blocks(&*self.inner, &self.config)
+	}
+
+	fn load_stored_blocks(&self) -> SubsystemResult<Option<StoredBlockRange>> {
+		load_stored_blocks(&*self.inner, &self.config)
+	}
+
+	/// Atomically write the list of operations, with later operations taking precedence over prior.
+	fn write<I>(&mut self, ops: I) -> SubsystemResult<()>
+	where
+		I: IntoIterator<Item = BackendWriteOp>,
+	{
+		let mut tx = DBTransaction::new();
+		for op in ops {
+			match op {
+				BackendWriteOp::WriteStoredBlockRange(stored_block_range) => {
+					tx.put_vec(
+						self.config.col_approval_data,
+						&STORED_BLOCKS_KEY,
+						stored_block_range.encode(),
+					);
+				},
+				BackendWriteOp::DeleteStoredBlockRange => {
+					tx.delete(self.config.col_approval_data, &STORED_BLOCKS_KEY);
+				},
+				BackendWriteOp::WriteBlocksAtHeight(h, blocks) => {
+					tx.put_vec(
+						self.config.col_approval_data,
+						&blocks_at_height_key(h),
+						blocks.encode(),
+					);
+				},
+				BackendWriteOp::DeleteBlocksAtHeight(h) => {
+					tx.delete(self.config.col_approval_data, &blocks_at_height_key(h));
+				},
+				BackendWriteOp::WriteBlockEntry(block_entry) => {
+					let block_entry: BlockEntry = block_entry.into();
+					tx.put_vec(
+						self.config.col_approval_data,
+						&block_entry_key(&block_entry.block_hash),
+						block_entry.encode(),
+					);
+				},
+				BackendWriteOp::DeleteBlockEntry(hash) => {
+					tx.delete(self.config.col_approval_data, &block_entry_key(&hash));
+				},
+				BackendWriteOp::WriteCandidateEntry(candidate_entry) => {
+					let candidate_entry: CandidateEntry = candidate_entry.into();
+					tx.put_vec(
+						self.config.col_approval_data,
+						&candidate_entry_key(&candidate_entry.candidate.hash()),
+						candidate_entry.encode(),
+					);
+				},
+				BackendWriteOp::DeleteCandidateEntry(candidate_hash) => {
+					tx.delete(self.config.col_approval_data, &candidate_entry_key(&candidate_hash));
+				},
+			}
+		}
+
+		self.inner.write(tx).map_err(|e| e.into())
+	}
+}
+
+impl V1ReadBackend for DbBackend {
+	fn load_candidate_entry_v1(
+		&self,
+		candidate_hash: &CandidateHash,
+		candidate_index: CandidateIndex,
+	) -> SubsystemResult<Option<persisted_entries::CandidateEntry>> {
+		load_candidate_entry_v1(&*self.inner, &self.config, candidate_hash)
+			.map(|e| e.map(|e| persisted_entries::CandidateEntry::from_v1(e, candidate_index)))
+	}
+
+	fn load_block_entry_v1(
+		&self,
+		block_hash: &Hash,
+	) -> SubsystemResult<Option<persisted_entries::BlockEntry>> {
+		load_block_entry_v1(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into))
+	}
+}
+
+impl V2ReadBackend for DbBackend {
+	fn load_candidate_entry_v2(
+		&self,
+		candidate_hash: &CandidateHash,
+		candidate_index: CandidateIndex,
+	) -> SubsystemResult<Option<persisted_entries::CandidateEntry>> {
+		load_candidate_entry_v2(&*self.inner, &self.config, candidate_hash)
+			.map(|e| e.map(|e| persisted_entries::CandidateEntry::from_v2(e, candidate_index)))
+	}
+
+	fn load_block_entry_v2(
+		&self,
+		block_hash: &Hash,
+	) -> SubsystemResult<Option<persisted_entries::BlockEntry>> {
+		load_block_entry_v2(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into))
+	}
+}
+
+pub(crate) fn load_decode<D: Decode>(
+	store: &dyn Database,
+	col_approval_data: u32,
+	key: &[u8],
+) -> Result<Option<D>> {
+	match store.get(col_approval_data, key)? {
+		None => Ok(None),
+		Some(raw) => D::decode(&mut &raw[..]).map(Some).map_err(Into::into),
+	}
+}
+
+/// The key a given block entry is stored under.
+pub(crate) fn block_entry_key(block_hash: &Hash) -> [u8; 46] {
+	const BLOCK_ENTRY_PREFIX: [u8; 14] = *b"Approvals_blck";
+
+	let mut key = [0u8; 14 + 32];
+	key[0..14].copy_from_slice(&BLOCK_ENTRY_PREFIX);
+	key[14..][..32].copy_from_slice(block_hash.as_ref());
+
+	key
+}
+
+/// The key a given candidate entry is stored under.
+pub(crate) fn candidate_entry_key(candidate_hash: &CandidateHash) -> [u8; 46] {
+	const CANDIDATE_ENTRY_PREFIX: [u8; 14] = *b"Approvals_cand";
+
+	let mut key = [0u8; 14 + 32];
+	key[0..14].copy_from_slice(&CANDIDATE_ENTRY_PREFIX);
+	key[14..][..32].copy_from_slice(candidate_hash.0.as_ref());
+
+	key
+}
+
+/// The key a set of block hashes corresponding to a block number is stored under.
+pub(crate) fn blocks_at_height_key(block_number: BlockNumber) -> [u8; 16] {
+	const BLOCKS_AT_HEIGHT_PREFIX: [u8; 12] = *b"Approvals_at";
+
+	let mut key = [0u8; 12 + 4];
+	key[0..12].copy_from_slice(&BLOCKS_AT_HEIGHT_PREFIX);
+	block_number.using_encoded(|s| key[12..16].copy_from_slice(s));
+
+	key
+}
+
+/// Return all blocks which have entries in the DB, ascending, by height.
+pub fn load_all_blocks(store: &dyn Database, config: &Config) -> SubsystemResult<Vec<Hash>> {
+	let mut hashes = Vec::new();
+	if let Some(stored_blocks) = load_stored_blocks(store, config)? {
+		for height in stored_blocks.0..stored_blocks.1 {
+			let blocks = load_blocks_at_height(store, config, &height)?;
+			hashes.extend(blocks);
+		}
+	}
+
+	Ok(hashes)
+}
+
+/// Load the stored-blocks key from the state.
+pub fn load_stored_blocks(
+	store: &dyn Database,
+	config: &Config,
+) -> SubsystemResult<Option<StoredBlockRange>> {
+	load_decode(store, config.col_approval_data, STORED_BLOCKS_KEY)
+		.map_err(|e| SubsystemError::with_origin("approval-voting", e))
+}
+
+/// Load a blocks-at-height entry for a given block number.
+pub fn load_blocks_at_height(
+	store: &dyn Database,
+	config: &Config,
+	block_number: &BlockNumber,
+) -> SubsystemResult<Vec<Hash>> {
+	load_decode(store, config.col_approval_data, &blocks_at_height_key(*block_number))
+		.map(|x| x.unwrap_or_default())
+		.map_err(|e| SubsystemError::with_origin("approval-voting", e))
+}
+
+/// Load a block entry from the aux store.
+pub fn load_block_entry(
+	store: &dyn Database,
+	config: &Config,
+	block_hash: &Hash,
+) -> SubsystemResult<Option<BlockEntry>> {
+	load_decode(store, config.col_approval_data, &block_entry_key(block_hash))
+		.map(|u: Option<BlockEntry>| u.map(|v| v.into()))
+		.map_err(|e| SubsystemError::with_origin("approval-voting", e))
+}
+
+/// Load a candidate entry from the aux store in current version format.
+pub fn load_candidate_entry(
+	store: &dyn Database,
+	config: &Config,
+	candidate_hash: &CandidateHash,
+) -> SubsystemResult<Option<CandidateEntry>> {
+	load_decode(store, config.col_approval_data, &candidate_entry_key(candidate_hash))
+		.map(|u: Option<CandidateEntry>| u.map(|v| v.into()))
+		.map_err(|e| SubsystemError::with_origin("approval-voting", e))
+}

pezkuwi/node/core/approval-voting/src/approval_db/mod.rs

@@ -0,0 +1,36 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Approval DB accessors and writers for on-disk persisted approval storage
+//! data.
+//!
+//! We persist data to disk although it is not intended to be used across runs of the
+//! program. This is because under medium to long periods of finality stalling, for whatever
+//! reason that may be, the amount of data we'd need to keep would be potentially too large
+//! for memory.
+//!
+//! With tens or hundreds of teyrchains, hundreds of validators, and parablocks
+//! in every relay chain block, there can be a humongous amount of information to reference
+//! at any given time.
+//!
+//! As such, we provide a function from this module to clear the database on start-up.
+//! In the future, we may use a temporary DB which doesn't need to be wiped, but for the
+//! time being we share the same DB with the rest of Substrate.
+
+pub mod common;
+pub mod v1;
+pub mod v2;
+pub mod v3;

pezkuwi/node/core/approval-voting/src/approval_db/v1/mod.rs

@@ -0,0 +1,91 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Version 1 of the DB schema.
+//!
+//! Note that the version here differs from the actual version of the teyrchains
+//! database (check `CURRENT_VERSION` in `node/service/src/teyrchains_db/upgrade.rs`).
+//! The code in this module implements the way approval voting works with
+//! its data in the database. Any breaking changes here will still
+//! require a db migration (check `node/service/src/teyrchains_db/upgrade.rs`).
+
+use codec::{Decode, Encode};
+use pezkuwi_node_primitives::approval::v1::{AssignmentCert, DelayTranche};
+use pezkuwi_primitives::{
+	BlockNumber, CandidateHash, CandidateReceiptV2 as CandidateReceipt, CoreIndex, GroupIndex,
+	Hash, SessionIndex, ValidatorIndex, ValidatorSignature,
+};
+use sp_consensus_slots::Slot;
+use std::collections::BTreeMap;
+
+use super::v2::Bitfield;
+
+/// Details pertaining to our assignment on a block.
+#[derive(Encode, Decode, Debug, Clone, PartialEq)]
+pub struct OurAssignment {
+	pub cert: AssignmentCert,
+	pub tranche: DelayTranche,
+	pub validator_index: ValidatorIndex,
+	// Whether the assignment has been triggered already.
+	pub triggered: bool,
+}
+use super::v2::TrancheEntry;
+
+/// Metadata regarding approval of a particular candidate within the context of some
+/// particular block.
+#[derive(Encode, Decode, Debug, Clone, PartialEq)]
+pub struct ApprovalEntry {
+	pub tranches: Vec<TrancheEntry>,
+	pub backing_group: GroupIndex,
+	pub our_assignment: Option<OurAssignment>,
+	pub our_approval_sig: Option<ValidatorSignature>,
+	// `n_validators` bits.
+	pub assignments: Bitfield,
+	pub approved: bool,
+}
+
+/// Metadata regarding approval of a particular candidate.
+#[derive(Encode, Decode, Debug, Clone, PartialEq)]
+pub struct CandidateEntry {
+	pub candidate: CandidateReceipt,
+	pub session: SessionIndex,
+	// Assignments are based on blocks, so we need to track assignments separately
+	// based on the block we are looking at.
+	pub block_assignments: BTreeMap<Hash, ApprovalEntry>,
+	pub approvals: Bitfield,
+}
+
+/// Metadata regarding approval of a particular block, by way of approval of the
+/// candidates contained within it.
+#[derive(Encode, Decode, Debug, Clone, PartialEq)]
+pub struct BlockEntry {
+	pub block_hash: Hash,
+	pub block_number: BlockNumber,
+	pub parent_hash: Hash,
+	pub session: SessionIndex,
+	pub slot: Slot,
+	/// Random bytes derived from the VRF submitted within the block by the block
+	/// author as a credential and used as input to approval assignment criteria.
+	pub relay_vrf_story: [u8; 32],
+	// The candidates included as-of this block and the index of the core they are
+	// leaving. Sorted ascending by core index.
+	pub candidates: Vec<(CoreIndex, CandidateHash)>,
+	// A bitfield where the i'th bit corresponds to the i'th candidate in `candidates`.
+	// The i'th bit is `true` iff the candidate has been approved in the context of this
+	// block. The block can be considered approved if the bitfield has all bits set to `true`.
+	pub approved_bitfield: Bitfield,
+	pub children: Vec<Hash>,
+}

pezkuwi/node/core/approval-voting/src/approval_db/v1/tests.rs

@@ -0,0 +1,567 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Tests for the aux-schema of approval voting.
+
+use super::{DbBackend, StoredBlockRange, *};
+use crate::{
+	backend::{Backend, OverlayedBackend},
+	ops::{add_block_entry, canonicalize, force_approve, NewCandidateInfo},
+};
+use pezkuwi_node_subsystem_util::database::Database;
+use pezkuwi_primitives::Id as ParaId;
+use std::{collections::HashMap, sync::Arc};
+
+use pezkuwi_primitives_test_helpers::{
+	dummy_candidate_receipt, dummy_candidate_receipt_bad_sig, dummy_hash,
+};
+
+const DATA_COL: u32 = 0;
+
+const NUM_COLUMNS: u32 = 1;
+
+const TEST_CONFIG: Config = Config { col_approval_data: DATA_COL };
+
+fn make_db() -> (DbBackend, Arc<dyn Database>) {
+	let db = kvdb_memorydb::create(NUM_COLUMNS);
+	let db = pezkuwi_node_subsystem_util::database::kvdb_impl::DbAdapter::new(db, &[]);
+	let db_writer: Arc<dyn Database> = Arc::new(db);
+	(DbBackend::new(db_writer.clone(), TEST_CONFIG), db_writer)
+}
+
+fn make_block_entry(
+	block_hash: Hash,
+	parent_hash: Hash,
+	block_number: BlockNumber,
+	candidates: Vec<(CoreIndex, CandidateHash)>,
+) -> BlockEntry {
+	BlockEntry {
+		block_hash,
+		parent_hash,
+		block_number,
+		session: 1,
+		slot: Slot::from(1),
+		relay_vrf_story: [0u8; 32],
+		approved_bitfield: make_bitvec(candidates.len()),
+		candidates,
+		children: Vec::new(),
+	}
+}
+
+fn make_candidate(para_id: ParaId, relay_parent: Hash) -> CandidateReceipt {
+	let mut c = dummy_candidate_receipt(dummy_hash());
+
+	c.descriptor.para_id = para_id;
+	c.descriptor.relay_parent = relay_parent;
+
+	c
+}
+
+#[test]
+fn read_write() {
+	let (mut db, store) = make_db();
+
+	let hash_a = Hash::repeat_byte(1);
+	let hash_b = Hash::repeat_byte(2);
+	let candidate_hash = dummy_candidate_receipt_bad_sig(dummy_hash(), None).hash();
+
+	let range = StoredBlockRange(10, 20);
+	let at_height = vec![hash_a, hash_b];
+
+	let block_entry =
+		make_block_entry(hash_a, Default::default(), 1, vec![(CoreIndex(0), candidate_hash)]);
+
+	let candidate_entry = CandidateEntry {
+		candidate: dummy_candidate_receipt_bad_sig(dummy_hash(), None),
+		session: 5,
+		block_assignments: vec![(
+			hash_a,
+			ApprovalEntry {
+				tranches: Vec::new(),
+				backing_group: GroupIndex(1),
+				our_assignment: None,
+				our_approval_sig: None,
+				assignments: Default::default(),
+				approved: false,
+			},
+		)]
+		.into_iter()
+		.collect(),
+		approvals: Default::default(),
+	};
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	overlay_db.write_stored_block_range(range.clone());
+	overlay_db.write_blocks_at_height(1, at_height.clone());
+	overlay_db.write_block_entry(block_entry.clone().into());
+	overlay_db.write_candidate_entry(candidate_entry.clone().into());
+
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	assert_eq!(load_stored_blocks(store.as_ref(), &TEST_CONFIG).unwrap(), Some(range));
+	assert_eq!(load_blocks_at_height(store.as_ref(), &TEST_CONFIG, &1).unwrap(), at_height);
+	assert_eq!(
+		load_block_entry(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap(),
+		Some(block_entry.into())
+	);
+	assert_eq!(
+		load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash).unwrap(),
+		Some(candidate_entry.into()),
+	);
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	overlay_db.delete_blocks_at_height(1);
+	overlay_db.delete_block_entry(&hash_a);
+	overlay_db.delete_candidate_entry(&candidate_hash);
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	assert!(load_blocks_at_height(store.as_ref(), &TEST_CONFIG, &1).unwrap().is_empty());
+	assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap().is_none());
+	assert!(load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash)
+		.unwrap()
+		.is_none());
+}
+
+#[test]
+fn add_block_entry_works() {
+	let (mut db, store) = make_db();
+
+	let parent_hash = Hash::repeat_byte(1);
+	let block_hash_a = Hash::repeat_byte(2);
+	let block_hash_b = Hash::repeat_byte(69);
+
+	let candidate_receipt_a = make_candidate(ParaId::from(1_u32), parent_hash);
+	let candidate_receipt_b = make_candidate(ParaId::from(2_u32), parent_hash);
+
+	let candidate_hash_a = candidate_receipt_a.hash();
+	let candidate_hash_b = candidate_receipt_b.hash();
+
+	let block_number = 10;
+
+	let block_entry_a = make_block_entry(
+		block_hash_a,
+		parent_hash,
+		block_number,
+		vec![(CoreIndex(0), candidate_hash_a)],
+	);
+
+	let block_entry_b = make_block_entry(
+		block_hash_b,
+		parent_hash,
+		block_number,
+		vec![(CoreIndex(0), candidate_hash_a), (CoreIndex(1), candidate_hash_b)],
+	);
+
+	let n_validators = 10;
+
+	let mut new_candidate_info = HashMap::new();
+	new_candidate_info
+		.insert(candidate_hash_a, NewCandidateInfo::new(candidate_receipt_a, GroupIndex(0), None));
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |h| {
+		new_candidate_info.get(h).map(|x| x.clone())
+	})
+	.unwrap();
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	new_candidate_info
+		.insert(candidate_hash_b, NewCandidateInfo::new(candidate_receipt_b, GroupIndex(1), None));
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |h| {
+		new_candidate_info.get(h).map(|x| x.clone())
+	})
+	.unwrap();
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	assert_eq!(
+		load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(),
+		Some(block_entry_a.into())
+	);
+	assert_eq!(
+		load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(),
+		Some(block_entry_b.into())
+	);
+
+	let candidate_entry_a = load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash_a)
+		.unwrap()
+		.unwrap();
+	assert_eq!(
+		candidate_entry_a.block_assignments.keys().collect::<Vec<_>>(),
+		vec![&block_hash_a, &block_hash_b]
+	);
+
+	let candidate_entry_b = load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash_b)
+		.unwrap()
+		.unwrap();
+	assert_eq!(candidate_entry_b.block_assignments.keys().collect::<Vec<_>>(), vec![&block_hash_b]);
+}
+
+#[test]
+fn add_block_entry_adds_child() {
+	let (mut db, store) = make_db();
+
+	let parent_hash = Hash::repeat_byte(1);
+	let block_hash_a = Hash::repeat_byte(2);
+	let block_hash_b = Hash::repeat_byte(69);
+
+	let mut block_entry_a = make_block_entry(block_hash_a, parent_hash, 1, Vec::new());
+
+	let block_entry_b = make_block_entry(block_hash_b, block_hash_a, 2, Vec::new());
+
+	let n_validators = 10;
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |_| None).unwrap();
+
+	add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |_| None).unwrap();
+
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	block_entry_a.children.push(block_hash_b);
+
+	assert_eq!(
+		load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(),
+		Some(block_entry_a.into())
+	);
+	assert_eq!(
+		load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(),
+		Some(block_entry_b.into())
+	);
+}
+
+#[test]
+fn canonicalize_works() {
+	let (mut db, store) = make_db();
+
+	//   -> B1 -> C1 -> D1
+	// A -> B2 -> C2 -> D2
+	//
+	// We'll canonicalize C1. Everything except D1 should disappear.
+	//
+	// Candidates:
+	// Cand1 in B2
+	// Cand2 in C2
+	// Cand3 in C2 and D1
+	// Cand4 in D1
+	// Cand5 in D2
+	// Only Cand3 and Cand4 should remain after canonicalize.
+
+	let n_validators = 10;
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	overlay_db.write_stored_block_range(StoredBlockRange(1, 5));
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	let genesis = Hash::repeat_byte(0);
+
+	let block_hash_a = Hash::repeat_byte(1);
+	let block_hash_b1 = Hash::repeat_byte(2);
+	let block_hash_b2 = Hash::repeat_byte(3);
+	let block_hash_c1 = Hash::repeat_byte(4);
+	let block_hash_c2 = Hash::repeat_byte(5);
+	let block_hash_d1 = Hash::repeat_byte(6);
+	let block_hash_d2 = Hash::repeat_byte(7);
+
+	let candidate_receipt_genesis = make_candidate(ParaId::from(1_u32), genesis);
+	let candidate_receipt_a = make_candidate(ParaId::from(2_u32), block_hash_a);
+	let candidate_receipt_b = make_candidate(ParaId::from(3_u32), block_hash_a);
+	let candidate_receipt_b1 = make_candidate(ParaId::from(4_u32), block_hash_b1);
+	let candidate_receipt_c1 = make_candidate(ParaId::from(5_u32), block_hash_c1);
+
+	let cand_hash_1 = candidate_receipt_genesis.hash();
+	let cand_hash_2 = candidate_receipt_a.hash();
+	let cand_hash_3 = candidate_receipt_b.hash();
+	let cand_hash_4 = candidate_receipt_b1.hash();
+	let cand_hash_5 = candidate_receipt_c1.hash();
+
+	let block_entry_a = make_block_entry(block_hash_a, genesis, 1, Vec::new());
+	let block_entry_b1 = make_block_entry(block_hash_b1, block_hash_a, 2, Vec::new());
+	let block_entry_b2 =
+		make_block_entry(block_hash_b2, block_hash_a, 2, vec![(CoreIndex(0), cand_hash_1)]);
+	let block_entry_c1 = make_block_entry(block_hash_c1, block_hash_b1, 3, Vec::new());
+	let block_entry_c2 = make_block_entry(
+		block_hash_c2,
+		block_hash_b2,
+		3,
+		vec![(CoreIndex(0), cand_hash_2), (CoreIndex(1), cand_hash_3)],
+	);
+	let block_entry_d1 = make_block_entry(
+		block_hash_d1,
+		block_hash_c1,
+		4,
+		vec![(CoreIndex(0), cand_hash_3), (CoreIndex(1), cand_hash_4)],
+	);
+	let block_entry_d2 =
+		make_block_entry(block_hash_d2, block_hash_c2, 4, vec![(CoreIndex(0), cand_hash_5)]);
+
+	let candidate_info = {
+		let mut candidate_info = HashMap::new();
+		candidate_info.insert(
+			cand_hash_1,
+			NewCandidateInfo::new(candidate_receipt_genesis, GroupIndex(1), None),
+		);
+
+		candidate_info
+			.insert(cand_hash_2, NewCandidateInfo::new(candidate_receipt_a, GroupIndex(2), None));
+
+		candidate_info
+			.insert(cand_hash_3, NewCandidateInfo::new(candidate_receipt_b, GroupIndex(3), None));
+
+		candidate_info
+			.insert(cand_hash_4, NewCandidateInfo::new(candidate_receipt_b1, GroupIndex(4), None));
+
+		candidate_info
+			.insert(cand_hash_5, NewCandidateInfo::new(candidate_receipt_c1, GroupIndex(5), None));
+
+		candidate_info
+	};
+
+	// now insert all the blocks.
+	let blocks = vec![
+		block_entry_a.clone(),
+		block_entry_b1.clone(),
+		block_entry_b2.clone(),
+		block_entry_c1.clone(),
+		block_entry_c2.clone(),
+		block_entry_d1.clone(),
+		block_entry_d2.clone(),
+	];
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	for block_entry in blocks {
+		add_block_entry(&mut overlay_db, block_entry.into(), n_validators, |h| {
+			candidate_info.get(h).map(|x| x.clone())
+		})
+		.unwrap();
+	}
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	let check_candidates_in_store = |expected: Vec<(CandidateHash, Option<Vec<_>>)>| {
+		for (c_hash, in_blocks) in expected {
+			let (entry, in_blocks) = match in_blocks {
+				None => {
+					assert!(load_candidate_entry(store.as_ref(), &TEST_CONFIG, &c_hash)
+						.unwrap()
+						.is_none());
+					continue
+				},
+				Some(i) => (
+					load_candidate_entry(store.as_ref(), &TEST_CONFIG, &c_hash).unwrap().unwrap(),
+					i,
+				),
+			};
+
+			assert_eq!(entry.block_assignments.len(), in_blocks.len());
+
+			for x in in_blocks {
+				assert!(entry.block_assignments.contains_key(&x));
+			}
+		}
+	};
+
+	let check_blocks_in_store = |expected: Vec<(Hash, Option<Vec<_>>)>| {
+		for (hash, with_candidates) in expected {
+			let (entry, with_candidates) = match with_candidates {
+				None => {
+					assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash)
+						.unwrap()
+						.is_none());
+					continue
+				},
+				Some(i) =>
+					(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash).unwrap().unwrap(), i),
+			};
+
+			assert_eq!(entry.candidates.len(), with_candidates.len());
+
+			for x in with_candidates {
+				assert!(entry.candidates.iter().any(|(_, c)| c == &x));
+			}
+		}
+	};
+
+	check_candidates_in_store(vec![
+		(cand_hash_1, Some(vec![block_hash_b2])),
+		(cand_hash_2, Some(vec![block_hash_c2])),
+		(cand_hash_3, Some(vec![block_hash_c2, block_hash_d1])),
+		(cand_hash_4, Some(vec![block_hash_d1])),
+		(cand_hash_5, Some(vec![block_hash_d2])),
+	]);
+
+	check_blocks_in_store(vec![
+		(block_hash_a, Some(vec![])),
+		(block_hash_b1, Some(vec![])),
+		(block_hash_b2, Some(vec![cand_hash_1])),
+		(block_hash_c1, Some(vec![])),
+		(block_hash_c2, Some(vec![cand_hash_2, cand_hash_3])),
+		(block_hash_d1, Some(vec![cand_hash_3, cand_hash_4])),
+		(block_hash_d2, Some(vec![cand_hash_5])),
+	]);
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	canonicalize(&mut overlay_db, 3, block_hash_c1).unwrap();
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	assert_eq!(
+		load_stored_blocks(store.as_ref(), &TEST_CONFIG).unwrap().unwrap(),
+		StoredBlockRange(4, 5)
+	);
+
+	check_candidates_in_store(vec![
+		(cand_hash_1, None),
+		(cand_hash_2, None),
+		(cand_hash_3, Some(vec![block_hash_d1])),
+		(cand_hash_4, Some(vec![block_hash_d1])),
+		(cand_hash_5, None),
+	]);
+
+	check_blocks_in_store(vec![
+		(block_hash_a, None),
+		(block_hash_b1, None),
+		(block_hash_b2, None),
+		(block_hash_c1, None),
+		(block_hash_c2, None),
+		(block_hash_d1, Some(vec![cand_hash_3, cand_hash_4])),
+		(block_hash_d2, None),
+	]);
+}
+
+#[test]
+fn force_approve_works() {
+	let (mut db, store) = make_db();
+	let n_validators = 10;
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	overlay_db.write_stored_block_range(StoredBlockRange(1, 4));
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	let candidate_hash = CandidateHash(Hash::repeat_byte(42));
+	let single_candidate_vec = vec![(CoreIndex(0), candidate_hash)];
+	let candidate_info = {
+		let mut candidate_info = HashMap::new();
+		candidate_info.insert(
+			candidate_hash,
+			NewCandidateInfo::new(
+				make_candidate(ParaId::from(1_u32), Default::default()),
+				GroupIndex(1),
+				None,
+			),
+		);
+
+		candidate_info
+	};
+
+	let block_hash_a = Hash::repeat_byte(1); // 1
+	let block_hash_b = Hash::repeat_byte(2);
+	let block_hash_c = Hash::repeat_byte(3);
+	let block_hash_d = Hash::repeat_byte(4); // 4
+
+	let block_entry_a =
+		make_block_entry(block_hash_a, Default::default(), 1, single_candidate_vec.clone());
+	let block_entry_b =
+		make_block_entry(block_hash_b, block_hash_a, 2, single_candidate_vec.clone());
+	let block_entry_c =
+		make_block_entry(block_hash_c, block_hash_b, 3, single_candidate_vec.clone());
+	let block_entry_d =
+		make_block_entry(block_hash_d, block_hash_c, 4, single_candidate_vec.clone());
+
+	let blocks = vec![
+		block_entry_a.clone(),
+		block_entry_b.clone(),
+		block_entry_c.clone(),
+		block_entry_d.clone(),
+	];
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	for block_entry in blocks {
+		add_block_entry(&mut overlay_db, block_entry.into(), n_validators, |h| {
+			candidate_info.get(h).map(|x| x.clone())
+		})
+		.unwrap();
+	}
+	let approved_hashes = force_approve(&mut overlay_db, block_hash_d, 2).unwrap();
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a,)
+		.unwrap()
+		.unwrap()
+		.approved_bitfield
+		.all());
+	assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b,)
+		.unwrap()
+		.unwrap()
+		.approved_bitfield
+		.all());
+	assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_c,)
+		.unwrap()
+		.unwrap()
+		.approved_bitfield
+		.not_any());
+	assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_d,)
+		.unwrap()
+		.unwrap()
+		.approved_bitfield
+		.not_any());
+	assert_eq!(approved_hashes, vec![block_hash_b, block_hash_a]);
+}
+
+#[test]
+fn load_all_blocks_works() {
+	let (mut db, store) = make_db();
+
+	let parent_hash = Hash::repeat_byte(1);
+	let block_hash_a = Hash::repeat_byte(2);
+	let block_hash_b = Hash::repeat_byte(69);
+	let block_hash_c = Hash::repeat_byte(42);
+
+	let block_number = 10;
+
+	let block_entry_a = make_block_entry(block_hash_a, parent_hash, block_number, vec![]);
+
+	let block_entry_b = make_block_entry(block_hash_b, parent_hash, block_number, vec![]);
+
+	let block_entry_c = make_block_entry(block_hash_c, block_hash_a, block_number + 1, vec![]);
+
+	let n_validators = 10;
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |_| None).unwrap();
+
+	// add C before B to test sorting.
+	add_block_entry(&mut overlay_db, block_entry_c.clone().into(), n_validators, |_| None).unwrap();
+
+	add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |_| None).unwrap();
+
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	assert_eq!(
+		load_all_blocks(store.as_ref(), &TEST_CONFIG).unwrap(),
+		vec![block_hash_a, block_hash_b, block_hash_c],
+	)
+}

pezkuwi/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs

@@ -0,0 +1,202 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Approval DB migration helpers.
+use super::*;
+use crate::{
+	approval_db::common::{
+		migration_helpers::{dummy_assignment_cert, make_bitvec},
+		Error, Result, StoredBlockRange,
+	},
+	backend::Backend,
+};
+
+use pezkuwi_node_primitives::approval::v1::AssignmentCertKind;
+use pezkuwi_node_subsystem_util::database::Database;
+use sp_application_crypto::sp_core::H256;
+use std::{collections::HashSet, sync::Arc};
+
+fn make_block_entry_v1(
+	block_hash: Hash,
+	parent_hash: Hash,
+	block_number: BlockNumber,
+	candidates: Vec<(CoreIndex, CandidateHash)>,
+) -> crate::approval_db::v1::BlockEntry {
+	crate::approval_db::v1::BlockEntry {
+		block_hash,
+		parent_hash,
+		block_number,
+		session: 1,
+		slot: Slot::from(1),
+		relay_vrf_story: [0u8; 32],
+		approved_bitfield: make_bitvec(candidates.len()),
+		candidates,
+		children: Vec::new(),
+	}
+}
+
+/// Migrates `OurAssignment`, `CandidateEntry` and `ApprovalEntry` to version 2.
+/// Returns on any error.
+/// Must only be used in teyrchains DB migration code - `pezkuwi-service` crate.
+pub fn v1_to_latest(db: Arc<dyn Database>, config: Config) -> Result<()> {
+	let mut backend = crate::DbBackend::new(db, config);
+	let all_blocks = backend
+		.load_all_blocks()
+		.map_err(|e| Error::InternalError(e))?
+		.iter()
+		.filter_map(|block_hash| {
+			backend
+				.load_block_entry_v1(block_hash)
+				.map_err(|e| Error::InternalError(e))
+				.ok()?
+		})
+		.collect::<Vec<_>>();
+
+	gum::info!(
+		target: crate::LOG_TARGET,
+		"Migrating candidate entries on top of {} blocks",
+		all_blocks.len()
+	);
+
+	let mut overlay = crate::OverlayedBackend::new(&backend);
+	let mut counter = 0;
+	// Get all candidate entries, approval entries and convert each of them.
+	for block in all_blocks {
+		for (candidate_index, (_core_index, candidate_hash)) in
+			block.candidates().iter().enumerate()
+		{
+			// Loading the candidate will also perform the conversion to the updated format and
+			// return that representation.
+			if let Some(candidate_entry) = backend
+				.load_candidate_entry_v1(&candidate_hash, candidate_index as CandidateIndex)
+				.map_err(|e| Error::InternalError(e))?
+			{
+				// Write the updated representation.
+				overlay.write_candidate_entry(candidate_entry);
+				counter += 1;
+			}
+		}
+		overlay.write_block_entry(block);
+	}
+
+	gum::info!(target: crate::LOG_TARGET, "Migrated {} entries", counter);
+
+	// Commit all changes to DB.
+	let write_ops = overlay.into_write_ops();
+	backend.write(write_ops).unwrap();
+
+	Ok(())
+}
+
+// Fills the db with dummy data in v1 scheme.
+pub fn v1_fill_test_data<F>(
+	db: Arc<dyn Database>,
+	config: Config,
+	dummy_candidate_create: F,
+) -> Result<HashSet<CandidateHash>>
+where
+	F: Fn(H256) -> CandidateReceipt<H256>,
+{
+	let mut backend = crate::DbBackend::new(db.clone(), config);
+	let mut overlay_db = crate::OverlayedBackend::new(&backend);
+	let mut expected_candidates = HashSet::new();
+
+	const RELAY_BLOCK_COUNT: u32 = 10;
+
+	let range = StoredBlockRange(1, 11);
+	overlay_db.write_stored_block_range(range.clone());
+
+	for relay_number in 1..=RELAY_BLOCK_COUNT {
+		let relay_hash = Hash::repeat_byte(relay_number as u8);
+		let assignment_core_index = CoreIndex(relay_number);
+		let candidate = dummy_candidate_create(relay_hash);
+		let candidate_hash = candidate.hash();
+
+		let at_height = vec![relay_hash];
+
+		let block_entry = make_block_entry_v1(
+			relay_hash,
+			Default::default(),
+			relay_number,
+			vec![(assignment_core_index, candidate_hash)],
+		);
+
+		let dummy_assignment = crate::approval_db::v1::OurAssignment {
+			cert: dummy_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }).into(),
+			tranche: 0,
+			validator_index: ValidatorIndex(0),
+			triggered: false,
+		};
+
+		let candidate_entry = crate::approval_db::v1::CandidateEntry {
+			candidate,
+			session: 123,
+			block_assignments: vec![(
+				relay_hash,
+				crate::approval_db::v1::ApprovalEntry {
+					tranches: Vec::new(),
+					backing_group: GroupIndex(1),
+					our_assignment: Some(dummy_assignment),
+					our_approval_sig: None,
+					assignments: Default::default(),
+					approved: false,
+				},
+			)]
+			.into_iter()
+			.collect(),
+			approvals: Default::default(),
+		};
+
+		overlay_db.write_blocks_at_height(relay_number, at_height.clone());
+		expected_candidates.insert(candidate_entry.candidate.hash());
+
+		db.write(write_candidate_entry_v1(candidate_entry, config)).unwrap();
+		db.write(write_block_entry_v1(block_entry, config)).unwrap();
+	}
+
+	let write_ops = overlay_db.into_write_ops();
+	backend.write(write_ops).unwrap();
+
+	Ok(expected_candidates)
+}
+
+// Low level DB helper to write a candidate entry in v1 scheme.
+fn write_candidate_entry_v1(
+	candidate_entry: crate::approval_db::v1::CandidateEntry,
+	config: Config,
+) -> DBTransaction {
+	let mut tx = DBTransaction::new();
+	tx.put_vec(
+		config.col_approval_data,
+		&candidate_entry_key(&candidate_entry.candidate.hash()),
+		candidate_entry.encode(),
+	);
+	tx
+}
+
+// Low level DB helper to write a block entry in v1 scheme.
+fn write_block_entry_v1(
+	block_entry: crate::approval_db::v1::BlockEntry,
+	config: Config,
+) -> DBTransaction {
+	let mut tx = DBTransaction::new();
+	tx.put_vec(
+		config.col_approval_data,
+		&block_entry_key(&block_entry.block_hash),
+		block_entry.encode(),
+	);
+	tx
+}

pezkuwi/node/core/approval-voting/src/approval_db/v2/mod.rs

@@ -0,0 +1,153 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Version 2 of the DB schema.
+
+use codec::{Decode, Encode};
+use pezkuwi_node_primitives::approval::{v1::DelayTranche, v2::AssignmentCertV2};
+use pezkuwi_node_subsystem::{SubsystemError, SubsystemResult};
+use pezkuwi_node_subsystem_util::database::{DBTransaction, Database};
+use pezkuwi_primitives::{
+	BlockNumber, CandidateHash, CandidateIndex, CandidateReceiptV2 as CandidateReceipt, CoreIndex,
+	GroupIndex, Hash, SessionIndex, ValidatorIndex, ValidatorSignature,
+};
+
+use sp_consensus_slots::Slot;
+
+use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec};
+use std::collections::BTreeMap;
+
+use crate::backend::V1ReadBackend;
+
+use super::common::{block_entry_key, candidate_entry_key, load_decode, Config};
+
+pub mod migration_helpers;
+#[cfg(test)]
+pub mod tests;
+
+// slot_duration * 2 + DelayTranche gives the number of delay tranches since the
+// unix epoch.
+#[derive(Encode, Decode, Clone, Copy, Debug, PartialEq)]
+pub struct Tick(u64);
+
+/// Convenience type definition
+pub type Bitfield = BitVec<u8, BitOrderLsb0>;
+
+/// Details pertaining to our assignment on a block.
+#[derive(Encode, Decode, Debug, Clone, PartialEq)]
+pub struct OurAssignment {
+	/// Our assignment certificate.
+	pub cert: AssignmentCertV2,
+	/// The tranche for which the assignment refers to.
+	pub tranche: DelayTranche,
+	/// Our validator index for the session in which the candidates were included.
+	pub validator_index: ValidatorIndex,
+	/// Whether the assignment has been triggered already.
+	pub triggered: bool,
+}
+
+/// Metadata regarding a specific tranche of assignments for a specific candidate.
+#[derive(Encode, Decode, Debug, Clone, PartialEq)]
+pub struct TrancheEntry {
+	pub tranche: DelayTranche,
+	// Assigned validators, and the instant we received their assignment, rounded
+	// to the nearest tick.
+	pub assignments: Vec<(ValidatorIndex, Tick)>,
+}
+
+/// Metadata regarding approval of a particular candidate within the context of some
+/// particular block.
+#[derive(Encode, Decode, Debug, Clone, PartialEq)]
+pub struct ApprovalEntry {
+	pub tranches: Vec<TrancheEntry>,
+	pub backing_group: GroupIndex,
+	pub our_assignment: Option<OurAssignment>,
+	pub our_approval_sig: Option<ValidatorSignature>,
+	// `n_validators` bits.
+	pub assigned_validators: Bitfield,
+	pub approved: bool,
+}
+
+/// Metadata regarding approval of a particular candidate.
+#[derive(Encode, Decode, Debug, Clone, PartialEq)]
+pub struct CandidateEntry {
+	pub candidate: CandidateReceipt,
+	pub session: SessionIndex,
+	// Assignments are based on blocks, so we need to track assignments separately
+	// based on the block we are looking at.
+	pub block_assignments: BTreeMap<Hash, ApprovalEntry>,
+	pub approvals: Bitfield,
+}
+
+/// Metadata regarding approval of a particular block, by way of approval of the
+/// candidates contained within it.
+#[derive(Encode, Decode, Debug, Clone, PartialEq)]
+pub struct BlockEntry {
+	pub block_hash: Hash,
+	pub block_number: BlockNumber,
+	pub parent_hash: Hash,
+	pub session: SessionIndex,
+	pub slot: Slot,
+	/// Random bytes derived from the VRF submitted within the block by the block
+	/// author as a credential and used as input to approval assignment criteria.
+	pub relay_vrf_story: [u8; 32],
+	// The candidates included as-of this block and the index of the core they are
+	// leaving. Sorted ascending by core index.
+	pub candidates: Vec<(CoreIndex, CandidateHash)>,
+	// A bitfield where the i'th bit corresponds to the i'th candidate in `candidates`.
+	// The i'th bit is `true` iff the candidate has been approved in the context of this
+	// block. The block can be considered approved if the bitfield has all bits set to `true`.
+	pub approved_bitfield: Bitfield,
+	pub children: Vec<Hash>,
+	// Assignments we already distributed. A 1 bit means the candidate index for which
+	// we already have sent out an assignment. We need this to avoid distributing
+	// multiple core assignments more than once.
+	pub distributed_assignments: Bitfield,
+}
+
+impl From<crate::Tick> for Tick {
+	fn from(tick: crate::Tick) -> Tick {
+		Tick(tick)
+	}
+}
+
+impl From<Tick> for crate::Tick {
+	fn from(tick: Tick) -> crate::Tick {
+		tick.0
+	}
+}
+
+/// Load a candidate entry from the aux store in v1 format.
+pub fn load_candidate_entry_v1(
+	store: &dyn Database,
+	config: &Config,
+	candidate_hash: &CandidateHash,
+) -> SubsystemResult<Option<super::v1::CandidateEntry>> {
+	load_decode(store, config.col_approval_data, &candidate_entry_key(candidate_hash))
+		.map(|u: Option<super::v1::CandidateEntry>| u.map(|v| v.into()))
+		.map_err(|e| SubsystemError::with_origin("approval-voting", e))
+}
+
+/// Load a block entry from the aux store in v1 format.
+pub fn load_block_entry_v1(
+	store: &dyn Database,
+	config: &Config,
+	block_hash: &Hash,
+) -> SubsystemResult<Option<super::v1::BlockEntry>> {
+	load_decode(store, config.col_approval_data, &block_entry_key(block_hash))
+		.map(|u: Option<super::v1::BlockEntry>| u.map(|v| v.into()))
+		.map_err(|e| SubsystemError::with_origin("approval-voting", e))
+}

pezkuwi/node/core/approval-voting/src/approval_db/v2/tests.rs

@@ -0,0 +1,586 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Tests for the aux-schema of approval voting.
+
+use crate::{
+	approval_db::{
+		common::{migration_helpers::make_bitvec, DbBackend, StoredBlockRange, *},
+		v2::*,
+		v3::{load_block_entry_v2, load_candidate_entry_v2},
+	},
+	backend::{Backend, OverlayedBackend},
+	ops::{add_block_entry, canonicalize, force_approve, NewCandidateInfo},
+};
+use pezkuwi_primitives::{
+	BlockNumber, CandidateHash, CandidateReceiptV2 as CandidateReceipt, CoreIndex, GroupIndex,
+	Hash, MutateDescriptorV2,
+};
+
+use pezkuwi_node_subsystem_util::database::Database;
+use pezkuwi_primitives::Id as ParaId;
+use sp_consensus_slots::Slot;
+use std::{collections::HashMap, sync::Arc};
+
+use pezkuwi_primitives_test_helpers::{
+	dummy_candidate_receipt_bad_sig, dummy_candidate_receipt_v2,
+	dummy_candidate_receipt_v2_bad_sig, dummy_hash,
+};
+
+const DATA_COL: u32 = 0;
+
+const NUM_COLUMNS: u32 = 1;
+
+const TEST_CONFIG: Config = Config { col_approval_data: DATA_COL };
+
+fn make_db() -> (DbBackend, Arc<dyn Database>) {
+	let db = kvdb_memorydb::create(NUM_COLUMNS);
+	let db = pezkuwi_node_subsystem_util::database::kvdb_impl::DbAdapter::new(db, &[]);
+	let db_writer: Arc<dyn Database> = Arc::new(db);
+	(DbBackend::new(db_writer.clone(), TEST_CONFIG), db_writer)
+}
+
+fn make_block_entry(
+	block_hash: Hash,
+	parent_hash: Hash,
+	block_number: BlockNumber,
+	candidates: Vec<(CoreIndex, CandidateHash)>,
+) -> BlockEntry {
+	BlockEntry {
+		block_hash,
+		parent_hash,
+		block_number,
+		session: 1,
+		slot: Slot::from(1),
+		relay_vrf_story: [0u8; 32],
+		approved_bitfield: make_bitvec(candidates.len()),
+		candidates,
+		children: Vec::new(),
+		distributed_assignments: Default::default(),
+	}
+}
+
+fn make_candidate(para_id: ParaId, relay_parent: Hash) -> CandidateReceipt {
+	let mut c = dummy_candidate_receipt_v2(dummy_hash());
+
+	c.descriptor.set_para_id(para_id);
+	c.descriptor.set_relay_parent(relay_parent);
+
+	c
+}
+
+#[test]
+fn read_write() {
+	let (mut db, store) = make_db();
+
+	let hash_a = Hash::repeat_byte(1);
+	let hash_b = Hash::repeat_byte(2);
+	let candidate_hash = dummy_candidate_receipt_bad_sig(dummy_hash(), None).hash();
+
+	let range = StoredBlockRange(10, 20);
+	let at_height = vec![hash_a, hash_b];
+
+	let block_entry =
+		make_block_entry(hash_a, Default::default(), 1, vec![(CoreIndex(0), candidate_hash)]);
+
+	let candidate_entry = CandidateEntry {
+		candidate: dummy_candidate_receipt_v2_bad_sig(dummy_hash(), None),
+		session: 5,
+		block_assignments: vec![(
+			hash_a,
+			ApprovalEntry {
+				tranches: Vec::new(),
+				backing_group: GroupIndex(1),
+				our_assignment: None,
+				our_approval_sig: None,
+				assigned_validators: Default::default(),
+				approved: false,
+			},
+		)]
+		.into_iter()
+		.collect(),
+		approvals: Default::default(),
+	};
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	overlay_db.write_stored_block_range(range.clone());
+	overlay_db.write_blocks_at_height(1, at_height.clone());
+	overlay_db.write_block_entry(block_entry.clone().into());
+	overlay_db.write_candidate_entry(crate::persisted_entries::CandidateEntry::from_v2(
+		candidate_entry.clone(),
+		0,
+	));
+
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	assert_eq!(load_stored_blocks(store.as_ref(), &TEST_CONFIG).unwrap(), Some(range));
+	assert_eq!(load_blocks_at_height(store.as_ref(), &TEST_CONFIG, &1).unwrap(), at_height);
+	assert_eq!(
+		load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap(),
+		Some(block_entry.into())
+	);
+	assert_eq!(
+		load_candidate_entry_v2(store.as_ref(), &TEST_CONFIG, &candidate_hash).unwrap(),
+		Some(candidate_entry.into()),
+	);
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	overlay_db.delete_blocks_at_height(1);
+	overlay_db.delete_block_entry(&hash_a);
+	overlay_db.delete_candidate_entry(&candidate_hash);
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	assert!(load_blocks_at_height(store.as_ref(), &TEST_CONFIG, &1).unwrap().is_empty());
+	assert!(load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap().is_none());
+	assert!(load_candidate_entry_v2(store.as_ref(), &TEST_CONFIG, &candidate_hash)
+		.unwrap()
+		.is_none());
+}
+
+#[test]
+fn add_block_entry_works() {
+	let (mut db, store) = make_db();
+
+	let parent_hash = Hash::repeat_byte(1);
+	let block_hash_a = Hash::repeat_byte(2);
+	let block_hash_b = Hash::repeat_byte(69);
+
+	let candidate_receipt_a = make_candidate(ParaId::from(1_u32), parent_hash);
+	let candidate_receipt_b = make_candidate(ParaId::from(2_u32), parent_hash);
+
+	let candidate_hash_a = candidate_receipt_a.hash();
+	let candidate_hash_b = candidate_receipt_b.hash();
+
+	let block_number = 10;
+
+	let block_entry_a = make_block_entry(
+		block_hash_a,
+		parent_hash,
+		block_number,
+		vec![(CoreIndex(0), candidate_hash_a)],
+	);
+
+	let block_entry_b = make_block_entry(
+		block_hash_b,
+		parent_hash,
+		block_number,
+		vec![(CoreIndex(0), candidate_hash_a), (CoreIndex(1), candidate_hash_b)],
+	);
+
+	let n_validators = 10;
+
+	let mut new_candidate_info = HashMap::new();
+	new_candidate_info
+		.insert(candidate_hash_a, NewCandidateInfo::new(candidate_receipt_a, GroupIndex(0), None));
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |h| {
+		new_candidate_info.get(h).map(|x| x.clone())
+	})
+	.unwrap();
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	new_candidate_info
+		.insert(candidate_hash_b, NewCandidateInfo::new(candidate_receipt_b, GroupIndex(1), None));
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |h| {
+		new_candidate_info.get(h).map(|x| x.clone())
+	})
+	.unwrap();
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	assert_eq!(
+		load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(),
+		Some(block_entry_a.into())
+	);
+	assert_eq!(
+		load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(),
+		Some(block_entry_b.into())
+	);
+
+	let candidate_entry_a =
+		load_candidate_entry_v2(store.as_ref(), &TEST_CONFIG, &candidate_hash_a)
+			.unwrap()
+			.unwrap();
+	assert_eq!(
+		candidate_entry_a.block_assignments.keys().collect::<Vec<_>>(),
+		vec![&block_hash_a, &block_hash_b]
+	);
+
+	let candidate_entry_b =
+		load_candidate_entry_v2(store.as_ref(), &TEST_CONFIG, &candidate_hash_b)
+			.unwrap()
+			.unwrap();
+	assert_eq!(candidate_entry_b.block_assignments.keys().collect::<Vec<_>>(), vec![&block_hash_b]);
+}
+
+#[test]
+fn add_block_entry_adds_child() {
+	let (mut db, store) = make_db();
+
+	let parent_hash = Hash::repeat_byte(1);
+	let block_hash_a = Hash::repeat_byte(2);
+	let block_hash_b = Hash::repeat_byte(69);
+
+	let mut block_entry_a = make_block_entry(block_hash_a, parent_hash, 1, Vec::new());
+
+	let block_entry_b = make_block_entry(block_hash_b, block_hash_a, 2, Vec::new());
+
+	let n_validators = 10;
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |_| None).unwrap();
+
+	add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |_| None).unwrap();
+
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	block_entry_a.children.push(block_hash_b);
+
+	assert_eq!(
+		load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(),
+		Some(block_entry_a.into())
+	);
+	assert_eq!(
+		load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(),
+		Some(block_entry_b.into())
+	);
+}
+
+#[test]
+fn canonicalize_works() {
+	let (mut db, store) = make_db();
+
+	//   -> B1 -> C1 -> D1
+	// A -> B2 -> C2 -> D2
+	//
+	// We'll canonicalize C1. Everything except D1 should disappear.
+	//
+	// Candidates:
+	// Cand1 in B2
+	// Cand2 in C2
+	// Cand3 in C2 and D1
+	// Cand4 in D1
+	// Cand5 in D2
+	// Only Cand3 and Cand4 should remain after canonicalize.
+
+	let n_validators = 10;
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	overlay_db.write_stored_block_range(StoredBlockRange(1, 5));
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	let genesis = Hash::repeat_byte(0);
+
+	let block_hash_a = Hash::repeat_byte(1);
+	let block_hash_b1 = Hash::repeat_byte(2);
+	let block_hash_b2 = Hash::repeat_byte(3);
+	let block_hash_c1 = Hash::repeat_byte(4);
+	let block_hash_c2 = Hash::repeat_byte(5);
+	let block_hash_d1 = Hash::repeat_byte(6);
+	let block_hash_d2 = Hash::repeat_byte(7);
+
+	let candidate_receipt_genesis = make_candidate(ParaId::from(1_u32), genesis);
+	let candidate_receipt_a = make_candidate(ParaId::from(2_u32), block_hash_a);
+	let candidate_receipt_b = make_candidate(ParaId::from(3_u32), block_hash_a);
+	let candidate_receipt_b1 = make_candidate(ParaId::from(4_u32), block_hash_b1);
+	let candidate_receipt_c1 = make_candidate(ParaId::from(5_u32), block_hash_c1);
+
+	let cand_hash_1 = candidate_receipt_genesis.hash();
+	let cand_hash_2 = candidate_receipt_a.hash();
+	let cand_hash_3 = candidate_receipt_b.hash();
+	let cand_hash_4 = candidate_receipt_b1.hash();
+	let cand_hash_5 = candidate_receipt_c1.hash();
+
+	let block_entry_a = make_block_entry(block_hash_a, genesis, 1, Vec::new());
+	let block_entry_b1 = make_block_entry(block_hash_b1, block_hash_a, 2, Vec::new());
+	let block_entry_b2 =
+		make_block_entry(block_hash_b2, block_hash_a, 2, vec![(CoreIndex(0), cand_hash_1)]);
+	let block_entry_c1 = make_block_entry(block_hash_c1, block_hash_b1, 3, Vec::new());
+	let block_entry_c2 = make_block_entry(
+		block_hash_c2,
+		block_hash_b2,
+		3,
+		vec![(CoreIndex(0), cand_hash_2), (CoreIndex(1), cand_hash_3)],
+	);
+	let block_entry_d1 = make_block_entry(
+		block_hash_d1,
+		block_hash_c1,
+		4,
+		vec![(CoreIndex(0), cand_hash_3), (CoreIndex(1), cand_hash_4)],
+	);
+	let block_entry_d2 =
+		make_block_entry(block_hash_d2, block_hash_c2, 4, vec![(CoreIndex(0), cand_hash_5)]);
+
+	let candidate_info = {
+		let mut candidate_info = HashMap::new();
+		candidate_info.insert(
+			cand_hash_1,
+			NewCandidateInfo::new(candidate_receipt_genesis, GroupIndex(1), None),
+		);
+
+		candidate_info
+			.insert(cand_hash_2, NewCandidateInfo::new(candidate_receipt_a, GroupIndex(2), None));
+
+		candidate_info
+			.insert(cand_hash_3, NewCandidateInfo::new(candidate_receipt_b, GroupIndex(3), None));
+
+		candidate_info
+			.insert(cand_hash_4, NewCandidateInfo::new(candidate_receipt_b1, GroupIndex(4), None));
+
+		candidate_info
+			.insert(cand_hash_5, NewCandidateInfo::new(candidate_receipt_c1, GroupIndex(5), None));
+
+		candidate_info
+	};
+
+	// now insert all the blocks.
+	let blocks = vec![
+		block_entry_a.clone(),
+		block_entry_b1.clone(),
+		block_entry_b2.clone(),
+		block_entry_c1.clone(),
+		block_entry_c2.clone(),
+		block_entry_d1.clone(),
+		block_entry_d2.clone(),
+	];
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	for block_entry in blocks {
+		add_block_entry(&mut overlay_db, block_entry.into(), n_validators, |h| {
+			candidate_info.get(h).map(|x| x.clone())
+		})
+		.unwrap();
+	}
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	let check_candidates_in_store = |expected: Vec<(CandidateHash, Option<Vec<_>>)>| {
+		for (c_hash, in_blocks) in expected {
+			let (entry, in_blocks) = match in_blocks {
+				None => {
+					assert!(load_candidate_entry_v2(store.as_ref(), &TEST_CONFIG, &c_hash)
+						.unwrap()
+						.is_none());
+					continue;
+				},
+				Some(i) => (
+					load_candidate_entry_v2(store.as_ref(), &TEST_CONFIG, &c_hash)
+						.unwrap()
+						.unwrap(),
+					i,
+				),
+			};
+
+			assert_eq!(entry.block_assignments.len(), in_blocks.len());
+
+			for x in in_blocks {
+				assert!(entry.block_assignments.contains_key(&x));
+			}
+		}
+	};
+
+	let check_blocks_in_store = |expected: Vec<(Hash, Option<Vec<_>>)>| {
+		for (hash, with_candidates) in expected {
+			let (entry, with_candidates) = match with_candidates {
+				None => {
+					assert!(load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &hash)
+						.unwrap()
+						.is_none());
+					continue;
+				},
+				Some(i) =>
+					(load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &hash).unwrap().unwrap(), i),
+			};
+
+			assert_eq!(entry.candidates.len(), with_candidates.len());
+
+			for x in with_candidates {
+				assert!(entry.candidates.iter().any(|(_, c)| c == &x));
+			}
+		}
+	};
+
+	check_candidates_in_store(vec![
+		(cand_hash_1, Some(vec![block_hash_b2])),
+		(cand_hash_2, Some(vec![block_hash_c2])),
+		(cand_hash_3, Some(vec![block_hash_c2, block_hash_d1])),
+		(cand_hash_4, Some(vec![block_hash_d1])),
+		(cand_hash_5, Some(vec![block_hash_d2])),
+	]);
+
+	check_blocks_in_store(vec![
+		(block_hash_a, Some(vec![])),
+		(block_hash_b1, Some(vec![])),
+		(block_hash_b2, Some(vec![cand_hash_1])),
+		(block_hash_c1, Some(vec![])),
+		(block_hash_c2, Some(vec![cand_hash_2, cand_hash_3])),
+		(block_hash_d1, Some(vec![cand_hash_3, cand_hash_4])),
+		(block_hash_d2, Some(vec![cand_hash_5])),
+	]);
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	canonicalize(&mut overlay_db, 3, block_hash_c1).unwrap();
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	assert_eq!(
+		load_stored_blocks(store.as_ref(), &TEST_CONFIG).unwrap().unwrap(),
+		StoredBlockRange(4, 5)
+	);
+
+	check_candidates_in_store(vec![
+		(cand_hash_1, None),
+		(cand_hash_2, None),
+		(cand_hash_3, Some(vec![block_hash_d1])),
+		(cand_hash_4, Some(vec![block_hash_d1])),
+		(cand_hash_5, None),
+	]);
+
+	check_blocks_in_store(vec![
+		(block_hash_a, None),
+		(block_hash_b1, None),
+		(block_hash_b2, None),
+		(block_hash_c1, None),
+		(block_hash_c2, None),
+		(block_hash_d1, Some(vec![cand_hash_3, cand_hash_4])),
+		(block_hash_d2, None),
+	]);
+}
+
+#[test]
+fn force_approve_works() {
+	let (mut db, store) = make_db();
+	let n_validators = 10;
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	overlay_db.write_stored_block_range(StoredBlockRange(1, 4));
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	let candidate_hash = CandidateHash(Hash::repeat_byte(42));
+	let single_candidate_vec = vec![(CoreIndex(0), candidate_hash)];
+	let candidate_info = {
+		let mut candidate_info = HashMap::new();
+		candidate_info.insert(
+			candidate_hash,
+			NewCandidateInfo::new(
+				make_candidate(ParaId::from(1_u32), Default::default()),
+				GroupIndex(1),
+				None,
+			),
+		);
+
+		candidate_info
+	};
+
+	let block_hash_a = Hash::repeat_byte(1); // 1
+	let block_hash_b = Hash::repeat_byte(2);
+	let block_hash_c = Hash::repeat_byte(3);
+	let block_hash_d = Hash::repeat_byte(4); // 4
+
+	let block_entry_a =
+		make_block_entry(block_hash_a, Default::default(), 1, single_candidate_vec.clone());
+	let block_entry_b =
+		make_block_entry(block_hash_b, block_hash_a, 2, single_candidate_vec.clone());
+	let block_entry_c =
+		make_block_entry(block_hash_c, block_hash_b, 3, single_candidate_vec.clone());
+	let block_entry_d =
+		make_block_entry(block_hash_d, block_hash_c, 4, single_candidate_vec.clone());
+
+	let blocks = vec![
+		block_entry_a.clone(),
+		block_entry_b.clone(),
+		block_entry_c.clone(),
+		block_entry_d.clone(),
+	];
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	for block_entry in blocks {
+		add_block_entry(&mut overlay_db, block_entry.into(), n_validators, |h| {
+			candidate_info.get(h).map(|x| x.clone())
+		})
+		.unwrap();
+	}
+	let approved_hashes = force_approve(&mut overlay_db, block_hash_d, 2).unwrap();
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	assert!(load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_a,)
+		.unwrap()
+		.unwrap()
+		.approved_bitfield
+		.all());
+	assert!(load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_b,)
+		.unwrap()
+		.unwrap()
+		.approved_bitfield
+		.all());
+	assert!(load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_c,)
+		.unwrap()
+		.unwrap()
+		.approved_bitfield
+		.not_any());
+	assert!(load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_d,)
+		.unwrap()
+		.unwrap()
+		.approved_bitfield
+		.not_any());
+	assert_eq!(approved_hashes, vec![block_hash_b, block_hash_a]);
+}
+
+#[test]
+fn load_all_blocks_works() {
+	let (mut db, store) = make_db();
+
+	let parent_hash = Hash::repeat_byte(1);
+	let block_hash_a = Hash::repeat_byte(2);
+	let block_hash_b = Hash::repeat_byte(69);
+	let block_hash_c = Hash::repeat_byte(42);
+
+	let block_number = 10;
+
+	let block_entry_a = make_block_entry(block_hash_a, parent_hash, block_number, vec![]);
+
+	let block_entry_b = make_block_entry(block_hash_b, parent_hash, block_number, vec![]);
+
+	let block_entry_c = make_block_entry(block_hash_c, block_hash_a, block_number + 1, vec![]);
+
+	let n_validators = 10;
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |_| None).unwrap();
+
+	// add C before B to test sorting.
+	add_block_entry(&mut overlay_db, block_entry_c.clone().into(), n_validators, |_| None).unwrap();
+
+	add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |_| None).unwrap();
+
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	assert_eq!(
+		load_all_blocks(store.as_ref(), &TEST_CONFIG).unwrap(),
+		vec![block_hash_a, block_hash_b, block_hash_c],
+	)
+}

pezkuwi/node/core/approval-voting/src/approval_db/v3/migration_helpers.rs

@@ -0,0 +1,237 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Approval DB migration helpers.
+use super::*;
+use crate::{
+	approval_db::common::{
+		block_entry_key, candidate_entry_key,
+		migration_helpers::{dummy_assignment_cert, make_bitvec},
+		Config, Error, Result, StoredBlockRange,
+	},
+	backend::{Backend, V2ReadBackend},
+};
+use pezkuwi_node_primitives::approval::v1::AssignmentCertKind;
+use pezkuwi_node_subsystem_util::database::Database;
+use sp_application_crypto::sp_core::H256;
+use std::{collections::HashSet, sync::Arc};
+
+/// Migrates `BlockEntry`, `CandidateEntry`, `ApprovalEntry` and `OurApproval` to version 3.
+/// Returns on any error.
+/// Must only be used in teyrchains DB migration code - `pezkuwi-service` crate.
+pub fn v2_to_latest(db: Arc<dyn Database>, config: Config) -> Result<()> {
+	let mut backend = crate::DbBackend::new(db, config);
+	let all_blocks = backend
+		.load_all_blocks()
+		.map_err(|e| Error::InternalError(e))?
+		.iter()
+		.filter_map(|block_hash| {
+			backend
+				.load_block_entry_v2(block_hash)
+				.map_err(|e| Error::InternalError(e))
+				.ok()?
+		})
+		.collect::<Vec<_>>();
+
+	gum::info!(
+		target: crate::LOG_TARGET,
+		"Migrating candidate entries on top of {} blocks",
+		all_blocks.len()
+	);
+
+	let mut overlay = crate::OverlayedBackend::new(&backend);
+	let mut counter = 0;
+	// Get all candidate entries, approval entries and convert each of them.
+	for block in all_blocks {
+		for (candidate_index, (_core_index, candidate_hash)) in
+			block.candidates().iter().enumerate()
+		{
+			// Loading the candidate will also perform the conversion to the updated format and
+			// return that representation.
+			if let Some(candidate_entry) = backend
+				.load_candidate_entry_v2(&candidate_hash, candidate_index as CandidateIndex)
+				.map_err(|e| Error::InternalError(e))?
+			{
+				// Write the updated representation.
+				overlay.write_candidate_entry(candidate_entry);
+				counter += 1;
+			}
+		}
+		overlay.write_block_entry(block);
+	}
+
+	gum::info!(target: crate::LOG_TARGET, "Migrated {} entries", counter);
+
+	// Commit all changes to DB.
+	let write_ops = overlay.into_write_ops();
+	backend.write(write_ops).unwrap();
+
+	Ok(())
+}
+
+// Checks if the migration doesn't leave the DB in an unsane state.
+// This function is to be used in tests.
+pub fn v1_to_latest_sanity_check(
+	db: Arc<dyn Database>,
+	config: Config,
+	expected_candidates: HashSet<CandidateHash>,
+) -> Result<()> {
+	let backend = crate::DbBackend::new(db, config);
+
+	let all_blocks = backend
+		.load_all_blocks()
+		.unwrap()
+		.iter()
+		.map(|block_hash| backend.load_block_entry(block_hash).unwrap().unwrap())
+		.collect::<Vec<_>>();
+
+	let mut candidates = HashSet::new();
+
+	// Iterate all blocks and approval entries.
+	for block in all_blocks {
+		for (_core_index, candidate_hash) in block.candidates() {
+			// Loading the candidate will also perform the conversion to the updated format and
+			// return that representation.
+			if let Some(candidate_entry) = backend.load_candidate_entry(&candidate_hash).unwrap() {
+				candidates.insert(candidate_entry.candidate.hash());
+			}
+		}
+	}
+
+	assert_eq!(candidates, expected_candidates);
+
+	Ok(())
+}
+
+// Fills the db with dummy data in v2 scheme.
+pub fn v2_fill_test_data<F>(
+	db: Arc<dyn Database>,
+	config: Config,
+	dummy_candidate_create: F,
+) -> Result<HashSet<CandidateHash>>
+where
+	F: Fn(H256) -> CandidateReceipt<H256>,
+{
+	let mut backend = crate::DbBackend::new(db.clone(), config);
+	let mut overlay_db = crate::OverlayedBackend::new(&backend);
+	let mut expected_candidates = HashSet::new();
+
+	const RELAY_BLOCK_COUNT: u32 = 10;
+
+	let range = StoredBlockRange(1, 11);
+	overlay_db.write_stored_block_range(range.clone());
+
+	for relay_number in 1..=RELAY_BLOCK_COUNT {
+		let relay_hash = Hash::repeat_byte(relay_number as u8);
+		let assignment_core_index = CoreIndex(relay_number);
+		let candidate = dummy_candidate_create(relay_hash);
+		let candidate_hash = candidate.hash();
+
+		let at_height = vec![relay_hash];
+
+		let block_entry = make_block_entry_v2(
+			relay_hash,
+			Default::default(),
+			relay_number,
+			vec![(assignment_core_index, candidate_hash)],
+		);
+
+		let dummy_assignment = crate::approval_db::v2::OurAssignment {
+			cert: dummy_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }).into(),
+			tranche: 0,
+			validator_index: ValidatorIndex(0),
+			triggered: false,
+		};
+
+		let candidate_entry = crate::approval_db::v2::CandidateEntry {
+			candidate,
+			session: 123,
+			block_assignments: vec![(
+				relay_hash,
+				crate::approval_db::v2::ApprovalEntry {
+					tranches: Vec::new(),
+					backing_group: GroupIndex(1),
+					our_assignment: Some(dummy_assignment),
+					our_approval_sig: None,
+					approved: false,
+					assigned_validators: make_bitvec(1),
+				},
+			)]
+			.into_iter()
+			.collect(),
+			approvals: Default::default(),
+		};
+
+		overlay_db.write_blocks_at_height(relay_number, at_height.clone());
+		expected_candidates.insert(candidate_entry.candidate.hash());
+
+		db.write(write_candidate_entry_v2(candidate_entry, config)).unwrap();
+		db.write(write_block_entry_v2(block_entry, config)).unwrap();
+	}
+
+	let write_ops = overlay_db.into_write_ops();
+	backend.write(write_ops).unwrap();
+
+	Ok(expected_candidates)
+}
+
+fn make_block_entry_v2(
+	block_hash: Hash,
+	parent_hash: Hash,
+	block_number: BlockNumber,
+	candidates: Vec<(CoreIndex, CandidateHash)>,
+) -> crate::approval_db::v2::BlockEntry {
+	crate::approval_db::v2::BlockEntry {
+		block_hash,
+		parent_hash,
+		block_number,
+		session: 1,
+		slot: Slot::from(1),
+		relay_vrf_story: [0u8; 32],
+		approved_bitfield: make_bitvec(candidates.len()),
+		distributed_assignments: make_bitvec(candidates.len()),
+		candidates,
+		children: Vec::new(),
+	}
+}
+
+// Low level DB helper to write a candidate entry in v1 scheme.
+fn write_candidate_entry_v2(
+	candidate_entry: crate::approval_db::v2::CandidateEntry,
+	config: Config,
+) -> DBTransaction {
+	let mut tx = DBTransaction::new();
+	tx.put_vec(
+		config.col_approval_data,
+		&candidate_entry_key(&candidate_entry.candidate.hash()),
+		candidate_entry.encode(),
+	);
+	tx
+}
+
+// Low level DB helper to write a block entry in v1 scheme.
+fn write_block_entry_v2(
+	block_entry: crate::approval_db::v2::BlockEntry,
+	config: Config,
+) -> DBTransaction {
+	let mut tx = DBTransaction::new();
+	tx.put_vec(
+		config.col_approval_data,
+		&block_entry_key(&block_entry.block_hash),
+		block_entry.encode(),
+	);
+	tx
+}

pezkuwi/node/core/approval-voting/src/approval_db/v3/mod.rs

@@ -0,0 +1,137 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Version 3 of the DB schema.
+//!
+//! Version 3 modifies the `our_approval` format of `ApprovalEntry`
+//! and adds a new field `pending_signatures` for `BlockEntry`
+
+use codec::{Decode, Encode};
+use pezkuwi_node_primitives::approval::v2::CandidateBitfield;
+use pezkuwi_node_subsystem::SubsystemResult;
+use pezkuwi_node_subsystem_util::database::{DBTransaction, Database};
+use pezkuwi_overseer::SubsystemError;
+use pezkuwi_primitives::{
+	BlockNumber, CandidateHash, CandidateIndex, CandidateReceiptV2 as CandidateReceipt, CoreIndex,
+	GroupIndex, Hash, SessionIndex, ValidatorIndex, ValidatorSignature,
+};
+
+use sp_consensus_slots::Slot;
+
+use std::collections::BTreeMap;
+
+use super::common::{block_entry_key, candidate_entry_key, load_decode, Config};
+
+/// Re-export this structs as v3 since they did not change between v2 and v3.
+pub use super::v2::{Bitfield, OurAssignment, Tick, TrancheEntry};
+
+pub mod migration_helpers;
+
+#[cfg(test)]
+pub mod tests;
+
+/// Metadata about our approval signature
+#[derive(Encode, Decode, Debug, Clone, PartialEq)]
+pub struct OurApproval {
+	/// The signature for the candidates hashes pointed by indices.
+	pub signature: ValidatorSignature,
+	/// The indices of the candidates signed in this approval.
+	pub signed_candidates_indices: CandidateBitfield,
+}
+
+/// Metadata regarding approval of a particular candidate within the context of some
+/// particular block.
+#[derive(Encode, Decode, Debug, Clone, PartialEq)]
+pub struct ApprovalEntry {
+	pub tranches: Vec<TrancheEntry>,
+	pub backing_group: GroupIndex,
+	pub our_assignment: Option<OurAssignment>,
+	pub our_approval_sig: Option<OurApproval>,
+	// `n_validators` bits.
+	pub assigned_validators: Bitfield,
+	pub approved: bool,
+}
+
+/// Metadata regarding approval of a particular candidate.
+#[derive(Encode, Decode, Debug, Clone, PartialEq)]
+pub struct CandidateEntry {
+	pub candidate: CandidateReceipt,
+	pub session: SessionIndex,
+	// Assignments are based on blocks, so we need to track assignments separately
+	// based on the block we are looking at.
+	pub block_assignments: BTreeMap<Hash, ApprovalEntry>,
+	pub approvals: Bitfield,
+}
+
+/// Metadata regarding approval of a particular block, by way of approval of the
+/// candidates contained within it.
+#[derive(Encode, Decode, Debug, Clone, PartialEq)]
+pub struct BlockEntry {
+	pub block_hash: Hash,
+	pub block_number: BlockNumber,
+	pub parent_hash: Hash,
+	pub session: SessionIndex,
+	pub slot: Slot,
+	/// Random bytes derived from the VRF submitted within the block by the block
+	/// author as a credential and used as input to approval assignment criteria.
+	pub relay_vrf_story: [u8; 32],
+	// The candidates included as-of this block and the index of the core they are
+	// leaving. Sorted ascending by core index.
+	pub candidates: Vec<(CoreIndex, CandidateHash)>,
+	// A bitfield where the i'th bit corresponds to the i'th candidate in `candidates`.
+	// The i'th bit is `true` iff the candidate has been approved in the context of this
+	// block. The block can be considered approved if the bitfield has all bits set to `true`.
+	pub approved_bitfield: Bitfield,
+	pub children: Vec<Hash>,
+	// A list of candidates we have checked, but didn't not sign and
+	// advertise the vote yet.
+	pub candidates_pending_signature: BTreeMap<CandidateIndex, CandidateSigningContext>,
+	// Assignments we already distributed. A 1 bit means the candidate index for which
+	// we already have sent out an assignment. We need this to avoid distributing
+	// multiple core assignments more than once.
+	pub distributed_assignments: Bitfield,
+}
+
+#[derive(Encode, Decode, Debug, Clone, PartialEq)]
+/// Context needed for creating an approval signature for a given candidate.
+pub struct CandidateSigningContext {
+	/// The candidate hash, to be included in the signature.
+	pub candidate_hash: CandidateHash,
+	/// The latest tick we have to create and send the approval.
+	pub sign_no_later_than_tick: Tick,
+}
+
+/// Load a candidate entry from the aux store in v2 format.
+pub fn load_candidate_entry_v2(
+	store: &dyn Database,
+	config: &Config,
+	candidate_hash: &CandidateHash,
+) -> SubsystemResult<Option<super::v2::CandidateEntry>> {
+	load_decode(store, config.col_approval_data, &candidate_entry_key(candidate_hash))
+		.map(|u: Option<super::v2::CandidateEntry>| u.map(|v| v.into()))
+		.map_err(|e| SubsystemError::with_origin("approval-voting", e))
+}
+
+/// Load a block entry from the aux store in v2 format.
+pub fn load_block_entry_v2(
+	store: &dyn Database,
+	config: &Config,
+	block_hash: &Hash,
+) -> SubsystemResult<Option<super::v2::BlockEntry>> {
+	load_decode(store, config.col_approval_data, &block_entry_key(block_hash))
+		.map(|u: Option<super::v2::BlockEntry>| u.map(|v| v.into()))
+		.map_err(|e| SubsystemError::with_origin("approval-voting", e))
+}

pezkuwi/node/core/approval-voting/src/approval_db/v3/tests.rs

@@ -0,0 +1,624 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Tests for the aux-schema of approval voting.
+
+use crate::{
+	approval_db::{
+		common::{migration_helpers::make_bitvec, DbBackend, StoredBlockRange, *},
+		v3::*,
+	},
+	backend::{Backend, OverlayedBackend},
+	ops::{add_block_entry, canonicalize, force_approve, NewCandidateInfo},
+};
+use pezkuwi_primitives::{
+	BlockNumber, CandidateHash, CandidateReceiptV2 as CandidateReceipt, CoreIndex, GroupIndex,
+	Hash, MutateDescriptorV2,
+};
+
+use pezkuwi_node_subsystem_util::database::Database;
+use pezkuwi_primitives::Id as ParaId;
+use sp_consensus_slots::Slot;
+use std::{collections::HashMap, sync::Arc};
+
+use pezkuwi_primitives_test_helpers::{
+	dummy_candidate_receipt_v2, dummy_candidate_receipt_v2_bad_sig, dummy_hash,
+};
+
+const DATA_COL: u32 = 0;
+
+const NUM_COLUMNS: u32 = 1;
+
+const TEST_CONFIG: Config = Config { col_approval_data: DATA_COL };
+
+fn make_db() -> (DbBackend, Arc<dyn Database>) {
+	let db = kvdb_memorydb::create(NUM_COLUMNS);
+	let db = pezkuwi_node_subsystem_util::database::kvdb_impl::DbAdapter::new(db, &[]);
+	let db_writer: Arc<dyn Database> = Arc::new(db);
+	(DbBackend::new(db_writer.clone(), TEST_CONFIG), db_writer)
+}
+
+fn make_block_entry(
+	block_hash: Hash,
+	parent_hash: Hash,
+	block_number: BlockNumber,
+	candidates: Vec<(CoreIndex, CandidateHash)>,
+) -> BlockEntry {
+	BlockEntry {
+		block_hash,
+		parent_hash,
+		block_number,
+		session: 1,
+		slot: Slot::from(1),
+		relay_vrf_story: [0u8; 32],
+		approved_bitfield: make_bitvec(candidates.len()),
+		candidates,
+		children: Vec::new(),
+		candidates_pending_signature: Default::default(),
+		distributed_assignments: Default::default(),
+	}
+}
+
+fn make_candidate(para_id: ParaId, relay_parent: Hash) -> CandidateReceipt {
+	let mut c = dummy_candidate_receipt_v2(dummy_hash());
+
+	c.descriptor.set_para_id(para_id);
+	c.descriptor.set_relay_parent(relay_parent);
+
+	c.into()
+}
+
+#[test]
+fn read_write() {
+	let (mut db, store) = make_db();
+
+	let hash_a = Hash::repeat_byte(1);
+	let hash_b = Hash::repeat_byte(2);
+	let candidate_hash = dummy_candidate_receipt_v2_bad_sig(dummy_hash(), None).hash();
+
+	let range = StoredBlockRange(10, 20);
+	let at_height = vec![hash_a, hash_b];
+
+	let block_entry =
+		make_block_entry(hash_a, Default::default(), 1, vec![(CoreIndex(0), candidate_hash)]);
+
+	let candidate_entry = CandidateEntry {
+		candidate: dummy_candidate_receipt_v2_bad_sig(dummy_hash(), None),
+		session: 5,
+		block_assignments: vec![(
+			hash_a,
+			ApprovalEntry {
+				tranches: Vec::new(),
+				backing_group: GroupIndex(1),
+				our_assignment: None,
+				our_approval_sig: None,
+				assigned_validators: Default::default(),
+				approved: false,
+			},
+		)]
+		.into_iter()
+		.collect(),
+		approvals: Default::default(),
+	};
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	overlay_db.write_stored_block_range(range.clone());
+	overlay_db.write_blocks_at_height(1, at_height.clone());
+	overlay_db.write_block_entry(block_entry.clone().into());
+	overlay_db.write_candidate_entry(candidate_entry.clone().into());
+
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	assert_eq!(load_stored_blocks(store.as_ref(), &TEST_CONFIG).unwrap(), Some(range));
+	assert_eq!(load_blocks_at_height(store.as_ref(), &TEST_CONFIG, &1).unwrap(), at_height);
+	assert_eq!(
+		load_block_entry(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap(),
+		Some(block_entry.into())
+	);
+	assert_eq!(
+		load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash).unwrap(),
+		Some(candidate_entry.into()),
+	);
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	overlay_db.delete_blocks_at_height(1);
+	overlay_db.delete_block_entry(&hash_a);
+	overlay_db.delete_candidate_entry(&candidate_hash);
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	assert!(load_blocks_at_height(store.as_ref(), &TEST_CONFIG, &1).unwrap().is_empty());
+	assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap().is_none());
+	assert!(load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash)
+		.unwrap()
+		.is_none());
+}
+
+#[test]
+fn add_block_entry_works() {
+	let (mut db, store) = make_db();
+
+	let parent_hash = Hash::repeat_byte(1);
+	let block_hash_a = Hash::repeat_byte(2);
+	let block_hash_b = Hash::repeat_byte(69);
+
+	let candidate_receipt_a = make_candidate(ParaId::from(1_u32), parent_hash);
+	let candidate_receipt_b = make_candidate(ParaId::from(2_u32), parent_hash);
+
+	let candidate_hash_a = candidate_receipt_a.hash();
+	let candidate_hash_b = candidate_receipt_b.hash();
+
+	let block_number = 10;
+
+	let block_entry_a = make_block_entry(
+		block_hash_a,
+		parent_hash,
+		block_number,
+		vec![(CoreIndex(0), candidate_hash_a)],
+	);
+
+	let block_entry_b = make_block_entry(
+		block_hash_b,
+		parent_hash,
+		block_number,
+		vec![(CoreIndex(0), candidate_hash_a), (CoreIndex(1), candidate_hash_b)],
+	);
+
+	let n_validators = 10;
+
+	let mut new_candidate_info = HashMap::new();
+	new_candidate_info
+		.insert(candidate_hash_a, NewCandidateInfo::new(candidate_receipt_a, GroupIndex(0), None));
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |h| {
+		new_candidate_info.get(h).map(|x| x.clone())
+	})
+	.unwrap();
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	new_candidate_info
+		.insert(candidate_hash_b, NewCandidateInfo::new(candidate_receipt_b, GroupIndex(1), None));
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |h| {
+		new_candidate_info.get(h).map(|x| x.clone())
+	})
+	.unwrap();
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	assert_eq!(
+		load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(),
+		Some(block_entry_a.into())
+	);
+	assert_eq!(
+		load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(),
+		Some(block_entry_b.into())
+	);
+
+	let candidate_entry_a = load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash_a)
+		.unwrap()
+		.unwrap();
+	assert_eq!(
+		candidate_entry_a.block_assignments.keys().collect::<Vec<_>>(),
+		vec![&block_hash_a, &block_hash_b]
+	);
+
+	let candidate_entry_b = load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash_b)
+		.unwrap()
+		.unwrap();
+	assert_eq!(candidate_entry_b.block_assignments.keys().collect::<Vec<_>>(), vec![&block_hash_b]);
+}
+
+#[test]
+fn add_block_entry_adds_child() {
+	let (mut db, store) = make_db();
+
+	let parent_hash = Hash::repeat_byte(1);
+	let block_hash_a = Hash::repeat_byte(2);
+	let block_hash_b = Hash::repeat_byte(69);
+
+	let mut block_entry_a = make_block_entry(block_hash_a, parent_hash, 1, Vec::new());
+
+	let block_entry_b = make_block_entry(block_hash_b, block_hash_a, 2, Vec::new());
+
+	let n_validators = 10;
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |_| None).unwrap();
+
+	add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |_| None).unwrap();
+
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	block_entry_a.children.push(block_hash_b);
+
+	assert_eq!(
+		load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(),
+		Some(block_entry_a.into())
+	);
+	assert_eq!(
+		load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(),
+		Some(block_entry_b.into())
+	);
+}
+
+#[test]
+fn canonicalize_works() {
+	let (mut db, store) = make_db();
+
+	//   -> B1 -> C1 -> D1 -> E1
+	// A -> B2 -> C2 -> D2 -> E2
+	//
+	// We'll canonicalize C1. Everything except D1 should disappear.
+	//
+	// Candidates:
+	// Cand1 in B2
+	// Cand2 in C2
+	// Cand3 in C2 and D1
+	// Cand4 in D1
+	// Cand5 in D2
+	// Only Cand3 and Cand4 should remain after canonicalize.
+
+	let n_validators = 10;
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	overlay_db.write_stored_block_range(StoredBlockRange(1, 5));
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	let genesis = Hash::repeat_byte(0);
+
+	let block_hash_a = Hash::repeat_byte(1);
+	let block_hash_b1 = Hash::repeat_byte(2);
+	let block_hash_b2 = Hash::repeat_byte(3);
+	let block_hash_c1 = Hash::repeat_byte(4);
+	let block_hash_c2 = Hash::repeat_byte(5);
+	let block_hash_d1 = Hash::repeat_byte(6);
+	let block_hash_d2 = Hash::repeat_byte(7);
+	let block_hash_e1 = Hash::repeat_byte(8);
+	let block_hash_e2 = Hash::repeat_byte(9);
+
+	let candidate_receipt_genesis = make_candidate(ParaId::from(1_u32), genesis);
+	let candidate_receipt_a = make_candidate(ParaId::from(2_u32), block_hash_a);
+	let candidate_receipt_b = make_candidate(ParaId::from(3_u32), block_hash_a);
+	let candidate_receipt_b1 = make_candidate(ParaId::from(4_u32), block_hash_b1);
+	let candidate_receipt_c1 = make_candidate(ParaId::from(5_u32), block_hash_c1);
+	let candidate_receipt_e1 = make_candidate(ParaId::from(6_u32), block_hash_e1);
+
+	let cand_hash_1 = candidate_receipt_genesis.hash();
+	let cand_hash_2 = candidate_receipt_a.hash();
+	let cand_hash_3 = candidate_receipt_b.hash();
+	let cand_hash_4 = candidate_receipt_b1.hash();
+	let cand_hash_5 = candidate_receipt_c1.hash();
+	let cand_hash_6 = candidate_receipt_e1.hash();
+
+	let block_entry_a = make_block_entry(block_hash_a, genesis, 1, Vec::new());
+	let block_entry_b1 = make_block_entry(block_hash_b1, block_hash_a, 2, Vec::new());
+	let block_entry_b2 =
+		make_block_entry(block_hash_b2, block_hash_a, 2, vec![(CoreIndex(0), cand_hash_1)]);
+	let block_entry_c1 = make_block_entry(block_hash_c1, block_hash_b1, 3, Vec::new());
+	let block_entry_c2 = make_block_entry(
+		block_hash_c2,
+		block_hash_b2,
+		3,
+		vec![(CoreIndex(0), cand_hash_2), (CoreIndex(1), cand_hash_3)],
+	);
+	let block_entry_d1 = make_block_entry(
+		block_hash_d1,
+		block_hash_c1,
+		4,
+		vec![(CoreIndex(0), cand_hash_3), (CoreIndex(1), cand_hash_4)],
+	);
+	let block_entry_d2 =
+		make_block_entry(block_hash_d2, block_hash_c2, 4, vec![(CoreIndex(0), cand_hash_5)]);
+
+	let block_entry_e1 =
+		make_block_entry(block_hash_e1, block_hash_d1, 5, vec![(CoreIndex(0), cand_hash_6)]);
+
+	let block_entry_e2 =
+		make_block_entry(block_hash_e2, block_hash_d2, 5, vec![(CoreIndex(0), cand_hash_6)]);
+
+	let candidate_info = {
+		let mut candidate_info = HashMap::new();
+		candidate_info.insert(
+			cand_hash_1,
+			NewCandidateInfo::new(candidate_receipt_genesis, GroupIndex(1), None),
+		);
+
+		candidate_info
+			.insert(cand_hash_2, NewCandidateInfo::new(candidate_receipt_a, GroupIndex(2), None));
+
+		candidate_info
+			.insert(cand_hash_3, NewCandidateInfo::new(candidate_receipt_b, GroupIndex(3), None));
+
+		candidate_info
+			.insert(cand_hash_4, NewCandidateInfo::new(candidate_receipt_b1, GroupIndex(4), None));
+
+		candidate_info
+			.insert(cand_hash_5, NewCandidateInfo::new(candidate_receipt_c1, GroupIndex(5), None));
+
+		candidate_info
+			.insert(cand_hash_6, NewCandidateInfo::new(candidate_receipt_e1, GroupIndex(6), None));
+		candidate_info
+	};
+
+	// now insert all the blocks.
+	let blocks = vec![
+		block_entry_a.clone(),
+		block_entry_b1.clone(),
+		block_entry_b2.clone(),
+		block_entry_c1.clone(),
+		block_entry_c2.clone(),
+		block_entry_d1.clone(),
+		block_entry_d2.clone(),
+		block_entry_e1.clone(),
+		block_entry_e2.clone(),
+	];
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	for block_entry in blocks {
+		add_block_entry(&mut overlay_db, block_entry.into(), n_validators, |h| {
+			candidate_info.get(h).map(|x| x.clone())
+		})
+		.unwrap();
+	}
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	let check_candidates_in_store = |expected: Vec<(CandidateHash, Option<Vec<_>>)>| {
+		for (c_hash, in_blocks) in expected {
+			let (entry, in_blocks) = match in_blocks {
+				None => {
+					assert!(load_candidate_entry(store.as_ref(), &TEST_CONFIG, &c_hash)
+						.unwrap()
+						.is_none());
+					continue;
+				},
+				Some(i) => (
+					load_candidate_entry(store.as_ref(), &TEST_CONFIG, &c_hash).unwrap().unwrap(),
+					i,
+				),
+			};
+
+			assert_eq!(entry.block_assignments.len(), in_blocks.len());
+
+			for x in in_blocks {
+				assert!(entry.block_assignments.contains_key(&x));
+			}
+		}
+	};
+
+	let check_blocks_in_store = |expected: Vec<(Hash, Option<Vec<_>>)>| {
+		for (hash, with_candidates) in expected {
+			let (entry, with_candidates) = match with_candidates {
+				None => {
+					assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash)
+						.unwrap()
+						.is_none());
+					continue;
+				},
+				Some(i) =>
+					(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash).unwrap().unwrap(), i),
+			};
+
+			assert_eq!(entry.candidates.len(), with_candidates.len());
+
+			for x in with_candidates {
+				assert!(entry.candidates.iter().any(|(_, c)| c == &x));
+			}
+		}
+	};
+
+	check_candidates_in_store(vec![
+		(cand_hash_1, Some(vec![block_hash_b2])),
+		(cand_hash_2, Some(vec![block_hash_c2])),
+		(cand_hash_3, Some(vec![block_hash_c2, block_hash_d1])),
+		(cand_hash_4, Some(vec![block_hash_d1])),
+		(cand_hash_5, Some(vec![block_hash_d2])),
+	]);
+
+	check_blocks_in_store(vec![
+		(block_hash_a, Some(vec![])),
+		(block_hash_b1, Some(vec![])),
+		(block_hash_b2, Some(vec![cand_hash_1])),
+		(block_hash_c1, Some(vec![])),
+		(block_hash_c2, Some(vec![cand_hash_2, cand_hash_3])),
+		(block_hash_d1, Some(vec![cand_hash_3, cand_hash_4])),
+		(block_hash_d2, Some(vec![cand_hash_5])),
+	]);
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	canonicalize(&mut overlay_db, 3, block_hash_c1).unwrap();
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	assert_eq!(
+		load_stored_blocks(store.as_ref(), &TEST_CONFIG).unwrap().unwrap(),
+		StoredBlockRange(4, 6)
+	);
+
+	check_candidates_in_store(vec![
+		(cand_hash_1, None),
+		(cand_hash_2, None),
+		(cand_hash_3, Some(vec![block_hash_d1])),
+		(cand_hash_4, Some(vec![block_hash_d1])),
+		(cand_hash_5, None),
+		(cand_hash_6, Some(vec![block_hash_e1])),
+	]);
+
+	check_blocks_in_store(vec![
+		(block_hash_a, None),
+		(block_hash_b1, None),
+		(block_hash_b2, None),
+		(block_hash_c1, None),
+		(block_hash_c2, None),
+		(block_hash_d1, Some(vec![cand_hash_3, cand_hash_4])),
+		(block_hash_e1, Some(vec![cand_hash_6])),
+		(block_hash_d2, None),
+	]);
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	canonicalize(&mut overlay_db, 4, block_hash_d1).unwrap();
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	assert_eq!(
+		load_stored_blocks(store.as_ref(), &TEST_CONFIG).unwrap().unwrap(),
+		StoredBlockRange(5, 6)
+	);
+
+	check_candidates_in_store(vec![
+		(cand_hash_1, None),
+		(cand_hash_2, None),
+		(cand_hash_3, None),
+		(cand_hash_4, None),
+		(cand_hash_5, None),
+		(cand_hash_6, Some(vec![block_hash_e1])),
+	]);
+
+	check_blocks_in_store(vec![
+		(block_hash_a, None),
+		(block_hash_b1, None),
+		(block_hash_b2, None),
+		(block_hash_c1, None),
+		(block_hash_c2, None),
+		(block_hash_d1, None),
+		(block_hash_e1, Some(vec![cand_hash_6])),
+		(block_hash_d2, None),
+	]);
+}
+
+#[test]
+fn force_approve_works() {
+	let (mut db, store) = make_db();
+	let n_validators = 10;
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	overlay_db.write_stored_block_range(StoredBlockRange(1, 4));
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	let candidate_hash = CandidateHash(Hash::repeat_byte(42));
+	let single_candidate_vec = vec![(CoreIndex(0), candidate_hash)];
+	let candidate_info = {
+		let mut candidate_info = HashMap::new();
+		candidate_info.insert(
+			candidate_hash,
+			NewCandidateInfo::new(
+				make_candidate(ParaId::from(1_u32), Default::default()),
+				GroupIndex(1),
+				None,
+			),
+		);
+
+		candidate_info
+	};
+
+	let block_hash_a = Hash::repeat_byte(1); // 1
+	let block_hash_b = Hash::repeat_byte(2);
+	let block_hash_c = Hash::repeat_byte(3);
+	let block_hash_d = Hash::repeat_byte(4); // 4
+
+	let block_entry_a =
+		make_block_entry(block_hash_a, Default::default(), 1, single_candidate_vec.clone());
+	let block_entry_b =
+		make_block_entry(block_hash_b, block_hash_a, 2, single_candidate_vec.clone());
+	let block_entry_c =
+		make_block_entry(block_hash_c, block_hash_b, 3, single_candidate_vec.clone());
+	let block_entry_d =
+		make_block_entry(block_hash_d, block_hash_c, 4, single_candidate_vec.clone());
+
+	let blocks = vec![
+		block_entry_a.clone(),
+		block_entry_b.clone(),
+		block_entry_c.clone(),
+		block_entry_d.clone(),
+	];
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	for block_entry in blocks {
+		add_block_entry(&mut overlay_db, block_entry.into(), n_validators, |h| {
+			candidate_info.get(h).map(|x| x.clone())
+		})
+		.unwrap();
+	}
+	let approved_hashes = force_approve(&mut overlay_db, block_hash_d, 2).unwrap();
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a,)
+		.unwrap()
+		.unwrap()
+		.approved_bitfield
+		.all());
+	assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b,)
+		.unwrap()
+		.unwrap()
+		.approved_bitfield
+		.all());
+	assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_c,)
+		.unwrap()
+		.unwrap()
+		.approved_bitfield
+		.not_any());
+	assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_d,)
+		.unwrap()
+		.unwrap()
+		.approved_bitfield
+		.not_any());
+	assert_eq!(approved_hashes, vec![block_hash_b, block_hash_a]);
+}
+
+#[test]
+fn load_all_blocks_works() {
+	let (mut db, store) = make_db();
+
+	let parent_hash = Hash::repeat_byte(1);
+	let block_hash_a = Hash::repeat_byte(2);
+	let block_hash_b = Hash::repeat_byte(69);
+	let block_hash_c = Hash::repeat_byte(42);
+
+	let block_number = 10;
+
+	let block_entry_a = make_block_entry(block_hash_a, parent_hash, block_number, vec![]);
+
+	let block_entry_b = make_block_entry(block_hash_b, parent_hash, block_number, vec![]);
+
+	let block_entry_c = make_block_entry(block_hash_c, block_hash_a, block_number + 1, vec![]);
+
+	let n_validators = 10;
+
+	let mut overlay_db = OverlayedBackend::new(&db);
+	add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |_| None).unwrap();
+
+	// add C before B to test sorting.
+	add_block_entry(&mut overlay_db, block_entry_c.clone().into(), n_validators, |_| None).unwrap();
+
+	add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |_| None).unwrap();
+
+	let write_ops = overlay_db.into_write_ops();
+	db.write(write_ops).unwrap();
+
+	assert_eq!(
+		load_all_blocks(store.as_ref(), &TEST_CONFIG).unwrap(),
+		vec![block_hash_a, block_hash_b, block_hash_c],
+	)
+}

pezkuwi/node/core/approval-voting/src/backend.rs

@@ -0,0 +1,249 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! An abstraction over storage used by the chain selection subsystem.
+//!
+//! This provides both a [`Backend`] trait and an [`OverlayedBackend`]
+//! struct which allows in-memory changes to be applied on top of a
+//! [`Backend`], maintaining consistency between queries and temporary writes,
+//! before any commit to the underlying storage is made.
+
+use pezkuwi_node_subsystem::SubsystemResult;
+use pezkuwi_primitives::{BlockNumber, CandidateHash, CandidateIndex, Hash};
+
+use std::collections::HashMap;
+
+use super::{
+	approval_db::common::StoredBlockRange,
+	persisted_entries::{BlockEntry, CandidateEntry},
+};
+
+#[derive(Debug)]
+pub enum BackendWriteOp {
+	WriteStoredBlockRange(StoredBlockRange),
+	WriteBlocksAtHeight(BlockNumber, Vec<Hash>),
+	WriteBlockEntry(BlockEntry),
+	WriteCandidateEntry(CandidateEntry),
+	DeleteStoredBlockRange,
+	DeleteBlocksAtHeight(BlockNumber),
+	DeleteBlockEntry(Hash),
+	DeleteCandidateEntry(CandidateHash),
+}
+
+/// An abstraction over backend storage for the logic of this subsystem.
+/// Implementation must always target latest storage version.
+pub trait Backend {
+	/// Load a block entry from the DB.
+	fn load_block_entry(&self, hash: &Hash) -> SubsystemResult<Option<BlockEntry>>;
+	/// Load a candidate entry from the DB.
+	fn load_candidate_entry(
+		&self,
+		candidate_hash: &CandidateHash,
+	) -> SubsystemResult<Option<CandidateEntry>>;
+
+	/// Load all blocks at a specific height.
+	fn load_blocks_at_height(&self, height: &BlockNumber) -> SubsystemResult<Vec<Hash>>;
+	/// Load all block from the DB.
+	fn load_all_blocks(&self) -> SubsystemResult<Vec<Hash>>;
+	/// Load stored block range form the DB.
+	fn load_stored_blocks(&self) -> SubsystemResult<Option<StoredBlockRange>>;
+	/// Atomically write the list of operations, with later operations taking precedence over prior.
+	fn write<I>(&mut self, ops: I) -> SubsystemResult<()>
+	where
+		I: IntoIterator<Item = BackendWriteOp>;
+}
+
+/// A read only backend to enable db migration from version 1 of DB.
+pub trait V1ReadBackend: Backend {
+	/// Load a candidate entry from the DB with scheme version 1.
+	fn load_candidate_entry_v1(
+		&self,
+		candidate_hash: &CandidateHash,
+		candidate_index: CandidateIndex,
+	) -> SubsystemResult<Option<CandidateEntry>>;
+
+	/// Load a block entry from the DB with scheme version 1.
+	fn load_block_entry_v1(&self, block_hash: &Hash) -> SubsystemResult<Option<BlockEntry>>;
+}
+
+/// A read only backend to enable db migration from version 2 of DB.
+pub trait V2ReadBackend: Backend {
+	/// Load a candidate entry from the DB with scheme version 1.
+	fn load_candidate_entry_v2(
+		&self,
+		candidate_hash: &CandidateHash,
+		candidate_index: CandidateIndex,
+	) -> SubsystemResult<Option<CandidateEntry>>;
+
+	/// Load a block entry from the DB with scheme version 1.
+	fn load_block_entry_v2(&self, block_hash: &Hash) -> SubsystemResult<Option<BlockEntry>>;
+}
+
+// Status of block range in the `OverlayedBackend`.
+#[derive(PartialEq)]
+enum BlockRangeStatus {
+	// Value has not been modified.
+	NotModified,
+	// Value has been deleted
+	Deleted,
+	// Value has been updated.
+	Inserted(StoredBlockRange),
+}
+
+/// An in-memory overlay over the backend.
+///
+/// This maintains read-only access to the underlying backend, but can be
+/// converted into a set of write operations which will, when written to
+/// the underlying backend, give the same view as the state of the overlay.
+pub struct OverlayedBackend<'a, B: 'a> {
+	inner: &'a B,
+	// `Some(None)` means deleted. Missing (`None`) means query inner.
+	stored_block_range: BlockRangeStatus,
+	// `None` means 'deleted', missing means query inner.
+	blocks_at_height: HashMap<BlockNumber, Option<Vec<Hash>>>,
+	// `None` means 'deleted', missing means query inner.
+	block_entries: HashMap<Hash, Option<BlockEntry>>,
+	// `None` means 'deleted', missing means query inner.
+	candidate_entries: HashMap<CandidateHash, Option<CandidateEntry>>,
+}
+
+impl<'a, B: 'a + Backend> OverlayedBackend<'a, B> {
+	pub fn new(backend: &'a B) -> Self {
+		OverlayedBackend {
+			inner: backend,
+			stored_block_range: BlockRangeStatus::NotModified,
+			blocks_at_height: HashMap::new(),
+			block_entries: HashMap::new(),
+			candidate_entries: HashMap::new(),
+		}
+	}
+
+	pub fn is_empty(&self) -> bool {
+		self.block_entries.is_empty() &&
+			self.candidate_entries.is_empty() &&
+			self.blocks_at_height.is_empty() &&
+			self.stored_block_range == BlockRangeStatus::NotModified
+	}
+
+	pub fn load_all_blocks(&self) -> SubsystemResult<Vec<Hash>> {
+		let mut hashes = Vec::new();
+		if let Some(stored_blocks) = self.load_stored_blocks()? {
+			for height in stored_blocks.0..stored_blocks.1 {
+				hashes.extend(self.load_blocks_at_height(&height)?);
+			}
+		}
+
+		Ok(hashes)
+	}
+
+	pub fn load_stored_blocks(&self) -> SubsystemResult<Option<StoredBlockRange>> {
+		match self.stored_block_range {
+			BlockRangeStatus::Inserted(ref value) => Ok(Some(value.clone())),
+			BlockRangeStatus::Deleted => Ok(None),
+			BlockRangeStatus::NotModified => self.inner.load_stored_blocks(),
+		}
+	}
+
+	pub fn load_blocks_at_height(&self, height: &BlockNumber) -> SubsystemResult<Vec<Hash>> {
+		if let Some(val) = self.blocks_at_height.get(&height) {
+			return Ok(val.clone().unwrap_or_default());
+		}
+
+		self.inner.load_blocks_at_height(height)
+	}
+
+	pub fn load_block_entry(&self, hash: &Hash) -> SubsystemResult<Option<BlockEntry>> {
+		if let Some(val) = self.block_entries.get(&hash) {
+			return Ok(val.clone());
+		}
+
+		self.inner.load_block_entry(hash)
+	}
+
+	pub fn load_candidate_entry(
+		&self,
+		candidate_hash: &CandidateHash,
+	) -> SubsystemResult<Option<CandidateEntry>> {
+		if let Some(val) = self.candidate_entries.get(&candidate_hash) {
+			return Ok(val.clone());
+		}
+
+		self.inner.load_candidate_entry(candidate_hash)
+	}
+
+	pub fn write_stored_block_range(&mut self, range: StoredBlockRange) {
+		self.stored_block_range = BlockRangeStatus::Inserted(range);
+	}
+
+	pub fn delete_stored_block_range(&mut self) {
+		self.stored_block_range = BlockRangeStatus::Deleted;
+	}
+
+	pub fn write_blocks_at_height(&mut self, height: BlockNumber, blocks: Vec<Hash>) {
+		self.blocks_at_height.insert(height, Some(blocks));
+	}
+
+	pub fn delete_blocks_at_height(&mut self, height: BlockNumber) {
+		self.blocks_at_height.insert(height, None);
+	}
+
+	pub fn write_block_entry(&mut self, entry: BlockEntry) {
+		self.block_entries.insert(entry.block_hash(), Some(entry));
+	}
+
+	pub fn delete_block_entry(&mut self, hash: &Hash) {
+		self.block_entries.insert(*hash, None);
+	}
+
+	pub fn write_candidate_entry(&mut self, entry: CandidateEntry) {
+		self.candidate_entries.insert(entry.candidate_receipt().hash(), Some(entry));
+	}
+
+	pub fn delete_candidate_entry(&mut self, hash: &CandidateHash) {
+		self.candidate_entries.insert(*hash, None);
+	}
+
+	/// Transform this backend into a set of write-ops to be written to the
+	/// inner backend.
+	pub fn into_write_ops(self) -> impl Iterator<Item = BackendWriteOp> {
+		let blocks_at_height_ops = self.blocks_at_height.into_iter().map(|(h, v)| match v {
+			Some(v) => BackendWriteOp::WriteBlocksAtHeight(h, v),
+			None => BackendWriteOp::DeleteBlocksAtHeight(h),
+		});
+
+		let block_entry_ops = self.block_entries.into_iter().map(|(h, v)| match v {
+			Some(v) => BackendWriteOp::WriteBlockEntry(v),
+			None => BackendWriteOp::DeleteBlockEntry(h),
+		});
+
+		let candidate_entry_ops = self.candidate_entries.into_iter().map(|(h, v)| match v {
+			Some(v) => BackendWriteOp::WriteCandidateEntry(v),
+			None => BackendWriteOp::DeleteCandidateEntry(h),
+		});
+
+		let stored_block_range_ops = match self.stored_block_range {
+			BlockRangeStatus::Inserted(val) => Some(BackendWriteOp::WriteStoredBlockRange(val)),
+			BlockRangeStatus::Deleted => Some(BackendWriteOp::DeleteStoredBlockRange),
+			BlockRangeStatus::NotModified => None,
+		};
+
+		stored_block_range_ops
+			.into_iter()
+			.chain(blocks_at_height_ops)
+			.chain(block_entry_ops)
+			.chain(candidate_entry_ops)
+	}
+}

pezkuwi/node/core/approval-voting/src/ops.rs

@@ -0,0 +1,428 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Middleware interface that leverages low-level database operations
+//! to provide a clean API for processing block and candidate imports.
+
+use pezkuwi_node_subsystem::{SubsystemError, SubsystemResult};
+
+use bitvec::order::Lsb0 as BitOrderLsb0;
+use pezkuwi_primitives::{
+	BlockNumber, CandidateHash, CandidateReceiptV2 as CandidateReceipt, GroupIndex, Hash,
+};
+
+use std::collections::{hash_map::Entry, BTreeMap, HashMap};
+
+use super::{
+	approval_db::{common::StoredBlockRange, v2::OurAssignment},
+	backend::{Backend, OverlayedBackend},
+	persisted_entries::{ApprovalEntry, BlockEntry, CandidateEntry},
+	LOG_TARGET,
+};
+
+/// Information about a new candidate necessary to instantiate the requisite
+/// candidate and approval entries.
+#[derive(Clone)]
+pub struct NewCandidateInfo {
+	candidate: CandidateReceipt,
+	backing_group: GroupIndex,
+	our_assignment: Option<OurAssignment>,
+}
+
+impl NewCandidateInfo {
+	/// Convenience constructor
+	pub fn new(
+		candidate: CandidateReceipt,
+		backing_group: GroupIndex,
+		our_assignment: Option<OurAssignment>,
+	) -> Self {
+		Self { candidate, backing_group, our_assignment }
+	}
+}
+
+fn visit_and_remove_block_entry(
+	block_hash: Hash,
+	overlayed_db: &mut OverlayedBackend<'_, impl Backend>,
+	visited_candidates: &mut HashMap<CandidateHash, CandidateEntry>,
+) -> SubsystemResult<Vec<Hash>> {
+	let block_entry = match overlayed_db.load_block_entry(&block_hash)? {
+		None => return Ok(Vec::new()),
+		Some(b) => b,
+	};
+
+	overlayed_db.delete_block_entry(&block_hash);
+	for (_, candidate_hash) in block_entry.candidates() {
+		let candidate = match visited_candidates.entry(*candidate_hash) {
+			Entry::Occupied(e) => e.into_mut(),
+			Entry::Vacant(e) => {
+				e.insert(match overlayed_db.load_candidate_entry(candidate_hash)? {
+					None => continue, // Should not happen except for corrupt DB
+					Some(c) => c,
+				})
+			},
+		};
+
+		candidate.block_assignments.remove(&block_hash);
+	}
+
+	Ok(block_entry.children)
+}
+
+/// Canonicalize some particular block, pruning everything before it and
+/// pruning any competing branches at the same height.
+pub fn canonicalize(
+	overlay_db: &mut OverlayedBackend<'_, impl Backend>,
+	canon_number: BlockNumber,
+	canon_hash: Hash,
+) -> SubsystemResult<()> {
+	let range = match overlay_db.load_stored_blocks()? {
+		None => return Ok(()),
+		Some(range) if range.0 > canon_number => return Ok(()),
+		Some(range) => range,
+	};
+
+	// Storing all candidates in memory is potentially heavy, but should be fine
+	// as long as finality doesn't stall for a long while. We could optimize this
+	// by keeping only the metadata about which blocks reference each candidate.
+	let mut visited_candidates = HashMap::new();
+
+	// All the block heights we visited but didn't necessarily delete everything from.
+	let mut visited_heights = HashMap::new();
+
+	// First visit everything before the height.
+	for i in range.0..canon_number {
+		let at_height = overlay_db.load_blocks_at_height(&i)?;
+		overlay_db.delete_blocks_at_height(i);
+
+		for b in at_height {
+			visit_and_remove_block_entry(b, overlay_db, &mut visited_candidates)?;
+		}
+	}
+
+	// Then visit everything at the height.
+	let pruned_branches = {
+		let at_height = overlay_db.load_blocks_at_height(&canon_number)?;
+		overlay_db.delete_blocks_at_height(canon_number);
+
+		// Note that while there may be branches descending from blocks at earlier heights,
+		// we have already covered them by removing everything at earlier heights.
+		let mut pruned_branches = Vec::new();
+
+		for b in at_height {
+			let children = visit_and_remove_block_entry(b, overlay_db, &mut visited_candidates)?;
+
+			if b != canon_hash {
+				pruned_branches.extend(children);
+			}
+		}
+
+		pruned_branches
+	};
+
+	// Follow all children of non-canonicalized blocks.
+	{
+		let mut frontier: Vec<(BlockNumber, Hash)> =
+			pruned_branches.into_iter().map(|h| (canon_number + 1, h)).collect();
+		while let Some((height, next_child)) = frontier.pop() {
+			let children =
+				visit_and_remove_block_entry(next_child, overlay_db, &mut visited_candidates)?;
+
+			// extend the frontier of branches to include the given height.
+			frontier.extend(children.into_iter().map(|h| (height + 1, h)));
+
+			// visit the at-height key for this deleted block's height.
+			let at_height = match visited_heights.entry(height) {
+				Entry::Occupied(e) => e.into_mut(),
+				Entry::Vacant(e) => e.insert(overlay_db.load_blocks_at_height(&height)?),
+			};
+			if let Some(i) = at_height.iter().position(|x| x == &next_child) {
+				at_height.remove(i);
+			}
+		}
+	}
+
+	// Update all `CandidateEntry`s, deleting all those which now have empty `block_assignments`.
+	for (candidate_hash, candidate) in visited_candidates.into_iter() {
+		if candidate.block_assignments.is_empty() {
+			overlay_db.delete_candidate_entry(&candidate_hash);
+		} else {
+			overlay_db.write_candidate_entry(candidate);
+		}
+	}
+
+	// Update all blocks-at-height keys, deleting all those which now have empty
+	// `block_assignments`.
+	for (h, at) in visited_heights.into_iter() {
+		if at.is_empty() {
+			overlay_db.delete_blocks_at_height(h);
+		} else {
+			overlay_db.write_blocks_at_height(h, at);
+		}
+	}
+
+	// due to the fork pruning, this range actually might go too far above where our actual highest
+	// block is, if a relatively short fork is canonicalized.
+	// TODO https://github.com/paritytech/polkadot/issues/3389
+	let new_range = StoredBlockRange(canon_number + 1, std::cmp::max(range.1, canon_number + 2));
+
+	overlay_db.write_stored_block_range(new_range);
+
+	Ok(())
+}
+
+/// Record a new block entry.
+///
+/// This will update the blocks-at-height mapping, the stored block range, if necessary,
+/// and add block and candidate entries. It will also add approval entries to existing
+/// candidate entries and add this as a child of any block entry corresponding to the
+/// parent hash.
+///
+/// Has no effect if there is already an entry for the block or `candidate_info` returns
+/// `None` for any of the candidates referenced by the block entry. In these cases,
+/// no information about new candidates will be referred to by this function.
+pub fn add_block_entry(
+	store: &mut OverlayedBackend<'_, impl Backend>,
+	entry: BlockEntry,
+	n_validators: usize,
+	candidate_info: impl Fn(&CandidateHash) -> Option<NewCandidateInfo>,
+) -> SubsystemResult<Vec<(CandidateHash, CandidateEntry)>> {
+	let session = entry.session();
+	let parent_hash = entry.parent_hash();
+	let number = entry.block_number();
+
+	// Update the stored block range.
+	{
+		let new_range = match store.load_stored_blocks()? {
+			None => Some(StoredBlockRange(number, number + 1)),
+			Some(range) if range.1 <= number => Some(StoredBlockRange(range.0, number + 1)),
+			Some(_) => None,
+		};
+
+		new_range.map(|n| store.write_stored_block_range(n));
+	};
+
+	// Update the blocks at height meta key.
+	{
+		let mut blocks_at_height = store.load_blocks_at_height(&number)?;
+		if blocks_at_height.contains(&entry.block_hash()) {
+			// seems we already have a block entry for this block. nothing to do here.
+			return Ok(Vec::new());
+		}
+
+		blocks_at_height.push(entry.block_hash());
+		store.write_blocks_at_height(number, blocks_at_height)
+	};
+
+	let mut candidate_entries = Vec::with_capacity(entry.candidates().len());
+
+	// read and write all updated entries.
+	{
+		for (_, candidate_hash) in entry.candidates() {
+			let NewCandidateInfo { candidate, backing_group, our_assignment } =
+				match candidate_info(candidate_hash) {
+					None => return Ok(Vec::new()),
+					Some(info) => info,
+				};
+
+			let mut candidate_entry =
+				store.load_candidate_entry(&candidate_hash)?.unwrap_or_else(move || {
+					CandidateEntry {
+						candidate,
+						session,
+						block_assignments: BTreeMap::new(),
+						approvals: bitvec::bitvec![u8, BitOrderLsb0; 0; n_validators],
+					}
+				});
+
+			candidate_entry.block_assignments.insert(
+				entry.block_hash(),
+				ApprovalEntry::new(
+					Vec::new(),
+					backing_group,
+					our_assignment.map(|v| v.into()),
+					None,
+					bitvec::bitvec![u8, BitOrderLsb0; 0; n_validators],
+					false,
+				),
+			);
+
+			store.write_candidate_entry(candidate_entry.clone());
+
+			candidate_entries.push((*candidate_hash, candidate_entry));
+		}
+	};
+
+	// Update the child index for the parent.
+	store.load_block_entry(&parent_hash)?.map(|mut e| {
+		e.children.push(entry.block_hash());
+		store.write_block_entry(e);
+	});
+
+	// Put the new block entry in.
+	store.write_block_entry(entry);
+
+	Ok(candidate_entries)
+}
+
+/// Forcibly approve all candidates included at up to the given relay-chain height in the indicated
+/// chain.
+pub fn force_approve(
+	store: &mut OverlayedBackend<'_, impl Backend>,
+	chain_head: Hash,
+	up_to: BlockNumber,
+) -> SubsystemResult<Vec<Hash>> {
+	#[derive(PartialEq, Eq)]
+	enum State {
+		WalkTo,
+		Approving,
+	}
+	let mut approved_hashes = Vec::new();
+
+	let mut cur_hash = chain_head;
+	let mut state = State::WalkTo;
+	let mut cur_block_number: BlockNumber = 0;
+
+	// iterate back to the `up_to` block, and then iterate backwards until all blocks
+	// are updated.
+	while let Some(mut entry) = store.load_block_entry(&cur_hash)? {
+		cur_block_number = entry.block_number();
+		if cur_block_number <= up_to {
+			if state == State::WalkTo {
+				gum::debug!(
+					target: LOG_TARGET,
+					block_hash = ?chain_head,
+					?cur_hash,
+					?cur_block_number,
+					"Start forced approval from block",
+				);
+			}
+			state = State::Approving;
+		}
+
+		cur_hash = entry.parent_hash();
+
+		match state {
+			State::WalkTo => {},
+			State::Approving => {
+				entry.approved_bitfield.iter_mut().for_each(|mut b| *b = true);
+				approved_hashes.push(entry.block_hash());
+				store.write_block_entry(entry);
+			},
+		}
+	}
+
+	if state == State::WalkTo {
+		gum::warn!(
+			target: LOG_TARGET,
+			?chain_head,
+			?cur_hash,
+			?cur_block_number,
+			?up_to,
+			"Missing block in the chain, cannot start force approval"
+		);
+	}
+
+	Ok(approved_hashes)
+}
+
+/// Revert to the block corresponding to the specified `hash`.
+/// The operation is not allowed for blocks older than the last finalized one.
+pub fn revert_to(
+	overlay: &mut OverlayedBackend<'_, impl Backend>,
+	hash: Hash,
+) -> SubsystemResult<()> {
+	let mut stored_range = overlay.load_stored_blocks()?.ok_or_else(|| {
+		SubsystemError::Context("no available blocks to infer revert point height".to_string())
+	})?;
+
+	let (children, children_height) = match overlay.load_block_entry(&hash)? {
+		Some(mut entry) => {
+			let children_height = entry.block_number() + 1;
+			let children = std::mem::take(&mut entry.children);
+			// Write revert point block entry without the children.
+			overlay.write_block_entry(entry);
+			(children, children_height)
+		},
+		None => {
+			let children_height = stored_range.0;
+			let children = overlay.load_blocks_at_height(&children_height)?;
+
+			let child_entry = children
+				.first()
+				.and_then(|hash| overlay.load_block_entry(hash).ok())
+				.flatten()
+				.ok_or_else(|| {
+					SubsystemError::Context("lookup failure for first block".to_string())
+				})?;
+
+			// The parent is expected to be the revert point
+			if child_entry.parent_hash() != hash {
+				return Err(SubsystemError::Context(
+					"revert below last finalized block or corrupted storage".to_string(),
+				));
+			}
+
+			(children, children_height)
+		},
+	};
+
+	let mut stack: Vec<_> = children.into_iter().map(|h| (h, children_height)).collect();
+	let mut range_end = stored_range.1;
+
+	while let Some((hash, number)) = stack.pop() {
+		let mut blocks_at_height = overlay.load_blocks_at_height(&number)?;
+		blocks_at_height.retain(|h| h != &hash);
+
+		// Check if we need to update the range top
+		if blocks_at_height.is_empty() && number < range_end {
+			range_end = number;
+		}
+
+		overlay.write_blocks_at_height(number, blocks_at_height);
+
+		if let Some(entry) = overlay.load_block_entry(&hash)? {
+			overlay.delete_block_entry(&hash);
+
+			// Cleanup the candidate entries by removing any reference to the
+			// removed block. If for a candidate entry the block block_assignments
+			// drops to zero then we remove the entry.
+			for (_, candidate_hash) in entry.candidates() {
+				if let Some(mut candidate_entry) = overlay.load_candidate_entry(candidate_hash)? {
+					candidate_entry.block_assignments.remove(&hash);
+					if candidate_entry.block_assignments.is_empty() {
+						overlay.delete_candidate_entry(candidate_hash);
+					} else {
+						overlay.write_candidate_entry(candidate_entry);
+					}
+				}
+			}
+
+			stack.extend(entry.children.into_iter().map(|h| (h, number + 1)));
+		}
+	}
+
+	// Check if our modifications to the dag has reduced the range top
+	if range_end != stored_range.1 {
+		if stored_range.0 < range_end {
+			stored_range.1 = range_end;
+			overlay.write_stored_block_range(stored_range);
+		} else {
+			overlay.delete_stored_block_range();
+		}
+	}
+
+	Ok(())
+}

pezkuwi/node/core/approval-voting/src/persisted_entries.rs

@@ -0,0 +1,769 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Entries pertaining to approval which need to be persisted.
+//!
+//! The actual persisting of data is handled by the `approval_db` module.
+//! Within that context, things are plain-old-data. Within this module,
+//! data and logic are intertwined.
+
+use itertools::Itertools;
+use pezkuwi_node_primitives::approval::{
+	v1::{DelayTranche, RelayVRFStory},
+	v2::{AssignmentCertV2, CandidateBitfield},
+};
+use pezkuwi_primitives::{
+	BlockNumber, CandidateHash, CandidateIndex, CandidateReceiptV2 as CandidateReceipt, CoreIndex,
+	GroupIndex, Hash, SessionIndex, ValidatorIndex, ValidatorSignature,
+};
+use sp_consensus_slots::Slot;
+
+use bitvec::{order::Lsb0 as BitOrderLsb0, slice::BitSlice};
+use std::collections::BTreeMap;
+
+use crate::approval_db::v2::Bitfield;
+
+use super::criteria::OurAssignment;
+
+use pezkuwi_node_primitives::approval::time::Tick;
+
+/// Metadata regarding a specific tranche of assignments for a specific candidate.
+#[derive(Debug, Clone, PartialEq)]
+pub struct TrancheEntry {
+	tranche: DelayTranche,
+	// Assigned validators, and the instant we received their assignment, rounded
+	// to the nearest tick.
+	assignments: Vec<(ValidatorIndex, Tick)>,
+}
+
+impl TrancheEntry {
+	/// Get the tranche of this entry.
+	pub fn tranche(&self) -> DelayTranche {
+		self.tranche
+	}
+
+	/// Get the assignments for this entry.
+	pub fn assignments(&self) -> &[(ValidatorIndex, Tick)] {
+		&self.assignments
+	}
+}
+
+impl From<crate::approval_db::v2::TrancheEntry> for TrancheEntry {
+	fn from(entry: crate::approval_db::v2::TrancheEntry) -> Self {
+		TrancheEntry {
+			tranche: entry.tranche,
+			assignments: entry.assignments.into_iter().map(|(v, t)| (v, t.into())).collect(),
+		}
+	}
+}
+
+impl From<TrancheEntry> for crate::approval_db::v2::TrancheEntry {
+	fn from(entry: TrancheEntry) -> Self {
+		Self {
+			tranche: entry.tranche,
+			assignments: entry.assignments.into_iter().map(|(v, t)| (v, t.into())).collect(),
+		}
+	}
+}
+
+impl From<crate::approval_db::v3::OurApproval> for OurApproval {
+	fn from(approval: crate::approval_db::v3::OurApproval) -> Self {
+		Self {
+			signature: approval.signature,
+			signed_candidates_indices: approval.signed_candidates_indices,
+		}
+	}
+}
+impl From<OurApproval> for crate::approval_db::v3::OurApproval {
+	fn from(approval: OurApproval) -> Self {
+		Self {
+			signature: approval.signature,
+			signed_candidates_indices: approval.signed_candidates_indices,
+		}
+	}
+}
+
+/// Metadata about our approval signature
+#[derive(Debug, Clone, PartialEq)]
+pub struct OurApproval {
+	/// The signature for the candidates hashes pointed by indices.
+	pub signature: ValidatorSignature,
+	/// The indices of the candidates signed in this approval.
+	pub signed_candidates_indices: CandidateBitfield,
+}
+
+impl OurApproval {
+	/// Converts a ValidatorSignature to an OurApproval.
+	/// It used in converting the database from v1 to latest.
+	pub fn from_v1(value: ValidatorSignature, candidate_index: CandidateIndex) -> Self {
+		Self { signature: value, signed_candidates_indices: candidate_index.into() }
+	}
+
+	/// Converts a ValidatorSignature to an OurApproval.
+	/// It used in converting the database from v2 to latest.
+	pub fn from_v2(value: ValidatorSignature, candidate_index: CandidateIndex) -> Self {
+		Self::from_v1(value, candidate_index)
+	}
+}
+/// Metadata regarding approval of a particular candidate within the context of some
+/// particular block.
+#[derive(Debug, Clone, PartialEq)]
+pub struct ApprovalEntry {
+	tranches: Vec<TrancheEntry>,
+	backing_group: GroupIndex,
+	our_assignment: Option<OurAssignment>,
+	our_approval_sig: Option<OurApproval>,
+	// `n_validators` bits.
+	assigned_validators: Bitfield,
+	approved: bool,
+}
+
+impl ApprovalEntry {
+	/// Convenience constructor
+	pub fn new(
+		tranches: Vec<TrancheEntry>,
+		backing_group: GroupIndex,
+		our_assignment: Option<OurAssignment>,
+		our_approval_sig: Option<OurApproval>,
+		// `n_validators` bits.
+		assigned_validators: Bitfield,
+		approved: bool,
+	) -> Self {
+		Self {
+			tranches,
+			backing_group,
+			our_assignment,
+			our_approval_sig,
+			assigned_validators,
+			approved,
+		}
+	}
+
+	// Access our assignment for this approval entry.
+	pub fn our_assignment(&self) -> Option<&OurAssignment> {
+		self.our_assignment.as_ref()
+	}
+
+	// Note that our assignment is triggered. No-op if already triggered.
+	pub fn trigger_our_assignment(
+		&mut self,
+		tick_now: Tick,
+	) -> Option<(AssignmentCertV2, ValidatorIndex, DelayTranche)> {
+		let our = self.our_assignment.as_mut().and_then(|a| {
+			if a.triggered() {
+				return None;
+			}
+			a.mark_triggered();
+
+			Some(a.clone())
+		});
+
+		our.map(|a| {
+			self.import_assignment(a.tranche(), a.validator_index(), tick_now, false);
+
+			(a.cert().clone(), a.validator_index(), a.tranche())
+		})
+	}
+
+	/// Import our local approval vote signature for this candidate.
+	pub fn import_approval_sig(&mut self, approval_sig: OurApproval) {
+		self.our_approval_sig = Some(approval_sig);
+	}
+
+	/// Whether a validator is already assigned.
+	pub fn is_assigned(&self, validator_index: ValidatorIndex) -> bool {
+		self.assigned_validators
+			.get(validator_index.0 as usize)
+			.map(|b| *b)
+			.unwrap_or(false)
+	}
+
+	/// Import an assignment. No-op if already assigned on the same tranche.
+	pub fn import_assignment(
+		&mut self,
+		tranche: DelayTranche,
+		validator_index: ValidatorIndex,
+		tick_now: Tick,
+		is_duplicate: bool,
+	) {
+		// linear search probably faster than binary. not many tranches typically.
+		let idx = match self.tranches.iter().position(|t| t.tranche >= tranche) {
+			Some(pos) => {
+				if self.tranches[pos].tranche > tranche {
+					self.tranches.insert(pos, TrancheEntry { tranche, assignments: Vec::new() });
+				}
+
+				pos
+			},
+			None => {
+				self.tranches.push(TrancheEntry { tranche, assignments: Vec::new() });
+
+				self.tranches.len() - 1
+			},
+		};
+		// At restart we might have duplicate assignments because approval-distribution is not
+		// persistent across restarts, so avoid adding duplicates.
+		// We already know if we have seen an assignment from this validator and since this
+		// function is on the hot path we can avoid iterating through tranches by using
+		// !is_duplicate to determine if it is already present in the vector and does not need
+		// adding.
+		if !is_duplicate {
+			self.tranches[idx].assignments.push((validator_index, tick_now));
+		}
+		self.assigned_validators.set(validator_index.0 as _, true);
+	}
+
+	// Produce a bitvec indicating the assignments of all validators up to and
+	// including `tranche`.
+	pub fn assignments_up_to(&self, tranche: DelayTranche) -> Bitfield {
+		self.tranches.iter().take_while(|e| e.tranche <= tranche).fold(
+			bitvec::bitvec![u8, BitOrderLsb0; 0; self.assigned_validators.len()],
+			|mut a, e| {
+				for &(v, _) in &e.assignments {
+					a.set(v.0 as _, true);
+				}
+
+				a
+			},
+		)
+	}
+
+	/// Whether the approval entry is approved
+	pub fn is_approved(&self) -> bool {
+		self.approved
+	}
+
+	/// Mark the approval entry as approved.
+	pub fn mark_approved(&mut self) {
+		self.approved = true;
+	}
+
+	/// Access the tranches.
+	pub fn tranches(&self) -> &[TrancheEntry] {
+		&self.tranches
+	}
+
+	/// Get the number of validators in this approval entry.
+	pub fn n_validators(&self) -> usize {
+		self.assigned_validators.len()
+	}
+
+	/// Get the number of assignments by validators, including the local validator.
+	pub fn n_assignments(&self) -> usize {
+		self.assigned_validators.count_ones()
+	}
+
+	/// Get the backing group index of the approval entry.
+	pub fn backing_group(&self) -> GroupIndex {
+		self.backing_group
+	}
+
+	/// Get the assignment cert & approval signature.
+	///
+	/// The approval signature will only be `Some` if the assignment is too.
+	pub fn local_statements(&self) -> (Option<OurAssignment>, Option<OurApproval>) {
+		let approval_sig = self.our_approval_sig.clone();
+		if let Some(our_assignment) = self.our_assignment.as_ref().filter(|a| a.triggered()) {
+			(Some(our_assignment.clone()), approval_sig)
+		} else {
+			(None, None)
+		}
+	}
+
+	// Convert an ApprovalEntry from v1 version to latest version
+	pub fn from_v1(
+		value: crate::approval_db::v1::ApprovalEntry,
+		candidate_index: CandidateIndex,
+	) -> Self {
+		ApprovalEntry {
+			tranches: value.tranches.into_iter().map(|tranche| tranche.into()).collect(),
+			backing_group: value.backing_group,
+			our_assignment: value.our_assignment.map(|assignment| assignment.into()),
+			our_approval_sig: value
+				.our_approval_sig
+				.map(|sig| OurApproval::from_v1(sig, candidate_index)),
+			assigned_validators: value.assignments,
+			approved: value.approved,
+		}
+	}
+
+	// Convert an ApprovalEntry from v1 version to latest version
+	pub fn from_v2(
+		value: crate::approval_db::v2::ApprovalEntry,
+		candidate_index: CandidateIndex,
+	) -> Self {
+		ApprovalEntry {
+			tranches: value.tranches.into_iter().map(|tranche| tranche.into()).collect(),
+			backing_group: value.backing_group,
+			our_assignment: value.our_assignment.map(|assignment| assignment.into()),
+			our_approval_sig: value
+				.our_approval_sig
+				.map(|sig| OurApproval::from_v2(sig, candidate_index)),
+			assigned_validators: value.assigned_validators,
+			approved: value.approved,
+		}
+	}
+}
+
+impl From<crate::approval_db::v3::ApprovalEntry> for ApprovalEntry {
+	fn from(entry: crate::approval_db::v3::ApprovalEntry) -> Self {
+		ApprovalEntry {
+			tranches: entry.tranches.into_iter().map(Into::into).collect(),
+			backing_group: entry.backing_group,
+			our_assignment: entry.our_assignment.map(Into::into),
+			our_approval_sig: entry.our_approval_sig.map(Into::into),
+			assigned_validators: entry.assigned_validators,
+			approved: entry.approved,
+		}
+	}
+}
+
+impl From<ApprovalEntry> for crate::approval_db::v3::ApprovalEntry {
+	fn from(entry: ApprovalEntry) -> Self {
+		Self {
+			tranches: entry.tranches.into_iter().map(Into::into).collect(),
+			backing_group: entry.backing_group,
+			our_assignment: entry.our_assignment.map(Into::into),
+			our_approval_sig: entry.our_approval_sig.map(Into::into),
+			assigned_validators: entry.assigned_validators,
+			approved: entry.approved,
+		}
+	}
+}
+
+/// Metadata regarding approval of a particular candidate.
+#[derive(Debug, Clone, PartialEq)]
+pub struct CandidateEntry {
+	pub candidate: CandidateReceipt,
+	pub session: SessionIndex,
+	// Assignments are based on blocks, so we need to track assignments separately
+	// based on the block we are looking at.
+	pub block_assignments: BTreeMap<Hash, ApprovalEntry>,
+	pub approvals: Bitfield,
+}
+
+impl CandidateEntry {
+	/// Access the bit-vec of approvals.
+	pub fn approvals(&self) -> &BitSlice<u8, BitOrderLsb0> {
+		&self.approvals
+	}
+
+	/// Note that a given validator has approved. Return the previous approval state.
+	pub fn mark_approval(&mut self, validator: ValidatorIndex) -> bool {
+		let prev = self.has_approved(validator);
+		self.approvals.set(validator.0 as usize, true);
+		prev
+	}
+
+	/// Query whether a given validator has approved the candidate.
+	pub fn has_approved(&self, validator: ValidatorIndex) -> bool {
+		self.approvals.get(validator.0 as usize).map(|b| *b).unwrap_or(false)
+	}
+
+	/// Get the candidate receipt.
+	pub fn candidate_receipt(&self) -> &CandidateReceipt {
+		&self.candidate
+	}
+
+	/// Get the approval entry, mutably, for this candidate under a specific block.
+	pub fn approval_entry_mut(&mut self, block_hash: &Hash) -> Option<&mut ApprovalEntry> {
+		self.block_assignments.get_mut(block_hash)
+	}
+
+	/// Get the approval entry for this candidate under a specific block.
+	pub fn approval_entry(&self, block_hash: &Hash) -> Option<&ApprovalEntry> {
+		self.block_assignments.get(block_hash)
+	}
+
+	/// Convert a CandidateEntry from a v1 to its latest equivalent.
+	pub fn from_v1(
+		value: crate::approval_db::v1::CandidateEntry,
+		candidate_index: CandidateIndex,
+	) -> Self {
+		Self {
+			approvals: value.approvals,
+			block_assignments: value
+				.block_assignments
+				.into_iter()
+				.map(|(h, ae)| (h, ApprovalEntry::from_v1(ae, candidate_index)))
+				.collect(),
+			candidate: value.candidate,
+			session: value.session,
+		}
+	}
+
+	/// Convert a CandidateEntry from a v2 to its latest equivalent.
+	pub fn from_v2(
+		value: crate::approval_db::v2::CandidateEntry,
+		candidate_index: CandidateIndex,
+	) -> Self {
+		Self {
+			approvals: value.approvals,
+			block_assignments: value
+				.block_assignments
+				.into_iter()
+				.map(|(h, ae)| (h, ApprovalEntry::from_v2(ae, candidate_index)))
+				.collect(),
+			candidate: value.candidate,
+			session: value.session,
+		}
+	}
+}
+
+impl From<crate::approval_db::v3::CandidateEntry> for CandidateEntry {
+	fn from(entry: crate::approval_db::v3::CandidateEntry) -> Self {
+		CandidateEntry {
+			candidate: entry.candidate,
+			session: entry.session,
+			block_assignments: entry
+				.block_assignments
+				.into_iter()
+				.map(|(h, ae)| (h, ae.into()))
+				.collect(),
+			approvals: entry.approvals,
+		}
+	}
+}
+
+impl From<CandidateEntry> for crate::approval_db::v3::CandidateEntry {
+	fn from(entry: CandidateEntry) -> Self {
+		Self {
+			candidate: entry.candidate,
+			session: entry.session,
+			block_assignments: entry
+				.block_assignments
+				.into_iter()
+				.map(|(h, ae)| (h, ae.into()))
+				.collect(),
+			approvals: entry.approvals,
+		}
+	}
+}
+
+/// Metadata regarding approval of a particular block, by way of approval of the
+/// candidates contained within it.
+#[derive(Debug, Clone, PartialEq)]
+pub struct BlockEntry {
+	block_hash: Hash,
+	parent_hash: Hash,
+	block_number: BlockNumber,
+	session: SessionIndex,
+	slot: Slot,
+	relay_vrf_story: RelayVRFStory,
+	// The candidates included as-of this block and the index of the core they are
+	// leaving.
+	candidates: Vec<(CoreIndex, CandidateHash)>,
+	// A bitfield where the i'th bit corresponds to the i'th candidate in `candidates`.
+	// The i'th bit is `true` iff the candidate has been approved in the context of this
+	// block. The block can be considered approved if the bitfield has all bits set to `true`.
+	pub approved_bitfield: Bitfield,
+	pub children: Vec<Hash>,
+	// A list of candidates we have checked, but didn't not sign and
+	// advertise the vote yet.
+	candidates_pending_signature: BTreeMap<CandidateIndex, CandidateSigningContext>,
+	// A list of assignments for which we already distributed the assignment.
+	// We use this to ensure we don't distribute multiple core assignments twice as we track
+	// individual wakeups for each core.
+	distributed_assignments: Bitfield,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct CandidateSigningContext {
+	pub candidate_hash: CandidateHash,
+	pub sign_no_later_than_tick: Tick,
+}
+
+impl BlockEntry {
+	/// Mark a candidate as fully approved in the bitfield.
+	pub fn mark_approved_by_hash(&mut self, candidate_hash: &CandidateHash) {
+		if let Some(p) = self.candidates.iter().position(|(_, h)| h == candidate_hash) {
+			self.approved_bitfield.set(p, true);
+		}
+	}
+
+	/// Whether a candidate is approved in the bitfield.
+	pub fn is_candidate_approved(&self, candidate_hash: &CandidateHash) -> bool {
+		self.candidates
+			.iter()
+			.position(|(_, h)| h == candidate_hash)
+			.and_then(|p| self.approved_bitfield.get(p).map(|b| *b))
+			.unwrap_or(false)
+	}
+
+	/// Whether the block entry is fully approved.
+	pub fn is_fully_approved(&self) -> bool {
+		self.approved_bitfield.all()
+	}
+
+	/// Iterate over all unapproved candidates.
+	pub fn unapproved_candidates(&self) -> impl Iterator<Item = CandidateHash> + '_ {
+		self.approved_bitfield.iter().enumerate().filter_map(move |(i, a)| {
+			if !*a {
+				Some(self.candidates[i].1)
+			} else {
+				None
+			}
+		})
+	}
+
+	/// Get the slot of the block.
+	pub fn slot(&self) -> Slot {
+		self.slot
+	}
+
+	/// Get the relay-vrf-story of the block.
+	pub fn relay_vrf_story(&self) -> RelayVRFStory {
+		self.relay_vrf_story.clone()
+	}
+
+	/// Get the session index of the block.
+	pub fn session(&self) -> SessionIndex {
+		self.session
+	}
+
+	/// Get the i'th candidate.
+	pub fn candidate(&self, i: usize) -> Option<&(CoreIndex, CandidateHash)> {
+		self.candidates.get(i)
+	}
+
+	/// Access the underlying candidates as a slice.
+	pub fn candidates(&self) -> &[(CoreIndex, CandidateHash)] {
+		&self.candidates
+	}
+
+	/// Access the block number of the block entry.
+	pub fn block_number(&self) -> BlockNumber {
+		self.block_number
+	}
+
+	/// Access the block hash of the block entry.
+	pub fn block_hash(&self) -> Hash {
+		self.block_hash
+	}
+
+	/// Access the parent hash of the block entry.
+	pub fn parent_hash(&self) -> Hash {
+		self.parent_hash
+	}
+
+	/// Mark distributed assignment for many candidate indices.
+	/// Returns `true` if an assignment was already distributed for the `candidates`.
+	pub fn mark_assignment_distributed(&mut self, candidates: CandidateBitfield) -> bool {
+		let bitfield = candidates.into_inner();
+		let total_one_bits = self.distributed_assignments.count_ones();
+
+		let new_len = std::cmp::max(self.distributed_assignments.len(), bitfield.len());
+		self.distributed_assignments.resize(new_len, false);
+		self.distributed_assignments |= bitfield;
+
+		// If an operation did not change our current bitfield, we return true.
+		let distributed = total_one_bits == self.distributed_assignments.count_ones();
+
+		distributed
+	}
+
+	/// Defer signing and issuing an approval for a candidate no later than the specified tick
+	pub fn defer_candidate_signature(
+		&mut self,
+		candidate_index: CandidateIndex,
+		candidate_hash: CandidateHash,
+		sign_no_later_than_tick: Tick,
+	) -> Option<CandidateSigningContext> {
+		self.candidates_pending_signature.insert(
+			candidate_index,
+			CandidateSigningContext { candidate_hash, sign_no_later_than_tick },
+		)
+	}
+
+	/// Returns the number of candidates waiting for an approval to be issued.
+	pub fn num_candidates_pending_signature(&self) -> usize {
+		self.candidates_pending_signature.len()
+	}
+
+	/// Return if we have candidates waiting for signature to be issued
+	pub fn has_candidates_pending_signature(&self) -> bool {
+		!self.candidates_pending_signature.is_empty()
+	}
+
+	/// Returns true if candidate hash is in the queue for a signature.
+	pub fn candidate_is_pending_signature(&self, candidate_hash: CandidateHash) -> bool {
+		self.candidates_pending_signature
+			.values()
+			.any(|context| context.candidate_hash == candidate_hash)
+	}
+
+	/// Candidate hashes  for candidates pending signatures
+	fn candidate_hashes_pending_signature(&self) -> Vec<CandidateHash> {
+		self.candidates_pending_signature
+			.values()
+			.map(|unsigned_approval| unsigned_approval.candidate_hash)
+			.collect()
+	}
+
+	/// Candidate indices for candidates pending signature
+	fn candidate_indices_pending_signature(&self) -> Option<CandidateBitfield> {
+		self.candidates_pending_signature
+			.keys()
+			.map(|val| *val)
+			.collect_vec()
+			.try_into()
+			.ok()
+	}
+
+	/// Returns a list of candidates hashes that need need signature created at the current tick:
+	/// This might happen in other of the two reasons:
+	/// 1. We queued more than max_approval_coalesce_count candidates.
+	/// 2. We have candidates that waiting in the queue past their `sign_no_later_than_tick`
+	///
+	/// Additionally, we also return the first tick when we will have to create a signature,
+	/// so that the caller can arm the timer if it is not already armed.
+	pub fn get_candidates_that_need_signature(
+		&self,
+		tick_now: Tick,
+		max_approval_coalesce_count: u32,
+	) -> (Option<(Vec<CandidateHash>, CandidateBitfield)>, Option<Tick>) {
+		let sign_no_later_than_tick = self
+			.candidates_pending_signature
+			.values()
+			.min_by(|a, b| a.sign_no_later_than_tick.cmp(&b.sign_no_later_than_tick))
+			.map(|val| val.sign_no_later_than_tick);
+
+		if let Some(sign_no_later_than_tick) = sign_no_later_than_tick {
+			if sign_no_later_than_tick <= tick_now ||
+				self.num_candidates_pending_signature() >= max_approval_coalesce_count as usize
+			{
+				(
+					self.candidate_indices_pending_signature().and_then(|candidate_indices| {
+						Some((self.candidate_hashes_pending_signature(), candidate_indices))
+					}),
+					Some(sign_no_later_than_tick),
+				)
+			} else {
+				// We can still wait for other candidates to queue in, so just make sure
+				// we wake up at the tick we have to sign the longest waiting candidate.
+				(Default::default(), Some(sign_no_later_than_tick))
+			}
+		} else {
+			// No cached candidates, nothing to do here, this just means the timer fired,
+			// but the signatures were already sent because we gathered more than
+			// max_approval_coalesce_count.
+			(Default::default(), sign_no_later_than_tick)
+		}
+	}
+
+	/// Clears the candidates pending signature because the approval was issued.
+	pub fn issued_approval(&mut self) {
+		self.candidates_pending_signature.clear();
+	}
+}
+
+impl From<crate::approval_db::v3::BlockEntry> for BlockEntry {
+	fn from(entry: crate::approval_db::v3::BlockEntry) -> Self {
+		BlockEntry {
+			block_hash: entry.block_hash,
+			parent_hash: entry.parent_hash,
+			block_number: entry.block_number,
+			session: entry.session,
+			slot: entry.slot,
+			relay_vrf_story: RelayVRFStory(entry.relay_vrf_story),
+			candidates: entry.candidates,
+			approved_bitfield: entry.approved_bitfield,
+			children: entry.children,
+			candidates_pending_signature: entry
+				.candidates_pending_signature
+				.into_iter()
+				.map(|(candidate_index, signing_context)| (candidate_index, signing_context.into()))
+				.collect(),
+			distributed_assignments: entry.distributed_assignments,
+		}
+	}
+}
+
+impl From<crate::approval_db::v1::BlockEntry> for BlockEntry {
+	fn from(entry: crate::approval_db::v1::BlockEntry) -> Self {
+		BlockEntry {
+			block_hash: entry.block_hash,
+			parent_hash: entry.parent_hash,
+			block_number: entry.block_number,
+			session: entry.session,
+			slot: entry.slot,
+			relay_vrf_story: RelayVRFStory(entry.relay_vrf_story),
+			candidates: entry.candidates,
+			approved_bitfield: entry.approved_bitfield,
+			children: entry.children,
+			distributed_assignments: Default::default(),
+			candidates_pending_signature: Default::default(),
+		}
+	}
+}
+
+impl From<crate::approval_db::v2::BlockEntry> for BlockEntry {
+	fn from(entry: crate::approval_db::v2::BlockEntry) -> Self {
+		BlockEntry {
+			block_hash: entry.block_hash,
+			parent_hash: entry.parent_hash,
+			block_number: entry.block_number,
+			session: entry.session,
+			slot: entry.slot,
+			relay_vrf_story: RelayVRFStory(entry.relay_vrf_story),
+			candidates: entry.candidates,
+			approved_bitfield: entry.approved_bitfield,
+			children: entry.children,
+			distributed_assignments: entry.distributed_assignments,
+			candidates_pending_signature: Default::default(),
+		}
+	}
+}
+
+impl From<BlockEntry> for crate::approval_db::v3::BlockEntry {
+	fn from(entry: BlockEntry) -> Self {
+		Self {
+			block_hash: entry.block_hash,
+			parent_hash: entry.parent_hash,
+			block_number: entry.block_number,
+			session: entry.session,
+			slot: entry.slot,
+			relay_vrf_story: entry.relay_vrf_story.0,
+			candidates: entry.candidates,
+			approved_bitfield: entry.approved_bitfield,
+			children: entry.children,
+			candidates_pending_signature: entry
+				.candidates_pending_signature
+				.into_iter()
+				.map(|(candidate_index, signing_context)| (candidate_index, signing_context.into()))
+				.collect(),
+			distributed_assignments: entry.distributed_assignments,
+		}
+	}
+}
+
+impl From<crate::approval_db::v3::CandidateSigningContext> for CandidateSigningContext {
+	fn from(signing_context: crate::approval_db::v3::CandidateSigningContext) -> Self {
+		Self {
+			candidate_hash: signing_context.candidate_hash,
+			sign_no_later_than_tick: signing_context.sign_no_later_than_tick.into(),
+		}
+	}
+}
+
+impl From<CandidateSigningContext> for crate::approval_db::v3::CandidateSigningContext {
+	fn from(signing_context: CandidateSigningContext) -> Self {
+		Self {
+			candidate_hash: signing_context.candidate_hash,
+			sign_no_later_than_tick: signing_context.sign_no_later_than_tick.into(),
+		}
+	}
+}

pezkuwi/node/core/av-store/Cargo.toml

@@ -0,0 +1,52 @@
+[package]
+name = "pezkuwi-node-core-av-store"
+description = "The Availability Store subsystem. Wrapper over the DB that stores availability data and chunks."
+version = "7.0.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+
+[lints]
+workspace = true
+
+[dependencies]
+bitvec = { workspace = true, default-features = true }
+futures = { workspace = true }
+futures-timer = { workspace = true }
+gum = { workspace = true, default-features = true }
+thiserror = { workspace = true }
+
+codec = { features = ["derive"], workspace = true, default-features = true }
+pezkuwi-erasure-coding = { workspace = true, default-features = true }
+pezkuwi-node-primitives = { workspace = true, default-features = true }
+pezkuwi-node-subsystem = { workspace = true, default-features = true }
+pezkuwi-node-subsystem-util = { workspace = true, default-features = true }
+pezkuwi-primitives = { workspace = true, default-features = true }
+sp-consensus = { workspace = true }
+
+[dev-dependencies]
+assert_matches = { workspace = true }
+kvdb-memorydb = { workspace = true }
+sp-tracing = { workspace = true }
+
+parking_lot = { workspace = true, default-features = true }
+pezkuwi-node-subsystem-test-helpers = { workspace = true }
+pezkuwi-primitives-test-helpers = { workspace = true }
+sp-core = { workspace = true, default-features = true }
+sp-keyring = { workspace = true, default-features = true }
+
+[features]
+runtime-benchmarks = [
+	"gum/runtime-benchmarks",
+	"pezkuwi-erasure-coding/runtime-benchmarks",
+	"pezkuwi-node-primitives/runtime-benchmarks",
+	"pezkuwi-node-subsystem-test-helpers/runtime-benchmarks",
+	"pezkuwi-node-subsystem-util/runtime-benchmarks",
+	"pezkuwi-node-subsystem/runtime-benchmarks",
+	"pezkuwi-primitives-test-helpers/runtime-benchmarks",
+	"pezkuwi-primitives/runtime-benchmarks",
+	"sp-consensus/runtime-benchmarks",
+	"sp-keyring/runtime-benchmarks",
+]

pezkuwi/node/core/av-store/src/metrics.rs

@@ -0,0 +1,158 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use pezkuwi_node_subsystem_util::metrics::{self, prometheus};
+
+#[derive(Clone)]
+pub(crate) struct MetricsInner {
+	received_availability_chunks_total: prometheus::Counter<prometheus::U64>,
+	pruning: prometheus::Histogram,
+	process_block_finalized: prometheus::Histogram,
+	block_activated: prometheus::Histogram,
+	process_message: prometheus::Histogram,
+	store_available_data: prometheus::Histogram,
+	store_chunk: prometheus::Histogram,
+	get_chunk: prometheus::Histogram,
+}
+
+/// Availability metrics.
+#[derive(Default, Clone)]
+pub struct Metrics(Option<MetricsInner>);
+
+impl Metrics {
+	pub(crate) fn on_chunks_received(&self, count: usize) {
+		if let Some(metrics) = &self.0 {
+			// assume usize fits into u64
+			let by = u64::try_from(count).unwrap_or_default();
+			metrics.received_availability_chunks_total.inc_by(by);
+		}
+	}
+
+	/// Provide a timer for `prune_povs` which observes on drop.
+	pub(crate) fn time_pruning(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.pruning.start_timer())
+	}
+
+	/// Provide a timer for `process_block_finalized` which observes on drop.
+	pub(crate) fn time_process_block_finalized(
+		&self,
+	) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.process_block_finalized.start_timer())
+	}
+
+	/// Provide a timer for `block_activated` which observes on drop.
+	pub(crate) fn time_block_activated(
+		&self,
+	) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.block_activated.start_timer())
+	}
+
+	/// Provide a timer for `process_message` which observes on drop.
+	pub(crate) fn time_process_message(
+		&self,
+	) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.process_message.start_timer())
+	}
+
+	/// Provide a timer for `store_available_data` which observes on drop.
+	pub(crate) fn time_store_available_data(
+		&self,
+	) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.store_available_data.start_timer())
+	}
+
+	/// Provide a timer for `store_chunk` which observes on drop.
+	pub(crate) fn time_store_chunk(
+		&self,
+	) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.store_chunk.start_timer())
+	}
+
+	/// Provide a timer for `get_chunk` which observes on drop.
+	pub(crate) fn time_get_chunk(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.get_chunk.start_timer())
+	}
+}
+
+impl metrics::Metrics for Metrics {
+	fn try_register(registry: &prometheus::Registry) -> Result<Self, prometheus::PrometheusError> {
+		let metrics = MetricsInner {
+			received_availability_chunks_total: prometheus::register(
+				prometheus::Counter::new(
+					"pezkuwi_teyrchain_received_availability_chunks_total",
+					"Number of availability chunks received.",
+				)?,
+				registry,
+			)?,
+			pruning: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_av_store_pruning",
+					"Time spent within `av_store::prune_all`",
+				))?,
+				registry,
+			)?,
+			process_block_finalized: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_av_store_process_block_finalized",
+					"Time spent within `av_store::process_block_finalized`",
+				))?,
+				registry,
+			)?,
+			block_activated: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_av_store_block_activated",
+					"Time spent within `av_store::process_block_activated`",
+				))?,
+				registry,
+			)?,
+			process_message: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_av_store_process_message",
+					"Time spent within `av_store::process_message`",
+				))?,
+				registry,
+			)?,
+			store_available_data: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_av_store_store_available_data",
+					"Time spent within `av_store::store_available_data`",
+				))?,
+				registry,
+			)?,
+			store_chunk: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_av_store_store_chunk",
+					"Time spent within `av_store::store_chunk`",
+				))?,
+				registry,
+			)?,
+			get_chunk: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"pezkuwi_teyrchain_av_store_get_chunk",
+						"Time spent fetching requested chunks.`",
+					)
+					.buckets(vec![
+						0.000625, 0.00125, 0.0025, 0.005, 0.0075, 0.01, 0.025, 0.05, 0.1, 0.25,
+						0.5, 1.0, 2.5, 5.0, 10.0,
+					]),
+				)?,
+				registry,
+			)?,
+		};
+		Ok(Metrics(Some(metrics)))
+	}
+}

pezkuwi/node/core/backing/Cargo.toml

@@ -0,0 +1,55 @@
+[package]
+name = "pezkuwi-node-core-backing"
+version = "7.0.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+description = "The Candidate Backing Subsystem. Tracks teyrchain candidates that can be backed, as well as the issuance of statements about candidates."
+homepage.workspace = true
+repository.workspace = true
+
+[lints]
+workspace = true
+
+[dependencies]
+bitvec = { features = ["alloc"], workspace = true }
+fatality = { workspace = true }
+futures = { workspace = true }
+gum = { workspace = true, default-features = true }
+pezkuwi-erasure-coding = { workspace = true, default-features = true }
+pezkuwi-node-primitives = { workspace = true, default-features = true }
+pezkuwi-node-subsystem = { workspace = true, default-features = true }
+pezkuwi-node-subsystem-util = { workspace = true, default-features = true }
+pezkuwi-primitives = { workspace = true, default-features = true }
+pezkuwi-statement-table = { workspace = true, default-features = true }
+pezkuwi-teyrchain-primitives = { workspace = true, default-features = true }
+schnellru = { workspace = true }
+sp-keystore = { workspace = true, default-features = true }
+thiserror = { workspace = true }
+
+[dev-dependencies]
+assert_matches = { workspace = true }
+futures = { features = ["thread-pool"], workspace = true }
+pezkuwi-node-subsystem-test-helpers = { workspace = true }
+pezkuwi-primitives = { workspace = true, features = ["test"] }
+pezkuwi-primitives-test-helpers = { workspace = true }
+sc-keystore = { workspace = true, default-features = true }
+sp-application-crypto = { workspace = true, default-features = true }
+sp-core = { workspace = true, default-features = true }
+sp-keyring = { workspace = true, default-features = true }
+sp-tracing = { workspace = true, default-features = true }
+
+[features]
+runtime-benchmarks = [
+	"gum/runtime-benchmarks",
+	"pezkuwi-erasure-coding/runtime-benchmarks",
+	"pezkuwi-node-primitives/runtime-benchmarks",
+	"pezkuwi-node-subsystem-test-helpers/runtime-benchmarks",
+	"pezkuwi-node-subsystem-util/runtime-benchmarks",
+	"pezkuwi-node-subsystem/runtime-benchmarks",
+	"pezkuwi-primitives-test-helpers/runtime-benchmarks",
+	"pezkuwi-primitives/runtime-benchmarks",
+	"pezkuwi-statement-table/runtime-benchmarks",
+	"pezkuwi-teyrchain-primitives/runtime-benchmarks",
+	"sp-keyring/runtime-benchmarks",
+]

pezkuwi/node/core/backing/src/error.rs

@@ -0,0 +1,132 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::collections::HashMap;
+
+use fatality::Nested;
+use futures::channel::{mpsc, oneshot};
+
+use pezkuwi_node_subsystem::{
+	messages::{StoreAvailableDataError, ValidationFailed},
+	RuntimeApiError, SubsystemError,
+};
+use pezkuwi_node_subsystem_util::{runtime, Error as UtilError};
+use pezkuwi_primitives::{BackedCandidate, ValidationCodeHash};
+
+use crate::{ParaId, LOG_TARGET};
+
+pub type Result<T> = std::result::Result<T, Error>;
+pub type FatalResult<T> = std::result::Result<T, FatalError>;
+
+/// Errors that can occur in candidate backing.
+#[allow(missing_docs)]
+#[fatality::fatality(splitable)]
+pub enum Error {
+	#[fatal]
+	#[error("Failed to spawn background task")]
+	FailedToSpawnBackgroundTask,
+
+	#[fatal(forward)]
+	#[error("Error while accessing runtime information")]
+	Runtime(#[from] runtime::Error),
+
+	#[fatal]
+	#[error(transparent)]
+	BackgroundValidationMpsc(#[from] mpsc::SendError),
+
+	#[error("Candidate is not found")]
+	CandidateNotFound,
+
+	#[error("CoreIndex cannot be determined for a candidate")]
+	CoreIndexUnavailable,
+
+	#[error("Signature is invalid")]
+	InvalidSignature,
+
+	#[error("Failed to send candidates {0:?}")]
+	Send(HashMap<ParaId, Vec<BackedCandidate>>),
+
+	#[error("FetchPoV failed")]
+	FetchPoV,
+
+	#[error("Fetching validation code by hash failed {0:?}, {1:?}")]
+	FetchValidationCode(ValidationCodeHash, RuntimeApiError),
+
+	#[error("Fetching Runtime API version failed {0:?}")]
+	FetchRuntimeApiVersion(RuntimeApiError),
+
+	#[error("No validation code {0:?}")]
+	NoValidationCode(ValidationCodeHash),
+
+	#[error("Candidate rejected by prospective teyrchains subsystem")]
+	RejectedByProspectiveTeyrchains,
+
+	#[error("ValidateFromExhaustive channel closed before receipt")]
+	ValidateFromExhaustive(#[source] oneshot::Canceled),
+
+	#[error("StoreAvailableData channel closed before receipt")]
+	StoreAvailableDataChannel(#[source] oneshot::Canceled),
+
+	#[error("RuntimeAPISubsystem channel closed before receipt")]
+	RuntimeApiUnavailable(#[source] oneshot::Canceled),
+
+	#[error("a channel was closed before receipt in try_join!")]
+	#[fatal]
+	JoinMultiple(#[source] oneshot::Canceled),
+
+	#[error("Obtaining erasure chunks failed")]
+	ObtainErasureChunks(#[from] pezkuwi_erasure_coding::Error),
+
+	#[error(transparent)]
+	ValidationFailed(#[from] ValidationFailed),
+
+	#[error(transparent)]
+	UtilError(#[from] UtilError),
+
+	#[error(transparent)]
+	SubsystemError(#[from] SubsystemError),
+
+	#[fatal]
+	#[error(transparent)]
+	OverseerExited(SubsystemError),
+
+	#[error("Availability store error")]
+	StoreAvailableData(#[source] StoreAvailableDataError),
+
+	#[error("Runtime API returned None for executor params")]
+	MissingExecutorParams,
+}
+
+/// Utility for eating top level errors and log them.
+///
+/// We basically always want to try and continue on error. This utility function is meant to
+/// consume top-level errors by simply logging them
+pub fn log_error(result: Result<()>) -> std::result::Result<(), FatalError> {
+	match result.into_nested()? {
+		Ok(()) => Ok(()),
+		Err(jfyi) => {
+			jfyi.log();
+			Ok(())
+		},
+	}
+}
+
+impl JfyiError {
+	/// Log a `JfyiError`.
+	pub fn log(self) {
+		gum::debug!(target: LOG_TARGET, error = ?self);
+	}
+}

pezkuwi/node/core/backing/src/metrics.rs

@@ -0,0 +1,107 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use pezkuwi_node_subsystem_util::metrics::{self, prometheus};
+
+#[derive(Clone)]
+pub(crate) struct MetricsInner {
+	pub(crate) signed_statements_total: prometheus::Counter<prometheus::U64>,
+	pub(crate) candidates_seconded_total: prometheus::Counter<prometheus::U64>,
+	pub(crate) process_second: prometheus::Histogram,
+	pub(crate) process_statement: prometheus::Histogram,
+	pub(crate) get_backed_candidates: prometheus::Histogram,
+}
+
+/// Candidate backing metrics.
+#[derive(Default, Clone)]
+pub struct Metrics(pub(crate) Option<MetricsInner>);
+
+impl Metrics {
+	pub fn on_statement_signed(&self) {
+		if let Some(metrics) = &self.0 {
+			metrics.signed_statements_total.inc();
+		}
+	}
+
+	pub fn on_candidate_seconded(&self) {
+		if let Some(metrics) = &self.0 {
+			metrics.candidates_seconded_total.inc();
+		}
+	}
+
+	/// Provide a timer for handling `CandidateBackingMessage:Second` which observes on drop.
+	pub fn time_process_second(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.process_second.start_timer())
+	}
+
+	/// Provide a timer for handling `CandidateBackingMessage::Statement` which observes on drop.
+	pub fn time_process_statement(
+		&self,
+	) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.process_statement.start_timer())
+	}
+
+	/// Provide a timer for handling `CandidateBackingMessage::GetBackedCandidates` which observes
+	/// on drop.
+	pub fn time_get_backed_candidates(
+		&self,
+	) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.get_backed_candidates.start_timer())
+	}
+}
+
+impl metrics::Metrics for Metrics {
+	fn try_register(registry: &prometheus::Registry) -> Result<Self, prometheus::PrometheusError> {
+		let metrics = MetricsInner {
+			signed_statements_total: prometheus::register(
+				prometheus::Counter::new(
+					"pezkuwi_teyrchain_candidate_backing_signed_statements_total",
+					"Number of statements signed.",
+				)?,
+				registry,
+			)?,
+			candidates_seconded_total: prometheus::register(
+				prometheus::Counter::new(
+					"pezkuwi_teyrchain_candidate_backing_candidates_seconded_total",
+					"Number of candidates seconded.",
+				)?,
+				registry,
+			)?,
+			process_second: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_candidate_backing_process_second",
+					"Time spent within `candidate_backing::process_second`",
+				))?,
+				registry,
+			)?,
+			process_statement: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_candidate_backing_process_statement",
+					"Time spent within `candidate_backing::process_statement`",
+				))?,
+				registry,
+			)?,
+			get_backed_candidates: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_candidate_backing_get_backed_candidates",
+					"Time spent within `candidate_backing::get_backed_candidates`",
+				))?,
+				registry,
+			)?,
+		};
+		Ok(Metrics(Some(metrics)))
+	}
+}

pezkuwi/node/core/bitfield-signing/Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+name = "pezkuwi-node-core-bitfield-signing"
+version = "7.0.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+description = "Bitfield signing subsystem for the Pezkuwi node"
+homepage.workspace = true
+repository.workspace = true
+
+[lints]
+workspace = true
+
+[dependencies]
+futures = { workspace = true }
+gum = { workspace = true, default-features = true }
+pezkuwi-node-subsystem = { workspace = true, default-features = true }
+pezkuwi-node-subsystem-util = { workspace = true, default-features = true }
+pezkuwi-primitives = { workspace = true, default-features = true }
+sp-keystore = { workspace = true, default-features = true }
+thiserror = { workspace = true }
+wasm-timer = { workspace = true }
+
+[dev-dependencies]
+pezkuwi-node-subsystem-test-helpers = { workspace = true }
+pezkuwi-primitives-test-helpers = { workspace = true }
+
+[features]
+runtime-benchmarks = [
+	"gum/runtime-benchmarks",
+	"pezkuwi-node-subsystem-test-helpers/runtime-benchmarks",
+	"pezkuwi-node-subsystem-util/runtime-benchmarks",
+	"pezkuwi-node-subsystem/runtime-benchmarks",
+	"pezkuwi-primitives-test-helpers/runtime-benchmarks",
+	"pezkuwi-primitives/runtime-benchmarks",
+]

pezkuwi/node/core/bitfield-signing/src/lib.rs

@@ -0,0 +1,277 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! The bitfield signing subsystem produces `SignedAvailabilityBitfield`s once per block.
+
+#![deny(unused_crate_dependencies)]
+#![warn(missing_docs)]
+#![recursion_limit = "256"]
+
+use futures::{
+	channel::{mpsc, oneshot},
+	future,
+	lock::Mutex,
+	FutureExt,
+};
+use pezkuwi_node_subsystem::{
+	messages::{AvailabilityStoreMessage, BitfieldDistributionMessage},
+	overseer, ActivatedLeaf, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError,
+	SubsystemResult,
+};
+use pezkuwi_node_subsystem_util::{
+	self as util, request_availability_cores, runtime::recv_runtime, Validator,
+};
+use pezkuwi_primitives::{AvailabilityBitfield, CoreState, Hash, ValidatorIndex};
+use sp_keystore::{Error as KeystoreError, KeystorePtr};
+use std::{collections::HashMap, time::Duration};
+use wasm_timer::{Delay, Instant};
+
+mod metrics;
+use self::metrics::Metrics;
+
+#[cfg(test)]
+mod tests;
+
+/// Delay between starting a bitfield signing job and its attempting to create a bitfield.
+const SPAWNED_TASK_DELAY: Duration = Duration::from_millis(1500);
+const LOG_TARGET: &str = "teyrchain::bitfield-signing";
+
+// TODO: use `fatality` (https://github.com/paritytech/polkadot/issues/5540).
+/// Errors we may encounter in the course of executing the `BitfieldSigningSubsystem`.
+#[derive(Debug, thiserror::Error)]
+#[allow(missing_docs)]
+pub enum Error {
+	#[error(transparent)]
+	Util(#[from] util::Error),
+
+	#[error(transparent)]
+	Io(#[from] std::io::Error),
+
+	#[error(transparent)]
+	Oneshot(#[from] oneshot::Canceled),
+
+	#[error(transparent)]
+	MpscSend(#[from] mpsc::SendError),
+
+	#[error(transparent)]
+	Runtime(#[from] util::runtime::Error),
+
+	#[error("Keystore failed: {0:?}")]
+	Keystore(KeystoreError),
+}
+
+/// If there is a candidate pending availability, query the Availability Store
+/// for whether we have the availability chunk for our validator index.
+async fn get_core_availability(
+	core: &CoreState,
+	validator_index: ValidatorIndex,
+	sender: &Mutex<&mut impl overseer::BitfieldSigningSenderTrait>,
+) -> Result<bool, Error> {
+	if let CoreState::Occupied(core) = core {
+		let (tx, rx) = oneshot::channel();
+		sender
+			.lock()
+			.await
+			.send_message(AvailabilityStoreMessage::QueryChunkAvailability(
+				core.candidate_hash,
+				validator_index,
+				tx,
+			))
+			.await;
+
+		let res = rx.await.map_err(Into::into);
+
+		gum::trace!(
+			target: LOG_TARGET,
+			para_id = %core.para_id(),
+			availability = ?res,
+			?core.candidate_hash,
+			"Candidate availability",
+		);
+
+		res
+	} else {
+		Ok(false)
+	}
+}
+
+/// - get the list of core states from the runtime
+/// - for each core, concurrently determine chunk availability (see `get_core_availability`)
+/// - return the bitfield if there were no errors at any point in this process (otherwise, it's
+///   prone to false negatives)
+async fn construct_availability_bitfield(
+	relay_parent: Hash,
+	validator_idx: ValidatorIndex,
+	sender: &mut impl overseer::BitfieldSigningSenderTrait,
+) -> Result<AvailabilityBitfield, Error> {
+	// get the set of availability cores from the runtime
+	let availability_cores =
+		{ recv_runtime(request_availability_cores(relay_parent, sender).await).await? };
+
+	// Wrap the sender in a Mutex to share it between the futures.
+	//
+	// We use a `Mutex` here to not `clone` the sender inside the future, because
+	// cloning the sender will always increase the capacity of the channel by one.
+	// (for the lifetime of the sender)
+	let sender = Mutex::new(sender);
+
+	// Handle all cores concurrently
+	// `try_join_all` returns all results in the same order as the input futures.
+	let results = future::try_join_all(
+		availability_cores
+			.iter()
+			.map(|core| get_core_availability(core, validator_idx, &sender)),
+	)
+	.await?;
+
+	let core_bits = FromIterator::from_iter(results.into_iter());
+	gum::debug!(
+		target: LOG_TARGET,
+		?relay_parent,
+		"Signing Bitfield for {core_count} cores: {core_bits}",
+		core_count = availability_cores.len(),
+		core_bits = core_bits,
+	);
+
+	Ok(AvailabilityBitfield(core_bits))
+}
+
+/// The bitfield signing subsystem.
+pub struct BitfieldSigningSubsystem {
+	keystore: KeystorePtr,
+	metrics: Metrics,
+}
+
+impl BitfieldSigningSubsystem {
+	/// Create a new instance of the `BitfieldSigningSubsystem`.
+	pub fn new(keystore: KeystorePtr, metrics: Metrics) -> Self {
+		Self { keystore, metrics }
+	}
+}
+
+#[overseer::subsystem(BitfieldSigning, error=SubsystemError, prefix=self::overseer)]
+impl<Context> BitfieldSigningSubsystem {
+	fn start(self, ctx: Context) -> SpawnedSubsystem {
+		let future = async move {
+			run(ctx, self.keystore, self.metrics)
+				.await
+				.map_err(|e| SubsystemError::with_origin("bitfield-signing", e))
+		}
+		.boxed();
+
+		SpawnedSubsystem { name: "bitfield-signing-subsystem", future }
+	}
+}
+
+#[overseer::contextbounds(BitfieldSigning, prefix = self::overseer)]
+async fn run<Context>(
+	mut ctx: Context,
+	keystore: KeystorePtr,
+	metrics: Metrics,
+) -> SubsystemResult<()> {
+	// Track spawned jobs per active leaf.
+	let mut running = HashMap::<Hash, future::AbortHandle>::new();
+
+	loop {
+		match ctx.recv().await? {
+			FromOrchestra::Signal(OverseerSignal::ActiveLeaves(update)) => {
+				// Abort jobs for deactivated leaves.
+				for leaf in &update.deactivated {
+					if let Some(handle) = running.remove(leaf) {
+						handle.abort();
+					}
+				}
+
+				if let Some(leaf) = update.activated {
+					let sender = ctx.sender().clone();
+					let leaf_hash = leaf.hash;
+
+					let (fut, handle) = future::abortable(handle_active_leaves_update(
+						sender,
+						leaf,
+						keystore.clone(),
+						metrics.clone(),
+					));
+
+					running.insert(leaf_hash, handle);
+
+					ctx.spawn("bitfield-signing-job", fut.map(drop).boxed())?;
+				}
+			},
+			FromOrchestra::Signal(OverseerSignal::BlockFinalized(..)) => {},
+			FromOrchestra::Signal(OverseerSignal::Conclude) => return Ok(()),
+			FromOrchestra::Communication { .. } => {},
+		}
+	}
+}
+
+async fn handle_active_leaves_update<Sender>(
+	mut sender: Sender,
+	leaf: ActivatedLeaf,
+	keystore: KeystorePtr,
+	metrics: Metrics,
+) -> Result<(), Error>
+where
+	Sender: overseer::BitfieldSigningSenderTrait,
+{
+	let wait_until = Instant::now() + SPAWNED_TASK_DELAY;
+
+	// now do all the work we can before we need to wait for the availability store
+	// if we're not a validator, we can just succeed effortlessly
+	let validator = match Validator::new(leaf.hash, keystore.clone(), &mut sender).await {
+		Ok(validator) => validator,
+		Err(util::Error::NotAValidator) => return Ok(()),
+		Err(err) => return Err(Error::Util(err)),
+	};
+
+	// wait a bit before doing anything else
+	Delay::new_at(wait_until).await?;
+
+	// this timer does not appear at the head of the function because we don't want to include
+	// SPAWNED_TASK_DELAY each time.
+	let _timer = metrics.time_run();
+
+	let bitfield =
+		match construct_availability_bitfield(leaf.hash, validator.index(), &mut sender).await {
+			Err(Error::Runtime(runtime_err)) => {
+				// Don't take down the node on runtime API errors.
+				gum::warn!(target: LOG_TARGET, err = ?runtime_err, "Encountered a runtime API error");
+				return Ok(());
+			},
+			Err(err) => return Err(err),
+			Ok(bitfield) => bitfield,
+		};
+
+	let signed_bitfield =
+		match validator.sign(keystore, bitfield).map_err(|e| Error::Keystore(e))? {
+			Some(b) => b,
+			None => {
+				gum::error!(
+					target: LOG_TARGET,
+					"Key was found at construction, but while signing it could not be found.",
+				);
+				return Ok(());
+			},
+		};
+
+	metrics.on_bitfield_signed();
+
+	sender
+		.send_message(BitfieldDistributionMessage::DistributeBitfield(leaf.hash, signed_bitfield))
+		.await;
+
+	Ok(())
+}

pezkuwi/node/core/bitfield-signing/src/metrics.rs

@@ -0,0 +1,68 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use pezkuwi_node_subsystem_util::metrics::{self, prometheus};
+
+#[derive(Clone)]
+pub(crate) struct MetricsInner {
+	pub(crate) bitfields_signed_total: prometheus::Counter<prometheus::U64>,
+	pub(crate) run: prometheus::Histogram,
+}
+
+/// Bitfield signing metrics.
+#[derive(Default, Clone)]
+pub struct Metrics(pub(crate) Option<MetricsInner>);
+
+impl Metrics {
+	pub fn on_bitfield_signed(&self) {
+		if let Some(metrics) = &self.0 {
+			metrics.bitfields_signed_total.inc();
+		}
+	}
+
+	/// Provide a timer for `prune_povs` which observes on drop.
+	pub fn time_run(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.run.start_timer())
+	}
+}
+
+impl metrics::Metrics for Metrics {
+	fn try_register(registry: &prometheus::Registry) -> Result<Self, prometheus::PrometheusError> {
+		let metrics = MetricsInner {
+			bitfields_signed_total: prometheus::register(
+				prometheus::Counter::new(
+					"pezkuwi_teyrchain_bitfields_signed_total",
+					"Number of bitfields signed.",
+				)?,
+				registry,
+			)?,
+			run: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"pezkuwi_teyrchain_bitfield_signing_run",
+						"Time spent within `bitfield_signing::run`",
+					)
+					.buckets(vec![
+						0.000625, 0.00125, 0.0025, 0.005, 0.0075, 0.01, 0.025, 0.05, 0.1, 0.25,
+						0.5, 1.0, 2.5, 5.0, 10.0,
+					]),
+				)?,
+				registry,
+			)?,
+		};
+		Ok(Metrics(Some(metrics)))
+	}
+}

pezkuwi/node/core/bitfield-signing/src/tests.rs

@@ -0,0 +1,80 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use super::*;
+use futures::{executor::block_on, pin_mut, StreamExt};
+use pezkuwi_node_subsystem::messages::{AllMessages, RuntimeApiMessage, RuntimeApiRequest};
+use pezkuwi_primitives::{CandidateHash, OccupiedCore};
+use pezkuwi_primitives_test_helpers::dummy_candidate_descriptor_v2;
+
+fn occupied_core(para_id: u32, candidate_hash: CandidateHash) -> CoreState {
+	CoreState::Occupied(OccupiedCore {
+		group_responsible: para_id.into(),
+		next_up_on_available: None,
+		occupied_since: 100_u32,
+		time_out_at: 200_u32,
+		next_up_on_time_out: None,
+		availability: Default::default(),
+		candidate_hash,
+		candidate_descriptor: dummy_candidate_descriptor_v2(Hash::zero()),
+	})
+}
+
+#[test]
+fn construct_availability_bitfield_works() {
+	block_on(async move {
+		let relay_parent = Hash::default();
+		let validator_index = ValidatorIndex(1u32);
+
+		let (mut sender, mut receiver) = pezkuwi_node_subsystem_test_helpers::sender_receiver();
+		let future =
+			construct_availability_bitfield(relay_parent, validator_index, &mut sender).fuse();
+		pin_mut!(future);
+
+		let hash_a = CandidateHash(Hash::repeat_byte(1));
+		let hash_b = CandidateHash(Hash::repeat_byte(2));
+
+		loop {
+			futures::select! {
+				m = receiver.next() => match m.unwrap() {
+					AllMessages::RuntimeApi(
+						RuntimeApiMessage::Request(rp, RuntimeApiRequest::AvailabilityCores(tx)),
+					) => {
+						assert_eq!(relay_parent, rp);
+						tx.send(Ok(vec![CoreState::Free, occupied_core(1, hash_a), occupied_core(2, hash_b)])).unwrap();
+					}
+					AllMessages::AvailabilityStore(
+						AvailabilityStoreMessage::QueryChunkAvailability(c_hash, vidx, tx),
+					) => {
+						assert_eq!(validator_index, vidx.into());
+
+						tx.send(c_hash == hash_a).unwrap();
+					},
+					o => panic!("Unknown message: {:?}", o),
+				},
+				r = future => match r {
+					Ok(r) => {
+						assert!(!r.0.get(0).unwrap());
+						assert!(r.0.get(1).unwrap());
+						assert!(!r.0.get(2).unwrap());
+						break
+					},
+					Err(e) => panic!("Failed: {:?}", e),
+				},
+			}
+		}
+	});
+}

pezkuwi/node/core/candidate-validation/Cargo.toml

@@ -0,0 +1,60 @@
+[package]
+name = "pezkuwi-node-core-candidate-validation"
+description = "Pezkuwi crate that implements the Candidate Validation subsystem. Handles requests to validate candidates according to a PVF."
+version = "7.0.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+
+[lints]
+workspace = true
+
+[dependencies]
+async-trait = { workspace = true }
+futures = { workspace = true }
+futures-timer = { workspace = true }
+gum = { workspace = true, default-features = true }
+
+codec = { features = ["bit-vec", "derive"], workspace = true }
+sp-application-crypto = { workspace = true }
+sp-keystore = { workspace = true }
+
+pezkuwi-node-metrics = { workspace = true, default-features = true }
+pezkuwi-node-primitives = { workspace = true, default-features = true }
+pezkuwi-node-subsystem = { workspace = true, default-features = true }
+pezkuwi-node-subsystem-util = { workspace = true, default-features = true }
+pezkuwi-overseer = { workspace = true, default-features = true }
+pezkuwi-primitives = { workspace = true, default-features = true }
+pezkuwi-teyrchain-primitives = { workspace = true, default-features = true }
+
+[target.'cfg(not(any(target_os = "android", target_os = "unknown")))'.dependencies]
+pezkuwi-node-core-pvf = { workspace = true, default-features = true }
+
+[dev-dependencies]
+assert_matches = { workspace = true }
+futures = { features = ["thread-pool"], workspace = true }
+pezkuwi-node-subsystem-test-helpers = { workspace = true }
+pezkuwi-primitives = { workspace = true, features = ["test"] }
+pezkuwi-primitives-test-helpers = { workspace = true }
+rstest = { workspace = true }
+sp-core = { workspace = true, default-features = true }
+sp-keyring = { workspace = true, default-features = true }
+sp-maybe-compressed-blob = { workspace = true, default-features = true }
+
+[features]
+runtime-benchmarks = [
+	"gum/runtime-benchmarks",
+	"pezkuwi-node-core-pvf/runtime-benchmarks",
+	"pezkuwi-node-metrics/runtime-benchmarks",
+	"pezkuwi-node-primitives/runtime-benchmarks",
+	"pezkuwi-node-subsystem-test-helpers/runtime-benchmarks",
+	"pezkuwi-node-subsystem-util/runtime-benchmarks",
+	"pezkuwi-node-subsystem/runtime-benchmarks",
+	"pezkuwi-overseer/runtime-benchmarks",
+	"pezkuwi-primitives-test-helpers/runtime-benchmarks",
+	"pezkuwi-primitives/runtime-benchmarks",
+	"pezkuwi-teyrchain-primitives/runtime-benchmarks",
+	"sp-keyring/runtime-benchmarks",
+]

pezkuwi/node/core/candidate-validation/src/metrics.rs

@@ -0,0 +1,95 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use super::{ValidationFailed, ValidationResult};
+use pezkuwi_node_metrics::metrics::{self, prometheus};
+
+#[derive(Clone)]
+pub(crate) struct MetricsInner {
+	pub(crate) validation_requests: prometheus::CounterVec<prometheus::U64>,
+	pub(crate) validate_from_exhaustive: prometheus::Histogram,
+	pub(crate) validate_candidate_exhaustive: prometheus::Histogram,
+}
+
+/// Candidate validation metrics.
+#[derive(Default, Clone)]
+pub struct Metrics(Option<MetricsInner>);
+
+impl Metrics {
+	pub fn on_validation_event(&self, event: &Result<ValidationResult, ValidationFailed>) {
+		if let Some(metrics) = &self.0 {
+			match event {
+				Ok(ValidationResult::Valid(_, _)) => {
+					metrics.validation_requests.with_label_values(&["valid"]).inc();
+				},
+				Ok(ValidationResult::Invalid(_)) => {
+					metrics.validation_requests.with_label_values(&["invalid"]).inc();
+				},
+				Err(_) => {
+					metrics.validation_requests.with_label_values(&["validation failure"]).inc();
+				},
+			}
+		}
+	}
+
+	/// Provide a timer for `validate_from_exhaustive` which observes on drop.
+	pub fn time_validate_from_exhaustive(
+		&self,
+	) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.validate_from_exhaustive.start_timer())
+	}
+
+	/// Provide a timer for `validate_candidate_exhaustive` which observes on drop.
+	pub fn time_validate_candidate_exhaustive(
+		&self,
+	) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0
+			.as_ref()
+			.map(|metrics| metrics.validate_candidate_exhaustive.start_timer())
+	}
+}
+
+impl metrics::Metrics for Metrics {
+	fn try_register(registry: &prometheus::Registry) -> Result<Self, prometheus::PrometheusError> {
+		let metrics = MetricsInner {
+			validation_requests: prometheus::register(
+				prometheus::CounterVec::new(
+					prometheus::Opts::new(
+						"pezkuwi_teyrchain_validation_requests_total",
+						"Number of validation requests served.",
+					),
+					&["validity"],
+				)?,
+				registry,
+			)?,
+			validate_from_exhaustive: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_candidate_validation_validate_from_exhaustive",
+					"Time spent within `candidate_validation::validate_from_exhaustive`",
+				))?,
+				registry,
+			)?,
+			validate_candidate_exhaustive: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_candidate_validation_validate_candidate_exhaustive",
+					"Time spent within `candidate_validation::validate_candidate_exhaustive`",
+				))?,
+				registry,
+			)?,
+		};
+		Ok(Metrics(Some(metrics)))
+	}
+}

pezkuwi/node/core/chain-api/Cargo.toml

@@ -0,0 +1,45 @@
+[package]
+name = "pezkuwi-node-core-chain-api"
+version = "7.0.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+description = "The Chain API subsystem provides access to chain related utility functions like block number to hash conversions."
+homepage.workspace = true
+repository.workspace = true
+
+[lints]
+workspace = true
+
+[dependencies]
+futures = { workspace = true }
+gum = { workspace = true, default-features = true }
+pezkuwi-node-metrics = { workspace = true, default-features = true }
+pezkuwi-node-subsystem = { workspace = true, default-features = true }
+pezkuwi-node-subsystem-types = { workspace = true, default-features = true }
+sc-client-api = { workspace = true, default-features = true }
+sc-consensus-babe = { workspace = true, default-features = true }
+
+[dev-dependencies]
+codec = { workspace = true, default-features = true }
+futures = { features = ["thread-pool"], workspace = true }
+maplit = { workspace = true }
+pezkuwi-node-primitives = { workspace = true, default-features = true }
+pezkuwi-node-subsystem-test-helpers = { workspace = true }
+pezkuwi-primitives = { workspace = true, default-features = true }
+sp-blockchain = { workspace = true, default-features = true }
+sp-core = { workspace = true, default-features = true }
+
+[features]
+runtime-benchmarks = [
+	"gum/runtime-benchmarks",
+	"pezkuwi-node-metrics/runtime-benchmarks",
+	"pezkuwi-node-primitives/runtime-benchmarks",
+	"pezkuwi-node-subsystem-test-helpers/runtime-benchmarks",
+	"pezkuwi-node-subsystem-types/runtime-benchmarks",
+	"pezkuwi-node-subsystem/runtime-benchmarks",
+	"pezkuwi-primitives/runtime-benchmarks",
+	"sc-client-api/runtime-benchmarks",
+	"sc-consensus-babe/runtime-benchmarks",
+	"sp-blockchain/runtime-benchmarks",
+]

pezkuwi/node/core/chain-api/src/lib.rs

@@ -0,0 +1,169 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Implements the Chain API Subsystem
+//!
+//! Provides access to the chain data. Every request may return an error.
+//! At the moment, the implementation requires `Client` to implement `HeaderBackend`,
+//! we may add more bounds in the future if we will need e.g. block bodies.
+//!
+//! Supported requests:
+//! * Block hash to number
+//! * Block hash to header
+//! * Block weight (cumulative)
+//! * Finalized block number to hash
+//! * Last finalized block number
+//! * Ancestors
+
+#![deny(unused_crate_dependencies, unused_results)]
+#![warn(missing_docs)]
+
+use std::sync::Arc;
+
+use futures::prelude::*;
+use sc_client_api::AuxStore;
+
+use futures::stream::StreamExt;
+use pezkuwi_node_subsystem::{
+	messages::ChainApiMessage, overseer, FromOrchestra, OverseerSignal, SpawnedSubsystem,
+	SubsystemError, SubsystemResult,
+};
+use pezkuwi_node_subsystem_types::ChainApiBackend;
+
+mod metrics;
+use self::metrics::Metrics;
+
+#[cfg(test)]
+mod tests;
+
+const LOG_TARGET: &str = "teyrchain::chain-api";
+
+/// The Chain API Subsystem implementation.
+pub struct ChainApiSubsystem<Client> {
+	client: Arc<Client>,
+	metrics: Metrics,
+}
+
+impl<Client> ChainApiSubsystem<Client> {
+	/// Create a new Chain API subsystem with the given client.
+	pub fn new(client: Arc<Client>, metrics: Metrics) -> Self {
+		ChainApiSubsystem { client, metrics }
+	}
+}
+
+#[overseer::subsystem(ChainApi, error = SubsystemError, prefix = self::overseer)]
+impl<Client, Context> ChainApiSubsystem<Client>
+where
+	Client: ChainApiBackend + AuxStore + 'static,
+{
+	fn start(self, ctx: Context) -> SpawnedSubsystem {
+		let future = run::<Client, Context>(ctx, self)
+			.map_err(|e| SubsystemError::with_origin("chain-api", e))
+			.boxed();
+		SpawnedSubsystem { future, name: "chain-api-subsystem" }
+	}
+}
+
+#[overseer::contextbounds(ChainApi, prefix = self::overseer)]
+async fn run<Client, Context>(
+	mut ctx: Context,
+	subsystem: ChainApiSubsystem<Client>,
+) -> SubsystemResult<()>
+where
+	Client: ChainApiBackend + AuxStore,
+{
+	loop {
+		match ctx.recv().await? {
+			FromOrchestra::Signal(OverseerSignal::Conclude) => return Ok(()),
+			FromOrchestra::Signal(OverseerSignal::ActiveLeaves(_)) => {},
+			FromOrchestra::Signal(OverseerSignal::BlockFinalized(..)) => {},
+			FromOrchestra::Communication { msg } => match msg {
+				ChainApiMessage::BlockNumber(hash, response_channel) => {
+					let _timer = subsystem.metrics.time_block_number();
+					let result =
+						subsystem.client.number(hash).await.map_err(|e| e.to_string().into());
+					subsystem.metrics.on_request(result.is_ok());
+					let _ = response_channel.send(result);
+				},
+				ChainApiMessage::BlockHeader(hash, response_channel) => {
+					let _timer = subsystem.metrics.time_block_header();
+					let result =
+						subsystem.client.header(hash).await.map_err(|e| e.to_string().into());
+					subsystem.metrics.on_request(result.is_ok());
+					let _ = response_channel.send(result);
+				},
+				ChainApiMessage::BlockWeight(hash, response_channel) => {
+					let _timer = subsystem.metrics.time_block_weight();
+					let result = sc_consensus_babe::block_weight(&*subsystem.client, hash)
+						.map_err(|e| e.to_string().into());
+					subsystem.metrics.on_request(result.is_ok());
+					let _ = response_channel.send(result);
+				},
+				ChainApiMessage::FinalizedBlockHash(number, response_channel) => {
+					let _timer = subsystem.metrics.time_finalized_block_hash();
+					// Note: we don't verify it's finalized
+					let result =
+						subsystem.client.hash(number).await.map_err(|e| e.to_string().into());
+					subsystem.metrics.on_request(result.is_ok());
+					let _ = response_channel.send(result);
+				},
+				ChainApiMessage::FinalizedBlockNumber(response_channel) => {
+					let _timer = subsystem.metrics.time_finalized_block_number();
+					let result = subsystem
+						.client
+						.info()
+						.await
+						.map_err(|e| e.to_string().into())
+						.map(|info| info.finalized_number);
+					subsystem.metrics.on_request(result.is_ok());
+					let _ = response_channel.send(result);
+				},
+				ChainApiMessage::Ancestors { hash, k, response_channel } => {
+					let _timer = subsystem.metrics.time_ancestors();
+					gum::trace!(target: LOG_TARGET, hash=%hash, k=k, "ChainApiMessage::Ancestors");
+
+					let next_parent_stream = futures::stream::unfold(
+						(hash, subsystem.client.clone()),
+						|(hash, client)| async move {
+							let maybe_header = client.header(hash).await;
+							match maybe_header {
+								// propagate the error
+								Err(e) => {
+									let e = e.to_string().into();
+									Some((Err(e), (hash, client)))
+								},
+								// fewer than `k` ancestors are available
+								Ok(None) => None,
+								Ok(Some(header)) => {
+									// stop at the genesis header.
+									if header.number == 0 {
+										None
+									} else {
+										Some((Ok(header.parent_hash), (header.parent_hash, client)))
+									}
+								},
+							}
+						},
+					);
+
+					let result = next_parent_stream.take(k).try_collect().await;
+					subsystem.metrics.on_request(result.is_ok());
+					let _ = response_channel.send(result);
+				},
+			},
+		}
+	}
+}

pezkuwi/node/core/chain-api/src/metrics.rs

@@ -0,0 +1,138 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use pezkuwi_node_metrics::metrics::{self, prometheus};
+
+#[derive(Clone)]
+pub(crate) struct MetricsInner {
+	pub(crate) chain_api_requests: prometheus::CounterVec<prometheus::U64>,
+	pub(crate) block_number: prometheus::Histogram,
+	pub(crate) block_header: prometheus::Histogram,
+	pub(crate) block_weight: prometheus::Histogram,
+	pub(crate) finalized_block_hash: prometheus::Histogram,
+	pub(crate) finalized_block_number: prometheus::Histogram,
+	pub(crate) ancestors: prometheus::Histogram,
+}
+
+/// Chain API metrics.
+#[derive(Default, Clone)]
+pub struct Metrics(pub(crate) Option<MetricsInner>);
+
+impl Metrics {
+	pub fn on_request(&self, succeeded: bool) {
+		if let Some(metrics) = &self.0 {
+			if succeeded {
+				metrics.chain_api_requests.with_label_values(&["succeeded"]).inc();
+			} else {
+				metrics.chain_api_requests.with_label_values(&["failed"]).inc();
+			}
+		}
+	}
+
+	/// Provide a timer for `block_number` which observes on drop.
+	pub fn time_block_number(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.block_number.start_timer())
+	}
+
+	/// Provide a timer for `block_header` which observes on drop.
+	pub fn time_block_header(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.block_header.start_timer())
+	}
+
+	/// Provide a timer for `block_weight` which observes on drop.
+	pub fn time_block_weight(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.block_weight.start_timer())
+	}
+
+	/// Provide a timer for `finalized_block_hash` which observes on drop.
+	pub fn time_finalized_block_hash(
+		&self,
+	) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.finalized_block_hash.start_timer())
+	}
+
+	/// Provide a timer for `finalized_block_number` which observes on drop.
+	pub fn time_finalized_block_number(
+		&self,
+	) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.finalized_block_number.start_timer())
+	}
+
+	/// Provide a timer for `ancestors` which observes on drop.
+	pub fn time_ancestors(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.ancestors.start_timer())
+	}
+}
+
+impl metrics::Metrics for Metrics {
+	fn try_register(registry: &prometheus::Registry) -> Result<Self, prometheus::PrometheusError> {
+		let metrics = MetricsInner {
+			chain_api_requests: prometheus::register(
+				prometheus::CounterVec::new(
+					prometheus::Opts::new(
+						"pezkuwi_teyrchain_chain_api_requests_total",
+						"Number of Chain API requests served.",
+					),
+					&["success"],
+				)?,
+				registry,
+			)?,
+			block_number: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_chain_api_block_number",
+					"Time spent within `chain_api::block_number`",
+				))?,
+				registry,
+			)?,
+			block_header: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_chain_api_block_headers",
+					"Time spent within `chain_api::block_headers`",
+				))?,
+				registry,
+			)?,
+			block_weight: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_chain_api_block_weight",
+					"Time spent within `chain_api::block_weight`",
+				))?,
+				registry,
+			)?,
+			finalized_block_hash: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_chain_api_finalized_block_hash",
+					"Time spent within `chain_api::finalized_block_hash`",
+				))?,
+				registry,
+			)?,
+			finalized_block_number: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_chain_api_finalized_block_number",
+					"Time spent within `chain_api::finalized_block_number`",
+				))?,
+				registry,
+			)?,
+			ancestors: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_chain_api_ancestors",
+					"Time spent within `chain_api::ancestors`",
+				))?,
+				registry,
+			)?,
+		};
+		Ok(Metrics(Some(metrics)))
+	}
+}

pezkuwi/node/core/chain-api/src/tests.rs

@@ -0,0 +1,373 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use super::*;
+
+use codec::Encode;
+use futures::{channel::oneshot, future::BoxFuture};
+use std::collections::BTreeMap;
+
+use pezkuwi_node_primitives::BlockWeight;
+use pezkuwi_node_subsystem_test_helpers::{make_subsystem_context, TestSubsystemContextHandle};
+use pezkuwi_node_subsystem_types::ChainApiBackend;
+use pezkuwi_primitives::{Block, BlockNumber, Hash, Header};
+use sp_blockchain::Info as BlockInfo;
+use sp_core::testing::TaskExecutor;
+
+#[derive(Clone)]
+struct TestClient {
+	blocks: BTreeMap<Hash, BlockNumber>,
+	block_weights: BTreeMap<Hash, BlockWeight>,
+	finalized_blocks: BTreeMap<BlockNumber, Hash>,
+	headers: BTreeMap<Hash, Header>,
+}
+
+const GENESIS: Hash = Hash::repeat_byte(0xAA);
+const ONE: Hash = Hash::repeat_byte(0x01);
+const TWO: Hash = Hash::repeat_byte(0x02);
+const THREE: Hash = Hash::repeat_byte(0x03);
+const FOUR: Hash = Hash::repeat_byte(0x04);
+const ERROR_PATH: Hash = Hash::repeat_byte(0xFF);
+
+fn default_header() -> Header {
+	Header {
+		parent_hash: Hash::zero(),
+		number: 100500,
+		state_root: Hash::zero(),
+		extrinsics_root: Hash::zero(),
+		digest: Default::default(),
+	}
+}
+
+impl Default for TestClient {
+	fn default() -> Self {
+		Self {
+			blocks: maplit::btreemap! {
+				GENESIS => 0,
+				ONE => 1,
+				TWO => 2,
+				THREE => 3,
+				FOUR => 4,
+			},
+			block_weights: maplit::btreemap! {
+				ONE => 0,
+				TWO => 1,
+				THREE => 1,
+				FOUR => 2,
+			},
+			finalized_blocks: maplit::btreemap! {
+				1 => ONE,
+				3 => THREE,
+			},
+			headers: maplit::btreemap! {
+				GENESIS => Header {
+					parent_hash: Hash::zero(), // Dummy parent with zero hash.
+					number: 0,
+					..default_header()
+				},
+				ONE => Header {
+					parent_hash: GENESIS,
+					number: 1,
+					..default_header()
+				},
+				TWO => Header {
+					parent_hash: ONE,
+					number: 2,
+					..default_header()
+				},
+				THREE => Header {
+					parent_hash: TWO,
+					number: 3,
+					..default_header()
+				},
+				FOUR => Header {
+					parent_hash: THREE,
+					number: 4,
+					..default_header()
+				},
+				ERROR_PATH => Header {
+					..default_header()
+				}
+			},
+		}
+	}
+}
+
+fn last_key_value<K: Clone, V: Clone>(map: &BTreeMap<K, V>) -> (K, V) {
+	assert!(!map.is_empty());
+	map.iter().last().map(|(k, v)| (k.clone(), v.clone())).unwrap()
+}
+
+impl sp_blockchain::HeaderBackend<Block> for TestClient {
+	fn info(&self) -> BlockInfo<Block> {
+		let genesis_hash = self.blocks.iter().next().map(|(h, _)| *h).unwrap();
+		let (best_hash, best_number) = last_key_value(&self.blocks);
+		let (finalized_number, finalized_hash) = last_key_value(&self.finalized_blocks);
+
+		BlockInfo {
+			best_hash,
+			best_number,
+			genesis_hash,
+			finalized_hash,
+			finalized_number,
+			number_leaves: 0,
+			finalized_state: None,
+			block_gap: None,
+		}
+	}
+	fn number(&self, hash: Hash) -> sp_blockchain::Result<Option<BlockNumber>> {
+		Ok(self.blocks.get(&hash).copied())
+	}
+	fn hash(&self, number: BlockNumber) -> sp_blockchain::Result<Option<Hash>> {
+		Ok(self.finalized_blocks.get(&number).copied())
+	}
+	fn header(&self, hash: Hash) -> sp_blockchain::Result<Option<Header>> {
+		if hash.is_zero() {
+			Err(sp_blockchain::Error::Backend("Zero hashes are illegal!".into()))
+		} else {
+			Ok(self.headers.get(&hash).cloned())
+		}
+	}
+	fn status(&self, _hash: Hash) -> sp_blockchain::Result<sp_blockchain::BlockStatus> {
+		unimplemented!()
+	}
+}
+
+fn test_harness(
+	test: impl FnOnce(
+		Arc<TestClient>,
+		TestSubsystemContextHandle<ChainApiMessage>,
+	) -> BoxFuture<'static, ()>,
+) {
+	let (ctx, ctx_handle) = make_subsystem_context(TaskExecutor::new());
+	let client = Arc::new(TestClient::default());
+
+	let subsystem = ChainApiSubsystem::new(client.clone(), Metrics(None));
+	let chain_api_task = run(ctx, subsystem).map(|x| x.unwrap());
+	let test_task = test(client, ctx_handle);
+
+	futures::executor::block_on(future::join(chain_api_task, test_task));
+}
+
+impl AuxStore for TestClient {
+	fn insert_aux<
+		'a,
+		'b: 'a,
+		'c: 'a,
+		I: IntoIterator<Item = &'a (&'c [u8], &'c [u8])>,
+		D: IntoIterator<Item = &'a &'b [u8]>,
+	>(
+		&self,
+		_insert: I,
+		_delete: D,
+	) -> sp_blockchain::Result<()> {
+		unimplemented!()
+	}
+
+	fn get_aux(&self, key: &[u8]) -> sp_blockchain::Result<Option<Vec<u8>>> {
+		Ok(self
+			.block_weights
+			.iter()
+			.find(|(hash, _)| sc_consensus_babe::aux_schema::block_weight_key(hash) == key)
+			.map(|(_, weight)| weight.encode()))
+	}
+}
+
+#[test]
+fn request_block_number() {
+	test_harness(|client, mut sender| {
+		async move {
+			let zero = Hash::zero();
+			let test_cases = [
+				(TWO, client.number(TWO).await.unwrap()),
+				(zero, client.number(zero).await.unwrap()), // not here
+			];
+			for (hash, expected) in &test_cases {
+				let (tx, rx) = oneshot::channel();
+
+				sender
+					.send(FromOrchestra::Communication {
+						msg: ChainApiMessage::BlockNumber(*hash, tx),
+					})
+					.await;
+
+				assert_eq!(rx.await.unwrap().unwrap(), *expected);
+			}
+
+			sender.send(FromOrchestra::Signal(OverseerSignal::Conclude)).await;
+		}
+		.boxed()
+	})
+}
+
+#[test]
+fn request_block_header() {
+	test_harness(|client, mut sender| {
+		async move {
+			const NOT_HERE: Hash = Hash::repeat_byte(0x5);
+			let test_cases = [
+				(TWO, client.header(TWO).await.unwrap()),
+				(NOT_HERE, client.header(NOT_HERE).await.unwrap()),
+			];
+			for (hash, expected) in &test_cases {
+				let (tx, rx) = oneshot::channel();
+
+				sender
+					.send(FromOrchestra::Communication {
+						msg: ChainApiMessage::BlockHeader(*hash, tx),
+					})
+					.await;
+
+				assert_eq!(rx.await.unwrap().unwrap(), *expected);
+			}
+
+			sender.send(FromOrchestra::Signal(OverseerSignal::Conclude)).await;
+		}
+		.boxed()
+	})
+}
+
+#[test]
+fn request_block_weight() {
+	test_harness(|client, mut sender| {
+		async move {
+			const NOT_HERE: Hash = Hash::repeat_byte(0x5);
+			let test_cases = [
+				(TWO, sc_consensus_babe::block_weight(&*client, TWO).unwrap()),
+				(FOUR, sc_consensus_babe::block_weight(&*client, FOUR).unwrap()),
+				(NOT_HERE, sc_consensus_babe::block_weight(&*client, NOT_HERE).unwrap()),
+			];
+			for (hash, expected) in &test_cases {
+				let (tx, rx) = oneshot::channel();
+
+				sender
+					.send(FromOrchestra::Communication {
+						msg: ChainApiMessage::BlockWeight(*hash, tx),
+					})
+					.await;
+
+				assert_eq!(rx.await.unwrap().unwrap(), *expected);
+			}
+
+			sender.send(FromOrchestra::Signal(OverseerSignal::Conclude)).await;
+		}
+		.boxed()
+	})
+}
+
+#[test]
+fn request_finalized_hash() {
+	test_harness(|client, mut sender| {
+		async move {
+			let test_cases = [
+				(1, client.hash(1).await.unwrap()), // not here
+				(2, client.hash(2).await.unwrap()),
+			];
+			for (number, expected) in &test_cases {
+				let (tx, rx) = oneshot::channel();
+
+				sender
+					.send(FromOrchestra::Communication {
+						msg: ChainApiMessage::FinalizedBlockHash(*number, tx),
+					})
+					.await;
+
+				assert_eq!(rx.await.unwrap().unwrap(), *expected);
+			}
+
+			sender.send(FromOrchestra::Signal(OverseerSignal::Conclude)).await;
+		}
+		.boxed()
+	})
+}
+
+#[test]
+fn request_last_finalized_number() {
+	test_harness(|client, mut sender| {
+		async move {
+			let (tx, rx) = oneshot::channel();
+
+			let expected = client.info().await.unwrap().finalized_number;
+			sender
+				.send(FromOrchestra::Communication {
+					msg: ChainApiMessage::FinalizedBlockNumber(tx),
+				})
+				.await;
+
+			assert_eq!(rx.await.unwrap().unwrap(), expected);
+
+			sender.send(FromOrchestra::Signal(OverseerSignal::Conclude)).await;
+		}
+		.boxed()
+	})
+}
+
+#[test]
+fn request_ancestors() {
+	test_harness(|_client, mut sender| {
+		async move {
+			let (tx, rx) = oneshot::channel();
+			sender
+				.send(FromOrchestra::Communication {
+					msg: ChainApiMessage::Ancestors { hash: THREE, k: 4, response_channel: tx },
+				})
+				.await;
+			assert_eq!(rx.await.unwrap().unwrap(), vec![TWO, ONE, GENESIS]);
+
+			// Limit the number of ancestors.
+			let (tx, rx) = oneshot::channel();
+			sender
+				.send(FromOrchestra::Communication {
+					msg: ChainApiMessage::Ancestors { hash: TWO, k: 1, response_channel: tx },
+				})
+				.await;
+			assert_eq!(rx.await.unwrap().unwrap(), vec![ONE]);
+
+			// Ancestor of block #1 is returned.
+			let (tx, rx) = oneshot::channel();
+			sender
+				.send(FromOrchestra::Communication {
+					msg: ChainApiMessage::Ancestors { hash: ONE, k: 10, response_channel: tx },
+				})
+				.await;
+			assert_eq!(rx.await.unwrap().unwrap(), vec![GENESIS]);
+
+			// No ancestors of genesis block.
+			let (tx, rx) = oneshot::channel();
+			sender
+				.send(FromOrchestra::Communication {
+					msg: ChainApiMessage::Ancestors { hash: GENESIS, k: 10, response_channel: tx },
+				})
+				.await;
+			assert_eq!(rx.await.unwrap().unwrap(), Vec::new());
+
+			let (tx, rx) = oneshot::channel();
+			sender
+				.send(FromOrchestra::Communication {
+					msg: ChainApiMessage::Ancestors {
+						hash: ERROR_PATH,
+						k: 2,
+						response_channel: tx,
+					},
+				})
+				.await;
+			assert!(rx.await.unwrap().is_err());
+
+			sender.send(FromOrchestra::Signal(OverseerSignal::Conclude)).await;
+		}
+		.boxed()
+	})
+}

pezkuwi/node/core/chain-selection/Cargo.toml

@@ -0,0 +1,40 @@
+[package]
+name = "pezkuwi-node-core-chain-selection"
+description = "Chain Selection Subsystem"
+version = "7.0.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+
+[lints]
+workspace = true
+
+[dependencies]
+codec = { workspace = true, default-features = true }
+futures = { workspace = true }
+futures-timer = { workspace = true }
+gum = { workspace = true, default-features = true }
+pezkuwi-node-primitives = { workspace = true, default-features = true }
+pezkuwi-node-subsystem = { workspace = true, default-features = true }
+pezkuwi-node-subsystem-util = { workspace = true, default-features = true }
+pezkuwi-primitives = { workspace = true, default-features = true }
+thiserror = { workspace = true }
+
+[dev-dependencies]
+assert_matches = { workspace = true }
+kvdb-memorydb = { workspace = true }
+parking_lot = { workspace = true, default-features = true }
+pezkuwi-node-subsystem-test-helpers = { workspace = true }
+sp-core = { workspace = true, default-features = true }
+
+[features]
+runtime-benchmarks = [
+	"gum/runtime-benchmarks",
+	"pezkuwi-node-primitives/runtime-benchmarks",
+	"pezkuwi-node-subsystem-test-helpers/runtime-benchmarks",
+	"pezkuwi-node-subsystem-util/runtime-benchmarks",
+	"pezkuwi-node-subsystem/runtime-benchmarks",
+	"pezkuwi-primitives/runtime-benchmarks",
+]

pezkuwi/node/core/chain-selection/src/backend.rs

@@ -0,0 +1,237 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! An abstraction over storage used by the chain selection subsystem.
+//!
+//! This provides both a [`Backend`] trait and an [`OverlayedBackend`]
+//! struct which allows in-memory changes to be applied on top of a
+//! [`Backend`], maintaining consistency between queries and temporary writes,
+//! before any commit to the underlying storage is made.
+
+use pezkuwi_primitives::{BlockNumber, Hash};
+
+use std::collections::HashMap;
+
+use crate::{BlockEntry, Error, LeafEntrySet, Timestamp};
+
+pub(super) enum BackendWriteOp {
+	WriteBlockEntry(BlockEntry),
+	WriteBlocksByNumber(BlockNumber, Vec<Hash>),
+	WriteViableLeaves(LeafEntrySet),
+	WriteStagnantAt(Timestamp, Vec<Hash>),
+	DeleteBlocksByNumber(BlockNumber),
+	DeleteBlockEntry(Hash),
+	DeleteStagnantAt(Timestamp),
+}
+
+/// An abstraction over backend storage for the logic of this subsystem.
+pub(super) trait Backend {
+	/// Load a block entry from the DB.
+	fn load_block_entry(&self, hash: &Hash) -> Result<Option<BlockEntry>, Error>;
+	/// Load the active-leaves set.
+	fn load_leaves(&self) -> Result<LeafEntrySet, Error>;
+	/// Load the stagnant list at the given timestamp.
+	fn load_stagnant_at(&self, timestamp: Timestamp) -> Result<Vec<Hash>, Error>;
+	/// Load all stagnant lists up to and including the given Unix timestamp
+	/// in ascending order. Stop fetching stagnant entries upon reaching `max_elements`.
+	fn load_stagnant_at_up_to(
+		&self,
+		up_to: Timestamp,
+		max_elements: usize,
+	) -> Result<Vec<(Timestamp, Vec<Hash>)>, Error>;
+	/// Load the earliest kept block number.
+	fn load_first_block_number(&self) -> Result<Option<BlockNumber>, Error>;
+	/// Load blocks by number.
+	fn load_blocks_by_number(&self, number: BlockNumber) -> Result<Vec<Hash>, Error>;
+
+	/// Atomically write the list of operations, with later operations taking precedence over prior.
+	fn write<I>(&mut self, ops: I) -> Result<(), Error>
+	where
+		I: IntoIterator<Item = BackendWriteOp>;
+}
+
+/// An in-memory overlay over the backend.
+///
+/// This maintains read-only access to the underlying backend, but can be
+/// converted into a set of write operations which will, when written to
+/// the underlying backend, give the same view as the state of the overlay.
+pub(super) struct OverlayedBackend<'a, B: 'a> {
+	inner: &'a B,
+
+	// `None` means 'deleted', missing means query inner.
+	block_entries: HashMap<Hash, Option<BlockEntry>>,
+	// `None` means 'deleted', missing means query inner.
+	blocks_by_number: HashMap<BlockNumber, Option<Vec<Hash>>>,
+	// 'None' means 'deleted', missing means query inner.
+	stagnant_at: HashMap<Timestamp, Option<Vec<Hash>>>,
+	// 'None' means query inner.
+	leaves: Option<LeafEntrySet>,
+}
+
+impl<'a, B: 'a + Backend> OverlayedBackend<'a, B> {
+	pub(super) fn new(backend: &'a B) -> Self {
+		OverlayedBackend {
+			inner: backend,
+			block_entries: HashMap::new(),
+			blocks_by_number: HashMap::new(),
+			stagnant_at: HashMap::new(),
+			leaves: None,
+		}
+	}
+
+	pub(super) fn load_block_entry(&self, hash: &Hash) -> Result<Option<BlockEntry>, Error> {
+		if let Some(val) = self.block_entries.get(&hash) {
+			return Ok(val.clone());
+		}
+
+		self.inner.load_block_entry(hash)
+	}
+
+	pub(super) fn load_blocks_by_number(&self, number: BlockNumber) -> Result<Vec<Hash>, Error> {
+		if let Some(val) = self.blocks_by_number.get(&number) {
+			return Ok(val.as_ref().map_or(Vec::new(), Clone::clone));
+		}
+
+		self.inner.load_blocks_by_number(number)
+	}
+
+	pub(super) fn load_leaves(&self) -> Result<LeafEntrySet, Error> {
+		if let Some(ref set) = self.leaves {
+			return Ok(set.clone());
+		}
+
+		self.inner.load_leaves()
+	}
+
+	pub(super) fn load_stagnant_at(&self, timestamp: Timestamp) -> Result<Vec<Hash>, Error> {
+		if let Some(val) = self.stagnant_at.get(&timestamp) {
+			return Ok(val.as_ref().map_or(Vec::new(), Clone::clone));
+		}
+
+		self.inner.load_stagnant_at(timestamp)
+	}
+
+	pub(super) fn write_block_entry(&mut self, entry: BlockEntry) {
+		self.block_entries.insert(entry.block_hash, Some(entry));
+	}
+
+	pub(super) fn delete_block_entry(&mut self, hash: &Hash) {
+		self.block_entries.insert(*hash, None);
+	}
+
+	pub(super) fn write_blocks_by_number(&mut self, number: BlockNumber, blocks: Vec<Hash>) {
+		if blocks.is_empty() {
+			self.blocks_by_number.insert(number, None);
+		} else {
+			self.blocks_by_number.insert(number, Some(blocks));
+		}
+	}
+
+	pub(super) fn delete_blocks_by_number(&mut self, number: BlockNumber) {
+		self.blocks_by_number.insert(number, None);
+	}
+
+	pub(super) fn write_leaves(&mut self, leaves: LeafEntrySet) {
+		self.leaves = Some(leaves);
+	}
+
+	pub(super) fn write_stagnant_at(&mut self, timestamp: Timestamp, hashes: Vec<Hash>) {
+		self.stagnant_at.insert(timestamp, Some(hashes));
+	}
+
+	pub(super) fn delete_stagnant_at(&mut self, timestamp: Timestamp) {
+		self.stagnant_at.insert(timestamp, None);
+	}
+
+	/// Transform this backend into a set of write-ops to be written to the
+	/// inner backend.
+	pub(super) fn into_write_ops(self) -> impl Iterator<Item = BackendWriteOp> {
+		let block_entry_ops = self.block_entries.into_iter().map(|(h, v)| match v {
+			Some(v) => BackendWriteOp::WriteBlockEntry(v),
+			None => BackendWriteOp::DeleteBlockEntry(h),
+		});
+
+		let blocks_by_number_ops = self.blocks_by_number.into_iter().map(|(n, v)| match v {
+			Some(v) => BackendWriteOp::WriteBlocksByNumber(n, v),
+			None => BackendWriteOp::DeleteBlocksByNumber(n),
+		});
+
+		let leaf_ops = self.leaves.into_iter().map(BackendWriteOp::WriteViableLeaves);
+
+		let stagnant_at_ops = self.stagnant_at.into_iter().map(|(n, v)| match v {
+			Some(v) => BackendWriteOp::WriteStagnantAt(n, v),
+			None => BackendWriteOp::DeleteStagnantAt(n),
+		});
+
+		block_entry_ops
+			.chain(blocks_by_number_ops)
+			.chain(leaf_ops)
+			.chain(stagnant_at_ops)
+	}
+}
+
+/// Attempt to find the given ancestor in the chain with given head.
+///
+/// If the ancestor is the most recently finalized block, and the `head` is
+/// a known unfinalized block, this will return `true`.
+///
+/// If the ancestor is an unfinalized block and `head` is known, this will
+/// return true if `ancestor` is in `head`'s chain.
+///
+/// If the ancestor is an older finalized block, this will return `false`.
+fn contains_ancestor(backend: &impl Backend, head: Hash, ancestor: Hash) -> Result<bool, Error> {
+	let mut current_hash = head;
+	loop {
+		if current_hash == ancestor {
+			return Ok(true);
+		}
+		match backend.load_block_entry(&current_hash)? {
+			Some(e) => current_hash = e.parent_hash,
+			None => break,
+		}
+	}
+
+	Ok(false)
+}
+
+/// This returns the best unfinalized leaf containing the required block.
+///
+/// If the required block is finalized but not the most recent finalized block,
+/// this will return `None`.
+///
+/// If the required block is unfinalized but not an ancestor of any viable leaf,
+/// this will return `None`.
+//
+// Note: this is O(N^2) in the depth of `required` and the number of leaves.
+// We expect the number of unfinalized blocks to be small, as in, to not exceed
+// single digits in practice, and exceedingly unlikely to surpass 1000.
+//
+// However, if we need to, we could implement some type of skip-list for
+// fast ancestry checks.
+pub(super) fn find_best_leaf_containing(
+	backend: &impl Backend,
+	required: Hash,
+) -> Result<Option<Hash>, Error> {
+	let leaves = backend.load_leaves()?;
+	for leaf in leaves.into_hashes_descending() {
+		if contains_ancestor(backend, leaf, required)? {
+			return Ok(Some(leaf));
+		}
+	}
+
+	// If there are no viable leaves containing the ancestor
+	Ok(None)
+}

pezkuwi/node/core/chain-selection/src/db_backend/mod.rs

@@ -0,0 +1,19 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! A database [`Backend`][crate::backend::Backend] for the chain selection subsystem.
+
+pub(super) mod v1;

pezkuwi/node/core/chain-selection/src/db_backend/v1.rs

@@ -0,0 +1,631 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! A database [`Backend`][crate::backend::Backend] for the chain selection subsystem.
+//!
+//! This stores the following schema:
+//!
+//! ```ignore
+//! ("CS_block_entry", Hash) -> BlockEntry;
+//! ("CS_block_height", BigEndianBlockNumber) -> Vec<Hash>;
+//! ("CS_stagnant_at", BigEndianTimestamp) -> Vec<Hash>;
+//! ("CS_leaves") -> LeafEntrySet;
+//! ```
+//!
+//! The big-endian encoding is used for creating iterators over the key-value DB which are
+//! accessible by prefix, to find the earliest block number stored as well as the all stagnant
+//! blocks.
+//!
+//! The `Vec`s stored are always non-empty. Empty `Vec`s are not stored on disk so there is no
+//! semantic difference between `None` and an empty `Vec`.
+
+use crate::{
+	backend::{Backend, BackendWriteOp},
+	Error,
+};
+
+use pezkuwi_node_primitives::BlockWeight;
+use pezkuwi_primitives::{BlockNumber, Hash};
+
+use codec::{Decode, Encode};
+use pezkuwi_node_subsystem_util::database::{DBTransaction, Database};
+
+use std::sync::Arc;
+
+const BLOCK_ENTRY_PREFIX: &[u8; 14] = b"CS_block_entry";
+const BLOCK_HEIGHT_PREFIX: &[u8; 15] = b"CS_block_height";
+const STAGNANT_AT_PREFIX: &[u8; 14] = b"CS_stagnant_at";
+const LEAVES_KEY: &[u8; 9] = b"CS_leaves";
+
+type Timestamp = u64;
+
+#[derive(Debug, Encode, Decode, Clone, PartialEq)]
+enum Approval {
+	#[codec(index = 0)]
+	Approved,
+	#[codec(index = 1)]
+	Unapproved,
+	#[codec(index = 2)]
+	Stagnant,
+}
+
+impl From<crate::Approval> for Approval {
+	fn from(x: crate::Approval) -> Self {
+		match x {
+			crate::Approval::Approved => Approval::Approved,
+			crate::Approval::Unapproved => Approval::Unapproved,
+			crate::Approval::Stagnant => Approval::Stagnant,
+		}
+	}
+}
+
+impl From<Approval> for crate::Approval {
+	fn from(x: Approval) -> crate::Approval {
+		match x {
+			Approval::Approved => crate::Approval::Approved,
+			Approval::Unapproved => crate::Approval::Unapproved,
+			Approval::Stagnant => crate::Approval::Stagnant,
+		}
+	}
+}
+
+#[derive(Debug, Encode, Decode, Clone, PartialEq)]
+struct ViabilityCriteria {
+	explicitly_reverted: bool,
+	approval: Approval,
+	earliest_unviable_ancestor: Option<Hash>,
+}
+
+impl From<crate::ViabilityCriteria> for ViabilityCriteria {
+	fn from(x: crate::ViabilityCriteria) -> Self {
+		ViabilityCriteria {
+			explicitly_reverted: x.explicitly_reverted,
+			approval: x.approval.into(),
+			earliest_unviable_ancestor: x.earliest_unviable_ancestor,
+		}
+	}
+}
+
+impl From<ViabilityCriteria> for crate::ViabilityCriteria {
+	fn from(x: ViabilityCriteria) -> crate::ViabilityCriteria {
+		crate::ViabilityCriteria {
+			explicitly_reverted: x.explicitly_reverted,
+			approval: x.approval.into(),
+			earliest_unviable_ancestor: x.earliest_unviable_ancestor,
+		}
+	}
+}
+
+#[derive(Encode, Decode)]
+struct LeafEntry {
+	weight: BlockWeight,
+	block_number: BlockNumber,
+	block_hash: Hash,
+}
+
+impl From<crate::LeafEntry> for LeafEntry {
+	fn from(x: crate::LeafEntry) -> Self {
+		LeafEntry { weight: x.weight, block_number: x.block_number, block_hash: x.block_hash }
+	}
+}
+
+impl From<LeafEntry> for crate::LeafEntry {
+	fn from(x: LeafEntry) -> crate::LeafEntry {
+		crate::LeafEntry {
+			weight: x.weight,
+			block_number: x.block_number,
+			block_hash: x.block_hash,
+		}
+	}
+}
+
+#[derive(Encode, Decode)]
+struct LeafEntrySet {
+	inner: Vec<LeafEntry>,
+}
+
+impl From<crate::LeafEntrySet> for LeafEntrySet {
+	fn from(x: crate::LeafEntrySet) -> Self {
+		LeafEntrySet { inner: x.inner.into_iter().map(Into::into).collect() }
+	}
+}
+
+impl From<LeafEntrySet> for crate::LeafEntrySet {
+	fn from(x: LeafEntrySet) -> crate::LeafEntrySet {
+		crate::LeafEntrySet { inner: x.inner.into_iter().map(Into::into).collect() }
+	}
+}
+
+#[derive(Debug, Encode, Decode, Clone, PartialEq)]
+struct BlockEntry {
+	block_hash: Hash,
+	block_number: BlockNumber,
+	parent_hash: Hash,
+	children: Vec<Hash>,
+	viability: ViabilityCriteria,
+	weight: BlockWeight,
+}
+
+impl From<crate::BlockEntry> for BlockEntry {
+	fn from(x: crate::BlockEntry) -> Self {
+		BlockEntry {
+			block_hash: x.block_hash,
+			block_number: x.block_number,
+			parent_hash: x.parent_hash,
+			children: x.children,
+			viability: x.viability.into(),
+			weight: x.weight,
+		}
+	}
+}
+
+impl From<BlockEntry> for crate::BlockEntry {
+	fn from(x: BlockEntry) -> crate::BlockEntry {
+		crate::BlockEntry {
+			block_hash: x.block_hash,
+			block_number: x.block_number,
+			parent_hash: x.parent_hash,
+			children: x.children,
+			viability: x.viability.into(),
+			weight: x.weight,
+		}
+	}
+}
+
+/// Configuration for the database backend.
+#[derive(Debug, Clone, Copy)]
+pub struct Config {
+	/// The column where block metadata is stored.
+	pub col_data: u32,
+}
+
+/// The database backend.
+pub struct DbBackend {
+	inner: Arc<dyn Database>,
+	config: Config,
+}
+
+impl DbBackend {
+	/// Create a new [`DbBackend`] with the supplied key-value store and
+	/// config.
+	pub fn new(db: Arc<dyn Database>, config: Config) -> Self {
+		DbBackend { inner: db, config }
+	}
+}
+
+impl Backend for DbBackend {
+	fn load_block_entry(&self, hash: &Hash) -> Result<Option<crate::BlockEntry>, Error> {
+		load_decode::<BlockEntry>(&*self.inner, self.config.col_data, &block_entry_key(hash))
+			.map(|o| o.map(Into::into))
+	}
+
+	fn load_leaves(&self) -> Result<crate::LeafEntrySet, Error> {
+		load_decode::<LeafEntrySet>(&*self.inner, self.config.col_data, LEAVES_KEY)
+			.map(|o| o.map(Into::into).unwrap_or_default())
+	}
+
+	fn load_stagnant_at(&self, timestamp: crate::Timestamp) -> Result<Vec<Hash>, Error> {
+		load_decode::<Vec<Hash>>(
+			&*self.inner,
+			self.config.col_data,
+			&stagnant_at_key(timestamp.into()),
+		)
+		.map(|o| o.unwrap_or_default())
+	}
+
+	fn load_stagnant_at_up_to(
+		&self,
+		up_to: crate::Timestamp,
+		max_elements: usize,
+	) -> Result<Vec<(crate::Timestamp, Vec<Hash>)>, Error> {
+		let stagnant_at_iter =
+			self.inner.iter_with_prefix(self.config.col_data, &STAGNANT_AT_PREFIX[..]);
+
+		let val = stagnant_at_iter
+			.filter_map(|r| match r {
+				Ok((k, v)) => {
+					match (decode_stagnant_at_key(&mut &k[..]), <Vec<_>>::decode(&mut &v[..]).ok())
+					{
+						(Some(at), Some(stagnant_at)) => Some(Ok((at, stagnant_at))),
+						_ => None,
+					}
+				},
+				Err(e) => Some(Err(e)),
+			})
+			.enumerate()
+			.take_while(|(idx, r)| {
+				r.as_ref().map_or(true, |(at, _)| *at <= up_to.into() && *idx < max_elements)
+			})
+			.map(|(_, v)| v)
+			.collect::<Result<Vec<_>, _>>()?;
+
+		Ok(val)
+	}
+
+	fn load_first_block_number(&self) -> Result<Option<BlockNumber>, Error> {
+		let blocks_at_height_iter =
+			self.inner.iter_with_prefix(self.config.col_data, &BLOCK_HEIGHT_PREFIX[..]);
+
+		let val = blocks_at_height_iter
+			.filter_map(|r| match r {
+				Ok((k, _)) => decode_block_height_key(&k[..]).map(Ok),
+				Err(e) => Some(Err(e)),
+			})
+			.next();
+
+		val.transpose().map_err(Error::from)
+	}
+
+	fn load_blocks_by_number(&self, number: BlockNumber) -> Result<Vec<Hash>, Error> {
+		load_decode::<Vec<Hash>>(&*self.inner, self.config.col_data, &block_height_key(number))
+			.map(|o| o.unwrap_or_default())
+	}
+
+	/// Atomically write the list of operations, with later operations taking precedence over prior.
+	fn write<I>(&mut self, ops: I) -> Result<(), Error>
+	where
+		I: IntoIterator<Item = BackendWriteOp>,
+	{
+		let mut tx = DBTransaction::new();
+		for op in ops {
+			match op {
+				BackendWriteOp::WriteBlockEntry(block_entry) => {
+					let block_entry: BlockEntry = block_entry.into();
+					tx.put_vec(
+						self.config.col_data,
+						&block_entry_key(&block_entry.block_hash),
+						block_entry.encode(),
+					);
+				},
+				BackendWriteOp::WriteBlocksByNumber(block_number, v) =>
+					if v.is_empty() {
+						tx.delete(self.config.col_data, &block_height_key(block_number));
+					} else {
+						tx.put_vec(
+							self.config.col_data,
+							&block_height_key(block_number),
+							v.encode(),
+						);
+					},
+				BackendWriteOp::WriteViableLeaves(leaves) => {
+					let leaves: LeafEntrySet = leaves.into();
+					if leaves.inner.is_empty() {
+						tx.delete(self.config.col_data, &LEAVES_KEY[..]);
+					} else {
+						tx.put_vec(self.config.col_data, &LEAVES_KEY[..], leaves.encode());
+					}
+				},
+				BackendWriteOp::WriteStagnantAt(timestamp, stagnant_at) => {
+					let timestamp: Timestamp = timestamp.into();
+					if stagnant_at.is_empty() {
+						tx.delete(self.config.col_data, &stagnant_at_key(timestamp));
+					} else {
+						tx.put_vec(
+							self.config.col_data,
+							&stagnant_at_key(timestamp),
+							stagnant_at.encode(),
+						);
+					}
+				},
+				BackendWriteOp::DeleteBlocksByNumber(block_number) => {
+					tx.delete(self.config.col_data, &block_height_key(block_number));
+				},
+				BackendWriteOp::DeleteBlockEntry(hash) => {
+					tx.delete(self.config.col_data, &block_entry_key(&hash));
+				},
+				BackendWriteOp::DeleteStagnantAt(timestamp) => {
+					let timestamp: Timestamp = timestamp.into();
+					tx.delete(self.config.col_data, &stagnant_at_key(timestamp));
+				},
+			}
+		}
+
+		self.inner.write(tx).map_err(Into::into)
+	}
+}
+
+fn load_decode<D: Decode>(
+	db: &dyn Database,
+	col_data: u32,
+	key: &[u8],
+) -> Result<Option<D>, Error> {
+	match db.get(col_data, key)? {
+		None => Ok(None),
+		Some(raw) => D::decode(&mut &raw[..]).map(Some).map_err(Into::into),
+	}
+}
+
+fn block_entry_key(hash: &Hash) -> [u8; 14 + 32] {
+	let mut key = [0; 14 + 32];
+	key[..14].copy_from_slice(BLOCK_ENTRY_PREFIX);
+	hash.using_encoded(|s| key[14..].copy_from_slice(s));
+	key
+}
+
+fn block_height_key(number: BlockNumber) -> [u8; 15 + 4] {
+	let mut key = [0; 15 + 4];
+	key[..15].copy_from_slice(BLOCK_HEIGHT_PREFIX);
+	key[15..].copy_from_slice(&number.to_be_bytes());
+	key
+}
+
+fn stagnant_at_key(timestamp: Timestamp) -> [u8; 14 + 8] {
+	let mut key = [0; 14 + 8];
+	key[..14].copy_from_slice(STAGNANT_AT_PREFIX);
+	key[14..].copy_from_slice(&timestamp.to_be_bytes());
+	key
+}
+
+fn decode_block_height_key(key: &[u8]) -> Option<BlockNumber> {
+	if key.len() != 15 + 4 {
+		return None;
+	}
+	if !key.starts_with(BLOCK_HEIGHT_PREFIX) {
+		return None;
+	}
+
+	let mut bytes = [0; 4];
+	bytes.copy_from_slice(&key[15..]);
+	Some(BlockNumber::from_be_bytes(bytes))
+}
+
+fn decode_stagnant_at_key(key: &[u8]) -> Option<Timestamp> {
+	if key.len() != 14 + 8 {
+		return None;
+	}
+	if !key.starts_with(STAGNANT_AT_PREFIX) {
+		return None;
+	}
+
+	let mut bytes = [0; 8];
+	bytes.copy_from_slice(&key[14..]);
+	Some(Timestamp::from_be_bytes(bytes))
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[cfg(test)]
+	fn test_db() -> Arc<dyn Database> {
+		let db = kvdb_memorydb::create(1);
+		let db = pezkuwi_node_subsystem_util::database::kvdb_impl::DbAdapter::new(db, &[0]);
+		Arc::new(db)
+	}
+
+	#[test]
+	fn block_height_key_decodes() {
+		let key = block_height_key(5);
+		assert_eq!(decode_block_height_key(&key), Some(5));
+	}
+
+	#[test]
+	fn stagnant_at_key_decodes() {
+		let key = stagnant_at_key(5);
+		assert_eq!(decode_stagnant_at_key(&key), Some(5));
+	}
+
+	#[test]
+	fn lower_block_height_key_lesser() {
+		for i in 0..256 {
+			for j in 1..=256 {
+				let key_a = block_height_key(i);
+				let key_b = block_height_key(i + j);
+
+				assert!(key_a < key_b);
+			}
+		}
+	}
+
+	#[test]
+	fn lower_stagnant_at_key_lesser() {
+		for i in 0..256 {
+			for j in 1..=256 {
+				let key_a = stagnant_at_key(i);
+				let key_b = stagnant_at_key(i + j);
+
+				assert!(key_a < key_b);
+			}
+		}
+	}
+
+	#[test]
+	fn write_read_block_entry() {
+		let db = test_db();
+		let config = Config { col_data: 0 };
+
+		let mut backend = DbBackend::new(db, config);
+
+		let block_entry = BlockEntry {
+			block_hash: Hash::repeat_byte(1),
+			block_number: 1,
+			parent_hash: Hash::repeat_byte(0),
+			children: vec![],
+			viability: ViabilityCriteria {
+				earliest_unviable_ancestor: None,
+				explicitly_reverted: false,
+				approval: Approval::Unapproved,
+			},
+			weight: 100,
+		};
+
+		backend
+			.write(vec![BackendWriteOp::WriteBlockEntry(block_entry.clone().into())])
+			.unwrap();
+
+		assert_eq!(
+			backend.load_block_entry(&block_entry.block_hash).unwrap().map(BlockEntry::from),
+			Some(block_entry),
+		);
+	}
+
+	#[test]
+	fn delete_block_entry() {
+		let db = test_db();
+		let config = Config { col_data: 0 };
+
+		let mut backend = DbBackend::new(db, config);
+
+		let block_entry = BlockEntry {
+			block_hash: Hash::repeat_byte(1),
+			block_number: 1,
+			parent_hash: Hash::repeat_byte(0),
+			children: vec![],
+			viability: ViabilityCriteria {
+				earliest_unviable_ancestor: None,
+				explicitly_reverted: false,
+				approval: Approval::Unapproved,
+			},
+			weight: 100,
+		};
+
+		backend
+			.write(vec![BackendWriteOp::WriteBlockEntry(block_entry.clone().into())])
+			.unwrap();
+
+		backend
+			.write(vec![BackendWriteOp::DeleteBlockEntry(block_entry.block_hash)])
+			.unwrap();
+
+		assert!(backend.load_block_entry(&block_entry.block_hash).unwrap().is_none());
+	}
+
+	#[test]
+	fn earliest_block_number() {
+		let db = test_db();
+		let config = Config { col_data: 0 };
+
+		let mut backend = DbBackend::new(db, config);
+
+		assert!(backend.load_first_block_number().unwrap().is_none());
+
+		backend
+			.write(vec![
+				BackendWriteOp::WriteBlocksByNumber(2, vec![Hash::repeat_byte(0)]),
+				BackendWriteOp::WriteBlocksByNumber(5, vec![Hash::repeat_byte(0)]),
+				BackendWriteOp::WriteBlocksByNumber(10, vec![Hash::repeat_byte(0)]),
+			])
+			.unwrap();
+
+		assert_eq!(backend.load_first_block_number().unwrap(), Some(2));
+
+		backend
+			.write(vec![
+				BackendWriteOp::WriteBlocksByNumber(2, vec![]),
+				BackendWriteOp::DeleteBlocksByNumber(5),
+			])
+			.unwrap();
+
+		assert_eq!(backend.load_first_block_number().unwrap(), Some(10));
+	}
+
+	#[test]
+	fn stagnant_at_up_to() {
+		let db = test_db();
+		let config = Config { col_data: 0 };
+
+		let mut backend = DbBackend::new(db, config);
+
+		// Prove that it's cheap
+		assert!(backend
+			.load_stagnant_at_up_to(Timestamp::max_value(), usize::MAX)
+			.unwrap()
+			.is_empty());
+
+		backend
+			.write(vec![
+				BackendWriteOp::WriteStagnantAt(2, vec![Hash::repeat_byte(1)]),
+				BackendWriteOp::WriteStagnantAt(5, vec![Hash::repeat_byte(2)]),
+				BackendWriteOp::WriteStagnantAt(10, vec![Hash::repeat_byte(3)]),
+			])
+			.unwrap();
+
+		assert_eq!(
+			backend.load_stagnant_at_up_to(Timestamp::max_value(), usize::MAX).unwrap(),
+			vec![
+				(2, vec![Hash::repeat_byte(1)]),
+				(5, vec![Hash::repeat_byte(2)]),
+				(10, vec![Hash::repeat_byte(3)]),
+			]
+		);
+
+		assert_eq!(
+			backend.load_stagnant_at_up_to(10, usize::MAX).unwrap(),
+			vec![
+				(2, vec![Hash::repeat_byte(1)]),
+				(5, vec![Hash::repeat_byte(2)]),
+				(10, vec![Hash::repeat_byte(3)]),
+			]
+		);
+
+		assert_eq!(
+			backend.load_stagnant_at_up_to(9, usize::MAX).unwrap(),
+			vec![(2, vec![Hash::repeat_byte(1)]), (5, vec![Hash::repeat_byte(2)]),]
+		);
+
+		assert_eq!(
+			backend.load_stagnant_at_up_to(9, 1).unwrap(),
+			vec![(2, vec![Hash::repeat_byte(1)]),]
+		);
+
+		backend.write(vec![BackendWriteOp::DeleteStagnantAt(2)]).unwrap();
+
+		assert_eq!(
+			backend.load_stagnant_at_up_to(5, usize::MAX).unwrap(),
+			vec![(5, vec![Hash::repeat_byte(2)]),]
+		);
+
+		backend.write(vec![BackendWriteOp::WriteStagnantAt(5, vec![])]).unwrap();
+
+		assert_eq!(
+			backend.load_stagnant_at_up_to(10, usize::MAX).unwrap(),
+			vec![(10, vec![Hash::repeat_byte(3)]),]
+		);
+	}
+
+	#[test]
+	fn write_read_blocks_at_height() {
+		let db = test_db();
+		let config = Config { col_data: 0 };
+
+		let mut backend = DbBackend::new(db, config);
+
+		backend
+			.write(vec![
+				BackendWriteOp::WriteBlocksByNumber(2, vec![Hash::repeat_byte(1)]),
+				BackendWriteOp::WriteBlocksByNumber(5, vec![Hash::repeat_byte(2)]),
+				BackendWriteOp::WriteBlocksByNumber(10, vec![Hash::repeat_byte(3)]),
+			])
+			.unwrap();
+
+		assert_eq!(backend.load_blocks_by_number(2).unwrap(), vec![Hash::repeat_byte(1)]);
+
+		assert_eq!(backend.load_blocks_by_number(3).unwrap(), vec![]);
+
+		backend
+			.write(vec![
+				BackendWriteOp::WriteBlocksByNumber(2, vec![]),
+				BackendWriteOp::DeleteBlocksByNumber(5),
+			])
+			.unwrap();
+
+		assert_eq!(backend.load_blocks_by_number(2).unwrap(), vec![]);
+
+		assert_eq!(backend.load_blocks_by_number(5).unwrap(), vec![]);
+
+		assert_eq!(backend.load_blocks_by_number(10).unwrap(), vec![Hash::repeat_byte(3)]);
+	}
+}

pezkuwi/node/core/chain-selection/src/lib.rs

@@ -0,0 +1,743 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Implements the Chain Selection Subsystem.
+
+use pezkuwi_node_primitives::BlockWeight;
+use pezkuwi_node_subsystem::{
+	errors::ChainApiError,
+	messages::{ChainApiMessage, ChainSelectionMessage},
+	overseer::{self, SubsystemSender},
+	FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError,
+};
+use pezkuwi_node_subsystem_util::database::Database;
+use pezkuwi_primitives::{BlockNumber, ConsensusLog, Hash, Header};
+
+use codec::Error as CodecError;
+use futures::{channel::oneshot, future::Either, prelude::*};
+
+use std::{
+	sync::Arc,
+	time::{Duration, SystemTime, UNIX_EPOCH},
+};
+
+use crate::backend::{Backend, BackendWriteOp, OverlayedBackend};
+
+mod backend;
+mod db_backend;
+mod tree;
+
+#[cfg(test)]
+mod tests;
+
+const LOG_TARGET: &str = "teyrchain::chain-selection";
+/// Timestamp based on the 1 Jan 1970 UNIX base, which is persistent across node restarts and OS
+/// reboots.
+type Timestamp = u64;
+
+// If a block isn't approved in 120 seconds, nodes will abandon it
+// and begin building on another chain.
+const STAGNANT_TIMEOUT: Timestamp = 120;
+// Delay pruning of the stagnant keys in prune only mode by 25 hours to avoid interception with the
+// finality
+const STAGNANT_PRUNE_DELAY: Timestamp = 25 * 60 * 60;
+// Maximum number of stagnant entries cleaned during one `STAGNANT_TIMEOUT` iteration
+const MAX_STAGNANT_ENTRIES: usize = 1000;
+
+#[derive(Debug, Clone)]
+enum Approval {
+	// Approved
+	Approved,
+	// Unapproved but not stagnant
+	Unapproved,
+	// Unapproved and stagnant.
+	Stagnant,
+}
+
+impl Approval {
+	fn is_stagnant(&self) -> bool {
+		matches!(*self, Approval::Stagnant)
+	}
+}
+
+#[derive(Debug, Clone)]
+struct ViabilityCriteria {
+	// Whether this block has been explicitly reverted by one of its descendants.
+	explicitly_reverted: bool,
+	// The approval state of this block specifically.
+	approval: Approval,
+	// The earliest unviable ancestor - the hash of the earliest unfinalized
+	// block in the ancestry which is explicitly reverted or stagnant.
+	earliest_unviable_ancestor: Option<Hash>,
+}
+
+impl ViabilityCriteria {
+	fn is_viable(&self) -> bool {
+		self.is_parent_viable() && self.is_explicitly_viable()
+	}
+
+	// Whether the current block is explicitly viable.
+	// That is, whether the current block is neither reverted nor stagnant.
+	fn is_explicitly_viable(&self) -> bool {
+		!self.explicitly_reverted && !self.approval.is_stagnant()
+	}
+
+	// Whether the parent is viable. This assumes that the parent
+	// descends from the finalized chain.
+	fn is_parent_viable(&self) -> bool {
+		self.earliest_unviable_ancestor.is_none()
+	}
+}
+
+// Light entries describing leaves of the chain.
+//
+// These are ordered first by weight and then by block number.
+#[derive(Debug, Clone, PartialEq)]
+struct LeafEntry {
+	weight: BlockWeight,
+	block_number: BlockNumber,
+	block_hash: Hash,
+}
+
+impl PartialOrd for LeafEntry {
+	fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+		let ord = self.weight.cmp(&other.weight).then(self.block_number.cmp(&other.block_number));
+
+		if !matches!(ord, std::cmp::Ordering::Equal) {
+			Some(ord)
+		} else {
+			None
+		}
+	}
+}
+
+#[derive(Debug, Default, Clone)]
+struct LeafEntrySet {
+	inner: Vec<LeafEntry>,
+}
+
+impl LeafEntrySet {
+	fn remove(&mut self, hash: &Hash) -> bool {
+		match self.inner.iter().position(|e| &e.block_hash == hash) {
+			None => false,
+			Some(i) => {
+				self.inner.remove(i);
+				true
+			},
+		}
+	}
+
+	fn insert(&mut self, new: LeafEntry) {
+		let mut pos = None;
+		for (i, e) in self.inner.iter().enumerate() {
+			if e == &new {
+				return;
+			}
+			if e < &new {
+				pos = Some(i);
+				break;
+			}
+		}
+
+		match pos {
+			None => self.inner.push(new),
+			Some(i) => self.inner.insert(i, new),
+		}
+	}
+
+	fn into_hashes_descending(self) -> impl Iterator<Item = Hash> {
+		self.inner.into_iter().map(|e| e.block_hash)
+	}
+}
+
+#[derive(Debug, Clone)]
+struct BlockEntry {
+	block_hash: Hash,
+	block_number: BlockNumber,
+	parent_hash: Hash,
+	children: Vec<Hash>,
+	viability: ViabilityCriteria,
+	weight: BlockWeight,
+}
+
+impl BlockEntry {
+	fn leaf_entry(&self) -> LeafEntry {
+		LeafEntry {
+			block_hash: self.block_hash,
+			block_number: self.block_number,
+			weight: self.weight,
+		}
+	}
+
+	fn non_viable_ancestor_for_child(&self) -> Option<Hash> {
+		if self.viability.is_viable() {
+			None
+		} else {
+			self.viability.earliest_unviable_ancestor.or(Some(self.block_hash))
+		}
+	}
+}
+
+#[derive(Debug, thiserror::Error)]
+#[allow(missing_docs)]
+pub enum Error {
+	#[error(transparent)]
+	ChainApi(#[from] ChainApiError),
+
+	#[error(transparent)]
+	Io(#[from] std::io::Error),
+
+	#[error(transparent)]
+	Oneshot(#[from] oneshot::Canceled),
+
+	#[error(transparent)]
+	Subsystem(#[from] SubsystemError),
+
+	#[error(transparent)]
+	Codec(#[from] CodecError),
+}
+
+impl Error {
+	fn trace(&self) {
+		match self {
+			// don't spam the log with spurious errors
+			Self::Oneshot(_) => gum::debug!(target: LOG_TARGET, err = ?self),
+			// it's worth reporting otherwise
+			_ => gum::warn!(target: LOG_TARGET, err = ?self),
+		}
+	}
+}
+
+/// A clock used for fetching the current timestamp.
+pub trait Clock {
+	/// Get the current timestamp.
+	fn timestamp_now(&self) -> Timestamp;
+}
+
+struct SystemClock;
+
+impl Clock for SystemClock {
+	fn timestamp_now(&self) -> Timestamp {
+		// `SystemTime` is notoriously non-monotonic, so our timers might not work
+		// exactly as expected. Regardless, stagnation is detected on the order of minutes,
+		// and slippage of a few seconds in either direction won't cause any major harm.
+		//
+		// The exact time that a block becomes stagnant in the local node is always expected
+		// to differ from other nodes due to network asynchrony and delays in block propagation.
+		// Non-monotonicity exacerbates that somewhat, but not meaningfully.
+
+		match SystemTime::now().duration_since(UNIX_EPOCH) {
+			Ok(d) => d.as_secs(),
+			Err(e) => {
+				gum::warn!(
+					target: LOG_TARGET,
+					err = ?e,
+					"Current time is before unix epoch. Validation will not work correctly."
+				);
+
+				0
+			},
+		}
+	}
+}
+
+/// The interval, in seconds to check for stagnant blocks.
+#[derive(Debug, Clone)]
+pub struct StagnantCheckInterval(Option<Duration>);
+
+impl Default for StagnantCheckInterval {
+	fn default() -> Self {
+		// 5 seconds is a reasonable balance between avoiding DB reads and
+		// ensuring validators are generally in agreement on stagnant blocks.
+		//
+		// Assuming a network delay of D, the longest difference in view possible
+		// between 2 validators is D + 5s.
+		const DEFAULT_STAGNANT_CHECK_INTERVAL: Duration = Duration::from_secs(5);
+
+		StagnantCheckInterval(Some(DEFAULT_STAGNANT_CHECK_INTERVAL))
+	}
+}
+
+impl StagnantCheckInterval {
+	/// Create a new stagnant-check interval wrapping the given duration.
+	pub fn new(interval: Duration) -> Self {
+		StagnantCheckInterval(Some(interval))
+	}
+
+	/// Create a `StagnantCheckInterval` which never triggers.
+	pub fn never() -> Self {
+		StagnantCheckInterval(None)
+	}
+
+	fn timeout_stream(&self) -> impl Stream<Item = ()> {
+		match self.0 {
+			Some(interval) => Either::Left({
+				let mut delay = futures_timer::Delay::new(interval);
+
+				futures::stream::poll_fn(move |cx| {
+					let poll = delay.poll_unpin(cx);
+					if poll.is_ready() {
+						delay.reset(interval)
+					}
+
+					poll.map(Some)
+				})
+			}),
+			None => Either::Right(futures::stream::pending()),
+		}
+	}
+}
+
+/// Mode of the stagnant check operations: check and prune or prune only
+#[derive(Debug, Clone)]
+pub enum StagnantCheckMode {
+	CheckAndPrune,
+	PruneOnly,
+}
+
+impl Default for StagnantCheckMode {
+	fn default() -> Self {
+		StagnantCheckMode::PruneOnly
+	}
+}
+
+/// Configuration for the chain selection subsystem.
+#[derive(Debug, Clone)]
+pub struct Config {
+	/// The column in the database that the storage should use.
+	pub col_data: u32,
+	/// How often to check for stagnant blocks.
+	pub stagnant_check_interval: StagnantCheckInterval,
+	/// Mode of stagnant checks
+	pub stagnant_check_mode: StagnantCheckMode,
+}
+
+/// The chain selection subsystem.
+pub struct ChainSelectionSubsystem {
+	config: Config,
+	db: Arc<dyn Database>,
+}
+
+impl ChainSelectionSubsystem {
+	/// Create a new instance of the subsystem with the given config
+	/// and key-value store.
+	pub fn new(config: Config, db: Arc<dyn Database>) -> Self {
+		ChainSelectionSubsystem { config, db }
+	}
+
+	/// Revert to the block corresponding to the specified `hash`.
+	/// The operation is not allowed for blocks older than the last finalized one.
+	pub fn revert_to(&self, hash: Hash) -> Result<(), Error> {
+		let config = db_backend::v1::Config { col_data: self.config.col_data };
+		let mut backend = db_backend::v1::DbBackend::new(self.db.clone(), config);
+
+		let ops = tree::revert_to(&backend, hash)?.into_write_ops();
+
+		backend.write(ops)
+	}
+}
+
+#[overseer::subsystem(ChainSelection, error = SubsystemError, prefix = self::overseer)]
+impl<Context> ChainSelectionSubsystem {
+	fn start(self, ctx: Context) -> SpawnedSubsystem {
+		let backend = db_backend::v1::DbBackend::new(
+			self.db,
+			db_backend::v1::Config { col_data: self.config.col_data },
+		);
+
+		SpawnedSubsystem {
+			future: run(
+				ctx,
+				backend,
+				self.config.stagnant_check_interval,
+				self.config.stagnant_check_mode,
+				Box::new(SystemClock),
+			)
+			.map(Ok)
+			.boxed(),
+			name: "chain-selection-subsystem",
+		}
+	}
+}
+
+#[overseer::contextbounds(ChainSelection, prefix = self::overseer)]
+async fn run<Context, B>(
+	mut ctx: Context,
+	mut backend: B,
+	stagnant_check_interval: StagnantCheckInterval,
+	stagnant_check_mode: StagnantCheckMode,
+	clock: Box<dyn Clock + Send + Sync>,
+) where
+	B: Backend,
+{
+	#![allow(clippy::all)]
+	loop {
+		let res = run_until_error(
+			&mut ctx,
+			&mut backend,
+			&stagnant_check_interval,
+			&stagnant_check_mode,
+			&*clock,
+		)
+		.await;
+		match res {
+			Err(e) => {
+				e.trace();
+				// All errors are considered fatal right now:
+				break;
+			},
+			Ok(()) => {
+				gum::info!(target: LOG_TARGET, "received `Conclude` signal, exiting");
+				break;
+			},
+		}
+	}
+}
+
+// Run the subsystem until an error is encountered or a `conclude` signal is received.
+// Most errors are non-fatal and should lead to another call to this function.
+//
+// A return value of `Ok` indicates that an exit should be made, while non-fatal errors
+// lead to another call to this function.
+#[overseer::contextbounds(ChainSelection, prefix = self::overseer)]
+async fn run_until_error<Context, B>(
+	ctx: &mut Context,
+	backend: &mut B,
+	stagnant_check_interval: &StagnantCheckInterval,
+	stagnant_check_mode: &StagnantCheckMode,
+	clock: &(dyn Clock + Sync),
+) -> Result<(), Error>
+where
+	B: Backend,
+{
+	let mut stagnant_check_stream = stagnant_check_interval.timeout_stream();
+	loop {
+		futures::select! {
+			msg = ctx.recv().fuse() => {
+				let msg = msg?;
+				match msg {
+					FromOrchestra::Signal(OverseerSignal::Conclude) => {
+						return Ok(())
+					}
+					FromOrchestra::Signal(OverseerSignal::ActiveLeaves(update)) => {
+						if let Some(leaf) = update.activated {
+							let write_ops = handle_active_leaf(
+								ctx.sender(),
+								&*backend,
+								clock.timestamp_now() + STAGNANT_TIMEOUT,
+								leaf.hash,
+							).await?;
+
+							backend.write(write_ops)?;
+						}
+					}
+					FromOrchestra::Signal(OverseerSignal::BlockFinalized(h, n)) => {
+						handle_finalized_block(backend, h, n)?
+					}
+					FromOrchestra::Communication { msg } => match msg {
+						ChainSelectionMessage::Approved(hash) => {
+							handle_approved_block(backend, hash)?
+						}
+						ChainSelectionMessage::Leaves(tx) => {
+							let leaves = load_leaves(ctx.sender(), &*backend).await?;
+							let _ = tx.send(leaves);
+						}
+						ChainSelectionMessage::BestLeafContaining(required, tx) => {
+							let best_containing = backend::find_best_leaf_containing(
+								&*backend,
+								required,
+							)?;
+
+							// note - this may be none if the finalized block is
+							// a leaf. this is fine according to the expected usage of the
+							// function. `None` responses should just `unwrap_or(required)`,
+							// so if the required block is the finalized block, then voilá.
+
+							let _ = tx.send(best_containing);
+						}
+						ChainSelectionMessage::RevertBlocks(blocks_to_revert) => {
+							let write_ops = handle_revert_blocks(backend, blocks_to_revert)?;
+							backend.write(write_ops)?;
+						}
+					}
+				}
+			}
+			_ = stagnant_check_stream.next().fuse() => {
+				match stagnant_check_mode {
+					StagnantCheckMode::CheckAndPrune => detect_stagnant(backend, clock.timestamp_now(), MAX_STAGNANT_ENTRIES),
+					StagnantCheckMode::PruneOnly => {
+						let now_timestamp = clock.timestamp_now();
+						prune_only_stagnant(backend, now_timestamp - STAGNANT_PRUNE_DELAY, MAX_STAGNANT_ENTRIES)
+					},
+				}?;
+			}
+		}
+	}
+}
+
+async fn fetch_finalized(
+	sender: &mut impl SubsystemSender<ChainApiMessage>,
+) -> Result<Option<(Hash, BlockNumber)>, Error> {
+	let (number_tx, number_rx) = oneshot::channel();
+
+	sender.send_message(ChainApiMessage::FinalizedBlockNumber(number_tx)).await;
+
+	let number = match number_rx.await? {
+		Ok(number) => number,
+		Err(err) => {
+			gum::warn!(target: LOG_TARGET, ?err, "Fetching finalized number failed");
+			return Ok(None);
+		},
+	};
+
+	let (hash_tx, hash_rx) = oneshot::channel();
+
+	sender.send_message(ChainApiMessage::FinalizedBlockHash(number, hash_tx)).await;
+
+	match hash_rx.await? {
+		Err(err) => {
+			gum::warn!(target: LOG_TARGET, number, ?err, "Fetching finalized block number failed");
+			Ok(None)
+		},
+		Ok(None) => {
+			gum::warn!(target: LOG_TARGET, number, "Missing hash for finalized block number");
+			Ok(None)
+		},
+		Ok(Some(h)) => Ok(Some((h, number))),
+	}
+}
+
+async fn fetch_header(
+	sender: &mut impl SubsystemSender<ChainApiMessage>,
+	hash: Hash,
+) -> Result<Option<Header>, Error> {
+	let (tx, rx) = oneshot::channel();
+	sender.send_message(ChainApiMessage::BlockHeader(hash, tx)).await;
+
+	Ok(rx.await?.unwrap_or_else(|err| {
+		gum::warn!(target: LOG_TARGET, ?hash, ?err, "Missing hash for finalized block number");
+		None
+	}))
+}
+
+async fn fetch_block_weight(
+	sender: &mut impl overseer::SubsystemSender<ChainApiMessage>,
+	hash: Hash,
+) -> Result<Option<BlockWeight>, Error> {
+	let (tx, rx) = oneshot::channel();
+	sender.send_message(ChainApiMessage::BlockWeight(hash, tx)).await;
+
+	let res = rx.await?;
+
+	Ok(res.unwrap_or_else(|err| {
+		gum::warn!(target: LOG_TARGET, ?hash, ?err, "Missing hash for finalized block number");
+		None
+	}))
+}
+
+// Handle a new active leaf.
+async fn handle_active_leaf(
+	sender: &mut impl overseer::ChainSelectionSenderTrait,
+	backend: &impl Backend,
+	stagnant_at: Timestamp,
+	hash: Hash,
+) -> Result<Vec<BackendWriteOp>, Error> {
+	let lower_bound = match backend.load_first_block_number()? {
+		Some(l) => {
+			// We want to iterate back to finalized, and first block number
+			// is assumed to be 1 above finalized - the implicit root of the
+			// tree.
+			l.saturating_sub(1)
+		},
+		None => fetch_finalized(sender).await?.map_or(1, |(_, n)| n),
+	};
+
+	let header = match fetch_header(sender, hash).await? {
+		None => {
+			gum::warn!(target: LOG_TARGET, ?hash, "Missing header for new head");
+			return Ok(Vec::new());
+		},
+		Some(h) => h,
+	};
+
+	let new_blocks = pezkuwi_node_subsystem_util::determine_new_blocks(
+		sender,
+		|h| backend.load_block_entry(h).map(|b| b.is_some()),
+		hash,
+		&header,
+		lower_bound,
+	)
+	.await?;
+
+	let mut overlay = OverlayedBackend::new(backend);
+
+	// determine_new_blocks gives blocks in descending order.
+	// for this, we want ascending order.
+	for (hash, header) in new_blocks.into_iter().rev() {
+		let weight = match fetch_block_weight(sender, hash).await? {
+			None => {
+				gum::warn!(
+					target: LOG_TARGET,
+					?hash,
+					"Missing block weight for new head. Skipping chain.",
+				);
+
+				// If we don't know the weight, we can't import the block.
+				// And none of its descendants either.
+				break;
+			},
+			Some(w) => w,
+		};
+
+		let reversion_logs = extract_reversion_logs(&header);
+		tree::import_block(
+			&mut overlay,
+			hash,
+			header.number,
+			header.parent_hash,
+			reversion_logs,
+			weight,
+			stagnant_at,
+		)?;
+	}
+
+	Ok(overlay.into_write_ops().collect())
+}
+
+// Extract all reversion logs from a header in ascending order.
+//
+// Ignores logs with number > the block header number.
+fn extract_reversion_logs(header: &Header) -> Vec<BlockNumber> {
+	let number = header.number;
+	let mut logs = header
+		.digest
+		.logs()
+		.iter()
+		.enumerate()
+		.filter_map(|(i, d)| match ConsensusLog::from_digest_item(d) {
+			Err(e) => {
+				gum::warn!(
+					target: LOG_TARGET,
+					err = ?e,
+					index = i,
+					block_hash = ?header.hash(),
+					"Digest item failed to encode"
+				);
+
+				None
+			},
+			Ok(Some(ConsensusLog::Revert(b))) if b <= number => Some(b),
+			Ok(Some(ConsensusLog::Revert(b))) => {
+				gum::warn!(
+					target: LOG_TARGET,
+					revert_target = b,
+					block_number = number,
+					block_hash = ?header.hash(),
+					"Block issued invalid revert digest targeting future"
+				);
+
+				None
+			},
+			Ok(_) => None,
+		})
+		.collect::<Vec<_>>();
+
+	logs.sort();
+
+	logs
+}
+
+/// Handle a finalized block event.
+fn handle_finalized_block(
+	backend: &mut impl Backend,
+	finalized_hash: Hash,
+	finalized_number: BlockNumber,
+) -> Result<(), Error> {
+	let ops = tree::finalize_block(&*backend, finalized_hash, finalized_number)?.into_write_ops();
+
+	backend.write(ops)
+}
+
+// Handle an approved block event.
+fn handle_approved_block(backend: &mut impl Backend, approved_block: Hash) -> Result<(), Error> {
+	let ops = {
+		let mut overlay = OverlayedBackend::new(&*backend);
+
+		tree::approve_block(&mut overlay, approved_block)?;
+
+		overlay.into_write_ops()
+	};
+
+	backend.write(ops)
+}
+
+// Here we revert a provided group of blocks. The most common cause for this is that
+// the dispute coordinator has notified chain selection of a dispute which concluded
+// against a candidate.
+fn handle_revert_blocks(
+	backend: &impl Backend,
+	blocks_to_revert: Vec<(BlockNumber, Hash)>,
+) -> Result<Vec<BackendWriteOp>, Error> {
+	let mut overlay = OverlayedBackend::new(backend);
+	for (block_number, block_hash) in blocks_to_revert {
+		tree::apply_single_reversion(&mut overlay, block_hash, block_number)?;
+	}
+
+	Ok(overlay.into_write_ops().collect())
+}
+
+fn detect_stagnant(
+	backend: &mut impl Backend,
+	now: Timestamp,
+	max_elements: usize,
+) -> Result<(), Error> {
+	let ops = {
+		let overlay = tree::detect_stagnant(&*backend, now, max_elements)?;
+
+		overlay.into_write_ops()
+	};
+
+	backend.write(ops)
+}
+
+fn prune_only_stagnant(
+	backend: &mut impl Backend,
+	up_to: Timestamp,
+	max_elements: usize,
+) -> Result<(), Error> {
+	let ops = {
+		let overlay = tree::prune_only_stagnant(&*backend, up_to, max_elements)?;
+
+		overlay.into_write_ops()
+	};
+
+	backend.write(ops)
+}
+
+// Load the leaves from the backend. If there are no leaves, then return
+// the finalized block.
+async fn load_leaves(
+	sender: &mut impl overseer::SubsystemSender<ChainApiMessage>,
+	backend: &impl Backend,
+) -> Result<Vec<Hash>, Error> {
+	let leaves: Vec<_> = backend.load_leaves()?.into_hashes_descending().collect();
+
+	if leaves.is_empty() {
+		Ok(fetch_finalized(sender).await?.map_or(Vec::new(), |(h, _)| vec![h]))
+	} else {
+		Ok(leaves)
+	}
+}

pezkuwi/node/core/chain-selection/src/tree.rs

@@ -0,0 +1,782 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Implements the tree-view over the data backend which we use to determine
+//! viable leaves.
+//!
+//! The metadata is structured as a tree, with the root implicitly being the
+//! finalized block, which is not stored as part of the tree.
+//!
+//! Each direct descendant of the finalized block acts as its own sub-tree,
+//! and as the finalized block advances, orphaned sub-trees are entirely pruned.
+
+use pezkuwi_node_primitives::BlockWeight;
+use pezkuwi_node_subsystem::ChainApiError;
+use pezkuwi_primitives::{BlockNumber, Hash};
+
+use std::collections::HashMap;
+
+use super::{Approval, BlockEntry, Error, LeafEntry, Timestamp, ViabilityCriteria, LOG_TARGET};
+use crate::backend::{Backend, OverlayedBackend};
+
+// A viability update to be applied to a block.
+struct ViabilityUpdate(Option<Hash>);
+
+impl ViabilityUpdate {
+	// Apply the viability update to a single block, yielding the updated
+	// block entry along with a vector of children and the updates to apply
+	// to them.
+	fn apply(self, mut entry: BlockEntry) -> (BlockEntry, Vec<(Hash, ViabilityUpdate)>) {
+		// 1. When an ancestor has changed from unviable to viable,
+		// we erase the `earliest_unviable_ancestor` of all descendants
+		// until encountering a explicitly unviable descendant D.
+		//
+		// We then update the `earliest_unviable_ancestor` for all
+		// descendants of D to be equal to D.
+		//
+		// 2. When an ancestor A has changed from viable to unviable,
+		// we update the `earliest_unviable_ancestor` for all blocks
+		// to A.
+		//
+		// The following algorithm covers both cases.
+		//
+		// Furthermore, if there has been any change in viability,
+		// it is necessary to visit every single descendant of the root
+		// block.
+		//
+		// If a block B was unviable and is now viable, then every descendant
+		// has an `earliest_unviable_ancestor` which must be updated either
+		// to nothing or to the new earliest unviable ancestor.
+		//
+		// If a block B was viable and is now unviable, then every descendant
+		// has an `earliest_unviable_ancestor` which needs to be set to B.
+
+		let maybe_earliest_unviable = self.0;
+		let next_earliest_unviable = {
+			if maybe_earliest_unviable.is_none() && !entry.viability.is_explicitly_viable() {
+				Some(entry.block_hash)
+			} else {
+				maybe_earliest_unviable
+			}
+		};
+		entry.viability.earliest_unviable_ancestor = maybe_earliest_unviable;
+
+		let recurse = entry
+			.children
+			.iter()
+			.cloned()
+			.map(move |c| (c, ViabilityUpdate(next_earliest_unviable)))
+			.collect();
+
+		(entry, recurse)
+	}
+}
+
+// Propagate viability update to descendants of the given block. This writes
+// the `base` entry as well as all descendants. If the parent of the block
+// entry is not viable, this will not affect any descendants.
+//
+// If the block entry provided is self-unviable, then it's assumed that an
+// unviability update needs to be propagated to descendants.
+//
+// If the block entry provided is self-viable, then it's assumed that a
+// viability update needs to be propagated to descendants.
+fn propagate_viability_update(
+	backend: &mut OverlayedBackend<impl Backend>,
+	base: BlockEntry,
+) -> Result<(), Error> {
+	enum BlockEntryRef {
+		Explicit(BlockEntry),
+		Hash(Hash),
+	}
+
+	if !base.viability.is_parent_viable() {
+		// If the parent of the block is still unviable,
+		// then the `earliest_viable_ancestor` will not change
+		// regardless of the change in the block here.
+		//
+		// Furthermore, in such cases, the set of viable leaves
+		// does not change at all.
+		backend.write_block_entry(base);
+		return Ok(());
+	}
+
+	let mut viable_leaves = backend.load_leaves()?;
+
+	// A mapping of Block Hash -> number
+	// Where the hash is the hash of a viable block which has
+	// at least 1 unviable child.
+	//
+	// The number is the number of known unviable children which is known
+	// as the pivot count.
+	let mut viability_pivots = HashMap::new();
+
+	// If the base block is itself explicitly unviable,
+	// this will change to a `Some(base_hash)` after the first
+	// invocation.
+	let viability_update = ViabilityUpdate(None);
+
+	// Recursively apply update to tree.
+	//
+	// As we go, we remove any blocks from the leaves which are no longer viable
+	// leaves. We also add blocks to the leaves-set which are obviously viable leaves.
+	// And we build up a frontier of blocks which may either be viable leaves or
+	// the ancestors of one.
+	let mut tree_frontier = vec![(BlockEntryRef::Explicit(base), viability_update)];
+	while let Some((entry_ref, update)) = tree_frontier.pop() {
+		let entry = match entry_ref {
+			BlockEntryRef::Explicit(entry) => entry,
+			BlockEntryRef::Hash(hash) => match backend.load_block_entry(&hash)? {
+				None => {
+					gum::warn!(
+						target: LOG_TARGET,
+						block_hash = ?hash,
+						"Missing expected block entry"
+					);
+
+					continue;
+				},
+				Some(entry) => entry,
+			},
+		};
+
+		let (new_entry, children) = update.apply(entry);
+
+		if new_entry.viability.is_viable() {
+			// A block which is viable has a parent which is obviously not
+			// in the viable leaves set.
+			viable_leaves.remove(&new_entry.parent_hash);
+
+			// Furthermore, if the block is viable and has no children,
+			// it is viable by definition.
+			if new_entry.children.is_empty() {
+				viable_leaves.insert(new_entry.leaf_entry());
+			}
+		} else {
+			// A block which is not viable is certainly not a viable leaf.
+			viable_leaves.remove(&new_entry.block_hash);
+
+			// When the parent is viable but the entry itself is not, that means
+			// that the parent is a viability pivot. As we visit the children
+			// of a viability pivot, we build up an exhaustive pivot count.
+			if new_entry.viability.is_parent_viable() {
+				*viability_pivots.entry(new_entry.parent_hash).or_insert(0) += 1;
+			}
+		}
+
+		backend.write_block_entry(new_entry);
+
+		tree_frontier
+			.extend(children.into_iter().map(|(h, update)| (BlockEntryRef::Hash(h), update)));
+	}
+
+	// Revisit the viability pivots now that we've traversed the entire subtree.
+	// After this point, the viable leaves set is fully updated. A proof follows.
+	//
+	// If the base has become unviable, then we've iterated into all descendants,
+	// made them unviable and removed them from the set. We know that the parent is
+	// viable as this function is a no-op otherwise, so we need to see if the parent
+	// has other children or not.
+	//
+	// If the base has become viable, then we've iterated into all descendants,
+	// and found all blocks which are viable and have no children. We've already added
+	// those blocks to the leaf set, but what we haven't detected
+	// is blocks which are viable and have children, but all of the children are
+	// unviable.
+	//
+	// The solution of viability pivots addresses both of these:
+	//
+	// When the base has become unviable, the parent's viability is unchanged and therefore
+	// any leaves descending from parent but not base are still in the viable leaves set.
+	// If the parent has only one child which is the base, the parent is now a viable leaf.
+	// We've already visited the base in recursive search so the set of pivots should
+	// contain only a single entry `(parent, 1)`. qed.
+	//
+	// When the base has become viable, we've already iterated into every descendant
+	// of the base and thus have collected a set of pivots whose corresponding pivot
+	// counts have already been exhaustively computed from their children. qed.
+	for (pivot, pivot_count) in viability_pivots {
+		match backend.load_block_entry(&pivot)? {
+			None => {
+				// This means the block is finalized. We might reach this
+				// code path when the base is a child of the finalized block
+				// and has become unviable.
+				//
+				// Each such child is the root of its own tree
+				// which, as an invariant, does not depend on the viability
+				// of the finalized block. So no siblings need to be inspected
+				// and we can ignore it safely.
+				//
+				// Furthermore, if the set of viable leaves is empty, the
+				// finalized block is implicitly the viable leaf.
+				continue;
+			},
+			Some(entry) =>
+				if entry.children.len() == pivot_count {
+					viable_leaves.insert(entry.leaf_entry());
+				},
+		}
+	}
+
+	backend.write_leaves(viable_leaves);
+
+	Ok(())
+}
+
+/// Imports a new block and applies any reversions to ancestors or the block itself.
+pub(crate) fn import_block(
+	backend: &mut OverlayedBackend<impl Backend>,
+	block_hash: Hash,
+	block_number: BlockNumber,
+	parent_hash: Hash,
+	reversion_logs: Vec<BlockNumber>,
+	weight: BlockWeight,
+	stagnant_at: Timestamp,
+) -> Result<(), Error> {
+	let block_entry =
+		add_block(backend, block_hash, block_number, parent_hash, weight, stagnant_at)?;
+	apply_reversions(backend, block_entry, reversion_logs)?;
+
+	Ok(())
+}
+
+// Load the given ancestor's block entry, in descending order from the `block_hash`.
+// The ancestor_number must be not higher than the `block_entry`'s.
+//
+// The returned entry will be `None` if the range is invalid or any block in the path had
+// no entry present. If any block entry was missing, it can safely be assumed to
+// be finalized.
+fn load_ancestor(
+	backend: &mut OverlayedBackend<impl Backend>,
+	block_entry: &BlockEntry,
+	ancestor_number: BlockNumber,
+) -> Result<Option<BlockEntry>, Error> {
+	let block_hash = block_entry.block_hash;
+	let block_number = block_entry.block_number;
+	if block_number == ancestor_number {
+		return Ok(Some(block_entry.clone()));
+	} else if block_number < ancestor_number {
+		return Ok(None);
+	}
+
+	let mut current_hash = block_hash;
+	let mut current_entry = None;
+
+	let segment_length = (block_number - ancestor_number) + 1;
+	for _ in 0..segment_length {
+		match backend.load_block_entry(&current_hash)? {
+			None => return Ok(None),
+			Some(entry) => {
+				let parent_hash = entry.parent_hash;
+				current_entry = Some(entry);
+				current_hash = parent_hash;
+			},
+		}
+	}
+
+	// Current entry should always be `Some` here.
+	Ok(current_entry)
+}
+
+// Add a new block to the tree, which is assumed to be unreverted and unapproved,
+// but not stagnant. It inherits viability from its parent, if any.
+//
+// This updates the parent entry, if any, and updates the viable leaves set accordingly.
+// This also schedules a stagnation-check update and adds the block to the blocks-by-number
+// mapping.
+fn add_block(
+	backend: &mut OverlayedBackend<impl Backend>,
+	block_hash: Hash,
+	block_number: BlockNumber,
+	parent_hash: Hash,
+	weight: BlockWeight,
+	stagnant_at: Timestamp,
+) -> Result<BlockEntry, Error> {
+	let mut leaves = backend.load_leaves()?;
+	let parent_entry = backend.load_block_entry(&parent_hash)?;
+
+	let inherited_viability =
+		parent_entry.as_ref().and_then(|parent| parent.non_viable_ancestor_for_child());
+
+	// 1. Add the block to the DB assuming it's not reverted.
+	let block_entry = BlockEntry {
+		block_hash,
+		block_number,
+		parent_hash,
+		children: Vec::new(),
+		viability: ViabilityCriteria {
+			earliest_unviable_ancestor: inherited_viability,
+			explicitly_reverted: false,
+			approval: Approval::Unapproved,
+		},
+		weight,
+	};
+	backend.write_block_entry(block_entry.clone());
+
+	// 2. Update leaves if inherited viability is fine.
+	if inherited_viability.is_none() {
+		leaves.remove(&parent_hash);
+		leaves.insert(LeafEntry { block_hash, block_number, weight });
+		backend.write_leaves(leaves);
+	}
+
+	// 3. Update and write the parent
+	if let Some(mut parent_entry) = parent_entry {
+		parent_entry.children.push(block_hash);
+		backend.write_block_entry(parent_entry);
+	}
+
+	// 4. Add to blocks-by-number.
+	let mut blocks_by_number = backend.load_blocks_by_number(block_number)?;
+	blocks_by_number.push(block_hash);
+	backend.write_blocks_by_number(block_number, blocks_by_number);
+
+	// 5. Add stagnation timeout.
+	let mut stagnant_at_list = backend.load_stagnant_at(stagnant_at)?;
+	stagnant_at_list.push(block_hash);
+	backend.write_stagnant_at(stagnant_at, stagnant_at_list);
+
+	Ok(block_entry)
+}
+
+/// Assuming that a block is already imported, accepts the number of the block
+/// as well as a list of reversions triggered by the block in ascending order.
+fn apply_reversions(
+	backend: &mut OverlayedBackend<impl Backend>,
+	block_entry: BlockEntry,
+	reversions: Vec<BlockNumber>,
+) -> Result<(), Error> {
+	// Note: since revert numbers are  in ascending order, the expensive propagation
+	// of unviability is only heavy on the first log.
+	for revert_number in reversions {
+		let maybe_block_entry = load_ancestor(backend, &block_entry, revert_number)?;
+		if let Some(entry) = &maybe_block_entry {
+			gum::trace!(
+				target: LOG_TARGET,
+				?revert_number,
+				revert_hash = ?entry.block_hash,
+				"Block marked as reverted via scraped on-chain reversions"
+			);
+		}
+		revert_single_block_entry_if_present(
+			backend,
+			maybe_block_entry,
+			None,
+			revert_number,
+			Some(block_entry.block_hash),
+			Some(block_entry.block_number),
+		)?;
+	}
+
+	Ok(())
+}
+
+/// Marks a single block as explicitly reverted, then propagates viability updates
+/// to all its children. This is triggered when the disputes subsystem signals that
+/// a dispute has concluded against a candidate.
+pub(crate) fn apply_single_reversion(
+	backend: &mut OverlayedBackend<impl Backend>,
+	revert_hash: Hash,
+	revert_number: BlockNumber,
+) -> Result<(), Error> {
+	gum::trace!(
+		target: LOG_TARGET,
+		?revert_number,
+		?revert_hash,
+		"Block marked as reverted via ChainSelectionMessage::RevertBlocks"
+	);
+	let maybe_block_entry = backend.load_block_entry(&revert_hash)?;
+	revert_single_block_entry_if_present(
+		backend,
+		maybe_block_entry,
+		Some(revert_hash),
+		revert_number,
+		None,
+		None,
+	)?;
+	Ok(())
+}
+
+fn revert_single_block_entry_if_present(
+	backend: &mut OverlayedBackend<impl Backend>,
+	maybe_block_entry: Option<BlockEntry>,
+	maybe_revert_hash: Option<Hash>,
+	revert_number: BlockNumber,
+	maybe_reporting_hash: Option<Hash>,
+	maybe_reporting_number: Option<BlockNumber>,
+) -> Result<(), Error> {
+	match maybe_block_entry {
+		None => {
+			gum::warn!(
+				target: LOG_TARGET,
+				?maybe_revert_hash,
+				revert_target = revert_number,
+				?maybe_reporting_hash,
+				?maybe_reporting_number,
+				"The hammer has dropped. \
+				The protocol has indicated that a finalized block be reverted. \
+				Please inform an adult.",
+			);
+		},
+		Some(mut block_entry) => {
+			gum::info!(
+				target: LOG_TARGET,
+				?maybe_revert_hash,
+				revert_target = revert_number,
+				?maybe_reporting_hash,
+				?maybe_reporting_number,
+				"Unfinalized block reverted due to a bad teyrchain block.",
+			);
+
+			block_entry.viability.explicitly_reverted = true;
+			// Marks children of reverted block as non-viable
+			propagate_viability_update(backend, block_entry)?;
+		},
+	}
+	Ok(())
+}
+
+/// Finalize a block with the given number and hash.
+///
+/// This will prune all sub-trees not descending from the given block,
+/// all block entries at or before the given height,
+/// and will update the viability of all sub-trees descending from the given
+/// block if the finalized block was not viable.
+///
+/// This is assumed to start with a fresh backend, and will produce
+/// an overlay over the backend with all the changes applied.
+pub(super) fn finalize_block<'a, B: Backend + 'a>(
+	backend: &'a B,
+	finalized_hash: Hash,
+	finalized_number: BlockNumber,
+) -> Result<OverlayedBackend<'a, B>, Error> {
+	let earliest_stored_number = backend.load_first_block_number()?;
+	let mut backend = OverlayedBackend::new(backend);
+
+	let earliest_stored_number = match earliest_stored_number {
+		None => {
+			// This implies that there are no unfinalized blocks and hence nothing
+			// to update.
+			return Ok(backend);
+		},
+		Some(e) => e,
+	};
+
+	let mut viable_leaves = backend.load_leaves()?;
+
+	// Walk all numbers up to the finalized number and remove those entries.
+	for number in earliest_stored_number..finalized_number {
+		let blocks_at = backend.load_blocks_by_number(number)?;
+		backend.delete_blocks_by_number(number);
+
+		for block in blocks_at {
+			viable_leaves.remove(&block);
+			backend.delete_block_entry(&block);
+		}
+	}
+
+	// Remove all blocks at the finalized height, with the exception of the finalized block,
+	// and their descendants, recursively.
+	{
+		let blocks_at_finalized_height = backend.load_blocks_by_number(finalized_number)?;
+		backend.delete_blocks_by_number(finalized_number);
+
+		let mut frontier: Vec<_> = blocks_at_finalized_height
+			.into_iter()
+			.filter(|h| h != &finalized_hash)
+			.map(|h| (h, finalized_number))
+			.collect();
+
+		while let Some((dead_hash, dead_number)) = frontier.pop() {
+			let entry = backend.load_block_entry(&dead_hash)?;
+			backend.delete_block_entry(&dead_hash);
+			viable_leaves.remove(&dead_hash);
+
+			// This does a few extra `clone`s but is unlikely to be
+			// a bottleneck. Code complexity is very low as a result.
+			let mut blocks_at_height = backend.load_blocks_by_number(dead_number)?;
+			blocks_at_height.retain(|h| h != &dead_hash);
+			backend.write_blocks_by_number(dead_number, blocks_at_height);
+
+			// Add all children to the frontier.
+			let next_height = dead_number + 1;
+			frontier.extend(entry.into_iter().flat_map(|e| e.children).map(|h| (h, next_height)));
+		}
+	}
+
+	// Visit and remove the finalized block, fetching its children.
+	let children_of_finalized = {
+		let finalized_entry = backend.load_block_entry(&finalized_hash)?;
+		backend.delete_block_entry(&finalized_hash);
+		viable_leaves.remove(&finalized_hash);
+
+		finalized_entry.into_iter().flat_map(|e| e.children)
+	};
+
+	backend.write_leaves(viable_leaves);
+
+	// Update the viability of each child.
+	for child in children_of_finalized {
+		if let Some(mut child) = backend.load_block_entry(&child)? {
+			// Finalized blocks are always viable.
+			child.viability.earliest_unviable_ancestor = None;
+
+			propagate_viability_update(&mut backend, child)?;
+		} else {
+			gum::debug!(
+				target: LOG_TARGET,
+				?finalized_hash,
+				finalized_number,
+				child_hash = ?child,
+				"Missing child of finalized block",
+			);
+
+			// No need to do anything, but this is an inconsistent state.
+		}
+	}
+
+	Ok(backend)
+}
+
+/// Mark a block as approved and update the viability of itself and its
+/// descendants accordingly.
+pub(super) fn approve_block(
+	backend: &mut OverlayedBackend<impl Backend>,
+	approved_hash: Hash,
+) -> Result<(), Error> {
+	if let Some(mut entry) = backend.load_block_entry(&approved_hash)? {
+		let was_viable = entry.viability.is_viable();
+		entry.viability.approval = Approval::Approved;
+		let is_viable = entry.viability.is_viable();
+
+		// Approval can change the viability in only one direction.
+		// If the viability has changed, then we propagate that to children
+		// and recalculate the viable leaf set.
+		if !was_viable && is_viable {
+			propagate_viability_update(backend, entry)?;
+		} else {
+			backend.write_block_entry(entry);
+		}
+	} else {
+		gum::debug!(
+			target: LOG_TARGET,
+			block_hash = ?approved_hash,
+			"Missing entry for freshly-approved block. Ignoring"
+		);
+	}
+
+	Ok(())
+}
+
+/// Check whether any blocks up to the given timestamp are stagnant and update
+/// accordingly.
+///
+/// This accepts a fresh backend and returns an overlay on top of it representing
+/// all changes made.
+pub(super) fn detect_stagnant<'a, B: 'a + Backend>(
+	backend: &'a B,
+	up_to: Timestamp,
+	max_elements: usize,
+) -> Result<OverlayedBackend<'a, B>, Error> {
+	let stagnant_up_to = backend.load_stagnant_at_up_to(up_to, max_elements)?;
+	let mut backend = OverlayedBackend::new(backend);
+
+	let (min_ts, max_ts) = match stagnant_up_to.len() {
+		0 => (0 as Timestamp, 0 as Timestamp),
+		1 => (stagnant_up_to[0].0, stagnant_up_to[0].0),
+		n => (stagnant_up_to[0].0, stagnant_up_to[n - 1].0),
+	};
+
+	// As this is in ascending order, only the earliest stagnant
+	// blocks will involve heavy viability propagations.
+	gum::debug!(
+		target: LOG_TARGET,
+		?up_to,
+		?min_ts,
+		?max_ts,
+		"Prepared {} stagnant entries for checking/pruning",
+		stagnant_up_to.len()
+	);
+
+	for (timestamp, maybe_stagnant) in stagnant_up_to {
+		backend.delete_stagnant_at(timestamp);
+
+		for block_hash in maybe_stagnant {
+			if let Some(mut entry) = backend.load_block_entry(&block_hash)? {
+				let was_viable = entry.viability.is_viable();
+				if let Approval::Unapproved = entry.viability.approval {
+					entry.viability.approval = Approval::Stagnant;
+				}
+				let is_viable = entry.viability.is_viable();
+				gum::trace!(
+					target: LOG_TARGET,
+					?block_hash,
+					?timestamp,
+					?was_viable,
+					?is_viable,
+					"Found existing stagnant entry"
+				);
+
+				if was_viable && !is_viable {
+					propagate_viability_update(&mut backend, entry)?;
+				} else {
+					backend.write_block_entry(entry);
+				}
+			} else {
+				gum::trace!(
+					target: LOG_TARGET,
+					?block_hash,
+					?timestamp,
+					"Found non-existing stagnant entry"
+				);
+			}
+		}
+	}
+
+	Ok(backend)
+}
+
+/// Prune stagnant entries at some timestamp without other checks
+/// This function is intended just to clean leftover entries when the real
+/// stagnant checks are disabled
+pub(super) fn prune_only_stagnant<'a, B: 'a + Backend>(
+	backend: &'a B,
+	up_to: Timestamp,
+	max_elements: usize,
+) -> Result<OverlayedBackend<'a, B>, Error> {
+	let stagnant_up_to = backend.load_stagnant_at_up_to(up_to, max_elements)?;
+	let mut backend = OverlayedBackend::new(backend);
+
+	let (min_ts, max_ts) = match stagnant_up_to.len() {
+		0 => (0 as Timestamp, 0 as Timestamp),
+		1 => (stagnant_up_to[0].0, stagnant_up_to[0].0),
+		n => (stagnant_up_to[0].0, stagnant_up_to[n - 1].0),
+	};
+
+	gum::debug!(
+		target: LOG_TARGET,
+		?up_to,
+		?min_ts,
+		?max_ts,
+		"Prepared {} stagnant entries for pruning",
+		stagnant_up_to.len()
+	);
+
+	for (timestamp, _) in stagnant_up_to {
+		backend.delete_stagnant_at(timestamp);
+	}
+
+	Ok(backend)
+}
+
+/// Revert the tree to the block relative to `hash`.
+///
+/// This accepts a fresh backend and returns an overlay on top of it representing
+/// all changes made.
+pub(super) fn revert_to<'a, B: Backend + 'a>(
+	backend: &'a B,
+	hash: Hash,
+) -> Result<OverlayedBackend<'a, B>, Error> {
+	let first_number = backend.load_first_block_number()?.unwrap_or_default();
+
+	let mut backend = OverlayedBackend::new(backend);
+
+	let mut entry = match backend.load_block_entry(&hash)? {
+		Some(entry) => entry,
+		None => {
+			// May be a revert to the last finalized block. If this is the case,
+			// then revert to this block should be handled specially since no
+			// information about finalized blocks is persisted within the tree.
+			//
+			// We use part of the information contained in the finalized block
+			// children (that are expected to be in the tree) to construct a
+			// dummy block entry for the last finalized block. This will be
+			// wiped as soon as the next block is finalized.
+
+			let blocks = backend.load_blocks_by_number(first_number)?;
+
+			let block = blocks
+				.first()
+				.and_then(|hash| backend.load_block_entry(hash).ok())
+				.flatten()
+				.ok_or_else(|| {
+					ChainApiError::from(format!(
+						"Lookup failure for block at height {}",
+						first_number
+					))
+				})?;
+
+			// The parent is expected to be the last finalized block.
+			if block.parent_hash != hash {
+				return Err(ChainApiError::from("Can't revert below last finalized block").into());
+			}
+
+			// The weight is set to the one of the first child. Even though this is
+			// not accurate, it does the job. The reason is that the revert point is
+			// the last finalized block, i.e. this is the best and only choice.
+			let block_number = first_number.saturating_sub(1);
+			let viability = ViabilityCriteria {
+				explicitly_reverted: false,
+				approval: Approval::Approved,
+				earliest_unviable_ancestor: None,
+			};
+			let entry = BlockEntry {
+				block_hash: hash,
+				block_number,
+				parent_hash: Hash::default(),
+				children: blocks,
+				viability,
+				weight: block.weight,
+			};
+			// This becomes the first entry according to the block number.
+			backend.write_blocks_by_number(block_number, vec![hash]);
+			entry
+		},
+	};
+
+	let mut stack: Vec<_> = std::mem::take(&mut entry.children)
+		.into_iter()
+		.map(|h| (h, entry.block_number + 1))
+		.collect();
+
+	// Write revert point block entry without the children.
+	backend.write_block_entry(entry.clone());
+
+	let mut viable_leaves = backend.load_leaves()?;
+
+	viable_leaves.insert(LeafEntry {
+		block_hash: hash,
+		block_number: entry.block_number,
+		weight: entry.weight,
+	});
+
+	while let Some((hash, number)) = stack.pop() {
+		let entry = backend.load_block_entry(&hash)?;
+		backend.delete_block_entry(&hash);
+
+		viable_leaves.remove(&hash);
+
+		let mut blocks_at_height = backend.load_blocks_by_number(number)?;
+		blocks_at_height.retain(|h| h != &hash);
+		backend.write_blocks_by_number(number, blocks_at_height);
+
+		stack.extend(entry.into_iter().flat_map(|e| e.children).map(|h| (h, number + 1)));
+	}
+
+	backend.write_leaves(viable_leaves);
+
+	Ok(backend)
+}

pezkuwi/node/core/dispute-coordinator/Cargo.toml

@@ -0,0 +1,64 @@
+[package]
+name = "pezkuwi-node-core-dispute-coordinator"
+version = "7.0.0"
+description = "The node-side components that participate in disputes"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+
+[lints]
+workspace = true
+
+[[bench]]
+name = "dispute-coordinator-regression-bench"
+path = "benches/dispute-coordinator-regression-bench.rs"
+harness = false
+required-features = ["subsystem-benchmarks"]
+
+[dependencies]
+codec = { workspace = true, default-features = true }
+fatality = { workspace = true }
+futures = { workspace = true }
+gum = { workspace = true, default-features = true }
+schnellru = { workspace = true }
+thiserror = { workspace = true }
+
+pezkuwi-node-primitives = { workspace = true, default-features = true }
+pezkuwi-node-subsystem = { workspace = true, default-features = true }
+pezkuwi-node-subsystem-util = { workspace = true, default-features = true }
+pezkuwi-primitives = { workspace = true, default-features = true }
+
+sc-keystore = { workspace = true, default-features = true }
+
+[dev-dependencies]
+assert_matches = { workspace = true }
+futures-timer = { workspace = true }
+kvdb-memorydb = { workspace = true }
+pezkuwi-node-subsystem-test-helpers = { workspace = true }
+pezkuwi-primitives = { workspace = true, features = ["test"] }
+pezkuwi-primitives-test-helpers = { workspace = true }
+sp-application-crypto = { workspace = true, default-features = true }
+sp-core = { workspace = true, default-features = true }
+sp-keyring = { workspace = true, default-features = true }
+sp-keystore = { workspace = true, default-features = true }
+sp-tracing = { workspace = true, default-features = true }
+
+pezkuwi-subsystem-bench = { workspace = true }
+
+[features]
+# If not enabled, the dispute coordinator will do nothing.
+disputes = []
+subsystem-benchmarks = []
+runtime-benchmarks = [
+	"gum/runtime-benchmarks",
+	"pezkuwi-node-primitives/runtime-benchmarks",
+	"pezkuwi-node-subsystem-test-helpers/runtime-benchmarks",
+	"pezkuwi-node-subsystem-util/runtime-benchmarks",
+	"pezkuwi-node-subsystem/runtime-benchmarks",
+	"pezkuwi-primitives-test-helpers/runtime-benchmarks",
+	"pezkuwi-primitives/runtime-benchmarks",
+	"pezkuwi-subsystem-bench/runtime-benchmarks",
+	"sp-keyring/runtime-benchmarks",
+]

pezkuwi/node/core/dispute-coordinator/benches/dispute-coordinator-regression-bench.rs

@@ -0,0 +1,86 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! dispute-coordinator throughput test
+//!
+//! Dispute Coordinator benchmark based on Kusama parameters and scale.
+//!
+//! Subsystems involved:
+//! - dispute-coordinator
+//! - dispute-distribution
+
+use pezkuwi_subsystem_bench::{
+	configuration::TestConfiguration,
+	disputes::{benchmark_dispute_coordinator, prepare_test, DisputesOptions, TestState},
+	usage::BenchmarkUsage,
+	utils::save_to_file,
+};
+use std::io::Write;
+
+const BENCH_COUNT: usize = 10;
+
+fn main() -> Result<(), String> {
+	let mut messages = vec![];
+	let mut config = TestConfiguration::default();
+	config.n_cores = 100;
+	config.n_validators = 500;
+	config.num_blocks = 10;
+	config.peer_bandwidth = 524288000000;
+	config.bandwidth = 524288000000;
+	config.latency = None;
+	config.connectivity = 100;
+	config.generate_pov_sizes();
+	let options = DisputesOptions { n_disputes: 50 };
+
+	println!("Benchmarking...");
+	let usages: Vec<BenchmarkUsage> = (0..BENCH_COUNT)
+		.map(|n| {
+			print!("\r[{}{}]", "#".repeat(n), "_".repeat(BENCH_COUNT - n));
+			std::io::stdout().flush().unwrap();
+			let state = TestState::new(&config, &options);
+			let mut env = prepare_test(&state, false);
+			env.runtime().block_on(benchmark_dispute_coordinator(&mut env, &state))
+		})
+		.collect();
+	println!("\rDone!{}", " ".repeat(BENCH_COUNT));
+
+	let average_usage = BenchmarkUsage::average(&usages);
+	save_to_file(
+		"charts/dispute-coordinator-regression-bench.json",
+		average_usage.to_chart_json().map_err(|e| e.to_string())?,
+	)
+	.map_err(|e| e.to_string())?;
+	println!("{}", average_usage);
+
+	// We expect some small variance for received and sent because the
+	// test messages are generated at every benchmark run and they contain
+	// random data so use 0.01 as the accepted variance.
+	messages.extend(average_usage.check_network_usage(&[
+		("Received from peers", 23.8, 0.01),
+		("Sent to peers", 227.1, 0.01),
+	]));
+	messages.extend(average_usage.check_cpu_usage(&[
+		("dispute-coordinator", 0.0026, 0.1),
+		("dispute-distribution", 0.0086, 0.1),
+	]));
+
+	if messages.is_empty() {
+		Ok(())
+	} else {
+		eprintln!("{}", messages.join("\n"));
+		Err("Regressions found".to_string())
+	}
+}

pezkuwi/node/core/dispute-coordinator/src/backend.rs

@@ -0,0 +1,171 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! An abstraction over storage used by the chain selection subsystem.
+//!
+//! This provides both a [`Backend`] trait and an [`OverlayedBackend`]
+//! struct which allows in-memory changes to be applied on top of a
+//! [`Backend`], maintaining consistency between queries and temporary writes,
+//! before any commit to the underlying storage is made.
+
+use pezkuwi_primitives::{CandidateHash, SessionIndex};
+
+use std::collections::HashMap;
+
+use super::db::v1::{CandidateVotes, RecentDisputes};
+use crate::error::FatalResult;
+
+#[derive(Debug)]
+pub enum BackendWriteOp {
+	WriteEarliestSession(SessionIndex),
+	WriteRecentDisputes(RecentDisputes),
+	WriteCandidateVotes(SessionIndex, CandidateHash, CandidateVotes),
+	DeleteCandidateVotes(SessionIndex, CandidateHash),
+}
+
+/// An abstraction over backend storage for the logic of this subsystem.
+pub trait Backend {
+	/// Load the earliest session, if any.
+	fn load_earliest_session(&self) -> FatalResult<Option<SessionIndex>>;
+
+	/// Load the recent disputes, if any.
+	fn load_recent_disputes(&self) -> FatalResult<Option<RecentDisputes>>;
+
+	/// Load the candidate votes for the specific session-candidate pair, if any.
+	fn load_candidate_votes(
+		&self,
+		session: SessionIndex,
+		candidate_hash: &CandidateHash,
+	) -> FatalResult<Option<CandidateVotes>>;
+
+	/// Atomically writes the list of operations, with later operations taking precedence over
+	/// prior.
+	fn write<I>(&mut self, ops: I) -> FatalResult<()>
+	where
+		I: IntoIterator<Item = BackendWriteOp>;
+}
+
+/// An in-memory overlay for the backend.
+///
+/// This maintains read-only access to the underlying backend, but can be converted into a set of
+/// write operations which will, when written to the underlying backend, give the same view as the
+/// state of the overlay.
+pub struct OverlayedBackend<'a, B: 'a> {
+	inner: &'a B,
+
+	// `None` means unchanged.
+	earliest_session: Option<SessionIndex>,
+	// `None` means unchanged.
+	recent_disputes: Option<RecentDisputes>,
+	// `None` means deleted, missing means query inner.
+	candidate_votes: HashMap<(SessionIndex, CandidateHash), Option<CandidateVotes>>,
+}
+
+impl<'a, B: 'a + Backend> OverlayedBackend<'a, B> {
+	pub fn new(backend: &'a B) -> Self {
+		Self {
+			inner: backend,
+			earliest_session: None,
+			recent_disputes: None,
+			candidate_votes: HashMap::new(),
+		}
+	}
+
+	/// Returns true if the are no write operations to perform.
+	pub fn is_empty(&self) -> bool {
+		self.earliest_session.is_none() &&
+			self.recent_disputes.is_none() &&
+			self.candidate_votes.is_empty()
+	}
+
+	/// Load the earliest session, if any.
+	pub fn load_earliest_session(&self) -> FatalResult<Option<SessionIndex>> {
+		if let Some(val) = self.earliest_session {
+			return Ok(Some(val));
+		}
+
+		self.inner.load_earliest_session()
+	}
+
+	/// Load the recent disputes, if any.
+	pub fn load_recent_disputes(&self) -> FatalResult<Option<RecentDisputes>> {
+		if let Some(val) = &self.recent_disputes {
+			return Ok(Some(val.clone()));
+		}
+
+		self.inner.load_recent_disputes()
+	}
+
+	/// Load the candidate votes for the specific session-candidate pair, if any.
+	pub fn load_candidate_votes(
+		&self,
+		session: SessionIndex,
+		candidate_hash: &CandidateHash,
+	) -> FatalResult<Option<CandidateVotes>> {
+		if let Some(val) = self.candidate_votes.get(&(session, *candidate_hash)) {
+			return Ok(val.clone());
+		}
+
+		self.inner.load_candidate_votes(session, candidate_hash)
+	}
+
+	/// Prepare a write to the "earliest session" field of the DB.
+	///
+	/// Later calls to this function will override earlier ones.
+	pub fn write_earliest_session(&mut self, session: SessionIndex) {
+		self.earliest_session = Some(session);
+	}
+
+	/// Prepare a write to the recent disputes stored in the DB.
+	///
+	/// Later calls to this function will override earlier ones.
+	pub fn write_recent_disputes(&mut self, recent_disputes: RecentDisputes) {
+		self.recent_disputes = Some(recent_disputes)
+	}
+
+	/// Prepare a write of the candidate votes under the indicated candidate.
+	///
+	/// Later calls to this function for the same candidate will override earlier ones.
+	pub fn write_candidate_votes(
+		&mut self,
+		session: SessionIndex,
+		candidate_hash: CandidateHash,
+		votes: CandidateVotes,
+	) {
+		self.candidate_votes.insert((session, candidate_hash), Some(votes));
+	}
+
+	/// Transform this backend into a set of write-ops to be written to the inner backend.
+	pub fn into_write_ops(self) -> impl Iterator<Item = BackendWriteOp> {
+		let earliest_session_ops = self
+			.earliest_session
+			.map(|s| BackendWriteOp::WriteEarliestSession(s))
+			.into_iter();
+
+		let recent_dispute_ops =
+			self.recent_disputes.map(|d| BackendWriteOp::WriteRecentDisputes(d)).into_iter();
+
+		let candidate_vote_ops =
+			self.candidate_votes
+				.into_iter()
+				.map(|((session, candidate), votes)| match votes {
+					Some(votes) => BackendWriteOp::WriteCandidateVotes(session, candidate, votes),
+					None => BackendWriteOp::DeleteCandidateVotes(session, candidate),
+				});
+
+		earliest_session_ops.chain(recent_dispute_ops).chain(candidate_vote_ops)
+	}
+}

pezkuwi/node/core/dispute-coordinator/src/db/mod.rs

@@ -0,0 +1,19 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Database component for the dispute coordinator.
+
+pub(super) mod v1;

pezkuwi/node/core/dispute-coordinator/src/db/v1.rs

@@ -0,0 +1,689 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! `V1` database for the dispute coordinator.
+//!
+//! Note that the version here differs from the actual version of the teyrchains
+//! database (check `CURRENT_VERSION` in `node/service/src/teyrchains_db/upgrade.rs`).
+//! The code in this module implements the way dispute coordinator works with
+//! the dispute data in the database. Any breaking changes here will still
+//! require a db migration (check `node/service/src/teyrchains_db/upgrade.rs`).
+
+use pezkuwi_node_primitives::DisputeStatus;
+use pezkuwi_node_subsystem_util::database::{DBTransaction, Database};
+use pezkuwi_primitives::{
+	CandidateHash, CandidateReceiptV2 as CandidateReceipt, Hash, InvalidDisputeStatementKind,
+	SessionIndex, ValidDisputeStatementKind, ValidatorIndex, ValidatorSignature,
+};
+
+use std::sync::Arc;
+
+use codec::{Decode, Encode};
+
+use crate::{
+	backend::{Backend, BackendWriteOp, OverlayedBackend},
+	error::{FatalError, FatalResult},
+	metrics::Metrics,
+	LOG_TARGET,
+};
+
+const RECENT_DISPUTES_KEY: &[u8; 15] = b"recent-disputes";
+const EARLIEST_SESSION_KEY: &[u8; 16] = b"earliest-session";
+const CANDIDATE_VOTES_SUBKEY: &[u8; 15] = b"candidate-votes";
+/// Until what session have votes been cleaned up already?
+const CLEANED_VOTES_WATERMARK_KEY: &[u8; 23] = b"cleaned-votes-watermark";
+
+/// Restrict number of cleanup operations.
+///
+/// On the first run we are starting at session 0 going up all the way to the current session -
+/// this should not be done at once, but rather in smaller batches so nodes won't get stalled by
+/// this.
+///
+/// 300 is with session duration of 1 hour and 30 teyrchains around <3_000_000 key purges in the
+/// worst case. Which is already quite a lot, at the same time we have around 21_000 sessions on
+/// Kusama. This means at 300 purged sessions per session, cleaning everything up will take
+/// around 3 days. Depending on how severe disk usage becomes, we might want to bump the batch
+/// size, at the cost of risking issues at session boundaries (performance).
+#[cfg(test)]
+const MAX_CLEAN_BATCH_SIZE: u32 = 10;
+#[cfg(not(test))]
+const MAX_CLEAN_BATCH_SIZE: u32 = 300;
+
+pub struct DbBackend {
+	inner: Arc<dyn Database>,
+	config: ColumnConfiguration,
+	metrics: Metrics,
+}
+
+impl DbBackend {
+	pub fn new(db: Arc<dyn Database>, config: ColumnConfiguration, metrics: Metrics) -> Self {
+		Self { inner: db, config, metrics }
+	}
+
+	/// Cleanup old votes.
+	///
+	/// Should be called whenever a new earliest session gets written.
+	fn add_vote_cleanup_tx(
+		&mut self,
+		tx: &mut DBTransaction,
+		earliest_session: SessionIndex,
+	) -> FatalResult<()> {
+		// Cleanup old votes in db:
+		let watermark = load_cleaned_votes_watermark(&*self.inner, &self.config)?.unwrap_or(0);
+		let clean_until = if earliest_session.saturating_sub(watermark) > MAX_CLEAN_BATCH_SIZE {
+			watermark + MAX_CLEAN_BATCH_SIZE
+		} else {
+			earliest_session
+		};
+		gum::trace!(
+			target: LOG_TARGET,
+			?watermark,
+			?clean_until,
+			?earliest_session,
+			?MAX_CLEAN_BATCH_SIZE,
+			"WriteEarliestSession"
+		);
+
+		for index in watermark..clean_until {
+			gum::trace!(
+			target: LOG_TARGET,
+			?index,
+			encoded = ?candidate_votes_session_prefix(index),
+			"Cleaning votes for session index"
+			);
+			tx.delete_prefix(self.config.col_dispute_data, &candidate_votes_session_prefix(index));
+		}
+		// New watermark:
+		tx.put_vec(self.config.col_dispute_data, CLEANED_VOTES_WATERMARK_KEY, clean_until.encode());
+		Ok(())
+	}
+}
+
+impl Backend for DbBackend {
+	/// Load the earliest session, if any.
+	fn load_earliest_session(&self) -> FatalResult<Option<SessionIndex>> {
+		load_earliest_session(&*self.inner, &self.config)
+	}
+
+	/// Load the recent disputes, if any.
+	fn load_recent_disputes(&self) -> FatalResult<Option<RecentDisputes>> {
+		load_recent_disputes(&*self.inner, &self.config)
+	}
+
+	/// Load the candidate votes for the specific session-candidate pair, if any.
+	fn load_candidate_votes(
+		&self,
+		session: SessionIndex,
+		candidate_hash: &CandidateHash,
+	) -> FatalResult<Option<CandidateVotes>> {
+		load_candidate_votes(&*self.inner, &self.config, session, candidate_hash)
+	}
+
+	/// Atomically writes the list of operations, with later operations taking precedence over
+	/// prior.
+	///
+	/// This also takes care of purging old votes (of obsolete sessions).
+	fn write<I>(&mut self, ops: I) -> FatalResult<()>
+	where
+		I: IntoIterator<Item = BackendWriteOp>,
+	{
+		let mut tx = DBTransaction::new();
+		// Make sure the whole process is timed, including the actual transaction flush:
+		let mut cleanup_timer = None;
+		for op in ops {
+			match op {
+				BackendWriteOp::WriteEarliestSession(session) => {
+					cleanup_timer = match cleanup_timer.take() {
+						None => Some(self.metrics.time_vote_cleanup()),
+						Some(t) => Some(t),
+					};
+					self.add_vote_cleanup_tx(&mut tx, session)?;
+
+					// Actually write the earliest session.
+					tx.put_vec(
+						self.config.col_dispute_data,
+						EARLIEST_SESSION_KEY,
+						session.encode(),
+					);
+				},
+				BackendWriteOp::WriteRecentDisputes(recent_disputes) => {
+					tx.put_vec(
+						self.config.col_dispute_data,
+						RECENT_DISPUTES_KEY,
+						recent_disputes.encode(),
+					);
+				},
+				BackendWriteOp::WriteCandidateVotes(session, candidate_hash, votes) => {
+					gum::trace!(target: LOG_TARGET, ?session, "Writing candidate votes");
+					tx.put_vec(
+						self.config.col_dispute_data,
+						&candidate_votes_key(session, &candidate_hash),
+						votes.encode(),
+					);
+				},
+				BackendWriteOp::DeleteCandidateVotes(session, candidate_hash) => {
+					tx.delete(
+						self.config.col_dispute_data,
+						&candidate_votes_key(session, &candidate_hash),
+					);
+				},
+			}
+		}
+
+		self.inner.write(tx).map_err(FatalError::DbWriteFailed)
+	}
+}
+
+fn candidate_votes_key(session: SessionIndex, candidate_hash: &CandidateHash) -> [u8; 15 + 4 + 32] {
+	let mut buf = [0u8; 15 + 4 + 32];
+	buf[..15].copy_from_slice(CANDIDATE_VOTES_SUBKEY);
+
+	// big-endian encoding is used to ensure lexicographic ordering.
+	buf[15..][..4].copy_from_slice(&session.to_be_bytes());
+	candidate_hash.using_encoded(|s| buf[(15 + 4)..].copy_from_slice(s));
+
+	buf
+}
+
+fn candidate_votes_session_prefix(session: SessionIndex) -> [u8; 15 + 4] {
+	let mut buf = [0u8; 15 + 4];
+	buf[..15].copy_from_slice(CANDIDATE_VOTES_SUBKEY);
+
+	// big-endian encoding is used to ensure lexicographic ordering.
+	buf[15..][..4].copy_from_slice(&session.to_be_bytes());
+	buf
+}
+
+/// Column configuration information for the DB.
+#[derive(Debug, Clone)]
+pub struct ColumnConfiguration {
+	/// The column in the key-value DB where data is stored.
+	pub col_dispute_data: u32,
+}
+
+/// Tracked votes on candidates, for the purposes of dispute resolution.
+#[derive(Debug, Clone, Encode, Decode)]
+pub struct CandidateVotes {
+	/// The receipt of the candidate itself.
+	pub candidate_receipt: CandidateReceipt,
+	/// Votes of validity, sorted by validator index.
+	pub valid: Vec<(ValidDisputeStatementKind, ValidatorIndex, ValidatorSignature)>,
+	/// Votes of invalidity, sorted by validator index.
+	pub invalid: Vec<(InvalidDisputeStatementKind, ValidatorIndex, ValidatorSignature)>,
+}
+
+impl From<CandidateVotes> for pezkuwi_node_primitives::CandidateVotes {
+	fn from(db_votes: CandidateVotes) -> pezkuwi_node_primitives::CandidateVotes {
+		pezkuwi_node_primitives::CandidateVotes {
+			candidate_receipt: db_votes.candidate_receipt,
+			valid: db_votes.valid.into_iter().map(|(kind, i, sig)| (i, (kind, sig))).collect(),
+			invalid: db_votes.invalid.into_iter().map(|(kind, i, sig)| (i, (kind, sig))).collect(),
+		}
+	}
+}
+
+impl From<pezkuwi_node_primitives::CandidateVotes> for CandidateVotes {
+	fn from(primitive_votes: pezkuwi_node_primitives::CandidateVotes) -> CandidateVotes {
+		CandidateVotes {
+			candidate_receipt: primitive_votes.candidate_receipt,
+			valid: primitive_votes
+				.valid
+				.into_iter()
+				.map(|(i, (kind, sig))| (kind, i, sig))
+				.collect(),
+			invalid: primitive_votes.invalid.into_iter().map(|(i, (k, sig))| (k, i, sig)).collect(),
+		}
+	}
+}
+
+/// The mapping for recent disputes; any which have not yet been pruned for being ancient.
+pub type RecentDisputes = std::collections::BTreeMap<(SessionIndex, CandidateHash), DisputeStatus>;
+
+/// Errors while accessing things from the DB.
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+	#[error(transparent)]
+	Io(#[from] std::io::Error),
+	#[error(transparent)]
+	Codec(#[from] codec::Error),
+}
+
+impl From<Error> for crate::error::Error {
+	fn from(err: Error) -> Self {
+		match err {
+			Error::Io(io) => Self::Io(io),
+			Error::Codec(e) => Self::Codec(e),
+		}
+	}
+}
+
+/// Result alias for DB errors.
+pub type Result<T> = std::result::Result<T, Error>;
+
+fn load_decode<D: Decode>(
+	db: &dyn Database,
+	col_dispute_data: u32,
+	key: &[u8],
+) -> Result<Option<D>> {
+	match db.get(col_dispute_data, key)? {
+		None => Ok(None),
+		Some(raw) => D::decode(&mut &raw[..]).map(Some).map_err(Into::into),
+	}
+}
+
+/// Load the candidate votes for the specific session-candidate pair, if any.
+pub(crate) fn load_candidate_votes(
+	db: &dyn Database,
+	config: &ColumnConfiguration,
+	session: SessionIndex,
+	candidate_hash: &CandidateHash,
+) -> FatalResult<Option<CandidateVotes>> {
+	load_decode(db, config.col_dispute_data, &candidate_votes_key(session, candidate_hash))
+		.map_err(|e| FatalError::DbReadFailed(e))
+}
+
+/// Load the earliest session, if any.
+pub(crate) fn load_earliest_session(
+	db: &dyn Database,
+	config: &ColumnConfiguration,
+) -> FatalResult<Option<SessionIndex>> {
+	load_decode(db, config.col_dispute_data, EARLIEST_SESSION_KEY)
+		.map_err(|e| FatalError::DbReadFailed(e))
+}
+
+/// Load the recent disputes, if any.
+pub(crate) fn load_recent_disputes(
+	db: &dyn Database,
+	config: &ColumnConfiguration,
+) -> FatalResult<Option<RecentDisputes>> {
+	load_decode(db, config.col_dispute_data, RECENT_DISPUTES_KEY)
+		.map_err(|e| FatalError::DbReadFailed(e))
+}
+
+/// Maybe prune data in the DB based on the provided session index.
+///
+/// This is intended to be called on every block, and as such will be used to populate the DB on
+/// first launch. If the on-disk data does not need to be pruned, only a single storage read
+/// will be performed.
+///
+/// If one or more ancient sessions are pruned, all metadata on candidates within the ancient
+/// session will be deleted.
+pub(crate) fn note_earliest_session(
+	overlay_db: &mut OverlayedBackend<'_, impl Backend>,
+	new_earliest_session: SessionIndex,
+) -> FatalResult<()> {
+	match overlay_db.load_earliest_session()? {
+		None => {
+			// First launch - write new-earliest.
+			overlay_db.write_earliest_session(new_earliest_session);
+		},
+		Some(prev_earliest) if new_earliest_session > prev_earliest => {
+			// Prune all data in the outdated sessions.
+			overlay_db.write_earliest_session(new_earliest_session);
+
+			// Clear recent disputes metadata.
+			{
+				let mut recent_disputes = overlay_db.load_recent_disputes()?.unwrap_or_default();
+
+				let lower_bound = (new_earliest_session, CandidateHash(Hash::repeat_byte(0x00)));
+
+				let new_recent_disputes = recent_disputes.split_off(&lower_bound);
+				// Any remaining disputes are considered ancient and must be pruned.
+				let pruned_disputes = recent_disputes;
+
+				if pruned_disputes.len() != 0 {
+					overlay_db.write_recent_disputes(new_recent_disputes);
+					// Note: Deleting old candidate votes is handled in `write` based on the
+					// earliest session.
+				}
+			}
+		},
+		Some(_) => {
+			// nothing to do.
+		},
+	}
+
+	Ok(())
+}
+
+/// Until what session votes have been cleaned up already.
+///
+/// That is the db has already been purged of votes for sessions older than the returned
+/// `SessionIndex`.
+fn load_cleaned_votes_watermark(
+	db: &dyn Database,
+	config: &ColumnConfiguration,
+) -> FatalResult<Option<SessionIndex>> {
+	load_decode(db, config.col_dispute_data, CLEANED_VOTES_WATERMARK_KEY)
+		.map_err(|e| FatalError::DbReadFailed(e))
+}
+
+#[cfg(test)]
+mod tests {
+
+	use super::*;
+	use pezkuwi_node_primitives::DISPUTE_WINDOW;
+	use pezkuwi_primitives::{Hash, Id as ParaId};
+	use pezkuwi_primitives_test_helpers::{
+		dummy_candidate_receipt, dummy_candidate_receipt_v2, dummy_hash,
+	};
+
+	fn make_db() -> DbBackend {
+		let db = kvdb_memorydb::create(1);
+		let db = pezkuwi_node_subsystem_util::database::kvdb_impl::DbAdapter::new(db, &[0]);
+		let store = Arc::new(db);
+		let config = ColumnConfiguration { col_dispute_data: 0 };
+		DbBackend::new(store, config, Metrics::default())
+	}
+
+	#[test]
+	fn max_clean_batch_size_is_honored() {
+		let mut backend = make_db();
+
+		let mut overlay_db = OverlayedBackend::new(&backend);
+		let current_session = MAX_CLEAN_BATCH_SIZE + DISPUTE_WINDOW.get() + 3;
+		let earliest_session = current_session - DISPUTE_WINDOW.get();
+
+		overlay_db.write_earliest_session(0);
+		let candidate_hash = CandidateHash(Hash::repeat_byte(1));
+
+		for session in 0..current_session + 1 {
+			overlay_db.write_candidate_votes(
+				session,
+				candidate_hash,
+				CandidateVotes {
+					candidate_receipt: dummy_candidate_receipt_v2(dummy_hash()),
+					valid: Vec::new(),
+					invalid: Vec::new(),
+				},
+			);
+		}
+		assert!(overlay_db.load_candidate_votes(0, &candidate_hash).unwrap().is_some());
+		assert!(overlay_db
+			.load_candidate_votes(MAX_CLEAN_BATCH_SIZE - 1, &candidate_hash)
+			.unwrap()
+			.is_some());
+		assert!(overlay_db
+			.load_candidate_votes(MAX_CLEAN_BATCH_SIZE, &candidate_hash)
+			.unwrap()
+			.is_some());
+
+		// Cleanup only works for votes that have been written already - so write.
+		let write_ops = overlay_db.into_write_ops();
+		backend.write(write_ops).unwrap();
+
+		let mut overlay_db = OverlayedBackend::new(&backend);
+
+		gum::trace!(target: LOG_TARGET, ?current_session, "Noting current session");
+		note_earliest_session(&mut overlay_db, earliest_session).unwrap();
+
+		let write_ops = overlay_db.into_write_ops();
+		backend.write(write_ops).unwrap();
+
+		let mut overlay_db = OverlayedBackend::new(&backend);
+
+		assert!(overlay_db
+			.load_candidate_votes(MAX_CLEAN_BATCH_SIZE - 1, &candidate_hash)
+			.unwrap()
+			.is_none());
+		// After batch size votes should still be there:
+		assert!(overlay_db
+			.load_candidate_votes(MAX_CLEAN_BATCH_SIZE, &candidate_hash)
+			.unwrap()
+			.is_some());
+
+		let current_session = current_session + 1;
+		let earliest_session = earliest_session + 1;
+
+		note_earliest_session(&mut overlay_db, earliest_session).unwrap();
+
+		let write_ops = overlay_db.into_write_ops();
+		backend.write(write_ops).unwrap();
+
+		let overlay_db = OverlayedBackend::new(&backend);
+
+		// All should be gone now:
+		assert!(overlay_db
+			.load_candidate_votes(earliest_session - 1, &candidate_hash)
+			.unwrap()
+			.is_none());
+		// Earliest session should still be there:
+		assert!(overlay_db
+			.load_candidate_votes(earliest_session, &candidate_hash)
+			.unwrap()
+			.is_some());
+		// Old current session should still be there as well:
+		assert!(overlay_db
+			.load_candidate_votes(current_session - 1, &candidate_hash)
+			.unwrap()
+			.is_some());
+	}
+
+	#[test]
+	fn overlay_pre_and_post_commit_consistency() {
+		let mut backend = make_db();
+
+		let mut overlay_db = OverlayedBackend::new(&backend);
+
+		overlay_db.write_earliest_session(0);
+		overlay_db.write_earliest_session(1);
+
+		overlay_db.write_recent_disputes(
+			vec![((0, CandidateHash(Hash::repeat_byte(0))), DisputeStatus::Active)]
+				.into_iter()
+				.collect(),
+		);
+
+		overlay_db.write_recent_disputes(
+			vec![((1, CandidateHash(Hash::repeat_byte(1))), DisputeStatus::Active)]
+				.into_iter()
+				.collect(),
+		);
+
+		overlay_db.write_candidate_votes(
+			1,
+			CandidateHash(Hash::repeat_byte(1)),
+			CandidateVotes {
+				candidate_receipt: dummy_candidate_receipt_v2(dummy_hash()),
+				valid: Vec::new(),
+				invalid: Vec::new(),
+			},
+		);
+		overlay_db.write_candidate_votes(
+			1,
+			CandidateHash(Hash::repeat_byte(1)),
+			CandidateVotes {
+				candidate_receipt: {
+					let mut receipt = dummy_candidate_receipt(dummy_hash());
+					receipt.descriptor.para_id = ParaId::from(5_u32);
+
+					receipt.into()
+				},
+				valid: Vec::new(),
+				invalid: Vec::new(),
+			},
+		);
+
+		// Test that overlay returns the correct values before committing.
+		assert_eq!(overlay_db.load_earliest_session().unwrap().unwrap(), 1);
+
+		assert_eq!(
+			overlay_db.load_recent_disputes().unwrap().unwrap(),
+			vec![((1, CandidateHash(Hash::repeat_byte(1))), DisputeStatus::Active),]
+				.into_iter()
+				.collect()
+		);
+
+		assert_eq!(
+			overlay_db
+				.load_candidate_votes(1, &CandidateHash(Hash::repeat_byte(1)))
+				.unwrap()
+				.unwrap()
+				.candidate_receipt
+				.descriptor
+				.para_id(),
+			ParaId::from(5),
+		);
+
+		let write_ops = overlay_db.into_write_ops();
+		backend.write(write_ops).unwrap();
+
+		// Test that subsequent writes were written.
+		assert_eq!(backend.load_earliest_session().unwrap().unwrap(), 1);
+
+		assert_eq!(
+			backend.load_recent_disputes().unwrap().unwrap(),
+			vec![((1, CandidateHash(Hash::repeat_byte(1))), DisputeStatus::Active),]
+				.into_iter()
+				.collect()
+		);
+
+		assert_eq!(
+			backend
+				.load_candidate_votes(1, &CandidateHash(Hash::repeat_byte(1)))
+				.unwrap()
+				.unwrap()
+				.candidate_receipt
+				.descriptor
+				.para_id(),
+			ParaId::from(5),
+		);
+	}
+
+	#[test]
+	fn overlay_preserves_candidate_votes_operation_order() {
+		let mut backend = make_db();
+
+		let mut overlay_db = OverlayedBackend::new(&backend);
+
+		overlay_db.write_candidate_votes(
+			1,
+			CandidateHash(Hash::repeat_byte(1)),
+			CandidateVotes {
+				candidate_receipt: dummy_candidate_receipt_v2(Hash::random()),
+				valid: Vec::new(),
+				invalid: Vec::new(),
+			},
+		);
+
+		let receipt = dummy_candidate_receipt_v2(dummy_hash());
+
+		overlay_db.write_candidate_votes(
+			1,
+			CandidateHash(Hash::repeat_byte(1)),
+			CandidateVotes {
+				candidate_receipt: receipt.clone(),
+				valid: Vec::new(),
+				invalid: Vec::new(),
+			},
+		);
+
+		let write_ops = overlay_db.into_write_ops();
+		backend.write(write_ops).unwrap();
+
+		assert_eq!(
+			backend
+				.load_candidate_votes(1, &CandidateHash(Hash::repeat_byte(1)))
+				.unwrap()
+				.unwrap()
+				.candidate_receipt,
+			receipt,
+		);
+	}
+
+	#[test]
+	fn note_earliest_session_prunes_old() {
+		let mut backend = make_db();
+
+		let hash_a = CandidateHash(Hash::repeat_byte(0x0a));
+		let hash_b = CandidateHash(Hash::repeat_byte(0x0b));
+		let hash_c = CandidateHash(Hash::repeat_byte(0x0c));
+		let hash_d = CandidateHash(Hash::repeat_byte(0x0d));
+
+		let prev_earliest_session = 0;
+		let new_earliest_session = 5;
+		let current_session = 5 + DISPUTE_WINDOW.get();
+
+		let super_old_no_dispute = 1;
+		let very_old = 3;
+		let slightly_old = 4;
+		let very_recent = current_session - 1;
+
+		let blank_candidate_votes = || CandidateVotes {
+			candidate_receipt: dummy_candidate_receipt_v2(dummy_hash()),
+			valid: Vec::new(),
+			invalid: Vec::new(),
+		};
+
+		let mut overlay_db = OverlayedBackend::new(&backend);
+		overlay_db.write_earliest_session(prev_earliest_session);
+		overlay_db.write_recent_disputes(
+			vec![
+				((very_old, hash_a), DisputeStatus::Active),
+				((slightly_old, hash_b), DisputeStatus::Active),
+				((new_earliest_session, hash_c), DisputeStatus::Active),
+				((very_recent, hash_d), DisputeStatus::Active),
+			]
+			.into_iter()
+			.collect(),
+		);
+
+		overlay_db.write_candidate_votes(super_old_no_dispute, hash_a, blank_candidate_votes());
+		overlay_db.write_candidate_votes(very_old, hash_a, blank_candidate_votes());
+
+		overlay_db.write_candidate_votes(slightly_old, hash_b, blank_candidate_votes());
+
+		overlay_db.write_candidate_votes(new_earliest_session, hash_c, blank_candidate_votes());
+
+		overlay_db.write_candidate_votes(very_recent, hash_d, blank_candidate_votes());
+
+		let write_ops = overlay_db.into_write_ops();
+		backend.write(write_ops).unwrap();
+
+		let mut overlay_db = OverlayedBackend::new(&backend);
+		note_earliest_session(&mut overlay_db, new_earliest_session).unwrap();
+
+		assert_eq!(overlay_db.load_earliest_session().unwrap(), Some(new_earliest_session));
+
+		assert_eq!(
+			overlay_db.load_recent_disputes().unwrap().unwrap(),
+			vec![
+				((new_earliest_session, hash_c), DisputeStatus::Active),
+				((very_recent, hash_d), DisputeStatus::Active),
+			]
+			.into_iter()
+			.collect(),
+		);
+
+		// Votes are only cleaned up after actual write:
+		let write_ops = overlay_db.into_write_ops();
+		backend.write(write_ops).unwrap();
+
+		let overlay_db = OverlayedBackend::new(&backend);
+
+		assert!(overlay_db
+			.load_candidate_votes(super_old_no_dispute, &hash_a)
+			.unwrap()
+			.is_none());
+		assert!(overlay_db.load_candidate_votes(very_old, &hash_a).unwrap().is_none());
+		assert!(overlay_db.load_candidate_votes(slightly_old, &hash_b).unwrap().is_none());
+		assert!(overlay_db
+			.load_candidate_votes(new_earliest_session, &hash_c)
+			.unwrap()
+			.is_some());
+		assert!(overlay_db.load_candidate_votes(very_recent, &hash_d).unwrap().is_some());
+	}
+}

pezkuwi/node/core/dispute-coordinator/src/error.rs

@@ -0,0 +1,132 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use fatality::Nested;
+use futures::channel::oneshot;
+
+use pezkuwi_node_subsystem::{errors::ChainApiError, SubsystemError};
+use pezkuwi_node_subsystem_util::runtime;
+
+use crate::{db, participation, LOG_TARGET};
+use codec::Error as CodecError;
+
+pub type Result<T> = std::result::Result<T, Error>;
+pub type FatalResult<T> = std::result::Result<T, FatalError>;
+pub type JfyiResult<T> = std::result::Result<T, JfyiError>;
+
+#[allow(missing_docs)]
+#[fatality::fatality(splitable)]
+pub enum Error {
+	/// We received a legacy `SubystemError::Context` error which is considered fatal.
+	#[fatal]
+	#[error("SubsystemError::Context error: {0}")]
+	SubsystemContext(String),
+
+	/// `ctx.spawn` failed with an error.
+	#[fatal]
+	#[error("Spawning a task failed: {0}")]
+	SpawnFailed(#[source] SubsystemError),
+
+	#[fatal]
+	#[error("Participation worker receiver exhausted.")]
+	ParticipationWorkerReceiverExhausted,
+
+	/// Receiving subsystem message from overseer failed.
+	#[fatal]
+	#[error("Receiving message from overseer failed: {0}")]
+	SubsystemReceive(#[source] SubsystemError),
+
+	#[fatal]
+	#[error("Writing to database failed: {0}")]
+	DbWriteFailed(std::io::Error),
+
+	#[fatal]
+	#[error("Reading from database failed: {0}")]
+	DbReadFailed(db::v1::Error),
+
+	#[fatal]
+	#[error("Oneshot for receiving block number from chain API got cancelled")]
+	CanceledBlockNumber,
+
+	#[fatal]
+	#[error("Retrieving block number from chain API failed with error: {0}")]
+	ChainApiBlockNumber(ChainApiError),
+
+	#[fatal]
+	#[error(transparent)]
+	ChainApiAncestors(ChainApiError),
+
+	#[fatal]
+	#[error("Chain API dropped response channel sender")]
+	ChainApiSenderDropped,
+
+	#[fatal(forward)]
+	#[error("Error while accessing runtime information {0}")]
+	Runtime(#[from] runtime::Error),
+
+	#[error(transparent)]
+	ChainApi(#[from] ChainApiError),
+
+	#[error(transparent)]
+	Io(#[from] std::io::Error),
+
+	#[error(transparent)]
+	Oneshot(#[from] oneshot::Canceled),
+
+	#[error("Could not send import confirmation (receiver canceled)")]
+	DisputeImportOneshotSend,
+
+	#[error(transparent)]
+	Subsystem(#[from] SubsystemError),
+
+	#[error(transparent)]
+	Codec(#[from] CodecError),
+
+	/// `RollingSessionWindow` was not able to retrieve `SessionInfo`s.
+	#[error("Session can't be fetched via `RuntimeInfo`")]
+	SessionInfo,
+
+	#[error(transparent)]
+	QueueError(#[from] participation::QueueError),
+}
+
+/// Utility for eating top level errors and log them.
+///
+/// We basically always want to try and continue on error. This utility function is meant to
+/// consume top-level errors by simply logging them
+pub fn log_error(result: Result<()>) -> std::result::Result<(), FatalError> {
+	match result.into_nested()? {
+		Ok(()) => Ok(()),
+		Err(jfyi) => {
+			jfyi.log();
+			Ok(())
+		},
+	}
+}
+
+impl JfyiError {
+	/// Log a `JfyiError`.
+	pub fn log(self) {
+		match self {
+			// don't spam the log with spurious errors
+			Self::Runtime(runtime::Error::RuntimeRequestCanceled(_)) | Self::Oneshot(_) => {
+				gum::debug!(target: LOG_TARGET, error = ?self)
+			},
+			// it's worth reporting otherwise
+			_ => gum::warn!(target: LOG_TARGET, error = ?self),
+		}
+	}
+}

pezkuwi/node/core/dispute-coordinator/src/import.rs

@@ -0,0 +1,636 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Vote import logic.
+//!
+//! This module encapsulates the actual logic for importing new votes and provides easy access of
+//! the current state for votes for a particular candidate.
+//!
+//! In particular there is `CandidateVoteState` which tells what can be concluded for a particular
+//! set of votes. E.g. whether a dispute is ongoing, whether it is confirmed, concluded, ..
+//!
+//! Then there is `ImportResult` which reveals information about what changed once additional votes
+//! got imported on top of an existing `CandidateVoteState` and reveals "dynamic" information, like
+//! whether due to the import a dispute was raised/got confirmed, ...
+
+use std::collections::{BTreeMap, HashMap, HashSet};
+
+use pezkuwi_node_primitives::{
+	disputes::ValidCandidateVotes, CandidateVotes, DisputeStatus, SignedDisputeStatement, Timestamp,
+};
+use pezkuwi_node_subsystem::overseer;
+use pezkuwi_node_subsystem_util::{runtime::RuntimeInfo, ControlledValidatorIndices};
+use pezkuwi_primitives::{
+	CandidateHash, CandidateReceiptV2 as CandidateReceipt, DisputeStatement, ExecutorParams, Hash,
+	IndexedVec, SessionIndex, SessionInfo, ValidDisputeStatementKind, ValidatorId, ValidatorIndex,
+	ValidatorSignature,
+};
+
+use crate::LOG_TARGET;
+
+/// (Session) environment of a candidate.
+pub struct CandidateEnvironment<'a> {
+	/// The session the candidate appeared in.
+	session_index: SessionIndex,
+	/// Session for above index.
+	session: &'a SessionInfo,
+	/// Executor parameters for the session.
+	executor_params: &'a ExecutorParams,
+	/// Validator indices controlled by this node.
+	controlled_indices: HashSet<ValidatorIndex>,
+	/// Indices of on-chain disabled validators at the `relay_parent` combined
+	/// with the off-chain state.
+	disabled_indices: HashSet<ValidatorIndex>,
+}
+
+#[overseer::contextbounds(DisputeCoordinator, prefix = self::overseer)]
+impl<'a> CandidateEnvironment<'a> {
+	/// Create `CandidateEnvironment`.
+	///
+	/// Return: `None` in case session is outside of session window.
+	pub async fn new<Context>(
+		ctx: &mut Context,
+		runtime_info: &'a mut RuntimeInfo,
+		session_index: SessionIndex,
+		relay_parent: Hash,
+		disabled_offchain: impl IntoIterator<Item = ValidatorIndex>,
+		controlled_indices: &mut ControlledValidatorIndices,
+	) -> Option<CandidateEnvironment<'a>> {
+		let disabled_onchain = runtime_info
+			.get_disabled_validators(ctx.sender(), relay_parent)
+			.await
+			.unwrap_or_else(|err| {
+				gum::info!(target: LOG_TARGET, ?err, "Failed to get disabled validators");
+				Vec::new()
+			});
+
+		let (session, executor_params) = match runtime_info
+			.get_session_info_by_index(ctx.sender(), relay_parent, session_index)
+			.await
+		{
+			Ok(extended_session_info) =>
+				(&extended_session_info.session_info, &extended_session_info.executor_params),
+			Err(_) => return None,
+		};
+
+		let n_validators = session.validators.len();
+		let byzantine_threshold = pezkuwi_primitives::byzantine_threshold(n_validators);
+		// combine on-chain with off-chain disabled validators
+		// process disabled validators in the following order:
+		// - on-chain disabled validators
+		// - prioritized order of off-chain disabled validators
+		// deduplicate the list and take at most `byzantine_threshold` validators
+		let disabled_indices = {
+			let mut d: HashSet<ValidatorIndex> = HashSet::new();
+			for v in disabled_onchain.into_iter().chain(disabled_offchain.into_iter()) {
+				if d.len() == byzantine_threshold {
+					break;
+				}
+				d.insert(v);
+			}
+			d
+		};
+
+		let controlled_indices = controlled_indices
+			.get(session_index, &session.validators)
+			.map_or(HashSet::new(), |index| HashSet::from([index]));
+
+		Some(Self { session_index, session, executor_params, controlled_indices, disabled_indices })
+	}
+
+	/// Validators in the candidate's session.
+	pub fn validators(&self) -> &IndexedVec<ValidatorIndex, ValidatorId> {
+		&self.session.validators
+	}
+
+	/// `SessionInfo` for the candidate's session.
+	pub fn session_info(&self) -> &SessionInfo {
+		&self.session
+	}
+
+	/// Executor parameters for the candidate's session
+	pub fn executor_params(&self) -> &ExecutorParams {
+		&self.executor_params
+	}
+
+	/// Retrieve `SessionIndex` for this environment.
+	pub fn session_index(&self) -> SessionIndex {
+		self.session_index
+	}
+
+	/// Indices controlled by this node.
+	pub fn controlled_indices(&'a self) -> &'a HashSet<ValidatorIndex> {
+		&self.controlled_indices
+	}
+
+	/// Indices of off-chain and on-chain disabled validators.
+	pub fn disabled_indices(&'a self) -> &'a HashSet<ValidatorIndex> {
+		&self.disabled_indices
+	}
+}
+
+/// Whether or not we already issued some statement about a candidate.
+pub enum OwnVoteState {
+	/// Our votes, if any.
+	Voted(Vec<(ValidatorIndex, (DisputeStatement, ValidatorSignature))>),
+
+	/// We are not a teyrchain validator in the session.
+	///
+	/// Hence we cannot vote.
+	CannotVote,
+}
+
+impl OwnVoteState {
+	fn new(votes: &CandidateVotes, env: &CandidateEnvironment) -> Self {
+		let controlled_indices = env.controlled_indices();
+		if controlled_indices.is_empty() {
+			return Self::CannotVote;
+		}
+
+		let our_valid_votes = controlled_indices
+			.iter()
+			.filter_map(|i| votes.valid.raw().get_key_value(i))
+			.map(|(index, (kind, sig))| {
+				(*index, (DisputeStatement::Valid(kind.clone()), sig.clone()))
+			});
+		let our_invalid_votes = controlled_indices
+			.iter()
+			.filter_map(|i| votes.invalid.get_key_value(i))
+			.map(|(index, (kind, sig))| (*index, (DisputeStatement::Invalid(*kind), sig.clone())));
+
+		Self::Voted(our_valid_votes.chain(our_invalid_votes).collect())
+	}
+
+	/// Is a vote from us missing but we are a validator able to vote?
+	fn vote_missing(&self) -> bool {
+		match self {
+			Self::Voted(votes) if votes.is_empty() => true,
+			Self::Voted(_) | Self::CannotVote => false,
+		}
+	}
+
+	/// Get own approval votes, if any.
+	///
+	/// Empty iterator means, no approval votes. `None` means, there will never be any (we cannot
+	/// vote).
+	fn approval_votes(
+		&self,
+	) -> Option<impl Iterator<Item = (ValidatorIndex, &ValidatorSignature)>> {
+		match self {
+			Self::Voted(votes) => Some(votes.iter().filter_map(|(index, (kind, sig))| {
+				if let DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) = kind {
+					Some((*index, sig))
+				} else {
+					None
+				}
+			})),
+			Self::CannotVote => None,
+		}
+	}
+
+	/// Get our votes if there are any.
+	///
+	/// Empty iterator means, no votes. `None` means, there will never be any (we cannot
+	/// vote).
+	fn votes(&self) -> Option<&Vec<(ValidatorIndex, (DisputeStatement, ValidatorSignature))>> {
+		match self {
+			Self::Voted(votes) => Some(&votes),
+			Self::CannotVote => None,
+		}
+	}
+}
+
+/// Complete state of votes for a candidate.
+///
+/// All votes + information whether a dispute is ongoing, confirmed, concluded, whether we already
+/// voted, ...
+pub struct CandidateVoteState<Votes> {
+	/// Votes already existing for the candidate + receipt.
+	votes: Votes,
+
+	/// Information about own votes:
+	own_vote: OwnVoteState,
+
+	/// Current dispute status, if there is any.
+	dispute_status: Option<DisputeStatus>,
+
+	/// Are there `byzantine threshold + 1` invalid votes
+	byzantine_threshold_against: bool,
+}
+
+impl CandidateVoteState<CandidateVotes> {
+	/// Create an empty `CandidateVoteState`
+	///
+	/// in case there have not been any previous votes.
+	pub fn new_from_receipt(candidate_receipt: CandidateReceipt) -> Self {
+		let votes = CandidateVotes {
+			candidate_receipt,
+			valid: ValidCandidateVotes::new(),
+			invalid: BTreeMap::new(),
+		};
+		Self {
+			votes,
+			own_vote: OwnVoteState::CannotVote,
+			dispute_status: None,
+			byzantine_threshold_against: false,
+		}
+	}
+
+	/// Create a new `CandidateVoteState` from already existing votes.
+	pub fn new(votes: CandidateVotes, env: &CandidateEnvironment, now: Timestamp) -> Self {
+		let own_vote = OwnVoteState::new(&votes, env);
+
+		let n_validators = env.validators().len();
+
+		let supermajority_threshold = pezkuwi_primitives::supermajority_threshold(n_validators);
+
+		// We have a dispute, if we have votes on both sides, with at least one invalid vote
+		// from non-disabled validator or with votes on both sides and confirmed.
+		let has_non_disabled_invalid_votes =
+			votes.invalid.keys().any(|i| !env.disabled_indices().contains(i));
+		let byzantine_threshold = pezkuwi_primitives::byzantine_threshold(n_validators);
+		let votes_on_both_sides = !votes.valid.raw().is_empty() && !votes.invalid.is_empty();
+		let is_confirmed =
+			votes_on_both_sides && (votes.voted_indices().len() > byzantine_threshold);
+		let is_disputed =
+			is_confirmed || (has_non_disabled_invalid_votes && !votes.valid.raw().is_empty());
+
+		let (dispute_status, byzantine_threshold_against) = if is_disputed {
+			let mut status = DisputeStatus::active();
+			if is_confirmed {
+				status = status.confirm();
+			};
+			let concluded_for = votes.valid.raw().len() >= supermajority_threshold;
+			if concluded_for {
+				status = status.conclude_for(now);
+			};
+
+			let concluded_against = votes.invalid.len() >= supermajority_threshold;
+			if concluded_against {
+				status = status.conclude_against(now);
+			};
+			(Some(status), votes.invalid.len() > byzantine_threshold)
+		} else {
+			(None, false)
+		};
+
+		Self { votes, own_vote, dispute_status, byzantine_threshold_against }
+	}
+
+	/// Import fresh statements.
+	///
+	/// Result will be a new state plus information about things that changed due to the import.
+	pub fn import_statements(
+		self,
+		env: &CandidateEnvironment,
+		statements: Vec<(SignedDisputeStatement, ValidatorIndex)>,
+		now: Timestamp,
+	) -> ImportResult {
+		let (mut votes, old_state) = self.into_old_state();
+
+		let mut new_invalid_voters = Vec::new();
+		let mut imported_invalid_votes = 0;
+		let mut imported_valid_votes = 0;
+
+		let expected_candidate_hash = votes.candidate_receipt.hash();
+
+		for (statement, val_index) in statements {
+			if env
+				.validators()
+				.get(val_index)
+				.map_or(true, |v| v != statement.validator_public())
+			{
+				gum::error!(
+					target: LOG_TARGET,
+					?val_index,
+					session= ?env.session_index,
+					claimed_key = ?statement.validator_public(),
+					"Validator index doesn't match claimed key",
+				);
+
+				continue;
+			}
+			if statement.candidate_hash() != &expected_candidate_hash {
+				gum::error!(
+					target: LOG_TARGET,
+					?val_index,
+					session= ?env.session_index,
+					given_candidate_hash = ?statement.candidate_hash(),
+					?expected_candidate_hash,
+					"Vote is for unexpected candidate!",
+				);
+				continue;
+			}
+			if statement.session_index() != env.session_index() {
+				gum::error!(
+					target: LOG_TARGET,
+					?val_index,
+					session= ?env.session_index,
+					given_candidate_hash = ?statement.candidate_hash(),
+					?expected_candidate_hash,
+					"Vote is for unexpected session!",
+				);
+				continue;
+			}
+
+			match statement.statement() {
+				DisputeStatement::Valid(valid_kind) => {
+					let fresh = votes.valid.insert_vote(
+						val_index,
+						valid_kind.clone(),
+						statement.into_validator_signature(),
+					);
+					if fresh {
+						imported_valid_votes += 1;
+					}
+				},
+				DisputeStatement::Invalid(invalid_kind) => {
+					let fresh = votes
+						.invalid
+						.insert(val_index, (*invalid_kind, statement.into_validator_signature()))
+						.is_none();
+					if fresh {
+						new_invalid_voters.push(val_index);
+						imported_invalid_votes += 1;
+					}
+				},
+			}
+		}
+
+		let new_state = Self::new(votes, env, now);
+
+		ImportResult {
+			old_state,
+			new_state,
+			imported_invalid_votes,
+			imported_valid_votes,
+			imported_approval_votes: 0,
+			new_invalid_voters,
+		}
+	}
+
+	/// Retrieve `CandidateReceipt` in `CandidateVotes`.
+	pub fn candidate_receipt(&self) -> &CandidateReceipt {
+		&self.votes.candidate_receipt
+	}
+
+	/// Returns true if all the invalid votes are from disabled validators.
+	pub fn invalid_votes_all_disabled(
+		&self,
+		mut is_disabled: impl FnMut(&ValidatorIndex) -> bool,
+	) -> bool {
+		self.votes.invalid.keys().all(|i| is_disabled(i))
+	}
+
+	/// Extract `CandidateVotes` for handling import of new statements.
+	fn into_old_state(self) -> (CandidateVotes, CandidateVoteState<()>) {
+		let CandidateVoteState { votes, own_vote, dispute_status, byzantine_threshold_against } =
+			self;
+		(
+			votes,
+			CandidateVoteState { votes: (), own_vote, dispute_status, byzantine_threshold_against },
+		)
+	}
+}
+
+impl<V> CandidateVoteState<V> {
+	/// Whether or not we have an ongoing dispute.
+	pub fn is_disputed(&self) -> bool {
+		self.dispute_status.is_some()
+	}
+
+	/// Whether there is an ongoing confirmed dispute.
+	///
+	/// This checks whether there is a dispute ongoing and we have more than byzantine threshold
+	/// votes.
+	pub fn is_confirmed(&self) -> bool {
+		self.dispute_status.map_or(false, |s| s.is_confirmed_concluded())
+	}
+
+	/// Are we a validator in the session, but have not yet voted?
+	pub fn own_vote_missing(&self) -> bool {
+		self.own_vote.vote_missing()
+	}
+
+	/// Own approval votes if any:
+	pub fn own_approval_votes(
+		&self,
+	) -> Option<impl Iterator<Item = (ValidatorIndex, &ValidatorSignature)>> {
+		self.own_vote.approval_votes()
+	}
+
+	/// Get own votes if there are any.
+	pub fn own_votes(
+		&self,
+	) -> Option<&Vec<(ValidatorIndex, (DisputeStatement, ValidatorSignature))>> {
+		self.own_vote.votes()
+	}
+
+	/// Whether or not there is a dispute and it has already enough valid votes to conclude.
+	pub fn has_concluded_for(&self) -> bool {
+		self.dispute_status.map_or(false, |s| s.has_concluded_for())
+	}
+
+	/// Whether or not there is a dispute and it has already enough invalid votes to conclude.
+	pub fn has_concluded_against(&self) -> bool {
+		self.dispute_status.map_or(false, |s| s.has_concluded_against())
+	}
+
+	/// Get access to the dispute status, in case there is one.
+	pub fn dispute_status(&self) -> &Option<DisputeStatus> {
+		&self.dispute_status
+	}
+
+	/// Access to underlying votes.
+	pub fn votes(&self) -> &V {
+		&self.votes
+	}
+}
+
+/// An ongoing statement/vote import.
+pub struct ImportResult {
+	/// The state we had before importing new statements.
+	old_state: CandidateVoteState<()>,
+	/// The new state after importing the new statements.
+	new_state: CandidateVoteState<CandidateVotes>,
+	/// New invalid voters as of this import.
+	new_invalid_voters: Vec<ValidatorIndex>,
+	/// Number of successfully imported valid votes.
+	imported_invalid_votes: u32,
+	/// Number of successfully imported invalid votes.
+	imported_valid_votes: u32,
+	/// Number of approval votes imported via `import_approval_votes()`.
+	///
+	/// And only those: If normal import included approval votes, those are not counted here.
+	///
+	/// In other words, without a call `import_approval_votes()` this will always be 0.
+	imported_approval_votes: u32,
+}
+
+impl ImportResult {
+	/// Whether or not anything has changed due to the import.
+	pub fn votes_changed(&self) -> bool {
+		self.imported_valid_votes != 0 || self.imported_invalid_votes != 0
+	}
+
+	/// The dispute state has changed in some way.
+	///
+	/// - freshly disputed
+	/// - freshly confirmed
+	/// - freshly concluded (valid or invalid)
+	pub fn dispute_state_changed(&self) -> bool {
+		self.is_freshly_disputed() || self.is_freshly_confirmed() || self.is_freshly_concluded()
+	}
+
+	/// State as it was before import.
+	pub fn old_state(&self) -> &CandidateVoteState<()> {
+		&self.old_state
+	}
+
+	/// State after import
+	pub fn new_state(&self) -> &CandidateVoteState<CandidateVotes> {
+		&self.new_state
+	}
+
+	/// New "invalid" voters encountered during import.
+	pub fn new_invalid_voters(&self) -> &Vec<ValidatorIndex> {
+		&self.new_invalid_voters
+	}
+
+	/// Number of imported valid votes.
+	pub fn imported_valid_votes(&self) -> u32 {
+		self.imported_valid_votes
+	}
+
+	/// Number of imported invalid votes.
+	pub fn imported_invalid_votes(&self) -> u32 {
+		self.imported_invalid_votes
+	}
+
+	/// Number of imported approval votes.
+	pub fn imported_approval_votes(&self) -> u32 {
+		self.imported_approval_votes
+	}
+
+	/// Whether we now have a dispute and did not prior to the import.
+	pub fn is_freshly_disputed(&self) -> bool {
+		!self.old_state().is_disputed() && self.new_state().is_disputed()
+	}
+
+	/// Whether we just surpassed the byzantine threshold.
+	pub fn is_freshly_confirmed(&self) -> bool {
+		!self.old_state().is_confirmed() && self.new_state().is_confirmed()
+	}
+
+	/// Whether or not any dispute just concluded valid due to the import.
+	pub fn is_freshly_concluded_for(&self) -> bool {
+		!self.old_state().has_concluded_for() && self.new_state().has_concluded_for()
+	}
+
+	/// Whether or not any dispute just concluded invalid due to the import.
+	pub fn is_freshly_concluded_against(&self) -> bool {
+		!self.old_state().has_concluded_against() && self.new_state().has_concluded_against()
+	}
+
+	/// Whether or not any dispute just concluded either invalid or valid due to the import.
+	pub fn is_freshly_concluded(&self) -> bool {
+		self.is_freshly_concluded_against() || self.is_freshly_concluded_for()
+	}
+
+	/// Whether or not the invalid vote count for the dispute went beyond the byzantine threshold
+	/// after the last import
+	pub fn has_fresh_byzantine_threshold_against(&self) -> bool {
+		!self.old_state().byzantine_threshold_against &&
+			self.new_state().byzantine_threshold_against
+	}
+
+	/// Modify this `ImportResult`s, by importing additional approval votes.
+	///
+	/// Both results and `new_state` will be changed as if those approval votes had been in the
+	/// original import.
+	pub fn import_approval_votes(
+		self,
+		env: &CandidateEnvironment,
+		approval_votes: HashMap<ValidatorIndex, (Vec<CandidateHash>, ValidatorSignature)>,
+		now: Timestamp,
+	) -> Self {
+		let Self {
+			old_state,
+			new_state,
+			new_invalid_voters,
+			mut imported_valid_votes,
+			imported_invalid_votes,
+			mut imported_approval_votes,
+		} = self;
+
+		let (mut votes, _) = new_state.into_old_state();
+
+		for (index, (candidate_hashes, sig)) in approval_votes.into_iter() {
+			debug_assert!(
+				{
+					let pub_key = &env.session_info().validators.get(index).expect("indices are validated by approval-voting subsystem; qed");
+					let session_index = env.session_index();
+					candidate_hashes.contains(&votes.candidate_receipt.hash()) && DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(candidate_hashes.clone()))
+						.check_signature(pub_key, *candidate_hashes.first().expect("Valid votes have at least one candidate; qed"), session_index, &sig)
+						.is_ok()
+				},
+				"Signature check for imported approval votes failed! This is a serious bug. Session: {:?}, candidate hash: {:?}, validator index: {:?}", env.session_index(), votes.candidate_receipt.hash(), index
+			);
+			if votes.valid.insert_vote(
+				index,
+				// There is a hidden dependency here between approval-voting and this subsystem.
+				// We should be able to start emitting
+				// ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates only after:
+				// 1. Runtime have been upgraded to know about the new format.
+				// 2. All nodes have been upgraded to know about the new format.
+				// Once those two requirements have been met we should be able to increase
+				// max_approval_coalesce_count to values greater than 1.
+				if candidate_hashes.len() > 1 {
+					ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(candidate_hashes)
+				} else {
+					ValidDisputeStatementKind::ApprovalChecking
+				},
+				sig,
+			) {
+				imported_valid_votes += 1;
+				imported_approval_votes += 1;
+			}
+		}
+
+		let new_state = CandidateVoteState::new(votes, env, now);
+
+		Self {
+			old_state,
+			new_state,
+			new_invalid_voters,
+			imported_valid_votes,
+			imported_invalid_votes,
+			imported_approval_votes,
+		}
+	}
+
+	/// All done, give me those votes.
+	///
+	/// Returns: `None` in case nothing has changed (import was redundant).
+	pub fn into_updated_votes(self) -> Option<CandidateVotes> {
+		if self.votes_changed() {
+			let CandidateVoteState { votes, .. } = self.new_state;
+			Some(votes)
+		} else {
+			None
+		}
+	}
+}

pezkuwi/node/core/dispute-coordinator/src/lib.rs

@@ -0,0 +1,649 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Implements the dispute coordinator subsystem.
+//!
+//! This is the central subsystem of the node-side components which participate in disputes.
+//! This subsystem wraps a database which tracks all statements observed by all validators over some
+//! window of sessions. Votes older than this session window are pruned.
+//!
+//! This subsystem will be the point which produce dispute votes, either positive or negative, based
+//! on locally-observed validation results as well as a sink for votes received by other subsystems.
+//! When importing a dispute vote from another node, this will trigger dispute participation to
+//! recover and validate the block.
+
+use std::sync::Arc;
+
+use error::FatalError;
+use futures::FutureExt;
+
+use gum::CandidateHash;
+use sc_keystore::LocalKeystore;
+
+use pezkuwi_node_primitives::{
+	CandidateVotes, DisputeMessage, DisputeMessageCheckError, SignedDisputeStatement,
+	DISPUTE_WINDOW,
+};
+use pezkuwi_node_subsystem::{
+	messages::DisputeDistributionMessage, overseer, ActivatedLeaf, FromOrchestra, OverseerSignal,
+	SpawnedSubsystem, SubsystemError,
+};
+use pezkuwi_node_subsystem_util::{
+	database::Database,
+	runtime::{Config as RuntimeInfoConfig, RuntimeInfo},
+	ControlledValidatorIndices,
+};
+use pezkuwi_primitives::{
+	DisputeStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, ValidatorIndex,
+};
+
+use crate::{
+	error::{FatalResult, Result},
+	metrics::Metrics,
+	status::{get_active_with_status, SystemClock},
+};
+use backend::{Backend, OverlayedBackend};
+use db::v1::DbBackend;
+use fatality::Split;
+
+use self::{
+	import::{CandidateEnvironment, CandidateVoteState},
+	participation::{ParticipationPriority, ParticipationRequest},
+	spam_slots::{SpamSlots, UnconfirmedDisputes},
+};
+
+pub(crate) mod backend;
+pub(crate) mod db;
+pub(crate) mod error;
+
+/// Subsystem after receiving the first active leaf.
+mod initialized;
+use initialized::{InitialData, Initialized};
+
+/// Provider of data scraped from chain.
+///
+/// If we have seen a candidate included somewhere, we should treat it as priority and will be able
+/// to provide an ordering for participation. Thus a dispute for a candidate where we can get some
+/// ordering is high-priority (we know it is a valid dispute) and those can be ordered by
+/// `participation` based on `relay_parent` block number and other metrics, so each validator will
+/// participate in disputes in a similar order, which ensures we will be resolving disputes, even
+/// under heavy load.
+mod scraping;
+use scraping::ChainScraper;
+
+/// When importing votes we will check via the `ordering` module, whether or not we know of the
+/// candidate to be included somewhere. If not, the votes might be spam, in this case we want to
+/// limit the amount of locally imported votes, to prevent DoS attacks/resource exhaustion. The
+/// `spam_slots` module helps keeping track of unconfirmed disputes per validators, if a spam slot
+/// gets full, we will drop any further potential spam votes from that validator and report back
+/// that the import failed. Which will lead to any honest validator to retry, thus the spam slots
+/// can be relatively small, as a drop is not fatal.
+mod spam_slots;
+
+/// Handling of participation requests via `Participation`.
+///
+/// `Participation` provides an API (`Participation::queue_participation`) for queuing of dispute
+/// participations and will process those participation requests, such that most important/urgent
+/// disputes will be resolved and processed first and more importantly it will order requests in a
+/// way so disputes will get resolved, even if there are lots of them.
+pub(crate) mod participation;
+
+/// Pure processing of vote imports.
+pub(crate) mod import;
+
+/// Metrics types.
+mod metrics;
+
+/// Status tracking of disputes (`DisputeStatus`).
+mod status;
+
+use crate::status::Clock;
+
+#[cfg(test)]
+mod tests;
+
+pub(crate) const LOG_TARGET: &str = "teyrchain::dispute-coordinator";
+
+/// An implementation of the dispute coordinator subsystem.
+pub struct DisputeCoordinatorSubsystem {
+	config: Config,
+	store: Arc<dyn Database>,
+	keystore: Arc<LocalKeystore>,
+	metrics: Metrics,
+}
+
+/// Configuration for the dispute coordinator subsystem.
+#[derive(Debug, Clone, Copy)]
+pub struct Config {
+	/// The data column in the store to use for dispute data.
+	pub col_dispute_data: u32,
+}
+
+impl Config {
+	fn column_config(&self) -> db::v1::ColumnConfiguration {
+		db::v1::ColumnConfiguration { col_dispute_data: self.col_dispute_data }
+	}
+}
+
+#[overseer::subsystem(DisputeCoordinator, error=SubsystemError, prefix=self::overseer)]
+impl<Context: Send> DisputeCoordinatorSubsystem {
+	fn start(self, ctx: Context) -> SpawnedSubsystem {
+		let future = async {
+			let backend = DbBackend::new(
+				self.store.clone(),
+				self.config.column_config(),
+				self.metrics.clone(),
+			);
+			self.run(ctx, backend, Box::new(SystemClock))
+				.await
+				.map_err(|e| SubsystemError::with_origin("dispute-coordinator", e))
+		}
+		.boxed();
+
+		SpawnedSubsystem { name: "dispute-coordinator-subsystem", future }
+	}
+}
+
+#[overseer::contextbounds(DisputeCoordinator, prefix = self::overseer)]
+impl DisputeCoordinatorSubsystem {
+	/// Create a new instance of the subsystem.
+	pub fn new(
+		store: Arc<dyn Database>,
+		config: Config,
+		keystore: Arc<LocalKeystore>,
+		metrics: Metrics,
+	) -> Self {
+		Self { store, config, keystore, metrics }
+	}
+
+	/// Initialize and afterwards run `Initialized::run`.
+	async fn run<B, Context>(
+		self,
+		mut ctx: Context,
+		backend: B,
+		clock: Box<dyn Clock>,
+	) -> FatalResult<()>
+	where
+		B: Backend + 'static,
+	{
+		let res = self.initialize(&mut ctx, backend, &*clock).await?;
+
+		let (participations, votes, first_leaf, initialized, backend) = match res {
+			// Concluded:
+			None => return Ok(()),
+			Some(r) => r,
+		};
+
+		initialized
+			.run(ctx, backend, Some(InitialData { participations, votes, leaf: first_leaf }), clock)
+			.await
+	}
+
+	/// Make sure to recover participations properly on startup.
+	async fn initialize<B, Context>(
+		self,
+		ctx: &mut Context,
+		mut backend: B,
+		clock: &dyn Clock,
+	) -> FatalResult<
+		Option<(
+			Vec<(ParticipationPriority, ParticipationRequest)>,
+			Vec<ScrapedOnChainVotes>,
+			ActivatedLeaf,
+			Initialized,
+			B,
+		)>,
+	>
+	where
+		B: Backend + 'static,
+	{
+		loop {
+			let first_leaf = match wait_for_first_leaf(ctx).await {
+				Ok(Some(activated_leaf)) => activated_leaf,
+				Ok(None) => continue,
+				Err(e) => {
+					e.split()?.log();
+					continue;
+				},
+			};
+
+			// `RuntimeInfo` cache should match `DISPUTE_WINDOW` so that we can
+			// keep all sessions for a dispute window
+			let mut runtime_info = RuntimeInfo::new_with_config(RuntimeInfoConfig {
+				keystore: None,
+				session_cache_lru_size: DISPUTE_WINDOW.get(),
+			});
+			let mut overlay_db = OverlayedBackend::new(&mut backend);
+			let (
+				participations,
+				votes,
+				spam_slots,
+				ordering_provider,
+				highest_session_seen,
+				gaps_in_cache,
+				offchain_disabled_validators,
+				controlled_validator_indices,
+			) = match self
+				.handle_startup(ctx, first_leaf.clone(), &mut runtime_info, &mut overlay_db, clock)
+				.await
+			{
+				Ok(v) => v,
+				Err(e) => {
+					e.split()?.log();
+					continue;
+				},
+			};
+			if !overlay_db.is_empty() {
+				let ops = overlay_db.into_write_ops();
+				backend.write(ops)?;
+			}
+
+			return Ok(Some((
+				participations,
+				votes,
+				first_leaf,
+				Initialized::new(
+					self,
+					runtime_info,
+					spam_slots,
+					ordering_provider,
+					highest_session_seen,
+					gaps_in_cache,
+					offchain_disabled_validators,
+					controlled_validator_indices,
+				),
+				backend,
+			)));
+		}
+	}
+
+	// Restores the subsystem's state before proceeding with the main event loop.
+	//
+	// - Prune any old disputes.
+	// - Find disputes we need to participate in.
+	// - Initialize spam slots & OrderingProvider.
+	async fn handle_startup<Context>(
+		&self,
+		ctx: &mut Context,
+		initial_head: ActivatedLeaf,
+		runtime_info: &mut RuntimeInfo,
+		overlay_db: &mut OverlayedBackend<'_, impl Backend>,
+		clock: &dyn Clock,
+	) -> Result<(
+		Vec<(ParticipationPriority, ParticipationRequest)>,
+		Vec<ScrapedOnChainVotes>,
+		SpamSlots,
+		ChainScraper,
+		SessionIndex,
+		bool,
+		initialized::OffchainDisabledValidators,
+		ControlledValidatorIndices,
+	)> {
+		let now = clock.now();
+
+		// We assume the highest session is the passed leaf. If we can't get the session index
+		// we can't initialize the subsystem so we'll wait for a new leaf
+		let highest_session = runtime_info
+			.get_session_index_for_child(ctx.sender(), initial_head.hash)
+			.await?;
+		let earliest_session = highest_session.saturating_sub(DISPUTE_WINDOW.get() - 1);
+
+		// Load recent disputes from the database
+		let recent_disputes = match overlay_db.load_recent_disputes() {
+			Ok(disputes) => disputes.unwrap_or_default(),
+			Err(e) => {
+				gum::error!(target: LOG_TARGET, "Failed initial load of recent disputes: {:?}", e);
+				return Err(e.into());
+			},
+		};
+
+		// Initialize offchain disabled validators from recent disputes
+		let offchain_disabled_validators = initialized::OffchainDisabledValidators::new_from_state(
+			&recent_disputes,
+			|session, candidate_hash| match overlay_db.load_candidate_votes(session, candidate_hash)
+			{
+				Ok(Some(votes)) => Some(votes.into()),
+				_ => None,
+			},
+			earliest_session,
+		);
+
+		let active_disputes = get_active_with_status(recent_disputes.into_iter(), now);
+
+		let mut gap_in_cache = false;
+		// Cache the sessions. A failure to fetch a session here is not that critical so we
+		// won't abort the initialization
+		for idx in earliest_session..=highest_session {
+			// Print disabled validators on startup if any
+			let disabled: Vec<u32> = offchain_disabled_validators.iter(idx).map(|i| i.0).collect();
+			if !disabled.is_empty() {
+				gum::info!(
+					target: LOG_TARGET,
+					disabled = ?disabled,
+					session = idx,
+					"Detected disabled validators on startup",
+				);
+			}
+
+			if let Err(e) = runtime_info
+				.get_session_info_by_index(ctx.sender(), initial_head.hash, idx)
+				.await
+			{
+				gum::debug!(
+					target: LOG_TARGET,
+					leaf_hash = ?initial_head.hash,
+					session_idx = idx,
+					err = ?e,
+					"Can't cache SessionInfo during subsystem initialization. Skipping session."
+				);
+				gap_in_cache = true;
+				continue;
+			};
+		}
+
+		// Prune obsolete disputes:
+		db::v1::note_earliest_session(overlay_db, earliest_session)?;
+
+		let mut participation_requests = Vec::new();
+		let mut spam_disputes: UnconfirmedDisputes = UnconfirmedDisputes::new();
+		let mut controlled_indices =
+			ControlledValidatorIndices::new(self.keystore.clone(), DISPUTE_WINDOW.get());
+		let leaf_hash = initial_head.hash;
+		let (scraper, votes) = ChainScraper::new(ctx.sender(), initial_head).await?;
+		for ((session, ref candidate_hash), _) in active_disputes {
+			let env = match CandidateEnvironment::new(
+				ctx,
+				runtime_info,
+				highest_session,
+				leaf_hash,
+				offchain_disabled_validators.iter(session),
+				&mut controlled_indices,
+			)
+			.await
+			{
+				None => {
+					gum::warn!(
+						target: LOG_TARGET,
+						session,
+						"We are lacking a `SessionInfo` for handling db votes on startup."
+					);
+
+					continue;
+				},
+				Some(env) => env,
+			};
+
+			let votes: CandidateVotes =
+				match overlay_db.load_candidate_votes(session, candidate_hash) {
+					Ok(Some(votes)) => votes.into(),
+					Ok(None) => continue,
+					Err(e) => {
+						gum::error!(
+							target: LOG_TARGET,
+							"Failed initial load of candidate votes: {:?}",
+							e
+						);
+						continue;
+					},
+				};
+			let vote_state = CandidateVoteState::new(votes, &env, now);
+			let is_disabled = |v: &ValidatorIndex| env.disabled_indices().contains(v);
+			let potential_spam =
+				is_potential_spam(&scraper, &vote_state, candidate_hash, is_disabled);
+			let is_included =
+				scraper.is_candidate_included(&vote_state.votes().candidate_receipt.hash());
+
+			if potential_spam {
+				gum::trace!(
+					target: LOG_TARGET,
+					?session,
+					?candidate_hash,
+					"Found potential spam dispute on startup"
+				);
+				spam_disputes
+					.insert((session, *candidate_hash), vote_state.votes().voted_indices());
+			} else {
+				// Participate if need be:
+				if vote_state.own_vote_missing() {
+					gum::trace!(
+						target: LOG_TARGET,
+						?session,
+						?candidate_hash,
+						"Found valid dispute, with no vote from us on startup - participating."
+					);
+					let request_timer = self.metrics.time_participation_pipeline();
+					participation_requests.push((
+						ParticipationPriority::with_priority_if(is_included),
+						ParticipationRequest::new(
+							vote_state.votes().candidate_receipt.clone(),
+							session,
+							env.executor_params().clone(),
+							request_timer,
+						),
+					));
+				}
+				// Else make sure our own vote is distributed:
+				else {
+					gum::trace!(
+						target: LOG_TARGET,
+						?session,
+						?candidate_hash,
+						"Found valid dispute, with vote from us on startup - send vote."
+					);
+					send_dispute_messages(ctx, &env, &vote_state).await;
+				}
+			}
+		}
+
+		Ok((
+			participation_requests,
+			votes,
+			SpamSlots::recover_from_state(spam_disputes),
+			scraper,
+			highest_session,
+			gap_in_cache,
+			offchain_disabled_validators,
+			controlled_indices,
+		))
+	}
+}
+
+/// Wait for `ActiveLeavesUpdate`, returns `None` if `Conclude` signal came first.
+#[overseer::contextbounds(DisputeCoordinator, prefix = self::overseer)]
+async fn wait_for_first_leaf<Context>(ctx: &mut Context) -> Result<Option<ActivatedLeaf>> {
+	loop {
+		match ctx.recv().await.map_err(FatalError::SubsystemReceive)? {
+			FromOrchestra::Signal(OverseerSignal::Conclude) => return Ok(None),
+			FromOrchestra::Signal(OverseerSignal::ActiveLeaves(update)) => {
+				if let Some(activated) = update.activated {
+					return Ok(Some(activated));
+				}
+			},
+			FromOrchestra::Signal(OverseerSignal::BlockFinalized(_, _)) => {},
+			FromOrchestra::Communication { msg } =>
+			// NOTE: We could technically actually handle a couple of message types, even if
+			// not initialized (e.g. all requests that only query the database). The problem
+			// is, we would deliver potentially outdated information, especially in the event
+			// of bugs where initialization fails for a while (e.g. `SessionInfo`s are not
+			// available). So instead of telling subsystems, everything is fine, because of an
+			// hour old database state, we should rather cancel contained oneshots and delay
+			// finality until we are fully functional.
+			{
+				gum::warn!(
+					target: LOG_TARGET,
+					?msg,
+					"Received msg before first active leaves update. This is not expected - message will be dropped."
+				)
+			},
+		}
+	}
+}
+
+/// Check whether a dispute for the given candidate could be spam.
+///
+/// That is the candidate could be made up.
+pub fn is_potential_spam(
+	scraper: &ChainScraper,
+	vote_state: &CandidateVoteState<CandidateVotes>,
+	candidate_hash: &CandidateHash,
+	is_disabled: impl FnMut(&ValidatorIndex) -> bool,
+) -> bool {
+	let is_disputed = vote_state.is_disputed();
+	let is_included = scraper.is_candidate_included(candidate_hash);
+	let is_backed = scraper.is_candidate_backed(candidate_hash);
+	let is_confirmed = vote_state.is_confirmed();
+	let all_invalid_votes_disabled = vote_state.invalid_votes_all_disabled(is_disabled);
+	let ignore_disabled = !is_confirmed && all_invalid_votes_disabled;
+
+	gum::trace!(
+		target: LOG_TARGET,
+		?candidate_hash,
+		?is_disputed,
+		?is_included,
+		?is_backed,
+		?is_confirmed,
+		?all_invalid_votes_disabled,
+		?ignore_disabled,
+		"Checking for potential spam"
+	);
+
+	(is_disputed && !is_included && !is_backed && !is_confirmed) || ignore_disabled
+}
+
+/// Tell dispute-distribution to send all our votes.
+///
+/// Should be called on startup for all active disputes where there are votes from us already.
+#[overseer::contextbounds(DisputeCoordinator, prefix = self::overseer)]
+async fn send_dispute_messages<Context>(
+	ctx: &mut Context,
+	env: &CandidateEnvironment<'_>,
+	vote_state: &CandidateVoteState<CandidateVotes>,
+) {
+	for own_vote in vote_state.own_votes().into_iter().flatten() {
+		let (validator_index, (kind, sig)) = own_vote;
+		let public_key = if let Some(key) = env.session_info().validators.get(*validator_index) {
+			key.clone()
+		} else {
+			gum::error!(
+				target: LOG_TARGET,
+				?validator_index,
+				session_index = ?env.session_index(),
+				"Could not find our own key in `SessionInfo`"
+			);
+			continue;
+		};
+		let our_vote_signed = SignedDisputeStatement::new_checked(
+			kind.clone(),
+			vote_state.votes().candidate_receipt.hash(),
+			env.session_index(),
+			public_key,
+			sig.clone(),
+		);
+		let our_vote_signed = match our_vote_signed {
+			Ok(signed) => signed,
+			Err(()) => {
+				gum::error!(
+					target: LOG_TARGET,
+					"Checking our own signature failed - db corruption?"
+				);
+				continue;
+			},
+		};
+		let dispute_message = match make_dispute_message(
+			env.session_info(),
+			vote_state.votes(),
+			our_vote_signed,
+			*validator_index,
+		) {
+			Err(err) => {
+				gum::debug!(target: LOG_TARGET, ?err, "Creating dispute message failed.");
+				continue;
+			},
+			Ok(dispute_message) => dispute_message,
+		};
+
+		ctx.send_message(DisputeDistributionMessage::SendDispute(dispute_message)).await;
+	}
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum DisputeMessageCreationError {
+	#[error("There was no opposite vote available")]
+	NoOppositeVote,
+	#[error("Found vote had an invalid validator index that could not be found")]
+	InvalidValidatorIndex,
+	#[error("Statement found in votes had invalid signature.")]
+	InvalidStoredStatement,
+	#[error(transparent)]
+	InvalidStatementCombination(DisputeMessageCheckError),
+}
+
+/// Create a `DisputeMessage` to be sent to `DisputeDistribution`.
+pub fn make_dispute_message(
+	info: &SessionInfo,
+	votes: &CandidateVotes,
+	our_vote: SignedDisputeStatement,
+	our_index: ValidatorIndex,
+) -> std::result::Result<DisputeMessage, DisputeMessageCreationError> {
+	let validators = &info.validators;
+
+	let (valid_statement, valid_index, invalid_statement, invalid_index) =
+		if let DisputeStatement::Valid(_) = our_vote.statement() {
+			let (validator_index, (statement_kind, validator_signature)) =
+				votes.invalid.iter().next().ok_or(DisputeMessageCreationError::NoOppositeVote)?;
+			let other_vote = SignedDisputeStatement::new_checked(
+				DisputeStatement::Invalid(*statement_kind),
+				*our_vote.candidate_hash(),
+				our_vote.session_index(),
+				validators
+					.get(*validator_index)
+					.ok_or(DisputeMessageCreationError::InvalidValidatorIndex)?
+					.clone(),
+				validator_signature.clone(),
+			)
+			.map_err(|()| DisputeMessageCreationError::InvalidStoredStatement)?;
+			(our_vote, our_index, other_vote, *validator_index)
+		} else {
+			let (validator_index, (statement_kind, validator_signature)) = votes
+				.valid
+				.raw()
+				.iter()
+				.next()
+				.ok_or(DisputeMessageCreationError::NoOppositeVote)?;
+			let other_vote = SignedDisputeStatement::new_checked(
+				DisputeStatement::Valid(statement_kind.clone()),
+				*our_vote.candidate_hash(),
+				our_vote.session_index(),
+				validators
+					.get(*validator_index)
+					.ok_or(DisputeMessageCreationError::InvalidValidatorIndex)?
+					.clone(),
+				validator_signature.clone(),
+			)
+			.map_err(|()| DisputeMessageCreationError::InvalidStoredStatement)?;
+			(other_vote, *validator_index, our_vote, our_index)
+		};
+
+	DisputeMessage::from_signed_statements(
+		valid_statement,
+		valid_index,
+		invalid_statement,
+		invalid_index,
+		votes.candidate_receipt.clone(),
+		info,
+	)
+	.map_err(DisputeMessageCreationError::InvalidStatementCombination)
+}

pezkuwi/node/core/dispute-coordinator/src/metrics.rs

@@ -0,0 +1,252 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use pezkuwi_node_subsystem_util::metrics::{self, prometheus};
+
+#[derive(Clone)]
+struct MetricsInner {
+	/// Number of opened disputes.
+	open: prometheus::Counter<prometheus::U64>,
+	/// Votes of all disputes.
+	votes: prometheus::CounterVec<prometheus::U64>,
+	/// Number of approval votes explicitly fetched from approval voting.
+	approval_votes: prometheus::Counter<prometheus::U64>,
+	/// Conclusion across all disputes.
+	concluded: prometheus::CounterVec<prometheus::U64>,
+	/// Number of participations that have been queued.
+	queued_participations: prometheus::CounterVec<prometheus::U64>,
+	/// How long vote cleanup batches take.
+	vote_cleanup_time: prometheus::Histogram,
+	/// Number of refrained participations.
+	refrained_participations: prometheus::Counter<prometheus::U64>,
+	/// Number of unactivated disputes.
+	unactivated: prometheus::Counter<prometheus::U64>,
+	/// Distribution of participation durations.
+	participation_durations: prometheus::Histogram,
+	/// Measures the duration of the full participation pipeline: From when
+	/// a participation request is first queued to when participation in the
+	/// requested dispute is complete.
+	participation_pipeline_durations: prometheus::Histogram,
+	/// Size of participation priority queue
+	participation_priority_queue_size: prometheus::Gauge<prometheus::U64>,
+	/// Size of participation best effort queue
+	participation_best_effort_queue_size: prometheus::Gauge<prometheus::U64>,
+}
+
+/// Candidate validation metrics.
+#[derive(Default, Clone)]
+pub struct Metrics(Option<MetricsInner>);
+
+impl Metrics {
+	pub(crate) fn on_open(&self) {
+		if let Some(metrics) = &self.0 {
+			metrics.open.inc();
+		}
+	}
+
+	pub(crate) fn on_valid_votes(&self, vote_count: u32) {
+		if let Some(metrics) = &self.0 {
+			metrics.votes.with_label_values(&["valid"]).inc_by(vote_count as _);
+		}
+	}
+
+	pub(crate) fn on_invalid_votes(&self, vote_count: u32) {
+		if let Some(metrics) = &self.0 {
+			metrics.votes.with_label_values(&["invalid"]).inc_by(vote_count as _);
+		}
+	}
+
+	pub(crate) fn on_approval_votes(&self, vote_count: u32) {
+		if let Some(metrics) = &self.0 {
+			metrics.approval_votes.inc_by(vote_count as _);
+		}
+	}
+
+	pub(crate) fn on_concluded_valid(&self) {
+		if let Some(metrics) = &self.0 {
+			metrics.concluded.with_label_values(&["valid"]).inc();
+		}
+	}
+
+	pub(crate) fn on_concluded_invalid(&self) {
+		if let Some(metrics) = &self.0 {
+			metrics.concluded.with_label_values(&["invalid"]).inc();
+		}
+	}
+
+	pub(crate) fn on_queued_priority_participation(&self) {
+		if let Some(metrics) = &self.0 {
+			metrics.queued_participations.with_label_values(&["priority"]).inc();
+		}
+	}
+
+	pub(crate) fn on_queued_best_effort_participation(&self) {
+		if let Some(metrics) = &self.0 {
+			metrics.queued_participations.with_label_values(&["best-effort"]).inc();
+		}
+	}
+
+	pub(crate) fn time_vote_cleanup(&self) -> Option<prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.vote_cleanup_time.start_timer())
+	}
+
+	pub(crate) fn on_refrained_participation(&self) {
+		if let Some(metrics) = &self.0 {
+			metrics.refrained_participations.inc();
+		}
+	}
+
+	/// Provide a timer for participation durations which updates on drop.
+	pub(crate) fn time_participation(
+		&self,
+	) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.participation_durations.start_timer())
+	}
+
+	/// Provide a timer for participation pipeline durations which updates on drop.
+	pub(crate) fn time_participation_pipeline(
+		&self,
+	) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0
+			.as_ref()
+			.map(|metrics| metrics.participation_pipeline_durations.start_timer())
+	}
+
+	/// Set the `priority_queue_size` metric
+	pub fn report_priority_queue_size(&self, size: u64) {
+		if let Some(metrics) = &self.0 {
+			metrics.participation_priority_queue_size.set(size);
+		}
+	}
+
+	/// Set the `best_effort_queue_size` metric
+	pub fn report_best_effort_queue_size(&self, size: u64) {
+		if let Some(metrics) = &self.0 {
+			metrics.participation_best_effort_queue_size.set(size);
+		}
+	}
+
+	pub(crate) fn on_unactivated_dispute(&self) {
+		if let Some(metrics) = &self.0 {
+			metrics.unactivated.inc();
+		}
+	}
+}
+
+impl metrics::Metrics for Metrics {
+	fn try_register(registry: &prometheus::Registry) -> Result<Self, prometheus::PrometheusError> {
+		let metrics = MetricsInner {
+			open: prometheus::register(
+				prometheus::Counter::with_opts(prometheus::Opts::new(
+					"pezkuwi_teyrchain_candidate_disputes_total",
+					"Total number of raised disputes.",
+				))?,
+				registry,
+			)?,
+			concluded: prometheus::register(
+				prometheus::CounterVec::new(
+					prometheus::Opts::new(
+						"pezkuwi_teyrchain_candidate_dispute_concluded",
+						"Concluded dispute votes, sorted by candidate is `valid` and `invalid`.",
+					),
+					&["validity"],
+				)?,
+				registry,
+			)?,
+			votes: prometheus::register(
+				prometheus::CounterVec::new(
+					prometheus::Opts::new(
+						"pezkuwi_teyrchain_candidate_dispute_votes",
+						"Accumulated dispute votes, sorted by candidate is `valid` and `invalid`.",
+					),
+					&["validity"],
+				)?,
+				registry,
+			)?,
+			approval_votes: prometheus::register(
+				prometheus::Counter::with_opts(prometheus::Opts::new(
+					"pezkuwi_teyrchain_dispute_candidate_approval_votes_fetched_total",
+					"Number of approval votes fetched from approval voting.",
+				))?,
+				registry,
+			)?,
+			queued_participations: prometheus::register(
+				prometheus::CounterVec::new(
+					prometheus::Opts::new(
+						"pezkuwi_teyrchain_dispute_participations",
+						"Total number of queued participations, grouped by priority and best-effort. (Not every queueing will necessarily lead to an actual participation because of duplicates.)",
+					),
+					&["priority"],
+				)?,
+				registry,
+			)?,
+			vote_cleanup_time: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"pezkuwi_teyrchain_dispute_coordinator_vote_cleanup",
+						"Time spent cleaning up old votes per batch.",
+					)
+					.buckets([0.01, 0.1, 0.5, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0].into()),
+				)?,
+				registry,
+			)?,
+			refrained_participations: prometheus::register(
+			prometheus::Counter::with_opts(
+				prometheus::Opts::new(
+					"pezkuwi_teyrchain_dispute_refrained_participations",
+					"Number of refrained participations. We refrain from participation if all of the following conditions are met: disputed candidate is not included, not backed and not confirmed.",
+				))?,
+				registry,
+			)?,
+			participation_durations: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"pezkuwi_teyrchain_dispute_participation_durations",
+						"Time spent within fn Participation::participate",
+					)
+				)?,
+				registry,
+			)?,
+			participation_pipeline_durations: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"pezkuwi_teyrchain_dispute_participation_pipeline_durations",
+						"Measures the duration of the full participation pipeline: From when a participation request is first queued to when participation in the requested dispute is complete.",
+					)
+				)?,
+				registry,
+			)?,
+			participation_priority_queue_size: prometheus::register(
+				prometheus::Gauge::new("pezkuwi_teyrchain_dispute_participation_priority_queue_size",
+				"Number of disputes waiting for local participation in the priority queue.")?,
+				registry,
+			)?,
+			participation_best_effort_queue_size: prometheus::register(
+				prometheus::Gauge::new("pezkuwi_teyrchain_dispute_participation_best_effort_queue_size",
+				"Number of disputes waiting for local participation in the best effort queue.")?,
+				registry,
+			)?,
+			unactivated: prometheus::register(
+				prometheus::Counter::with_opts(prometheus::Opts::new(
+					"pezkuwi_teyrchain_dispute_unactivated_total",
+					"Total number of disputes that were unactivated due to all raising parties being disabled.",
+				))?,
+				registry,
+			)?,
+		};
+		Ok(Metrics(Some(metrics)))
+	}
+}

pezkuwi/node/core/dispute-coordinator/src/participation/mod.rs

@@ -0,0 +1,447 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::collections::HashSet;
+#[cfg(test)]
+use std::time::Duration;
+
+use futures::{
+	channel::{mpsc, oneshot},
+	FutureExt, SinkExt,
+};
+#[cfg(test)]
+use futures_timer::Delay;
+
+use pezkuwi_node_primitives::ValidationResult;
+use pezkuwi_node_subsystem::{
+	messages::{AvailabilityRecoveryMessage, CandidateValidationMessage, PvfExecKind},
+	overseer, ActiveLeavesUpdate, RecoveryError,
+};
+use pezkuwi_node_subsystem_util::runtime::get_validation_code_by_hash;
+use pezkuwi_primitives::{
+	BlockNumber, CandidateHash, CandidateReceiptV2 as CandidateReceipt, Hash, SessionIndex,
+};
+
+use crate::LOG_TARGET;
+
+use crate::error::{FatalError, FatalResult, Result};
+
+#[cfg(test)]
+mod tests;
+#[cfg(test)]
+pub use tests::{participation_full_happy_path, participation_missing_availability};
+
+mod queues;
+use queues::Queues;
+pub use queues::{ParticipationPriority, ParticipationRequest, QueueError};
+
+use crate::metrics::Metrics;
+use pezkuwi_node_subsystem_util::metrics::prometheus::prometheus;
+
+/// How many participation processes do we want to run in parallel the most.
+///
+/// This should be a relatively low value, while we might have a speedup once we fetched the data,
+/// due to multi-core architectures, but the fetching itself can not be improved by parallel
+/// requests. This means that higher numbers make it harder for a single dispute to resolve fast.
+#[cfg(not(test))]
+const MAX_PARALLEL_PARTICIPATIONS: usize = 3;
+#[cfg(test)]
+pub(crate) const MAX_PARALLEL_PARTICIPATIONS: usize = 1;
+
+/// Keep track of disputes we need to participate in.
+///
+/// - Prioritize and queue participations
+/// - Dequeue participation requests in order and launch participation worker.
+pub struct Participation {
+	/// Participations currently being processed.
+	running_participations: HashSet<CandidateHash>,
+	/// Priority and best effort queues.
+	queue: Queues,
+	/// Sender to be passed to worker tasks.
+	worker_sender: WorkerMessageSender,
+	/// Some recent block for retrieving validation code from chain.
+	recent_block: Option<(BlockNumber, Hash)>,
+	/// Metrics handle cloned from Initialized
+	metrics: Metrics,
+}
+
+/// Message from worker tasks.
+#[derive(Debug)]
+pub struct WorkerMessage(ParticipationStatement);
+
+/// Sender use by worker tasks.
+pub type WorkerMessageSender = mpsc::Sender<WorkerMessage>;
+
+/// Receiver to receive messages from worker tasks.
+pub type WorkerMessageReceiver = mpsc::Receiver<WorkerMessage>;
+
+/// Statement as result of the validation process.
+#[derive(Debug)]
+pub struct ParticipationStatement {
+	/// Relevant session.
+	pub session: SessionIndex,
+	/// The candidate the worker has been spawned for.
+	pub candidate_hash: CandidateHash,
+	/// Used receipt.
+	pub candidate_receipt: CandidateReceipt,
+	/// Actual result.
+	pub outcome: ParticipationOutcome,
+}
+
+/// Outcome of the validation process.
+#[derive(Copy, Clone, Debug)]
+pub enum ParticipationOutcome {
+	/// Candidate was found to be valid.
+	Valid,
+	/// Candidate was found to be invalid.
+	Invalid,
+	/// Candidate was found to be unavailable.
+	Unavailable,
+	/// Something went wrong (bug), details can be found in the logs.
+	Error,
+}
+
+impl ParticipationOutcome {
+	/// If validation was successful, get whether the candidate was valid or invalid.
+	pub fn validity(self) -> Option<bool> {
+		match self {
+			Self::Valid => Some(true),
+			Self::Invalid => Some(false),
+			Self::Unavailable | Self::Error => None,
+		}
+	}
+}
+
+impl WorkerMessage {
+	fn from_request(req: ParticipationRequest, outcome: ParticipationOutcome) -> Self {
+		let session = req.session();
+		let (candidate_hash, candidate_receipt) = req.into_candidate_info();
+		Self(ParticipationStatement { session, candidate_hash, candidate_receipt, outcome })
+	}
+}
+
+#[overseer::contextbounds(DisputeCoordinator, prefix = self::overseer)]
+impl Participation {
+	/// Get ready for managing dispute participation requests.
+	///
+	/// The passed in sender will be used by background workers to communicate back their results.
+	/// The calling context should make sure to call `Participation::on_worker_message()` for the
+	/// received messages.
+	pub fn new(sender: WorkerMessageSender, metrics: Metrics) -> Self {
+		Self {
+			running_participations: HashSet::new(),
+			queue: Queues::new(metrics.clone()),
+			worker_sender: sender,
+			recent_block: None,
+			metrics,
+		}
+	}
+
+	/// Queue a dispute for the node to participate in.
+	///
+	/// If capacity is available right now and we already got some relay chain head via
+	/// `on_active_leaves_update`, the participation will be launched right away.
+	///
+	/// Returns: false, if queues are already full.
+	pub async fn queue_participation<Context>(
+		&mut self,
+		ctx: &mut Context,
+		priority: ParticipationPriority,
+		mut req: ParticipationRequest,
+	) -> Result<()> {
+		// Participation already running - we can ignore that request, discarding its timer:
+		if self.running_participations.contains(req.candidate_hash()) {
+			req.discard_timer();
+			return Ok(());
+		}
+		// Available capacity - participate right away (if we already have a recent block):
+		if let Some((_, h)) = self.recent_block {
+			if self.running_participations.len() < MAX_PARALLEL_PARTICIPATIONS {
+				self.fork_participation(ctx, req, h)?;
+				return Ok(());
+			}
+		}
+		// Out of capacity/no recent block yet - queue:
+		self.queue.queue(ctx.sender(), priority, req).await
+	}
+
+	/// Message from a worker task was received - get the outcome.
+	///
+	/// Call this function to keep participations going and to receive `ParticipationStatement`s.
+	///
+	/// This message has to be called for each received worker message, in order to make sure
+	/// enough participation processes are running at any given time.
+	///
+	/// Returns: The received `ParticipationStatement` or a fatal error, in case
+	/// something went wrong when dequeuing more requests (tasks could not be spawned).
+	pub async fn get_participation_result<Context>(
+		&mut self,
+		ctx: &mut Context,
+		msg: WorkerMessage,
+	) -> FatalResult<ParticipationStatement> {
+		let WorkerMessage(statement) = msg;
+		self.running_participations.remove(&statement.candidate_hash);
+		let recent_block = self.recent_block.expect("We never ever reset recent_block to `None` and we already received a result, so it must have been set before. qed.");
+		self.dequeue_until_capacity(ctx, recent_block.1).await?;
+		Ok(statement)
+	}
+
+	/// Process active leaves update.
+	///
+	/// Make sure we to dequeue participations if that became possible and update most recent
+	/// block.
+	pub async fn process_active_leaves_update<Context>(
+		&mut self,
+		ctx: &mut Context,
+		update: &ActiveLeavesUpdate,
+	) -> FatalResult<()> {
+		if let Some(activated) = &update.activated {
+			match self.recent_block {
+				None => {
+					self.recent_block = Some((activated.number, activated.hash));
+					// Work got potentially unblocked:
+					self.dequeue_until_capacity(ctx, activated.hash).await?;
+				},
+				Some((number, _)) if activated.number > number => {
+					self.recent_block = Some((activated.number, activated.hash));
+				},
+				Some(_) => {},
+			}
+		}
+		Ok(())
+	}
+
+	/// Moving any request concerning the given candidates from best-effort to
+	/// priority, ignoring any candidates that don't have any queued participation requests.
+	pub async fn bump_to_priority_for_candidates<Context>(
+		&mut self,
+		ctx: &mut Context,
+		included_receipts: &Vec<CandidateReceipt>,
+	) -> Result<()> {
+		for receipt in included_receipts {
+			self.queue.prioritize_if_present(ctx.sender(), receipt).await?;
+		}
+		Ok(())
+	}
+
+	/// Dequeue until `MAX_PARALLEL_PARTICIPATIONS` is reached.
+	async fn dequeue_until_capacity<Context>(
+		&mut self,
+		ctx: &mut Context,
+		recent_head: Hash,
+	) -> FatalResult<()> {
+		while self.running_participations.len() < MAX_PARALLEL_PARTICIPATIONS {
+			if let Some(req) = self.queue.dequeue() {
+				self.fork_participation(ctx, req, recent_head)?;
+			} else {
+				break;
+			}
+		}
+		Ok(())
+	}
+
+	/// Fork a participation task in the background.
+	fn fork_participation<Context>(
+		&mut self,
+		ctx: &mut Context,
+		req: ParticipationRequest,
+		recent_head: Hash,
+	) -> FatalResult<()> {
+		gum::trace!(
+			target: LOG_TARGET,
+			candidate_hash = ?req.candidate_hash(),
+			session = req.session(),
+			"Forking participation"
+		);
+
+		let participation_timer = self.metrics.time_participation();
+		if self.running_participations.insert(*req.candidate_hash()) {
+			let sender = ctx.sender().clone();
+			ctx.spawn(
+				"participation-worker",
+				participate(
+					self.worker_sender.clone(),
+					sender,
+					recent_head,
+					req,
+					participation_timer,
+				)
+				.boxed(),
+			)
+			.map_err(FatalError::SpawnFailed)?;
+		}
+		Ok(())
+	}
+}
+
+async fn participate(
+	mut result_sender: WorkerMessageSender,
+	mut sender: impl overseer::DisputeCoordinatorSenderTrait,
+	block_hash: Hash,
+	req: ParticipationRequest, // Sends metric data via request_timer field when dropped
+	_participation_timer: Option<prometheus::HistogramTimer>, // Sends metric data when dropped
+) {
+	#[cfg(test)]
+	// Hack for tests, so we get recovery messages not too early.
+	Delay::new(Duration::from_millis(100)).await;
+	// in order to validate a candidate we need to start by recovering the
+	// available data
+	let (recover_available_data_tx, recover_available_data_rx) = oneshot::channel();
+	sender
+		.send_message(AvailabilityRecoveryMessage::RecoverAvailableData(
+			req.candidate_receipt().clone(),
+			req.session(),
+			None,
+			None,
+			recover_available_data_tx,
+		))
+		.await;
+
+	let available_data = match recover_available_data_rx.await {
+		Err(oneshot::Canceled) => {
+			gum::warn!(
+				target: LOG_TARGET,
+				"`Oneshot` got cancelled when recovering available data {:?}",
+				req.candidate_hash(),
+			);
+			send_result(&mut result_sender, req, ParticipationOutcome::Error).await;
+			return;
+		},
+		Ok(Ok(data)) => data,
+		Ok(Err(RecoveryError::Invalid)) => {
+			gum::debug!(
+				target: LOG_TARGET,
+				candidate_hash = ?req.candidate_hash(),
+				session = req.session(),
+				"Invalid availability data during participation"
+			);
+			// the available data was recovered but it is invalid, therefore we'll
+			// vote negatively for the candidate dispute
+			send_result(&mut result_sender, req, ParticipationOutcome::Invalid).await;
+			return;
+		},
+		Ok(Err(RecoveryError::Unavailable)) | Ok(Err(RecoveryError::ChannelClosed)) => {
+			gum::debug!(
+				target: LOG_TARGET,
+				candidate_hash = ?req.candidate_hash(),
+				session = req.session(),
+				"Can't fetch availability data in participation"
+			);
+			send_result(&mut result_sender, req, ParticipationOutcome::Unavailable).await;
+			return;
+		},
+	};
+
+	// we also need to fetch the validation code which we can reference by its
+	// hash as taken from the candidate descriptor
+	let validation_code = match get_validation_code_by_hash(
+		&mut sender,
+		block_hash,
+		req.candidate_receipt().descriptor.validation_code_hash(),
+	)
+	.await
+	{
+		Ok(Some(code)) => code,
+		Ok(None) => {
+			gum::warn!(
+				target: LOG_TARGET,
+				"Validation code unavailable for code hash {:?} in the state of block {:?}",
+				req.candidate_receipt().descriptor.validation_code_hash(),
+				block_hash,
+			);
+
+			send_result(&mut result_sender, req, ParticipationOutcome::Error).await;
+			return;
+		},
+		Err(err) => {
+			gum::warn!(target: LOG_TARGET, ?err, "Error when fetching validation code.");
+			send_result(&mut result_sender, req, ParticipationOutcome::Error).await;
+			return;
+		},
+	};
+
+	// Issue a request to validate the candidate with the provided exhaustive
+	// parameters
+	//
+	// We use the approval execution timeout because this is intended to
+	// be run outside of backing and therefore should be subject to the
+	// same level of leeway.
+	let (validation_tx, validation_rx) = oneshot::channel();
+	sender
+		.send_message(CandidateValidationMessage::ValidateFromExhaustive {
+			validation_data: available_data.validation_data,
+			validation_code,
+			candidate_receipt: req.candidate_receipt().clone(),
+			pov: available_data.pov,
+			executor_params: req.executor_params(),
+			exec_kind: PvfExecKind::Dispute,
+			response_sender: validation_tx,
+		})
+		.await;
+
+	// we cast votes (either positive or negative) depending on the outcome of
+	// the validation and if valid, whether the commitments hash matches
+	match validation_rx.await {
+		Err(oneshot::Canceled) => {
+			gum::warn!(
+				target: LOG_TARGET,
+				"`Oneshot` got cancelled when validating candidate {:?}",
+				req.candidate_hash(),
+			);
+			send_result(&mut result_sender, req, ParticipationOutcome::Error).await;
+			return;
+		},
+		Ok(Err(err)) => {
+			gum::warn!(
+				target: LOG_TARGET,
+				"Candidate {:?} validation failed with: {:?}",
+				req.candidate_hash(),
+				err,
+			);
+
+			send_result(&mut result_sender, req, ParticipationOutcome::Error).await;
+		},
+
+		Ok(Ok(ValidationResult::Invalid(invalid))) => {
+			gum::warn!(
+				target: LOG_TARGET,
+				"Candidate {:?} considered invalid: {:?}",
+				req.candidate_hash(),
+				invalid,
+			);
+
+			send_result(&mut result_sender, req, ParticipationOutcome::Invalid).await;
+		},
+		Ok(Ok(ValidationResult::Valid(_, _))) => {
+			send_result(&mut result_sender, req, ParticipationOutcome::Valid).await;
+		},
+	}
+}
+
+/// Helper function for sending the result back and report any error.
+async fn send_result(
+	sender: &mut WorkerMessageSender,
+	req: ParticipationRequest,
+	outcome: ParticipationOutcome,
+) {
+	if let Err(err) = sender.feed(WorkerMessage::from_request(req, outcome)).await {
+		gum::error!(
+			target: LOG_TARGET,
+			?err,
+			"Sending back participation result failed. Dispute coordinator not working properly!"
+		);
+	}
+}

pezkuwi/node/core/dispute-coordinator/src/participation/queues/mod.rs

@@ -0,0 +1,474 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+	cmp::Ordering,
+	collections::{btree_map::Entry, BTreeMap},
+};
+
+use futures::channel::oneshot;
+use pezkuwi_node_subsystem::{messages::ChainApiMessage, overseer};
+use pezkuwi_primitives::{
+	BlockNumber, CandidateHash, CandidateReceiptV2 as CandidateReceipt, ExecutorParams, Hash,
+	SessionIndex,
+};
+
+use crate::{
+	error::{FatalError, FatalResult, Result},
+	LOG_TARGET,
+};
+
+use crate::metrics::Metrics;
+use pezkuwi_node_subsystem_util::metrics::prometheus::prometheus;
+
+#[cfg(test)]
+mod tests;
+
+/// How many potential garbage disputes we want to queue, before starting to drop requests.
+#[cfg(not(test))]
+const BEST_EFFORT_QUEUE_SIZE: usize = 100;
+#[cfg(test)]
+const BEST_EFFORT_QUEUE_SIZE: usize = 3;
+
+/// How many priority disputes can be queued.
+///
+/// Once the queue exceeds that size, we will start to drop the newest participation requests in
+/// the queue. Note that for each vote import the request will be re-added, if there is free
+/// capacity. This limit just serves as a safe guard, it is not expected to ever really be reached.
+///
+/// For 100 teyrchains, this would allow for every single candidate in 100 blocks on
+/// two forks to get disputed, which should be plenty to deal with any realistic attack.
+#[cfg(not(test))]
+const PRIORITY_QUEUE_SIZE: usize = 20_000;
+#[cfg(test)]
+const PRIORITY_QUEUE_SIZE: usize = 2;
+
+/// Queues for dispute participation.
+/// In both queues we have a strict ordering of candidates and participation will
+/// happen in that order. Refer to `CandidateComparator` for details on the ordering.
+pub struct Queues {
+	/// Set of best effort participation requests.
+	best_effort: BTreeMap<CandidateComparator, ParticipationRequest>,
+
+	/// Priority queue.
+	priority: BTreeMap<CandidateComparator, ParticipationRequest>,
+
+	/// Handle for recording queues data in metrics
+	metrics: Metrics,
+}
+
+/// A dispute participation request that can be queued.
+#[derive(Debug)]
+pub struct ParticipationRequest {
+	candidate_hash: CandidateHash,
+	candidate_receipt: CandidateReceipt,
+	session: SessionIndex,
+	executor_params: ExecutorParams,
+	request_timer: Option<prometheus::HistogramTimer>, // Sends metric data when request is dropped
+}
+
+/// Whether a `ParticipationRequest` should be put on best-effort or the priority queue.
+#[derive(Debug)]
+pub enum ParticipationPriority {
+	BestEffort,
+	Priority,
+}
+
+impl ParticipationPriority {
+	/// Create `ParticipationPriority` with either `Priority`
+	///
+	/// or `BestEffort`.
+	pub fn with_priority_if(is_priority: bool) -> Self {
+		if is_priority {
+			Self::Priority
+		} else {
+			Self::BestEffort
+		}
+	}
+
+	/// Whether or not this is a priority entry.
+	///
+	/// If false, it is best effort.
+	pub fn is_priority(&self) -> bool {
+		match self {
+			Self::Priority => true,
+			Self::BestEffort => false,
+		}
+	}
+}
+
+/// What can go wrong when queuing a request.
+#[derive(Debug, thiserror::Error)]
+pub enum QueueError {
+	#[error("Request could not be queued, because best effort queue was already full.")]
+	BestEffortFull,
+	#[error("Request could not be queued, because priority queue was already full.")]
+	PriorityFull,
+}
+
+impl ParticipationRequest {
+	/// Create a new `ParticipationRequest` to be queued.
+	pub fn new(
+		candidate_receipt: CandidateReceipt,
+		session: SessionIndex,
+		executor_params: ExecutorParams,
+		request_timer: Option<prometheus::HistogramTimer>,
+	) -> Self {
+		Self {
+			candidate_hash: candidate_receipt.hash(),
+			candidate_receipt,
+			session,
+			executor_params,
+			request_timer,
+		}
+	}
+
+	pub fn candidate_receipt(&'_ self) -> &'_ CandidateReceipt {
+		&self.candidate_receipt
+	}
+	pub fn candidate_hash(&'_ self) -> &'_ CandidateHash {
+		&self.candidate_hash
+	}
+	pub fn session(&self) -> SessionIndex {
+		self.session
+	}
+	pub fn executor_params(&self) -> ExecutorParams {
+		self.executor_params.clone()
+	}
+	pub fn discard_timer(&mut self) {
+		if let Some(timer) = self.request_timer.take() {
+			timer.stop_and_discard();
+		}
+	}
+	pub fn into_candidate_info(self) -> (CandidateHash, CandidateReceipt) {
+		let Self { candidate_hash, candidate_receipt, .. } = self;
+		(candidate_hash, candidate_receipt)
+	}
+}
+
+// We want to compare and clone participation requests in unit tests, so we
+// only implement Eq and Clone for tests.
+#[cfg(test)]
+impl PartialEq for ParticipationRequest {
+	fn eq(&self, other: &Self) -> bool {
+		let ParticipationRequest {
+			candidate_receipt,
+			candidate_hash,
+			session,
+			executor_params,
+			request_timer: _,
+		} = self;
+		candidate_receipt == other.candidate_receipt() &&
+			candidate_hash == other.candidate_hash() &&
+			*session == other.session() &&
+			executor_params.hash() == other.executor_params.hash()
+	}
+}
+#[cfg(test)]
+impl Eq for ParticipationRequest {}
+
+impl Queues {
+	/// Create new `Queues`.
+	pub fn new(metrics: Metrics) -> Self {
+		Self { best_effort: BTreeMap::new(), priority: BTreeMap::new(), metrics }
+	}
+
+	/// Will put message in queue, either priority or best effort depending on priority.
+	///
+	/// If the message was already previously present on best effort, it will be moved to priority
+	/// if it is considered priority now.
+	///
+	/// Returns error in case a queue was found full already.
+	pub async fn queue(
+		&mut self,
+		sender: &mut impl overseer::DisputeCoordinatorSenderTrait,
+		priority: ParticipationPriority,
+		req: ParticipationRequest,
+	) -> Result<()> {
+		let comparator = CandidateComparator::new(sender, &req.candidate_receipt).await?;
+
+		self.queue_with_comparator(comparator, priority, req)?;
+		Ok(())
+	}
+
+	/// Get the next best request for dispute participation if any.
+	/// First the priority queue is considered and then the best effort one.
+	pub fn dequeue(&mut self) -> Option<ParticipationRequest> {
+		if let Some(req) = self.pop_priority() {
+			self.metrics.report_priority_queue_size(self.priority.len() as u64);
+			return Some(req.1);
+		}
+		if let Some(req) = self.pop_best_effort() {
+			self.metrics.report_best_effort_queue_size(self.best_effort.len() as u64);
+			return Some(req.1);
+		}
+		None
+	}
+
+	/// Reprioritizes any participation requests pertaining to the
+	/// passed candidates from best effort to priority.
+	pub async fn prioritize_if_present(
+		&mut self,
+		sender: &mut impl overseer::DisputeCoordinatorSenderTrait,
+		receipt: &CandidateReceipt,
+	) -> Result<()> {
+		let comparator = CandidateComparator::new(sender, receipt).await?;
+		self.prioritize_with_comparator(comparator)?;
+		Ok(())
+	}
+
+	fn prioritize_with_comparator(
+		&mut self,
+		comparator: CandidateComparator,
+	) -> std::result::Result<(), QueueError> {
+		if self.priority.len() >= PRIORITY_QUEUE_SIZE {
+			return Err(QueueError::PriorityFull);
+		}
+		if let Some(request) = self.best_effort.remove(&comparator) {
+			self.priority.insert(comparator, request);
+			// Report changes to both queue sizes
+			self.metrics.report_priority_queue_size(self.priority.len() as u64);
+			self.metrics.report_best_effort_queue_size(self.best_effort.len() as u64);
+		}
+		Ok(())
+	}
+
+	/// Will put message in queue, either priority or best effort depending on priority.
+	///
+	/// If the message was already previously present on best effort, it will be moved to priority
+	/// if it is considered priority now.
+	///
+	/// Returns error in case a queue was found full already.
+	///
+	///  # Request timers
+	///
+	/// [`ParticipationRequest`]s contain request timers.
+	/// Where an old request would be replaced by a new one, we keep the old request.
+	/// This prevents request timers from resetting on each new request.
+	fn queue_with_comparator(
+		&mut self,
+		comparator: CandidateComparator,
+		priority: ParticipationPriority,
+		mut req: ParticipationRequest,
+	) -> std::result::Result<(), QueueError> {
+		if priority.is_priority() {
+			if self.priority.len() >= PRIORITY_QUEUE_SIZE {
+				return Err(QueueError::PriorityFull);
+			}
+			// Remove any best effort entry, using it to replace our new
+			// request.
+			if let Some(older_request) = self.best_effort.remove(&comparator) {
+				req.discard_timer();
+				req = older_request;
+			}
+			// Keeping old request if any.
+			match self.priority.entry(comparator) {
+				Entry::Occupied(_) => req.discard_timer(),
+				Entry::Vacant(vac) => {
+					gum::trace!(
+						target: LOG_TARGET,
+						candidate_hash = ?req.candidate_hash(),
+						"Added to priority participation queue"
+					);
+					vac.insert(req);
+				},
+			}
+			self.metrics.report_priority_queue_size(self.priority.len() as u64);
+			self.metrics.report_best_effort_queue_size(self.best_effort.len() as u64);
+		} else {
+			if self.priority.contains_key(&comparator) {
+				// The candidate is already in priority queue - don't
+				// add in in best effort too.
+				return Ok(());
+			}
+			if self.best_effort.len() >= BEST_EFFORT_QUEUE_SIZE {
+				return Err(QueueError::BestEffortFull);
+			}
+			// Keeping old request if any.
+			match self.best_effort.entry(comparator) {
+				Entry::Occupied(_) => req.discard_timer(),
+				Entry::Vacant(vac) => {
+					gum::trace!(
+						target: LOG_TARGET,
+						candidate_hash = ?req.candidate_hash(),
+						"Added to best effort participation queue"
+					);
+					vac.insert(req);
+				},
+			}
+			self.metrics.report_best_effort_queue_size(self.best_effort.len() as u64);
+		}
+		Ok(())
+	}
+
+	/// Get best from the best effort queue.
+	fn pop_best_effort(&mut self) -> Option<(CandidateComparator, ParticipationRequest)> {
+		return Self::pop_impl(&mut self.best_effort);
+	}
+
+	/// Get best priority queue entry.
+	fn pop_priority(&mut self) -> Option<(CandidateComparator, ParticipationRequest)> {
+		return Self::pop_impl(&mut self.priority);
+	}
+
+	// `pop_best_effort` and `pop_priority` do the same but on different `BTreeMap`s. This function
+	// has the extracted implementation
+	fn pop_impl(
+		target: &mut BTreeMap<CandidateComparator, ParticipationRequest>,
+	) -> Option<(CandidateComparator, ParticipationRequest)> {
+		// Once https://github.com/rust-lang/rust/issues/62924 is there, we can use a simple:
+		// target.pop_first().
+		if let Some((comparator, _)) = target.iter().next() {
+			let comparator = *comparator;
+			target
+				.remove(&comparator)
+				.map(|participation_request| (comparator, participation_request))
+		} else {
+			None
+		}
+	}
+}
+
+/// `Comparator` for ordering of disputes for candidates.
+///
+/// This `comparator` makes it possible to order disputes based on age and to ensure some fairness
+/// between chains in case of equally old disputes.
+///
+/// Objective ordering between nodes is important in case of lots disputes, so nodes will pull in
+/// the same direction and work on resolving the same disputes first. This ensures that we will
+/// conclude some disputes, even if there are lots of them. While any objective ordering would
+/// suffice for this goal, ordering by age ensures we are not only resolving disputes, but also
+/// resolve the oldest one first, which are also the most urgent and important ones to resolve.
+///
+/// Note: That by `oldest` we mean oldest in terms of relay chain block number, for any block
+/// number that has not yet been finalized. If a block has been finalized already it should be
+/// treated as low priority when it comes to disputes, as even in the case of a negative outcome,
+/// we are already too late. The ordering mechanism here serves to prevent this from happening in
+/// the first place.
+#[derive(Copy, Clone)]
+#[cfg_attr(test, derive(Debug))]
+struct CandidateComparator {
+	/// Block number of the relay parent. It's wrapped in an `Option<>` because there are cases
+	/// when it can't be obtained. For example when the node is lagging behind and new leaves are
+	/// received with a slight delay. Candidates with unknown relay parent are treated with the
+	/// lowest priority.
+	///
+	/// The order enforced by `CandidateComparator` is important because we want to participate in
+	/// the oldest disputes first.
+	///
+	/// Note: In theory it would make more sense to use the `BlockNumber` of the including
+	/// block, as inclusion time is the actual relevant event when it comes to ordering. The
+	/// problem is, that a candidate can get included multiple times on forks, so the `BlockNumber`
+	/// of the including block is not unique. We could theoretically work around that problem, by
+	/// just using the lowest `BlockNumber` of all available including blocks - the problem is,
+	/// that is not stable. If a new fork appears after the fact, we would start ordering the same
+	/// candidate differently, which would result in the same candidate getting queued twice.
+	relay_parent_block_number: Option<BlockNumber>,
+	/// By adding the `CandidateHash`, we can guarantee a unique ordering across candidates with
+	/// the same relay parent block number. Candidates without `relay_parent_block_number` are
+	/// ordered by the `candidate_hash` (and treated with the lowest priority, as already
+	/// mentioned).
+	candidate_hash: CandidateHash,
+}
+
+impl CandidateComparator {
+	/// Create a candidate comparator based on given (fake) values.
+	///
+	/// Useful for testing.
+	#[cfg(test)]
+	pub fn new_dummy(block_number: Option<BlockNumber>, candidate_hash: CandidateHash) -> Self {
+		Self { relay_parent_block_number: block_number, candidate_hash }
+	}
+
+	/// Create a candidate comparator for a given candidate.
+	///
+	/// Returns:
+	/// 	- `Ok(CandidateComparator{Some(relay_parent_block_number), candidate_hash})` when the
+	/// 	relay parent can be obtained. This is the happy case.
+	/// - `Ok(CandidateComparator{None, candidate_hash})` in case the candidate's relay parent
+	/// 	can't be obtained.
+	/// 	- `FatalError` in case the chain API call fails with an unexpected error.
+	pub async fn new(
+		sender: &mut impl overseer::DisputeCoordinatorSenderTrait,
+		candidate: &CandidateReceipt,
+	) -> FatalResult<Self> {
+		let candidate_hash = candidate.hash();
+		let n = get_block_number(sender, candidate.descriptor().relay_parent()).await?;
+
+		if n.is_none() {
+			gum::warn!(
+				target: LOG_TARGET,
+				candidate_hash = ?candidate_hash,
+				"Candidate's relay_parent could not be found via chain API - `CandidateComparator` \
+				with an empty relay parent block number will be provided!"
+			);
+		}
+
+		Ok(CandidateComparator { relay_parent_block_number: n, candidate_hash })
+	}
+}
+
+impl PartialEq for CandidateComparator {
+	fn eq(&self, other: &CandidateComparator) -> bool {
+		Ordering::Equal == self.cmp(other)
+	}
+}
+
+impl Eq for CandidateComparator {}
+
+impl PartialOrd for CandidateComparator {
+	fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+		Some(self.cmp(other))
+	}
+}
+
+impl Ord for CandidateComparator {
+	fn cmp(&self, other: &Self) -> Ordering {
+		return match (self.relay_parent_block_number, other.relay_parent_block_number) {
+			(None, None) => {
+				// No relay parents for both -> compare hashes
+				self.candidate_hash.cmp(&other.candidate_hash)
+			},
+			(Some(self_relay_parent_block_num), Some(other_relay_parent_block_num)) => {
+				match self_relay_parent_block_num.cmp(&other_relay_parent_block_num) {
+					// if the relay parent is the same for both -> compare hashes
+					Ordering::Equal => self.candidate_hash.cmp(&other.candidate_hash),
+					// if not - return the result from comparing the relay parent block numbers
+					o => return o,
+				}
+			},
+			(Some(_), None) => {
+				// Candidates with known relay parents are always with priority
+				Ordering::Less
+			},
+			(None, Some(_)) => {
+				// Ditto
+				Ordering::Greater
+			},
+		};
+	}
+}
+
+async fn get_block_number(
+	sender: &mut impl overseer::DisputeCoordinatorSenderTrait,
+	relay_parent: Hash,
+) -> FatalResult<Option<BlockNumber>> {
+	let (tx, rx) = oneshot::channel();
+	sender.send_message(ChainApiMessage::BlockNumber(relay_parent, tx)).await;
+	rx.await
+		.map_err(|_| FatalError::ChainApiSenderDropped)?
+		.map_err(FatalError::ChainApiAncestors)
+}

pezkuwi/node/core/dispute-coordinator/src/participation/queues/tests.rs

@@ -0,0 +1,214 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::{metrics::Metrics, ParticipationPriority};
+use assert_matches::assert_matches;
+use pezkuwi_primitives::{BlockNumber, Hash};
+use pezkuwi_primitives_test_helpers::{dummy_candidate_receipt_v2, dummy_hash};
+
+use super::{CandidateComparator, ParticipationRequest, QueueError, Queues};
+
+/// Make a `ParticipationRequest` based on the given commitments hash.
+fn make_participation_request(hash: Hash) -> ParticipationRequest {
+	let mut receipt = dummy_candidate_receipt_v2(dummy_hash());
+	// make it differ:
+	receipt.commitments_hash = hash;
+	let request_timer = Metrics::default().time_participation_pipeline();
+	ParticipationRequest::new(receipt, 1, Default::default(), request_timer)
+}
+
+/// Make dummy comparator for request, based on the given block number.
+fn make_dummy_comparator(
+	req: &ParticipationRequest,
+	relay_parent: Option<BlockNumber>,
+) -> CandidateComparator {
+	CandidateComparator::new_dummy(relay_parent, *req.candidate_hash())
+}
+
+/// Make a partial clone of the given `ParticipationRequest`, just missing
+/// the `request_timer` field. We prefer this helper to implementing Clone
+/// for `ParticipationRequest`, since we only clone requests in tests.
+fn clone_request(request: &ParticipationRequest) -> ParticipationRequest {
+	ParticipationRequest {
+		candidate_receipt: request.candidate_receipt.clone(),
+		candidate_hash: request.candidate_hash,
+		session: request.session,
+		executor_params: request.executor_params.clone(),
+		request_timer: None,
+	}
+}
+
+/// Check that dequeuing acknowledges order.
+///
+/// Any priority item will be dequeued before any best effort items, priority and best effort with
+/// known parent block number items will be processed in order. Best effort items without known
+/// parent block number should be treated with lowest priority.
+#[test]
+fn ordering_works_as_expected() {
+	let metrics = Metrics::default();
+	let mut queue = Queues::new(metrics.clone());
+	let req1 = make_participation_request(Hash::repeat_byte(0x01));
+	let req_prio = make_participation_request(Hash::repeat_byte(0x02));
+	let req3 = make_participation_request(Hash::repeat_byte(0x03));
+	let req_prio_2 = make_participation_request(Hash::repeat_byte(0x04));
+	let req5_unknown_parent = make_participation_request(Hash::repeat_byte(0x05));
+	let req_full = make_participation_request(Hash::repeat_byte(0x06));
+	let req_prio_full = make_participation_request(Hash::repeat_byte(0x07));
+	queue
+		.queue_with_comparator(
+			make_dummy_comparator(&req1, Some(1)),
+			ParticipationPriority::BestEffort,
+			clone_request(&req1),
+		)
+		.unwrap();
+	queue
+		.queue_with_comparator(
+			make_dummy_comparator(&req_prio, Some(1)),
+			ParticipationPriority::Priority,
+			clone_request(&req_prio),
+		)
+		.unwrap();
+	queue
+		.queue_with_comparator(
+			make_dummy_comparator(&req3, Some(2)),
+			ParticipationPriority::BestEffort,
+			clone_request(&req3),
+		)
+		.unwrap();
+	queue
+		.queue_with_comparator(
+			make_dummy_comparator(&req_prio_2, Some(2)),
+			ParticipationPriority::Priority,
+			clone_request(&req_prio_2),
+		)
+		.unwrap();
+	queue
+		.queue_with_comparator(
+			make_dummy_comparator(&req5_unknown_parent, None),
+			ParticipationPriority::BestEffort,
+			clone_request(&req5_unknown_parent),
+		)
+		.unwrap();
+	assert_matches!(
+		queue.queue_with_comparator(
+			make_dummy_comparator(&req_prio_full, Some(3)),
+			ParticipationPriority::Priority,
+			req_prio_full,
+		),
+		Err(QueueError::PriorityFull)
+	);
+	assert_matches!(
+		queue.queue_with_comparator(
+			make_dummy_comparator(&req_full, Some(3)),
+			ParticipationPriority::BestEffort,
+			req_full,
+		),
+		Err(QueueError::BestEffortFull)
+	);
+
+	// Prioritized queue is ordered correctly
+	assert_eq!(queue.dequeue(), Some(req_prio));
+	assert_eq!(queue.dequeue(), Some(req_prio_2));
+	// So is the best-effort
+	assert_eq!(queue.dequeue(), Some(req1));
+	assert_eq!(queue.dequeue(), Some(req3));
+	assert_eq!(queue.dequeue(), Some(req5_unknown_parent));
+
+	assert_matches!(queue.dequeue(), None);
+}
+
+/// No matter how often a candidate gets queued, it should only ever get dequeued once.
+#[test]
+fn candidate_is_only_dequeued_once() {
+	let metrics = Metrics::default();
+	let mut queue = Queues::new(metrics.clone());
+	let req1 = make_participation_request(Hash::repeat_byte(0x01));
+	let req_prio = make_participation_request(Hash::repeat_byte(0x02));
+	let req_best_effort_then_prio = make_participation_request(Hash::repeat_byte(0x03));
+	let req_prio_then_best_effort = make_participation_request(Hash::repeat_byte(0x04));
+
+	queue
+		.queue_with_comparator(
+			make_dummy_comparator(&req1, None),
+			ParticipationPriority::BestEffort,
+			clone_request(&req1),
+		)
+		.unwrap();
+	queue
+		.queue_with_comparator(
+			make_dummy_comparator(&req_prio, Some(1)),
+			ParticipationPriority::Priority,
+			clone_request(&req_prio),
+		)
+		.unwrap();
+	// Insert same best effort again:
+	queue
+		.queue_with_comparator(
+			make_dummy_comparator(&req1, None),
+			ParticipationPriority::BestEffort,
+			clone_request(&req1),
+		)
+		.unwrap();
+	// insert same prio again:
+	queue
+		.queue_with_comparator(
+			make_dummy_comparator(&req_prio, Some(1)),
+			ParticipationPriority::Priority,
+			clone_request(&req_prio),
+		)
+		.unwrap();
+	// Insert first as best effort:
+	queue
+		.queue_with_comparator(
+			make_dummy_comparator(&req_best_effort_then_prio, Some(2)),
+			ParticipationPriority::BestEffort,
+			clone_request(&req_best_effort_then_prio),
+		)
+		.unwrap();
+	// Then as prio:
+	queue
+		.queue_with_comparator(
+			make_dummy_comparator(&req_best_effort_then_prio, Some(2)),
+			ParticipationPriority::Priority,
+			clone_request(&req_best_effort_then_prio),
+		)
+		.unwrap();
+
+	// Make space in prio:
+	assert_eq!(queue.dequeue(), Some(req_prio));
+
+	// Insert first as prio:
+	queue
+		.queue_with_comparator(
+			make_dummy_comparator(&req_prio_then_best_effort, Some(3)),
+			ParticipationPriority::Priority,
+			clone_request(&req_prio_then_best_effort),
+		)
+		.unwrap();
+	// Then as best effort:
+	queue
+		.queue_with_comparator(
+			make_dummy_comparator(&req_prio_then_best_effort, Some(3)),
+			ParticipationPriority::BestEffort,
+			clone_request(&req_prio_then_best_effort),
+		)
+		.unwrap();
+
+	assert_eq!(queue.dequeue(), Some(req_best_effort_then_prio));
+	assert_eq!(queue.dequeue(), Some(req_prio_then_best_effort));
+	assert_eq!(queue.dequeue(), Some(req1));
+	assert_matches!(queue.dequeue(), None);
+}

pezkuwi/node/core/dispute-coordinator/src/participation/tests.rs

@@ -0,0 +1,543 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use assert_matches::assert_matches;
+use futures::StreamExt;
+use pezkuwi_node_subsystem_util::TimeoutExt;
+use std::{sync::Arc, time::Duration};
+
+use sp_core::testing::TaskExecutor;
+
+use super::*;
+use codec::Encode;
+use pezkuwi_node_primitives::{AvailableData, BlockData, InvalidCandidate, PoV};
+use pezkuwi_node_subsystem::{
+	messages::{
+		AllMessages, ChainApiMessage, DisputeCoordinatorMessage, PvfExecKind, RuntimeApiMessage,
+		RuntimeApiRequest,
+	},
+	ActiveLeavesUpdate, SpawnGlue,
+};
+use pezkuwi_node_subsystem_test_helpers::{
+	make_subsystem_context, mock::new_leaf, TestSubsystemContext, TestSubsystemContextHandle,
+};
+use pezkuwi_primitives::{
+	BlakeTwo256, CandidateCommitments, HashT, Header, PersistedValidationData, ValidationCode,
+};
+use pezkuwi_primitives_test_helpers::{
+	dummy_candidate_commitments, dummy_candidate_receipt_bad_sig, dummy_digest, dummy_hash,
+};
+
+type VirtualOverseer = TestSubsystemContextHandle<DisputeCoordinatorMessage>;
+
+pub fn make_our_subsystem_context<S>(
+	spawner: S,
+) -> (
+	TestSubsystemContext<DisputeCoordinatorMessage, SpawnGlue<S>>,
+	TestSubsystemContextHandle<DisputeCoordinatorMessage>,
+) {
+	make_subsystem_context(spawner)
+}
+
+#[overseer::contextbounds(DisputeCoordinator, prefix = self::overseer)]
+async fn participate<Context>(ctx: &mut Context, participation: &mut Participation) -> Result<()> {
+	let commitments = CandidateCommitments::default();
+	participate_with_commitments_hash(ctx, participation, commitments.hash()).await
+}
+
+#[overseer::contextbounds(DisputeCoordinator, prefix = self::overseer)]
+async fn participate_with_commitments_hash<Context>(
+	ctx: &mut Context,
+	participation: &mut Participation,
+	commitments_hash: Hash,
+) -> Result<()> {
+	let candidate_receipt = {
+		let mut receipt = dummy_candidate_receipt_bad_sig(dummy_hash(), dummy_hash());
+		receipt.commitments_hash = commitments_hash;
+		receipt
+	}
+	.into();
+	let session = 1;
+
+	let request_timer = participation.metrics.time_participation_pipeline();
+	let req =
+		ParticipationRequest::new(candidate_receipt, session, Default::default(), request_timer);
+
+	participation
+		.queue_participation(ctx, ParticipationPriority::BestEffort, req)
+		.await
+}
+
+#[overseer::contextbounds(DisputeCoordinator, prefix = self::overseer)]
+async fn activate_leaf<Context>(
+	ctx: &mut Context,
+	participation: &mut Participation,
+	block_number: BlockNumber,
+) -> FatalResult<()> {
+	let block_header = Header {
+		parent_hash: BlakeTwo256::hash(&block_number.encode()),
+		number: block_number,
+		digest: dummy_digest(),
+		state_root: dummy_hash(),
+		extrinsics_root: dummy_hash(),
+	};
+
+	let block_hash = block_header.hash();
+
+	participation
+		.process_active_leaves_update(
+			ctx,
+			&ActiveLeavesUpdate::start_work(new_leaf(block_hash, block_number)),
+		)
+		.await
+}
+
+/// Full participation happy path as seen via the overseer.
+pub async fn participation_full_happy_path(
+	ctx_handle: &mut VirtualOverseer,
+	expected_commitments_hash: Hash,
+) {
+	recover_available_data(ctx_handle).await;
+	fetch_validation_code(ctx_handle).await;
+
+	assert_matches!(
+	ctx_handle.recv().await,
+	AllMessages::CandidateValidation(
+		CandidateValidationMessage::ValidateFromExhaustive { candidate_receipt, exec_kind, response_sender, .. }
+		) if exec_kind == PvfExecKind::Dispute => {
+			if expected_commitments_hash != candidate_receipt.commitments_hash {
+				response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::CommitmentsHashMismatch))).unwrap();
+			} else {
+				response_sender.send(Ok(ValidationResult::Valid(dummy_candidate_commitments(None), PersistedValidationData::default()))).unwrap();
+			}
+	},
+	"overseer did not receive candidate validation message",
+	);
+}
+
+/// Full participation with failing availability recovery.
+pub async fn participation_missing_availability(ctx_handle: &mut VirtualOverseer) {
+	assert_matches!(
+		ctx_handle.recv().await,
+		AllMessages::AvailabilityRecovery(
+			AvailabilityRecoveryMessage::RecoverAvailableData(_, _, _, _, tx)
+		) => {
+			tx.send(Err(RecoveryError::Unavailable)).unwrap();
+		},
+		"overseer did not receive recover available data message",
+	);
+}
+
+async fn recover_available_data(virtual_overseer: &mut VirtualOverseer) {
+	let pov_block = PoV { block_data: BlockData(Vec::new()) };
+
+	let available_data = AvailableData {
+		pov: Arc::new(pov_block),
+		validation_data: PersistedValidationData::default(),
+	};
+
+	assert_matches!(
+		virtual_overseer.recv().await,
+		AllMessages::AvailabilityRecovery(
+			AvailabilityRecoveryMessage::RecoverAvailableData(_, _, _, _, tx)
+		) => {
+			tx.send(Ok(available_data)).unwrap();
+		},
+		"overseer did not receive recover available data message",
+	);
+}
+
+/// Handles validation code fetch, returns the received relay parent hash.
+async fn fetch_validation_code(virtual_overseer: &mut VirtualOverseer) -> Hash {
+	let validation_code = ValidationCode(Vec::new());
+
+	assert_matches!(
+		virtual_overseer.recv().await,
+		AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+			hash,
+			RuntimeApiRequest::ValidationCodeByHash(
+				_,
+				tx,
+			)
+		)) => {
+			tx.send(Ok(Some(validation_code))).unwrap();
+			hash
+		},
+		"overseer did not receive runtime API request for validation code",
+	)
+}
+
+#[test]
+fn same_req_wont_get_queued_if_participation_is_already_running() {
+	futures::executor::block_on(async {
+		let (mut ctx, mut ctx_handle) = make_our_subsystem_context(TaskExecutor::new());
+
+		let (sender, mut worker_receiver) = mpsc::channel(1);
+		let mut participation = Participation::new(sender, Metrics::default());
+		activate_leaf(&mut ctx, &mut participation, 10).await.unwrap();
+		participate(&mut ctx, &mut participation).await.unwrap();
+		for _ in 0..MAX_PARALLEL_PARTICIPATIONS {
+			participate(&mut ctx, &mut participation).await.unwrap();
+		}
+
+		assert_matches!(
+			ctx_handle.recv().await,
+			AllMessages::AvailabilityRecovery(
+				AvailabilityRecoveryMessage::RecoverAvailableData(_, _, _, _, tx)
+			) => {
+				tx.send(Err(RecoveryError::Unavailable)).unwrap();
+			},
+			"overseer did not receive recover available data message",
+		);
+
+		let result = participation
+			.get_participation_result(&mut ctx, worker_receiver.next().await.unwrap())
+			.await
+			.unwrap();
+
+		assert_matches!(
+			result.outcome,
+			ParticipationOutcome::Unavailable => {}
+		);
+
+		// we should not have any further results nor recovery requests:
+		assert_matches!(ctx_handle.recv().timeout(Duration::from_millis(10)).await, None);
+		assert_matches!(worker_receiver.next().timeout(Duration::from_millis(10)).await, None);
+	})
+}
+
+#[test]
+fn reqs_get_queued_when_out_of_capacity() {
+	let (mut ctx, mut ctx_handle) = make_our_subsystem_context(TaskExecutor::new());
+
+	let test = async {
+		let (sender, mut worker_receiver) = mpsc::channel(1);
+		let mut participation = Participation::new(sender, Metrics::default());
+		activate_leaf(&mut ctx, &mut participation, 10).await.unwrap();
+		participate(&mut ctx, &mut participation).await.unwrap();
+		for i in 0..MAX_PARALLEL_PARTICIPATIONS {
+			participate_with_commitments_hash(
+				&mut ctx,
+				&mut participation,
+				Hash::repeat_byte(i as _),
+			)
+			.await
+			.unwrap();
+		}
+
+		for _ in 0..MAX_PARALLEL_PARTICIPATIONS + 1 {
+			let result = participation
+				.get_participation_result(&mut ctx, worker_receiver.next().await.unwrap())
+				.await
+				.unwrap();
+			assert_matches!(
+				result.outcome,
+				ParticipationOutcome::Unavailable => {}
+			);
+		}
+		// we should not have any further recovery requests:
+		assert_matches!(worker_receiver.next().timeout(Duration::from_millis(10)).await, None);
+	};
+
+	let request_handler = async {
+		let mut recover_available_data_msg_count = 0;
+		let mut block_number_msg_count = 0;
+
+		while recover_available_data_msg_count < MAX_PARALLEL_PARTICIPATIONS + 1 ||
+			block_number_msg_count < 1
+		{
+			match ctx_handle.recv().await {
+				AllMessages::AvailabilityRecovery(
+					AvailabilityRecoveryMessage::RecoverAvailableData(_, _, _, _, tx),
+				) => {
+					tx.send(Err(RecoveryError::Unavailable)).unwrap();
+					recover_available_data_msg_count += 1;
+				},
+				AllMessages::ChainApi(ChainApiMessage::BlockNumber(_, tx)) => {
+					tx.send(Ok(None)).unwrap();
+					block_number_msg_count += 1;
+				},
+				_ => assert!(false, "Received unexpected message"),
+			}
+		}
+
+		// we should not have any further results
+		assert_matches!(ctx_handle.recv().timeout(Duration::from_millis(10)).await, None);
+	};
+
+	futures::executor::block_on(async {
+		futures::join!(test, request_handler);
+	});
+}
+
+#[test]
+fn reqs_get_queued_on_no_recent_block() {
+	let (mut ctx, mut ctx_handle) = make_our_subsystem_context(TaskExecutor::new());
+	let (mut unblock_test, mut wait_for_verification) = mpsc::channel(0);
+	let test = async {
+		let (sender, _worker_receiver) = mpsc::channel(1);
+		let mut participation = Participation::new(sender, Metrics::default());
+		participate(&mut ctx, &mut participation).await.unwrap();
+
+		// We have initiated participation but we'll block `active_leaf` so that we can check that
+		// the participation is queued in race-free way
+		let _ = wait_for_verification.next().await.unwrap();
+
+		activate_leaf(&mut ctx, &mut participation, 10).await.unwrap();
+	};
+
+	// Responds to messages from the test and verifies its behaviour
+	let request_handler = async {
+		// If we receive `BlockNumber` request this implicitly proves that the participation is
+		// queued
+		assert_matches!(
+			ctx_handle.recv().await,
+			AllMessages::ChainApi(ChainApiMessage::BlockNumber(_, tx)) => {
+				tx.send(Ok(None)).unwrap();
+			},
+			"overseer did not receive `ChainApiMessage::BlockNumber` message",
+		);
+
+		assert!(ctx_handle.recv().timeout(Duration::from_millis(10)).await.is_none());
+
+		// No activity so the participation is queued => unblock the test
+		unblock_test.send(()).await.unwrap();
+
+		// after activating at least one leaf the recent block
+		// state should be available which should lead to trying
+		// to participate by first trying to recover the available
+		// data
+		assert_matches!(
+			ctx_handle.recv().await,
+			AllMessages::AvailabilityRecovery(AvailabilityRecoveryMessage::RecoverAvailableData(
+				..
+			)),
+			"overseer did not receive recover available data message",
+		);
+	};
+
+	futures::executor::block_on(async {
+		futures::join!(test, request_handler);
+	});
+}
+
+#[test]
+fn cannot_participate_if_cannot_recover_available_data() {
+	futures::executor::block_on(async {
+		let (mut ctx, mut ctx_handle) = make_our_subsystem_context(TaskExecutor::new());
+
+		let (sender, mut worker_receiver) = mpsc::channel(1);
+		let mut participation = Participation::new(sender, Metrics::default());
+		activate_leaf(&mut ctx, &mut participation, 10).await.unwrap();
+		participate(&mut ctx, &mut participation).await.unwrap();
+
+		assert_matches!(
+			ctx_handle.recv().await,
+			AllMessages::AvailabilityRecovery(
+				AvailabilityRecoveryMessage::RecoverAvailableData(_, _, _, _, tx)
+			) => {
+				tx.send(Err(RecoveryError::Unavailable)).unwrap();
+			},
+			"overseer did not receive recover available data message",
+		);
+		let result = participation
+			.get_participation_result(&mut ctx, worker_receiver.next().await.unwrap())
+			.await
+			.unwrap();
+		assert_matches!(
+			result.outcome,
+			ParticipationOutcome::Unavailable => {}
+		);
+	})
+}
+
+#[test]
+fn cannot_participate_if_cannot_recover_validation_code() {
+	futures::executor::block_on(async {
+		let (mut ctx, mut ctx_handle) = make_our_subsystem_context(TaskExecutor::new());
+
+		let (sender, mut worker_receiver) = mpsc::channel(1);
+		let mut participation = Participation::new(sender, Metrics::default());
+		activate_leaf(&mut ctx, &mut participation, 10).await.unwrap();
+		participate(&mut ctx, &mut participation).await.unwrap();
+		recover_available_data(&mut ctx_handle).await;
+
+		assert_matches!(
+			ctx_handle.recv().await,
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+				_,
+				RuntimeApiRequest::ValidationCodeByHash(
+					_,
+					tx,
+				)
+			)) => {
+				tx.send(Ok(None)).unwrap();
+			},
+			"overseer did not receive runtime API request for validation code",
+		);
+
+		let result = participation
+			.get_participation_result(&mut ctx, worker_receiver.next().await.unwrap())
+			.await
+			.unwrap();
+		assert_matches!(
+			result.outcome,
+			ParticipationOutcome::Error => {}
+		);
+	})
+}
+
+#[test]
+fn cast_invalid_vote_if_available_data_is_invalid() {
+	futures::executor::block_on(async {
+		let (mut ctx, mut ctx_handle) = make_our_subsystem_context(TaskExecutor::new());
+
+		let (sender, mut worker_receiver) = mpsc::channel(1);
+		let mut participation = Participation::new(sender, Metrics::default());
+		activate_leaf(&mut ctx, &mut participation, 10).await.unwrap();
+		participate(&mut ctx, &mut participation).await.unwrap();
+
+		assert_matches!(
+			ctx_handle.recv().await,
+			AllMessages::AvailabilityRecovery(
+				AvailabilityRecoveryMessage::RecoverAvailableData(_, _, _, _, tx)
+			) => {
+				tx.send(Err(RecoveryError::Invalid)).unwrap();
+			},
+			"overseer did not receive recover available data message",
+		);
+
+		let result = participation
+			.get_participation_result(&mut ctx, worker_receiver.next().await.unwrap())
+			.await
+			.unwrap();
+		assert_matches!(
+			result.outcome,
+			ParticipationOutcome::Invalid => {}
+		);
+	})
+}
+
+#[test]
+fn cast_invalid_vote_if_validation_fails_or_is_invalid() {
+	futures::executor::block_on(async {
+		let (mut ctx, mut ctx_handle) = make_our_subsystem_context(TaskExecutor::new());
+
+		let (sender, mut worker_receiver) = mpsc::channel(1);
+		let mut participation = Participation::new(sender, Metrics::default());
+		activate_leaf(&mut ctx, &mut participation, 10).await.unwrap();
+		participate(&mut ctx, &mut participation).await.unwrap();
+
+		recover_available_data(&mut ctx_handle).await;
+		assert_eq!(
+			fetch_validation_code(&mut ctx_handle).await,
+			participation.recent_block.unwrap().1
+		);
+
+		assert_matches!(
+			ctx_handle.recv().await,
+			AllMessages::CandidateValidation(
+				CandidateValidationMessage::ValidateFromExhaustive { exec_kind, response_sender, .. }
+			) if exec_kind == PvfExecKind::Dispute => {
+				response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::Timeout))).unwrap();
+			},
+			"overseer did not receive candidate validation message",
+		);
+
+		let result = participation
+			.get_participation_result(&mut ctx, worker_receiver.next().await.unwrap())
+			.await
+			.unwrap();
+		assert_matches!(
+			result.outcome,
+			ParticipationOutcome::Invalid => {}
+		);
+	})
+}
+
+#[test]
+fn cast_invalid_vote_if_commitments_dont_match() {
+	futures::executor::block_on(async {
+		let (mut ctx, mut ctx_handle) = make_our_subsystem_context(TaskExecutor::new());
+
+		let (sender, mut worker_receiver) = mpsc::channel(1);
+		let mut participation = Participation::new(sender, Metrics::default());
+		activate_leaf(&mut ctx, &mut participation, 10).await.unwrap();
+		participate(&mut ctx, &mut participation).await.unwrap();
+
+		recover_available_data(&mut ctx_handle).await;
+		assert_eq!(
+			fetch_validation_code(&mut ctx_handle).await,
+			participation.recent_block.unwrap().1
+		);
+
+		assert_matches!(
+			ctx_handle.recv().await,
+			AllMessages::CandidateValidation(
+				CandidateValidationMessage::ValidateFromExhaustive { exec_kind, response_sender, .. }
+			) if exec_kind == PvfExecKind::Dispute => {
+				response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::CommitmentsHashMismatch))).unwrap();
+			},
+			"overseer did not receive candidate validation message",
+		);
+
+		let result = participation
+			.get_participation_result(&mut ctx, worker_receiver.next().await.unwrap())
+			.await
+			.unwrap();
+		assert_matches!(
+			result.outcome,
+			ParticipationOutcome::Invalid => {}
+		);
+	})
+}
+
+#[test]
+fn cast_valid_vote_if_validation_passes() {
+	futures::executor::block_on(async {
+		let (mut ctx, mut ctx_handle) = make_our_subsystem_context(TaskExecutor::new());
+
+		let (sender, mut worker_receiver) = mpsc::channel(1);
+		let mut participation = Participation::new(sender, Metrics::default());
+		activate_leaf(&mut ctx, &mut participation, 10).await.unwrap();
+		participate(&mut ctx, &mut participation).await.unwrap();
+
+		recover_available_data(&mut ctx_handle).await;
+		assert_eq!(
+			fetch_validation_code(&mut ctx_handle).await,
+			participation.recent_block.unwrap().1
+		);
+
+		assert_matches!(
+			ctx_handle.recv().await,
+			AllMessages::CandidateValidation(
+				CandidateValidationMessage::ValidateFromExhaustive { exec_kind, response_sender, .. }
+			) if exec_kind == PvfExecKind::Dispute => {
+				response_sender.send(Ok(ValidationResult::Valid(dummy_candidate_commitments(None), PersistedValidationData::default()))).unwrap();
+			},
+			"overseer did not receive candidate validation message",
+		);
+
+		let result = participation
+			.get_participation_result(&mut ctx, worker_receiver.next().await.unwrap())
+			.await
+			.unwrap();
+		assert_matches!(
+			result.outcome,
+			ParticipationOutcome::Valid => {}
+		);
+	})
+}

pezkuwi/node/core/dispute-coordinator/src/scraping/candidates.rs

@@ -0,0 +1,169 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use pezkuwi_primitives::{BlockNumber, CandidateHash};
+use std::collections::{BTreeMap, HashMap, HashSet};
+
+/// Keeps `CandidateHash` in reference counted way.
+/// Each `insert` saves a value with `reference count == 1` or increases the reference
+/// count if the value already exists.
+/// Each `remove` decreases the reference count for the corresponding `CandidateHash`.
+/// If the reference count reaches 0 - the value is removed.
+struct RefCountedCandidates {
+	candidates: HashMap<CandidateHash, usize>,
+}
+
+impl RefCountedCandidates {
+	pub fn new() -> Self {
+		Self { candidates: HashMap::new() }
+	}
+	// If `CandidateHash` doesn't exist in the `HashMap` it is created and its reference
+	// count is set to 1.
+	// If `CandidateHash` already exists in the `HashMap` its reference count is increased.
+	pub fn insert(&mut self, candidate: CandidateHash) {
+		*self.candidates.entry(candidate).or_default() += 1;
+	}
+
+	// If a `CandidateHash` with reference count equals to 1 is about to be removed - the
+	// candidate is dropped from the container too.
+	// If a `CandidateHash` with reference count bigger than 1 is about to be removed - the
+	// reference count is decreased and the candidate remains in the container.
+	pub fn remove(&mut self, candidate: &CandidateHash) {
+		match self.candidates.get_mut(candidate) {
+			Some(v) if *v > 1 => *v -= 1,
+			Some(v) => {
+				assert!(*v == 1);
+				self.candidates.remove(candidate);
+			},
+			None => {},
+		}
+	}
+
+	pub fn contains(&self, candidate: &CandidateHash) -> bool {
+		self.candidates.contains_key(&candidate)
+	}
+}
+
+#[cfg(test)]
+mod ref_counted_candidates_tests {
+	use super::*;
+	use pezkuwi_primitives::{BlakeTwo256, HashT};
+
+	#[test]
+	fn element_is_removed_when_refcount_reaches_zero() {
+		let mut container = RefCountedCandidates::new();
+
+		let zero = CandidateHash(BlakeTwo256::hash(&vec![0]));
+		let one = CandidateHash(BlakeTwo256::hash(&vec![1]));
+		// add two separate candidates
+		container.insert(zero); // refcount == 1
+		container.insert(one);
+
+		// and increase the reference count for the first
+		container.insert(zero); // refcount == 2
+
+		assert!(container.contains(&zero));
+		assert!(container.contains(&one));
+
+		// remove once -> refcount == 1
+		container.remove(&zero);
+		assert!(container.contains(&zero));
+		assert!(container.contains(&one));
+
+		// remove once -> refcount == 0
+		container.remove(&zero);
+		assert!(!container.contains(&zero));
+		assert!(container.contains(&one));
+
+		// remove the other element
+		container.remove(&one);
+		assert!(!container.contains(&zero));
+		assert!(!container.contains(&one));
+	}
+}
+
+/// Keeps track of scraped candidates. Supports `insert`, `remove_up_to_height` and `contains`
+/// operations.
+pub struct ScrapedCandidates {
+	/// Main data structure which keeps the candidates we know about. `contains` does lookups only
+	/// here.
+	candidates: RefCountedCandidates,
+	/// Keeps track at which block number a candidate was inserted. Used in `remove_up_to_height`.
+	/// Without this tracking we won't be able to remove all candidates before block X.
+	candidates_by_block_number: BTreeMap<BlockNumber, HashSet<CandidateHash>>,
+}
+
+impl ScrapedCandidates {
+	pub fn new() -> Self {
+		Self {
+			candidates: RefCountedCandidates::new(),
+			candidates_by_block_number: BTreeMap::new(),
+		}
+	}
+
+	pub fn contains(&self, candidate_hash: &CandidateHash) -> bool {
+		self.candidates.contains(candidate_hash)
+	}
+
+	// Removes all candidates up to a given height. The candidates at the block height are NOT
+	// removed.
+	pub fn remove_up_to_height(&mut self, height: &BlockNumber) {
+		let not_stale = self.candidates_by_block_number.split_off(&height);
+		let stale = std::mem::take(&mut self.candidates_by_block_number);
+		self.candidates_by_block_number = not_stale;
+		for candidate in stale.values().flatten() {
+			self.candidates.remove(candidate);
+		}
+	}
+
+	pub fn insert(&mut self, block_number: BlockNumber, candidate_hash: CandidateHash) {
+		if self
+			.candidates_by_block_number
+			.entry(block_number)
+			.or_default()
+			.insert(candidate_hash)
+		{
+			self.candidates.insert(candidate_hash);
+		}
+	}
+
+	// Used only for tests to verify the pruning doesn't leak data.
+	#[cfg(test)]
+	pub fn candidates_by_block_number_is_empty(&self) -> bool {
+		self.candidates_by_block_number.is_empty()
+	}
+}
+
+#[cfg(test)]
+mod scraped_candidates_tests {
+	use super::*;
+	use pezkuwi_primitives::{BlakeTwo256, HashT};
+
+	#[test]
+	fn stale_candidates_are_removed() {
+		let mut candidates = ScrapedCandidates::new();
+		let target = CandidateHash(BlakeTwo256::hash(&vec![1, 2, 3]));
+		candidates.insert(1, target);
+		// Repeated inserts at same height should be fine:
+		candidates.insert(1, target);
+
+		assert!(candidates.contains(&target));
+
+		candidates.remove_up_to_height(&2);
+		assert!(!candidates.contains(&target));
+		assert!(candidates.candidates_by_block_number_is_empty());
+	}
+}

pezkuwi/node/core/dispute-coordinator/src/scraping/mod.rs

@@ -0,0 +1,501 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::collections::{btree_map::Entry, BTreeMap, HashSet};
+
+use futures::channel::oneshot;
+use schnellru::{ByLength, LruMap};
+
+use pezkuwi_node_primitives::{DISPUTE_CANDIDATE_LIFETIME_AFTER_FINALIZATION, MAX_FINALITY_LAG};
+use pezkuwi_node_subsystem::{
+	messages::ChainApiMessage, overseer, ActivatedLeaf, ActiveLeavesUpdate, ChainApiError,
+	RuntimeApiError, SubsystemSender,
+};
+use pezkuwi_node_subsystem_util::runtime::{
+	self, get_candidate_events, get_on_chain_votes, get_unapplied_slashes,
+};
+use pezkuwi_primitives::{
+	slashing::PendingSlashes, BlockNumber, CandidateEvent, CandidateHash,
+	CandidateReceiptV2 as CandidateReceipt, Hash, ScrapedOnChainVotes, SessionIndex,
+};
+
+use crate::{
+	error::{FatalError, FatalResult, Result},
+	LOG_TARGET,
+};
+
+#[cfg(test)]
+mod tests;
+
+mod candidates;
+
+/// Number of hashes to keep in the LRU.
+///
+///
+/// When traversing the ancestry of a block we will stop once we hit a hash that we find in the
+/// `last_observed_blocks` LRU. This means, this value should the very least be as large as the
+/// number of expected forks for keeping chain scraping efficient. Making the LRU much larger than
+/// that has very limited use.
+/// In cases of high load when finality lags, forks could appear anywhere from the last finalized
+/// block to best, hence this number needs to be large enough to hold all the hashes from best to
+/// finalized.
+const LRU_OBSERVED_BLOCKS_CAPACITY: u32 = 2 * MAX_FINALITY_LAG;
+
+/// `ScrapedUpdates`
+///
+/// Updates to `on_chain_votes` and included receipts for new active leaf and its unprocessed
+/// ancestors.
+///
+/// `on_chain_votes`: New votes as seen on chain
+/// `included_receipts`: Newly included teyrchain block candidate receipts as seen on chain
+pub struct ScrapedUpdates {
+	pub on_chain_votes: Vec<ScrapedOnChainVotes>,
+	pub included_receipts: Vec<CandidateReceipt>,
+	pub unapplied_slashes: Vec<(SessionIndex, CandidateHash, PendingSlashes)>,
+}
+
+impl ScrapedUpdates {
+	pub fn new() -> Self {
+		Self {
+			on_chain_votes: Vec::new(),
+			included_receipts: Vec::new(),
+			unapplied_slashes: Vec::new(),
+		}
+	}
+}
+
+/// A structure meant to facilitate chain reversions in the event of a dispute
+/// concluding against a candidate. Each candidate hash maps to a number of
+/// block heights, which in turn map to vectors of blocks at those heights.
+pub struct Inclusions {
+	inclusions_inner: BTreeMap<CandidateHash, BTreeMap<BlockNumber, HashSet<Hash>>>,
+	/// Keeps track at which block number a candidate was inserted. Used in `remove_up_to_height`.
+	/// Without this tracking we won't be able to remove all candidates before block X.
+	candidates_by_block_number: BTreeMap<BlockNumber, HashSet<CandidateHash>>,
+}
+
+impl Inclusions {
+	pub fn new() -> Self {
+		Self { inclusions_inner: BTreeMap::new(), candidates_by_block_number: BTreeMap::new() }
+	}
+
+	pub fn insert(
+		&mut self,
+		candidate_hash: CandidateHash,
+		block_number: BlockNumber,
+		block_hash: Hash,
+	) {
+		self.inclusions_inner
+			.entry(candidate_hash)
+			.or_default()
+			.entry(block_number)
+			.or_default()
+			.insert(block_hash);
+
+		self.candidates_by_block_number
+			.entry(block_number)
+			.or_default()
+			.insert(candidate_hash);
+	}
+
+	/// Removes all candidates up to a given height.
+	///
+	/// The candidates at the block height are NOT removed.
+	pub fn remove_up_to_height(&mut self, height: &BlockNumber) {
+		let not_stale = self.candidates_by_block_number.split_off(&height);
+		let stale = std::mem::take(&mut self.candidates_by_block_number);
+		self.candidates_by_block_number = not_stale;
+
+		for candidate in stale.into_values().flatten() {
+			match self.inclusions_inner.entry(candidate) {
+				Entry::Vacant(_) => {
+					// Rare case where same candidate was present on multiple heights, but all are
+					// pruned at the same time. This candidate was already pruned in the previous
+					// occurrence so it is skipped now.
+				},
+				Entry::Occupied(mut e) => {
+					let mut blocks_including = std::mem::take(e.get_mut());
+					// Returns everything after the given key, including the key. This works because
+					// the blocks are sorted in ascending order.
+					blocks_including = blocks_including.split_off(&height);
+					if blocks_including.is_empty() {
+						e.remove_entry();
+					} else {
+						*e.get_mut() = blocks_including;
+					}
+				},
+			}
+		}
+	}
+
+	pub fn get(&self, candidate: &CandidateHash) -> Vec<(BlockNumber, Hash)> {
+		let mut inclusions_as_vec: Vec<(BlockNumber, Hash)> = Vec::new();
+		if let Some(blocks_including) = self.inclusions_inner.get(candidate) {
+			for (height, blocks_at_height) in blocks_including.iter() {
+				for block in blocks_at_height {
+					inclusions_as_vec.push((*height, *block));
+				}
+			}
+		}
+		inclusions_as_vec
+	}
+
+	pub fn contains(&self, candidate: &CandidateHash) -> bool {
+		self.inclusions_inner.get(candidate).is_some()
+	}
+}
+
+/// Chain scraper
+///
+/// Scrapes unfinalized chain in order to collect information from blocks. Chain scraping
+/// during disputes enables critical spam prevention. It does so by updating two important
+/// criteria determining whether a vote sent during dispute distribution is potential
+/// spam. Namely, whether the candidate being voted on is backed or included.
+///
+/// Concretely:
+///
+/// - Monitors for `CandidateIncluded` events to keep track of candidates that have been included on
+///   chains.
+/// - Monitors for `CandidateBacked` events to keep track of all backed candidates.
+/// - Calls `FetchOnChainVotes` for each block to gather potentially missed votes from chain.
+///
+/// With this information it provides a `CandidateComparator` and as a return value of
+/// `process_active_leaves_update` any scraped votes.
+///
+/// Scraped candidates are available `DISPUTE_CANDIDATE_LIFETIME_AFTER_FINALIZATION` more blocks
+/// after finalization as a precaution not to prune them prematurely. Besides the newly scraped
+/// candidates `DISPUTE_CANDIDATE_LIFETIME_AFTER_FINALIZATION` finalized blocks are parsed as
+/// another precaution to have their `CandidateReceipts` available in case a dispute is raised on
+/// them,
+pub struct ChainScraper {
+	/// All candidates we have seen backed.
+	backed_candidates: candidates::ScrapedCandidates,
+
+	/// Maps included candidate hashes to one or more relay block heights and hashes.
+	/// These correspond to all the relay blocks which marked a candidate as included,
+	/// and are needed to apply reversions in case a dispute is concluded against the
+	/// candidate.
+	inclusions: Inclusions,
+
+	/// Latest relay blocks observed by the provider.
+	///
+	/// This is used to avoid redundant scraping of ancestry. We assume that ancestors of cached
+	/// blocks are already processed, i.e. we have saved corresponding included candidates.
+	last_observed_blocks: LruMap<Hash, ()>,
+}
+
+impl ChainScraper {
+	/// Limits the number of ancestors received for a single request.
+	pub(crate) const ANCESTRY_CHUNK_SIZE: u32 = 10;
+	/// Limits the overall number of ancestors walked through for a given head.
+	///
+	/// As long as we have `MAX_FINALITY_LAG` this makes sense as a value.
+	pub(crate) const ANCESTRY_SIZE_LIMIT: u32 = MAX_FINALITY_LAG;
+
+	/// Create a properly initialized `OrderingProvider`.
+	///
+	/// Returns: `Self` and any scraped votes.
+	pub async fn new<Sender>(
+		sender: &mut Sender,
+		initial_head: ActivatedLeaf,
+	) -> Result<(Self, Vec<ScrapedOnChainVotes>)>
+	where
+		Sender: overseer::DisputeCoordinatorSenderTrait,
+	{
+		let mut s = Self {
+			backed_candidates: candidates::ScrapedCandidates::new(),
+			inclusions: Inclusions::new(),
+			last_observed_blocks: LruMap::new(ByLength::new(LRU_OBSERVED_BLOCKS_CAPACITY)),
+		};
+		let update =
+			ActiveLeavesUpdate { activated: Some(initial_head), deactivated: Default::default() };
+		let updates = s.process_active_leaves_update(sender, &update).await?;
+		Ok((s, updates.on_chain_votes))
+	}
+
+	/// Check whether we have seen a candidate included on any chain.
+	pub fn is_candidate_included(&self, candidate_hash: &CandidateHash) -> bool {
+		self.inclusions.contains(candidate_hash)
+	}
+
+	/// Check whether the candidate is backed
+	pub fn is_candidate_backed(&self, candidate_hash: &CandidateHash) -> bool {
+		self.backed_candidates.contains(candidate_hash)
+	}
+
+	/// Query active leaves for any candidate `CandidateEvent::CandidateIncluded` events.
+	///
+	/// and updates current heads, so we can query candidates for all non finalized blocks.
+	///
+	/// Returns: On chain votes and included candidate receipts for the leaf and any
+	/// ancestors we might not yet have seen.
+	pub async fn process_active_leaves_update<Sender>(
+		&mut self,
+		sender: &mut Sender,
+		update: &ActiveLeavesUpdate,
+	) -> Result<ScrapedUpdates>
+	where
+		Sender: overseer::DisputeCoordinatorSenderTrait,
+	{
+		let activated = match update.activated.as_ref() {
+			Some(activated) => activated,
+			None => return Ok(ScrapedUpdates::new()),
+		};
+
+		// Fetch ancestry up to `SCRAPED_FINALIZED_BLOCKS_COUNT` blocks beyond
+		// the last finalized one
+		let ancestors = self
+			.get_relevant_block_ancestors(sender, activated.hash, activated.number)
+			.await?;
+
+		// Ancestors block numbers are consecutive in the descending order.
+		let earliest_block_number = activated.number - ancestors.len() as u32;
+		let block_numbers = (earliest_block_number..=activated.number).rev();
+
+		let block_hashes = std::iter::once(activated.hash).chain(ancestors);
+
+		let mut scraped_updates = ScrapedUpdates::new();
+		for (block_number, block_hash) in block_numbers.zip(block_hashes) {
+			gum::trace!(target: LOG_TARGET, ?block_number, ?block_hash, "In ancestor processing.");
+
+			let receipts_for_block =
+				self.process_candidate_events(sender, block_number, block_hash).await?;
+			scraped_updates.included_receipts.extend(receipts_for_block);
+
+			if let Some(votes) = get_on_chain_votes(sender, block_hash).await? {
+				scraped_updates.on_chain_votes.push(votes);
+			}
+		}
+
+		// for unapplied slashes, we only look at the latest activated hash,
+		// it should accumulate them all
+		match get_unapplied_slashes(sender, activated.hash).await {
+			Ok(unapplied_slashes) => {
+				scraped_updates.unapplied_slashes = unapplied_slashes;
+			},
+			Err(runtime::Error::RuntimeRequest(RuntimeApiError::NotSupported { .. })) => {
+				gum::debug!(
+					target: LOG_TARGET,
+					block_hash = ?activated.hash,
+					"Fetching unapplied slashes not yet supported.",
+				);
+			},
+			Err(error) => {
+				gum::warn!(
+					target: LOG_TARGET,
+					block_hash = ?activated.hash,
+					?error,
+					"Error fetching unapplied slashes.",
+				);
+			},
+		}
+
+		self.last_observed_blocks.insert(activated.hash, ());
+
+		Ok(scraped_updates)
+	}
+
+	/// Prune finalized candidates.
+	///
+	/// We keep each candidate for `DISPUTE_CANDIDATE_LIFETIME_AFTER_FINALIZATION` blocks after
+	/// finalization. After that we treat it as low priority.
+	pub fn process_finalized_block(&mut self, finalized_block_number: &BlockNumber) {
+		// `DISPUTE_CANDIDATE_LIFETIME_AFTER_FINALIZATION - 1` because
+		// `finalized_block_number`counts to the candidate lifetime.
+		match finalized_block_number.checked_sub(DISPUTE_CANDIDATE_LIFETIME_AFTER_FINALIZATION - 1)
+		{
+			Some(key_to_prune) => {
+				self.backed_candidates.remove_up_to_height(&key_to_prune);
+				self.inclusions.remove_up_to_height(&key_to_prune);
+			},
+			None => {
+				// Nothing to prune. We are still in the beginning of the chain and there are not
+				// enough finalized blocks yet.
+			},
+		}
+		{}
+	}
+
+	/// Process candidate events of a block.
+	///
+	/// Keep track of all included and backed candidates.
+	///
+	/// Returns freshly included candidate receipts
+	async fn process_candidate_events<Sender>(
+		&mut self,
+		sender: &mut Sender,
+		block_number: BlockNumber,
+		block_hash: Hash,
+	) -> Result<Vec<CandidateReceipt>>
+	where
+		Sender: overseer::DisputeCoordinatorSenderTrait,
+	{
+		let events = get_candidate_events(sender, block_hash).await?;
+		let mut included_receipts: Vec<CandidateReceipt> = Vec::new();
+		// Get included and backed events:
+		for ev in events {
+			match ev {
+				CandidateEvent::CandidateIncluded(receipt, _, _, _) => {
+					let candidate_hash = receipt.hash();
+					gum::trace!(
+						target: LOG_TARGET,
+						?candidate_hash,
+						?block_number,
+						"Processing included event"
+					);
+					self.inclusions.insert(candidate_hash, block_number, block_hash);
+					included_receipts.push(receipt);
+				},
+				CandidateEvent::CandidateBacked(receipt, _, _, _) => {
+					let candidate_hash = receipt.hash();
+					gum::trace!(
+						target: LOG_TARGET,
+						?candidate_hash,
+						?block_number,
+						"Processing backed event"
+					);
+					self.backed_candidates.insert(block_number, candidate_hash);
+				},
+				_ => {
+					// skip the rest
+				},
+			}
+		}
+		Ok(included_receipts)
+	}
+
+	/// Returns ancestors of `head` in the descending order, stopping
+	/// either at the block present in cache or at `SCRAPED_FINALIZED_BLOCKS_COUNT -1` blocks after
+	/// the last finalized one (called `target_ancestor`).
+	///
+	/// Both `head` and the `target_ancestor` blocks are **not** included in the result.
+	async fn get_relevant_block_ancestors<Sender>(
+		&mut self,
+		sender: &mut Sender,
+		mut head: Hash,
+		mut head_number: BlockNumber,
+	) -> Result<Vec<Hash>>
+	where
+		Sender: overseer::DisputeCoordinatorSenderTrait,
+	{
+		let target_ancestor = get_finalized_block_number(sender)
+			.await?
+			.saturating_sub(DISPUTE_CANDIDATE_LIFETIME_AFTER_FINALIZATION);
+
+		let mut ancestors = Vec::new();
+
+		// If head_number <= target_ancestor + 1 the ancestry will be empty.
+		if self.last_observed_blocks.get(&head).is_some() || head_number <= target_ancestor + 1 {
+			return Ok(ancestors);
+		}
+
+		loop {
+			let hashes = get_block_ancestors(sender, head, Self::ANCESTRY_CHUNK_SIZE).await?;
+
+			let earliest_block_number = match head_number.checked_sub(hashes.len() as u32) {
+				Some(number) => number,
+				None => {
+					// It's assumed that it's impossible to retrieve
+					// more than N ancestors for block number N.
+					gum::error!(
+						target: LOG_TARGET,
+						"Received {} ancestors for block number {} from Chain API",
+						hashes.len(),
+						head_number,
+					);
+					return Ok(ancestors);
+				},
+			};
+			// The reversed order is parent, grandparent, etc. excluding the head.
+			let block_numbers = (earliest_block_number..head_number).rev();
+
+			for (block_number, hash) in block_numbers.zip(&hashes) {
+				// Return if we either met target/cached block or
+				// hit the size limit for the returned ancestry of head.
+				if self.last_observed_blocks.get(hash).is_some() ||
+					block_number <= target_ancestor ||
+					ancestors.len() >= Self::ANCESTRY_SIZE_LIMIT as usize
+				{
+					return Ok(ancestors);
+				}
+
+				ancestors.push(*hash);
+			}
+
+			match hashes.last() {
+				Some(last_hash) => {
+					head = *last_hash;
+					head_number = earliest_block_number;
+				},
+				None => break,
+			}
+		}
+		return Ok(ancestors);
+	}
+
+	pub fn get_blocks_including_candidate(
+		&self,
+		candidate: &CandidateHash,
+	) -> Vec<(BlockNumber, Hash)> {
+		self.inclusions.get(candidate)
+	}
+}
+
+async fn get_finalized_block_number<Sender>(sender: &mut Sender) -> FatalResult<BlockNumber>
+where
+	Sender: overseer::DisputeCoordinatorSenderTrait,
+{
+	let (number_tx, number_rx) = oneshot::channel();
+	send_message_fatal(sender, ChainApiMessage::FinalizedBlockNumber(number_tx), number_rx).await
+}
+
+async fn get_block_ancestors<Sender>(
+	sender: &mut Sender,
+	head: Hash,
+	num_ancestors: BlockNumber,
+) -> FatalResult<Vec<Hash>>
+where
+	Sender: overseer::DisputeCoordinatorSenderTrait,
+{
+	let (tx, rx) = oneshot::channel();
+	sender
+		.send_message(ChainApiMessage::Ancestors {
+			hash: head,
+			k: num_ancestors as usize,
+			response_channel: tx,
+		})
+		.await;
+
+	rx.await
+		.or(Err(FatalError::ChainApiSenderDropped))?
+		.map_err(FatalError::ChainApiAncestors)
+}
+
+async fn send_message_fatal<Sender, Response>(
+	sender: &mut Sender,
+	message: ChainApiMessage,
+	receiver: oneshot::Receiver<std::result::Result<Response, ChainApiError>>,
+) -> FatalResult<Response>
+where
+	Sender: SubsystemSender<ChainApiMessage>,
+{
+	sender.send_message(message).await;
+
+	receiver
+		.await
+		.map_err(|_| FatalError::ChainApiSenderDropped)?
+		.map_err(FatalError::ChainApiAncestors)
+}

pezkuwi/node/core/dispute-coordinator/src/spam_slots.rs

@@ -0,0 +1,135 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::collections::{BTreeSet, HashMap};
+
+use pezkuwi_primitives::{CandidateHash, SessionIndex, ValidatorIndex};
+
+use crate::LOG_TARGET;
+
+/// Type used for counting potential spam votes.
+type SpamCount = u32;
+
+/// How many unconfirmed disputes a validator is allowed to import (per session).
+///
+/// Unconfirmed means: Node has not seen the candidate be included on any chain, it has not cast a
+/// vote itself on that dispute, the dispute has not yet reached more than a third of
+/// validator's votes and the including relay chain block has not yet been finalized.
+///
+/// Exact number of `MAX_SPAM_VOTES` is not that important here. It is important that the number is
+/// low enough to not cause resource exhaustion (disk & memory) on the importing validator, even if
+/// multiple validators fully make use of their assigned spam slots.
+///
+/// Also if things are working properly, this number cannot really be too low either, as all
+/// relevant disputes _should_ have been seen as included by enough validators. (Otherwise the
+/// candidate would not have been available in the first place and could not have been included.)
+/// So this is really just a fallback mechanism if things go terribly wrong.
+#[cfg(not(test))]
+const MAX_SPAM_VOTES: SpamCount = 50;
+#[cfg(test)]
+const MAX_SPAM_VOTES: SpamCount = 1;
+
+/// Spam slots for raised disputes concerning unknown candidates.
+pub struct SpamSlots {
+	/// Counts per validator and session.
+	///
+	/// Must not exceed `MAX_SPAM_VOTES`.
+	slots: HashMap<(SessionIndex, ValidatorIndex), SpamCount>,
+
+	/// All unconfirmed candidates we are aware of right now.
+	unconfirmed: UnconfirmedDisputes,
+}
+
+/// Unconfirmed disputes to be passed at initialization.
+pub type UnconfirmedDisputes = HashMap<(SessionIndex, CandidateHash), BTreeSet<ValidatorIndex>>;
+
+impl SpamSlots {
+	/// Recover `SpamSlots` from state on startup.
+	///
+	/// Initialize based on already existing active disputes.
+	pub fn recover_from_state(unconfirmed_disputes: UnconfirmedDisputes) -> Self {
+		let mut slots: HashMap<(SessionIndex, ValidatorIndex), SpamCount> = HashMap::new();
+		for ((session, _), validators) in unconfirmed_disputes.iter() {
+			for validator in validators {
+				let spam_vote_count = slots.entry((*session, *validator)).or_default();
+				*spam_vote_count += 1;
+				if *spam_vote_count > MAX_SPAM_VOTES {
+					gum::debug!(
+						target: LOG_TARGET,
+						?session,
+						?validator,
+						count = ?spam_vote_count,
+						"Import exceeded spam slot for validator"
+					);
+				}
+			}
+		}
+
+		Self { slots, unconfirmed: unconfirmed_disputes }
+	}
+
+	/// Increase a "voting invalid" validator's spam slot.
+	///
+	/// This function should get called for any validator's invalidity vote for any not yet
+	/// confirmed dispute.
+	///
+	/// Returns: `true` if validator still had vacant spam slots, `false` otherwise.
+	pub fn add_unconfirmed(
+		&mut self,
+		session: SessionIndex,
+		candidate: CandidateHash,
+		validator: ValidatorIndex,
+	) -> bool {
+		let spam_vote_count = self.slots.entry((session, validator)).or_default();
+		if *spam_vote_count >= MAX_SPAM_VOTES {
+			return false;
+		}
+		let validators = self.unconfirmed.entry((session, candidate)).or_default();
+
+		if validators.insert(validator) {
+			// We only increment spam slots once per candidate, as each validator has to provide an
+			// opposing vote for sending out its own vote. Therefore, receiving multiple votes for
+			// a single candidate is expected and should not get punished here.
+			*spam_vote_count += 1;
+		}
+
+		true
+	}
+
+	/// Clear out spam slots for a given candidate in a session.
+	///
+	/// This effectively reduces the spam slot count for all validators participating in a dispute
+	/// for that candidate. You should call this function once a dispute became obsolete or got
+	/// confirmed and thus votes for it should no longer be treated as potential spam.
+	pub fn clear(&mut self, key: &(SessionIndex, CandidateHash)) {
+		if let Some(validators) = self.unconfirmed.remove(key) {
+			let (session, _) = key;
+			for validator in validators {
+				if let Some(spam_vote_count) = self.slots.remove(&(*session, validator)) {
+					let new = spam_vote_count - 1;
+					if new > 0 {
+						self.slots.insert((*session, validator), new);
+					}
+				}
+			}
+		}
+	}
+	/// Prune all spam slots for sessions older than the given index.
+	pub fn prune_old(&mut self, oldest_index: SessionIndex) {
+		self.unconfirmed.retain(|(session, _), _| *session >= oldest_index);
+		self.slots.retain(|(session, _), _| *session >= oldest_index);
+	}
+}

pezkuwi/node/core/dispute-coordinator/src/status.rs

@@ -0,0 +1,58 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use pezkuwi_node_primitives::{dispute_is_inactive, DisputeStatus, Timestamp};
+use pezkuwi_primitives::{CandidateHash, SessionIndex};
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use crate::LOG_TARGET;
+
+/// Get active disputes as iterator, preserving its `DisputeStatus`.
+pub fn get_active_with_status(
+	recent_disputes: impl Iterator<Item = ((SessionIndex, CandidateHash), DisputeStatus)>,
+	now: Timestamp,
+) -> impl Iterator<Item = ((SessionIndex, CandidateHash), DisputeStatus)> {
+	recent_disputes.filter(move |(_, status)| !dispute_is_inactive(status, &now))
+}
+
+pub trait Clock: Send + Sync {
+	fn now(&self) -> Timestamp;
+}
+
+pub struct SystemClock;
+
+impl Clock for SystemClock {
+	fn now(&self) -> Timestamp {
+		// `SystemTime` is notoriously non-monotonic, so our timers might not work
+		// exactly as expected.
+		//
+		// Regardless, disputes are considered active based on an order of minutes,
+		// so a few seconds of slippage in either direction shouldn't affect the
+		// amount of work the node is doing significantly.
+		match SystemTime::now().duration_since(UNIX_EPOCH) {
+			Ok(d) => d.as_secs(),
+			Err(e) => {
+				gum::warn!(
+					target: LOG_TARGET,
+					err = ?e,
+					"Current time is before unix epoch. Validation will not work correctly."
+				);
+
+				0
+			},
+		}
+	}
+}

pezkuwi/node/core/prospective-teyrchains/Cargo.toml

@@ -0,0 +1,42 @@
+[package]
+name = "pezkuwi-node-core-prospective-teyrchains"
+version = "6.0.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+description = "The Prospective Teyrchains subsystem. Tracks and handles prospective teyrchain fragments."
+homepage.workspace = true
+repository.workspace = true
+
+[lints]
+workspace = true
+
+[dependencies]
+fatality = { workspace = true }
+futures = { workspace = true }
+gum = { workspace = true, default-features = true }
+thiserror = { workspace = true }
+
+pezkuwi-node-subsystem = { workspace = true, default-features = true }
+pezkuwi-node-subsystem-util = { workspace = true, default-features = true }
+pezkuwi-primitives = { workspace = true, default-features = true }
+
+[dev-dependencies]
+assert_matches = { workspace = true }
+pezkuwi-node-subsystem-test-helpers = { workspace = true }
+pezkuwi-primitives = { workspace = true, features = ["test"] }
+pezkuwi-primitives-test-helpers = { workspace = true }
+rand = { workspace = true }
+rstest = { workspace = true }
+sp-core = { workspace = true, default-features = true }
+sp-tracing = { workspace = true }
+
+[features]
+runtime-benchmarks = [
+	"gum/runtime-benchmarks",
+	"pezkuwi-node-subsystem-test-helpers/runtime-benchmarks",
+	"pezkuwi-node-subsystem-util/runtime-benchmarks",
+	"pezkuwi-node-subsystem/runtime-benchmarks",
+	"pezkuwi-primitives-test-helpers/runtime-benchmarks",
+	"pezkuwi-primitives/runtime-benchmarks",
+]

pezkuwi/node/core/prospective-teyrchains/src/error.rs

@@ -0,0 +1,72 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Error types.
+
+use futures::channel::oneshot;
+
+use pezkuwi_node_subsystem::{
+	errors::{ChainApiError, RuntimeApiError},
+	SubsystemError,
+};
+use pezkuwi_node_subsystem_util::runtime;
+
+use crate::LOG_TARGET;
+use fatality::Nested;
+
+#[allow(missing_docs)]
+#[fatality::fatality(splitable)]
+pub enum Error {
+	#[fatal]
+	#[error("Receiving message from overseer failed: {0}")]
+	SubsystemReceive(#[source] SubsystemError),
+
+	#[error("Error while accessing runtime information")]
+	Runtime(#[from] runtime::Error),
+
+	#[error(transparent)]
+	RuntimeApi(#[from] RuntimeApiError),
+
+	#[error(transparent)]
+	ChainApi(#[from] ChainApiError),
+
+	#[error("Request to chain API subsystem dropped")]
+	ChainApiRequestCanceled(oneshot::Canceled),
+
+	#[error("Request to runtime API subsystem dropped")]
+	RuntimeApiRequestCanceled(oneshot::Canceled),
+}
+
+/// General `Result` type.
+pub type Result<R> = std::result::Result<R, Error>;
+/// Result for non-fatal only failures.
+pub type JfyiErrorResult<T> = std::result::Result<T, JfyiError>;
+/// Result for fatal only failures.
+pub type FatalResult<T> = std::result::Result<T, FatalError>;
+
+/// Utility for eating top level errors and log them.
+///
+/// We basically always want to try and continue on error. This utility function is meant to
+/// consume top-level errors by simply logging them
+pub fn log_error(result: Result<()>, ctx: &'static str) -> FatalResult<()> {
+	match result.into_nested()? {
+		Ok(()) => Ok(()),
+		Err(jfyi) => {
+			gum::debug!(target: LOG_TARGET, error = ?jfyi, ctx);
+			Ok(())
+		},
+	}
+}

pezkuwi/node/core/prospective-teyrchains/src/metrics.rs

@@ -0,0 +1,158 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use pezkuwi_node_subsystem::prometheus::Opts;
+use pezkuwi_node_subsystem_util::metrics::{
+	self,
+	prometheus::{self, Gauge, GaugeVec, U64},
+};
+
+#[derive(Clone)]
+pub(crate) struct MetricsInner {
+	time_active_leaves_update: prometheus::Histogram,
+	time_introduce_seconded_candidate: prometheus::Histogram,
+	time_candidate_backed: prometheus::Histogram,
+	time_hypothetical_membership: prometheus::Histogram,
+	candidate_count: prometheus::GaugeVec<U64>,
+	active_leaves_count: prometheus::GaugeVec<U64>,
+	implicit_view_candidate_count: prometheus::Gauge<U64>,
+}
+
+/// Candidate backing metrics.
+#[derive(Default, Clone)]
+pub struct Metrics(pub(crate) Option<MetricsInner>);
+
+impl Metrics {
+	/// Provide a timer for handling `ActiveLeavesUpdate` which observes on drop.
+	pub fn time_handle_active_leaves_update(
+		&self,
+	) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.time_active_leaves_update.start_timer())
+	}
+
+	/// Provide a timer for handling `IntroduceSecondedCandidate` which observes on drop.
+	pub fn time_introduce_seconded_candidate(
+		&self,
+	) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0
+			.as_ref()
+			.map(|metrics| metrics.time_introduce_seconded_candidate.start_timer())
+	}
+
+	/// Provide a timer for handling `CandidateBacked` which observes on drop.
+	pub fn time_candidate_backed(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.time_candidate_backed.start_timer())
+	}
+
+	/// Provide a timer for handling `GetHypotheticalMembership` which observes on drop.
+	pub fn time_hypothetical_membership_request(
+		&self,
+	) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0
+			.as_ref()
+			.map(|metrics| metrics.time_hypothetical_membership.start_timer())
+	}
+
+	/// Record number of candidates across all fragment chains. First param is the connected
+	/// candidates count, second param is the unconnected candidates count.
+	pub fn record_candidate_count(&self, connected_count: u64, unconnected_count: u64) {
+		self.0.as_ref().map(|metrics| {
+			metrics.candidate_count.with_label_values(&["connected"]).set(connected_count);
+			metrics
+				.candidate_count
+				.with_label_values(&["unconnected"])
+				.set(unconnected_count);
+		});
+	}
+
+	/// Record the number of candidates present in the implicit view of the subsystem.
+	pub fn record_candidate_count_in_implicit_view(&self, count: u64) {
+		self.0.as_ref().map(|metrics| {
+			metrics.implicit_view_candidate_count.set(count);
+		});
+	}
+
+	/// Record the number of active/inactive leaves kept by the subsystem.
+	pub fn record_leaves_count(&self, active_count: u64, inactive_count: u64) {
+		self.0.as_ref().map(|metrics| {
+			metrics.active_leaves_count.with_label_values(&["active"]).set(active_count);
+			metrics.active_leaves_count.with_label_values(&["inactive"]).set(inactive_count);
+		});
+	}
+}
+
+impl metrics::Metrics for Metrics {
+	fn try_register(registry: &prometheus::Registry) -> Result<Self, prometheus::PrometheusError> {
+		let metrics = MetricsInner {
+			time_active_leaves_update: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_prospective_teyrchains_time_active_leaves_update",
+					"Time spent within `prospective_teyrchains::handle_active_leaves_update`",
+				))?,
+				registry,
+			)?,
+			time_introduce_seconded_candidate: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_prospective_teyrchains_time_introduce_seconded_candidate",
+					"Time spent within `prospective_teyrchains::handle_introduce_seconded_candidate`",
+				))?,
+				registry,
+			)?,
+			time_candidate_backed: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_prospective_teyrchains_time_candidate_backed",
+					"Time spent within `prospective_teyrchains::handle_candidate_backed`",
+				))?,
+				registry,
+			)?,
+			time_hypothetical_membership: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_prospective_teyrchains_time_hypothetical_membership",
+					"Time spent responding to `GetHypotheticalMembership`",
+				))?,
+				registry,
+			)?,
+			candidate_count: prometheus::register(
+				GaugeVec::new(
+					Opts::new(
+						"pezkuwi_teyrchain_prospective_teyrchains_candidate_count",
+						"Number of candidates present across all fragment chains, split by connected and unconnected"
+					),
+					&["type"],
+				)?,
+				registry,
+			)?,
+			active_leaves_count: prometheus::register(
+				GaugeVec::new(
+					Opts::new(
+						"pezkuwi_teyrchain_prospective_teyrchains_active_leaves_count",
+						"Number of leaves kept by the subsystem, split by active/inactive"
+					),
+					&["type"],
+				)?,
+				registry,
+			)?,
+			implicit_view_candidate_count: prometheus::register(
+				Gauge::new(
+					"pezkuwi_teyrchain_prospective_teyrchains_implicit_view_candidate_count", 
+					"Number of candidates present in the implicit view"
+				)?,
+				registry
+			)?,
+		};
+		Ok(Metrics(Some(metrics)))
+	}
+}

pezkuwi/node/core/provisioner/Cargo.toml

@@ -0,0 +1,45 @@
+[package]
+name = "pezkuwi-node-core-provisioner"
+version = "7.0.0"
+description = "Responsible for assembling a relay chain block from a set of available teyrchain candidates"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+
+[lints]
+workspace = true
+
+[dependencies]
+bitvec = { features = ["alloc"], workspace = true }
+fatality = { workspace = true }
+futures = { workspace = true }
+futures-timer = { workspace = true }
+gum = { workspace = true, default-features = true }
+pezkuwi-node-primitives = { workspace = true, default-features = true }
+pezkuwi-node-subsystem = { workspace = true, default-features = true }
+pezkuwi-node-subsystem-util = { workspace = true, default-features = true }
+pezkuwi-primitives = { workspace = true, default-features = true }
+sc-consensus-slots = { workspace = true }
+schnellru = { workspace = true }
+thiserror = { workspace = true }
+
+[dev-dependencies]
+pezkuwi-node-subsystem-test-helpers = { workspace = true }
+pezkuwi-primitives = { workspace = true, features = ["test"] }
+pezkuwi-primitives-test-helpers = { workspace = true }
+sp-application-crypto = { workspace = true, default-features = true }
+sp-keystore = { workspace = true, default-features = true }
+
+[features]
+runtime-benchmarks = [
+	"gum/runtime-benchmarks",
+	"pezkuwi-node-primitives/runtime-benchmarks",
+	"pezkuwi-node-subsystem-test-helpers/runtime-benchmarks",
+	"pezkuwi-node-subsystem-util/runtime-benchmarks",
+	"pezkuwi-node-subsystem/runtime-benchmarks",
+	"pezkuwi-primitives-test-helpers/runtime-benchmarks",
+	"pezkuwi-primitives/runtime-benchmarks",
+	"sc-consensus-slots/runtime-benchmarks",
+]

pezkuwi/node/core/provisioner/src/disputes/mod.rs

@@ -0,0 +1,48 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! The disputes module is responsible for selecting dispute votes to be sent with the inherent
+//! data.
+
+use crate::LOG_TARGET;
+use futures::channel::oneshot;
+use pezkuwi_node_primitives::CandidateVotes;
+use pezkuwi_node_subsystem::{messages::DisputeCoordinatorMessage, overseer};
+use pezkuwi_primitives::{CandidateHash, SessionIndex};
+
+/// Request the relevant dispute statements for a set of disputes identified by `CandidateHash` and
+/// the `SessionIndex`.
+async fn request_votes(
+	sender: &mut impl overseer::ProvisionerSenderTrait,
+	disputes_to_query: Vec<(SessionIndex, CandidateHash)>,
+) -> Vec<(SessionIndex, CandidateHash, CandidateVotes)> {
+	let (tx, rx) = oneshot::channel();
+	// Bounded by block production - `ProvisionerMessage::RequestInherentData`.
+	sender.send_unbounded_message(DisputeCoordinatorMessage::QueryCandidateVotes(
+		disputes_to_query,
+		tx,
+	));
+
+	match rx.await {
+		Ok(v) => v,
+		Err(oneshot::Canceled) => {
+			gum::warn!(target: LOG_TARGET, "Unable to query candidate votes");
+			Vec::new()
+		},
+	}
+}
+
+pub(crate) mod prioritized_selection;

pezkuwi/node/core/provisioner/src/disputes/prioritized_selection/mod.rs

@@ -0,0 +1,501 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! This module uses different approach for selecting dispute votes. It queries the Runtime
+//! about the votes already known onchain and tries to select only relevant votes. Refer to
+//! the documentation of `select_disputes` for more details about the actual implementation.
+
+use crate::{error::GetOnchainDisputesError, metrics, LOG_TARGET};
+use futures::channel::oneshot;
+use pezkuwi_node_primitives::{dispute_is_inactive, CandidateVotes, DisputeStatus, Timestamp};
+use pezkuwi_node_subsystem::{
+	errors::RuntimeApiError,
+	messages::{DisputeCoordinatorMessage, RuntimeApiMessage, RuntimeApiRequest},
+	overseer, ActivatedLeaf,
+};
+use pezkuwi_primitives::{
+	supermajority_threshold, CandidateHash, DisputeState, DisputeStatement, DisputeStatementSet,
+	Hash, MultiDisputeStatementSet, SessionIndex, ValidDisputeStatementKind, ValidatorIndex,
+};
+use std::{
+	collections::{BTreeMap, HashMap},
+	time::{SystemTime, UNIX_EPOCH},
+};
+
+#[cfg(test)]
+mod tests;
+
+/// The maximum number of disputes Provisioner will include in the inherent data.
+/// Serves as a protection not to flood the Runtime with excessive data.
+#[cfg(not(test))]
+pub const MAX_DISPUTE_VOTES_FORWARDED_TO_RUNTIME: usize = 200_000;
+#[cfg(test)]
+pub const MAX_DISPUTE_VOTES_FORWARDED_TO_RUNTIME: usize = 200;
+
+/// Controls how much dispute votes to be fetched from the `dispute-coordinator` per iteration in
+/// `fn vote_selection`. The purpose is to fetch the votes in batches until
+/// `MAX_DISPUTE_VOTES_FORWARDED_TO_RUNTIME` is reached. If all votes are fetched in single call
+/// we might fetch votes which we never use. This will create unnecessary load on
+/// `dispute-coordinator`.
+///
+/// This value should be less than `MAX_DISPUTE_VOTES_FORWARDED_TO_RUNTIME`. Increase it in case
+/// `provisioner` sends too many `QueryCandidateVotes` messages to `dispute-coordinator`.
+#[cfg(not(test))]
+const VOTES_SELECTION_BATCH_SIZE: usize = 1_100;
+#[cfg(test)]
+const VOTES_SELECTION_BATCH_SIZE: usize = 11;
+
+/// Implements the `select_disputes` function which selects dispute votes which should
+/// be sent to the Runtime.
+///
+/// # How the prioritization works
+///
+/// Generally speaking disputes can be described as:
+///   * Active vs Inactive
+///   * Known vs Unknown onchain
+///   * Offchain vs Onchain
+///   * Concluded onchain vs Unconcluded onchain
+///
+/// Provisioner fetches all disputes from `dispute-coordinator` and separates them in multiple
+/// partitions. Please refer to `struct PartitionedDisputes` for details about the actual
+/// partitions. Each partition has got a priority implicitly assigned to it and the disputes are
+/// selected based on this priority (e.g. disputes in partition 1, then if there is space - disputes
+/// from partition 2 and so on).
+///
+/// # Votes selection
+///
+/// Besides the prioritization described above the votes in each partition are filtered too.
+/// Provisioner fetches all onchain votes and filters them out from all partitions. As a result the
+/// Runtime receives only fresh votes (votes it didn't know about).
+///
+/// # How the onchain votes are fetched
+///
+/// The logic outlined above relies on `RuntimeApiRequest::Disputes` message from the Runtime. The
+/// user check the Runtime version before calling `select_disputes`. If the function is used with
+/// old runtime an error is logged and the logic will continue with empty onchain votes `HashMap`.
+pub async fn select_disputes<Sender>(
+	sender: &mut Sender,
+	metrics: &metrics::Metrics,
+	leaf: &ActivatedLeaf,
+) -> MultiDisputeStatementSet
+where
+	Sender: overseer::ProvisionerSenderTrait,
+{
+	gum::trace!(
+		target: LOG_TARGET,
+		?leaf,
+		"Selecting disputes for inherent data using prioritized selection"
+	);
+
+	// Fetch the onchain disputes. We'll do a prioritization based on them.
+	let onchain = match get_onchain_disputes(sender, leaf.hash).await {
+		Ok(r) => {
+			gum::trace!(
+				target: LOG_TARGET,
+				?leaf,
+				"Successfully fetched {} onchain disputes",
+				r.len()
+			);
+			r
+		},
+		Err(GetOnchainDisputesError::NotSupported(runtime_api_err, relay_parent)) => {
+			// Runtime version is checked before calling this method, so the error below should
+			// never happen!
+			gum::error!(
+				target: LOG_TARGET,
+				?runtime_api_err,
+				?relay_parent,
+				"Can't fetch onchain disputes, because TeyrchainHost runtime api version is old. Will continue with empty onchain disputes set.",
+			);
+			HashMap::new()
+		},
+		Err(GetOnchainDisputesError::Channel) => {
+			// This error usually means the node is shutting down. Log just in case.
+			gum::debug!(
+				target: LOG_TARGET,
+				"Channel error occurred while fetching onchain disputes. Will continue with empty onchain disputes set.",
+			);
+			HashMap::new()
+		},
+		Err(GetOnchainDisputesError::Execution(runtime_api_err, parent_hash)) => {
+			gum::warn!(
+				target: LOG_TARGET,
+				?runtime_api_err,
+				?parent_hash,
+				"Unexpected execution error occurred while fetching onchain votes. Will continue with empty onchain disputes set.",
+			);
+			HashMap::new()
+		},
+	};
+	metrics.on_fetched_onchain_disputes(onchain.keys().len() as u64);
+
+	gum::trace!(target: LOG_TARGET, ?leaf, "Fetching recent disputes");
+	let recent_disputes = request_disputes(sender).await;
+	gum::trace!(
+		target: LOG_TARGET,
+		?leaf,
+		"Got {} recent disputes and {} onchain disputes.",
+		recent_disputes.len(),
+		onchain.len(),
+	);
+
+	gum::trace!(target: LOG_TARGET, ?leaf, "Filtering recent disputes");
+	// Filter out unconfirmed disputes. However if the dispute is already onchain - don't skip it.
+	// In this case we'd better push as much fresh votes as possible to bring it to conclusion
+	// faster.
+	let recent_disputes = recent_disputes
+		.into_iter()
+		.filter(|(key, dispute_status)| {
+			dispute_status.is_confirmed_concluded() || onchain.contains_key(key)
+		})
+		.collect::<BTreeMap<_, _>>();
+	gum::trace!(target: LOG_TARGET, ?leaf, "Partitioning recent disputes");
+	let partitioned = partition_recent_disputes(recent_disputes, &onchain);
+	metrics.on_partition_recent_disputes(&partitioned);
+
+	if partitioned.inactive_unknown_onchain.len() > 0 {
+		gum::warn!(
+			target: LOG_TARGET,
+			?leaf,
+			"Got {} inactive unknown onchain disputes. This should not happen in normal conditions!",
+			partitioned.inactive_unknown_onchain.len()
+		);
+	}
+
+	gum::trace!(target: LOG_TARGET, ?leaf, "Vote selection for recent disputes");
+	let result = vote_selection(sender, partitioned, &onchain).await;
+
+	gum::trace!(target: LOG_TARGET, ?leaf, "Convert to multi dispute statement set");
+	make_multi_dispute_statement_set(metrics, result)
+}
+
+/// Selects dispute votes from `PartitionedDisputes` which should be sent to the runtime. Votes
+/// which are already onchain are filtered out. Result should be sorted by `(SessionIndex,
+/// CandidateHash)` which is enforced by the `BTreeMap`. This is a requirement from the runtime.
+async fn vote_selection<Sender>(
+	sender: &mut Sender,
+	partitioned: PartitionedDisputes,
+	onchain: &HashMap<(SessionIndex, CandidateHash), DisputeState>,
+) -> BTreeMap<(SessionIndex, CandidateHash), CandidateVotes>
+where
+	Sender: overseer::ProvisionerSenderTrait,
+{
+	// fetch in batches until there are enough votes
+	let mut disputes = partitioned.into_iter().collect::<Vec<_>>();
+	let mut total_votes_len = 0;
+	let mut result = BTreeMap::new();
+	let mut request_votes_counter = 0;
+	while !disputes.is_empty() {
+		gum::trace!(target: LOG_TARGET, "has to process {} disputes left", disputes.len());
+		let batch_size = std::cmp::min(VOTES_SELECTION_BATCH_SIZE, disputes.len());
+		let batch = Vec::from_iter(disputes.drain(0..batch_size));
+
+		// Filter votes which are already onchain
+		request_votes_counter += 1;
+		gum::trace!(target: LOG_TARGET, "requesting onchain votes",);
+		let votes = super::request_votes(sender, batch)
+			.await
+			.into_iter()
+			.map(|(session_index, candidate_hash, mut votes)| {
+				let onchain_state =
+					if let Some(onchain_state) = onchain.get(&(session_index, candidate_hash)) {
+						onchain_state
+					} else {
+						// onchain knows nothing about this dispute - add all votes
+						return (session_index, candidate_hash, votes);
+					};
+
+				votes.valid.retain(|validator_idx, (statement_kind, _)| {
+					is_vote_worth_to_keep(
+						validator_idx,
+						DisputeStatement::Valid(statement_kind.clone()),
+						&onchain_state,
+					)
+				});
+				votes.invalid.retain(|validator_idx, (statement_kind, _)| {
+					is_vote_worth_to_keep(
+						validator_idx,
+						DisputeStatement::Invalid(*statement_kind),
+						&onchain_state,
+					)
+				});
+				(session_index, candidate_hash, votes)
+			})
+			.collect::<Vec<_>>();
+		gum::trace!(target: LOG_TARGET, "got {} onchain votes after processing", votes.len());
+
+		// Check if votes are within the limit
+		for (session_index, candidate_hash, selected_votes) in votes {
+			let votes_len = selected_votes.valid.raw().len() + selected_votes.invalid.len();
+			if votes_len + total_votes_len > MAX_DISPUTE_VOTES_FORWARDED_TO_RUNTIME {
+				// we are done - no more votes can be added. Importantly, we don't add any votes for
+				// a dispute here if we can't fit them all. This gives us an important invariant,
+				// that backing votes for disputes make it into the provisioned vote set.
+				gum::trace!(
+					target: LOG_TARGET,
+					?request_votes_counter,
+					?total_votes_len,
+					"vote_selection DisputeCoordinatorMessage::QueryCandidateVotes counter",
+				);
+
+				return result;
+			}
+
+			result.insert((session_index, candidate_hash), selected_votes);
+			total_votes_len += votes_len
+		}
+	}
+
+	gum::trace!(
+		target: LOG_TARGET,
+		?request_votes_counter,
+		?total_votes_len,
+		"vote_selection DisputeCoordinatorMessage::QueryCandidateVotes counter",
+	);
+
+	result
+}
+
+/// Contains disputes by partitions. Check the field comments for further details.
+#[derive(Default)]
+pub(crate) struct PartitionedDisputes {
+	/// Concluded and inactive disputes which are completely unknown for the Runtime.
+	/// Hopefully this should never happen.
+	/// Will be sent to the Runtime with FIRST priority.
+	pub inactive_unknown_onchain: Vec<(SessionIndex, CandidateHash)>,
+	/// Disputes which are INACTIVE locally but they are unconcluded for the Runtime.
+	/// A dispute can have enough local vote to conclude and at the same time the
+	/// Runtime knows nothing about them at treats it as unconcluded. This discrepancy
+	/// should be treated with high priority.
+	/// Will be sent to the Runtime with SECOND priority.
+	pub inactive_unconcluded_onchain: Vec<(SessionIndex, CandidateHash)>,
+	/// Active disputes completely unknown onchain.
+	/// Will be sent to the Runtime with THIRD priority.
+	pub active_unknown_onchain: Vec<(SessionIndex, CandidateHash)>,
+	/// Active disputes unconcluded onchain.
+	/// Will be sent to the Runtime with FOURTH priority.
+	pub active_unconcluded_onchain: Vec<(SessionIndex, CandidateHash)>,
+	/// Active disputes concluded onchain. New votes are not that important for
+	/// this partition.
+	/// Will be sent to the Runtime with FIFTH priority.
+	pub active_concluded_onchain: Vec<(SessionIndex, CandidateHash)>,
+	/// Inactive disputes which has concluded onchain. These are not interesting and
+	/// won't be sent to the Runtime.
+	/// Will be DROPPED
+	pub inactive_concluded_onchain: Vec<(SessionIndex, CandidateHash)>,
+}
+
+impl PartitionedDisputes {
+	fn new() -> PartitionedDisputes {
+		Default::default()
+	}
+
+	fn into_iter(self) -> impl Iterator<Item = (SessionIndex, CandidateHash)> {
+		self.inactive_unknown_onchain
+			.into_iter()
+			.chain(self.inactive_unconcluded_onchain.into_iter())
+			.chain(self.active_unknown_onchain.into_iter())
+			.chain(self.active_unconcluded_onchain.into_iter())
+			.chain(self.active_concluded_onchain.into_iter())
+		// inactive_concluded_onchain is dropped on purpose
+	}
+}
+
+fn secs_since_epoch() -> Timestamp {
+	match SystemTime::now().duration_since(UNIX_EPOCH) {
+		Ok(d) => d.as_secs(),
+		Err(e) => {
+			gum::warn!(
+				target: LOG_TARGET,
+				err = ?e,
+				"Error getting system time."
+			);
+			0
+		},
+	}
+}
+
+fn concluded_onchain(onchain_state: &DisputeState) -> bool {
+	// Check if there are enough onchain votes for or against to conclude the dispute
+	let supermajority = supermajority_threshold(onchain_state.validators_for.len());
+
+	onchain_state.validators_for.count_ones() >= supermajority ||
+		onchain_state.validators_against.count_ones() >= supermajority
+}
+
+fn partition_recent_disputes(
+	recent: BTreeMap<(SessionIndex, CandidateHash), DisputeStatus>,
+	onchain: &HashMap<(SessionIndex, CandidateHash), DisputeState>,
+) -> PartitionedDisputes {
+	let mut partitioned = PartitionedDisputes::new();
+	let time_now = &secs_since_epoch();
+
+	for ((session_index, candidate_hash), dispute_state) in recent {
+		let key = (session_index, candidate_hash);
+		if dispute_is_inactive(&dispute_state, time_now) {
+			match onchain.get(&key) {
+				Some(onchain_state) =>
+					if concluded_onchain(onchain_state) {
+						partitioned
+							.inactive_concluded_onchain
+							.push((session_index, candidate_hash));
+					} else {
+						partitioned
+							.inactive_unconcluded_onchain
+							.push((session_index, candidate_hash));
+					},
+				None => partitioned.inactive_unknown_onchain.push((session_index, candidate_hash)),
+			}
+		} else {
+			match onchain.get(&(session_index, candidate_hash)) {
+				Some(d) => match concluded_onchain(d) {
+					true =>
+						partitioned.active_concluded_onchain.push((session_index, candidate_hash)),
+					false =>
+						partitioned.active_unconcluded_onchain.push((session_index, candidate_hash)),
+				},
+				None => partitioned.active_unknown_onchain.push((session_index, candidate_hash)),
+			}
+		}
+	}
+
+	partitioned
+}
+
+/// Determines if a vote is worth to be kept, based on the onchain disputes
+fn is_vote_worth_to_keep(
+	validator_index: &ValidatorIndex,
+	dispute_statement: DisputeStatement,
+	onchain_state: &DisputeState,
+) -> bool {
+	let (offchain_vote, valid_kind) = match dispute_statement {
+		DisputeStatement::Valid(kind) => (true, Some(kind)),
+		DisputeStatement::Invalid(_) => (false, None),
+	};
+	// We want to keep all backing votes. This maximizes the number of backers
+	// punished when misbehaving.
+	if let Some(kind) = valid_kind {
+		match kind {
+			ValidDisputeStatementKind::BackingValid(_) |
+			ValidDisputeStatementKind::BackingSeconded(_) => return true,
+			_ => (),
+		}
+	}
+
+	let in_validators_for = onchain_state
+		.validators_for
+		.get(validator_index.0 as usize)
+		.as_deref()
+		.copied()
+		.unwrap_or(false);
+	let in_validators_against = onchain_state
+		.validators_against
+		.get(validator_index.0 as usize)
+		.as_deref()
+		.copied()
+		.unwrap_or(false);
+
+	if in_validators_for && in_validators_against {
+		// The validator has double voted and runtime knows about this. Ignore this vote.
+		return false;
+	}
+
+	if offchain_vote && in_validators_against || !offchain_vote && in_validators_for {
+		// offchain vote differs from the onchain vote
+		// we need this vote to punish the offending validator
+		return true;
+	}
+
+	// The vote is valid. Return true if it is not seen onchain.
+	!in_validators_for && !in_validators_against
+}
+
+/// Request disputes identified by `CandidateHash` and the `SessionIndex`.
+async fn request_disputes(
+	sender: &mut impl overseer::ProvisionerSenderTrait,
+) -> BTreeMap<(SessionIndex, CandidateHash), DisputeStatus> {
+	let (tx, rx) = oneshot::channel();
+	let msg = DisputeCoordinatorMessage::RecentDisputes(tx);
+
+	// Bounded by block production - `ProvisionerMessage::RequestInherentData`.
+	sender.send_unbounded_message(msg);
+
+	let recent_disputes = rx.await.unwrap_or_else(|err| {
+		gum::warn!(target: LOG_TARGET, err=?err, "Unable to gather recent disputes");
+		BTreeMap::new()
+	});
+	recent_disputes
+}
+
+// This function produces the return value for `pub fn select_disputes()`
+fn make_multi_dispute_statement_set(
+	metrics: &metrics::Metrics,
+	dispute_candidate_votes: BTreeMap<(SessionIndex, CandidateHash), CandidateVotes>,
+) -> MultiDisputeStatementSet {
+	// Transform all `CandidateVotes` into `MultiDisputeStatementSet`.
+	dispute_candidate_votes
+		.into_iter()
+		.map(|((session_index, candidate_hash), votes)| {
+			let valid_statements = votes
+				.valid
+				.into_iter()
+				.map(|(i, (s, sig))| (DisputeStatement::Valid(s), i, sig));
+
+			let invalid_statements = votes
+				.invalid
+				.into_iter()
+				.map(|(i, (s, sig))| (DisputeStatement::Invalid(s), i, sig));
+
+			metrics.inc_valid_statements_by(valid_statements.len());
+			metrics.inc_invalid_statements_by(invalid_statements.len());
+			metrics.inc_dispute_statement_sets_by(1);
+
+			DisputeStatementSet {
+				candidate_hash,
+				session: session_index,
+				statements: valid_statements.chain(invalid_statements).collect(),
+			}
+		})
+		.collect()
+}
+
+/// Gets the on-chain disputes at a given block number and returns them as a `HashMap` so that
+/// searching in them is cheap.
+pub async fn get_onchain_disputes<Sender>(
+	sender: &mut Sender,
+	relay_parent: Hash,
+) -> Result<HashMap<(SessionIndex, CandidateHash), DisputeState>, GetOnchainDisputesError>
+where
+	Sender: overseer::ProvisionerSenderTrait,
+{
+	gum::trace!(target: LOG_TARGET, ?relay_parent, "Fetching on-chain disputes");
+	let (tx, rx) = oneshot::channel();
+	sender
+		.send_message(RuntimeApiMessage::Request(relay_parent, RuntimeApiRequest::Disputes(tx)))
+		.await;
+
+	rx.await
+		.map_err(|_| GetOnchainDisputesError::Channel)
+		.and_then(|res| {
+			res.map_err(|e| match e {
+				RuntimeApiError::Execution { .. } =>
+					GetOnchainDisputesError::Execution(e, relay_parent),
+				RuntimeApiError::NotSupported { .. } =>
+					GetOnchainDisputesError::NotSupported(e, relay_parent),
+			})
+		})
+		.map(|v| v.into_iter().map(|e| ((e.0, e.1), e.2)).collect())
+}

pezkuwi/node/core/provisioner/src/disputes/prioritized_selection/tests.rs

@@ -0,0 +1,764 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use super::super::{
+	super::{tests::common::test_harness, *},
+	prioritized_selection::*,
+};
+use bitvec::prelude::*;
+use futures::channel::mpsc;
+use pezkuwi_node_primitives::{CandidateVotes, DisputeStatus, ACTIVE_DURATION_SECS};
+use pezkuwi_node_subsystem::messages::{
+	AllMessages, DisputeCoordinatorMessage, RuntimeApiMessage, RuntimeApiRequest,
+};
+use pezkuwi_node_subsystem_test_helpers::{mock::new_leaf, TestSubsystemSender};
+use pezkuwi_primitives::{
+	CandidateHash, CandidateReceiptV2 as CandidateReceipt, DisputeState,
+	InvalidDisputeStatementKind, SessionIndex, ValidDisputeStatementKind, ValidatorSignature,
+};
+
+//
+// Unit tests for various functions
+//
+#[test]
+fn should_keep_vote_behaves() {
+	let onchain_state = DisputeState {
+		validators_for: bitvec![u8, Lsb0; 1, 0, 1, 0, 1],
+		validators_against: bitvec![u8, Lsb0; 0, 1, 0, 0, 1],
+		start: 1,
+		concluded_at: None,
+	};
+
+	let local_valid_known = (ValidatorIndex(0), ValidDisputeStatementKind::Explicit);
+	let local_valid_unknown = (ValidatorIndex(3), ValidDisputeStatementKind::Explicit);
+
+	let local_invalid_known = (ValidatorIndex(1), InvalidDisputeStatementKind::Explicit);
+	let local_invalid_unknown = (ValidatorIndex(3), InvalidDisputeStatementKind::Explicit);
+
+	assert_eq!(
+		is_vote_worth_to_keep(
+			&local_valid_known.0,
+			DisputeStatement::Valid(local_valid_known.1),
+			&onchain_state
+		),
+		false
+	);
+	assert_eq!(
+		is_vote_worth_to_keep(
+			&local_valid_unknown.0,
+			DisputeStatement::Valid(local_valid_unknown.1),
+			&onchain_state
+		),
+		true
+	);
+	assert_eq!(
+		is_vote_worth_to_keep(
+			&local_invalid_known.0,
+			DisputeStatement::Invalid(local_invalid_known.1),
+			&onchain_state
+		),
+		false
+	);
+	assert_eq!(
+		is_vote_worth_to_keep(
+			&local_invalid_unknown.0,
+			DisputeStatement::Invalid(local_invalid_unknown.1),
+			&onchain_state
+		),
+		true
+	);
+
+	//double voting - onchain knows
+	let local_double_vote_onchain_knows =
+		(ValidatorIndex(4), InvalidDisputeStatementKind::Explicit);
+	assert_eq!(
+		is_vote_worth_to_keep(
+			&local_double_vote_onchain_knows.0,
+			DisputeStatement::Invalid(local_double_vote_onchain_knows.1),
+			&onchain_state
+		),
+		false
+	);
+
+	//double voting - onchain doesn't know
+	let local_double_vote_onchain_doesnt_knows =
+		(ValidatorIndex(0), InvalidDisputeStatementKind::Explicit);
+	assert_eq!(
+		is_vote_worth_to_keep(
+			&local_double_vote_onchain_doesnt_knows.0,
+			DisputeStatement::Invalid(local_double_vote_onchain_doesnt_knows.1),
+			&onchain_state
+		),
+		true
+	);
+
+	// empty onchain state
+	let empty_onchain_state = DisputeState {
+		validators_for: BitVec::new(),
+		validators_against: BitVec::new(),
+		start: 1,
+		concluded_at: None,
+	};
+	assert_eq!(
+		is_vote_worth_to_keep(
+			&local_double_vote_onchain_doesnt_knows.0,
+			DisputeStatement::Invalid(local_double_vote_onchain_doesnt_knows.1),
+			&empty_onchain_state
+		),
+		true
+	);
+}
+
+#[test]
+fn partitioning_happy_case() {
+	let mut input = BTreeMap::<(SessionIndex, CandidateHash), DisputeStatus>::new();
+	let mut onchain = HashMap::<(u32, CandidateHash), DisputeState>::new();
+	let time_now = secs_since_epoch();
+
+	// Create one dispute for each partition
+	let inactive_unknown_onchain = (
+		(0, CandidateHash(Hash::random())),
+		DisputeStatus::ConcludedFor(time_now - ACTIVE_DURATION_SECS * 2),
+	);
+	input.insert(inactive_unknown_onchain.0, inactive_unknown_onchain.1);
+
+	let inactive_unconcluded_onchain = (
+		(1, CandidateHash(Hash::random())),
+		DisputeStatus::ConcludedFor(time_now - ACTIVE_DURATION_SECS * 2),
+	);
+	input.insert(inactive_unconcluded_onchain.0, inactive_unconcluded_onchain.1);
+	onchain.insert(
+		inactive_unconcluded_onchain.0,
+		DisputeState {
+			validators_for: bitvec![u8, Lsb0; 1, 1, 1, 0, 0, 0, 0, 0, 0],
+			validators_against: bitvec![u8, Lsb0; 0, 0, 0, 0, 0, 0, 0, 0, 0],
+			start: 1,
+			concluded_at: None,
+		},
+	);
+
+	let active_unknown_onchain = ((2, CandidateHash(Hash::random())), DisputeStatus::Active);
+	input.insert(active_unknown_onchain.0, active_unknown_onchain.1);
+
+	let active_unconcluded_onchain = ((3, CandidateHash(Hash::random())), DisputeStatus::Active);
+	input.insert(active_unconcluded_onchain.0, active_unconcluded_onchain.1);
+	onchain.insert(
+		active_unconcluded_onchain.0,
+		DisputeState {
+			validators_for: bitvec![u8, Lsb0; 1, 1, 1, 0, 0, 0, 0, 0, 0],
+			validators_against: bitvec![u8, Lsb0; 0, 0, 0, 0, 0, 0, 0, 0, 0],
+			start: 1,
+			concluded_at: None,
+		},
+	);
+
+	let active_concluded_onchain = ((4, CandidateHash(Hash::random())), DisputeStatus::Active);
+	input.insert(active_concluded_onchain.0, active_concluded_onchain.1);
+	onchain.insert(
+		active_concluded_onchain.0,
+		DisputeState {
+			validators_for: bitvec![u8, Lsb0; 1, 1, 1, 1, 1, 1, 1, 1, 0],
+			validators_against: bitvec![u8, Lsb0; 0, 0, 0, 0, 0, 0, 0, 0, 0],
+			start: 1,
+			concluded_at: Some(3),
+		},
+	);
+
+	let inactive_concluded_onchain = (
+		(5, CandidateHash(Hash::random())),
+		DisputeStatus::ConcludedFor(time_now - ACTIVE_DURATION_SECS * 2),
+	);
+	input.insert(inactive_concluded_onchain.0, inactive_concluded_onchain.1);
+	onchain.insert(
+		inactive_concluded_onchain.0,
+		DisputeState {
+			validators_for: bitvec![u8, Lsb0; 1, 1, 1, 1, 1, 1, 1, 0, 0],
+			validators_against: bitvec![u8, Lsb0; 0, 0, 0, 0, 0, 0, 0, 0, 0],
+			start: 1,
+			concluded_at: Some(3),
+		},
+	);
+
+	let result = partition_recent_disputes(input, &onchain);
+
+	// Check results
+	assert_eq!(result.inactive_unknown_onchain.len(), 1);
+	assert_eq!(result.inactive_unknown_onchain.get(0).unwrap(), &inactive_unknown_onchain.0);
+
+	assert_eq!(result.inactive_unconcluded_onchain.len(), 1);
+	assert_eq!(
+		result.inactive_unconcluded_onchain.get(0).unwrap(),
+		&inactive_unconcluded_onchain.0
+	);
+
+	assert_eq!(result.active_unknown_onchain.len(), 1);
+	assert_eq!(result.active_unknown_onchain.get(0).unwrap(), &active_unknown_onchain.0);
+
+	assert_eq!(result.active_unconcluded_onchain.len(), 1);
+	assert_eq!(result.active_unconcluded_onchain.get(0).unwrap(), &active_unconcluded_onchain.0);
+
+	assert_eq!(result.active_concluded_onchain.len(), 1);
+	assert_eq!(result.active_concluded_onchain.get(0).unwrap(), &active_concluded_onchain.0);
+
+	assert_eq!(result.inactive_concluded_onchain.len(), 1);
+	assert_eq!(result.inactive_concluded_onchain.get(0).unwrap(), &inactive_concluded_onchain.0);
+}
+
+// This test verifies the double voting behavior. Currently we don't care if a supermajority is
+// achieved with or without the 'help' of a double vote (a validator voting for and against at the
+// same time). This makes the test a bit pointless but anyway I'm leaving it here to make this
+// decision explicit and have the test code ready in case this behavior needs to be further tested
+// in the future. Link to the PR with the discussions: https://github.com/paritytech/polkadot/pull/5567
+#[test]
+fn partitioning_doubled_onchain_vote() {
+	let mut input = BTreeMap::<(SessionIndex, CandidateHash), DisputeStatus>::new();
+	let mut onchain = HashMap::<(u32, CandidateHash), DisputeState>::new();
+
+	// Dispute A relies on a 'double onchain vote' to conclude. Validator with index 0 has voted
+	// both `for` and `against`. Despite that this dispute should be considered 'can conclude
+	// onchain'.
+	let dispute_a = ((3, CandidateHash(Hash::random())), DisputeStatus::Active);
+	// Dispute B has supermajority + 1 votes, so the doubled onchain vote doesn't affect it. It
+	// should be considered as 'can conclude onchain'.
+	let dispute_b = ((4, CandidateHash(Hash::random())), DisputeStatus::Active);
+	input.insert(dispute_a.0, dispute_a.1);
+	input.insert(dispute_b.0, dispute_b.1);
+	onchain.insert(
+		dispute_a.0,
+		DisputeState {
+			validators_for: bitvec![u8, Lsb0; 1, 1, 1, 1, 1, 1, 1, 0, 0],
+			validators_against: bitvec![u8, Lsb0; 1, 0, 0, 0, 0, 0, 0, 0, 0],
+			start: 1,
+			concluded_at: None,
+		},
+	);
+	onchain.insert(
+		dispute_b.0,
+		DisputeState {
+			validators_for: bitvec![u8, Lsb0; 1, 1, 1, 1, 1, 1, 1, 1, 0],
+			validators_against: bitvec![u8, Lsb0; 1, 0, 0, 0, 0, 0, 0, 0, 0],
+			start: 1,
+			concluded_at: None,
+		},
+	);
+
+	let result = partition_recent_disputes(input, &onchain);
+
+	assert_eq!(result.active_unconcluded_onchain.len(), 0);
+	assert_eq!(result.active_concluded_onchain.len(), 2);
+}
+
+#[test]
+fn partitioning_duplicated_dispute() {
+	let mut input = BTreeMap::<(SessionIndex, CandidateHash), DisputeStatus>::new();
+	let mut onchain = HashMap::<(u32, CandidateHash), DisputeState>::new();
+
+	let some_dispute = ((3, CandidateHash(Hash::random())), DisputeStatus::Active);
+	input.insert(some_dispute.0, some_dispute.1);
+	input.insert(some_dispute.0, some_dispute.1);
+	onchain.insert(
+		some_dispute.0,
+		DisputeState {
+			validators_for: bitvec![u8, Lsb0; 1, 1, 1, 0, 0, 0, 0, 0, 0],
+			validators_against: bitvec![u8, Lsb0; 0, 0, 0, 0, 0, 0, 0, 0, 0],
+			start: 1,
+			concluded_at: None,
+		},
+	);
+
+	let result = partition_recent_disputes(input, &onchain);
+
+	assert_eq!(result.active_unconcluded_onchain.len(), 1);
+	assert_eq!(result.active_unconcluded_onchain.get(0).unwrap(), &some_dispute.0);
+}
+
+//
+// end-to-end tests for select_disputes()
+//
+
+async fn mock_overseer(
+	mut receiver: mpsc::UnboundedReceiver<AllMessages>,
+	disputes_db: &mut TestDisputes,
+	vote_queries_count: &mut usize,
+) {
+	while let Some(from_job) = receiver.next().await {
+		match from_job {
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+				_,
+				RuntimeApiRequest::Disputes(sender),
+			)) => {
+				let _ = sender.send(Ok(disputes_db
+					.onchain_disputes
+					.clone()
+					.into_iter()
+					.map(|(k, v)| (k.0, k.1, v))
+					.collect::<Vec<_>>()));
+			},
+			AllMessages::RuntimeApi(_) => panic!("Unexpected RuntimeApi request"),
+			AllMessages::DisputeCoordinator(DisputeCoordinatorMessage::RecentDisputes(sender)) => {
+				let _ = sender.send(disputes_db.local_disputes.clone());
+			},
+			AllMessages::DisputeCoordinator(DisputeCoordinatorMessage::QueryCandidateVotes(
+				disputes,
+				sender,
+			)) => {
+				*vote_queries_count += 1;
+				let mut res = Vec::new();
+				for d in disputes.iter() {
+					let v = disputes_db.votes_db.get(d).unwrap().clone();
+					res.push((d.0, d.1, v));
+				}
+
+				let _ = sender.send(res);
+			},
+			_ => panic!("Unexpected message: {:?}", from_job),
+		}
+	}
+}
+
+fn leaf() -> ActivatedLeaf {
+	new_leaf(Hash::repeat_byte(0xAA), 0xAA)
+}
+
+struct TestDisputes {
+	pub local_disputes: BTreeMap<(SessionIndex, CandidateHash), DisputeStatus>,
+	pub votes_db: HashMap<(SessionIndex, CandidateHash), CandidateVotes>,
+	pub onchain_disputes: HashMap<(u32, CandidateHash), DisputeState>,
+	validators_count: usize,
+}
+
+impl TestDisputes {
+	pub fn new(validators_count: usize) -> TestDisputes {
+		TestDisputes {
+			local_disputes: BTreeMap::<(SessionIndex, CandidateHash), DisputeStatus>::new(),
+			votes_db: HashMap::<(SessionIndex, CandidateHash), CandidateVotes>::new(),
+			onchain_disputes: HashMap::<(u32, CandidateHash), DisputeState>::new(),
+			validators_count,
+		}
+	}
+
+	// Offchain disputes are on node side
+	fn add_offchain_dispute(
+		&mut self,
+		dispute: (SessionIndex, CandidateHash, DisputeStatus),
+		local_votes_count: usize,
+		dummy_receipt: CandidateReceipt,
+	) {
+		self.local_disputes.insert((dispute.0, dispute.1), dispute.2);
+		self.votes_db.insert(
+			(dispute.0, dispute.1),
+			CandidateVotes {
+				candidate_receipt: dummy_receipt,
+				valid: TestDisputes::generate_local_votes(
+					ValidDisputeStatementKind::Explicit,
+					0,
+					local_votes_count,
+				)
+				.into_iter()
+				.collect(),
+				invalid: BTreeMap::new(),
+			},
+		);
+	}
+
+	fn add_onchain_dispute(
+		&mut self,
+		dispute: (SessionIndex, CandidateHash, DisputeStatus),
+		onchain_votes_count: usize,
+	) {
+		let concluded_at = match dispute.2 {
+			DisputeStatus::Active | DisputeStatus::Confirmed => None,
+			DisputeStatus::ConcludedAgainst(_) | DisputeStatus::ConcludedFor(_) => Some(1),
+		};
+		self.onchain_disputes.insert(
+			(dispute.0, dispute.1),
+			DisputeState {
+				validators_for: TestDisputes::generate_bitvec(
+					self.validators_count,
+					0,
+					onchain_votes_count,
+				),
+				validators_against: bitvec![u8, Lsb0; 0; self.validators_count],
+				start: 1,
+				concluded_at,
+			},
+		);
+	}
+
+	pub fn add_unconfirmed_disputes_concluded_onchain(
+		&mut self,
+		dispute_count: usize,
+	) -> (SessionIndex, usize) {
+		let local_votes_count = self.validators_count * 90 / 100;
+		let onchain_votes_count = self.validators_count * 80 / 100;
+		let session_idx = 0;
+		let lf = leaf();
+		let dummy_receipt = pezkuwi_primitives_test_helpers::dummy_candidate_receipt_v2(lf.hash);
+		for _ in 0..dispute_count {
+			let d = (session_idx, CandidateHash(Hash::random()), DisputeStatus::Active);
+			self.add_offchain_dispute(d, local_votes_count, dummy_receipt.clone());
+			self.add_onchain_dispute(d, onchain_votes_count);
+		}
+
+		(session_idx, (local_votes_count - onchain_votes_count) * dispute_count)
+	}
+
+	pub fn add_unconfirmed_disputes_unconcluded_onchain(
+		&mut self,
+		dispute_count: usize,
+	) -> (SessionIndex, usize) {
+		let local_votes_count = self.validators_count * 90 / 100;
+		let onchain_votes_count = self.validators_count * 40 / 100;
+		let session_idx = 1;
+		let lf = leaf();
+		let dummy_receipt = pezkuwi_primitives_test_helpers::dummy_candidate_receipt_v2(lf.hash);
+		for _ in 0..dispute_count {
+			let d = (session_idx, CandidateHash(Hash::random()), DisputeStatus::Active);
+			self.add_offchain_dispute(d, local_votes_count, dummy_receipt.clone());
+			self.add_onchain_dispute(d, onchain_votes_count);
+		}
+
+		(session_idx, (local_votes_count - onchain_votes_count) * dispute_count)
+	}
+
+	pub fn add_confirmed_disputes_unknown_onchain(
+		&mut self,
+		dispute_count: usize,
+	) -> (SessionIndex, usize) {
+		let local_votes_count = self.validators_count * 90 / 100;
+		let session_idx = 2;
+		let lf = leaf();
+		let dummy_receipt = pezkuwi_primitives_test_helpers::dummy_candidate_receipt_v2(lf.hash);
+		for _ in 0..dispute_count {
+			let d = (session_idx, CandidateHash(Hash::random()), DisputeStatus::Confirmed);
+			self.add_offchain_dispute(d, local_votes_count, dummy_receipt.clone());
+		}
+		(session_idx, local_votes_count * dispute_count)
+	}
+
+	pub fn add_concluded_disputes_known_onchain(
+		&mut self,
+		dispute_count: usize,
+	) -> (SessionIndex, usize) {
+		let local_votes_count = self.validators_count * 90 / 100;
+		let onchain_votes_count = self.validators_count * 75 / 100;
+		let session_idx = 3;
+		let lf = leaf();
+		let dummy_receipt = pezkuwi_primitives_test_helpers::dummy_candidate_receipt_v2(lf.hash);
+		for _ in 0..dispute_count {
+			let d = (session_idx, CandidateHash(Hash::random()), DisputeStatus::ConcludedFor(0));
+			self.add_offchain_dispute(d, local_votes_count, dummy_receipt.clone());
+			self.add_onchain_dispute(d, onchain_votes_count);
+		}
+		(session_idx, (local_votes_count - onchain_votes_count) * dispute_count)
+	}
+
+	pub fn add_concluded_disputes_unknown_onchain(
+		&mut self,
+		dispute_count: usize,
+	) -> (SessionIndex, usize) {
+		let local_votes_count = self.validators_count * 90 / 100;
+		let session_idx = 4;
+		let lf = leaf();
+		let dummy_receipt = pezkuwi_primitives_test_helpers::dummy_candidate_receipt_v2(lf.hash);
+		for _ in 0..dispute_count {
+			let d = (session_idx, CandidateHash(Hash::random()), DisputeStatus::ConcludedFor(0));
+			self.add_offchain_dispute(d, local_votes_count, dummy_receipt.clone());
+		}
+		(session_idx, local_votes_count * dispute_count)
+	}
+
+	pub fn add_unconfirmed_disputes_known_onchain(
+		&mut self,
+		dispute_count: usize,
+	) -> (SessionIndex, usize) {
+		let local_votes_count = self.validators_count * 10 / 100;
+		let onchain_votes_count = self.validators_count * 10 / 100;
+		let session_idx = 5;
+		let lf = leaf();
+		let dummy_receipt = pezkuwi_primitives_test_helpers::dummy_candidate_receipt_v2(lf.hash);
+		for _ in 0..dispute_count {
+			let d = (session_idx, CandidateHash(Hash::random()), DisputeStatus::Active);
+			self.add_offchain_dispute(d, local_votes_count, dummy_receipt.clone());
+			self.add_onchain_dispute(d, onchain_votes_count);
+		}
+
+		(session_idx, (local_votes_count - onchain_votes_count) * dispute_count)
+	}
+
+	pub fn add_unconfirmed_disputes_unknown_onchain(
+		&mut self,
+		dispute_count: usize,
+	) -> (SessionIndex, usize) {
+		let local_votes_count = self.validators_count * 10 / 100;
+		let session_idx = 6;
+		let lf = leaf();
+		let dummy_receipt = pezkuwi_primitives_test_helpers::dummy_candidate_receipt_v2(lf.hash);
+		for _ in 0..dispute_count {
+			let d = (session_idx, CandidateHash(Hash::random()), DisputeStatus::Active);
+			self.add_offchain_dispute(d, local_votes_count, dummy_receipt.clone());
+		}
+
+		(session_idx, local_votes_count * dispute_count)
+	}
+
+	fn generate_local_votes<T: Clone>(
+		statement_kind: T,
+		start_idx: usize,
+		count: usize,
+	) -> BTreeMap<ValidatorIndex, (T, ValidatorSignature)> {
+		assert!(start_idx < count);
+		(start_idx..count)
+			.map(|idx| {
+				(
+					ValidatorIndex(idx as u32),
+					(statement_kind.clone(), pezkuwi_primitives_test_helpers::dummy_signature()),
+				)
+			})
+			.collect::<BTreeMap<_, _>>()
+	}
+
+	fn generate_bitvec(
+		validator_count: usize,
+		start_idx: usize,
+		count: usize,
+	) -> BitVec<u8, bitvec::order::Lsb0> {
+		assert!(start_idx < count);
+		assert!(start_idx + count < validator_count);
+		let mut res = bitvec![u8, Lsb0; 0; validator_count];
+		for idx in start_idx..count {
+			res.set(idx, true);
+		}
+
+		res
+	}
+}
+
+#[test]
+fn normal_flow() {
+	const VALIDATOR_COUNT: usize = 10;
+	const DISPUTES_PER_BATCH: usize = 2;
+	const ACCEPTABLE_RUNTIME_VOTES_QUERIES_COUNT: usize = 1;
+
+	let mut input = TestDisputes::new(VALIDATOR_COUNT);
+
+	// active, concluded onchain
+	let (third_idx, third_votes) =
+		input.add_unconfirmed_disputes_concluded_onchain(DISPUTES_PER_BATCH);
+
+	// active unconcluded onchain
+	let (first_idx, first_votes) =
+		input.add_unconfirmed_disputes_unconcluded_onchain(DISPUTES_PER_BATCH);
+
+	//concluded disputes unknown onchain
+	let (fifth_idx, fifth_votes) = input.add_concluded_disputes_unknown_onchain(DISPUTES_PER_BATCH);
+
+	// concluded disputes known onchain - these should be ignored
+	let (_, _) = input.add_concluded_disputes_known_onchain(DISPUTES_PER_BATCH);
+
+	// confirmed disputes unknown onchain
+	let (second_idx, second_votes) =
+		input.add_confirmed_disputes_unknown_onchain(DISPUTES_PER_BATCH);
+
+	let metrics = metrics::Metrics::new_dummy();
+	let mut vote_queries: usize = 0;
+	test_harness(
+		|r| mock_overseer(r, &mut input, &mut vote_queries),
+		|mut tx: TestSubsystemSender| async move {
+			let lf = leaf();
+			let result = select_disputes(&mut tx, &metrics, &lf).await;
+
+			assert!(!result.is_empty());
+
+			assert_eq!(result.len(), 4 * DISPUTES_PER_BATCH);
+
+			// Naive checks that the result is partitioned correctly
+			let (first_batch, rest): (Vec<DisputeStatementSet>, Vec<DisputeStatementSet>) =
+				result.into_iter().partition(|d| d.session == first_idx);
+			assert_eq!(first_batch.len(), DISPUTES_PER_BATCH);
+
+			let (second_batch, rest): (Vec<DisputeStatementSet>, Vec<DisputeStatementSet>) =
+				rest.into_iter().partition(|d| d.session == second_idx);
+			assert_eq!(second_batch.len(), DISPUTES_PER_BATCH);
+
+			let (third_batch, rest): (Vec<DisputeStatementSet>, Vec<DisputeStatementSet>) =
+				rest.into_iter().partition(|d| d.session == third_idx);
+			assert_eq!(third_batch.len(), DISPUTES_PER_BATCH);
+
+			let (fifth_batch, rest): (Vec<DisputeStatementSet>, Vec<DisputeStatementSet>) =
+				rest.into_iter().partition(|d| d.session == fifth_idx);
+			assert_eq!(fifth_batch.len(), DISPUTES_PER_BATCH);
+
+			// Ensure there are no more disputes - fourth_batch should be dropped
+			assert_eq!(rest.len(), 0);
+
+			assert_eq!(
+				first_batch.iter().map(|d| d.statements.len()).fold(0, |acc, v| acc + v),
+				first_votes
+			);
+			assert_eq!(
+				second_batch.iter().map(|d| d.statements.len()).fold(0, |acc, v| acc + v),
+				second_votes
+			);
+			assert_eq!(
+				third_batch.iter().map(|d| d.statements.len()).fold(0, |acc, v| acc + v),
+				third_votes
+			);
+			assert_eq!(
+				fifth_batch.iter().map(|d| d.statements.len()).fold(0, |acc, v| acc + v),
+				fifth_votes
+			);
+		},
+	);
+	assert!(vote_queries <= ACCEPTABLE_RUNTIME_VOTES_QUERIES_COUNT);
+}
+
+#[test]
+fn many_batches() {
+	const VALIDATOR_COUNT: usize = 10;
+	const DISPUTES_PER_PARTITION: usize = 10;
+
+	// 10 disputes per partition * 4 partitions = 40 disputes
+	// BATCH_SIZE = 11
+	// => There should be no more than 40 / 11 queries ( ~4 )
+	const ACCEPTABLE_RUNTIME_VOTES_QUERIES_COUNT: usize = 4;
+
+	let mut input = TestDisputes::new(VALIDATOR_COUNT);
+
+	// active which can conclude onchain
+	input.add_unconfirmed_disputes_concluded_onchain(DISPUTES_PER_PARTITION);
+
+	// active which can't conclude onchain
+	input.add_unconfirmed_disputes_unconcluded_onchain(DISPUTES_PER_PARTITION);
+
+	//concluded disputes unknown onchain
+	input.add_concluded_disputes_unknown_onchain(DISPUTES_PER_PARTITION);
+
+	// concluded disputes known onchain
+	input.add_concluded_disputes_known_onchain(DISPUTES_PER_PARTITION);
+
+	// confirmed disputes unknown onchain
+	input.add_confirmed_disputes_unknown_onchain(DISPUTES_PER_PARTITION);
+
+	let metrics = metrics::Metrics::new_dummy();
+	let mut vote_queries: usize = 0;
+	test_harness(
+		|r| mock_overseer(r, &mut input, &mut vote_queries),
+		|mut tx: TestSubsystemSender| async move {
+			let lf = leaf();
+			let result = select_disputes(&mut tx, &metrics, &lf).await;
+
+			assert!(!result.is_empty());
+
+			let vote_count = result.iter().map(|d| d.statements.len()).fold(0, |acc, v| acc + v);
+
+			assert!(
+				MAX_DISPUTE_VOTES_FORWARDED_TO_RUNTIME - VALIDATOR_COUNT <= vote_count &&
+					vote_count <= MAX_DISPUTE_VOTES_FORWARDED_TO_RUNTIME,
+				"vote_count: {}",
+				vote_count
+			);
+		},
+	);
+
+	assert!(
+		vote_queries <= ACCEPTABLE_RUNTIME_VOTES_QUERIES_COUNT,
+		"vote_queries: {} ACCEPTABLE_RUNTIME_VOTES_QUERIES_COUNT: {}",
+		vote_queries,
+		ACCEPTABLE_RUNTIME_VOTES_QUERIES_COUNT
+	);
+}
+
+#[test]
+fn votes_above_limit() {
+	const VALIDATOR_COUNT: usize = 10;
+	const DISPUTES_PER_PARTITION: usize = 50;
+	const ACCEPTABLE_RUNTIME_VOTES_QUERIES_COUNT: usize = 4;
+
+	let mut input = TestDisputes::new(VALIDATOR_COUNT);
+
+	// active which can conclude onchain
+	let (_, second_votes) =
+		input.add_unconfirmed_disputes_concluded_onchain(DISPUTES_PER_PARTITION);
+
+	// active which can't conclude onchain
+	let (_, first_votes) =
+		input.add_unconfirmed_disputes_unconcluded_onchain(DISPUTES_PER_PARTITION);
+
+	//concluded disputes unknown onchain
+	let (_, third_votes) = input.add_concluded_disputes_unknown_onchain(DISPUTES_PER_PARTITION);
+
+	assert!(
+		first_votes + second_votes + third_votes > 3 * MAX_DISPUTE_VOTES_FORWARDED_TO_RUNTIME,
+		"Total relevant votes generated: {}",
+		first_votes + second_votes + third_votes
+	);
+
+	let metrics = metrics::Metrics::new_dummy();
+	let mut vote_queries: usize = 0;
+	test_harness(
+		|r| mock_overseer(r, &mut input, &mut vote_queries),
+		|mut tx: TestSubsystemSender| async move {
+			let lf = leaf();
+			let result = select_disputes(&mut tx, &metrics, &lf).await;
+
+			assert!(!result.is_empty());
+
+			let vote_count = result.iter().map(|d| d.statements.len()).fold(0, |acc, v| acc + v);
+
+			assert!(
+				MAX_DISPUTE_VOTES_FORWARDED_TO_RUNTIME - VALIDATOR_COUNT <= vote_count &&
+					vote_count <= MAX_DISPUTE_VOTES_FORWARDED_TO_RUNTIME,
+				"vote_count: {}",
+				vote_count
+			);
+		},
+	);
+
+	assert!(
+		vote_queries <= ACCEPTABLE_RUNTIME_VOTES_QUERIES_COUNT,
+		"vote_queries: {} ACCEPTABLE_RUNTIME_VOTES_QUERIES_COUNT: {}",
+		vote_queries,
+		ACCEPTABLE_RUNTIME_VOTES_QUERIES_COUNT
+	);
+}
+
+#[test]
+fn unconfirmed_are_handled_correctly() {
+	const VALIDATOR_COUNT: usize = 10;
+	const DISPUTES_PER_PARTITION: usize = 50;
+
+	let mut input = TestDisputes::new(VALIDATOR_COUNT);
+
+	// Add unconfirmed known onchain -> this should be pushed
+	let (pushed_idx, _) = input.add_unconfirmed_disputes_known_onchain(DISPUTES_PER_PARTITION);
+
+	// Add unconfirmed unknown onchain -> this should be ignored
+	input.add_unconfirmed_disputes_unknown_onchain(DISPUTES_PER_PARTITION);
+
+	let metrics = metrics::Metrics::new_dummy();
+	let mut vote_queries: usize = 0;
+	test_harness(
+		|r| mock_overseer(r, &mut input, &mut vote_queries),
+		|mut tx: TestSubsystemSender| async move {
+			let lf = leaf();
+			let result = select_disputes(&mut tx, &metrics, &lf).await;
+
+			assert!(result.len() == DISPUTES_PER_PARTITION);
+			result.iter().for_each(|d| assert!(d.session == pushed_idx));
+		},
+	);
+}

pezkuwi/node/core/provisioner/src/error.rs

@@ -0,0 +1,123 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+///! Error types for provisioner module
+use fatality::Nested;
+use futures::channel::{mpsc, oneshot};
+use pezkuwi_node_subsystem::errors::{ChainApiError, RuntimeApiError, SubsystemError};
+use pezkuwi_node_subsystem_util as util;
+use pezkuwi_primitives::Hash;
+
+pub type FatalResult<T> = std::result::Result<T, FatalError>;
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// Errors in the provisioner.
+#[allow(missing_docs)]
+#[fatality::fatality(splitable)]
+pub enum Error {
+	#[fatal(forward)]
+	#[error("Error while accessing runtime information")]
+	Runtime(#[from] util::runtime::Error),
+
+	#[error(transparent)]
+	Util(#[from] util::Error),
+
+	#[error("failed to get availability cores")]
+	CanceledAvailabilityCores(#[source] oneshot::Canceled),
+
+	#[error("failed to get persisted validation data")]
+	CanceledPersistedValidationData(#[source] oneshot::Canceled),
+
+	#[error("failed to get block number")]
+	CanceledBlockNumber(#[source] oneshot::Canceled),
+
+	#[error("failed to get session index")]
+	CanceledSessionIndex(#[source] oneshot::Canceled),
+
+	#[error("failed to get node features")]
+	CanceledNodeFeatures(#[source] oneshot::Canceled),
+
+	#[error("failed to get backed candidates")]
+	CanceledBackedCandidates(#[source] oneshot::Canceled),
+
+	#[error("failed to get votes on dispute")]
+	CanceledCandidateVotes(#[source] oneshot::Canceled),
+
+	#[error("failed to get backable candidates from prospective teyrchains")]
+	CanceledBackableCandidates(#[source] oneshot::Canceled),
+
+	#[error(transparent)]
+	ChainApi(#[from] ChainApiError),
+
+	#[error(transparent)]
+	RuntimeApi(#[from] RuntimeApiError),
+
+	#[error("failed to send message to ChainAPI")]
+	ChainApiMessageSend(#[source] mpsc::SendError),
+
+	#[error("failed to send message to CandidateBacking to get backed candidates")]
+	GetBackedCandidatesSend(#[source] mpsc::SendError),
+
+	#[error("Send inherent data timeout.")]
+	SendInherentDataTimeout,
+
+	#[error("failed to send return message with Inherents")]
+	InherentDataReturnChannel,
+
+	#[fatal]
+	#[error("Failed to spawn background task")]
+	FailedToSpawnBackgroundTask,
+
+	#[error(transparent)]
+	SubsystemError(#[from] SubsystemError),
+
+	#[fatal]
+	#[error(transparent)]
+	OverseerExited(SubsystemError),
+}
+
+/// Used by `get_onchain_disputes` to represent errors related to fetching on-chain disputes from
+/// the Runtime
+#[allow(dead_code)] // Remove when promoting to stable
+#[fatality::fatality]
+pub enum GetOnchainDisputesError {
+	#[fatal]
+	#[error("runtime subsystem is down")]
+	Channel,
+
+	#[error("runtime execution error occurred while fetching onchain disputes for parent {1}")]
+	Execution(#[source] RuntimeApiError, Hash),
+
+	#[error("runtime doesn't support RuntimeApiRequest::Disputes for parent {1}")]
+	NotSupported(#[source] RuntimeApiError, Hash),
+}
+
+pub fn log_error(result: Result<()>) -> std::result::Result<(), FatalError> {
+	match result.into_nested()? {
+		Ok(()) => Ok(()),
+		Err(jfyi) => {
+			jfyi.log();
+			Ok(())
+		},
+	}
+}
+
+impl JfyiError {
+	/// Log a `JfyiError`.
+	pub fn log(self) {
+		gum::debug!(target: super::LOG_TARGET, error = ?self);
+	}
+}

pezkuwi/node/core/provisioner/src/lib.rs

@@ -0,0 +1,802 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! The provisioner is responsible for assembling a relay chain block
+//! from a set of available teyrchain candidates of its choice.
+
+#![deny(missing_docs, unused_crate_dependencies)]
+
+use bitvec::vec::BitVec;
+use futures::{
+	channel::oneshot::{self, Canceled},
+	future::BoxFuture,
+	prelude::*,
+	stream::FuturesUnordered,
+	FutureExt,
+};
+use futures_timer::Delay;
+use pezkuwi_node_subsystem::{
+	messages::{
+		Ancestors, CandidateBackingMessage, ChainApiMessage, ProspectiveTeyrchainsMessage,
+		ProvisionableData, ProvisionerInherentData, ProvisionerMessage,
+	},
+	overseer, ActivatedLeaf, ActiveLeavesUpdate, FromOrchestra, OverseerSignal, SpawnedSubsystem,
+	SubsystemError,
+};
+use pezkuwi_node_subsystem_util::{request_availability_cores, TimeoutExt};
+use pezkuwi_primitives::{
+	BackedCandidate, CandidateEvent, CandidateHash, CoreIndex, CoreState, Hash, Id as ParaId,
+	SignedAvailabilityBitfield, ValidatorIndex,
+};
+use sc_consensus_slots::time_until_next_slot;
+use schnellru::{ByLength, LruMap};
+use std::{
+	collections::{BTreeMap, HashMap},
+	time::Duration,
+};
+mod disputes;
+mod error;
+mod metrics;
+
+pub use self::metrics::*;
+use error::{Error, FatalResult};
+
+#[cfg(test)]
+mod tests;
+
+/// How long to wait before proposing.
+const PRE_PROPOSE_TIMEOUT: std::time::Duration = core::time::Duration::from_millis(2000);
+/// Some timeout to ensure task won't hang around in the background forever on issues.
+const SEND_INHERENT_DATA_TIMEOUT: std::time::Duration = core::time::Duration::from_millis(500);
+
+const LOG_TARGET: &str = "teyrchain::provisioner";
+
+/// The provisioner subsystem.
+pub struct ProvisionerSubsystem {
+	metrics: Metrics,
+}
+
+impl ProvisionerSubsystem {
+	/// Create a new instance of the `ProvisionerSubsystem`.
+	pub fn new(metrics: Metrics) -> Self {
+		Self { metrics }
+	}
+}
+
+/// A per-relay-parent state for the provisioning subsystem.
+pub struct PerRelayParent {
+	leaf: ActivatedLeaf,
+	signed_bitfields: Vec<SignedAvailabilityBitfield>,
+	is_inherent_ready: bool,
+	awaiting_inherent: Vec<oneshot::Sender<ProvisionerInherentData>>,
+}
+
+impl PerRelayParent {
+	fn new(leaf: ActivatedLeaf) -> Self {
+		Self {
+			leaf,
+			signed_bitfields: Vec::new(),
+			is_inherent_ready: false,
+			awaiting_inherent: Vec::new(),
+		}
+	}
+}
+
+type InherentDelays = FuturesUnordered<BoxFuture<'static, Hash>>;
+type SlotDelays = FuturesUnordered<BoxFuture<'static, Hash>>;
+type InherentReceivers =
+	FuturesUnordered<BoxFuture<'static, (Hash, Result<ProvisionerInherentData, Canceled>)>>;
+
+#[overseer::subsystem(Provisioner, error=SubsystemError, prefix=self::overseer)]
+impl<Context> ProvisionerSubsystem {
+	fn start(self, ctx: Context) -> SpawnedSubsystem {
+		let future = async move {
+			run(ctx, self.metrics)
+				.await
+				.map_err(|e| SubsystemError::with_origin("provisioner", e))
+		}
+		.boxed();
+
+		SpawnedSubsystem { name: "provisioner-subsystem", future }
+	}
+}
+
+#[overseer::contextbounds(Provisioner, prefix = self::overseer)]
+async fn run<Context>(mut ctx: Context, metrics: Metrics) -> FatalResult<()> {
+	let mut inherent_delays = InherentDelays::new();
+	let mut inherent_receivers = InherentReceivers::new();
+	let mut slot_delays = SlotDelays::new();
+	let mut per_relay_parent = HashMap::new();
+	let mut inherents = LruMap::new(ByLength::new(16));
+
+	loop {
+		let result = run_iteration(
+			&mut ctx,
+			&mut per_relay_parent,
+			&mut inherent_delays,
+			&mut inherent_receivers,
+			&mut inherents,
+			&mut slot_delays,
+			&metrics,
+		)
+		.await;
+
+		match result {
+			Ok(()) => break,
+			err => crate::error::log_error(err)?,
+		}
+	}
+
+	Ok(())
+}
+
+#[overseer::contextbounds(Provisioner, prefix = self::overseer)]
+async fn run_iteration<Context>(
+	ctx: &mut Context,
+	per_relay_parent: &mut HashMap<Hash, PerRelayParent>,
+	inherent_delays: &mut InherentDelays,
+	inherent_receivers: &mut InherentReceivers,
+	inherents: &mut LruMap<Hash, ProvisionerInherentData>,
+	slot_delays: &mut SlotDelays,
+	metrics: &Metrics,
+) -> Result<(), Error> {
+	loop {
+		futures::select! {
+			from_overseer = ctx.recv().fuse() => {
+				// Map the error to ensure that the subsystem exits when the overseer is gone.
+				match from_overseer.map_err(Error::OverseerExited)? {
+					FromOrchestra::Signal(OverseerSignal::ActiveLeaves(update)) =>
+						handle_active_leaves_update(ctx, update, per_relay_parent, inherent_delays, slot_delays, inherents, metrics).await?,
+					FromOrchestra::Signal(OverseerSignal::BlockFinalized(..)) => {},
+					FromOrchestra::Signal(OverseerSignal::Conclude) => return Ok(()),
+					FromOrchestra::Communication { msg } => {
+						handle_communication(ctx, per_relay_parent, msg, metrics).await?;
+					},
+				}
+			},
+			hash = slot_delays.select_next_some() => {
+				gum::debug!(target: LOG_TARGET, leaf_hash=?hash, "Slot start, preparing debug inherent");
+
+				let Some(state) = per_relay_parent.get_mut(&hash) else {
+					continue
+				};
+
+				// Create the inherent data just to record the backed candidates.
+				let (inherent_tx, inherent_rx) = oneshot::channel();
+				let task = async move {
+					match inherent_rx.await {
+						Ok(res) => (hash, Ok(res)),
+						Err(e) => (hash, Err(e)),
+					}
+				}
+				.boxed();
+
+				inherent_receivers.push(task);
+
+				send_inherent_data_bg(ctx, &state, vec![inherent_tx], metrics.clone()).await?;
+			},
+			(hash, inherent_data) = inherent_receivers.select_next_some() => {
+				let Ok(inherent_data) = inherent_data else {
+					continue
+				};
+
+				gum::trace!(
+					target: LOG_TARGET,
+					relay_parent = ?hash,
+					"Debug Inherent Data became ready"
+				);
+				inherents.insert(hash, inherent_data);
+			}
+			hash = inherent_delays.select_next_some() => {
+				if let Some(state) = per_relay_parent.get_mut(&hash) {
+					state.is_inherent_ready = true;
+
+					gum::trace!(
+						target: LOG_TARGET,
+						relay_parent = ?hash,
+						"Inherent Data became ready"
+					);
+
+					let return_senders = std::mem::take(&mut state.awaiting_inherent);
+					if !return_senders.is_empty() {
+						send_inherent_data_bg(ctx, &state, return_senders, metrics.clone()).await?;
+					}
+				}
+			}
+		}
+	}
+}
+
+#[overseer::contextbounds(Provisioner, prefix = self::overseer)]
+async fn handle_active_leaves_update<Context>(
+	ctx: &mut Context,
+	update: ActiveLeavesUpdate,
+	per_relay_parent: &mut HashMap<Hash, PerRelayParent>,
+	inherent_delays: &mut InherentDelays,
+	slot_delays: &mut SlotDelays,
+	inherents: &mut LruMap<Hash, ProvisionerInherentData>,
+	metrics: &Metrics,
+) -> Result<(), Error> {
+	gum::trace!(target: LOG_TARGET, "Handle ActiveLeavesUpdate");
+	for deactivated in &update.deactivated {
+		per_relay_parent.remove(deactivated);
+	}
+
+	let Some(leaf) = update.activated else { return Ok(()) };
+
+	gum::trace!(target: LOG_TARGET, leaf_hash=?leaf.hash, "Adding delay");
+	let delay_fut = Delay::new(PRE_PROPOSE_TIMEOUT).map(move |_| leaf.hash).boxed();
+	per_relay_parent.insert(leaf.hash, PerRelayParent::new(leaf.clone()));
+	inherent_delays.push(delay_fut);
+
+	let slot_delay = time_until_next_slot(Duration::from_millis(6000));
+	gum::debug!(target: LOG_TARGET, leaf_hash=?leaf.hash, "Expecting next slot in {}ms", slot_delay.as_millis());
+
+	let slot_delay_task =
+		Delay::new(slot_delay + PRE_PROPOSE_TIMEOUT).map(move |_| leaf.hash).boxed();
+	slot_delays.push(slot_delay_task);
+
+	let Ok(Ok(candidate_events)) =
+		pezkuwi_node_subsystem_util::request_candidate_events(leaf.hash, ctx.sender())
+			.await
+			.await
+	else {
+		gum::warn!(target: LOG_TARGET, leaf_hash=?leaf.hash, "Failed to fetch candidate events");
+
+		return Ok(());
+	};
+
+	let in_block_count = candidate_events
+		.into_iter()
+		.filter(|event| matches!(event, CandidateEvent::CandidateBacked(_, _, _, _)))
+		.count() as isize;
+
+	let (tx, rx) = oneshot::channel();
+	ctx.send_message(ChainApiMessage::BlockHeader(leaf.hash, tx)).await;
+
+	let Ok(Some(header)) = rx.await.unwrap_or_else(|err| {
+		gum::warn!(target: LOG_TARGET, hash = ?leaf.hash, ?err, "Missing header for block");
+		Ok(None)
+	}) else {
+		return Ok(());
+	};
+
+	gum::trace!(target: LOG_TARGET, hash = ?header.parent_hash, "Looking up debug inherent");
+
+	// Now, let's get the candidate count from our own inherent built earlier.
+	// The inherent is stored under the parent hash.
+	let Some(inherent) = inherents.get(&header.parent_hash) else { return Ok(()) };
+
+	let diff = inherent.backed_candidates.len() as isize - in_block_count;
+	gum::debug!(target: LOG_TARGET, 
+		 ?diff,
+		 ?in_block_count,
+		 local_count = ?inherent.backed_candidates.len(),
+		 leaf_hash=?leaf.hash, "Offchain vs on-chain backing update");
+
+	metrics.observe_backable_vs_in_block(diff);
+	Ok(())
+}
+
+#[overseer::contextbounds(Provisioner, prefix = self::overseer)]
+async fn handle_communication<Context>(
+	ctx: &mut Context,
+	per_relay_parent: &mut HashMap<Hash, PerRelayParent>,
+	message: ProvisionerMessage,
+	metrics: &Metrics,
+) -> Result<(), Error> {
+	match message {
+		ProvisionerMessage::RequestInherentData(relay_parent, return_sender) => {
+			gum::trace!(target: LOG_TARGET, ?relay_parent, "Inherent data got requested.");
+
+			if let Some(state) = per_relay_parent.get_mut(&relay_parent) {
+				if state.is_inherent_ready {
+					gum::trace!(target: LOG_TARGET, ?relay_parent, "Calling send_inherent_data.");
+					send_inherent_data_bg(ctx, &state, vec![return_sender], metrics.clone())
+						.await?;
+				} else {
+					gum::trace!(
+						target: LOG_TARGET,
+						?relay_parent,
+						"Queuing inherent data request (inherent data not yet ready)."
+					);
+					state.awaiting_inherent.push(return_sender);
+				}
+			}
+		},
+		ProvisionerMessage::ProvisionableData(relay_parent, data) => {
+			if let Some(state) = per_relay_parent.get_mut(&relay_parent) {
+				let _timer = metrics.time_provisionable_data();
+
+				gum::trace!(target: LOG_TARGET, ?relay_parent, "Received provisionable data: {:?}", &data);
+
+				note_provisionable_data(state, data);
+			}
+		},
+	}
+
+	Ok(())
+}
+
+#[overseer::contextbounds(Provisioner, prefix = self::overseer)]
+async fn send_inherent_data_bg<Context>(
+	ctx: &mut Context,
+	per_relay_parent: &PerRelayParent,
+	return_senders: Vec<oneshot::Sender<ProvisionerInherentData>>,
+	metrics: Metrics,
+) -> Result<(), Error> {
+	let leaf = per_relay_parent.leaf.clone();
+	let signed_bitfields = per_relay_parent.signed_bitfields.clone();
+	let mut sender = ctx.sender().clone();
+
+	let bg = async move {
+		let _timer = metrics.time_request_inherent_data();
+
+		gum::trace!(
+			target: LOG_TARGET,
+			relay_parent = ?leaf.hash,
+			"Sending inherent data in background."
+		);
+
+		let send_result =
+			send_inherent_data(&leaf, &signed_bitfields, return_senders, &mut sender, &metrics) // Make sure call is not taking forever:
+				.timeout(SEND_INHERENT_DATA_TIMEOUT)
+				.map(|v| match v {
+					Some(r) => r,
+					None => Err(Error::SendInherentDataTimeout),
+				});
+
+		match send_result.await {
+			Err(err) => {
+				if let Error::CanceledBackedCandidates(_) = err {
+					gum::debug!(
+						target: LOG_TARGET,
+						err = ?err,
+						"Failed to assemble or send inherent data - block got likely obsoleted already."
+					);
+				} else {
+					gum::warn!(target: LOG_TARGET, err = ?err, "failed to assemble or send inherent data");
+				}
+				metrics.on_inherent_data_request(Err(()));
+			},
+			Ok(()) => {
+				metrics.on_inherent_data_request(Ok(()));
+				gum::debug!(
+					target: LOG_TARGET,
+					signed_bitfield_count = signed_bitfields.len(),
+					leaf_hash = ?leaf.hash,
+					"inherent data sent successfully"
+				);
+				metrics.observe_inherent_data_bitfields_count(signed_bitfields.len());
+			},
+		}
+	};
+
+	ctx.spawn("send-inherent-data", bg.boxed())
+		.map_err(|_| Error::FailedToSpawnBackgroundTask)?;
+
+	Ok(())
+}
+
+fn note_provisionable_data(
+	per_relay_parent: &mut PerRelayParent,
+	provisionable_data: ProvisionableData,
+) {
+	match provisionable_data {
+		ProvisionableData::Bitfield(_, signed_bitfield) =>
+			per_relay_parent.signed_bitfields.push(signed_bitfield),
+		// We choose not to punish these forms of misbehavior for the time being.
+		// Risks from misbehavior are sufficiently mitigated at the protocol level
+		// via reputation changes. Punitive actions here may become desirable
+		// enough to dedicate time to in the future.
+		ProvisionableData::MisbehaviorReport(_, _, _) => {},
+		// We wait and do nothing here, preferring to initiate a dispute after the
+		// parablock candidate is included for the following reasons:
+		//
+		// 1. A dispute for a candidate triggered at any point before the candidate
+		// has been made available, including the backing stage, can't be
+		// guaranteed to conclude. Non-concluding disputes are unacceptable.
+		// 2. Candidates which haven't been made available don't pose a security
+		// risk as they can not be included, approved, or finalized.
+		//
+		// Currently we rely on approval checkers to trigger disputes for bad
+		// parablocks once they are included. But we can do slightly better by
+		// allowing disagreeing backers to record their disagreement and initiate a
+		// dispute once the parablock in question has been included. This potential
+		// change is tracked by: https://github.com/paritytech/polkadot/issues/3232
+		ProvisionableData::Dispute(_, _) => {},
+	}
+}
+
+type CoreAvailability = BitVec<u8, bitvec::order::Lsb0>;
+
+/// The provisioner is the subsystem best suited to choosing which specific
+/// backed candidates and availability bitfields should be assembled into the
+/// block. To engage this functionality, a
+/// `ProvisionerMessage::RequestInherentData` is sent; the response is a set of
+/// non-conflicting candidates and the appropriate bitfields. Non-conflicting
+/// means that there are never two distinct teyrchain candidates included for
+/// the same teyrchain and that new teyrchain candidates cannot be included
+/// until the previous one either gets declared available or expired.
+///
+/// The main complication here is going to be around handling
+/// occupied-core-assumptions. We might have candidates that are only
+/// includable when some bitfields are included. And we might have candidates
+/// that are not includable when certain bitfields are included.
+///
+/// When we're choosing bitfields to include, the rule should be simple:
+/// maximize availability. So basically, include all bitfields. And then
+/// choose a coherent set of candidates along with that.
+async fn send_inherent_data(
+	leaf: &ActivatedLeaf,
+	bitfields: &[SignedAvailabilityBitfield],
+	return_senders: Vec<oneshot::Sender<ProvisionerInherentData>>,
+	from_job: &mut impl overseer::ProvisionerSenderTrait,
+	metrics: &Metrics,
+) -> Result<(), Error> {
+	gum::trace!(
+		target: LOG_TARGET,
+		relay_parent = ?leaf.hash,
+		"Requesting availability cores"
+	);
+	let availability_cores = request_availability_cores(leaf.hash, from_job)
+		.await
+		.await
+		.map_err(|err| Error::CanceledAvailabilityCores(err))??;
+
+	gum::trace!(
+		target: LOG_TARGET,
+		relay_parent = ?leaf.hash,
+		"Selecting disputes"
+	);
+
+	let disputes = disputes::prioritized_selection::select_disputes(from_job, metrics, leaf).await;
+
+	gum::trace!(
+		target: LOG_TARGET,
+		relay_parent = ?leaf.hash,
+		"Selected disputes"
+	);
+
+	let bitfields = select_availability_bitfields(&availability_cores, bitfields, &leaf.hash);
+
+	gum::trace!(
+		target: LOG_TARGET,
+		relay_parent = ?leaf.hash,
+		"Selected bitfields"
+	);
+
+	let candidates = select_candidates(&availability_cores, &bitfields, leaf, from_job).await?;
+
+	gum::trace!(
+		target: LOG_TARGET,
+		relay_parent = ?leaf.hash,
+		"Selected candidates"
+	);
+
+	gum::debug!(
+		target: LOG_TARGET,
+		availability_cores_len = availability_cores.len(),
+		disputes_count = disputes.len(),
+		bitfields_count = bitfields.len(),
+		candidates_count = candidates.len(),
+		leaf_hash = ?leaf.hash,
+		"inherent data prepared",
+	);
+
+	let inherent_data =
+		ProvisionerInherentData { bitfields, backed_candidates: candidates, disputes };
+
+	gum::trace!(
+		target: LOG_TARGET,
+		relay_parent = ?leaf.hash,
+		"Sending back inherent data to requesters."
+	);
+
+	for return_sender in return_senders {
+		return_sender
+			.send(inherent_data.clone())
+			.map_err(|_data| Error::InherentDataReturnChannel)?;
+	}
+
+	Ok(())
+}
+
+/// In general, we want to pick all the bitfields. However, we have the following constraints:
+///
+/// - not more than one per validator
+/// - each 1 bit must correspond to an occupied core
+///
+/// If we have too many, an arbitrary selection policy is fine. For purposes of maximizing
+/// availability, we pick the one with the greatest number of 1 bits.
+///
+/// Note: This does not enforce any sorting precondition on the output; the ordering there will be
+/// unrelated to the sorting of the input.
+fn select_availability_bitfields(
+	cores: &[CoreState],
+	bitfields: &[SignedAvailabilityBitfield],
+	leaf_hash: &Hash,
+) -> Vec<SignedAvailabilityBitfield> {
+	let mut selected: BTreeMap<ValidatorIndex, SignedAvailabilityBitfield> = BTreeMap::new();
+
+	gum::debug!(
+		target: LOG_TARGET,
+		bitfields_count = bitfields.len(),
+		?leaf_hash,
+		"bitfields count before selection"
+	);
+
+	'a: for bitfield in bitfields.iter().cloned() {
+		if bitfield.payload().0.len() != cores.len() {
+			gum::debug!(target: LOG_TARGET, ?leaf_hash, "dropping bitfield due to length mismatch");
+			continue;
+		}
+
+		let is_better = selected
+			.get(&bitfield.validator_index())
+			.map_or(true, |b| b.payload().0.count_ones() < bitfield.payload().0.count_ones());
+
+		if !is_better {
+			gum::trace!(
+				target: LOG_TARGET,
+				val_idx = bitfield.validator_index().0,
+				?leaf_hash,
+				"dropping bitfield due to duplication - the better one is kept"
+			);
+			continue;
+		}
+
+		for (idx, _) in cores.iter().enumerate().filter(|v| !v.1.is_occupied()) {
+			// Bit is set for an unoccupied core - invalid
+			if *bitfield.payload().0.get(idx).as_deref().unwrap_or(&false) {
+				gum::debug!(
+					target: LOG_TARGET,
+					val_idx = bitfield.validator_index().0,
+					?leaf_hash,
+					"dropping invalid bitfield - bit is set for an unoccupied core"
+				);
+				continue 'a;
+			}
+		}
+
+		let _ = selected.insert(bitfield.validator_index(), bitfield);
+	}
+
+	gum::debug!(
+		target: LOG_TARGET,
+		?leaf_hash,
+		"selected {} of all {} bitfields (each bitfield is from a unique validator)",
+		selected.len(),
+		bitfields.len()
+	);
+
+	selected.into_values().collect()
+}
+
+/// Requests backable candidates from Prospective Teyrchains subsystem
+/// based on core states.
+async fn request_backable_candidates(
+	availability_cores: &[CoreState],
+	bitfields: &[SignedAvailabilityBitfield],
+	relay_parent: &ActivatedLeaf,
+	sender: &mut impl overseer::ProvisionerSenderTrait,
+) -> Result<HashMap<ParaId, Vec<(CandidateHash, Hash)>>, Error> {
+	let block_number_under_construction = relay_parent.number + 1;
+
+	// Record how many cores are scheduled for each paraid. Use a BTreeMap because
+	// we'll need to iterate through them.
+	let mut scheduled_cores_per_para: BTreeMap<ParaId, usize> = BTreeMap::new();
+	// The on-chain ancestors of a para present in availability-cores.
+	let mut ancestors: HashMap<ParaId, Ancestors> =
+		HashMap::with_capacity(availability_cores.len());
+
+	for (core_idx, core) in availability_cores.iter().enumerate() {
+		let core_idx = CoreIndex(core_idx as u32);
+		match core {
+			CoreState::Scheduled(scheduled_core) => {
+				*scheduled_cores_per_para.entry(scheduled_core.para_id).or_insert(0) += 1;
+			},
+			CoreState::Occupied(occupied_core) => {
+				let is_available = bitfields_indicate_availability(
+					core_idx.0 as usize,
+					bitfields,
+					&occupied_core.availability,
+				);
+
+				if is_available {
+					ancestors
+						.entry(occupied_core.para_id())
+						.or_default()
+						.insert(occupied_core.candidate_hash);
+
+					if let Some(ref scheduled_core) = occupied_core.next_up_on_available {
+						// Request a new backable candidate for the newly scheduled para id.
+						*scheduled_cores_per_para.entry(scheduled_core.para_id).or_insert(0) += 1;
+					}
+				} else if occupied_core.time_out_at <= block_number_under_construction {
+					// Timed out before being available.
+
+					if let Some(ref scheduled_core) = occupied_core.next_up_on_time_out {
+						// Candidate's availability timed out, practically same as scheduled.
+						*scheduled_cores_per_para.entry(scheduled_core.para_id).or_insert(0) += 1;
+					}
+				} else {
+					// Not timed out and not available.
+					ancestors
+						.entry(occupied_core.para_id())
+						.or_default()
+						.insert(occupied_core.candidate_hash);
+				}
+			},
+			CoreState::Free => continue,
+		};
+	}
+
+	let mut selected_candidates: HashMap<ParaId, Vec<(CandidateHash, Hash)>> =
+		HashMap::with_capacity(scheduled_cores_per_para.len());
+
+	for (para_id, core_count) in scheduled_cores_per_para {
+		let para_ancestors = ancestors.remove(&para_id).unwrap_or_default();
+
+		let response = get_backable_candidates(
+			relay_parent.hash,
+			para_id,
+			para_ancestors,
+			core_count as u32,
+			sender,
+		)
+		.await?;
+
+		if response.is_empty() {
+			gum::debug!(
+				target: LOG_TARGET,
+				leaf_hash = ?relay_parent.hash,
+				?para_id,
+				"No backable candidate returned by prospective teyrchains",
+			);
+			continue;
+		}
+
+		selected_candidates.insert(para_id, response);
+	}
+
+	Ok(selected_candidates)
+}
+
+/// Determine which cores are free, and then to the degree possible, pick a candidate appropriate to
+/// each free core.
+async fn select_candidates(
+	availability_cores: &[CoreState],
+	bitfields: &[SignedAvailabilityBitfield],
+	leaf: &ActivatedLeaf,
+	sender: &mut impl overseer::ProvisionerSenderTrait,
+) -> Result<Vec<BackedCandidate>, Error> {
+	let relay_parent = leaf.hash;
+	gum::trace!(
+		target: LOG_TARGET,
+		leaf_hash=?relay_parent,
+		"before GetBackedCandidates"
+	);
+
+	let selected_candidates =
+		request_backable_candidates(availability_cores, bitfields, leaf, sender).await?;
+	gum::debug!(target: LOG_TARGET, ?selected_candidates, "Got backable candidates");
+
+	// now get the backed candidates corresponding to these candidate receipts
+	let (tx, rx) = oneshot::channel();
+	sender.send_unbounded_message(CandidateBackingMessage::GetBackableCandidates(
+		selected_candidates.clone(),
+		tx,
+	));
+	let candidates = rx.await.map_err(|err| Error::CanceledBackedCandidates(err))?;
+	gum::trace!(
+		target: LOG_TARGET,
+		leaf_hash=?relay_parent,
+		"Got {} backed candidates", candidates.len()
+	);
+
+	// keep only one candidate with validation code.
+	let mut with_validation_code = false;
+	// merge the candidates into a common collection, preserving the order
+	let mut merged_candidates = Vec::with_capacity(availability_cores.len());
+
+	for para_candidates in candidates.into_values() {
+		for candidate in para_candidates {
+			if candidate.candidate().commitments.new_validation_code.is_some() {
+				if with_validation_code {
+					break;
+				} else {
+					with_validation_code = true;
+				}
+			}
+
+			merged_candidates.push(candidate);
+		}
+	}
+
+	gum::debug!(
+		target: LOG_TARGET,
+		n_candidates = merged_candidates.len(),
+		n_cores = availability_cores.len(),
+		?relay_parent,
+		"Selected backed candidates",
+	);
+
+	Ok(merged_candidates)
+}
+
+/// Requests backable candidates from Prospective Teyrchains based on
+/// the given ancestors in the fragment chain. The ancestors may not be ordered.
+async fn get_backable_candidates(
+	relay_parent: Hash,
+	para_id: ParaId,
+	ancestors: Ancestors,
+	count: u32,
+	sender: &mut impl overseer::ProvisionerSenderTrait,
+) -> Result<Vec<(CandidateHash, Hash)>, Error> {
+	let (tx, rx) = oneshot::channel();
+	sender
+		.send_message(ProspectiveTeyrchainsMessage::GetBackableCandidates(
+			relay_parent,
+			para_id,
+			count,
+			ancestors,
+			tx,
+		))
+		.await;
+
+	rx.await.map_err(Error::CanceledBackableCandidates)
+}
+
+/// The availability bitfield for a given core is the transpose
+/// of a set of signed availability bitfields. It goes like this:
+///
+/// - construct a transverse slice along `core_idx`
+/// - bitwise-or it with the availability slice
+/// - count the 1 bits, compare to the total length; true on 2/3+
+fn bitfields_indicate_availability(
+	core_idx: usize,
+	bitfields: &[SignedAvailabilityBitfield],
+	availability: &CoreAvailability,
+) -> bool {
+	let mut availability = availability.clone();
+	let availability_len = availability.len();
+
+	for bitfield in bitfields {
+		let validator_idx = bitfield.validator_index().0 as usize;
+		match availability.get_mut(validator_idx) {
+			None => {
+				// in principle, this function might return a `Result<bool, Error>` so that we can
+				// more clearly express this error condition however, in practice, that would just
+				// push off an error-handling routine which would look a whole lot like this one.
+				// simpler to just handle the error internally here.
+				gum::warn!(
+					target: LOG_TARGET,
+					validator_idx = %validator_idx,
+					availability_len = %availability_len,
+					"attempted to set a transverse bit at idx {} which is greater than bitfield size {}",
+					validator_idx,
+					availability_len,
+				);
+
+				return false;
+			},
+			Some(mut bit_mut) => *bit_mut |= bitfield.payload().0[core_idx],
+		}
+	}
+
+	3 * availability.count_ones() >= 2 * availability.len()
+}

pezkuwi/node/core/provisioner/src/metrics.rs

@@ -0,0 +1,246 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::disputes::prioritized_selection::PartitionedDisputes;
+use pezkuwi_node_subsystem_util::metrics::{self, prometheus};
+
+#[derive(Clone)]
+struct MetricsInner {
+	/// Tracks successful/unsuccessful inherent data requests
+	inherent_data_requests: prometheus::CounterVec<prometheus::U64>,
+	/// How much time the `RequestInherentData` processing takes
+	request_inherent_data_duration: prometheus::Histogram,
+	/// How much time `ProvisionableData` processing takes
+	provisionable_data_duration: prometheus::Histogram,
+	/// Bitfields array length in `ProvisionerInherentData` (the result for `RequestInherentData`)
+	inherent_data_response_bitfields: prometheus::Histogram,
+
+	/// The following metrics track how many disputes/votes the runtime will have to process. These
+	/// will count all recent statements meaning every dispute from last sessions: 10 min on
+	/// Pezkuwichain, 60 min on Kusama and 4 hours on Pezkuwi. The metrics are updated only when
+	/// the node authors a block, so values vary across nodes.
+	inherent_data_dispute_statement_sets: prometheus::Counter<prometheus::U64>,
+	inherent_data_dispute_statements: prometheus::CounterVec<prometheus::U64>,
+
+	/// The disputes received from `disputes-coordinator` by partition
+	partitioned_disputes: prometheus::CounterVec<prometheus::U64>,
+
+	/// The disputes fetched from the runtime.
+	fetched_onchain_disputes: prometheus::Counter<prometheus::U64>,
+
+	/// The difference between the number of backed candidates in a block and the number of
+	/// backable candidates on the node side.
+	backable_vs_in_block: prometheus::Histogram,
+}
+
+/// Provisioner metrics.
+#[derive(Default, Clone)]
+pub struct Metrics(Option<MetricsInner>);
+
+impl Metrics {
+	/// Creates new dummy `Metrics` instance. Used for testing only.
+	#[cfg(test)]
+	pub fn new_dummy() -> Metrics {
+		Metrics(None)
+	}
+
+	pub(crate) fn on_inherent_data_request(&self, response: Result<(), ()>) {
+		if let Some(metrics) = &self.0 {
+			match response {
+				Ok(()) => metrics.inherent_data_requests.with_label_values(&["succeeded"]).inc(),
+				Err(()) => metrics.inherent_data_requests.with_label_values(&["failed"]).inc(),
+			}
+		}
+	}
+
+	/// Provide a timer for `request_inherent_data` which observes on drop.
+	pub(crate) fn time_request_inherent_data(
+		&self,
+	) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0
+			.as_ref()
+			.map(|metrics| metrics.request_inherent_data_duration.start_timer())
+	}
+
+	/// Provide a timer for `provisionable_data` which observes on drop.
+	pub(crate) fn time_provisionable_data(
+		&self,
+	) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.provisionable_data_duration.start_timer())
+	}
+
+	pub(crate) fn observe_inherent_data_bitfields_count(&self, bitfields_count: usize) {
+		self.0.as_ref().map(|metrics| {
+			metrics.inherent_data_response_bitfields.observe(bitfields_count as f64)
+		});
+	}
+
+	pub(crate) fn inc_valid_statements_by(&self, votes: usize) {
+		if let Some(metrics) = &self.0 {
+			metrics
+				.inherent_data_dispute_statements
+				.with_label_values(&["valid"])
+				.inc_by(votes.try_into().unwrap_or(0));
+		}
+	}
+
+	pub(crate) fn inc_invalid_statements_by(&self, votes: usize) {
+		if let Some(metrics) = &self.0 {
+			metrics
+				.inherent_data_dispute_statements
+				.with_label_values(&["invalid"])
+				.inc_by(votes.try_into().unwrap_or(0));
+		}
+	}
+
+	pub(crate) fn inc_dispute_statement_sets_by(&self, disputes: usize) {
+		if let Some(metrics) = &self.0 {
+			metrics
+				.inherent_data_dispute_statement_sets
+				.inc_by(disputes.try_into().unwrap_or(0));
+		}
+	}
+
+	pub(crate) fn on_partition_recent_disputes(&self, disputes: &PartitionedDisputes) {
+		if let Some(metrics) = &self.0 {
+			let PartitionedDisputes {
+				inactive_unknown_onchain,
+				inactive_unconcluded_onchain: inactive_unconcluded_known_onchain,
+				active_unknown_onchain,
+				active_unconcluded_onchain,
+				active_concluded_onchain,
+				inactive_concluded_onchain: inactive_concluded_known_onchain,
+			} = disputes;
+
+			metrics
+				.partitioned_disputes
+				.with_label_values(&["inactive_unknown_onchain"])
+				.inc_by(inactive_unknown_onchain.len().try_into().unwrap_or(0));
+			metrics
+				.partitioned_disputes
+				.with_label_values(&["inactive_unconcluded_known_onchain"])
+				.inc_by(inactive_unconcluded_known_onchain.len().try_into().unwrap_or(0));
+			metrics
+				.partitioned_disputes
+				.with_label_values(&["active_unknown_onchain"])
+				.inc_by(active_unknown_onchain.len().try_into().unwrap_or(0));
+			metrics
+				.partitioned_disputes
+				.with_label_values(&["active_unconcluded_onchain"])
+				.inc_by(active_unconcluded_onchain.len().try_into().unwrap_or(0));
+			metrics
+				.partitioned_disputes
+				.with_label_values(&["active_concluded_onchain"])
+				.inc_by(active_concluded_onchain.len().try_into().unwrap_or(0));
+			metrics
+				.partitioned_disputes
+				.with_label_values(&["inactive_concluded_known_onchain"])
+				.inc_by(inactive_concluded_known_onchain.len().try_into().unwrap_or(0));
+		}
+	}
+
+	pub(crate) fn on_fetched_onchain_disputes(&self, onchain_count: u64) {
+		if let Some(metrics) = &self.0 {
+			metrics.fetched_onchain_disputes.inc_by(onchain_count);
+		}
+	}
+
+	pub(crate) fn observe_backable_vs_in_block(&self, diff: isize) {
+		self.0.as_ref().map(|metrics| metrics.backable_vs_in_block.observe(diff as f64));
+	}
+}
+
+impl metrics::Metrics for Metrics {
+	fn try_register(registry: &prometheus::Registry) -> Result<Self, prometheus::PrometheusError> {
+		let metrics = MetricsInner {
+			inherent_data_requests: prometheus::register(
+				prometheus::CounterVec::new(
+					prometheus::Opts::new(
+						"pezkuwi_teyrchain_inherent_data_requests_total",
+						"Number of InherentData requests served by provisioner.",
+					),
+					&["success"],
+				)?,
+				registry,
+			)?,
+			request_inherent_data_duration: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_provisioner_request_inherent_data_time",
+					"Time spent within `provisioner::request_inherent_data`",
+				))?,
+				registry,
+			)?,
+			provisionable_data_duration: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_provisioner_provisionable_data_time",
+					"Time spent within `provisioner::provisionable_data`",
+				))?,
+				registry,
+			)?,
+			inherent_data_dispute_statements: prometheus::register(
+				prometheus::CounterVec::new(
+					prometheus::Opts::new(
+						"pezkuwi_teyrchain_inherent_data_dispute_statements",
+						"Number of dispute statements passed to `create_inherent()`.",
+					),
+					&["validity"],
+				)?,
+				&registry,
+			)?,
+			inherent_data_dispute_statement_sets: prometheus::register(
+				prometheus::Counter::new(
+					"pezkuwi_teyrchain_inherent_data_dispute_statement_sets",
+					"Number of dispute statements sets passed to `create_inherent()`.",
+				)?,
+				registry,
+			)?,
+			inherent_data_response_bitfields: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"pezkuwi_teyrchain_provisioner_inherent_data_response_bitfields_sent",
+						"Number of inherent bitfields sent in response to `ProvisionerMessage::RequestInherentData`.",
+					).buckets(vec![0.0, 25.0, 50.0, 100.0, 150.0, 200.0, 250.0, 300.0, 400.0, 500.0, 600.0]),
+				)?,
+				registry,
+			)?,
+			backable_vs_in_block: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"pezkuwi_teyrchain_provisioner_backable_vs_in_block",
+						"Difference between number of backable blocks and number of backed candidates in block",
+					).buckets(vec![-100.0, -50.0, -40.0, -30.0, -20.0, -15.0, -10.0, -5.0, 0.0, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, 100.0]),
+				)?,
+				registry,
+			)?,
+			partitioned_disputes: prometheus::register(
+				prometheus::CounterVec::new(
+					prometheus::Opts::new(
+						"pezkuwi_teyrchain_provisioner_partitioned_disputes",
+						"Number of disputes partitioned by type.",
+					),
+					&["partition"],
+				)?,
+				&registry,
+			)?,
+			fetched_onchain_disputes: prometheus::register(
+				prometheus::Counter::new("pezkuwi_teyrchain_fetched_onchain_disputes", "Number of disputes fetched from the runtime"
+				)?,
+				&registry,
+			)?,
+		};
+		Ok(Metrics(Some(metrics)))
+	}
+}

pezkuwi/node/core/pvf-checker/Cargo.toml

@@ -0,0 +1,44 @@
+[package]
+name = "pezkuwi-node-core-pvf-checker"
+description = "Pezkuwi crate that implements the PVF pre-checking subsystem. Responsible for checking and voting for PVFs that are pending approval."
+version = "7.0.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+
+[lints]
+workspace = true
+
+[dependencies]
+futures = { workspace = true }
+gum = { workspace = true, default-features = true }
+
+pezkuwi-node-subsystem = { workspace = true, default-features = true }
+pezkuwi-node-subsystem-util = { workspace = true, default-features = true }
+pezkuwi-primitives = { workspace = true, default-features = true }
+
+sp-keystore = { workspace = true, default-features = true }
+
+[dev-dependencies]
+futures-timer = { workspace = true }
+pezkuwi-node-subsystem-test-helpers = { workspace = true }
+pezkuwi-primitives-test-helpers = { workspace = true }
+sc-keystore = { workspace = true, default-features = true }
+sp-application-crypto = { workspace = true, default-features = true }
+sp-core = { workspace = true, default-features = true }
+sp-keyring = { workspace = true, default-features = true }
+sp-runtime = { workspace = true, default-features = true }
+
+[features]
+runtime-benchmarks = [
+	"gum/runtime-benchmarks",
+	"pezkuwi-node-subsystem-test-helpers/runtime-benchmarks",
+	"pezkuwi-node-subsystem-util/runtime-benchmarks",
+	"pezkuwi-node-subsystem/runtime-benchmarks",
+	"pezkuwi-primitives-test-helpers/runtime-benchmarks",
+	"pezkuwi-primitives/runtime-benchmarks",
+	"sp-keyring/runtime-benchmarks",
+	"sp-runtime/runtime-benchmarks",
+]

pezkuwi/node/core/pvf-checker/src/interest_view.rs

@@ -0,0 +1,156 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use pezkuwi_primitives::{Hash, ValidationCodeHash};
+use std::collections::{
+	btree_map::{self, BTreeMap},
+	HashSet,
+};
+
+/// Whether the PVF passed pre-checking or not.
+#[derive(Copy, Clone, Debug)]
+pub enum Judgement {
+	Valid,
+	Invalid,
+}
+
+impl Judgement {
+	/// Whether the PVF is valid or not.
+	pub fn is_valid(&self) -> bool {
+		match self {
+			Judgement::Valid => true,
+			Judgement::Invalid => false,
+		}
+	}
+}
+
+/// Data about a particular validation code.
+#[derive(Default, Debug)]
+struct PvfData {
+	/// If `Some` then the PVF pre-checking was run for this PVF. If `None` we are either waiting
+	/// for the judgement to come in or the PVF pre-checking failed.
+	judgement: Option<Judgement>,
+
+	/// The set of block hashes where this PVF was seen.
+	seen_in: HashSet<Hash>,
+}
+
+impl PvfData {
+	/// Initialize a new `PvfData` which is awaiting for the initial judgement.
+	fn pending(origin: Hash) -> Self {
+		// Preallocate the hashset with 5 items. This is the anticipated maximum leaves we can
+		// deal at the same time. In the vast majority of the cases it will have length of 1.
+		let mut seen_in = HashSet::with_capacity(5);
+		seen_in.insert(origin);
+		Self { judgement: None, seen_in }
+	}
+
+	/// Mark the `PvfData` as seen in the provided relay-chain block referenced by `relay_hash`.
+	pub fn seen_in(&mut self, relay_hash: Hash) {
+		self.seen_in.insert(relay_hash);
+	}
+
+	/// Removes the given `relay_hash` from the set of seen in, and returns if the set is now empty.
+	pub fn remove_origin(&mut self, relay_hash: &Hash) -> bool {
+		self.seen_in.remove(relay_hash);
+		self.seen_in.is_empty()
+	}
+}
+
+/// The result of [`InterestView::on_leaves_update`].
+pub struct OnLeavesUpdateOutcome {
+	/// The list of PVFs that we first seen in the activated block.
+	pub newcomers: Vec<ValidationCodeHash>,
+	/// The number of PVFs that were removed from the view.
+	pub left_num: usize,
+}
+
+/// A structure that keeps track of relevant PVFs and judgements about them. A relevant PVF is one
+/// that resides in at least a single active leaf.
+#[derive(Debug)]
+pub struct InterestView {
+	active_leaves: BTreeMap<Hash, HashSet<ValidationCodeHash>>,
+	pvfs: BTreeMap<ValidationCodeHash, PvfData>,
+}
+
+impl InterestView {
+	pub fn new() -> Self {
+		Self { active_leaves: BTreeMap::new(), pvfs: BTreeMap::new() }
+	}
+
+	pub fn on_leaves_update(
+		&mut self,
+		activated: Option<(Hash, Vec<ValidationCodeHash>)>,
+		deactivated: &[Hash],
+	) -> OnLeavesUpdateOutcome {
+		let mut newcomers = Vec::new();
+
+		if let Some((leaf, pending_pvfs)) = activated {
+			for pvf in &pending_pvfs {
+				match self.pvfs.entry(*pvf) {
+					btree_map::Entry::Vacant(v) => {
+						v.insert(PvfData::pending(leaf));
+						newcomers.push(*pvf);
+					},
+					btree_map::Entry::Occupied(mut o) => {
+						o.get_mut().seen_in(leaf);
+					},
+				}
+			}
+			self.active_leaves.entry(leaf).or_default().extend(pending_pvfs);
+		}
+
+		let mut left_num = 0;
+		for leaf in deactivated {
+			let pvfs = self.active_leaves.remove(leaf);
+			for pvf in pvfs.into_iter().flatten() {
+				if let btree_map::Entry::Occupied(mut o) = self.pvfs.entry(pvf) {
+					let now_empty = o.get_mut().remove_origin(leaf);
+					if now_empty {
+						left_num += 1;
+						o.remove();
+					}
+				}
+			}
+		}
+
+		OnLeavesUpdateOutcome { newcomers, left_num }
+	}
+
+	/// Handles a new judgement for the given `pvf`.
+	///
+	/// Returns `Err` if the given PVF hash is not known.
+	pub fn on_judgement(
+		&mut self,
+		subject: ValidationCodeHash,
+		judgement: Judgement,
+	) -> Result<(), ()> {
+		match self.pvfs.get_mut(&subject) {
+			Some(data) => {
+				data.judgement = Some(judgement);
+				Ok(())
+			},
+			None => Err(()),
+		}
+	}
+
+	/// Returns all PVFs that previously received a judgement.
+	pub fn judgements(&self) -> impl Iterator<Item = (ValidationCodeHash, Judgement)> + '_ {
+		self.pvfs
+			.iter()
+			.filter_map(|(code_hash, data)| data.judgement.map(|judgement| (*code_hash, judgement)))
+	}
+}

pezkuwi/node/core/pvf-checker/src/lib.rs

@@ -0,0 +1,559 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Implements the PVF pre-checking subsystem.
+//!
+//! This subsystem is responsible for scanning the chain for PVFs that are pending for the approval
+//! as well as submitting statements regarding them passing or not the PVF pre-checking.
+
+use futures::{channel::oneshot, future::BoxFuture, prelude::*, stream::FuturesUnordered};
+
+use pezkuwi_node_subsystem::{
+	messages::{CandidateValidationMessage, PreCheckOutcome, PvfCheckerMessage, RuntimeApiMessage},
+	overseer, ActiveLeavesUpdate, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError,
+	SubsystemResult, SubsystemSender,
+};
+use pezkuwi_primitives::{
+	BlockNumber, Hash, PvfCheckStatement, SessionIndex, ValidationCodeHash, ValidatorId,
+	ValidatorIndex,
+};
+use sp_keystore::KeystorePtr;
+use std::collections::HashSet;
+
+const LOG_TARGET: &str = "teyrchain::pvf-checker";
+
+mod interest_view;
+mod metrics;
+mod runtime_api;
+
+#[cfg(test)]
+mod tests;
+
+use self::{
+	interest_view::{InterestView, Judgement},
+	metrics::Metrics,
+};
+
+/// PVF pre-checking subsystem.
+pub struct PvfCheckerSubsystem {
+	keystore: KeystorePtr,
+	metrics: Metrics,
+}
+
+impl PvfCheckerSubsystem {
+	pub fn new(keystore: KeystorePtr, metrics: Metrics) -> Self {
+		PvfCheckerSubsystem { keystore, metrics }
+	}
+}
+
+#[overseer::subsystem(PvfChecker, error=SubsystemError, prefix = self::overseer)]
+impl<Context> PvfCheckerSubsystem {
+	fn start(self, ctx: Context) -> SpawnedSubsystem {
+		let future = run(ctx, self.keystore, self.metrics)
+			.map_err(|e| SubsystemError::with_origin("pvf-checker", e))
+			.boxed();
+
+		SpawnedSubsystem { name: "pvf-checker-subsystem", future }
+	}
+}
+
+/// A struct that holds the credentials required to sign the PVF check statements. These credentials
+/// are implicitly to pinned to a session where our node acts as a validator.
+struct SigningCredentials {
+	/// The validator public key.
+	validator_key: ValidatorId,
+	/// The validator index in the current session.
+	validator_index: ValidatorIndex,
+}
+
+struct State {
+	/// If `Some` then our node is in the active validator set during the current session.
+	///
+	/// Updated when a new session index is detected in one of the heads.
+	credentials: Option<SigningCredentials>,
+
+	/// The number and the hash of the most recent block that we have seen.
+	///
+	/// This is only updated when the PVF pre-checking API is detected in a new leaf block.
+	recent_block: Option<(BlockNumber, Hash)>,
+
+	/// The session index of the most recent session that we have seen.
+	///
+	/// This is only updated when the PVF pre-checking API is detected in a new leaf block.
+	latest_session: Option<SessionIndex>,
+
+	/// The set of PVF hashes that we cast a vote for within the current session.
+	voted: HashSet<ValidationCodeHash>,
+
+	/// The collection of PVFs that are observed throughout the active heads.
+	view: InterestView,
+
+	/// The container for the futures that are waiting for the outcome of the pre-checking.
+	///
+	/// Here are some fun facts about these futures:
+	///
+	/// - Pre-checking can take quite some time, in the matter of tens of seconds, so the futures
+	///   here can soak for quite some time.
+	/// - Pre-checking of one PVF can take drastically more time than pre-checking of another PVF.
+	///   This leads to results coming out of order.
+	///
+	/// Resolving to `None` means that the request was dropped before replying.
+	currently_checking:
+		FuturesUnordered<BoxFuture<'static, Option<(PreCheckOutcome, ValidationCodeHash)>>>,
+}
+
+#[overseer::contextbounds(PvfChecker, prefix = self::overseer)]
+async fn run<Context>(
+	mut ctx: Context,
+	keystore: KeystorePtr,
+	metrics: Metrics,
+) -> SubsystemResult<()> {
+	let mut state = State {
+		credentials: None,
+		recent_block: None,
+		latest_session: None,
+		voted: HashSet::with_capacity(16),
+		view: InterestView::new(),
+		currently_checking: FuturesUnordered::new(),
+	};
+
+	loop {
+		let mut sender = ctx.sender().clone();
+		futures::select! {
+			precheck_response = state.currently_checking.select_next_some() => {
+				if let Some((outcome, validation_code_hash)) = precheck_response {
+					handle_pvf_check(
+						&mut state,
+						&mut sender,
+						&keystore,
+						&metrics,
+						outcome,
+						validation_code_hash,
+					).await;
+				} else {
+					// See note in `initiate_precheck` for why this is possible and why we do not
+					// care here.
+				}
+			}
+			from_overseer = ctx.recv().fuse() => {
+				let outcome = handle_from_overseer(
+					&mut state,
+					&mut sender,
+					&keystore,
+					&metrics,
+					from_overseer?,
+				)
+				.await;
+				if let Some(Conclude) = outcome {
+					return Ok(());
+				}
+			}
+		}
+	}
+}
+
+/// Handle an incoming PVF pre-check result from the candidate-validation subsystem.
+async fn handle_pvf_check(
+	state: &mut State,
+	sender: &mut impl overseer::PvfCheckerSenderTrait,
+	keystore: &KeystorePtr,
+	metrics: &Metrics,
+	outcome: PreCheckOutcome,
+	validation_code_hash: ValidationCodeHash,
+) {
+	gum::debug!(
+		target: LOG_TARGET,
+		?validation_code_hash,
+		"Received pre-check result: {:?}",
+		outcome,
+	);
+
+	let judgement = match outcome {
+		PreCheckOutcome::Valid => Judgement::Valid,
+		PreCheckOutcome::Invalid => Judgement::Invalid,
+		PreCheckOutcome::Failed => {
+			// Always vote against in case of failures. Voting against a PVF when encountering a
+			// timeout (or an unlikely node-specific issue) can be considered safe, since
+			// there is no slashing for being on the wrong side on a pre-check vote.
+			//
+			// Also, by being more strict here, we can safely be more lenient during preparation and
+			// avoid the risk of getting slashed there.
+			gum::info!(
+				target: LOG_TARGET,
+				?validation_code_hash,
+				"Pre-check failed, voting against",
+			);
+			Judgement::Invalid
+		},
+	};
+
+	match state.view.on_judgement(validation_code_hash, judgement) {
+		Ok(()) => (),
+		Err(()) => {
+			gum::debug!(
+				target: LOG_TARGET,
+				?validation_code_hash,
+				"received judgement for an unknown (or removed) PVF hash",
+			);
+			return;
+		},
+	}
+
+	match (state.credentials.as_ref(), state.recent_block, state.latest_session) {
+		// Note, the availability of credentials implies the availability of the recent block and
+		// the session index.
+		(Some(credentials), Some(recent_block), Some(session_index)) => {
+			sign_and_submit_pvf_check_statement(
+				sender,
+				keystore,
+				&mut state.voted,
+				credentials,
+				metrics,
+				recent_block.1,
+				session_index,
+				judgement,
+				validation_code_hash,
+			)
+			.await;
+		},
+		_ => (),
+	}
+}
+
+/// A marker for the outer loop that the subsystem should stop.
+struct Conclude;
+
+async fn handle_from_overseer(
+	state: &mut State,
+	sender: &mut impl overseer::PvfCheckerSenderTrait,
+	keystore: &KeystorePtr,
+	metrics: &Metrics,
+	from_overseer: FromOrchestra<PvfCheckerMessage>,
+) -> Option<Conclude> {
+	match from_overseer {
+		FromOrchestra::Signal(OverseerSignal::Conclude) => {
+			gum::info!(target: LOG_TARGET, "Received `Conclude` signal, exiting");
+			Some(Conclude)
+		},
+		FromOrchestra::Signal(OverseerSignal::BlockFinalized(_, _)) => {
+			// ignore
+			None
+		},
+		FromOrchestra::Signal(OverseerSignal::ActiveLeaves(update)) => {
+			handle_leaves_update(state, sender, keystore, metrics, update).await;
+			None
+		},
+		FromOrchestra::Communication { msg } => match msg {
+				// uninhabited type, thus statically unreachable.
+			},
+	}
+}
+
+async fn handle_leaves_update(
+	state: &mut State,
+	sender: &mut impl overseer::PvfCheckerSenderTrait,
+	keystore: &KeystorePtr,
+	metrics: &Metrics,
+	update: ActiveLeavesUpdate,
+) {
+	if let Some(activated) = update.activated {
+		let ActivationEffect { new_session_index, recent_block, pending_pvfs } =
+			match examine_activation(state, sender, keystore, activated.hash, activated.number)
+				.await
+			{
+				None => {
+					// None indicates that the pre-checking runtime API is not supported.
+					return;
+				},
+				Some(e) => e,
+			};
+
+		// Note that this is not necessarily the newly activated leaf.
+		let recent_block_hash = recent_block.1;
+		state.recent_block = Some(recent_block);
+
+		// Update the PVF view and get the previously unseen PVFs and start working on them.
+		let outcome = state
+			.view
+			.on_leaves_update(Some((activated.hash, pending_pvfs)), &update.deactivated);
+		metrics.on_pvf_observed(outcome.newcomers.len());
+		metrics.on_pvf_left(outcome.left_num);
+		for newcomer in outcome.newcomers {
+			initiate_precheck(state, sender, activated.hash, newcomer, metrics).await;
+		}
+
+		if let Some((new_session_index, credentials)) = new_session_index {
+			// New session change:
+			// - update the session index
+			// - reset the set of all PVFs we voted.
+			// - set (or reset) the credentials.
+			state.latest_session = Some(new_session_index);
+			state.voted.clear();
+			state.credentials = credentials;
+
+			// If our node is a validator in the new session, we need to re-sign and submit all
+			// previously obtained judgements.
+			if let Some(ref credentials) = state.credentials {
+				for (code_hash, judgement) in state.view.judgements() {
+					sign_and_submit_pvf_check_statement(
+						sender,
+						keystore,
+						&mut state.voted,
+						credentials,
+						metrics,
+						recent_block_hash,
+						new_session_index,
+						judgement,
+						code_hash,
+					)
+					.await;
+				}
+			}
+		}
+	} else {
+		state.view.on_leaves_update(None, &update.deactivated);
+	}
+}
+
+struct ActivationEffect {
+	/// If the activated leaf is in a new session, the index of the new session. If the new session
+	/// has a validator in the set our node happened to have private key for, the signing
+	new_session_index: Option<(SessionIndex, Option<SigningCredentials>)>,
+	/// This is the block hash and number of the newly activated block if it's "better" than the
+	/// last one we've seen. The block is better if it's number is higher or if there are no blocks
+	/// observed whatsoever. If the leaf is not better then this holds the existing recent block.
+	recent_block: (BlockNumber, Hash),
+	/// The full list of PVFs that are pending pre-checking according to the runtime API. In case
+	/// the API returned an error this list is empty.
+	pending_pvfs: Vec<ValidationCodeHash>,
+}
+
+/// Examines the new leaf and returns the effects of the examination.
+///
+/// Returns `None` if the PVF pre-checking runtime API is not supported for the given leaf hash.
+async fn examine_activation(
+	state: &mut State,
+	sender: &mut impl overseer::PvfCheckerSenderTrait,
+	keystore: &KeystorePtr,
+	leaf_hash: Hash,
+	leaf_number: BlockNumber,
+) -> Option<ActivationEffect> {
+	gum::debug!(
+		target: LOG_TARGET,
+		"Examining activation of leaf {:?} ({})",
+		leaf_hash,
+		leaf_number,
+	);
+
+	let pending_pvfs = match runtime_api::pvfs_require_precheck(sender, leaf_hash).await {
+		Err(runtime_api::RuntimeRequestError::NotSupported) => return None,
+		Err(_) => {
+			gum::debug!(
+				target: LOG_TARGET,
+				relay_parent = ?leaf_hash,
+				"cannot fetch PVFs that require pre-checking from runtime API",
+			);
+			Vec::new()
+		},
+		Ok(v) => v,
+	};
+
+	let recent_block = match state.recent_block {
+		Some((recent_block_num, recent_block_hash)) if leaf_number < recent_block_num => {
+			// the existing recent block is not worse than the new activation, so leave it.
+			(recent_block_num, recent_block_hash)
+		},
+		_ => (leaf_number, leaf_hash),
+	};
+
+	let new_session_index = match runtime_api::session_index_for_child(sender, leaf_hash).await {
+		Ok(session_index) =>
+			if state.latest_session.map_or(true, |l| l < session_index) {
+				let signing_credentials =
+					check_signing_credentials(sender, keystore, leaf_hash).await;
+				Some((session_index, signing_credentials))
+			} else {
+				None
+			},
+		Err(e) => {
+			gum::warn!(
+				target: LOG_TARGET,
+				relay_parent = ?leaf_hash,
+				"cannot fetch session index from runtime API: {:?}",
+				e,
+			);
+			None
+		},
+	};
+
+	Some(ActivationEffect { new_session_index, recent_block, pending_pvfs })
+}
+
+/// Checks the active validators for the given leaf. If we have a signing key for one of them,
+/// returns the [`SigningCredentials`].
+async fn check_signing_credentials(
+	sender: &mut impl SubsystemSender<RuntimeApiMessage>,
+	keystore: &KeystorePtr,
+	leaf: Hash,
+) -> Option<SigningCredentials> {
+	let validators = match runtime_api::validators(sender, leaf).await {
+		Ok(v) => v,
+		Err(e) => {
+			gum::warn!(
+				target: LOG_TARGET,
+				relay_parent = ?leaf,
+				"error occurred during requesting validators: {:?}",
+				e
+			);
+			return None;
+		},
+	};
+
+	pezkuwi_node_subsystem_util::signing_key_and_index(&validators, keystore).map(
+		|(validator_key, validator_index)| SigningCredentials { validator_key, validator_index },
+	)
+}
+
+/// Signs and submits a vote for or against a given validation code.
+///
+/// If the validator already voted for the given code, this function does nothing.
+async fn sign_and_submit_pvf_check_statement(
+	sender: &mut impl overseer::PvfCheckerSenderTrait,
+	keystore: &KeystorePtr,
+	voted: &mut HashSet<ValidationCodeHash>,
+	credentials: &SigningCredentials,
+	metrics: &Metrics,
+	relay_parent: Hash,
+	session_index: SessionIndex,
+	judgement: Judgement,
+	validation_code_hash: ValidationCodeHash,
+) {
+	gum::debug!(
+		target: LOG_TARGET,
+		?validation_code_hash,
+		?relay_parent,
+		"submitting a PVF check statement for validation code = {:?}",
+		judgement,
+	);
+
+	metrics.on_vote_submission_started();
+
+	if voted.contains(&validation_code_hash) {
+		gum::trace!(
+			target: LOG_TARGET,
+			relay_parent = ?relay_parent,
+			?validation_code_hash,
+			"already voted for this validation code",
+		);
+		metrics.on_vote_duplicate();
+		return;
+	}
+
+	voted.insert(validation_code_hash);
+
+	let stmt = PvfCheckStatement {
+		accept: judgement.is_valid(),
+		session_index,
+		subject: validation_code_hash,
+		validator_index: credentials.validator_index,
+	};
+	let signature = match pezkuwi_node_subsystem_util::sign(
+		keystore,
+		&credentials.validator_key,
+		&stmt.signing_payload(),
+	) {
+		Ok(Some(signature)) => signature,
+		Ok(None) => {
+			gum::warn!(
+				target: LOG_TARGET,
+				?relay_parent,
+				validator_index = ?credentials.validator_index,
+				?validation_code_hash,
+				"private key for signing is not available",
+			);
+			return;
+		},
+		Err(e) => {
+			gum::warn!(
+				target: LOG_TARGET,
+				?relay_parent,
+				validator_index = ?credentials.validator_index,
+				?validation_code_hash,
+				"error signing the statement: {:?}",
+				e,
+			);
+			return;
+		},
+	};
+
+	match runtime_api::submit_pvf_check_statement(sender, relay_parent, stmt, signature).await {
+		Ok(()) => {
+			metrics.on_vote_submitted();
+		},
+		Err(e) => {
+			gum::warn!(
+				target: LOG_TARGET,
+				?relay_parent,
+				?validation_code_hash,
+				"error occurred during submitting a vote: {:?}",
+				e,
+			);
+		},
+	}
+}
+
+/// Sends a request to the candidate-validation subsystem to validate the given PVF.
+///
+/// The relay-parent is used as an anchor from where to fetch the PVF code. The request will be put
+/// into the `currently_checking` set.
+async fn initiate_precheck(
+	state: &mut State,
+	sender: &mut impl overseer::PvfCheckerSenderTrait,
+	relay_parent: Hash,
+	validation_code_hash: ValidationCodeHash,
+	metrics: &Metrics,
+) {
+	gum::debug!(target: LOG_TARGET, ?validation_code_hash, ?relay_parent, "initiating a precheck",);
+
+	let (tx, rx) = oneshot::channel();
+	sender
+		.send_message(CandidateValidationMessage::PreCheck {
+			relay_parent,
+			validation_code_hash,
+			response_sender: tx,
+		})
+		.await;
+
+	let timer = metrics.time_pre_check_judgement();
+	state.currently_checking.push(Box::pin(async move {
+		let _timer = timer;
+		match rx.await {
+			Ok(accept) => Some((accept, validation_code_hash)),
+			Err(oneshot::Canceled) => {
+				// Pre-checking request dropped before replying. That can happen in case the
+				// overseer is shutting down. Our part of shutdown will be handled by the
+				// overseer conclude signal. Log it here just in case.
+				gum::debug!(
+					target: LOG_TARGET,
+					?validation_code_hash,
+					?relay_parent,
+					"precheck request was canceled",
+				);
+				None
+			},
+		}
+	}));
+}

pezkuwi/node/core/pvf-checker/src/metrics.rs

@@ -0,0 +1,130 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Metrics definitions for the PVF pre-checking subsystem.
+
+use pezkuwi_node_subsystem_util::metrics::{self, prometheus};
+
+#[derive(Clone)]
+struct MetricsInner {
+	pre_check_judgement: prometheus::Histogram,
+	votes_total: prometheus::Counter<prometheus::U64>,
+	votes_started: prometheus::Counter<prometheus::U64>,
+	votes_duplicate: prometheus::Counter<prometheus::U64>,
+	pvfs_observed: prometheus::Counter<prometheus::U64>,
+	pvfs_left: prometheus::Counter<prometheus::U64>,
+}
+
+#[derive(Default, Clone)]
+pub struct Metrics(Option<MetricsInner>);
+
+impl Metrics {
+	/// Time between sending the pre-check request to receiving the response.
+	pub(crate) fn time_pre_check_judgement(
+		&self,
+	) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.pre_check_judgement.start_timer())
+	}
+
+	/// Called when a PVF vote/statement is submitted.
+	pub(crate) fn on_vote_submitted(&self) {
+		if let Some(metrics) = &self.0 {
+			metrics.votes_total.inc();
+		}
+	}
+
+	/// Called when a PVF vote/statement is started submission.
+	pub(crate) fn on_vote_submission_started(&self) {
+		if let Some(metrics) = &self.0 {
+			metrics.votes_started.inc();
+		}
+	}
+
+	/// Called when the vote is a duplicate.
+	pub(crate) fn on_vote_duplicate(&self) {
+		if let Some(metrics) = &self.0 {
+			metrics.votes_duplicate.inc();
+		}
+	}
+
+	/// Called when a new PVF is observed.
+	pub(crate) fn on_pvf_observed(&self, num: usize) {
+		if let Some(metrics) = &self.0 {
+			metrics.pvfs_observed.inc_by(num as u64);
+		}
+	}
+
+	/// Called when a PVF left the view.
+	pub(crate) fn on_pvf_left(&self, num: usize) {
+		if let Some(metrics) = &self.0 {
+			metrics.pvfs_left.inc_by(num as u64);
+		}
+	}
+}
+
+impl metrics::Metrics for Metrics {
+	fn try_register(registry: &prometheus::Registry) -> Result<Self, prometheus::PrometheusError> {
+		let metrics = MetricsInner {
+			pre_check_judgement: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"pezkuwi_pvf_precheck_judgement",
+						"Time between sending the pre-check request to receiving the response.",
+					)
+					.buckets(vec![0.1, 0.5, 1.0, 10.0, 20.0, 30.0, 40.0, 50.0, 60.0]),
+				)?,
+				registry,
+			)?,
+			votes_total: prometheus::register(
+				prometheus::Counter::new(
+					"pezkuwi_pvf_precheck_votes_total",
+					"The total number of votes submitted.",
+				)?,
+				registry,
+			)?,
+			votes_started: prometheus::register(
+				prometheus::Counter::new(
+					"pezkuwi_pvf_precheck_votes_started",
+					"The number of votes that are pending submission",
+				)?,
+				registry,
+			)?,
+			votes_duplicate: prometheus::register(
+				prometheus::Counter::new(
+					"pezkuwi_pvf_precheck_votes_duplicate",
+					"The number of votes that are submitted more than once for the same code within\
+the same session.",
+				)?,
+				registry,
+			)?,
+			pvfs_observed: prometheus::register(
+				prometheus::Counter::new(
+					"pezkuwi_pvf_precheck_pvfs_observed",
+					"The number of new PVFs observed.",
+				)?,
+				registry,
+			)?,
+			pvfs_left: prometheus::register(
+				prometheus::Counter::new(
+					"pezkuwi_pvf_precheck_pvfs_left",
+					"The number of PVFs removed from the view.",
+				)?,
+				registry,
+			)?,
+		};
+		Ok(Self(Some(metrics)))
+	}
+}

pezkuwi/node/core/pvf-checker/src/runtime_api.rs

@@ -0,0 +1,108 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::LOG_TARGET;
+use futures::channel::oneshot;
+use pezkuwi_node_subsystem::{
+	errors::RuntimeApiError as RuntimeApiSubsystemError,
+	messages::{RuntimeApiMessage, RuntimeApiRequest},
+	SubsystemSender,
+};
+use pezkuwi_primitives::{
+	Hash, PvfCheckStatement, SessionIndex, ValidationCodeHash, ValidatorId, ValidatorSignature,
+};
+
+pub(crate) async fn session_index_for_child(
+	sender: &mut impl SubsystemSender<RuntimeApiMessage>,
+	relay_parent: Hash,
+) -> Result<SessionIndex, RuntimeRequestError> {
+	let (tx, rx) = oneshot::channel();
+	runtime_api_request(sender, relay_parent, RuntimeApiRequest::SessionIndexForChild(tx), rx).await
+}
+
+pub(crate) async fn validators(
+	sender: &mut impl SubsystemSender<RuntimeApiMessage>,
+	relay_parent: Hash,
+) -> Result<Vec<ValidatorId>, RuntimeRequestError> {
+	let (tx, rx) = oneshot::channel();
+	runtime_api_request(sender, relay_parent, RuntimeApiRequest::Validators(tx), rx).await
+}
+
+pub(crate) async fn submit_pvf_check_statement(
+	sender: &mut impl SubsystemSender<RuntimeApiMessage>,
+	relay_parent: Hash,
+	stmt: PvfCheckStatement,
+	signature: ValidatorSignature,
+) -> Result<(), RuntimeRequestError> {
+	let (tx, rx) = oneshot::channel();
+	runtime_api_request(
+		sender,
+		relay_parent,
+		RuntimeApiRequest::SubmitPvfCheckStatement(stmt, signature, tx),
+		rx,
+	)
+	.await
+}
+
+pub(crate) async fn pvfs_require_precheck(
+	sender: &mut impl SubsystemSender<RuntimeApiMessage>,
+	relay_parent: Hash,
+) -> Result<Vec<ValidationCodeHash>, RuntimeRequestError> {
+	let (tx, rx) = oneshot::channel();
+	runtime_api_request(sender, relay_parent, RuntimeApiRequest::PvfsRequirePrecheck(tx), rx).await
+}
+
+#[derive(Debug)]
+pub(crate) enum RuntimeRequestError {
+	NotSupported,
+	ApiError,
+	CommunicationError,
+}
+
+pub(crate) async fn runtime_api_request<T>(
+	sender: &mut impl SubsystemSender<RuntimeApiMessage>,
+	relay_parent: Hash,
+	request: RuntimeApiRequest,
+	receiver: oneshot::Receiver<Result<T, RuntimeApiSubsystemError>>,
+) -> Result<T, RuntimeRequestError> {
+	sender
+		.send_message(RuntimeApiMessage::Request(relay_parent, request).into())
+		.await;
+
+	receiver
+		.await
+		.map_err(|_| {
+			gum::debug!(target: LOG_TARGET, ?relay_parent, "Runtime API request dropped");
+			RuntimeRequestError::CommunicationError
+		})
+		.and_then(|res| {
+			res.map_err(|e| {
+				use RuntimeApiSubsystemError::*;
+				match e {
+					Execution { .. } => {
+						gum::debug!(
+							target: LOG_TARGET,
+							?relay_parent,
+							err = ?e,
+							"Runtime API request internal error"
+						);
+						RuntimeRequestError::ApiError
+					},
+					NotSupported { .. } => RuntimeRequestError::NotSupported,
+				}
+			})
+		})
+}

pezkuwi/node/core/pvf-checker/src/tests.rs

@@ -0,0 +1,930 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use futures::{channel::oneshot, future::BoxFuture, prelude::*};
+use pezkuwi_node_subsystem::{
+	messages::{
+		AllMessages, CandidateValidationMessage, PreCheckOutcome, PvfCheckerMessage,
+		RuntimeApiMessage, RuntimeApiRequest,
+	},
+	ActiveLeavesUpdate, FromOrchestra, OverseerSignal, RuntimeApiError,
+};
+use pezkuwi_node_subsystem_test_helpers::{
+	make_subsystem_context, mock::new_leaf, TestSubsystemContextHandle,
+};
+use pezkuwi_primitives::{
+	BlockNumber, Hash, Header, PvfCheckStatement, SessionIndex, ValidationCode, ValidationCodeHash,
+	ValidatorId,
+};
+use pezkuwi_primitives_test_helpers::{dummy_digest, dummy_hash, validator_pubkeys};
+use sp_application_crypto::AppCrypto;
+use sp_core::testing::TaskExecutor;
+use sp_keyring::Sr25519Keyring;
+use sp_keystore::Keystore;
+use sp_runtime::traits::AppVerify;
+use std::{collections::HashMap, sync::Arc, time::Duration};
+
+type VirtualOverseer = TestSubsystemContextHandle<PvfCheckerMessage>;
+
+fn dummy_validation_code_hash(discriminator: u8) -> ValidationCodeHash {
+	ValidationCode(vec![discriminator]).hash()
+}
+
+struct StartsNewSession {
+	session_index: SessionIndex,
+	validators: Vec<Sr25519Keyring>,
+}
+
+#[derive(Debug, Clone)]
+struct FakeLeaf {
+	block_hash: Hash,
+	block_number: BlockNumber,
+	pvfs: Vec<ValidationCodeHash>,
+}
+
+impl FakeLeaf {
+	fn new(parent_hash: Hash, block_number: BlockNumber, pvfs: Vec<ValidationCodeHash>) -> Self {
+		let block_header = Header {
+			parent_hash,
+			number: block_number,
+			digest: dummy_digest(),
+			state_root: dummy_hash(),
+			extrinsics_root: dummy_hash(),
+		};
+		let block_hash = block_header.hash();
+		Self { block_hash, block_number, pvfs }
+	}
+
+	fn descendant(&self, pvfs: Vec<ValidationCodeHash>) -> FakeLeaf {
+		FakeLeaf::new(self.block_hash, self.block_number + 1, pvfs)
+	}
+}
+
+struct LeafState {
+	/// The session index at which this leaf was activated.
+	session_index: SessionIndex,
+
+	/// The list of PVFs that are pending in this leaf.
+	pvfs: Vec<ValidationCodeHash>,
+}
+
+/// The state we model about a session.
+struct SessionState {
+	validators: Vec<ValidatorId>,
+}
+
+struct TestState {
+	leaves: HashMap<Hash, LeafState>,
+	sessions: HashMap<SessionIndex, SessionState>,
+	last_session_index: SessionIndex,
+}
+
+const OUR_VALIDATOR: Sr25519Keyring = Sr25519Keyring::Alice;
+
+impl TestState {
+	fn new() -> Self {
+		// Initialize the default session 1. No validators are present there.
+		let last_session_index = 1;
+		let mut sessions = HashMap::new();
+		sessions.insert(last_session_index, SessionState { validators: vec![] });
+
+		let mut leaves = HashMap::new();
+		leaves.insert(dummy_hash(), LeafState { session_index: last_session_index, pvfs: vec![] });
+
+		Self { leaves, sessions, last_session_index }
+	}
+
+	/// A convenience function to receive a message from the overseer and returning `None` if
+	/// nothing was received within a reasonable (for local tests anyway) timeout.
+	async fn recv_timeout(&mut self, handle: &mut VirtualOverseer) -> Option<AllMessages> {
+		futures::select! {
+			msg = handle.recv().fuse() => {
+				Some(msg)
+			}
+			_ = futures_timer::Delay::new(Duration::from_millis(500)).fuse() => {
+				None
+			}
+		}
+	}
+
+	async fn send_conclude(&mut self, handle: &mut VirtualOverseer) {
+		// To ensure that no messages are left in the queue there is no better way to just wait.
+		match self.recv_timeout(handle).await {
+			Some(msg) => {
+				panic!("we supposed to conclude, but received a message: {:#?}", msg);
+			},
+			None => {
+				// No messages are received. We are good.
+			},
+		}
+
+		handle.send(FromOrchestra::Signal(OverseerSignal::Conclude)).await;
+	}
+
+	/// Convenience function to invoke [`active_leaves_update`] with the new leaf that starts a new
+	/// session and there are no deactivated leaves.
+	///
+	/// Returns the block hash of the newly activated leaf.
+	async fn activate_leaf_with_session(
+		&mut self,
+		handle: &mut VirtualOverseer,
+		leaf: FakeLeaf,
+		starts_new_session: StartsNewSession,
+	) {
+		self.active_leaves_update(handle, Some(leaf), Some(starts_new_session), &[])
+			.await
+	}
+
+	/// Convenience function to invoke [`active_leaves_update`] with a new leaf. The leaf does not
+	/// start a new session and there are no deactivated leaves.
+	async fn activate_leaf(&mut self, handle: &mut VirtualOverseer, leaf: FakeLeaf) {
+		self.active_leaves_update(handle, Some(leaf), None, &[]).await
+	}
+
+	async fn deactivate_leaves(
+		&mut self,
+		handle: &mut VirtualOverseer,
+		deactivated: impl IntoIterator<Item = &Hash>,
+	) {
+		self.active_leaves_update(handle, None, None, deactivated).await
+	}
+
+	/// Sends an `ActiveLeavesUpdate` message to the overseer and also updates the test state to
+	/// record leaves and session changes.
+	///
+	/// NOTE: This function may stall if there is an unhandled message for the overseer.
+	async fn active_leaves_update(
+		&mut self,
+		handle: &mut VirtualOverseer,
+		fake_leaf: Option<FakeLeaf>,
+		starts_new_session: Option<StartsNewSession>,
+		deactivated: impl IntoIterator<Item = &Hash>,
+	) {
+		if let Some(new_session) = starts_new_session {
+			assert!(fake_leaf.is_some(), "Session can be started only with an activated leaf");
+			self.last_session_index = new_session.session_index;
+			let prev = self.sessions.insert(
+				new_session.session_index,
+				SessionState { validators: validator_pubkeys(&new_session.validators) },
+			);
+			assert!(prev.is_none(), "Session {} already exists", new_session.session_index);
+		}
+
+		let activated = if let Some(activated_leaf) = fake_leaf {
+			self.leaves.insert(
+				activated_leaf.block_hash,
+				LeafState {
+					session_index: self.last_session_index,
+					pvfs: activated_leaf.pvfs.clone(),
+				},
+			);
+
+			Some(new_leaf(activated_leaf.block_hash, activated_leaf.block_number))
+		} else {
+			None
+		};
+
+		handle
+			.send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate {
+				activated,
+				deactivated: deactivated.into_iter().cloned().collect(),
+			})))
+			.await;
+	}
+
+	/// Expects that the subsystem has sent a `Validators` Runtime API request. Answers with the
+	/// mocked validators for the requested leaf.
+	async fn expect_validators(&mut self, handle: &mut VirtualOverseer) {
+		match self.recv_timeout(handle).await.expect("timeout waiting for a message") {
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+				relay_parent,
+				RuntimeApiRequest::Validators(tx),
+			)) => match self.leaves.get(&relay_parent) {
+				Some(leaf) => {
+					let session_index = leaf.session_index;
+					let session = self.sessions.get(&session_index).unwrap();
+					tx.send(Ok(session.validators.clone())).unwrap();
+				},
+				None => {
+					panic!("a request to an unknown relay parent has been made");
+				},
+			},
+			msg => panic!("Unexpected message was received: {:#?}", msg),
+		}
+	}
+
+	/// Expects that the subsystem has sent a `SessionIndexForChild` Runtime API request. Answers
+	/// with the mocked session index for the requested leaf.
+	async fn expect_session_for_child(&mut self, handle: &mut VirtualOverseer) {
+		match self.recv_timeout(handle).await.expect("timeout waiting for a message") {
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+				relay_parent,
+				RuntimeApiRequest::SessionIndexForChild(tx),
+			)) => match self.leaves.get(&relay_parent) {
+				Some(leaf) => {
+					tx.send(Ok(leaf.session_index)).unwrap();
+				},
+				None => {
+					panic!("a request to an unknown relay parent has been made");
+				},
+			},
+			msg => panic!("Unexpected message was received: {:#?}", msg),
+		}
+	}
+
+	/// Expects that the subsystem has sent a `PvfsRequirePrecheck` Runtime API request. Answers
+	/// with the mocked PVF set for the requested leaf.
+	async fn expect_pvfs_require_precheck(
+		&mut self,
+		handle: &mut VirtualOverseer,
+	) -> ExpectPvfsRequirePrecheck<'_> {
+		match self.recv_timeout(handle).await.expect("timeout waiting for a message") {
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+				relay_parent,
+				RuntimeApiRequest::PvfsRequirePrecheck(tx),
+			)) => ExpectPvfsRequirePrecheck { test_state: self, relay_parent, tx },
+			msg => panic!("Unexpected message was received: {:#?}", msg),
+		}
+	}
+
+	/// Expects that the subsystem has sent a pre-checking request to candidate-validation. Returns
+	/// a mocked handle for the request.
+	async fn expect_candidate_precheck(
+		&mut self,
+		handle: &mut VirtualOverseer,
+	) -> ExpectCandidatePrecheck {
+		match self.recv_timeout(handle).await.expect("timeout waiting for a message") {
+			AllMessages::CandidateValidation(CandidateValidationMessage::PreCheck {
+				relay_parent,
+				validation_code_hash,
+				response_sender,
+				..
+			}) => ExpectCandidatePrecheck { relay_parent, validation_code_hash, tx: response_sender },
+			msg => panic!("Unexpected message was received: {:#?}", msg),
+		}
+	}
+
+	/// Expects that the subsystem has sent a `SubmitPvfCheckStatement` runtime API request. Returns
+	/// a mocked handle for the request.
+	async fn expect_submit_vote(&mut self, handle: &mut VirtualOverseer) -> ExpectSubmitVote {
+		match self.recv_timeout(handle).await.expect("timeout waiting for a message") {
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+				relay_parent,
+				RuntimeApiRequest::SubmitPvfCheckStatement(stmt, signature, tx),
+			)) => {
+				let signing_payload = stmt.signing_payload();
+				assert!(signature.verify(&signing_payload[..], &OUR_VALIDATOR.public().into()));
+
+				ExpectSubmitVote { relay_parent, stmt, tx }
+			},
+			msg => panic!("Unexpected message was received: {:#?}", msg),
+		}
+	}
+}
+
+#[must_use]
+struct ExpectPvfsRequirePrecheck<'a> {
+	test_state: &'a mut TestState,
+	relay_parent: Hash,
+	tx: oneshot::Sender<Result<Vec<ValidationCodeHash>, RuntimeApiError>>,
+}
+
+impl<'a> ExpectPvfsRequirePrecheck<'a> {
+	fn reply_mock(self) {
+		match self.test_state.leaves.get(&self.relay_parent) {
+			Some(leaf) => {
+				self.tx.send(Ok(leaf.pvfs.clone())).unwrap();
+			},
+			None => {
+				panic!(
+					"a request to an unknown relay parent has been made: {:#?}",
+					self.relay_parent
+				);
+			},
+		}
+	}
+
+	fn reply_not_supported(self) {
+		self.tx
+			.send(Err(RuntimeApiError::NotSupported { runtime_api_name: "pvfs_require_precheck" }))
+			.unwrap();
+	}
+}
+
+#[must_use]
+struct ExpectCandidatePrecheck {
+	relay_parent: Hash,
+	validation_code_hash: ValidationCodeHash,
+	tx: oneshot::Sender<PreCheckOutcome>,
+}
+
+impl ExpectCandidatePrecheck {
+	fn reply(self, outcome: PreCheckOutcome) {
+		self.tx.send(outcome).unwrap();
+	}
+}
+
+#[must_use]
+struct ExpectSubmitVote {
+	relay_parent: Hash,
+	stmt: PvfCheckStatement,
+	tx: oneshot::Sender<Result<(), RuntimeApiError>>,
+}
+
+impl ExpectSubmitVote {
+	fn reply_ok(self) {
+		self.tx.send(Ok(())).unwrap();
+	}
+}
+
+fn test_harness(test: impl FnOnce(TestState, VirtualOverseer) -> BoxFuture<'static, ()>) {
+	let pool = TaskExecutor::new();
+	let (ctx, handle) = make_subsystem_context::<PvfCheckerMessage, _>(pool.clone());
+	let keystore = Arc::new(sc_keystore::LocalKeystore::in_memory());
+
+	// Add OUR_VALIDATOR (which is Alice) to the keystore.
+	Keystore::sr25519_generate_new(&*keystore, ValidatorId::ID, Some(&OUR_VALIDATOR.to_seed()))
+		.expect("Generating keys for our node failed");
+
+	let subsystem_task = crate::run(ctx, keystore, crate::Metrics::default()).map(|x| x.unwrap());
+
+	let test_state = TestState::new();
+	let test_task = test(test_state, handle);
+
+	futures::executor::block_on(future::join(subsystem_task, test_task));
+}
+
+#[test]
+fn concludes_correctly() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn reacts_to_new_pvfs_in_heads() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			let block = FakeLeaf::new(dummy_hash(), 1, vec![dummy_validation_code_hash(1)]);
+
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					block.clone(),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			let pre_check = test_state.expect_candidate_precheck(&mut handle).await;
+			assert_eq!(pre_check.relay_parent, block.block_hash);
+			pre_check.reply(PreCheckOutcome::Valid);
+
+			let vote = test_state.expect_submit_vote(&mut handle).await;
+			assert_eq!(vote.relay_parent, block.block_hash);
+			assert_eq!(vote.stmt.accept, true);
+			assert_eq!(vote.stmt.session_index, 2);
+			assert_eq!(vote.stmt.validator_index, 0.into());
+			assert_eq!(vote.stmt.subject, dummy_validation_code_hash(1));
+			vote.reply_ok();
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn no_new_session_no_validators_request() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					FakeLeaf::new(dummy_hash(), 1, vec![]),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			test_state
+				.activate_leaf(&mut handle, FakeLeaf::new(dummy_hash(), 2, vec![]))
+				.await;
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn activation_of_descendant_leaves_pvfs_in_view() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			let block_1 = FakeLeaf::new(dummy_hash(), 1, vec![dummy_validation_code_hash(1)]);
+			let block_2 = block_1.descendant(vec![dummy_validation_code_hash(1)]);
+
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					block_1.clone(),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			test_state
+				.expect_candidate_precheck(&mut handle)
+				.await
+				.reply(PreCheckOutcome::Valid);
+			test_state.expect_submit_vote(&mut handle).await.reply_ok();
+
+			// Now we deactivate the first block and activate it's descendant.
+			test_state
+				.active_leaves_update(
+					&mut handle,
+					Some(block_2),
+					None, // no new session started
+					&[block_1.block_hash],
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn reactivating_pvf_leads_to_second_check() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			let pvf = dummy_validation_code_hash(1);
+			let block_1 = FakeLeaf::new(dummy_hash(), 1, vec![pvf]);
+			let block_2 = block_1.descendant(vec![]);
+			let block_3 = block_2.descendant(vec![pvf]);
+
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					block_1.clone(),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+			test_state
+				.expect_candidate_precheck(&mut handle)
+				.await
+				.reply(PreCheckOutcome::Valid);
+			test_state.expect_submit_vote(&mut handle).await.reply_ok();
+
+			// Now activate a descendant leaf, where the PVF is not present.
+			test_state
+				.active_leaves_update(
+					&mut handle,
+					Some(block_2.clone()),
+					None,
+					&[block_1.block_hash],
+				)
+				.await;
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+
+			// Now the third block is activated, where the PVF is present.
+			test_state.activate_leaf(&mut handle, block_3).await;
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state
+				.expect_candidate_precheck(&mut handle)
+				.await
+				.reply(PreCheckOutcome::Valid);
+
+			// We do not vote here, because the PVF was already voted on within this session.
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn dont_double_vote_for_pvfs_in_view() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			let pvf = dummy_validation_code_hash(1);
+			let block_1_1 = FakeLeaf::new([1; 32].into(), 1, vec![pvf]);
+			let block_2_1 = FakeLeaf::new([2; 32].into(), 1, vec![pvf]);
+			let block_1_2 = block_1_1.descendant(vec![pvf]);
+
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					block_1_1.clone(),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			// Pre-checking will take quite some time.
+			let pre_check = test_state.expect_candidate_precheck(&mut handle).await;
+
+			// Activate a sibiling leaf, has the same PVF.
+			test_state.activate_leaf(&mut handle, block_2_1).await;
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+
+			// Now activate a descendant leaf with the same PVF.
+			test_state
+				.active_leaves_update(
+					&mut handle,
+					Some(block_1_2.clone()),
+					None,
+					&[block_1_1.block_hash],
+				)
+				.await;
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+
+			// Now finish the pre-checking request.
+			pre_check.reply(PreCheckOutcome::Valid);
+			test_state.expect_submit_vote(&mut handle).await.reply_ok();
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn judgements_come_out_of_order() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			let pvf_1 = dummy_validation_code_hash(1);
+			let pvf_2 = dummy_validation_code_hash(2);
+
+			let block_1 = FakeLeaf::new([1; 32].into(), 1, vec![pvf_1]);
+			let block_2 = FakeLeaf::new([2; 32].into(), 1, vec![pvf_2]);
+
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					block_1.clone(),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			let pre_check_1 = test_state.expect_candidate_precheck(&mut handle).await;
+
+			// Activate a sibiling leaf, has the second PVF.
+			test_state.activate_leaf(&mut handle, block_2).await;
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+
+			let pre_check_2 = test_state.expect_candidate_precheck(&mut handle).await;
+
+			// Resolve the PVF pre-checks out of order.
+			pre_check_2.reply(PreCheckOutcome::Valid);
+			pre_check_1.reply(PreCheckOutcome::Invalid);
+
+			// Catch the vote for the second PVF.
+			let vote_2 = test_state.expect_submit_vote(&mut handle).await;
+			assert_eq!(vote_2.stmt.accept, true);
+			assert_eq!(vote_2.stmt.subject, pvf_2.clone());
+			vote_2.reply_ok();
+
+			// Catch the vote for the first PVF.
+			let vote_1 = test_state.expect_submit_vote(&mut handle).await;
+			assert_eq!(vote_1.stmt.accept, false);
+			assert_eq!(vote_1.stmt.subject, pvf_1.clone());
+			vote_1.reply_ok();
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn dont_vote_until_a_validator() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					FakeLeaf::new(dummy_hash(), 1, vec![dummy_validation_code_hash(1)]),
+					StartsNewSession { session_index: 2, validators: vec![Sr25519Keyring::Bob] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			test_state
+				.expect_candidate_precheck(&mut handle)
+				.await
+				.reply(PreCheckOutcome::Invalid);
+
+			// Now a leaf brings a new session. In this session our validator comes into the active
+			// set. That means it will cast a vote for each judgement it has.
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					FakeLeaf::new(dummy_hash(), 2, vec![dummy_validation_code_hash(1)]),
+					StartsNewSession {
+						session_index: 3,
+						validators: vec![Sr25519Keyring::Bob, OUR_VALIDATOR],
+					},
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+			let vote = test_state.expect_submit_vote(&mut handle).await;
+			assert_eq!(vote.stmt.accept, false);
+			assert_eq!(vote.stmt.session_index, 3);
+			assert_eq!(vote.stmt.validator_index, 1.into());
+			assert_eq!(vote.stmt.subject, dummy_validation_code_hash(1));
+			vote.reply_ok();
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn resign_on_session_change() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			let pvf_1 = dummy_validation_code_hash(1);
+			let pvf_2 = dummy_validation_code_hash(2);
+
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					FakeLeaf::new(dummy_hash(), 1, vec![pvf_1, pvf_2]),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			let pre_check_1 = test_state.expect_candidate_precheck(&mut handle).await;
+			assert_eq!(pre_check_1.validation_code_hash, pvf_1);
+			pre_check_1.reply(PreCheckOutcome::Valid);
+			let pre_check_2 = test_state.expect_candidate_precheck(&mut handle).await;
+			assert_eq!(pre_check_2.validation_code_hash, pvf_2);
+			pre_check_2.reply(PreCheckOutcome::Invalid);
+
+			test_state.expect_submit_vote(&mut handle).await.reply_ok();
+			test_state.expect_submit_vote(&mut handle).await.reply_ok();
+
+			// So far so good. Now we change the session.
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					FakeLeaf::new(dummy_hash(), 2, vec![pvf_1, pvf_2]),
+					StartsNewSession { session_index: 3, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			// The votes should be re-signed and re-submitted.
+			let mut statements = Vec::new();
+			let vote_1 = test_state.expect_submit_vote(&mut handle).await;
+			statements.push(vote_1.stmt.clone());
+			vote_1.reply_ok();
+			let vote_2 = test_state.expect_submit_vote(&mut handle).await;
+			statements.push(vote_2.stmt.clone());
+			vote_2.reply_ok();
+
+			// Find and check the votes.
+			// Unfortunately, the order of revoting is not deterministic so we have to resort to
+			// a bit of trickery.
+			assert_eq!(statements.iter().find(|s| s.subject == pvf_1).unwrap().accept, true);
+			assert_eq!(statements.iter().find(|s| s.subject == pvf_2).unwrap().accept, false);
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn dont_resign_if_not_us() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			let pvf_1 = dummy_validation_code_hash(1);
+			let pvf_2 = dummy_validation_code_hash(2);
+
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					FakeLeaf::new(dummy_hash(), 1, vec![pvf_1, pvf_2]),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			let pre_check_1 = test_state.expect_candidate_precheck(&mut handle).await;
+			assert_eq!(pre_check_1.validation_code_hash, pvf_1);
+			pre_check_1.reply(PreCheckOutcome::Valid);
+			let pre_check_2 = test_state.expect_candidate_precheck(&mut handle).await;
+			assert_eq!(pre_check_2.validation_code_hash, pvf_2);
+			pre_check_2.reply(PreCheckOutcome::Invalid);
+
+			test_state.expect_submit_vote(&mut handle).await.reply_ok();
+			test_state.expect_submit_vote(&mut handle).await.reply_ok();
+
+			// So far so good. Now we change the session.
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					FakeLeaf::new(dummy_hash(), 2, vec![pvf_1, pvf_2]),
+					StartsNewSession {
+						session_index: 3,
+						// not us
+						validators: vec![Sr25519Keyring::Bob],
+					},
+				)
+				.await;
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			// We do not expect any votes to be re-signed.
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn api_not_supported() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					FakeLeaf::new(dummy_hash(), 1, vec![dummy_validation_code_hash(1)]),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_not_supported();
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn not_supported_api_becomes_supported() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					FakeLeaf::new(dummy_hash(), 1, vec![dummy_validation_code_hash(1)]),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_not_supported();
+
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					FakeLeaf::new(dummy_hash(), 1, vec![dummy_validation_code_hash(1)]),
+					StartsNewSession { session_index: 3, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+			test_state
+				.expect_candidate_precheck(&mut handle)
+				.await
+				.reply(PreCheckOutcome::Valid);
+			test_state.expect_submit_vote(&mut handle).await.reply_ok();
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+#[test]
+fn unexpected_pvf_check_judgement() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			let block_1 = FakeLeaf::new(dummy_hash(), 1, vec![dummy_validation_code_hash(1)]);
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					block_1.clone(),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			// Catch the pre-check request, but don't reply just yet.
+			let pre_check = test_state.expect_candidate_precheck(&mut handle).await;
+
+			// Now deactivate the leaf and reply to the precheck request.
+			test_state.deactivate_leaves(&mut handle, &[block_1.block_hash]).await;
+			pre_check.reply(PreCheckOutcome::Invalid);
+
+			// the subsystem must remain silent.
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}
+
+// Check that we do not abstain for a nondeterministic failure. Currently, this means the behavior
+// is the same as if the pre-check returned `PreCheckOutcome::Invalid`.
+#[test]
+fn dont_abstain_for_nondeterministic_pvfcheck_failure() {
+	test_harness(|mut test_state, mut handle| {
+		async move {
+			let block_1 = FakeLeaf::new(dummy_hash(), 1, vec![dummy_validation_code_hash(1)]);
+			test_state
+				.activate_leaf_with_session(
+					&mut handle,
+					block_1.clone(),
+					StartsNewSession { session_index: 2, validators: vec![OUR_VALIDATOR] },
+				)
+				.await;
+
+			test_state.expect_pvfs_require_precheck(&mut handle).await.reply_mock();
+			test_state.expect_session_for_child(&mut handle).await;
+			test_state.expect_validators(&mut handle).await;
+
+			// Catch the pre-check request, but don't reply just yet.
+			let pre_check = test_state.expect_candidate_precheck(&mut handle).await;
+
+			// Now deactivate the leaf and reply to the precheck request.
+			test_state.deactivate_leaves(&mut handle, &[block_1.block_hash]).await;
+			pre_check.reply(PreCheckOutcome::Failed);
+
+			// the subsystem must remain silent.
+
+			test_state.send_conclude(&mut handle).await;
+		}
+		.boxed()
+	});
+}