feat: Rebrand Polkadot/Substrate references to PezkuwiChain

This commit systematically rebrands various references from Parity Technologies'
Polkadot/Substrate ecosystem to PezkuwiChain within the kurdistan-sdk.

Key changes include:
- Updated external repository URLs (zombienet-sdk, parity-db, parity-scale-codec, wasm-instrument) to point to pezkuwichain forks.
- Modified internal documentation and code comments to reflect PezkuwiChain naming and structure.
- Replaced direct references to  with  or specific paths within the  for XCM, Pezkuwi, and other modules.
- Cleaned up deprecated  issue and PR references in various  and  files, particularly in  and  modules.
- Adjusted image and logo URLs in documentation to point to PezkuwiChain assets.
- Removed or rephrased comments related to external Polkadot/Substrate PRs and issues.

This is a significant step towards fully customizing the SDK for the PezkuwiChain ecosystem.

bizinikiwi/client/service/Cargo.toml

@@ -0,0 +1,121 @@
+[package]
+name = "pezsc-service"
+version = "0.35.0"
+authors.workspace = true
+edition.workspace = true
+license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
+homepage.workspace = true
+repository.workspace = true
+description = "Bizinikiwi service. Starts a thread that spins up the network, client, and extrinsic pool. Manages communication between them."
+readme = "README.md"
+
+[lints]
+workspace = true
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[dependencies]
+async-trait = { workspace = true }
+codec = { workspace = true, default-features = true }
+directories = { workspace = true }
+exit-future = { workspace = true }
+futures = { workspace = true }
+futures-timer = { workspace = true }
+jsonrpsee = { features = ["server"], workspace = true }
+log = { workspace = true, default-features = true }
+parking_lot = { workspace = true, default-features = true }
+pin-project = { workspace = true }
+prometheus-endpoint = { workspace = true, default-features = true }
+rand = { workspace = true, default-features = true }
+pezsc-chain-spec = { workspace = true, default-features = true }
+pezsc-client-api = { workspace = true, default-features = true }
+pezsc-client-db = { workspace = true }
+pezsc-consensus = { workspace = true, default-features = true }
+pezsc-executor = { workspace = true, default-features = true }
+pezsc-informant = { workspace = true, default-features = true }
+pezsc-keystore = { workspace = true, default-features = true }
+pezsc-network = { workspace = true, default-features = true }
+pezsc-network-common = { workspace = true, default-features = true }
+pezsc-network-light = { workspace = true, default-features = true }
+pezsc-network-sync = { workspace = true, default-features = true }
+pezsc-network-transactions = { workspace = true, default-features = true }
+pezsc-network-types = { workspace = true, default-features = true }
+pezsc-rpc = { workspace = true, default-features = true }
+pezsc-rpc-server = { workspace = true, default-features = true }
+pezsc-rpc-spec-v2 = { workspace = true, default-features = true }
+pezsc-sysinfo = { workspace = true, default-features = true }
+pezsc-telemetry = { workspace = true, default-features = true }
+pezsc-tracing = { workspace = true, default-features = true }
+pezsc-transaction-pool = { workspace = true, default-features = true }
+pezsc-transaction-pool-api = { workspace = true, default-features = true }
+pezsc-utils = { workspace = true, default-features = true }
+schnellru = { workspace = true }
+serde = { workspace = true, default-features = true }
+serde_json = { workspace = true, default-features = true }
+pezsp-api = { workspace = true, default-features = true }
+pezsp-blockchain = { workspace = true, default-features = true }
+pezsp-consensus = { workspace = true, default-features = true }
+pezsp-core = { workspace = true, default-features = true }
+pezsp-externalities = { workspace = true, default-features = true }
+pezsp-keystore = { workspace = true, default-features = true }
+pezsp-runtime = { workspace = true, default-features = true }
+pezsp-session = { workspace = true, default-features = true }
+pezsp-state-machine = { workspace = true, default-features = true }
+pezsp-storage = { workspace = true, default-features = true }
+pezsp-transaction-pool = { workspace = true, default-features = true }
+pezsp-transaction-storage-proof = { workspace = true, default-features = true }
+pezsp-trie = { workspace = true, default-features = true }
+pezsp-version = { workspace = true, default-features = true }
+static_init = { workspace = true }
+tempfile = { workspace = true }
+thiserror = { workspace = true }
+tokio = { features = [
+	"parking_lot",
+	"rt-multi-thread",
+	"time",
+], workspace = true, default-features = true }
+tracing = { workspace = true, default-features = true }
+tracing-futures = { workspace = true }
+
+[dev-dependencies]
+bizinikiwi-test-runtime = { workspace = true }
+bizinikiwi-test-runtime-client = { workspace = true }
+
+[features]
+default = ["rocksdb"]
+# The RocksDB feature activates the RocksDB database backend. If it is not activated, and you pass
+# a path to a database, an error will be produced at runtime.
+rocksdb = ["pezsc-client-db/rocksdb"]
+runtime-benchmarks = [
+	"pezsc-chain-spec/runtime-benchmarks",
+	"pezsc-client-api/runtime-benchmarks",
+	"pezsc-client-db/runtime-benchmarks",
+	"pezsc-consensus/runtime-benchmarks",
+	"pezsc-executor/runtime-benchmarks",
+	"pezsc-informant/runtime-benchmarks",
+	"pezsc-network-common/runtime-benchmarks",
+	"pezsc-network-light/runtime-benchmarks",
+	"pezsc-network-sync/runtime-benchmarks",
+	"pezsc-network-transactions/runtime-benchmarks",
+	"pezsc-network/runtime-benchmarks",
+	"pezsc-rpc-server/runtime-benchmarks",
+	"pezsc-rpc-spec-v2/runtime-benchmarks",
+	"pezsc-rpc/runtime-benchmarks",
+	"pezsc-sysinfo/runtime-benchmarks",
+	"pezsc-tracing/runtime-benchmarks",
+	"pezsc-transaction-pool-api/runtime-benchmarks",
+	"pezsc-transaction-pool/runtime-benchmarks",
+	"pezsp-api/runtime-benchmarks",
+	"pezsp-blockchain/runtime-benchmarks",
+	"pezsp-consensus/runtime-benchmarks",
+	"pezsp-runtime/runtime-benchmarks",
+	"pezsp-session/runtime-benchmarks",
+	"pezsp-state-machine/runtime-benchmarks",
+	"pezsp-transaction-pool/runtime-benchmarks",
+	"pezsp-transaction-storage-proof/runtime-benchmarks",
+	"pezsp-trie/runtime-benchmarks",
+	"pezsp-version/runtime-benchmarks",
+	"bizinikiwi-test-runtime-client/runtime-benchmarks",
+	"bizinikiwi-test-runtime/runtime-benchmarks",
+]

bizinikiwi/client/service/README.md

@@ -0,0 +1,4 @@
+Bizinikiwi service. Starts a thread that spins up the network, client, and extrinsic pool.
+Manages communication between them.
+
+License: GPL-3.0-or-later WITH Classpath-exception-2.0

bizinikiwi/client/service/src/chain_ops/check_block.rs

@@ -0,0 +1,54 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::error::Error;
+use codec::Encode;
+use pezsc_client_api::{BlockBackend, HeaderBackend};
+use pezsc_consensus::import_queue::ImportQueue;
+use pezsp_runtime::{generic::BlockId, traits::Block as BlockT};
+
+use crate::chain_ops::import_blocks;
+use std::sync::Arc;
+
+/// Re-validate known block.
+pub async fn check_block<B, IQ, C>(
+	client: Arc<C>,
+	import_queue: IQ,
+	block_id: BlockId<B>,
+) -> Result<(), Error>
+where
+	C: BlockBackend<B> + HeaderBackend<B> + Send + Sync + 'static,
+	B: BlockT + for<'de> serde::Deserialize<'de>,
+	IQ: ImportQueue<B> + 'static,
+{
+	let maybe_block = client
+		.block_hash_from_id(&block_id)?
+		.map(|hash| client.block(hash))
+		.transpose()?
+		.flatten();
+	match maybe_block {
+		Some(block) => {
+			let mut buf = Vec::new();
+			1u64.encode_to(&mut buf);
+			block.encode_to(&mut buf);
+			let reader = std::io::Cursor::new(buf);
+			import_blocks(client, import_queue, reader, true, true).await
+		},
+		None => Err("Unknown block")?,
+	}
+}

bizinikiwi/client/service/src/chain_ops/export_blocks.rs

@@ -0,0 +1,107 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::error::Error;
+use codec::Encode;
+use futures::{future, prelude::*};
+use log::info;
+use pezsp_runtime::{
+	generic::BlockId,
+	traits::{Block as BlockT, NumberFor, One, SaturatedConversion, Zero},
+};
+
+use pezsc_client_api::{BlockBackend, HeaderBackend, UsageProvider};
+use std::{io::Write, pin::Pin, sync::Arc, task::Poll};
+
+/// Performs the blocks export.
+pub fn export_blocks<B, C>(
+	client: Arc<C>,
+	mut output: impl Write + 'static,
+	from: NumberFor<B>,
+	to: Option<NumberFor<B>>,
+	binary: bool,
+) -> Pin<Box<dyn Future<Output = Result<(), Error>>>>
+where
+	C: HeaderBackend<B> + BlockBackend<B> + UsageProvider<B> + 'static,
+	B: BlockT,
+{
+	let mut block = from;
+
+	let last = match to {
+		Some(v) if v.is_zero() => One::one(),
+		Some(v) => v,
+		None => client.usage_info().chain.best_number,
+	};
+
+	let mut wrote_header = false;
+
+	// Exporting blocks is implemented as a future, because we want the operation to be
+	// interruptible.
+	//
+	// Every time we write a block to the output, the `Future` re-schedules itself and returns
+	// `Poll::Pending`.
+	// This makes it possible either to interleave other operations in-between the block exports,
+	// or to stop the operation completely.
+	let export = future::poll_fn(move |cx| {
+		let client = &client;
+
+		if last < block {
+			return Poll::Ready(Err("Invalid block range specified".into()));
+		}
+
+		if !wrote_header {
+			info!("Exporting blocks from #{} to #{}", block, last);
+			if binary {
+				let last_: u64 = last.saturated_into::<u64>();
+				let block_: u64 = block.saturated_into::<u64>();
+				let len: u64 = last_ - block_ + 1;
+				output.write_all(&len.encode())?;
+			}
+			wrote_header = true;
+		}
+
+		match client
+			.block_hash_from_id(&BlockId::number(block))?
+			.map(|hash| client.block(hash))
+			.transpose()?
+			.flatten()
+		{
+			Some(block) =>
+				if binary {
+					output.write_all(&block.encode())?;
+				} else {
+					serde_json::to_writer(&mut output, &block)
+						.map_err(|e| format!("Error writing JSON: {}", e))?;
+				},
+			None => return Poll::Ready(Ok(())),
+		}
+		if (block % 10000u32.into()).is_zero() {
+			info!("#{}", block);
+		}
+		if block == last {
+			return Poll::Ready(Ok(()));
+		}
+		block += One::one();
+
+		// Re-schedule the task in order to continue the operation.
+		cx.waker().wake_by_ref();
+		Poll::Pending
+	});
+
+	Box::pin(export)
+}

bizinikiwi/client/service/src/chain_ops/export_raw_state.rs

@@ -0,0 +1,63 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::error::Error;
+use pezsc_client_api::{StorageProvider, UsageProvider};
+use pezsp_core::storage::{well_known_keys, ChildInfo, Storage, StorageChild, StorageKey, StorageMap};
+use pezsp_runtime::traits::Block as BlockT;
+
+use std::{
+	collections::{BTreeMap, HashMap},
+	sync::Arc,
+};
+
+/// Export the raw state at the given `block`. If `block` is `None`, the
+/// best block will be used.
+pub fn export_raw_state<B, BA, C>(client: Arc<C>, hash: B::Hash) -> Result<Storage, Error>
+where
+	C: UsageProvider<B> + StorageProvider<B, BA>,
+	B: BlockT,
+	BA: pezsc_client_api::backend::Backend<B>,
+{
+	let mut top = BTreeMap::new();
+	let mut children_default = HashMap::new();
+
+	for (key, value) in client.storage_pairs(hash, None, None)? {
+		// Remove all default child storage roots from the top storage and collect the child storage
+		// pairs.
+		if key.0.starts_with(well_known_keys::DEFAULT_CHILD_STORAGE_KEY_PREFIX) {
+			let child_root_key = StorageKey(
+				key.0[well_known_keys::DEFAULT_CHILD_STORAGE_KEY_PREFIX.len()..].to_vec(),
+			);
+			let child_info = ChildInfo::new_default(&child_root_key.0);
+			let mut pairs = StorageMap::new();
+			for child_key in client.child_storage_keys(hash, child_info.clone(), None, None)? {
+				if let Some(child_value) = client.child_storage(hash, &child_info, &child_key)? {
+					pairs.insert(child_key.0, child_value.0);
+				}
+			}
+
+			children_default.insert(child_root_key.0, StorageChild { child_info, data: pairs });
+			continue;
+		}
+
+		top.insert(key.0, value.0);
+	}
+
+	Ok(Storage { top, children_default })
+}

bizinikiwi/client/service/src/chain_ops/import_blocks.rs

@@ -0,0 +1,496 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{error, error::Error};
+use codec::{Decode, IoReader as CodecIoReader};
+use futures::{future, prelude::*};
+use futures_timer::Delay;
+use log::{info, warn};
+use pezsc_chain_spec::ChainSpec;
+use pezsc_client_api::HeaderBackend;
+use pezsc_consensus::import_queue::{
+	BlockImportError, BlockImportStatus, ImportQueue, IncomingBlock, Link,
+};
+use serde_json::{de::IoRead as JsonIoRead, Deserializer, StreamDeserializer};
+use pezsp_consensus::BlockOrigin;
+use pezsp_runtime::{
+	generic::SignedBlock,
+	traits::{
+		Block as BlockT, CheckedDiv, Header, MaybeSerializeDeserialize, NumberFor, Saturating, Zero,
+	},
+};
+use std::{
+	io::Read,
+	pin::Pin,
+	sync::{
+		atomic::{AtomicBool, AtomicU64, Ordering},
+		Arc,
+	},
+	task::Poll,
+	time::{Duration, Instant},
+};
+
+/// Number of blocks we will add to the queue before waiting for the queue to catch up.
+const MAX_PENDING_BLOCKS: u64 = 10_000;
+
+/// Number of milliseconds to wait until next poll.
+const DELAY_TIME: u64 = 200;
+
+/// Number of milliseconds that must have passed between two updates.
+const TIME_BETWEEN_UPDATES: u64 = 3_000;
+
+/// Build a chain spec json
+pub fn build_spec(spec: &dyn ChainSpec, raw: bool) -> error::Result<String> {
+	spec.as_json(raw).map_err(Into::into)
+}
+
+/// Helper enum that wraps either a binary decoder (from parity-scale-codec), or a JSON decoder
+/// (from serde_json). Implements the Iterator Trait, calling `next()` will decode the next
+/// SignedBlock and return it.
+enum BlockIter<R, B>
+where
+	R: std::io::Read,
+{
+	Binary {
+		// Total number of blocks we are expecting to decode.
+		num_expected_blocks: u64,
+		// Number of blocks we have decoded thus far.
+		read_block_count: u64,
+		// Reader to the data, used for decoding new blocks.
+		reader: CodecIoReader<R>,
+	},
+	Json {
+		// Number of blocks we have decoded thus far.
+		read_block_count: u64,
+		// Stream to the data, used for decoding new blocks.
+		reader: StreamDeserializer<'static, JsonIoRead<R>, SignedBlock<B>>,
+	},
+}
+
+impl<R, B> BlockIter<R, B>
+where
+	R: Read + 'static,
+	B: BlockT + MaybeSerializeDeserialize,
+{
+	fn new(input: R, binary: bool) -> Result<Self, String> {
+		if binary {
+			let mut reader = CodecIoReader(input);
+			// If the file is encoded in binary format, it is expected to first specify the number
+			// of blocks that are going to be decoded. We read it and add it to our enum struct.
+			let num_expected_blocks: u64 = Decode::decode(&mut reader)
+				.map_err(|e| format!("Failed to decode the number of blocks: {:?}", e))?;
+			Ok(BlockIter::Binary { num_expected_blocks, read_block_count: 0, reader })
+		} else {
+			let stream_deser = Deserializer::from_reader(input).into_iter::<SignedBlock<B>>();
+			Ok(BlockIter::Json { reader: stream_deser, read_block_count: 0 })
+		}
+	}
+
+	/// Returns the number of blocks read thus far.
+	fn read_block_count(&self) -> u64 {
+		match self {
+			BlockIter::Binary { read_block_count, .. } |
+			BlockIter::Json { read_block_count, .. } => *read_block_count,
+		}
+	}
+
+	/// Returns the total number of blocks to be imported, if possible.
+	fn num_expected_blocks(&self) -> Option<u64> {
+		match self {
+			BlockIter::Binary { num_expected_blocks, .. } => Some(*num_expected_blocks),
+			BlockIter::Json { .. } => None,
+		}
+	}
+}
+
+impl<R, B> Iterator for BlockIter<R, B>
+where
+	R: Read + 'static,
+	B: BlockT + MaybeSerializeDeserialize,
+{
+	type Item = Result<SignedBlock<B>, String>;
+
+	fn next(&mut self) -> Option<Self::Item> {
+		match self {
+			BlockIter::Binary { num_expected_blocks, read_block_count, reader } => {
+				if read_block_count < num_expected_blocks {
+					let block_result: Result<SignedBlock<B>, _> =
+						SignedBlock::<B>::decode(reader).map_err(|e| e.to_string());
+					*read_block_count += 1;
+					Some(block_result)
+				} else {
+					// `read_block_count` == `num_expected_blocks` so we've read enough blocks.
+					None
+				}
+			},
+			BlockIter::Json { reader, read_block_count } => {
+				let res = Some(reader.next()?.map_err(|e| e.to_string()));
+				*read_block_count += 1;
+				res
+			},
+		}
+	}
+}
+
+/// Imports the SignedBlock to the queue.
+fn import_block_to_queue<TBl, TImpQu>(
+	signed_block: SignedBlock<TBl>,
+	queue: &mut TImpQu,
+	force: bool,
+) where
+	TBl: BlockT + MaybeSerializeDeserialize,
+	TImpQu: 'static + ImportQueue<TBl>,
+{
+	let (header, extrinsics) = signed_block.block.deconstruct();
+	let hash = header.hash();
+	// import queue handles verification and importing it into the client.
+	queue.service_ref().import_blocks(
+		BlockOrigin::File,
+		vec![IncomingBlock::<TBl> {
+			hash,
+			header: Some(header),
+			body: Some(extrinsics),
+			indexed_body: None,
+			justifications: signed_block.justifications,
+			origin: None,
+			allow_missing_state: false,
+			import_existing: force,
+			state: None,
+			skip_execution: false,
+		}],
+	);
+}
+
+/// Returns true if we have imported every block we were supposed to import, else returns false.
+fn importing_is_done(
+	num_expected_blocks: Option<u64>,
+	read_block_count: u64,
+	imported_blocks: u64,
+) -> bool {
+	if let Some(num_expected_blocks) = num_expected_blocks {
+		imported_blocks >= num_expected_blocks
+	} else {
+		imported_blocks >= read_block_count
+	}
+}
+
+/// Structure used to log the block importing speed.
+struct Speedometer<B: BlockT> {
+	best_number: NumberFor<B>,
+	last_number: Option<NumberFor<B>>,
+	last_update: Instant,
+}
+
+impl<B: BlockT> Speedometer<B> {
+	/// Creates a fresh Speedometer.
+	fn new() -> Self {
+		Self {
+			best_number: NumberFor::<B>::from(0u32),
+			last_number: None,
+			last_update: Instant::now(),
+		}
+	}
+
+	/// Calculates `(best_number - last_number) / (now - last_update)` and
+	/// logs the speed of import.
+	fn display_speed(&self) {
+		// Number of milliseconds elapsed since last time.
+		let elapsed_ms = {
+			let elapsed = self.last_update.elapsed();
+			let since_last_millis = elapsed.as_secs() * 1000;
+			let since_last_subsec_millis = elapsed.subsec_millis() as u64;
+			since_last_millis + since_last_subsec_millis
+		};
+
+		// Number of blocks that have been imported since last time.
+		let diff = match self.last_number {
+			None => return,
+			Some(n) => self.best_number.saturating_sub(n),
+		};
+
+		if let Ok(diff) = TryInto::<u128>::try_into(diff) {
+			// If the number of blocks can be converted to a regular integer, then it's easy: just
+			// do the math and turn it into a `f64`.
+			let speed = diff
+				.saturating_mul(10_000)
+				.checked_div(u128::from(elapsed_ms))
+				.map_or(0.0, |s| s as f64) /
+				10.0;
+			info!("📦 Current best block: {} ({:4.1} bps)", self.best_number, speed);
+		} else {
+			// If the number of blocks can't be converted to a regular integer, then we need a more
+			// algebraic approach and we stay within the realm of integers.
+			let one_thousand = NumberFor::<B>::from(1_000u32);
+			let elapsed =
+				NumberFor::<B>::from(<u32 as TryFrom<_>>::try_from(elapsed_ms).unwrap_or(u32::MAX));
+
+			let speed = diff
+				.saturating_mul(one_thousand)
+				.checked_div(&elapsed)
+				.unwrap_or_else(Zero::zero);
+			info!("📦 Current best block: {} ({} bps)", self.best_number, speed)
+		}
+	}
+
+	/// Updates the Speedometer.
+	fn update(&mut self, best_number: NumberFor<B>) {
+		self.last_number = Some(self.best_number);
+		self.best_number = best_number;
+		self.last_update = Instant::now();
+	}
+
+	// If more than TIME_BETWEEN_UPDATES has elapsed since last update,
+	// then print and update the speedometer.
+	fn notify_user(&mut self, best_number: NumberFor<B>) {
+		let delta = Duration::from_millis(TIME_BETWEEN_UPDATES);
+		if Instant::now().duration_since(self.last_update) >= delta {
+			self.display_speed();
+			self.update(best_number);
+		}
+	}
+}
+
+/// Different State that the `import_blocks` future could be in.
+enum ImportState<R, B>
+where
+	R: Read + 'static,
+	B: BlockT + MaybeSerializeDeserialize,
+{
+	/// We are reading from the [`BlockIter`] structure, adding those blocks to the queue if
+	/// possible.
+	Reading { block_iter: BlockIter<R, B> },
+	/// The queue is full (contains at least MAX_PENDING_BLOCKS blocks) and we are waiting for it
+	/// to catch up.
+	WaitingForImportQueueToCatchUp {
+		block_iter: BlockIter<R, B>,
+		delay: Delay,
+		block: SignedBlock<B>,
+	},
+	// We have added all the blocks to the queue but they are still being processed.
+	WaitingForImportQueueToFinish {
+		num_expected_blocks: Option<u64>,
+		read_block_count: u64,
+		delay: Delay,
+	},
+}
+
+/// Starts the process of importing blocks.
+pub fn import_blocks<B, IQ, C>(
+	client: Arc<C>,
+	mut import_queue: IQ,
+	input: impl Read + Send + 'static,
+	force: bool,
+	binary: bool,
+) -> Pin<Box<dyn Future<Output = Result<(), Error>> + Send>>
+where
+	C: HeaderBackend<B> + Send + Sync + 'static,
+	B: BlockT + for<'de> serde::Deserialize<'de>,
+	IQ: ImportQueue<B> + 'static,
+{
+	struct WaitLink {
+		imported_blocks: AtomicU64,
+		has_error: AtomicBool,
+	}
+
+	impl WaitLink {
+		fn new() -> WaitLink {
+			WaitLink { imported_blocks: AtomicU64::new(0), has_error: AtomicBool::new(false) }
+		}
+	}
+
+	impl<B: BlockT> Link<B> for WaitLink {
+		fn blocks_processed(
+			&self,
+			imported: usize,
+			_num_expected_blocks: usize,
+			results: Vec<(Result<BlockImportStatus<NumberFor<B>>, BlockImportError>, B::Hash)>,
+		) {
+			self.imported_blocks.fetch_add(imported as u64, Ordering::AcqRel);
+
+			for result in results {
+				if let (Err(err), hash) = result {
+					warn!("There was an error importing block with hash {:?}: {}", hash, err);
+					self.has_error.store(true, Ordering::Release);
+					break;
+				}
+			}
+		}
+	}
+
+	let mut link = WaitLink::new();
+	let block_iter_res: Result<BlockIter<_, B>, String> = BlockIter::new(input, binary);
+
+	let block_iter = match block_iter_res {
+		Ok(block_iter) => block_iter,
+		Err(e) => {
+			// We've encountered an error while creating the block iterator
+			// so we can just return a future that returns an error.
+			return future::ready(Err(Error::Other(e))).boxed();
+		},
+	};
+
+	let mut state = Some(ImportState::Reading { block_iter });
+	let mut speedometer = Speedometer::<B>::new();
+
+	// Importing blocks is implemented as a future, because we want the operation to be
+	// interruptible.
+	//
+	// Every time we read a block from the input or import a bunch of blocks from the import
+	// queue, the `Future` re-schedules itself and returns `Poll::Pending`.
+	// This makes it possible either to interleave other operations in-between the block imports,
+	// or to stop the operation completely.
+	let import = future::poll_fn(move |cx| {
+		let client = &client;
+		let queue = &mut import_queue;
+		match state.take().expect("state should never be None; qed") {
+			ImportState::Reading { mut block_iter } => {
+				match block_iter.next() {
+					None => {
+						// The iterator is over: we now need to wait for the import queue to finish.
+						let num_expected_blocks = block_iter.num_expected_blocks();
+						let read_block_count = block_iter.read_block_count();
+						let delay = Delay::new(Duration::from_millis(DELAY_TIME));
+						state = Some(ImportState::WaitingForImportQueueToFinish {
+							num_expected_blocks,
+							read_block_count,
+							delay,
+						});
+					},
+					Some(block_result) => {
+						let read_block_count = block_iter.read_block_count();
+						match block_result {
+							Ok(block) => {
+								if read_block_count - link.imported_blocks.load(Ordering::Acquire) >=
+									MAX_PENDING_BLOCKS
+								{
+									// The queue is full, so do not add this block and simply wait
+									// until the queue has made some progress.
+									let delay = Delay::new(Duration::from_millis(DELAY_TIME));
+									state = Some(ImportState::WaitingForImportQueueToCatchUp {
+										block_iter,
+										delay,
+										block,
+									});
+								} else {
+									// Queue is not full, we can keep on adding blocks to the queue.
+									import_block_to_queue(block, queue, force);
+									state = Some(ImportState::Reading { block_iter });
+								}
+							},
+							Err(e) =>
+								return Poll::Ready(Err(Error::Other(format!(
+									"Error reading block #{}: {}",
+									read_block_count, e
+								)))),
+						}
+					},
+				}
+			},
+			ImportState::WaitingForImportQueueToCatchUp { block_iter, mut delay, block } => {
+				let read_block_count = block_iter.read_block_count();
+				if read_block_count - link.imported_blocks.load(Ordering::Acquire) >=
+					MAX_PENDING_BLOCKS
+				{
+					// Queue is still full, so wait until there is room to insert our block.
+					match Pin::new(&mut delay).poll(cx) {
+						Poll::Pending => {
+							state = Some(ImportState::WaitingForImportQueueToCatchUp {
+								block_iter,
+								delay,
+								block,
+							});
+							return Poll::Pending;
+						},
+						Poll::Ready(_) => {
+							delay.reset(Duration::from_millis(DELAY_TIME));
+						},
+					}
+					state = Some(ImportState::WaitingForImportQueueToCatchUp {
+						block_iter,
+						delay,
+						block,
+					});
+				} else {
+					// Queue is no longer full, so we can add our block to the queue.
+					import_block_to_queue(block, queue, force);
+					// Switch back to Reading state.
+					state = Some(ImportState::Reading { block_iter });
+				}
+			},
+			ImportState::WaitingForImportQueueToFinish {
+				num_expected_blocks,
+				read_block_count,
+				mut delay,
+			} => {
+				// All the blocks have been added to the queue, which doesn't mean they
+				// have all been properly imported.
+				if importing_is_done(
+					num_expected_blocks,
+					read_block_count,
+					link.imported_blocks.load(Ordering::Acquire),
+				) {
+					// Importing is done, we can log the result and return.
+					info!(
+						"🎉 Imported {} blocks. Best: #{}",
+						read_block_count,
+						client.info().best_number
+					);
+					return Poll::Ready(Ok(()));
+				} else {
+					// Importing is not done, we still have to wait for the queue to finish.
+					// Wait for the delay, because we know the queue is lagging behind.
+					match Pin::new(&mut delay).poll(cx) {
+						Poll::Pending => {
+							state = Some(ImportState::WaitingForImportQueueToFinish {
+								num_expected_blocks,
+								read_block_count,
+								delay,
+							});
+							return Poll::Pending;
+						},
+						Poll::Ready(_) => {
+							delay.reset(Duration::from_millis(DELAY_TIME));
+						},
+					}
+
+					state = Some(ImportState::WaitingForImportQueueToFinish {
+						num_expected_blocks,
+						read_block_count,
+						delay,
+					});
+				}
+			},
+		}
+
+		queue.poll_actions(cx, &mut link);
+
+		let best_number = client.info().best_number;
+		speedometer.notify_user(best_number);
+
+		if link.has_error.load(Ordering::Acquire) {
+			return Poll::Ready(Err(Error::Other(format!(
+				"Stopping after #{} blocks because of an error",
+				link.imported_blocks.load(Ordering::Acquire)
+			))));
+		}
+
+		cx.waker().wake_by_ref();
+		Poll::Pending
+	});
+	Box::pin(import)
+}

bizinikiwi/client/service/src/chain_ops/mod.rs

@@ -0,0 +1,31 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Chain utilities.
+
+mod check_block;
+mod export_blocks;
+mod export_raw_state;
+mod import_blocks;
+mod revert_chain;
+
+pub use check_block::*;
+pub use export_blocks::*;
+pub use export_raw_state::*;
+pub use import_blocks::*;
+pub use revert_chain::*;

bizinikiwi/client/service/src/chain_ops/revert_chain.rs

@@ -0,0 +1,52 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::error::Error;
+use log::info;
+use pezsc_client_api::{Backend, UsageProvider};
+use pezsp_runtime::traits::{Block as BlockT, NumberFor, Zero};
+use std::sync::Arc;
+
+/// Performs a revert of `blocks` blocks.
+pub fn revert_chain<B, BA, C>(
+	client: Arc<C>,
+	backend: Arc<BA>,
+	blocks: NumberFor<B>,
+) -> Result<(), Error>
+where
+	B: BlockT,
+	C: UsageProvider<B>,
+	BA: Backend<B>,
+{
+	let reverted = backend.revert(blocks, false)?;
+	let info = client.usage_info().chain;
+
+	if reverted.0.is_zero() {
+		info!("There aren't any non-finalized blocks to revert.");
+	} else {
+		info!("Reverted {} blocks. Best: #{} ({})", reverted.0, info.best_number, info.best_hash);
+
+		if reverted.0 > blocks {
+			info!(
+				"Number of reverted blocks is higher than requested \
+				because of reverted leaves higher than the best block."
+			)
+		}
+	}
+	Ok(())
+}

bizinikiwi/client/service/src/client/block_rules.rs

@@ -0,0 +1,74 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Client fixed chain specification rules
+
+use std::collections::{HashMap, HashSet};
+
+use pezsp_runtime::traits::{Block as BlockT, NumberFor};
+
+use pezsc_client_api::{BadBlocks, ForkBlocks};
+
+/// Chain specification rules lookup result.
+pub enum LookupResult<B: BlockT> {
+	/// Specification rules do not contain any special rules about this block
+	NotSpecial,
+	/// The block is known to be bad and should not be imported
+	KnownBad,
+	/// There is a specified canonical block hash for the given height
+	Expected(B::Hash),
+}
+
+/// Chain-specific block filtering rules.
+///
+/// This holds known bad blocks and known good forks, and
+/// is usually part of the chain spec.
+pub struct BlockRules<B: BlockT> {
+	bad: HashSet<B::Hash>,
+	forks: HashMap<NumberFor<B>, B::Hash>,
+}
+
+impl<B: BlockT> BlockRules<B> {
+	/// New block rules with provided black and white lists.
+	pub fn new(fork_blocks: ForkBlocks<B>, bad_blocks: BadBlocks<B>) -> Self {
+		Self {
+			bad: bad_blocks.unwrap_or_default(),
+			forks: fork_blocks.unwrap_or_default().into_iter().collect(),
+		}
+	}
+
+	/// Mark a new block as bad.
+	pub fn mark_bad(&mut self, hash: B::Hash) {
+		self.bad.insert(hash);
+	}
+
+	/// Check if there's any rule affecting the given block.
+	pub fn lookup(&self, number: NumberFor<B>, hash: &B::Hash) -> LookupResult<B> {
+		if let Some(hash_for_height) = self.forks.get(&number) {
+			if hash_for_height != hash {
+				return LookupResult::Expected(*hash_for_height);
+			}
+		}
+
+		if self.bad.contains(hash) {
+			return LookupResult::KnownBad;
+		}
+
+		LookupResult::NotSpecial
+	}
+}

bizinikiwi/client/service/src/client/call_executor.rs

@@ -0,0 +1,268 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use super::{code_provider::CodeProvider, ClientConfig};
+use pezsc_client_api::{
+	backend, call_executor::CallExecutor, execution_extensions::ExecutionExtensions, HeaderBackend,
+	TrieCacheContext,
+};
+use pezsc_executor::{RuntimeVersion, RuntimeVersionOf};
+use pezsp_api::ProofRecorder;
+use pezsp_core::traits::{CallContext, CodeExecutor};
+use pezsp_externalities::Extensions;
+use pezsp_runtime::{
+	generic::BlockId,
+	traits::{Block as BlockT, HashingFor},
+};
+use pezsp_state_machine::{backend::AsTrieBackend, OverlayedChanges, StateMachine, StorageProof};
+use std::{cell::RefCell, sync::Arc};
+
+/// Call executor that executes methods locally, querying all required
+/// data from local backend.
+pub struct LocalCallExecutor<Block: BlockT, B, E> {
+	backend: Arc<B>,
+	executor: E,
+	code_provider: CodeProvider<Block, B, E>,
+	execution_extensions: Arc<ExecutionExtensions<Block>>,
+}
+
+impl<Block: BlockT, B, E> LocalCallExecutor<Block, B, E>
+where
+	E: CodeExecutor + RuntimeVersionOf + Clone + 'static,
+	B: backend::Backend<Block>,
+{
+	/// Creates new instance of local call executor.
+	pub fn new(
+		backend: Arc<B>,
+		executor: E,
+		client_config: ClientConfig<Block>,
+		execution_extensions: ExecutionExtensions<Block>,
+	) -> pezsp_blockchain::Result<Self> {
+		let code_provider = CodeProvider::new(&client_config, executor.clone(), backend.clone())?;
+
+		Ok(LocalCallExecutor {
+			backend,
+			executor,
+			code_provider,
+			execution_extensions: Arc::new(execution_extensions),
+		})
+	}
+}
+
+impl<Block: BlockT, B, E> Clone for LocalCallExecutor<Block, B, E>
+where
+	E: Clone,
+{
+	fn clone(&self) -> Self {
+		LocalCallExecutor {
+			backend: self.backend.clone(),
+			executor: self.executor.clone(),
+			code_provider: self.code_provider.clone(),
+			execution_extensions: self.execution_extensions.clone(),
+		}
+	}
+}
+
+impl<B, E, Block> CallExecutor<Block> for LocalCallExecutor<Block, B, E>
+where
+	B: backend::Backend<Block>,
+	E: CodeExecutor + RuntimeVersionOf + Clone + 'static,
+	Block: BlockT,
+{
+	type Error = E::Error;
+
+	type Backend = B;
+
+	fn execution_extensions(&self) -> &ExecutionExtensions<Block> {
+		&self.execution_extensions
+	}
+
+	fn call(
+		&self,
+		at_hash: Block::Hash,
+		method: &str,
+		call_data: &[u8],
+		context: CallContext,
+	) -> pezsp_blockchain::Result<Vec<u8>> {
+		let mut changes = OverlayedChanges::default();
+		let at_number =
+			self.backend.blockchain().expect_block_number_from_id(&BlockId::Hash(at_hash))?;
+		let state = self.backend.state_at(at_hash, context.into())?;
+
+		let state_runtime_code = pezsp_state_machine::backend::BackendRuntimeCode::new(&state);
+		let runtime_code =
+			state_runtime_code.runtime_code().map_err(pezsp_blockchain::Error::RuntimeCode)?;
+
+		let runtime_code = self.code_provider.maybe_override_code(runtime_code, &state, at_hash)?.0;
+
+		let mut extensions = self.execution_extensions.extensions(at_hash, at_number);
+
+		let mut sm = StateMachine::new(
+			&state,
+			&mut changes,
+			&self.executor,
+			method,
+			call_data,
+			&mut extensions,
+			&runtime_code,
+			context,
+		)
+		.set_parent_hash(at_hash);
+
+		sm.execute().map_err(Into::into)
+	}
+
+	fn contextual_call(
+		&self,
+		at_hash: Block::Hash,
+		method: &str,
+		call_data: &[u8],
+		changes: &RefCell<OverlayedChanges<HashingFor<Block>>>,
+		recorder: &Option<ProofRecorder<Block>>,
+		call_context: CallContext,
+		extensions: &RefCell<Extensions>,
+	) -> Result<Vec<u8>, pezsp_blockchain::Error> {
+		let state = self.backend.state_at(at_hash, call_context.into())?;
+
+		let changes = &mut *changes.borrow_mut();
+
+		// It is important to extract the runtime code here before we create the proof
+		// recorder to not record it. We also need to fetch the runtime code from `state` to
+		// make sure we use the caching layers.
+		let state_runtime_code = pezsp_state_machine::backend::BackendRuntimeCode::new(&state);
+
+		let runtime_code =
+			state_runtime_code.runtime_code().map_err(pezsp_blockchain::Error::RuntimeCode)?;
+		let runtime_code = self.code_provider.maybe_override_code(runtime_code, &state, at_hash)?.0;
+		let mut extensions = extensions.borrow_mut();
+
+		match recorder {
+			Some(recorder) => {
+				let trie_state = state.as_trie_backend();
+
+				let backend = pezsp_state_machine::TrieBackendBuilder::wrap(&trie_state)
+					.with_recorder(recorder.clone())
+					.build();
+
+				let mut state_machine = StateMachine::new(
+					&backend,
+					changes,
+					&self.executor,
+					method,
+					call_data,
+					&mut extensions,
+					&runtime_code,
+					call_context,
+				)
+				.set_parent_hash(at_hash);
+				state_machine.execute()
+			},
+			None => {
+				let mut state_machine = StateMachine::new(
+					&state,
+					changes,
+					&self.executor,
+					method,
+					call_data,
+					&mut extensions,
+					&runtime_code,
+					call_context,
+				)
+				.set_parent_hash(at_hash);
+				state_machine.execute()
+			},
+		}
+		.map_err(Into::into)
+	}
+
+	fn runtime_version(&self, at_hash: Block::Hash) -> pezsp_blockchain::Result<RuntimeVersion> {
+		let state = self.backend.state_at(at_hash, backend::TrieCacheContext::Untrusted)?;
+		let state_runtime_code = pezsp_state_machine::backend::BackendRuntimeCode::new(&state);
+
+		let runtime_code =
+			state_runtime_code.runtime_code().map_err(pezsp_blockchain::Error::RuntimeCode)?;
+		self.code_provider
+			.maybe_override_code(runtime_code, &state, at_hash)
+			.map(|(_, v)| v)
+	}
+
+	fn prove_execution(
+		&self,
+		at_hash: Block::Hash,
+		method: &str,
+		call_data: &[u8],
+	) -> pezsp_blockchain::Result<(Vec<u8>, StorageProof)> {
+		let at_number =
+			self.backend.blockchain().expect_block_number_from_id(&BlockId::Hash(at_hash))?;
+		let state = self.backend.state_at(at_hash, TrieCacheContext::Untrusted)?;
+
+		let trie_backend = state.as_trie_backend();
+
+		let state_runtime_code = pezsp_state_machine::backend::BackendRuntimeCode::new(trie_backend);
+		let runtime_code =
+			state_runtime_code.runtime_code().map_err(pezsp_blockchain::Error::RuntimeCode)?;
+		let runtime_code = self.code_provider.maybe_override_code(runtime_code, &state, at_hash)?.0;
+
+		pezsp_state_machine::prove_execution_on_trie_backend(
+			trie_backend,
+			&mut Default::default(),
+			&self.executor,
+			method,
+			call_data,
+			&runtime_code,
+			&mut self.execution_extensions.extensions(at_hash, at_number),
+		)
+		.map_err(Into::into)
+	}
+}
+
+impl<B, E, Block> RuntimeVersionOf for LocalCallExecutor<Block, B, E>
+where
+	E: RuntimeVersionOf,
+	Block: BlockT,
+{
+	fn runtime_version(
+		&self,
+		ext: &mut dyn pezsp_externalities::Externalities,
+		runtime_code: &pezsp_core::traits::RuntimeCode,
+	) -> Result<pezsp_version::RuntimeVersion, pezsc_executor::error::Error> {
+		RuntimeVersionOf::runtime_version(&self.executor, ext, runtime_code)
+	}
+}
+
+impl<Block, B, E> pezsp_version::GetRuntimeVersionAt<Block> for LocalCallExecutor<Block, B, E>
+where
+	B: backend::Backend<Block>,
+	E: CodeExecutor + RuntimeVersionOf + Clone + 'static,
+	Block: BlockT,
+{
+	fn runtime_version(&self, at: Block::Hash) -> Result<pezsp_version::RuntimeVersion, String> {
+		CallExecutor::runtime_version(self, at).map_err(|e| e.to_string())
+	}
+}
+
+impl<Block, B, E> pezsp_version::GetNativeVersion for LocalCallExecutor<Block, B, E>
+where
+	B: backend::Backend<Block>,
+	E: CodeExecutor + pezsp_version::GetNativeVersion + Clone + 'static,
+	Block: BlockT,
+{
+	fn native_version(&self) -> &pezsp_version::NativeVersion {
+		self.executor.native_version()
+	}
+}

bizinikiwi/client/service/src/client/code_provider.rs

@@ -0,0 +1,352 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use super::{client::ClientConfig, wasm_override::WasmOverride, wasm_substitutes::WasmSubstitutes};
+use pezsc_client_api::{backend, TrieCacheContext};
+use pezsc_executor::{RuntimeVersion, RuntimeVersionOf};
+use pezsp_core::traits::{FetchRuntimeCode, RuntimeCode};
+use pezsp_runtime::traits::Block as BlockT;
+use pezsp_state_machine::{Ext, OverlayedChanges};
+use std::sync::Arc;
+
+/// Provider for fetching `:code` of a block.
+///
+/// As a node can run with code overrides or substitutes, this will ensure that these are taken into
+/// account before returning the actual `code` for a block.
+pub struct CodeProvider<Block: BlockT, Backend, Executor> {
+	backend: Arc<Backend>,
+	executor: Arc<Executor>,
+	wasm_override: Arc<Option<WasmOverride>>,
+	wasm_substitutes: WasmSubstitutes<Block, Executor, Backend>,
+}
+
+impl<Block: BlockT, Backend, Executor: Clone> Clone for CodeProvider<Block, Backend, Executor> {
+	fn clone(&self) -> Self {
+		Self {
+			backend: self.backend.clone(),
+			executor: self.executor.clone(),
+			wasm_override: self.wasm_override.clone(),
+			wasm_substitutes: self.wasm_substitutes.clone(),
+		}
+	}
+}
+
+impl<Block, Backend, Executor> CodeProvider<Block, Backend, Executor>
+where
+	Block: BlockT,
+	Backend: backend::Backend<Block>,
+	Executor: RuntimeVersionOf,
+{
+	/// Create a new instance.
+	pub fn new(
+		client_config: &ClientConfig<Block>,
+		executor: Executor,
+		backend: Arc<Backend>,
+	) -> pezsp_blockchain::Result<Self> {
+		let wasm_override = client_config
+			.wasm_runtime_overrides
+			.as_ref()
+			.map(|p| WasmOverride::new(p.clone(), &executor))
+			.transpose()?;
+
+		let executor = Arc::new(executor);
+
+		let wasm_substitutes = WasmSubstitutes::new(
+			client_config.wasm_runtime_substitutes.clone(),
+			executor.clone(),
+			backend.clone(),
+		)?;
+
+		Ok(Self { backend, executor, wasm_override: Arc::new(wasm_override), wasm_substitutes })
+	}
+
+	/// Returns the `:code` for the given `block`.
+	///
+	/// This takes into account potential overrides/substitutes.
+	pub fn code_at_ignoring_overrides(&self, block: Block::Hash) -> pezsp_blockchain::Result<Vec<u8>> {
+		let state = self.backend.state_at(block, TrieCacheContext::Untrusted)?;
+
+		let state_runtime_code = pezsp_state_machine::backend::BackendRuntimeCode::new(&state);
+		let runtime_code =
+			state_runtime_code.runtime_code().map_err(pezsp_blockchain::Error::RuntimeCode)?;
+
+		self.maybe_override_code_internal(runtime_code, &state, block, true)
+			.and_then(|r| {
+				r.0.fetch_runtime_code().map(Into::into).ok_or_else(|| {
+					pezsp_blockchain::Error::Backend("Could not find `:code` in backend.".into())
+				})
+			})
+	}
+
+	/// Maybe override the given `onchain_code`.
+	///
+	/// This takes into account potential overrides/substitutes.
+	pub fn maybe_override_code<'a>(
+		&'a self,
+		onchain_code: RuntimeCode<'a>,
+		state: &Backend::State,
+		hash: Block::Hash,
+	) -> pezsp_blockchain::Result<(RuntimeCode<'a>, RuntimeVersion)> {
+		self.maybe_override_code_internal(onchain_code, state, hash, false)
+	}
+
+	/// Maybe override the given `onchain_code`.
+	///
+	/// This takes into account potential overrides(depending on `ignore_overrides`)/substitutes.
+	fn maybe_override_code_internal<'a>(
+		&'a self,
+		onchain_code: RuntimeCode<'a>,
+		state: &Backend::State,
+		hash: Block::Hash,
+		ignore_overrides: bool,
+	) -> pezsp_blockchain::Result<(RuntimeCode<'a>, RuntimeVersion)> {
+		let on_chain_version = self.on_chain_runtime_version(&onchain_code, state)?;
+		let code_and_version = if let Some(d) = self.wasm_override.as_ref().as_ref().and_then(|o| {
+			if ignore_overrides {
+				return None;
+			}
+
+			o.get(
+				&on_chain_version.spec_version,
+				onchain_code.heap_pages,
+				&on_chain_version.spec_name,
+			)
+		}) {
+			tracing::debug!(target: "code-provider::overrides", block = ?hash, "using WASM override");
+			d
+		} else if let Some(s) =
+			self.wasm_substitutes
+				.get(on_chain_version.spec_version, onchain_code.heap_pages, hash)
+		{
+			tracing::debug!(target: "code-provider::substitutes", block = ?hash, "Using WASM substitute");
+			s
+		} else {
+			tracing::debug!(
+				target: "code-provider",
+				block = ?hash,
+				"Neither WASM override nor substitute available, using onchain code",
+			);
+			(onchain_code, on_chain_version)
+		};
+
+		Ok(code_and_version)
+	}
+
+	/// Returns the on chain runtime version.
+	fn on_chain_runtime_version(
+		&self,
+		code: &RuntimeCode,
+		state: &Backend::State,
+	) -> pezsp_blockchain::Result<RuntimeVersion> {
+		let mut overlay = OverlayedChanges::default();
+
+		let mut ext = Ext::new(&mut overlay, state, None);
+
+		self.executor
+			.runtime_version(&mut ext, code)
+			.map_err(|e| pezsp_blockchain::Error::VersionInvalid(e.to_string()))
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use backend::Backend;
+	use pezsc_client_api::{in_mem, HeaderBackend};
+	use pezsc_executor::WasmExecutor;
+	use pezsp_core::{
+		testing::TaskExecutor,
+		traits::{FetchRuntimeCode, WrappedRuntimeCode},
+	};
+	use std::collections::HashMap;
+	use bizinikiwi_test_runtime_client::{runtime, GenesisInit};
+
+	#[test]
+	fn no_override_no_substitutes_work() {
+		let executor = WasmExecutor::default();
+
+		let code_fetcher = WrappedRuntimeCode(bizinikiwi_test_runtime::wasm_binary_unwrap().into());
+		let onchain_code = RuntimeCode {
+			code_fetcher: &code_fetcher,
+			heap_pages: Some(128),
+			hash: vec![0, 0, 0, 0],
+		};
+
+		let backend = Arc::new(in_mem::Backend::<runtime::Block>::new());
+
+		// wasm_runtime_overrides is `None` here because we construct the
+		// LocalCallExecutor directly later on
+		let client_config = ClientConfig::default();
+
+		let genesis_block_builder = crate::GenesisBlockBuilder::new(
+			&bizinikiwi_test_runtime_client::GenesisParameters::default().genesis_storage(),
+			!client_config.no_genesis,
+			backend.clone(),
+			executor.clone(),
+		)
+		.expect("Creates genesis block builder");
+
+		// client is used for the convenience of creating and inserting the genesis block.
+		let _client =
+			crate::client::new_with_backend::<_, _, runtime::Block, _, runtime::RuntimeApi>(
+				backend.clone(),
+				executor.clone(),
+				genesis_block_builder,
+				Box::new(TaskExecutor::new()),
+				None,
+				None,
+				client_config.clone(),
+			)
+			.expect("Creates a client");
+
+		let executor = Arc::new(executor);
+
+		let code_provider = CodeProvider {
+			backend: backend.clone(),
+			executor: executor.clone(),
+			wasm_override: Arc::new(None),
+			wasm_substitutes: WasmSubstitutes::new(Default::default(), executor, backend.clone())
+				.unwrap(),
+		};
+
+		let check = code_provider
+			.maybe_override_code(
+				onchain_code,
+				&backend
+					.state_at(backend.blockchain().info().genesis_hash, TrieCacheContext::Untrusted)
+					.unwrap(),
+				backend.blockchain().info().genesis_hash,
+			)
+			.expect("RuntimeCode override")
+			.0;
+
+		assert_eq!(code_fetcher.fetch_runtime_code(), check.fetch_runtime_code());
+	}
+
+	#[test]
+	fn should_get_override_if_exists() {
+		let executor = WasmExecutor::default();
+
+		let overrides = crate::client::wasm_override::dummy_overrides();
+		let onchain_code = WrappedRuntimeCode(bizinikiwi_test_runtime::wasm_binary_unwrap().into());
+		let onchain_code = RuntimeCode {
+			code_fetcher: &onchain_code,
+			heap_pages: Some(128),
+			hash: vec![0, 0, 0, 0],
+		};
+
+		let backend = Arc::new(in_mem::Backend::<runtime::Block>::new());
+
+		// wasm_runtime_overrides is `None` here because we construct the
+		// LocalCallExecutor directly later on
+		let client_config = ClientConfig::default();
+
+		let genesis_block_builder = crate::GenesisBlockBuilder::new(
+			&bizinikiwi_test_runtime_client::GenesisParameters::default().genesis_storage(),
+			!client_config.no_genesis,
+			backend.clone(),
+			executor.clone(),
+		)
+		.expect("Creates genesis block builder");
+
+		// client is used for the convenience of creating and inserting the genesis block.
+		let _client =
+			crate::client::new_with_backend::<_, _, runtime::Block, _, runtime::RuntimeApi>(
+				backend.clone(),
+				executor.clone(),
+				genesis_block_builder,
+				Box::new(TaskExecutor::new()),
+				None,
+				None,
+				client_config.clone(),
+			)
+			.expect("Creates a client");
+
+		let executor = Arc::new(executor);
+
+		let code_provider = CodeProvider {
+			backend: backend.clone(),
+			executor: executor.clone(),
+			wasm_override: Arc::new(Some(overrides)),
+			wasm_substitutes: WasmSubstitutes::new(Default::default(), executor, backend.clone())
+				.unwrap(),
+		};
+
+		let check = code_provider
+			.maybe_override_code(
+				onchain_code,
+				&backend
+					.state_at(backend.blockchain().info().genesis_hash, TrieCacheContext::Untrusted)
+					.unwrap(),
+				backend.blockchain().info().genesis_hash,
+			)
+			.expect("RuntimeCode override")
+			.0;
+
+		assert_eq!(Some(vec![2, 2, 2, 2, 2, 2, 2, 2]), check.fetch_runtime_code().map(Into::into));
+	}
+
+	#[test]
+	fn returns_runtime_version_from_substitute() {
+		const SUBSTITUTE_SPEC_NAME: &str = "substitute-spec-name-cool";
+
+		let executor = WasmExecutor::default();
+
+		let backend = Arc::new(in_mem::Backend::<runtime::Block>::new());
+
+		// Let's only override the `spec_name` for our testing purposes.
+		let substitute = pezsp_version::embed::embed_runtime_version(
+			&bizinikiwi_test_runtime::WASM_BINARY_BLOATY.unwrap(),
+			pezsp_version::RuntimeVersion {
+				spec_name: SUBSTITUTE_SPEC_NAME.into(),
+				..bizinikiwi_test_runtime::VERSION
+			},
+		)
+		.unwrap();
+
+		let client_config = crate::client::ClientConfig {
+			wasm_runtime_substitutes: vec![(0, substitute)].into_iter().collect::<HashMap<_, _>>(),
+			..Default::default()
+		};
+
+		let genesis_block_builder = crate::GenesisBlockBuilder::new(
+			&bizinikiwi_test_runtime_client::GenesisParameters::default().genesis_storage(),
+			!client_config.no_genesis,
+			backend.clone(),
+			executor.clone(),
+		)
+		.expect("Creates genesis block builder");
+
+		// client is used for the convenience of creating and inserting the genesis block.
+		let client =
+			crate::client::new_with_backend::<_, _, runtime::Block, _, runtime::RuntimeApi>(
+				backend.clone(),
+				executor.clone(),
+				genesis_block_builder,
+				Box::new(TaskExecutor::new()),
+				None,
+				None,
+				client_config,
+			)
+			.expect("Creates a client");
+
+		let version = client.runtime_version_at(client.chain_info().genesis_hash).unwrap();
+
+		assert_eq!(SUBSTITUTE_SPEC_NAME, &*version.spec_name);
+	}
+}

bizinikiwi/client/service/src/client/mod.rs

@@ -0,0 +1,59 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Bizinikiwi Client and associated logic.
+//!
+//! The [`Client`] is one of the most important components of Bizinikiwi. It mainly comprises two
+//! parts:
+//!
+//! - A database containing the blocks and chain state, generally referred to as
+//! the [`Backend`](pezsc_client_api::backend::Backend).
+//! - A runtime environment, generally referred to as the
+//! [`Executor`](pezsc_client_api::call_executor::CallExecutor).
+//!
+//! # Initialization
+//!
+//! Creating a [`Client`] is done by calling the `new` method and passing to it a
+//! [`Backend`](pezsc_client_api::backend::Backend) and an
+//! [`Executor`](pezsc_client_api::call_executor::CallExecutor).
+//!
+//! The former is typically provided by the `sc-client-db` crate.
+//!
+//! The latter typically requires passing one of:
+//!
+//! - A [`LocalCallExecutor`] running the runtime locally.
+//! - A `RemoteCallExecutor` that will ask a third-party to perform the executions.
+//! - A `RemoteOrLocalCallExecutor` combination of the two.
+//!
+//! Additionally, the fourth generic parameter of the `Client` is a marker type representing
+//! the ways in which the runtime can interface with the outside. Any code that builds a `Client`
+//! is responsible for putting the right marker.
+
+mod block_rules;
+mod call_executor;
+mod client;
+mod code_provider;
+mod notification_pinning;
+mod wasm_override;
+mod wasm_substitutes;
+
+pub use call_executor::LocalCallExecutor;
+pub use client::{Client, ClientConfig};
+pub(crate) use code_provider::CodeProvider;
+
+pub use self::client::new_with_backend;

bizinikiwi/client/service/src/client/notification_pinning.rs

@@ -0,0 +1,353 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Notification pinning related logic.
+//!
+//! This file contains a worker that should be started when a new client instance is created.
+//! The goal is to avoid pruning of blocks that have active notifications in the node. Every
+//! recipient of notifications should receive the chance to act upon them. In addition, notification
+//! listeners can hold onto a [`pezsc_client_api::UnpinHandle`] to keep a block pinned. Once the handle
+//! is dropped, a message is sent and the worker unpins the respective block.
+use std::{
+	marker::PhantomData,
+	sync::{Arc, Weak},
+};
+
+use futures::StreamExt;
+use pezsc_client_api::{Backend, UnpinWorkerMessage};
+
+use pezsc_utils::mpsc::TracingUnboundedReceiver;
+use schnellru::Limiter;
+use pezsp_runtime::traits::Block as BlockT;
+
+const LOG_TARGET: &str = "db::notification_pinning";
+const NOTIFICATION_PINNING_LIMIT: usize = 1024;
+
+/// A limiter which automatically unpins blocks that leave the data structure.
+#[derive(Clone, Debug)]
+struct UnpinningByLengthLimiter<Block: BlockT, B: Backend<Block>> {
+	max_length: usize,
+	backend: Weak<B>,
+	_phantom: PhantomData<Block>,
+}
+
+impl<Block: BlockT, B: Backend<Block>> UnpinningByLengthLimiter<Block, B> {
+	/// Creates a new length limiter with a given `max_length`.
+	pub fn new(max_length: usize, backend: Weak<B>) -> UnpinningByLengthLimiter<Block, B> {
+		UnpinningByLengthLimiter { max_length, backend, _phantom: PhantomData::<Block>::default() }
+	}
+}
+
+impl<Block: BlockT, B: Backend<Block>> Limiter<Block::Hash, u32>
+	for UnpinningByLengthLimiter<Block, B>
+{
+	type KeyToInsert<'a> = Block::Hash;
+	type LinkType = usize;
+
+	fn is_over_the_limit(&self, length: usize) -> bool {
+		length > self.max_length
+	}
+
+	fn on_insert(
+		&mut self,
+		_length: usize,
+		key: Self::KeyToInsert<'_>,
+		value: u32,
+	) -> Option<(Block::Hash, u32)> {
+		log::debug!(target: LOG_TARGET, "Pinning block based on notification. hash = {key}");
+		if self.max_length > 0 {
+			Some((key, value))
+		} else {
+			None
+		}
+	}
+
+	fn on_replace(
+		&mut self,
+		_length: usize,
+		_old_key: &mut Block::Hash,
+		_new_key: Block::Hash,
+		_old_value: &mut u32,
+		_new_value: &mut u32,
+	) -> bool {
+		true
+	}
+
+	fn on_removed(&mut self, key: &mut Block::Hash, references: &mut u32) {
+		// If reference count was larger than 0 on removal,
+		// the item was removed due to capacity limitations.
+		// Since the cache should be large enough for pinned items,
+		// we want to know about these evictions.
+		if *references > 0 {
+			log::warn!(
+				target: LOG_TARGET,
+				"Notification block pinning limit reached. Unpinning block with hash = {key:?}"
+			);
+			if let Some(backend) = self.backend.upgrade() {
+				(0..*references).for_each(|_| backend.unpin_block(*key));
+			}
+		} else {
+			log::trace!(
+				target: LOG_TARGET,
+				"Unpinned block. hash = {key:?}",
+			)
+		}
+	}
+
+	fn on_cleared(&mut self) {}
+
+	fn on_grow(&mut self, _new_memory_usage: usize) -> bool {
+		true
+	}
+}
+
+/// Worker for the handling of notification pinning.
+///
+/// It receives messages from a receiver and pins/unpins based on the incoming messages.
+/// All notification related unpinning should go through this worker. If the maximum number of
+/// notification pins is reached, the block from the oldest notification is unpinned.
+pub struct NotificationPinningWorker<Block: BlockT, Back: Backend<Block>> {
+	unpin_message_rx: TracingUnboundedReceiver<UnpinWorkerMessage<Block>>,
+	task_backend: Weak<Back>,
+	pinned_blocks: schnellru::LruMap<Block::Hash, u32, UnpinningByLengthLimiter<Block, Back>>,
+}
+
+impl<Block: BlockT, Back: Backend<Block>> NotificationPinningWorker<Block, Back> {
+	/// Creates a new `NotificationPinningWorker`.
+	pub fn new(
+		unpin_message_rx: TracingUnboundedReceiver<UnpinWorkerMessage<Block>>,
+		task_backend: Arc<Back>,
+	) -> Self {
+		let pinned_blocks =
+			schnellru::LruMap::<Block::Hash, u32, UnpinningByLengthLimiter<Block, Back>>::new(
+				UnpinningByLengthLimiter::new(
+					NOTIFICATION_PINNING_LIMIT,
+					Arc::downgrade(&task_backend),
+				),
+			);
+		Self { unpin_message_rx, task_backend: Arc::downgrade(&task_backend), pinned_blocks }
+	}
+
+	fn handle_announce_message(&mut self, hash: Block::Hash) {
+		if let Some(entry) = self.pinned_blocks.get_or_insert(hash, Default::default) {
+			*entry = *entry + 1;
+		}
+	}
+
+	fn handle_unpin_message(&mut self, hash: Block::Hash) -> Result<(), ()> {
+		if let Some(refcount) = self.pinned_blocks.peek_mut(&hash) {
+			*refcount = *refcount - 1;
+			if *refcount == 0 {
+				self.pinned_blocks.remove(&hash);
+			}
+			if let Some(backend) = self.task_backend.upgrade() {
+				log::debug!(target: LOG_TARGET, "Reducing pinning refcount for block hash = {hash:?}");
+				backend.unpin_block(hash);
+			} else {
+				log::debug!(target: LOG_TARGET, "Terminating unpin-worker, backend reference was dropped.");
+				return Err(());
+			}
+		} else {
+			log::debug!(target: LOG_TARGET, "Received unpin message for already unpinned block. hash = {hash:?}");
+		}
+		Ok(())
+	}
+
+	/// Start working on the received messages.
+	///
+	/// The worker maintains a map which keeps track of the pinned blocks and their reference count.
+	/// Depending upon the received message, it acts to pin/unpin the block.
+	pub async fn run(mut self) {
+		while let Some(message) = self.unpin_message_rx.next().await {
+			match message {
+				UnpinWorkerMessage::AnnouncePin(hash) => self.handle_announce_message(hash),
+				UnpinWorkerMessage::Unpin(hash) =>
+					if self.handle_unpin_message(hash).is_err() {
+						return;
+					},
+			}
+		}
+		log::debug!(target: LOG_TARGET, "Terminating unpin-worker, stream terminated.")
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use std::sync::Arc;
+
+	use pezsc_client_api::{Backend, UnpinWorkerMessage};
+	use pezsc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver};
+	use pezsp_core::H256;
+	use pezsp_runtime::traits::Block as BlockT;
+
+	type Block = bizinikiwi_test_runtime_client::runtime::Block;
+
+	use super::{NotificationPinningWorker, UnpinningByLengthLimiter};
+
+	impl<Block: BlockT, Back: Backend<Block>> NotificationPinningWorker<Block, Back> {
+		fn new_with_limit(
+			unpin_message_rx: TracingUnboundedReceiver<UnpinWorkerMessage<Block>>,
+			task_backend: Arc<Back>,
+			limit: usize,
+		) -> Self {
+			let pinned_blocks =
+				schnellru::LruMap::<Block::Hash, u32, UnpinningByLengthLimiter<Block, Back>>::new(
+					UnpinningByLengthLimiter::new(limit, Arc::downgrade(&task_backend)),
+				);
+			Self { unpin_message_rx, task_backend: Arc::downgrade(&task_backend), pinned_blocks }
+		}
+
+		fn lru(
+			&self,
+		) -> &schnellru::LruMap<Block::Hash, u32, UnpinningByLengthLimiter<Block, Back>> {
+			&self.pinned_blocks
+		}
+	}
+
+	#[test]
+	fn pinning_worker_handles_base_case() {
+		let (_tx, rx) = tracing_unbounded("testing", 1000);
+
+		let backend = Arc::new(pezsc_client_api::in_mem::Backend::<Block>::new());
+
+		let hash = H256::random();
+
+		let mut worker = NotificationPinningWorker::new(rx, backend.clone());
+
+		// Block got pinned and unpin message should unpin in the backend.
+		let _ = backend.pin_block(hash);
+		assert_eq!(backend.pin_refs(&hash), Some(1));
+
+		worker.handle_announce_message(hash);
+		assert_eq!(worker.lru().len(), 1);
+
+		let _ = worker.handle_unpin_message(hash);
+
+		assert_eq!(backend.pin_refs(&hash), Some(0));
+		assert!(worker.lru().is_empty());
+	}
+
+	#[test]
+	fn pinning_worker_handles_multiple_pins() {
+		let (_tx, rx) = tracing_unbounded("testing", 1000);
+
+		let backend = Arc::new(pezsc_client_api::in_mem::Backend::<Block>::new());
+
+		let hash = H256::random();
+
+		let mut worker = NotificationPinningWorker::new(rx, backend.clone());
+		// Block got pinned multiple times.
+		let _ = backend.pin_block(hash);
+		let _ = backend.pin_block(hash);
+		let _ = backend.pin_block(hash);
+		assert_eq!(backend.pin_refs(&hash), Some(3));
+
+		worker.handle_announce_message(hash);
+		worker.handle_announce_message(hash);
+		worker.handle_announce_message(hash);
+		assert_eq!(worker.lru().len(), 1);
+
+		let _ = worker.handle_unpin_message(hash);
+		assert_eq!(backend.pin_refs(&hash), Some(2));
+		let _ = worker.handle_unpin_message(hash);
+		assert_eq!(backend.pin_refs(&hash), Some(1));
+		let _ = worker.handle_unpin_message(hash);
+		assert_eq!(backend.pin_refs(&hash), Some(0));
+		assert!(worker.lru().is_empty());
+
+		let _ = worker.handle_unpin_message(hash);
+		assert_eq!(backend.pin_refs(&hash), Some(0));
+	}
+
+	#[test]
+	fn pinning_worker_handles_too_many_unpins() {
+		let (_tx, rx) = tracing_unbounded("testing", 1000);
+
+		let backend = Arc::new(pezsc_client_api::in_mem::Backend::<Block>::new());
+
+		let hash = H256::random();
+		let hash2 = H256::random();
+
+		let mut worker = NotificationPinningWorker::new(rx, backend.clone());
+		// Block was announced once but unpinned multiple times. The worker should ignore the
+		// additional unpins.
+		let _ = backend.pin_block(hash);
+		let _ = backend.pin_block(hash);
+		let _ = backend.pin_block(hash);
+		assert_eq!(backend.pin_refs(&hash), Some(3));
+
+		worker.handle_announce_message(hash);
+		assert_eq!(worker.lru().len(), 1);
+
+		let _ = worker.handle_unpin_message(hash);
+		assert_eq!(backend.pin_refs(&hash), Some(2));
+		let _ = worker.handle_unpin_message(hash);
+		assert_eq!(backend.pin_refs(&hash), Some(2));
+		assert!(worker.lru().is_empty());
+
+		let _ = worker.handle_unpin_message(hash2);
+		assert!(worker.lru().is_empty());
+		assert_eq!(backend.pin_refs(&hash2), None);
+	}
+
+	#[test]
+	fn pinning_worker_should_evict_when_limit_reached() {
+		let (_tx, rx) = tracing_unbounded("testing", 1000);
+
+		let backend = Arc::new(pezsc_client_api::in_mem::Backend::<Block>::new());
+
+		let hash1 = H256::random();
+		let hash2 = H256::random();
+		let hash3 = H256::random();
+		let hash4 = H256::random();
+
+		// Only two items fit into the cache.
+		let mut worker = NotificationPinningWorker::new_with_limit(rx, backend.clone(), 2);
+
+		// Multiple blocks are announced but the cache size is too small. We expect that blocks
+		// are evicted by the cache and unpinned in the backend.
+		let _ = backend.pin_block(hash1);
+		let _ = backend.pin_block(hash2);
+		let _ = backend.pin_block(hash3);
+		assert_eq!(backend.pin_refs(&hash1), Some(1));
+		assert_eq!(backend.pin_refs(&hash2), Some(1));
+		assert_eq!(backend.pin_refs(&hash3), Some(1));
+
+		worker.handle_announce_message(hash1);
+		assert!(worker.lru().peek(&hash1).is_some());
+		worker.handle_announce_message(hash2);
+		assert!(worker.lru().peek(&hash2).is_some());
+		worker.handle_announce_message(hash3);
+		assert!(worker.lru().peek(&hash3).is_some());
+		assert!(worker.lru().peek(&hash2).is_some());
+		assert_eq!(worker.lru().len(), 2);
+
+		// Hash 1 should have gotten unpinned, since its oldest.
+		assert_eq!(backend.pin_refs(&hash1), Some(0));
+		assert_eq!(backend.pin_refs(&hash2), Some(1));
+		assert_eq!(backend.pin_refs(&hash3), Some(1));
+
+		// Hash 2 is getting bumped.
+		worker.handle_announce_message(hash2);
+		assert_eq!(worker.lru().peek(&hash2), Some(&2));
+
+		// Since hash 2 was accessed, evict hash 3.
+		worker.handle_announce_message(hash4);
+		assert_eq!(worker.lru().peek(&hash3), None);
+	}
+}

bizinikiwi/client/service/src/client/wasm_override.rs

@@ -0,0 +1,347 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! # WASM Local Blob-Override
+//!
+//! WASM Local blob override provides tools to replace on-chain WASM with custom WASM.
+//! These customized WASM blobs may include functionality that is not included in the
+//! on-chain WASM, such as tracing or debugging information. This extra information is especially
+//! useful in external scenarios, like exchanges or archive nodes.
+//!
+//! ## Usage
+//!
+//! WASM overrides may be enabled with the `--wasm-runtime-overrides` argument. The argument
+//! expects a path to a directory that holds custom WASM.
+//!
+//! Any file ending in '.wasm' will be scraped and instantiated as a WASM blob. WASM can be built by
+//! compiling the required runtime with the changes needed. For example, compiling a runtime with
+//! tracing enabled would produce a WASM blob that can used.
+//!
+//! A custom WASM blob will override on-chain WASM if the spec version matches. If it is
+//! required to overrides multiple runtimes, multiple WASM blobs matching each of the spec versions
+//! needed must be provided in the given directory.
+
+use pezsc_executor::RuntimeVersionOf;
+use pezsp_blockchain::Result;
+use pezsp_core::traits::{FetchRuntimeCode, RuntimeCode, WrappedRuntimeCode};
+use pezsp_state_machine::BasicExternalities;
+use pezsp_version::RuntimeVersion;
+use std::{
+	collections::{hash_map::DefaultHasher, HashMap},
+	fs,
+	hash::Hasher as _,
+	path::{Path, PathBuf},
+	time::{Duration, Instant},
+};
+
+/// The interval in that we will print a warning when a wasm blob `spec_name`
+/// doesn't match with the on-chain `spec_name`.
+const WARN_INTERVAL: Duration = Duration::from_secs(30);
+
+/// Auxiliary structure that holds a wasm blob and its hash.
+#[derive(Debug)]
+struct WasmBlob {
+	/// The actual wasm blob, aka the code.
+	code: Vec<u8>,
+	/// The hash of [`Self::code`].
+	hash: Vec<u8>,
+	/// The path where this blob was found.
+	path: PathBuf,
+	/// The runtime version of this blob.
+	version: RuntimeVersion,
+	/// When was the last time we have warned about the wasm blob having
+	/// a wrong `spec_name`?
+	last_warn: parking_lot::Mutex<Option<Instant>>,
+}
+
+impl WasmBlob {
+	fn new(code: Vec<u8>, hash: Vec<u8>, path: PathBuf, version: RuntimeVersion) -> Self {
+		Self { code, hash, path, version, last_warn: Default::default() }
+	}
+
+	fn runtime_code(&self, heap_pages: Option<u64>) -> RuntimeCode<'_> {
+		RuntimeCode { code_fetcher: self, hash: self.hash.clone(), heap_pages }
+	}
+}
+
+/// Make a hash out of a byte string using the default rust hasher
+fn make_hash<K: std::hash::Hash + ?Sized>(val: &K) -> Vec<u8> {
+	let mut state = DefaultHasher::new();
+	val.hash(&mut state);
+	state.finish().to_le_bytes().to_vec()
+}
+
+impl FetchRuntimeCode for WasmBlob {
+	fn fetch_runtime_code(&self) -> Option<std::borrow::Cow<'_, [u8]>> {
+		Some(self.code.as_slice().into())
+	}
+}
+
+#[derive(Debug, thiserror::Error)]
+#[allow(missing_docs)]
+pub enum WasmOverrideError {
+	#[error("Failed to get runtime version: {0}")]
+	VersionInvalid(String),
+
+	#[error("WASM override IO error")]
+	Io(PathBuf, #[source] std::io::Error),
+
+	#[error("Overwriting WASM requires a directory where local \
+	WASM is stored. {} is not a directory", .0.display())]
+	NotADirectory(PathBuf),
+
+	#[error("Duplicate WASM Runtimes found: \n{}\n", .0.join("\n") )]
+	DuplicateRuntime(Vec<String>),
+}
+
+impl From<WasmOverrideError> for pezsp_blockchain::Error {
+	fn from(err: WasmOverrideError) -> Self {
+		Self::Application(Box::new(err))
+	}
+}
+
+/// Scrapes WASM from a folder and returns WASM from that folder
+/// if the runtime spec version matches.
+#[derive(Debug)]
+pub struct WasmOverride {
+	// Map of runtime spec version -> Wasm Blob
+	overrides: HashMap<u32, WasmBlob>,
+}
+
+impl WasmOverride {
+	pub fn new<P, E>(path: P, executor: &E) -> Result<Self>
+	where
+		P: AsRef<Path>,
+		E: RuntimeVersionOf,
+	{
+		let overrides = Self::scrape_overrides(path.as_ref(), executor)?;
+		Ok(Self { overrides })
+	}
+
+	/// Gets an override by it's runtime spec version.
+	///
+	/// Returns `None` if an override for a spec version does not exist.
+	pub fn get<'a, 'b: 'a>(
+		&'b self,
+		spec: &u32,
+		pages: Option<u64>,
+		spec_name: &str,
+	) -> Option<(RuntimeCode<'a>, RuntimeVersion)> {
+		self.overrides.get(spec).and_then(|w| {
+			if spec_name == &*w.version.spec_name {
+				Some((w.runtime_code(pages), w.version.clone()))
+			} else {
+				let mut last_warn = w.last_warn.lock();
+				let now = Instant::now();
+
+				if last_warn.map_or(true, |l| l + WARN_INTERVAL <= now) {
+					*last_warn = Some(now);
+
+					tracing::warn!(
+						target = "wasm_overrides",
+						on_chain_spec_name = %spec_name,
+						override_spec_name = %w.version,
+						spec_version = %spec,
+						wasm_file = %w.path.display(),
+						"On chain and override `spec_name` do not match! Ignoring override.",
+					);
+				}
+
+				None
+			}
+		})
+	}
+
+	/// Scrapes a folder for WASM runtimes.
+	/// Returns a hashmap of the runtime version and wasm runtime code.
+	fn scrape_overrides<E>(dir: &Path, executor: &E) -> Result<HashMap<u32, WasmBlob>>
+	where
+		E: RuntimeVersionOf,
+	{
+		let handle_err = |e: std::io::Error| -> pezsp_blockchain::Error {
+			WasmOverrideError::Io(dir.to_owned(), e).into()
+		};
+
+		if !dir.is_dir() {
+			return Err(WasmOverrideError::NotADirectory(dir.to_owned()).into());
+		}
+
+		let mut overrides = HashMap::new();
+		let mut duplicates = Vec::new();
+		for entry in fs::read_dir(dir).map_err(handle_err)? {
+			let entry = entry.map_err(handle_err)?;
+			let path = entry.path();
+			if let Some("wasm") = path.extension().and_then(|e| e.to_str()) {
+				let code = fs::read(&path).map_err(handle_err)?;
+				let code_hash = make_hash(&code);
+				let version = Self::runtime_version(executor, &code, &code_hash, Some(128))?;
+				tracing::info!(
+					target: "wasm_overrides",
+					version = %version,
+					file = %path.display(),
+					"Found wasm override.",
+				);
+
+				let wasm = WasmBlob::new(code, code_hash, path.clone(), version.clone());
+
+				if let Some(other) = overrides.insert(version.spec_version, wasm) {
+					tracing::info!(
+						target: "wasm_overrides",
+						first = %other.path.display(),
+						second = %path.display(),
+						%version,
+						"Found duplicate spec version for runtime.",
+					);
+					duplicates.push(path.display().to_string());
+				}
+			}
+		}
+
+		if !duplicates.is_empty() {
+			return Err(WasmOverrideError::DuplicateRuntime(duplicates).into());
+		}
+
+		Ok(overrides)
+	}
+
+	fn runtime_version<E>(
+		executor: &E,
+		code: &[u8],
+		code_hash: &[u8],
+		heap_pages: Option<u64>,
+	) -> Result<RuntimeVersion>
+	where
+		E: RuntimeVersionOf,
+	{
+		let mut ext = BasicExternalities::default();
+		executor
+			.runtime_version(
+				&mut ext,
+				&RuntimeCode {
+					code_fetcher: &WrappedRuntimeCode(code.into()),
+					heap_pages,
+					hash: code_hash.into(),
+				},
+			)
+			.map_err(|e| WasmOverrideError::VersionInvalid(e.to_string()).into())
+	}
+}
+
+/// Returns a WasmOverride struct filled with dummy data for testing.
+#[cfg(test)]
+pub fn dummy_overrides() -> WasmOverride {
+	let version = RuntimeVersion { spec_name: "test".into(), ..Default::default() };
+	let mut overrides = HashMap::new();
+	overrides.insert(
+		0,
+		WasmBlob::new(vec![0, 0, 0, 0, 0, 0, 0, 0], vec![0], PathBuf::new(), version.clone()),
+	);
+	overrides.insert(
+		1,
+		WasmBlob::new(vec![1, 1, 1, 1, 1, 1, 1, 1], vec![1], PathBuf::new(), version.clone()),
+	);
+	overrides
+		.insert(2, WasmBlob::new(vec![2, 2, 2, 2, 2, 2, 2, 2], vec![2], PathBuf::new(), version));
+
+	WasmOverride { overrides }
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use pezsc_executor::{HeapAllocStrategy, WasmExecutor};
+	use std::fs::{self, File};
+
+	fn executor() -> WasmExecutor {
+		WasmExecutor::builder()
+			.with_onchain_heap_alloc_strategy(HeapAllocStrategy::Static { extra_pages: 128 })
+			.with_offchain_heap_alloc_strategy(HeapAllocStrategy::Static { extra_pages: 128 })
+			.with_max_runtime_instances(1)
+			.with_runtime_cache_size(2)
+			.build()
+	}
+
+	fn wasm_test<F>(fun: F)
+	where
+		F: Fn(&Path, &[u8], &WasmExecutor),
+	{
+		let exec = executor();
+		let bytes = bizinikiwi_test_runtime::wasm_binary_unwrap();
+		let dir = tempfile::tempdir().expect("Create a temporary directory");
+		fun(dir.path(), bytes, &exec);
+		dir.close().expect("Temporary Directory should close");
+	}
+
+	#[test]
+	fn should_get_runtime_version() {
+		let executor = executor();
+
+		let version = WasmOverride::runtime_version(
+			&executor,
+			bizinikiwi_test_runtime::wasm_binary_unwrap(),
+			&[1],
+			Some(128),
+		)
+		.expect("should get the `RuntimeVersion` of the test-runtime wasm blob");
+		assert_eq!(version.spec_version, 2);
+	}
+
+	#[test]
+	fn should_scrape_wasm() {
+		wasm_test(|dir, wasm_bytes, exec| {
+			fs::write(dir.join("test.wasm"), wasm_bytes).expect("Create test file");
+			let overrides =
+				WasmOverride::scrape_overrides(dir, exec).expect("HashMap of u32 and WasmBlob");
+			let wasm = overrides.get(&2).expect("WASM binary");
+			assert_eq!(wasm.code, bizinikiwi_test_runtime::wasm_binary_unwrap().to_vec())
+		});
+	}
+
+	#[test]
+	fn should_check_for_duplicates() {
+		wasm_test(|dir, wasm_bytes, exec| {
+			fs::write(dir.join("test0.wasm"), wasm_bytes).expect("Create test file");
+			fs::write(dir.join("test1.wasm"), wasm_bytes).expect("Create test file");
+			let scraped = WasmOverride::scrape_overrides(dir, exec);
+
+			match scraped {
+				Err(pezsp_blockchain::Error::Application(e)) => {
+					match e.downcast_ref::<WasmOverrideError>() {
+						Some(WasmOverrideError::DuplicateRuntime(duplicates)) => {
+							assert_eq!(duplicates.len(), 1);
+						},
+						_ => panic!("Test should end with Msg Error Variant"),
+					}
+				},
+				_ => panic!("Test should end in error"),
+			}
+		});
+	}
+
+	#[test]
+	fn should_ignore_non_wasm() {
+		wasm_test(|dir, wasm_bytes, exec| {
+			File::create(dir.join("README.md")).expect("Create test file");
+			File::create(dir.join("LICENSE")).expect("Create a test file");
+			fs::write(dir.join("test0.wasm"), wasm_bytes).expect("Create test file");
+			let scraped =
+				WasmOverride::scrape_overrides(dir, exec).expect("HashMap of u32 and WasmBlob");
+			assert_eq!(scraped.len(), 1);
+		});
+	}
+}

bizinikiwi/client/service/src/client/wasm_substitutes.rs

@@ -0,0 +1,163 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! # WASM substitutes
+
+use pezsc_client_api::backend;
+use pezsc_executor::RuntimeVersionOf;
+use pezsp_blockchain::{HeaderBackend, Result};
+use pezsp_core::traits::{FetchRuntimeCode, RuntimeCode, WrappedRuntimeCode};
+use pezsp_runtime::traits::{Block as BlockT, NumberFor};
+use pezsp_state_machine::BasicExternalities;
+use pezsp_version::RuntimeVersion;
+use std::{
+	collections::{hash_map::DefaultHasher, HashMap},
+	hash::Hasher as _,
+	sync::Arc,
+};
+
+/// A wasm substitute for the on chain wasm.
+#[derive(Debug)]
+struct WasmSubstitute<Block: BlockT> {
+	code: Vec<u8>,
+	hash: Vec<u8>,
+	/// The block number on which we should start using the substitute.
+	block_number: NumberFor<Block>,
+	version: RuntimeVersion,
+}
+
+impl<Block: BlockT> WasmSubstitute<Block> {
+	fn new(code: Vec<u8>, block_number: NumberFor<Block>, version: RuntimeVersion) -> Self {
+		let hash = make_hash(&code);
+		Self { code, hash, block_number, version }
+	}
+
+	fn runtime_code(&self, heap_pages: Option<u64>) -> RuntimeCode<'_> {
+		RuntimeCode { code_fetcher: self, hash: self.hash.clone(), heap_pages }
+	}
+
+	/// Returns `true` when the substitute matches for the given `hash`.
+	fn matches(
+		&self,
+		hash: <Block as BlockT>::Hash,
+		backend: &impl backend::Backend<Block>,
+	) -> bool {
+		let requested_block_number = backend.blockchain().number(hash).ok().flatten();
+
+		Some(self.block_number) <= requested_block_number
+	}
+}
+
+/// Make a hash out of a byte string using the default rust hasher
+fn make_hash<K: std::hash::Hash + ?Sized>(val: &K) -> Vec<u8> {
+	let mut state = DefaultHasher::new();
+	val.hash(&mut state);
+	state.finish().to_le_bytes().to_vec()
+}
+
+impl<Block: BlockT> FetchRuntimeCode for WasmSubstitute<Block> {
+	fn fetch_runtime_code(&self) -> Option<std::borrow::Cow<'_, [u8]>> {
+		Some(self.code.as_slice().into())
+	}
+}
+
+#[derive(Debug, thiserror::Error)]
+#[allow(missing_docs)]
+pub enum WasmSubstituteError {
+	#[error("Failed to get runtime version: {0}")]
+	VersionInvalid(String),
+}
+
+impl From<WasmSubstituteError> for pezsp_blockchain::Error {
+	fn from(err: WasmSubstituteError) -> Self {
+		Self::Application(Box::new(err))
+	}
+}
+
+/// Substitutes the on-chain wasm with some hard coded blobs.
+#[derive(Debug)]
+pub struct WasmSubstitutes<Block: BlockT, Executor, Backend> {
+	/// spec_version -> WasmSubstitute
+	substitutes: Arc<HashMap<u32, WasmSubstitute<Block>>>,
+	executor: Arc<Executor>,
+	backend: Arc<Backend>,
+}
+
+impl<Block: BlockT, Executor: Clone, Backend> Clone for WasmSubstitutes<Block, Executor, Backend> {
+	fn clone(&self) -> Self {
+		Self {
+			substitutes: self.substitutes.clone(),
+			executor: self.executor.clone(),
+			backend: self.backend.clone(),
+		}
+	}
+}
+
+impl<Executor, Backend, Block> WasmSubstitutes<Block, Executor, Backend>
+where
+	Executor: RuntimeVersionOf,
+	Backend: backend::Backend<Block>,
+	Block: BlockT,
+{
+	/// Create a new instance.
+	pub fn new(
+		substitutes: HashMap<NumberFor<Block>, Vec<u8>>,
+		executor: Arc<Executor>,
+		backend: Arc<Backend>,
+	) -> Result<Self> {
+		let substitutes = substitutes
+			.into_iter()
+			.map(|(block_number, code)| {
+				let runtime_code = RuntimeCode {
+					code_fetcher: &WrappedRuntimeCode((&code).into()),
+					heap_pages: None,
+					hash: make_hash(&code),
+				};
+				let version = Self::runtime_version(&executor, &runtime_code)?;
+				let spec_version = version.spec_version;
+
+				let substitute = WasmSubstitute::new(code, block_number, version);
+
+				Ok((spec_version, substitute))
+			})
+			.collect::<Result<HashMap<_, _>>>()?;
+
+		Ok(Self { executor, substitutes: Arc::new(substitutes), backend })
+	}
+
+	/// Get a substitute.
+	///
+	/// Returns `None` if there isn't any substitute required.
+	pub fn get(
+		&self,
+		spec: u32,
+		pages: Option<u64>,
+		hash: Block::Hash,
+	) -> Option<(RuntimeCode<'_>, RuntimeVersion)> {
+		let s = self.substitutes.get(&spec)?;
+		s.matches(hash, &*self.backend)
+			.then(|| (s.runtime_code(pages), s.version.clone()))
+	}
+
+	fn runtime_version(executor: &Executor, code: &RuntimeCode) -> Result<RuntimeVersion> {
+		let mut ext = BasicExternalities::default();
+		executor
+			.runtime_version(&mut ext, code)
+			.map_err(|e| WasmSubstituteError::VersionInvalid(e.to_string()).into())
+	}
+}

bizinikiwi/client/service/src/config.rs

@@ -0,0 +1,371 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Service configuration.
+
+pub use jsonrpsee::server::BatchRequestConfig as RpcBatchRequestConfig;
+use prometheus_endpoint::Registry;
+use pezsc_chain_spec::ChainSpec;
+pub use pezsc_client_db::{BlocksPruning, Database, DatabaseSource, PruningMode};
+pub use pezsc_executor::{WasmExecutionMethod, WasmtimeInstantiationStrategy};
+pub use pezsc_network::{
+	config::{
+		MultiaddrWithPeerId, NetworkConfiguration, NodeKeyConfig, NonDefaultSetConfig, ProtocolId,
+		Role, SetConfig, SyncMode, TransportConfig,
+	},
+	request_responses::{
+		IncomingRequest, OutgoingResponse, ProtocolConfig as RequestResponseConfig,
+	},
+	Multiaddr,
+};
+pub use pezsc_rpc_server::{
+	IpNetwork, RpcEndpoint, RpcMethods, SubscriptionIdProvider as RpcSubscriptionIdProvider,
+};
+pub use pezsc_telemetry::TelemetryEndpoints;
+pub use pezsc_transaction_pool::TransactionPoolOptions;
+use pezsp_core::crypto::SecretString;
+use std::{
+	io, iter,
+	net::SocketAddr,
+	num::NonZeroU32,
+	path::{Path, PathBuf},
+};
+use tempfile::TempDir;
+
+/// Service configuration.
+#[derive(Debug)]
+pub struct Configuration {
+	/// Implementation name
+	pub impl_name: String,
+	/// Implementation version (see sc-cli to see an example of format)
+	pub impl_version: String,
+	/// Node role.
+	pub role: Role,
+	/// Handle to the tokio runtime. Will be used to spawn futures by the task manager.
+	pub tokio_handle: tokio::runtime::Handle,
+	/// Extrinsic pool configuration.
+	pub transaction_pool: TransactionPoolOptions,
+	/// Network configuration.
+	pub network: NetworkConfiguration,
+	/// Configuration for the keystore.
+	pub keystore: KeystoreConfig,
+	/// Configuration for the database.
+	pub database: DatabaseSource,
+	/// Maximum size of internal trie cache in bytes.
+	///
+	/// If `None` is given the cache is disabled.
+	pub trie_cache_maximum_size: Option<usize>,
+	/// Force the trie cache to be in memory.
+	pub warm_up_trie_cache: Option<TrieCacheWarmUpStrategy>,
+	/// State pruning settings.
+	pub state_pruning: Option<PruningMode>,
+	/// Number of blocks to keep in the db.
+	///
+	/// NOTE: only finalized blocks are subject for removal!
+	pub blocks_pruning: BlocksPruning,
+	/// Chain configuration.
+	pub chain_spec: Box<dyn ChainSpec>,
+	/// Runtime executor configuration.
+	pub executor: ExecutorConfiguration,
+	/// Directory where local WASM runtimes live. These runtimes take precedence
+	/// over on-chain runtimes when the spec version matches. Set to `None` to
+	/// disable overrides (default).
+	pub wasm_runtime_overrides: Option<PathBuf>,
+	/// RPC configuration.
+	pub rpc: RpcConfiguration,
+	/// Prometheus endpoint configuration. `None` if disabled.
+	pub prometheus_config: Option<PrometheusConfig>,
+	/// Telemetry service URL. `None` if disabled.
+	pub telemetry_endpoints: Option<TelemetryEndpoints>,
+	/// Should offchain workers be executed.
+	pub offchain_worker: OffchainWorkerConfig,
+	/// Enable authoring even when offline.
+	pub force_authoring: bool,
+	/// Disable GRANDPA when running in validator mode
+	pub disable_grandpa: bool,
+	/// Development key seed.
+	///
+	/// When running in development mode, the seed will be used to generate authority keys by the
+	/// keystore.
+	///
+	/// Should only be set when `node` is running development mode.
+	pub dev_key_seed: Option<String>,
+	/// Tracing targets
+	pub tracing_targets: Option<String>,
+	/// Tracing receiver
+	pub tracing_receiver: pezsc_tracing::TracingReceiver,
+	/// Announce block automatically after they have been imported
+	pub announce_block: bool,
+	/// Data path root for the configured chain.
+	pub data_path: PathBuf,
+	/// Base path of the configuration. This is shared between chains.
+	pub base_path: BasePath,
+}
+
+/// Warmup strategy for the trie cache.
+#[derive(Debug, Clone, Copy)]
+pub enum TrieCacheWarmUpStrategy {
+	/// Warm up the cache in a non-blocking way.
+	NonBlocking,
+	/// Warm up the cache in a blocking way.
+	Blocking,
+}
+
+impl TrieCacheWarmUpStrategy {
+	/// Returns true if the warmup strategy is blocking.
+	pub(crate) fn is_blocking(&self) -> bool {
+		matches!(self, Self::Blocking)
+	}
+}
+
+/// Type for tasks spawned by the executor.
+#[derive(PartialEq)]
+pub enum TaskType {
+	/// Regular non-blocking futures. Polling the task is expected to be a lightweight operation.
+	Async,
+	/// The task might perform a lot of expensive CPU operations and/or call `thread::sleep`.
+	Blocking,
+}
+
+/// Configuration of the client keystore.
+#[derive(Debug, Clone)]
+pub enum KeystoreConfig {
+	/// Keystore at a path on-disk. Recommended for native nodes.
+	Path {
+		/// The path of the keystore.
+		path: PathBuf,
+		/// Node keystore's password.
+		password: Option<SecretString>,
+	},
+	/// In-memory keystore. Recommended for in-browser nodes.
+	InMemory,
+}
+
+impl KeystoreConfig {
+	/// Returns the path for the keystore.
+	pub fn path(&self) -> Option<&Path> {
+		match self {
+			Self::Path { path, .. } => Some(path),
+			Self::InMemory => None,
+		}
+	}
+}
+/// Configuration of the database of the client.
+#[derive(Debug, Clone, Default)]
+pub struct OffchainWorkerConfig {
+	/// If this is allowed.
+	pub enabled: bool,
+	/// allow writes from the runtime to the offchain worker database.
+	pub indexing_enabled: bool,
+}
+
+/// Configuration of the Prometheus endpoint.
+#[derive(Debug, Clone)]
+pub struct PrometheusConfig {
+	/// Port to use.
+	pub port: SocketAddr,
+	/// A metrics registry to use. Useful for setting the metric prefix.
+	pub registry: Registry,
+}
+
+impl PrometheusConfig {
+	/// Create a new config using the default registry.
+	pub fn new_with_default_registry(port: SocketAddr, chain_id: String) -> Self {
+		let param = iter::once((String::from("chain"), chain_id)).collect();
+		Self {
+			port,
+			registry: Registry::new_custom(None, Some(param))
+				.expect("this can only fail if the prefix is empty"),
+		}
+	}
+}
+
+impl Configuration {
+	/// Returns a string displaying the node role.
+	pub fn display_role(&self) -> String {
+		self.role.to_string()
+	}
+
+	/// Returns the prometheus metrics registry, if available.
+	pub fn prometheus_registry(&self) -> Option<&Registry> {
+		self.prometheus_config.as_ref().map(|config| &config.registry)
+	}
+
+	/// Returns the network protocol id from the chain spec, or the default.
+	pub fn protocol_id(&self) -> ProtocolId {
+		let protocol_id_full = match self.chain_spec.protocol_id() {
+			Some(pid) => pid,
+			None => {
+				log::warn!(
+					"Using default protocol ID {:?} because none is configured in the \
+					chain specs",
+					crate::DEFAULT_PROTOCOL_ID
+				);
+				crate::DEFAULT_PROTOCOL_ID
+			},
+		};
+		ProtocolId::from(protocol_id_full)
+	}
+
+	/// Returns true if the genesis state writing will be skipped while initializing the genesis
+	/// block.
+	pub fn no_genesis(&self) -> bool {
+		matches!(self.network.sync_mode, SyncMode::LightState { .. } | SyncMode::Warp { .. })
+	}
+
+	/// Returns the database config for creating the backend.
+	pub fn db_config(&self) -> pezsc_client_db::DatabaseSettings {
+		pezsc_client_db::DatabaseSettings {
+			trie_cache_maximum_size: self.trie_cache_maximum_size,
+			state_pruning: self.state_pruning.clone(),
+			source: self.database.clone(),
+			blocks_pruning: self.blocks_pruning,
+			metrics_registry: self.prometheus_registry().cloned(),
+		}
+	}
+}
+
+#[static_init::dynamic(drop, lazy)]
+static mut BASE_PATH_TEMP: Option<TempDir> = None;
+
+/// The base path that is used for everything that needs to be written on disk to run a node.
+#[derive(Clone, Debug)]
+pub struct BasePath {
+	path: PathBuf,
+}
+
+impl BasePath {
+	/// Create a `BasePath` instance using a temporary directory prefixed with "bizinikiwi" and use
+	/// it as base path.
+	///
+	/// Note: The temporary directory will be created automatically and deleted when the program
+	/// exits. Every call to this function will return the same path for the lifetime of the
+	/// program.
+	pub fn new_temp_dir() -> io::Result<BasePath> {
+		let mut temp = BASE_PATH_TEMP.write();
+
+		match &*temp {
+			Some(p) => Ok(Self::new(p.path())),
+			None => {
+				let temp_dir = tempfile::Builder::new().prefix("bizinikiwi").tempdir()?;
+				let path = PathBuf::from(temp_dir.path());
+
+				*temp = Some(temp_dir);
+				Ok(Self::new(path))
+			},
+		}
+	}
+
+	/// Create a `BasePath` instance based on an existing path on disk.
+	///
+	/// Note: this function will not ensure that the directory exist nor create the directory. It
+	/// will also not delete the directory when the instance is dropped.
+	pub fn new<P: Into<PathBuf>>(path: P) -> BasePath {
+		Self { path: path.into() }
+	}
+
+	/// Create a base path from values describing the project.
+	pub fn from_project(qualifier: &str, organization: &str, application: &str) -> BasePath {
+		BasePath::new(
+			directories::ProjectDirs::from(qualifier, organization, application)
+				.expect("app directories exist on all supported platforms; qed")
+				.data_local_dir(),
+		)
+	}
+
+	/// Retrieve the base path.
+	pub fn path(&self) -> &Path {
+		&self.path
+	}
+
+	/// Returns the configuration directory inside this base path.
+	///
+	/// The path looks like `$base_path/chains/$chain_id`
+	pub fn config_dir(&self, chain_id: &str) -> PathBuf {
+		self.path().join("chains").join(chain_id)
+	}
+}
+
+impl From<PathBuf> for BasePath {
+	fn from(path: PathBuf) -> Self {
+		BasePath::new(path)
+	}
+}
+
+/// RPC configuration.
+#[derive(Debug)]
+pub struct RpcConfiguration {
+	/// JSON-RPC server endpoints.
+	pub addr: Option<Vec<RpcEndpoint>>,
+	/// Maximum number of connections for JSON-RPC server.
+	pub max_connections: u32,
+	/// CORS settings for HTTP & WS servers. `None` if all origins are allowed.
+	pub cors: Option<Vec<String>>,
+	/// RPC methods to expose (by default only a safe subset or all of them).
+	pub methods: RpcMethods,
+	/// Maximum payload of a rpc request
+	pub max_request_size: u32,
+	/// Maximum payload of a rpc response.
+	pub max_response_size: u32,
+	/// Custom JSON-RPC subscription ID provider.
+	///
+	/// Default: [`crate::RandomStringSubscriptionId`].
+	pub id_provider: Option<Box<dyn RpcSubscriptionIdProvider>>,
+	/// Maximum allowed subscriptions per rpc connection
+	pub max_subs_per_conn: u32,
+	/// JSON-RPC server default port.
+	pub port: u16,
+	/// The number of messages the JSON-RPC server is allowed to keep in memory.
+	pub message_buffer_capacity: u32,
+	/// JSON-RPC server batch config.
+	pub batch_config: RpcBatchRequestConfig,
+	/// RPC rate limit per minute.
+	pub rate_limit: Option<NonZeroU32>,
+	/// RPC rate limit whitelisted ip addresses.
+	pub rate_limit_whitelisted_ips: Vec<IpNetwork>,
+	/// RPC rate limit trust proxy headers.
+	pub rate_limit_trust_proxy_headers: bool,
+	/// RPC logger capacity (default: 1024).
+	pub request_logger_limit: u32,
+}
+
+/// Runtime executor configuration.
+#[derive(Debug, Clone)]
+pub struct ExecutorConfiguration {
+	/// Wasm execution method.
+	pub wasm_method: WasmExecutionMethod,
+	/// The size of the instances cache.
+	///
+	/// The default value is 8.
+	pub max_runtime_instances: usize,
+	/// The default number of 64KB pages to allocate for Wasm execution
+	pub default_heap_pages: Option<u64>,
+	/// Maximum number of different runtime versions that can be cached.
+	pub runtime_cache_size: u8,
+}
+
+impl Default for ExecutorConfiguration {
+	fn default() -> Self {
+		Self {
+			wasm_method: WasmExecutionMethod::default(),
+			max_runtime_instances: 8,
+			default_heap_pages: None,
+			runtime_cache_size: 2,
+		}
+	}
+}

bizinikiwi/client/service/src/error.rs

@@ -0,0 +1,77 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Errors that can occur during the service operation.
+
+use pezsc_keystore;
+use pezsp_blockchain;
+use pezsp_consensus;
+
+/// Service Result typedef.
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// Service errors.
+#[derive(Debug, thiserror::Error)]
+#[allow(missing_docs)]
+#[non_exhaustive]
+pub enum Error {
+	#[error(transparent)]
+	Client(#[from] pezsp_blockchain::Error),
+
+	#[error(transparent)]
+	Io(#[from] std::io::Error),
+
+	#[error(transparent)]
+	Consensus(#[from] pezsp_consensus::Error),
+
+	#[error(transparent)]
+	Network(#[from] pezsc_network::error::Error),
+
+	#[error(transparent)]
+	Keystore(#[from] pezsc_keystore::Error),
+
+	#[error(transparent)]
+	Telemetry(#[from] pezsc_telemetry::Error),
+
+	#[error("Best chain selection strategy (SelectChain) is not provided.")]
+	SelectChainRequired,
+
+	#[error("Tasks executor hasn't been provided.")]
+	TaskExecutorRequired,
+
+	#[error("Prometheus metrics error: {0}")]
+	Prometheus(#[from] prometheus_endpoint::PrometheusError),
+
+	#[error("Application: {0}")]
+	Application(#[from] Box<dyn std::error::Error + Send + Sync + 'static>),
+
+	#[error("Other: {0}")]
+	Other(String),
+}
+
+impl<'a> From<&'a str> for Error {
+	fn from(s: &'a str) -> Self {
+		Error::Other(s.into())
+	}
+}
+
+impl From<String> for Error {
+	fn from(s: String) -> Self {
+		Error::Other(s)
+	}
+}

bizinikiwi/client/service/src/lib.rs

@@ -0,0 +1,620 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Bizinikiwi service. Starts a thread that spins up the network, client, and extrinsic pool.
+//! Manages communication between them.
+
+#![warn(missing_docs)]
+#![recursion_limit = "1024"]
+
+pub mod chain_ops;
+pub mod client;
+pub mod config;
+pub mod error;
+
+mod builder;
+mod metrics;
+mod task_manager;
+
+use crate::config::Multiaddr;
+use std::{
+	collections::HashMap,
+	net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6},
+};
+
+use codec::{Decode, Encode};
+use futures::{pin_mut, FutureExt, StreamExt};
+use jsonrpsee::RpcModule;
+use log::{debug, error, trace, warn};
+use pezsc_client_api::{blockchain::HeaderBackend, BlockBackend, BlockchainEvents, ProofProvider};
+use pezsc_network::{
+	config::MultiaddrWithPeerId, service::traits::NetworkService, NetworkBackend, NetworkBlock,
+	NetworkPeers, NetworkStateInfo,
+};
+use pezsc_network_sync::SyncingService;
+use pezsc_network_types::PeerId;
+use pezsc_rpc_server::Server;
+use pezsc_utils::mpsc::TracingUnboundedReceiver;
+use pezsp_blockchain::HeaderMetadata;
+use pezsp_consensus::SyncOracle;
+use pezsp_runtime::traits::{Block as BlockT, Header as HeaderT};
+
+pub use self::{
+	builder::{
+		build_default_block_downloader, build_default_syncing_engine, build_network,
+		build_network_advanced, build_pezkuwi_syncing_strategy, gen_rpc_module, init_telemetry,
+		new_client, new_db_backend, new_full_client, new_full_parts, new_full_parts_record_import,
+		new_full_parts_with_genesis_builder, new_wasm_executor,
+		propagate_transaction_notifications, spawn_tasks, BuildNetworkAdvancedParams,
+		BuildNetworkParams, DefaultSyncingEngineConfig, KeystoreContainer, SpawnTasksParams,
+		TFullBackend, TFullCallExecutor, TFullClient,
+	},
+	client::{ClientConfig, LocalCallExecutor},
+	error::Error,
+	metrics::MetricsService,
+};
+#[allow(deprecated)]
+pub use builder::new_native_or_wasm_executor;
+
+pub use pezsc_chain_spec::{
+	construct_genesis_block, resolve_state_version_from_wasm, BuildGenesisBlock,
+	GenesisBlockBuilder,
+};
+
+pub use config::{
+	BasePath, BlocksPruning, Configuration, DatabaseSource, PruningMode, Role, RpcMethods, TaskType,
+};
+pub use pezsc_chain_spec::{
+	ChainSpec, ChainType, Extension as ChainSpecExtension, GenericChainSpec, NoExtension,
+	Properties,
+};
+
+use crate::config::RpcConfiguration;
+use prometheus_endpoint::Registry;
+pub use pezsc_consensus::ImportQueue;
+pub use pezsc_executor::NativeExecutionDispatch;
+pub use pezsc_network_sync::WarpSyncConfig;
+#[doc(hidden)]
+pub use pezsc_network_transactions::config::{TransactionImport, TransactionImportFuture};
+pub use pezsc_rpc::{RandomIntegerSubscriptionId, RandomStringSubscriptionId};
+pub use pezsc_tracing::TracingReceiver;
+pub use pezsc_transaction_pool::TransactionPoolOptions;
+pub use pezsc_transaction_pool_api::{error::IntoPoolError, InPoolTransaction, TransactionPool};
+#[doc(hidden)]
+pub use std::{ops::Deref, result::Result, sync::Arc};
+pub use task_manager::{
+	SpawnEssentialTaskHandle, SpawnTaskHandle, Task, TaskManager, TaskRegistry, DEFAULT_GROUP_NAME,
+};
+use tokio::runtime::Handle;
+
+const DEFAULT_PROTOCOL_ID: &str = "sup";
+
+/// A running RPC service that can perform in-memory RPC queries.
+#[derive(Clone)]
+pub struct RpcHandlers {
+	// This is legacy and may be removed at some point, it was for WASM stuff before smoldot was a
+	// thing. https://github.com/pezkuwichain/kurdistan-sdk/issues/121#discussion_r1694971805
+	rpc_module: Arc<RpcModule<()>>,
+
+	// This can be used to introspect the port the RPC server is listening on. SDK consumers are
+	// depending on this and it should be supported even if in-memory query support is removed.
+	listen_addresses: Vec<Multiaddr>,
+}
+
+impl RpcHandlers {
+	/// Create PRC handlers instance.
+	pub fn new(rpc_module: Arc<RpcModule<()>>, listen_addresses: Vec<Multiaddr>) -> Self {
+		Self { rpc_module, listen_addresses }
+	}
+
+	/// Starts an RPC query.
+	///
+	/// The query is passed as a string and must be valid JSON-RPC request object.
+	///
+	/// Returns a response and a stream if the call successful, fails if the
+	/// query could not be decoded as a JSON-RPC request object.
+	///
+	/// If the request subscribes you to events, the `stream` can be used to
+	/// retrieve the events.
+	pub async fn rpc_query(
+		&self,
+		json_query: &str,
+	) -> Result<(String, tokio::sync::mpsc::Receiver<String>), serde_json::Error> {
+		// Because `tokio::sync::mpsc::channel` is used under the hood
+		// it will panic if it's set to usize::MAX.
+		//
+		// This limit is used to prevent panics and is large enough.
+		const TOKIO_MPSC_MAX_SIZE: usize = tokio::sync::Semaphore::MAX_PERMITS;
+
+		self.rpc_module.raw_json_request(json_query, TOKIO_MPSC_MAX_SIZE).await
+	}
+
+	/// Provides access to the underlying `RpcModule`
+	pub fn handle(&self) -> Arc<RpcModule<()>> {
+		self.rpc_module.clone()
+	}
+
+	/// Provides access to listen addresses
+	pub fn listen_addresses(&self) -> &[Multiaddr] {
+		&self.listen_addresses[..]
+	}
+}
+
+/// An incomplete set of chain components, but enough to run the chain ops subcommands.
+pub struct PartialComponents<Client, Backend, SelectChain, ImportQueue, TransactionPool, Other> {
+	/// A shared client instance.
+	pub client: Arc<Client>,
+	/// A shared backend instance.
+	pub backend: Arc<Backend>,
+	/// The chain task manager.
+	pub task_manager: TaskManager,
+	/// A keystore container instance.
+	pub keystore_container: KeystoreContainer,
+	/// A chain selection algorithm instance.
+	pub select_chain: SelectChain,
+	/// An import queue.
+	pub import_queue: ImportQueue,
+	/// A shared transaction pool.
+	pub transaction_pool: Arc<TransactionPool>,
+	/// Everything else that needs to be passed into the main build function.
+	pub other: Other,
+}
+
+/// Builds a future that continuously polls the network.
+async fn build_network_future<
+	B: BlockT,
+	C: BlockchainEvents<B>
+		+ HeaderBackend<B>
+		+ BlockBackend<B>
+		+ HeaderMetadata<B, Error = pezsp_blockchain::Error>
+		+ ProofProvider<B>
+		+ Send
+		+ Sync
+		+ 'static,
+	H: pezsc_network_common::ExHashT,
+	N: NetworkBackend<B, <B as BlockT>::Hash>,
+>(
+	network: N,
+	client: Arc<C>,
+	sync_service: Arc<SyncingService<B>>,
+	announce_imported_blocks: bool,
+) {
+	let mut imported_blocks_stream = client.import_notification_stream().fuse();
+
+	// Stream of finalized blocks reported by the client.
+	let mut finality_notification_stream = client.finality_notification_stream().fuse();
+
+	let network_run = network.run().fuse();
+	pin_mut!(network_run);
+
+	loop {
+		futures::select! {
+			// List of blocks that the client has imported.
+			notification = imported_blocks_stream.next() => {
+				let notification = match notification {
+					Some(n) => n,
+					// If this stream is shut down, that means the client has shut down, and the
+					// most appropriate thing to do for the network future is to shut down too.
+					None => {
+						debug!("Block import stream has terminated, shutting down the network future.");
+						return
+					},
+				};
+
+				if announce_imported_blocks {
+					sync_service.announce_block(notification.hash, None);
+				}
+
+				if notification.is_new_best {
+					sync_service.new_best_block_imported(
+						notification.hash,
+						*notification.header.number(),
+					);
+				}
+			}
+
+			// List of blocks that the client has finalized.
+			notification = finality_notification_stream.select_next_some() => {
+				sync_service.on_block_finalized(notification.hash, notification.header);
+			}
+
+			// Drive the network. Shut down the network future if `NetworkWorker` has terminated.
+			_ = network_run => {
+				debug!("`NetworkWorker` has terminated, shutting down the network future.");
+				return
+			}
+		}
+	}
+}
+
+/// Builds a future that processes system RPC requests.
+pub async fn build_system_rpc_future<
+	B: BlockT,
+	C: BlockchainEvents<B>
+		+ HeaderBackend<B>
+		+ BlockBackend<B>
+		+ HeaderMetadata<B, Error = pezsp_blockchain::Error>
+		+ ProofProvider<B>
+		+ Send
+		+ Sync
+		+ 'static,
+	H: pezsc_network_common::ExHashT,
+>(
+	role: Role,
+	network_service: Arc<dyn NetworkService>,
+	sync_service: Arc<SyncingService<B>>,
+	client: Arc<C>,
+	mut rpc_rx: TracingUnboundedReceiver<pezsc_rpc::system::Request<B>>,
+	should_have_peers: bool,
+) {
+	// Current best block at initialization, to report to the RPC layer.
+	let starting_block = client.info().best_number;
+
+	loop {
+		// Answer incoming RPC requests.
+		let Some(req) = rpc_rx.next().await else {
+			debug!("RPC requests stream has terminated, shutting down the system RPC future.");
+			return;
+		};
+
+		match req {
+			pezsc_rpc::system::Request::Health(sender) => match sync_service.peers_info().await {
+				Ok(info) => {
+					let _ = sender.send(pezsc_rpc::system::Health {
+						peers: info.len(),
+						is_syncing: sync_service.is_major_syncing(),
+						should_have_peers,
+					});
+				},
+				Err(_) => log::error!("`SyncingEngine` shut down"),
+			},
+			pezsc_rpc::system::Request::LocalPeerId(sender) => {
+				let _ = sender.send(network_service.local_peer_id().to_base58());
+			},
+			pezsc_rpc::system::Request::LocalListenAddresses(sender) => {
+				let peer_id = (network_service.local_peer_id()).into();
+				let p2p_proto_suffix = pezsc_network::multiaddr::Protocol::P2p(peer_id);
+				let addresses = network_service
+					.listen_addresses()
+					.iter()
+					.map(|addr| addr.clone().with(p2p_proto_suffix.clone()).to_string())
+					.collect();
+				let _ = sender.send(addresses);
+			},
+			pezsc_rpc::system::Request::Peers(sender) => match sync_service.peers_info().await {
+				Ok(info) => {
+					let _ = sender.send(
+						info.into_iter()
+							.map(|(peer_id, p)| pezsc_rpc::system::PeerInfo {
+								peer_id: peer_id.to_base58(),
+								roles: format!("{:?}", p.roles),
+								best_hash: p.best_hash,
+								best_number: p.best_number,
+							})
+							.collect(),
+					);
+				},
+				Err(_) => log::error!("`SyncingEngine` shut down"),
+			},
+			pezsc_rpc::system::Request::NetworkState(sender) => {
+				let network_state = network_service.network_state().await;
+				if let Ok(network_state) = network_state {
+					if let Ok(network_state) = serde_json::to_value(network_state) {
+						let _ = sender.send(network_state);
+					}
+				} else {
+					break;
+				}
+			},
+			pezsc_rpc::system::Request::NetworkAddReservedPeer(peer_addr, sender) => {
+				let result = match MultiaddrWithPeerId::try_from(peer_addr) {
+					Ok(peer) => network_service.add_reserved_peer(peer),
+					Err(err) => Err(err.to_string()),
+				};
+				let x = result.map_err(pezsc_rpc::system::error::Error::MalformattedPeerArg);
+				let _ = sender.send(x);
+			},
+			pezsc_rpc::system::Request::NetworkRemoveReservedPeer(peer_id, sender) => {
+				let _ = match peer_id.parse::<PeerId>() {
+					Ok(peer_id) => {
+						network_service.remove_reserved_peer(peer_id);
+						sender.send(Ok(()))
+					},
+					Err(e) => sender.send(Err(pezsc_rpc::system::error::Error::MalformattedPeerArg(
+						e.to_string(),
+					))),
+				};
+			},
+			pezsc_rpc::system::Request::NetworkReservedPeers(sender) => {
+				let Ok(reserved_peers) = network_service.reserved_peers().await else {
+					break;
+				};
+
+				let _ =
+					sender.send(reserved_peers.iter().map(|peer_id| peer_id.to_base58()).collect());
+			},
+			pezsc_rpc::system::Request::NodeRoles(sender) => {
+				use pezsc_rpc::system::NodeRole;
+
+				let node_role = match role {
+					Role::Authority { .. } => NodeRole::Authority,
+					Role::Full => NodeRole::Full,
+				};
+
+				let _ = sender.send(vec![node_role]);
+			},
+			pezsc_rpc::system::Request::SyncState(sender) => {
+				use pezsc_rpc::system::SyncState;
+
+				match sync_service.status().await.map(|status| status.best_seen_block) {
+					Ok(best_seen_block) => {
+						let best_number = client.info().best_number;
+						let _ = sender.send(SyncState {
+							starting_block,
+							current_block: best_number,
+							highest_block: best_seen_block.unwrap_or(best_number),
+						});
+					},
+					Err(_) => log::error!("`SyncingEngine` shut down"),
+				}
+			},
+		}
+	}
+
+	debug!("`NetworkWorker` has terminated, shutting down the system RPC future.");
+}
+
+/// Starts RPC servers.
+pub fn start_rpc_servers<R>(
+	rpc_configuration: &RpcConfiguration,
+	registry: Option<&Registry>,
+	tokio_handle: &Handle,
+	gen_rpc_module: R,
+	rpc_id_provider: Option<Box<dyn pezsc_rpc_server::SubscriptionIdProvider>>,
+) -> Result<Server, error::Error>
+where
+	R: Fn() -> Result<RpcModule<()>, Error>,
+{
+	let endpoints: Vec<pezsc_rpc_server::RpcEndpoint> = if let Some(endpoints) =
+		rpc_configuration.addr.as_ref()
+	{
+		endpoints.clone()
+	} else {
+		let ipv6 =
+			SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::LOCALHOST, rpc_configuration.port, 0, 0));
+		let ipv4 = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, rpc_configuration.port));
+
+		vec![
+			pezsc_rpc_server::RpcEndpoint {
+				batch_config: rpc_configuration.batch_config,
+				cors: rpc_configuration.cors.clone(),
+				listen_addr: ipv4,
+				max_buffer_capacity_per_connection: rpc_configuration.message_buffer_capacity,
+				max_connections: rpc_configuration.max_connections,
+				max_payload_in_mb: rpc_configuration.max_request_size,
+				max_payload_out_mb: rpc_configuration.max_response_size,
+				max_subscriptions_per_connection: rpc_configuration.max_subs_per_conn,
+				rpc_methods: rpc_configuration.methods.into(),
+				rate_limit: rpc_configuration.rate_limit,
+				rate_limit_trust_proxy_headers: rpc_configuration.rate_limit_trust_proxy_headers,
+				rate_limit_whitelisted_ips: rpc_configuration.rate_limit_whitelisted_ips.clone(),
+				retry_random_port: true,
+				is_optional: false,
+			},
+			pezsc_rpc_server::RpcEndpoint {
+				batch_config: rpc_configuration.batch_config,
+				cors: rpc_configuration.cors.clone(),
+				listen_addr: ipv6,
+				max_buffer_capacity_per_connection: rpc_configuration.message_buffer_capacity,
+				max_connections: rpc_configuration.max_connections,
+				max_payload_in_mb: rpc_configuration.max_request_size,
+				max_payload_out_mb: rpc_configuration.max_response_size,
+				max_subscriptions_per_connection: rpc_configuration.max_subs_per_conn,
+				rpc_methods: rpc_configuration.methods.into(),
+				rate_limit: rpc_configuration.rate_limit,
+				rate_limit_trust_proxy_headers: rpc_configuration.rate_limit_trust_proxy_headers,
+				rate_limit_whitelisted_ips: rpc_configuration.rate_limit_whitelisted_ips.clone(),
+				retry_random_port: true,
+				is_optional: true,
+			},
+		]
+	};
+
+	let metrics = pezsc_rpc_server::RpcMetrics::new(registry)?;
+	let rpc_api = gen_rpc_module()?;
+
+	let server_config = pezsc_rpc_server::Config {
+		endpoints,
+		rpc_api,
+		metrics,
+		id_provider: rpc_id_provider,
+		tokio_handle: tokio_handle.clone(),
+		request_logger_limit: rpc_configuration.request_logger_limit,
+	};
+
+	// TODO: https://github.com/pezkuwichain/kurdistan-sdk/issues/12
+	//
+	// `block_in_place` is a hack to allow callers to call `block_on` prior to
+	// calling `start_rpc_servers`.
+	match tokio::task::block_in_place(|| {
+		tokio_handle.block_on(pezsc_rpc_server::start_server(server_config))
+	}) {
+		Ok(server) => Ok(server),
+		Err(e) => Err(Error::Application(e)),
+	}
+}
+
+/// Transaction pool adapter.
+pub struct TransactionPoolAdapter<C, P> {
+	pool: Arc<P>,
+	client: Arc<C>,
+}
+
+impl<C, P> TransactionPoolAdapter<C, P> {
+	/// Constructs a new instance of [`TransactionPoolAdapter`].
+	pub fn new(pool: Arc<P>, client: Arc<C>) -> Self {
+		Self { pool, client }
+	}
+}
+
+/// Get transactions for propagation.
+///
+/// Function extracted to simplify the test and prevent creating `ServiceFactory`.
+fn transactions_to_propagate<Pool, B, H, E>(pool: &Pool) -> Vec<(H, Arc<B::Extrinsic>)>
+where
+	Pool: TransactionPool<Block = B, Hash = H, Error = E>,
+	B: BlockT,
+	H: std::hash::Hash + Eq + pezsp_runtime::traits::Member + pezsp_runtime::traits::MaybeSerialize,
+	E: IntoPoolError + From<pezsc_transaction_pool_api::error::Error>,
+{
+	pool.ready()
+		.filter(|t| t.is_propagable())
+		.map(|t| {
+			let hash = t.hash().clone();
+			let ex = t.data().clone();
+			(hash, ex)
+		})
+		.collect()
+}
+
+impl<B, H, C, Pool, E> pezsc_network_transactions::config::TransactionPool<H, B>
+	for TransactionPoolAdapter<C, Pool>
+where
+	C: HeaderBackend<B>
+		+ BlockBackend<B>
+		+ HeaderMetadata<B, Error = pezsp_blockchain::Error>
+		+ ProofProvider<B>
+		+ Send
+		+ Sync
+		+ 'static,
+	Pool: 'static + TransactionPool<Block = B, Hash = H, Error = E>,
+	B: BlockT,
+	H: std::hash::Hash + Eq + pezsp_runtime::traits::Member + pezsp_runtime::traits::MaybeSerialize,
+	E: 'static + IntoPoolError + From<pezsc_transaction_pool_api::error::Error>,
+{
+	fn transactions(&self) -> Vec<(H, Arc<B::Extrinsic>)> {
+		transactions_to_propagate(&*self.pool)
+	}
+
+	fn hash_of(&self, transaction: &B::Extrinsic) -> H {
+		self.pool.hash_of(transaction)
+	}
+
+	fn import(&self, transaction: B::Extrinsic) -> TransactionImportFuture {
+		let encoded = transaction.encode();
+		let uxt = match Decode::decode(&mut &encoded[..]) {
+			Ok(uxt) => uxt,
+			Err(e) => {
+				debug!(target: pezsc_transaction_pool::LOG_TARGET, "Transaction invalid: {:?}", e);
+				return Box::pin(futures::future::ready(TransactionImport::Bad));
+			},
+		};
+
+		let start = std::time::Instant::now();
+		let pool = self.pool.clone();
+		let client = self.client.clone();
+		Box::pin(async move {
+			match pool
+				.submit_one(
+					client.info().best_hash,
+					pezsc_transaction_pool_api::TransactionSource::External,
+					uxt,
+				)
+				.await
+			{
+				Ok(_) => {
+					let elapsed = start.elapsed();
+					trace!(target: pezsc_transaction_pool::LOG_TARGET, "import transaction: {elapsed:?}");
+					TransactionImport::NewGood
+				},
+				Err(e) => match e.into_pool_error() {
+					Ok(pezsc_transaction_pool_api::error::Error::AlreadyImported(_)) =>
+						TransactionImport::KnownGood,
+					Ok(_) => TransactionImport::Bad,
+					Err(_) => {
+						// it is not bad at least, just some internal node logic error, so peer is
+						// innocent.
+						TransactionImport::KnownGood
+					},
+				},
+			}
+		})
+	}
+
+	fn on_broadcasted(&self, propagations: HashMap<H, Vec<String>>) {
+		self.pool.on_broadcasted(propagations)
+	}
+
+	fn transaction(&self, hash: &H) -> Option<Arc<B::Extrinsic>> {
+		self.pool.ready_transaction(hash).and_then(
+			// Only propagable transactions should be resolved for network service.
+			|tx| tx.is_propagable().then(|| tx.data().clone()),
+		)
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use futures::executor::block_on;
+	use pezsc_transaction_pool::BasicPool;
+	use pezsp_consensus::SelectChain;
+	use bizinikiwi_test_runtime_client::{
+		prelude::*,
+		runtime::{ExtrinsicBuilder, Transfer, TransferData},
+	};
+
+	#[test]
+	fn should_not_propagate_transactions_that_are_marked_as_such() {
+		// given
+		let (client, longest_chain) = TestClientBuilder::new().build_with_longest_chain();
+		let client = Arc::new(client);
+		let spawner = pezsp_core::testing::TaskExecutor::new();
+		let pool = Arc::from(BasicPool::new_full(
+			Default::default(),
+			true.into(),
+			None,
+			spawner,
+			client.clone(),
+		));
+		let source = pezsp_runtime::transaction_validity::TransactionSource::External;
+		let best = block_on(longest_chain.best_chain()).unwrap();
+		let transaction = Transfer {
+			amount: 5,
+			nonce: 0,
+			from: Sr25519Keyring::Alice.into(),
+			to: Sr25519Keyring::Bob.into(),
+		}
+		.into_unchecked_extrinsic();
+		block_on(pool.submit_one(best.hash(), source, transaction.clone())).unwrap();
+		block_on(pool.submit_one(
+			best.hash(),
+			source,
+			ExtrinsicBuilder::new_call_do_not_propagate().nonce(1).build(),
+		))
+		.unwrap();
+		assert_eq!(pool.status().ready, 2);
+
+		// when
+		let transactions = transactions_to_propagate(&*pool);
+
+		// then
+		assert_eq!(transactions.len(), 1);
+		assert!(TransferData::try_from(&*transactions[0].1).is_ok());
+	}
+}

bizinikiwi/client/service/src/metrics.rs

@@ -0,0 +1,293 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use futures_timer::Delay;
+use prometheus_endpoint::{register, Gauge, GaugeVec, Opts, PrometheusError, Registry, U64};
+use pezsc_client_api::{ClientInfo, UsageProvider};
+use pezsc_network::{config::Role, NetworkStatus, NetworkStatusProvider};
+use pezsc_network_sync::{SyncStatus, SyncStatusProvider};
+use pezsc_telemetry::{telemetry, TelemetryHandle, BIZINIKIWI_INFO};
+use pezsc_transaction_pool_api::{MaintainedTransactionPool, PoolStatus};
+use pezsc_utils::metrics::register_globals;
+use pezsp_api::ProvideRuntimeApi;
+use pezsp_runtime::traits::{Block, NumberFor, SaturatedConversion, UniqueSaturatedInto};
+use std::{
+	sync::Arc,
+	time::{Duration, Instant, SystemTime},
+};
+
+struct PrometheusMetrics {
+	// generic info
+	block_height: GaugeVec<U64>,
+	number_leaves: Gauge<U64>,
+	ready_transactions_number: Gauge<U64>,
+
+	// I/O
+	database_cache: Gauge<U64>,
+	state_cache: Gauge<U64>,
+}
+
+impl PrometheusMetrics {
+	fn setup(
+		registry: &Registry,
+		name: &str,
+		version: &str,
+		roles: u64,
+	) -> Result<Self, PrometheusError> {
+		register(
+			Gauge::<U64>::with_opts(
+				Opts::new(
+					"bizinikiwi_build_info",
+					"A metric with a constant '1' value labeled by name, version",
+				)
+				.const_label("name", name)
+				.const_label("version", version),
+			)?,
+			registry,
+		)?
+		.set(1);
+
+		register(
+			Gauge::<U64>::new("bizinikiwi_node_roles", "The roles the node is running as")?,
+			registry,
+		)?
+		.set(roles);
+
+		register_globals(registry)?;
+
+		let start_time_since_epoch =
+			SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap_or_default();
+		register(
+			Gauge::<U64>::new(
+				"bizinikiwi_process_start_time_seconds",
+				"Number of seconds between the UNIX epoch and the moment the process started",
+			)?,
+			registry,
+		)?
+		.set(start_time_since_epoch.as_secs());
+
+		Ok(Self {
+			// generic internals
+			block_height: register(
+				GaugeVec::new(
+					Opts::new("bizinikiwi_block_height", "Block height info of the chain"),
+					&["status"],
+				)?,
+				registry,
+			)?,
+
+			number_leaves: register(
+				Gauge::new("bizinikiwi_number_leaves", "Number of known chain leaves (aka forks)")?,
+				registry,
+			)?,
+
+			ready_transactions_number: register(
+				Gauge::new(
+					"bizinikiwi_ready_transactions_number",
+					"Number of transactions in the ready queue",
+				)?,
+				registry,
+			)?,
+
+			// I/ O
+			database_cache: register(
+				Gauge::new("bizinikiwi_database_cache_bytes", "RocksDB cache size in bytes")?,
+				registry,
+			)?,
+			state_cache: register(
+				Gauge::new("bizinikiwi_state_cache_bytes", "State cache size in bytes")?,
+				registry,
+			)?,
+		})
+	}
+}
+
+/// A `MetricsService` periodically sends general client and
+/// network state to the telemetry as well as (optionally)
+/// a Prometheus endpoint.
+pub struct MetricsService {
+	metrics: Option<PrometheusMetrics>,
+	last_update: Instant,
+	last_total_bytes_inbound: u64,
+	last_total_bytes_outbound: u64,
+	telemetry: Option<TelemetryHandle>,
+}
+
+impl MetricsService {
+	/// Creates a `MetricsService` that only sends information
+	/// to the telemetry.
+	pub fn new(telemetry: Option<TelemetryHandle>) -> Self {
+		MetricsService {
+			metrics: None,
+			last_total_bytes_inbound: 0,
+			last_total_bytes_outbound: 0,
+			last_update: Instant::now(),
+			telemetry,
+		}
+	}
+
+	/// Creates a `MetricsService` that sends metrics
+	/// to prometheus alongside the telemetry.
+	pub fn with_prometheus(
+		telemetry: Option<TelemetryHandle>,
+		registry: &Registry,
+		role: Role,
+		node_name: &str,
+		impl_version: &str,
+	) -> Result<Self, PrometheusError> {
+		let role_bits = match role {
+			Role::Full => 1u64,
+			// 2u64 used to represent light client role
+			Role::Authority { .. } => 4u64,
+		};
+
+		PrometheusMetrics::setup(registry, node_name, impl_version, role_bits).map(|p| {
+			MetricsService {
+				metrics: Some(p),
+				last_total_bytes_inbound: 0,
+				last_total_bytes_outbound: 0,
+				last_update: Instant::now(),
+				telemetry,
+			}
+		})
+	}
+
+	/// Returns a never-ending `Future` that performs the
+	/// metric and telemetry updates with information from
+	/// the given sources.
+	pub async fn run<TBl, TExPool, TCl, TNet, TSync>(
+		mut self,
+		client: Arc<TCl>,
+		transactions: Arc<TExPool>,
+		network: TNet,
+		syncing: TSync,
+	) where
+		TBl: Block,
+		TCl: ProvideRuntimeApi<TBl> + UsageProvider<TBl>,
+		TExPool: MaintainedTransactionPool<Block = TBl, Hash = <TBl as Block>::Hash>,
+		TNet: NetworkStatusProvider,
+		TSync: SyncStatusProvider<TBl>,
+	{
+		let mut timer = Delay::new(Duration::from_secs(0));
+		let timer_interval = Duration::from_secs(5);
+
+		loop {
+			// Wait for the next tick of the timer.
+			(&mut timer).await;
+
+			// Try to get the latest network information.
+			let net_status = network.status().await.ok();
+
+			// Try to get the latest syncing information.
+			let sync_status = syncing.status().await.ok();
+
+			// Update / Send the metrics.
+			self.update(&client.usage_info(), &transactions.status(), net_status, sync_status);
+
+			// Schedule next tick.
+			timer.reset(timer_interval);
+		}
+	}
+
+	fn update<T: Block>(
+		&mut self,
+		info: &ClientInfo<T>,
+		txpool_status: &PoolStatus,
+		net_status: Option<NetworkStatus>,
+		sync_status: Option<SyncStatus<T>>,
+	) {
+		let now = Instant::now();
+		let elapsed = (now - self.last_update).as_secs();
+		self.last_update = now;
+
+		let best_number = info.chain.best_number.saturated_into::<u64>();
+		let best_hash = info.chain.best_hash;
+		let finalized_number: u64 = info.chain.finalized_number.saturated_into::<u64>();
+
+		// Update/send metrics that are always available.
+		telemetry!(
+			self.telemetry;
+			BIZINIKIWI_INFO;
+			"system.interval";
+			"height" => best_number,
+			"best" => ?best_hash,
+			"txcount" => txpool_status.ready,
+			"finalized_height" => finalized_number,
+			"finalized_hash" => ?info.chain.finalized_hash,
+			"used_state_cache_size" => info.usage.as_ref()
+				.map(|usage| usage.memory.state_cache.as_bytes())
+				.unwrap_or(0),
+		);
+
+		if let Some(metrics) = self.metrics.as_ref() {
+			metrics.block_height.with_label_values(&["finalized"]).set(finalized_number);
+			metrics.block_height.with_label_values(&["best"]).set(best_number);
+
+			if let Ok(leaves) = u64::try_from(info.chain.number_leaves) {
+				metrics.number_leaves.set(leaves);
+			}
+
+			metrics.ready_transactions_number.set(txpool_status.ready as u64);
+
+			if let Some(info) = info.usage.as_ref() {
+				metrics.database_cache.set(info.memory.database_cache.as_bytes() as u64);
+				metrics.state_cache.set(info.memory.state_cache.as_bytes() as u64);
+			}
+		}
+
+		// Update/send network status information, if any.
+		if let Some(net_status) = net_status {
+			let num_peers = net_status.num_connected_peers;
+			let total_bytes_inbound = net_status.total_bytes_inbound;
+			let total_bytes_outbound = net_status.total_bytes_outbound;
+
+			let diff_bytes_inbound = total_bytes_inbound - self.last_total_bytes_inbound;
+			let diff_bytes_outbound = total_bytes_outbound - self.last_total_bytes_outbound;
+			let (avg_bytes_per_sec_inbound, avg_bytes_per_sec_outbound) = if elapsed > 0 {
+				self.last_total_bytes_inbound = total_bytes_inbound;
+				self.last_total_bytes_outbound = total_bytes_outbound;
+				(diff_bytes_inbound / elapsed, diff_bytes_outbound / elapsed)
+			} else {
+				(diff_bytes_inbound, diff_bytes_outbound)
+			};
+
+			telemetry!(
+				self.telemetry;
+				BIZINIKIWI_INFO;
+				"system.interval";
+				"peers" => num_peers,
+				"bandwidth_download" => avg_bytes_per_sec_inbound,
+				"bandwidth_upload" => avg_bytes_per_sec_outbound,
+			);
+		}
+
+		if let Some(sync_status) = sync_status {
+			if let Some(metrics) = self.metrics.as_ref() {
+				let best_seen_block: Option<u64> =
+					sync_status.best_seen_block.map(|num: NumberFor<T>| {
+						UniqueSaturatedInto::<u64>::unique_saturated_into(num)
+					});
+
+				metrics
+					.block_height
+					.with_label_values(&["sync_target"])
+					.set(best_seen_block.unwrap_or(best_number));
+			}
+		}
+	}
+}

bizinikiwi/client/service/src/task_manager/mod.rs

@@ -0,0 +1,551 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Bizinikiwi service tasks management module.
+
+use crate::{config::TaskType, Error};
+use exit_future::Signal;
+use futures::{
+	future::{pending, select, try_join_all, BoxFuture, Either},
+	Future, FutureExt, StreamExt,
+};
+use parking_lot::Mutex;
+use prometheus_endpoint::{
+	exponential_buckets, register, CounterVec, HistogramOpts, HistogramVec, Opts, PrometheusError,
+	Registry, U64,
+};
+use pezsc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender};
+use std::{
+	collections::{hash_map::Entry, HashMap},
+	panic,
+	pin::Pin,
+	result::Result,
+	sync::Arc,
+};
+use tokio::runtime::Handle;
+use tracing_futures::Instrument;
+
+mod prometheus_future;
+#[cfg(test)]
+mod tests;
+
+/// Default task group name.
+pub const DEFAULT_GROUP_NAME: &str = "default";
+
+/// The name of a group a task belongs to.
+///
+/// This name is passed belong-side the task name to the prometheus metrics and can be used
+/// to group tasks.
+pub enum GroupName {
+	/// Sets the group name to `default`.
+	Default,
+	/// Use the specifically given name as group name.
+	Specific(&'static str),
+}
+
+impl From<Option<&'static str>> for GroupName {
+	fn from(name: Option<&'static str>) -> Self {
+		match name {
+			Some(name) => Self::Specific(name),
+			None => Self::Default,
+		}
+	}
+}
+
+impl From<&'static str> for GroupName {
+	fn from(name: &'static str) -> Self {
+		Self::Specific(name)
+	}
+}
+
+/// An handle for spawning tasks in the service.
+#[derive(Clone)]
+pub struct SpawnTaskHandle {
+	on_exit: exit_future::Exit,
+	tokio_handle: Handle,
+	metrics: Option<Metrics>,
+	task_registry: TaskRegistry,
+}
+
+impl SpawnTaskHandle {
+	/// Spawns the given task with the given name and a group name.
+	/// If group is not specified `DEFAULT_GROUP_NAME` will be used.
+	///
+	/// Note that the `name` is a `&'static str`. The reason for this choice is that
+	/// statistics about this task are getting reported to the Prometheus endpoint (if enabled), and
+	/// that therefore the set of possible task names must be bounded.
+	///
+	/// In other words, it would be a bad idea for someone to do for example
+	/// `spawn(format!("{:?}", some_public_key))`.
+	pub fn spawn(
+		&self,
+		name: &'static str,
+		group: impl Into<GroupName>,
+		task: impl Future<Output = ()> + Send + 'static,
+	) {
+		self.spawn_inner(name, group, task, TaskType::Async)
+	}
+
+	/// Spawns the blocking task with the given name. See also `spawn`.
+	pub fn spawn_blocking(
+		&self,
+		name: &'static str,
+		group: impl Into<GroupName>,
+		task: impl Future<Output = ()> + Send + 'static,
+	) {
+		self.spawn_inner(name, group, task, TaskType::Blocking)
+	}
+
+	/// Helper function that implements the spawning logic. See `spawn` and `spawn_blocking`.
+	fn spawn_inner(
+		&self,
+		name: &'static str,
+		group: impl Into<GroupName>,
+		task: impl Future<Output = ()> + Send + 'static,
+		task_type: TaskType,
+	) {
+		let on_exit = self.on_exit.clone();
+		let metrics = self.metrics.clone();
+		let registry = self.task_registry.clone();
+
+		let group = match group.into() {
+			GroupName::Specific(var) => var,
+			// If no group is specified use default.
+			GroupName::Default => DEFAULT_GROUP_NAME,
+		};
+
+		let task_type_label = match task_type {
+			TaskType::Blocking => "blocking",
+			TaskType::Async => "async",
+		};
+
+		// Note that we increase the started counter here and not within the future. This way,
+		// we could properly visualize on Prometheus situations where the spawning doesn't work.
+		if let Some(metrics) = &self.metrics {
+			metrics.tasks_spawned.with_label_values(&[name, group, task_type_label]).inc();
+			// We do a dummy increase in order for the task to show up in metrics.
+			metrics
+				.tasks_ended
+				.with_label_values(&[name, "finished", group, task_type_label])
+				.inc_by(0);
+		}
+
+		let future = async move {
+			// Register the task and keep the "token" alive until the task is ended. Then this
+			// "token" will unregister this task.
+			let _registry_token = registry.register_task(name, group);
+
+			if let Some(metrics) = metrics {
+				// Add some wrappers around `task`.
+				let task = {
+					let poll_duration =
+						metrics.poll_duration.with_label_values(&[name, group, task_type_label]);
+					let poll_start =
+						metrics.poll_start.with_label_values(&[name, group, task_type_label]);
+					let inner =
+						prometheus_future::with_poll_durations(poll_duration, poll_start, task);
+					// The logic of `AssertUnwindSafe` here is ok considering that we throw
+					// away the `Future` after it has panicked.
+					panic::AssertUnwindSafe(inner).catch_unwind()
+				};
+				futures::pin_mut!(task);
+
+				match select(on_exit, task).await {
+					Either::Right((Err(payload), _)) => {
+						metrics
+							.tasks_ended
+							.with_label_values(&[name, "panic", group, task_type_label])
+							.inc();
+						panic::resume_unwind(payload)
+					},
+					Either::Right((Ok(()), _)) => {
+						metrics
+							.tasks_ended
+							.with_label_values(&[name, "finished", group, task_type_label])
+							.inc();
+					},
+					Either::Left(((), _)) => {
+						// The `on_exit` has triggered.
+						metrics
+							.tasks_ended
+							.with_label_values(&[name, "interrupted", group, task_type_label])
+							.inc();
+					},
+				}
+			} else {
+				futures::pin_mut!(task);
+				let _ = select(on_exit, task).await;
+			}
+		}
+		.in_current_span();
+
+		match task_type {
+			TaskType::Async => {
+				self.tokio_handle.spawn(future);
+			},
+			TaskType::Blocking => {
+				let handle = self.tokio_handle.clone();
+				self.tokio_handle.spawn_blocking(move || {
+					handle.block_on(future);
+				});
+			},
+		}
+	}
+}
+
+impl pezsp_core::traits::SpawnNamed for SpawnTaskHandle {
+	fn spawn_blocking(
+		&self,
+		name: &'static str,
+		group: Option<&'static str>,
+		future: BoxFuture<'static, ()>,
+	) {
+		self.spawn_inner(name, group, future, TaskType::Blocking)
+	}
+
+	fn spawn(
+		&self,
+		name: &'static str,
+		group: Option<&'static str>,
+		future: BoxFuture<'static, ()>,
+	) {
+		self.spawn_inner(name, group, future, TaskType::Async)
+	}
+}
+
+/// A wrapper over `SpawnTaskHandle` that will notify a receiver whenever any
+/// task spawned through it fails. The service should be on the receiver side
+/// and will shut itself down whenever it receives any message, i.e. an
+/// essential task has failed.
+#[derive(Clone)]
+pub struct SpawnEssentialTaskHandle {
+	essential_failed_tx: TracingUnboundedSender<()>,
+	inner: SpawnTaskHandle,
+}
+
+impl SpawnEssentialTaskHandle {
+	/// Creates a new `SpawnEssentialTaskHandle`.
+	pub fn new(
+		essential_failed_tx: TracingUnboundedSender<()>,
+		spawn_task_handle: SpawnTaskHandle,
+	) -> SpawnEssentialTaskHandle {
+		SpawnEssentialTaskHandle { essential_failed_tx, inner: spawn_task_handle }
+	}
+
+	/// Spawns the given task with the given name.
+	///
+	/// See also [`SpawnTaskHandle::spawn`].
+	pub fn spawn(
+		&self,
+		name: &'static str,
+		group: impl Into<GroupName>,
+		task: impl Future<Output = ()> + Send + 'static,
+	) {
+		self.spawn_inner(name, group, task, TaskType::Async)
+	}
+
+	/// Spawns the blocking task with the given name.
+	///
+	/// See also [`SpawnTaskHandle::spawn_blocking`].
+	pub fn spawn_blocking(
+		&self,
+		name: &'static str,
+		group: impl Into<GroupName>,
+		task: impl Future<Output = ()> + Send + 'static,
+	) {
+		self.spawn_inner(name, group, task, TaskType::Blocking)
+	}
+
+	fn spawn_inner(
+		&self,
+		name: &'static str,
+		group: impl Into<GroupName>,
+		task: impl Future<Output = ()> + Send + 'static,
+		task_type: TaskType,
+	) {
+		let essential_failed = self.essential_failed_tx.clone();
+		let essential_task = std::panic::AssertUnwindSafe(task).catch_unwind().map(move |_| {
+			log::error!("Essential task `{}` failed. Shutting down service.", name);
+			let _ = essential_failed.close();
+		});
+
+		let _ = self.inner.spawn_inner(name, group, essential_task, task_type);
+	}
+}
+
+impl pezsp_core::traits::SpawnEssentialNamed for SpawnEssentialTaskHandle {
+	fn spawn_essential_blocking(
+		&self,
+		name: &'static str,
+		group: Option<&'static str>,
+		future: BoxFuture<'static, ()>,
+	) {
+		self.spawn_blocking(name, group, future);
+	}
+
+	fn spawn_essential(
+		&self,
+		name: &'static str,
+		group: Option<&'static str>,
+		future: BoxFuture<'static, ()>,
+	) {
+		self.spawn(name, group, future);
+	}
+}
+
+/// Helper struct to manage background/async tasks in Service.
+pub struct TaskManager {
+	/// A future that resolves when the service has exited, this is useful to
+	/// make sure any internally spawned futures stop when the service does.
+	on_exit: exit_future::Exit,
+	/// A signal that makes the exit future above resolve, fired on drop.
+	_signal: Signal,
+	/// Tokio runtime handle that is used to spawn futures.
+	tokio_handle: Handle,
+	/// Prometheus metric where to report the polling times.
+	metrics: Option<Metrics>,
+	/// Send a signal when a spawned essential task has concluded. The next time
+	/// the service future is polled it should complete with an error.
+	essential_failed_tx: TracingUnboundedSender<()>,
+	/// A receiver for spawned essential-tasks concluding.
+	essential_failed_rx: TracingUnboundedReceiver<()>,
+	/// Things to keep alive until the task manager is dropped.
+	keep_alive: Box<dyn std::any::Any + Send>,
+	/// A list of other `TaskManager`'s to terminate and gracefully shutdown when the parent
+	/// terminates and gracefully shutdown. Also ends the parent `future()` if a child's essential
+	/// task fails.
+	children: Vec<TaskManager>,
+	/// The registry of all running tasks.
+	task_registry: TaskRegistry,
+}
+
+impl TaskManager {
+	/// If a Prometheus registry is passed, it will be used to report statistics about the
+	/// service tasks.
+	pub fn new(
+		tokio_handle: Handle,
+		prometheus_registry: Option<&Registry>,
+	) -> Result<Self, PrometheusError> {
+		let (signal, on_exit) = exit_future::signal();
+
+		// A side-channel for essential tasks to communicate shutdown.
+		let (essential_failed_tx, essential_failed_rx) =
+			tracing_unbounded("mpsc_essential_tasks", 100);
+
+		let metrics = prometheus_registry.map(Metrics::register).transpose()?;
+
+		Ok(Self {
+			on_exit,
+			_signal: signal,
+			tokio_handle,
+			metrics,
+			essential_failed_tx,
+			essential_failed_rx,
+			keep_alive: Box::new(()),
+			children: Vec::new(),
+			task_registry: Default::default(),
+		})
+	}
+
+	/// Get a handle for spawning tasks.
+	pub fn spawn_handle(&self) -> SpawnTaskHandle {
+		SpawnTaskHandle {
+			on_exit: self.on_exit.clone(),
+			tokio_handle: self.tokio_handle.clone(),
+			metrics: self.metrics.clone(),
+			task_registry: self.task_registry.clone(),
+		}
+	}
+
+	/// Get a handle for spawning essential tasks.
+	pub fn spawn_essential_handle(&self) -> SpawnEssentialTaskHandle {
+		SpawnEssentialTaskHandle::new(self.essential_failed_tx.clone(), self.spawn_handle())
+	}
+
+	/// Return a future that will end with success if the signal to terminate was sent
+	/// (`self.terminate()`) or with an error if an essential task fails.
+	///
+	/// # Warning
+	///
+	/// This function will not wait until the end of the remaining task.
+	pub fn future<'a>(
+		&'a mut self,
+	) -> Pin<Box<dyn Future<Output = Result<(), Error>> + Send + 'a>> {
+		Box::pin(async move {
+			let mut t1 = self.essential_failed_rx.next().fuse();
+			let mut t2 = self.on_exit.clone().fuse();
+			let mut t3 = try_join_all(
+				self.children
+					.iter_mut()
+					.map(|x| x.future())
+					// Never end this future if there is no error because if there is no children,
+					// it must not stop
+					.chain(std::iter::once(pending().boxed())),
+			)
+			.fuse();
+
+			futures::select! {
+				_ = t1 => Err(Error::Other("Essential task failed.".into())),
+				_ = t2 => Ok(()),
+				res = t3 => Err(res.map(|_| ()).expect_err("this future never ends; qed")),
+			}
+		})
+	}
+
+	/// Set what the task manager should keep alive, can be called multiple times.
+	pub fn keep_alive<T: 'static + Send>(&mut self, to_keep_alive: T) {
+		// allows this fn to safely called multiple times.
+		use std::mem;
+		let old = mem::replace(&mut self.keep_alive, Box::new(()));
+		self.keep_alive = Box::new((to_keep_alive, old));
+	}
+
+	/// Register another TaskManager to terminate and gracefully shutdown when the parent
+	/// terminates and gracefully shutdown. Also ends the parent `future()` if a child's essential
+	/// task fails. (But don't end the parent if a child's normal task fails.)
+	pub fn add_child(&mut self, child: TaskManager) {
+		self.children.push(child);
+	}
+
+	/// Consume `self` and return the [`TaskRegistry`].
+	///
+	/// This [`TaskRegistry`] can be used to check for still running tasks after this task manager
+	/// was dropped.
+	pub fn into_task_registry(self) -> TaskRegistry {
+		self.task_registry
+	}
+}
+
+#[derive(Clone)]
+struct Metrics {
+	// This list is ordered alphabetically
+	poll_duration: HistogramVec,
+	poll_start: CounterVec<U64>,
+	tasks_spawned: CounterVec<U64>,
+	tasks_ended: CounterVec<U64>,
+}
+
+impl Metrics {
+	fn register(registry: &Registry) -> Result<Self, PrometheusError> {
+		Ok(Self {
+			poll_duration: register(HistogramVec::new(
+				HistogramOpts {
+					common_opts: Opts::new(
+						"bizinikiwi_tasks_polling_duration",
+						"Duration in seconds of each invocation of Future::poll"
+					),
+					buckets: exponential_buckets(0.001, 4.0, 9)
+						.expect("function parameters are constant and always valid; qed"),
+				},
+				&["task_name", "task_group", "kind"]
+			)?, registry)?,
+			poll_start: register(CounterVec::new(
+				Opts::new(
+					"bizinikiwi_tasks_polling_started_total",
+					"Total number of times we started invoking Future::poll"
+				),
+				&["task_name", "task_group", "kind"]
+			)?, registry)?,
+			tasks_spawned: register(CounterVec::new(
+				Opts::new(
+					"bizinikiwi_tasks_spawned_total",
+					"Total number of tasks that have been spawned on the Service"
+				),
+				&["task_name", "task_group", "kind"]
+			)?, registry)?,
+			tasks_ended: register(CounterVec::new(
+				Opts::new(
+					"bizinikiwi_tasks_ended_total",
+					"Total number of tasks for which Future::poll has returned Ready(()) or panicked"
+				),
+				&["task_name", "reason", "task_group", "kind"]
+			)?, registry)?,
+		})
+	}
+}
+
+/// Ensures that a [`Task`] is unregistered when this object is dropped.
+struct UnregisterOnDrop {
+	task: Task,
+	registry: TaskRegistry,
+}
+
+impl Drop for UnregisterOnDrop {
+	fn drop(&mut self) {
+		let mut tasks = self.registry.tasks.lock();
+
+		if let Entry::Occupied(mut entry) = (*tasks).entry(self.task.clone()) {
+			*entry.get_mut() -= 1;
+
+			if *entry.get() == 0 {
+				entry.remove();
+			}
+		}
+	}
+}
+
+/// Represents a running async task in the [`TaskManager`].
+///
+/// As a task is identified by a name and a group, it is totally valid that there exists multiple
+/// tasks with the same name and group.
+#[derive(Clone, Hash, Eq, PartialEq)]
+pub struct Task {
+	/// The name of the task.
+	pub name: &'static str,
+	/// The group this task is associated to.
+	pub group: &'static str,
+}
+
+impl Task {
+	/// Returns if the `group` is the [`DEFAULT_GROUP_NAME`].
+	pub fn is_default_group(&self) -> bool {
+		self.group == DEFAULT_GROUP_NAME
+	}
+}
+
+/// Keeps track of all running [`Task`]s in [`TaskManager`].
+#[derive(Clone, Default)]
+pub struct TaskRegistry {
+	tasks: Arc<Mutex<HashMap<Task, usize>>>,
+}
+
+impl TaskRegistry {
+	/// Register a task with the given `name` and `group`.
+	///
+	/// Returns [`UnregisterOnDrop`] that ensures that the task is unregistered when this value is
+	/// dropped.
+	fn register_task(&self, name: &'static str, group: &'static str) -> UnregisterOnDrop {
+		let task = Task { name, group };
+
+		{
+			let mut tasks = self.tasks.lock();
+
+			*(*tasks).entry(task.clone()).or_default() += 1;
+		}
+
+		UnregisterOnDrop { task, registry: self.clone() }
+	}
+
+	/// Returns the running tasks.
+	///
+	/// As a task is only identified by its `name` and `group`, there can be duplicate tasks. The
+	/// number per task represents the concurrently running tasks with the same identifier.
+	pub fn running_tasks(&self) -> HashMap<Task, usize> {
+		(*self.tasks.lock()).clone()
+	}
+}

bizinikiwi/client/service/src/task_manager/prometheus_future.rs

@@ -0,0 +1,74 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Wrapper around a `Future` that reports statistics about when the `Future` is polled.
+
+use futures::prelude::*;
+use prometheus_endpoint::{Counter, Histogram, U64};
+use std::{
+	fmt,
+	pin::Pin,
+	task::{Context, Poll},
+};
+
+/// Wraps around a `Future`. Report the polling duration to the `Histogram` and when the polling
+/// starts to the `Counter`.
+pub fn with_poll_durations<T>(
+	poll_duration: Histogram,
+	poll_start: Counter<U64>,
+	inner: T,
+) -> PrometheusFuture<T> {
+	PrometheusFuture { inner, poll_duration, poll_start }
+}
+
+/// Wraps around `Future` and adds diagnostics to it.
+#[pin_project::pin_project]
+#[derive(Clone)]
+pub struct PrometheusFuture<T> {
+	/// The inner future doing the actual work.
+	#[pin]
+	inner: T,
+	poll_duration: Histogram,
+	poll_start: Counter<U64>,
+}
+
+impl<T> Future for PrometheusFuture<T>
+where
+	T: Future,
+{
+	type Output = T::Output;
+
+	fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Self::Output> {
+		let this = self.project();
+
+		this.poll_start.inc();
+		let _timer = this.poll_duration.start_timer();
+		Future::poll(this.inner, cx)
+
+		// `_timer` is dropped here and will observe the duration
+	}
+}
+
+impl<T> fmt::Debug for PrometheusFuture<T>
+where
+	T: fmt::Debug,
+{
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		fmt::Debug::fmt(&self.inner, f)
+	}
+}

bizinikiwi/client/service/src/task_manager/tests.rs

@@ -0,0 +1,252 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::task_manager::TaskManager;
+use futures::{future::FutureExt, pin_mut, select};
+use parking_lot::Mutex;
+use std::{any::Any, sync::Arc, time::Duration};
+
+#[derive(Clone, Debug)]
+struct DropTester(Arc<Mutex<usize>>);
+
+struct DropTesterRef(DropTester);
+
+impl DropTester {
+	fn new() -> DropTester {
+		DropTester(Arc::new(Mutex::new(0)))
+	}
+
+	fn new_ref(&self) -> DropTesterRef {
+		*self.0.lock() += 1;
+		DropTesterRef(self.clone())
+	}
+
+	fn wait_on_drop(&self) {
+		while *self != 0 {
+			std::thread::sleep(std::time::Duration::from_millis(10));
+		}
+	}
+}
+
+impl PartialEq<usize> for DropTester {
+	fn eq(&self, other: &usize) -> bool {
+		&*self.0.lock() == other
+	}
+}
+
+impl Drop for DropTesterRef {
+	fn drop(&mut self) {
+		*(self.0).0.lock() -= 1;
+	}
+}
+
+#[test]
+fn ensure_drop_tester_working() {
+	let drop_tester = DropTester::new();
+	assert_eq!(drop_tester, 0);
+	let drop_tester_ref_1 = drop_tester.new_ref();
+	assert_eq!(drop_tester, 1);
+	let drop_tester_ref_2 = drop_tester.new_ref();
+	assert_eq!(drop_tester, 2);
+	drop(drop_tester_ref_1);
+	assert_eq!(drop_tester, 1);
+	drop(drop_tester_ref_2);
+	assert_eq!(drop_tester, 0);
+}
+
+async fn run_background_task(_keep_alive: impl Any) {
+	loop {
+		tokio::time::sleep(Duration::from_secs(1)).await;
+	}
+}
+
+async fn run_background_task_blocking(duration: Duration, _keep_alive: impl Any) {
+	loop {
+		// block for X sec (not interruptible)
+		std::thread::sleep(duration);
+		// await for 1 sec (interruptible)
+		tokio::time::sleep(Duration::from_secs(1)).await;
+	}
+}
+
+fn new_task_manager(tokio_handle: tokio::runtime::Handle) -> TaskManager {
+	TaskManager::new(tokio_handle, None).unwrap()
+}
+
+#[test]
+fn ensure_tasks_are_awaited_on_shutdown() {
+	let drop_tester = DropTester::new();
+	{
+		let runtime = tokio::runtime::Runtime::new().unwrap();
+		let handle = runtime.handle().clone();
+
+		let task_manager = new_task_manager(handle);
+		let spawn_handle = task_manager.spawn_handle();
+		spawn_handle.spawn("task1", None, run_background_task(drop_tester.new_ref()));
+		spawn_handle.spawn("task2", None, run_background_task(drop_tester.new_ref()));
+		assert_eq!(drop_tester, 2);
+		// allow the tasks to even start
+		runtime.block_on(async { tokio::time::sleep(Duration::from_secs(1)).await });
+		assert_eq!(drop_tester, 2);
+	}
+	drop_tester.wait_on_drop();
+}
+
+#[test]
+fn ensure_keep_alive_during_shutdown() {
+	let drop_tester = DropTester::new();
+	{
+		let runtime = tokio::runtime::Runtime::new().unwrap();
+		let handle = runtime.handle().clone();
+
+		let mut task_manager = new_task_manager(handle);
+		let spawn_handle = task_manager.spawn_handle();
+		task_manager.keep_alive(drop_tester.new_ref());
+		spawn_handle.spawn("task1", None, run_background_task(()));
+		assert_eq!(drop_tester, 1);
+		// allow the tasks to even start
+		runtime.block_on(async { tokio::time::sleep(Duration::from_secs(1)).await });
+		assert_eq!(drop_tester, 1);
+	}
+	drop_tester.wait_on_drop();
+}
+
+#[test]
+fn ensure_blocking_futures_are_awaited_on_shutdown() {
+	let drop_tester = DropTester::new();
+	{
+		let runtime = tokio::runtime::Runtime::new().unwrap();
+		let handle = runtime.handle().clone();
+
+		let task_manager = new_task_manager(handle);
+		let spawn_handle = task_manager.spawn_handle();
+		spawn_handle.spawn(
+			"task1",
+			None,
+			run_background_task_blocking(Duration::from_secs(3), drop_tester.new_ref()),
+		);
+		spawn_handle.spawn(
+			"task2",
+			None,
+			run_background_task_blocking(Duration::from_secs(3), drop_tester.new_ref()),
+		);
+		assert_eq!(drop_tester, 2);
+		// allow the tasks to even start
+		runtime.block_on(async { tokio::time::sleep(Duration::from_secs(1)).await });
+		assert_eq!(drop_tester, 2);
+	}
+	assert_eq!(drop_tester, 0);
+}
+
+#[test]
+fn ensure_task_manager_future_ends_with_error_when_essential_task_fails() {
+	let drop_tester = DropTester::new();
+	{
+		let runtime = tokio::runtime::Runtime::new().unwrap();
+		let handle = runtime.handle().clone();
+
+		let mut task_manager = new_task_manager(handle);
+		let spawn_handle = task_manager.spawn_handle();
+		let spawn_essential_handle = task_manager.spawn_essential_handle();
+		spawn_handle.spawn("task1", None, run_background_task(drop_tester.new_ref()));
+		spawn_handle.spawn("task2", None, run_background_task(drop_tester.new_ref()));
+		assert_eq!(drop_tester, 2);
+		// allow the tasks to even start
+		runtime.block_on(async { tokio::time::sleep(Duration::from_secs(1)).await });
+		assert_eq!(drop_tester, 2);
+		spawn_essential_handle.spawn("task3", None, async { panic!("task failed") });
+		runtime
+			.block_on(task_manager.future())
+			.expect_err("future()'s Result must be Err");
+		assert_eq!(drop_tester, 2);
+	}
+	drop_tester.wait_on_drop();
+}
+
+#[test]
+fn ensure_task_manager_future_ends_with_error_when_childs_essential_task_fails() {
+	let drop_tester = DropTester::new();
+	{
+		let runtime = tokio::runtime::Runtime::new().unwrap();
+		let handle = runtime.handle().clone();
+
+		let mut task_manager = new_task_manager(handle.clone());
+		let child_1 = new_task_manager(handle.clone());
+		let spawn_handle_child_1 = child_1.spawn_handle();
+		let spawn_essential_handle_child_1 = child_1.spawn_essential_handle();
+		let child_2 = new_task_manager(handle.clone());
+		let spawn_handle_child_2 = child_2.spawn_handle();
+		task_manager.add_child(child_1);
+		task_manager.add_child(child_2);
+		let spawn_handle = task_manager.spawn_handle();
+		spawn_handle.spawn("task1", None, run_background_task(drop_tester.new_ref()));
+		spawn_handle.spawn("task2", None, run_background_task(drop_tester.new_ref()));
+		spawn_handle_child_1.spawn("task3", None, run_background_task(drop_tester.new_ref()));
+		spawn_handle_child_2.spawn("task4", None, run_background_task(drop_tester.new_ref()));
+		assert_eq!(drop_tester, 4);
+		// allow the tasks to even start
+		runtime.block_on(async { tokio::time::sleep(Duration::from_secs(1)).await });
+		assert_eq!(drop_tester, 4);
+		spawn_essential_handle_child_1.spawn("task5", None, async { panic!("task failed") });
+		runtime
+			.block_on(task_manager.future())
+			.expect_err("future()'s Result must be Err");
+		assert_eq!(drop_tester, 4);
+	}
+	drop_tester.wait_on_drop();
+}
+
+#[test]
+fn ensure_task_manager_future_continues_when_childs_not_essential_task_fails() {
+	let drop_tester = DropTester::new();
+	{
+		let runtime = tokio::runtime::Runtime::new().unwrap();
+		let handle = runtime.handle().clone();
+
+		let mut task_manager = new_task_manager(handle.clone());
+		let child_1 = new_task_manager(handle.clone());
+		let spawn_handle_child_1 = child_1.spawn_handle();
+		let child_2 = new_task_manager(handle.clone());
+		let spawn_handle_child_2 = child_2.spawn_handle();
+		task_manager.add_child(child_1);
+		task_manager.add_child(child_2);
+		let spawn_handle = task_manager.spawn_handle();
+		spawn_handle.spawn("task1", None, run_background_task(drop_tester.new_ref()));
+		spawn_handle.spawn("task2", None, run_background_task(drop_tester.new_ref()));
+		spawn_handle_child_1.spawn("task3", None, run_background_task(drop_tester.new_ref()));
+		spawn_handle_child_2.spawn("task4", None, run_background_task(drop_tester.new_ref()));
+		assert_eq!(drop_tester, 4);
+		// allow the tasks to even start
+		runtime.block_on(async { tokio::time::sleep(Duration::from_secs(1)).await });
+		assert_eq!(drop_tester, 4);
+		spawn_handle_child_1.spawn("task5", None, async { panic!("task failed") });
+		runtime.block_on(async {
+			let t1 = task_manager.future().fuse();
+			let t2 = tokio::time::sleep(Duration::from_secs(3)).fuse();
+
+			pin_mut!(t1, t2);
+
+			select! {
+				res = t1 => panic!("task should not have stopped: {:?}", res),
+				_ = t2 => {},
+			}
+		});
+		assert_eq!(drop_tester, 4);
+	}
+	drop_tester.wait_on_drop();
+}

bizinikiwi/client/service/test/Cargo.toml

@@ -0,0 +1,67 @@
+[package]
+name = "pezsc-service-test"
+version = "2.0.0"
+authors.workspace = true
+edition.workspace = true
+license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
+publish = false
+homepage.workspace = true
+repository.workspace = true
+
+[lints]
+workspace = true
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[dependencies]
+array-bytes = { workspace = true, default-features = true }
+async-channel = { workspace = true }
+codec = { workspace = true, default-features = true }
+fdlimit = { workspace = true }
+futures = { workspace = true }
+log = { workspace = true, default-features = true }
+parking_lot = { workspace = true, default-features = true }
+pezsc-block-builder = { workspace = true, default-features = true }
+pezsc-client-api = { workspace = true, default-features = true }
+pezsc-client-db = { workspace = true }
+pezsc-consensus = { workspace = true, default-features = true }
+pezsc-executor = { workspace = true, default-features = true }
+pezsc-network = { workspace = true, default-features = true }
+pezsc-network-sync = { workspace = true, default-features = true }
+pezsc-service = { workspace = true, default-features = true }
+pezsc-transaction-pool-api = { workspace = true, default-features = true }
+pezsp-api = { workspace = true, default-features = true }
+pezsp-blockchain = { workspace = true, default-features = true }
+pezsp-consensus = { workspace = true, default-features = true }
+pezsp-core = { workspace = true, default-features = true }
+pezsp-io = { workspace = true, default-features = true }
+pezsp-runtime = { workspace = true, default-features = true }
+pezsp-state-machine = { workspace = true, default-features = true }
+pezsp-storage = { workspace = true, default-features = true }
+pezsp-tracing = { workspace = true, default-features = true }
+bizinikiwi-test-runtime = { workspace = true }
+bizinikiwi-test-runtime-client = { workspace = true }
+tempfile = { workspace = true }
+tokio = { features = ["time"], workspace = true, default-features = true }
+
+[features]
+runtime-benchmarks = [
+	"pezsc-block-builder/runtime-benchmarks",
+	"pezsc-client-api/runtime-benchmarks",
+	"pezsc-client-db/runtime-benchmarks",
+	"pezsc-consensus/runtime-benchmarks",
+	"pezsc-executor/runtime-benchmarks",
+	"pezsc-network-sync/runtime-benchmarks",
+	"pezsc-network/runtime-benchmarks",
+	"pezsc-service/runtime-benchmarks",
+	"pezsc-transaction-pool-api/runtime-benchmarks",
+	"pezsp-api/runtime-benchmarks",
+	"pezsp-blockchain/runtime-benchmarks",
+	"pezsp-consensus/runtime-benchmarks",
+	"pezsp-io/runtime-benchmarks",
+	"pezsp-runtime/runtime-benchmarks",
+	"pezsp-state-machine/runtime-benchmarks",
+	"bizinikiwi-test-runtime-client/runtime-benchmarks",
+	"bizinikiwi-test-runtime/runtime-benchmarks",
+]

bizinikiwi/client/service/test/src/client/db.rs

@@ -0,0 +1,58 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use pezsp_core::offchain::{storage::InMemOffchainStorage, OffchainStorage};
+use std::sync::Arc;
+
+type TestBackend = pezsc_client_api::in_mem::Backend<bizinikiwi_test_runtime::Block>;
+
+#[test]
+fn test_leaves_with_complex_block_tree() {
+	let backend = Arc::new(TestBackend::new());
+
+	bizinikiwi_test_runtime_client::trait_tests::test_leaves_for_backend(backend);
+}
+
+#[test]
+fn test_blockchain_query_by_number_gets_canonical() {
+	let backend = Arc::new(TestBackend::new());
+
+	bizinikiwi_test_runtime_client::trait_tests::test_blockchain_query_by_number_gets_canonical(
+		backend,
+	);
+}
+
+#[test]
+fn in_memory_offchain_storage() {
+	let mut storage = InMemOffchainStorage::default();
+	assert_eq!(storage.get(b"A", b"B"), None);
+	assert_eq!(storage.get(b"B", b"A"), None);
+
+	storage.set(b"A", b"B", b"C");
+	assert_eq!(storage.get(b"A", b"B"), Some(b"C".to_vec()));
+	assert_eq!(storage.get(b"B", b"A"), None);
+
+	storage.compare_and_set(b"A", b"B", Some(b"X"), b"D");
+	assert_eq!(storage.get(b"A", b"B"), Some(b"C".to_vec()));
+	storage.compare_and_set(b"A", b"B", Some(b"C"), b"D");
+	assert_eq!(storage.get(b"A", b"B"), Some(b"D".to_vec()));
+
+	assert!(!storage.compare_and_set(b"B", b"A", Some(b""), b"Y"));
+	assert!(storage.compare_and_set(b"B", b"A", None, b"X"));
+	assert_eq!(storage.get(b"B", b"A"), Some(b"X".to_vec()));
+}

bizinikiwi/client/service/test/src/lib.rs

@@ -0,0 +1,571 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Service integration test utils.
+
+use futures::{task::Poll, Future, TryFutureExt as _};
+use log::{debug, info};
+use parking_lot::Mutex;
+use pezsc_client_api::{Backend, CallExecutor};
+use pezsc_network::{
+	config::{MultiaddrWithPeerId, NetworkConfiguration, TransportConfig},
+	multiaddr, NetworkBlock, NetworkPeers, NetworkStateInfo,
+};
+use pezsc_network_sync::SyncingService;
+use pezsc_service::{
+	client::Client,
+	config::{
+		BasePath, DatabaseSource, ExecutorConfiguration, KeystoreConfig, RpcBatchRequestConfig,
+		RpcConfiguration,
+	},
+	BlocksPruning, ChainSpecExtension, Configuration, Error, GenericChainSpec, Role,
+	SpawnTaskHandle, TaskManager,
+};
+use pezsc_transaction_pool_api::TransactionPool;
+use pezsp_blockchain::HeaderBackend;
+use pezsp_runtime::traits::Block as BlockT;
+use std::{iter, net::Ipv4Addr, pin::Pin, sync::Arc, task::Context, time::Duration};
+use tempfile::TempDir;
+use tokio::{runtime::Runtime, time};
+
+#[cfg(test)]
+mod client;
+
+/// Maximum duration of single wait call.
+const MAX_WAIT_TIME: Duration = Duration::from_secs(60 * 3);
+
+struct TestNet<E, F, U> {
+	runtime: Runtime,
+	authority_nodes: Vec<(usize, F, U, MultiaddrWithPeerId)>,
+	full_nodes: Vec<(usize, F, U, MultiaddrWithPeerId)>,
+	chain_spec: GenericChainSpec<E>,
+	base_port: u16,
+	nodes: usize,
+}
+
+impl<E, F, U> Drop for TestNet<E, F, U> {
+	fn drop(&mut self) {
+		// Drop the nodes before dropping the runtime, as the runtime otherwise waits for all
+		// futures to be ended and we run into a dead lock.
+		self.full_nodes.drain(..);
+		self.authority_nodes.drain(..);
+	}
+}
+
+pub trait TestNetNode: Clone + Future<Output = Result<(), Error>> + Send + 'static {
+	type Block: BlockT;
+	type Backend: Backend<Self::Block>;
+	type Executor: CallExecutor<Self::Block> + Send + Sync;
+	type RuntimeApi: Send + Sync;
+	type TransactionPool: TransactionPool<Block = Self::Block>;
+
+	fn client(&self) -> Arc<Client<Self::Backend, Self::Executor, Self::Block, Self::RuntimeApi>>;
+	fn transaction_pool(&self) -> Arc<Self::TransactionPool>;
+	fn network(&self) -> Arc<dyn pezsc_network::service::traits::NetworkService>;
+	fn sync(&self) -> &Arc<SyncingService<Self::Block>>;
+	fn spawn_handle(&self) -> SpawnTaskHandle;
+}
+
+pub struct TestNetComponents<TBl: BlockT, TBackend, TExec, TRtApi, TExPool> {
+	task_manager: Arc<Mutex<TaskManager>>,
+	client: Arc<Client<TBackend, TExec, TBl, TRtApi>>,
+	transaction_pool: Arc<TExPool>,
+	network: Arc<dyn pezsc_network::service::traits::NetworkService>,
+	sync: Arc<SyncingService<TBl>>,
+}
+
+impl<TBl: BlockT, TBackend, TExec, TRtApi, TExPool>
+	TestNetComponents<TBl, TBackend, TExec, TRtApi, TExPool>
+{
+	pub fn new(
+		task_manager: TaskManager,
+		client: Arc<Client<TBackend, TExec, TBl, TRtApi>>,
+		network: Arc<dyn pezsc_network::service::traits::NetworkService>,
+		sync: Arc<SyncingService<TBl>>,
+		transaction_pool: Arc<TExPool>,
+	) -> Self {
+		Self {
+			client,
+			sync,
+			transaction_pool,
+			network,
+			task_manager: Arc::new(Mutex::new(task_manager)),
+		}
+	}
+}
+
+impl<TBl: BlockT, TBackend, TExec, TRtApi, TExPool> Clone
+	for TestNetComponents<TBl, TBackend, TExec, TRtApi, TExPool>
+{
+	fn clone(&self) -> Self {
+		Self {
+			task_manager: self.task_manager.clone(),
+			client: self.client.clone(),
+			transaction_pool: self.transaction_pool.clone(),
+			network: self.network.clone(),
+			sync: self.sync.clone(),
+		}
+	}
+}
+
+impl<TBl: BlockT, TBackend, TExec, TRtApi, TExPool> Future
+	for TestNetComponents<TBl, TBackend, TExec, TRtApi, TExPool>
+{
+	type Output = Result<(), Error>;
+
+	fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Self::Output> {
+		Pin::new(&mut self.task_manager.lock().future()).poll(cx)
+	}
+}
+
+impl<TBl, TBackend, TExec, TRtApi, TExPool> TestNetNode
+	for TestNetComponents<TBl, TBackend, TExec, TRtApi, TExPool>
+where
+	TBl: BlockT,
+	TBackend: pezsc_client_api::Backend<TBl> + Send + Sync + 'static,
+	TExec: CallExecutor<TBl> + Send + Sync + 'static,
+	TRtApi: Send + Sync + 'static,
+	TExPool: TransactionPool<Block = TBl> + Send + Sync + 'static,
+{
+	type Block = TBl;
+	type Backend = TBackend;
+	type Executor = TExec;
+	type RuntimeApi = TRtApi;
+	type TransactionPool = TExPool;
+
+	fn client(&self) -> Arc<Client<Self::Backend, Self::Executor, Self::Block, Self::RuntimeApi>> {
+		self.client.clone()
+	}
+	fn transaction_pool(&self) -> Arc<Self::TransactionPool> {
+		self.transaction_pool.clone()
+	}
+	fn network(&self) -> Arc<dyn pezsc_network::service::traits::NetworkService> {
+		self.network.clone()
+	}
+	fn sync(&self) -> &Arc<SyncingService<Self::Block>> {
+		&self.sync
+	}
+	fn spawn_handle(&self) -> SpawnTaskHandle {
+		self.task_manager.lock().spawn_handle()
+	}
+}
+
+impl<E, F, U> TestNet<E, F, U>
+where
+	F: Clone + Send + 'static,
+	U: Clone + Send + 'static,
+{
+	pub fn run_until_all_full<FP>(&mut self, full_predicate: FP)
+	where
+		FP: Send + Fn(usize, &F) -> bool + 'static,
+	{
+		let full_nodes = self.full_nodes.clone();
+		let future = async move {
+			let mut interval = time::interval(Duration::from_millis(100));
+			loop {
+				interval.tick().await;
+
+				if full_nodes.iter().all(|(id, service, _, _)| full_predicate(*id, service)) {
+					break;
+				}
+			}
+		};
+
+		if self
+			.runtime
+			.block_on(async move { time::timeout(MAX_WAIT_TIME, future).await })
+			.is_err()
+		{
+			panic!("Waited for too long");
+		}
+	}
+}
+
+fn node_config<E: ChainSpecExtension + Clone + 'static + Send + Sync>(
+	index: usize,
+	spec: &GenericChainSpec<E>,
+	role: Role,
+	tokio_handle: tokio::runtime::Handle,
+	key_seed: Option<String>,
+	base_port: u16,
+	root: &TempDir,
+) -> Configuration {
+	let root = root.path().join(format!("node-{}", index));
+
+	let mut network_config = NetworkConfiguration::new(
+		format!("Node {}", index),
+		"network/test/0.1",
+		Default::default(),
+		None,
+	);
+
+	network_config.allow_non_globals_in_dht = true;
+
+	network_config.listen_addresses.push(
+		iter::once(multiaddr::Protocol::Ip4(Ipv4Addr::new(127, 0, 0, 1)))
+			.chain(iter::once(multiaddr::Protocol::Tcp(base_port + index as u16)))
+			.collect(),
+	);
+
+	network_config.transport =
+		TransportConfig::Normal { enable_mdns: false, allow_private_ip: true };
+
+	Configuration {
+		impl_name: String::from("network-test-impl"),
+		impl_version: String::from("0.1"),
+		role,
+		tokio_handle,
+		transaction_pool: Default::default(),
+		network: network_config,
+		keystore: KeystoreConfig::Path { path: root.join("key"), password: None },
+		database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128 },
+		trie_cache_maximum_size: Some(16 * 1024 * 1024),
+		warm_up_trie_cache: None,
+		state_pruning: Default::default(),
+		blocks_pruning: BlocksPruning::KeepFinalized,
+		chain_spec: Box::new((*spec).clone()),
+		executor: ExecutorConfiguration::default(),
+		wasm_runtime_overrides: Default::default(),
+		rpc: RpcConfiguration {
+			addr: Default::default(),
+			max_connections: Default::default(),
+			cors: None,
+			methods: Default::default(),
+			max_request_size: Default::default(),
+			max_response_size: Default::default(),
+			id_provider: Default::default(),
+			max_subs_per_conn: Default::default(),
+			port: 9944,
+			message_buffer_capacity: Default::default(),
+			batch_config: RpcBatchRequestConfig::Unlimited,
+			rate_limit: None,
+			rate_limit_whitelisted_ips: Default::default(),
+			rate_limit_trust_proxy_headers: Default::default(),
+			request_logger_limit: 1024,
+		},
+		prometheus_config: None,
+		telemetry_endpoints: None,
+		offchain_worker: Default::default(),
+		force_authoring: false,
+		disable_grandpa: false,
+		dev_key_seed: key_seed,
+		tracing_targets: None,
+		tracing_receiver: Default::default(),
+		announce_block: true,
+		base_path: BasePath::new(root.clone()),
+		data_path: root,
+	}
+}
+
+impl<E, F, U> TestNet<E, F, U>
+where
+	F: TestNetNode,
+	E: ChainSpecExtension + Clone + 'static + Send + Sync,
+{
+	fn new(
+		temp: &TempDir,
+		spec: GenericChainSpec<E>,
+		full: impl Iterator<Item = impl FnOnce(Configuration) -> Result<(F, U), Error>>,
+		authorities: impl Iterator<Item = (String, impl FnOnce(Configuration) -> Result<(F, U), Error>)>,
+		base_port: u16,
+	) -> TestNet<E, F, U> {
+		pezsp_tracing::try_init_simple();
+		fdlimit::raise_fd_limit().unwrap();
+		let runtime = Runtime::new().expect("Error creating tokio runtime");
+		let mut net = TestNet {
+			runtime,
+			authority_nodes: Default::default(),
+			full_nodes: Default::default(),
+			chain_spec: spec,
+			base_port,
+			nodes: 0,
+		};
+		net.insert_nodes(temp, full, authorities);
+		net
+	}
+
+	fn insert_nodes(
+		&mut self,
+		temp: &TempDir,
+		full: impl Iterator<Item = impl FnOnce(Configuration) -> Result<(F, U), Error>>,
+		authorities: impl Iterator<Item = (String, impl FnOnce(Configuration) -> Result<(F, U), Error>)>,
+	) {
+		self.runtime.block_on(async {
+			let handle = self.runtime.handle().clone();
+
+			for (key, authority) in authorities {
+				let node_config = node_config(
+					self.nodes,
+					&self.chain_spec,
+					Role::Authority,
+					handle.clone(),
+					Some(key),
+					self.base_port,
+					temp,
+				);
+				let addr = node_config.network.listen_addresses.first().unwrap().clone();
+				let (service, user_data) =
+					authority(node_config).expect("Error creating test node service");
+
+				handle.spawn(service.clone().map_err(|_| ()));
+				let addr = MultiaddrWithPeerId {
+					multiaddr: addr,
+					peer_id: service.network().local_peer_id(),
+				};
+				self.authority_nodes.push((self.nodes, service, user_data, addr));
+				self.nodes += 1;
+			}
+
+			for full in full {
+				let node_config = node_config(
+					self.nodes,
+					&self.chain_spec,
+					Role::Full,
+					handle.clone(),
+					None,
+					self.base_port,
+					temp,
+				);
+				let addr = node_config.network.listen_addresses.first().unwrap().clone();
+				let (service, user_data) =
+					full(node_config).expect("Error creating test node service");
+
+				handle.spawn(service.clone().map_err(|_| ()));
+				let addr = MultiaddrWithPeerId {
+					multiaddr: addr,
+					peer_id: service.network().local_peer_id(),
+				};
+				self.full_nodes.push((self.nodes, service, user_data, addr));
+				self.nodes += 1;
+			}
+		});
+	}
+}
+
+fn tempdir_with_prefix(prefix: &str) -> TempDir {
+	tempfile::Builder::new()
+		.prefix(prefix)
+		.tempdir()
+		.expect("Error creating test dir")
+}
+
+pub fn connectivity<E, Fb, F>(spec: GenericChainSpec<E>, full_builder: Fb)
+where
+	E: ChainSpecExtension + Clone + 'static + Send + Sync,
+	Fb: Fn(Configuration) -> Result<F, Error>,
+	F: TestNetNode,
+{
+	const NUM_FULL_NODES: usize = 5;
+
+	let expected_full_connections = NUM_FULL_NODES - 1;
+
+	{
+		let temp = tempdir_with_prefix("bizinikiwi-connectivity-test");
+		{
+			let mut network = TestNet::new(
+				&temp,
+				spec.clone(),
+				(0..NUM_FULL_NODES).map(|_| |cfg| full_builder(cfg).map(|s| (s, ()))),
+				// Note: this iterator is empty but we can't just use `iter::empty()`, otherwise
+				// the type of the closure cannot be inferred.
+				(0..0).map(|_| (String::new(), { |cfg| full_builder(cfg).map(|s| (s, ())) })),
+				30400,
+			);
+			info!("Checking star topology");
+			let first_address = network.full_nodes[0].3.clone();
+			for (_, service, _, _) in network.full_nodes.iter().skip(1) {
+				service
+					.network()
+					.add_reserved_peer(first_address.clone())
+					.expect("Error adding reserved peer");
+			}
+
+			network.run_until_all_full(move |_index, service| {
+				let connected = service.network().sync_num_connected();
+				debug!("Got {}/{} full connections...", connected, expected_full_connections);
+				connected == expected_full_connections
+			});
+		};
+
+		temp.close().expect("Error removing temp dir");
+	}
+	{
+		let temp = tempdir_with_prefix("bizinikiwi-connectivity-test");
+		{
+			let mut network = TestNet::new(
+				&temp,
+				spec,
+				(0..NUM_FULL_NODES).map(|_| |cfg| full_builder(cfg).map(|s| (s, ()))),
+				// Note: this iterator is empty but we can't just use `iter::empty()`, otherwise
+				// the type of the closure cannot be inferred.
+				(0..0).map(|_| (String::new(), { |cfg| full_builder(cfg).map(|s| (s, ())) })),
+				30400,
+			);
+			info!("Checking linked topology");
+			let mut address = network.full_nodes[0].3.clone();
+			for i in 0..NUM_FULL_NODES {
+				if i != 0 {
+					if let Some((_, service, _, node_id)) = network.full_nodes.get(i) {
+						service
+							.network()
+							.add_reserved_peer(address)
+							.expect("Error adding reserved peer");
+						address = node_id.clone();
+					}
+				}
+			}
+
+			network.run_until_all_full(move |_index, service| {
+				let connected = service.network().sync_num_connected();
+				debug!("Got {}/{} full connections...", connected, expected_full_connections);
+				connected == expected_full_connections
+			});
+		}
+		temp.close().expect("Error removing temp dir");
+	}
+}
+
+pub fn sync<E, Fb, F, B, ExF, U>(
+	spec: GenericChainSpec<E>,
+	full_builder: Fb,
+	mut make_block_and_import: B,
+	mut extrinsic_factory: ExF,
+) where
+	Fb: Fn(Configuration) -> Result<(F, U), Error>,
+	F: TestNetNode,
+	B: FnMut(&F, &mut U),
+	ExF: FnMut(&F, &U) -> <F::Block as BlockT>::Extrinsic,
+	U: Clone + Send + 'static,
+	E: ChainSpecExtension + Clone + 'static + Send + Sync,
+{
+	const NUM_FULL_NODES: usize = 10;
+	const NUM_BLOCKS: usize = 512;
+	let temp = tempdir_with_prefix("bizinikiwi-sync-test");
+	let mut network = TestNet::new(
+		&temp,
+		spec,
+		(0..NUM_FULL_NODES).map(|_| |cfg| full_builder(cfg)),
+		// Note: this iterator is empty but we can't just use `iter::empty()`, otherwise
+		// the type of the closure cannot be inferred.
+		(0..0).map(|_| (String::new(), { |cfg| full_builder(cfg) })),
+		30500,
+	);
+	info!("Checking block sync");
+	let first_address = {
+		let &mut (_, ref first_service, ref mut first_user_data, _) = &mut network.full_nodes[0];
+		for i in 0..NUM_BLOCKS {
+			if i % 128 == 0 {
+				info!("Generating #{}", i + 1);
+			}
+
+			make_block_and_import(first_service, first_user_data);
+		}
+		let info = network.full_nodes[0].1.client().info();
+		network.full_nodes[0]
+			.1
+			.sync()
+			.new_best_block_imported(info.best_hash, info.best_number);
+		network.full_nodes[0].3.clone()
+	};
+
+	info!("Running sync");
+	for (_, service, _, _) in network.full_nodes.iter().skip(1) {
+		service
+			.network()
+			.add_reserved_peer(first_address.clone())
+			.expect("Error adding reserved peer");
+	}
+
+	network.run_until_all_full(|_index, service| {
+		service.client().info().best_number == (NUM_BLOCKS as u32).into()
+	});
+
+	info!("Checking extrinsic propagation");
+	let first_service = network.full_nodes[0].1.clone();
+	let first_user_data = &network.full_nodes[0].2;
+	let best_block = first_service.client().info().best_hash;
+	let extrinsic = extrinsic_factory(&first_service, first_user_data);
+	let source = pezsc_transaction_pool_api::TransactionSource::External;
+
+	futures::executor::block_on(
+		first_service.transaction_pool().submit_one(best_block, source, extrinsic),
+	)
+	.expect("failed to submit extrinsic");
+
+	network.run_until_all_full(|_index, service| service.transaction_pool().ready().count() == 1);
+}
+
+pub fn consensus<E, Fb, F>(
+	spec: GenericChainSpec<E>,
+	full_builder: Fb,
+	authorities: impl IntoIterator<Item = String>,
+) where
+	Fb: Fn(Configuration) -> Result<F, Error>,
+	F: TestNetNode,
+	E: ChainSpecExtension + Clone + 'static + Send + Sync,
+{
+	const NUM_FULL_NODES: usize = 10;
+	const NUM_BLOCKS: usize = 10; // 10 * 2 sec block production time = ~20 seconds
+	let temp = tempdir_with_prefix("bizinikiwi-consensus-test");
+	let mut network = TestNet::new(
+		&temp,
+		spec,
+		(0..NUM_FULL_NODES / 2).map(|_| |cfg| full_builder(cfg).map(|s| (s, ()))),
+		authorities
+			.into_iter()
+			.map(|key| (key, { |cfg| full_builder(cfg).map(|s| (s, ())) })),
+		30600,
+	);
+
+	info!("Checking consensus");
+	let first_address = network.authority_nodes[0].3.clone();
+	for (_, service, _, _) in network.full_nodes.iter() {
+		service
+			.network()
+			.add_reserved_peer(first_address.clone())
+			.expect("Error adding reserved peer");
+	}
+	for (_, service, _, _) in network.authority_nodes.iter().skip(1) {
+		service
+			.network()
+			.add_reserved_peer(first_address.clone())
+			.expect("Error adding reserved peer");
+	}
+	network.run_until_all_full(|_index, service| {
+		service.client().info().finalized_number >= (NUM_BLOCKS as u32 / 2).into()
+	});
+
+	info!("Adding more peers");
+	network.insert_nodes(
+		&temp,
+		(0..NUM_FULL_NODES / 2).map(|_| |cfg| full_builder(cfg).map(|s| (s, ()))),
+		// Note: this iterator is empty but we can't just use `iter::empty()`, otherwise
+		// the type of the closure cannot be inferred.
+		(0..0).map(|_| (String::new(), { |cfg| full_builder(cfg).map(|s| (s, ())) })),
+	);
+	for (_, service, _, _) in network.full_nodes.iter() {
+		service
+			.network()
+			.add_reserved_peer(first_address.clone())
+			.expect("Error adding reserved peer");
+	}
+
+	network.run_until_all_full(|_index, service| {
+		service.client().info().finalized_number >= (NUM_BLOCKS as u32).into()
+	});
+}