feat: initialize Kurdistan SDK - independent fork of Polkadot SDK

pezkuwi/teyrchain/test-teyrchains/adder/collator/src/cli.rs

@@ -0,0 +1,117 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Pezkuwi CLI library.
+
+use clap::Parser;
+use sc_cli::SubstrateCli;
+use std::path::PathBuf;
+
+/// Sub-commands supported by the collator.
+#[derive(Debug, Parser)]
+pub enum Subcommand {
+	/// Export the genesis state of the teyrchain.
+	#[command(name = "export-genesis-state")]
+	ExportGenesisState(ExportGenesisHeadCommand),
+
+	/// Export the genesis wasm of the teyrchain.
+	#[command(name = "export-genesis-wasm")]
+	ExportGenesisWasm(ExportGenesisWasmCommand),
+}
+
+/// Command for exporting the genesis head data of the teyrchain
+#[derive(Debug, Parser)]
+pub struct ExportGenesisHeadCommand {
+	/// Output file name or stdout if unspecified.
+	#[arg()]
+	pub output: Option<PathBuf>,
+}
+
+/// Command for exporting the genesis wasm file.
+#[derive(Debug, Parser)]
+pub struct ExportGenesisWasmCommand {
+	/// Output file name or stdout if unspecified.
+	#[arg()]
+	pub output: Option<PathBuf>,
+}
+
+#[allow(missing_docs)]
+#[derive(Debug, Parser)]
+#[group(skip)]
+pub struct RunCmd {
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub base: sc_cli::RunCmd,
+
+	/// Id of the teyrchain this collator collates for.
+	#[arg(long)]
+	pub teyrchain_id: Option<u32>,
+}
+
+#[allow(missing_docs)]
+#[derive(Debug, Parser)]
+pub struct Cli {
+	#[command(subcommand)]
+	pub subcommand: Option<Subcommand>,
+
+	#[clap(flatten)]
+	pub run: RunCmd,
+}
+
+impl SubstrateCli for Cli {
+	fn impl_name() -> String {
+		"Parity Pezkuwi".into()
+	}
+
+	fn impl_version() -> String {
+		"0.0.0".into()
+	}
+
+	fn description() -> String {
+		env!("CARGO_PKG_DESCRIPTION").into()
+	}
+
+	fn author() -> String {
+		env!("CARGO_PKG_AUTHORS").into()
+	}
+
+	fn support_url() -> String {
+		"https://github.com/pezkuwichain/pezkuwi-sdk/issues/new".into()
+	}
+
+	fn copyright_start_year() -> i32 {
+		2017
+	}
+
+	fn executable_name() -> String {
+		"adder-collator".into()
+	}
+
+	fn load_spec(&self, id: &str) -> std::result::Result<Box<dyn sc_service::ChainSpec>, String> {
+		let id = if id.is_empty() { "pezkuwichain" } else { id };
+		Ok(match id {
+			"pezkuwichain-staging" =>
+				Box::new(pezkuwi_service::chain_spec::pezkuwichain_staging_testnet_config()?),
+			"pezkuwichain-local" =>
+				Box::new(pezkuwi_service::chain_spec::pezkuwichain_local_testnet_config()?),
+			"pezkuwichain" => Box::new(pezkuwi_service::chain_spec::pezkuwichain_config()?),
+			path => {
+				let path = std::path::PathBuf::from(path);
+				Box::new(pezkuwi_service::PezkuwichainChainSpec::from_json_file(path)?)
+			},
+		})
+	}
+}

pezkuwi/teyrchain/test-teyrchains/adder/collator/src/lib.rs

@@ -0,0 +1,382 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Collator for the adder test teyrchain.
+
+use codec::{Decode, Encode};
+use futures::channel::oneshot;
+use futures_timer::Delay;
+use pezkuwi_node_primitives::{
+	Collation, CollationResult, CollationSecondedSignal, CollatorFn, MaybeCompressedPoV, PoV,
+	Statement,
+};
+use pezkuwi_primitives::{CollatorId, CollatorPair};
+use sp_core::{traits::SpawnNamed, Pair};
+use std::{
+	collections::HashMap,
+	sync::{
+		atomic::{AtomicU32, Ordering},
+		Arc, Mutex,
+	},
+	time::Duration,
+};
+use test_teyrchain_adder::{execute, hash_state, BlockData, HeadData};
+
+/// The amount we add when producing a new block.
+///
+/// This is a constant to make tests easily reproducible.
+const ADD: u64 = 2;
+
+/// Calculates the head and state for the block with the given `number`.
+fn calculate_head_and_state_for_number(number: u64) -> (HeadData, u64) {
+	let mut head =
+		HeadData { number: 0, parent_hash: Default::default(), post_state: hash_state(0) };
+
+	let mut state = 0u64;
+
+	while head.number < number {
+		let block = BlockData { state, add: ADD };
+		head = execute(head.hash(), head.clone(), &block).expect("Produces valid block");
+		state = state.wrapping_add(ADD);
+	}
+
+	(head, state)
+}
+
+/// The state of the adder teyrchain.
+struct State {
+	head_to_state: HashMap<Arc<HeadData>, u64>,
+	number_to_head: HashMap<u64, Arc<HeadData>>,
+	/// Block number of the best block.
+	best_block: u64,
+}
+
+impl State {
+	/// Init the genesis state.
+	fn genesis() -> Self {
+		let genesis_state = Arc::new(calculate_head_and_state_for_number(0).0);
+
+		Self {
+			head_to_state: vec![(genesis_state.clone(), 0)].into_iter().collect(),
+			number_to_head: vec![(0, genesis_state)].into_iter().collect(),
+			best_block: 0,
+		}
+	}
+
+	/// Advance the state and produce a new block based on the given `parent_head`.
+	///
+	/// Returns the new [`BlockData`] and the new [`HeadData`].
+	fn advance(&mut self, parent_head: HeadData) -> (BlockData, HeadData) {
+		self.best_block = parent_head.number;
+
+		let block = BlockData {
+			state: self
+				.head_to_state
+				.get(&parent_head)
+				.copied()
+				.unwrap_or_else(|| calculate_head_and_state_for_number(parent_head.number).1),
+			add: ADD,
+		};
+
+		let new_head =
+			execute(parent_head.hash(), parent_head, &block).expect("Produces valid block");
+
+		let new_head_arc = Arc::new(new_head.clone());
+		self.head_to_state.insert(new_head_arc.clone(), block.state.wrapping_add(ADD));
+		self.number_to_head.insert(new_head.number, new_head_arc);
+
+		(block, new_head)
+	}
+}
+
+/// Local collator state so we can compute how fast we are advancing
+/// per relay parent.
+#[derive(Default)]
+pub struct LocalCollatorState {
+	/// First relay block number on which we've built on.
+	first_relay_parent: Option<u32>,
+	/// Last relay block number on which we've built on.
+	last_relay_parent: Option<u32>,
+}
+
+impl LocalCollatorState {
+	fn advance(&mut self, new_relay_parent: u32, best_block: u64) {
+		match (self.first_relay_parent, self.last_relay_parent) {
+			(Some(first_relay_parent), Some(last_relay_parent)) => {
+				// Compute the teyrchain velocity when relay parent changes vs our last
+				// recorded relay parent. We do this to only print out the velocity
+				// once per relay parent.
+				if new_relay_parent > last_relay_parent {
+					let building_for = (new_relay_parent - first_relay_parent) as f32;
+					// Round it up, as we don't expect perfect runs in CI.
+					let velocity = (best_block as f32 / building_for).ceil() as u32;
+
+					log::info!("Teyrchain velocity: {:}", velocity);
+				}
+			},
+			_ => {},
+		}
+
+		if self.first_relay_parent.is_none() {
+			self.first_relay_parent = Some(new_relay_parent);
+		}
+		self.last_relay_parent = Some(new_relay_parent);
+	}
+}
+
+/// The collator of the adder teyrchain.
+pub struct Collator {
+	state: Arc<Mutex<State>>,
+	key: CollatorPair,
+	seconded_collations: Arc<AtomicU32>,
+	collator_state: Arc<Mutex<LocalCollatorState>>,
+}
+
+impl Collator {
+	/// Create a new collator instance with the state initialized as genesis.
+	pub fn new() -> Self {
+		Self {
+			state: Arc::new(Mutex::new(State::genesis())),
+			key: CollatorPair::generate().0,
+			seconded_collations: Arc::new(AtomicU32::new(0)),
+			collator_state: Default::default(),
+		}
+	}
+
+	/// Get the SCALE encoded genesis head of the adder teyrchain.
+	pub fn genesis_head(&self) -> Vec<u8> {
+		self.state
+			.lock()
+			.unwrap()
+			.number_to_head
+			.get(&0)
+			.expect("Genesis header exists")
+			.encode()
+	}
+
+	/// Get the validation code of the adder teyrchain.
+	pub fn validation_code(&self) -> &[u8] {
+		test_teyrchain_adder::wasm_binary_unwrap()
+	}
+
+	/// Get the collator key.
+	pub fn collator_key(&self) -> CollatorPair {
+		self.key.clone()
+	}
+
+	/// Get the collator id.
+	pub fn collator_id(&self) -> CollatorId {
+		self.key.public()
+	}
+
+	/// Create the collation function.
+	///
+	/// This collation function can be plugged into the overseer to generate collations for the
+	/// adder teyrchain.
+	pub fn create_collation_function(
+		&self,
+		spawner: impl SpawnNamed + Clone + 'static,
+	) -> CollatorFn {
+		use futures::FutureExt as _;
+
+		let state = self.state.clone();
+		let collator_state = self.collator_state.clone();
+		let seconded_collations = self.seconded_collations.clone();
+
+		Box::new(move |relay_parent, validation_data| {
+			let parent = HeadData::decode(&mut &validation_data.parent_head.0[..])
+				.expect("Decodes parent head");
+
+			collator_state
+				.lock()
+				.unwrap()
+				.advance(validation_data.relay_parent_number, parent.number);
+
+			let (block_data, head_data) = state.lock().unwrap().advance(parent);
+
+			log::info!(
+				"created a new collation on relay-parent({}): {:?}",
+				relay_parent,
+				block_data,
+			);
+
+			let pov = PoV { block_data: block_data.encode().into() };
+
+			let collation = Collation {
+				upward_messages: Default::default(),
+				horizontal_messages: Default::default(),
+				new_validation_code: None,
+				head_data: head_data.encode().into(),
+				proof_of_validity: MaybeCompressedPoV::Raw(pov.clone()),
+				processed_downward_messages: 0,
+				hrmp_watermark: validation_data.relay_parent_number,
+			};
+
+			let compressed_pov = pezkuwi_node_primitives::maybe_compress_pov(pov);
+
+			let (result_sender, recv) = oneshot::channel::<CollationSecondedSignal>();
+			let seconded_collations = seconded_collations.clone();
+			spawner.spawn(
+				"adder-collator-seconded",
+				None,
+				async move {
+					if let Ok(res) = recv.await {
+						if !matches!(
+							res.statement.payload(),
+							Statement::Seconded(s) if s.descriptor.pov_hash() == compressed_pov.hash(),
+						) {
+							log::error!(
+								"Seconded statement should match our collation: {:?}",
+								res.statement.payload()
+							);
+							std::process::exit(-1);
+						}
+
+						seconded_collations.fetch_add(1, Ordering::Relaxed);
+					}
+				}
+				.boxed(),
+			);
+
+			async move { Some(CollationResult { collation, result_sender: Some(result_sender) }) }
+				.boxed()
+		})
+	}
+
+	/// Wait until `blocks` are built and enacted.
+	pub async fn wait_for_blocks(&self, blocks: u64) {
+		let start_block = self.state.lock().unwrap().best_block;
+		loop {
+			Delay::new(Duration::from_secs(1)).await;
+
+			let current_block = self.state.lock().unwrap().best_block;
+
+			if start_block + blocks <= current_block {
+				return;
+			}
+		}
+	}
+
+	/// Wait until `seconded` collations of this collator are seconded by a teyrchain validator.
+	///
+	/// The internal counter isn't de-duplicating the collations when counting the number of
+	/// seconded collations. This means when one collation is seconded by X validators, we record X
+	/// seconded messages.
+	pub async fn wait_for_seconded_collations(&self, seconded: u32) {
+		let seconded_collations = self.seconded_collations.clone();
+		loop {
+			Delay::new(Duration::from_secs(1)).await;
+
+			if seconded <= seconded_collations.load(Ordering::Relaxed) {
+				return;
+			}
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	use futures::executor::block_on;
+	use pezkuwi_primitives::PersistedValidationData;
+	use pezkuwi_teyrchain_primitives::primitives::{ValidationParams, ValidationResult};
+
+	#[test]
+	fn collator_works() {
+		let spawner = sp_core::testing::TaskExecutor::new();
+		let collator = Collator::new();
+		let collation_function = collator.create_collation_function(spawner);
+
+		for i in 0..5 {
+			let parent_head =
+				collator.state.lock().unwrap().number_to_head.get(&i).unwrap().clone();
+
+			let validation_data = PersistedValidationData {
+				parent_head: parent_head.encode().into(),
+				..Default::default()
+			};
+
+			let collation =
+				block_on(collation_function(Default::default(), &validation_data)).unwrap();
+			validate_collation(&collator, (*parent_head).clone(), collation.collation);
+		}
+	}
+
+	fn validate_collation(collator: &Collator, parent_head: HeadData, collation: Collation) {
+		use pezkuwi_node_core_pvf::testing::validate_candidate;
+
+		let block_data = match collation.proof_of_validity {
+			MaybeCompressedPoV::Raw(pov) => pov.block_data,
+			MaybeCompressedPoV::Compressed(_) => panic!("Only works with uncompressed povs"),
+		};
+
+		let ret_buf = validate_candidate(
+			collator.validation_code(),
+			&ValidationParams {
+				parent_head: parent_head.encode().into(),
+				block_data,
+				relay_parent_number: 1,
+				relay_parent_storage_root: Default::default(),
+			}
+			.encode(),
+		)
+		.unwrap();
+		let ret = ValidationResult::decode(&mut &ret_buf[..]).unwrap();
+
+		let new_head = HeadData::decode(&mut &ret.head_data.0[..]).unwrap();
+		assert_eq!(
+			**collator
+				.state
+				.lock()
+				.unwrap()
+				.number_to_head
+				.get(&(parent_head.number + 1))
+				.unwrap(),
+			new_head
+		);
+	}
+
+	#[test]
+	fn advance_to_state_when_parent_head_is_missing() {
+		let collator = Collator::new();
+
+		let mut head = calculate_head_and_state_for_number(10).0;
+
+		for i in 1..10 {
+			head = collator.state.lock().unwrap().advance(head).1;
+			assert_eq!(10 + i, head.number);
+		}
+
+		let collator = Collator::new();
+		let mut second_head = collator
+			.state
+			.lock()
+			.unwrap()
+			.number_to_head
+			.get(&0)
+			.cloned()
+			.unwrap()
+			.as_ref()
+			.clone();
+
+		for _ in 1..20 {
+			second_head = collator.state.lock().unwrap().advance(second_head.clone()).1;
+		}
+
+		assert_eq!(second_head, head);
+	}
+}

pezkuwi/teyrchain/test-teyrchains/adder/collator/src/main.rs

@@ -0,0 +1,142 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Collator for the adder test teyrchain.
+
+use pezkuwi_cli::{Error, Result};
+use pezkuwi_node_primitives::CollationGenerationConfig;
+use pezkuwi_node_subsystem::messages::{CollationGenerationMessage, CollatorProtocolMessage};
+use pezkuwi_primitives::Id as ParaId;
+use sc_cli::{Error as SubstrateCliError, SubstrateCli};
+use sp_core::hexdisplay::HexDisplay;
+use std::{
+	collections::HashSet,
+	fs,
+	io::{self, Write},
+};
+use test_teyrchain_adder_collator::Collator;
+
+/// The teyrchain ID to collate for in case it wasn't set explicitly through CLI.
+const DEFAULT_PARA_ID: ParaId = ParaId::new(100);
+
+mod cli;
+use cli::Cli;
+
+fn main() -> Result<()> {
+	let cli = Cli::from_args();
+
+	match cli.subcommand {
+		Some(cli::Subcommand::ExportGenesisState(params)) => {
+			let collator = Collator::new();
+			let output_buf =
+				format!("0x{:?}", HexDisplay::from(&collator.genesis_head())).into_bytes();
+
+			if let Some(output) = params.output {
+				std::fs::write(output, output_buf)?;
+			} else {
+				std::io::stdout().write_all(&output_buf)?;
+			}
+
+			Ok::<_, Error>(())
+		},
+		Some(cli::Subcommand::ExportGenesisWasm(params)) => {
+			let collator = Collator::new();
+			let output_buf =
+				format!("0x{:?}", HexDisplay::from(&collator.validation_code())).into_bytes();
+
+			if let Some(output) = params.output {
+				fs::write(output, output_buf)?;
+			} else {
+				io::stdout().write_all(&output_buf)?;
+			}
+
+			Ok(())
+		},
+		None => {
+			let runner = cli.create_runner(&cli.run.base).map_err(|e| {
+				SubstrateCliError::Application(
+					Box::new(e) as Box<(dyn 'static + Send + Sync + std::error::Error)>
+				)
+			})?;
+
+			runner.run_node_until_exit(|config| async move {
+				let collator = Collator::new();
+
+				let full_node = pezkuwi_service::build_full(
+					config,
+					pezkuwi_service::NewFullParams {
+						is_teyrchain_node: pezkuwi_service::IsTeyrchainNode::Collator(
+							collator.collator_key(),
+						),
+						enable_beefy: false,
+						force_authoring_backoff: false,
+						telemetry_worker_handle: None,
+
+						// Collators don't spawn PVF workers, so we can disable version checks.
+						node_version: None,
+						secure_validator_mode: false,
+						workers_path: None,
+						workers_names: None,
+
+						overseer_gen: pezkuwi_service::CollatorOverseerGen,
+						overseer_message_channel_capacity_override: None,
+						malus_finality_delay: None,
+						hwbench: None,
+						execute_workers_max_num: None,
+						prepare_workers_hard_max_num: None,
+						prepare_workers_soft_max_num: None,
+						keep_finalized_for: None,
+						invulnerable_ah_collators: HashSet::new(),
+						collator_protocol_hold_off: None,
+					},
+				)
+				.map_err(|e| e.to_string())?;
+				let mut overseer_handle = full_node
+					.overseer_handle
+					.expect("Overseer handle should be initialized for collators");
+
+				let genesis_head_hex =
+					format!("0x{:?}", HexDisplay::from(&collator.genesis_head()));
+				let validation_code_hex =
+					format!("0x{:?}", HexDisplay::from(&collator.validation_code()));
+
+				let para_id = cli.run.teyrchain_id.map(ParaId::from).unwrap_or(DEFAULT_PARA_ID);
+
+				log::info!("Running adder collator for teyrchain id: {}", para_id);
+				log::info!("Genesis state: {}", genesis_head_hex);
+				log::info!("Validation code: {}", validation_code_hex);
+
+				let config = CollationGenerationConfig {
+					key: collator.collator_key(),
+					collator: Some(
+						collator.create_collation_function(full_node.task_manager.spawn_handle()),
+					),
+					para_id,
+				};
+				overseer_handle
+					.send_msg(CollationGenerationMessage::Initialize(config), "Collator")
+					.await;
+
+				overseer_handle
+					.send_msg(CollatorProtocolMessage::CollateOn(para_id), "Collator")
+					.await;
+
+				Ok(full_node.task_manager)
+			})
+		},
+	}?;
+	Ok(())
+}