Consensus utilities and rearchitecture for more dynamic collators (#2382)

* implement a proposer utility for consensus

* tidy up deps of new proposer crate

* implement a collator-service crate

* rewrite cumulus-collator to use new service struct

* implement a module for relay-chain-driven collators

* adapt start_collator to use the new relay_chain_driven module

* move collator-service to a public submodule

* create an interface trait for the proposer

* begin aura reimplementation

* address review comments

* update substrrate git ref

* update polkadot-primitives refs

* rough draft of aura collation using standalone fns

* add a ServiceInterface

* port aura reimpl to use new service trait

* add an import queue utility crate

* remove import queue crate in favor of module in common

* implement new verification queue for aura

* implement remaining behaviors

* split 'collate' into smaller functions that could be pub

* add telemetry

* fix doc job?

* Specify async-trait patch version

Co-authored-by: Bastian Köcher <git@kchr.de>

* remove 'fn@' in doc string.

Co-authored-by: Bastian Köcher <git@kchr.de>

* update variable names to be more readable

* refactor proposer errors to anyhow/thiserror

* remove manual span instrumentation

Co-authored-by: Bastian Köcher <git@kchr.de>

* make slot_claim private

* fix unused import

* fmt

* fmt

* make clippy happy

---------

Co-authored-by: Bastian Köcher <git@kchr.de>

client/consensus/proposer/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "cumulus-client-consensus-proposer"
+description = "A Substrate `Proposer` for building parachain blocks"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2021"
+
+[dependencies]
+anyhow = "1.0"
+async-trait = "0.1.68"
+thiserror = "1.0.40"
+
+# Substrate
+sp-consensus = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-inherents = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-state-machine = { git = "https://github.com/paritytech/substrate", branch = "master" }
+
+# Cumulus
+cumulus-primitives-parachain-inherent = { path = "../../../primitives/parachain-inherent" }

client/consensus/proposer/src/lib.rs

@@ -0,0 +1,137 @@
+// Copyright 2023 Parity Technologies (UK) Ltd.
+// This file is part of Cumulus.
+
+// Cumulus is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Cumulus is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Cumulus.  If not, see <http://www.gnu.org/licenses/>.
+
+//! The Cumulus [`Proposer`] is a wrapper around a Substrate [`sp_consensus::Environment`]
+//! for creating new parachain blocks.
+//!
+//! This utility is designed to be composed within any collator consensus algorithm.
+
+use async_trait::async_trait;
+
+use cumulus_primitives_parachain_inherent::ParachainInherentData;
+use sp_consensus::{EnableProofRecording, Environment, Proposal, Proposer as SubstrateProposer};
+use sp_inherents::InherentData;
+use sp_runtime::{traits::Block as BlockT, Digest};
+use sp_state_machine::StorageProof;
+
+use std::{fmt::Debug, time::Duration};
+
+/// Errors that can occur when proposing a parachain block.
+#[derive(thiserror::Error, Debug)]
+#[error(transparent)]
+pub struct Error {
+	inner: anyhow::Error,
+}
+
+impl Error {
+	/// Create an error tied to the creation of a proposer.
+	pub fn proposer_creation(err: impl Into<anyhow::Error>) -> Self {
+		Error { inner: err.into().context("Proposer Creation") }
+	}
+
+	/// Create an error tied to the proposing logic itself.
+	pub fn proposing(err: impl Into<anyhow::Error>) -> Self {
+		Error { inner: err.into().context("Proposing") }
+	}
+}
+
+/// A type alias for easily referring to the type of a proposal produced by a specific
+/// [`Proposer`].
+pub type ProposalOf<B, T> = Proposal<B, <T as ProposerInterface<B>>::Transaction, StorageProof>;
+
+/// An interface for proposers.
+#[async_trait]
+pub trait ProposerInterface<Block: BlockT> {
+	/// The underlying DB transaction type produced with the block proposal.
+	type Transaction: Default + Send + 'static;
+
+	/// Propose a collation using the supplied `InherentData` and the provided
+	/// `ParachainInherentData`.
+	///
+	/// Also specify any required inherent digests, the maximum proposal duration,
+	/// and the block size limit in bytes. See the documentation on [`sp_consensus::Proposer::propose`]
+	/// for more details on how to interpret these parameters.
+	///
+	/// The `InherentData` and `Digest` are left deliberately general in order to accommodate
+	/// all possible collator selection algorithms or inherent creation mechanisms,
+	/// while the `ParachainInherentData` is made explicit so it will be constructed appropriately.
+	///
+	/// If the `InherentData` passed into this function already has a `ParachainInherentData`,
+	/// this should throw an error.
+	async fn propose(
+		&mut self,
+		parent_header: &Block::Header,
+		paras_inherent_data: &ParachainInherentData,
+		other_inherent_data: InherentData,
+		inherent_digests: Digest,
+		max_duration: Duration,
+		block_size_limit: Option<usize>,
+	) -> Result<Proposal<Block, Self::Transaction, StorageProof>, Error>;
+}
+
+/// A simple wrapper around a Substrate proposer for creating collations.
+pub struct Proposer<B, T> {
+	inner: T,
+	_marker: std::marker::PhantomData<B>,
+}
+
+impl<B, T> Proposer<B, T> {
+	/// Create a new Cumulus [`Proposer`].
+	pub fn new(inner: T) -> Self {
+		Proposer { inner, _marker: std::marker::PhantomData }
+	}
+}
+
+#[async_trait]
+impl<B, T> ProposerInterface<B> for Proposer<B, T>
+where
+	B: sp_runtime::traits::Block,
+	T: Environment<B> + Send,
+	T::Error: Send + Sync + 'static,
+	T::Proposer: SubstrateProposer<B, ProofRecording = EnableProofRecording, Proof = StorageProof>,
+	<T::Proposer as SubstrateProposer<B>>::Error: Send + Sync + 'static,
+{
+	type Transaction = <<T as Environment<B>>::Proposer as SubstrateProposer<B>>::Transaction;
+
+	async fn propose(
+		&mut self,
+		parent_header: &B::Header,
+		paras_inherent_data: &ParachainInherentData,
+		other_inherent_data: InherentData,
+		inherent_digests: Digest,
+		max_duration: Duration,
+		block_size_limit: Option<usize>,
+	) -> Result<Proposal<B, Self::Transaction, StorageProof>, Error> {
+		let proposer = self
+			.inner
+			.init(parent_header)
+			.await
+			.map_err(|e| Error::proposer_creation(anyhow::Error::new(e)))?;
+
+		let mut inherent_data = other_inherent_data;
+		inherent_data
+			.put_data(
+				cumulus_primitives_parachain_inherent::INHERENT_IDENTIFIER,
+				&paras_inherent_data,
+			)
+			.map_err(|e| Error::proposing(anyhow::Error::new(e)))?;
+
+		proposer
+			.propose(inherent_data, inherent_digests, max_duration, block_size_limit)
+			.await
+			.map_err(|e| Error::proposing(anyhow::Error::new(e)).into())
+	}
+}