mirror of
https://github.com/pezkuwichain/pezkuwi-subxt.git
synced 2026-04-26 21:37:56 +00:00
Request based collation fetching (#2621)
* Introduce collation fetching protocol also move to mod.rs * Allow `PeerId`s in requests to network bridge. * Fix availability distribution tests. * Move CompressedPoV to primitives. * Request based collator protocol: validator side - Missing: tests - Collator side - don't connect, if not connected * Fixes. * Basic request based collator side. * Minor fix on collator side. * Don't connect in requests in collation protocol. Also some cleanup. * Fix PoV distribution * Bump substrate * Add back metrics + whitespace fixes. * Add back missing spans. * More cleanup. * Guide update. * Fix tests * Handle results in tests. * Fix weird compilation issue. * Add missing ) * Get rid of dead code. * Get rid of redundant import. * Fix runtime build. * Cleanup. * Fix wasm build. * Format fixes. Thanks @andronik !
This commit is contained in:
Robert Klotzner
@@ -0,0 +1,140 @@
|
||||
// Copyright 2021 Parity Technologies (UK) Ltd.
|
||||
// This file is part of Polkadot.
|
||||
|
||||
// Polkadot is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// Polkadot is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
//! Overview over request/responses as used in `Polkadot`.
|
||||
//!
|
||||
//! enum Protocol .... List of all supported protocols.
|
||||
//!
|
||||
//! enum Requests .... List of all supported requests, each entry matches one in protocols, but
|
||||
//! has the actual request as payload.
|
||||
//!
|
||||
//! struct IncomingRequest .... wrapper for incoming requests, containing a sender for sending
|
||||
//! responses.
|
||||
//!
|
||||
//! struct OutgoingRequest .... wrapper for outgoing requests, containing a sender used by the
|
||||
//! networking code for delivering responses/delivery errors.
|
||||
//!
|
||||
//! trait `IsRequest` .... A trait describing a particular request. It is used for gathering meta
|
||||
//! data, like what is the corresponding response type.
|
||||
//!
|
||||
//! Versioned (v1 module): The actual requests and responses as sent over the network.
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::channel::mpsc;
|
||||
use strum::EnumIter;
|
||||
|
||||
pub use sc_network::config as network;
|
||||
pub use sc_network::config::RequestResponseConfig;
|
||||
|
||||
/// All requests that can be sent to the network bridge.
|
||||
pub mod request;
|
||||
pub use request::{IncomingRequest, OutgoingRequest, Requests, Recipient, OutgoingResult};
|
||||
|
||||
///// Multiplexer for incoming requests.
|
||||
// pub mod multiplexer;
|
||||
|
||||
/// Actual versioned requests and responses, that are sent over the wire.
|
||||
pub mod v1;
|
||||
|
||||
/// A protocol per subsystem seems to make the most sense, this way we don't need any dispatching
|
||||
/// within protocols.
|
||||
#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, EnumIter)]
|
||||
pub enum Protocol {
|
||||
/// Protocol for availability fetching, used by availability distribution.
|
||||
AvailabilityFetching,
|
||||
/// Protocol for fetching collations from collators.
|
||||
CollationFetching,
|
||||
}
|
||||
|
||||
/// Default request timeout in seconds.
|
||||
///
|
||||
/// When decreasing this value, take into account that the very first request might need to open a
|
||||
/// connection, which can be slow. If this causes problems, we should ensure connectivity via peer
|
||||
/// sets.
|
||||
const DEFAULT_REQUEST_TIMEOUT: Duration = Duration::from_secs(3);
|
||||
|
||||
/// Request timeout where we can assume the connection is already open (e.g. we have peers in a
|
||||
/// peer set as well).
|
||||
const DEFAULT_REQUEST_TIMEOUT_CONNECTED: Duration = Duration::from_secs(1);
|
||||
|
||||
impl Protocol {
|
||||
/// Get a configuration for a given Request response protocol.
|
||||
///
|
||||
/// Returns a receiver for messages received on this protocol and the requested
|
||||
/// `ProtocolConfig`.
|
||||
///
|
||||
/// See also `dispatcher::RequestDispatcher`, which makes use of this function and provides a more
|
||||
/// high-level interface.
|
||||
pub fn get_config(
|
||||
self,
|
||||
) -> (
|
||||
mpsc::Receiver<network::IncomingRequest>,
|
||||
RequestResponseConfig,
|
||||
) {
|
||||
let p_name = self.into_protocol_name();
|
||||
let (tx, rx) = mpsc::channel(self.get_channel_size());
|
||||
let cfg = match self {
|
||||
Protocol::AvailabilityFetching => RequestResponseConfig {
|
||||
name: p_name,
|
||||
max_request_size: 1_000,
|
||||
max_response_size: 100_000,
|
||||
request_timeout: DEFAULT_REQUEST_TIMEOUT,
|
||||
inbound_queue: Some(tx),
|
||||
},
|
||||
Protocol::CollationFetching => RequestResponseConfig {
|
||||
name: p_name,
|
||||
max_request_size: 1_000,
|
||||
/// Collations are expected to be around 10Meg, probably much smaller with
|
||||
/// compression. So 10Meg should be sufficient, we might be able to reduce this
|
||||
/// further.
|
||||
max_response_size: 10_000_000,
|
||||
// Taken from initial implementation in collator protocol:
|
||||
request_timeout: DEFAULT_REQUEST_TIMEOUT_CONNECTED,
|
||||
inbound_queue: Some(tx),
|
||||
},
|
||||
};
|
||||
(rx, cfg)
|
||||
}
|
||||
|
||||
// Channel sizes for the supported protocols.
|
||||
fn get_channel_size(self) -> usize {
|
||||
match self {
|
||||
// Hundreds of validators will start requesting their chunks once they see a candidate
|
||||
// awaiting availability on chain. Given that they will see that block at different
|
||||
// times (due to network delays), 100 seems big enough to accomodate for "bursts",
|
||||
// assuming we can service requests relatively quickly, which would need to be measured
|
||||
// as well.
|
||||
Protocol::AvailabilityFetching => 100,
|
||||
// 10 seems reasonable, considering group sizes of max 10 validators.
|
||||
Protocol::CollationFetching => 10,
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the protocol name of this protocol, as understood by substrate networking.
|
||||
pub fn into_protocol_name(self) -> Cow<'static, str> {
|
||||
self.get_protocol_name_static().into()
|
||||
}
|
||||
|
||||
/// Get the protocol name associated with each peer set as static str.
|
||||
pub const fn get_protocol_name_static(self) -> &'static str {
|
||||
match self {
|
||||
Protocol::AvailabilityFetching => "/polkadot/req_availability/1",
|
||||
Protocol::CollationFetching => "/polkadot/req_collation/1",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -40,6 +40,8 @@ pub trait IsRequest {
|
||||
pub enum Requests {
|
||||
/// Request an availability chunk from a node.
|
||||
AvailabilityFetching(OutgoingRequest<v1::AvailabilityFetchingRequest>),
|
||||
/// Fetch a collation from a collator which previously announced it.
|
||||
CollationFetching(OutgoingRequest<v1::CollationFetchingRequest>),
|
||||
}
|
||||
|
||||
impl Requests {
|
||||
@@ -47,6 +49,7 @@ impl Requests {
|
||||
pub fn get_protocol(&self) -> Protocol {
|
||||
match self {
|
||||
Self::AvailabilityFetching(_) => Protocol::AvailabilityFetching,
|
||||
Self::CollationFetching(_) => Protocol::CollationFetching,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,10 +63,20 @@ impl Requests {
|
||||
pub fn encode_request(self) -> (Protocol, OutgoingRequest<Vec<u8>>) {
|
||||
match self {
|
||||
Self::AvailabilityFetching(r) => r.encode_request(),
|
||||
Self::CollationFetching(r) => r.encode_request(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Potential recipients of an outgoing request.
|
||||
#[derive(Debug, Eq, Hash, PartialEq)]
|
||||
pub enum Recipient {
|
||||
/// Recipient is a regular peer and we know its peer id.
|
||||
Peer(PeerId),
|
||||
/// Recipient is a validator, we address it via this `AuthorityDiscoveryId`.
|
||||
Authority(AuthorityDiscoveryId),
|
||||
}
|
||||
|
||||
/// A request to be sent to the network bridge, including a sender for sending responses/failures.
|
||||
///
|
||||
/// The network implementation will make use of that sender for informing the requesting subsystem
|
||||
@@ -71,7 +84,7 @@ impl Requests {
|
||||
#[derive(Debug)]
|
||||
pub struct OutgoingRequest<Req> {
|
||||
/// Intendent recipient of this request.
|
||||
pub peer: AuthorityDiscoveryId,
|
||||
pub peer: Recipient,
|
||||
/// The actual request to send over the wire.
|
||||
pub payload: Req,
|
||||
/// Sender which is used by networking to get us back a response.
|
||||
@@ -90,6 +103,9 @@ pub enum RequestError {
|
||||
Canceled(oneshot::Canceled),
|
||||
}
|
||||
|
||||
/// Responses received for an `OutgoingRequest`.
|
||||
pub type OutgoingResult<Res> = Result<Res, RequestError>;
|
||||
|
||||
impl<Req> OutgoingRequest<Req>
|
||||
where
|
||||
Req: IsRequest + Encode,
|
||||
@@ -100,11 +116,11 @@ where
|
||||
/// It will contain a sender that is used by the networking for sending back responses. The
|
||||
/// connected receiver is returned as the second element in the returned tuple.
|
||||
pub fn new(
|
||||
peer: AuthorityDiscoveryId,
|
||||
peer: Recipient,
|
||||
payload: Req,
|
||||
) -> (
|
||||
Self,
|
||||
impl Future<Output = Result<Req::Response, RequestError>>,
|
||||
impl Future<Output = OutgoingResult<Req::Response>>,
|
||||
) {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let r = Self {
|
||||
@@ -201,7 +217,7 @@ where
|
||||
/// Future for actually receiving a typed response for an OutgoingRequest.
|
||||
async fn receive_response<Req>(
|
||||
rec: oneshot::Receiver<Result<Vec<u8>, network::RequestFailure>>,
|
||||
) -> Result<Req::Response, RequestError>
|
||||
) -> OutgoingResult<Req::Response>
|
||||
where
|
||||
Req: IsRequest,
|
||||
Req::Response: Decode,
|
||||
|
||||
@@ -18,7 +18,8 @@
|
||||
|
||||
use parity_scale_codec::{Decode, Encode};
|
||||
|
||||
use polkadot_primitives::v1::{CandidateHash, ErasureChunk, ValidatorIndex};
|
||||
use polkadot_primitives::v1::{CandidateHash, CandidateReceipt, ErasureChunk, ValidatorIndex, CompressedPoV, Hash};
|
||||
use polkadot_primitives::v1::Id as ParaId;
|
||||
|
||||
use super::request::IsRequest;
|
||||
use super::Protocol;
|
||||
@@ -78,3 +79,25 @@ impl IsRequest for AvailabilityFetchingRequest {
|
||||
type Response = AvailabilityFetchingResponse;
|
||||
const PROTOCOL: Protocol = Protocol::AvailabilityFetching;
|
||||
}
|
||||
|
||||
/// Request the advertised collation at that relay-parent.
|
||||
#[derive(Debug, Clone, Encode, Decode)]
|
||||
pub struct CollationFetchingRequest {
|
||||
/// Relay parent we want a collation for.
|
||||
pub relay_parent: Hash,
|
||||
/// The `ParaId` of the collation.
|
||||
pub para_id: ParaId,
|
||||
}
|
||||
|
||||
/// Responses as sent by collators.
|
||||
#[derive(Debug, Clone, Encode, Decode)]
|
||||
pub enum CollationFetchingResponse {
|
||||
/// Deliver requested collation.
|
||||
#[codec(index = 0)]
|
||||
Collation(CandidateReceipt, CompressedPoV),
|
||||
}
|
||||
|
||||
impl IsRequest for CollationFetchingRequest {
|
||||
type Response = CollationFetchingResponse;
|
||||
const PROTOCOL: Protocol = Protocol::CollationFetching;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user