pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-09 20:11:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Network sync refactoring (part 4) (#11412)

Browse Source 
* Remove direct dependency of `sc-network` on `sc-network-light`

* Move `WarpSyncProvider` trait and surrounding data structures into `sc-network-common`

* Move `WarpSyncProvider` trait and surrounding data structures into `sc-network-common`

* Create `sync` module in `sc-network-common`, create `ChainSync` trait there (not used yet), move a bunch of associated data structures from `sc-network-sync`

* Switch from concrete implementation to `ChainSync` trait from `sc-network-common`

* Introduce `OpaqueStateRequest`/`OpaqueStateResponse` to remove generics from `StateSync` trait

* Introduce `OpaqueBlockRequest`/`OpaqueBlockResponse`, make `scheme` module of `sc-network-sync` private

* Surface `sc-network-sync` into `sc-service` and make `sc-network` not depend on it anymore

* Remove now unnecessary dependency from `sc-network`

* Replace crate links with just text since dependencies are gone now

* Remove `warp_sync` re-export from `sc-network-common`

* Update copyright in network-related files

* Address review comments about documentation

* Apply review suggestion

* Rename `extra_requests` module to `metrics`

Co-authored-by: Bastian Köcher <info@kchr.de>
This commit is contained in:
Nazar Mokrynskyi
2022-07-12 23:34:17 +03:00
committed by GitHub
parent 4b8d784210
commit 5896072b86
35 changed files with 1286 additions and 1041 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/Cargo.lock
Generated
+10 -3
View File
@@ -8429,6 +8429,7 @@ dependencies = [
"sc-consensus",
"sc-keystore",
"sc-network",
"sc-network-common",
"sc-network-gossip",
"sc-network-test",
"sc-telemetry",
@@ -8553,7 +8554,6 @@ dependencies = [
"sp-blockchain",
"sp-consensus",
"sp-core",
"sp-finality-grandpa",
"sp-runtime",
"sp-test-primitives",
"sp-tracing",
@@ -8571,12 +8571,17 @@ dependencies = [
name = "sc-network-common"
version = "0.10.0-dev"
dependencies = [
"bitflags",
"futures",
"libp2p",
"parity-scale-codec",
"prost-build",
"sc-consensus",
"sc-peerset",
"smallvec",
"sp-consensus",
"sp-finality-grandpa",
"sp-runtime",
]
[[package]]
@@ -8621,8 +8626,6 @@ dependencies = [
name = "sc-network-sync"
version = "0.10.0-dev"
dependencies = [
"bitflags",
"either",
"fork-tree",
"futures",
"libp2p",
@@ -8667,6 +8670,8 @@ dependencies = [
"sc-consensus",
"sc-network",
"sc-network-common",
"sc-network-light",
"sc-network-sync",
"sc-service",
"sp-blockchain",
"sp-consensus",
@@ -8849,6 +8854,8 @@ dependencies = [
"sc-keystore",
"sc-network",
"sc-network-common",
"sc-network-light",
"sc-network-sync",
"sc-offchain",
"sc-rpc",
"sc-rpc-server",
substrate/client/beefy/src/tests.rs
+2 -12
View File
@@ -28,7 +28,6 @@ use sc_chain_spec::{ChainSpec, GenericChainSpec};
use sc_client_api::HeaderBackend;
use sc_consensus::BoxJustificationImport;
use sc_keystore::LocalKeystore;
use sc_network::config::ProtocolConfig;
use sc_network_test::{
Block, BlockImportAdapter, FullPeerConfig, PassThroughVerifier, Peer, PeersClient,
TestNetFactory,
@@ -111,6 +110,7 @@ pub(crate) struct PeerData {
pub(crate) beefy_link_half: Mutex<Option<BeefyLinkHalf>>,
}
#[derive(Default)]
pub(crate) struct BeefyTestNet {
peers: Vec<BeefyPeer>,
}
@@ -166,17 +166,7 @@ impl TestNetFactory for BeefyTestNet {
type BlockImport = PeersClient;
type PeerData = PeerData;
/// Create new test network with peers and given config.
fn from_config(_config: &ProtocolConfig) -> Self {
BeefyTestNet { peers: Vec::new() }
}
fn make_verifier(
&self,
_client: PeersClient,
_cfg: &ProtocolConfig,
_: &PeerData,
) -> Self::Verifier {
fn make_verifier(&self, _client: PeersClient, _: &PeerData) -> Self::Verifier {
PassThroughVerifier::new(false) // use non-instant finality.
}
substrate/client/consensus/aura/src/lib.rs
+2 -12
View File
@@ -566,7 +566,6 @@ mod tests {
use sc_consensus::BoxJustificationImport;
use sc_consensus_slots::{BackoffAuthoringOnFinalizedHeadLagging, SimpleSlotWorker};
use sc_keystore::LocalKeystore;
use sc_network::config::ProtocolConfig;
use sc_network_test::{Block as TestBlock, *};
use sp_application_crypto::key_types::AURA;
use sp_consensus::{
@@ -645,6 +644,7 @@ mod tests {
>;
type AuraPeer = Peer<(), PeersClient>;
#[derive(Default)]
pub struct AuraTestNet {
peers: Vec<AuraPeer>,
}
@@ -654,17 +654,7 @@ mod tests {
type PeerData = ();
type BlockImport = PeersClient;
/// Create new test network with peers and given config.
fn from_config(_config: &ProtocolConfig) -> Self {
AuraTestNet { peers: Vec::new() }
}
fn make_verifier(
&self,
client: PeersClient,
_cfg: &ProtocolConfig,
_peer_data: &(),
) -> Self::Verifier {
fn make_verifier(&self, client: PeersClient, _peer_data: &()) -> Self::Verifier {
let client = client.as_client();
let slot_duration = slot_duration(&*client).expect("slot duration available");
substrate/client/consensus/babe/src/tests.rs
+2 -13
View File
@@ -29,7 +29,6 @@ use sc_client_api::{backend::TransactionFor, BlockchainEvents, Finalizer};
use sc_consensus::{BoxBlockImport, BoxJustificationImport};
use sc_consensus_slots::BackoffAuthoringOnFinalizedHeadLagging;
use sc_keystore::LocalKeystore;
use sc_network::config::ProtocolConfig;
use sc_network_test::{Block as TestBlock, *};
use sp_application_crypto::key_types::BABE;
use sp_consensus::{AlwaysCanAuthor, DisableProofRecording, NoNetwork as DummyOracle, Proposal};
@@ -220,6 +219,7 @@ where
type BabePeer = Peer<Option<PeerData>, BabeBlockImport>;
#[derive(Default)]
pub struct BabeTestNet {
peers: Vec<BabePeer>,
}
@@ -278,12 +278,6 @@ impl TestNetFactory for BabeTestNet {
type PeerData = Option<PeerData>;
type BlockImport = BabeBlockImport;
/// Create new test network with peers and given config.
fn from_config(_config: &ProtocolConfig) -> Self {
debug!(target: "babe", "Creating test network from config");
BabeTestNet { peers: Vec::new() }
}
fn make_block_import(
&self,
client: PeersClient,
@@ -309,12 +303,7 @@ impl TestNetFactory for BabeTestNet {
)
}
fn make_verifier(
&self,
client: PeersClient,
_cfg: &ProtocolConfig,
maybe_link: &Option<PeerData>,
) -> Self::Verifier {
fn make_verifier(&self, client: PeersClient, maybe_link: &Option<PeerData>) -> Self::Verifier {
use substrate_test_runtime_client::DefaultTestClientBuilderExt;
let client = client.as_client();
substrate/client/finality-grandpa/Cargo.toml
+1
View File
@@ -36,6 +36,7 @@ sc-consensus = { version = "0.10.0-dev", path = "../consensus/common" }
sc-keystore = { version = "4.0.0-dev", path = "../keystore" }
sc-network = { version = "0.10.0-dev", path = "../network" }
sc-network-gossip = { version = "0.10.0-dev", path = "../network-gossip" }
sc-network-common = { version = "0.10.0-dev", path = "../network/common" }
sc-telemetry = { version = "4.0.0-dev", path = "../telemetry" }
sc-utils = { version = "4.0.0-dev", path = "../utils" }
sp-api = { version = "4.0.0-dev", path = "../../primitives/api" }
substrate/client/finality-grandpa/src/tests.rs
+3 -17
View File
@@ -28,7 +28,7 @@ use sc_consensus::{
BlockImport, BlockImportParams, BoxJustificationImport, ForkChoiceStrategy, ImportResult,
ImportedAux,
};
use sc_network::config::{ProtocolConfig, Role};
use sc_network::config::Role;
use sc_network_test::{
Block, BlockImportAdapter, FullPeerConfig, Hash, PassThroughVerifier, Peer, PeersClient,
PeersFullClient, TestClient, TestNetFactory,
@@ -73,6 +73,7 @@ type GrandpaBlockImport = crate::GrandpaBlockImport<
LongestChain<substrate_test_runtime_client::Backend, Block>,
>;
#[derive(Default)]
struct GrandpaTestNet {
peers: Vec<GrandpaPeer>,
test_config: TestApi,
@@ -110,16 +111,6 @@ impl TestNetFactory for GrandpaTestNet {
type PeerData = PeerData;
type BlockImport = GrandpaBlockImport;
/// Create new test network with peers and given config.
fn from_config(_config: &ProtocolConfig) -> Self {
GrandpaTestNet { peers: Vec::new(), test_config: Default::default() }
}
fn default_config() -> ProtocolConfig {
// This is unused.
ProtocolConfig::default()
}
fn add_full_peer(&mut self) {
self.add_full_peer_with_config(FullPeerConfig {
notifications_protocols: vec![grandpa_protocol_name::NAME.into()],
@@ -128,12 +119,7 @@ impl TestNetFactory for GrandpaTestNet {
})
}
fn make_verifier(
&self,
_client: PeersClient,
_cfg: &ProtocolConfig,
_: &PeerData,
) -> Self::Verifier {
fn make_verifier(&self, _client: PeersClient, _: &PeerData) -> Self::Verifier {
PassThroughVerifier::new(false) // use non-instant finality.
}
substrate/client/finality-grandpa/src/warp_proof.rs
+1 -1
View File
@@ -23,7 +23,7 @@ use crate::{
BlockNumberOps, GrandpaJustification, SharedAuthoritySet,
};
use sc_client_api::Backend as ClientBackend;
use sc_network::warp_request_handler::{EncodedProof, VerificationResult, WarpSyncProvider};
use sc_network_common::sync::warp::{EncodedProof, VerificationResult, WarpSyncProvider};
use sp_blockchain::{Backend as BlockchainBackend, HeaderBackend};
use sp_finality_grandpa::{AuthorityList, SetId, GRANDPA_ENGINE_ID};
use sp_runtime::{
substrate/client/network/Cargo.toml
+2 -3
View File
@@ -51,15 +51,12 @@ sc-block-builder = { version = "0.10.0-dev", path = "../block-builder" }
sc-client-api = { version = "4.0.0-dev", path = "../api" }
sc-consensus = { version = "0.10.0-dev", path = "../consensus/common" }
sc-network-common = { version = "0.10.0-dev", path = "./common" }
sc-network-light = { version = "0.10.0-dev", path = "./light" }
sc-network-sync = { version = "0.10.0-dev", path = "./sync" }
sc-peerset = { version = "4.0.0-dev", path = "../peerset" }
sc-utils = { version = "4.0.0-dev", path = "../utils" }
sp-arithmetic = { version = "5.0.0", path = "../../primitives/arithmetic" }
sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" }
sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/common" }
sp-core = { version = "6.0.0", path = "../../primitives/core" }
sp-finality-grandpa = { version = "4.0.0-dev", path = "../../primitives/finality-grandpa" }
sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" }
[dev-dependencies]
@@ -67,6 +64,8 @@ assert_matches = "1.3"
async-std = "1.11.0"
rand = "0.7.2"
tempfile = "3.1.0"
sc-network-light = { version = "0.10.0-dev", path = "./light" }
sc-network-sync = { version = "0.10.0-dev", path = "./sync" }
sp-test-primitives = { version = "2.0.0", path = "../../primitives/test-primitives" }
sp-tracing = { version = "5.0.0", path = "../../primitives/tracing" }
substrate-test-runtime = { version = "2.0.0", path = "../../test-utils/runtime" }
substrate/client/network/common/Cargo.toml
+5
View File
@@ -17,10 +17,15 @@ targets = ["x86_64-unknown-linux-gnu"]
prost-build = "0.10"
[dependencies]
bitflags = "1.3.2"
codec = { package = "parity-scale-codec", version = "3.0.0", features = [
"derive",
] }
futures = "0.3.21"
libp2p = "0.46.1"
smallvec = "1.8.0"
sc-consensus = { version = "0.10.0-dev", path = "../../consensus/common" }
sc-peerset = { version = "4.0.0-dev", path = "../../peerset" }
sp-consensus = { version = "0.10.0-dev", path = "../../../primitives/consensus/common" }
sp-finality-grandpa = { version = "4.0.0-dev", path = "../../../primitives/finality-grandpa" }
sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" }
substrate/client/network/common/src/lib.rs
+1
View File
@@ -21,3 +21,4 @@
pub mod config;
pub mod message;
pub mod request_responses;
pub mod sync;
substrate/client/network/common/src/sync.rs
+394
View File
@@ -0,0 +1,394 @@
// This file is part of Substrate.
// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//! Abstract interfaces and data structures related to network sync.
pub mod message;
pub mod metrics;
pub mod warp;
use libp2p::PeerId;
use message::{BlockAnnounce, BlockData, BlockRequest, BlockResponse};
use sc_consensus::{BlockImportError, BlockImportStatus, IncomingBlock};
use sp_consensus::BlockOrigin;
use sp_runtime::{
traits::{Block as BlockT, NumberFor},
Justifications,
};
use std::{any::Any, fmt, fmt::Formatter, task::Poll};
use warp::{EncodedProof, WarpProofRequest, WarpSyncProgress};
/// The sync status of a peer we are trying to sync with
#[derive(Debug)]
pub struct PeerInfo<Block: BlockT> {
/// Their best block hash.
pub best_hash: Block::Hash,
/// Their best block number.
pub best_number: NumberFor<Block>,
}
/// Reported sync state.
#[derive(Clone, Eq, PartialEq, Debug)]
pub enum SyncState {
/// Initial sync is complete, keep-up sync is active.
Idle,
/// Actively catching up with the chain.
Downloading,
}
/// Reported state download progress.
#[derive(Clone, Eq, PartialEq, Debug)]
pub struct StateDownloadProgress {
/// Estimated download percentage.
pub percentage: u32,
/// Total state size in bytes downloaded so far.
pub size: u64,
}
/// Syncing status and statistics.
#[derive(Clone)]
pub struct SyncStatus<Block: BlockT> {
/// Current global sync state.
pub state: SyncState,
/// Target sync block number.
pub best_seen_block: Option<NumberFor<Block>>,
/// Number of peers participating in syncing.
pub num_peers: u32,
/// Number of blocks queued for import
pub queued_blocks: u32,
/// State sync status in progress, if any.
pub state_sync: Option<StateDownloadProgress>,
/// Warp sync in progress, if any.
pub warp_sync: Option<WarpSyncProgress<Block>>,
}
/// A peer did not behave as expected and should be reported.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BadPeer(pub PeerId, pub sc_peerset::ReputationChange);
impl fmt::Display for BadPeer {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "Bad peer {}; Reputation change: {:?}", self.0, self.1)
}
}
impl std::error::Error for BadPeer {}
/// Result of [`ChainSync::on_block_data`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum OnBlockData<Block: BlockT> {
/// The block should be imported.
Import(BlockOrigin, Vec<IncomingBlock<Block>>),
/// A new block request needs to be made to the given peer.
Request(PeerId, BlockRequest<Block>),
}
/// Result of [`ChainSync::on_block_justification`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum OnBlockJustification<Block: BlockT> {
/// The justification needs no further handling.
Nothing,
/// The justification should be imported.
Import {
peer: PeerId,
hash: Block::Hash,
number: NumberFor<Block>,
justifications: Justifications,
},
}
/// Result of [`ChainSync::on_state_data`].
#[derive(Debug)]
pub enum OnStateData<Block: BlockT> {
/// The block and state that should be imported.
Import(BlockOrigin, IncomingBlock<Block>),
/// A new state request needs to be made to the given peer.
Continue,
}
/// Result of [`ChainSync::poll_block_announce_validation`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PollBlockAnnounceValidation<H> {
/// The announcement failed at validation.
///
/// The peer reputation should be decreased.
Failure {
/// Who sent the processed block announcement?
who: PeerId,
/// Should the peer be disconnected?
disconnect: bool,
},
/// The announcement does not require further handling.
Nothing {
/// Who sent the processed block announcement?
who: PeerId,
/// Was this their new best block?
is_best: bool,
/// The announcement.
announce: BlockAnnounce<H>,
},
/// The announcement header should be imported.
ImportHeader {
/// Who sent the processed block announcement?
who: PeerId,
/// Was this their new best block?
is_best: bool,
/// The announcement.
announce: BlockAnnounce<H>,
},
/// The block announcement should be skipped.
Skip,
}
/// Operation mode.
#[derive(Debug, PartialEq, Eq)]
pub enum SyncMode {
// Sync headers only
Light,
// Sync headers and block bodies
Full,
// Sync headers and the last finalied state
LightState { storage_chain_mode: bool, skip_proofs: bool },
// Warp sync mode.
Warp,
}
#[derive(Debug)]
pub struct Metrics {
pub queued_blocks: u32,
pub fork_targets: u32,
pub justifications: metrics::Metrics,
}
/// Wrapper for implementation-specific state request.
///
/// NOTE: Implementation must be able to encode and decode it for network purposes.
pub struct OpaqueStateRequest(pub Box<dyn Any + Send>);
impl fmt::Debug for OpaqueStateRequest {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
f.debug_struct("OpaqueStateRequest").finish()
}
}
/// Wrapper for implementation-specific state response.
///
/// NOTE: Implementation must be able to encode and decode it for network purposes.
pub struct OpaqueStateResponse(pub Box<dyn Any + Send>);
impl fmt::Debug for OpaqueStateResponse {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
f.debug_struct("OpaqueStateResponse").finish()
}
}
/// Wrapper for implementation-specific block request.
///
/// NOTE: Implementation must be able to encode and decode it for network purposes.
pub struct OpaqueBlockRequest(pub Box<dyn Any + Send>);
impl fmt::Debug for OpaqueBlockRequest {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
f.debug_struct("OpaqueBlockRequest").finish()
}
}
/// Wrapper for implementation-specific block response.
///
/// NOTE: Implementation must be able to encode and decode it for network purposes.
pub struct OpaqueBlockResponse(pub Box<dyn Any + Send>);
impl fmt::Debug for OpaqueBlockResponse {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
f.debug_struct("OpaqueBlockResponse").finish()
}
}
/// Something that represents the syncing strategy to download past and future blocks of the chain.
pub trait ChainSync<Block: BlockT>: Send {
/// Returns the state of the sync of the given peer.
///
/// Returns `None` if the peer is unknown.
fn peer_info(&self, who: &PeerId) -> Option<PeerInfo<Block>>;
/// Returns the current sync status.
fn status(&self) -> SyncStatus<Block>;
/// Number of active forks requests. This includes
/// requests that are pending or could be issued right away.
fn num_sync_requests(&self) -> usize;
/// Number of downloaded blocks.
fn num_downloaded_blocks(&self) -> usize;
/// Returns the current number of peers stored within this state machine.
fn num_peers(&self) -> usize;
/// Handle a new connected peer.
///
/// Call this method whenever we connect to a new peer.
fn new_peer(
&mut self,
who: PeerId,
best_hash: Block::Hash,
best_number: NumberFor<Block>,
) -> Result<Option<BlockRequest<Block>>, BadPeer>;
/// Signal that a new best block has been imported.
fn update_chain_info(&mut self, best_hash: &Block::Hash, best_number: NumberFor<Block>);
/// Schedule a justification request for the given block.
fn request_justification(&mut self, hash: &Block::Hash, number: NumberFor<Block>);
/// Clear all pending justification requests.
fn clear_justification_requests(&mut self);
/// Request syncing for the given block from given set of peers.
fn set_sync_fork_request(
&mut self,
peers: Vec<PeerId>,
hash: &Block::Hash,
number: NumberFor<Block>,
);
/// Get an iterator over all scheduled justification requests.
fn justification_requests(
&mut self,
) -> Box<dyn Iterator<Item = (PeerId, BlockRequest<Block>)> + '_>;
/// Get an iterator over all block requests of all peers.
fn block_requests(&mut self) -> Box<dyn Iterator<Item = (&PeerId, BlockRequest<Block>)> + '_>;
/// Get a state request, if any.
fn state_request(&mut self) -> Option<(PeerId, OpaqueStateRequest)>;
/// Get a warp sync request, if any.
fn warp_sync_request(&mut self) -> Option<(PeerId, WarpProofRequest<Block>)>;
/// Handle a response from the remote to a block request that we made.
///
/// `request` must be the original request that triggered `response`.
/// or `None` if data comes from the block announcement.
///
/// If this corresponds to a valid block, this outputs the block that
/// must be imported in the import queue.
fn on_block_data(
&mut self,
who: &PeerId,
request: Option<BlockRequest<Block>>,
response: BlockResponse<Block>,
) -> Result<OnBlockData<Block>, BadPeer>;
/// Handle a response from the remote to a state request that we made.
fn on_state_data(
&mut self,
who: &PeerId,
response: OpaqueStateResponse,
) -> Result<OnStateData<Block>, BadPeer>;
/// Handle a response from the remote to a warp proof request that we made.
fn on_warp_sync_data(&mut self, who: &PeerId, response: EncodedProof) -> Result<(), BadPeer>;
/// Handle a response from the remote to a justification request that we made.
///
/// `request` must be the original request that triggered `response`.
fn on_block_justification(
&mut self,
who: PeerId,
response: BlockResponse<Block>,
) -> Result<OnBlockJustification<Block>, BadPeer>;
/// A batch of blocks have been processed, with or without errors.
///
/// Call this when a batch of blocks have been processed by the import
/// queue, with or without errors.
fn on_blocks_processed(
&mut self,
imported: usize,
count: usize,
results: Vec<(Result<BlockImportStatus<NumberFor<Block>>, BlockImportError>, Block::Hash)>,
) -> Box<dyn Iterator<Item = Result<(PeerId, BlockRequest<Block>), BadPeer>>>;
/// Call this when a justification has been processed by the import queue,
/// with or without errors.
fn on_justification_import(
&mut self,
hash: Block::Hash,
number: NumberFor<Block>,
success: bool,
);
/// Notify about finalization of the given block.
fn on_block_finalized(&mut self, hash: &Block::Hash, number: NumberFor<Block>);
/// Push a block announce validation.
///
/// It is required that [`ChainSync::poll_block_announce_validation`] is called
/// to check for finished block announce validations.
fn push_block_announce_validation(
&mut self,
who: PeerId,
hash: Block::Hash,
announce: BlockAnnounce<Block::Header>,
is_best: bool,
);
/// Poll block announce validation.
///
/// Block announce validations can be pushed by using
/// [`ChainSync::push_block_announce_validation`].
///
/// This should be polled until it returns [`Poll::Pending`].
///
/// If [`PollBlockAnnounceValidation::ImportHeader`] is returned, then the caller MUST try to
/// import passed header (call `on_block_data`). The network request isn't sent in this case.
fn poll_block_announce_validation(
&mut self,
cx: &mut std::task::Context,
) -> Poll<PollBlockAnnounceValidation<Block::Header>>;
/// Call when a peer has disconnected.
/// Canceled obsolete block request may result in some blocks being ready for
/// import, so this functions checks for such blocks and returns them.
fn peer_disconnected(&mut self, who: &PeerId) -> Option<OnBlockData<Block>>;
/// Return some key metrics.
fn metrics(&self) -> Metrics;
/// Create implementation-specific block request.
fn create_opaque_block_request(&self, request: &BlockRequest<Block>) -> OpaqueBlockRequest;
/// Encode implementation-specific block request.
fn encode_block_request(&self, request: &OpaqueBlockRequest) -> Result<Vec<u8>, String>;
/// Decode implementation-specific block response.
fn decode_block_response(&self, response: &[u8]) -> Result<OpaqueBlockResponse, String>;
/// Access blocks from implementation-specific block response.
fn block_response_into_blocks(
&self,
request: &BlockRequest<Block>,
response: OpaqueBlockResponse,
) -> Result<Vec<BlockData<Block>>, String>;
/// Encode implementation-specific state request.
fn encode_state_request(&self, request: &OpaqueStateRequest) -> Result<Vec<u8>, String>;
/// Decode implementation-specific state response.
fn decode_state_response(&self, response: &[u8]) -> Result<OpaqueStateResponse, String>;
}
substrate/client/network/sync/src/message.rs → substrate/client/network/common/src/sync/message.rs
+1 -1
View File
@@ -124,8 +124,8 @@ impl<H: HeaderT> BlockAnnounce<H> {
/// Generic types.
pub mod generic {
use super::{BlockAttributes, BlockState, Direction};
use crate::message::RequestId;
use codec::{Decode, Encode, Input, Output};
use sc_network_common::message::RequestId;
use sp_runtime::{EncodedJustification, Justifications};
/// Block data sent in the response.
substrate/client/network/common/src/sync/metrics.rs
+25
View File
@@ -0,0 +1,25 @@
// This file is part of Substrate.
// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
#[derive(Debug)]
pub struct Metrics {
pub pending_requests: u32,
pub active_requests: u32,
pub importing_requests: u32,
pub failed_requests: u32,
}
substrate/client/network/common/src/sync/warp.rs
+94
View File
@@ -0,0 +1,94 @@
// Copyright 2022 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
use codec::{Decode, Encode};
pub use sp_finality_grandpa::{AuthorityList, SetId};
use sp_runtime::traits::{Block as BlockT, NumberFor};
use std::fmt;
/// Scale-encoded warp sync proof response.
pub struct EncodedProof(pub Vec<u8>);
/// Warp sync request
#[derive(Encode, Decode, Debug)]
pub struct WarpProofRequest<B: BlockT> {
/// Start collecting proofs from this block.
pub begin: B::Hash,
}
/// Proof verification result.
pub enum VerificationResult<Block: BlockT> {
/// Proof is valid, but the target was not reached.
Partial(SetId, AuthorityList, Block::Hash),
/// Target finality is proved.
Complete(SetId, AuthorityList, Block::Header),
}
/// Warp sync backend. Handles retrieveing and verifying warp sync proofs.
pub trait WarpSyncProvider<Block: BlockT>: Send + Sync {
/// Generate proof starting at given block hash. The proof is accumulated until maximum proof
/// size is reached.
fn generate(
&self,
start: Block::Hash,
) -> Result<EncodedProof, Box<dyn std::error::Error + Send + Sync>>;
/// Verify warp proof against current set of authorities.
fn verify(
&self,
proof: &EncodedProof,
set_id: SetId,
authorities: AuthorityList,
) -> Result<VerificationResult<Block>, Box<dyn std::error::Error + Send + Sync>>;
/// Get current list of authorities. This is supposed to be genesis authorities when starting
/// sync.
fn current_authorities(&self) -> AuthorityList;
}
/// Reported warp sync phase.
#[derive(Clone, Eq, PartialEq, Debug)]
pub enum WarpSyncPhase<Block: BlockT> {
/// Waiting for peers to connect.
AwaitingPeers,
/// Downloading and verifying grandpa warp proofs.
DownloadingWarpProofs,
/// Downloading state data.
DownloadingState,
/// Importing state.
ImportingState,
/// Downloading block history.
DownloadingBlocks(NumberFor<Block>),
}
impl<Block: BlockT> fmt::Display for WarpSyncPhase<Block> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Self::AwaitingPeers => write!(f, "Waiting for peers"),
Self::DownloadingWarpProofs => write!(f, "Downloading finality proofs"),
Self::DownloadingState => write!(f, "Downloading state"),
Self::ImportingState => write!(f, "Importing state"),
Self::DownloadingBlocks(n) => write!(f, "Downloading block history (#{})", n),
}
}
}
/// Reported warp sync progress.
#[derive(Clone, Eq, PartialEq, Debug)]
pub struct WarpSyncProgress<Block: BlockT> {
/// Estimated download percentage.
pub phase: WarpSyncPhase<Block>,
/// Total bytes downloaded so far.
pub total_bytes: u64,
}
substrate/client/network/src/behaviour.rs
+35 -33
View File
@@ -38,7 +38,7 @@ use libp2p::{
NetworkBehaviour,
};
use log::debug;
use prost::Message;
use sc_client_api::{BlockBackend, ProofProvider};
use sc_consensus::import_queue::{IncomingBlock, Origin};
use sc_network_common::{config::ProtocolId, request_responses::ProtocolConfig};
@@ -382,42 +382,44 @@ where
.events
.push_back(BehaviourOut::JustificationImport(origin, hash, nb, justification)),
CustomMessageOutcome::BlockRequest { target, request, pending_response } => {
let mut buf = Vec::with_capacity(request.encoded_len());
if let Err(err) = request.encode(&mut buf) {
log::warn!(
target: "sync",
"Failed to encode block request {:?}: {:?}",
request, err
);
return
match self.substrate.encode_block_request(&request) {
Ok(data) => {
self.request_responses.send_request(
&target,
&self.block_request_protocol_name,
data,
pending_response,
IfDisconnected::ImmediateError,
);
},
Err(err) => {
log::warn!(
target: "sync",
"Failed to encode block request {:?}: {:?}",
request, err
);
},
}
self.request_responses.send_request(
&target,
&self.block_request_protocol_name,
buf,
pending_response,
IfDisconnected::ImmediateError,
);
},
CustomMessageOutcome::StateRequest { target, request, pending_response } => {
let mut buf = Vec::with_capacity(request.encoded_len());
if let Err(err) = request.encode(&mut buf) {
log::warn!(
target: "sync",
"Failed to encode state request {:?}: {:?}",
request, err
);
return
match self.substrate.encode_state_request(&request) {
Ok(data) => {
self.request_responses.send_request(
&target,
&self.state_request_protocol_name,
data,
pending_response,
IfDisconnected::ImmediateError,
);
},
Err(err) => {
log::warn!(
target: "sync",
"Failed to encode state request {:?}: {:?}",
request, err
);
},
}
self.request_responses.send_request(
&target,
&self.state_request_protocol_name,
buf,
pending_response,
IfDisconnected::ImmediateError,
);
},
CustomMessageOutcome::WarpSyncRequest { target, request, pending_response } =>
match &self.warp_sync_protocol_name {
substrate/client/network/src/bitswap.rs
+2 -3
View File
@@ -1,4 +1,4 @@
// Copyright 2021 Parity Technologies (UK) Ltd.
// Copyright 2022 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
@@ -118,8 +118,7 @@ where
fn upgrade_outbound(self, mut socket: TSocket, _info: Self::Info) -> Self::Future {
Box::pin(async move {
let mut data = Vec::with_capacity(self.encoded_len());
self.encode(&mut data)?;
let data = self.encode_to_vec();
upgrade::write_length_prefixed(&mut socket, data).await
})
}
substrate/client/network/src/config.rs
+17 -16
View File
@@ -26,16 +26,11 @@ pub use sc_network_common::{
request_responses::{
IncomingRequest, OutgoingResponse, ProtocolConfig as RequestResponseConfig,
},
sync::warp::WarpSyncProvider,
};
pub use sc_network_sync::warp_request_handler::WarpSyncProvider;
pub use libp2p::{build_multiaddr, core::PublicKey, identity};
// Note: this re-export shouldn't be part of the public API of the crate and will be removed in
// the future.
#[doc(hidden)]
pub use crate::protocol::ProtocolConfig;
use crate::ExHashT;
use core::{fmt, iter};
@@ -46,7 +41,7 @@ use libp2p::{
};
use prometheus_endpoint::Registry;
use sc_consensus::ImportQueue;
use sp_consensus::block_validation::BlockAnnounceValidator;
use sc_network_common::sync::ChainSync;
use sp_runtime::traits::Block as BlockT;
use std::{
borrow::Cow,
@@ -101,8 +96,14 @@ where
/// valid.
pub import_queue: Box<dyn ImportQueue<B>>,
/// Type to check incoming block announcements.
pub block_announce_validator: Box<dyn BlockAnnounceValidator<B> + Send>,
/// Factory function that creates a new instance of chain sync.
pub create_chain_sync: Box<
dyn FnOnce(
sc_network_common::sync::SyncMode,
Arc<Client>,
Option<Arc<dyn WarpSyncProvider<B>>>,
) -> crate::error::Result<Box<dyn ChainSync<B>>>,
>,
/// Registry for recording prometheus metrics to.
pub metrics_registry: Option<Registry>,
@@ -114,26 +115,26 @@ where
/// block requests, if enabled.
///
/// Can be constructed either via
/// [`sc_network_sync::block_request_handler::generate_protocol_config`] allowing outgoing but
/// not incoming requests, or constructed via [`sc_network_sync::block_request_handler::
/// BlockRequestHandler::new`] allowing both outgoing and incoming requests.
/// `sc_network_sync::block_request_handler::generate_protocol_config` allowing outgoing but
/// not incoming requests, or constructed via `sc_network_sync::block_request_handler::
/// BlockRequestHandler::new` allowing both outgoing and incoming requests.
pub block_request_protocol_config: RequestResponseConfig,
/// Request response configuration for the light client request protocol.
///
/// Can be constructed either via
/// [`sc_network_light::light_client_requests::generate_protocol_config`] allowing outgoing but
/// `sc_network_light::light_client_requests::generate_protocol_config` allowing outgoing but
/// not incoming requests, or constructed via
/// [`sc_network_light::light_client_requests::handler::LightClientRequestHandler::new`]
/// `sc_network_light::light_client_requests::handler::LightClientRequestHandler::new`
/// allowing both outgoing and incoming requests.
pub light_client_request_protocol_config: RequestResponseConfig,
/// Request response configuration for the state request protocol.
///
/// Can be constructed either via
/// [`sc_network_sync::block_request_handler::generate_protocol_config`] allowing outgoing but
/// `sc_network_sync::state_request_handler::generate_protocol_config` allowing outgoing but
/// not incoming requests, or constructed via
/// [`crate::state_request_handler::StateRequestHandler::new`] allowing
/// `sc_network_sync::state_request_handler::StateRequestHandler::new` allowing
/// both outgoing and incoming requests.
pub state_request_protocol_config: RequestResponseConfig,
substrate/client/network/src/lib.rs
+2 -6
View File
@@ -266,13 +266,9 @@ pub use protocol::{
event::{DhtEvent, Event, ObservedRole},
PeerInfo,
};
pub use sc_network_light::light_client_requests;
pub use sc_network_sync::{
block_request_handler,
state::StateDownloadProgress,
state_request_handler,
pub use sc_network_common::sync::{
warp::{WarpSyncPhase, WarpSyncProgress},
warp_request_handler, SyncState,
StateDownloadProgress, SyncState,
};
pub use service::{
DecodingError, IfDisconnected, KademliaKey, Keypair, NetworkService, NetworkWorker,
substrate/client/network/src/protocol.rs
+111 -199
View File
@@ -20,7 +20,6 @@ use crate::{
config, error,
request_responses::RequestFailure,
utils::{interval, LruHashSet},
warp_request_handler::{EncodedProof, WarpSyncProvider},
};
use bytes::Bytes;
@@ -42,21 +41,23 @@ use message::{
};
use notifications::{Notifications, NotificationsOut};
use prometheus_endpoint::{register, Gauge, GaugeVec, Opts, PrometheusError, Registry, U64};
use prost::Message as _;
use sc_client_api::{BlockBackend, HeaderBackend, ProofProvider};
use sc_consensus::import_queue::{BlockImportError, BlockImportStatus, IncomingBlock, Origin};
use sc_network_common::config::ProtocolId;
use sc_network_sync::{
message::{
BlockAnnounce, BlockAttributes, BlockData, BlockRequest, BlockResponse, BlockState,
FromBlock,
use sc_network_common::{
config::ProtocolId,
sync::{
message::{
BlockAnnounce, BlockAttributes, BlockData, BlockRequest, BlockResponse, BlockState,
},
warp::{EncodedProof, WarpProofRequest},
BadPeer, ChainSync, OnBlockData, OnBlockJustification, OnStateData, OpaqueBlockRequest,
OpaqueBlockResponse, OpaqueStateRequest, OpaqueStateResponse, PollBlockAnnounceValidation,
SyncStatus,
},
schema::v1::StateResponse,
BadPeer, ChainSync, OnBlockData, OnBlockJustification, OnStateData,
PollBlockAnnounceValidation, Status as SyncStatus,
};
use sp_arithmetic::traits::SaturatedConversion;
use sp_consensus::{block_validation::BlockAnnounceValidator, BlockOrigin};
use sp_blockchain::HeaderMetadata;
use sp_consensus::BlockOrigin;
use sp_runtime::{
generic::BlockId,
traits::{Block as BlockT, CheckedSub, Header as HeaderT, NumberFor, Zero},
@@ -79,7 +80,6 @@ pub mod event;
pub mod message;
pub use notifications::{NotificationsSink, NotifsHandlerError, Ready};
use sp_blockchain::HeaderMetadata;
/// Interval at which we perform time based maintenance
const TICK_TIMEOUT: time::Duration = time::Duration::from_millis(1100);
@@ -167,11 +167,12 @@ pub struct Protocol<B: BlockT, Client> {
tick_timeout: Pin<Box<dyn Stream<Item = ()> + Send>>,
/// Pending list of messages to return from `poll` as a priority.
pending_messages: VecDeque<CustomMessageOutcome<B>>,
config: ProtocolConfig,
/// Assigned roles.
roles: Roles,
genesis_hash: B::Hash,
/// State machine that handles the list of in-progress requests. Only full node peers are
/// registered.
sync: ChainSync<B, Client>,
chain_sync: Box<dyn ChainSync<B>>,
// All connected peers. Contains both full and light node peers.
peers: HashMap<PeerId, Peer<B>>,
chain: Arc<Client>,
@@ -231,38 +232,6 @@ pub struct PeerInfo<B: BlockT> {
pub best_number: <B::Header as HeaderT>::Number,
}
/// Configuration for the Substrate-specific part of the networking layer.
#[derive(Clone)]
pub struct ProtocolConfig {
/// Assigned roles.
pub roles: Roles,
/// Maximum number of peers to ask the same blocks in parallel.
pub max_parallel_downloads: u32,
/// Enable state sync.
pub sync_mode: config::SyncMode,
}
impl ProtocolConfig {
fn sync_mode(&self) -> sc_network_sync::SyncMode {
if self.roles.is_light() {
sc_network_sync::SyncMode::Light
} else {
match self.sync_mode {
config::SyncMode::Full => sc_network_sync::SyncMode::Full,
config::SyncMode::Fast { skip_proofs, storage_chain_mode } =>
sc_network_sync::SyncMode::LightState { skip_proofs, storage_chain_mode },
config::SyncMode::Warp => sc_network_sync::SyncMode::Warp,
}
}
}
}
impl Default for ProtocolConfig {
fn default() -> ProtocolConfig {
Self { roles: Roles::FULL, max_parallel_downloads: 5, sync_mode: config::SyncMode::Full }
}
}
/// Handshake sent when we open a block announces substream.
#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
struct BlockAnnouncesHandshake<B: BlockT> {
@@ -278,12 +247,12 @@ struct BlockAnnouncesHandshake<B: BlockT> {
impl<B: BlockT> BlockAnnouncesHandshake<B> {
fn build(
protocol_config: &ProtocolConfig,
roles: Roles,
best_number: NumberFor<B>,
best_hash: B::Hash,
genesis_hash: B::Hash,
) -> Self {
Self { genesis_hash, roles: protocol_config.roles, best_number, best_hash }
Self { genesis_hash, roles, best_number, best_hash }
}
}
@@ -300,24 +269,15 @@ where
{
/// Create a new instance.
pub fn new(
config: ProtocolConfig,
roles: Roles,
chain: Arc<Client>,
protocol_id: ProtocolId,
network_config: &config::NetworkConfiguration,
notifications_protocols_handshakes: Vec<Vec<u8>>,
block_announce_validator: Box<dyn BlockAnnounceValidator<B> + Send>,
metrics_registry: Option<&Registry>,
warp_sync_provider: Option<Arc<dyn WarpSyncProvider<B>>>,
) -> error::Result<(Protocol<B, Client>, sc_peerset::PeersetHandle, Vec<(PeerId, Multiaddr)>)> {
chain_sync: Box<dyn ChainSync<B>>,
) -> error::Result<(Self, sc_peerset::PeersetHandle, Vec<(PeerId, Multiaddr)>)> {
let info = chain.info();
let sync = ChainSync::new(
config.sync_mode(),
chain.clone(),
block_announce_validator,
config.max_parallel_downloads,
warp_sync_provider,
)
.map_err(Box::new)?;
let boot_node_ids = {
let mut list = HashSet::new();
@@ -405,7 +365,7 @@ where
let genesis_hash = info.genesis_hash;
let block_announces_handshake =
BlockAnnouncesHandshake::<B>::build(&config, best_number, best_hash, genesis_hash)
BlockAnnouncesHandshake::<B>::build(roles, best_number, best_hash, genesis_hash)
.encode();
let sync_protocol_config = notifications::ProtocolConfig {
@@ -438,11 +398,11 @@ where
let protocol = Self {
tick_timeout: Box::pin(interval(TICK_TIMEOUT)),
pending_messages: VecDeque::new(),
config,
roles,
peers: HashMap::new(),
chain,
genesis_hash: info.genesis_hash,
sync,
chain_sync,
important_peers,
default_peers_set_num_full: network_config.default_peers_set_num_full as usize,
default_peers_set_num_light: {
@@ -510,49 +470,49 @@ where
/// Current global sync state.
pub fn sync_state(&self) -> SyncStatus<B> {
self.sync.status()
self.chain_sync.status()
}
/// Target sync block number.
pub fn best_seen_block(&self) -> Option<NumberFor<B>> {
self.sync.status().best_seen_block
self.chain_sync.status().best_seen_block
}
/// Number of peers participating in syncing.
pub fn num_sync_peers(&self) -> u32 {
self.sync.status().num_peers
self.chain_sync.status().num_peers
}
/// Number of blocks in the import queue.
pub fn num_queued_blocks(&self) -> u32 {
self.sync.status().queued_blocks
self.chain_sync.status().queued_blocks
}
/// Number of downloaded blocks.
pub fn num_downloaded_blocks(&self) -> usize {
self.sync.num_downloaded_blocks()
self.chain_sync.num_downloaded_blocks()
}
/// Number of active sync requests.
pub fn num_sync_requests(&self) -> usize {
self.sync.num_sync_requests()
self.chain_sync.num_sync_requests()
}
/// Inform sync about new best imported block.
pub fn new_best_block_imported(&mut self, hash: B::Hash, number: NumberFor<B>) {
debug!(target: "sync", "New best block imported {:?}/#{}", hash, number);
self.sync.update_chain_info(&hash, number);
self.chain_sync.update_chain_info(&hash, number);
self.behaviour.set_notif_protocol_handshake(
HARDCODED_PEERSETS_SYNC,
BlockAnnouncesHandshake::<B>::build(&self.config, number, hash, self.genesis_hash)
BlockAnnouncesHandshake::<B>::build(self.roles, number, hash, self.genesis_hash)
.encode(),
);
}
fn update_peer_info(&mut self, who: &PeerId) {
if let Some(info) = self.sync.peer_info(who) {
if let Some(info) = self.chain_sync.peer_info(who) {
if let Some(ref mut peer) = self.peers.get_mut(who) {
peer.info.best_hash = info.best_hash;
peer.info.best_number = info.best_number;
@@ -565,14 +525,6 @@ where
self.peers.iter().map(|(id, peer)| (id, &peer.info))
}
fn prepare_block_request(
&mut self,
who: PeerId,
request: BlockRequest<B>,
) -> CustomMessageOutcome<B> {
prepare_block_request::<B>(&mut self.peers, who, request)
}
/// Called by peer when it is disconnecting.
///
/// Returns a result if the handshake of this peer was indeed accepted.
@@ -584,7 +536,9 @@ where
}
if let Some(_peer_data) = self.peers.remove(&peer) {
if let Some(OnBlockData::Import(origin, blocks)) = self.sync.peer_disconnected(&peer) {
if let Some(OnBlockData::Import(origin, blocks)) =
self.chain_sync.peer_disconnected(&peer)
{
self.pending_messages
.push_back(CustomMessageOutcome::BlockImport(origin, blocks));
}
@@ -605,62 +559,9 @@ where
&mut self,
peer_id: PeerId,
request: BlockRequest<B>,
response: sc_network_sync::schema::v1::BlockResponse,
response: OpaqueBlockResponse,
) -> CustomMessageOutcome<B> {
let blocks = response
.blocks
.into_iter()
.map(|block_data| {
Ok(BlockData::<B> {
hash: Decode::decode(&mut block_data.hash.as_ref())?,
header: if !block_data.header.is_empty() {
Some(Decode::decode(&mut block_data.header.as_ref())?)
} else {
None
},
body: if request.fields.contains(BlockAttributes::BODY) {
Some(
block_data
.body
.iter()
.map(|body| Decode::decode(&mut body.as_ref()))
.collect::<Result<Vec<_>, _>>()?,
)
} else {
None
},
indexed_body: if request.fields.contains(BlockAttributes::INDEXED_BODY) {
Some(block_data.indexed_body)
} else {
None
},
receipt: if !block_data.receipt.is_empty() {
Some(block_data.receipt)
} else {
None
},
message_queue: if !block_data.message_queue.is_empty() {
Some(block_data.message_queue)
} else {
None
},
justification: if !block_data.justification.is_empty() {
Some(block_data.justification)
} else if block_data.is_empty_justification {
Some(Vec::new())
} else {
None
},
justifications: if !block_data.justifications.is_empty() {
Some(DecodeAll::decode_all(&mut block_data.justifications.as_ref())?)
} else {
None
},
})
})
.collect::<Result<Vec<_>, codec::Error>>();
let blocks = match blocks {
let blocks = match self.chain_sync.block_response_into_blocks(&request, response) {
Ok(blocks) => blocks,
Err(err) => {
debug!(target: "sync", "Failed to decode block response from {}: {}", peer_id, err);
@@ -690,7 +591,7 @@ where
);
if request.fields == BlockAttributes::JUSTIFICATION {
match self.sync.on_block_justification(peer_id, block_response) {
match self.chain_sync.on_block_justification(peer_id, block_response) {
Ok(OnBlockJustification::Nothing) => CustomMessageOutcome::None,
Ok(OnBlockJustification::Import { peer, hash, number, justifications }) =>
CustomMessageOutcome::JustificationImport(peer, hash, number, justifications),
@@ -701,10 +602,11 @@ where
},
}
} else {
match self.sync.on_block_data(&peer_id, Some(request), block_response) {
match self.chain_sync.on_block_data(&peer_id, Some(request), block_response) {
Ok(OnBlockData::Import(origin, blocks)) =>
CustomMessageOutcome::BlockImport(origin, blocks),
Ok(OnBlockData::Request(peer, req)) => self.prepare_block_request(peer, req),
Ok(OnBlockData::Request(peer, req)) =>
prepare_block_request(self.chain_sync.as_ref(), &mut self.peers, peer, req),
Err(BadPeer(id, repu)) => {
self.behaviour.disconnect_peer(&id, HARDCODED_PEERSETS_SYNC);
self.peerset_handle.report_peer(id, repu);
@@ -719,9 +621,9 @@ where
pub fn on_state_response(
&mut self,
peer_id: PeerId,
response: StateResponse,
response: OpaqueStateResponse,
) -> CustomMessageOutcome<B> {
match self.sync.on_state_data(&peer_id, response) {
match self.chain_sync.on_state_data(&peer_id, response) {
Ok(OnStateData::Import(origin, block)) =>
CustomMessageOutcome::BlockImport(origin, vec![block]),
Ok(OnStateData::Continue) => CustomMessageOutcome::None,
@@ -738,9 +640,9 @@ where
pub fn on_warp_sync_response(
&mut self,
peer_id: PeerId,
response: crate::warp_request_handler::EncodedProof,
response: EncodedProof,
) -> CustomMessageOutcome<B> {
match self.sync.on_warp_sync_data(&peer_id, response) {
match self.chain_sync.on_warp_sync_data(&peer_id, response) {
Ok(()) => CustomMessageOutcome::None,
Err(BadPeer(id, repu)) => {
self.behaviour.disconnect_peer(&id, HARDCODED_PEERSETS_SYNC);
@@ -798,7 +700,7 @@ where
return Err(())
}
if self.config.roles.is_light() {
if self.roles.is_light() {
// we're not interested in light peers
if status.roles.is_light() {
debug!(target: "sync", "Peer {} is unable to serve light requests", who);
@@ -821,14 +723,15 @@ where
}
}
if status.roles.is_full() && self.sync.num_peers() >= self.default_peers_set_num_full {
if status.roles.is_full() && self.chain_sync.num_peers() >= self.default_peers_set_num_full
{
debug!(target: "sync", "Too many full nodes, rejecting {}", who);
self.behaviour.disconnect_peer(&who, HARDCODED_PEERSETS_SYNC);
return Err(())
}
if status.roles.is_light() &&
(self.peers.len() - self.sync.num_peers()) >= self.default_peers_set_num_light
(self.peers.len() - self.chain_sync.num_peers()) >= self.default_peers_set_num_light
{
// Make sure that not all slots are occupied by light clients.
debug!(target: "sync", "Too many light nodes, rejecting {}", who);
@@ -849,7 +752,7 @@ where
};
let req = if peer.info.roles.is_full() {
match self.sync.new_peer(who, peer.info.best_hash, peer.info.best_number) {
match self.chain_sync.new_peer(who, peer.info.best_hash, peer.info.best_number) {
Ok(req) => req,
Err(BadPeer(id, repu)) => {
self.behaviour.disconnect_peer(&id, HARDCODED_PEERSETS_SYNC);
@@ -868,8 +771,12 @@ where
.push_back(CustomMessageOutcome::PeerNewBest(who, status.best_number));
if let Some(req) = req {
let event = self.prepare_block_request(who, req);
self.pending_messages.push_back(event);
self.pending_messages.push_back(prepare_block_request(
self.chain_sync.as_ref(),
&mut self.peers,
who,
req,
));
}
Ok(())
@@ -953,7 +860,7 @@ where
};
if peer.info.roles.is_full() {
self.sync.push_block_announce_validation(who, hash, announce, is_best);
self.chain_sync.push_block_announce_validation(who, hash, announce, is_best);
}
}
@@ -1010,7 +917,7 @@ where
// to import header from announced block let's construct response to request that normally
// would have been sent over network (but it is not in our case)
let blocks_to_import = self.sync.on_block_data(
let blocks_to_import = self.chain_sync.on_block_data(
&who,
None,
BlockResponse::<B> {
@@ -1035,7 +942,8 @@ where
match blocks_to_import {
Ok(OnBlockData::Import(origin, blocks)) =>
CustomMessageOutcome::BlockImport(origin, blocks),
Ok(OnBlockData::Request(peer, req)) => self.prepare_block_request(peer, req),
Ok(OnBlockData::Request(peer, req)) =>
prepare_block_request(self.chain_sync.as_ref(), &mut self.peers, peer, req),
Err(BadPeer(id, repu)) => {
self.behaviour.disconnect_peer(&id, HARDCODED_PEERSETS_SYNC);
self.peerset_handle.report_peer(id, repu);
@@ -1047,7 +955,7 @@ where
/// Call this when a block has been finalized. The sync layer may have some additional
/// requesting to perform.
pub fn on_block_finalized(&mut self, hash: B::Hash, header: &B::Header) {
self.sync.on_block_finalized(&hash, *header.number())
self.chain_sync.on_block_finalized(&hash, *header.number())
}
/// Request a justification for the given block.
@@ -1055,12 +963,12 @@ where
/// Uses `protocol` to queue a new justification request and tries to dispatch all pending
/// requests.
pub fn request_justification(&mut self, hash: &B::Hash, number: NumberFor<B>) {
self.sync.request_justification(hash, number)
self.chain_sync.request_justification(hash, number)
}
/// Clear all pending justification requests.
pub fn clear_justification_requests(&mut self) {
self.sync.clear_justification_requests();
self.chain_sync.clear_justification_requests();
}
/// Request syncing for the given block from given set of peers.
@@ -1072,7 +980,7 @@ where
hash: &B::Hash,
number: NumberFor<B>,
) {
self.sync.set_sync_fork_request(peers, hash, number)
self.chain_sync.set_sync_fork_request(peers, hash, number)
}
/// A batch of blocks have been processed, with or without errors.
@@ -1084,11 +992,12 @@ where
count: usize,
results: Vec<(Result<BlockImportStatus<NumberFor<B>>, BlockImportError>, B::Hash)>,
) {
let results = self.sync.on_blocks_processed(imported, count, results);
let results = self.chain_sync.on_blocks_processed(imported, count, results);
for result in results {
match result {
Ok((id, req)) => {
self.pending_messages.push_back(prepare_block_request(
self.chain_sync.as_ref(),
&mut self.peers,
id,
req,
@@ -1111,7 +1020,7 @@ where
number: NumberFor<B>,
success: bool,
) {
self.sync.on_justification_import(hash, number, success);
self.chain_sync.on_justification_import(hash, number, success);
if !success {
info!("💔 Invalid justification provided by {} for #{}", who, hash);
self.behaviour.disconnect_peer(&who, HARDCODED_PEERSETS_SYNC);
@@ -1228,12 +1137,22 @@ where
}
}
/// Encode implementation-specific block request.
pub fn encode_block_request(&self, request: &OpaqueBlockRequest) -> Result<Vec<u8>, String> {
self.chain_sync.encode_block_request(request)
}
/// Encode implementation-specific state request.
pub fn encode_state_request(&self, request: &OpaqueStateRequest) -> Result<Vec<u8>, String> {
self.chain_sync.encode_state_request(request)
}
fn report_metrics(&self) {
if let Some(metrics) = &self.metrics {
let n = u64::try_from(self.peers.len()).unwrap_or(std::u64::MAX);
metrics.peers.set(n);
let m = self.sync.metrics();
let m = self.chain_sync.metrics();
metrics.fork_targets.set(m.fork_targets.into());
metrics.queued_blocks.set(m.queued_blocks.into());
@@ -1259,6 +1178,7 @@ where
}
fn prepare_block_request<B: BlockT>(
chain_sync: &dyn ChainSync<B>,
peers: &mut HashMap<PeerId, Peer<B>>,
who: PeerId,
request: BlockRequest<B>,
@@ -1269,19 +1189,7 @@ fn prepare_block_request<B: BlockT>(
peer.request = Some((PeerRequest::Block(request.clone()), rx));
}
let request = sc_network_sync::schema::v1::BlockRequest {
fields: request.fields.to_be_u32(),
from_block: match request.from {
FromBlock::Hash(h) =>
Some(sc_network_sync::schema::v1::block_request::FromBlock::Hash(h.encode())),
FromBlock::Number(n) =>
Some(sc_network_sync::schema::v1::block_request::FromBlock::Number(n.encode())),
},
to_block: request.to.map(|h| h.encode()).unwrap_or_default(),
direction: request.direction as i32,
max_blocks: request.max.unwrap_or(0),
support_multiple_justifications: true,
};
let request = chain_sync.create_opaque_block_request(&request);
CustomMessageOutcome::BlockRequest { target: who, request, pending_response: tx }
}
@@ -1289,7 +1197,7 @@ fn prepare_block_request<B: BlockT>(
fn prepare_state_request<B: BlockT>(
peers: &mut HashMap<PeerId, Peer<B>>,
who: PeerId,
request: sc_network_sync::schema::v1::StateRequest,
request: OpaqueStateRequest,
) -> CustomMessageOutcome<B> {
let (tx, rx) = oneshot::channel();
@@ -1302,7 +1210,7 @@ fn prepare_state_request<B: BlockT>(
fn prepare_warp_sync_request<B: BlockT>(
peers: &mut HashMap<PeerId, Peer<B>>,
who: PeerId,
request: crate::warp_request_handler::Request<B>,
request: WarpProofRequest<B>,
) -> CustomMessageOutcome<B> {
let (tx, rx) = oneshot::channel();
@@ -1346,19 +1254,19 @@ pub enum CustomMessageOutcome<B: BlockT> {
/// A new block request must be emitted.
BlockRequest {
target: PeerId,
request: sc_network_sync::schema::v1::BlockRequest,
request: OpaqueBlockRequest,
pending_response: oneshot::Sender<Result<Vec<u8>, RequestFailure>>,
},
/// A new storage request must be emitted.
StateRequest {
target: PeerId,
request: sc_network_sync::schema::v1::StateRequest,
request: OpaqueStateRequest,
pending_response: oneshot::Sender<Result<Vec<u8>, RequestFailure>>,
},
/// A new warp sync request must be emitted.
WarpSyncRequest {
target: PeerId,
request: crate::warp_request_handler::Request<B>,
request: WarpProofRequest<B>,
pending_response: oneshot::Sender<Result<Vec<u8>, RequestFailure>>,
},
/// Peer has a reported a new head of chain.
@@ -1455,10 +1363,8 @@ where
let (req, _) = peer.request.take().unwrap();
match req {
PeerRequest::Block(req) => {
let protobuf_response =
match sc_network_sync::schema::v1::BlockResponse::decode(
&resp[..],
) {
let response =
match self.chain_sync.decode_block_response(&resp[..]) {
Ok(proto) => proto,
Err(e) => {
debug!(
@@ -1474,13 +1380,11 @@ where
},
};
finished_block_requests.push((*id, req, protobuf_response));
finished_block_requests.push((*id, req, response));
},
PeerRequest::State => {
let protobuf_response =
match sc_network_sync::schema::v1::StateResponse::decode(
&resp[..],
) {
let response =
match self.chain_sync.decode_state_response(&resp[..]) {
Ok(proto) => proto,
Err(e) => {
debug!(
@@ -1496,7 +1400,7 @@ where
},
};
finished_state_requests.push((*id, protobuf_response));
finished_state_requests.push((*id, response));
},
PeerRequest::WarpProof => {
finished_warp_sync_requests.push((*id, resp));
@@ -1555,12 +1459,12 @@ where
}
}
}
for (id, req, protobuf_response) in finished_block_requests {
let ev = self.on_block_response(id, req, protobuf_response);
for (id, req, response) in finished_block_requests {
let ev = self.on_block_response(id, req, response);
self.pending_messages.push_back(ev);
}
for (id, protobuf_response) in finished_state_requests {
let ev = self.on_state_response(id, protobuf_response);
for (id, response) in finished_state_requests {
let ev = self.on_state_response(id, response);
self.pending_messages.push_back(ev);
}
for (id, response) in finished_warp_sync_requests {
@@ -1572,25 +1476,32 @@ where
self.tick();
}
for (id, request) in self.sync.block_requests() {
let event = prepare_block_request(&mut self.peers, *id, request);
for (id, request) in self
.chain_sync
.block_requests()
.map(|(peer_id, request)| (*peer_id, request))
.collect::<Vec<_>>()
{
let event =
prepare_block_request(self.chain_sync.as_ref(), &mut self.peers, id, request);
self.pending_messages.push_back(event);
}
if let Some((id, request)) = self.sync.state_request() {
if let Some((id, request)) = self.chain_sync.state_request() {
let event = prepare_state_request(&mut self.peers, id, request);
self.pending_messages.push_back(event);
}
for (id, request) in self.sync.justification_requests() {
let event = prepare_block_request(&mut self.peers, id, request);
for (id, request) in self.chain_sync.justification_requests().collect::<Vec<_>>() {
let event =
prepare_block_request(self.chain_sync.as_ref(), &mut self.peers, id, request);
self.pending_messages.push_back(event);
}
if let Some((id, request)) = self.sync.warp_sync_request() {
if let Some((id, request)) = self.chain_sync.warp_sync_request() {
let event = prepare_warp_sync_request(&mut self.peers, id, request);
self.pending_messages.push_back(event);
}
// Check if there is any block announcement validation finished.
while let Poll::Ready(result) = self.sync.poll_block_announce_validation(cx) {
while let Poll::Ready(result) = self.chain_sync.poll_block_announce_validation(cx) {
match self.process_block_announce_validation_result(result) {
CustomMessageOutcome::None => {},
outcome => self.pending_messages.push_back(outcome),
@@ -1771,7 +1682,8 @@ where
// Make sure that the newly added block announce validation future was
// polled once to be registered in the task.
if let Poll::Ready(res) = self.sync.poll_block_announce_validation(cx) {
if let Poll::Ready(res) = self.chain_sync.poll_block_announce_validation(cx)
{
self.process_block_announce_validation_result(res)
} else {
CustomMessageOutcome::None
substrate/client/network/src/protocol/message.rs
+6 -4
View File
@@ -63,10 +63,12 @@ pub mod generic {
use bitflags::bitflags;
use codec::{Decode, Encode, Input, Output};
use sc_client_api::StorageProof;
use sc_network_common::message::RequestId;
use sc_network_sync::message::{
generic::{BlockRequest, BlockResponse},
BlockAnnounce,
use sc_network_common::{
message::RequestId,
sync::message::{
generic::{BlockRequest, BlockResponse},
BlockAnnounce,
},
};
use sp_runtime::ConsensusEngineId;
substrate/client/network/src/service.rs
+18 -9
View File
@@ -30,7 +30,7 @@
use crate::{
behaviour::{self, Behaviour, BehaviourOut},
bitswap::Bitswap,
config::{parse_str_addr, Params, TransportConfig},
config::{self, parse_str_addr, Params, TransportConfig},
discovery::DiscoveryConfig,
error::Error,
network_state::{
@@ -60,7 +60,7 @@ use metrics::{Histogram, HistogramVec, MetricSources, Metrics};
use parking_lot::Mutex;
use sc_client_api::{BlockBackend, ProofProvider};
use sc_consensus::{BlockImportError, BlockImportStatus, ImportQueue, Link};
use sc_network_sync::{Status as SyncStatus, SyncState};
use sc_network_common::sync::{SyncMode, SyncState, SyncStatus};
use sc_peerset::PeersetHandle;
use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender};
use sp_blockchain::{HeaderBackend, HeaderMetadata};
@@ -207,13 +207,23 @@ where
None => (None, None),
};
let (protocol, peerset_handle, mut known_addresses) = Protocol::new(
protocol::ProtocolConfig {
roles: From::from(&params.role),
max_parallel_downloads: params.network_config.max_parallel_downloads,
sync_mode: params.network_config.sync_mode.clone(),
let chain_sync = (params.create_chain_sync)(
if params.role.is_light() {
SyncMode::Light
} else {
match params.network_config.sync_mode {
config::SyncMode::Full => SyncMode::Full,
config::SyncMode::Fast { skip_proofs, storage_chain_mode } =>
SyncMode::LightState { skip_proofs, storage_chain_mode },
config::SyncMode::Warp => SyncMode::Warp,
}
},
params.chain.clone(),
warp_sync_provider,
)?;
let (protocol, peerset_handle, mut known_addresses) = Protocol::new(
From::from(&params.role),
params.chain.clone(),
params.protocol_id.clone(),
&params.network_config,
iter::once(Vec::new())
@@ -222,9 +232,8 @@ where
.map(|_| default_notif_handshake_message.clone()),
)
.collect(),
params.block_announce_validator,
params.metrics_registry.as_ref(),
warp_sync_provider,
chain_sync,
)?;
// List of multiaddresses that we know in the network.
substrate/client/network/src/service/tests.rs
+18 -6
View File
@@ -16,15 +16,17 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
config, state_request_handler::StateRequestHandler, Event, NetworkService, NetworkWorker,
};
use crate::{config, Event, NetworkService, NetworkWorker};
use futures::prelude::*;
use libp2p::PeerId;
use sc_network_common::config::ProtocolId;
use sc_network_light::light_client_requests::handler::LightClientRequestHandler;
use sc_network_sync::block_request_handler::BlockRequestHandler;
use sc_network_sync::{
block_request_handler::BlockRequestHandler, state_request_handler::StateRequestHandler,
ChainSync,
};
use sp_consensus::block_validation::DefaultBlockAnnounceValidator;
use sp_runtime::traits::{Block as BlockT, Header as _};
use std::{borrow::Cow, sync::Arc, time::Duration};
use substrate_test_runtime_client::{TestClientBuilder, TestClientBuilderExt as _};
@@ -109,6 +111,7 @@ fn build_test_full_node(
protocol_config
};
let max_parallel_downloads = config.max_parallel_downloads;
let worker = NetworkWorker::new(config::Params {
role: config::Role::Full,
executor: None,
@@ -120,8 +123,17 @@ fn build_test_full_node(
transaction_pool: Arc::new(crate::config::EmptyTransactionPool),
protocol_id,
import_queue,
block_announce_validator: Box::new(
sp_consensus::block_validation::DefaultBlockAnnounceValidator,
create_chain_sync: Box::new(
move |sync_mode, chain, warp_sync_provider| match ChainSync::new(
sync_mode,
chain,
Box::new(DefaultBlockAnnounceValidator),
max_parallel_downloads,
warp_sync_provider,
) {
Ok(chain_sync) => Ok(Box::new(chain_sync)),
Err(error) => Err(Box::new(error).into()),
},
),
metrics_registry: None,
block_request_protocol_config,
substrate/client/network/sync/Cargo.toml
-2
View File
@@ -17,11 +17,9 @@ targets = ["x86_64-unknown-linux-gnu"]
prost-build = "0.10"
[dependencies]
bitflags = "1.3.2"
codec = { package = "parity-scale-codec", version = "3.0.0", features = [
"derive",
] }
either = "1.5.3"
futures = "0.3.21"
libp2p = "0.46.1"
log = "0.4.17"
substrate/client/network/sync/src/block_request_handler.rs
+2 -4
View File
@@ -17,10 +17,7 @@
//! Helper for handling (i.e. answering) block requests from a remote peer via the
//! `crate::request_responses::RequestResponsesBehaviour`.
use crate::{
message::BlockAttributes,
schema::v1::{block_request::FromBlock, BlockResponse, Direction},
};
use crate::schema::v1::{block_request::FromBlock, BlockResponse, Direction};
use codec::{Decode, Encode};
use futures::{
channel::{mpsc, oneshot},
@@ -34,6 +31,7 @@ use sc_client_api::BlockBackend;
use sc_network_common::{
config::ProtocolId,
request_responses::{IncomingRequest, OutgoingResponse, ProtocolConfig},
sync::message::BlockAttributes,
};
use sp_blockchain::HeaderBackend;
use sp_runtime::{
substrate/client/network/sync/src/blocks.rs
+2 -2
View File
@@ -16,9 +16,9 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::message;
use libp2p::PeerId;
use log::trace;
use sc_network_common::sync::message;
use sp_runtime::traits::{Block as BlockT, NumberFor, One};
use std::{
cmp,
@@ -245,8 +245,8 @@ impl<B: BlockT> BlockCollection<B> {
#[cfg(test)]
mod test {
use super::{BlockCollection, BlockData, BlockRangeState};
use crate::message;
use libp2p::PeerId;
use sc_network_common::sync::message;
use sp_core::H256;
use sp_runtime::testing::{Block as RawBlock, ExtrinsicWrapper};
substrate/client/network/sync/src/extra_requests.rs
+1 -10
View File
@@ -20,6 +20,7 @@ use crate::{PeerSync, PeerSyncState};
use fork_tree::ForkTree;
use libp2p::PeerId;
use log::{debug, trace, warn};
use sc_network_common::sync::metrics::Metrics;
use sp_blockchain::Error as ClientError;
use sp_runtime::traits::{Block as BlockT, NumberFor, Zero};
use std::{
@@ -56,15 +57,6 @@ pub(crate) struct ExtraRequests<B: BlockT> {
request_type_name: &'static str,
}
#[derive(Debug)]
pub struct Metrics {
pub pending_requests: u32,
pub active_requests: u32,
pub importing_requests: u32,
pub failed_requests: u32,
_priv: (),
}
impl<B: BlockT> ExtraRequests<B> {
pub(crate) fn new(request_type_name: &'static str) -> Self {
Self {
@@ -258,7 +250,6 @@ impl<B: BlockT> ExtraRequests<B> {
active_requests: self.active_requests.len().try_into().unwrap_or(std::u32::MAX),
failed_requests: self.failed_requests.len().try_into().unwrap_or(std::u32::MAX),
importing_requests: self.importing_requests.len().try_into().unwrap_or(std::u32::MAX),
_priv: (),
}
}
}
substrate/client/network/sync/src/lib.rs
+443 -523
View File
File diff suppressed because it is too large Load Diff
substrate/client/network/sync/src/schema.rs
+1 -1
View File
@@ -18,6 +18,6 @@
//! Include sources generated from protobuf definitions.
pub mod v1 {
pub(crate) mod v1 {
include!(concat!(env!("OUT_DIR"), "/api.v1.rs"));
}
substrate/client/network/sync/src/state.rs
+1 -9
View File
@@ -23,6 +23,7 @@ use codec::{Decode, Encode};
use log::debug;
use sc_client_api::{CompactProof, ProofProvider};
use sc_consensus::ImportedState;
use sc_network_common::sync::StateDownloadProgress;
use smallvec::SmallVec;
use sp_core::storage::well_known_keys;
use sp_runtime::traits::{Block as BlockT, Header, NumberFor};
@@ -42,15 +43,6 @@ pub struct StateSync<B: BlockT, Client> {
skip_proof: bool,
}
/// Reported state download progress.
#[derive(Clone, Eq, PartialEq, Debug)]
pub struct StateDownloadProgress {
/// Estimated download percentage.
pub percentage: u32,
/// Total state size in bytes downloaded so far.
pub size: u64,
}
/// Import state chunk result.
pub enum ImportResult<B: BlockT> {
/// State is complete and ready for import.
substrate/client/network/sync/src/warp.rs
+5 -40
View File
@@ -18,60 +18,25 @@
//! Warp sync support.
pub use crate::warp_request_handler::{
EncodedProof, Request as WarpProofRequest, VerificationResult, WarpSyncProvider,
};
use crate::{
schema::v1::{StateRequest, StateResponse},
state::{ImportResult, StateSync},
};
use sc_client_api::ProofProvider;
use sc_network_common::sync::warp::{
EncodedProof, VerificationResult, WarpProofRequest, WarpSyncPhase, WarpSyncProgress,
WarpSyncProvider,
};
use sp_blockchain::HeaderBackend;
use sp_finality_grandpa::{AuthorityList, SetId};
use sp_runtime::traits::{Block as BlockT, NumberFor, Zero};
use std::{fmt, sync::Arc};
use std::sync::Arc;
enum Phase<B: BlockT, Client> {
WarpProof { set_id: SetId, authorities: AuthorityList, last_hash: B::Hash },
State(StateSync<B, Client>),
}
/// Reported warp sync phase.
#[derive(Clone, Eq, PartialEq, Debug)]
pub enum WarpSyncPhase<B: BlockT> {
/// Waiting for peers to connect.
AwaitingPeers,
/// Downloading and verifying grandpa warp proofs.
DownloadingWarpProofs,
/// Downloading state data.
DownloadingState,
/// Importing state.
ImportingState,
/// Downloading block history.
DownloadingBlocks(NumberFor<B>),
}
impl<B: BlockT> fmt::Display for WarpSyncPhase<B> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Self::AwaitingPeers => write!(f, "Waiting for peers"),
Self::DownloadingWarpProofs => write!(f, "Downloading finality proofs"),
Self::DownloadingState => write!(f, "Downloading state"),
Self::ImportingState => write!(f, "Importing state"),
Self::DownloadingBlocks(n) => write!(f, "Downloading block history (#{})", n),
}
}
}
/// Reported warp sync progress.
#[derive(Clone, Eq, PartialEq, Debug)]
pub struct WarpSyncProgress<B: BlockT> {
/// Estimated download percentage.
pub phase: WarpSyncPhase<B>,
/// Total bytes downloaded so far.
pub total_bytes: u64,
}
/// Import warp proof result.
pub enum WarpProofImportResult {
/// Import was successful.
substrate/client/network/sync/src/warp_request_handler.rs
+4 -43
View File
@@ -1,4 +1,4 @@
// Copyright 2021 Parity Technologies (UK) Ltd.
// Copyright 2022 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
@@ -16,7 +16,7 @@
//! Helper for handling (i.e. answering) grandpa warp sync requests from a remote peer.
use codec::{Decode, Encode};
use codec::Decode;
use futures::{
channel::{mpsc, oneshot},
stream::StreamExt,
@@ -27,52 +27,13 @@ use sc_network_common::{
request_responses::{
IncomingRequest, OutgoingResponse, ProtocolConfig as RequestResponseConfig,
},
sync::warp::{EncodedProof, WarpProofRequest, WarpSyncProvider},
};
use sp_runtime::traits::Block as BlockT;
use std::{sync::Arc, time::Duration};
pub use sp_finality_grandpa::{AuthorityList, SetId};
/// Scale-encoded warp sync proof response.
pub struct EncodedProof(pub Vec<u8>);
/// Warp sync request
#[derive(Encode, Decode, Debug)]
pub struct Request<B: BlockT> {
/// Start collecting proofs from this block.
pub begin: B::Hash,
}
const MAX_RESPONSE_SIZE: u64 = 16 * 1024 * 1024;
/// Proof verification result.
pub enum VerificationResult<Block: BlockT> {
/// Proof is valid, but the target was not reached.
Partial(SetId, AuthorityList, Block::Hash),
/// Target finality is proved.
Complete(SetId, AuthorityList, Block::Header),
}
/// Warp sync backend. Handles retrieveing and verifying warp sync proofs.
pub trait WarpSyncProvider<B: BlockT>: Send + Sync {
/// Generate proof starting at given block hash. The proof is accumulated until maximum proof
/// size is reached.
fn generate(
&self,
start: B::Hash,
) -> Result<EncodedProof, Box<dyn std::error::Error + Send + Sync>>;
/// Verify warp proof against current set of authorities.
fn verify(
&self,
proof: &EncodedProof,
set_id: SetId,
authorities: AuthorityList,
) -> Result<VerificationResult<B>, Box<dyn std::error::Error + Send + Sync>>;
/// Get current list of authorities. This is supposed to be genesis authorities when starting
/// sync.
fn current_authorities(&self) -> AuthorityList;
}
/// Generates a [`RequestResponseConfig`] for the grandpa warp sync request protocol, refusing
/// incoming requests.
pub fn generate_request_response_config(protocol_id: ProtocolId) -> RequestResponseConfig {
@@ -115,7 +76,7 @@ impl<TBlock: BlockT> RequestHandler<TBlock> {
payload: Vec<u8>,
pending_response: oneshot::Sender<OutgoingResponse>,
) -> Result<(), HandleRequestError> {
let request = Request::<TBlock>::decode(&mut &payload[..])?;
let request = WarpProofRequest::<TBlock>::decode(&mut &payload[..])?;
let EncodedProof(proof) = self
.backend
substrate/client/network/test/Cargo.toml
+2
View File
@@ -26,6 +26,8 @@ sc-client-api = { version = "4.0.0-dev", path = "../../api" }
sc-consensus = { version = "0.10.0-dev", path = "../../consensus/common" }
sc-network = { version = "0.10.0-dev", path = "../" }
sc-network-common = { version = "0.10.0-dev", path = "../common" }
sc-network-light = { version = "0.10.0-dev", path = "../light" }
sc-network-sync = { version = "0.10.0-dev", path = "../sync" }
sc-service = { version = "0.10.0-dev", default-features = false, features = ["test-helpers"], path = "../../service" }
sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" }
sp-consensus = { version = "0.10.0-dev", path = "../../../primitives/consensus/common" }
substrate/client/network/test/src/lib.rs
+48 -62
View File
@@ -48,16 +48,21 @@ use sc_consensus::{
};
pub use sc_network::config::EmptyTransactionPool;
use sc_network::{
block_request_handler::BlockRequestHandler,
config::{
MultiaddrWithPeerId, NetworkConfiguration, NonDefaultSetConfig, NonReservedPeerMode,
ProtocolConfig, Role, SyncMode, TransportConfig,
MultiaddrWithPeerId, NetworkConfiguration, NonDefaultSetConfig, NonReservedPeerMode, Role,
SyncMode, TransportConfig,
},
light_client_requests::handler::LightClientRequestHandler,
state_request_handler::StateRequestHandler,
warp_request_handler, Multiaddr, NetworkService, NetworkWorker,
Multiaddr, NetworkService, NetworkWorker,
};
pub use sc_network_common::config::ProtocolId;
use sc_network_common::sync::warp::{
AuthorityList, EncodedProof, SetId, VerificationResult, WarpSyncProvider,
};
use sc_network_light::light_client_requests::handler::LightClientRequestHandler;
use sc_network_sync::{
block_request_handler::BlockRequestHandler, state_request_handler::StateRequestHandler,
warp_request_handler, ChainSync,
};
use sc_service::client::Client;
use sp_blockchain::{
well_known_cache_keys::{self, Id as CacheKeyId},
@@ -638,27 +643,26 @@ impl<B: BlockT> VerifierAdapter<B> {
struct TestWarpSyncProvider<B: BlockT>(Arc<dyn HeaderBackend<B>>);
impl<B: BlockT> warp_request_handler::WarpSyncProvider<B> for TestWarpSyncProvider<B> {
impl<B: BlockT> WarpSyncProvider<B> for TestWarpSyncProvider<B> {
fn generate(
&self,
_start: B::Hash,
) -> Result<warp_request_handler::EncodedProof, Box<dyn std::error::Error + Send + Sync>> {
) -> Result<EncodedProof, Box<dyn std::error::Error + Send + Sync>> {
let info = self.0.info();
let best_header = self.0.header(BlockId::hash(info.best_hash)).unwrap().unwrap();
Ok(warp_request_handler::EncodedProof(best_header.encode()))
Ok(EncodedProof(best_header.encode()))
}
fn verify(
&self,
proof: &warp_request_handler::EncodedProof,
_set_id: warp_request_handler::SetId,
_authorities: warp_request_handler::AuthorityList,
) -> Result<warp_request_handler::VerificationResult<B>, Box<dyn std::error::Error + Send + Sync>>
{
let warp_request_handler::EncodedProof(encoded) = proof;
proof: &EncodedProof,
_set_id: SetId,
_authorities: AuthorityList,
) -> Result<VerificationResult<B>, Box<dyn std::error::Error + Send + Sync>> {
let EncodedProof(encoded) = proof;
let header = B::Header::decode(&mut encoded.as_slice()).unwrap();
Ok(warp_request_handler::VerificationResult::Complete(0, Default::default(), header))
Ok(VerificationResult::Complete(0, Default::default(), header))
}
fn current_authorities(&self) -> warp_request_handler::AuthorityList {
fn current_authorities(&self) -> AuthorityList {
Default::default()
}
}
@@ -688,7 +692,7 @@ pub struct FullPeerConfig {
pub storage_chain: bool,
}
pub trait TestNetFactory: Sized
pub trait TestNetFactory: Default + Sized
where
<Self::BlockImport as BlockImport<Block>>::Transaction: Send,
{
@@ -696,14 +700,8 @@ where
type BlockImport: BlockImport<Block, Error = ConsensusError> + Clone + Send + Sync + 'static;
type PeerData: Default;
/// These two need to be implemented!
fn from_config(config: &ProtocolConfig) -> Self;
fn make_verifier(
&self,
client: PeersClient,
config: &ProtocolConfig,
peer_data: &Self::PeerData,
) -> Self::Verifier;
/// This one needs to be implemented!
fn make_verifier(&self, client: PeersClient, peer_data: &Self::PeerData) -> Self::Verifier;
/// Get reference to peer.
fn peer(&mut self, i: usize) -> &mut Peer<Self::PeerData, Self::BlockImport>;
@@ -723,15 +721,10 @@ where
Self::PeerData,
);
fn default_config() -> ProtocolConfig {
ProtocolConfig::default()
}
/// Create new test network with this many peers.
fn new(n: usize) -> Self {
trace!(target: "test_network", "Creating test network");
let config = Self::default_config();
let mut net = Self::from_config(&config);
let mut net = Self::default();
for i in 0..n {
trace!(target: "test_network", "Adding peer {}", i);
@@ -767,11 +760,8 @@ where
let (block_import, justification_import, data) = self
.make_block_import(PeersClient { client: client.clone(), backend: backend.clone() });
let verifier = self.make_verifier(
PeersClient { client: client.clone(), backend: backend.clone() },
&Default::default(),
&data,
);
let verifier = self
.make_verifier(PeersClient { client: client.clone(), backend: backend.clone() }, &data);
let verifier = VerifierAdapter::new(verifier);
let import_queue = Box::new(BasicQueue::new(
@@ -845,6 +835,10 @@ where
protocol_config
};
let max_parallel_downloads = network_config.max_parallel_downloads;
let block_announce_validator = config
.block_announce_validator
.unwrap_or_else(|| Box::new(DefaultBlockAnnounceValidator));
let network = NetworkWorker::new(sc_network::config::Params {
role: if config.is_authority { Role::Authority } else { Role::Full },
executor: None,
@@ -856,9 +850,18 @@ where
transaction_pool: Arc::new(EmptyTransactionPool),
protocol_id,
import_queue,
block_announce_validator: config
.block_announce_validator
.unwrap_or_else(|| Box::new(DefaultBlockAnnounceValidator)),
create_chain_sync: Box::new(move |sync_mode, chain, warp_sync_provider| {
match ChainSync::new(
sync_mode,
chain,
block_announce_validator,
max_parallel_downloads,
warp_sync_provider,
) {
Ok(chain_sync) => Ok(Box::new(chain_sync)),
Err(error) => Err(Box::new(error).into()),
}
}),
metrics_registry: None,
block_request_protocol_config,
state_request_protocol_config,
@@ -1012,6 +1015,7 @@ where
}
}
#[derive(Default)]
pub struct TestNet {
peers: Vec<Peer<(), PeersClient>>,
}
@@ -1021,17 +1025,7 @@ impl TestNetFactory for TestNet {
type PeerData = ();
type BlockImport = PeersClient;
/// Create new test network with peers and given config.
fn from_config(_config: &ProtocolConfig) -> Self {
TestNet { peers: Vec::new() }
}
fn make_verifier(
&self,
_client: PeersClient,
_config: &ProtocolConfig,
_peer_data: &(),
) -> Self::Verifier {
fn make_verifier(&self, _client: PeersClient, _peer_data: &()) -> Self::Verifier {
PassThroughVerifier::new(false)
}
@@ -1081,6 +1075,7 @@ impl JustificationImport<Block> for ForceFinalized {
}
}
#[derive(Default)]
pub struct JustificationTestNet(TestNet);
impl TestNetFactory for JustificationTestNet {
@@ -1088,17 +1083,8 @@ impl TestNetFactory for JustificationTestNet {
type PeerData = ();
type BlockImport = PeersClient;
fn from_config(config: &ProtocolConfig) -> Self {
JustificationTestNet(TestNet::from_config(config))
}
fn make_verifier(
&self,
client: PeersClient,
config: &ProtocolConfig,
peer_data: &(),
) -> Self::Verifier {
self.0.make_verifier(client, config, peer_data)
fn make_verifier(&self, client: PeersClient, peer_data: &()) -> Self::Verifier {
self.0.make_verifier(client, peer_data)
}
fn peer(&mut self, i: usize) -> &mut Peer<Self::PeerData, Self::BlockImport> {
substrate/client/service/Cargo.toml
+3 -1
View File
@@ -50,8 +50,10 @@ sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/comm
sc-consensus = { version = "0.10.0-dev", path = "../../client/consensus/common" }
sp-inherents = { version = "4.0.0-dev", path = "../../primitives/inherents" }
sp-storage = { version = "6.0.0", path = "../../primitives/storage" }
sc-network-common = { version = "0.10.0-dev", path = "../network/common" }
sc-network = { version = "0.10.0-dev", path = "../network" }
sc-network-common = { version = "0.10.0-dev", path = "../network/common" }
sc-network-light = { version = "0.10.0-dev", path = "../network/light" }
sc-network-sync = { version = "0.10.0-dev", path = "../network/sync" }
sc-chain-spec = { version = "4.0.0-dev", path = "../chain-spec" }
sc-client-api = { version = "4.0.0-dev", path = "../api" }
sp-api = { version = "4.0.0-dev", path = "../../primitives/api" }
substrate/client/service/src/builder.rs
+22 -6
View File
@@ -38,13 +38,17 @@ use sc_consensus::import_queue::ImportQueue;
use sc_executor::RuntimeVersionOf;
use sc_keystore::LocalKeystore;
use sc_network::{
block_request_handler::{self, BlockRequestHandler},
config::{Role, SyncMode},
light_client_requests::{self, handler::LightClientRequestHandler},
state_request_handler::{self, StateRequestHandler},
warp_request_handler::{self, RequestHandler as WarpSyncRequestHandler, WarpSyncProvider},
NetworkService,
};
use sc_network_common::sync::warp::WarpSyncProvider;
use sc_network_light::light_client_requests::{self, handler::LightClientRequestHandler};
use sc_network_sync::{
block_request_handler::{self, BlockRequestHandler},
state_request_handler::{self, StateRequestHandler},
warp_request_handler::{self, RequestHandler as WarpSyncRequestHandler},
ChainSync,
};
use sc_rpc::{
author::AuthorApiServer,
chain::ChainApiServer,
@@ -801,6 +805,7 @@ where
}
};
let max_parallel_downloads = config.network.max_parallel_downloads;
let network_params = sc_network::config::Params {
role: config.role.clone(),
executor: {
@@ -818,9 +823,20 @@ where
network_config: config.network.clone(),
chain: client.clone(),
transaction_pool: transaction_pool_adapter as _,
import_queue: Box::new(import_queue),
protocol_id,
block_announce_validator,
import_queue: Box::new(import_queue),
create_chain_sync: Box::new(
move |sync_mode, chain, warp_sync_provider| match ChainSync::new(
sync_mode,
chain,
block_announce_validator,
max_parallel_downloads,
warp_sync_provider,
) {
Ok(chain_sync) => Ok(Box::new(chain_sync)),
Err(error) => Err(Box::new(error).into()),
},
),
metrics_registry: config.prometheus_config.as_ref().map(|config| config.registry.clone()),
block_request_protocol_config,
state_request_protocol_config,