pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-09 19:01:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove v0 node-side parachains code (#1609)

Browse Source 
* clean out v0 consensus crates

* remove service dependencies on old consensus code

* fix cli

* kill adder-collator

* bump Cargo.lock
This commit is contained in:
Robert Habermeier
2020-08-24 13:43:01 +02:00
committed by GitHub
parent 591e9b7454
commit 430cf6e6f2
40 changed files with 40 additions and 11469 deletions
Download Patch File Download Diff File Expand all files Collapse all files
polkadot/Cargo.lock
Generated
+7 -254
View File
@@ -2683,7 +2683,7 @@ dependencies = [
"sp-trie",
"sp-version",
"static_assertions",
"substrate-wasm-builder-runner 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"substrate-wasm-builder-runner",
"tiny-keccak 1.5.0",
]
@@ -4729,7 +4729,6 @@ dependencies = [
"nix 0.17.0",
"parity-util-mem",
"polkadot-cli",
"polkadot-collator",
"polkadot-service",
"tempfile",
]
@@ -4785,32 +4784,6 @@ dependencies = [
"streamunordered",
]
[[package]]
name = "polkadot-availability-store"
version = "0.8.22"
dependencies = [
"derive_more 0.99.9",
"exit-future",
"futures 0.3.5",
"kvdb",
"kvdb-memorydb",
"kvdb-rocksdb",
"log 0.4.11",
"parity-scale-codec",
"parking_lot 0.9.0",
"polkadot-erasure-coding",
"polkadot-primitives",
"sc-client-api",
"sc-keystore",
"sc-network",
"sp-api",
"sp-blockchain",
"sp-consensus",
"sp-core",
"sp-runtime",
"tokio 0.2.21",
]
[[package]]
name = "polkadot-cli"
version = "0.8.22"
@@ -4837,34 +4810,6 @@ dependencies = [
"wasm-bindgen-futures",
]
[[package]]
name = "polkadot-collator"
version = "0.8.22"
dependencies = [
"futures 0.3.5",
"futures-timer 2.0.2",
"log 0.4.11",
"parity-scale-codec",
"polkadot-cli",
"polkadot-network",
"polkadot-primitives",
"polkadot-service",
"polkadot-service-new",
"polkadot-validation",
"sc-cli",
"sc-client-api",
"sc-executor",
"sc-network",
"sc-service",
"sp-api",
"sp-blockchain",
"sp-consensus",
"sp-core",
"sp-keyring",
"sp-runtime",
"tokio 0.2.21",
]
[[package]]
name = "polkadot-core-primitives"
version = "0.7.30"
@@ -4887,36 +4832,6 @@ dependencies = [
"sp-trie",
]
[[package]]
name = "polkadot-network"
version = "0.8.22"
dependencies = [
"arrayvec 0.4.12",
"bytes 0.5.5",
"derive_more 0.14.1",
"exit-future",
"futures 0.3.5",
"futures-timer 2.0.2",
"log 0.4.11",
"parity-scale-codec",
"parking_lot 0.9.0",
"polkadot-availability-store",
"polkadot-erasure-coding",
"polkadot-primitives",
"polkadot-validation",
"rand 0.7.3",
"sc-network",
"sc-network-gossip",
"sp-api",
"sp-blockchain",
"sp-consensus",
"sp-core",
"sp-keyring",
"sp-runtime",
"sp-state-machine",
"wasm-timer",
]
[[package]]
name = "polkadot-network-bridge"
version = "0.1.0"
@@ -4938,27 +4853,6 @@ dependencies = [
"streamunordered",
]
[[package]]
name = "polkadot-network-test"
version = "0.8.22"
dependencies = [
"futures 0.3.5",
"log 0.4.11",
"parking_lot 0.10.2",
"polkadot-test-runtime-client",
"rand 0.7.3",
"sc-block-builder",
"sc-client-api",
"sc-consensus",
"sc-network",
"sc-network-test",
"sc-service",
"sp-blockchain",
"sp-consensus",
"sp-core",
"sp-runtime",
]
[[package]]
name = "polkadot-node-collation-generation"
version = "0.1.0"
@@ -5396,7 +5290,7 @@ dependencies = [
"sp-trie",
"sp-version",
"static_assertions",
"substrate-wasm-builder-runner 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"substrate-wasm-builder-runner",
"tiny-keccak 1.5.0",
"trie-db",
]
@@ -5508,8 +5402,6 @@ dependencies = [
"pallet-transaction-payment-rpc-runtime-api",
"parity-scale-codec",
"parking_lot 0.9.0",
"polkadot-availability-store",
"polkadot-network",
"polkadot-primitives",
"polkadot-rpc",
"polkadot-runtime",
@@ -5568,7 +5460,6 @@ dependencies = [
"pallet-transaction-payment-rpc-runtime-api",
"parity-scale-codec",
"parking_lot 0.9.0",
"polkadot-network",
"polkadot-node-core-proposer",
"polkadot-node-subsystem",
"polkadot-overseer",
@@ -5699,7 +5590,7 @@ dependencies = [
"sp-transaction-pool",
"sp-trie",
"sp-version",
"substrate-wasm-builder-runner 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"substrate-wasm-builder-runner",
"tiny-keccak 1.5.0",
]
@@ -5739,14 +5630,11 @@ dependencies = [
"pallet-balances",
"pallet-staking",
"pallet-transaction-payment",
"polkadot-availability-store",
"polkadot-network",
"polkadot-primitives",
"polkadot-rpc",
"polkadot-runtime-common",
"polkadot-service",
"polkadot-test-runtime",
"polkadot-validation",
"rand 0.7.3",
"sc-authority-discovery",
"sc-chain-spec",
@@ -5780,25 +5668,16 @@ dependencies = [
name = "polkadot-validation"
version = "0.8.22"
dependencies = [
"ansi_term 0.12.1",
"bitvec",
"derive_more 0.14.1",
"exit-future",
"futures 0.3.5",
"futures-timer 2.0.2",
"log 0.4.11",
"parity-scale-codec",
"parking_lot 0.9.0",
"polkadot-availability-store",
"polkadot-erasure-coding",
"polkadot-parachain",
"polkadot-primitives",
"polkadot-statement-table",
"sc-basic-authorship",
"sc-block-builder",
"sc-client-api",
"sc-finality-grandpa",
"sc-keystore",
"sp-api",
"sp-blockchain",
"sp-consensus",
@@ -5811,7 +5690,6 @@ dependencies = [
"sp-transaction-pool",
"sp-trie",
"substrate-prometheus-endpoint",
"tokio 0.2.21",
]
[[package]]
@@ -6515,7 +6393,7 @@ dependencies = [
"sp-std",
"sp-transaction-pool",
"sp-version",
"substrate-wasm-builder-runner 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"substrate-wasm-builder-runner",
]
[[package]]
@@ -7238,33 +7116,6 @@ dependencies = [
"wasm-timer",
]
[[package]]
name = "sc-network-test"
version = "0.8.0-rc6"
source = "git+https://github.com/paritytech/substrate#dfe2871b272d2bb343c8fb2b1f0bb671324e52e9"
dependencies = [
"env_logger",
"futures 0.3.5",
"futures-timer 3.0.2",
"libp2p",
"log 0.4.11",
"parking_lot 0.10.2",
"rand 0.7.3",
"sc-block-builder",
"sc-client-api",
"sc-consensus",
"sc-network",
"sc-service",
"sp-blockchain",
"sp-consensus",
"sp-consensus-babe",
"sp-core",
"sp-runtime",
"substrate-test-runtime",
"substrate-test-runtime-client",
"tempfile",
]
[[package]]
name = "sc-offchain"
version = "2.0.0-rc6"
@@ -8135,20 +7986,6 @@ dependencies = [
"wasm-timer",
]
[[package]]
name = "sp-consensus-aura"
version = "0.8.0-rc6"
source = "git+https://github.com/paritytech/substrate#dfe2871b272d2bb343c8fb2b1f0bb671324e52e9"
dependencies = [
"parity-scale-codec",
"sp-api",
"sp-application-crypto",
"sp-inherents",
"sp-runtime",
"sp-std",
"sp-timestamp",
]
[[package]]
name = "sp-consensus-babe"
version = "0.8.0-rc6"
@@ -8819,67 +8656,6 @@ dependencies = [
"sp-state-machine",
]
[[package]]
name = "substrate-test-runtime"
version = "2.0.0-rc6"
source = "git+https://github.com/paritytech/substrate#dfe2871b272d2bb343c8fb2b1f0bb671324e52e9"
dependencies = [
"cfg-if",
"frame-executive",
"frame-support",
"frame-system",
"frame-system-rpc-runtime-api",
"log 0.4.11",
"memory-db",
"pallet-babe",
"pallet-timestamp",
"parity-scale-codec",
"parity-util-mem",
"sc-service",
"serde",
"sp-api",
"sp-application-crypto",
"sp-block-builder",
"sp-consensus-aura",
"sp-consensus-babe",
"sp-core",
"sp-finality-grandpa",
"sp-inherents",
"sp-io",
"sp-keyring",
"sp-offchain",
"sp-runtime",
"sp-runtime-interface",
"sp-session",
"sp-std",
"sp-transaction-pool",
"sp-trie",
"sp-version",
"substrate-wasm-builder-runner 1.0.6 (git+https://github.com/paritytech/substrate)",
"trie-db",
]
[[package]]
name = "substrate-test-runtime-client"
version = "2.0.0-rc6"
source = "git+https://github.com/paritytech/substrate#dfe2871b272d2bb343c8fb2b1f0bb671324e52e9"
dependencies = [
"futures 0.3.5",
"parity-scale-codec",
"sc-block-builder",
"sc-client-api",
"sc-consensus",
"sc-light",
"sc-service",
"sp-api",
"sp-blockchain",
"sp-consensus",
"sp-core",
"sp-runtime",
"substrate-test-client",
"substrate-test-runtime",
]
[[package]]
name = "substrate-test-utils"
version = "2.0.0-rc6"
@@ -8900,11 +8676,6 @@ dependencies = [
"syn 1.0.33",
]
[[package]]
name = "substrate-wasm-builder-runner"
version = "1.0.6"
source = "git+https://github.com/paritytech/substrate#dfe2871b272d2bb343c8fb2b1f0bb671324e52e9"
[[package]]
name = "substrate-wasm-builder-runner"
version = "1.0.6"
@@ -9012,33 +8783,15 @@ dependencies = [
"polkadot-parachain",
"sp-io",
"sp-std",
"substrate-wasm-builder-runner 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"substrate-wasm-builder-runner",
"tiny-keccak 1.5.0",
]
[[package]]
name = "test-parachain-adder-collator"
version = "0.1.0"
dependencies = [
"futures 0.3.5",
"parity-scale-codec",
"parking_lot 0.10.2",
"polkadot-collator",
"polkadot-parachain",
"polkadot-primitives",
"polkadot-service",
"sc-client-api",
"sp-api",
"sp-core",
"sp-runtime",
"test-parachain-adder",
]
[[package]]
name = "test-parachain-halt"
version = "0.8.22"
dependencies = [
"substrate-wasm-builder-runner 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"substrate-wasm-builder-runner",
]
[[package]]
@@ -10209,7 +9962,7 @@ dependencies = [
"sp-trie",
"sp-version",
"static_assertions",
"substrate-wasm-builder-runner 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"substrate-wasm-builder-runner",
"tiny-keccak 1.5.0",
]
polkadot/Cargo.toml
-8
View File
@@ -10,8 +10,6 @@ edition = "2018"
[dependencies]
cli = { package = "polkadot-cli", path = "cli" }
# It looks like this is the only way to pass features to it
collator = { package = "polkadot-collator", path = "collator" }
futures = "0.3.4"
service = { package = "polkadot-service", path = "service" }
parity-util-mem = { version = "*", default-features = false, features = ["jemalloc-global"] }
@@ -23,13 +21,9 @@ tempfile = "3.1.0"
[workspace]
members = [
"availability-store",
"cli",
"collator",
"core-primitives",
"erasure-coding",
"network",
"network/test",
"primitives",
"runtime/common",
"runtime/parachains",
@@ -69,7 +63,6 @@ members = [
"parachain/test-parachains",
"parachain/test-parachains/adder",
"parachain/test-parachains/adder/collator",
]
[badges]
@@ -83,5 +76,4 @@ panic = "unwind"
runtime-benchmarks=["cli/runtime-benchmarks"]
service-rewr= [
"cli/service-rewr",
"collator/service-rewr",
]
polkadot/availability-store/Cargo.toml
-30
View File
@@ -1,30 +0,0 @@
[package]
name = "polkadot-availability-store"
description = "Persistent database for parachain data"
version = "0.8.22"
authors = ["Parity Technologies <admin@parity.io>"]
edition = "2018"
[dependencies]
polkadot-primitives = { path = "../primitives" }
polkadot-erasure-coding = { path = "../erasure-coding" }
parking_lot = "0.9.0"
derive_more = "0.99"
log = "0.4.8"
futures = "0.3.4"
tokio = { version = "0.2.13", features = ["rt-core"] }
exit-future = "0.2.0"
codec = { package = "parity-scale-codec", version = "1.3.4", features = ["derive"] }
sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" }
consensus_common = { package = "sp-consensus", git = "https://github.com/paritytech/substrate", branch = "master" }
client = { package = "sc-client-api", git = "https://github.com/paritytech/substrate", branch = "master", version = "2.0.0-rc5" }
sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
keystore = { package = "sc-keystore", git = "https://github.com/paritytech/substrate", branch = "master" }
sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
kvdb = "0.7.0"
kvdb-memorydb = "0.7.0"
[target.'cfg(not(target_os = "unknown"))'.dependencies]
kvdb-rocksdb = "0.9.0"
polkadot/availability-store/src/lib.rs
-352
View File
@@ -1,352 +0,0 @@
// Copyright 2018-2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Persistent database for parachain data: PoV block data, erasure-coding chunks and outgoing messages.
//!
//! This will be written into during the block validation pipeline, and queried
//! by networking code in order to circulate required data and maintain availability
//! of it.
#![warn(missing_docs)]
use futures::prelude::*;
use futures::channel::{mpsc, oneshot};
use keystore::KeyStorePtr;
use polkadot_primitives::v0::{
Hash, Block,
PoVBlock, AbridgedCandidateReceipt, ErasureChunk,
ParachainHost, AvailableData, OmittedValidationData,
};
use sp_runtime::traits::HashFor;
use sp_blockchain::Result as ClientResult;
use client::{
BlockchainEvents, BlockBackend,
};
use sp_api::{ApiExt, ProvideRuntimeApi};
use codec::{Encode, Decode};
use sp_core::traits::SpawnNamed;
use log::warn;
use std::sync::Arc;
use std::collections::HashSet;
use std::path::PathBuf;
use std::io;
use std::pin::Pin;
mod worker;
mod store;
pub use worker::AvailabilityBlockImport;
pub use store::AwaitedFrontierEntry;
use worker::{
Worker, WorkerHandle, IncludedParachainBlocks, WorkerMsg, MakeAvailable, Chunks
};
use store::Store as InnerStore;
const LOG_TARGET: &str = "availability";
/// Configuration for the availability store.
pub struct Config {
/// Cache size in bytes. If `None` default is used.
pub cache_size: Option<usize>,
/// Path to the database.
pub path: PathBuf,
}
/// An abstraction around networking for the availablity-store.
///
/// Currently it is not possible to use the networking code in the availability store
/// core directly due to a number of loop dependencies it requires:
///
/// `availability-store` -> `network` -> `availability-store`
///
/// `availability-store` -> `network` -> `validation` -> `availability-store`
///
/// So we provide this trait that gets implemented for a type in
/// the [`network`] module or a mock in tests.
///
/// [`network`]: ../polkadot_network/index.html
pub trait ErasureNetworking {
/// Errors that can occur when fetching erasure chunks.
type Error: std::fmt::Debug + 'static;
/// Fetch an erasure chunk from the networking service.
fn fetch_erasure_chunk(
&self,
candidate_hash: &Hash,
index: u32,
) -> Pin<Box<dyn Future<Output = Result<ErasureChunk, Self::Error>> + Send>>;
/// Distributes an erasure chunk to the correct validator node.
fn distribute_erasure_chunk(
&self,
candidate_hash: Hash,
chunk: ErasureChunk,
);
}
/// Data that, when combined with an `AbridgedCandidateReceipt`, is enough
/// to fully re-execute a block.
#[derive(Debug, Encode, Decode, PartialEq)]
pub struct ExecutionData {
/// The `PoVBlock`.
pub pov_block: PoVBlock,
/// The data omitted from the `AbridgedCandidateReceipt`.
pub omitted_validation: OmittedValidationData,
}
/// Handle to the availability store.
///
/// This provides a proxying API that
/// * in case of write operations provides async methods that send data to
/// the background worker and resolve when that data is processed by the worker
/// * in case of read opeartions queries the underlying storage synchronously.
#[derive(Clone)]
pub struct Store {
inner: InnerStore,
worker: Arc<WorkerHandle>,
to_worker: mpsc::UnboundedSender<WorkerMsg>,
}
impl Store {
/// Create a new `Store` with given config on disk.
///
/// Creating a store among other things starts a background worker thread that
/// handles most of the write operations to the storage.
#[cfg(not(target_os = "unknown"))]
pub fn new<EN>(config: Config, network: EN) -> io::Result<Self>
where EN: ErasureNetworking + Send + Sync + Clone + 'static
{
let inner = InnerStore::new(config)?;
let worker = Arc::new(Worker::start(inner.clone(), network));
let to_worker = worker.to_worker().clone();
Ok(Self {
inner,
worker,
to_worker,
})
}
/// Create a new in-memory `Store`. Useful for tests.
///
/// Creating a store among other things starts a background worker thread
/// that handles most of the write operations to the storage.
pub fn new_in_memory<EN>(network: EN) -> Self
where EN: ErasureNetworking + Send + Sync + Clone + 'static
{
let inner = InnerStore::new_in_memory();
let worker = Arc::new(Worker::start(inner.clone(), network));
let to_worker = worker.to_worker().clone();
Self {
inner,
worker,
to_worker,
}
}
/// Obtain a [`BlockImport`] implementation to import blocks into this store.
///
/// This block import will act upon all newly imported blocks sending information
/// about parachain heads included in them to this `Store`'s background worker.
/// The user may create multiple instances of [`BlockImport`]s with this call.
///
/// [`BlockImport`]: https://substrate.dev/rustdocs/v1.0/substrate_consensus_common/trait.BlockImport.html
pub fn block_import<I, P>(
&self,
wrapped_block_import: I,
client: Arc<P>,
spawner: impl SpawnNamed,
keystore: KeyStorePtr,
) -> ClientResult<AvailabilityBlockImport<I, P>>
where
P: ProvideRuntimeApi<Block> + BlockchainEvents<Block> + BlockBackend<Block> + Send + Sync + 'static,
P::Api: ParachainHost<Block>,
P::Api: ApiExt<Block, Error=sp_blockchain::Error>,
// Rust bug: https://github.com/rust-lang/rust/issues/24159
sp_api::StateBackendFor<P, Block>: sp_api::StateBackend<HashFor<Block>>,
{
let to_worker = self.to_worker.clone();
let import = AvailabilityBlockImport::new(
client,
wrapped_block_import,
spawner,
keystore,
to_worker,
);
Ok(import)
}
/// Make some data available provisionally.
///
/// Validators with the responsibility of maintaining availability
/// for a block or collators collating a block will call this function
/// in order to persist that data to disk and so it can be queried and provided
/// to other nodes in the network.
///
/// Determination of invalidity is beyond the scope of this function.
///
/// This method will send the data to the background worker, allowing the caller to
/// asynchronously wait for the result.
pub async fn make_available(&self, candidate_hash: Hash, available_data: AvailableData)
-> io::Result<()>
{
let (s, r) = oneshot::channel();
let msg = WorkerMsg::MakeAvailable(MakeAvailable {
candidate_hash,
available_data,
result: s,
});
let _ = self.to_worker.unbounded_send(msg);
if let Ok(Ok(())) = r.await {
Ok(())
} else {
Err(io::Error::new(io::ErrorKind::Other, format!("adding erasure chunks failed")))
}
}
/// Get a set of all chunks we are waiting for.
pub fn awaited_chunks(&self) -> Option<HashSet<AwaitedFrontierEntry>> {
self.inner.awaited_chunks()
}
/// Adds an erasure chunk to storage.
///
/// The chunk should be checked for validity against the root of encoding
/// and its proof prior to calling this.
///
/// This method will send the chunk to the background worker, allowing the caller to
/// asynchronously wait for the result.
pub async fn add_erasure_chunk(
&self,
candidate: AbridgedCandidateReceipt,
n_validators: u32,
chunk: ErasureChunk,
) -> io::Result<()> {
self.add_erasure_chunks(candidate, n_validators, std::iter::once(chunk)).await
}
/// Adds a set of erasure chunks to storage.
///
/// The chunks should be checked for validity against the root of encoding
/// and its proof prior to calling this.
///
/// This method will send the chunks to the background worker, allowing the caller to
/// asynchronously wait for the result.
pub async fn add_erasure_chunks<I>(
&self,
candidate: AbridgedCandidateReceipt,
n_validators: u32,
chunks: I,
) -> io::Result<()>
where I: IntoIterator<Item = ErasureChunk>
{
let candidate_hash = candidate.hash();
self.add_candidate(candidate).await?;
let (s, r) = oneshot::channel();
let chunks = chunks.into_iter().collect();
let msg = WorkerMsg::Chunks(Chunks {
candidate_hash,
chunks,
n_validators,
result: s,
});
let _ = self.to_worker.unbounded_send(msg);
if let Ok(Ok(())) = r.await {
Ok(())
} else {
Err(io::Error::new(io::ErrorKind::Other, format!("adding erasure chunks failed")))
}
}
/// Queries an erasure chunk by the candidate hash and validator index.
pub fn get_erasure_chunk(
&self,
candidate_hash: &Hash,
validator_index: usize,
) -> Option<ErasureChunk> {
self.inner.get_erasure_chunk(candidate_hash, validator_index)
}
/// Note a validator's index and a number of validators at a relay parent in the
/// store.
///
/// This should be done before adding erasure chunks with this relay parent.
pub fn note_validator_index_and_n_validators(
&self,
relay_parent: &Hash,
validator_index: u32,
n_validators: u32,
) -> io::Result<()> {
self.inner.note_validator_index_and_n_validators(
relay_parent,
validator_index,
n_validators,
)
}
// Stores a candidate receipt.
async fn add_candidate(
&self,
candidate: AbridgedCandidateReceipt,
) -> io::Result<()> {
let (s, r) = oneshot::channel();
let msg = WorkerMsg::IncludedParachainBlocks(IncludedParachainBlocks {
blocks: vec![crate::worker::IncludedParachainBlock {
candidate,
available_data: None,
}],
result: s,
});
let _ = self.to_worker.unbounded_send(msg);
if let Ok(Ok(())) = r.await {
Ok(())
} else {
Err(io::Error::new(io::ErrorKind::Other, format!("adding erasure chunks failed")))
}
}
/// Queries a candidate receipt by its hash.
pub fn get_candidate(&self, candidate_hash: &Hash)
-> Option<AbridgedCandidateReceipt>
{
self.inner.get_candidate(candidate_hash)
}
/// Query execution data by pov-block hash.
pub fn execution_data(&self, candidate_hash: &Hash)
-> Option<ExecutionData>
{
self.inner.execution_data(candidate_hash)
}
}
polkadot/availability-store/src/store.rs
-620
View File
@@ -1,620 +0,0 @@
// Copyright 2018-2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
#[cfg(not(target_os = "unknown"))]
use kvdb_rocksdb::{Database, DatabaseConfig};
use kvdb::{KeyValueDB, DBTransaction};
use codec::{Encode, Decode};
use polkadot_erasure_coding as erasure;
use polkadot_primitives::v0::{
Hash, ErasureChunk, AvailableData, AbridgedCandidateReceipt,
};
use parking_lot::Mutex;
use log::{trace, warn};
use std::collections::HashSet;
use std::sync::Arc;
use std::iter::FromIterator;
use std::io;
use crate::{LOG_TARGET, Config, ExecutionData};
mod columns {
pub const DATA: u32 = 0;
pub const META: u32 = 1;
pub const NUM_COLUMNS: u32 = 2;
}
#[derive(Clone)]
pub struct Store {
inner: Arc<dyn KeyValueDB>,
candidate_descendents_lock: Arc<Mutex<()>>
}
// data keys
fn execution_data_key(candidate_hash: &Hash) -> Vec<u8> {
(candidate_hash, 0i8).encode()
}
fn erasure_chunks_key(candidate_hash: &Hash) -> Vec<u8> {
(candidate_hash, 1i8).encode()
}
fn candidate_key(candidate_hash: &Hash) -> Vec<u8> {
(candidate_hash, 2i8).encode()
}
fn candidates_with_relay_parent_key(relay_block: &Hash) -> Vec<u8> {
(relay_block, 4i8).encode()
}
// meta keys
const AWAITED_CHUNKS_KEY: [u8; 14] = *b"awaited_chunks";
fn validator_index_and_n_validators_key(relay_parent: &Hash) -> Vec<u8> {
(relay_parent, 1i8).encode()
}
fn available_chunks_key(candidate_hash: &Hash) -> Vec<u8> {
(candidate_hash, 2i8).encode()
}
/// An entry in the awaited frontier of chunks we are interested in.
#[derive(Encode, Decode, Debug, Hash, PartialEq, Eq, Clone)]
pub struct AwaitedFrontierEntry {
/// The hash of the candidate for which we want to fetch a chunk for.
/// There will be duplicate entries in the case of multiple candidates with
/// the same erasure-root, but this is unlikely.
pub candidate_hash: Hash,
/// Although the relay-parent is implicitly referenced by the candidate hash,
/// we include it here as well for convenience in pruning the set.
pub relay_parent: Hash,
/// The index of the validator we represent.
pub validator_index: u32,
}
impl Store {
/// Create a new `Store` with given condig on disk.
#[cfg(not(target_os = "unknown"))]
pub(super) fn new(config: Config) -> io::Result<Self> {
let mut db_config = DatabaseConfig::with_columns(columns::NUM_COLUMNS);
if let Some(cache_size) = config.cache_size {
let mut memory_budget = std::collections::HashMap::new();
for i in 0..columns::NUM_COLUMNS {
memory_budget.insert(i, cache_size / columns::NUM_COLUMNS as usize);
}
db_config.memory_budget = memory_budget;
}
let path = config.path.to_str().ok_or_else(|| io::Error::new(
io::ErrorKind::Other,
format!("Bad database path: {:?}", config.path),
))?;
let db = Database::open(&db_config, &path)?;
Ok(Store {
inner: Arc::new(db),
candidate_descendents_lock: Arc::new(Mutex::new(())),
})
}
/// Create a new `Store` in-memory. Useful for tests.
pub(super) fn new_in_memory() -> Self {
Store {
inner: Arc::new(::kvdb_memorydb::create(columns::NUM_COLUMNS)),
candidate_descendents_lock: Arc::new(Mutex::new(())),
}
}
/// Make some data available provisionally.
pub(crate) fn make_available(&self, candidate_hash: Hash, available_data: AvailableData)
-> io::Result<()>
{
let mut tx = DBTransaction::new();
// at the moment, these structs are identical. later, we will also
// keep outgoing message queues available, and these are not needed
// for execution.
let AvailableData { pov_block, omitted_validation } = available_data;
let execution_data = ExecutionData {
pov_block,
omitted_validation,
};
tx.put_vec(
columns::DATA,
execution_data_key(&candidate_hash).as_slice(),
execution_data.encode(),
);
self.inner.write(tx)
}
/// Get a set of all chunks we are waiting for.
pub fn awaited_chunks(&self) -> Option<HashSet<AwaitedFrontierEntry>> {
self.query_inner(columns::META, &AWAITED_CHUNKS_KEY).map(|vec: Vec<AwaitedFrontierEntry>| {
HashSet::from_iter(vec.into_iter())
})
}
/// Adds a set of candidates hashes that were included in a relay block by the block's parent.
///
/// If we already possess the receipts for these candidates _and_ our position at the specified
/// relay chain the awaited frontier of the erasure chunks will also be extended.
///
/// This method modifies the erasure chunks awaited frontier by adding this validator's
/// chunks from `candidates` to it. In order to do so the information about this validator's
/// position at parent `relay_parent` should be known to the store prior to calling this
/// method, in other words `note_validator_index_and_n_validators` should be called for
/// the given `relay_parent` before calling this function.
pub(crate) fn note_candidates_with_relay_parent(
&self,
relay_parent: &Hash,
candidates: &[Hash],
) -> io::Result<()> {
let mut tx = DBTransaction::new();
let dbkey = candidates_with_relay_parent_key(relay_parent);
// This call can race against another call to `note_candidates_with_relay_parent`
// with a different set of descendents.
let _lock = self.candidate_descendents_lock.lock();
if let Some((validator_index, _)) = self.get_validator_index_and_n_validators(relay_parent) {
let candidates = candidates.clone();
let awaited_frontier: Vec<AwaitedFrontierEntry> = self
.query_inner(columns::META, &AWAITED_CHUNKS_KEY)
.unwrap_or_else(|| Vec::new());
let mut awaited_frontier: HashSet<AwaitedFrontierEntry> =
HashSet::from_iter(awaited_frontier.into_iter());
awaited_frontier.extend(candidates.iter().cloned().map(|candidate_hash| {
AwaitedFrontierEntry {
relay_parent: relay_parent.clone(),
candidate_hash,
validator_index,
}
}));
let awaited_frontier = Vec::from_iter(awaited_frontier.into_iter());
tx.put_vec(columns::META, &AWAITED_CHUNKS_KEY, awaited_frontier.encode());
}
let mut descendent_candidates = self.get_candidates_with_relay_parent(relay_parent);
descendent_candidates.extend(candidates.iter().cloned());
tx.put_vec(columns::DATA, &dbkey, descendent_candidates.encode());
self.inner.write(tx)
}
/// Make a validator's index and a number of validators at a relay parent available.
pub(crate) fn note_validator_index_and_n_validators(
&self,
relay_parent: &Hash,
validator_index: u32,
n_validators: u32,
) -> io::Result<()> {
let mut tx = DBTransaction::new();
let dbkey = validator_index_and_n_validators_key(relay_parent);
tx.put_vec(columns::META, &dbkey, (validator_index, n_validators).encode());
self.inner.write(tx)
}
/// Query a validator's index and n_validators by relay parent.
pub(crate) fn get_validator_index_and_n_validators(&self, relay_parent: &Hash) -> Option<(u32, u32)> {
let dbkey = validator_index_and_n_validators_key(relay_parent);
self.query_inner(columns::META, &dbkey)
}
/// Add a set of chunks.
///
/// The same as `add_erasure_chunk` but adds a set of chunks in one atomic transaction.
pub fn add_erasure_chunks<I>(
&self,
n_validators: u32,
candidate_hash: &Hash,
chunks: I,
) -> io::Result<()>
where I: IntoIterator<Item = ErasureChunk>
{
if let Some(receipt) = self.get_candidate(candidate_hash) {
let mut tx = DBTransaction::new();
let dbkey = erasure_chunks_key(candidate_hash);
let mut v = self.query_inner(columns::DATA, &dbkey).unwrap_or(Vec::new());
let av_chunks_key = available_chunks_key(candidate_hash);
let mut have_chunks = self.query_inner(columns::META, &av_chunks_key).unwrap_or(Vec::new());
let awaited_frontier: Option<Vec<AwaitedFrontierEntry>> = self.query_inner(
columns::META,
&AWAITED_CHUNKS_KEY,
);
for chunk in chunks.into_iter() {
if !have_chunks.contains(&chunk.index) {
have_chunks.push(chunk.index);
}
v.push(chunk);
}
if let Some(mut awaited_frontier) = awaited_frontier {
awaited_frontier.retain(|entry| {
!(
entry.relay_parent == receipt.relay_parent &&
&entry.candidate_hash == candidate_hash &&
have_chunks.contains(&entry.validator_index)
)
});
tx.put_vec(columns::META, &AWAITED_CHUNKS_KEY, awaited_frontier.encode());
}
// If there are no block data in the store at this point,
// check that they can be reconstructed now and add them to store if they can.
if self.execution_data(&candidate_hash).is_none() {
if let Ok(available_data) = erasure::reconstruct_v0(
n_validators as usize,
v.iter().map(|chunk| (chunk.chunk.as_ref(), chunk.index as usize)),
)
{
self.make_available(*candidate_hash, available_data)?;
}
}
tx.put_vec(columns::DATA, &dbkey, v.encode());
tx.put_vec(columns::META, &av_chunks_key, have_chunks.encode());
self.inner.write(tx)
} else {
trace!(target: LOG_TARGET, "Candidate with hash {} not found", candidate_hash);
Ok(())
}
}
/// Queries an erasure chunk by its block's relay-parent, the candidate hash, and index.
pub fn get_erasure_chunk(
&self,
candidate_hash: &Hash,
index: usize,
) -> Option<ErasureChunk> {
self.query_inner(columns::DATA, &erasure_chunks_key(candidate_hash))
.and_then(|chunks: Vec<ErasureChunk>| {
chunks.iter()
.find(|chunk: &&ErasureChunk| chunk.index == index as u32)
.map(|chunk| chunk.clone())
})
}
/// Stores a candidate receipt.
pub fn add_candidate(
&self,
receipt: &AbridgedCandidateReceipt,
) -> io::Result<()> {
let candidate_hash = receipt.hash();
let dbkey = candidate_key(&candidate_hash);
let mut tx = DBTransaction::new();
tx.put_vec(columns::DATA, &dbkey, receipt.encode());
self.inner.write(tx)
}
/// Queries a candidate receipt by the relay parent hash and its hash.
pub(crate) fn get_candidate(&self, candidate_hash: &Hash)
-> Option<AbridgedCandidateReceipt>
{
self.query_inner(columns::DATA, &candidate_key(candidate_hash))
}
/// Note that a set of candidates have been included in a finalized block with given hash and parent hash.
pub(crate) fn candidates_finalized(
&self,
relay_parent: Hash,
finalized_candidates: HashSet<Hash>,
) -> io::Result<()> {
let mut tx = DBTransaction::new();
let awaited_frontier: Option<Vec<AwaitedFrontierEntry>> = self
.query_inner(columns::META, &AWAITED_CHUNKS_KEY);
if let Some(mut awaited_frontier) = awaited_frontier {
awaited_frontier.retain(|entry| entry.relay_parent != relay_parent);
tx.put_vec(columns::META, &AWAITED_CHUNKS_KEY, awaited_frontier.encode());
}
let candidates = self.get_candidates_with_relay_parent(&relay_parent);
for candidate in candidates.into_iter().filter(|c| !finalized_candidates.contains(c)) {
// we only delete this data for candidates which were not finalized.
// we keep all data for the finalized chain forever at the moment.
tx.delete(columns::DATA, execution_data_key(&candidate).as_slice());
tx.delete(columns::DATA, &erasure_chunks_key(&candidate));
tx.delete(columns::DATA, &candidate_key(&candidate));
tx.delete(columns::META, &available_chunks_key(&candidate));
}
self.inner.write(tx)
}
/// Query execution data by relay parent and candidate hash.
pub(crate) fn execution_data(&self, candidate_hash: &Hash) -> Option<ExecutionData> {
self.query_inner(columns::DATA, &execution_data_key(candidate_hash))
}
/// Get candidates which pinned to the environment of the given relay parent.
/// Note that this is not necessarily the same as candidates that were included in a direct
/// descendent of the given relay-parent.
fn get_candidates_with_relay_parent(&self, relay_parent: &Hash) -> Vec<Hash> {
let key = candidates_with_relay_parent_key(relay_parent);
self.query_inner(columns::DATA, &key[..]).unwrap_or_default()
}
fn query_inner<T: Decode>(&self, column: u32, key: &[u8]) -> Option<T> {
match self.inner.get(column, key) {
Ok(Some(raw)) => {
let res = T::decode(&mut &raw[..]).expect("all stored data serialized correctly; qed");
Some(res)
}
Ok(None) => None,
Err(e) => {
warn!(target: LOG_TARGET, "Error reading from the availability store: {:?}", e);
None
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use polkadot_erasure_coding::{self as erasure};
use polkadot_primitives::v0::{
Id as ParaId, BlockData, AvailableData, PoVBlock, OmittedValidationData,
};
fn available_data(block_data: &[u8]) -> AvailableData {
AvailableData {
pov_block: PoVBlock {
block_data: BlockData(block_data.to_vec()),
},
omitted_validation: OmittedValidationData {
global_validation: Default::default(),
local_validation: Default::default(),
}
}
}
fn execution_data(available: &AvailableData) -> ExecutionData {
let AvailableData { pov_block, omitted_validation } = available.clone();
ExecutionData { pov_block, omitted_validation }
}
#[test]
fn finalization_removes_unneeded() {
let relay_parent = [1; 32].into();
let para_id_1 = 5.into();
let para_id_2 = 6.into();
let mut candidate_1 = AbridgedCandidateReceipt::default();
let mut candidate_2 = AbridgedCandidateReceipt::default();
candidate_1.parachain_index = para_id_1;
candidate_1.commitments.erasure_root = [6; 32].into();
candidate_1.relay_parent = relay_parent;
candidate_2.parachain_index = para_id_2;
candidate_2.commitments.erasure_root = [6; 32].into();
candidate_2.relay_parent = relay_parent;
let candidate_1_hash = candidate_1.hash();
let candidate_2_hash = candidate_2.hash();
let available_data_1 = available_data(&[1, 2, 3]);
let available_data_2 = available_data(&[4, 5, 6]);
let erasure_chunk_1 = ErasureChunk {
chunk: vec![10, 20, 30],
index: 1,
proof: vec![],
};
let erasure_chunk_2 = ErasureChunk {
chunk: vec![40, 50, 60],
index: 1,
proof: vec![],
};
let store = Store::new_in_memory();
store.make_available(candidate_1_hash, available_data_1.clone()).unwrap();
store.make_available(candidate_2_hash, available_data_2.clone()).unwrap();
store.add_candidate(&candidate_1).unwrap();
store.add_candidate(&candidate_2).unwrap();
store.note_candidates_with_relay_parent(&relay_parent, &[candidate_1_hash, candidate_2_hash]).unwrap();
assert!(store.add_erasure_chunks(3, &candidate_1_hash, vec![erasure_chunk_1.clone()]).is_ok());
assert!(store.add_erasure_chunks(3, &candidate_2_hash, vec![erasure_chunk_2.clone()]).is_ok());
assert_eq!(store.execution_data(&candidate_1_hash).unwrap(), execution_data(&available_data_1));
assert_eq!(store.execution_data(&candidate_2_hash).unwrap(), execution_data(&available_data_2));
assert_eq!(store.get_erasure_chunk(&candidate_1_hash, 1).as_ref(), Some(&erasure_chunk_1));
assert_eq!(store.get_erasure_chunk(&candidate_2_hash, 1), Some(erasure_chunk_2));
assert_eq!(store.get_candidate(&candidate_1_hash), Some(candidate_1.clone()));
assert_eq!(store.get_candidate(&candidate_2_hash), Some(candidate_2.clone()));
store.candidates_finalized(relay_parent, [candidate_1_hash].iter().cloned().collect()).unwrap();
assert_eq!(store.get_erasure_chunk(&candidate_1_hash, 1).as_ref(), Some(&erasure_chunk_1));
assert!(store.get_erasure_chunk(&candidate_2_hash, 1).is_none());
assert_eq!(store.get_candidate(&candidate_1_hash), Some(candidate_1));
assert_eq!(store.get_candidate(&candidate_2_hash), None);
assert_eq!(store.execution_data(&candidate_1_hash).unwrap(), execution_data(&available_data_1));
assert!(store.execution_data(&candidate_2_hash).is_none());
}
#[test]
fn erasure_coding() {
let relay_parent: Hash = [1; 32].into();
let para_id: ParaId = 5.into();
let available_data = available_data(&[42; 8]);
let n_validators = 5;
let erasure_chunks = erasure::obtain_chunks_v0(
n_validators,
&available_data,
).unwrap();
let branches = erasure::branches(erasure_chunks.as_ref());
let mut candidate = AbridgedCandidateReceipt::default();
candidate.parachain_index = para_id;
candidate.commitments.erasure_root = [6; 32].into();
candidate.relay_parent = relay_parent;
let candidate_hash = candidate.hash();
let chunks: Vec<_> = erasure_chunks
.iter()
.zip(branches.map(|(proof, _)| proof))
.enumerate()
.map(|(index, (chunk, proof))| ErasureChunk {
chunk: chunk.clone(),
proof,
index: index as u32,
})
.collect();
let store = Store::new_in_memory();
store.add_candidate(&candidate).unwrap();
store.add_erasure_chunks(n_validators as u32, &candidate_hash, vec![chunks[0].clone()]).unwrap();
assert_eq!(store.get_erasure_chunk(&candidate_hash, 0), Some(chunks[0].clone()));
assert!(store.execution_data(&candidate_hash).is_none());
store.add_erasure_chunks(n_validators as u32, &candidate_hash, chunks).unwrap();
assert_eq!(store.execution_data(&candidate_hash), Some(execution_data(&available_data)));
}
#[test]
fn add_validator_index_works() {
let relay_parent = [42; 32].into();
let store = Store::new_in_memory();
store.note_validator_index_and_n_validators(&relay_parent, 42, 24).unwrap();
assert_eq!(store.get_validator_index_and_n_validators(&relay_parent).unwrap(), (42, 24));
}
#[test]
fn add_candidates_in_relay_block_works() {
let relay_parent = [42; 32].into();
let store = Store::new_in_memory();
let candidates = vec![[1; 32].into(), [2; 32].into(), [3; 32].into()];
store.note_candidates_with_relay_parent(&relay_parent, &candidates).unwrap();
assert_eq!(store.get_candidates_with_relay_parent(&relay_parent), candidates);
}
#[test]
fn awaited_chunks_works() {
use std::iter::FromIterator;
let validator_index = 3;
let n_validators = 10;
let relay_parent = [42; 32].into();
let erasure_root_1 = [11; 32].into();
let erasure_root_2 = [12; 32].into();
let mut receipt_1 = AbridgedCandidateReceipt::default();
let mut receipt_2 = AbridgedCandidateReceipt::default();
receipt_1.parachain_index = 1.into();
receipt_1.commitments.erasure_root = erasure_root_1;
receipt_1.relay_parent = relay_parent;
receipt_2.parachain_index = 2.into();
receipt_2.commitments.erasure_root = erasure_root_2;
receipt_2.relay_parent = relay_parent;
let receipt_1_hash = receipt_1.hash();
let receipt_2_hash = receipt_2.hash();
let chunk = ErasureChunk {
chunk: vec![1, 2, 3],
index: validator_index,
proof: Vec::new(),
};
let candidates = vec![receipt_1_hash, receipt_2_hash];
let store = Store::new_in_memory();
store.note_validator_index_and_n_validators(
&relay_parent,
validator_index,
n_validators
).unwrap();
store.add_candidate(&receipt_1).unwrap();
store.add_candidate(&receipt_2).unwrap();
// We are waiting for chunks from two candidates.
store.note_candidates_with_relay_parent(&relay_parent, &candidates).unwrap();
let awaited_frontier = store.awaited_chunks().unwrap();
warn!(target: "availability", "awaited {:?}", awaited_frontier);
let expected: HashSet<_> = candidates
.clone()
.into_iter()
.map(|c| AwaitedFrontierEntry {
relay_parent,
candidate_hash: c,
validator_index,
})
.collect();
assert_eq!(awaited_frontier, expected);
// We add chunk from one of the candidates.
store.add_erasure_chunks(n_validators, &receipt_1_hash, vec![chunk]).unwrap();
let awaited_frontier = store.awaited_chunks().unwrap();
// Now we wait for the other chunk that we haven't received yet.
let expected: HashSet<_> = vec![AwaitedFrontierEntry {
relay_parent,
candidate_hash: receipt_2_hash,
validator_index,
}].into_iter().collect();
assert_eq!(awaited_frontier, expected);
// Finalizing removes awaited candidates from frontier.
store.candidates_finalized(relay_parent, HashSet::from_iter(candidates.into_iter())).unwrap();
assert_eq!(store.awaited_chunks().unwrap().len(), 0);
}
}
polkadot/availability-store/src/worker.rs
-941
View File
@@ -1,941 +0,0 @@
// Copyright 2018-2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
use std::collections::{HashMap, HashSet};
use std::io;
use std::sync::Arc;
use std::thread;
use log::{error, info, trace, warn};
use sp_blockchain::Result as ClientResult;
use sp_runtime::traits::{Header as HeaderT, Block as BlockT, HashFor, BlakeTwo256};
use sp_api::{ApiExt, ProvideRuntimeApi};
use client::{
BlockchainEvents, BlockBackend,
blockchain::ProvideCache,
};
use consensus_common::{
self, BlockImport, BlockCheckParams, BlockImportParams, Error as ConsensusError,
ImportResult,
import_queue::CacheKeyId,
};
use sp_core::traits::SpawnNamed;
use polkadot_primitives::v0::{
Block, BlockId, Hash,
ParachainHost, ValidatorId, AbridgedCandidateReceipt, AvailableData,
ValidatorPair, ErasureChunk,
};
use futures::{prelude::*, future::select, channel::{mpsc, oneshot}};
use futures::future::AbortHandle;
use keystore::KeyStorePtr;
use tokio::runtime::{Handle, Runtime as LocalRuntime};
use crate::{LOG_TARGET, ErasureNetworking};
use crate::store::Store;
/// Errors that may occur.
#[derive(Debug, derive_more::Display, derive_more::From)]
pub(crate) enum Error {
#[from]
StoreError(io::Error),
#[display(fmt = "Validator's id and number of validators at block with parent {} not found", relay_parent)]
IdAndNValidatorsNotFound { relay_parent: Hash },
}
/// Used in testing to interact with the worker thread.
#[cfg(test)]
pub(crate) struct WithWorker(Box<dyn FnOnce(&mut Worker) + Send>);
#[cfg(test)]
impl std::fmt::Debug for WithWorker {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(f, "<boxed closure>")
}
}
/// Messages sent to the `Worker`.
///
/// Messages are sent in a number of different scenarios,
/// for instance, when:
/// * importing blocks in `BlockImport` implementation,
/// * recieving finality notifications,
/// * when the `Store` api is used by outside code.
#[derive(Debug)]
pub(crate) enum WorkerMsg {
IncludedParachainBlocks(IncludedParachainBlocks),
Chunks(Chunks),
CandidatesFinalized(CandidatesFinalized),
MakeAvailable(MakeAvailable),
#[cfg(test)]
WithWorker(WithWorker),
}
/// A notification of a parachain block included in the relay chain.
#[derive(Debug)]
pub(crate) struct IncludedParachainBlock {
/// The abridged candidate receipt, extracted from a relay-chain block.
pub candidate: AbridgedCandidateReceipt,
/// The data to keep available from the candidate, if known.
pub available_data: Option<AvailableData>,
}
/// The receipts of the heads included into the block with a given parent.
#[derive(Debug)]
pub(crate) struct IncludedParachainBlocks {
/// The blocks themselves.
pub blocks: Vec<IncludedParachainBlock>,
/// A sender to signal the result asynchronously.
pub result: oneshot::Sender<Result<(), Error>>,
}
/// We have received chunks we requested.
#[derive(Debug)]
pub(crate) struct Chunks {
/// The hash of the parachain candidate these chunks belong to.
pub candidate_hash: Hash,
/// The chunks
pub chunks: Vec<ErasureChunk>,
/// The number of validators present at the candidate's relay-parent.
pub n_validators: u32,
/// A sender to signal the result asynchronously.
pub result: oneshot::Sender<Result<(), Error>>,
}
/// These candidates have been finalized, so unneded availability may be now pruned
#[derive(Debug)]
pub(crate) struct CandidatesFinalized {
/// The relay parent of the block that was finalized.
relay_parent: Hash,
/// The hashes of candidates that were finalized in this block.
included_candidates: HashSet<Hash>,
}
/// The message that corresponds to `make_available` call of the crate API.
#[derive(Debug)]
pub(crate) struct MakeAvailable {
/// The hash of the candidate for which we are publishing data.
pub candidate_hash: Hash,
/// The data to make available.
pub available_data: AvailableData,
/// A sender to signal the result asynchronously.
pub result: oneshot::Sender<Result<(), Error>>,
}
/// Description of a chunk we are listening for.
#[derive(Hash, Debug, PartialEq, Eq)]
struct ListeningKey {
candidate_hash: Hash,
index: u32,
}
/// An availability worker with it's inner state.
pub(super) struct Worker {
availability_store: Store,
listening_for: HashMap<ListeningKey, AbortHandle>,
sender: mpsc::UnboundedSender<WorkerMsg>,
}
/// The handle to the `Worker`.
pub(super) struct WorkerHandle {
thread: Option<thread::JoinHandle<io::Result<()>>>,
sender: mpsc::UnboundedSender<WorkerMsg>,
exit_signal: Option<exit_future::Signal>,
}
impl WorkerHandle {
pub(crate) fn to_worker(&self) -> &mpsc::UnboundedSender<WorkerMsg> {
&self.sender
}
}
impl Drop for WorkerHandle {
fn drop(&mut self) {
if let Some(signal) = self.exit_signal.take() {
let _ = signal.fire();
}
if let Some(thread) = self.thread.take() {
if let Err(_) = thread.join() {
error!(target: LOG_TARGET, "Errored stopping the thread");
}
}
}
}
fn fetch_candidates<P>(client: &P, extrinsics: Vec<<Block as BlockT>::Extrinsic>, parent: &BlockId)
-> ClientResult<Option<Vec<AbridgedCandidateReceipt>>>
where
P: ProvideRuntimeApi<Block>,
P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
// Rust bug: https://github.com/rust-lang/rust/issues/24159
sp_api::StateBackendFor<P, Block>: sp_api::StateBackend<HashFor<Block>>,
{
let api = client.runtime_api();
let candidates = if api.has_api_with::<dyn ParachainHost<Block, Error = ()>, _>(
parent,
|version| version >= 2,
).map_err(|e| ConsensusError::ChainLookup(e.to_string()))? {
api.get_heads(&parent, extrinsics)
.map_err(|e| ConsensusError::ChainLookup(e.to_string()))?
} else {
None
};
Ok(candidates)
}
/// Creates a task to prune entries in availability store upon block finalization.
async fn prune_unneeded_availability<P, S>(client: Arc<P>, mut sender: S)
where
P: ProvideRuntimeApi<Block> + BlockchainEvents<Block> + BlockBackend<Block> + Send + Sync + 'static,
P::Api: ParachainHost<Block> + ApiExt<Block, Error=sp_blockchain::Error>,
S: Sink<WorkerMsg> + Clone + Send + Sync + Unpin,
// Rust bug: https://github.com/rust-lang/rust/issues/24159
sp_api::StateBackendFor<P, Block>: sp_api::StateBackend<HashFor<Block>>,
{
let mut finality_notification_stream = client.finality_notification_stream();
while let Some(notification) = finality_notification_stream.next().await {
let hash = notification.hash;
let parent_hash = notification.header.parent_hash;
let extrinsics = match client.block_body(&BlockId::hash(hash)) {
Ok(Some(extrinsics)) => extrinsics,
Ok(None) => {
error!(
target: LOG_TARGET,
"No block body found for imported block {:?}",
hash,
);
continue;
}
Err(e) => {
error!(
target: LOG_TARGET,
"Failed to get block body for imported block {:?}: {:?}",
hash,
e,
);
continue;
}
};
let included_candidates = match fetch_candidates(
&*client,
extrinsics,
&BlockId::hash(parent_hash),
) {
Ok(Some(candidates)) => candidates
.into_iter()
.map(|c| c.hash())
.collect(),
Ok(None) => {
warn!(
target: LOG_TARGET,
"Failed to extract candidates from block body of imported block {:?}", hash
);
continue;
}
Err(e) => {
warn!(
target: LOG_TARGET,
"Failed to fetch block body for imported block {:?}: {:?}", hash, e
);
continue;
}
};
let msg = WorkerMsg::CandidatesFinalized(CandidatesFinalized {
relay_parent: parent_hash,
included_candidates
});
if let Err(_) = sender.send(msg).await {
break;
}
}
}
impl Worker {
// Called on startup of the worker to initiate fetch from network for all awaited chunks.
fn initiate_all_fetches<EN: ErasureNetworking>(
&mut self,
runtime_handle: &Handle,
erasure_network: &EN,
sender: &mut mpsc::UnboundedSender<WorkerMsg>,
) {
if let Some(awaited_chunks) = self.availability_store.awaited_chunks() {
for awaited_chunk in awaited_chunks {
if let Err(e) = self.initiate_fetch(
runtime_handle,
erasure_network,
sender,
awaited_chunk.relay_parent,
awaited_chunk.candidate_hash,
) {
warn!(target: LOG_TARGET, "Failed to register network listener: {}", e);
}
}
}
}
// initiates a fetch from network for the described chunk, with our local index.
fn initiate_fetch<EN: ErasureNetworking>(
&mut self,
runtime_handle: &Handle,
erasure_network: &EN,
sender: &mut mpsc::UnboundedSender<WorkerMsg>,
relay_parent: Hash,
candidate_hash: Hash,
) -> Result<(), Error> {
let (local_id, n_validators) = self.availability_store
.get_validator_index_and_n_validators(&relay_parent)
.ok_or(Error::IdAndNValidatorsNotFound { relay_parent })?;
// fast exit for if we already have the chunk.
if self.availability_store.get_erasure_chunk(&candidate_hash, local_id as _).is_some() {
return Ok(())
}
trace!(
target: LOG_TARGET,
"Initiating fetch for erasure-chunk at parent {} with candidate-hash {}",
relay_parent,
candidate_hash,
);
let fut = erasure_network.fetch_erasure_chunk(&candidate_hash, local_id);
let mut sender = sender.clone();
let (fut, signal) = future::abortable(async move {
let chunk = match fut.await {
Ok(chunk) => chunk,
Err(e) => {
warn!(target: LOG_TARGET, "Unable to fetch erasure-chunk from network: {:?}", e);
return
}
};
let (s, _) = oneshot::channel();
let _ = sender.send(WorkerMsg::Chunks(Chunks {
candidate_hash,
chunks: vec![chunk],
n_validators,
result: s,
})).await;
}.map(drop).boxed());
let key = ListeningKey {
candidate_hash,
index: local_id,
};
self.listening_for.insert(key, signal);
let _ = runtime_handle.spawn(fut);
Ok(())
}
fn on_parachain_blocks_received<EN: ErasureNetworking>(
&mut self,
runtime_handle: &Handle,
erasure_network: &EN,
sender: &mut mpsc::UnboundedSender<WorkerMsg>,
blocks: Vec<IncludedParachainBlock>,
) -> Result<(), Error> {
// First we have to add the receipts themselves.
for IncludedParachainBlock { candidate, available_data }
in blocks.into_iter()
{
let _ = self.availability_store.add_candidate(&candidate);
if let Some(_available_data) = available_data {
// Should we be breaking block into chunks here and gossiping it and so on?
}
// This leans on the codebase-wide assumption that the `relay_parent`
// of all candidates in a block matches the parent hash of that block.
//
// In the future this will not always be true.
let candidate_hash = candidate.hash();
let _ = self.availability_store.note_candidates_with_relay_parent(
&candidate.relay_parent,
&[candidate_hash],
);
if let Err(e) = self.initiate_fetch(
runtime_handle,
erasure_network,
sender,
candidate.relay_parent,
candidate_hash,
) {
warn!(target: LOG_TARGET, "Failed to register chunk listener: {}", e);
}
}
Ok(())
}
// Handles chunks that were required.
fn on_chunks(
&mut self,
candidate_hash: Hash,
chunks: Vec<ErasureChunk>,
n_validators: u32,
) -> Result<(), Error> {
for c in &chunks {
let key = ListeningKey {
candidate_hash,
index: c.index,
};
// remove bookkeeping so network does not attempt to fetch
// any longer.
if let Some(exit_signal) = self.listening_for.remove(&key) {
exit_signal.abort();
}
}
self.availability_store.add_erasure_chunks(
n_validators,
&candidate_hash,
chunks,
)?;
Ok(())
}
/// Starts a worker with a given availability store and a gossip messages provider.
pub fn start<EN: ErasureNetworking + Send + 'static>(
availability_store: Store,
erasure_network: EN,
) -> WorkerHandle {
let (sender, mut receiver) = mpsc::unbounded();
let mut worker = Worker {
availability_store,
listening_for: HashMap::new(),
sender: sender.clone(),
};
let sender = sender.clone();
let (signal, exit) = exit_future::signal();
let handle = thread::spawn(move || -> io::Result<()> {
let mut runtime = LocalRuntime::new()?;
let mut sender = worker.sender.clone();
let runtime_handle = runtime.handle().clone();
// On startup, initiates fetch from network for all
// entries in the awaited frontier.
worker.initiate_all_fetches(runtime.handle(), &erasure_network, &mut sender);
let process_notification = async move {
while let Some(msg) = receiver.next().await {
trace!(target: LOG_TARGET, "Received message {:?}", msg);
let res = match msg {
WorkerMsg::IncludedParachainBlocks(msg) => {
let IncludedParachainBlocks {
blocks,
result,
} = msg;
let res = worker.on_parachain_blocks_received(
&runtime_handle,
&erasure_network,
&mut sender,
blocks,
);
let _ = result.send(res);
Ok(())
}
WorkerMsg::Chunks(msg) => {
let Chunks {
candidate_hash,
chunks,
n_validators,
result,
} = msg;
let res = worker.on_chunks(
candidate_hash,
chunks,
n_validators,
);
let _ = result.send(res);
Ok(())
}
WorkerMsg::CandidatesFinalized(msg) => {
let CandidatesFinalized { relay_parent, included_candidates } = msg;
worker.availability_store.candidates_finalized(
relay_parent,
included_candidates,
)
}
WorkerMsg::MakeAvailable(msg) => {
let MakeAvailable { candidate_hash, available_data, result } = msg;
let res = worker.availability_store
.make_available(candidate_hash, available_data)
.map_err(|e| e.into());
let _ = result.send(res);
Ok(())
}
#[cfg(test)]
WorkerMsg::WithWorker(with_worker) => {
(with_worker.0)(&mut worker);
Ok(())
}
};
if let Err(_) = res {
warn!(target: LOG_TARGET, "An error occured while processing a message");
}
}
};
runtime.spawn(select(process_notification.boxed(), exit.clone()).map(drop));
runtime.block_on(exit);
info!(target: LOG_TARGET, "Availability worker exiting");
Ok(())
});
WorkerHandle {
thread: Some(handle),
sender,
exit_signal: Some(signal),
}
}
}
/// Implementer of the [`BlockImport`] trait.
///
/// Used to embed `availability-store` logic into the block imporing pipeline.
///
/// [`BlockImport`]: https://substrate.dev/rustdocs/v1.0/substrate_consensus_common/trait.BlockImport.html
pub struct AvailabilityBlockImport<I, P> {
inner: I,
client: Arc<P>,
keystore: KeyStorePtr,
to_worker: mpsc::UnboundedSender<WorkerMsg>,
exit_signal: AbortHandle,
}
impl<I, P> Drop for AvailabilityBlockImport<I, P> {
fn drop(&mut self) {
self.exit_signal.abort();
}
}
impl<I, P> BlockImport<Block> for AvailabilityBlockImport<I, P> where
I: BlockImport<Block, Transaction = sp_api::TransactionFor<P, Block>> + Send + Sync,
I::Error: Into<ConsensusError>,
P: ProvideRuntimeApi<Block> + ProvideCache<Block>,
P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
// Rust bug: https://github.com/rust-lang/rust/issues/24159
sp_api::StateBackendFor<P, Block>: sp_api::StateBackend<BlakeTwo256>
{
type Error = ConsensusError;
type Transaction = sp_api::TransactionFor<P, Block>;
fn import_block(
&mut self,
block: BlockImportParams<Block, Self::Transaction>,
new_cache: HashMap<CacheKeyId, Vec<u8>>,
) -> Result<ImportResult, Self::Error> {
trace!(
target: LOG_TARGET,
"Importing block #{}, ({})",
block.header.number(),
block.post_hash(),
);
if let Some(ref extrinsics) = block.body {
let parent_id = BlockId::hash(*block.header.parent_hash());
// Extract our local position i from the validator set of the parent.
let validators = self.client.runtime_api().validators(&parent_id)
.map_err(|e| ConsensusError::ChainLookup(e.to_string()))?;
let our_id = self.our_id(&validators);
// Use a runtime API to extract all included erasure-roots from the imported block.
let candidates = fetch_candidates(&*self.client, extrinsics.clone(), &parent_id)
.map_err(|e| ConsensusError::ChainLookup(e.to_string()))?;
match candidates {
Some(candidates) => {
match our_id {
Some(our_id) => {
trace!(
target: LOG_TARGET,
"Our validator id is {}, the candidates included are {:?}",
our_id,
candidates,
);
let (s, _) = oneshot::channel();
// Inform the worker about the included parachain blocks.
let blocks = candidates
.into_iter()
.map(|c| IncludedParachainBlock {
candidate: c,
available_data: None,
})
.collect();
let msg = WorkerMsg::IncludedParachainBlocks(IncludedParachainBlocks {
blocks,
result: s,
});
let _ = self.to_worker.unbounded_send(msg);
}
None => (),
}
}
None => {
trace!(
target: LOG_TARGET,
"No parachain heads were included in block {}", block.header.hash()
);
},
}
}
self.inner.import_block(block, new_cache).map_err(Into::into)
}
fn check_block(
&mut self,
block: BlockCheckParams<Block>,
) -> Result<ImportResult, Self::Error> {
self.inner.check_block(block).map_err(Into::into)
}
}
impl<I, P> AvailabilityBlockImport<I, P> {
pub(crate) fn new(
client: Arc<P>,
block_import: I,
spawner: impl SpawnNamed,
keystore: KeyStorePtr,
to_worker: mpsc::UnboundedSender<WorkerMsg>,
) -> Self
where
P: ProvideRuntimeApi<Block> + BlockBackend<Block> + BlockchainEvents<Block> + Send + Sync + 'static,
P::Api: ParachainHost<Block>,
P::Api: ApiExt<Block, Error = sp_blockchain::Error>,
// Rust bug: https://github.com/rust-lang/rust/issues/24159
sp_api::StateBackendFor<P, Block>: sp_api::StateBackend<HashFor<Block>>,
{
// This is not the right place to spawn the finality future,
// it would be more appropriate to spawn it in the `start` method of the `Worker`.
// However, this would make the type of the `Worker` and the `Store` itself
// dependent on the types of client and executor, which would prove
// not not so handy in the testing code.
let (prune_available, exit_signal) = future::abortable(prune_unneeded_availability(
client.clone(),
to_worker.clone(),
));
spawner.spawn("polkadot-prune-availibility", prune_available.map(drop).boxed());
AvailabilityBlockImport {
client,
inner: block_import,
to_worker,
keystore,
exit_signal,
}
}
fn our_id(&self, validators: &[ValidatorId]) -> Option<u32> {
let keystore = self.keystore.read();
validators
.iter()
.enumerate()
.find_map(|(i, v)| {
keystore.key_pair::<ValidatorPair>(&v).map(|_| i as u32).ok()
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use futures::channel::oneshot;
use std::sync::Arc;
use std::pin::Pin;
use tokio::runtime::Runtime;
use parking_lot::Mutex;
use crate::store::AwaitedFrontierEntry;
#[derive(Default, Clone)]
struct TestErasureNetwork {
chunk_receivers: Arc<Mutex<HashMap<
(Hash, u32),
oneshot::Receiver<ErasureChunk>
>>>,
}
impl TestErasureNetwork {
// adds a receiver. this returns a sender for the erasure-chunk
// along with an exit future that fires when the erasure chunk has
// been fully-processed
fn add_receiver(&self, candidate_hash: Hash, index: u32)
-> oneshot::Sender<ErasureChunk>
{
let (sender, receiver) = oneshot::channel();
self.chunk_receivers.lock().insert((candidate_hash, index), receiver);
sender
}
}
impl ErasureNetworking for TestErasureNetwork {
type Error = String;
fn fetch_erasure_chunk(&self, candidate_hash: &Hash, index: u32)
-> Pin<Box<dyn Future<Output = Result<ErasureChunk, Self::Error>> + Send>>
{
match self.chunk_receivers.lock().remove(&(*candidate_hash, index)) {
Some(receiver) => receiver.then(|x| match x {
Ok(x) => future::ready(Ok(x)).left_future(),
Err(_) => future::pending().right_future(),
}).boxed(),
None => future::pending().boxed(),
}
}
fn distribute_erasure_chunk(
&self,
_candidate_hash: Hash,
_chunk: ErasureChunk
) {}
}
// This test tests that as soon as the worker receives info about new parachain blocks
// included it registers gossip listeners for it's own chunks. Upon receiving the awaited
// chunk messages the corresponding listeners are deregistered and these chunks are removed
// from the awaited chunks set.
#[test]
fn receiving_gossip_chunk_removes_from_frontier() {
let mut runtime = Runtime::new().unwrap();
let relay_parent = [1; 32].into();
let local_id = 2;
let n_validators = 4;
let store = Store::new_in_memory();
let mut candidate = AbridgedCandidateReceipt::default();
candidate.relay_parent = relay_parent;
let candidate_hash = candidate.hash();
// Tell the store our validator's position and the number of validators at given point.
store.note_validator_index_and_n_validators(&relay_parent, local_id, n_validators).unwrap();
let network = TestErasureNetwork::default();
let chunk_sender = network.add_receiver(candidate_hash, local_id);
// At this point we shouldn't be waiting for any chunks.
assert!(store.awaited_chunks().is_none());
let (s, r) = oneshot::channel();
let msg = WorkerMsg::IncludedParachainBlocks(IncludedParachainBlocks {
blocks: vec![IncludedParachainBlock {
candidate,
available_data: None,
}],
result: s,
});
let handle = Worker::start(store.clone(), network);
// Tell the worker that the new blocks have been included into the relay chain.
// This should trigger the registration of gossip message listeners for the
// chunk topics.
handle.sender.unbounded_send(msg).unwrap();
runtime.block_on(r).unwrap().unwrap();
// Make sure that at this point we are waiting for the appropriate chunk.
assert_eq!(
store.awaited_chunks().unwrap(),
vec![AwaitedFrontierEntry {
relay_parent,
candidate_hash,
validator_index: local_id,
}].into_iter().collect()
);
// Complete the chunk request.
chunk_sender.send(ErasureChunk {
chunk: vec![1, 2, 3],
index: local_id as u32,
proof: vec![],
}).unwrap();
// wait until worker thread has de-registered the listener for a
// particular chunk.
loop {
let (s, r) = oneshot::channel();
handle.sender.unbounded_send(WorkerMsg::WithWorker(WithWorker(Box::new(move |worker| {
let key = ListeningKey {
candidate_hash,
index: local_id,
};
let is_waiting = worker.listening_for.contains_key(&key);
s.send(!is_waiting).unwrap(); // tell the test thread `true` if we are not waiting.
})))).unwrap();
if runtime.block_on(r).unwrap() {
break
}
}
// The awaited chunk has been received so at this point we no longer wait for any chunks.
assert_eq!(store.awaited_chunks().unwrap().len(), 0);
}
#[test]
fn included_parachain_blocks_registers_listener() {
let mut runtime = Runtime::new().unwrap();
let relay_parent = [1; 32].into();
let erasure_root_1 = [2; 32].into();
let erasure_root_2 = [3; 32].into();
let pov_block_hash_1 = [4; 32].into();
let pov_block_hash_2 = [5; 32].into();
let local_id = 2;
let n_validators = 4;
let mut candidate_1 = AbridgedCandidateReceipt::default();
candidate_1.commitments.erasure_root = erasure_root_1;
candidate_1.pov_block_hash = pov_block_hash_1;
candidate_1.relay_parent = relay_parent;
let candidate_1_hash = candidate_1.hash();
let mut candidate_2 = AbridgedCandidateReceipt::default();
candidate_2.commitments.erasure_root = erasure_root_2;
candidate_2.pov_block_hash = pov_block_hash_2;
candidate_2.relay_parent = relay_parent;
let candidate_2_hash = candidate_2.hash();
let store = Store::new_in_memory();
// Tell the store our validator's position and the number of validators at given point.
store.note_validator_index_and_n_validators(&relay_parent, local_id, n_validators).unwrap();
// Let the store know about the candidates
store.add_candidate(&candidate_1).unwrap();
store.add_candidate(&candidate_2).unwrap();
// And let the store know about the chunk from the second candidate.
store.add_erasure_chunks(
n_validators,
&candidate_2_hash,
vec![ErasureChunk {
chunk: vec![1, 2, 3],
index: local_id,
proof: Vec::default(),
}],
).unwrap();
let network = TestErasureNetwork::default();
let _ = network.add_receiver(candidate_1_hash, local_id);
let _ = network.add_receiver(candidate_2_hash, local_id);
let handle = Worker::start(store.clone(), network.clone());
{
let (s, r) = oneshot::channel();
// Tell the worker to listen for chunks from candidate 2 (we alredy have a chunk from it).
let listen_msg_2 = WorkerMsg::IncludedParachainBlocks(IncludedParachainBlocks {
blocks: vec![IncludedParachainBlock {
candidate: candidate_2,
available_data: None,
}],
result: s,
});
handle.sender.unbounded_send(listen_msg_2).unwrap();
runtime.block_on(r).unwrap().unwrap();
// The receiver for this chunk left intact => listener not registered.
assert!(network.chunk_receivers.lock().contains_key(&(candidate_2_hash, local_id)));
// more directly:
let (s, r) = oneshot::channel();
handle.sender.unbounded_send(WorkerMsg::WithWorker(WithWorker(Box::new(move |worker| {
let key = ListeningKey {
candidate_hash: candidate_2_hash,
index: local_id,
};
let _ = s.send(worker.listening_for.contains_key(&key));
})))).unwrap();
assert!(!runtime.block_on(r).unwrap());
}
{
let (s, r) = oneshot::channel();
// Tell the worker to listen for chunks from candidate 1.
// (we don't have a chunk from it yet).
let listen_msg_1 = WorkerMsg::IncludedParachainBlocks(IncludedParachainBlocks {
blocks: vec![IncludedParachainBlock {
candidate: candidate_1,
available_data: None,
}],
result: s,
});
handle.sender.unbounded_send(listen_msg_1).unwrap();
runtime.block_on(r).unwrap().unwrap();
// The receiver taken => listener registered.
assert!(!network.chunk_receivers.lock().contains_key(&(candidate_1_hash, local_id)));
// more directly:
let (s, r) = oneshot::channel();
handle.sender.unbounded_send(WorkerMsg::WithWorker(WithWorker(Box::new(move |worker| {
let key = ListeningKey {
candidate_hash: candidate_1_hash,
index: local_id,
};
let _ = s.send(worker.listening_for.contains_key(&key));
})))).unwrap();
assert!(runtime.block_on(r).unwrap());
}
}
}
polkadot/cli/src/command.rs
-2
View File
@@ -144,9 +144,7 @@ pub fn run() -> Result<()> {
_ => service::build_full(
config,
None,
None,
authority_discovery_enabled,
6000,
grandpa_pause,
).map(|r| r.0),
}
polkadot/collator/Cargo.toml
-37
View File
@@ -1,37 +0,0 @@
[package]
name = "polkadot-collator"
version = "0.8.22"
authors = ["Parity Technologies <admin@parity.io>"]
description = "Collator node implementation"
edition = "2018"
[dependencies]
futures = "0.3.4"
sc-service = { git = "https://github.com/paritytech/substrate", branch = "master" }
sc-executor = { git = "https://github.com/paritytech/substrate", branch = "master" }
sc-cli = { git = "https://github.com/paritytech/substrate", branch = "master" }
sc-client-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master" }
consensus_common = { package = "sp-consensus", git = "https://github.com/paritytech/substrate", branch = "master" }
polkadot-primitives = { path = "../primitives" }
polkadot-cli = { path = "../cli" }
polkadot-network = { path = "../network" }
polkadot-validation = { path = "../validation" }
polkadot-service = { path = "../service", optional = true}
polkadot-service-new = { path = "../node/service", optional = true }
log = "0.4.8"
tokio = "0.2.13"
futures-timer = "2.0"
codec = { package = "parity-scale-codec", version = "1.3.4" }
[dev-dependencies]
keyring = { package = "sp-keyring", git = "https://github.com/paritytech/substrate", branch = "master" }
[features]
default = ["service-old"]
service-old = [ "polkadot-service" ]
service-rewr = [ "polkadot-service-new" ]
polkadot/collator/README.adoc
-5
View File
@@ -1,5 +0,0 @@
= Polkadot Collator
placeholder
//TODO Write content :) (https://github.com/paritytech/polkadot/issues/159)
polkadot/collator/src/lib.rs
-496
View File
@@ -1,496 +0,0 @@
// Copyright 2017-2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Collation node logic.
//!
//! A collator node lives on a distinct parachain and submits a proposal for
//! a state transition, along with a proof for its validity
//! (what we might call a witness or block data).
//!
//! One of collators' other roles is to route messages between chains.
//! Each parachain produces a list of "egress" posts of messages for each other
//! parachain on each block, for a total of N^2 lists all together.
//!
//! We will refer to the egress list at relay chain block X of parachain A with
//! destination B as egress(X)[A -> B]
//!
//! On every block, each parachain will be intended to route messages from some
//! subset of all the other parachains. (NOTE: in practice this is not done until PoC-3)
//!
//! Since the egress information is unique to every block, when routing from a
//! parachain a collator must gather all egress posts from that parachain
//! up to the last point in history that messages were successfully routed
//! from that parachain, accounting for relay chain blocks where no candidate
//! from the collator's parachain was produced.
//!
//! In the case that all parachains route to each other and a candidate for the
//! collator's parachain was included in the last relay chain block, the collator
//! only has to gather egress posts from other parachains one block back in relay
//! chain history.
//!
//! This crate defines traits which provide context necessary for collation logic
//! to be performed, as the collation logic itself.
use std::collections::HashSet;
use std::fmt;
use std::sync::Arc;
use std::time::Duration;
use std::pin::Pin;
use futures::{future, Future, Stream, FutureExt, StreamExt};
use sp_core::Pair;
use polkadot_primitives::v0::{
BlockId, Hash, Block, DownwardMessage,
BlockData, DutyRoster, HeadData, Id as ParaId,
PoVBlock, ValidatorId, CollatorPair, LocalValidationData, GlobalValidationData,
Collation, CollationInfo, collator_signature_payload,
};
use polkadot_cli::service::{self, Role};
pub use polkadot_cli::service::Configuration;
pub use polkadot_cli::Cli;
pub use polkadot_validation::SignedStatement;
pub use polkadot_primitives::v0::CollatorId;
pub use sc_network::PeerId;
pub use service::{RuntimeApiCollection, Client};
pub use sc_cli::SubstrateCli;
#[cfg(not(feature = "service-rewr"))]
use polkadot_service::{FullNodeHandles, AbstractClient, ClientHandle};
#[cfg(feature = "service-rewr")]
use polkadot_service_new::{
self as polkadot_service,
Error as ServiceError, FullNodeHandles, AbstractClient,
};
use sc_service::SpawnTaskHandle;
use sp_core::traits::SpawnNamed;
use sp_runtime::traits::BlakeTwo256;
use consensus_common::SyncOracle;
use sc_client_api::Backend as BackendT;
const COLLATION_TIMEOUT: Duration = Duration::from_secs(30);
/// An abstraction over the `Network` with useful functions for a `Collator`.
pub trait Network: Send + Sync {
/// Create a `Stream` of checked statements for the given `relay_parent`.
///
/// The returned stream will not terminate, so it is required to make sure that the stream is
/// dropped when it is not required anymore. Otherwise, it will stick around in memory
/// infinitely.
fn checked_statements(&self, relay_parent: Hash) -> Pin<Box<dyn Stream<Item=SignedStatement> + Send>>;
}
impl Network for polkadot_network::protocol::Service {
fn checked_statements(&self, relay_parent: Hash) -> Pin<Box<dyn Stream<Item=SignedStatement> + Send>> {
polkadot_network::protocol::Service::checked_statements(self, relay_parent).boxed()
}
}
/// Collation errors.
#[derive(Debug)]
pub enum Error {
/// Error on the relay-chain side of things.
Polkadot(String),
}
impl fmt::Display for Error {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
Error::Polkadot(ref err) => write!(f, "Polkadot node error: {}", err),
}
}
}
/// Something that can build a `ParachainContext`.
pub trait BuildParachainContext {
/// The parachain context produced by the `build` function.
type ParachainContext: self::ParachainContext;
/// Build the `ParachainContext`.
fn build<SP, Client, Backend>(
self,
client: Arc<Client>,
spawner: SP,
network: impl Network + SyncOracle + Clone + 'static,
) -> Result<Self::ParachainContext, ()>
where
SP: SpawnNamed + Clone + Send + Sync + 'static,
Backend: BackendT<Block>,
Backend::State: sp_api::StateBackend<BlakeTwo256>,
Client: polkadot_service::AbstractClient<Block, Backend> + 'static,
Client::Api: RuntimeApiCollection<StateBackend = Backend::State>,
;
}
/// Parachain context needed for collation.
///
/// This can be implemented through an externally attached service or a stub.
/// This is expected to be a lightweight, shared type like an Arc.
pub trait ParachainContext: Clone {
type ProduceCandidate: Future<Output = Option<(BlockData, HeadData)>>;
/// Produce a candidate, given the relay parent hash, the latest ingress queue information
/// and the last parachain head.
fn produce_candidate(
&mut self,
relay_parent: Hash,
global_validation: GlobalValidationData,
local_validation: LocalValidationData,
downward_messages: Vec<DownwardMessage>,
) -> Self::ProduceCandidate;
}
/// Produce a candidate for the parachain, with given contexts, parent head, and signing key.
pub async fn collate<P>(
relay_parent: Hash,
local_id: ParaId,
global_validation: GlobalValidationData,
local_validation_data: LocalValidationData,
downward_messages: Vec<DownwardMessage>,
mut para_context: P,
key: Arc<CollatorPair>,
) -> Option<Collation>
where
P: ParachainContext,
P::ProduceCandidate: Send,
{
let (block_data, head_data) = para_context.produce_candidate(
relay_parent,
global_validation,
local_validation_data,
downward_messages,
).await?;
let pov_block = PoVBlock {
block_data,
};
let pov_block_hash = pov_block.hash();
let signature = key.sign(&collator_signature_payload(
&relay_parent,
&local_id,
&pov_block_hash,
));
let info = CollationInfo {
parachain_index: local_id,
relay_parent,
collator: key.public(),
signature,
head_data,
pov_block_hash,
};
let collation = Collation {
info,
pov: pov_block,
};
Some(collation)
}
/// Build a collator service based on the `ClientHandle`.
#[cfg(feature = "service-rewr")]
pub fn build_collator_service<P>(
spawner: SpawnTaskHandle,
handles: FullNodeHandles,
client: impl ClientHandle,
para_id: ParaId,
key: Arc<CollatorPair>,
build_parachain_context: P,
) -> Result<Pin<Box<dyn Future<Output = ()> + Send + 'static>>, polkadot_service::Error>
where
P: BuildParachainContext,
P::ParachainContext: Send + 'static,
<P::ParachainContext as ParachainContext>::ProduceCandidate: Send,
{
Err("Collator is not functional with the new service yet".into())
}
struct BuildCollationWork<P> {
handles: polkadot_service::FullNodeHandles,
para_id: ParaId,
key: Arc<CollatorPair>,
build_parachain_context: P,
spawner: SpawnTaskHandle,
}
impl<P> polkadot_service::ExecuteWithClient for BuildCollationWork<P>
where
P: BuildParachainContext,
P::ParachainContext: Send + 'static,
<P::ParachainContext as ParachainContext>::ProduceCandidate: Send,
{
type Output = Result<Pin<Box<dyn Future<Output = ()> + Send + 'static>>, polkadot_service::Error>;
fn execute_with_client<Client, Api, Backend>(self, client: Arc<Client>) -> Self::Output
where<Api as sp_api::ApiExt<Block>>::StateBackend: sp_api::StateBackend<BlakeTwo256>,
Backend: sc_client_api::Backend<Block>,
Backend::State: sp_api::StateBackend<BlakeTwo256>,
Api: RuntimeApiCollection<StateBackend = Backend::State>,
Client: AbstractClient<Block, Backend, Api = Api> + 'static,
{
let polkadot_network = self.handles
.polkadot_network
.ok_or_else(|| "Collator cannot run when Polkadot-specific networking has not been started")?;
// We don't require this here, but we need to make sure that the validation service is started.
// This service makes sure the collator is joining the correct gossip topics and receives the appropiate
// messages.
self.handles.validation_service_handle
.ok_or_else(|| "Collator cannot run when validation networking has not been started")?;
let parachain_context = match self.build_parachain_context.build(
client.clone(),
self.spawner.clone(),
polkadot_network.clone(),
) {
Ok(ctx) => ctx,
Err(()) => {
return Err("Could not build the parachain context!".into())
}
};
let key = self.key;
let para_id = self.para_id;
let spawner = self.spawner;
let res = async move {
let mut notification_stream = client.import_notification_stream();
while let Some(notification) = notification_stream.next().await {
macro_rules! try_fr {
($e:expr) => {
match $e {
Ok(x) => x,
Err(e) => return future::Either::Left(future::err(Error::Polkadot(
format!("{:?}", e)
))),
}
}
}
let relay_parent = notification.hash;
let id = BlockId::hash(relay_parent);
let network = polkadot_network.clone();
let client = client.clone();
let key = key.clone();
let parachain_context = parachain_context.clone();
let work = future::lazy(move |_| {
let api = client.runtime_api();
let global_validation = try_fr!(api.global_validation_data(&id));
let local_validation = match try_fr!(api.local_validation_data(&id, para_id)) {
Some(local_validation) => local_validation,
None => return future::Either::Left(future::ok(())),
};
let downward_messages = try_fr!(api.downward_messages(&id, para_id));
let validators = try_fr!(api.validators(&id));
let targets = compute_targets(
para_id,
validators.as_slice(),
try_fr!(api.duty_roster(&id)),
);
let collation_work = collate(
relay_parent,
para_id,
global_validation,
local_validation,
downward_messages,
parachain_context,
key,
).map(move |collation| {
match collation {
Some(collation) => network.distribute_collation(targets, collation),
None => log::trace!("Skipping collation as `collate` returned `None`"),
}
Ok(())
});
future::Either::Right(collation_work)
});
let deadlined = future::select(
work.then(|f| f).boxed(),
futures_timer::Delay::new(COLLATION_TIMEOUT)
);
let silenced = deadlined
.map(|either| {
match either {
future::Either::Right(_) => log::warn!("Collation failure: timeout"),
future::Either::Left((Err(e), _)) => {
log::error!("Collation failed: {:?}", e)
}
future::Either::Left((Ok(()), _)) => {},
}
});
let future = silenced.map(drop);
spawner.spawn("collation-work", future);
}
};
Ok(res.boxed())
}
}
/// Build a collator service based on the `ClientHandle`.
#[cfg(not(feature = "service-rewr"))]
pub fn build_collator_service<P>(
spawner: SpawnTaskHandle,
handles: FullNodeHandles,
client: impl ClientHandle,
para_id: ParaId,
key: Arc<CollatorPair>,
build_parachain_context: P,
) -> Result<Pin<Box<dyn Future<Output = ()> + Send + 'static>>, polkadot_service::Error>
where
P: BuildParachainContext,
P::ParachainContext: Send + 'static,
<P::ParachainContext as ParachainContext>::ProduceCandidate: Send,
{
client.execute_with(BuildCollationWork {
handles,
para_id,
key,
build_parachain_context,
spawner,
})
}
/// Async function that will run the collator node with the given `RelayChainContext` and `ParachainContext`
/// built by the given `BuildParachainContext` and arguments to the underlying polkadot node.
pub fn start_collator<P>(
build_parachain_context: P,
para_id: ParaId,
key: Arc<CollatorPair>,
config: Configuration,
) -> Result<
(Pin<Box<dyn Future<Output = ()> + Send>>, sc_service::TaskManager),
polkadot_service::Error
>
where
P: 'static + BuildParachainContext,
P::ParachainContext: Send + 'static,
<P::ParachainContext as ParachainContext>::ProduceCandidate: Send,
{
if matches!(config.role, Role::Light) {
return Err(
polkadot_service::Error::Other("light nodes are unsupported as collator".into())
.into());
}
let (task_manager, client, handles) = polkadot_service::build_full(
config,
Some((key.public(), para_id)),
None,
false,
6000,
None,
)?;
let future = build_collator_service(
task_manager.spawn_handle(),
handles,
client,
para_id,
key,
build_parachain_context,
)?;
Ok((future, task_manager))
}
#[cfg(not(feature = "service-rewr"))]
fn compute_targets(para_id: ParaId, session_keys: &[ValidatorId], roster: DutyRoster) -> HashSet<ValidatorId> {
use polkadot_primitives::v0::Chain;
roster.validator_duty.iter().enumerate()
.filter(|&(_, c)| c == &Chain::Parachain(para_id))
.filter_map(|(i, _)| session_keys.get(i))
.cloned()
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[derive(Clone)]
struct DummyParachainContext;
impl ParachainContext for DummyParachainContext {
type ProduceCandidate = future::Ready<Option<(BlockData, HeadData)>>;
fn produce_candidate(
&mut self,
_relay_parent: Hash,
_global: GlobalValidationData,
_local_validation: LocalValidationData,
_: Vec<DownwardMessage>,
) -> Self::ProduceCandidate {
// send messages right back.
future::ready(Some((
BlockData(vec![1, 2, 3, 4, 5,]),
HeadData(vec![9, 9, 9]),
)))
}
}
struct BuildDummyParachainContext;
impl BuildParachainContext for BuildDummyParachainContext {
type ParachainContext = DummyParachainContext;
fn build<SP, Client, Backend>(
self,
_: Arc<Client>,
_: SP,
_: impl Network + Clone + 'static,
) -> Result<Self::ParachainContext, ()>
where
SP: SpawnNamed + Clone + Send + Sync + 'static,
Backend: BackendT<Block>,
Backend::State: sp_api::StateBackend<BlakeTwo256>,
Client: polkadot_service::AbstractClient<Block, Backend> + 'static,
Client::Api: RuntimeApiCollection<StateBackend = Backend::State>,
{
Ok(DummyParachainContext)
}
}
// Make sure that the future returned by `start_collator` implements `Send`.
#[test]
fn start_collator_is_send() {
fn check_send<T: Send>(_: T) {}
let cli = Cli::from_iter(&["-dev"]);
let task_executor = |_, _| async {};
let config = cli.create_configuration(&cli.run.base, task_executor.into()).unwrap();
check_send(start_collator(
BuildDummyParachainContext,
0.into(),
Arc::new(CollatorPair::generate().0),
config,
));
}
}
polkadot/network/Cargo.toml
-34
View File
@@ -1,34 +0,0 @@
[package]
name = "polkadot-network"
version = "0.8.22"
authors = ["Parity Technologies <admin@parity.io>"]
description = "Polkadot-specific networking protocol"
edition = "2018"
[dependencies]
arrayvec = "0.4.12"
bytes = "0.5"
parking_lot = "0.9.0"
derive_more = "0.14.1"
av_store = { package = "polkadot-availability-store", path = "../availability-store" }
polkadot-validation = { path = "../validation" }
polkadot-primitives = { path = "../primitives" }
polkadot-erasure-coding = { path = "../erasure-coding" }
codec = { package = "parity-scale-codec", version = "1.3.4", default-features = false, features = ["derive"] }
sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" }
sc-network-gossip = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-consensus = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master" }
futures = "0.3.5"
log = "0.4.8"
exit-future = "0.2.0"
futures-timer = "2.0"
sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
wasm-timer = "0.2.4"
rand = "0.7.3"
[dev-dependencies]
sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-state-machine = { git = "https://github.com/paritytech/substrate", branch = "master" }
polkadot/network/README.adoc
-5
View File
@@ -1,5 +0,0 @@
= Polkadot Network
placeholder
//TODO Write content :) (https://github.com/paritytech/polkadot/issues/159)
polkadot/network/src/legacy/collator_pool.rs
-331
View File
@@ -1,331 +0,0 @@
// Copyright 2018-2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Bridge between the network and consensus service for getting collations to it.
use codec::{Encode, Decode};
use polkadot_primitives::v0::{Hash, CollatorId, Id as ParaId, Collation};
use sc_network::PeerId;
use futures::channel::oneshot;
use std::collections::hash_map::{HashMap, Entry};
use std::time::Duration;
use wasm_timer::Instant;
const COLLATION_LIFETIME: Duration = Duration::from_secs(60 * 5);
/// The role of the collator. Whether they're the primary or backup for this parachain.
#[derive(PartialEq, Debug, Clone, Copy, Encode, Decode)]
pub enum Role {
/// Primary collators should send collations whenever it's time.
Primary = 0,
/// Backup collators should not.
Backup = 1,
}
/// A maintenance action for the collator set.
#[derive(PartialEq, Debug)]
#[allow(dead_code)]
pub enum Action {
/// Disconnect the given collator.
Disconnect(CollatorId),
/// Give the collator a new role.
NewRole(CollatorId, Role),
}
struct CollationSlot {
live_at: Instant,
entries: SlotEntries,
}
impl CollationSlot {
fn blank_now() -> Self {
CollationSlot {
live_at: Instant::now(),
entries: SlotEntries::Blank,
}
}
fn stay_alive(&self, now: Instant) -> bool {
self.live_at + COLLATION_LIFETIME > now
}
}
#[derive(Debug)]
enum SlotEntries {
Blank,
// not queried yet
Pending(Vec<Collation>),
// waiting for next to arrive.
Awaiting(Vec<oneshot::Sender<Collation>>),
}
impl SlotEntries {
fn received_collation(&mut self, collation: Collation) {
*self = match std::mem::replace(self, SlotEntries::Blank) {
SlotEntries::Blank => SlotEntries::Pending(vec![collation]),
SlotEntries::Pending(mut cs) => {
cs.push(collation);
SlotEntries::Pending(cs)
}
SlotEntries::Awaiting(senders) => {
for sender in senders {
let _ = sender.send(collation.clone());
}
SlotEntries::Blank
}
};
}
fn await_with(&mut self, sender: oneshot::Sender<Collation>) {
*self = match ::std::mem::replace(self, SlotEntries::Blank) {
SlotEntries::Blank => SlotEntries::Awaiting(vec![sender]),
SlotEntries::Awaiting(mut senders) => {
senders.push(sender);
SlotEntries::Awaiting(senders)
}
SlotEntries::Pending(mut cs) => {
let next_collation = cs.pop().expect("empty variant is always `Blank`; qed");
let _ = sender.send(next_collation);
if cs.is_empty() {
SlotEntries::Blank
} else {
SlotEntries::Pending(cs)
}
}
};
}
}
struct ParachainCollators {
primary: CollatorId,
backup: Vec<CollatorId>,
}
/// Manages connected collators and role assignments from the perspective of a validator.
#[derive(Default)]
pub struct CollatorPool {
collators: HashMap<CollatorId, (ParaId, PeerId)>,
parachain_collators: HashMap<ParaId, ParachainCollators>,
collations: HashMap<(Hash, ParaId), CollationSlot>,
}
impl CollatorPool {
/// Create a new `CollatorPool` object.
pub fn new() -> Self {
CollatorPool {
collators: HashMap::new(),
parachain_collators: HashMap::new(),
collations: HashMap::new(),
}
}
/// Call when a new collator is authenticated. Returns the role.
pub fn on_new_collator(&mut self, collator_id: CollatorId, para_id: ParaId, peer_id: PeerId) -> Role {
self.collators.insert(collator_id.clone(), (para_id, peer_id));
match self.parachain_collators.entry(para_id) {
Entry::Vacant(vacant) => {
vacant.insert(ParachainCollators {
primary: collator_id,
backup: Vec::new(),
});
Role::Primary
},
Entry::Occupied(mut occupied) => {
occupied.get_mut().backup.push(collator_id);
Role::Backup
}
}
}
/// Called when a collator disconnects. If it was the primary, returns a new primary for that
/// parachain.
pub fn on_disconnect(&mut self, collator_id: CollatorId) -> Option<CollatorId> {
self.collators.remove(&collator_id).and_then(|(para_id, _)| match self.parachain_collators.entry(para_id) {
Entry::Vacant(_) => None,
Entry::Occupied(mut occ) => {
if occ.get().primary == collator_id {
if occ.get().backup.is_empty() {
occ.remove();
None
} else {
let mut collators = occ.get_mut();
collators.primary = collators.backup.pop().expect("backup non-empty; qed");
Some(collators.primary.clone())
}
} else {
let pos = occ.get().backup.iter().position(|a| a == &collator_id)
.expect("registered collator always present in backup if not primary; qed");
occ.get_mut().backup.remove(pos);
None
}
}
})
}
/// Called when a collation is received.
/// The collator should be registered for the parachain of the collation as a precondition of this function.
/// The collation should have been checked for integrity of signature before passing to this function.
pub fn on_collation(&mut self, collator_id: CollatorId, relay_parent: Hash, collation: Collation) {
log::debug!(
target: "collator-pool", "On collation from collator {} for relay parent {}",
collator_id,
relay_parent,
);
if let Some((para_id, _)) = self.collators.get(&collator_id) {
debug_assert_eq!(para_id, &collation.info.parachain_index);
// TODO: punish if not primary? (https://github.com/paritytech/polkadot/issues/213)
self.collations.entry((relay_parent, para_id.clone()))
.or_insert_with(CollationSlot::blank_now)
.entries
.received_collation(collation);
}
}
/// Wait for a collation from a parachain.
pub fn await_collation(&mut self, relay_parent: Hash, para_id: ParaId, sender: oneshot::Sender<Collation>) {
self.collations.entry((relay_parent, para_id))
.or_insert_with(CollationSlot::blank_now)
.entries
.await_with(sender);
}
/// Call periodically to perform collator set maintenance.
/// Returns a set of actions to perform on the network level.
pub fn maintain_peers(&mut self) -> Vec<Action> {
// TODO: rearrange periodically to new primary, evaluate based on latency etc.
// https://github.com/paritytech/polkadot/issues/214
Vec::new()
}
/// called when a block with given hash has been imported.
pub fn collect_garbage(&mut self, chain_head: Option<&Hash>) {
let now = Instant::now();
self.collations.retain(|&(ref h, _), slot| chain_head != Some(h) && slot.stay_alive(now));
}
/// Convert the given `CollatorId` to a `PeerId`.
pub fn collator_id_to_peer_id(&self, collator_id: &CollatorId) -> Option<&PeerId> {
self.collators.get(collator_id).map(|ids| &ids.1)
}
}
#[cfg(test)]
mod tests {
use super::*;
use sp_core::crypto::UncheckedInto;
use polkadot_primitives::v0::{CollationInfo, BlockData, PoVBlock};
use futures::executor::block_on;
fn make_pov(block_data: Vec<u8>) -> PoVBlock {
PoVBlock {
block_data: BlockData(block_data),
}
}
#[test]
fn disconnect_primary_gives_new_primary() {
let mut pool = CollatorPool::new();
let para_id: ParaId = 5.into();
let bad_primary: CollatorId = [0; 32].unchecked_into();
let good_backup: CollatorId = [1; 32].unchecked_into();
assert_eq!(pool.on_new_collator(bad_primary.clone(), para_id.clone(), PeerId::random()), Role::Primary);
assert_eq!(pool.on_new_collator(good_backup.clone(), para_id.clone(), PeerId::random()), Role::Backup);
assert_eq!(pool.on_disconnect(bad_primary), Some(good_backup.clone()));
assert_eq!(pool.on_disconnect(good_backup), None);
}
#[test]
fn disconnect_backup_removes_from_pool() {
let mut pool = CollatorPool::new();
let para_id: ParaId = 5.into();
let primary = [0; 32].unchecked_into();
let backup: CollatorId = [1; 32].unchecked_into();
assert_eq!(pool.on_new_collator(primary, para_id.clone(), PeerId::random()), Role::Primary);
assert_eq!(pool.on_new_collator(backup.clone(), para_id.clone(), PeerId::random()), Role::Backup);
assert_eq!(pool.on_disconnect(backup), None);
assert!(pool.parachain_collators.get(&para_id).unwrap().backup.is_empty());
}
#[test]
fn await_before_collation() {
let mut pool = CollatorPool::new();
let para_id: ParaId = 5.into();
let peer_id = PeerId::random();
let primary: CollatorId = [0; 32].unchecked_into();
let relay_parent = [1; 32].into();
assert_eq!(pool.on_new_collator(primary.clone(), para_id.clone(), peer_id.clone()), Role::Primary);
let (tx1, rx1) = oneshot::channel();
let (tx2, rx2) = oneshot::channel();
pool.await_collation(relay_parent, para_id, tx1);
pool.await_collation(relay_parent, para_id, tx2);
let mut collation_info = CollationInfo::default();
collation_info.parachain_index = para_id;
collation_info.collator = primary.clone().into();
pool.on_collation(primary.clone(), relay_parent, Collation {
info: collation_info,
pov: make_pov(vec![4, 5, 6]),
});
block_on(rx1).unwrap();
block_on(rx2).unwrap();
assert_eq!(pool.collators.get(&primary).map(|ids| &ids.1).unwrap(), &peer_id);
}
#[test]
fn collate_before_await() {
let mut pool = CollatorPool::new();
let para_id: ParaId = 5.into();
let primary: CollatorId = [0; 32].unchecked_into();
let relay_parent = [1; 32].into();
assert_eq!(pool.on_new_collator(primary.clone(), para_id.clone(), PeerId::random()), Role::Primary);
let mut collation_info = CollationInfo::default();
collation_info.parachain_index = para_id;
collation_info.collator = primary.clone();
pool.on_collation(primary.clone(), relay_parent, Collation {
info: collation_info,
pov: make_pov(vec![4, 5, 6]),
});
let (tx, rx) = oneshot::channel();
pool.await_collation(relay_parent, para_id, tx);
block_on(rx).unwrap();
}
#[test]
fn slot_stay_alive() {
let slot = CollationSlot::blank_now();
let now = slot.live_at;
assert!(slot.stay_alive(now));
assert!(slot.stay_alive(now + Duration::from_secs(10)));
assert!(!slot.stay_alive(now + COLLATION_LIFETIME));
assert!(!slot.stay_alive(now + COLLATION_LIFETIME + Duration::from_secs(10)));
}
}
polkadot/network/src/legacy/gossip/attestation.rs
-347
View File
@@ -1,347 +0,0 @@
// Copyright 2019-2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Gossip messages and structures for dealing with attestations (statements of
//! validity of invalidity on parachain candidates).
//!
//! This follows the same principles as other gossip modules (see parent
//! documentation for more details) by being aware of our current chain
//! heads and accepting only information relative to them. Attestations are localized to
//! relay chain head, so this is easily doable.
//!
//! This module also provides a filter, so we can only broadcast messages to
//! peers that are relevant to chain heads they have advertised.
//!
//! Furthermore, since attestations are bottlenecked by the `Candidate` statement,
//! we only accept attestations which are themselves `Candidate` messages, or reference
//! a `Candidate` we are aware of. Otherwise, it is possible we could be forced to
//! consider an infinite amount of attestations produced by a misbehaving validator.
use sc_network_gossip::{ValidationResult as GossipValidationResult};
use sc_network::ReputationChange;
use polkadot_validation::GenericStatement;
use polkadot_primitives::v0::Hash;
use std::collections::HashMap;
use log::warn;
use super::{
cost, benefit, attestation_topic, MAX_CHAIN_HEADS, LeavesVec,
ChainContext, Known, MessageValidationData, GossipStatement,
};
/// Meta-data that we keep about a candidate in the `Knowledge`.
#[derive(Debug, Clone)]
pub(super) struct CandidateMeta {
/// The hash of the pov-block data.
pub(super) pov_block_hash: Hash,
}
// knowledge about attestations on a single parent-hash.
#[derive(Default)]
pub(super) struct Knowledge {
candidates: HashMap<Hash, CandidateMeta>,
}
impl Knowledge {
// whether the peer is aware of a candidate with given hash.
fn is_aware_of(&self, candidate_hash: &Hash) -> bool {
self.candidates.contains_key(candidate_hash)
}
// Get candidate meta data for a candidate by hash.
fn candidate_meta(&self, candidate_hash: &Hash) -> Option<&CandidateMeta> {
self.candidates.get(candidate_hash)
}
// note that the peer is aware of a candidate with given hash. this should
// be done after observing an incoming candidate message via gossip.
fn note_aware(&mut self, candidate_hash: Hash, candidate_meta: CandidateMeta) {
self.candidates.insert(candidate_hash, candidate_meta);
}
}
#[derive(Default)]
pub(super) struct PeerData {
live: HashMap<Hash, Knowledge>,
}
impl PeerData {
/// Update leaves, returning a list of which leaves are new.
pub(super) fn update_leaves(&mut self, leaves: &LeavesVec) -> LeavesVec {
let mut new = LeavesVec::new();
self.live.retain(|k, _| leaves.contains(k));
for &leaf in leaves {
self.live.entry(leaf).or_insert_with(|| {
new.push(leaf);
Default::default()
});
}
new
}
#[cfg(test)]
pub(super) fn note_aware_under_leaf(
&mut self,
relay_chain_leaf: &Hash,
candidate_hash: Hash,
meta: CandidateMeta,
) {
if let Some(knowledge) = self.live.get_mut(relay_chain_leaf) {
knowledge.note_aware(candidate_hash, meta);
}
}
pub(super) fn knowledge_at_mut(&mut self, parent_hash: &Hash) -> Option<&mut Knowledge> {
self.live.get_mut(parent_hash)
}
}
/// An impartial view of what topics and data are valid based on attestation session data.
pub(super) struct View {
leaf_work: Vec<(Hash, LeafView)>, // hashes of the best DAG-leaves paired with validation data.
topics: HashMap<Hash, Hash>, // maps topic hashes to block hashes.
}
impl Default for View {
fn default() -> Self {
View {
leaf_work: Vec::with_capacity(MAX_CHAIN_HEADS),
topics: Default::default(),
}
}
}
impl View {
fn leaf_view(&self, relay_chain_leaf: &Hash) -> Option<&LeafView> {
self.leaf_work.iter()
.find_map(|&(ref h, ref leaf)| if h == relay_chain_leaf { Some(leaf) } else { None } )
}
fn leaf_view_mut(&mut self, relay_chain_leaf: &Hash) -> Option<&mut LeafView> {
self.leaf_work.iter_mut()
.find_map(|&mut (ref h, ref mut leaf)| if h == relay_chain_leaf { Some(leaf) } else { None } )
}
/// Get our leaves-set. Guaranteed to have length <= MAX_CHAIN_HEADS.
pub(super) fn neighbor_info<'a>(&'a self) -> impl Iterator<Item=Hash> + 'a + Clone {
self.leaf_work.iter().take(MAX_CHAIN_HEADS).map(|(p, _)| p.clone())
}
/// Note new leaf in our local view and validation data necessary to check signatures
/// of statements issued under this leaf.
///
/// This will be pruned later on a call to `prune_old_leaves`, when this leaf
/// is not a leaf anymore.
pub(super) fn new_local_leaf(
&mut self,
validation_data: MessageValidationData,
) {
let relay_chain_leaf = validation_data.signing_context.parent_hash.clone();
self.leaf_work.push((
validation_data.signing_context.parent_hash.clone(),
LeafView {
validation_data,
knowledge: Default::default(),
},
));
self.topics.insert(attestation_topic(relay_chain_leaf), relay_chain_leaf);
self.topics.insert(super::pov_block_topic(relay_chain_leaf), relay_chain_leaf);
}
/// Prune old leaf-work that fails the leaf predicate.
pub(super) fn prune_old_leaves<F: Fn(&Hash) -> bool>(&mut self, is_leaf: F) {
let leaf_work = &mut self.leaf_work;
leaf_work.retain(|&(ref relay_chain_leaf, _)| is_leaf(relay_chain_leaf));
self.topics.retain(|_, v| leaf_work.iter().find(|(p, _)| p == v).is_some());
}
/// Whether a message topic is considered live relative to our view. non-live
/// topics do not pertain to our perceived leaves, and are uninteresting to us.
pub(super) fn is_topic_live(&self, topic: &Hash) -> bool {
self.topics.contains_key(topic)
}
/// The relay-chain block hash corresponding to a topic.
pub(super) fn topic_block(&self, topic: &Hash) -> Option<&Hash> {
self.topics.get(topic)
}
#[cfg(test)]
pub(super) fn note_aware_under_leaf(
&mut self,
relay_chain_leaf: &Hash,
candidate_hash: Hash,
meta: CandidateMeta,
) {
if let Some(view) = self.leaf_view_mut(relay_chain_leaf) {
view.knowledge.note_aware(candidate_hash, meta);
}
}
/// Validate the signature on an attestation statement of some kind. Should be done before
/// any repropagation of that statement.
pub(super) fn validate_statement_signature<C: ChainContext + ?Sized>(
&mut self,
message: GossipStatement,
chain: &C,
)
-> (GossipValidationResult<Hash>, ReputationChange)
{
// message must reference one of our chain heads and
// if message is not a `Candidate` we should have the candidate available
// in `attestation_view`.
match self.leaf_view(&message.relay_chain_leaf) {
None => {
let cost = match chain.is_known(&message.relay_chain_leaf) {
Some(Known::Leaf) => {
warn!(
target: "network",
"Leaf block {} not considered live for attestation",
message.relay_chain_leaf,
);
cost::NONE
}
Some(Known::Old) => cost::PAST_MESSAGE,
_ => cost::FUTURE_MESSAGE,
};
(GossipValidationResult::Discard, cost)
}
Some(view) => {
// first check that we are capable of receiving this message
// in a DoS-proof manner.
let benefit = match message.signed_statement.statement {
GenericStatement::Candidate(_) => benefit::NEW_CANDIDATE,
GenericStatement::Valid(ref h) | GenericStatement::Invalid(ref h) => {
if !view.knowledge.is_aware_of(h) {
let cost = cost::ATTESTATION_NO_CANDIDATE;
return (GossipValidationResult::Discard, cost);
}
benefit::NEW_ATTESTATION
}
};
// validate signature.
let res = view.validation_data.check_statement(
&message.signed_statement,
);
match res {
Ok(()) => {
let topic = attestation_topic(message.relay_chain_leaf);
(GossipValidationResult::ProcessAndKeep(topic), benefit)
}
Err(()) => (GossipValidationResult::Discard, cost::BAD_SIGNATURE),
}
}
}
}
/// Validate a pov-block message.
pub(super) fn validate_pov_block_message<C: ChainContext + ?Sized>(
&mut self,
message: &super::GossipPoVBlock,
chain: &C,
)
-> (GossipValidationResult<Hash>, ReputationChange)
{
match self.leaf_view(&message.relay_chain_leaf) {
None => {
let cost = match chain.is_known(&message.relay_chain_leaf) {
Some(Known::Leaf) => {
warn!(
target: "network",
"Leaf block {} not considered live for attestation",
message.relay_chain_leaf,
);
cost::NONE
}
Some(Known::Old) => cost::POV_BLOCK_UNWANTED,
_ => cost::FUTURE_MESSAGE,
};
(GossipValidationResult::Discard, cost)
}
Some(view) => {
// we only accept pov-blocks for candidates that we have
// and consider active.
match view.knowledge.candidate_meta(&message.candidate_hash) {
None => (GossipValidationResult::Discard, cost::POV_BLOCK_UNWANTED),
Some(meta) => {
// check that the pov-block hash is actually correct.
if meta.pov_block_hash == message.pov_block.hash() {
let topic = super::pov_block_topic(message.relay_chain_leaf);
(GossipValidationResult::ProcessAndKeep(topic), benefit::NEW_POV_BLOCK)
} else {
(GossipValidationResult::Discard, cost::POV_BLOCK_BAD_DATA)
}
}
}
}
}
}
/// whether it's allowed to send a statement to a peer with given knowledge
/// about the relay parent the statement refers to.
pub(super) fn statement_allowed(
&mut self,
statement: &GossipStatement,
peer_knowledge: &mut Knowledge,
) -> bool {
let signed = &statement.signed_statement;
let relay_chain_leaf = &statement.relay_chain_leaf;
match signed.statement {
GenericStatement::Valid(ref h) | GenericStatement::Invalid(ref h) => {
// `valid` and `invalid` statements can only be propagated after
// a candidate message is known by that peer.
peer_knowledge.is_aware_of(h)
}
GenericStatement::Candidate(ref c) => {
// if we are sending a `Candidate` message we should make sure that
// attestation_view and their_view reflects that we know about the candidate.
let hash = c.hash();
let meta = CandidateMeta { pov_block_hash: c.pov_block_hash };
peer_knowledge.note_aware(hash, meta.clone());
if let Some(attestation_view) = self.leaf_view_mut(&relay_chain_leaf) {
attestation_view.knowledge.note_aware(hash, meta);
}
// at this point, the peer hasn't seen the message or the candidate
// and has knowledge of the relevant relay-chain parent.
true
}
}
}
/// whether it's allowed to send a pov-block to a peer.
pub(super) fn pov_block_allowed(
&mut self,
statement: &super::GossipPoVBlock,
peer_knowledge: &mut Knowledge,
) -> bool {
peer_knowledge.is_aware_of(&statement.candidate_hash)
}
}
struct LeafView {
validation_data: MessageValidationData,
knowledge: Knowledge,
}
polkadot/network/src/legacy/gossip/mod.rs
-1249
View File
File diff suppressed because it is too large Load Diff
polkadot/network/src/legacy/local_collations.rs
-214
View File
@@ -1,214 +0,0 @@
// Copyright 2018-2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Local collations to be circulated to validators.
//!
//! Collations are attempted to be repropagated when a new validator connects,
//! a validator changes his session key, or when they are generated.
use polkadot_primitives::v0::{Hash, ValidatorId};
use crate::legacy::collator_pool::Role;
use std::collections::{HashMap, HashSet};
use std::time::Duration;
use wasm_timer::Instant;
use rand::seq::SliceRandom;
const LIVE_FOR: Duration = Duration::from_secs(60 * 5);
struct LocalCollation<C> {
targets: HashSet<ValidatorId>,
collation: C,
live_since: Instant,
}
/// Tracker for locally collated values and which validators to send them to.
pub struct LocalCollations<C> {
primary_for: HashSet<ValidatorId>,
local_collations: HashMap<Hash, LocalCollation<C>>,
}
impl<C: Clone> Default for LocalCollations<C> {
fn default() -> Self {
Self::new()
}
}
impl<C: Clone> LocalCollations<C> {
/// Create a new `LocalCollations` tracker.
pub fn new() -> Self {
LocalCollations {
primary_for: HashSet::new(),
local_collations: HashMap::new(),
}
}
/// Validator gave us a new role. If the new role is "primary", this function might return
/// a set of collations to send to that validator.
pub fn note_validator_role(&mut self, key: ValidatorId, role: Role) -> Vec<(Hash, C)> {
match role {
Role::Backup => {
self.primary_for.remove(&key);
Vec::new()
}
Role::Primary => {
let new_primary = self.primary_for.insert(key.clone());
if new_primary {
self.collations_targeting(&key)
} else {
Vec::new()
}
}
}
}
/// Fresh session key from a validator. Returns a vector of collations to send
/// to the validator.
pub fn fresh_key(&mut self, old_key: &ValidatorId, new_key: &ValidatorId) -> Vec<(Hash, C)> {
if self.primary_for.remove(old_key) {
self.primary_for.insert(new_key.clone());
self.collations_targeting(new_key)
} else {
Vec::new()
}
}
/// Validator disconnected.
pub fn on_disconnect(&mut self, key: &ValidatorId) {
self.primary_for.remove(key);
}
/// Mark collations relevant to the given parent hash as obsolete.
pub fn collect_garbage(&mut self, relay_parent: Option<&Hash>) {
if let Some(relay_parent) = relay_parent {
self.local_collations.remove(relay_parent);
}
let now = Instant::now();
self.local_collations.retain(|_, v| v.live_since + LIVE_FOR > now);
}
/// Add a collation. Returns an iterator of session keys to send to and lazy copies of the collation.
pub fn add_collation<'a>(
&'a mut self,
relay_parent: Hash,
targets: HashSet<ValidatorId>,
collation: C
) -> impl Iterator<Item=(ValidatorId, C)> + 'a {
self.local_collations.insert(relay_parent, LocalCollation {
targets,
collation,
live_since: Instant::now(),
});
let local = self.local_collations.get(&relay_parent)
.expect("just inserted to this key; qed");
let borrowed_collation = &local.collation;
// If we are conected to multiple validators,
// make sure we always send the collation to one of the validators
// we are registered as backup. This ensures that one collator that
// is primary at multiple validators, desn't block the Parachain from progressing.
let mut rng = rand::thread_rng();
let diff = local.targets.difference(&self.primary_for).collect::<Vec<_>>();
local.targets
.intersection(&self.primary_for)
.chain(diff.choose(&mut rng).map(|r| r.clone()))
.map(move |k| (k.clone(), borrowed_collation.clone()))
}
fn collations_targeting(&self, key: &ValidatorId) -> Vec<(Hash, C)> {
self.local_collations.iter()
.filter(|&(_, ref v)| v.targets.contains(key))
.map(|(h, v)| (*h, v.collation.clone()))
.collect()
}
}
#[cfg(test)]
mod tests {
use super::*;
use sp_core::crypto::UncheckedInto;
use polkadot_primitives::v0::ValidatorId;
#[test]
fn add_validator_with_ready_collation() {
let key: ValidatorId = [1; 32].unchecked_into();
let relay_parent = [2; 32].into();
let targets = {
let mut set = HashSet::new();
set.insert(key.clone());
set
};
let mut tracker = LocalCollations::new();
assert!(tracker.add_collation(relay_parent, targets, 5).next().is_some());
assert_eq!(tracker.note_validator_role(key, Role::Primary), vec![(relay_parent, 5)]);
}
#[test]
fn rename_with_ready() {
let orig_key: ValidatorId = [1; 32].unchecked_into();
let new_key: ValidatorId = [2; 32].unchecked_into();
let relay_parent = [255; 32].into();
let targets = {
let mut set = HashSet::new();
set.insert(new_key.clone());
set
};
let mut tracker: LocalCollations<u8> = LocalCollations::new();
assert!(tracker.add_collation(relay_parent, targets, 5).next().is_some());
assert!(tracker.note_validator_role(orig_key.clone(), Role::Primary).is_empty());
assert_eq!(tracker.fresh_key(&orig_key, &new_key), vec![(relay_parent, 5u8)]);
}
#[test]
fn collecting_garbage() {
let relay_parent_a = [255; 32].into();
let relay_parent_b = [222; 32].into();
let mut tracker: LocalCollations<u8> = LocalCollations::new();
assert!(tracker.add_collation(relay_parent_a, HashSet::new(), 5).next().is_none());
assert!(tracker.add_collation(relay_parent_b, HashSet::new(), 69).next().is_none());
let live_since = Instant::now() - LIVE_FOR - Duration::from_secs(10);
tracker.local_collations.get_mut(&relay_parent_b).unwrap().live_since = live_since;
tracker.collect_garbage(Some(&relay_parent_a));
// first one pruned because of relay parent, other because of time.
assert!(tracker.local_collations.is_empty());
}
#[test]
fn add_collation_with_connected_target() {
let key: ValidatorId = [1; 32].unchecked_into();
let relay_parent = [2; 32].into();
let targets = {
let mut set = HashSet::new();
set.insert(key.clone());
set
};
let mut tracker = LocalCollations::new();
assert!(tracker.note_validator_role(key.clone(), Role::Primary).is_empty());
assert_eq!(tracker.add_collation(relay_parent, targets, 5).next(), Some((key, 5)));
}
}
polkadot/network/src/legacy/mod.rs
-83
View File
@@ -1,83 +0,0 @@
// Copyright 2017-2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Polkadot-specific network implementation.
//!
//! This manages routing for parachain statements, parachain block and outgoing message
//! data fetching, communication between collators and validators, and more.
pub mod collator_pool;
pub mod local_collations;
pub mod gossip;
use codec::Decode;
use futures::prelude::*;
use polkadot_primitives::v0::Hash;
use sc_network::PeerId;
use sc_network_gossip::TopicNotification;
use log::debug;
use std::pin::Pin;
use std::task::{Context as PollContext, Poll};
use self::gossip::GossipMessage;
/// Basic gossip functionality that a network has to fulfill.
pub trait GossipService {
/// Get a stream of gossip messages for a given hash.
fn gossip_messages_for(&self, topic: Hash) -> GossipMessageStream;
/// Gossip a message on given topic.
fn gossip_message(&self, topic: Hash, message: GossipMessage);
/// Send a message to a specific peer we're connected to.
fn send_message(&self, who: PeerId, message: GossipMessage);
}
/// A stream of gossip messages and an optional sender for a topic.
pub struct GossipMessageStream {
topic_stream: Pin<Box<dyn Stream<Item = TopicNotification> + Send>>,
}
impl GossipMessageStream {
/// Create a new instance with the given topic stream.
pub fn new(topic_stream: Pin<Box<dyn Stream<Item = TopicNotification> + Send>>) -> Self {
Self {
topic_stream,
}
}
}
impl Stream for GossipMessageStream {
type Item = (GossipMessage, Option<PeerId>);
fn poll_next(self: Pin<&mut Self>, cx: &mut PollContext) -> Poll<Option<Self::Item>> {
let this = Pin::into_inner(self);
loop {
let msg = match Pin::new(&mut this.topic_stream).poll_next(cx) {
Poll::Ready(Some(msg)) => msg,
Poll::Ready(None) => return Poll::Ready(None),
Poll::Pending => return Poll::Pending,
};
debug!(target: "validation", "Processing statement for live validation leaf-work");
if let Ok(gmsg) = GossipMessage::decode(&mut &msg.message[..]) {
return Poll::Ready(Some((gmsg, msg.sender)))
}
}
}
}
polkadot/network/src/lib.rs
-54
View File
@@ -1,54 +0,0 @@
// Copyright 2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! High-level network protocols for Polkadot.
//!
//! This manages routing for parachain statements, parachain block and outgoing message
//! data fetching, communication between collators and validators, and more.
#![recursion_limit="256"]
use polkadot_primitives::v0::{Block, Hash, BlakeTwo256, HashT};
pub mod legacy;
pub mod protocol;
/// Specialization of the network service for the polkadot block type.
pub type PolkadotNetworkService = sc_network::NetworkService<Block, Hash>;
mod cost {
use sc_network::ReputationChange as Rep;
pub(super) const UNEXPECTED_MESSAGE: Rep = Rep::new(-200, "Polkadot: Unexpected message");
pub(super) const INVALID_FORMAT: Rep = Rep::new(-200, "Polkadot: Bad message");
pub(super) const UNKNOWN_PEER: Rep = Rep::new(-50, "Polkadot: Unknown peer");
pub(super) const BAD_COLLATION: Rep = Rep::new(-1000, "Polkadot: Bad collation");
}
mod benefit {
use sc_network::ReputationChange as Rep;
pub(super) const VALID_FORMAT: Rep = Rep::new(20, "Polkadot: Valid message format");
pub(super) const GOOD_COLLATION: Rep = Rep::new(100, "Polkadot: Good collation");
}
/// Compute gossip topic for the erasure chunk messages given the hash of the
/// candidate they correspond to.
fn erasure_coding_topic(candidate_hash: &Hash) -> Hash {
let mut v = candidate_hash.as_ref().to_vec();
v.extend(b"erasure_chunks");
BlakeTwo256::hash(&v[..])
}
polkadot/network/src/protocol/mod.rs
-1561
View File
File diff suppressed because it is too large Load Diff
polkadot/network/src/protocol/tests.rs
-573
View File
@@ -1,573 +0,0 @@
// Copyright 2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//! Tests for the protocol.
use super::*;
use crate::legacy::gossip::GossipPoVBlock;
use parking_lot::Mutex;
use polkadot_primitives::v0::{
Block,
Id as ParaId, Chain, DutyRoster, ParachainHost, ValidatorId,
Retriable, CollatorId, AbridgedCandidateReceipt,
GlobalValidationData, LocalValidationData, SigningContext,
PoVBlock, BlockData, ValidationCode,
};
use polkadot_validation::{SharedTable, TableRouter};
use av_store::Store as AvailabilityStore;
use sc_network_gossip::TopicNotification;
use sp_api::{ApiRef, ProvideRuntimeApi};
use sp_runtime::traits::Block as BlockT;
use sp_core::{crypto::Pair, testing::TaskExecutor};
use sp_keyring::Sr25519Keyring;
use futures::executor::LocalPool;
use futures::task::LocalSpawnExt;
#[derive(Default)]
pub struct MockNetworkOps {
recorded: Mutex<Recorded>,
}
#[derive(Default)]
struct Recorded {
peer_reputations: HashMap<PeerId, i32>,
notifications: Vec<(PeerId, Message)>,
}
// Test setup registers receivers of gossip messages as well as signals that
// fire when they are taken.
type GossipStreamEntry = (mpsc::UnboundedReceiver<TopicNotification>, oneshot::Sender<()>);
#[derive(Default, Clone)]
struct MockGossip {
inner: Arc<Mutex<HashMap<Hash, GossipStreamEntry>>>,
gossip_messages: Arc<Mutex<HashMap<Hash, GossipMessage>>>,
}
impl MockGossip {
fn add_gossip_stream(&self, topic: Hash)
-> (mpsc::UnboundedSender<TopicNotification>, oneshot::Receiver<()>)
{
let (tx, rx) = mpsc::unbounded();
let (o_tx, o_rx) = oneshot::channel();
self.inner.lock().insert(topic, (rx, o_tx));
(tx, o_rx)
}
}
impl NetworkServiceOps for MockNetworkOps {
fn report_peer(&self, peer: PeerId, value: sc_network::ReputationChange) {
let mut recorded = self.recorded.lock();
let total_rep = recorded.peer_reputations.entry(peer).or_insert(0);
*total_rep = total_rep.saturating_add(value.value);
}
fn write_notification(
&self,
peer: PeerId,
engine_id: ConsensusEngineId,
notification: Vec<u8>,
) {
assert_eq!(engine_id, POLKADOT_ENGINE_ID);
let message = Message::decode(&mut &notification[..]).expect("invalid notification");
self.recorded.lock().notifications.push((peer, message));
}
}
impl crate::legacy::GossipService for MockGossip {
fn gossip_messages_for(&self, topic: Hash) -> crate::legacy::GossipMessageStream {
crate::legacy::GossipMessageStream::new(match self.inner.lock().remove(&topic) {
None => Box::pin(stream::empty()),
Some((rx, o_rx)) => {
let _ = o_rx.send(());
Box::pin(rx)
}
})
}
fn gossip_message(&self, topic: Hash, message: GossipMessage) {
self.gossip_messages.lock().insert(topic, message);
}
fn send_message(&self, _who: PeerId, _message: GossipMessage) {
}
}
impl GossipOps for MockGossip {
fn new_local_leaf(&self, _: crate::legacy::gossip::MessageValidationData) -> crate::legacy::gossip::NewLeafActions {
crate::legacy::gossip::NewLeafActions::new()
}
fn register_availability_store(&self, _store: av_store::Store) {}
}
#[derive(Default)]
struct ApiData {
validators: Vec<ValidatorId>,
duties: Vec<Chain>,
active_parachains: Vec<(ParaId, Option<(CollatorId, Retriable)>)>,
}
#[derive(Default, Clone)]
struct TestApi {
data: Arc<Mutex<ApiData>>,
}
#[derive(Default)]
struct RuntimeApi {
data: Arc<Mutex<ApiData>>,
}
impl ProvideRuntimeApi<Block> for TestApi {
type Api = RuntimeApi;
fn runtime_api<'a>(&'a self) -> ApiRef<'a, Self::Api> {
RuntimeApi { data: self.data.clone() }.into()
}
}
sp_api::mock_impl_runtime_apis! {
impl ParachainHost<Block> for RuntimeApi {
type Error = sp_blockchain::Error;
fn validators(&self) -> Vec<ValidatorId> {
self.data.lock().validators.clone()
}
fn duty_roster(&self) -> DutyRoster {
DutyRoster {
validator_duty: self.data.lock().duties.clone(),
}
}
fn active_parachains(&self) -> Vec<(ParaId, Option<(CollatorId, Retriable)>)> {
self.data.lock().active_parachains.clone()
}
fn parachain_code(_: ParaId) -> Option<ValidationCode> {
Some(ValidationCode(Vec::new()))
}
fn global_validation_data() -> GlobalValidationData {
Default::default()
}
fn local_validation_data(_: ParaId) -> Option<LocalValidationData> {
Some(Default::default())
}
fn get_heads(_: Vec<<Block as BlockT>::Extrinsic>) -> Option<Vec<AbridgedCandidateReceipt>> {
Some(Vec::new())
}
fn signing_context() -> SigningContext {
SigningContext {
session_index: Default::default(),
parent_hash: Default::default(),
}
}
fn downward_messages(_: ParaId) -> Vec<polkadot_primitives::v0::DownwardMessage> {
Vec::new()
}
}
}
impl super::Service<MockNetworkOps> {
async fn connect_peer(&mut self, peer: PeerId, role: ObservedRole) {
self.sender.send(ServiceToWorkerMsg::PeerConnected(peer, role)).await.unwrap();
}
async fn peer_message(&mut self, peer: PeerId, message: Message) {
let bytes = message.encode().into();
self.sender.send(ServiceToWorkerMsg::PeerMessage(peer, vec![bytes])).await.unwrap();
}
async fn disconnect_peer(&mut self, peer: PeerId) {
self.sender.send(ServiceToWorkerMsg::PeerDisconnected(peer)).await.unwrap();
}
async fn synchronize<T: Send + 'static>(
&mut self,
callback: impl FnOnce(&mut ProtocolHandler) -> T + Send + 'static,
) -> T {
let (tx, rx) = oneshot::channel();
let msg = ServiceToWorkerMsg::Synchronize(Box::new(move |proto| {
let res = callback(proto);
if let Err(_) = tx.send(res) {
log::warn!(target: "p_net", "Failed to send synchronization result");
}
}));
self.sender.send(msg).await.expect("Worker thread unexpectedly hung up");
rx.await.expect("Worker thread failed to send back result")
}
}
fn test_setup(config: Config) -> (
Service<MockNetworkOps>,
MockGossip,
LocalPool,
impl Future<Output = ()> + 'static,
) {
let pool = LocalPool::new();
let network_ops = Arc::new(MockNetworkOps::default());
let mock_gossip = MockGossip::default();
let (worker_tx, worker_rx) = mpsc::channel(0);
let api = Arc::new(TestApi::default());
let worker_task = worker_loop(
config,
network_ops.clone(),
mock_gossip.clone(),
api.clone(),
worker_rx,
TaskExecutor::new(),
);
let service = Service {
sender: worker_tx,
network_service: network_ops,
};
(service, mock_gossip, pool, worker_task)
}
#[test]
fn worker_task_shuts_down_when_sender_dropped() {
let (service, _gossip, mut pool, worker_task) = test_setup(Config { collating_for: None });
drop(service);
let _ = pool.run_until(worker_task);
}
/// Given the async nature of `select!` that is being used in the main loop of the worker
/// and that consensus instances use their own channels, we don't know when the synchronize message
/// is handled. This helper functions checks multiple times that the given instance is dropped. Even
/// if the first round fails, the second one should be successful as the consensus instance drop
/// should be already handled this time.
fn wait_for_instance_drop(service: &mut Service<MockNetworkOps>, pool: &mut LocalPool, instance: Hash) {
let mut try_counter = 0;
let max_tries = 3;
while try_counter < max_tries {
let dropped = pool.run_until(service.synchronize(move |proto| {
!proto.consensus_instances.contains_key(&instance)
}));
if dropped {
return;
}
try_counter += 1;
}
panic!("Consensus instance `{}` wasn't dropped!", instance);
}
#[test]
fn consensus_instances_cleaned_up() {
let (mut service, _gossip, mut pool, worker_task) = test_setup(Config { collating_for: None });
let relay_parent = [0; 32].into();
let signing_context = SigningContext {
session_index: Default::default(),
parent_hash: relay_parent,
};
let table = Arc::new(SharedTable::new(
Vec::new(),
HashMap::new(),
None,
signing_context,
AvailabilityStore::new_in_memory(service.clone()),
None,
None,
));
pool.spawner().spawn_local(worker_task).unwrap();
let router = pool.run_until(
service.build_table_router(table, &[])
).unwrap();
drop(router);
wait_for_instance_drop(&mut service, &mut pool, relay_parent);
}
#[test]
fn collation_is_received_with_dropped_router() {
let (mut service, gossip, mut pool, worker_task) = test_setup(Config { collating_for: None });
let relay_parent = [0; 32].into();
let topic = crate::legacy::gossip::attestation_topic(relay_parent);
let signing_context = SigningContext {
session_index: Default::default(),
parent_hash: relay_parent,
};
let table = Arc::new(SharedTable::new(
vec![Sr25519Keyring::Alice.public().into()],
HashMap::new(),
Some(Arc::new(Sr25519Keyring::Alice.pair().into())),
signing_context,
AvailabilityStore::new_in_memory(service.clone()),
None,
None,
));
pool.spawner().spawn_local(worker_task).unwrap();
let router = pool.run_until(
service.build_table_router(table, &[])
).unwrap();
let receipt = AbridgedCandidateReceipt { relay_parent, ..Default::default() };
let local_collation_future = router.local_collation(
receipt,
PoVBlock { block_data: BlockData(Vec::new()) },
(0, &[]),
);
// Drop the router and make sure that the consensus instance is still alive
drop(router);
assert!(pool.run_until(service.synchronize(move |proto| {
proto.consensus_instances.contains_key(&relay_parent)
})));
// The gossip message should still be unknown
assert!(!gossip.gossip_messages.lock().contains_key(&topic));
pool.run_until(local_collation_future).unwrap();
// Make sure the instance is now dropped and the message was gossiped
wait_for_instance_drop(&mut service, &mut pool, relay_parent);
assert!(pool.run_until(service.synchronize(move |_| {
gossip.gossip_messages.lock().contains_key(&topic)
})));
}
#[test]
fn validator_peer_cleaned_up() {
let (mut service, _gossip, mut pool, worker_task) = test_setup(Config { collating_for: None });
let peer = PeerId::random();
let validator_key = Sr25519Keyring::Alice.pair();
let validator_id = ValidatorId::from(validator_key.public());
pool.spawner().spawn_local(worker_task).unwrap();
pool.run_until(async move {
service.connect_peer(peer.clone(), ObservedRole::Authority).await;
service.peer_message(peer.clone(), Message::Status(Status {
version: VERSION,
collating_for: None,
})).await;
service.peer_message(peer.clone(), Message::ValidatorId(validator_id.clone())).await;
let p = peer.clone();
let v = validator_id.clone();
let (peer_has_key, reverse_lookup) = service.synchronize(move |proto| {
let peer_has_key = proto.peers.get(&p).map_or(
false,
|p_data| p_data.session_keys.as_slice().contains(&v),
);
let reverse_lookup = proto.connected_validators.get(&v).map_or(
false,
|reps| reps.contains(&p),
);
(peer_has_key, reverse_lookup)
}).await;
assert!(peer_has_key);
assert!(reverse_lookup);
service.disconnect_peer(peer.clone()).await;
let p = peer.clone();
let v = validator_id.clone();
let (peer_removed, rev_removed) = service.synchronize(move |proto| {
let peer_removed = !proto.peers.contains_key(&p);
let reverse_mapping_removed = !proto.connected_validators.contains_key(&v);
(peer_removed, reverse_mapping_removed)
}).await;
assert!(peer_removed);
assert!(rev_removed);
});
}
#[test]
fn validator_key_spillover_cleaned() {
let (mut service, _gossip, mut pool, worker_task) = test_setup(Config { collating_for: None });
let peer = PeerId::random();
let make_validator_id = |ring: Sr25519Keyring| ValidatorId::from(ring.public());
// We will push 1 extra beyond what is normally kept.
assert_eq!(RECENT_SESSIONS, 3);
let key_a = make_validator_id(Sr25519Keyring::Alice);
let key_b = make_validator_id(Sr25519Keyring::Bob);
let key_c = make_validator_id(Sr25519Keyring::Charlie);
let key_d = make_validator_id(Sr25519Keyring::Dave);
let keys = vec![key_a, key_b, key_c, key_d];
pool.spawner().spawn_local(worker_task).unwrap();
pool.run_until(async move {
service.connect_peer(peer.clone(), ObservedRole::Authority).await;
service.peer_message(peer.clone(), Message::Status(Status {
version: VERSION,
collating_for: None,
})).await;
for key in &keys {
service.peer_message(peer.clone(), Message::ValidatorId(key.clone())).await;
}
let p = peer.clone();
let active_keys = keys[1..].to_vec();
let discarded_key = keys[0].clone();
assert!(service.synchronize(move |proto| {
let active_correct = proto.peers.get(&p).map_or(false, |p_data| {
p_data.session_keys.as_slice() == &active_keys[..]
});
let active_lookup = active_keys.iter().all(|k| {
proto.connected_validators.get(&k).map_or(false, |m| m.contains(&p))
});
let discarded = !proto.connected_validators.contains_key(&discarded_key);
active_correct && active_lookup && discarded
}).await);
});
}
#[test]
fn fetches_pov_block_from_gossip() {
let (service, gossip, mut pool, worker_task) = test_setup(Config { collating_for: None });
let relay_parent = [255; 32].into();
let pov_block = PoVBlock {
block_data: BlockData(vec![1, 2, 3]),
};
let mut candidate = AbridgedCandidateReceipt::default();
candidate.relay_parent = relay_parent;
candidate.pov_block_hash = pov_block.hash();
let candidate_hash = candidate.hash();
let signing_context = SigningContext {
session_index: Default::default(),
parent_hash: relay_parent,
};
let table = Arc::new(SharedTable::new(
Vec::new(),
HashMap::new(),
None,
signing_context,
AvailabilityStore::new_in_memory(service.clone()),
None,
None,
));
let spawner = pool.spawner();
spawner.spawn_local(worker_task).unwrap();
let topic = crate::legacy::gossip::pov_block_topic(relay_parent);
let (mut gossip_tx, _gossip_taken_rx) = gossip.add_gossip_stream(topic);
let test_work = async move {
let router = service.build_table_router(table, &[]).await.unwrap();
let pov_block_listener = router.fetch_pov_block(&candidate);
let message = GossipMessage::PoVBlock(GossipPoVBlock {
relay_chain_leaf: relay_parent,
candidate_hash,
pov_block,
}).encode();
gossip_tx.send(TopicNotification { message, sender: None }).await.unwrap();
pov_block_listener.await
};
pool.run_until(test_work).unwrap();
}
#[test]
fn validator_sends_key_and_role_to_collator_on_status() {
let (service, _gossip, mut pool, worker_task) = test_setup(Config { collating_for: None });
let peer = PeerId::random();
let peer_clone = peer.clone();
let validator_key = Sr25519Keyring::Alice.pair();
let validator_id = ValidatorId::from(validator_key.public());
let validator_id_clone = validator_id.clone();
let collator_id = CollatorId::from(Sr25519Keyring::Bob.public());
let para_id = ParaId::from(100);
let mut service_clone = service.clone();
pool.spawner().spawn_local(worker_task).unwrap();
pool.run_until(async move {
service_clone.synchronize(move |proto| { proto.local_keys.insert(validator_id_clone); }).await;
service_clone.connect_peer(peer_clone.clone(), ObservedRole::Authority).await;
service_clone.peer_message(peer_clone.clone(), Message::Status(Status {
version: VERSION,
collating_for: Some((collator_id, para_id)),
})).await;
});
let expected_msg = Message::ValidatorId(validator_id.clone());
let validator_id_pos = service.network_service.recorded.lock().notifications.iter().position(|(p, notification)| {
peer == *p && *notification == expected_msg
});
let expected_msg = Message::CollatorRole(CollatorRole::Primary);
let collator_role_pos = service.network_service.recorded.lock().notifications.iter().position(|(p, notification)| {
peer == *p && *notification == expected_msg
});
assert!(validator_id_pos < collator_role_pos);
}
#[test]
fn collator_state_send_key_updates_state_correctly() {
let mut state = CollatorState::Fresh;
state.send_key(Sr25519Keyring::Alice.public().into(), |_| {});
assert!(matches!(state, CollatorState::Primed(None)));
let mut state = CollatorState::RolePending(CollatorRole::Primary);
let mut counter = 0;
state.send_key(Sr25519Keyring::Alice.public().into(), |msg| {
match (counter, msg) {
(0, Message::ValidatorId(_)) => {
counter += 1;
},
(1, Message::CollatorRole(CollatorRole::Primary)) => {},
err @ _ => panic!("Unexpected message: {:?}", err),
}
});
assert!(matches!(state, CollatorState::Primed(Some(CollatorRole::Primary))));
}
polkadot/network/test/Cargo.toml
-23
View File
@@ -1,23 +0,0 @@
[package]
name = "polkadot-network-test"
version = "0.8.22"
license = "GPL-3.0"
authors = ["Parity Technologies <admin@parity.io>"]
edition = "2018"
[dependencies]
log = "0.4.8"
parking_lot = "0.10.0"
futures = "0.3.1"
rand = "0.7.2"
sc-consensus = { git = "https://github.com/paritytech/substrate", branch = "master" }
sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" }
sc-service = { git = "https://github.com/paritytech/substrate", features = ["test-helpers"], branch = "master" }
sc-network-test = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-consensus = { git = "https://github.com/paritytech/substrate", branch = "master" }
sc-client-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
sc-block-builder = { git = "https://github.com/paritytech/substrate", branch = "master" }
polkadot-test-runtime-client = { path = "../../runtime/test-runtime/client" }
polkadot/network/test/src/block_import.rs
-106
View File
@@ -1,106 +0,0 @@
// Copyright 2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Testing block import logic.
use sp_consensus::ImportedAux;
use sp_consensus::import_queue::{
import_single_block, BasicQueue, BlockImportError, BlockImportResult, IncomingBlock,
};
use polkadot_test_runtime_client::{self, prelude::*};
use polkadot_test_runtime_client::runtime::{Block, Hash};
use sp_runtime::generic::BlockId;
use super::*;
fn prepare_good_block() -> (TestClient, Hash, u64, PeerId, IncomingBlock<Block>) {
let mut client = polkadot_test_runtime_client::new();
let mut builder = client.new_block(Default::default()).unwrap();
let extrinsics = polkadot_test_runtime_client::needed_extrinsics(0);
for extrinsic in &extrinsics {
builder.push(extrinsic.clone()).unwrap();
}
let block = builder.build().unwrap().block;
client.import(BlockOrigin::File, block).unwrap();
let (hash, number) = (client.block_hash(1).unwrap().unwrap(), 1);
let header = client.header(&BlockId::Number(1)).unwrap();
let justification = client.justification(&BlockId::Number(1)).unwrap();
let peer_id = PeerId::random();
(client, hash, number, peer_id.clone(), IncomingBlock {
hash,
header,
body: Some(extrinsics),
justification,
origin: Some(peer_id.clone()),
allow_missing_state: false,
import_existing: false,
})
}
#[test]
fn import_single_good_block_works() {
let (_, _hash, number, peer_id, block) = prepare_good_block();
let mut expected_aux = ImportedAux::default();
expected_aux.is_new_best = true;
match import_single_block(&mut polkadot_test_runtime_client::new(), BlockOrigin::File, block, &mut PassThroughVerifier::new(true)) {
Ok(BlockImportResult::ImportedUnknown(ref num, ref aux, ref org))
if *num == number as u32 && *aux == expected_aux && *org == Some(peer_id) => {}
r @ _ => panic!("{:?}", r)
}
}
#[test]
fn import_single_good_known_block_is_ignored() {
let (mut client, _hash, number, _, block) = prepare_good_block();
match import_single_block(&mut client, BlockOrigin::File, block, &mut PassThroughVerifier::new(true)) {
Ok(BlockImportResult::ImportedKnown(ref n)) if *n == number as u32 => {}
_ => panic!()
}
}
#[test]
fn import_single_good_block_without_header_fails() {
let (_, _, _, peer_id, mut block) = prepare_good_block();
block.header = None;
match import_single_block(&mut polkadot_test_runtime_client::new(), BlockOrigin::File, block, &mut PassThroughVerifier::new(true)) {
Err(BlockImportError::IncompleteHeader(ref org)) if *org == Some(peer_id) => {}
_ => panic!()
}
}
#[test]
fn async_import_queue_drops() {
let executor = sp_core::testing::TaskExecutor::new();
// Perform this test multiple times since it exhibits non-deterministic behavior.
for _ in 0..100 {
let verifier = PassThroughVerifier::new(true);
let queue = BasicQueue::new(
verifier,
Box::new(polkadot_test_runtime_client::new()),
None,
None,
&executor,
None
);
drop(queue);
}
}
polkadot/network/test/src/lib.rs
-884
View File
@@ -1,884 +0,0 @@
// Copyright 2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
#![allow(missing_docs)]
#[cfg(test)]
mod block_import;
use std::{collections::HashMap, pin::Pin, sync::Arc, marker::PhantomData, task::{Poll, Context as FutureContext}};
use log::trace;
use sc_network::config::{build_multiaddr, FinalityProofProvider, Role};
use sp_blockchain::{
Result as ClientResult, well_known_cache_keys::{self, Id as CacheKeyId}, Info as BlockchainInfo,
HeaderBackend,
};
use sc_client_api::{
BlockchainEvents, BlockImportNotification,
FinalityNotifications, ImportNotifications,
FinalityNotification,
client::BlockBackend,
backend::{TransactionFor, AuxStore, Backend, Finalizer},
};
use sc_consensus::LongestChain;
use sc_block_builder::{BlockBuilder, BlockBuilderProvider};
use sp_consensus::block_validation::DefaultBlockAnnounceValidator;
use sp_consensus::import_queue::{
BasicQueue, BoxJustificationImport, Verifier, BoxFinalityProofImport,
};
use sp_consensus::block_import::{BlockImport, ImportResult};
use sp_consensus::Error as ConsensusError;
use sp_consensus::{BlockOrigin, BlockImportParams, BlockCheckParams, JustificationImport};
use futures::prelude::*;
use sc_network::{NetworkWorker, NetworkService, config::ProtocolId};
use sc_network::config::{
NetworkConfiguration, TransportConfig, BoxFinalityProofRequestBuilder, TransactionImport,
TransactionImportFuture
};
use parking_lot::Mutex;
use sp_core::H256;
use sc_network::{PeerId, config::{ProtocolConfig, TransactionPool}};
use sp_runtime::generic::BlockId;
use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor};
use sp_runtime::Justification;
pub use sc_network_test::PassThroughVerifier;
use sc_service::client::Client;
pub use polkadot_test_runtime_client::runtime::{Block, Extrinsic, Hash};
pub use polkadot_test_runtime_client::{TestClient, TestClientBuilder, TestClientBuilderExt};
pub type PeersFullClient = Client<
polkadot_test_runtime_client::Backend,
polkadot_test_runtime_client::Executor,
Block,
polkadot_test_runtime_client::runtime::RuntimeApi
>;
pub type PeersLightClient = Client<
polkadot_test_runtime_client::LightBackend,
polkadot_test_runtime_client::LightExecutor,
Block,
polkadot_test_runtime_client::runtime::RuntimeApi
>;
#[derive(Clone)]
pub enum PeersClient {
Full(Arc<PeersFullClient>, Arc<polkadot_test_runtime_client::Backend>),
Light(Arc<PeersLightClient>, Arc<polkadot_test_runtime_client::LightBackend>),
}
impl PeersClient {
pub fn as_full(&self) -> Option<Arc<PeersFullClient>> {
match *self {
PeersClient::Full(ref client, ref _backend) => Some(client.clone()),
_ => None,
}
}
pub fn as_block_import<Transaction>(&self) -> BlockImportAdapter<Transaction> {
match *self {
PeersClient::Full(ref client, ref _backend) =>
BlockImportAdapter::new_full(client.clone()),
PeersClient::Light(ref client, ref _backend) =>
BlockImportAdapter::Light(Arc::new(Mutex::new(client.clone())), PhantomData),
}
}
pub fn get_aux(&self, key: &[u8]) -> ClientResult<Option<Vec<u8>>> {
match *self {
PeersClient::Full(ref client, ref _backend) => client.get_aux(key),
PeersClient::Light(ref client, ref _backend) => client.get_aux(key),
}
}
pub fn info(&self) -> BlockchainInfo<Block> {
match *self {
PeersClient::Full(ref client, ref _backend) => client.chain_info(),
PeersClient::Light(ref client, ref _backend) => client.chain_info(),
}
}
pub fn header(&self, block: &BlockId<Block>) -> ClientResult<Option<<Block as BlockT>::Header>> {
match *self {
PeersClient::Full(ref client, ref _backend) => client.header(block),
PeersClient::Light(ref client, ref _backend) => client.header(block),
}
}
pub fn justification(&self, block: &BlockId<Block>) -> ClientResult<Option<Justification>> {
match *self {
PeersClient::Full(ref client, ref _backend) => client.justification(block),
PeersClient::Light(ref client, ref _backend) => client.justification(block),
}
}
pub fn finality_notification_stream(&self) -> FinalityNotifications<Block> {
match *self {
PeersClient::Full(ref client, ref _backend) => client.finality_notification_stream(),
PeersClient::Light(ref client, ref _backend) => client.finality_notification_stream(),
}
}
pub fn import_notification_stream(&self) -> ImportNotifications<Block>{
match *self {
PeersClient::Full(ref client, ref _backend) => client.import_notification_stream(),
PeersClient::Light(ref client, ref _backend) => client.import_notification_stream(),
}
}
pub fn finalize_block(
&self,
id: BlockId<Block>,
justification: Option<Justification>,
notify: bool
) -> ClientResult<()> {
match *self {
PeersClient::Full(ref client, ref _backend) => client.finalize_block(id, justification, notify),
PeersClient::Light(ref client, ref _backend) => client.finalize_block(id, justification, notify),
}
}
}
pub struct Peer<D> {
pub data: D,
client: PeersClient,
/// We keep a copy of the verifier so that we can invoke it for locally-generated blocks,
/// instead of going through the import queue.
verifier: VerifierAdapter<Block>,
/// We keep a copy of the block_import so that we can invoke it for locally-generated blocks,
/// instead of going through the import queue.
block_import: BlockImportAdapter<()>,
select_chain: Option<LongestChain<polkadot_test_runtime_client::Backend, Block>>,
backend: Option<Arc<polkadot_test_runtime_client::Backend>>,
network: NetworkWorker<Block, <Block as BlockT>::Hash>,
imported_blocks_stream: Pin<Box<dyn Stream<Item = BlockImportNotification<Block>> + Send>>,
finality_notification_stream: Pin<Box<dyn Stream<Item = FinalityNotification<Block>> + Send>>,
}
impl<D> Peer<D> {
/// Get this peer ID.
pub fn id(&self) -> &PeerId {
self.network.service().local_peer_id()
}
/// Returns true if we're major syncing.
pub fn is_major_syncing(&self) -> bool {
self.network.service().is_major_syncing()
}
// Returns a clone of the local SelectChain, only available on full nodes
pub fn select_chain(&self) -> Option<LongestChain<polkadot_test_runtime_client::Backend, Block>> {
self.select_chain.clone()
}
/// Returns the number of peers we're connected to.
pub fn num_peers(&self) -> usize {
self.network.num_connected_peers()
}
/// Returns true if we have no peer.
pub fn is_offline(&self) -> bool {
self.num_peers() == 0
}
/// Request a justification for the given block.
pub fn request_justification(&self, hash: &<Block as BlockT>::Hash, number: NumberFor<Block>) {
self.network.service().request_justification(hash, number);
}
/// Announces an important block on the network.
pub fn announce_block(&self, hash: <Block as BlockT>::Hash, data: Vec<u8>) {
self.network.service().announce_block(hash, data);
}
/// Request explicit fork sync.
pub fn set_sync_fork_request(&self, peers: Vec<PeerId>, hash: <Block as BlockT>::Hash, number: NumberFor<Block>) {
self.network.service().set_sync_fork_request(peers, hash, number);
}
/// Add blocks to the peer -- edit the block before adding
pub fn generate_blocks<F>(&mut self, count: usize, origin: BlockOrigin, edit_block: F) -> H256
where F: FnMut(BlockBuilder<Block, PeersFullClient, polkadot_test_runtime_client::Backend>) -> Block
{
let best_hash = self.client.info().best_hash;
self.generate_blocks_at(BlockId::Hash(best_hash), count, origin, edit_block, false)
}
/// Add blocks to the peer -- edit the block before adding. The chain will
/// start at the given block iD.
fn generate_blocks_at<F>(
&mut self,
at: BlockId<Block>,
count: usize,
origin: BlockOrigin,
mut edit_block: F,
headers_only: bool,
) -> H256 where F: FnMut(BlockBuilder<Block, PeersFullClient, polkadot_test_runtime_client::Backend>) -> Block {
let full_client = self.client.as_full()
.expect("blocks could only be generated by full clients");
let mut at = full_client.header(&at).unwrap().unwrap().hash();
for _ in 0..count {
let builder = full_client.new_block_at(
&BlockId::Hash(at),
Default::default(),
false,
).unwrap();
let block = edit_block(builder);
let hash = block.header.hash();
trace!(
target: "test_network",
"Generating {}, (#{}, parent={})",
hash,
block.header.number,
block.header.parent_hash,
);
let header = block.header.clone();
let (import_block, cache) = self.verifier.verify(
origin,
header.clone(),
None,
if headers_only { None } else { Some(block.extrinsics) },
).unwrap();
let cache = if let Some(cache) = cache {
cache.into_iter().collect()
} else {
Default::default()
};
self.block_import.import_block(import_block, cache).expect("block_import failed");
at = hash;
}
self.network.update_chain();
self.network.service().announce_block(at.clone(), Vec::new());
at
}
/// Push blocks to the peer (simplified: with or without a TX)
pub fn push_blocks(&mut self, count: usize, with_tx: bool) -> H256 {
let best_hash = self.client.info().best_hash;
self.push_blocks_at(BlockId::Hash(best_hash), count, with_tx)
}
/// Push blocks to the peer (simplified: with or without a TX)
pub fn push_headers(&mut self, count: usize) -> H256 {
let best_hash = self.client.info().best_hash;
self.generate_tx_blocks_at(BlockId::Hash(best_hash), count, false, true)
}
/// Push blocks to the peer (simplified: with or without a TX) starting from
/// given hash.
pub fn push_blocks_at(&mut self, at: BlockId<Block>, count: usize, with_tx: bool) -> H256 {
self.generate_tx_blocks_at(at, count, with_tx, false)
}
/// Push blocks/headers to the peer (simplified: with or without a TX) starting from
/// given hash.
fn generate_tx_blocks_at(&mut self, at: BlockId<Block>, count: usize, with_tx: bool, headers_only:bool) -> H256 {
if with_tx {
self.generate_blocks_at(
at,
count,
BlockOrigin::File,
|builder| builder.build().unwrap().block,
headers_only
)
} else {
self.generate_blocks_at(
at,
count,
BlockOrigin::File,
|builder| builder.build().unwrap().block,
headers_only,
)
}
}
/// Get a reference to the client.
pub fn client(&self) -> &PeersClient {
&self.client
}
/// Get a reference to the network service.
pub fn network_service(&self) -> &Arc<NetworkService<Block, <Block as BlockT>::Hash>> {
&self.network.service()
}
/// Test helper to compare the blockchain state of multiple (networked)
/// clients.
/// Potentially costly, as it creates in-memory copies of both blockchains in order
/// to compare them. If you have easier/softer checks that are sufficient, e.g.
/// by using .info(), you should probably use it instead of this.
pub fn blockchain_canon_equals(&self, other: &Self) -> bool {
if let (Some(mine), Some(others)) = (self.backend.clone(), other.backend.clone()) {
mine.blockchain().info().best_hash == others.blockchain().info().best_hash
} else {
false
}
}
/// Count the total number of imported blocks.
pub fn blocks_count(&self) -> u64 {
self.backend.as_ref().map(
|backend| backend.blockchain().info().best_number as u64
).unwrap_or(0)
}
/// Return a collection of block hashes that failed verification
pub fn failed_verifications(&self) -> HashMap<<Block as BlockT>::Hash, String> {
self.verifier.failed_verifications.lock().clone()
}
}
pub struct EmptyTransactionPool;
impl TransactionPool<Hash, Block> for EmptyTransactionPool {
fn transactions(&self) -> Vec<(Hash, Extrinsic)> {
Vec::new()
}
fn hash_of(&self, _transaction: &Extrinsic) -> Hash {
Hash::default()
}
fn import(&self, _transaction: Extrinsic) -> TransactionImportFuture {
Box::pin(futures::future::ready(TransactionImport::None))
}
fn on_broadcasted(&self, _: HashMap<Hash, Vec<String>>) {}
fn transaction(&self, _h: &Hash) -> Option<Extrinsic> { None }
}
/// Implements `BlockImport` for any `Transaction`. Internally the transaction is
/// "converted", aka the field is set to `None`.
///
/// This is required as the `TestNetFactory` trait does not distinguish between
/// full and light nodes.
pub enum BlockImportAdapter<Transaction> {
Full(
Arc<Mutex<dyn BlockImport<
Block,
Transaction = TransactionFor<polkadot_test_runtime_client::Backend, Block>,
Error = ConsensusError
> + Send>>,
PhantomData<Transaction>,
),
Light(
Arc<Mutex<dyn BlockImport<
Block,
Transaction = TransactionFor<polkadot_test_runtime_client::LightBackend, Block>,
Error = ConsensusError
> + Send>>,
PhantomData<Transaction>,
),
}
impl<Transaction> BlockImportAdapter<Transaction> {
/// Create a new instance of `Self::Full`.
pub fn new_full(
full: impl BlockImport<
Block,
Transaction = TransactionFor<polkadot_test_runtime_client::Backend, Block>,
Error = ConsensusError
>
+ 'static
+ Send
) -> Self {
Self::Full(Arc::new(Mutex::new(full)), PhantomData)
}
/// Create a new instance of `Self::Light`.
pub fn new_light(
light: impl BlockImport<
Block,
Transaction = TransactionFor<polkadot_test_runtime_client::LightBackend, Block>,
Error = ConsensusError
>
+ 'static
+ Send
) -> Self {
Self::Light(Arc::new(Mutex::new(light)), PhantomData)
}
}
impl<Transaction> Clone for BlockImportAdapter<Transaction> {
fn clone(&self) -> Self {
match self {
Self::Full(full, _) => Self::Full(full.clone(), PhantomData),
Self::Light(light, _) => Self::Light(light.clone(), PhantomData),
}
}
}
impl<Transaction> BlockImport<Block> for BlockImportAdapter<Transaction> {
type Error = ConsensusError;
type Transaction = Transaction;
fn check_block(
&mut self,
block: BlockCheckParams<Block>,
) -> Result<ImportResult, Self::Error> {
match self {
Self::Full(full, _) => full.lock().check_block(block),
Self::Light(light, _) => light.lock().check_block(block),
}
}
fn import_block(
&mut self,
block: BlockImportParams<Block, Transaction>,
cache: HashMap<well_known_cache_keys::Id, Vec<u8>>,
) -> Result<ImportResult, Self::Error> {
match self {
Self::Full(full, _) => full.lock().import_block(block.convert_transaction(), cache),
Self::Light(light, _) => light.lock().import_block(block.convert_transaction(), cache),
}
}
}
/// Implements `Verifier` on an `Arc<Mutex<impl Verifier>>`. Used internally.
#[derive(Clone)]
struct VerifierAdapter<B: BlockT> {
verifier: Arc<Mutex<Box<dyn Verifier<B>>>>,
failed_verifications: Arc<Mutex<HashMap<B::Hash, String>>>,
}
impl<B: BlockT> Verifier<B> for VerifierAdapter<B> {
fn verify(
&mut self,
origin: BlockOrigin,
header: B::Header,
justification: Option<Justification>,
body: Option<Vec<B::Extrinsic>>
) -> Result<(BlockImportParams<B, ()>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String> {
let hash = header.hash();
self.verifier.lock().verify(origin, header, justification, body).map_err(|e| {
self.failed_verifications.lock().insert(hash, e.clone());
e
})
}
}
impl<B: BlockT> VerifierAdapter<B> {
fn new(verifier: Arc<Mutex<Box<dyn Verifier<B>>>>) -> VerifierAdapter<B> {
VerifierAdapter {
verifier,
failed_verifications: Default::default(),
}
}
}
pub trait TestNetFactory: Sized {
type Verifier: 'static + Verifier<Block>;
type PeerData: Default;
/// These two need to be implemented!
fn from_config(config: &ProtocolConfig) -> Self;
fn make_verifier(
&self,
client: PeersClient,
config: &ProtocolConfig,
peer_data: &Self::PeerData,
) -> Self::Verifier;
/// Get reference to peer.
fn peer(&mut self, i: usize) -> &mut Peer<Self::PeerData>;
fn peers(&self) -> &Vec<Peer<Self::PeerData>>;
fn mut_peers<F: FnOnce(&mut Vec<Peer<Self::PeerData>>)>(
&mut self,
closure: F,
);
/// Get custom block import handle for fresh client, along with peer data.
fn make_block_import<Transaction>(&self, client: PeersClient)
-> (
BlockImportAdapter<Transaction>,
Option<BoxJustificationImport<Block>>,
Option<BoxFinalityProofImport<Block>>,
Option<BoxFinalityProofRequestBuilder<Block>>,
Self::PeerData,
)
{
(client.as_block_import(), None, None, None, Default::default())
}
/// Get finality proof provider (if supported).
fn make_finality_proof_provider(
&self,
_client: PeersClient,
) -> Option<Arc<dyn FinalityProofProvider<Block>>> {
None
}
fn default_config() -> ProtocolConfig {
ProtocolConfig::default()
}
/// Create new test network with this many peers.
fn new(n: usize) -> Self {
trace!(target: "test_network", "Creating test network");
let mut net = Self::from_config(&Default::default());
for i in 0..n {
trace!(target: "test_network", "Adding peer {}", i);
net.add_full_peer();
}
net
}
fn add_full_peer(&mut self,) {
self.add_full_peer_with_states(None)
}
/// Add a full peer.
fn add_full_peer_with_states(&mut self, keep_blocks: Option<u32>) {
let test_client_builder = match keep_blocks {
Some(keep_blocks) => TestClientBuilder::with_pruning_window(keep_blocks),
None => TestClientBuilder::with_default_backend(),
};
let backend = test_client_builder.backend();
let (c, longest_chain) = test_client_builder.build_with_longest_chain();
let client = Arc::new(c);
let (
block_import,
justification_import,
finality_proof_import,
finality_proof_request_builder,
data,
) = self.make_block_import(PeersClient::Full(client.clone(), backend.clone()));
let verifier = self.make_verifier(
PeersClient::Full(client.clone(), backend.clone()),
&Default::default(),
&data,
);
let verifier = VerifierAdapter::new(Arc::new(Mutex::new(Box::new(verifier) as Box<_>)));
let import_queue = Box::new(BasicQueue::new(
verifier.clone(),
Box::new(block_import.clone()),
justification_import,
finality_proof_import,
&sp_core::testing::TaskExecutor::new(),
None,
));
let listen_addr = build_multiaddr![Memory(rand::random::<u64>())];
let mut network_config = NetworkConfiguration::new(
"test-node",
"test-client",
Default::default(),
None,
);
network_config.listen_addresses = vec![listen_addr.clone()];
network_config.transport = TransportConfig::MemoryOnly;
let network = NetworkWorker::new(sc_network::config::Params {
role: Role::Full,
executor: None,
network_config,
chain: client.clone(),
finality_proof_provider: self.make_finality_proof_provider(
PeersClient::Full(client.clone(), backend.clone()),
),
finality_proof_request_builder,
on_demand: None,
transaction_pool: Arc::new(EmptyTransactionPool),
protocol_id: ProtocolId::from(&b"test-protocol-name"[..]),
import_queue,
block_announce_validator: Box::new(DefaultBlockAnnounceValidator),
metrics_registry: None,
}).unwrap();
self.mut_peers(|peers| {
for peer in peers.iter_mut() {
peer.network.add_known_address(network.service().local_peer_id().clone(), listen_addr.clone());
}
let imported_blocks_stream = Box::pin(client.import_notification_stream().fuse());
let finality_notification_stream = Box::pin(client.finality_notification_stream().fuse());
peers.push(Peer {
data,
client: PeersClient::Full(client, backend.clone()),
select_chain: Some(longest_chain),
backend: Some(backend),
imported_blocks_stream,
finality_notification_stream,
block_import,
verifier,
network,
});
});
}
/// Add a light peer.
fn add_light_peer(&mut self) {
let (c, backend) = polkadot_test_runtime_client::new_light();
let client = Arc::new(c);
let (
block_import,
justification_import,
finality_proof_import,
finality_proof_request_builder,
data,
) = self.make_block_import(PeersClient::Light(client.clone(), backend.clone()));
let verifier = self.make_verifier(
PeersClient::Light(client.clone(), backend.clone()),
&Default::default(),
&data,
);
let verifier = VerifierAdapter::new(Arc::new(Mutex::new(Box::new(verifier) as Box<_>)));
let import_queue = Box::new(BasicQueue::new(
verifier.clone(),
Box::new(block_import.clone()),
justification_import,
finality_proof_import,
&sp_core::testing::TaskExecutor::new(),
None,
));
let listen_addr = build_multiaddr![Memory(rand::random::<u64>())];
let mut network_config = NetworkConfiguration::new(
"test-node",
"test-client",
Default::default(),
None,
);
network_config.listen_addresses = vec![listen_addr.clone()];
network_config.transport = TransportConfig::MemoryOnly;
let network = NetworkWorker::new(sc_network::config::Params {
role: Role::Full,
executor: None,
network_config,
chain: client.clone(),
finality_proof_provider: self.make_finality_proof_provider(
PeersClient::Light(client.clone(), backend.clone())
),
finality_proof_request_builder,
on_demand: None,
transaction_pool: Arc::new(EmptyTransactionPool),
protocol_id: ProtocolId::from(&b"test-protocol-name"[..]),
import_queue,
block_announce_validator: Box::new(DefaultBlockAnnounceValidator),
metrics_registry: None,
}).unwrap();
self.mut_peers(|peers| {
for peer in peers.iter_mut() {
peer.network.add_known_address(network.service().local_peer_id().clone(), listen_addr.clone());
}
let imported_blocks_stream = Box::pin(client.import_notification_stream().fuse());
let finality_notification_stream = Box::pin(client.finality_notification_stream().fuse());
peers.push(Peer {
data,
verifier,
select_chain: None,
backend: None,
block_import,
client: PeersClient::Light(client, backend),
imported_blocks_stream,
finality_notification_stream,
network,
});
});
}
/// Polls the testnet until all nodes are in sync.
///
/// Must be executed in a task context.
fn poll_until_sync(&mut self, cx: &mut FutureContext) -> Poll<()> {
self.poll(cx);
// Return `NotReady` if there's a mismatch in the highest block number.
let mut highest = None;
for peer in self.peers().iter() {
if peer.is_major_syncing() || peer.network.num_queued_blocks() != 0 {
return Poll::Pending
}
if peer.network.num_sync_requests() != 0 {
return Poll::Pending
}
match (highest, peer.client.info().best_hash) {
(None, b) => highest = Some(b),
(Some(ref a), ref b) if a == b => {},
(Some(_), _) => return Poll::Pending
}
}
Poll::Ready(())
}
/// Polls the testnet until theres' no activiy of any kind.
///
/// Must be executed in a task context.
fn poll_until_idle(&mut self, cx: &mut FutureContext) -> Poll<()> {
self.poll(cx);
for peer in self.peers().iter() {
if peer.is_major_syncing() || peer.network.num_queued_blocks() != 0 {
return Poll::Pending
}
if peer.network.num_sync_requests() != 0 {
return Poll::Pending
}
}
Poll::Ready(())
}
/// Blocks the current thread until we are sync'ed.
///
/// Calls `poll_until_sync` repeatedly.
fn block_until_sync(&mut self) {
futures::executor::block_on(futures::future::poll_fn::<(), _>(|cx| self.poll_until_sync(cx)));
}
/// Blocks the current thread until there are no pending packets.
///
/// Calls `poll_until_idle` repeatedly with the runtime passed as parameter.
fn block_until_idle(&mut self) {
futures::executor::block_on(futures::future::poll_fn::<(), _>(|cx| self.poll_until_idle(cx)));
}
/// Polls the testnet. Processes all the pending actions.
fn poll(&mut self, cx: &mut FutureContext) {
self.mut_peers(|peers| {
for peer in peers {
trace!(target: "sync", "-- Polling {}", peer.id());
if let Poll::Ready(()) = peer.network.poll_unpin(cx) {
panic!("NetworkWorker has terminated unexpectedly.")
}
trace!(target: "sync", "-- Polling complete {}", peer.id());
// We poll `imported_blocks_stream`.
while let Poll::Ready(Some(notification)) = peer.imported_blocks_stream.as_mut().poll_next(cx) {
peer.network.service().announce_block(notification.hash, Vec::new());
}
// We poll `finality_notification_stream`, but we only take the last event.
let mut last = None;
while let Poll::Ready(Some(item)) = peer.finality_notification_stream.as_mut().poll_next(cx) {
last = Some(item);
}
if let Some(notification) = last {
peer.network.on_block_finalized(notification.hash, notification.header);
}
}
});
}
}
pub struct TestNet {
peers: Vec<Peer<()>>,
}
impl TestNetFactory for TestNet {
type Verifier = PassThroughVerifier;
type PeerData = ();
/// Create new test network with peers and given config.
fn from_config(_config: &ProtocolConfig) -> Self {
TestNet {
peers: Vec::new(),
}
}
fn make_verifier(&self, _client: PeersClient, _config: &ProtocolConfig, _peer_data: &())
-> Self::Verifier
{
PassThroughVerifier::new(false)
}
fn peer(&mut self, i: usize) -> &mut Peer<()> {
&mut self.peers[i]
}
fn peers(&self) -> &Vec<Peer<()>> {
&self.peers
}
fn mut_peers<F: FnOnce(&mut Vec<Peer<()>>)>(&mut self, closure: F) {
closure(&mut self.peers);
}
}
pub struct ForceFinalized(PeersClient);
impl JustificationImport<Block> for ForceFinalized {
type Error = ConsensusError;
fn import_justification(
&mut self,
hash: H256,
_number: NumberFor<Block>,
justification: Justification,
) -> Result<(), Self::Error> {
self.0.finalize_block(BlockId::Hash(hash), Some(justification), true)
.map_err(|_| ConsensusError::InvalidJustification.into())
}
}
pub struct JustificationTestNet(TestNet);
impl TestNetFactory for JustificationTestNet {
type Verifier = PassThroughVerifier;
type PeerData = ();
fn from_config(config: &ProtocolConfig) -> Self {
JustificationTestNet(TestNet::from_config(config))
}
fn make_verifier(&self, client: PeersClient, config: &ProtocolConfig, peer_data: &()) -> Self::Verifier {
self.0.make_verifier(client, config, peer_data)
}
fn peer(&mut self, i: usize) -> &mut Peer<Self::PeerData> {
self.0.peer(i)
}
fn peers(&self) -> &Vec<Peer<Self::PeerData>> {
self.0.peers()
}
fn mut_peers<F: FnOnce(
&mut Vec<Peer<Self::PeerData>>,
)>(&mut self, closure: F) {
self.0.mut_peers(closure)
}
fn make_block_import<Transaction>(&self, client: PeersClient)
-> (
BlockImportAdapter<Transaction>,
Option<BoxJustificationImport<Block>>,
Option<BoxFinalityProofImport<Block>>,
Option<BoxFinalityProofRequestBuilder<Block>>,
Self::PeerData,
)
{
(
client.as_block_import(),
Some(Box::new(ForceFinalized(client))),
None,
None,
Default::default(),
)
}
}
polkadot/node/service/Cargo.toml
-1
View File
@@ -18,7 +18,6 @@ polkadot-overseer = { path = "../overseer" }
polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../subsystem" }
kusama-runtime = { path = "../../runtime/kusama" }
westend-runtime = { path = "../../runtime/westend" }
polkadot-network = { path = "../../network", optional = true }
polkadot-rpc = { path = "../../rpc" }
polkadot-node-core-proposer = { path = "../core/proposer" }
sp-io = { git = "https://github.com/paritytech/substrate", branch = "master" }
polkadot/node/test-service/Cargo.toml
-3
View File
@@ -13,9 +13,6 @@ rand = "0.7.3"
tempfile = "3.1.0"
# Polkadot dependencies
av_store = { package = "polkadot-availability-store", path = "../../availability-store" }
consensus = { package = "polkadot-validation", path = "../../validation" }
polkadot-network = { path = "../../network" }
polkadot-primitives = { path = "../../primitives" }
polkadot-rpc = { path = "../../rpc" }
polkadot-runtime-common = { path = "../../runtime/common" }
polkadot/node/test-service/src/lib.rs
+1 -5
View File
@@ -63,9 +63,7 @@ native_executor_instance!(
pub fn polkadot_test_new_full(
config: Configuration,
collating_for: Option<(CollatorId, ParaId)>,
max_block_data_size: Option<u64>,
authority_discovery_enabled: bool,
slot_duration: u64,
) -> Result<
(
TaskManager,
@@ -80,9 +78,7 @@ pub fn polkadot_test_new_full(
new_full::<polkadot_test_runtime::RuntimeApi, PolkadotTestExecutor>(
config,
collating_for,
max_block_data_size,
authority_discovery_enabled,
slot_duration,
None,
true,
)?;
@@ -211,7 +207,7 @@ pub fn run_test_node(
let multiaddr = config.network.listen_addresses[0].clone();
let authority_discovery_enabled = false;
let (task_manager, client, handles, network, rpc_handlers) =
polkadot_test_new_full(config, None, None, authority_discovery_enabled, 6000)
polkadot_test_new_full(config, None, authority_discovery_enabled)
.expect("could not create Polkadot test service");
let peer_id = network.local_peer_id().clone();
polkadot/parachain/test-parachains/adder/collator/Cargo.toml
-19
View File
@@ -1,19 +0,0 @@
[package]
name = "test-parachain-adder-collator"
version = "0.1.0"
authors = ["Parity Technologies <admin@parity.io>"]
edition = "2018"
[dependencies]
adder = { package = "test-parachain-adder", path = ".." }
parachain = { package = "polkadot-parachain", path = "../../.." }
collator = { package = "polkadot-collator", path = "../../../../collator" }
primitives = { package = "polkadot-primitives", path = "../../../../primitives" }
service = { package = "polkadot-service", path = "../../../../service" }
sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
client-api = { package = "sc-client-api", git = "https://github.com/paritytech/substrate", branch = "master" }
sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
parking_lot = "0.10.0"
codec = { package = "parity-scale-codec", version = "1.3.4" }
futures = "0.3.4"
polkadot/parachain/test-parachains/adder/collator/src/main.rs
-161
View File
@@ -1,161 +0,0 @@
// Copyright 2018-2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Collator for polkadot
use std::collections::HashMap;
use std::sync::Arc;
use adder::{HeadData as AdderHead, BlockData as AdderBody};
use sp_core::{traits::SpawnNamed, Pair};
use codec::{Encode, Decode};
use primitives::v0::{
Block, Hash, DownwardMessage,
HeadData, BlockData, Id as ParaId, LocalValidationData, GlobalValidationData,
};
use collator::{ParachainContext, Network, BuildParachainContext, Cli, SubstrateCli};
use parking_lot::Mutex;
use futures::future::{Ready, ready, FutureExt};
use sp_runtime::traits::BlakeTwo256;
use client_api::Backend as BackendT;
const GENESIS: AdderHead = AdderHead {
number: 0,
parent_hash: [0; 32],
post_state: [
1, 27, 77, 3, 221, 140, 1, 241, 4, 145, 67, 207, 156, 76, 129, 126, 75,
22, 127, 29, 27, 131, 229, 198, 240, 241, 13, 137, 186, 30, 123, 206
],
};
const GENESIS_BODY: AdderBody = AdderBody {
state: 0,
add: 0,
};
#[derive(Clone)]
struct AdderContext {
db: Arc<Mutex<HashMap<AdderHead, AdderBody>>>,
/// We store it here to make sure that our interfaces require the correct bounds.
_network: Option<Arc<dyn Network>>,
}
/// The parachain context.
impl ParachainContext for AdderContext {
type ProduceCandidate = Ready<Option<(BlockData, HeadData)>>;
fn produce_candidate(
&mut self,
_relay_parent: Hash,
_global_validation: GlobalValidationData,
local_validation: LocalValidationData,
_: Vec<DownwardMessage>,
) -> Self::ProduceCandidate
{
let adder_head = match AdderHead::decode(&mut &local_validation.parent_head.0[..]).ok() {
Some(res) => res,
None => return ready(None),
};
let mut db = self.db.lock();
let last_body = if adder_head == GENESIS {
GENESIS_BODY
} else {
db.get(&adder_head)
.expect("All past bodies stored since this is the only collator")
.clone()
};
let next_body = AdderBody {
state: last_body.state.overflowing_add(last_body.add).0,
add: adder_head.number % 100,
};
let next_head = adder::execute(adder_head.hash(), adder_head, &next_body)
.expect("good execution params; qed");
let encoded_head = HeadData(next_head.encode());
let encoded_body = BlockData(next_body.encode());
println!(
"Created collation for #{}, post-state={}",
next_head.number,
next_body.state.overflowing_add(next_body.add).0,
);
db.insert(next_head.clone(), next_body);
ready(Some((encoded_body, encoded_head)))
}
}
impl BuildParachainContext for AdderContext {
type ParachainContext = Self;
fn build<SP, Client, Backend>(
self,
_: Arc<Client>,
_: SP,
network: impl Network + Clone + 'static,
) -> Result<Self::ParachainContext, ()>
where
SP: SpawnNamed + Clone + Send + Sync + 'static,
Backend: BackendT<Block>,
Backend::State: sp_api::StateBackend<BlakeTwo256>,
Client: service::AbstractClient<Block, Backend> + 'static,
Client::Api: service::RuntimeApiCollection<StateBackend = Backend::State>,
{
Ok(Self { _network: Some(Arc::new(network)), ..self })
}
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
let key = Arc::new(Pair::from_seed(&[1; 32]));
let id: ParaId = 100.into();
println!("Starting adder collator with genesis: ");
{
let encoded = GENESIS.encode();
println!("Dec: {:?}", encoded);
print!("Hex: 0x");
for byte in encoded {
print!("{:02x}", byte);
}
println!();
}
let context = AdderContext {
db: Arc::new(Mutex::new(HashMap::new())),
_network: None,
};
let cli = Cli::from_iter(&["-dev"]);
let runner = cli.create_runner(&cli.run.base)?;
runner.async_run(|config| {
let (future, task_manager) = collator::start_collator(
context,
id,
key,
config,
)?;
Ok((future.map(Ok), task_manager))
})?;
Ok(())
}
polkadot/service/Cargo.toml
+1 -3
View File
@@ -12,13 +12,11 @@ log = "0.4.8"
futures = "0.3.4"
slog = "2.5.2"
hex-literal = "0.2.1"
av_store = { package = "polkadot-availability-store", path = "../availability-store", optional = true }
consensus = { package = "polkadot-validation", path = "../validation", optional = true }
polkadot-primitives = { path = "../primitives" }
polkadot-runtime = { path = "../runtime/polkadot" }
kusama-runtime = { path = "../runtime/kusama" }
westend-runtime = { path = "../runtime/westend" }
polkadot-network = { path = "../network", optional = true }
polkadot-rpc = { path = "../rpc" }
sp-io = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
@@ -67,4 +65,4 @@ env_logger = "0.7.0"
default = ["db", "full-node"]
db = ["service/db"]
runtime-benchmarks = ["polkadot-runtime/runtime-benchmarks", "kusama-runtime/runtime-benchmarks", "westend-runtime/runtime-benchmarks"]
full-node = ["av_store", "consensus", "polkadot-network"]
full-node = ["consensus"]
polkadot/service/src/lib.rs
+3 -128
View File
@@ -22,9 +22,7 @@ mod client;
use std::sync::Arc;
use std::time::Duration;
use polkadot_primitives::v0::{self as parachain, Hash, BlockId};
#[cfg(feature = "full-node")]
use polkadot_network::{legacy::gossip::Known, protocol as network_protocol};
use polkadot_primitives::v0 as parachain;
use service::error::Error as ServiceError;
use grandpa::{self, FinalityProofProvider as GrandpaFinalityProofProvider};
use sc_executor::native_executor_instance;
@@ -249,9 +247,7 @@ pub fn new_partial<RuntimeApi, Executor>(config: &mut Configuration, test: bool)
pub fn new_full<RuntimeApi, Executor>(
mut config: Configuration,
collating_for: Option<(CollatorId, parachain::Id)>,
max_block_data_size: Option<u64>,
authority_discovery_enabled: bool,
slot_duration: u64,
grandpa_pause: Option<(u32, u32)>,
test: bool,
) -> Result<(
@@ -275,10 +271,6 @@ pub fn new_full<RuntimeApi, Executor>(
let role = config.role.clone();
let is_authority = role.is_authority() && !is_collator;
let force_authoring = config.force_authoring;
let db_path = match config.database.path() {
Some(path) => std::path::PathBuf::from(path),
None => return Err("Starting a Polkadot service with a custom database isn't supported".to_string().into()),
};
let disable_grandpa = config.disable_grandpa;
let name = config.network.node_name.clone();
@@ -333,107 +325,16 @@ pub fn new_full<RuntimeApi, Executor>(
let shared_voter_state = rpc_setup;
let known_oracle = client.clone();
let mut handles = FullNodeHandles::default();
let gossip_validator_select_chain = select_chain.clone();
let is_known = move |block_hash: &Hash| {
use consensus_common::BlockStatus;
match known_oracle.block_status(&BlockId::hash(*block_hash)) {
Err(_) | Ok(BlockStatus::Unknown) | Ok(BlockStatus::Queued) => None,
Ok(BlockStatus::KnownBad) => Some(Known::Bad),
Ok(BlockStatus::InChainWithState) | Ok(BlockStatus::InChainPruned) => {
match gossip_validator_select_chain.leaves() {
Err(_) => None,
Ok(leaves) => if leaves.contains(block_hash) {
Some(Known::Leaf)
} else {
Some(Known::Old)
},
}
}
}
};
let polkadot_network_service = network_protocol::start(
network.clone(),
network_protocol::Config {
collating_for,
},
(is_known, client.clone()),
client.clone(),
task_manager.spawn_handle(),
).map_err(|e| format!("Could not spawn network worker: {:?}", e))?;
let authority_handles = if is_collator || role.is_authority() {
let availability_store = {
use std::path::PathBuf;
let mut path = PathBuf::from(db_path);
path.push("availability");
#[cfg(not(target_os = "unknown"))]
{
av_store::Store::new(
::av_store::Config {
cache_size: None,
path,
},
polkadot_network_service.clone(),
)?
}
#[cfg(target_os = "unknown")]
av_store::Store::new_in_memory(gossip)
};
polkadot_network_service.register_availability_store(availability_store.clone());
let (validation_service_handle, validation_service) = consensus::ServiceBuilder {
client: client.clone(),
network: polkadot_network_service.clone(),
collators: polkadot_network_service.clone(),
spawner: task_manager.spawn_handle(),
availability_store: availability_store.clone(),
select_chain: select_chain.clone(),
keystore: keystore.clone(),
max_block_data_size,
}.build();
task_manager.spawn_essential_handle().spawn("validation-service", Box::pin(validation_service));
handles.validation_service_handle = Some(validation_service_handle.clone());
Some((validation_service_handle, availability_store))
} else {
None
};
if role.is_authority() {
let (validation_service_handle, availability_store) = authority_handles
.clone()
.expect("Authority handles are set for authority nodes; qed");
let proposer = consensus::ProposerFactory::new(
client.clone(),
transaction_pool,
validation_service_handle,
slot_duration,
prometheus_registry.as_ref(),
);
let can_author_with =
consensus_common::CanAuthorWithNativeVersion::new(client.executor().clone());
let block_import = availability_store.block_import(
block_import,
client.clone(),
task_manager.spawn_handle(),
keystore.clone(),
)?;
let babe_config = babe::BabeParams {
keystore: keystore.clone(),
client: client.clone(),
@@ -556,8 +457,7 @@ pub fn new_full<RuntimeApi, Executor>(
network_starter.start_network();
handles.polkadot_network = Some(polkadot_network_service);
Ok((task_manager, client, handles, network, rpc_handlers))
Ok((task_manager, client, FullNodeHandles, network, rpc_handlers))
}
/// Builds a new service for a light client.
@@ -693,9 +593,7 @@ where
pub fn polkadot_new_full(
config: Configuration,
collating_for: Option<(CollatorId, parachain::Id)>,
max_block_data_size: Option<u64>,
authority_discovery_enabled: bool,
slot_duration: u64,
grandpa_pause: Option<(u32, u32)>,
)
-> Result<(
@@ -707,9 +605,7 @@ pub fn polkadot_new_full(
let (service, client, handles, _, _) = new_full::<polkadot_runtime::RuntimeApi, PolkadotExecutor>(
config,
collating_for,
max_block_data_size,
authority_discovery_enabled,
slot_duration,
grandpa_pause,
false,
)?;
@@ -722,9 +618,7 @@ pub fn polkadot_new_full(
pub fn kusama_new_full(
config: Configuration,
collating_for: Option<(CollatorId, parachain::Id)>,
max_block_data_size: Option<u64>,
authority_discovery_enabled: bool,
slot_duration: u64,
grandpa_pause: Option<(u32, u32)>,
) -> Result<(
TaskManager,
@@ -735,9 +629,7 @@ pub fn kusama_new_full(
let (service, client, handles, _, _) = new_full::<kusama_runtime::RuntimeApi, KusamaExecutor>(
config,
collating_for,
max_block_data_size,
authority_discovery_enabled,
slot_duration,
grandpa_pause,
false,
)?;
@@ -750,9 +642,7 @@ pub fn kusama_new_full(
pub fn westend_new_full(
config: Configuration,
collating_for: Option<(CollatorId, parachain::Id)>,
max_block_data_size: Option<u64>,
authority_discovery_enabled: bool,
slot_duration: u64,
grandpa_pause: Option<(u32, u32)>,
)
-> Result<(
@@ -764,9 +654,7 @@ pub fn westend_new_full(
let (service, client, handles, _, _) = new_full::<westend_runtime::RuntimeApi, WestendExecutor>(
config,
collating_for,
max_block_data_size,
authority_discovery_enabled,
slot_duration,
grandpa_pause,
false,
)?;
@@ -778,12 +666,7 @@ pub fn westend_new_full(
/// of the node may use.
#[cfg(feature = "full-node")]
#[derive(Default)]
pub struct FullNodeHandles {
/// A handle to the Polkadot networking protocol.
pub polkadot_network: Option<network_protocol::Service>,
/// A handle to the validation service.
pub validation_service_handle: Option<consensus::ServiceHandle>,
}
pub struct FullNodeHandles;
/// Build a new light node.
pub fn build_light(config: Configuration) -> Result<(TaskManager, RpcHandlers), ServiceError> {
@@ -801,18 +684,14 @@ pub fn build_light(config: Configuration) -> Result<(TaskManager, RpcHandlers),
pub fn build_full(
config: Configuration,
collating_for: Option<(CollatorId, parachain::Id)>,
max_block_data_size: Option<u64>,
authority_discovery_enabled: bool,
slot_duration: u64,
grandpa_pause: Option<(u32, u32)>,
) -> Result<(TaskManager, Client, FullNodeHandles), ServiceError> {
if config.chain_spec.is_kusama() {
new_full::<kusama_runtime::RuntimeApi, KusamaExecutor>(
config,
collating_for,
max_block_data_size,
authority_discovery_enabled,
slot_duration,
grandpa_pause,
false,
).map(|(task_manager, client, handles, _, _)| (task_manager, Client::Kusama(client), handles))
@@ -820,9 +699,7 @@ pub fn build_full(
new_full::<westend_runtime::RuntimeApi, WestendExecutor>(
config,
collating_for,
max_block_data_size,
authority_discovery_enabled,
slot_duration,
grandpa_pause,
false,
).map(|(task_manager, client, handles, _, _)| (task_manager, Client::Westend(client), handles))
@@ -830,9 +707,7 @@ pub fn build_full(
new_full::<polkadot_runtime::RuntimeApi, PolkadotExecutor>(
config,
collating_for,
max_block_data_size,
authority_discovery_enabled,
slot_duration,
grandpa_pause,
false,
).map(|(task_manager, client, handles, _, _)| (task_manager, Client::Polkadot(client), handles))
polkadot/validation/Cargo.toml
+11 -21
View File
@@ -5,37 +5,27 @@ authors = ["Parity Technologies <admin@parity.io>"]
edition = "2018"
[dependencies]
ansi_term = "0.12.1"
futures = "0.3.4"
futures-timer = "2.0"
parking_lot = "0.9.0"
tokio = { version = "0.2.13", features = ["rt-core", "blocking"] }
derive_more = "0.14.1"
log = "0.4.8"
exit-future = "0.2.0"
codec = { package = "parity-scale-codec", version = "1.3.4", default-features = false, features = ["derive"] }
availability_store = { package = "polkadot-availability-store", path = "../availability-store" }
parachain = { package = "polkadot-parachain", path = "../parachain" }
polkadot-primitives = { path = "../primitives" }
polkadot-erasure-coding = { path = "../erasure-coding" }
table = { package = "polkadot-statement-table", path = "../statement-table" }
parachain = { package = "polkadot-parachain", path = "../parachain" }
sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-timestamp = { git = "https://github.com/paritytech/substrate", branch = "master" }
sc-basic-authorship = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
consensus = { package = "sp-consensus", git = "https://github.com/paritytech/substrate", branch = "master" }
runtime_primitives = { package = "sp-runtime", git = "https://github.com/paritytech/substrate", branch = "master" }
futures = "0.3.4"
log = "0.4.8"
derive_more = "0.14.1"
codec = { package = "parity-scale-codec", version = "1.3.4", default-features = false, features = ["derive"] }
grandpa = { package = "sc-finality-grandpa", git = "https://github.com/paritytech/substrate", branch = "master" }
inherents = { package = "sp-inherents", git = "https://github.com/paritytech/substrate", branch = "master" }
consensus = { package = "sp-consensus", git = "https://github.com/paritytech/substrate", branch = "master" }
primitives = { package = "sp-core", git = "https://github.com/paritytech/substrate", branch = "master" }
txpool-api = { package = "sp-transaction-pool", git = "https://github.com/paritytech/substrate", branch = "master" }
sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
sc-client-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-timestamp = { git = "https://github.com/paritytech/substrate", branch = "master" }
block-builder = { package = "sc-block-builder", git = "https://github.com/paritytech/substrate", branch = "master" }
trie = { package = "sp-trie", git = "https://github.com/paritytech/substrate", branch = "master" }
runtime_primitives = { package = "sp-runtime", git = "https://github.com/paritytech/substrate", branch = "master" }
bitvec = { version = "0.17.4", default-features = false, features = ["alloc"] }
babe-primitives = { package = "sp-consensus-babe", git = "https://github.com/paritytech/substrate", branch = "master" }
keystore = { package = "sc-keystore", git = "https://github.com/paritytech/substrate", branch = "master" }
prometheus-endpoint = { package = "substrate-prometheus-endpoint", git = "https://github.com/paritytech/substrate", branch = "master" }
sc-basic-authorship = { git = "https://github.com/paritytech/substrate", branch = "master" }
[dev-dependencies]
sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" }
polkadot/validation/src/block_production.rs
+17 -82
View File
@@ -22,36 +22,25 @@
use std::{
pin::Pin,
sync::Arc,
time::{self, Duration, Instant},
time::Duration,
};
use sp_blockchain::HeaderBackend;
use block_builder::{BlockBuilderApi, BlockBuilderProvider};
use consensus::{Proposal, RecordProof};
use polkadot_primitives::v0::{Block, Header};
use polkadot_primitives::v0::{
ParachainHost, NEW_HEADS_IDENTIFIER,
};
use polkadot_primitives::v0::{NEW_HEADS_IDENTIFIER, Block, Header, AttestedCandidate};
use runtime_primitives::traits::{DigestFor, HashFor};
use futures_timer::Delay;
use txpool_api::TransactionPool;
use futures::prelude::*;
use inherents::InherentData;
use sp_timestamp::TimestampInherentData;
use sp_api::{ApiExt, ProvideRuntimeApi};
use prometheus_endpoint::Registry as PrometheusRegistry;
use crate::{
Error,
dynamic_inclusion::DynamicInclusion,
validation_service::ServiceHandle,
};
use crate::Error;
// Polkadot proposer factory.
pub struct ProposerFactory<Client, TxPool, Backend> {
service_handle: ServiceHandle,
babe_slot_duration: u64,
factory: sc_basic_authorship::ProposerFactory<TxPool, Backend, Client>,
}
@@ -60,8 +49,6 @@ impl<Client, TxPool, Backend> ProposerFactory<Client, TxPool, Backend> {
pub fn new(
client: Arc<Client>,
transaction_pool: Arc<TxPool>,
service_handle: ServiceHandle,
babe_slot_duration: u64,
prometheus: Option<&PrometheusRegistry>,
) -> Self {
let factory = sc_basic_authorship::ProposerFactory::new(
@@ -70,8 +57,6 @@ impl<Client, TxPool, Backend> ProposerFactory<Client, TxPool, Backend> {
prometheus,
);
ProposerFactory {
service_handle,
babe_slot_duration,
factory,
}
}
@@ -82,7 +67,7 @@ impl<Client, TxPool, Backend> consensus::Environment<Block>
where
TxPool: TransactionPool<Block=Block> + 'static,
Client: BlockBuilderProvider<Backend, Block, Client> + ProvideRuntimeApi<Block> + HeaderBackend<Block> + Send + Sync + 'static,
Client::Api: ParachainHost<Block> + BlockBuilderApi<Block>
Client::Api: BlockBuilderApi<Block>
+ ApiExt<Block, Error = sp_blockchain::Error>,
Backend: sc_client_api::Backend<
Block,
@@ -101,37 +86,24 @@ where
&mut self,
parent_header: &Header,
) -> Self::CreateProposer {
let parent_hash = parent_header.hash();
let slot_duration = self.babe_slot_duration.clone();
let proposer = self.factory.init(parent_header).into_inner();
let proposer = self.factory.init(parent_header)
.into_inner()
.map_err(Into::into)
.map(|proposer| Proposer { proposer });
let maybe_proposer = self.service_handle
.clone()
.get_validation_instance(parent_hash)
.and_then(move |tracker| future::ready(proposer
.map_err(Into::into)
.map(|proposer| Proposer {
tracker,
slot_duration,
proposer,
})
));
Box::pin(maybe_proposer)
Box::pin(future::ready(proposer))
}
}
/// The Polkadot proposer logic.
pub struct Proposer<Client, TxPool: TransactionPool<Block=Block>, Backend> {
tracker: crate::validation_service::ValidationInstanceHandle,
slot_duration: u64,
proposer: sc_basic_authorship::Proposer<Backend, Block, Client, TxPool>,
}
impl<Client, TxPool, Backend> consensus::Proposer<Block> for Proposer<Client, TxPool, Backend> where
TxPool: TransactionPool<Block=Block> + 'static,
Client: BlockBuilderProvider<Backend, Block, Client> + ProvideRuntimeApi<Block> + HeaderBackend<Block> + Send + Sync + 'static,
Client::Api: ParachainHost<Block> + BlockBuilderApi<Block> + ApiExt<Block, Error = sp_blockchain::Error>,
Client::Api: BlockBuilderApi<Block> + ApiExt<Block, Error = sp_blockchain::Error>,
Backend: sc_client_api::Backend<Block, State = sp_api::StateBackendFor<Client, Block>> + 'static,
// Rust bug: https://github.com/rust-lang/rust/issues/24159
sp_api::StateBackendFor<Client, Block>: sp_api::StateBackend<HashFor<Block>> + Send,
@@ -140,8 +112,11 @@ impl<Client, TxPool, Backend> consensus::Proposer<Block> for Proposer<Client, Tx
type Transaction = sp_api::TransactionFor<Client, Block>;
type Proposal = Pin<
Box<
dyn Future<Output = Result<Proposal<Block, sp_api::TransactionFor<Client, Block>>, Error>>
+ Send
dyn Future<Output = Result<
Proposal<Block, sp_api::TransactionFor<Client, Block>>,
Self::Error,
>>
+ Send
>
>;
@@ -152,57 +127,17 @@ impl<Client, TxPool, Backend> consensus::Proposer<Block> for Proposer<Client, Tx
max_duration: Duration,
record_proof: RecordProof,
) -> Self::Proposal {
const SLOT_DURATION_DENOMINATOR: u64 = 3; // wait up to 1/3 of the slot for candidates.
let initial_included = self.tracker.table().includable_count();
let now = Instant::now();
let dynamic_inclusion = DynamicInclusion::new(
self.tracker.table().num_parachains(),
self.tracker.started(),
Duration::from_millis(self.slot_duration / SLOT_DURATION_DENOMINATOR),
);
async move {
let enough_candidates = dynamic_inclusion.acceptable_in(
now,
initial_included,
).unwrap_or_else(|| Duration::from_millis(1));
let believed_timestamp = match inherent_data.timestamp_inherent_data() {
Ok(timestamp) => timestamp,
Err(e) => return Err(Error::InherentError(e)),
};
let deadline_diff = max_duration - max_duration / 3;
// set up delay until next allowed timestamp.
let current_timestamp = current_timestamp();
if current_timestamp < believed_timestamp {
Delay::new(Duration::from_millis(current_timestamp - believed_timestamp))
.await;
}
Delay::new(enough_candidates).await;
let proposed_candidates = self.tracker.table().proposed_set();
let mut inherent_data = inherent_data;
inherent_data.put_data(NEW_HEADS_IDENTIFIER, &proposed_candidates)
inherent_data.put_data(NEW_HEADS_IDENTIFIER, &Vec::<AttestedCandidate>::new())
.map_err(Error::InherentError)?;
self.proposer.propose(
inherent_data,
inherent_digests.clone(),
deadline_diff,
max_duration,
record_proof
).await.map_err(Into::into)
}.boxed()
}
}
fn current_timestamp() -> u64 {
time::SystemTime::now().duration_since(time::UNIX_EPOCH)
.expect("now always later than unix epoch; qed")
.as_millis() as u64
}
polkadot/validation/src/collation.rs
-117
View File
@@ -1,117 +0,0 @@
// Copyright 2017-2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Validator-side view of collation.
//!
//! This module contains type definitions, a trait for a batch of collators, and a trait for
//! attempting to fetch a collation repeatedly until a valid one is obtained.
use std::sync::Arc;
use polkadot_primitives::v0::{
BlakeTwo256, Block, Hash, HashT,
CollatorId, ParachainHost, Id as ParaId, Collation, ErasureChunk, CollationInfo,
};
use polkadot_erasure_coding as erasure;
use sp_api::ProvideRuntimeApi;
use futures::prelude::*;
use log::debug;
use primitives::traits::SpawnNamed;
/// Encapsulates connections to collators and allows collation on any parachain.
///
/// This is expected to be a lightweight, shared type like an `Arc`.
pub trait Collators: Clone {
/// Errors when producing collations.
type Error: std::fmt::Debug;
/// A full collation.
type Collation: Future<Output=Result<Collation, Self::Error>>;
/// Collate on a specific parachain, building on a given relay chain parent hash.
///
/// The returned collation should be checked for basic validity in the signature
/// and will be checked for state-transition validity by the consumer of this trait.
///
/// This does not have to guarantee local availability, as a valid collation
/// will be passed to the `TableRouter` instance.
///
/// The returned future may be prematurely concluded if the `relay_parent` goes
/// out of date.
fn collate(&self, parachain: ParaId, relay_parent: Hash) -> Self::Collation;
/// Note a bad collator. TODO: take proof (https://github.com/paritytech/polkadot/issues/217)
fn note_bad_collator(&self, collator: CollatorId);
}
/// A future which resolves when a collation is available.
pub async fn collation_fetch<C: Collators, P>(
validation_pool: Option<crate::pipeline::ValidationPool>,
parachain: ParaId,
relay_parent: Hash,
collators: C,
client: Arc<P>,
max_block_data_size: Option<u64>,
n_validators: usize,
spawner: impl SpawnNamed + Clone + 'static,
) -> Result<(CollationInfo, crate::pipeline::FullOutput), C::Error>
where
P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
C: Collators + Unpin,
P: ProvideRuntimeApi<Block>,
<C as Collators>::Collation: Unpin,
{
loop {
let collation = collators.collate(parachain, relay_parent).await?;
let Collation { info, pov } = collation;
let res = crate::pipeline::full_output_validation_with_api(
validation_pool.as_ref(),
&*client,
&info,
&pov,
&relay_parent,
max_block_data_size,
n_validators,
spawner.clone(),
);
match res {
Ok(full_output) => {
return Ok((info, full_output))
}
Err(e) => {
debug!("Failed to validate parachain due to API error: {}", e);
// just continue if we got a bad collation or failed to validate
collators.note_bad_collator(info.collator)
}
}
}
}
/// Validate an erasure chunk against an expected root.
pub fn validate_chunk(
root: &Hash,
chunk: &ErasureChunk,
) -> Result<(), ()> {
let expected = erasure::branch_hash(root, &chunk.proof, chunk.index as usize).map_err(|_| ())?;
let got = BlakeTwo256::hash(&chunk.chunk);
if expected != got {
return Err(())
}
Ok(())
}
polkadot/validation/src/dynamic_inclusion.rs
-130
View File
@@ -1,130 +0,0 @@
// Copyright 2017-2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Dynamic inclusion threshold over time.
use std::time::{Duration, Instant};
fn duration_to_micros(duration: &Duration) -> u64 {
duration.as_secs() * 1_000_000 + (duration.subsec_nanos() / 1000) as u64
}
/// Dynamic inclusion threshold over time.
///
/// The acceptable proportion of parachains which must have parachain candidates
/// reduces over time (eventually going to zero).
#[derive(Debug, Clone)]
pub struct DynamicInclusion {
start: Instant,
y: u64,
m: u64,
}
impl DynamicInclusion {
/// Constructs a new dynamic inclusion threshold calculator based on the time now,
/// how many parachain candidates are required at the beginning, and when an empty
/// block will be allowed.
pub fn new(initial: usize, start: Instant, allow_empty: Duration) -> Self {
// linear function f(n_candidates) -> valid after microseconds
// f(0) = allow_empty
// f(initial) = 0
// m is actually the negative slope to avoid using signed arithmetic.
let (y, m) = if initial != 0 {
let y = duration_to_micros(&allow_empty);
(y, y / initial as u64)
} else {
(0, 0)
};
DynamicInclusion {
start,
y,
m,
}
}
/// Returns the duration from `now` after which the amount of included parachain candidates
/// would be enough, or `None` if it is sufficient now.
///
/// Panics if `now` is earlier than the `start`.
pub fn acceptable_in(&self, now: Instant, included: usize) -> Option<Duration> {
let elapsed = now.duration_since(self.start);
let elapsed = duration_to_micros(&elapsed);
let valid_after = self.y.saturating_sub(self.m * included as u64);
if elapsed >= valid_after {
None
} else {
Some(Duration::from_millis((valid_after - elapsed) as u64 / 1000))
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn full_immediately_allowed() {
let now = Instant::now();
let dynamic = DynamicInclusion::new(
10,
now,
Duration::from_millis(4000),
);
assert!(dynamic.acceptable_in(now, 10).is_none());
assert!(dynamic.acceptable_in(now, 11).is_none());
assert!(dynamic.acceptable_in(now + Duration::from_millis(2000), 10).is_none());
}
#[test]
fn half_allowed_halfway() {
let now = Instant::now();
let dynamic = DynamicInclusion::new(
10,
now,
Duration::from_millis(4000),
);
assert_eq!(dynamic.acceptable_in(now, 5), Some(Duration::from_millis(2000)));
assert!(dynamic.acceptable_in(now + Duration::from_millis(2000), 5).is_none());
assert!(dynamic.acceptable_in(now + Duration::from_millis(3000), 5).is_none());
assert!(dynamic.acceptable_in(now + Duration::from_millis(4000), 5).is_none());
}
#[test]
fn zero_initial_is_flat() {
let now = Instant::now();
let dynamic = DynamicInclusion::new(
0,
now,
Duration::from_secs(10_000),
);
for i in 0..10_001 {
let now = now + Duration::from_secs(i);
assert!(dynamic.acceptable_in(now, 0).is_none());
assert!(dynamic.acceptable_in(now, 1).is_none());
assert!(dynamic.acceptable_in(now, 10).is_none());
}
}
}
polkadot/validation/src/error.rs
-62
View File
@@ -16,8 +16,6 @@
//! Errors that can occur during the validation process.
use polkadot_primitives::v0::{ValidatorId, Hash};
/// Error type for validation
#[derive(Debug, derive_more::Display, derive_more::From)]
pub enum Error {
@@ -25,66 +23,9 @@ pub enum Error {
Client(sp_blockchain::Error),
/// Consensus error
Consensus(consensus::error::Error),
/// A wasm-validation error.
WasmValidation(parachain::wasm_executor::ValidationError),
/// An I/O error.
Io(std::io::Error),
/// An error in the availability erasure-coding.
ErasureCoding(polkadot_erasure_coding::Error),
#[display(fmt = "Invalid duty roster length: expected {}, got {}", expected, got)]
InvalidDutyRosterLength {
/// Expected roster length
expected: usize,
/// Actual roster length
got: usize,
},
/// Local account not a validator at this block
#[display(fmt = "Local account ID ({:?}) not a validator at this block.", _0)]
NotValidator(ValidatorId),
/// Unexpected error checking inherents
#[display(fmt = "Unexpected error while checking inherents: {}", _0)]
InherentError(inherents::Error),
/// Proposer destroyed before finishing proposing or evaluating
#[display(fmt = "Proposer destroyed before finishing proposing or evaluating")]
PrematureDestruction,
/// Failed to build the table router.
#[display(fmt = "Failed to build the table router: {}", _0)]
CouldNotBuildTableRouter(String),
/// Timer failed
#[display(fmt = "Timer failed: {}", _0)]
Timer(std::io::Error),
#[display(fmt = "Failed to compute deadline of now + {:?}", _0)]
DeadlineComputeFailure(std::time::Duration),
#[display(fmt = "Validation service is down.")]
ValidationServiceDown,
/// PoV-block in collation doesn't match provided.
#[display(fmt = "PoV hash mismatch. Expected {:?}, got {:?}", _0, _1)]
PoVHashMismatch(Hash, Hash),
/// Collator signature is invalid.
#[display(fmt = "Invalid collator signature on collation")]
InvalidCollatorSignature,
/// Head-data too large.
#[display(fmt = "Head data size of {} exceeded maximum of {}", _0, _1)]
HeadDataTooLarge(usize, usize),
/// Head-data mismatch after validation.
#[display(fmt = "Validation produced a different parachain header")]
HeadDataMismatch,
/// Relay parent of candidate not allowed.
#[display(fmt = "Relay parent {} of candidate not allowed in this context.", _0)]
DisallowedRelayParent(Hash),
/// Commitments in candidate match commitments produced by validation.
#[display(fmt = "Commitments in candidate receipt do not match those produced by validation")]
CommitmentsMismatch,
/// The parachain for which validation work is being done is not active.
#[display(fmt = "Parachain {:?} is not active", _0)]
InactiveParachain(polkadot_primitives::v0::Id),
/// Block data is too big
#[display(fmt = "Block data is too big (maximum allowed size: {}, actual size: {})", size, max_size)]
BlockDataTooBig { size: u64, max_size: u64 },
Join(tokio::task::JoinError),
/// Could not cover fee for an operation e.g. for sending `UpwardMessage`.
#[display(fmt = "Parachain could not cover fee for an operation e.g. for sending an `UpwardMessage`.")]
CouldNotCoverFee,
}
impl std::error::Error for Error {
@@ -92,9 +33,6 @@ impl std::error::Error for Error {
match self {
Error::Client(ref err) => Some(err),
Error::Consensus(ref err) => Some(err),
Error::WasmValidation(ref err) => Some(err),
Error::ErasureCoding(ref err) => Some(err),
Error::Io(ref err) => Some(err),
_ => None,
}
}
polkadot/validation/src/lib.rs
-197
View File
@@ -29,207 +29,10 @@
//!
//! Groups themselves may be compromised by malicious authorities.
use std::{
collections::{HashMap, HashSet},
sync::Arc,
};
use codec::Encode;
use polkadot_primitives::v0::{
Id as ParaId, Chain, DutyRoster, AbridgedCandidateReceipt,
CompactStatement as PrimitiveStatement,
PoVBlock, ErasureChunk, ValidatorSignature, ValidatorIndex,
ValidatorPair, ValidatorId, SigningContext,
};
use primitives::Pair;
use futures::prelude::*;
pub use self::block_production::ProposerFactory;
pub use self::collation::Collators;
pub use self::error::Error;
pub use self::shared_table::{
SharedTable, ParachainWork, PrimedParachainWork, Validated, Statement, SignedStatement,
GenericStatement,
};
pub use self::validation_service::{ServiceHandle, ServiceBuilder};
pub use parachain::wasm_executor::run_worker as run_validation_worker;
mod dynamic_inclusion;
mod error;
mod shared_table;
pub mod block_production;
pub mod collation;
pub mod pipeline;
pub mod validation_service;
/// A handle to a statement table router.
///
/// This is expected to be a lightweight, shared type like an `Arc`.
/// Once all instances are dropped, consensus networking for this router
/// should be cleaned up.
pub trait TableRouter: Clone {
/// Errors when fetching data from the network.
type Error: std::fmt::Debug;
/// Future that drives sending of the local collation to the network.
type SendLocalCollation: Future<Output=Result<(), Self::Error>>;
/// Future that resolves when candidate data is fetched.
type FetchValidationProof: Future<Output=Result<PoVBlock, Self::Error>>;
/// Call with local candidate data. This will sign, import, and broadcast a statement about the candidate.
fn local_collation(
&self,
receipt: AbridgedCandidateReceipt,
pov_block: PoVBlock,
chunks: (ValidatorIndex, &[ErasureChunk]),
) -> Self::SendLocalCollation;
/// Fetch validation proof for a specific candidate.
///
/// This future must conclude once all `Clone`s of this `TableRouter` have
/// been cleaned up.
fn fetch_pov_block(&self, candidate: &AbridgedCandidateReceipt) -> Self::FetchValidationProof;
}
/// A long-lived network which can create parachain statement and BFT message routing processes on demand.
pub trait Network {
/// The error type of asynchronously building the table router.
type Error: std::fmt::Debug;
/// The table router type. This should handle importing of any statements,
/// routing statements to peers, and driving completion of any `StatementProducers`.
type TableRouter: TableRouter;
/// The future used for asynchronously building the table router.
/// This should not fail.
type BuildTableRouter: Future<Output=Result<Self::TableRouter,Self::Error>>;
/// Instantiate a table router using the given shared table.
/// Also pass through any outgoing messages to be broadcast to peers.
#[must_use]
fn build_table_router(
&self,
table: Arc<SharedTable>,
authorities: &[ValidatorId],
) -> Self::BuildTableRouter;
}
/// The local duty of a validator.
#[derive(Debug)]
pub struct LocalDuty {
validation: Chain,
index: ValidatorIndex,
}
/// Information about a specific group.
#[derive(Debug, Clone, Default)]
pub struct GroupInfo {
/// Authorities meant to check validity of candidates.
validity_guarantors: HashSet<ValidatorId>,
/// Number of votes needed for validity.
needed_validity: usize,
}
/// Sign a table statement against a parent hash.
/// The actual message signed is the encoded statement concatenated with the
/// parent hash.
pub fn sign_table_statement(
statement: &Statement,
key: &ValidatorPair,
signing_context: &SigningContext,
) -> ValidatorSignature {
let mut encoded = PrimitiveStatement::from(statement).encode();
encoded.extend(signing_context.encode());
key.sign(&encoded)
}
/// Check signature on table statement.
pub fn check_statement(
statement: &Statement,
signature: &ValidatorSignature,
signer: ValidatorId,
signing_context: &SigningContext,
) -> bool {
use runtime_primitives::traits::AppVerify;
let mut encoded = PrimitiveStatement::from(statement).encode();
encoded.extend(signing_context.encode());
signature.verify(&encoded[..], &signer)
}
/// Compute group info out of a duty roster and a local authority set.
pub fn make_group_info(
roster: DutyRoster,
authorities: &[ValidatorId],
local_id: Option<ValidatorId>,
) -> Result<(HashMap<ParaId, GroupInfo>, Option<LocalDuty>), Error> {
if roster.validator_duty.len() != authorities.len() {
return Err(Error::InvalidDutyRosterLength {
expected: authorities.len(),
got: roster.validator_duty.len()
});
}
let mut local_validation = None;
let mut local_index = 0;
let mut map = HashMap::new();
let duty_iter = authorities.iter().zip(&roster.validator_duty);
for (i, (authority, v_duty)) in duty_iter.enumerate() {
if Some(authority) == local_id.as_ref() {
local_validation = Some(v_duty.clone());
local_index = i;
}
match *v_duty {
Chain::Relay => {}, // does nothing for now.
Chain::Parachain(ref id) => {
map.entry(id.clone()).or_insert_with(GroupInfo::default)
.validity_guarantors
.insert(authority.clone());
}
}
}
for live_group in map.values_mut() {
let validity_len = live_group.validity_guarantors.len();
live_group.needed_validity = validity_len / 2 + validity_len % 2;
}
let local_duty = local_validation.map(|v| LocalDuty {
validation: v,
index: local_index as u32,
});
Ok((map, local_duty))
}
#[cfg(test)]
mod tests {
use super::*;
use sp_keyring::Sr25519Keyring;
#[test]
fn sign_and_check_statement() {
let statement: Statement = GenericStatement::Valid([1; 32].into());
let parent_hash = [2; 32].into();
let signing_context = SigningContext {
session_index: Default::default(),
parent_hash,
};
let sig = sign_table_statement(&statement, &Sr25519Keyring::Alice.pair().into(), &signing_context);
let wrong_signing_context = SigningContext {
session_index: Default::default(),
parent_hash: [0xff; 32].into(),
};
assert!(check_statement(&statement, &sig, Sr25519Keyring::Alice.public().into(), &signing_context));
assert!(!check_statement(&statement, &sig, Sr25519Keyring::Alice.public().into(), &wrong_signing_context));
assert!(!check_statement(&statement, &sig, Sr25519Keyring::Bob.public().into(), &signing_context));
}
}
polkadot/validation/src/pipeline.rs
-362
View File
@@ -1,362 +0,0 @@
// Copyright 2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! The pipeline of validation functions a parachain block must pass through before
//! it can be voted for.
use codec::Encode;
use polkadot_erasure_coding as erasure;
use polkadot_primitives::v0::{
CollationInfo, PoVBlock, LocalValidationData, GlobalValidationData, OmittedValidationData,
AvailableData, FeeSchedule, CandidateCommitments, ErasureChunk, ParachainHost,
Id as ParaId, AbridgedCandidateReceipt, ValidationCode,
};
use polkadot_primitives::v0::{Block, BlockId, Balance, Hash};
use parachain::{
wasm_executor::{self, ExecutionMode},
primitives::{UpwardMessage, ValidationParams},
};
use runtime_primitives::traits::{BlakeTwo256, Hash as HashT};
use sp_api::ProvideRuntimeApi;
use crate::Error;
use primitives::traits::SpawnNamed;
pub use parachain::wasm_executor::ValidationPool;
/// Does basic checks of a collation. Provide the encoded PoV-block.
pub fn basic_checks(
collation: &CollationInfo,
expected_relay_parent: &Hash,
max_block_data_size: Option<u64>,
encoded_pov: &[u8],
) -> Result<(), Error> {
if &collation.relay_parent != expected_relay_parent {
return Err(Error::DisallowedRelayParent(collation.relay_parent));
}
if let Some(max_size) = max_block_data_size {
if encoded_pov.len() as u64 > max_size {
return Err(Error::BlockDataTooBig { size: encoded_pov.len() as _, max_size });
}
}
let hash = BlakeTwo256::hash(encoded_pov);
if hash != collation.pov_block_hash {
return Err(Error::PoVHashMismatch(collation.pov_block_hash, hash));
}
if let Err(()) = collation.check_signature() {
return Err(Error::InvalidCollatorSignature);
}
Ok(())
}
/// Data from a fully-outputted validation of a parachain candidate. This contains
/// all outputs and commitments of the validation as well as all additional data to make available.
pub struct FullOutput {
/// Data about the candidate to keep available in the network.
pub available_data: AvailableData,
/// Commitments issued alongside the candidate to be placed on-chain.
pub commitments: CandidateCommitments,
/// All erasure-chunks associated with the available data. Each validator
/// should keep their chunk (by index). Other chunks do not need to be
/// kept available long-term, but should be distributed to other validators.
pub erasure_chunks: Vec<ErasureChunk>,
/// The number of validators that were present at this validation.
pub n_validators: usize,
}
impl FullOutput {
/// Check consistency of the outputs produced by the validation pipeline against
/// data contained within a candidate receipt.
pub fn check_consistency(&self, receipt: &AbridgedCandidateReceipt) -> Result<(), Error> {
if self.commitments != receipt.commitments {
Err(Error::CommitmentsMismatch)
} else {
Ok(())
}
}
}
/// The successful result of validating a collation. If the full commitments of the
/// validation are needed, call `full_output`. Otherwise, safely drop this value.
pub struct ValidatedCandidate<'a> {
pov_block: &'a PoVBlock,
global_validation: &'a GlobalValidationData,
local_validation: &'a LocalValidationData,
upward_messages: Vec<UpwardMessage>,
fees: Balance,
processed_downward_messages: u32,
}
impl<'a> ValidatedCandidate<'a> {
/// Fully-compute the commitments and outputs of the candidate. Provide the number
/// of validators. This computes the erasure-coding.
pub fn full_output(self, n_validators: usize) -> Result<FullOutput, Error> {
let ValidatedCandidate {
pov_block,
global_validation,
local_validation,
upward_messages,
fees,
processed_downward_messages,
} = self;
let omitted_validation = OmittedValidationData {
global_validation: global_validation.clone(),
local_validation: local_validation.clone(),
};
let available_data = AvailableData {
pov_block: pov_block.clone(),
omitted_validation,
};
let erasure_chunks = erasure::obtain_chunks_v0(
n_validators,
&available_data,
)?;
let branches = erasure::branches(erasure_chunks.as_ref());
let erasure_root = branches.root();
let chunks: Vec<_> = erasure_chunks
.iter()
.zip(branches.map(|(proof, _)| proof))
.enumerate()
.map(|(index, (chunk, proof))| ErasureChunk {
// branches borrows the original chunks, but this clone could probably be dodged.
chunk: chunk.clone(),
index: index as u32,
proof,
})
.collect();
let commitments = CandidateCommitments {
upward_messages,
fees,
erasure_root,
new_validation_code: None,
processed_downward_messages,
};
Ok(FullOutput {
available_data,
commitments,
erasure_chunks: chunks,
n_validators,
})
}
}
/// Validate that the given `UpwardMessage`s are covered by the given `free_balance`.
///
/// Will return an error if the `free_balance` does not cover the required fees to the
/// given `msgs`. On success it returns the fees that need to be charged for the `msgs`.
fn validate_upward_messages(
msgs: &[UpwardMessage],
fee_schedule: FeeSchedule,
free_balance: Balance,
) -> Result<Balance, Error> {
msgs.iter().try_fold(Balance::from(0u128), |fees_charged, msg| {
let fees = fee_schedule.compute_message_fee(msg.data.len());
let fees_charged = fees_charged.saturating_add(fees);
if fees_charged > free_balance {
Err(Error::CouldNotCoverFee)
} else {
Ok(fees_charged)
}
})
}
/// Does full checks of a collation, with provided PoV-block and contextual data.
pub fn validate<'a>(
validation_pool: Option<&'_ ValidationPool>,
collation: &'a CollationInfo,
pov_block: &'a PoVBlock,
local_validation: &'a LocalValidationData,
global_validation: &'a GlobalValidationData,
validation_code: &ValidationCode,
spawner: impl SpawnNamed + 'static,
) -> Result<ValidatedCandidate<'a>, Error> {
if collation.head_data.0.len() > global_validation.max_head_data_size as _ {
return Err(Error::HeadDataTooLarge(
collation.head_data.0.len(),
global_validation.max_head_data_size as _,
));
}
let params = ValidationParams {
parent_head: local_validation.parent_head.clone(),
block_data: pov_block.block_data.clone(),
relay_chain_height: global_validation.block_number,
hrmp_mqc_heads: Vec::new(),
};
// TODO: remove when ext does not do this.
let fee_schedule = FeeSchedule {
base: 0,
per_byte: 0,
};
let execution_mode = validation_pool
.map(ExecutionMode::Remote)
.unwrap_or(ExecutionMode::Local);
match wasm_executor::validate_candidate(
&validation_code.0,
params,
execution_mode,
spawner,
) {
Ok(result) => {
if result.head_data == collation.head_data {
let fees = validate_upward_messages(
&result.upward_messages,
fee_schedule,
local_validation.balance,
)?;
Ok(ValidatedCandidate {
pov_block,
global_validation,
local_validation,
upward_messages: result.upward_messages,
fees,
processed_downward_messages: result.processed_downward_messages,
})
} else {
Err(Error::HeadDataMismatch)
}
}
Err(e) => Err(e.into()),
}
}
/// Extracts validation parameters from a Polkadot runtime API for a specific parachain.
pub fn validation_params<P>(api: &P, relay_parent: Hash, para_id: ParaId)
-> Result<(LocalValidationData, GlobalValidationData, ValidationCode), Error>
where
P: ProvideRuntimeApi<Block>,
P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
{
let api = api.runtime_api();
let relay_parent = BlockId::hash(relay_parent);
// fetch all necessary data from runtime.
let local_validation = api.local_validation_data(&relay_parent, para_id)?
.ok_or_else(|| Error::InactiveParachain(para_id))?;
let global_validation = api.global_validation_data(&relay_parent)?;
let validation_code = api.parachain_code(&relay_parent, para_id)?
.ok_or_else(|| Error::InactiveParachain(para_id))?;
Ok((local_validation, global_validation, validation_code))
}
/// Does full-pipeline validation of a collation with provided contextual parameters.
pub fn full_output_validation_with_api<P>(
validation_pool: Option<&ValidationPool>,
api: &P,
collation: &CollationInfo,
pov_block: &PoVBlock,
expected_relay_parent: &Hash,
max_block_data_size: Option<u64>,
n_validators: usize,
spawner: impl SpawnNamed + 'static,
) -> Result<FullOutput, Error> where
P: ProvideRuntimeApi<Block>,
P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
{
let para_id = collation.parachain_index;
let (local_validation, global_validation, validation_code)
= validation_params(&*api, collation.relay_parent, para_id)?;
// put the parameters through the validation pipeline, producing
// erasure chunks.
let encoded_pov = pov_block.encode();
basic_checks(
&collation,
&expected_relay_parent,
max_block_data_size,
&encoded_pov,
)
.and_then(|()| {
let res = validate(
validation_pool,
&collation,
&pov_block,
&local_validation,
&global_validation,
&validation_code,
spawner,
);
match res {
Err(ref err) => log::debug!(
target: "validation",
"Failed to validate PoVBlock for parachain ({}): {:?}",
para_id,
err,
),
Ok(_) => log::debug!(
target: "validation",
"Successfully validated PoVBlock for parachain ({}).",
para_id,
),
}
res
})
.and_then(|validated| validated.full_output(n_validators))
}
#[cfg(test)]
mod tests {
use super::*;
use parachain::primitives::ParachainDispatchOrigin;
fn add_msg(size: usize, msgs: &mut Vec<UpwardMessage>) {
let msg = UpwardMessage { data: vec![0; size], origin: ParachainDispatchOrigin::Parachain };
msgs.push(msg);
}
#[test]
fn validate_upward_messages_works() {
let fee_schedule = FeeSchedule {
base: 1000,
per_byte: 10,
};
let free_balance = 1_000_000;
let mut msgs = Vec::new();
add_msg(100, &mut msgs);
assert_eq!(2000, validate_upward_messages(&msgs, fee_schedule, free_balance).unwrap());
add_msg(100, &mut msgs);
assert_eq!(4000, validate_upward_messages(&msgs, fee_schedule, free_balance).unwrap());
add_msg((1_000_000 - 4000 - 1000) / 10, &mut msgs);
assert_eq!(1_000_000, validate_upward_messages(&msgs, fee_schedule, free_balance).unwrap());
// cannot pay fee.
add_msg(1, &mut msgs);
let err = validate_upward_messages(&msgs, fee_schedule, free_balance).unwrap_err();
assert!(matches!(err, Error::CouldNotCoverFee));
}
}
polkadot/validation/src/shared_table/includable.rs
-122
View File
@@ -1,122 +0,0 @@
// Copyright 2017-2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Implements a future which resolves when all of the candidates referenced are includable.
use std::collections::HashMap;
use futures::channel::oneshot;
use polkadot_primitives::v0::Hash;
/// Track includability of a set of candidates,
pub(super) fn track<I: IntoIterator<Item=(Hash, bool)>>(candidates: I)
-> (IncludabilitySender, oneshot::Receiver<()>) {
let (tx, rx) = oneshot::channel();
let tracking: HashMap<_, _> = candidates.into_iter().collect();
let includable_count = tracking.values().filter(|x| **x).count();
let mut sender = IncludabilitySender {
tracking,
includable_count,
sender: Some(tx),
};
sender.try_complete();
(sender, rx)
}
/// The sending end of the includability sender.
pub(super) struct IncludabilitySender {
tracking: HashMap<Hash, bool>,
includable_count: usize,
sender: Option<oneshot::Sender<()>>,
}
impl IncludabilitySender {
/// update the inner candidate. wakes up the task as necessary.
/// returns `Err(Canceled)` if the other end has hung up.
///
/// returns `true` when this is completed and should be destroyed.
pub fn update_candidate(&mut self, candidate: Hash, includable: bool) -> bool {
use std::collections::hash_map::Entry;
match self.tracking.entry(candidate) {
Entry::Vacant(_) => {}
Entry::Occupied(mut entry) => {
let old = entry.insert(includable);
if !old && includable {
self.includable_count += 1;
} else if old && !includable {
self.includable_count -= 1;
}
}
}
self.try_complete()
}
/// whether the sender is completed.
pub fn is_complete(&self) -> bool {
self.sender.is_none()
}
fn try_complete(&mut self) -> bool {
if self.includable_count == self.tracking.len() {
if let Some(sender) = self.sender.take() {
let _ = sender.send(());
}
true
} else {
false
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use futures::executor::block_on;
#[test]
fn it_works() {
let hash1 = [1; 32].into();
let hash2 = [2; 32].into();
let hash3 = [3; 32].into();
let (mut sender, recv) = track([
(hash1, true),
(hash2, true),
(hash2, false), // overwrite should favor latter.
(hash3, true),
].iter().cloned());
assert!(!sender.is_complete());
// true -> false transition is possible and should be handled.
sender.update_candidate(hash1, false);
assert!(!sender.is_complete());
sender.update_candidate(hash2, true);
assert!(!sender.is_complete());
sender.update_candidate(hash1, true);
assert!(sender.is_complete());
block_on(recv).unwrap();
}
}
polkadot/validation/src/shared_table/mod.rs
-1045
View File
File diff suppressed because it is too large Load Diff
polkadot/validation/src/validation_service/mod.rs
-802
View File
@@ -1,802 +0,0 @@
// Copyright 2017-2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! The validation service is a long-running future that creates and manages parachain attestation
//! instances.
//!
//! As soon as we import a new chain head, we start a parachain attestation session on top of it.
//! The block authorship service may want access to the attestation session, and for that reason
//! we expose a `ServiceHandle` which can be used to request a copy of it.
//!
//! In fact, the import notification and request from the block production pipeline may race to be
//! the first one to create the instant, but the import notification will usually win.
//!
//! These attestation sessions are kept live until they are periodically garbage-collected.
use std::{time::{Duration, Instant}, sync::Arc, pin::Pin, collections::HashMap};
use crate::pipeline::FullOutput;
use sc_client_api::{BlockchainEvents, BlockBackend};
use consensus::SelectChain;
use futures::prelude::*;
use polkadot_primitives::v0::{
Block, Hash, BlockId,
Chain, ParachainHost, Id as ParaId, ValidatorIndex, ValidatorId, ValidatorPair,
CollationInfo, SigningContext,
};
use keystore::KeyStorePtr;
use sp_api::{ProvideRuntimeApi, ApiExt};
use runtime_primitives::traits::HashFor;
use availability_store::Store as AvailabilityStore;
use primitives::traits::SpawnNamed;
use ansi_term::Colour;
use log::{warn, info, debug, trace};
use super::{Network, Collators, SharedTable, TableRouter};
use crate::Error;
use crate::pipeline::ValidationPool;
// Remote processes may request for a validation instance to be cloned or instantiated.
// They send a oneshot channel.
type ValidationInstanceRequest = (
Hash,
futures::channel::oneshot::Sender<Result<ValidationInstanceHandle, Error>>,
);
/// A handle to a single instance of parachain validation, which is pinned to
/// a specific relay-chain block. This is the instance that should be used when
/// constructing any
#[derive(Clone)]
pub(crate) struct ValidationInstanceHandle {
table: Arc<SharedTable>,
started: Instant,
}
impl ValidationInstanceHandle {
/// Access the underlying table of attestations on parachain candidates.
pub(crate) fn table(&self) -> &Arc<SharedTable> {
&self.table
}
/// The moment we started this validation instance.
pub(crate) fn started(&self) -> Instant {
self.started.clone()
}
}
/// A handle to the service. This can be used to create a block-production environment.
#[derive(Clone)]
pub struct ServiceHandle {
sender: futures::channel::mpsc::Sender<ValidationInstanceRequest>,
}
impl ServiceHandle {
/// Requests instantiation or cloning of a validation instance from the service.
///
/// This can fail if the service task has shut down for some reason.
pub(crate) async fn get_validation_instance(self, relay_parent: Hash)
-> Result<ValidationInstanceHandle, Error>
{
let mut sender = self.sender;
let instance_rx = loop {
let (instance_tx, instance_rx) = futures::channel::oneshot::channel();
match sender.send((relay_parent, instance_tx)).await {
Ok(()) => break instance_rx,
Err(e) => if !e.is_full() {
// Sink::send should be doing `poll_ready` before start-send,
// so this should only happen when there is a race.
return Err(Error::ValidationServiceDown)
},
}
};
instance_rx.map_err(|_| Error::ValidationServiceDown).await.and_then(|x| x)
}
}
fn interval(duration: Duration) -> impl Stream<Item=()> + Send + Unpin {
stream::unfold((), move |_| {
futures_timer::Delay::new(duration).map(|_| Some(((), ())))
}).map(drop)
}
/// A builder for the validation service.
pub struct ServiceBuilder<C, N, P, SC, SP> {
/// The underlying blockchain client.
pub client: Arc<P>,
/// A handle to the network object used to communicate.
pub network: N,
/// A handle to the collator pool we are using.
pub collators: C,
/// A handle to a background executor.
pub spawner: SP,
/// A handle to the availability store.
pub availability_store: AvailabilityStore,
/// A chain selector for determining active leaves in the block-DAG.
pub select_chain: SC,
/// The keystore which holds the signing keys.
pub keystore: KeyStorePtr,
/// The maximum block-data size in bytes.
pub max_block_data_size: Option<u64>,
}
impl<C, N, P, SC, SP> ServiceBuilder<C, N, P, SC, SP> where
C: Collators + Send + Sync + Unpin + 'static,
C::Collation: Send + Unpin + 'static,
P: BlockchainEvents<Block> + BlockBackend<Block>,
P: ProvideRuntimeApi<Block> + Send + Sync + 'static,
P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
N: Network + Send + Sync + 'static,
N::TableRouter: Send + 'static + Sync,
N::BuildTableRouter: Send + Unpin + 'static,
<N::TableRouter as TableRouter>::SendLocalCollation: Send,
SC: SelectChain<Block> + 'static,
SP: SpawnNamed + Clone + 'static,
// Rust bug: https://github.com/rust-lang/rust/issues/24159
sp_api::StateBackendFor<P, Block>: sp_api::StateBackend<HashFor<Block>>,
{
/// Build the service - this consists of a handle to it, as well as a background
/// future to be run to completion.
pub fn build(self) -> (ServiceHandle, impl Future<Output = ()> + Send + 'static) {
const TIMER_INTERVAL: Duration = Duration::from_secs(30);
const CHAN_BUFFER: usize = 10;
enum Message {
CollectGarbage,
// relay-parent, receiver for instance.
RequestInstance(ValidationInstanceRequest),
// new chain heads - import notification.
NotifyImport(sc_client_api::BlockImportNotification<Block>),
}
let validation_pool = Some(ValidationPool::new());
let mut parachain_validation = ParachainValidationInstances {
client: self.client.clone(),
network: self.network,
spawner: self.spawner.clone(),
availability_store: self.availability_store,
live_instances: HashMap::new(),
validation_pool: validation_pool.clone(),
collation_fetch: DefaultCollationFetch {
collators: self.collators,
validation_pool,
spawner: self.spawner,
},
};
let client = self.client;
let select_chain = self.select_chain;
let keystore = self.keystore;
let max_block_data_size = self.max_block_data_size;
let (tx, rx) = futures::channel::mpsc::channel(CHAN_BUFFER);
let interval = interval(TIMER_INTERVAL).map(|_| Message::CollectGarbage);
let import_notifications = client.import_notification_stream().map(Message::NotifyImport);
let instance_requests = rx.map(Message::RequestInstance);
let service = ServiceHandle { sender: tx };
let background_work = async move {
let message_stream = futures::stream::select(interval, instance_requests);
let mut message_stream = futures::stream::select(import_notifications, message_stream);
while let Some(message) = message_stream.next().await {
match message {
Message::CollectGarbage => {
match select_chain.leaves() {
Ok(leaves) => {
parachain_validation.retain(|h| leaves.contains(h));
}
Err(e) => {
warn!("Error fetching leaves from client: {:?}", e);
}
}
}
Message::RequestInstance((relay_parent, sender)) => {
// Upstream will handle the failure case.
let _ = sender.send(parachain_validation.get_or_instantiate(
relay_parent,
&keystore,
max_block_data_size,
).await);
}
Message::NotifyImport(notification) => {
let relay_parent = notification.hash;
if notification.is_new_best {
let res = parachain_validation.get_or_instantiate(
relay_parent,
&keystore,
max_block_data_size,
).await;
if let Err(e) = res {
warn!(
"Unable to start parachain validation on top of {:?}: {}",
relay_parent, e
);
}
}
}
}
}
};
(service, background_work)
}
}
/// Abstraction over `collation_fetch`.
pub(crate) trait CollationFetch {
/// Error type used by `collation_fetch`.
type Error: std::fmt::Debug;
/// Fetch a collation for the given `parachain`.
fn collation_fetch<P>(
self,
parachain: ParaId,
relay_parent: Hash,
client: Arc<P>,
max_block_data_size: Option<u64>,
n_validators: usize,
) -> Pin<Box<dyn Future<Output = Result<(CollationInfo, FullOutput), Self::Error>> + Send>>
where
P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
P: ProvideRuntimeApi<Block> + Send + Sync + 'static;
}
#[derive(Clone)]
struct DefaultCollationFetch<C, S> {
collators: C,
validation_pool: Option<ValidationPool>,
spawner: S,
}
impl<C, S> CollationFetch for DefaultCollationFetch<C, S>
where
C: Collators + Send + Sync + Unpin + 'static,
C::Collation: Send + Unpin + 'static,
S: SpawnNamed + Clone + 'static,
{
type Error = C::Error;
fn collation_fetch<P>(
self,
parachain: ParaId,
relay_parent: Hash,
client: Arc<P>,
max_block_data_size: Option<u64>,
n_validators: usize,
) -> Pin<Box<dyn Future<Output = Result<(CollationInfo, FullOutput), Self::Error>> + Send>>
where
P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
P: ProvideRuntimeApi<Block> + Send + Sync + 'static,
{
crate::collation::collation_fetch(
self.validation_pool,
parachain,
relay_parent,
self.collators,
client,
max_block_data_size,
n_validators,
self.spawner,
).boxed()
}
}
// finds the first key we are capable of signing with out of the given set of validators,
// if any.
fn signing_key(validators: &[ValidatorId], keystore: &KeyStorePtr) -> Option<Arc<ValidatorPair>> {
let keystore = keystore.read();
validators.iter()
.find_map(|v| {
keystore.key_pair::<ValidatorPair>(&v).ok()
})
.map(|pair| Arc::new(pair))
}
/// A live instance that is related to a relay chain validation round.
///
/// It stores the `instance_handle` and the `_table_router`.
struct LiveInstance<TR> {
instance_handle: ValidationInstanceHandle,
/// Make sure we keep the table router alive, to respond/receive consensus messages.
_table_router: TR,
}
/// Constructs parachain-agreement instances.
pub(crate) struct ParachainValidationInstances<N: Network, P, SP, CF> {
/// The client instance.
client: Arc<P>,
/// The backing network handle.
network: N,
/// handle to spawner
spawner: SP,
/// Store for extrinsic data.
availability_store: AvailabilityStore,
/// Live agreements. Maps relay chain parent hashes to attestation
/// instances.
live_instances: HashMap<Hash, LiveInstance<N::TableRouter>>,
/// The underlying validation pool of processes to use.
/// Only `None` in tests.
validation_pool: Option<ValidationPool>,
/// Used to fetch a collation.
collation_fetch: CF,
}
impl<N, P, SP, CF> ParachainValidationInstances<N, P, SP, CF> where
N: Network,
N::Error: 'static,
P: ProvideRuntimeApi<Block> + Send + Sync + 'static,
P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
N::TableRouter: Send + 'static + Sync,
<N::TableRouter as TableRouter>::SendLocalCollation: Send,
N::BuildTableRouter: Unpin + Send + 'static,
SP: SpawnNamed + Send + 'static,
CF: CollationFetch + Clone + Send + Sync + 'static,
// Rust bug: https://github.com/rust-lang/rust/issues/24159
sp_api::StateBackendFor<P, Block>: sp_api::StateBackend<HashFor<Block>>,
{
/// Get an attestation table for given parent hash.
///
/// This starts a parachain agreement process on top of the parent hash if
/// one has not already started.
///
/// Additionally, this will trigger broadcast of data to the new block's duty
/// roster.
async fn get_or_instantiate(
&mut self,
parent_hash: Hash,
keystore: &KeyStorePtr,
max_block_data_size: Option<u64>,
) -> Result<ValidationInstanceHandle, Error> {
use primitives::Pair;
if let Some(instance) = self.live_instances.get(&parent_hash) {
return Ok(instance.instance_handle.clone());
}
let id = BlockId::hash(parent_hash);
let validators = self.client.runtime_api().validators(&id)?;
let sign_with = signing_key(&validators[..], keystore);
let duty_roster = self.client.runtime_api().duty_roster(&id)?;
let (group_info, local_duty) = crate::make_group_info(
duty_roster,
&validators,
sign_with.as_ref().map(|k| k.public()),
)?;
if let Some(ref duty) = local_duty {
info!(
"✍️ Starting parachain attestation session (parent: {}) with active duty {}",
parent_hash,
Colour::Red.bold().paint(format!("{:?}", duty)),
);
} else {
debug!(
"✍️ Starting parachain attestation session (parent: {}). No local duty..",
parent_hash,
);
}
let active_parachains = self.client.runtime_api().active_parachains(&id)?;
debug!(target: "validation", "Active parachains: {:?}", active_parachains);
// If we are a validator, we need to store our index in this round in availability store.
// This will tell which erasure chunk we should store.
if let Some(ref local_duty) = local_duty {
if let Err(e) = self.availability_store.note_validator_index_and_n_validators(
&parent_hash,
local_duty.index,
validators.len() as u32,
) {
warn!(
target: "validation",
"Failed to add validator index and n_validators to the availability-store: {:?}", e
)
}
}
let api = self.client.runtime_api();
let signing_context = if api.has_api_with::<dyn ParachainHost<Block, Error = ()>, _>(
&BlockId::hash(parent_hash),
|version| version >= 3,
)? {
api.signing_context(&id)?
} else {
trace!(
target: "validation",
"Expected runtime with ParachainHost version >= 3",
);
SigningContext {
session_index: 0,
parent_hash,
}
};
let table = Arc::new(SharedTable::new(
validators.clone(),
group_info,
sign_with,
signing_context,
self.availability_store.clone(),
max_block_data_size,
self.validation_pool.clone(),
));
// The router will join the consensus gossip network. This is important
// to receive messages sent for the current round.
let router = match self.network.build_table_router(
table.clone(),
&validators,
).await {
Ok(res) => res,
Err(e) => {
warn!(target: "validation", "Failed to build router: {:?}", e);
return Err(Error::CouldNotBuildTableRouter(format!("{:?}", e)))
}
};
if let Some((Chain::Parachain(id), index)) = local_duty.map(|d| (d.validation, d.index)) {
let n_validators = validators.len();
let availability_store = self.availability_store.clone();
let client = self.client.clone();
let collation_fetch = self.collation_fetch.clone();
let router = router.clone();
self.spawner.spawn(
"polkadot-parachain-validation-work",
launch_work(
move || collation_fetch.collation_fetch(id, parent_hash, client, max_block_data_size, n_validators),
availability_store,
router,
n_validators,
index,
).boxed(),
);
}
let tracker = ValidationInstanceHandle {
table,
started: Instant::now(),
};
let live_instance = LiveInstance {
instance_handle: tracker.clone(),
_table_router: router,
};
self.live_instances.insert(parent_hash, live_instance);
Ok(tracker)
}
/// Retain validation sessions matching predicate.
fn retain<F: FnMut(&Hash) -> bool>(&mut self, mut pred: F) {
self.live_instances.retain(|k, _| pred(k))
}
}
// launch parachain work asynchronously.
async fn launch_work<CFF, E>(
collation_fetch: impl FnOnce() -> CFF,
availability_store: AvailabilityStore,
router: impl TableRouter,
n_validators: usize,
local_id: ValidatorIndex,
) where
E: std::fmt::Debug,
CFF: Future<Output = Result<(CollationInfo, FullOutput), E>> + Send,
{
// fetch a local collation from connected collators.
let (collation_info, full_output) = match collation_fetch().await {
Ok(res) => res,
Err(e) => {
warn!(target: "validation", "Failed to collate candidate: {:?}", e);
return
}
};
let crate::pipeline::FullOutput {
commitments,
erasure_chunks,
available_data,
..
} = full_output;
let receipt = collation_info.into_receipt(commitments);
let pov_block = available_data.pov_block.clone();
if let Err(e) = availability_store.make_available(
receipt.hash(),
available_data,
).await {
warn!(
target: "validation",
"Failed to make parachain block data available: {}",
e,
);
}
if let Err(e) = availability_store.clone().add_erasure_chunks(
receipt.clone(),
n_validators as _,
erasure_chunks.clone(),
).await {
warn!(target: "validation", "Failed to add erasure chunks: {}", e);
}
if let Err(e) = router.local_collation(
receipt,
pov_block,
(local_id, &erasure_chunks),
).await {
warn!(target: "validation", "Failed to send local collation: {:?}", e);
}
}
#[cfg(test)]
mod tests {
use super::*;
use futures::{executor, future::ready, channel::mpsc};
use availability_store::ErasureNetworking;
use polkadot_primitives::v0::{
PoVBlock, AbridgedCandidateReceipt, ErasureChunk, ValidatorIndex,
CollationInfo, DutyRoster, GlobalValidationData, LocalValidationData,
Retriable, CollatorId, BlockData, Chain, AvailableData, SigningContext, ValidationCode,
};
use runtime_primitives::traits::Block as BlockT;
use std::pin::Pin;
use sp_keyring::sr25519::Keyring;
use primitives::testing::TaskExecutor;
/// Events fired while running mock implementations to follow execution.
enum Events {
BuildTableRouter,
CollationFetch,
LocalCollation,
}
#[derive(Clone)]
struct MockNetwork(mpsc::UnboundedSender<Events>);
impl Network for MockNetwork {
type Error = String;
type TableRouter = MockTableRouter;
type BuildTableRouter = Pin<Box<dyn Future<Output = Result<MockTableRouter, String>> + Send>>;
fn build_table_router(
&self,
_: Arc<SharedTable>,
_: &[ValidatorId],
) -> Self::BuildTableRouter {
let event_sender = self.0.clone();
async move {
event_sender.unbounded_send(Events::BuildTableRouter).expect("Send `BuildTableRouter`");
Ok(MockTableRouter(event_sender))
}.boxed()
}
}
#[derive(Clone)]
struct MockTableRouter(mpsc::UnboundedSender<Events>);
impl TableRouter for MockTableRouter {
type Error = String;
type SendLocalCollation = Pin<Box<dyn Future<Output = Result<(), String>> + Send>>;
type FetchValidationProof = Box<dyn Future<Output = Result<PoVBlock, String>> + Unpin>;
fn local_collation(
&self,
_: AbridgedCandidateReceipt,
_: PoVBlock,
_: (ValidatorIndex, &[ErasureChunk]),
) -> Self::SendLocalCollation {
let sender = self.0.clone();
async move {
sender.unbounded_send(Events::LocalCollation).expect("Send `LocalCollation`");
Ok(())
}.boxed()
}
fn fetch_pov_block(&self, _: &AbridgedCandidateReceipt) -> Self::FetchValidationProof {
unimplemented!("Not required in tests")
}
}
#[derive(Clone)]
struct MockErasureNetworking;
impl ErasureNetworking for MockErasureNetworking {
type Error = String;
fn fetch_erasure_chunk(
&self,
_: &Hash,
_: u32,
) -> Pin<Box<dyn Future<Output = Result<ErasureChunk, Self::Error>> + Send>> {
ready(Err("Not required in tests".to_string())).boxed()
}
fn distribute_erasure_chunk(&self, _: Hash, _: ErasureChunk) {
unimplemented!("Not required in tests")
}
}
#[derive(Clone)]
struct MockCollationFetch(mpsc::UnboundedSender<Events>);
impl CollationFetch for MockCollationFetch {
type Error = ();
fn collation_fetch<P>(
self,
parachain: ParaId,
relay_parent: Hash,
_: Arc<P>,
_: Option<u64>,
n_validators: usize,
) -> Pin<Box<dyn Future<Output = Result<(CollationInfo, FullOutput), ()>> + Send>> {
let info = CollationInfo {
parachain_index: parachain,
relay_parent,
collator: Default::default(),
signature: Default::default(),
head_data: Default::default(),
pov_block_hash: Default::default(),
};
let available_data = AvailableData {
pov_block: PoVBlock { block_data: BlockData(Vec::new()) },
omitted_validation: Default::default(),
};
let full_output = FullOutput {
available_data,
commitments: Default::default(),
erasure_chunks: Default::default(),
n_validators,
};
let sender = self.0;
async move {
sender.unbounded_send(Events::CollationFetch).expect("`CollationFetch` event send");
Ok((info, full_output))
}.boxed()
}
}
#[derive(Clone)]
struct MockRuntimeApi {
validators: Vec<ValidatorId>,
duty_roster: DutyRoster,
}
impl ProvideRuntimeApi<Block> for MockRuntimeApi {
type Api = Self;
fn runtime_api<'a>(&'a self) -> sp_api::ApiRef<'a, Self::Api> {
self.clone().into()
}
}
sp_api::mock_impl_runtime_apis! {
impl ParachainHost<Block> for MockRuntimeApi {
type Error = sp_blockchain::Error;
fn validators(&self) -> Vec<ValidatorId> { self.validators.clone() }
fn duty_roster(&self) -> DutyRoster { self.duty_roster.clone() }
fn active_parachains() -> Vec<(ParaId, Option<(CollatorId, Retriable)>)> { vec![(ParaId::from(1), None)] }
fn global_validation_data() -> GlobalValidationData { Default::default() }
fn local_validation_data(_: ParaId) -> Option<LocalValidationData> { None }
fn parachain_code(_: ParaId) -> Option<ValidationCode> { None }
fn get_heads(_: Vec<<Block as BlockT>::Extrinsic>) -> Option<Vec<AbridgedCandidateReceipt>> {
None
}
fn signing_context() -> SigningContext {
Default::default()
}
fn downward_messages(_: ParaId) -> Vec<polkadot_primitives::v0::DownwardMessage> {
Vec::new()
}
}
}
#[test]
fn launch_work_is_executed_properly() {
let executor = TaskExecutor::new();
let keystore = keystore::Store::new_in_memory();
// Make sure `Bob` key is in the keystore, so this mocked node will be a parachain validator.
keystore.write().insert_ephemeral_from_seed::<ValidatorPair>(&Keyring::Bob.to_seed())
.expect("Insert key into keystore");
let validators = vec![ValidatorId::from(Keyring::Alice.public()), ValidatorId::from(Keyring::Bob.public())];
let validator_duty = vec![Chain::Relay, Chain::Parachain(1.into())];
let duty_roster = DutyRoster { validator_duty };
let (events_sender, events) = mpsc::unbounded();
let mut parachain_validation = ParachainValidationInstances {
client: Arc::new(MockRuntimeApi { validators, duty_roster }),
network: MockNetwork(events_sender.clone()),
collation_fetch: MockCollationFetch(events_sender.clone()),
spawner: executor.clone(),
availability_store: AvailabilityStore::new_in_memory(MockErasureNetworking),
live_instances: HashMap::new(),
validation_pool: None,
};
executor::block_on(parachain_validation.get_or_instantiate(Default::default(), &keystore, None))
.expect("Creates new validation round");
assert!(parachain_validation.live_instances.contains_key(&Default::default()));
let mut events = executor::block_on_stream(events);
assert!(matches!(events.next().unwrap(), Events::BuildTableRouter));
assert!(matches!(events.next().unwrap(), Events::CollationFetch));
assert!(matches!(events.next().unwrap(), Events::LocalCollation));
drop(events_sender);
drop(parachain_validation);
assert!(events.next().is_none());
}
#[test]
fn router_is_built_on_relay_chain_validator() {
let executor = TaskExecutor::new();
let keystore = keystore::Store::new_in_memory();
// Make sure `Alice` key is in the keystore, so this mocked node will be a relay-chain validator.
keystore.write().insert_ephemeral_from_seed::<ValidatorPair>(&Keyring::Alice.to_seed())
.expect("Insert key into keystore");
let validators = vec![ValidatorId::from(Keyring::Alice.public()), ValidatorId::from(Keyring::Bob.public())];
let validator_duty = vec![Chain::Relay, Chain::Parachain(1.into())];
let duty_roster = DutyRoster { validator_duty };
let (events_sender, events) = mpsc::unbounded();
let mut parachain_validation = ParachainValidationInstances {
client: Arc::new(MockRuntimeApi { validators, duty_roster }),
network: MockNetwork(events_sender.clone()),
collation_fetch: MockCollationFetch(events_sender.clone()),
spawner: executor.clone(),
availability_store: AvailabilityStore::new_in_memory(MockErasureNetworking),
live_instances: HashMap::new(),
validation_pool: None,
};
executor::block_on(parachain_validation.get_or_instantiate(Default::default(), &keystore, None))
.expect("Creates new validation round");
assert!(parachain_validation.live_instances.contains_key(&Default::default()));
let mut events = executor::block_on_stream(events);
assert!(matches!(events.next().unwrap(), Events::BuildTableRouter));
drop(events_sender);
drop(parachain_validation);
assert!(events.next().is_none());
}
}