pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-05-30 11:41:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

A more comprehensive model for PoV-Blocks and Candidate receipts (#843)

Browse Source 
* encode the candidate statement as only the hash

* refactor CandidateReceipt and CollationInfo

* introduce an abridged candidate receipt type

* erasure coding stores candidate receipt

* store omitted data instead and introduce AvailableData type

* refactor availability-store schema

* tweak schema and APIs a bit more

* get availability-store tests passing

* accept AbridgedCandidateReceipt in `set_heads`

* change statement type in primitives to be hash-only

* fix parachains runtime tests

* fix bad merge

* rewrite validation pipeline

* remove evaluation module

* use abridged candidate hash as canonical

* statement table uses abridged candidate receipts

* kill availability_store::Data struct

* port shared table to new validation pipelines

* extract full validation pipeline to helper

* remove old validation pipeline from collation module

* polkadot-validation compiles

* polkadot-validation tests compile

* make local collation available in validation service

* port legacy network code

* polkadot-network fully ported

* network: ensure fresh statement is propagated

* remove pov_block_hash from LocalValidationData

* remove candidate_hash field from AttestedCandidate and update runtime

* port runtimes to new ParachainHost definition

* port over polkadot-collator

* fix test compilation

* better fix

* remove unrelated validation work dispatch fix

* address grumbles

* fix equality check
This commit is contained in:
Robert Habermeier
2020-02-25 15:16:58 -08:00
committed by GitHub
parent 1f9d2af08e
commit b7d30aa379
29 changed files with 1718 additions and 1704 deletions
Download Patch File Download Diff File Expand all files Collapse all files
polkadot/availability-store/src/lib.rs
+67 -81
View File
@@ -28,7 +28,8 @@ use keystore::KeyStorePtr;
use polkadot_primitives::{
Hash, Block,
parachain::{
Id as ParaId, BlockData, CandidateReceipt, ErasureChunk, ParachainHost
PoVBlock, AbridgedCandidateReceipt, ErasureChunk,
ParachainHost, AvailableData, OmittedValidationData,
},
};
use sp_runtime::traits::{BlakeTwo256, Hash as HashT, HasherFor};
@@ -37,6 +38,7 @@ use client::{
BlockchainEvents, BlockBody,
};
use sp_api::{ApiExt, ProvideRuntimeApi};
use codec::{Encode, Decode};
use log::warn;
@@ -50,9 +52,10 @@ mod worker;
mod store;
pub use worker::AvailabilityBlockImport;
pub use store::AwaitedFrontierEntry;
use worker::{
Worker, WorkerHandle, Chunks, ParachainBlocks, WorkerMsg, MakeAvailable,
Worker, WorkerHandle, Chunks, IncludedParachainBlocks, WorkerMsg, MakeAvailable,
};
use store::{Store as InnerStore};
@@ -116,15 +119,14 @@ pub trait ProvideGossipMessages {
);
}
/// Some data to keep available about a parachain block candidate.
#[derive(Debug)]
pub struct Data {
/// The relay chain parent hash this should be localized to.
pub relay_parent: Hash,
/// The parachain index for this candidate.
pub parachain_id: ParaId,
/// Block data.
pub block_data: BlockData,
/// Data which, when combined with an `AbridgedCandidateReceipt`, is enough
/// to fully re-execute a block.
#[derive(Debug, Encode, Decode, PartialEq)]
pub struct ExecutionData {
/// The `PoVBlock`.
pub pov_block: PoVBlock,
/// The data omitted from the `AbridgedCandidateReceipt`.
pub omitted_validation: OmittedValidationData,
}
/// Handle to the availability store.
@@ -220,17 +222,17 @@ impl Store {
/// in order to persist that data to disk and so it can be queried and provided
/// to other nodes in the network.
///
/// The message data of `Data` is optional but is expected
/// to be present with the exception of the case where there is no message data
/// due to the block's invalidity. Determination of invalidity is beyond the
/// scope of this function.
/// Determination of invalidity is beyond the scope of this function.
///
/// This method will send the `Data` to the background worker, allowing caller to
/// asynchrounously wait for the result.
pub async fn make_available(&self, data: Data) -> io::Result<()> {
/// This method will send the data to the background worker, allowing the caller to
/// asynchronously wait for the result.
pub async fn make_available(&self, candidate_hash: Hash, available_data: AvailableData)
-> io::Result<()>
{
let (s, r) = oneshot::channel();
let msg = WorkerMsg::MakeAvailable(MakeAvailable {
data,
candidate_hash,
available_data,
result: s,
});
@@ -244,41 +246,11 @@ impl Store {
}
/// Get a set of all chunks we are waiting for grouped by
/// `(relay_parent, erasure_root, candidate_hash, our_id)`.
pub fn awaited_chunks(&self) -> Option<HashSet<(Hash, Hash, Hash, u32)>> {
/// Get a set of all chunks we are waiting for.
pub fn awaited_chunks(&self) -> Option<HashSet<AwaitedFrontierEntry>> {
self.inner.awaited_chunks()
}
/// Qery which candidates were included in the relay chain block by block's parent.
pub fn get_candidates_in_relay_block(&self, relay_block: &Hash) -> Option<Vec<Hash>> {
self.inner.get_candidates_in_relay_block(relay_block)
}
/// Make a validator's index and a number of validators at a relay parent available.
///
/// This information is needed before the `add_candidates_in_relay_block` is called
/// since that call forms the awaited frontier of chunks.
/// In the current implementation this function is called in the `get_or_instantiate` at
/// the start of the parachain agreement process on top of some parent hash.
pub fn add_validator_index_and_n_validators(
&self,
relay_parent: &Hash,
validator_index: u32,
n_validators: u32,
) -> io::Result<()> {
self.inner.add_validator_index_and_n_validators(
relay_parent,
validator_index,
n_validators,
)
}
/// Query a validator's index and n_validators by relay parent.
pub fn get_validator_index_and_n_validators(&self, relay_parent: &Hash) -> Option<(u32, u32)> {
self.inner.get_validator_index_and_n_validators(relay_parent)
}
/// Adds an erasure chunk to storage.
///
/// The chunk should be checked for validity against the root of encoding
@@ -288,11 +260,10 @@ impl Store {
/// asynchrounously wait for the result.
pub async fn add_erasure_chunk(
&self,
relay_parent: Hash,
receipt: CandidateReceipt,
candidate: AbridgedCandidateReceipt,
chunk: ErasureChunk,
) -> io::Result<()> {
self.add_erasure_chunks(relay_parent, receipt, vec![chunk]).await
self.add_erasure_chunks(candidate, vec![chunk]).await
}
/// Adds a set of erasure chunks to storage.
@@ -304,16 +275,17 @@ impl Store {
/// asynchrounously waiting for the result.
pub async fn add_erasure_chunks<I>(
&self,
relay_parent: Hash,
receipt: CandidateReceipt,
candidate: AbridgedCandidateReceipt,
chunks: I,
) -> io::Result<()>
where I: IntoIterator<Item = ErasureChunk>
{
self.add_candidate(relay_parent, receipt.clone()).await?;
let candidate_hash = candidate.hash();
let relay_parent = candidate.relay_parent;
self.add_candidate(candidate).await?;
let (s, r) = oneshot::channel();
let chunks = chunks.into_iter().collect();
let candidate_hash = receipt.hash();
let msg = WorkerMsg::Chunks(Chunks {
relay_parent,
candidate_hash,
@@ -330,27 +302,44 @@ impl Store {
}
}
/// Queries an erasure chunk by its block's parent and hash and index.
/// Queries an erasure chunk by the candidate hash and validator index.
pub fn get_erasure_chunk(
&self,
relay_parent: &Hash,
block_data_hash: Hash,
index: usize,
candidate_hash: &Hash,
validator_index: usize,
) -> Option<ErasureChunk> {
self.inner.get_erasure_chunk(relay_parent, block_data_hash, index)
self.inner.get_erasure_chunk(candidate_hash, validator_index)
}
/// Stores a candidate receipt.
pub async fn add_candidate(
/// Note a validator's index and a number of validators at a relay parent in the
/// store.
///
/// This should be done before adding erasure chunks with this relay parent.
pub fn note_validator_index_and_n_validators(
&self,
relay_parent: Hash,
receipt: CandidateReceipt,
relay_parent: &Hash,
validator_index: u32,
n_validators: u32,
) -> io::Result<()> {
self.inner.note_validator_index_and_n_validators(
relay_parent,
validator_index,
n_validators,
)
}
// Stores a candidate receipt.
async fn add_candidate(
&self,
candidate: AbridgedCandidateReceipt,
) -> io::Result<()> {
let (s, r) = oneshot::channel();
let msg = WorkerMsg::ParachainBlocks(ParachainBlocks {
relay_parent,
blocks: vec![(receipt, None)],
let msg = WorkerMsg::IncludedParachainBlocks(IncludedParachainBlocks {
blocks: vec![crate::worker::IncludedParachainBlock {
candidate,
available_data: None,
}],
result: s,
});
@@ -363,20 +352,17 @@ impl Store {
}
}
/// Queries a candidate receipt by it's hash.
pub fn get_candidate(&self, candidate_hash: &Hash) -> Option<CandidateReceipt> {
/// Queries a candidate receipt by its hash.
pub fn get_candidate(&self, candidate_hash: &Hash)
-> Option<AbridgedCandidateReceipt>
{
self.inner.get_candidate(candidate_hash)
}
/// Query block data.
pub fn block_data(&self, relay_parent: Hash, block_data_hash: Hash) -> Option<BlockData> {
self.inner.block_data(relay_parent, block_data_hash)
}
/// Query block data by corresponding candidate receipt's hash.
pub fn block_data_by_candidate(&self, relay_parent: Hash, candidate_hash: Hash)
-> Option<BlockData>
/// Query execution data by pov-block hash.
pub fn execution_data(&self, candidate_hash: &Hash)
-> Option<ExecutionData>
{
self.inner.block_data_by_candidate(relay_parent, candidate_hash)
self.inner.execution_data(candidate_hash)
}
}
polkadot/availability-store/src/store.rs
+220 -230
View File
@@ -21,8 +21,11 @@ use codec::{Encode, Decode};
use polkadot_erasure_coding::{self as erasure};
use polkadot_primitives::{
Hash,
parachain::{BlockData, CandidateReceipt, ErasureChunk},
parachain::{
ErasureChunk, AvailableData, AbridgedCandidateReceipt,
},
};
use parking_lot::Mutex;
use log::{trace, warn};
use std::collections::HashSet;
@@ -30,7 +33,7 @@ use std::sync::Arc;
use std::iter::FromIterator;
use std::io;
use crate::{LOG_TARGET, Data, Config};
use crate::{LOG_TARGET, Config, ExecutionData};
mod columns {
pub const DATA: u32 = 0;
@@ -41,42 +44,51 @@ mod columns {
#[derive(Clone)]
pub struct Store {
inner: Arc<dyn KeyValueDB>,
candidate_descendents_lock: Arc<Mutex<()>>
}
fn block_data_key(relay_parent: &Hash, block_data_hash: &Hash) -> Vec<u8> {
(relay_parent, block_data_hash, 0i8).encode()
// data keys
fn execution_data_key(candidate_hash: &Hash) -> Vec<u8> {
(candidate_hash, 0i8).encode()
}
fn erasure_chunks_key(relay_parent: &Hash, block_data_hash: &Hash) -> Vec<u8> {
(relay_parent, block_data_hash, 1i8).encode()
}
fn awaited_chunks_key() -> Vec<u8> {
"awaited_chunks_key".encode()
}
fn available_chunks_key(relay_parent: &Hash, erasure_root: &Hash) -> Vec<u8> {
(relay_parent, erasure_root, 2i8).encode()
}
fn block_to_candidate_key(block_data_hash: &Hash) -> Vec<u8> {
(block_data_hash, 1i8).encode()
fn erasure_chunks_key(candidate_hash: &Hash) -> Vec<u8> {
(candidate_hash, 1i8).encode()
}
fn candidate_key(candidate_hash: &Hash) -> Vec<u8> {
(candidate_hash, 2i8).encode()
}
fn validator_index_and_n_validators_key(relay_parent: &Hash) -> Vec<u8> {
(relay_parent, 3i8).encode()
fn available_chunks_key(relay_parent: &Hash, erasure_root: &Hash) -> Vec<u8> {
(relay_parent, erasure_root, 3i8).encode()
}
fn candidates_in_relay_chain_block_key(relay_block: &Hash) -> Vec<u8> {
fn candidates_with_relay_parent_key(relay_block: &Hash) -> Vec<u8> {
(relay_block, 4i8).encode()
}
fn erasure_roots_in_relay_chain_block_key(relay_block: &Hash) -> Vec<u8> {
(relay_block, 5i8).encode()
// meta keys
fn awaited_chunks_key() -> [u8; 14] {
*b"awaited_chunks"
}
fn validator_index_and_n_validators_key(relay_parent: &Hash) -> Vec<u8> {
(relay_parent, 1i8).encode()
}
/// An entry in the awaited frontier of chunks we are interested in.
#[derive(Encode, Decode, Debug, Hash, PartialEq, Eq, Clone)]
pub struct AwaitedFrontierEntry {
/// The relay-chain parent block hash.
pub relay_parent: Hash,
/// The erasure-chunk trie root we are comparing against.
///
/// We index by erasure-root because there may be multiple candidates
/// with the same erasure root.
pub erasure_root: Hash,
/// The index of the validator we represent.
pub validator_index: u32,
}
impl Store {
@@ -103,6 +115,7 @@ impl Store {
Ok(Store {
inner: Arc::new(db),
candidate_descendents_lock: Arc::new(Mutex::new(())),
})
}
@@ -110,31 +123,37 @@ impl Store {
pub(super) fn new_in_memory() -> Self {
Store {
inner: Arc::new(::kvdb_memorydb::create(columns::NUM_COLUMNS)),
candidate_descendents_lock: Arc::new(Mutex::new(())),
}
}
/// Make some data available provisionally.
pub(crate) fn make_available(&self, data: Data) -> io::Result<()> {
pub(crate) fn make_available(&self, candidate_hash: Hash, available_data: AvailableData)
-> io::Result<()>
{
let mut tx = DBTransaction::new();
// note the meta key.
let mut v = self.query_inner(columns::META, data.relay_parent.as_ref()).unwrap_or(Vec::new());
v.push(data.block_data.hash());
tx.put_vec(columns::META, &data.relay_parent[..], v.encode());
// at the moment, these structs are identical. later, we will also
// keep outgoing message queues available, and these are not needed
// for execution.
let AvailableData { pov_block, omitted_validation } = available_data;
let execution_data = ExecutionData {
pov_block,
omitted_validation,
};
tx.put_vec(
columns::DATA,
block_data_key(&data.relay_parent, &data.block_data.hash()).as_slice(),
data.block_data.encode()
execution_data_key(&candidate_hash).as_slice(),
execution_data.encode(),
);
self.inner.write(tx)
}
/// Get a set of all chunks we are waiting for grouped by
/// `(relay_parent, erasure_root, candidate_hash, our_id)`.
pub fn awaited_chunks(&self) -> Option<HashSet<(Hash, Hash, Hash, u32)>> {
self.query_inner(columns::META, &awaited_chunks_key()).map(|vec: Vec<(Hash, Hash, Hash, u32)>| {
/// Get a set of all chunks we are waiting for.
pub fn awaited_chunks(&self) -> Option<HashSet<AwaitedFrontierEntry>> {
self.query_inner(columns::META, &awaited_chunks_key()).map(|vec: Vec<AwaitedFrontierEntry>| {
HashSet::from_iter(vec.into_iter())
})
}
@@ -147,60 +166,49 @@ impl Store {
/// This method modifies the erasure chunks awaited frontier by adding this validator's
/// chunks from `candidates` to it. In order to do so the information about this validator's
/// position at parent `relay_parent` should be known to the store prior to calling this
/// method, in other words `add_validator_index_and_n_validators` should be called for
/// method, in other words `note_validator_index_and_n_validators` should be called for
/// the given `relay_parent` before calling this function.
pub(crate) fn add_candidates_in_relay_block(
pub(crate) fn note_candidates_with_relay_parent(
&self,
relay_parent: &Hash,
candidates: Vec<Hash>,
candidates: &[Hash],
) -> io::Result<()> {
let mut tx = DBTransaction::new();
let dbkey = candidates_in_relay_chain_block_key(relay_parent);
let dbkey = candidates_with_relay_parent_key(relay_parent);
// This call can race against another call to `note_candidates_with_relay_parent`
// with a different set of descendents.
let _lock = self.candidate_descendents_lock.lock();
if let Some((validator_index, _)) = self.get_validator_index_and_n_validators(relay_parent) {
let candidates = candidates.clone();
let awaited_frontier: Vec<(Hash, Hash, Hash, u32)> = self
let awaited_frontier: Vec<AwaitedFrontierEntry> = self
.query_inner(columns::META, &awaited_chunks_key())
.unwrap_or_else(|| Vec::new());
let mut awaited_frontier: HashSet<(Hash, Hash, Hash, u32)> =
let mut awaited_frontier: HashSet<AwaitedFrontierEntry> =
HashSet::from_iter(awaited_frontier.into_iter());
awaited_frontier.extend(candidates.into_iter().filter_map(|candidate| {
self.get_candidate(&candidate)
.map(|receipt| (relay_parent.clone(), receipt.erasure_root, candidate, validator_index))
awaited_frontier.extend(candidates.iter().filter_map(|candidate| {
self.get_candidate(&candidate).map(|receipt| AwaitedFrontierEntry {
relay_parent: relay_parent.clone(),
erasure_root: receipt.commitments.erasure_root,
validator_index,
})
}));
let awaited_frontier = Vec::from_iter(awaited_frontier.into_iter());
tx.put_vec(columns::META, &awaited_chunks_key(), awaited_frontier.encode());
}
tx.put_vec(columns::DATA, &dbkey, candidates.encode());
self.inner.write(tx)
}
/// Qery which candidates were included in the relay chain block by block's parent.
pub fn get_candidates_in_relay_block(&self, relay_block: &Hash) -> Option<Vec<Hash>> {
let dbkey = candidates_in_relay_chain_block_key(relay_block);
self.query_inner(columns::DATA, &dbkey)
}
/// Adds a set of erasure chunk roots that were included in a relay block by block's parent.
pub(crate) fn add_erasure_roots_in_relay_block(
&self,
relay_parent: &Hash,
erasure_roots: Vec<Hash>,
) -> io::Result<()> {
let mut tx = DBTransaction::new();
let dbkey = erasure_roots_in_relay_chain_block_key(relay_parent);
tx.put_vec(columns::DATA, &dbkey, erasure_roots.encode());
let mut descendent_candidates = self.get_candidates_with_relay_parent(relay_parent);
descendent_candidates.extend(candidates.iter().cloned());
tx.put_vec(columns::DATA, &dbkey, descendent_candidates.encode());
self.inner.write(tx)
}
/// Make a validator's index and a number of validators at a relay parent available.
pub(crate) fn add_validator_index_and_n_validators(
pub(crate) fn note_validator_index_and_n_validators(
&self,
relay_parent: &Hash,
validator_index: u32,
@@ -215,7 +223,7 @@ impl Store {
}
/// Query a validator's index and n_validators by relay parent.
pub fn get_validator_index_and_n_validators(&self, relay_parent: &Hash) -> Option<(u32, u32)> {
pub(crate) fn get_validator_index_and_n_validators(&self, relay_parent: &Hash) -> Option<(u32, u32)> {
let dbkey = validator_index_and_n_validators_key(relay_parent);
self.query_inner(columns::META, &dbkey)
@@ -224,11 +232,9 @@ impl Store {
/// Add a set of chunks.
///
/// The same as `add_erasure_chunk` but adds a set of chunks in one atomic transaction.
/// Checks that all chunks have the same `relay_parent`, `block_data_hash` and `parachain_id` fields.
pub fn add_erasure_chunks<I>(
&self,
n_validators: u32,
relay_parent: &Hash,
candidate_hash: &Hash,
chunks: I,
) -> io::Result<()>
@@ -236,16 +242,19 @@ impl Store {
{
if let Some(receipt) = self.get_candidate(candidate_hash) {
let mut tx = DBTransaction::new();
let dbkey = erasure_chunks_key(relay_parent, &receipt.block_data_hash);
let dbkey = erasure_chunks_key(candidate_hash);
let mut v = self.query_inner(columns::DATA, &dbkey).unwrap_or(Vec::new());
let av_chunks_key = available_chunks_key(relay_parent, &receipt.erasure_root);
let av_chunks_key = available_chunks_key(
&receipt.relay_parent,
&receipt.commitments.erasure_root,
);
let mut have_chunks = self.query_inner(columns::META, &av_chunks_key).unwrap_or(Vec::new());
let awaited_frontier: Option<Vec<(Hash, Hash, Hash, u32)>> = self.query_inner(
let awaited_frontier: Option<Vec<AwaitedFrontierEntry>> = self.query_inner(
columns::META,
&awaited_chunks_key()
&awaited_chunks_key(),
);
for chunk in chunks.into_iter() {
@@ -256,31 +265,23 @@ impl Store {
}
if let Some(mut awaited_frontier) = awaited_frontier {
awaited_frontier.retain(|&(p, r, c, index)| {
awaited_frontier.retain(|entry| {
!(
*relay_parent == p &&
r == receipt.erasure_root &&
c == receipt.hash() &&
have_chunks.contains(&index)
entry.relay_parent == receipt.relay_parent &&
entry.erasure_root == receipt.commitments.erasure_root &&
have_chunks.contains(&entry.validator_index)
)
});
tx.put_vec(columns::META, &awaited_chunks_key(), awaited_frontier.encode());
}
// If therea are no block data and messages in the store at this point,
// If therea are no block data in the store at this point,
// check that they can be reconstructed now and add them to store if they can.
if let Ok(None) = self.inner.get(
columns::DATA,
&block_data_key(&relay_parent, &receipt.block_data_hash)
) {
if let Ok(block_data) = erasure::reconstruct(
if self.execution_data(&candidate_hash).is_none() {
if let Ok(available_data) = erasure::reconstruct(
n_validators as usize,
v.iter().map(|chunk| (chunk.chunk.as_ref(), chunk.index as usize))) {
self.make_available(Data {
relay_parent: *relay_parent,
parachain_id: receipt.parachain_index,
block_data,
})?;
self.make_available(*candidate_hash, available_data)?;
}
}
@@ -294,14 +295,13 @@ impl Store {
}
}
/// Queries an erasure chunk by its block's parent and hash and index.
/// Queries an erasure chunk by its block's relay-parent, the candidate hash, and index.
pub fn get_erasure_chunk(
&self,
relay_parent: &Hash,
block_data_hash: Hash,
candidate_hash: &Hash,
index: usize,
) -> Option<ErasureChunk> {
self.query_inner(columns::DATA, &erasure_chunks_key(&relay_parent, &block_data_hash))
self.query_inner(columns::DATA, &erasure_chunks_key(candidate_hash))
.and_then(|chunks: Vec<ErasureChunk>| {
chunks.iter()
.find(|chunk: &&ErasureChunk| chunk.index == index as u32)
@@ -310,70 +310,66 @@ impl Store {
}
/// Stores a candidate receipt.
pub fn add_candidate(&self, receipt: &CandidateReceipt) -> io::Result<()> {
let dbkey = candidate_key(&receipt.hash());
pub fn add_candidate(
&self,
receipt: &AbridgedCandidateReceipt,
) -> io::Result<()> {
let candidate_hash = receipt.hash();
let dbkey = candidate_key(&candidate_hash);
let mut tx = DBTransaction::new();
tx.put_vec(columns::DATA, &dbkey, receipt.encode());
tx.put_vec(columns::META, &block_to_candidate_key(&receipt.block_data_hash), receipt.hash().encode());
self.inner.write(tx)
}
/// Queries a candidate receipt by it's hash.
pub fn get_candidate(&self, candidate_hash: &Hash) -> Option<CandidateReceipt> {
/// Queries a candidate receipt by the relay parent hash and its hash.
pub(crate) fn get_candidate(&self, candidate_hash: &Hash)
-> Option<AbridgedCandidateReceipt>
{
self.query_inner(columns::DATA, &candidate_key(candidate_hash))
}
/// Note that a set of candidates have been included in a finalized block with given hash and parent hash.
pub fn candidates_finalized(
pub(crate) fn candidates_finalized(
&self,
parent: Hash,
relay_parent: Hash,
finalized_candidates: HashSet<Hash>,
) -> io::Result<()> {
let mut tx = DBTransaction::new();
let v = self.query_inner(columns::META, &parent[..]).unwrap_or(Vec::new());
tx.delete(columns::META, &parent[..]);
let awaited_frontier: Option<Vec<(Hash, Hash, Hash, u32)>> = self
let awaited_frontier: Option<Vec<AwaitedFrontierEntry>> = self
.query_inner(columns::META, &awaited_chunks_key());
if let Some(mut awaited_frontier) = awaited_frontier {
awaited_frontier.retain(|&(p, c, _, _)| (p != parent && !finalized_candidates.contains(&c)));
awaited_frontier.retain(|entry| entry.relay_parent != relay_parent);
tx.put_vec(columns::META, &awaited_chunks_key(), awaited_frontier.encode());
}
for block_data_hash in v {
if let Some(candidate_hash) = self.block_hash_to_candidate_hash(block_data_hash) {
if !finalized_candidates.contains(&candidate_hash) {
tx.delete(columns::DATA, block_data_key(&parent, &block_data_hash).as_slice());
tx.delete(columns::DATA, &erasure_chunks_key(&parent, &block_data_hash));
tx.delete(columns::DATA, &candidate_key(&candidate_hash));
tx.delete(columns::META, &block_to_candidate_key(&block_data_hash));
}
}
let candidates = self.get_candidates_with_relay_parent(&relay_parent);
for candidate in candidates.into_iter().filter(|c| !finalized_candidates.contains(c)) {
// we only delete this data for candidates which were not finalized.
// we keep all data for the finalized chain forever at the moment.
tx.delete(columns::DATA, execution_data_key(&candidate).as_slice());
tx.delete(columns::DATA, &erasure_chunks_key(&candidate));
tx.delete(columns::DATA, &candidate_key(&candidate));
}
self.inner.write(tx)
}
/// Query block data.
pub fn block_data(&self, relay_parent: Hash, block_data_hash: Hash) -> Option<BlockData> {
self.query_inner(columns::DATA, &block_data_key(&relay_parent, &block_data_hash))
/// Query execution data by relay parent and candidate hash.
pub(crate) fn execution_data(&self, candidate_hash: &Hash) -> Option<ExecutionData> {
self.query_inner(columns::DATA, &execution_data_key(candidate_hash))
}
/// Query block data by corresponding candidate receipt's hash.
pub fn block_data_by_candidate(&self, relay_parent: Hash, candidate_hash: Hash) -> Option<BlockData> {
let receipt_key = candidate_key(&candidate_hash);
self.query_inner(columns::DATA, &receipt_key[..]).and_then(|receipt: CandidateReceipt| {
self.block_data(relay_parent, receipt.block_data_hash)
})
}
fn block_hash_to_candidate_hash(&self, block_hash: Hash) -> Option<Hash> {
self.query_inner(columns::META, &block_to_candidate_key(&block_hash))
/// Get candidates which pinned to the environment of the given relay parent.
/// Note that this is not necessarily the same as candidates that were included in a direct
/// descendent of the given relay-parent.
fn get_candidates_with_relay_parent(&self, relay_parent: &Hash) -> Vec<Hash> {
let key = candidates_with_relay_parent_key(relay_parent);
self.query_inner(columns::DATA, &key[..]).unwrap_or_default()
}
fn query_inner<T: Decode>(&self, column: u32, key: &[u8]) -> Option<T> {
@@ -394,8 +390,27 @@ impl Store {
#[cfg(test)]
mod tests {
use super::*;
use polkadot_erasure_coding as erasure;
use polkadot_primitives::parachain::Id as ParaId;
use polkadot_erasure_coding::{self as erasure};
use polkadot_primitives::parachain::{
Id as ParaId, BlockData, AvailableData, PoVBlock, OmittedValidationData,
};
fn available_data(block_data: &[u8]) -> AvailableData {
AvailableData {
pov_block: PoVBlock {
block_data: BlockData(block_data.to_vec()),
},
omitted_validation: OmittedValidationData {
global_validation: Default::default(),
local_validation: Default::default(),
}
}
}
fn execution_data(available: &AvailableData) -> ExecutionData {
let AvailableData { pov_block, omitted_validation } = available.clone();
ExecutionData { pov_block, omitted_validation }
}
#[test]
fn finalization_removes_unneeded() {
@@ -404,8 +419,23 @@ mod tests {
let para_id_1 = 5.into();
let para_id_2 = 6.into();
let block_data_1 = BlockData(vec![1, 2, 3]);
let block_data_2 = BlockData(vec![4, 5, 6]);
let mut candidate_1 = AbridgedCandidateReceipt::default();
let mut candidate_2 = AbridgedCandidateReceipt::default();
candidate_1.parachain_index = para_id_1;
candidate_1.commitments.erasure_root = [6; 32].into();
candidate_1.relay_parent = relay_parent;
candidate_2.parachain_index = para_id_2;
candidate_2.commitments.erasure_root = [6; 32].into();
candidate_2.relay_parent = relay_parent;
let candidate_1_hash = candidate_1.hash();
let candidate_2_hash = candidate_2.hash();
let available_data_1 = available_data(&[1, 2, 3]);
let available_data_2 = available_data(&[4, 5, 6]);
let erasure_chunk_1 = ErasureChunk {
chunk: vec![10, 20, 30],
@@ -420,112 +450,59 @@ mod tests {
};
let store = Store::new_in_memory();
store.make_available(Data {
relay_parent,
parachain_id: para_id_1,
block_data: block_data_1.clone(),
}).unwrap();
store.make_available(candidate_1_hash, available_data_1.clone()).unwrap();
store.make_available(Data {
relay_parent,
parachain_id: para_id_2,
block_data: block_data_2.clone(),
}).unwrap();
let candidate_1 = CandidateReceipt {
parachain_index: para_id_1,
collator: Default::default(),
signature: Default::default(),
head_data: Default::default(),
parent_head: Default::default(),
fees: 0,
block_data_hash: block_data_1.hash(),
upward_messages: Vec::new(),
erasure_root: [6; 32].into(),
};
let candidate_2 = CandidateReceipt {
parachain_index: para_id_2,
collator: Default::default(),
signature: Default::default(),
head_data: Default::default(),
parent_head: Default::default(),
fees: 0,
block_data_hash: block_data_2.hash(),
upward_messages: Vec::new(),
erasure_root: [6; 32].into(),
};
store.make_available(candidate_2_hash, available_data_2.clone()).unwrap();
store.add_candidate(&candidate_1).unwrap();
store.add_candidate(&candidate_2).unwrap();
assert!(store.add_erasure_chunks(3, &relay_parent, &candidate_1.hash(), vec![erasure_chunk_1.clone()]).is_ok());
assert!(store.add_erasure_chunks(3, &relay_parent, &candidate_2.hash(), vec![erasure_chunk_2.clone()]).is_ok());
store.note_candidates_with_relay_parent(&relay_parent, &[candidate_1_hash, candidate_2_hash]).unwrap();
assert_eq!(store.block_data(relay_parent, block_data_1.hash()).unwrap(), block_data_1);
assert_eq!(store.block_data(relay_parent, block_data_2.hash()).unwrap(), block_data_2);
assert!(store.add_erasure_chunks(3, &candidate_1_hash, vec![erasure_chunk_1.clone()]).is_ok());
assert!(store.add_erasure_chunks(3, &candidate_2_hash, vec![erasure_chunk_2.clone()]).is_ok());
assert_eq!(store.get_erasure_chunk(&relay_parent, block_data_1.hash(), 1).as_ref(), Some(&erasure_chunk_1));
assert_eq!(store.get_erasure_chunk(&relay_parent, block_data_2.hash(), 1), Some(erasure_chunk_2));
assert_eq!(store.execution_data(&candidate_1_hash).unwrap(), execution_data(&available_data_1));
assert_eq!(store.execution_data(&candidate_2_hash).unwrap(), execution_data(&available_data_2));
assert_eq!(store.get_candidate(&candidate_1.hash()), Some(candidate_1.clone()));
assert_eq!(store.get_candidate(&candidate_2.hash()), Some(candidate_2.clone()));
assert_eq!(store.get_erasure_chunk(&candidate_1_hash, 1).as_ref(), Some(&erasure_chunk_1));
assert_eq!(store.get_erasure_chunk(&candidate_2_hash, 1), Some(erasure_chunk_2));
assert_eq!(store.block_data_by_candidate(relay_parent, candidate_1.hash()).unwrap(), block_data_1);
assert_eq!(store.block_data_by_candidate(relay_parent, candidate_2.hash()).unwrap(), block_data_2);
assert_eq!(store.get_candidate(&candidate_1_hash), Some(candidate_1.clone()));
assert_eq!(store.get_candidate(&candidate_2_hash), Some(candidate_2.clone()));
store.candidates_finalized(relay_parent, [candidate_1.hash()].iter().cloned().collect()).unwrap();
store.candidates_finalized(relay_parent, [candidate_1_hash].iter().cloned().collect()).unwrap();
assert_eq!(store.get_erasure_chunk(&relay_parent, block_data_1.hash(), 1).as_ref(), Some(&erasure_chunk_1));
assert!(store.get_erasure_chunk(&relay_parent, block_data_2.hash(), 1).is_none());
assert_eq!(store.get_erasure_chunk(&candidate_1_hash, 1).as_ref(), Some(&erasure_chunk_1));
assert!(store.get_erasure_chunk(&candidate_2_hash, 1).is_none());
assert_eq!(store.get_candidate(&candidate_1.hash()), Some(candidate_1));
assert_eq!(store.get_candidate(&candidate_2.hash()), None);
assert_eq!(store.get_candidate(&candidate_1_hash), Some(candidate_1));
assert_eq!(store.get_candidate(&candidate_2_hash), None);
assert_eq!(store.block_data(relay_parent, block_data_1.hash()).unwrap(), block_data_1);
assert!(store.block_data(relay_parent, block_data_2.hash()).is_none());
}
#[test]
fn queues_available_by_queue_root() {
let relay_parent = [1; 32].into();
let para_id = 5.into();
let block_data = BlockData(vec![1, 2, 3]);
let store = Store::new_in_memory();
store.make_available(Data {
relay_parent,
parachain_id: para_id,
block_data,
}).unwrap();
assert_eq!(store.execution_data(&candidate_1_hash).unwrap(), execution_data(&available_data_1));
assert!(store.execution_data(&candidate_2_hash).is_none());
}
#[test]
fn erasure_coding() {
let relay_parent: Hash = [1; 32].into();
let para_id: ParaId = 5.into();
let block_data = BlockData(vec![42; 8]);
let block_data_hash = block_data.hash();
let available_data = available_data(&[42; 8]);
let n_validators = 5;
let erasure_chunks = erasure::obtain_chunks(
n_validators,
&block_data,
&available_data,
).unwrap();
let branches = erasure::branches(erasure_chunks.as_ref());
let candidate = CandidateReceipt {
parachain_index: para_id,
collator: Default::default(),
signature: Default::default(),
head_data: Default::default(),
parent_head: Default::default(),
fees: 0,
block_data_hash: block_data.hash(),
upward_messages: Vec::new(),
erasure_root: [6; 32].into(),
};
let mut candidate = AbridgedCandidateReceipt::default();
candidate.parachain_index = para_id;
candidate.commitments.erasure_root = [6; 32].into();
candidate.relay_parent = relay_parent;
let candidate_hash = candidate.hash();
let chunks: Vec<_> = erasure_chunks
.iter()
@@ -541,13 +518,13 @@ mod tests {
let store = Store::new_in_memory();
store.add_candidate(&candidate).unwrap();
store.add_erasure_chunks(n_validators as u32, &relay_parent, &candidate.hash(), vec![chunks[0].clone()]).unwrap();
assert_eq!(store.get_erasure_chunk(&relay_parent, block_data_hash, 0), Some(chunks[0].clone()));
store.add_erasure_chunks(n_validators as u32, &candidate_hash, vec![chunks[0].clone()]).unwrap();
assert_eq!(store.get_erasure_chunk(&candidate_hash, 0), Some(chunks[0].clone()));
assert!(store.block_data(relay_parent, block_data_hash).is_none());
assert!(store.execution_data(&candidate_hash).is_none());
store.add_erasure_chunks(n_validators as u32, &relay_parent, &candidate.hash(), chunks).unwrap();
assert_eq!(store.block_data(relay_parent, block_data_hash), Some(block_data));
store.add_erasure_chunks(n_validators as u32, &candidate_hash, chunks).unwrap();
assert_eq!(store.execution_data(&candidate_hash), Some(execution_data(&available_data)));
}
#[test]
@@ -555,7 +532,7 @@ mod tests {
let relay_parent = [42; 32].into();
let store = Store::new_in_memory();
store.add_validator_index_and_n_validators(&relay_parent, 42, 24).unwrap();
store.note_validator_index_and_n_validators(&relay_parent, 42, 24).unwrap();
assert_eq!(store.get_validator_index_and_n_validators(&relay_parent).unwrap(), (42, 24));
}
@@ -566,8 +543,8 @@ mod tests {
let candidates = vec![[1; 32].into(), [2; 32].into(), [3; 32].into()];
store.add_candidates_in_relay_block(&relay_parent, candidates.clone()).unwrap();
assert_eq!(store.get_candidates_in_relay_block(&relay_parent).unwrap(), candidates);
store.note_candidates_with_relay_parent(&relay_parent, &candidates).unwrap();
assert_eq!(store.get_candidates_with_relay_parent(&relay_parent), candidates);
}
#[test]
@@ -578,25 +555,32 @@ mod tests {
let relay_parent = [42; 32].into();
let erasure_root_1 = [11; 32].into();
let erasure_root_2 = [12; 32].into();
let mut receipt_1 = CandidateReceipt::default();
let mut receipt_2 = CandidateReceipt::default();
let mut receipt_1 = AbridgedCandidateReceipt::default();
let mut receipt_2 = AbridgedCandidateReceipt::default();
receipt_1.parachain_index = 1.into();
receipt_1.erasure_root = erasure_root_1;
receipt_1.commitments.erasure_root = erasure_root_1;
receipt_1.relay_parent = relay_parent;
receipt_2.parachain_index = 2.into();
receipt_2.erasure_root = erasure_root_2;
receipt_2.commitments.erasure_root = erasure_root_2;
receipt_2.relay_parent = relay_parent;
let receipt_1_hash = receipt_1.hash();
let receipt_2_hash = receipt_2.hash();
let chunk = ErasureChunk {
chunk: vec![1, 2, 3],
index: validator_index,
proof: Vec::new(),
};
let candidates = vec![receipt_1.hash(), receipt_2.hash()];
let candidates = vec![receipt_1_hash, receipt_2_hash];
let erasure_roots = vec![erasure_root_1, erasure_root_2];
let store = Store::new_in_memory();
store.add_validator_index_and_n_validators(
store.note_validator_index_and_n_validators(
&relay_parent,
validator_index,
n_validators
@@ -605,7 +589,7 @@ mod tests {
store.add_candidate(&receipt_2).unwrap();
// We are waiting for chunks from two candidates.
store.add_candidates_in_relay_block(&relay_parent, candidates.clone()).unwrap();
store.note_candidates_with_relay_parent(&relay_parent, &candidates).unwrap();
let awaited_frontier = store.awaited_chunks().unwrap();
warn!(target: "availability", "awaited {:?}", awaited_frontier);
@@ -613,18 +597,24 @@ mod tests {
.clone()
.into_iter()
.zip(erasure_roots.iter())
.map(|(c, e)| (relay_parent, *e, c, validator_index))
.map(|(_c, &e)| AwaitedFrontierEntry {
relay_parent,
erasure_root: e,
validator_index,
})
.collect();
assert_eq!(awaited_frontier, expected);
// We add chunk from one of the candidates.
store.add_erasure_chunks(n_validators, &relay_parent, &receipt_1.hash(), vec![chunk]).unwrap();
store.add_erasure_chunks(n_validators, &receipt_1_hash, vec![chunk]).unwrap();
let awaited_frontier = store.awaited_chunks().unwrap();
// Now we wait for the other chunk that we haven't received yet.
let expected: HashSet<_> = vec![
(relay_parent, erasure_roots[1], candidates[1], validator_index)
].into_iter().collect();
let expected: HashSet<_> = vec![AwaitedFrontierEntry {
relay_parent,
erasure_root: erasure_roots[1],
validator_index,
}].into_iter().collect();
assert_eq!(awaited_frontier, expected);
polkadot/availability-store/src/worker.rs
+101 -124
View File
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::io;
use std::sync::Arc;
use std::thread;
@@ -34,15 +34,15 @@ use consensus_common::{
};
use polkadot_primitives::{Block, BlockId, Hash};
use polkadot_primitives::parachain::{
CandidateReceipt, ParachainHost, ValidatorId,
ValidatorPair, ErasureChunk, PoVBlock,
ParachainHost, ValidatorId, AbridgedCandidateReceipt, AvailableData,
ValidatorPair, ErasureChunk,
};
use futures::{prelude::*, future::select, channel::{mpsc, oneshot}, task::{Spawn, SpawnExt}};
use keystore::KeyStorePtr;
use tokio::runtime::{Handle, Runtime as LocalRuntime};
use crate::{LOG_TARGET, Data, ProvideGossipMessages, erasure_coding_topic};
use crate::{LOG_TARGET, ProvideGossipMessages, erasure_coding_topic};
use crate::store::Store;
/// Errors that may occur.
@@ -65,32 +65,27 @@ pub(crate) enum Error {
/// * when the `Store` api is used by outside code.
#[derive(Debug)]
pub(crate) enum WorkerMsg {
ErasureRoots(ErasureRoots),
ParachainBlocks(ParachainBlocks),
IncludedParachainBlocks(IncludedParachainBlocks),
ListenForChunks(ListenForChunks),
Chunks(Chunks),
CandidatesFinalized(CandidatesFinalized),
MakeAvailable(MakeAvailable),
}
/// The erasure roots of the heads included in the block with a given parent.
/// A notification of a parachain block included in the relay chain.
#[derive(Debug)]
pub(crate) struct ErasureRoots {
/// The relay parent of the block these roots belong to.
pub relay_parent: Hash,
/// The roots themselves.
pub erasure_roots: Vec<Hash>,
/// A sender to signal the result asynchronously.
pub result: oneshot::Sender<Result<(), Error>>,
pub(crate) struct IncludedParachainBlock {
/// The abridged candidate receipt, extracted from a relay-chain block.
pub candidate: AbridgedCandidateReceipt,
/// The data to keep available from the candidate, if known.
pub available_data: Option<AvailableData>,
}
/// The receipts of the heads included into the block with a given parent.
#[derive(Debug)]
pub(crate) struct ParachainBlocks {
/// The relay parent of the block these parachain blocks belong to.
pub relay_parent: Hash,
pub(crate) struct IncludedParachainBlocks {
/// The blocks themselves.
pub blocks: Vec<(CandidateReceipt, Option<PoVBlock>)>,
pub blocks: Vec<IncludedParachainBlock>,
/// A sender to signal the result asynchronously.
pub result: oneshot::Sender<Result<(), Error>>,
}
@@ -98,8 +93,6 @@ pub(crate) struct ParachainBlocks {
/// Listen gossip for these chunks.
#[derive(Debug)]
pub(crate) struct ListenForChunks {
/// The relay parent of the block the chunks from we want to listen to.
pub relay_parent: Hash,
/// The hash of the candidate chunk belongs to.
pub candidate_hash: Hash,
/// The index of the chunk we need.
@@ -126,15 +119,17 @@ pub(crate) struct Chunks {
pub(crate) struct CandidatesFinalized {
/// The relay parent of the block that was finalized.
relay_parent: Hash,
/// The parachain heads that were finalized in this block.
candidate_hashes: Vec<Hash>,
/// The hashes of candidates that were finalized in this block.
included_candidates: HashSet<Hash>,
}
/// The message that corresponds to `make_available` call of the crate API.
#[derive(Debug)]
pub(crate) struct MakeAvailable {
/// The data being made available.
pub data: Data,
/// The hash of the candidate for which we are publishing data.
pub candidate_hash: Hash,
/// The data to make available.
pub available_data: AvailableData,
/// A sender to signal the result asynchronously.
pub result: oneshot::Sender<Result<(), Error>>,
}
@@ -205,7 +200,7 @@ where
fn fetch_candidates<P>(client: &P, extrinsics: Vec<<Block as BlockT>::Extrinsic>, parent: &BlockId)
-> ClientResult<Option<Vec<CandidateReceipt>>>
-> ClientResult<Option<Vec<AbridgedCandidateReceipt>>>
where
P: ProvideRuntimeApi<Block>,
P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
@@ -262,12 +257,15 @@ where
}
};
let candidate_hashes = match fetch_candidates(
let included_candidates = match fetch_candidates(
&*client,
extrinsics,
&BlockId::hash(parent_hash)
&BlockId::hash(parent_hash),
) {
Ok(Some(candidates)) => candidates.into_iter().map(|c| c.hash()).collect(),
Ok(Some(candidates)) => candidates
.into_iter()
.map(|c| c.hash())
.collect(),
Ok(None) => {
warn!(
target: LOG_TARGET,
@@ -286,7 +284,7 @@ where
let msg = WorkerMsg::CandidatesFinalized(CandidatesFinalized {
relay_parent: parent_hash,
candidate_hashes
included_candidates
});
if let Err(_) = sender.send(msg).await {
@@ -315,12 +313,12 @@ where
sender: &mut mpsc::UnboundedSender<WorkerMsg>,
) {
if let Some(awaited_chunks) = self.availability_store.awaited_chunks() {
for chunk in awaited_chunks {
for awaited_chunk in awaited_chunks {
if let Err(e) = self.register_chunks_listener(
runtime_handle,
sender,
chunk.0,
chunk.1,
awaited_chunk.relay_parent,
awaited_chunk.erasure_root,
) {
warn!(target: LOG_TARGET, "Failed to register gossip listener: {}", e);
}
@@ -366,33 +364,36 @@ where
&mut self,
runtime_handle: &Handle,
sender: &mut mpsc::UnboundedSender<WorkerMsg>,
relay_parent: Hash,
blocks: Vec<(CandidateReceipt, Option<PoVBlock>)>,
blocks: Vec<IncludedParachainBlock>,
) -> Result<(), Error> {
let hashes: Vec<_> = blocks.iter().map(|(c, _)| c.hash()).collect();
// First we have to add the receipts themselves.
for (candidate, block) in blocks.into_iter() {
for IncludedParachainBlock { candidate, available_data }
in blocks.into_iter()
{
let _ = self.availability_store.add_candidate(&candidate);
if let Some(_block) = block {
if let Some(_available_data) = available_data {
// Should we be breaking block into chunks here and gossiping it and so on?
}
if let Err(e) = self.register_chunks_listener(
runtime_handle,
sender,
relay_parent,
candidate.erasure_root
candidate.relay_parent,
candidate.commitments.erasure_root,
) {
warn!(target: LOG_TARGET, "Failed to register chunk listener: {}", e);
}
}
let _ = self.availability_store.add_candidates_in_relay_block(
&relay_parent,
hashes
);
// This leans on the codebase-wide assumption that the `relay_parent`
// of all candidates in a block matches the parent hash of that block.
//
// In the future this will not always be true.
let _ = self.availability_store.note_candidates_with_relay_parent(
&candidate.relay_parent,
&[candidate.hash()],
);
}
Ok(())
}
@@ -415,7 +416,11 @@ where
.ok_or(Error::CandidateNotFound { candidate_hash })?;
for chunk in &chunks {
let topic = erasure_coding_topic(relay_parent, receipt.erasure_root, chunk.index);
let topic = erasure_coding_topic(
relay_parent,
receipt.commitments.erasure_root,
chunk.index,
);
// need to remove gossip listener and stop it.
if let Some(signal) = self.registered_gossip_streams.remove(&topic) {
let _ = signal.fire();
@@ -424,7 +429,6 @@ where
self.availability_store.add_erasure_chunks(
n_validators,
&relay_parent,
&candidate_hash,
chunks,
)?;
@@ -432,17 +436,6 @@ where
Ok(())
}
// Adds the erasure roots into the store.
fn on_erasure_roots_received(
&mut self,
relay_parent: Hash,
erasure_roots: Vec<Hash>
) -> Result<(), Error> {
self.availability_store.add_erasure_roots_in_relay_block(&relay_parent, erasure_roots)?;
Ok(())
}
// Processes the `ListenForChunks` message.
//
// When the worker receives a `ListenForChunk` message, it double-checks that
@@ -451,7 +444,6 @@ where
&mut self,
runtime_handle: &Handle,
sender: &mut mpsc::UnboundedSender<WorkerMsg>,
relay_parent: Hash,
candidate_hash: Hash,
id: usize
) -> Result<(), Error> {
@@ -459,13 +451,13 @@ where
.ok_or(Error::CandidateNotFound { candidate_hash })?;
if self.availability_store
.get_erasure_chunk(&relay_parent, candidate.block_data_hash, id)
.get_erasure_chunk(&candidate_hash, id)
.is_none() {
if let Err(e) = self.register_chunks_listener(
runtime_handle,
sender,
relay_parent,
candidate.erasure_root
candidate.relay_parent,
candidate.commitments.erasure_root
) {
warn!(target: LOG_TARGET, "Failed to register a gossip listener: {}", e);
}
@@ -498,7 +490,7 @@ where
let runtime_handle = runtime.handle().clone();
// On startup, registers listeners (gossip streams) for all
// (relay_parent, erasure-root, i) in the awaited frontier.
// entries in the awaited frontier.
worker.register_listeners(runtime.handle(), &mut sender);
let process_notification = async move {
@@ -506,18 +498,8 @@ where
trace!(target: LOG_TARGET, "Received message {:?}", msg);
let res = match msg {
WorkerMsg::ErasureRoots(msg) => {
let ErasureRoots { relay_parent, erasure_roots, result} = msg;
let res = worker.on_erasure_roots_received(
relay_parent,
erasure_roots,
);
let _ = result.send(res);
Ok(())
}
WorkerMsg::ListenForChunks(msg) => {
let ListenForChunks {
relay_parent,
candidate_hash,
index,
result,
@@ -526,7 +508,6 @@ where
let res = worker.on_listen_for_chunks_received(
&runtime_handle,
&mut sender,
relay_parent,
candidate_hash,
index as usize,
);
@@ -536,9 +517,8 @@ where
}
Ok(())
}
WorkerMsg::ParachainBlocks(msg) => {
let ParachainBlocks {
relay_parent,
WorkerMsg::IncludedParachainBlocks(msg) => {
let IncludedParachainBlocks {
blocks,
result,
} = msg;
@@ -546,7 +526,6 @@ where
let res = worker.on_parachain_blocks_received(
&runtime_handle,
&mut sender,
relay_parent,
blocks,
);
@@ -565,16 +544,17 @@ where
Ok(())
}
WorkerMsg::CandidatesFinalized(msg) => {
let CandidatesFinalized { relay_parent, candidate_hashes } = msg;
let CandidatesFinalized { relay_parent, included_candidates } = msg;
worker.availability_store.candidates_finalized(
relay_parent,
candidate_hashes.into_iter().collect(),
included_candidates,
)
}
WorkerMsg::MakeAvailable(msg) => {
let MakeAvailable { data, result } = msg;
let res = worker.availability_store.make_available(data)
let MakeAvailable { candidate_hash, available_data, result } = msg;
let res = worker.availability_store
.make_available(candidate_hash, available_data)
.map_err(|e| e.into());
let _ = result.send(res);
Ok(())
@@ -651,7 +631,6 @@ impl<I, P> BlockImport<Block> for AvailabilityBlockImport<I, P> where
);
if let Some(ref extrinsics) = block.body {
let relay_parent = *block.header.parent_hash();
let parent_id = BlockId::hash(*block.header.parent_hash());
// Extract our local position i from the validator set of the parent.
let validators = self.client.runtime_api().validators(&parent_id)
@@ -675,16 +654,15 @@ impl<I, P> BlockImport<Block> for AvailabilityBlockImport<I, P> where
);
for candidate in &candidates {
let candidate_hash = candidate.hash();
// If we don't yet have our chunk of this candidate,
// tell the worker to listen for one.
if self.availability_store.get_erasure_chunk(
&relay_parent,
candidate.block_data_hash,
&candidate_hash,
our_id as usize,
).is_none() {
let msg = WorkerMsg::ListenForChunks(ListenForChunks {
relay_parent,
candidate_hash: candidate.hash(),
candidate_hash,
index: our_id as u32,
result: None,
});
@@ -693,27 +671,19 @@ impl<I, P> BlockImport<Block> for AvailabilityBlockImport<I, P> where
}
}
let erasure_roots: Vec<_> = candidates
.iter()
.map(|c| c.erasure_root)
.collect();
// Inform the worker about new (relay_parent, erasure_roots) pairs
let (s, _) = oneshot::channel();
let msg = WorkerMsg::ErasureRoots(ErasureRoots {
relay_parent,
erasure_roots,
result: s,
});
let _ = self.to_worker.unbounded_send(msg);
let (s, _) = oneshot::channel();
// Inform the worker about the included parachain blocks.
let msg = WorkerMsg::ParachainBlocks(ParachainBlocks {
relay_parent,
blocks: candidates.into_iter().map(|c| (c, None)).collect(),
let blocks = candidates
.into_iter()
.map(|c| IncludedParachainBlock {
candidate: c,
available_data: None,
})
.collect();
let msg = WorkerMsg::IncludedParachainBlocks(IncludedParachainBlocks {
blocks,
result: s,
});
@@ -806,6 +776,7 @@ mod tests {
use std::sync::{Arc, Mutex, Condvar};
use std::pin::Pin;
use tokio::runtime::Runtime;
use crate::store::AwaitedFrontierEntry;
// Just contains topic->channel mapping to give to outer code on `gossip_messages_for` calls.
struct TestGossipMessages {
@@ -866,7 +837,7 @@ mod tests {
let store = Store::new_in_memory();
// Tell the store our validator's position and the number of validators at given point.
store.add_validator_index_and_n_validators(&relay_parent, local_id, n_validators).unwrap();
store.note_validator_index_and_n_validators(&relay_parent, local_id, n_validators).unwrap();
let (gossip_sender, gossip_receiver) = mpsc::unbounded();
@@ -879,9 +850,10 @@ mod tests {
].into_iter().collect()))
};
let mut candidate = CandidateReceipt::default();
let mut candidate = AbridgedCandidateReceipt::default();
candidate.erasure_root = erasure_root;
candidate.commitments.erasure_root = erasure_root;
candidate.relay_parent = relay_parent;
let candidate_hash = candidate.hash();
// At this point we shouldn't be waiting for any chunks.
@@ -889,9 +861,11 @@ mod tests {
let (s, r) = oneshot::channel();
let msg = WorkerMsg::ParachainBlocks(ParachainBlocks {
relay_parent,
blocks: vec![(candidate, None)],
let msg = WorkerMsg::IncludedParachainBlocks(IncludedParachainBlocks {
blocks: vec![IncludedParachainBlock {
candidate,
available_data: None,
}],
result: s,
});
@@ -907,7 +881,11 @@ mod tests {
// Make sure that at this point we are waiting for the appropriate chunk.
assert_eq!(
store.awaited_chunks().unwrap(),
vec![(relay_parent, erasure_root, candidate_hash, local_id)].into_iter().collect()
vec![AwaitedFrontierEntry {
relay_parent,
erasure_root,
validator_index: local_id,
}].into_iter().collect()
);
let msg = (
@@ -941,25 +919,27 @@ mod tests {
let relay_parent = [1; 32].into();
let erasure_root_1 = [2; 32].into();
let erasure_root_2 = [3; 32].into();
let block_data_hash_1 = [4; 32].into();
let block_data_hash_2 = [5; 32].into();
let pov_block_hash_1 = [4; 32].into();
let pov_block_hash_2 = [5; 32].into();
let local_id = 2;
let n_validators = 4;
let mut candidate_1 = CandidateReceipt::default();
candidate_1.erasure_root = erasure_root_1;
candidate_1.block_data_hash = block_data_hash_1;
let mut candidate_1 = AbridgedCandidateReceipt::default();
candidate_1.commitments.erasure_root = erasure_root_1;
candidate_1.pov_block_hash = pov_block_hash_1;
candidate_1.relay_parent = relay_parent;
let candidate_1_hash = candidate_1.hash();
let mut candidate_2 = CandidateReceipt::default();
candidate_2.erasure_root = erasure_root_2;
candidate_2.block_data_hash = block_data_hash_2;
let mut candidate_2 = AbridgedCandidateReceipt::default();
candidate_2.commitments.erasure_root = erasure_root_2;
candidate_2.pov_block_hash = pov_block_hash_2;
candidate_2.relay_parent = relay_parent;
let candidate_2_hash = candidate_2.hash();
let store = Store::new_in_memory();
// Tell the store our validator's position and the number of validators at given point.
store.add_validator_index_and_n_validators(&relay_parent, local_id, n_validators).unwrap();
store.note_validator_index_and_n_validators(&relay_parent, local_id, n_validators).unwrap();
// Let the store know about the candidates
store.add_candidate(&candidate_1).unwrap();
@@ -968,7 +948,6 @@ mod tests {
// And let the store know about the chunk from the second candidate.
store.add_erasure_chunks(
n_validators,
&relay_parent,
&candidate_2_hash,
vec![ErasureChunk {
chunk: vec![1, 2, 3],
@@ -999,7 +978,6 @@ mod tests {
let (s2, r2) = oneshot::channel();
// Tell the worker to listen for chunks from candidate 2 (we alredy have a chunk from it).
let listen_msg_2 = WorkerMsg::ListenForChunks(ListenForChunks {
relay_parent,
candidate_hash: candidate_2_hash,
index: local_id as u32,
result: Some(s2),
@@ -1016,7 +994,6 @@ mod tests {
// Tell the worker to listen for chunks from candidate 1.
// (we don't have a chunk from it yet).
let listen_msg_1 = WorkerMsg::ListenForChunks(ListenForChunks {
relay_parent,
candidate_hash: candidate_1_hash,
index: local_id as u32,
result: Some(s1),
polkadot/collator/src/lib.rs
+23 -20
View File
@@ -57,7 +57,7 @@ use polkadot_primitives::{
BlockId, Hash, Block,
parachain::{
self, BlockData, DutyRoster, HeadData, Id as ParaId,
PoVBlock, Status as ParachainStatus, ValidatorId, CollatorPair,
PoVBlock, ValidatorId, CollatorPair, LocalValidationData
}
};
use polkadot_cli::{
@@ -167,7 +167,7 @@ pub trait ParachainContext: Clone {
fn produce_candidate(
&mut self,
relay_parent: Hash,
status: ParachainStatus,
status: LocalValidationData,
) -> Self::ProduceCandidate;
}
@@ -175,7 +175,7 @@ pub trait ParachainContext: Clone {
pub async fn collate<P>(
relay_parent: Hash,
local_id: ParaId,
parachain_status: ParachainStatus,
local_validation_data: LocalValidationData,
mut para_context: P,
key: Arc<CollatorPair>,
)
@@ -186,25 +186,32 @@ pub async fn collate<P>(
{
let (block_data, head_data) = para_context.produce_candidate(
relay_parent,
parachain_status,
local_validation_data,
).map_err(Error::Collator).await?;
let block_data_hash = block_data.hash();
let signature = key.sign(block_data_hash.as_ref());
let pov_block = PoVBlock {
block_data,
};
let pov_block_hash = pov_block.hash();
let signature = key.sign(&parachain::collator_signature_payload(
&relay_parent,
&local_id,
&pov_block_hash,
));
let info = parachain::CollationInfo {
parachain_index: local_id,
relay_parent,
collator: key.public(),
signature,
head_data,
block_data_hash,
upward_messages: Vec::new(),
pov_block_hash,
};
let collation = parachain::Collation {
info,
pov: PoVBlock {
block_data,
},
pov: pov_block,
};
Ok(collation)
@@ -318,8 +325,8 @@ fn run_collator_node<S, P, Extrinsic>(
let work = future::lazy(move |_| {
let api = client.runtime_api();
let status = match try_fr!(api.parachain_status(&id, para_id)) {
Some(status) => status,
let local_validation = match try_fr!(api.local_validation_data(&id, para_id)) {
Some(local_validation) => local_validation,
None => return future::Either::Left(future::ok(())),
};
@@ -334,19 +341,17 @@ fn run_collator_node<S, P, Extrinsic>(
let collation_work = collate(
relay_parent,
para_id,
status,
local_validation,
parachain_context,
key,
).map_ok(move |collation| {
network.with_spec(move |spec, ctx| {
let res = spec.add_local_collation(
spec.add_local_collation(
ctx,
relay_parent,
targets,
collation,
);
tokio::spawn(res.boxed());
})
});
@@ -442,8 +447,6 @@ pub fn run_collator<P>(
#[cfg(test)]
mod tests {
use polkadot_primitives::parachain::FeeSchedule;
use keyring::Sr25519Keyring;
use super::*;
#[derive(Clone)]
@@ -455,7 +458,7 @@ mod tests {
fn produce_candidate(
&mut self,
_relay_parent: Hash,
_status: ParachainStatus,
_local_validation: LocalValidationData,
) -> Self::ProduceCandidate {
// send messages right back.
future::ok((
polkadot/erasure-coding/src/lib.rs
+27 -10
View File
@@ -27,7 +27,7 @@
use codec::{Encode, Decode};
use reed_solomon::galois_16::{self, ReedSolomon};
use primitives::{Hash as H256, BlakeTwo256, HashT};
use primitives::parachain::BlockData;
use primitives::parachain::AvailableData;
use sp_core::Blake2Hasher;
use trie::{EMPTY_PREFIX, MemoryDB, Trie, TrieMut, trie_types::{TrieDBMut, TrieDB}};
@@ -66,6 +66,8 @@ pub enum Error {
BranchOutOfBounds,
}
impl std::error::Error for Error { }
#[derive(Debug, PartialEq)]
struct CodeParams {
data_shards: usize,
@@ -125,11 +127,11 @@ fn code_params(n_validators: usize) -> Result<CodeParams, Error> {
/// Obtain erasure-coded chunks, one for each validator.
///
/// Works only up to 65536 validators, and `n_validators` must be non-zero.
pub fn obtain_chunks(n_validators: usize, block_data: &BlockData)
pub fn obtain_chunks(n_validators: usize, available_data: &AvailableData)
-> Result<Vec<Vec<u8>>, Error>
{
let params = code_params(n_validators)?;
let encoded = block_data.encode();
let encoded = available_data.encode();
if encoded.is_empty() {
return Err(Error::BadPayload);
@@ -151,7 +153,7 @@ pub fn obtain_chunks(n_validators: usize, block_data: &BlockData)
///
/// Works only up to 65536 validators, and `n_validators` must be non-zero.
pub fn reconstruct<'a, I: 'a>(n_validators: usize, chunks: I)
-> Result<BlockData, Error>
-> Result<AvailableData, Error>
where I: IntoIterator<Item=(&'a [u8], usize)>
{
let params = code_params(n_validators)?;
@@ -341,6 +343,7 @@ impl<'a, I: Iterator<Item=&'a [u8]>> codec::Input for ShardInput<'a, I> {
#[cfg(test)]
mod tests {
use super::*;
use primitives::parachain::{BlockData, PoVBlock};
#[test]
fn field_order_is_right_size() {
@@ -400,11 +403,18 @@ mod tests {
}
#[test]
fn round_trip_block_data() {
let block_data = BlockData((0..255).collect());
fn round_trip_works() {
let pov_block = PoVBlock {
block_data: BlockData((0..255).collect()),
};
let available_data = AvailableData {
pov_block,
omitted_validation: Default::default(),
};
let chunks = obtain_chunks(
10,
&block_data,
&available_data,
).unwrap();
assert_eq!(chunks.len(), 10);
@@ -420,16 +430,23 @@ mod tests {
].iter().cloned(),
).unwrap();
assert_eq!(reconstructed, block_data);
assert_eq!(reconstructed, available_data);
}
#[test]
fn construct_valid_branches() {
let block_data = BlockData(vec![2; 256]);
let pov_block = PoVBlock {
block_data: BlockData(vec![2; 256]),
};
let available_data = AvailableData {
pov_block,
omitted_validation: Default::default(),
};
let chunks = obtain_chunks(
10,
&block_data,
&available_data,
).unwrap();
assert_eq!(chunks.len(), 10);
polkadot/network/src/legacy/collator_pool.rs
+9 -23
View File
@@ -236,7 +236,7 @@ impl CollatorPool {
mod tests {
use super::*;
use sp_core::crypto::UncheckedInto;
use polkadot_primitives::parachain::{CandidateReceipt, BlockData, PoVBlock, HeadData};
use polkadot_primitives::parachain::{CollationInfo, BlockData, PoVBlock};
use futures::executor::block_on;
fn make_pov(block_data: Vec<u8>) -> PoVBlock {
@@ -284,18 +284,11 @@ mod tests {
let (tx2, rx2) = oneshot::channel();
pool.await_collation(relay_parent, para_id, tx1);
pool.await_collation(relay_parent, para_id, tx2);
let mut collation_info = CollationInfo::default();
collation_info.parachain_index = para_id;
collation_info.collator = primary.clone().into();
pool.on_collation(primary.clone(), relay_parent, Collation {
info: CandidateReceipt {
parachain_index: para_id,
collator: primary.clone().into(),
signature: Default::default(),
head_data: HeadData(vec![1, 2, 3]),
parent_head: HeadData(vec![]),
fees: 0,
block_data_hash: [3; 32].into(),
upward_messages: Vec::new(),
erasure_root: [1u8; 32].into(),
}.into(),
info: collation_info,
pov: make_pov(vec![4, 5, 6]),
});
@@ -313,18 +306,11 @@ mod tests {
assert_eq!(pool.on_new_collator(primary.clone(), para_id.clone(), PeerId::random()), Role::Primary);
let mut collation_info = CollationInfo::default();
collation_info.parachain_index = para_id;
collation_info.collator = primary.clone();
pool.on_collation(primary.clone(), relay_parent, Collation {
info: CandidateReceipt {
parachain_index: para_id,
collator: primary,
signature: Default::default(),
head_data: HeadData(vec![1, 2, 3]),
parent_head: HeadData(vec![]),
fees: 0,
block_data_hash: [3; 32].into(),
upward_messages: Vec::new(),
erasure_root: [1u8; 32].into(),
}.into(),
info: collation_info,
pov: make_pov(vec![4, 5, 6]),
});
polkadot/network/src/legacy/gossip/mod.rs
+10 -22
View File
@@ -540,7 +540,7 @@ impl<C: ?Sized + ChainContext> Inner<C> {
if let Some(store) = &self.availability_store {
if let Some(receipt) = store.get_candidate(&msg.candidate_hash) {
let chunk_hash = erasure::branch_hash(
&receipt.erasure_root,
&receipt.commitments.erasure_root,
&msg.chunk.proof,
msg.chunk.index as usize
);
@@ -552,15 +552,15 @@ impl<C: ?Sized + ChainContext> Inner<C> {
)
} else {
if let Some(awaited_chunks) = store.awaited_chunks() {
if awaited_chunks.contains(&(
msg.relay_parent,
receipt.erasure_root,
receipt.hash(),
msg.chunk.index,
)) {
let frontier_entry = av_store::AwaitedFrontierEntry {
relay_parent: msg.relay_parent,
erasure_root: receipt.commitments.erasure_root,
validator_index: msg.chunk.index,
};
if awaited_chunks.contains(&frontier_entry) {
let topic = av_store::erasure_coding_topic(
msg.relay_parent,
receipt.erasure_root,
receipt.commitments.erasure_root,
msg.chunk.index,
);
@@ -722,8 +722,7 @@ mod tests {
use sc_network_gossip::Validator as ValidatorT;
use std::sync::mpsc;
use parking_lot::Mutex;
use polkadot_primitives::parachain::{CandidateReceipt, HeadData};
use sp_core::crypto::UncheckedInto;
use polkadot_primitives::parachain::AbridgedCandidateReceipt;
use sp_core::sr25519::Signature as Sr25519Signature;
use polkadot_validation::GenericStatement;
@@ -807,18 +806,7 @@ mod tests {
validator_context.clear();
let candidate_receipt = CandidateReceipt {
parachain_index: 5.into(),
collator: [255; 32].unchecked_into(),
head_data: HeadData(vec![9, 9, 9]),
parent_head: HeadData(vec![]),
signature: Default::default(),
fees: 1_000_000,
block_data_hash: [20u8; 32].into(),
upward_messages: Vec::new(),
erasure_root: [1u8; 32].into(),
};
let candidate_receipt = AbridgedCandidateReceipt::default();
let statement = GossipMessage::Statement(GossipStatement {
relay_chain_leaf: hash_a,
signed_statement: SignedStatement {
polkadot/network/src/legacy/mod.rs
+7 -20
View File
@@ -30,7 +30,7 @@ use futures::channel::oneshot;
use futures::prelude::*;
use polkadot_primitives::{Block, Hash, Header};
use polkadot_primitives::parachain::{
Id as ParaId, CollatorId, CandidateReceipt, Collation, PoVBlock,
Id as ParaId, CollatorId, AbridgedCandidateReceipt, Collation, PoVBlock,
ValidatorId, ErasureChunk,
};
use sc_network::{
@@ -168,7 +168,7 @@ struct PoVBlockRequest {
attempted_peers: HashSet<ValidatorId>,
validation_leaf: Hash,
candidate_hash: Hash,
block_data_hash: Hash,
pov_block_hash: Hash,
sender: oneshot::Sender<PoVBlock>,
}
@@ -178,7 +178,7 @@ impl PoVBlockRequest {
//
// If `Ok(())` is returned, that indicates that the request has been processed.
fn process_response(self, pov_block: PoVBlock) -> Result<(), Self> {
if pov_block.block_data.hash() != self.block_data_hash {
if pov_block.hash() != self.pov_block_hash {
return Err(self);
}
@@ -292,7 +292,7 @@ impl PolkadotProtocol {
fn fetch_pov_block(
&mut self,
ctx: &mut dyn Context<Block>,
candidate: &CandidateReceipt,
candidate: &AbridgedCandidateReceipt,
relay_parent: Hash,
) -> oneshot::Receiver<PoVBlock> {
let (tx, rx) = oneshot::channel();
@@ -301,7 +301,7 @@ impl PolkadotProtocol {
attempted_peers: Default::default(),
validation_leaf: relay_parent,
candidate_hash: candidate.hash(),
block_data_hash: candidate.block_data_hash,
pov_block_hash: candidate.pov_block_hash,
sender: tx,
});
@@ -608,7 +608,7 @@ impl Specialization<Block> for PolkadotProtocol {
attempted_peers: Default::default(),
validation_leaf: Default::default(),
candidate_hash: Default::default(),
block_data_hash: Default::default(),
pov_block_hash: Default::default(),
sender,
}));
}
@@ -738,7 +738,7 @@ impl PolkadotProtocol {
relay_parent: Hash,
targets: HashSet<ValidatorId>,
collation: Collation,
) -> impl Future<Output = ()> {
) {
debug!(target: "p_net", "Importing local collation on relay parent {:?} and parachain {:?}",
relay_parent, collation.info.parachain_index);
@@ -756,19 +756,6 @@ impl PolkadotProtocol {
warn!(target: "polkadot_network", "Encountered tracked but disconnected validator {:?}", primary),
}
}
let availability_store = self.availability_store.clone();
let collation_cloned = collation.clone();
async move {
if let Some(availability_store) = availability_store {
let _ = availability_store.make_available(av_store::Data {
relay_parent,
parachain_id: collation_cloned.info.parachain_index,
block_data: collation_cloned.pov.block_data.clone(),
}).await;
}
}
}
/// Give the network protocol a handle to an availability store, used for
polkadot/network/src/legacy/router.rs
+13 -13
View File
@@ -29,7 +29,7 @@ use polkadot_validation::{
};
use polkadot_primitives::{Block, Hash};
use polkadot_primitives::parachain::{
CandidateReceipt, ParachainHost, ValidatorIndex, Collation, PoVBlock, ErasureChunk,
AbridgedCandidateReceipt, ParachainHost, ValidatorIndex, PoVBlock, ErasureChunk,
};
use sp_api::ProvideRuntimeApi;
@@ -204,17 +204,17 @@ impl<P: ProvideRuntimeApi<Block> + Send + Sync + 'static, T> Router<P, T> where
// store the data before broadcasting statements, so other peers can fetch.
knowledge.lock().note_candidate(
candidate_hash,
Some(validated.0.pov_block().clone()),
Some(validated.pov_block().clone()),
);
// propagate the statement.
// consider something more targeted than gossip in the future.
let statement = GossipStatement::new(
parent_hash,
match table.import_validated(validated.0) {
None => return,
Some(s) => s,
}
parent_hash,
match table.import_validated(validated) {
None => return,
Some(s) => s,
}
);
network.gossip_message(attestation_topic, statement.into());
@@ -238,16 +238,16 @@ impl<P: ProvideRuntimeApi<Block> + Send, T> TableRouter for Router<P, T> where
// We have fetched from a collator and here the receipt should have been already formed.
fn local_collation(
&self,
collation: Collation,
receipt: CandidateReceipt,
receipt: AbridgedCandidateReceipt,
pov_block: PoVBlock,
chunks: (ValidatorIndex, &[ErasureChunk])
) -> Self::SendLocalCollation {
// produce a signed statement
let hash = receipt.hash();
let erasure_root = receipt.erasure_root;
let erasure_root = receipt.commitments.erasure_root;
let validated = Validated::collated_local(
receipt,
collation.pov.clone(),
pov_block.clone(),
);
let statement = GossipStatement::new(
@@ -259,7 +259,7 @@ impl<P: ProvideRuntimeApi<Block> + Send, T> TableRouter for Router<P, T> where
);
// give to network to make available.
self.fetcher.knowledge().lock().note_candidate(hash, Some(collation.pov));
self.fetcher.knowledge().lock().note_candidate(hash, Some(pov_block));
self.network().gossip_message(self.attestation_topic, statement.into());
for chunk in chunks.1 {
@@ -279,7 +279,7 @@ impl<P: ProvideRuntimeApi<Block> + Send, T> TableRouter for Router<P, T> where
future::ready(Ok(()))
}
fn fetch_pov_block(&self, candidate: &CandidateReceipt) -> Self::FetchValidationProof {
fn fetch_pov_block(&self, candidate: &AbridgedCandidateReceipt) -> Self::FetchValidationProof {
self.fetcher.fetch_pov_block(candidate)
}
}
polkadot/network/src/legacy/tests/validation.rs
+24 -20
View File
@@ -28,8 +28,9 @@ use crate::legacy::{PolkadotProtocol, NetworkService, GossipService, GossipMessa
use polkadot_validation::{SharedTable, Network};
use polkadot_primitives::{Block, BlockNumber, Hash, Header, BlockId};
use polkadot_primitives::parachain::{
Id as ParaId, Chain, DutyRoster, ParachainHost, ValidatorId, Status,
FeeSchedule, HeadData, Retriable, CollatorId, ErasureChunk, CandidateReceipt,
Id as ParaId, Chain, DutyRoster, ParachainHost, ValidatorId,
FeeSchedule, HeadData, Retriable, CollatorId, ErasureChunk, AbridgedCandidateReceipt,
GlobalValidationSchedule, LocalValidationData,
};
use parking_lot::Mutex;
use sp_blockchain::Result as ClientResult;
@@ -270,23 +271,6 @@ impl ParachainHost<Block> for RuntimeApi {
Ok(NativeOrEncoded::Native(self.data.lock().active_parachains.clone()))
}
fn ParachainHost_parachain_status_runtime_api_impl(
&self,
_at: &BlockId,
_: ExecutionContext,
_: Option<ParaId>,
_: Vec<u8>,
) -> ClientResult<NativeOrEncoded<Option<Status>>> {
Ok(NativeOrEncoded::Native(Some(Status {
head_data: HeadData(Vec::new()),
balance: 0,
fee_schedule: FeeSchedule {
base: 0,
per_byte: 0,
}
})))
}
fn ParachainHost_parachain_code_runtime_api_impl(
&self,
_at: &BlockId,
@@ -297,13 +281,33 @@ impl ParachainHost<Block> for RuntimeApi {
Ok(NativeOrEncoded::Native(Some(Vec::new())))
}
fn ParachainHost_global_validation_schedule_runtime_api_impl(
&self,
_at: &BlockId,
_: ExecutionContext,
_: Option<()>,
_: Vec<u8>,
) -> ClientResult<NativeOrEncoded<GlobalValidationSchedule>> {
Ok(NativeOrEncoded::Native(Default::default()))
}
fn ParachainHost_local_validation_data_runtime_api_impl(
&self,
_at: &BlockId,
_: ExecutionContext,
_: Option<ParaId>,
_: Vec<u8>,
) -> ClientResult<NativeOrEncoded<Option<LocalValidationData>>> {
Ok(NativeOrEncoded::Native(Some(Default::default())))
}
fn ParachainHost_get_heads_runtime_api_impl(
&self,
_at: &BlockId,
_: ExecutionContext,
_extrinsics: Option<Vec<<Block as BlockT>::Extrinsic>>,
_: Vec<u8>,
) -> ClientResult<NativeOrEncoded<Option<Vec<CandidateReceipt>>>> {
) -> ClientResult<NativeOrEncoded<Option<Vec<AbridgedCandidateReceipt>>>> {
Ok(NativeOrEncoded::Native(Some(Vec::new())))
}
}
polkadot/network/src/legacy/validation.rs
+2 -2
View File
@@ -25,7 +25,7 @@ use polkadot_validation::{
};
use polkadot_primitives::{Block, Hash};
use polkadot_primitives::parachain::{
Id as ParaId, Collation, ParachainHost, CandidateReceipt, CollatorId,
Id as ParaId, Collation, ParachainHost, AbridgedCandidateReceipt, CollatorId,
ValidatorId, PoVBlock,
};
use sp_api::ProvideRuntimeApi;
@@ -557,7 +557,7 @@ impl<P: ProvideRuntimeApi<Block> + Send, T> LeafWorkDataFetcher<P, T> where
T: Clone + Executor + Send + 'static,
{
/// Fetch PoV block for the given candidate receipt.
pub fn fetch_pov_block(&self, candidate: &CandidateReceipt)
pub fn fetch_pov_block(&self, candidate: &AbridgedCandidateReceipt)
-> Pin<Box<dyn Future<Output = Result<PoVBlock, io::Error>> + Send>> {
let parent_hash = self.parent_hash;
polkadot/network/src/protocol.rs
+19 -17
View File
@@ -31,7 +31,7 @@ use log::{debug, trace};
use polkadot_primitives::{
Hash, Block,
parachain::{
PoVBlock, ValidatorId, ValidatorIndex, Collation, CandidateReceipt,
PoVBlock, ValidatorId, ValidatorIndex, Collation, AbridgedCandidateReceipt,
ErasureChunk, ParachainHost, Id as ParaId, CollatorId,
},
};
@@ -72,13 +72,13 @@ enum ServiceToWorkerMsg {
DropConsensusNetworking(Hash),
LocalCollation(
Hash, // relay-parent
Collation,
CandidateReceipt,
AbridgedCandidateReceipt,
PoVBlock,
(ValidatorIndex, Vec<ErasureChunk>),
),
FetchPoVBlock(
Hash, // relay-parent
CandidateReceipt,
AbridgedCandidateReceipt,
oneshot::Sender<PoVBlock>,
),
AwaitCollation(
@@ -650,7 +650,7 @@ async fn worker_loop<Api, Sp>(
ServiceToWorkerMsg::DropConsensusNetworking(relay_parent) => {
consensus_instances.remove(&relay_parent);
}
ServiceToWorkerMsg::LocalCollation(relay_parent, collation, receipt, chunks) => {
ServiceToWorkerMsg::LocalCollation(relay_parent, receipt, pov_block, chunks) => {
let instance = match consensus_instances.get(&relay_parent) {
None => continue,
Some(instance) => instance,
@@ -658,8 +658,8 @@ async fn worker_loop<Api, Sp>(
distribute_local_collation(
instance,
collation,
receipt,
pov_block,
chunks,
&gossip_handle,
);
@@ -686,7 +686,7 @@ async fn statement_import_loop<Api>(
table: Arc<SharedTable>,
api: Arc<Api>,
weak_router: Weak<RouterInner>,
validator: RegisteredMessageValidator,
gossip_handle: RegisteredMessageValidator,
mut exit: exit_future::Exit,
executor: impl Spawn,
) where
@@ -694,7 +694,7 @@ async fn statement_import_loop<Api>(
Api::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
{
let topic = crate::legacy::router::attestation_topic(relay_parent);
let mut checked_messages = validator.gossip_messages_for(topic)
let mut checked_messages = gossip_handle.gossip_messages_for(topic)
.filter_map(|msg| match msg.0 {
crate::legacy::gossip::GossipMessage::Statement(s) => future::ready(Some(s.signed_statement)),
_ => future::ready(None),
@@ -758,8 +758,10 @@ async fn statement_import_loop<Api>(
if let Some(producer) = producer {
trace!(target: "validation", "driving statement work to completion");
let work = producer.prime(api.clone()).validate();
let table = table.clone();
let gossip_handle = gossip_handle.clone();
let work = producer.prime(api.clone()).validate();
let work = future::select(work.boxed(), exit.clone()).map(drop);
let _ = executor.spawn(work);
}
@@ -773,17 +775,17 @@ async fn statement_import_loop<Api>(
// group.
fn distribute_local_collation(
instance: &ConsensusNetworkingInstance,
collation: Collation,
receipt: CandidateReceipt,
receipt: AbridgedCandidateReceipt,
pov_block: PoVBlock,
chunks: (ValidatorIndex, Vec<ErasureChunk>),
gossip_handle: &RegisteredMessageValidator,
) {
// produce a signed statement.
let hash = receipt.hash();
let erasure_root = receipt.erasure_root;
let erasure_root = receipt.commitments.erasure_root;
let validated = Validated::collated_local(
receipt,
collation.pov.clone(),
pov_block,
);
let statement = crate::legacy::gossip::GossipStatement::new(
@@ -906,14 +908,14 @@ impl TableRouter for Router {
fn local_collation(
&self,
collation: Collation,
receipt: CandidateReceipt,
receipt: AbridgedCandidateReceipt,
pov_block: PoVBlock,
chunks: (ValidatorIndex, &[ErasureChunk]),
) -> Self::SendLocalCollation {
let message = ServiceToWorkerMsg::LocalCollation(
self.inner.relay_parent.clone(),
collation,
receipt,
pov_block,
(chunks.0, chunks.1.to_vec()),
);
let mut sender = self.inner.sender.clone();
@@ -922,7 +924,7 @@ impl TableRouter for Router {
})
}
fn fetch_pov_block(&self, candidate: &CandidateReceipt) -> Self::FetchValidationProof {
fn fetch_pov_block(&self, candidate: &AbridgedCandidateReceipt) -> Self::FetchValidationProof {
let (tx, rx) = oneshot::channel();
let message = ServiceToWorkerMsg::FetchPoVBlock(
self.inner.relay_parent.clone(),
polkadot/parachain/src/wasm_executor/mod.rs
+7 -3
View File
@@ -79,7 +79,7 @@ pub enum Error {
#[display(fmt = "IO error: {}", _0)]
Io(std::io::Error),
#[display(fmt = "System error: {}", _0)]
System(Box<dyn std::error::Error>),
System(Box<dyn std::error::Error + Send>),
#[display(fmt = "WASM worker error: {}", _0)]
External(String),
#[display(fmt = "Shared memory error: {}", _0)]
@@ -129,10 +129,14 @@ pub fn validate_candidate<E: Externalities + 'static>(
},
#[cfg(target_os = "unknown")]
ExecutionMode::Remote =>
Err(Error::System("Remote validator not available".to_string().into())),
Err(Error::System(Box::<dyn std::error::Error + Send + Sync>::from(
"Remote validator not available".to_string()
) as Box<_>)),
#[cfg(target_os = "unknown")]
ExecutionMode::RemoteTest =>
Err(Error::System("Remote validator not available".to_string().into())),
Err(Error::System(Box::<dyn std::error::Error + Send + Sync>::from(
"Remote validator not available".to_string()
) as Box<_>)),
}
}
polkadot/primitives/src/parachain.rs
+347 -114
View File
@@ -28,12 +28,10 @@ use serde::{Serialize, Deserialize};
#[cfg(feature = "std")]
use primitives::bytes;
use primitives::RuntimeDebug;
use runtime_primitives::traits::{Block as BlockT};
use inherents::InherentIdentifier;
use application_crypto::KeyTypeId;
#[cfg(feature = "std")]
use trie::TrieConfiguration;
pub use polkadot_parachain::{
Id, ParachainDispatchOrigin, LOWEST_USER_ID, UpwardMessage,
};
@@ -164,98 +162,143 @@ pub struct DutyRoster {
pub validator_duty: Vec<Chain>,
}
/// Compute a trie root for a set of messages, given the raw message data.
#[cfg(feature = "std")]
pub fn message_queue_root<A, I: IntoIterator<Item=A>>(messages: I) -> Hash
where A: AsRef<[u8]>
{
trie::trie_types::Layout::<primitives::Blake2Hasher>::ordered_trie_root(messages)
}
/// Candidate receipt type.
#[derive(PartialEq, Eq, Clone, Encode, Decode)]
#[cfg_attr(feature = "std", derive(Debug))]
pub struct CollationInfo {
/// The ID of the parachain this is a candidate for.
pub parachain_index: Id,
/// The collator's relay-chain account ID
pub collator: CollatorId,
/// Signature on blake2-256 of the block data by collator.
pub signature: CollatorSignature,
/// The head-data
pub head_data: HeadData,
/// blake2-256 Hash of block data.
pub block_data_hash: Hash,
/// Messages destined to be interpreted by the Relay chain itself.
pub upward_messages: Vec<UpwardMessage>,
}
impl From<CandidateReceipt> for CollationInfo {
fn from(receipt: CandidateReceipt) -> Self {
CollationInfo {
parachain_index: receipt.parachain_index,
collator: receipt.collator,
signature: receipt.signature,
head_data: receipt.head_data,
block_data_hash: receipt.block_data_hash,
upward_messages: receipt.upward_messages,
}
}
}
impl CollationInfo {
/// Check integrity vs. provided block data.
pub fn check_signature(&self) -> Result<(), ()> {
use runtime_primitives::traits::AppVerify;
if self.signature.verify(self.block_data_hash.as_ref(), &self.collator) {
Ok(())
} else {
Err(())
}
}
}
/// Candidate receipt type.
/// Extra data which is needed along with the other fields in a `CandidateReceipt`
/// to fully validate the candidate.
///
/// These are global parameters that apply to all parachain candidates in a block.
#[derive(PartialEq, Eq, Clone, Encode, Decode)]
#[cfg_attr(feature = "std", derive(Debug, Default))]
pub struct CandidateReceipt {
/// The ID of the parachain this is a candidate for.
pub parachain_index: Id,
/// The collator's relay-chain account ID
pub collator: CollatorId,
/// Signature on blake2-256 of the block data by collator.
pub signature: CollatorSignature,
/// The head-data
pub head_data: HeadData,
pub struct GlobalValidationSchedule {
/// The maximum code size permitted, in bytes.
pub max_code_size: u32,
/// The maximum head-data size permitted, in bytes.
pub max_head_data_size: u32,
}
/// Extra data which is needed along with the other fields in a `CandidateReceipt`
/// to fully validate the candidate. These fields are parachain-specific.
#[derive(PartialEq, Eq, Clone, Encode, Decode)]
#[cfg_attr(feature = "std", derive(Debug, Default))]
pub struct LocalValidationData {
/// The parent head-data.
pub parent_head: HeadData,
/// The balance of the parachain at the moment of validation.
pub balance: Balance,
}
/// Commitments made in a `CandidateReceipt`. Many of these are outputs of validation.
#[derive(PartialEq, Eq, Clone, Encode, Decode)]
#[cfg_attr(feature = "std", derive(Debug, Default))]
pub struct CandidateCommitments {
/// Fees paid from the chain to the relay chain validators
pub fees: Balance,
/// blake2-256 Hash of block data.
pub block_data_hash: Hash,
/// Messages destined to be interpreted by the Relay chain itself.
pub upward_messages: Vec<UpwardMessage>,
/// The root of a block's erasure encoding Merkle tree.
pub erasure_root: Hash,
}
impl CandidateReceipt {
/// Get the blake2_256 hash
pub fn hash(&self) -> Hash {
use runtime_primitives::traits::{BlakeTwo256, Hash};
BlakeTwo256::hash_of(self)
}
/// Get a collator signature payload on a relay-parent, block-data combo.
pub fn collator_signature_payload(
relay_parent: &Hash,
parachain_index: &Id,
pov_block_hash: &Hash,
) -> [u8; 68] {
// 32-byte hash length is protected in a test below.
let mut payload = [0u8; 68];
payload[0..32].copy_from_slice(relay_parent.as_ref());
u32::from(*parachain_index).using_encoded(|s| payload[32..32 + s.len()].copy_from_slice(s));
payload[36..68].copy_from_slice(pov_block_hash.as_ref());
payload
}
fn check_collator_signature(
relay_parent: &Hash,
parachain_index: &Id,
pov_block_hash: &Hash,
collator: &CollatorId,
signature: &CollatorSignature,
) -> Result<(),()> {
use runtime_primitives::traits::AppVerify;
let payload = collator_signature_payload(relay_parent, parachain_index, pov_block_hash);
if signature.verify(&payload[..], collator) {
Ok(())
} else {
Err(())
}
}
/// All data pertaining to the execution of a parachain candidate.
#[derive(PartialEq, Eq, Clone, Encode, Decode)]
#[cfg_attr(feature = "std", derive(Debug, Default))]
pub struct CandidateReceipt {
/// The ID of the parachain this is a candidate for.
pub parachain_index: Id,
/// The hash of the relay-chain block this should be executed in
/// the context of.
pub relay_parent: Hash,
/// The head-data
pub head_data: HeadData,
/// The collator's relay-chain account ID
pub collator: CollatorId,
/// Signature on blake2-256 of the block data by collator.
pub signature: CollatorSignature,
/// The hash of the PoV-block.
pub pov_block_hash: Hash,
/// The global validation schedule.
pub global_validation: GlobalValidationSchedule,
/// The local validation data.
pub local_validation: LocalValidationData,
/// Commitments made as a result of validation.
pub commitments: CandidateCommitments,
}
impl CandidateReceipt {
/// Check integrity vs. provided block data.
pub fn check_signature(&self) -> Result<(), ()> {
use runtime_primitives::traits::AppVerify;
check_collator_signature(
&self.relay_parent,
&self.parachain_index,
&self.pov_block_hash,
&self.collator,
&self.signature,
)
}
if self.signature.verify(self.block_data_hash.as_ref(), &self.collator) {
Ok(())
} else {
Err(())
}
/// Abridge this `CandidateReceipt`, splitting it into an `AbridgedCandidateReceipt`
/// and its omitted component.
pub fn abridge(self) -> (AbridgedCandidateReceipt, OmittedValidationData) {
let CandidateReceipt {
parachain_index,
relay_parent,
head_data,
collator,
signature,
pov_block_hash,
global_validation,
local_validation,
commitments,
} = self;
let abridged = AbridgedCandidateReceipt {
parachain_index,
relay_parent,
head_data,
collator,
signature,
pov_block_hash,
commitments,
};
let omitted = OmittedValidationData {
global_validation,
local_validation,
};
(abridged, omitted)
}
}
@@ -265,17 +308,6 @@ impl PartialOrd for CandidateReceipt {
}
}
impl PartialEq<CollationInfo> for CandidateReceipt {
fn eq(&self, info: &CollationInfo) -> bool {
self.parachain_index == info.parachain_index &&
self.collator == info.collator &&
self.signature == info.signature &&
self.head_data == info.head_data &&
self.block_data_hash == info.block_data_hash &&
self.upward_messages == info.upward_messages
}
}
impl Ord for CandidateReceipt {
fn cmp(&self, other: &Self) -> Ordering {
// TODO: compare signatures or something more sane
@@ -285,6 +317,181 @@ impl Ord for CandidateReceipt {
}
}
/// All the data which is omitted in an `AbridgedCandidateReceipt`, but that
/// is necessary for validation of the parachain candidate.
#[derive(PartialEq, Eq, Clone, Encode, Decode)]
#[cfg_attr(feature = "std", derive(Debug, Default))]
pub struct OmittedValidationData {
/// The global validation schedule.
pub global_validation: GlobalValidationSchedule,
/// The local validation data.
pub local_validation: LocalValidationData,
}
/// An abridged candidate-receipt.
///
/// Much info in a candidate-receipt is duplicated from the relay-chain state.
/// When submitting to the relay-chain, this data should be omitted as it can
/// be re-generated from relay-chain state.
#[derive(PartialEq, Eq, Clone, Encode, Decode)]
#[cfg_attr(feature = "std", derive(Debug, Default))]
pub struct AbridgedCandidateReceipt {
/// The ID of the parachain this is a candidate for.
pub parachain_index: Id,
/// The hash of the relay-chain block this should be executed in
/// the context of.
// NOTE: the fact that the hash includes this value means that code depends
// on this for deduplication. Removing this field is likely to break things.
pub relay_parent: Hash,
/// The head-data
pub head_data: HeadData,
/// The collator's relay-chain account ID
pub collator: CollatorId,
/// Signature on blake2-256 of the block data by collator.
pub signature: CollatorSignature,
/// The hash of the pov-block.
pub pov_block_hash: Hash,
/// Commitments made as a result of validation.
pub commitments: CandidateCommitments,
}
impl AbridgedCandidateReceipt {
/// Compute the hash of the abridged candidate receipt.
///
/// This is often used as the canonical hash of the receipt, rather than
/// the hash of the full receipt. The reason being that all data in the full
/// receipt is comitted to in the abridged receipt; this receipt references
/// the relay-chain block in which context it should be executed, which implies
/// any blockchain state that must be referenced.
pub fn hash(&self) -> Hash {
use runtime_primitives::traits::{BlakeTwo256, Hash};
BlakeTwo256::hash_of(self)
}
/// Combine the abridged candidate receipt with the omitted data,
/// forming a full `CandidateReceipt`.
pub fn complete(self, omitted: OmittedValidationData) -> CandidateReceipt {
let AbridgedCandidateReceipt {
parachain_index,
relay_parent,
head_data,
collator,
signature,
pov_block_hash,
commitments,
} = self;
let OmittedValidationData {
global_validation,
local_validation,
} = omitted;
CandidateReceipt {
parachain_index,
relay_parent,
head_data,
collator,
signature,
pov_block_hash,
local_validation,
global_validation,
commitments,
}
}
/// Clone the relevant portions of the `CandidateReceipt` to form a `CollationInfo`.
pub fn to_collation_info(&self) -> CollationInfo {
let AbridgedCandidateReceipt {
parachain_index,
relay_parent,
head_data,
collator,
signature,
pov_block_hash,
commitments: _commitments,
} = self;
CollationInfo {
parachain_index: *parachain_index,
relay_parent: *relay_parent,
head_data: head_data.clone(),
collator: collator.clone(),
signature: signature.clone(),
pov_block_hash: *pov_block_hash,
}
}
}
impl PartialOrd for AbridgedCandidateReceipt {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(other))
}
}
impl Ord for AbridgedCandidateReceipt {
fn cmp(&self, other: &Self) -> Ordering {
// TODO: compare signatures or something more sane
// https://github.com/paritytech/polkadot/issues/222
self.parachain_index.cmp(&other.parachain_index)
.then_with(|| self.head_data.cmp(&other.head_data))
}
}
/// A collation sent by a collator.
#[derive(PartialEq, Eq, Clone, Encode, Decode)]
#[cfg_attr(feature = "std", derive(Debug, Default))]
pub struct CollationInfo {
/// The ID of the parachain this is a candidate for.
pub parachain_index: Id,
/// The relay-chain block hash this block should execute in the
/// context of.
pub relay_parent: Hash,
/// The collator's relay-chain account ID
pub collator: CollatorId,
/// Signature on blake2-256 of the block data by collator.
pub signature: CollatorSignature,
/// The head-data
pub head_data: HeadData,
/// blake2-256 Hash of the pov-block
pub pov_block_hash: Hash,
}
impl CollationInfo {
/// Check integrity vs. a pov-block.
pub fn check_signature(&self) -> Result<(), ()> {
check_collator_signature(
&self.relay_parent,
&self.parachain_index,
&self.pov_block_hash,
&self.collator,
&self.signature,
)
}
/// Turn this into an `AbridgedCandidateReceipt` by supplying a set of commitments.
pub fn into_receipt(self, commitments: CandidateCommitments) -> AbridgedCandidateReceipt {
let CollationInfo {
parachain_index,
relay_parent,
collator,
signature,
head_data,
pov_block_hash,
} = self;
AbridgedCandidateReceipt {
parachain_index,
relay_parent,
collator,
signature,
head_data,
pov_block_hash,
commitments,
}
}
}
/// A full collation.
#[derive(PartialEq, Eq, Clone)]
#[cfg_attr(feature = "std", derive(Debug, Encode, Decode))]
@@ -303,6 +510,27 @@ pub struct PoVBlock {
pub block_data: BlockData,
}
impl PoVBlock {
/// Compute hash of block data.
#[cfg(feature = "std")]
pub fn hash(&self) -> Hash {
use runtime_primitives::traits::{BlakeTwo256, Hash};
BlakeTwo256::hash_of(&self)
}
}
/// The data which is kept available about a particular parachain block.
#[derive(PartialEq, Eq, Clone)]
#[cfg_attr(feature = "std", derive(Debug, Encode, Decode))]
pub struct AvailableData {
/// The PoV block.
pub pov_block: PoVBlock,
/// Data which is omitted from an abridged candidate receipt
/// that is necessary for validation.
pub omitted_validation: OmittedValidationData,
// In the future, outgoing messages as well.
}
/// Parachain block data.
///
/// contains everything required to validate para-block, may contain block and witness data
@@ -350,13 +578,14 @@ pub struct ValidationCode(#[cfg_attr(feature = "std", serde(with="bytes"))] pub
#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))]
pub struct Activity(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>);
/// Statements which can be made about parachain candidates.
/// Statements which can be made about parachain candidates. These are the
/// actual values which are signed by
#[derive(Clone, PartialEq, Eq, Encode)]
#[cfg_attr(feature = "std", derive(Debug))]
pub enum Statement {
/// Proposal of a parachain candidate.
#[codec(index = "1")]
Candidate(CandidateReceipt),
Candidate(Hash),
/// State that a parachain candidate is valid.
#[codec(index = "2")]
Valid(Hash),
@@ -380,11 +609,12 @@ pub enum ValidityAttestation {
Explicit(ValidatorSignature),
}
/// An attested candidate.
/// An attested candidate. This is submitted to the relay chain by a block author.
#[derive(Clone, PartialEq, Decode, Encode, RuntimeDebug)]
pub struct AttestedCandidate {
/// The candidate data.
pub candidate: CandidateReceipt,
/// The candidate data. This is abridged, because the omitted data
/// is already present within the relay chain state.
pub candidate: AbridgedCandidateReceipt,
/// Validity attestations.
pub validity_votes: Vec<ValidityAttestation>,
/// Indices of the corresponding validity votes.
@@ -393,7 +623,7 @@ pub struct AttestedCandidate {
impl AttestedCandidate {
/// Get the candidate.
pub fn candidate(&self) -> &CandidateReceipt {
pub fn candidate(&self) -> &AbridgedCandidateReceipt {
&self.candidate
}
@@ -424,20 +654,6 @@ impl FeeSchedule {
}
}
/// Current Status of a parachain.
#[derive(PartialEq, Eq, Clone, Encode, Decode)]
#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))]
pub struct Status {
/// The head of the parachain.
pub head_data: HeadData,
/// The current balance of the parachain.
pub balance: Balance,
/// The fee schedule for messages coming from this parachain.
pub fee_schedule: FeeSchedule,
}
use runtime_primitives::traits::{Block as BlockT};
sp_api::decl_runtime_apis! {
/// The API for querying the state of parachains on-chain.
#[api_version(2)]
@@ -448,12 +664,16 @@ sp_api::decl_runtime_apis! {
fn duty_roster() -> DutyRoster;
/// Get the currently active parachains.
fn active_parachains() -> Vec<(Id, Option<(CollatorId, Retriable)>)>;
/// Get the given parachain's status.
fn parachain_status(id: Id) -> Option<Status>;
/// Get the global validation schedule that all parachains should
/// be validated under.
fn global_validation_schedule() -> GlobalValidationSchedule;
/// Get the local validation data for a particular parachain.
fn local_validation_data(id: Id) -> Option<LocalValidationData>;
/// Get the given parachain's head code blob.
fn parachain_code(id: Id) -> Option<Vec<u8>>;
/// Extract the heads that were set by this set of extrinsics.
fn get_heads(extrinsics: Vec<<Block as BlockT>::Extrinsic>) -> Option<Vec<CandidateReceipt>>;
/// Extract the abridged head that was set in the extrinsics.
fn get_heads(extrinsics: Vec<<Block as BlockT>::Extrinsic>)
-> Option<Vec<AbridgedCandidateReceipt>>;
}
}
@@ -476,4 +696,17 @@ mod tests {
assert!(zero_b.leading_zeros() >= zero_u.leading_zeros());
}
#[test]
fn collator_signature_payload_is_valid() {
// if this fails, collator signature verification code has to be updated.
let h = Hash::default();
assert_eq!(h.as_ref().len(), 32);
let _payload = collator_signature_payload(
&[1; 32].into(),
&5u32.into(),
&[2; 32].into(),
);
}
}
polkadot/runtime/common/src/attestations.rs
+2 -2
View File
@@ -25,7 +25,7 @@ use frame_support::{
decl_storage, decl_module, decl_error, ensure, dispatch::DispatchResult, traits::Get
};
use primitives::{Hash, parachain::{AttestedCandidate, CandidateReceipt, Id as ParaId}};
use primitives::{Hash, parachain::{AttestedCandidate, AbridgedCandidateReceipt, Id as ParaId}};
use sp_runtime::RuntimeDebug;
use sp_staking::SessionIndex;
@@ -50,7 +50,7 @@ pub struct IncludedBlocks<T: Trait> {
/// Attestations kept over time on a parachain block.
#[derive(Encode, Decode)]
pub struct BlockAttestations<T: Trait> {
receipt: CandidateReceipt,
receipt: AbridgedCandidateReceipt,
valid: Vec<T::AccountId>, // stash account ID of voter.
invalid: Vec<T::AccountId>, // stash account ID of voter.
}
polkadot/runtime/common/src/parachains.rs
+134 -137
View File
@@ -18,7 +18,7 @@
use rstd::prelude::*;
use rstd::result;
use codec::{Encode, Decode};
use codec::{Decode, Encode};
use sp_runtime::traits::{
Hash as HashT, BlakeTwo256, Saturating, One, Dispatchable,
@@ -29,8 +29,9 @@ use primitives::{
Balance,
parachain::{
self, Id as ParaId, Chain, DutyRoster, AttestedCandidate, Statement, ParachainDispatchOrigin,
UpwardMessage, ValidatorId, ActiveParas, CollatorId, Retriable,
NEW_HEADS_IDENTIFIER,
UpwardMessage, ValidatorId, ActiveParas, CollatorId, Retriable, OmittedValidationData,
CandidateReceipt, GlobalValidationSchedule, AbridgedCandidateReceipt,
LocalValidationData, NEW_HEADS_IDENTIFIER,
},
};
use frame_support::{
@@ -211,6 +212,10 @@ decl_error! {
ParentMismatch,
/// Head data was too large.
HeadDataTooLarge,
/// Para does not have enough balance to pay fees.
CannotPayFees,
/// Unexpected relay-parent for a candidate receipt.
UnexpectedRelayParent,
}
}
@@ -232,6 +237,11 @@ decl_module! {
let mut proceeded = Vec::with_capacity(heads.len());
let schedule = GlobalValidationSchedule {
max_code_size: T::MaxCodeSize::get(),
max_head_data_size: T::MaxHeadDataSize::get(),
};
if !active_parachains.is_empty() {
// perform integrity checks before writing to storage.
{
@@ -256,7 +266,7 @@ decl_module! {
Self::check_upward_messages(
id,
&head.candidate.upward_messages,
&head.candidate.commitments.upward_messages,
MAX_QUEUE_COUNT,
WATERMARK_QUEUE_SIZE,
)?;
@@ -267,7 +277,11 @@ decl_module! {
}
}
let para_blocks = Self::check_candidates(&heads, &active_parachains)?;
let para_blocks = Self::check_candidates(
&schedule,
&heads,
&active_parachains,
)?;
<attestations::Module<T>>::note_included(&heads, para_blocks);
@@ -275,6 +289,9 @@ decl_module! {
&heads,
);
// note: we dispatch new messages _after_ the call to `check_candidates`
// which deducts any fees. if that were not the case, an upward message
// could be dispatched and spend money that invalidated a candidate.
Self::dispatch_upward_messages(
MAX_QUEUE_COUNT,
WATERMARK_QUEUE_SIZE,
@@ -397,7 +414,7 @@ impl<T: Trait> Module<T> {
// Queue up upwards messages (from parachains to relay chain).
Self::queue_upward_messages(
id,
&head.candidate.upward_messages,
&head.candidate.commitments.upward_messages,
&mut ordered_needs_dispatch,
);
}
@@ -433,7 +450,8 @@ impl<T: Trait> Module<T> {
}
}
/// Simple FIFO dispatcher.
/// Simple FIFO dispatcher. This must be called after parachain fees are checked,
/// as dispatched messages may spend parachain funds.
fn dispatch_upward_messages(
max_queue_count: usize,
watermark_queue_size: usize,
@@ -540,18 +558,19 @@ impl<T: Trait> Module<T> {
(DutyRoster { validator_duty: roles_val, }, orig_seed)
}
/// Get the parachain status necessary for validation.
pub fn parachain_status(id: &parachain::Id) -> Option<parachain::Status> {
let balance = T::ParachainCurrency::free_balance(*id);
Self::parachain_head(id).map(|head_data| parachain::Status {
head_data: parachain::HeadData(head_data),
balance,
// TODO: https://github.com/paritytech/polkadot/issues/92
// plug in some real values here. most likely governable.
fee_schedule: parachain::FeeSchedule {
base: 0,
per_byte: 0,
}
/// Get the global validation schedule for all parachains.
pub fn global_validation_schedule() -> GlobalValidationSchedule {
GlobalValidationSchedule {
max_code_size: T::MaxCodeSize::get(),
max_head_data_size: T::MaxHeadDataSize::get(),
}
}
/// Get the local validation schedule for a particular parachain.
pub fn local_validation_data(id: &parachain::Id) -> Option<LocalValidationData> {
Self::parachain_head(id).map(|parent_head| LocalValidationData {
parent_head: primitives::parachain::HeadData(parent_head),
balance: T::ParachainCurrency::free_balance(*id),
})
}
@@ -563,6 +582,7 @@ impl<T: Trait> Module<T> {
// check the attestations on these candidates. The candidates should have been checked
// that each candidates' chain ID is valid.
fn check_candidates(
schedule: &GlobalValidationSchedule,
attested_candidates: &[AttestedCandidate],
active_parachains: &[(ParaId, Option<(CollatorId, Retriable)>)]
) -> rstd::result::Result<IncludedBlocks<T>, sp_runtime::DispatchError>
@@ -635,6 +655,29 @@ impl<T: Trait> Module<T> {
sorted_duties
};
// computes the omitted validation data for a particular parachain.
let full_candidate = |abridged: &AbridgedCandidateReceipt|
-> rstd::result::Result<CandidateReceipt, sp_runtime::DispatchError>
{
let para_id = abridged.parachain_index;
let parent_head = match Self::parachain_head(&para_id)
.map(primitives::parachain::HeadData)
{
Some(p) => p,
None => Err(Error::<T>::ParentMismatch)?,
};
let omitted = OmittedValidationData {
global_validation: schedule.clone(),
local_validation: LocalValidationData {
parent_head,
balance: T::ParachainCurrency::free_balance(para_id),
},
};
Ok(abridged.clone().complete(omitted))
};
let sorted_validators = make_sorted_duties(&duty_roster.validator_duty);
let parent_hash = <system::Module<T>>::parent_hash();
@@ -644,18 +687,20 @@ impl<T: Trait> Module<T> {
let mut para_block_hashes = Vec::new();
let max_head_data_size = T::MaxHeadDataSize::get();
for candidate in attested_candidates {
let para_id = candidate.parachain_index();
let validator_group = validator_groups.group_for(para_id)
.ok_or(Error::<T>::NoValidatorGroup)?;
let actual_head = Self::parachain_head(&para_id)
.map(primitives::parachain::HeadData);
// NOTE: when changing this to allow older blocks,
// care must be taken in the availability store pruning to ensure that
// data is stored correctly. A block containing a candidate C can be
// orphaned before a block containing C is finalized. Care must be taken
// not to prune the data for C simply because an orphaned block contained
// it.
ensure!(
actual_head.as_ref() == Some(&candidate.candidate.parent_head),
Error::<T>::ParentMismatch,
candidate.candidate().relay_parent.as_ref() == parent_hash.as_ref(),
Error::<T>::UnexpectedRelayParent,
);
ensure!(
@@ -669,14 +714,21 @@ impl<T: Trait> Module<T> {
);
ensure!(
max_head_data_size >= candidate.candidate().head_data.0.len() as _,
schedule.max_head_data_size >= candidate.candidate().head_data.0.len() as _,
Error::<T>::HeadDataTooLarge,
);
let fees = candidate.candidate().fees;
let full_candidate = full_candidate(candidate.candidate())?;
let fees = full_candidate.commitments.fees;
ensure!(
full_candidate.local_validation.balance >= full_candidate.commitments.fees,
Error::<T>::CannotPayFees,
);
T::ParachainCurrency::deduct(para_id, fees)?;
let mut candidate_hash = None;
let candidate_hash = candidate.candidate().hash();
let mut encoded_implicit = None;
let mut encoded_explicit = None;
@@ -702,18 +754,14 @@ impl<T: Trait> Module<T> {
let (payload, sig) = match validity_attestation {
ValidityAttestation::Implicit(sig) => {
let payload = encoded_implicit.get_or_insert_with(|| localized_payload(
Statement::Candidate(candidate.candidate.clone()),
Statement::Candidate(candidate_hash),
));
(payload, sig)
}
ValidityAttestation::Explicit(sig) => {
let hash = candidate_hash
.get_or_insert_with(|| candidate.candidate.hash())
.clone();
let payload = encoded_explicit.get_or_insert_with(|| localized_payload(
Statement::Valid(hash),
Statement::Valid(candidate_hash),
));
(payload, sig)
@@ -726,12 +774,12 @@ impl<T: Trait> Module<T> {
);
}
para_block_hashes.push(candidate_hash.unwrap_or_else(|| candidate.candidate().hash()));
ensure!(
candidate.validity_votes.len() == expected_votes_len,
Error::<T>::UntaggedVotes
);
para_block_hashes.push(candidate_hash);
}
Ok(IncludedBlocks {
@@ -833,7 +881,7 @@ mod tests {
use primitives::{
parachain::{
CandidateReceipt, HeadData, ValidityAttestation, ValidatorId, Info as ParaInfo,
Scheduling,
Scheduling, LocalValidationData, CandidateCommitments,
},
BlockNumber,
};
@@ -981,7 +1029,7 @@ mod tests {
type RewardRemainder = ();
type CurrencyToVote = ();
type Event = ();
type Currency = balances::Module<Test>;
type Currency = Balances;
type Slash = ();
type Reward = ();
type SessionsPerEra = SessionsPerEra;
@@ -1006,7 +1054,7 @@ mod tests {
impl slots::Trait for Test {
type Event = ();
type Currency = balances::Module<Test>;
type Currency = Balances;
type Parachains = registrar::Module<Test>;
type EndingPeriod = EndingPeriod;
type LeasePeriod = LeasePeriod;
@@ -1022,7 +1070,7 @@ mod tests {
impl registrar::Trait for Test {
type Event = ();
type Origin = Origin;
type Currency = balances::Module<Test>;
type Currency = Balances;
type ParathreadDeposit = ParathreadDeposit;
type SwapAux = slots::Module<Test>;
type QueueSize = QueueSize;
@@ -1037,7 +1085,7 @@ mod tests {
impl Trait for Test {
type Origin = Origin;
type Call = Call;
type ParachainCurrency = balances::Module<Test>;
type ParachainCurrency = Balances;
type Randomness = RandomnessCollectiveFlip;
type ActiveParachains = registrar::Module<Test>;
type Registrar = registrar::Module<Test>;
@@ -1046,6 +1094,7 @@ mod tests {
}
type Parachains = Module<Test>;
type Balances = balances::Module<Test>;
type System = system::Module<Test>;
type RandomnessCollectiveFlip = randomness_collective_flip::Module<Test>;
type Registrar = registrar::Module<Test>;
@@ -1125,6 +1174,38 @@ mod tests {
ParachainsCall::set_heads(v)
}
// creates a template candidate which pins to correct relay-chain state.
fn raw_candidate(para_id: ParaId) -> CandidateReceipt {
CandidateReceipt {
parachain_index: para_id,
relay_parent: System::parent_hash(),
head_data: Default::default(),
collator: Default::default(),
signature: Default::default(),
pov_block_hash: Default::default(),
global_validation: GlobalValidationSchedule {
max_code_size: <Test as Trait>::MaxCodeSize::get(),
max_head_data_size: <Test as Trait>::MaxHeadDataSize::get(),
},
local_validation: LocalValidationData {
parent_head: HeadData(Parachains::parachain_head(&para_id).unwrap()),
balance: <Balances as ParachainCurrency<u64>>::free_balance(para_id),
},
commitments: CandidateCommitments::default(),
}
}
// makes a blank attested candidate from a `CandidateReceipt`.
fn make_blank_attested(candidate: CandidateReceipt) -> AttestedCandidate {
let (candidate, _) = candidate.abridge();
AttestedCandidate {
validity_votes: vec![],
validator_indices: BitVec::new(),
candidate,
}
}
fn make_attestations(candidate: &mut AttestedCandidate) {
let mut vote_implicit = false;
let parent_hash = <system::Module<Test>>::parent_hash();
@@ -1150,7 +1231,7 @@ mod tests {
let key = extract_key(authorities[idx].clone());
let statement = if vote_implicit {
Statement::Candidate(candidate.candidate.clone())
Statement::Candidate(candidate_hash.clone())
} else {
Statement::Valid(candidate_hash.clone())
};
@@ -1176,23 +1257,12 @@ mod tests {
id: u32,
upward_messages: Vec<(ParachainDispatchOrigin, Vec<u8>)>
) -> AttestedCandidate {
AttestedCandidate {
validity_votes: vec![],
validator_indices: BitVec::new(),
candidate: CandidateReceipt {
parachain_index: id.into(),
collator: Default::default(),
signature: Default::default(),
head_data: HeadData(vec![1, 2, 3]),
parent_head: HeadData(vec![]),
fees: 0,
block_data_hash: Default::default(),
upward_messages: upward_messages.into_iter()
.map(|x| UpwardMessage { origin: x.0, data: x.1 })
.collect(),
erasure_root: [1u8; 32].into(),
}
}
let mut raw_candidate = raw_candidate(id.into());
raw_candidate.commitments.upward_messages = upward_messages.into_iter()
.map(|x| UpwardMessage { origin: x.0, data: x.1 })
.collect();
make_blank_attested(raw_candidate)
}
fn init_block() {
@@ -1586,23 +1656,7 @@ mod tests {
new_test_ext(parachains.clone()).execute_with(|| {
run_to_block(2);
let candidate = AttestedCandidate {
validity_votes: vec![],
validator_indices: BitVec::new(),
candidate: CandidateReceipt {
parachain_index: 0.into(),
collator: Default::default(),
signature: Default::default(),
head_data: HeadData(vec![1, 2, 3]),
parent_head: HeadData(vec![]),
fees: 0,
block_data_hash: Default::default(),
upward_messages: vec![],
erasure_root: [1u8; 32].into(),
},
};
let candidate = make_blank_attested(raw_candidate(0.into()));
assert!(Parachains::dispatch(set_heads(vec![candidate]), Origin::NONE).is_err());
})
}
@@ -1618,37 +1672,8 @@ mod tests {
run_to_block(2);
assert_eq!(Parachains::active_parachains().len(), 2);
let mut candidate_a = AttestedCandidate {
validity_votes: vec![],
validator_indices: BitVec::new(),
candidate: CandidateReceipt {
parachain_index: 0.into(),
collator: Default::default(),
signature: Default::default(),
head_data: HeadData(vec![1, 2, 3]),
parent_head: HeadData(vec![]),
fees: 0,
block_data_hash: Default::default(),
upward_messages: vec![],
erasure_root: [1u8; 32].into(),
}
};
let mut candidate_b = AttestedCandidate {
validity_votes: vec![],
validator_indices: BitVec::new(),
candidate: CandidateReceipt {
parachain_index: 1.into(),
collator: Default::default(),
signature: Default::default(),
head_data: HeadData(vec![2, 3, 4]),
parent_head: HeadData(vec![]),
fees: 0,
block_data_hash: Default::default(),
upward_messages: vec![],
erasure_root: [1u8; 32].into(),
}
};
let mut candidate_a = make_blank_attested(raw_candidate(0.into()));
let mut candidate_b = make_blank_attested(raw_candidate(1.into()));
make_attestations(&mut candidate_a);
make_attestations(&mut candidate_b);
@@ -1674,22 +1699,8 @@ mod tests {
new_test_ext(parachains.clone()).execute_with(|| {
run_to_block(2);
let mut candidate = AttestedCandidate {
validity_votes: vec![],
validator_indices: BitVec::new(),
candidate: CandidateReceipt {
parachain_index: 0.into(),
collator: Default::default(),
signature: Default::default(),
head_data: HeadData(vec![1, 2, 3]),
parent_head: HeadData(vec![]),
fees: 0,
block_data_hash: Default::default(),
upward_messages: vec![],
erasure_root: [1u8; 32].into(),
}
};
let mut candidate = make_blank_attested(raw_candidate(0.into()));
make_attestations(&mut candidate);
let mut double_validity = candidate.clone();
@@ -1712,22 +1723,8 @@ mod tests {
new_test_ext(parachains.clone()).execute_with(|| {
run_to_block(2);
let mut candidate = AttestedCandidate {
validity_votes: vec![],
validator_indices: BitVec::new(),
candidate: CandidateReceipt {
parachain_index: 0.into(),
collator: Default::default(),
signature: Default::default(),
head_data: HeadData(vec![1, 2, 3]),
parent_head: HeadData(vec![]),
fees: 0,
block_data_hash: Default::default(),
upward_messages: vec![],
erasure_root: [1u8; 32].into(),
}
};
let mut candidate = make_blank_attested(raw_candidate(0.into()));
make_attestations(&mut candidate);
// Change the last vote index to make it not corresponding to the assigned group.
polkadot/runtime/common/src/registrar.rs
+26 -12
View File
@@ -651,7 +651,8 @@ mod tests {
use primitives::{
parachain::{
ValidatorId, Info as ParaInfo, Scheduling, LOWEST_USER_ID, AttestedCandidate,
CandidateReceipt, HeadData, ValidityAttestation, Statement, Chain, CollatorPair,
CandidateReceipt, HeadData, ValidityAttestation, Statement, Chain,
CollatorPair, CandidateCommitments, GlobalValidationSchedule, LocalValidationData,
},
Balance, BlockNumber,
};
@@ -894,20 +895,33 @@ mod tests {
LOWEST_USER_ID + i
}
fn attest(id: ParaId, collator: &CollatorPair, head_data: &[u8], parent_head: &[u8], block_data: &[u8]) -> AttestedCandidate {
let block_data_hash = BlakeTwo256::hash(block_data);
fn attest(id: ParaId, collator: &CollatorPair, head_data: &[u8], block_data: &[u8]) -> AttestedCandidate {
let pov_block_hash = BlakeTwo256::hash(block_data);
let relay_parent = System::parent_hash();
let candidate = CandidateReceipt {
parachain_index: id,
collator: collator.public(),
signature: block_data_hash.using_encoded(|d| collator.sign(d)),
relay_parent,
head_data: HeadData(head_data.to_vec()),
parent_head: HeadData(parent_head.to_vec()),
fees: 0,
block_data_hash,
upward_messages: vec![],
erasure_root: [1; 32].into(),
collator: collator.public(),
signature: pov_block_hash.using_encoded(|d| collator.sign(d)),
pov_block_hash,
global_validation: GlobalValidationSchedule {
max_code_size: <Test as parachains::Trait>::MaxCodeSize::get(),
max_head_data_size: <Test as parachains::Trait>::MaxHeadDataSize::get(),
},
local_validation: LocalValidationData {
balance: Balances::free_balance(&id.into_account()),
parent_head: HeadData(Parachains::parachain_head(&id).unwrap()),
},
commitments: CandidateCommitments {
fees: 0,
upward_messages: vec![],
erasure_root: [1; 32].into(),
},
};
let payload = (Statement::Valid(candidate.hash()), System::parent_hash()).encode();
let (candidate, _) = candidate.abridge();
let candidate_hash = candidate.hash();
let payload = (Statement::Valid(candidate_hash), System::parent_hash()).encode();
let roster = Parachains::calculate_duty_roster().0.validator_duty;
AttestedCandidate {
candidate,
@@ -1175,7 +1189,7 @@ mod tests {
(user_id(0), Some((col.clone(), Retriable::WithRetries(0))))
]);
assert_ok!(Parachains::set_heads(Origin::NONE, vec![
attest(user_id(0), &Sr25519Keyring::One.pair().into(), &[3; 3], &[3; 3], &[0; 0])
attest(user_id(0), &Sr25519Keyring::One.pair().into(), &[3; 3], &[0; 0])
]));
run_to_block(6);
polkadot/runtime/kusama/src/lib.rs
+9 -4
View File
@@ -25,7 +25,7 @@ use sp_core::u32_trait::{_1, _2, _3, _4, _5};
use codec::{Encode, Decode};
use primitives::{
AccountId, AccountIndex, Balance, BlockNumber, Hash, Nonce, Signature, Moment,
parachain::{self, ActiveParas, CandidateReceipt}, ValidityError,
parachain::{self, ActiveParas, AbridgedCandidateReceipt}, ValidityError,
};
use runtime_common::{attestations, claims, parachains, registrar, slots,
impls::{CurrencyToVoteHandler, TargetedFeeAdjustment, ToAuthor},
@@ -766,13 +766,18 @@ sp_api::impl_runtime_apis! {
fn active_parachains() -> Vec<(parachain::Id, Option<(parachain::CollatorId, parachain::Retriable)>)> {
Registrar::active_paras()
}
fn parachain_status(id: parachain::Id) -> Option<parachain::Status> {
Parachains::parachain_status(&id)
fn global_validation_schedule() -> parachain::GlobalValidationSchedule {
Parachains::global_validation_schedule()
}
fn local_validation_data(id: parachain::Id) -> Option<parachain::LocalValidationData> {
Parachains::local_validation_data(&id)
}
fn parachain_code(id: parachain::Id) -> Option<Vec<u8>> {
Parachains::parachain_code(&id)
}
fn get_heads(extrinsics: Vec<<Block as BlockT>::Extrinsic>) -> Option<Vec<CandidateReceipt>> {
fn get_heads(extrinsics: Vec<<Block as BlockT>::Extrinsic>)
-> Option<Vec<AbridgedCandidateReceipt>>
{
extrinsics
.into_iter()
.find_map(|ex| match UncheckedExtrinsic::decode(&mut ex.encode().as_slice()) {
polkadot/runtime/polkadot/src/lib.rs
+9 -4
View File
@@ -31,7 +31,7 @@ use sp_core::u32_trait::{_1, _2, _3, _4, _5};
use codec::{Encode, Decode};
use primitives::{
AccountId, AccountIndex, Balance, BlockNumber, Hash, Nonce, Signature, Moment,
parachain::{self, ActiveParas, CandidateReceipt}, ValidityError,
parachain::{self, ActiveParas, AbridgedCandidateReceipt}, ValidityError,
};
use sp_runtime::{
create_runtime_str, generic, impl_opaque_keys,
@@ -688,13 +688,18 @@ sp_api::impl_runtime_apis! {
fn active_parachains() -> Vec<(parachain::Id, Option<(parachain::CollatorId, parachain::Retriable)>)> {
Registrar::active_paras()
}
fn parachain_status(id: parachain::Id) -> Option<parachain::Status> {
Parachains::parachain_status(&id)
fn global_validation_schedule() -> parachain::GlobalValidationSchedule {
Parachains::global_validation_schedule()
}
fn local_validation_data(id: parachain::Id) -> Option<parachain::LocalValidationData> {
Parachains::local_validation_data(&id)
}
fn parachain_code(id: parachain::Id) -> Option<Vec<u8>> {
Parachains::parachain_code(&id)
}
fn get_heads(extrinsics: Vec<<Block as BlockT>::Extrinsic>) -> Option<Vec<CandidateReceipt>> {
fn get_heads(extrinsics: Vec<<Block as BlockT>::Extrinsic>)
-> Option<Vec<AbridgedCandidateReceipt>>
{
extrinsics
.into_iter()
.find_map(|ex| match UncheckedExtrinsic::decode(&mut ex.encode().as_slice()) {
polkadot/statement-table/src/lib.rs
+21 -11
View File
@@ -19,18 +19,28 @@ pub mod generic;
pub use generic::Table;
use primitives::parachain::{
Id, CandidateReceipt, Statement as PrimitiveStatement, ValidatorSignature, ValidatorIndex,
Id, AbridgedCandidateReceipt, Statement as PrimitiveStatement, ValidatorSignature, ValidatorIndex,
};
use primitives::Hash;
/// Statements about candidates on the network.
pub type Statement = generic::Statement<CandidateReceipt, Hash>;
pub type Statement = generic::Statement<AbridgedCandidateReceipt, Hash>;
/// Signed statements about candidates.
pub type SignedStatement = generic::SignedStatement<CandidateReceipt, Hash, ValidatorIndex, ValidatorSignature>;
pub type SignedStatement = generic::SignedStatement<
AbridgedCandidateReceipt,
Hash,
ValidatorIndex,
ValidatorSignature,
>;
/// Kinds of misbehavior, along with proof.
pub type Misbehavior = generic::Misbehavior<CandidateReceipt, Hash, ValidatorIndex, ValidatorSignature>;
pub type Misbehavior = generic::Misbehavior<
AbridgedCandidateReceipt,
Hash,
ValidatorIndex,
ValidatorSignature,
>;
/// A summary of import of a statement.
pub type Summary = generic::Summary<Hash, Id>;
@@ -50,13 +60,13 @@ impl<C: Context> generic::Context for C {
type Digest = Hash;
type GroupId = Id;
type Signature = ValidatorSignature;
type Candidate = CandidateReceipt;
type Candidate = AbridgedCandidateReceipt;
fn candidate_digest(candidate: &CandidateReceipt) -> Hash {
fn candidate_digest(candidate: &AbridgedCandidateReceipt) -> Hash {
candidate.hash()
}
fn candidate_group(candidate: &CandidateReceipt) -> Id {
fn candidate_group(candidate: &AbridgedCandidateReceipt) -> Id {
candidate.parachain_index.clone()
}
@@ -69,12 +79,12 @@ impl<C: Context> generic::Context for C {
}
}
impl From<Statement> for PrimitiveStatement {
fn from(s: Statement) -> PrimitiveStatement {
match s {
impl<'a> From<&'a Statement> for PrimitiveStatement {
fn from(s: &'a Statement) -> PrimitiveStatement {
match *s {
generic::Statement::Valid(s) => PrimitiveStatement::Valid(s),
generic::Statement::Invalid(s) => PrimitiveStatement::Invalid(s),
generic::Statement::Candidate(s) => PrimitiveStatement::Candidate(s),
generic::Statement::Candidate(ref s) => PrimitiveStatement::Candidate(s.hash()),
}
}
}
polkadot/test-parachains/adder/collator/src/main.rs
+3 -3
View File
@@ -24,7 +24,7 @@ use sp_core::Pair;
use codec::{Encode, Decode};
use primitives::{
Hash,
parachain::{HeadData, BlockData, Id as ParaId, Status as ParachainStatus},
parachain::{HeadData, BlockData, Id as ParaId, LocalValidationData},
};
use collator::{
InvalidHead, ParachainContext, Network, BuildParachainContext, load_spec, Configuration,
@@ -60,10 +60,10 @@ impl ParachainContext for AdderContext {
fn produce_candidate(
&mut self,
_relay_parent: Hash,
status: ParachainStatus,
local_validation: LocalValidationData,
) -> Self::ProduceCandidate
{
let adder_head = match AdderHead::decode(&mut &status.head_data.0[..]) {
let adder_head = match AdderHead::decode(&mut &local_validation.parent_head.0[..]) {
Ok(adder_head) => adder_head,
Err(_) => return err(InvalidHead)
};
polkadot/validation/src/block_production.rs
+5 -25
View File
@@ -29,7 +29,7 @@ use sp_blockchain::HeaderBackend;
use block_builder::BlockBuilderApi;
use codec::Encode;
use consensus::{Proposal, RecordProof};
use polkadot_primitives::{Hash, Block, BlockId, BlockNumber, Header};
use polkadot_primitives::{Hash, Block, BlockId, Header};
use polkadot_primitives::parachain::{
ParachainHost, AttestedCandidate, NEW_HEADS_IDENTIFIER,
};
@@ -47,6 +47,9 @@ use crate::validation_service::ServiceHandle;
use crate::dynamic_inclusion::DynamicInclusion;
use crate::Error;
// block size limit.
pub(crate) const MAX_TRANSACTIONS_SIZE: usize = 4 * 1024 * 1024;
// Polkadot proposer factory.
pub struct ProposerFactory<Client, TxPool, Backend> {
client: Arc<Client>,
@@ -100,7 +103,6 @@ where
parent_header: &Header,
) -> Self::CreateProposer {
let parent_hash = parent_header.hash();
let parent_number = parent_header.number;
let parent_id = BlockId::hash(parent_hash);
let client = self.client.clone();
@@ -114,9 +116,7 @@ where
.and_then(move |tracker| future::ready(Ok(Proposer {
client,
tracker,
parent_hash,
parent_id,
parent_number,
transaction_pool,
slot_duration,
backend,
@@ -129,9 +129,7 @@ where
/// The Polkadot proposer logic.
pub struct Proposer<Client, TxPool, Backend> {
client: Arc<Client>,
parent_hash: Hash,
parent_id: BlockId,
parent_number: BlockNumber,
tracker: crate::validation_service::ValidationInstanceHandle,
transaction_pool: Arc<TxPool>,
slot_duration: u64,
@@ -172,8 +170,6 @@ impl<Client, TxPool, Backend> consensus::Proposer<Block> for Proposer<Client, Tx
Duration::from_millis(self.slot_duration / SLOT_DURATION_DENOMINATOR),
);
let parent_hash = self.parent_hash.clone();
let parent_number = self.parent_number.clone();
let parent_id = self.parent_id.clone();
let client = self.client.clone();
let transaction_pool = self.transaction_pool.clone();
@@ -198,13 +194,10 @@ impl<Client, TxPool, Backend> consensus::Proposer<Block> for Proposer<Client, Tx
};
let data = CreateProposalData {
parent_hash,
parent_number,
parent_id,
client,
transaction_pool,
table,
believed_minimum_timestamp: believed_timestamp,
inherent_data: Some(inherent_data),
inherent_digests,
// leave some time for the proposal finalisation
@@ -239,13 +232,10 @@ fn current_timestamp() -> u64 {
/// Inner data of the create proposal.
struct CreateProposalData<Client, TxPool, Backend> {
parent_hash: Hash,
parent_number: BlockNumber,
parent_id: BlockId,
client: Arc<Client>,
transaction_pool: Arc<TxPool>,
table: Arc<crate::SharedTable>,
believed_minimum_timestamp: u64,
inherent_data: Option<InherentData>,
inherent_digests: DigestFor<Block>,
deadline: Instant,
@@ -298,7 +288,7 @@ impl<Client, TxPool, Backend> CreateProposalData<Client, TxPool, Backend> where
let ready_iter = self.transaction_pool.ready();
for ready in ready_iter.take(MAX_TRANSACTIONS) {
let encoded_size = ready.data().encode().len();
if pending_size + encoded_size >= crate::evaluation::MAX_TRANSACTIONS_SIZE {
if pending_size + encoded_size >= MAX_TRANSACTIONS_SIZE {
break;
}
if Instant::now() > self.deadline {
@@ -339,16 +329,6 @@ impl<Client, TxPool, Backend> CreateProposalData<Client, TxPool, Backend> where
.join(", ")
);
// TODO: full re-evaluation (https://github.com/paritytech/polkadot/issues/216)
let active_parachains = runtime_api.active_parachains(&self.parent_id)?;
assert!(crate::evaluation::evaluate_initial(
&new_block,
self.believed_minimum_timestamp,
&self.parent_hash,
self.parent_number,
&active_parachains[..],
).is_ok());
Ok(Proposal { block: new_block, storage_changes, proof })
}
}
polkadot/validation/src/collation.rs
+17 -499
View File
@@ -22,22 +22,15 @@
use std::sync::Arc;
use polkadot_primitives::{
BlakeTwo256, Block, Hash, HashT, BlockId, Balance,
BlakeTwo256, Block, Hash, HashT,
parachain::{
CollatorId, CandidateReceipt, CollationInfo,
ParachainHost, Id as ParaId, Collation, FeeSchedule, ErasureChunk,
HeadData, PoVBlock,
CollatorId, ParachainHost, Id as ParaId, Collation, ErasureChunk, CollationInfo,
},
};
use polkadot_erasure_coding as erasure;
use sp_api::ProvideRuntimeApi;
use parachain::{
wasm_executor::{self, ExecutionMode}, UpwardMessage,
};
use trie::TrieConfiguration;
use futures::prelude::*;
use log::debug;
use parking_lot::Mutex;
/// Encapsulates connections to collators and allows collation on any parachain.
///
@@ -67,530 +60,55 @@ pub trait Collators: Clone {
/// A future which resolves when a collation is available.
pub async fn collation_fetch<C: Collators, P>(
parachain: ParaId,
relay_parent_hash: Hash,
relay_parent: Hash,
collators: C,
client: Arc<P>,
max_block_data_size: Option<u64>,
) -> Result<(Collation, HeadData, Balance),C::Error>
n_validators: usize,
) -> Result<(CollationInfo, crate::pipeline::FullOutput), C::Error>
where
P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
C: Collators + Unpin,
P: ProvideRuntimeApi<Block>,
<C as Collators>::Collation: Unpin,
{
let relay_parent = BlockId::hash(relay_parent_hash);
loop {
let collation = collators.collate(parachain, relay_parent_hash)
.await?;
let res = validate_collation(
let collation = collators.collate(parachain, relay_parent).await?;
let Collation { info, pov } = collation;
let res = crate::pipeline::full_output_validation_with_api(
&*client,
&info,
&pov,
&relay_parent,
&collation,
max_block_data_size,
n_validators,
);
match res {
Ok((parent_head, fees)) => {
return Ok((collation, parent_head, fees))
Ok(full_output) => {
return Ok((info, full_output))
}
Err(e) => {
debug!("Failed to validate parachain due to API error: {}", e);
// just continue if we got a bad collation or failed to validate
collators.note_bad_collator(collation.info.collator)
collators.note_bad_collator(info.collator)
}
}
}
}
// Errors that can occur when validating a parachain.
#[derive(Debug, derive_more::Display, derive_more::From)]
pub enum Error {
/// Client error
Client(sp_blockchain::Error),
/// Wasm validation error
WasmValidation(wasm_executor::Error),
/// Erasure-encoding error.
Erasure(erasure::Error),
/// Collated for inactive parachain
#[display(fmt = "Collated for inactive parachain: {:?}", _0)]
InactiveParachain(ParaId),
/// Unexpected egress root
#[display(fmt = "Got unexpected egress root to {:?}. (expected: {:?}, got {:?})", id, expected, got)]
EgressRootMismatch { id: ParaId, expected: Hash, got: Hash },
/// Unexpected ingress root
#[display(fmt = "Got unexpected ingress root to {:?}. (expected: {:?}, got {:?})", id, expected, got)]
IngressRootMismatch { id: ParaId, expected: Hash, got: Hash },
/// Ingress from wrong chain
#[display(fmt = "Got ingress from wrong chain. (expected: {:?}, got {:?})", expected, got)]
IngressChainMismatch { expected: ParaId, got: ParaId },
/// Ingress canonicality mismatch
#[display(fmt = "Got data for {} roots, expected {}", expected, got)]
IngressCanonicalityMismatch { expected: usize, got: usize },
/// Missing or extra egress root
#[display(fmt = "Missing or extra egress root. (expected: {:?}, got {:?})", expected, got)]
MissingEgressRoot { expected: Option<ParaId>, got: Option<ParaId>, },
/// Parachain validation produced wrong head data
#[display(fmt = "Parachain validation produced wrong head data (expected: {:?}, got {:?})", expected, got)]
WrongHeadData { expected: Vec<u8>, got: Vec<u8> },
/// Block data is too big
#[display(fmt = "Block data is too big (maximum allowed size: {}, actual size: {})", size, max_size)]
BlockDataTooBig { size: u64, max_size: u64 },
/// Parachain validation produced wrong relay-chain messages
#[display(fmt = "Parachain validation produced wrong relay-chain messages (expected: {:?}, got {:?})", expected, got)]
UpwardMessagesInvalid { expected: Vec<UpwardMessage>, got: Vec<UpwardMessage> },
/// Parachain validation produced wrong fees to charge to parachain.
#[display(fmt = "Parachain validation produced wrong relay-chain fees (expected: {:?}, got {:?})", expected, got)]
FeesChargedInvalid { expected: Balance, got: Balance },
/// Candidate block has an erasure-encoded root that mismatches the actual
/// erasure-encoded root of block data and extrinsics.
#[display(fmt = "Got unexpected erasure root (expected: {:?}, got {:?})", expected, got)]
ErasureRootMismatch { expected: Hash, got: Hash },
/// Candidate block collation info doesn't match candidate receipt.
#[display(fmt = "Got receipt mismatch for candidate {:?}", candidate)]
CandidateReceiptMismatch { candidate: Hash },
/// The parent header given in the candidate did not match current relay-chain
/// state.
#[display(fmt = "Got unexpected parachain parent.")]
ParentMismatch { expected: HeadData, got: HeadData },
}
impl std::error::Error for Error {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
Error::Client(ref err) => Some(err),
Error::WasmValidation(ref err) => Some(err),
_ => None,
}
}
}
/// Compute a trie root for a set of messages, given the raw message data.
pub fn message_queue_root<A, I: IntoIterator<Item=A>>(messages: I) -> Hash
where A: AsRef<[u8]>
{
trie::trie_types::Layout::<primitives::Blake2Hasher>::ordered_trie_root(messages)
}
struct ExternalitiesInner {
upward: Vec<UpwardMessage>,
fees_charged: Balance,
free_balance: Balance,
fee_schedule: FeeSchedule,
}
impl wasm_executor::Externalities for ExternalitiesInner {
fn post_upward_message(&mut self, message: UpwardMessage) -> Result<(), String> {
self.apply_message_fee(message.data.len())?;
self.upward.push(message);
Ok(())
}
}
impl ExternalitiesInner {
fn new(free_balance: Balance, fee_schedule: FeeSchedule) -> Self {
Self {
free_balance,
fee_schedule,
fees_charged: 0,
upward: Vec::new(),
}
}
fn apply_message_fee(&mut self, message_len: usize) -> Result<(), String> {
let fee = self.fee_schedule.compute_fee(message_len);
let new_fees_charged = self.fees_charged.saturating_add(fee);
if new_fees_charged > self.free_balance {
Err("could not cover fee.".into())
} else {
self.fees_charged = new_fees_charged;
Ok(())
}
}
// Performs final checks of validity, producing the outgoing message data.
fn final_checks(
&mut self,
upward_messages: &[UpwardMessage],
fees_charged: Option<Balance>,
) -> Result<Balance, Error> {
if self.upward != upward_messages {
return Err(Error::UpwardMessagesInvalid {
expected: upward_messages.to_vec(),
got: self.upward.clone(),
});
}
if let Some(fees_charged) = fees_charged {
if self.fees_charged != fees_charged {
return Err(Error::FeesChargedInvalid {
expected: fees_charged.clone(),
got: self.fees_charged.clone(),
});
}
}
Ok(self.fees_charged)
}
}
#[derive(Clone)]
struct Externalities(Arc<Mutex<ExternalitiesInner>>);
impl Externalities {
fn new(free_balance: Balance, fee_schedule: FeeSchedule) -> Self {
Self(Arc::new(Mutex::new(
ExternalitiesInner::new(free_balance, fee_schedule)
)))
}
}
impl wasm_executor::Externalities for Externalities {
fn post_upward_message(&mut self, message: UpwardMessage) -> Result<(), String> {
self.0.lock().post_upward_message(message)
}
}
/// Validate an erasure chunk against an expected root.
pub fn validate_chunk(
root: &Hash,
chunk: &ErasureChunk,
) -> Result<(), Error> {
let expected = erasure::branch_hash(root, &chunk.proof, chunk.index as usize)?;
) -> Result<(), ()> {
let expected = erasure::branch_hash(root, &chunk.proof, chunk.index as usize).map_err(|_| ())?;
let got = BlakeTwo256::hash(&chunk.chunk);
if expected != got {
return Err(Error::ErasureRootMismatch {
expected,
got,
})
return Err(())
}
Ok(())
}
// A utility function that implements most of the collation validation logic.
//
// Reused by `validate_collation` and `validate_receipt`.
// Returns outgoing messages, parent nead data, and fees charged for later reuse.
fn do_validation<P>(
client: &P,
relay_parent: &BlockId,
pov_block: &PoVBlock,
para_id: ParaId,
max_block_data_size: Option<u64>,
fees_charged: Option<Balance>,
head_data: &HeadData,
upward_messages: &Vec<UpwardMessage>,
) -> Result<(HeadData, Balance), Error> where
P: ProvideRuntimeApi<Block>,
P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
{
use parachain::ValidationParams;
if let Some(max_size) = max_block_data_size {
let block_data_size = pov_block.block_data.0.len() as u64;
if block_data_size > max_size {
return Err(Error::BlockDataTooBig { size: block_data_size, max_size });
}
}
let api = client.runtime_api();
let validation_code = api.parachain_code(relay_parent, para_id)?
.ok_or_else(|| Error::InactiveParachain(para_id))?;
let chain_status = api.parachain_status(relay_parent, para_id)?
.ok_or_else(|| Error::InactiveParachain(para_id))?;
let params = ValidationParams {
parent_head: chain_status.head_data.0.clone(),
block_data: pov_block.block_data.0.clone(),
};
let ext = Externalities::new(chain_status.balance, chain_status.fee_schedule);
match wasm_executor::validate_candidate(
&validation_code,
params,
ext.clone(),
ExecutionMode::Remote,
) {
Ok(result) => {
if result.head_data == head_data.0 {
let fees = ext.0.lock().final_checks(
upward_messages,
fees_charged
)?;
Ok((chain_status.head_data, fees))
} else {
Err(Error::WrongHeadData {
expected: head_data.0.clone(),
got: result.head_data
})
}
}
Err(e) => Err(e.into())
}
}
/// Produce a `CandidateReceipt` and erasure encoding chunks with a given collation.
///
/// To produce a `CandidateReceipt` among other things the root of erasure encoding of
/// the block data and messages needs to be known. To avoid redundant re-computations
/// of erasure encoding this method creates an encoding and produces a candidate with
/// encoding's root returning both for re-use.
pub fn produce_receipt_and_chunks(
n_validators: usize,
parent_head: HeadData,
pov: &PoVBlock,
fees: Balance,
info: &CollationInfo,
) -> Result<(CandidateReceipt, Vec<ErasureChunk>), Error>
{
let erasure_chunks = erasure::obtain_chunks(
n_validators,
&pov.block_data,
)?;
let branches = erasure::branches(erasure_chunks.as_ref());
let erasure_root = branches.root();
let chunks: Vec<_> = erasure_chunks
.iter()
.zip(branches.map(|(proof, _)| proof))
.enumerate()
.map(|(index, (chunk, proof))| ErasureChunk {
// branches borrows the original chunks, but this clone could probably be dodged.
chunk: chunk.clone(),
index: index as u32,
proof,
})
.collect();
let receipt = CandidateReceipt {
parachain_index: info.parachain_index,
collator: info.collator.clone(),
signature: info.signature.clone(),
head_data: info.head_data.clone(),
parent_head,
fees,
block_data_hash: info.block_data_hash.clone(),
upward_messages: info.upward_messages.clone(),
erasure_root,
};
Ok((receipt, chunks))
}
/// Check if a given candidate receipt is valid with a given collation.
///
/// This assumes that basic validity checks have been done:
/// - Block data hash is the same as linked in collation info and a receipt.
pub fn validate_receipt<P>(
client: &P,
relay_parent: &BlockId,
pov_block: &PoVBlock,
receipt: &CandidateReceipt,
max_block_data_size: Option<u64>,
) -> Result<Vec<ErasureChunk>, Error> where
P: ProvideRuntimeApi<Block>,
P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
{
let (parent_head, _fees) = do_validation(
client,
relay_parent,
pov_block,
receipt.parachain_index,
max_block_data_size,
Some(receipt.fees),
&receipt.head_data,
&receipt.upward_messages,
)?;
if parent_head != receipt.parent_head {
return Err(Error::ParentMismatch {
expected: receipt.parent_head.clone(),
got: parent_head,
});
}
let api = client.runtime_api();
let validators = api.validators(&relay_parent)?;
let n_validators = validators.len();
let (validated_receipt, chunks) = produce_receipt_and_chunks(
n_validators,
parent_head,
pov_block,
receipt.fees,
&receipt.clone().into(),
)?;
if validated_receipt.erasure_root != receipt.erasure_root {
return Err(Error::ErasureRootMismatch {
expected: validated_receipt.erasure_root,
got: receipt.erasure_root,
});
}
Ok(chunks)
}
/// Check whether a given collation is valid. Returns `Ok` on success, error otherwise.
/// Returns outgoing messages, parent head-data, and fees.
///
/// This assumes that basic validity checks have been done:
/// - Block data hash is the same as linked in collation info.
pub fn validate_collation<P>(
client: &P,
relay_parent: &BlockId,
collation: &Collation,
max_block_data_size: Option<u64>,
) -> Result<(HeadData, Balance), Error> where
P: ProvideRuntimeApi<Block>,
P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
{
let para_id = collation.info.parachain_index;
debug!("Validating collation for parachain {} at relay parent: {}", para_id, relay_parent);
do_validation(
client,
relay_parent,
&collation.pov,
para_id,
max_block_data_size,
None,
&collation.info.head_data,
&collation.info.upward_messages,
)
}
#[cfg(test)]
mod tests {
use super::*;
use parachain::wasm_executor::Externalities as ExternalitiesTrait;
use parachain::ParachainDispatchOrigin;
use polkadot_primitives::parachain::{CandidateReceipt, HeadData};
#[test]
fn ext_checks_upward_messages() {
let ext = || ExternalitiesInner {
upward: vec![
UpwardMessage{ data: vec![42], origin: ParachainDispatchOrigin::Parachain },
],
fees_charged: 0,
free_balance: 1_000_000,
fee_schedule: FeeSchedule {
base: 1000,
per_byte: 10,
},
};
let receipt = CandidateReceipt {
parachain_index: 5.into(),
collator: Default::default(),
signature: Default::default(),
head_data: HeadData(Vec::new()),
parent_head: HeadData(Vec::new()),
fees: 0,
block_data_hash: Default::default(),
upward_messages: vec![
UpwardMessage{ data: vec![42], origin: ParachainDispatchOrigin::Signed },
UpwardMessage{ data: vec![69], origin: ParachainDispatchOrigin::Parachain },
],
erasure_root: [1u8; 32].into(),
};
assert!(ext().final_checks(
&receipt.upward_messages,
Some(receipt.fees),
).is_err());
let receipt = CandidateReceipt {
parachain_index: 5.into(),
collator: Default::default(),
signature: Default::default(),
head_data: HeadData(Vec::new()),
parent_head: HeadData(Vec::new()),
fees: 0,
block_data_hash: Default::default(),
upward_messages: vec![
UpwardMessage{ data: vec![42], origin: ParachainDispatchOrigin::Signed },
],
erasure_root: [1u8; 32].into(),
};
assert!(ext().final_checks(
&receipt.upward_messages,
Some(receipt.fees),
).is_err());
let receipt = CandidateReceipt {
parachain_index: 5.into(),
collator: Default::default(),
signature: Default::default(),
head_data: HeadData(Vec::new()),
parent_head: HeadData(Vec::new()),
fees: 0,
block_data_hash: Default::default(),
upward_messages: vec![
UpwardMessage{ data: vec![69], origin: ParachainDispatchOrigin::Parachain },
],
erasure_root: [1u8; 32].into(),
};
assert!(ext().final_checks(
&receipt.upward_messages,
Some(receipt.fees),
).is_err());
let receipt = CandidateReceipt {
parachain_index: 5.into(),
collator: Default::default(),
signature: Default::default(),
head_data: HeadData(Vec::new()),
parent_head: HeadData(Vec::new()),
fees: 0,
block_data_hash: Default::default(),
upward_messages: vec![
UpwardMessage{ data: vec![42], origin: ParachainDispatchOrigin::Parachain },
],
erasure_root: [1u8; 32].into(),
};
assert!(ext().final_checks(
&receipt.upward_messages,
Some(receipt.fees),
).is_ok());
}
#[test]
fn ext_checks_fees_and_updates_correctly() {
let mut ext = ExternalitiesInner {
upward: vec![
UpwardMessage{ data: vec![42], origin: ParachainDispatchOrigin::Parachain },
],
fees_charged: 0,
free_balance: 1_000_000,
fee_schedule: FeeSchedule {
base: 1000,
per_byte: 10,
},
};
ext.apply_message_fee(100).unwrap();
assert_eq!(ext.fees_charged, 2000);
ext.post_upward_message(UpwardMessage {
origin: ParachainDispatchOrigin::Signed,
data: vec![0u8; 100],
}).unwrap();
assert_eq!(ext.fees_charged, 4000);
ext.apply_message_fee((1_000_000 - 4000 - 1000) / 10).unwrap();
assert_eq!(ext.fees_charged, 1_000_000);
// cannot pay fee.
assert!(ext.apply_message_fee(1).is_err());
}
}
polkadot/validation/src/error.rs
+34 -1
View File
@@ -16,7 +16,7 @@
//! Errors that can occur during the validation process.
use polkadot_primitives::parachain::ValidatorId;
use polkadot_primitives::{parachain::ValidatorId, Hash};
/// Error type for validation
#[derive(Debug, derive_more::Display, derive_more::From)]
@@ -25,6 +25,12 @@ pub enum Error {
Client(sp_blockchain::Error),
/// Consensus error
Consensus(consensus::error::Error),
/// A wasm-validation error.
WasmValidation(parachain::wasm_executor::Error),
/// An I/O error.
Io(std::io::Error),
/// An error in the availability erasure-coding.
ErasureCoding(polkadot_erasure_coding::Error),
#[display(fmt = "Invalid duty roster length: expected {}, got {}", expected, got)]
InvalidDutyRosterLength {
/// Expected roster length
@@ -48,6 +54,30 @@ pub enum Error {
DeadlineComputeFailure(std::time::Duration),
#[display(fmt = "Validation service is down.")]
ValidationServiceDown,
/// PoV-block in collation doesn't match provided.
#[display(fmt = "PoV hash mismatch. Expected {:?}, got {:?}", _0, _1)]
PoVHashMismatch(Hash, Hash),
/// Collator signature is invalid.
#[display(fmt = "Invalid collator signature on collation")]
InvalidCollatorSignature,
/// Head-data too large.
#[display(fmt = "Head data size of {} exceeded maximum of {}", _0, _1)]
HeadDataTooLarge(usize, usize),
/// Head-data mismatch after validation.
#[display(fmt = "Validation produced a different parachain header")]
HeadDataMismatch,
/// Relay parent of candidate not allowed.
#[display(fmt = "Relay parent {} of candidate not allowed in this context.", _0)]
DisallowedRelayParent(Hash),
/// Commitments in candidate match commitments produced by validation.
#[display(fmt = "Commitments in candidate receipt do not match those produced by validation")]
CommitmentsMismatch,
/// The parachain for which validation work is being done is not active.
#[display(fmt = "Parachain {:?} is not active", _0)]
InactiveParachain(polkadot_primitives::parachain::Id),
/// Block data is too big
#[display(fmt = "Block data is too big (maximum allowed size: {}, actual size: {})", size, max_size)]
BlockDataTooBig { size: u64, max_size: u64 },
Join(tokio::task::JoinError)
}
@@ -56,6 +86,9 @@ impl std::error::Error for Error {
match self {
Error::Client(ref err) => Some(err),
Error::Consensus(ref err) => Some(err),
Error::WasmValidation(ref err) => Some(err),
Error::ErasureCoding(ref err) => Some(err),
Error::Io(ref err) => Some(err),
_ => None,
}
}
polkadot/validation/src/evaluation.rs
-95
View File
@@ -1,95 +0,0 @@
// Copyright 2017-2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Polkadot block evaluation and evaluation errors.
// block size limit.
pub(crate) const MAX_TRANSACTIONS_SIZE: usize = 4 * 1024 * 1024;
use codec::Encode;
use polkadot_primitives::{Block, Hash, BlockNumber};
use polkadot_primitives::parachain::{Id as ParaId, CollatorId, Retriable};
/// Result type alias for block evaluation
pub type Result<T> = std::result::Result<T, Error>;
/// Error type for block evaluation
#[derive(Debug, derive_more::Display, derive_more::From)]
pub enum Error {
/// Client error
Client(sp_blockchain::Error),
/// Too many parachain candidates in proposal
#[display(fmt = "Proposal included {} candidates for {} parachains", expected, got)]
TooManyCandidates { expected: usize, got: usize },
/// Proposal included unregistered parachain
#[display(fmt = "Proposal included unregistered parachain {:?}", _0)]
UnknownParachain(ParaId),
/// Proposal had wrong parent hash
#[display(fmt = "Proposal had wrong parent hash. Expected {:?}, got {:?}", expected, got)]
WrongParentHash { expected: Hash, got: Hash },
/// Proposal had wrong number
#[display(fmt = "Proposal had wrong number. Expected {:?}, got {:?}", expected, got)]
WrongNumber { expected: BlockNumber, got: BlockNumber },
/// Proposal exceeded the maximum size
#[display(
fmt = "Proposal exceeded the maximum size of {} by {} bytes.",
MAX_TRANSACTIONS_SIZE, MAX_TRANSACTIONS_SIZE.saturating_sub(*_0)
)]
ProposalTooLarge(usize),
}
impl std::error::Error for Error {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
Error::Client(ref err) => Some(err),
_ => None,
}
}
}
/// Attempt to evaluate a substrate block as a polkadot block, returning error
/// upon any initial validity checks failing.
pub fn evaluate_initial(
proposal: &Block,
_now: u64,
parent_hash: &Hash,
parent_number: BlockNumber,
_active_parachains: &[(ParaId, Option<(CollatorId, Retriable)>)],
) -> Result<()> {
let transactions_size = proposal.extrinsics.iter().fold(0, |a, tx| {
a + Encode::encode(tx).len()
});
if transactions_size > MAX_TRANSACTIONS_SIZE {
return Err(Error::ProposalTooLarge(transactions_size))
}
if proposal.header.parent_hash != *parent_hash {
return Err(Error::WrongParentHash {
expected: *parent_hash,
got: proposal.header.parent_hash
});
}
if proposal.header.number != parent_number + 1 {
return Err(Error::WrongNumber {
expected: parent_number + 1,
got: proposal.header.number
});
}
Ok(())
}
polkadot/validation/src/lib.rs
+12 -21
View File
@@ -33,13 +33,12 @@ use std::{
collections::{HashMap, HashSet},
sync::Arc,
};
use codec::Encode;
use polkadot_primitives::Hash;
use polkadot_primitives::parachain::{
Id as ParaId, Chain, DutyRoster, CandidateReceipt,
Id as ParaId, Chain, DutyRoster, AbridgedCandidateReceipt,
Statement as PrimitiveStatement,
Collation, PoVBlock, ErasureChunk, ValidatorSignature, ValidatorIndex,
PoVBlock, ErasureChunk, ValidatorSignature, ValidatorIndex,
ValidatorPair, ValidatorId,
};
use primitives::Pair;
@@ -47,10 +46,7 @@ use primitives::Pair;
use futures::prelude::*;
pub use self::block_production::ProposerFactory;
pub use self::collation::{
validate_collation, message_queue_root, Collators,
produce_receipt_and_chunks,
};
pub use self::collation::Collators;
pub use self::error::Error;
pub use self::shared_table::{
SharedTable, ParachainWork, PrimedParachainWork, Validated, Statement, SignedStatement,
@@ -62,13 +58,13 @@ pub use self::validation_service::{ServiceHandle, ServiceBuilder};
pub use parachain::wasm_executor::{run_worker as run_validation_worker};
mod dynamic_inclusion;
mod evaluation;
mod error;
mod shared_table;
pub mod collation;
pub mod validation_service;
pub mod block_production;
pub mod collation;
pub mod pipeline;
pub mod validation_service;
/// A handle to a statement table router.
///
@@ -83,12 +79,11 @@ pub trait TableRouter: Clone {
/// Future that resolves when candidate data is fetched.
type FetchValidationProof: Future<Output=Result<PoVBlock, Self::Error>>;
/// Call with local candidate data. This will make the data available on the network,
/// and sign, import, and broadcast a statement about the candidate.
/// Call with local candidate data. This will sign, import, and broadcast a statement about the candidate.
fn local_collation(
&self,
collation: Collation,
receipt: CandidateReceipt,
receipt: AbridgedCandidateReceipt,
pov_block: PoVBlock,
chunks: (ValidatorIndex, &[ErasureChunk]),
) -> Self::SendLocalCollation;
@@ -96,7 +91,7 @@ pub trait TableRouter: Clone {
///
/// This future must conclude once all `Clone`s of this `TableRouter` have
/// been cleaned up.
fn fetch_pov_block(&self, candidate: &CandidateReceipt) -> Self::FetchValidationProof;
fn fetch_pov_block(&self, candidate: &AbridgedCandidateReceipt) -> Self::FetchValidationProof;
}
/// A long-lived network which can create parachain statement and BFT message routing processes on demand.
@@ -141,10 +136,7 @@ pub struct GroupInfo {
/// The actual message signed is the encoded statement concatenated with the
/// parent hash.
pub fn sign_table_statement(statement: &Statement, key: &ValidatorPair, parent_hash: &Hash) -> ValidatorSignature {
// we sign using the primitive statement type because that's what the runtime
// expects. These types probably encode the same way so this clone could be optimized
// out in the future.
let mut encoded = PrimitiveStatement::from(statement.clone()).encode();
let mut encoded = PrimitiveStatement::from(statement).encode();
encoded.extend(parent_hash.as_ref());
key.sign(&encoded)
@@ -159,7 +151,7 @@ pub fn check_statement(
) -> bool {
use runtime_primitives::traits::AppVerify;
let mut encoded = PrimitiveStatement::from(statement.clone()).encode();
let mut encoded = PrimitiveStatement::from(statement).encode();
encoded.extend(parent_hash.as_ref());
signature.verify(&encoded[..], &signer)
@@ -211,7 +203,6 @@ pub fn make_group_info(
});
Ok((map, local_duty))
}
#[cfg(test)]
polkadot/validation/src/pipeline.rs
+375
View File
@@ -0,0 +1,375 @@
// Copyright 2020 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! The pipeline of validation functions a parachain block must pass through before
//! it can be voted for.
use std::sync::Arc;
use codec::Encode;
use polkadot_erasure_coding as erasure;
use polkadot_primitives::parachain::{
CollationInfo, PoVBlock, LocalValidationData, GlobalValidationSchedule, OmittedValidationData,
AvailableData, FeeSchedule, CandidateCommitments, ErasureChunk, HeadData, ParachainHost,
Id as ParaId, AbridgedCandidateReceipt,
};
use polkadot_primitives::{Block, BlockId, Balance, Hash};
use parachain::{
wasm_executor::{self, ExecutionMode},
UpwardMessage, ValidationParams,
};
use runtime_primitives::traits::{BlakeTwo256, Hash as HashT};
use sp_api::ProvideRuntimeApi;
use parking_lot::Mutex;
use crate::Error;
/// Does basic checks of a collation. Provide the encoded PoV-block.
pub fn basic_checks(
collation: &CollationInfo,
expected_relay_parent: &Hash,
max_block_data_size: Option<u64>,
encoded_pov: &[u8],
) -> Result<(), Error> {
if &collation.relay_parent != expected_relay_parent {
return Err(Error::DisallowedRelayParent(collation.relay_parent));
}
if let Some(max_size) = max_block_data_size {
if encoded_pov.len() as u64 > max_size {
return Err(Error::BlockDataTooBig { size: encoded_pov.len() as _, max_size });
}
}
let hash = BlakeTwo256::hash(encoded_pov);
if hash != collation.pov_block_hash {
return Err(Error::PoVHashMismatch(collation.pov_block_hash, hash));
}
if let Err(()) = collation.check_signature() {
return Err(Error::InvalidCollatorSignature);
}
Ok(())
}
struct ExternalitiesInner {
upward: Vec<UpwardMessage>,
fees_charged: Balance,
free_balance: Balance,
fee_schedule: FeeSchedule,
}
impl wasm_executor::Externalities for ExternalitiesInner {
fn post_upward_message(&mut self, message: UpwardMessage) -> Result<(), String> {
self.apply_message_fee(message.data.len())?;
self.upward.push(message);
Ok(())
}
}
impl ExternalitiesInner {
fn new(free_balance: Balance, fee_schedule: FeeSchedule) -> Self {
Self {
free_balance,
fee_schedule,
fees_charged: 0,
upward: Vec::new(),
}
}
fn apply_message_fee(&mut self, message_len: usize) -> Result<(), String> {
let fee = self.fee_schedule.compute_fee(message_len);
let new_fees_charged = self.fees_charged.saturating_add(fee);
if new_fees_charged > self.free_balance {
Err("could not cover fee.".into())
} else {
self.fees_charged = new_fees_charged;
Ok(())
}
}
// Returns the noted outputs of execution so far - upward messages and balances.
fn outputs(self) -> (Vec<UpwardMessage>, Balance) {
(self.upward, self.fees_charged)
}
}
#[derive(Clone)]
struct Externalities(Arc<Mutex<ExternalitiesInner>>);
impl Externalities {
fn new(free_balance: Balance, fee_schedule: FeeSchedule) -> Self {
Self(Arc::new(Mutex::new(
ExternalitiesInner::new(free_balance, fee_schedule)
)))
}
}
impl wasm_executor::Externalities for Externalities {
fn post_upward_message(&mut self, message: UpwardMessage) -> Result<(), String> {
self.0.lock().post_upward_message(message)
}
}
/// Data from a fully-outputted validation of a parachain candidate. This contains
/// all outputs and commitments of the validation as well as all additional data to make available.
pub struct FullOutput {
/// Data about the candidate to keep available in the network.
pub available_data: AvailableData,
/// Commitments issued alongside the candidate to be placed on-chain.
pub commitments: CandidateCommitments,
/// All erasure-chunks associated with the available data. Each validator
/// should keep their chunk (by index). Other chunks do not need to be
/// kept available long-term, but should be distributed to other validators.
pub erasure_chunks: Vec<ErasureChunk>,
}
impl FullOutput {
/// Check consistency of the outputs produced by the validation pipeline against
/// data contained within a candidate receipt.
pub fn check_consistency(&self, receipt: &AbridgedCandidateReceipt) -> Result<(), Error> {
if self.commitments != receipt.commitments {
Err(Error::CommitmentsMismatch)
} else {
Ok(())
}
}
}
/// The successful result of validating a collation. If the full commitments of the
/// validation are needed, call `full_output`. Otherwise, safely drop this value.
pub struct ValidatedCandidate<'a> {
pov_block: &'a PoVBlock,
global_validation: &'a GlobalValidationSchedule,
parent_head: &'a HeadData,
balance: Balance,
upward_messages: Vec<UpwardMessage>,
fees: Balance,
}
impl<'a> ValidatedCandidate<'a> {
/// Fully-compute the commitments and outputs of the candidate. Provide the number
/// of validators. This computes the erasure-coding.
pub fn full_output(self, n_validators: usize) -> Result<FullOutput, Error> {
let ValidatedCandidate {
pov_block,
global_validation,
parent_head,
balance,
upward_messages,
fees,
} = self;
let omitted_validation = OmittedValidationData {
global_validation: global_validation.clone(),
local_validation: LocalValidationData {
parent_head: parent_head.clone(),
balance,
},
};
let available_data = AvailableData {
pov_block: pov_block.clone(),
omitted_validation,
};
let erasure_chunks = erasure::obtain_chunks(
n_validators,
&available_data,
)?;
let branches = erasure::branches(erasure_chunks.as_ref());
let erasure_root = branches.root();
let chunks: Vec<_> = erasure_chunks
.iter()
.zip(branches.map(|(proof, _)| proof))
.enumerate()
.map(|(index, (chunk, proof))| ErasureChunk {
// branches borrows the original chunks, but this clone could probably be dodged.
chunk: chunk.clone(),
index: index as u32,
proof,
})
.collect();
let commitments = CandidateCommitments {
upward_messages,
fees,
erasure_root,
};
Ok(FullOutput {
available_data,
commitments,
erasure_chunks: chunks,
})
}
}
/// Does full checks of a collation, with provided PoV-block and contextual data.
pub fn validate<'a>(
collation: &'a CollationInfo,
pov_block: &'a PoVBlock,
local_validation: &'a LocalValidationData,
global_validation: &'a GlobalValidationSchedule,
validation_code: &[u8],
) -> Result<ValidatedCandidate<'a>, Error> {
if collation.head_data.0.len() > global_validation.max_head_data_size as _ {
return Err(Error::HeadDataTooLarge(
collation.head_data.0.len(),
global_validation.max_head_data_size as _,
));
}
let params = ValidationParams {
parent_head: collation.head_data.0.clone(),
block_data: pov_block.block_data.0.clone(),
};
// TODO: remove when ext does not do this.
let fee_schedule = FeeSchedule {
base: 0,
per_byte: 0,
};
let ext = Externalities::new(local_validation.balance, fee_schedule);
match wasm_executor::validate_candidate(
&validation_code,
params,
ext.clone(),
ExecutionMode::Remote,
) {
Ok(result) => {
if result.head_data == collation.head_data.0 {
let (upward_messages, fees) = Arc::try_unwrap(ext.0)
.map_err(|_| "<non-unique>")
.expect("Wasm executor drops passed externalities on completion; \
call has concluded; qed")
.into_inner()
.outputs();
Ok(ValidatedCandidate {
pov_block,
global_validation,
parent_head: &local_validation.parent_head,
balance: local_validation.balance,
upward_messages,
fees,
})
} else {
Err(Error::HeadDataMismatch)
}
}
Err(e) => Err(e.into()),
}
}
/// Extracts validation parameters from a Polkadot runtime API for a specific parachain.
pub fn validation_params<P>(api: &P, relay_parent: Hash, para_id: ParaId)
-> Result<(LocalValidationData, GlobalValidationSchedule, Vec<u8>), Error>
where
P: ProvideRuntimeApi<Block>,
P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
{
let api = api.runtime_api();
let relay_parent = BlockId::hash(relay_parent);
// fetch all necessary data from runtime.
let local_validation = api.local_validation_data(&relay_parent, para_id)?
.ok_or_else(|| Error::InactiveParachain(para_id))?;
let global_validation = api.global_validation_schedule(&relay_parent)?;
let validation_code = api.parachain_code(&relay_parent, para_id)?
.ok_or_else(|| Error::InactiveParachain(para_id))?;
Ok((local_validation, global_validation, validation_code))
}
/// Does full-pipeline validation of a collation with provided contextual parameters.
pub fn full_output_validation_with_api<P>(
api: &P,
collation: &CollationInfo,
pov_block: &PoVBlock,
expected_relay_parent: &Hash,
max_block_data_size: Option<u64>,
n_validators: usize,
) -> Result<FullOutput, Error> where
P: ProvideRuntimeApi<Block>,
P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
{
let para_id = collation.parachain_index;
let (local_validation, global_validation, validation_code)
= validation_params(&*api, collation.relay_parent, para_id)?;
// put the parameters through the validation pipeline, producing
// erasure chunks.
let encoded_pov = pov_block.encode();
basic_checks(
&collation,
&expected_relay_parent,
max_block_data_size,
&encoded_pov,
)
.and_then(|()| validate(
&collation,
&pov_block,
&local_validation,
&global_validation,
&validation_code,
))
.and_then(|validated| validated.full_output(n_validators))
}
#[cfg(test)]
mod tests {
use super::*;
use parachain::wasm_executor::Externalities as ExternalitiesTrait;
use parachain::ParachainDispatchOrigin;
#[test]
fn ext_checks_fees_and_updates_correctly() {
let mut ext = ExternalitiesInner {
upward: vec![
UpwardMessage{ data: vec![42], origin: ParachainDispatchOrigin::Parachain },
],
fees_charged: 0,
free_balance: 1_000_000,
fee_schedule: FeeSchedule {
base: 1000,
per_byte: 10,
},
};
ext.apply_message_fee(100).unwrap();
assert_eq!(ext.fees_charged, 2000);
ext.post_upward_message(UpwardMessage {
origin: ParachainDispatchOrigin::Signed,
data: vec![0u8; 100],
}).unwrap();
assert_eq!(ext.fees_charged, 4000);
ext.apply_message_fee((1_000_000 - 4000 - 1000) / 10).unwrap();
assert_eq!(ext.fees_charged, 1_000_000);
// cannot pay fee.
assert!(ext.apply_message_fee(1).is_err());
}
}
polkadot/validation/src/shared_table/mod.rs
+153 -154
View File
@@ -22,10 +22,10 @@ use std::sync::Arc;
use availability_store::{Store as AvailabilityStore};
use table::{self, Table, Context as TableContextTrait};
use polkadot_primitives::{Block, BlockId, Hash};
use polkadot_primitives::{Block, Hash};
use polkadot_primitives::parachain::{
Id as ParaId, CandidateReceipt, ValidatorPair, ValidatorId,
AttestedCandidate, ParachainHost, PoVBlock, ValidatorIndex, ErasureChunk,
Id as ParaId, AbridgedCandidateReceipt, ValidatorPair, ValidatorId,
AttestedCandidate, ParachainHost, PoVBlock, ValidatorIndex,
};
use parking_lot::Mutex;
@@ -39,6 +39,9 @@ use self::includable::IncludabilitySender;
use primitives::Pair;
use sp_api::ProvideRuntimeApi;
use crate::pipeline::FullOutput;
use crate::Error;
mod includable;
pub use table::{SignedStatement, Statement};
@@ -188,8 +191,8 @@ impl SharedTableInner {
availability_store: self.availability_store.clone(),
relay_parent: context.parent_hash.clone(),
work,
local_index: local_index as usize,
max_block_data_size,
n_validators: context.validators.len(),
})
}
@@ -232,7 +235,7 @@ impl Validated {
/// Note that we've collated a candidate.
/// outgoing message required.
pub fn collated_local(
receipt: CandidateReceipt,
receipt: AbridgedCandidateReceipt,
collation: PoVBlock,
) -> Self {
Validated {
@@ -253,9 +256,9 @@ impl Validated {
pub struct ParachainWork<Fetch> {
work: Work<Fetch>,
relay_parent: Hash,
local_index: usize,
availability_store: AvailabilityStore,
max_block_data_size: Option<u64>,
n_validators: usize,
}
impl<Fetch: Future + Unpin> ParachainWork<Fetch> {
@@ -264,48 +267,47 @@ impl<Fetch: Future + Unpin> ParachainWork<Fetch> {
pub fn prime<P: ProvideRuntimeApi<Block>>(self, api: Arc<P>)
-> PrimedParachainWork<
Fetch,
impl Send + FnMut(&BlockId, &PoVBlock, &CandidateReceipt) -> Result<ErasureChunk, ()> + Unpin,
impl Send + FnMut(&PoVBlock, &AbridgedCandidateReceipt)
-> Result<FullOutput, Error> + Unpin,
>
where
P: Send + Sync + 'static,
P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
{
let max_block_data_size = self.max_block_data_size;
let local_index = self.local_index;
let n_validators = self.n_validators;
let expected_relay_parent = self.relay_parent;
let validate = move |id: &_, pov_block: &_, receipt: &_| {
let res = crate::collation::validate_receipt(
let validate = move |pov_block: &PoVBlock, candidate: &AbridgedCandidateReceipt| {
let collation_info = candidate.to_collation_info();
let full_output = crate::pipeline::full_output_validation_with_api(
&*api,
id,
&collation_info,
pov_block,
receipt,
&expected_relay_parent,
max_block_data_size,
);
n_validators,
)?;
match res {
Ok(mut chunks) => {
Ok(chunks.swap_remove(local_index))
}
Err(e) => {
debug!(target: "validation", "Encountered bad collation: {}", e);
Err(())
}
}
full_output.check_consistency(candidate)?;
Ok(full_output)
};
PrimedParachainWork { inner: self, validate }
}
/// Prime the parachain work with a custom validation function.
#[cfg(test)]
pub fn prime_with<F>(self, validate: F) -> PrimedParachainWork<Fetch, F>
where F: FnMut(&BlockId, &PoVBlock, &CandidateReceipt) -> Result<ErasureChunk, ()>
where F: FnMut(&PoVBlock, &AbridgedCandidateReceipt)
-> Result<FullOutput, Error>
{
PrimedParachainWork { inner: self, validate }
}
}
struct Work<Fetch> {
candidate_receipt: CandidateReceipt,
candidate_receipt: AbridgedCandidateReceipt,
fetch: Fetch
}
@@ -318,46 +320,62 @@ pub struct PrimedParachainWork<Fetch, F> {
impl<Fetch, F, Err> PrimedParachainWork<Fetch, F>
where
Fetch: Future<Output=Result<PoVBlock,Err>> + Unpin,
F: FnMut(&BlockId, &PoVBlock, &CandidateReceipt) -> Result<ErasureChunk, ()> + Unpin,
Err: From<::std::io::Error>,
F: FnMut(&PoVBlock, &AbridgedCandidateReceipt) -> Result<FullOutput, Error> + Unpin,
Err: From<std::io::Error>,
{
pub async fn validate(mut self) -> Result<(Validated, Option<ErasureChunk>), Err> {
let candidate = &self.inner.work.candidate_receipt;
pub async fn validate(self) -> Result<Validated, Err> {
let candidate = self.inner.work.candidate_receipt;
let pov_block = self.inner.work.fetch.await?;
let validation_res = (self.validate)(
&BlockId::hash(self.inner.relay_parent),
&pov_block,
&candidate,
);
let mut validate = self.validate;
let relay_parent = self.inner.relay_parent;
// create a wrapper around the custom validation function that does
// some more general pre and post checks.
let mut validate = move |pov_block: &_, candidate: &AbridgedCandidateReceipt| {
if candidate.relay_parent != relay_parent {
return Err(Error::DisallowedRelayParent(candidate.relay_parent));
}
let full_output = validate(pov_block, candidate)?;
if full_output.commitments != candidate.commitments {
return Err(Error::CommitmentsMismatch);
}
Ok(full_output)
};
let validation_res = (validate)(&pov_block, &candidate);
let candidate_hash = candidate.hash();
debug!(target: "validation", "Making validity statement about candidate {}: is_good? {:?}",
candidate_hash, validation_res.is_ok());
match validation_res {
Err(()) => Ok((
Validated {
Err(err) => {
debug!(target: "validation", "candidate is invalid: {}", err);
Ok(Validated {
statement: GenericStatement::Invalid(candidate_hash),
result: Validation::Invalid(pov_block),
},
None,
)),
Ok(our_chunk) => {
self.inner.availability_store.add_erasure_chunk(
self.inner.relay_parent,
candidate.clone(),
our_chunk.clone(),
})
}
Ok(full_output) => {
// make data and all erasure chunks available. The chunk
// must be fully available before we add the chunks.
self.inner.availability_store.make_available(
candidate_hash,
full_output.available_data,
).await?;
self.inner.availability_store.add_erasure_chunks(
candidate,
full_output.erasure_chunks,
).await?;
Ok((
Validated {
statement: GenericStatement::Valid(candidate_hash),
result: Validation::Valid(pov_block),
},
Some(our_chunk),
))
Ok(Validated {
statement: GenericStatement::Valid(candidate_hash),
result: Validation::Valid(pov_block),
})
}
}
}
@@ -461,7 +479,8 @@ impl SharedTable {
}
/// Sign and import the result of candidate validation. Returns `None` if the table
/// was instantiated without a local key.
/// was instantiated without a local key. Otherwise, returns a copy of the signed
/// statement.
pub fn import_validated(&self, validated: Validated)
-> Option<SignedStatement>
{
@@ -485,7 +504,7 @@ impl SharedTable {
///
/// Deadlocks if called recursively.
pub fn with_candidate<F, U>(&self, digest: &Hash, f: F) -> U
where F: FnOnce(Option<&CandidateReceipt>) -> U
where F: FnOnce(Option<&AbridgedCandidateReceipt>) -> U
{
let inner = self.inner.lock();
f(inner.table.get_candidate(digest))
@@ -570,8 +589,9 @@ impl SharedTable {
mod tests {
use super::*;
use sp_keyring::Sr25519Keyring;
use primitives::crypto::UncheckedInto;
use polkadot_primitives::parachain::{BlockData, Collation, HeadData};
use polkadot_primitives::parachain::{
BlockData, ErasureChunk, AvailableData,
};
use polkadot_erasure_coding::{self as erasure};
use availability_store::ProvideGossipMessages;
use futures::future;
@@ -613,12 +633,12 @@ mod tests {
fn local_collation(
&self,
_collation: Collation,
_candidate: CandidateReceipt,
_candidate: AbridgedCandidateReceipt,
_pov_block: PoVBlock,
_chunks: (ValidatorIndex, &[ErasureChunk])
) -> Self::SendLocalCollation { future::ready(Ok(())) }
fn fetch_pov_block(&self, _candidate: &CandidateReceipt) -> Self::FetchValidationProof {
fn fetch_pov_block(&self, _candidate: &AbridgedCandidateReceipt) -> Self::FetchValidationProof {
future::ok(pov_block_with_data(vec![1, 2, 3, 4, 5]))
}
}
@@ -652,17 +672,9 @@ mod tests {
None,
);
let candidate = CandidateReceipt {
parachain_index: para_id,
collator: [1; 32].unchecked_into(),
signature: Default::default(),
head_data: HeadData(vec![1, 2, 3, 4]),
parent_head: HeadData(vec![]),
fees: 1_000_000,
block_data_hash: [2; 32].into(),
upward_messages: Vec::new(),
erasure_root: [1u8; 32].into(),
};
let mut candidate = AbridgedCandidateReceipt::default();
candidate.parachain_index = para_id;
candidate.relay_parent = parent_hash;
let candidate_statement = GenericStatement::Candidate(candidate);
@@ -708,17 +720,9 @@ mod tests {
None,
);
let candidate = CandidateReceipt {
parachain_index: para_id,
collator: [1; 32].unchecked_into(),
signature: Default::default(),
head_data: HeadData(vec![1, 2, 3, 4]),
parent_head: HeadData(vec![]),
fees: 1_000_000,
block_data_hash: [2; 32].into(),
upward_messages: Vec::new(),
erasure_root: [1u8; 32].into(),
};
let mut candidate = AbridgedCandidateReceipt::default();
candidate.parachain_index = para_id;
candidate.relay_parent = parent_hash;
let candidate_statement = GenericStatement::Candidate(candidate);
@@ -741,25 +745,18 @@ mod tests {
let relay_parent = [0; 32].into();
let para_id = 5.into();
let pov_block = pov_block_with_data(vec![1, 2, 3]);
let block_data_hash = [2; 32].into();
let pov_block_hash = pov_block.hash();
let local_index = 0;
let n_validators = 2;
let candidate = CandidateReceipt {
parachain_index: para_id,
collator: [1; 32].unchecked_into(),
signature: Default::default(),
head_data: HeadData(vec![1, 2, 3, 4]),
parent_head: HeadData(vec![]),
fees: 1_000_000,
block_data_hash,
upward_messages: Vec::new(),
erasure_root: [1u8; 32].into(),
};
let mut candidate = AbridgedCandidateReceipt::default();
candidate.parachain_index = para_id;
candidate.relay_parent = relay_parent;
candidate.pov_block_hash = pov_block_hash;
let hash = candidate.hash();
let candidate_hash = candidate.hash();
store.add_validator_index_and_n_validators(
store.note_validator_index_and_n_validators(
&relay_parent,
local_index as u32,
n_validators as u32,
@@ -770,25 +767,37 @@ mod tests {
candidate_receipt: candidate,
fetch: future::ok(pov_block.clone()),
},
local_index,
relay_parent,
availability_store: store.clone(),
max_block_data_size: None,
n_validators,
};
let validated = block_on(producer.prime_with(|_, _, _| Ok(
ErasureChunk {
chunk: vec![1, 2, 3],
index: local_index as u32,
proof: vec![],
for i in 0..n_validators {
assert!(store.get_erasure_chunk(&candidate_hash, i).is_none());
}
let validated = block_on(producer.prime_with(|_, _| Ok(
FullOutput {
available_data: AvailableData {
pov_block: pov_block.clone(),
omitted_validation: Default::default(),
},
erasure_chunks: (0..n_validators).map(|i| ErasureChunk {
chunk: vec![1, 2, 3],
index: i as u32,
proof: vec![],
}).collect(),
commitments: Default::default(),
}
)).validate()).unwrap();
assert_eq!(validated.0.pov_block(), &pov_block);
assert_eq!(validated.0.statement, GenericStatement::Valid(hash));
assert_eq!(validated.pov_block(), &pov_block);
assert_eq!(validated.statement, GenericStatement::Valid(candidate_hash));
assert!(store.get_erasure_chunk(&relay_parent, block_data_hash, local_index).is_some());
assert!(store.get_erasure_chunk(&relay_parent, block_data_hash, local_index + 1).is_none());
for i in 0..n_validators {
assert!(store.get_erasure_chunk(&candidate_hash, i).is_some());
}
}
#[test]
@@ -797,25 +806,25 @@ mod tests {
let relay_parent = [0; 32].into();
let para_id = 5.into();
let pov_block = pov_block_with_data(vec![1, 2, 3]);
let block_data_hash = pov_block.block_data.hash();
let pov_block_hash = pov_block.hash();
let local_index = 0;
let n_validators = 2;
let candidate = CandidateReceipt {
parachain_index: para_id,
collator: [1; 32].unchecked_into(),
signature: Default::default(),
head_data: HeadData(vec![1, 2, 3, 4]),
parent_head: HeadData(vec![]),
fees: 1_000_000,
block_data_hash: [2; 32].into(),
upward_messages: Vec::new(),
erasure_root: [1u8; 32].into(),
let mut candidate = AbridgedCandidateReceipt::default();
candidate.parachain_index = para_id;
candidate.pov_block_hash = pov_block_hash;
candidate.relay_parent = relay_parent;
let candidate_hash = candidate.hash();
let available_data = AvailableData {
pov_block: pov_block.clone(),
omitted_validation: Default::default(),
};
let chunks = erasure::obtain_chunks(n_validators, &pov_block.block_data).unwrap();
let chunks = erasure::obtain_chunks(n_validators, &available_data).unwrap();
store.add_validator_index_and_n_validators(
store.note_validator_index_and_n_validators(
&relay_parent,
local_index as u32,
n_validators as u32,
@@ -826,26 +835,30 @@ mod tests {
candidate_receipt: candidate,
fetch: future::ok::<_, ::std::io::Error>(pov_block.clone()),
},
local_index,
relay_parent,
availability_store: store.clone(),
max_block_data_size: None,
n_validators,
};
let validated = block_on(producer.prime_with(|_, _, _| Ok(
ErasureChunk {
chunk: chunks[local_index].clone(),
index: local_index as u32,
proof: vec![],
let validated = block_on(producer.prime_with(|_, _| Ok(
FullOutput {
available_data: AvailableData {
pov_block: pov_block.clone(),
omitted_validation: Default::default(),
},
)).validate()).unwrap();
erasure_chunks: (0..n_validators).map(|i| ErasureChunk {
chunk: chunks[i].clone(),
index: i as u32,
proof: vec![],
}).collect(),
commitments: Default::default(),
}
)).validate()).unwrap();
assert_eq!(validated.0.pov_block(), &pov_block);
assert_eq!(validated.pov_block(), &pov_block);
// This works since there are only two validators and one erasure chunk should be
// enough to reconstruct the block data.
assert_eq!(store.block_data(relay_parent, block_data_hash).unwrap(), pov_block.block_data);
// TODO: check that a message queue is included by root.
assert_eq!(store.execution_data(&candidate_hash).unwrap().pov_block, pov_block);
}
#[test]
@@ -877,19 +890,11 @@ mod tests {
None,
);
let candidate = CandidateReceipt {
parachain_index: para_id,
collator: [1; 32].unchecked_into(),
signature: Default::default(),
head_data: HeadData(vec![1, 2, 3, 4]),
parent_head: HeadData(vec![]),
fees: 1_000_000,
block_data_hash: [2; 32].into(),
upward_messages: Vec::new(),
erasure_root: [1u8; 32].into(),
};
let mut candidate = AbridgedCandidateReceipt::default();
candidate.parachain_index = para_id;
candidate.relay_parent = parent_hash;
let hash = candidate.hash();
let candidate_hash = candidate.hash();
let candidate_statement = GenericStatement::Candidate(candidate);
let signature = crate::sign_table_statement(&candidate_statement, &validity_other_key.into(), &parent_hash);
@@ -904,7 +909,8 @@ mod tests {
signed_statement.clone(),
).expect("should produce work");
assert!(shared_table.inner.lock().validated.get(&hash).expect("validation has started").is_in_progress());
assert!(shared_table.inner.lock().validated.get(&candidate_hash)
.expect("validation has started").is_in_progress());
let b = shared_table.import_remote_statement(
&DummyRouter,
@@ -943,25 +949,18 @@ mod tests {
None,
);
let candidate = CandidateReceipt {
parachain_index: para_id,
collator: [1; 32].unchecked_into(),
signature: Default::default(),
head_data: HeadData(vec![1, 2, 3, 4]),
parent_head: HeadData(vec![]),
fees: 1_000_000,
block_data_hash: [2; 32].into(),
upward_messages: Vec::new(),
erasure_root: [1u8; 32].into(),
};
let mut candidate = AbridgedCandidateReceipt::default();
candidate.parachain_index = para_id;
candidate.relay_parent = parent_hash;
let hash = candidate.hash();
let candidate_hash = candidate.hash();
let signed_statement = shared_table.import_validated(Validated::collated_local(
candidate,
pov_block,
)).unwrap();
assert!(shared_table.inner.lock().validated.get(&hash).expect("validation has started").is_done());
assert!(shared_table.inner.lock().validated.get(&candidate_hash)
.expect("validation has started").is_done());
let a = shared_table.import_remote_statement(
&DummyRouter,
polkadot/validation/src/validation_service/mod.rs
+42 -37
View File
@@ -322,7 +322,7 @@ impl<C, N, P, SP> ParachainValidationInstances<C, N, P, SP> where
// If we are a validator, we need to store our index in this round in availability store.
// This will tell which erasure chunk we should store.
if let Some(ref local_duty) = local_duty {
if let Err(e) = self.availability_store.add_validator_index_and_n_validators(
if let Err(e) = self.availability_store.note_validator_index_and_n_validators(
&parent_hash,
local_duty.index,
validators.len() as u32,
@@ -374,7 +374,7 @@ impl<C, N, P, SP> ParachainValidationInstances<C, N, P, SP> where
validation_para: ParaId,
build_router: N::BuildTableRouter,
max_block_data_size: Option<u64>,
authorities_num: usize,
n_validators: usize,
local_id: ValidatorIndex,
) {
let (collators, client) = (self.collators.clone(), self.client.clone());
@@ -388,49 +388,54 @@ impl<C, N, P, SP> ParachainValidationInstances<C, N, P, SP> where
collators,
client.clone(),
max_block_data_size,
n_validators,
);
collation_work.then(move |result| match result {
Ok((collation, parent_head, fees_charged)) => {
match crate::collation::produce_receipt_and_chunks(
authorities_num,
parent_head,
&collation.pov,
fees_charged,
&collation.info,
) {
Ok((receipt, chunks)) => {
// Apparently the `async move` block is the only way to convince
// the compiler that we are not moving values out of borrowed context.
let av_clone = availability_store.clone();
let chunks_clone = chunks.clone();
let receipt_clone = receipt.clone();
Ok((collation_info, full_output)) => {
let crate::pipeline::FullOutput {
commitments,
erasure_chunks,
available_data,
} = full_output;
let res = async move {
if let Err(e) = av_clone.clone().add_erasure_chunks(
relay_parent.clone(),
receipt_clone,
chunks_clone,
).await {
warn!(target: "validation", "Failed to add erasure chunks: {}", e);
}
}
.unit_error()
.then(move |_| {
router.local_collation(
collation,
receipt,
(local_id, &chunks),
).map_err(|e| warn!(target: "validation", "Failed to send local collation: {:?}", e))
});
let receipt = collation_info.into_receipt(commitments);
res.boxed()
// Apparently the `async move` block is the only way to convince
// the compiler that we are not moving values out of borrowed context.
let av_clone = availability_store.clone();
let receipt_clone = receipt.clone();
let erasure_chunks_clone = erasure_chunks.clone();
let pov_block = available_data.pov_block.clone();
let res = async move {
if let Err(e) = av_clone.make_available(
receipt_clone.hash(),
available_data,
).await {
warn!(
target: "validation",
"Failed to make parachain block data available: {}",
e,
);
}
Err(e) => {
warn!(target: "validation", "Failed to produce a receipt: {:?}", e);
Box::pin(ready(Ok(())))
if let Err(e) = av_clone.clone().add_erasure_chunks(
receipt_clone,
erasure_chunks_clone,
).await {
warn!(target: "validation", "Failed to add erasure chunks: {}", e);
}
}
.unit_error()
.then(move |_| {
router.local_collation(
receipt,
pov_block,
(local_id, &erasure_chunks),
).map_err(|e| warn!(target: "validation", "Failed to send local collation: {:?}", e))
});
res.boxed()
}
Err(e) => {
warn!(target: "validation", "Failed to collate candidate: {:?}", e);