diff --git a/polkadot/availability-store/src/lib.rs b/polkadot/availability-store/src/lib.rs index f726b1c0cb..e7716b2869 100644 --- a/polkadot/availability-store/src/lib.rs +++ b/polkadot/availability-store/src/lib.rs @@ -28,7 +28,8 @@ use keystore::KeyStorePtr; use polkadot_primitives::{ Hash, Block, parachain::{ - Id as ParaId, BlockData, CandidateReceipt, ErasureChunk, ParachainHost + PoVBlock, AbridgedCandidateReceipt, ErasureChunk, + ParachainHost, AvailableData, OmittedValidationData, }, }; use sp_runtime::traits::{BlakeTwo256, Hash as HashT, HasherFor}; @@ -37,6 +38,7 @@ use client::{ BlockchainEvents, BlockBody, }; use sp_api::{ApiExt, ProvideRuntimeApi}; +use codec::{Encode, Decode}; use log::warn; @@ -50,9 +52,10 @@ mod worker; mod store; pub use worker::AvailabilityBlockImport; +pub use store::AwaitedFrontierEntry; use worker::{ - Worker, WorkerHandle, Chunks, ParachainBlocks, WorkerMsg, MakeAvailable, + Worker, WorkerHandle, Chunks, IncludedParachainBlocks, WorkerMsg, MakeAvailable, }; use store::{Store as InnerStore}; @@ -116,15 +119,14 @@ pub trait ProvideGossipMessages { ); } -/// Some data to keep available about a parachain block candidate. -#[derive(Debug)] -pub struct Data { - /// The relay chain parent hash this should be localized to. - pub relay_parent: Hash, - /// The parachain index for this candidate. - pub parachain_id: ParaId, - /// Block data. - pub block_data: BlockData, +/// Data which, when combined with an `AbridgedCandidateReceipt`, is enough +/// to fully re-execute a block. +#[derive(Debug, Encode, Decode, PartialEq)] +pub struct ExecutionData { + /// The `PoVBlock`. + pub pov_block: PoVBlock, + /// The data omitted from the `AbridgedCandidateReceipt`. + pub omitted_validation: OmittedValidationData, } /// Handle to the availability store. @@ -220,17 +222,17 @@ impl Store { /// in order to persist that data to disk and so it can be queried and provided /// to other nodes in the network. /// - /// The message data of `Data` is optional but is expected - /// to be present with the exception of the case where there is no message data - /// due to the block's invalidity. Determination of invalidity is beyond the - /// scope of this function. + /// Determination of invalidity is beyond the scope of this function. /// - /// This method will send the `Data` to the background worker, allowing caller to - /// asynchrounously wait for the result. - pub async fn make_available(&self, data: Data) -> io::Result<()> { + /// This method will send the data to the background worker, allowing the caller to + /// asynchronously wait for the result. + pub async fn make_available(&self, candidate_hash: Hash, available_data: AvailableData) + -> io::Result<()> + { let (s, r) = oneshot::channel(); let msg = WorkerMsg::MakeAvailable(MakeAvailable { - data, + candidate_hash, + available_data, result: s, }); @@ -244,41 +246,11 @@ impl Store { } - /// Get a set of all chunks we are waiting for grouped by - /// `(relay_parent, erasure_root, candidate_hash, our_id)`. - pub fn awaited_chunks(&self) -> Option> { + /// Get a set of all chunks we are waiting for. + pub fn awaited_chunks(&self) -> Option> { self.inner.awaited_chunks() } - /// Qery which candidates were included in the relay chain block by block's parent. - pub fn get_candidates_in_relay_block(&self, relay_block: &Hash) -> Option> { - self.inner.get_candidates_in_relay_block(relay_block) - } - - /// Make a validator's index and a number of validators at a relay parent available. - /// - /// This information is needed before the `add_candidates_in_relay_block` is called - /// since that call forms the awaited frontier of chunks. - /// In the current implementation this function is called in the `get_or_instantiate` at - /// the start of the parachain agreement process on top of some parent hash. - pub fn add_validator_index_and_n_validators( - &self, - relay_parent: &Hash, - validator_index: u32, - n_validators: u32, - ) -> io::Result<()> { - self.inner.add_validator_index_and_n_validators( - relay_parent, - validator_index, - n_validators, - ) - } - - /// Query a validator's index and n_validators by relay parent. - pub fn get_validator_index_and_n_validators(&self, relay_parent: &Hash) -> Option<(u32, u32)> { - self.inner.get_validator_index_and_n_validators(relay_parent) - } - /// Adds an erasure chunk to storage. /// /// The chunk should be checked for validity against the root of encoding @@ -288,11 +260,10 @@ impl Store { /// asynchrounously wait for the result. pub async fn add_erasure_chunk( &self, - relay_parent: Hash, - receipt: CandidateReceipt, + candidate: AbridgedCandidateReceipt, chunk: ErasureChunk, ) -> io::Result<()> { - self.add_erasure_chunks(relay_parent, receipt, vec![chunk]).await + self.add_erasure_chunks(candidate, vec![chunk]).await } /// Adds a set of erasure chunks to storage. @@ -304,16 +275,17 @@ impl Store { /// asynchrounously waiting for the result. pub async fn add_erasure_chunks( &self, - relay_parent: Hash, - receipt: CandidateReceipt, + candidate: AbridgedCandidateReceipt, chunks: I, ) -> io::Result<()> where I: IntoIterator { - self.add_candidate(relay_parent, receipt.clone()).await?; + let candidate_hash = candidate.hash(); + let relay_parent = candidate.relay_parent; + + self.add_candidate(candidate).await?; let (s, r) = oneshot::channel(); let chunks = chunks.into_iter().collect(); - let candidate_hash = receipt.hash(); let msg = WorkerMsg::Chunks(Chunks { relay_parent, candidate_hash, @@ -330,27 +302,44 @@ impl Store { } } - /// Queries an erasure chunk by its block's parent and hash and index. + /// Queries an erasure chunk by the candidate hash and validator index. pub fn get_erasure_chunk( &self, - relay_parent: &Hash, - block_data_hash: Hash, - index: usize, + candidate_hash: &Hash, + validator_index: usize, ) -> Option { - self.inner.get_erasure_chunk(relay_parent, block_data_hash, index) + self.inner.get_erasure_chunk(candidate_hash, validator_index) } - /// Stores a candidate receipt. - pub async fn add_candidate( + /// Note a validator's index and a number of validators at a relay parent in the + /// store. + /// + /// This should be done before adding erasure chunks with this relay parent. + pub fn note_validator_index_and_n_validators( &self, - relay_parent: Hash, - receipt: CandidateReceipt, + relay_parent: &Hash, + validator_index: u32, + n_validators: u32, + ) -> io::Result<()> { + self.inner.note_validator_index_and_n_validators( + relay_parent, + validator_index, + n_validators, + ) + } + + // Stores a candidate receipt. + async fn add_candidate( + &self, + candidate: AbridgedCandidateReceipt, ) -> io::Result<()> { let (s, r) = oneshot::channel(); - let msg = WorkerMsg::ParachainBlocks(ParachainBlocks { - relay_parent, - blocks: vec![(receipt, None)], + let msg = WorkerMsg::IncludedParachainBlocks(IncludedParachainBlocks { + blocks: vec![crate::worker::IncludedParachainBlock { + candidate, + available_data: None, + }], result: s, }); @@ -363,20 +352,17 @@ impl Store { } } - /// Queries a candidate receipt by it's hash. - pub fn get_candidate(&self, candidate_hash: &Hash) -> Option { + /// Queries a candidate receipt by its hash. + pub fn get_candidate(&self, candidate_hash: &Hash) + -> Option + { self.inner.get_candidate(candidate_hash) } - /// Query block data. - pub fn block_data(&self, relay_parent: Hash, block_data_hash: Hash) -> Option { - self.inner.block_data(relay_parent, block_data_hash) - } - - /// Query block data by corresponding candidate receipt's hash. - pub fn block_data_by_candidate(&self, relay_parent: Hash, candidate_hash: Hash) - -> Option + /// Query execution data by pov-block hash. + pub fn execution_data(&self, candidate_hash: &Hash) + -> Option { - self.inner.block_data_by_candidate(relay_parent, candidate_hash) + self.inner.execution_data(candidate_hash) } } diff --git a/polkadot/availability-store/src/store.rs b/polkadot/availability-store/src/store.rs index d7aa60443a..03a7855151 100644 --- a/polkadot/availability-store/src/store.rs +++ b/polkadot/availability-store/src/store.rs @@ -21,8 +21,11 @@ use codec::{Encode, Decode}; use polkadot_erasure_coding::{self as erasure}; use polkadot_primitives::{ Hash, - parachain::{BlockData, CandidateReceipt, ErasureChunk}, + parachain::{ + ErasureChunk, AvailableData, AbridgedCandidateReceipt, + }, }; +use parking_lot::Mutex; use log::{trace, warn}; use std::collections::HashSet; @@ -30,7 +33,7 @@ use std::sync::Arc; use std::iter::FromIterator; use std::io; -use crate::{LOG_TARGET, Data, Config}; +use crate::{LOG_TARGET, Config, ExecutionData}; mod columns { pub const DATA: u32 = 0; @@ -41,42 +44,51 @@ mod columns { #[derive(Clone)] pub struct Store { inner: Arc, + candidate_descendents_lock: Arc> } -fn block_data_key(relay_parent: &Hash, block_data_hash: &Hash) -> Vec { - (relay_parent, block_data_hash, 0i8).encode() +// data keys +fn execution_data_key(candidate_hash: &Hash) -> Vec { + (candidate_hash, 0i8).encode() } -fn erasure_chunks_key(relay_parent: &Hash, block_data_hash: &Hash) -> Vec { - (relay_parent, block_data_hash, 1i8).encode() -} - -fn awaited_chunks_key() -> Vec { - "awaited_chunks_key".encode() -} - -fn available_chunks_key(relay_parent: &Hash, erasure_root: &Hash) -> Vec { - (relay_parent, erasure_root, 2i8).encode() -} - -fn block_to_candidate_key(block_data_hash: &Hash) -> Vec { - (block_data_hash, 1i8).encode() +fn erasure_chunks_key(candidate_hash: &Hash) -> Vec { + (candidate_hash, 1i8).encode() } fn candidate_key(candidate_hash: &Hash) -> Vec { (candidate_hash, 2i8).encode() } -fn validator_index_and_n_validators_key(relay_parent: &Hash) -> Vec { - (relay_parent, 3i8).encode() +fn available_chunks_key(relay_parent: &Hash, erasure_root: &Hash) -> Vec { + (relay_parent, erasure_root, 3i8).encode() } -fn candidates_in_relay_chain_block_key(relay_block: &Hash) -> Vec { +fn candidates_with_relay_parent_key(relay_block: &Hash) -> Vec { (relay_block, 4i8).encode() } -fn erasure_roots_in_relay_chain_block_key(relay_block: &Hash) -> Vec { - (relay_block, 5i8).encode() +// meta keys +fn awaited_chunks_key() -> [u8; 14] { + *b"awaited_chunks" +} + +fn validator_index_and_n_validators_key(relay_parent: &Hash) -> Vec { + (relay_parent, 1i8).encode() +} + +/// An entry in the awaited frontier of chunks we are interested in. +#[derive(Encode, Decode, Debug, Hash, PartialEq, Eq, Clone)] +pub struct AwaitedFrontierEntry { + /// The relay-chain parent block hash. + pub relay_parent: Hash, + /// The erasure-chunk trie root we are comparing against. + /// + /// We index by erasure-root because there may be multiple candidates + /// with the same erasure root. + pub erasure_root: Hash, + /// The index of the validator we represent. + pub validator_index: u32, } impl Store { @@ -103,6 +115,7 @@ impl Store { Ok(Store { inner: Arc::new(db), + candidate_descendents_lock: Arc::new(Mutex::new(())), }) } @@ -110,31 +123,37 @@ impl Store { pub(super) fn new_in_memory() -> Self { Store { inner: Arc::new(::kvdb_memorydb::create(columns::NUM_COLUMNS)), + candidate_descendents_lock: Arc::new(Mutex::new(())), } } /// Make some data available provisionally. - pub(crate) fn make_available(&self, data: Data) -> io::Result<()> { + pub(crate) fn make_available(&self, candidate_hash: Hash, available_data: AvailableData) + -> io::Result<()> + { let mut tx = DBTransaction::new(); - // note the meta key. - let mut v = self.query_inner(columns::META, data.relay_parent.as_ref()).unwrap_or(Vec::new()); - v.push(data.block_data.hash()); - tx.put_vec(columns::META, &data.relay_parent[..], v.encode()); + // at the moment, these structs are identical. later, we will also + // keep outgoing message queues available, and these are not needed + // for execution. + let AvailableData { pov_block, omitted_validation } = available_data; + let execution_data = ExecutionData { + pov_block, + omitted_validation, + }; tx.put_vec( columns::DATA, - block_data_key(&data.relay_parent, &data.block_data.hash()).as_slice(), - data.block_data.encode() + execution_data_key(&candidate_hash).as_slice(), + execution_data.encode(), ); self.inner.write(tx) } - /// Get a set of all chunks we are waiting for grouped by - /// `(relay_parent, erasure_root, candidate_hash, our_id)`. - pub fn awaited_chunks(&self) -> Option> { - self.query_inner(columns::META, &awaited_chunks_key()).map(|vec: Vec<(Hash, Hash, Hash, u32)>| { + /// Get a set of all chunks we are waiting for. + pub fn awaited_chunks(&self) -> Option> { + self.query_inner(columns::META, &awaited_chunks_key()).map(|vec: Vec| { HashSet::from_iter(vec.into_iter()) }) } @@ -147,60 +166,49 @@ impl Store { /// This method modifies the erasure chunks awaited frontier by adding this validator's /// chunks from `candidates` to it. In order to do so the information about this validator's /// position at parent `relay_parent` should be known to the store prior to calling this - /// method, in other words `add_validator_index_and_n_validators` should be called for + /// method, in other words `note_validator_index_and_n_validators` should be called for /// the given `relay_parent` before calling this function. - pub(crate) fn add_candidates_in_relay_block( + pub(crate) fn note_candidates_with_relay_parent( &self, relay_parent: &Hash, - candidates: Vec, + candidates: &[Hash], ) -> io::Result<()> { let mut tx = DBTransaction::new(); - let dbkey = candidates_in_relay_chain_block_key(relay_parent); + let dbkey = candidates_with_relay_parent_key(relay_parent); + + // This call can race against another call to `note_candidates_with_relay_parent` + // with a different set of descendents. + let _lock = self.candidate_descendents_lock.lock(); if let Some((validator_index, _)) = self.get_validator_index_and_n_validators(relay_parent) { let candidates = candidates.clone(); - let awaited_frontier: Vec<(Hash, Hash, Hash, u32)> = self + let awaited_frontier: Vec = self .query_inner(columns::META, &awaited_chunks_key()) .unwrap_or_else(|| Vec::new()); - let mut awaited_frontier: HashSet<(Hash, Hash, Hash, u32)> = + let mut awaited_frontier: HashSet = HashSet::from_iter(awaited_frontier.into_iter()); - awaited_frontier.extend(candidates.into_iter().filter_map(|candidate| { - self.get_candidate(&candidate) - .map(|receipt| (relay_parent.clone(), receipt.erasure_root, candidate, validator_index)) + awaited_frontier.extend(candidates.iter().filter_map(|candidate| { + self.get_candidate(&candidate).map(|receipt| AwaitedFrontierEntry { + relay_parent: relay_parent.clone(), + erasure_root: receipt.commitments.erasure_root, + validator_index, + }) })); let awaited_frontier = Vec::from_iter(awaited_frontier.into_iter()); tx.put_vec(columns::META, &awaited_chunks_key(), awaited_frontier.encode()); } - tx.put_vec(columns::DATA, &dbkey, candidates.encode()); - self.inner.write(tx) - } - - /// Qery which candidates were included in the relay chain block by block's parent. - pub fn get_candidates_in_relay_block(&self, relay_block: &Hash) -> Option> { - let dbkey = candidates_in_relay_chain_block_key(relay_block); - - self.query_inner(columns::DATA, &dbkey) - } - - /// Adds a set of erasure chunk roots that were included in a relay block by block's parent. - pub(crate) fn add_erasure_roots_in_relay_block( - &self, - relay_parent: &Hash, - erasure_roots: Vec, - ) -> io::Result<()> { - let mut tx = DBTransaction::new(); - let dbkey = erasure_roots_in_relay_chain_block_key(relay_parent); - - tx.put_vec(columns::DATA, &dbkey, erasure_roots.encode()); + let mut descendent_candidates = self.get_candidates_with_relay_parent(relay_parent); + descendent_candidates.extend(candidates.iter().cloned()); + tx.put_vec(columns::DATA, &dbkey, descendent_candidates.encode()); self.inner.write(tx) } /// Make a validator's index and a number of validators at a relay parent available. - pub(crate) fn add_validator_index_and_n_validators( + pub(crate) fn note_validator_index_and_n_validators( &self, relay_parent: &Hash, validator_index: u32, @@ -215,7 +223,7 @@ impl Store { } /// Query a validator's index and n_validators by relay parent. - pub fn get_validator_index_and_n_validators(&self, relay_parent: &Hash) -> Option<(u32, u32)> { + pub(crate) fn get_validator_index_and_n_validators(&self, relay_parent: &Hash) -> Option<(u32, u32)> { let dbkey = validator_index_and_n_validators_key(relay_parent); self.query_inner(columns::META, &dbkey) @@ -224,11 +232,9 @@ impl Store { /// Add a set of chunks. /// /// The same as `add_erasure_chunk` but adds a set of chunks in one atomic transaction. - /// Checks that all chunks have the same `relay_parent`, `block_data_hash` and `parachain_id` fields. pub fn add_erasure_chunks( &self, n_validators: u32, - relay_parent: &Hash, candidate_hash: &Hash, chunks: I, ) -> io::Result<()> @@ -236,16 +242,19 @@ impl Store { { if let Some(receipt) = self.get_candidate(candidate_hash) { let mut tx = DBTransaction::new(); - let dbkey = erasure_chunks_key(relay_parent, &receipt.block_data_hash); + let dbkey = erasure_chunks_key(candidate_hash); let mut v = self.query_inner(columns::DATA, &dbkey).unwrap_or(Vec::new()); - let av_chunks_key = available_chunks_key(relay_parent, &receipt.erasure_root); + let av_chunks_key = available_chunks_key( + &receipt.relay_parent, + &receipt.commitments.erasure_root, + ); let mut have_chunks = self.query_inner(columns::META, &av_chunks_key).unwrap_or(Vec::new()); - let awaited_frontier: Option> = self.query_inner( + let awaited_frontier: Option> = self.query_inner( columns::META, - &awaited_chunks_key() + &awaited_chunks_key(), ); for chunk in chunks.into_iter() { @@ -256,31 +265,23 @@ impl Store { } if let Some(mut awaited_frontier) = awaited_frontier { - awaited_frontier.retain(|&(p, r, c, index)| { + awaited_frontier.retain(|entry| { !( - *relay_parent == p && - r == receipt.erasure_root && - c == receipt.hash() && - have_chunks.contains(&index) + entry.relay_parent == receipt.relay_parent && + entry.erasure_root == receipt.commitments.erasure_root && + have_chunks.contains(&entry.validator_index) ) }); tx.put_vec(columns::META, &awaited_chunks_key(), awaited_frontier.encode()); } - // If therea are no block data and messages in the store at this point, + // If therea are no block data in the store at this point, // check that they can be reconstructed now and add them to store if they can. - if let Ok(None) = self.inner.get( - columns::DATA, - &block_data_key(&relay_parent, &receipt.block_data_hash) - ) { - if let Ok(block_data) = erasure::reconstruct( + if self.execution_data(&candidate_hash).is_none() { + if let Ok(available_data) = erasure::reconstruct( n_validators as usize, v.iter().map(|chunk| (chunk.chunk.as_ref(), chunk.index as usize))) { - self.make_available(Data { - relay_parent: *relay_parent, - parachain_id: receipt.parachain_index, - block_data, - })?; + self.make_available(*candidate_hash, available_data)?; } } @@ -294,14 +295,13 @@ impl Store { } } - /// Queries an erasure chunk by its block's parent and hash and index. + /// Queries an erasure chunk by its block's relay-parent, the candidate hash, and index. pub fn get_erasure_chunk( &self, - relay_parent: &Hash, - block_data_hash: Hash, + candidate_hash: &Hash, index: usize, ) -> Option { - self.query_inner(columns::DATA, &erasure_chunks_key(&relay_parent, &block_data_hash)) + self.query_inner(columns::DATA, &erasure_chunks_key(candidate_hash)) .and_then(|chunks: Vec| { chunks.iter() .find(|chunk: &&ErasureChunk| chunk.index == index as u32) @@ -310,70 +310,66 @@ impl Store { } /// Stores a candidate receipt. - pub fn add_candidate(&self, receipt: &CandidateReceipt) -> io::Result<()> { - let dbkey = candidate_key(&receipt.hash()); + pub fn add_candidate( + &self, + receipt: &AbridgedCandidateReceipt, + ) -> io::Result<()> { + let candidate_hash = receipt.hash(); + let dbkey = candidate_key(&candidate_hash); let mut tx = DBTransaction::new(); tx.put_vec(columns::DATA, &dbkey, receipt.encode()); - tx.put_vec(columns::META, &block_to_candidate_key(&receipt.block_data_hash), receipt.hash().encode()); self.inner.write(tx) } - /// Queries a candidate receipt by it's hash. - pub fn get_candidate(&self, candidate_hash: &Hash) -> Option { + /// Queries a candidate receipt by the relay parent hash and its hash. + pub(crate) fn get_candidate(&self, candidate_hash: &Hash) + -> Option + { self.query_inner(columns::DATA, &candidate_key(candidate_hash)) } /// Note that a set of candidates have been included in a finalized block with given hash and parent hash. - pub fn candidates_finalized( + pub(crate) fn candidates_finalized( &self, - parent: Hash, + relay_parent: Hash, finalized_candidates: HashSet, ) -> io::Result<()> { let mut tx = DBTransaction::new(); - let v = self.query_inner(columns::META, &parent[..]).unwrap_or(Vec::new()); - tx.delete(columns::META, &parent[..]); - - let awaited_frontier: Option> = self + let awaited_frontier: Option> = self .query_inner(columns::META, &awaited_chunks_key()); if let Some(mut awaited_frontier) = awaited_frontier { - awaited_frontier.retain(|&(p, c, _, _)| (p != parent && !finalized_candidates.contains(&c))); + awaited_frontier.retain(|entry| entry.relay_parent != relay_parent); tx.put_vec(columns::META, &awaited_chunks_key(), awaited_frontier.encode()); } - for block_data_hash in v { - if let Some(candidate_hash) = self.block_hash_to_candidate_hash(block_data_hash) { - if !finalized_candidates.contains(&candidate_hash) { - tx.delete(columns::DATA, block_data_key(&parent, &block_data_hash).as_slice()); - tx.delete(columns::DATA, &erasure_chunks_key(&parent, &block_data_hash)); - tx.delete(columns::DATA, &candidate_key(&candidate_hash)); - tx.delete(columns::META, &block_to_candidate_key(&block_data_hash)); - } - } + let candidates = self.get_candidates_with_relay_parent(&relay_parent); + + for candidate in candidates.into_iter().filter(|c| !finalized_candidates.contains(c)) { + // we only delete this data for candidates which were not finalized. + // we keep all data for the finalized chain forever at the moment. + tx.delete(columns::DATA, execution_data_key(&candidate).as_slice()); + tx.delete(columns::DATA, &erasure_chunks_key(&candidate)); + tx.delete(columns::DATA, &candidate_key(&candidate)); } self.inner.write(tx) } - /// Query block data. - pub fn block_data(&self, relay_parent: Hash, block_data_hash: Hash) -> Option { - self.query_inner(columns::DATA, &block_data_key(&relay_parent, &block_data_hash)) + /// Query execution data by relay parent and candidate hash. + pub(crate) fn execution_data(&self, candidate_hash: &Hash) -> Option { + self.query_inner(columns::DATA, &execution_data_key(candidate_hash)) } - /// Query block data by corresponding candidate receipt's hash. - pub fn block_data_by_candidate(&self, relay_parent: Hash, candidate_hash: Hash) -> Option { - let receipt_key = candidate_key(&candidate_hash); - - self.query_inner(columns::DATA, &receipt_key[..]).and_then(|receipt: CandidateReceipt| { - self.block_data(relay_parent, receipt.block_data_hash) - }) - } - - fn block_hash_to_candidate_hash(&self, block_hash: Hash) -> Option { - self.query_inner(columns::META, &block_to_candidate_key(&block_hash)) + /// Get candidates which pinned to the environment of the given relay parent. + /// Note that this is not necessarily the same as candidates that were included in a direct + /// descendent of the given relay-parent. + fn get_candidates_with_relay_parent(&self, relay_parent: &Hash) -> Vec { + let key = candidates_with_relay_parent_key(relay_parent); + self.query_inner(columns::DATA, &key[..]).unwrap_or_default() } fn query_inner(&self, column: u32, key: &[u8]) -> Option { @@ -394,8 +390,27 @@ impl Store { #[cfg(test)] mod tests { use super::*; - use polkadot_erasure_coding as erasure; - use polkadot_primitives::parachain::Id as ParaId; + use polkadot_erasure_coding::{self as erasure}; + use polkadot_primitives::parachain::{ + Id as ParaId, BlockData, AvailableData, PoVBlock, OmittedValidationData, + }; + + fn available_data(block_data: &[u8]) -> AvailableData { + AvailableData { + pov_block: PoVBlock { + block_data: BlockData(block_data.to_vec()), + }, + omitted_validation: OmittedValidationData { + global_validation: Default::default(), + local_validation: Default::default(), + } + } + } + + fn execution_data(available: &AvailableData) -> ExecutionData { + let AvailableData { pov_block, omitted_validation } = available.clone(); + ExecutionData { pov_block, omitted_validation } + } #[test] fn finalization_removes_unneeded() { @@ -404,8 +419,23 @@ mod tests { let para_id_1 = 5.into(); let para_id_2 = 6.into(); - let block_data_1 = BlockData(vec![1, 2, 3]); - let block_data_2 = BlockData(vec![4, 5, 6]); + let mut candidate_1 = AbridgedCandidateReceipt::default(); + let mut candidate_2 = AbridgedCandidateReceipt::default(); + + candidate_1.parachain_index = para_id_1; + candidate_1.commitments.erasure_root = [6; 32].into(); + candidate_1.relay_parent = relay_parent; + + candidate_2.parachain_index = para_id_2; + candidate_2.commitments.erasure_root = [6; 32].into(); + candidate_2.relay_parent = relay_parent; + + + let candidate_1_hash = candidate_1.hash(); + let candidate_2_hash = candidate_2.hash(); + + let available_data_1 = available_data(&[1, 2, 3]); + let available_data_2 = available_data(&[4, 5, 6]); let erasure_chunk_1 = ErasureChunk { chunk: vec![10, 20, 30], @@ -420,112 +450,59 @@ mod tests { }; let store = Store::new_in_memory(); - store.make_available(Data { - relay_parent, - parachain_id: para_id_1, - block_data: block_data_1.clone(), - }).unwrap(); + store.make_available(candidate_1_hash, available_data_1.clone()).unwrap(); - store.make_available(Data { - relay_parent, - parachain_id: para_id_2, - block_data: block_data_2.clone(), - }).unwrap(); - - let candidate_1 = CandidateReceipt { - parachain_index: para_id_1, - collator: Default::default(), - signature: Default::default(), - head_data: Default::default(), - parent_head: Default::default(), - fees: 0, - block_data_hash: block_data_1.hash(), - upward_messages: Vec::new(), - erasure_root: [6; 32].into(), - }; - - let candidate_2 = CandidateReceipt { - parachain_index: para_id_2, - collator: Default::default(), - signature: Default::default(), - head_data: Default::default(), - parent_head: Default::default(), - fees: 0, - block_data_hash: block_data_2.hash(), - upward_messages: Vec::new(), - erasure_root: [6; 32].into(), - }; + store.make_available(candidate_2_hash, available_data_2.clone()).unwrap(); store.add_candidate(&candidate_1).unwrap(); store.add_candidate(&candidate_2).unwrap(); - assert!(store.add_erasure_chunks(3, &relay_parent, &candidate_1.hash(), vec![erasure_chunk_1.clone()]).is_ok()); - assert!(store.add_erasure_chunks(3, &relay_parent, &candidate_2.hash(), vec![erasure_chunk_2.clone()]).is_ok()); + store.note_candidates_with_relay_parent(&relay_parent, &[candidate_1_hash, candidate_2_hash]).unwrap(); - assert_eq!(store.block_data(relay_parent, block_data_1.hash()).unwrap(), block_data_1); - assert_eq!(store.block_data(relay_parent, block_data_2.hash()).unwrap(), block_data_2); + assert!(store.add_erasure_chunks(3, &candidate_1_hash, vec![erasure_chunk_1.clone()]).is_ok()); + assert!(store.add_erasure_chunks(3, &candidate_2_hash, vec![erasure_chunk_2.clone()]).is_ok()); - assert_eq!(store.get_erasure_chunk(&relay_parent, block_data_1.hash(), 1).as_ref(), Some(&erasure_chunk_1)); - assert_eq!(store.get_erasure_chunk(&relay_parent, block_data_2.hash(), 1), Some(erasure_chunk_2)); + assert_eq!(store.execution_data(&candidate_1_hash).unwrap(), execution_data(&available_data_1)); + assert_eq!(store.execution_data(&candidate_2_hash).unwrap(), execution_data(&available_data_2)); - assert_eq!(store.get_candidate(&candidate_1.hash()), Some(candidate_1.clone())); - assert_eq!(store.get_candidate(&candidate_2.hash()), Some(candidate_2.clone())); + assert_eq!(store.get_erasure_chunk(&candidate_1_hash, 1).as_ref(), Some(&erasure_chunk_1)); + assert_eq!(store.get_erasure_chunk(&candidate_2_hash, 1), Some(erasure_chunk_2)); - assert_eq!(store.block_data_by_candidate(relay_parent, candidate_1.hash()).unwrap(), block_data_1); - assert_eq!(store.block_data_by_candidate(relay_parent, candidate_2.hash()).unwrap(), block_data_2); + assert_eq!(store.get_candidate(&candidate_1_hash), Some(candidate_1.clone())); + assert_eq!(store.get_candidate(&candidate_2_hash), Some(candidate_2.clone())); - store.candidates_finalized(relay_parent, [candidate_1.hash()].iter().cloned().collect()).unwrap(); + store.candidates_finalized(relay_parent, [candidate_1_hash].iter().cloned().collect()).unwrap(); - assert_eq!(store.get_erasure_chunk(&relay_parent, block_data_1.hash(), 1).as_ref(), Some(&erasure_chunk_1)); - assert!(store.get_erasure_chunk(&relay_parent, block_data_2.hash(), 1).is_none()); + assert_eq!(store.get_erasure_chunk(&candidate_1_hash, 1).as_ref(), Some(&erasure_chunk_1)); + assert!(store.get_erasure_chunk(&candidate_2_hash, 1).is_none()); - assert_eq!(store.get_candidate(&candidate_1.hash()), Some(candidate_1)); - assert_eq!(store.get_candidate(&candidate_2.hash()), None); + assert_eq!(store.get_candidate(&candidate_1_hash), Some(candidate_1)); + assert_eq!(store.get_candidate(&candidate_2_hash), None); - assert_eq!(store.block_data(relay_parent, block_data_1.hash()).unwrap(), block_data_1); - assert!(store.block_data(relay_parent, block_data_2.hash()).is_none()); - } - - #[test] - fn queues_available_by_queue_root() { - let relay_parent = [1; 32].into(); - let para_id = 5.into(); - let block_data = BlockData(vec![1, 2, 3]); - - let store = Store::new_in_memory(); - store.make_available(Data { - relay_parent, - parachain_id: para_id, - block_data, - }).unwrap(); + assert_eq!(store.execution_data(&candidate_1_hash).unwrap(), execution_data(&available_data_1)); + assert!(store.execution_data(&candidate_2_hash).is_none()); } #[test] fn erasure_coding() { let relay_parent: Hash = [1; 32].into(); let para_id: ParaId = 5.into(); - let block_data = BlockData(vec![42; 8]); - let block_data_hash = block_data.hash(); + let available_data = available_data(&[42; 8]); let n_validators = 5; let erasure_chunks = erasure::obtain_chunks( n_validators, - &block_data, + &available_data, ).unwrap(); let branches = erasure::branches(erasure_chunks.as_ref()); - let candidate = CandidateReceipt { - parachain_index: para_id, - collator: Default::default(), - signature: Default::default(), - head_data: Default::default(), - parent_head: Default::default(), - fees: 0, - block_data_hash: block_data.hash(), - upward_messages: Vec::new(), - erasure_root: [6; 32].into(), - }; + let mut candidate = AbridgedCandidateReceipt::default(); + candidate.parachain_index = para_id; + candidate.commitments.erasure_root = [6; 32].into(); + candidate.relay_parent = relay_parent; + + let candidate_hash = candidate.hash(); let chunks: Vec<_> = erasure_chunks .iter() @@ -541,13 +518,13 @@ mod tests { let store = Store::new_in_memory(); store.add_candidate(&candidate).unwrap(); - store.add_erasure_chunks(n_validators as u32, &relay_parent, &candidate.hash(), vec![chunks[0].clone()]).unwrap(); - assert_eq!(store.get_erasure_chunk(&relay_parent, block_data_hash, 0), Some(chunks[0].clone())); + store.add_erasure_chunks(n_validators as u32, &candidate_hash, vec![chunks[0].clone()]).unwrap(); + assert_eq!(store.get_erasure_chunk(&candidate_hash, 0), Some(chunks[0].clone())); - assert!(store.block_data(relay_parent, block_data_hash).is_none()); + assert!(store.execution_data(&candidate_hash).is_none()); - store.add_erasure_chunks(n_validators as u32, &relay_parent, &candidate.hash(), chunks).unwrap(); - assert_eq!(store.block_data(relay_parent, block_data_hash), Some(block_data)); + store.add_erasure_chunks(n_validators as u32, &candidate_hash, chunks).unwrap(); + assert_eq!(store.execution_data(&candidate_hash), Some(execution_data(&available_data))); } #[test] @@ -555,7 +532,7 @@ mod tests { let relay_parent = [42; 32].into(); let store = Store::new_in_memory(); - store.add_validator_index_and_n_validators(&relay_parent, 42, 24).unwrap(); + store.note_validator_index_and_n_validators(&relay_parent, 42, 24).unwrap(); assert_eq!(store.get_validator_index_and_n_validators(&relay_parent).unwrap(), (42, 24)); } @@ -566,8 +543,8 @@ mod tests { let candidates = vec![[1; 32].into(), [2; 32].into(), [3; 32].into()]; - store.add_candidates_in_relay_block(&relay_parent, candidates.clone()).unwrap(); - assert_eq!(store.get_candidates_in_relay_block(&relay_parent).unwrap(), candidates); + store.note_candidates_with_relay_parent(&relay_parent, &candidates).unwrap(); + assert_eq!(store.get_candidates_with_relay_parent(&relay_parent), candidates); } #[test] @@ -578,25 +555,32 @@ mod tests { let relay_parent = [42; 32].into(); let erasure_root_1 = [11; 32].into(); let erasure_root_2 = [12; 32].into(); - let mut receipt_1 = CandidateReceipt::default(); - let mut receipt_2 = CandidateReceipt::default(); + let mut receipt_1 = AbridgedCandidateReceipt::default(); + let mut receipt_2 = AbridgedCandidateReceipt::default(); + receipt_1.parachain_index = 1.into(); - receipt_1.erasure_root = erasure_root_1; + receipt_1.commitments.erasure_root = erasure_root_1; + receipt_1.relay_parent = relay_parent; + receipt_2.parachain_index = 2.into(); - receipt_2.erasure_root = erasure_root_2; + receipt_2.commitments.erasure_root = erasure_root_2; + receipt_2.relay_parent = relay_parent; + + let receipt_1_hash = receipt_1.hash(); + let receipt_2_hash = receipt_2.hash(); let chunk = ErasureChunk { chunk: vec![1, 2, 3], index: validator_index, proof: Vec::new(), }; - let candidates = vec![receipt_1.hash(), receipt_2.hash()]; + let candidates = vec![receipt_1_hash, receipt_2_hash]; let erasure_roots = vec![erasure_root_1, erasure_root_2]; let store = Store::new_in_memory(); - store.add_validator_index_and_n_validators( + store.note_validator_index_and_n_validators( &relay_parent, validator_index, n_validators @@ -605,7 +589,7 @@ mod tests { store.add_candidate(&receipt_2).unwrap(); // We are waiting for chunks from two candidates. - store.add_candidates_in_relay_block(&relay_parent, candidates.clone()).unwrap(); + store.note_candidates_with_relay_parent(&relay_parent, &candidates).unwrap(); let awaited_frontier = store.awaited_chunks().unwrap(); warn!(target: "availability", "awaited {:?}", awaited_frontier); @@ -613,18 +597,24 @@ mod tests { .clone() .into_iter() .zip(erasure_roots.iter()) - .map(|(c, e)| (relay_parent, *e, c, validator_index)) + .map(|(_c, &e)| AwaitedFrontierEntry { + relay_parent, + erasure_root: e, + validator_index, + }) .collect(); assert_eq!(awaited_frontier, expected); // We add chunk from one of the candidates. - store.add_erasure_chunks(n_validators, &relay_parent, &receipt_1.hash(), vec![chunk]).unwrap(); + store.add_erasure_chunks(n_validators, &receipt_1_hash, vec![chunk]).unwrap(); let awaited_frontier = store.awaited_chunks().unwrap(); // Now we wait for the other chunk that we haven't received yet. - let expected: HashSet<_> = vec![ - (relay_parent, erasure_roots[1], candidates[1], validator_index) - ].into_iter().collect(); + let expected: HashSet<_> = vec![AwaitedFrontierEntry { + relay_parent, + erasure_root: erasure_roots[1], + validator_index, + }].into_iter().collect(); assert_eq!(awaited_frontier, expected); diff --git a/polkadot/availability-store/src/worker.rs b/polkadot/availability-store/src/worker.rs index 0811ef8b02..90f1e14a2a 100644 --- a/polkadot/availability-store/src/worker.rs +++ b/polkadot/availability-store/src/worker.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::io; use std::sync::Arc; use std::thread; @@ -34,15 +34,15 @@ use consensus_common::{ }; use polkadot_primitives::{Block, BlockId, Hash}; use polkadot_primitives::parachain::{ - CandidateReceipt, ParachainHost, ValidatorId, - ValidatorPair, ErasureChunk, PoVBlock, + ParachainHost, ValidatorId, AbridgedCandidateReceipt, AvailableData, + ValidatorPair, ErasureChunk, }; use futures::{prelude::*, future::select, channel::{mpsc, oneshot}, task::{Spawn, SpawnExt}}; use keystore::KeyStorePtr; use tokio::runtime::{Handle, Runtime as LocalRuntime}; -use crate::{LOG_TARGET, Data, ProvideGossipMessages, erasure_coding_topic}; +use crate::{LOG_TARGET, ProvideGossipMessages, erasure_coding_topic}; use crate::store::Store; /// Errors that may occur. @@ -65,32 +65,27 @@ pub(crate) enum Error { /// * when the `Store` api is used by outside code. #[derive(Debug)] pub(crate) enum WorkerMsg { - ErasureRoots(ErasureRoots), - ParachainBlocks(ParachainBlocks), + IncludedParachainBlocks(IncludedParachainBlocks), ListenForChunks(ListenForChunks), Chunks(Chunks), CandidatesFinalized(CandidatesFinalized), MakeAvailable(MakeAvailable), } -/// The erasure roots of the heads included in the block with a given parent. +/// A notification of a parachain block included in the relay chain. #[derive(Debug)] -pub(crate) struct ErasureRoots { - /// The relay parent of the block these roots belong to. - pub relay_parent: Hash, - /// The roots themselves. - pub erasure_roots: Vec, - /// A sender to signal the result asynchronously. - pub result: oneshot::Sender>, +pub(crate) struct IncludedParachainBlock { + /// The abridged candidate receipt, extracted from a relay-chain block. + pub candidate: AbridgedCandidateReceipt, + /// The data to keep available from the candidate, if known. + pub available_data: Option, } /// The receipts of the heads included into the block with a given parent. #[derive(Debug)] -pub(crate) struct ParachainBlocks { - /// The relay parent of the block these parachain blocks belong to. - pub relay_parent: Hash, +pub(crate) struct IncludedParachainBlocks { /// The blocks themselves. - pub blocks: Vec<(CandidateReceipt, Option)>, + pub blocks: Vec, /// A sender to signal the result asynchronously. pub result: oneshot::Sender>, } @@ -98,8 +93,6 @@ pub(crate) struct ParachainBlocks { /// Listen gossip for these chunks. #[derive(Debug)] pub(crate) struct ListenForChunks { - /// The relay parent of the block the chunks from we want to listen to. - pub relay_parent: Hash, /// The hash of the candidate chunk belongs to. pub candidate_hash: Hash, /// The index of the chunk we need. @@ -126,15 +119,17 @@ pub(crate) struct Chunks { pub(crate) struct CandidatesFinalized { /// The relay parent of the block that was finalized. relay_parent: Hash, - /// The parachain heads that were finalized in this block. - candidate_hashes: Vec, + /// The hashes of candidates that were finalized in this block. + included_candidates: HashSet, } /// The message that corresponds to `make_available` call of the crate API. #[derive(Debug)] pub(crate) struct MakeAvailable { - /// The data being made available. - pub data: Data, + /// The hash of the candidate for which we are publishing data. + pub candidate_hash: Hash, + /// The data to make available. + pub available_data: AvailableData, /// A sender to signal the result asynchronously. pub result: oneshot::Sender>, } @@ -205,7 +200,7 @@ where fn fetch_candidates

(client: &P, extrinsics: Vec<::Extrinsic>, parent: &BlockId) - -> ClientResult>> + -> ClientResult>> where P: ProvideRuntimeApi, P::Api: ParachainHost, @@ -262,12 +257,15 @@ where } }; - let candidate_hashes = match fetch_candidates( + let included_candidates = match fetch_candidates( &*client, extrinsics, - &BlockId::hash(parent_hash) + &BlockId::hash(parent_hash), ) { - Ok(Some(candidates)) => candidates.into_iter().map(|c| c.hash()).collect(), + Ok(Some(candidates)) => candidates + .into_iter() + .map(|c| c.hash()) + .collect(), Ok(None) => { warn!( target: LOG_TARGET, @@ -286,7 +284,7 @@ where let msg = WorkerMsg::CandidatesFinalized(CandidatesFinalized { relay_parent: parent_hash, - candidate_hashes + included_candidates }); if let Err(_) = sender.send(msg).await { @@ -315,12 +313,12 @@ where sender: &mut mpsc::UnboundedSender, ) { if let Some(awaited_chunks) = self.availability_store.awaited_chunks() { - for chunk in awaited_chunks { + for awaited_chunk in awaited_chunks { if let Err(e) = self.register_chunks_listener( runtime_handle, sender, - chunk.0, - chunk.1, + awaited_chunk.relay_parent, + awaited_chunk.erasure_root, ) { warn!(target: LOG_TARGET, "Failed to register gossip listener: {}", e); } @@ -366,33 +364,36 @@ where &mut self, runtime_handle: &Handle, sender: &mut mpsc::UnboundedSender, - relay_parent: Hash, - blocks: Vec<(CandidateReceipt, Option)>, + blocks: Vec, ) -> Result<(), Error> { - let hashes: Vec<_> = blocks.iter().map(|(c, _)| c.hash()).collect(); - // First we have to add the receipts themselves. - for (candidate, block) in blocks.into_iter() { + for IncludedParachainBlock { candidate, available_data } + in blocks.into_iter() + { let _ = self.availability_store.add_candidate(&candidate); - if let Some(_block) = block { + if let Some(_available_data) = available_data { // Should we be breaking block into chunks here and gossiping it and so on? } if let Err(e) = self.register_chunks_listener( runtime_handle, sender, - relay_parent, - candidate.erasure_root + candidate.relay_parent, + candidate.commitments.erasure_root, ) { warn!(target: LOG_TARGET, "Failed to register chunk listener: {}", e); } - } - let _ = self.availability_store.add_candidates_in_relay_block( - &relay_parent, - hashes - ); + // This leans on the codebase-wide assumption that the `relay_parent` + // of all candidates in a block matches the parent hash of that block. + // + // In the future this will not always be true. + let _ = self.availability_store.note_candidates_with_relay_parent( + &candidate.relay_parent, + &[candidate.hash()], + ); + } Ok(()) } @@ -415,7 +416,11 @@ where .ok_or(Error::CandidateNotFound { candidate_hash })?; for chunk in &chunks { - let topic = erasure_coding_topic(relay_parent, receipt.erasure_root, chunk.index); + let topic = erasure_coding_topic( + relay_parent, + receipt.commitments.erasure_root, + chunk.index, + ); // need to remove gossip listener and stop it. if let Some(signal) = self.registered_gossip_streams.remove(&topic) { let _ = signal.fire(); @@ -424,7 +429,6 @@ where self.availability_store.add_erasure_chunks( n_validators, - &relay_parent, &candidate_hash, chunks, )?; @@ -432,17 +436,6 @@ where Ok(()) } - // Adds the erasure roots into the store. - fn on_erasure_roots_received( - &mut self, - relay_parent: Hash, - erasure_roots: Vec - ) -> Result<(), Error> { - self.availability_store.add_erasure_roots_in_relay_block(&relay_parent, erasure_roots)?; - - Ok(()) - } - // Processes the `ListenForChunks` message. // // When the worker receives a `ListenForChunk` message, it double-checks that @@ -451,7 +444,6 @@ where &mut self, runtime_handle: &Handle, sender: &mut mpsc::UnboundedSender, - relay_parent: Hash, candidate_hash: Hash, id: usize ) -> Result<(), Error> { @@ -459,13 +451,13 @@ where .ok_or(Error::CandidateNotFound { candidate_hash })?; if self.availability_store - .get_erasure_chunk(&relay_parent, candidate.block_data_hash, id) + .get_erasure_chunk(&candidate_hash, id) .is_none() { if let Err(e) = self.register_chunks_listener( runtime_handle, sender, - relay_parent, - candidate.erasure_root + candidate.relay_parent, + candidate.commitments.erasure_root ) { warn!(target: LOG_TARGET, "Failed to register a gossip listener: {}", e); } @@ -498,7 +490,7 @@ where let runtime_handle = runtime.handle().clone(); // On startup, registers listeners (gossip streams) for all - // (relay_parent, erasure-root, i) in the awaited frontier. + // entries in the awaited frontier. worker.register_listeners(runtime.handle(), &mut sender); let process_notification = async move { @@ -506,18 +498,8 @@ where trace!(target: LOG_TARGET, "Received message {:?}", msg); let res = match msg { - WorkerMsg::ErasureRoots(msg) => { - let ErasureRoots { relay_parent, erasure_roots, result} = msg; - let res = worker.on_erasure_roots_received( - relay_parent, - erasure_roots, - ); - let _ = result.send(res); - Ok(()) - } WorkerMsg::ListenForChunks(msg) => { let ListenForChunks { - relay_parent, candidate_hash, index, result, @@ -526,7 +508,6 @@ where let res = worker.on_listen_for_chunks_received( &runtime_handle, &mut sender, - relay_parent, candidate_hash, index as usize, ); @@ -536,9 +517,8 @@ where } Ok(()) } - WorkerMsg::ParachainBlocks(msg) => { - let ParachainBlocks { - relay_parent, + WorkerMsg::IncludedParachainBlocks(msg) => { + let IncludedParachainBlocks { blocks, result, } = msg; @@ -546,7 +526,6 @@ where let res = worker.on_parachain_blocks_received( &runtime_handle, &mut sender, - relay_parent, blocks, ); @@ -565,16 +544,17 @@ where Ok(()) } WorkerMsg::CandidatesFinalized(msg) => { - let CandidatesFinalized { relay_parent, candidate_hashes } = msg; + let CandidatesFinalized { relay_parent, included_candidates } = msg; worker.availability_store.candidates_finalized( relay_parent, - candidate_hashes.into_iter().collect(), + included_candidates, ) } WorkerMsg::MakeAvailable(msg) => { - let MakeAvailable { data, result } = msg; - let res = worker.availability_store.make_available(data) + let MakeAvailable { candidate_hash, available_data, result } = msg; + let res = worker.availability_store + .make_available(candidate_hash, available_data) .map_err(|e| e.into()); let _ = result.send(res); Ok(()) @@ -651,7 +631,6 @@ impl BlockImport for AvailabilityBlockImport where ); if let Some(ref extrinsics) = block.body { - let relay_parent = *block.header.parent_hash(); let parent_id = BlockId::hash(*block.header.parent_hash()); // Extract our local position i from the validator set of the parent. let validators = self.client.runtime_api().validators(&parent_id) @@ -675,16 +654,15 @@ impl BlockImport for AvailabilityBlockImport where ); for candidate in &candidates { + let candidate_hash = candidate.hash(); // If we don't yet have our chunk of this candidate, // tell the worker to listen for one. if self.availability_store.get_erasure_chunk( - &relay_parent, - candidate.block_data_hash, + &candidate_hash, our_id as usize, ).is_none() { let msg = WorkerMsg::ListenForChunks(ListenForChunks { - relay_parent, - candidate_hash: candidate.hash(), + candidate_hash, index: our_id as u32, result: None, }); @@ -693,27 +671,19 @@ impl BlockImport for AvailabilityBlockImport where } } - let erasure_roots: Vec<_> = candidates - .iter() - .map(|c| c.erasure_root) - .collect(); - - // Inform the worker about new (relay_parent, erasure_roots) pairs - let (s, _) = oneshot::channel(); - let msg = WorkerMsg::ErasureRoots(ErasureRoots { - relay_parent, - erasure_roots, - result: s, - }); - - let _ = self.to_worker.unbounded_send(msg); - let (s, _) = oneshot::channel(); // Inform the worker about the included parachain blocks. - let msg = WorkerMsg::ParachainBlocks(ParachainBlocks { - relay_parent, - blocks: candidates.into_iter().map(|c| (c, None)).collect(), + let blocks = candidates + .into_iter() + .map(|c| IncludedParachainBlock { + candidate: c, + available_data: None, + }) + .collect(); + + let msg = WorkerMsg::IncludedParachainBlocks(IncludedParachainBlocks { + blocks, result: s, }); @@ -806,6 +776,7 @@ mod tests { use std::sync::{Arc, Mutex, Condvar}; use std::pin::Pin; use tokio::runtime::Runtime; + use crate::store::AwaitedFrontierEntry; // Just contains topic->channel mapping to give to outer code on `gossip_messages_for` calls. struct TestGossipMessages { @@ -866,7 +837,7 @@ mod tests { let store = Store::new_in_memory(); // Tell the store our validator's position and the number of validators at given point. - store.add_validator_index_and_n_validators(&relay_parent, local_id, n_validators).unwrap(); + store.note_validator_index_and_n_validators(&relay_parent, local_id, n_validators).unwrap(); let (gossip_sender, gossip_receiver) = mpsc::unbounded(); @@ -879,9 +850,10 @@ mod tests { ].into_iter().collect())) }; - let mut candidate = CandidateReceipt::default(); + let mut candidate = AbridgedCandidateReceipt::default(); - candidate.erasure_root = erasure_root; + candidate.commitments.erasure_root = erasure_root; + candidate.relay_parent = relay_parent; let candidate_hash = candidate.hash(); // At this point we shouldn't be waiting for any chunks. @@ -889,9 +861,11 @@ mod tests { let (s, r) = oneshot::channel(); - let msg = WorkerMsg::ParachainBlocks(ParachainBlocks { - relay_parent, - blocks: vec![(candidate, None)], + let msg = WorkerMsg::IncludedParachainBlocks(IncludedParachainBlocks { + blocks: vec![IncludedParachainBlock { + candidate, + available_data: None, + }], result: s, }); @@ -907,7 +881,11 @@ mod tests { // Make sure that at this point we are waiting for the appropriate chunk. assert_eq!( store.awaited_chunks().unwrap(), - vec![(relay_parent, erasure_root, candidate_hash, local_id)].into_iter().collect() + vec![AwaitedFrontierEntry { + relay_parent, + erasure_root, + validator_index: local_id, + }].into_iter().collect() ); let msg = ( @@ -941,25 +919,27 @@ mod tests { let relay_parent = [1; 32].into(); let erasure_root_1 = [2; 32].into(); let erasure_root_2 = [3; 32].into(); - let block_data_hash_1 = [4; 32].into(); - let block_data_hash_2 = [5; 32].into(); + let pov_block_hash_1 = [4; 32].into(); + let pov_block_hash_2 = [5; 32].into(); let local_id = 2; let n_validators = 4; - let mut candidate_1 = CandidateReceipt::default(); - candidate_1.erasure_root = erasure_root_1; - candidate_1.block_data_hash = block_data_hash_1; + let mut candidate_1 = AbridgedCandidateReceipt::default(); + candidate_1.commitments.erasure_root = erasure_root_1; + candidate_1.pov_block_hash = pov_block_hash_1; + candidate_1.relay_parent = relay_parent; let candidate_1_hash = candidate_1.hash(); - let mut candidate_2 = CandidateReceipt::default(); - candidate_2.erasure_root = erasure_root_2; - candidate_2.block_data_hash = block_data_hash_2; + let mut candidate_2 = AbridgedCandidateReceipt::default(); + candidate_2.commitments.erasure_root = erasure_root_2; + candidate_2.pov_block_hash = pov_block_hash_2; + candidate_2.relay_parent = relay_parent; let candidate_2_hash = candidate_2.hash(); let store = Store::new_in_memory(); // Tell the store our validator's position and the number of validators at given point. - store.add_validator_index_and_n_validators(&relay_parent, local_id, n_validators).unwrap(); + store.note_validator_index_and_n_validators(&relay_parent, local_id, n_validators).unwrap(); // Let the store know about the candidates store.add_candidate(&candidate_1).unwrap(); @@ -968,7 +948,6 @@ mod tests { // And let the store know about the chunk from the second candidate. store.add_erasure_chunks( n_validators, - &relay_parent, &candidate_2_hash, vec![ErasureChunk { chunk: vec![1, 2, 3], @@ -999,7 +978,6 @@ mod tests { let (s2, r2) = oneshot::channel(); // Tell the worker to listen for chunks from candidate 2 (we alredy have a chunk from it). let listen_msg_2 = WorkerMsg::ListenForChunks(ListenForChunks { - relay_parent, candidate_hash: candidate_2_hash, index: local_id as u32, result: Some(s2), @@ -1016,7 +994,6 @@ mod tests { // Tell the worker to listen for chunks from candidate 1. // (we don't have a chunk from it yet). let listen_msg_1 = WorkerMsg::ListenForChunks(ListenForChunks { - relay_parent, candidate_hash: candidate_1_hash, index: local_id as u32, result: Some(s1), diff --git a/polkadot/collator/src/lib.rs b/polkadot/collator/src/lib.rs index 8c91071f93..e0f230452c 100644 --- a/polkadot/collator/src/lib.rs +++ b/polkadot/collator/src/lib.rs @@ -57,7 +57,7 @@ use polkadot_primitives::{ BlockId, Hash, Block, parachain::{ self, BlockData, DutyRoster, HeadData, Id as ParaId, - PoVBlock, Status as ParachainStatus, ValidatorId, CollatorPair, + PoVBlock, ValidatorId, CollatorPair, LocalValidationData } }; use polkadot_cli::{ @@ -167,7 +167,7 @@ pub trait ParachainContext: Clone { fn produce_candidate( &mut self, relay_parent: Hash, - status: ParachainStatus, + status: LocalValidationData, ) -> Self::ProduceCandidate; } @@ -175,7 +175,7 @@ pub trait ParachainContext: Clone { pub async fn collate

( relay_parent: Hash, local_id: ParaId, - parachain_status: ParachainStatus, + local_validation_data: LocalValidationData, mut para_context: P, key: Arc, ) @@ -186,25 +186,32 @@ pub async fn collate

( { let (block_data, head_data) = para_context.produce_candidate( relay_parent, - parachain_status, + local_validation_data, ).map_err(Error::Collator).await?; - let block_data_hash = block_data.hash(); - let signature = key.sign(block_data_hash.as_ref()); + let pov_block = PoVBlock { + block_data, + }; + + let pov_block_hash = pov_block.hash(); + let signature = key.sign(¶chain::collator_signature_payload( + &relay_parent, + &local_id, + &pov_block_hash, + )); + let info = parachain::CollationInfo { parachain_index: local_id, + relay_parent, collator: key.public(), signature, head_data, - block_data_hash, - upward_messages: Vec::new(), + pov_block_hash, }; let collation = parachain::Collation { info, - pov: PoVBlock { - block_data, - }, + pov: pov_block, }; Ok(collation) @@ -318,8 +325,8 @@ fn run_collator_node( let work = future::lazy(move |_| { let api = client.runtime_api(); - let status = match try_fr!(api.parachain_status(&id, para_id)) { - Some(status) => status, + let local_validation = match try_fr!(api.local_validation_data(&id, para_id)) { + Some(local_validation) => local_validation, None => return future::Either::Left(future::ok(())), }; @@ -334,19 +341,17 @@ fn run_collator_node( let collation_work = collate( relay_parent, para_id, - status, + local_validation, parachain_context, key, ).map_ok(move |collation| { network.with_spec(move |spec, ctx| { - let res = spec.add_local_collation( + spec.add_local_collation( ctx, relay_parent, targets, collation, ); - - tokio::spawn(res.boxed()); }) }); @@ -442,8 +447,6 @@ pub fn run_collator

( #[cfg(test)] mod tests { - use polkadot_primitives::parachain::FeeSchedule; - use keyring::Sr25519Keyring; use super::*; #[derive(Clone)] @@ -455,7 +458,7 @@ mod tests { fn produce_candidate( &mut self, _relay_parent: Hash, - _status: ParachainStatus, + _local_validation: LocalValidationData, ) -> Self::ProduceCandidate { // send messages right back. future::ok(( diff --git a/polkadot/erasure-coding/src/lib.rs b/polkadot/erasure-coding/src/lib.rs index 0829f38d53..2beaa788e2 100644 --- a/polkadot/erasure-coding/src/lib.rs +++ b/polkadot/erasure-coding/src/lib.rs @@ -27,7 +27,7 @@ use codec::{Encode, Decode}; use reed_solomon::galois_16::{self, ReedSolomon}; use primitives::{Hash as H256, BlakeTwo256, HashT}; -use primitives::parachain::BlockData; +use primitives::parachain::AvailableData; use sp_core::Blake2Hasher; use trie::{EMPTY_PREFIX, MemoryDB, Trie, TrieMut, trie_types::{TrieDBMut, TrieDB}}; @@ -66,6 +66,8 @@ pub enum Error { BranchOutOfBounds, } +impl std::error::Error for Error { } + #[derive(Debug, PartialEq)] struct CodeParams { data_shards: usize, @@ -125,11 +127,11 @@ fn code_params(n_validators: usize) -> Result { /// Obtain erasure-coded chunks, one for each validator. /// /// Works only up to 65536 validators, and `n_validators` must be non-zero. -pub fn obtain_chunks(n_validators: usize, block_data: &BlockData) +pub fn obtain_chunks(n_validators: usize, available_data: &AvailableData) -> Result>, Error> { let params = code_params(n_validators)?; - let encoded = block_data.encode(); + let encoded = available_data.encode(); if encoded.is_empty() { return Err(Error::BadPayload); @@ -151,7 +153,7 @@ pub fn obtain_chunks(n_validators: usize, block_data: &BlockData) /// /// Works only up to 65536 validators, and `n_validators` must be non-zero. pub fn reconstruct<'a, I: 'a>(n_validators: usize, chunks: I) - -> Result + -> Result where I: IntoIterator { let params = code_params(n_validators)?; @@ -341,6 +343,7 @@ impl<'a, I: Iterator> codec::Input for ShardInput<'a, I> { #[cfg(test)] mod tests { use super::*; + use primitives::parachain::{BlockData, PoVBlock}; #[test] fn field_order_is_right_size() { @@ -400,11 +403,18 @@ mod tests { } #[test] - fn round_trip_block_data() { - let block_data = BlockData((0..255).collect()); + fn round_trip_works() { + let pov_block = PoVBlock { + block_data: BlockData((0..255).collect()), + }; + + let available_data = AvailableData { + pov_block, + omitted_validation: Default::default(), + }; let chunks = obtain_chunks( 10, - &block_data, + &available_data, ).unwrap(); assert_eq!(chunks.len(), 10); @@ -420,16 +430,23 @@ mod tests { ].iter().cloned(), ).unwrap(); - assert_eq!(reconstructed, block_data); + assert_eq!(reconstructed, available_data); } #[test] fn construct_valid_branches() { - let block_data = BlockData(vec![2; 256]); + let pov_block = PoVBlock { + block_data: BlockData(vec![2; 256]), + }; + + let available_data = AvailableData { + pov_block, + omitted_validation: Default::default(), + }; let chunks = obtain_chunks( 10, - &block_data, + &available_data, ).unwrap(); assert_eq!(chunks.len(), 10); diff --git a/polkadot/network/src/legacy/collator_pool.rs b/polkadot/network/src/legacy/collator_pool.rs index 4ca1d03711..a0c0a0458e 100644 --- a/polkadot/network/src/legacy/collator_pool.rs +++ b/polkadot/network/src/legacy/collator_pool.rs @@ -236,7 +236,7 @@ impl CollatorPool { mod tests { use super::*; use sp_core::crypto::UncheckedInto; - use polkadot_primitives::parachain::{CandidateReceipt, BlockData, PoVBlock, HeadData}; + use polkadot_primitives::parachain::{CollationInfo, BlockData, PoVBlock}; use futures::executor::block_on; fn make_pov(block_data: Vec) -> PoVBlock { @@ -284,18 +284,11 @@ mod tests { let (tx2, rx2) = oneshot::channel(); pool.await_collation(relay_parent, para_id, tx1); pool.await_collation(relay_parent, para_id, tx2); + let mut collation_info = CollationInfo::default(); + collation_info.parachain_index = para_id; + collation_info.collator = primary.clone().into(); pool.on_collation(primary.clone(), relay_parent, Collation { - info: CandidateReceipt { - parachain_index: para_id, - collator: primary.clone().into(), - signature: Default::default(), - head_data: HeadData(vec![1, 2, 3]), - parent_head: HeadData(vec![]), - fees: 0, - block_data_hash: [3; 32].into(), - upward_messages: Vec::new(), - erasure_root: [1u8; 32].into(), - }.into(), + info: collation_info, pov: make_pov(vec![4, 5, 6]), }); @@ -313,18 +306,11 @@ mod tests { assert_eq!(pool.on_new_collator(primary.clone(), para_id.clone(), PeerId::random()), Role::Primary); + let mut collation_info = CollationInfo::default(); + collation_info.parachain_index = para_id; + collation_info.collator = primary.clone(); pool.on_collation(primary.clone(), relay_parent, Collation { - info: CandidateReceipt { - parachain_index: para_id, - collator: primary, - signature: Default::default(), - head_data: HeadData(vec![1, 2, 3]), - parent_head: HeadData(vec![]), - fees: 0, - block_data_hash: [3; 32].into(), - upward_messages: Vec::new(), - erasure_root: [1u8; 32].into(), - }.into(), + info: collation_info, pov: make_pov(vec![4, 5, 6]), }); diff --git a/polkadot/network/src/legacy/gossip/mod.rs b/polkadot/network/src/legacy/gossip/mod.rs index 43d5e80036..c88a5292d0 100644 --- a/polkadot/network/src/legacy/gossip/mod.rs +++ b/polkadot/network/src/legacy/gossip/mod.rs @@ -540,7 +540,7 @@ impl Inner { if let Some(store) = &self.availability_store { if let Some(receipt) = store.get_candidate(&msg.candidate_hash) { let chunk_hash = erasure::branch_hash( - &receipt.erasure_root, + &receipt.commitments.erasure_root, &msg.chunk.proof, msg.chunk.index as usize ); @@ -552,15 +552,15 @@ impl Inner { ) } else { if let Some(awaited_chunks) = store.awaited_chunks() { - if awaited_chunks.contains(&( - msg.relay_parent, - receipt.erasure_root, - receipt.hash(), - msg.chunk.index, - )) { + let frontier_entry = av_store::AwaitedFrontierEntry { + relay_parent: msg.relay_parent, + erasure_root: receipt.commitments.erasure_root, + validator_index: msg.chunk.index, + }; + if awaited_chunks.contains(&frontier_entry) { let topic = av_store::erasure_coding_topic( msg.relay_parent, - receipt.erasure_root, + receipt.commitments.erasure_root, msg.chunk.index, ); @@ -722,8 +722,7 @@ mod tests { use sc_network_gossip::Validator as ValidatorT; use std::sync::mpsc; use parking_lot::Mutex; - use polkadot_primitives::parachain::{CandidateReceipt, HeadData}; - use sp_core::crypto::UncheckedInto; + use polkadot_primitives::parachain::AbridgedCandidateReceipt; use sp_core::sr25519::Signature as Sr25519Signature; use polkadot_validation::GenericStatement; @@ -807,18 +806,7 @@ mod tests { validator_context.clear(); - let candidate_receipt = CandidateReceipt { - parachain_index: 5.into(), - collator: [255; 32].unchecked_into(), - head_data: HeadData(vec![9, 9, 9]), - parent_head: HeadData(vec![]), - signature: Default::default(), - fees: 1_000_000, - block_data_hash: [20u8; 32].into(), - upward_messages: Vec::new(), - erasure_root: [1u8; 32].into(), - }; - + let candidate_receipt = AbridgedCandidateReceipt::default(); let statement = GossipMessage::Statement(GossipStatement { relay_chain_leaf: hash_a, signed_statement: SignedStatement { diff --git a/polkadot/network/src/legacy/mod.rs b/polkadot/network/src/legacy/mod.rs index fe7018dfb0..3e65263bc9 100644 --- a/polkadot/network/src/legacy/mod.rs +++ b/polkadot/network/src/legacy/mod.rs @@ -30,7 +30,7 @@ use futures::channel::oneshot; use futures::prelude::*; use polkadot_primitives::{Block, Hash, Header}; use polkadot_primitives::parachain::{ - Id as ParaId, CollatorId, CandidateReceipt, Collation, PoVBlock, + Id as ParaId, CollatorId, AbridgedCandidateReceipt, Collation, PoVBlock, ValidatorId, ErasureChunk, }; use sc_network::{ @@ -168,7 +168,7 @@ struct PoVBlockRequest { attempted_peers: HashSet, validation_leaf: Hash, candidate_hash: Hash, - block_data_hash: Hash, + pov_block_hash: Hash, sender: oneshot::Sender, } @@ -178,7 +178,7 @@ impl PoVBlockRequest { // // If `Ok(())` is returned, that indicates that the request has been processed. fn process_response(self, pov_block: PoVBlock) -> Result<(), Self> { - if pov_block.block_data.hash() != self.block_data_hash { + if pov_block.hash() != self.pov_block_hash { return Err(self); } @@ -292,7 +292,7 @@ impl PolkadotProtocol { fn fetch_pov_block( &mut self, ctx: &mut dyn Context, - candidate: &CandidateReceipt, + candidate: &AbridgedCandidateReceipt, relay_parent: Hash, ) -> oneshot::Receiver { let (tx, rx) = oneshot::channel(); @@ -301,7 +301,7 @@ impl PolkadotProtocol { attempted_peers: Default::default(), validation_leaf: relay_parent, candidate_hash: candidate.hash(), - block_data_hash: candidate.block_data_hash, + pov_block_hash: candidate.pov_block_hash, sender: tx, }); @@ -608,7 +608,7 @@ impl Specialization for PolkadotProtocol { attempted_peers: Default::default(), validation_leaf: Default::default(), candidate_hash: Default::default(), - block_data_hash: Default::default(), + pov_block_hash: Default::default(), sender, })); } @@ -738,7 +738,7 @@ impl PolkadotProtocol { relay_parent: Hash, targets: HashSet, collation: Collation, - ) -> impl Future { + ) { debug!(target: "p_net", "Importing local collation on relay parent {:?} and parachain {:?}", relay_parent, collation.info.parachain_index); @@ -756,19 +756,6 @@ impl PolkadotProtocol { warn!(target: "polkadot_network", "Encountered tracked but disconnected validator {:?}", primary), } } - - let availability_store = self.availability_store.clone(); - let collation_cloned = collation.clone(); - - async move { - if let Some(availability_store) = availability_store { - let _ = availability_store.make_available(av_store::Data { - relay_parent, - parachain_id: collation_cloned.info.parachain_index, - block_data: collation_cloned.pov.block_data.clone(), - }).await; - } - } } /// Give the network protocol a handle to an availability store, used for diff --git a/polkadot/network/src/legacy/router.rs b/polkadot/network/src/legacy/router.rs index 4a969a4870..ddffe72f0c 100644 --- a/polkadot/network/src/legacy/router.rs +++ b/polkadot/network/src/legacy/router.rs @@ -29,7 +29,7 @@ use polkadot_validation::{ }; use polkadot_primitives::{Block, Hash}; use polkadot_primitives::parachain::{ - CandidateReceipt, ParachainHost, ValidatorIndex, Collation, PoVBlock, ErasureChunk, + AbridgedCandidateReceipt, ParachainHost, ValidatorIndex, PoVBlock, ErasureChunk, }; use sp_api::ProvideRuntimeApi; @@ -204,17 +204,17 @@ impl + Send + Sync + 'static, T> Router where // store the data before broadcasting statements, so other peers can fetch. knowledge.lock().note_candidate( candidate_hash, - Some(validated.0.pov_block().clone()), + Some(validated.pov_block().clone()), ); // propagate the statement. // consider something more targeted than gossip in the future. let statement = GossipStatement::new( - parent_hash, - match table.import_validated(validated.0) { - None => return, - Some(s) => s, - } + parent_hash, + match table.import_validated(validated) { + None => return, + Some(s) => s, + } ); network.gossip_message(attestation_topic, statement.into()); @@ -238,16 +238,16 @@ impl + Send, T> TableRouter for Router where // We have fetched from a collator and here the receipt should have been already formed. fn local_collation( &self, - collation: Collation, - receipt: CandidateReceipt, + receipt: AbridgedCandidateReceipt, + pov_block: PoVBlock, chunks: (ValidatorIndex, &[ErasureChunk]) ) -> Self::SendLocalCollation { // produce a signed statement let hash = receipt.hash(); - let erasure_root = receipt.erasure_root; + let erasure_root = receipt.commitments.erasure_root; let validated = Validated::collated_local( receipt, - collation.pov.clone(), + pov_block.clone(), ); let statement = GossipStatement::new( @@ -259,7 +259,7 @@ impl + Send, T> TableRouter for Router where ); // give to network to make available. - self.fetcher.knowledge().lock().note_candidate(hash, Some(collation.pov)); + self.fetcher.knowledge().lock().note_candidate(hash, Some(pov_block)); self.network().gossip_message(self.attestation_topic, statement.into()); for chunk in chunks.1 { @@ -279,7 +279,7 @@ impl + Send, T> TableRouter for Router where future::ready(Ok(())) } - fn fetch_pov_block(&self, candidate: &CandidateReceipt) -> Self::FetchValidationProof { + fn fetch_pov_block(&self, candidate: &AbridgedCandidateReceipt) -> Self::FetchValidationProof { self.fetcher.fetch_pov_block(candidate) } } diff --git a/polkadot/network/src/legacy/tests/validation.rs b/polkadot/network/src/legacy/tests/validation.rs index a85e80faa6..49d63d8fa6 100644 --- a/polkadot/network/src/legacy/tests/validation.rs +++ b/polkadot/network/src/legacy/tests/validation.rs @@ -28,8 +28,9 @@ use crate::legacy::{PolkadotProtocol, NetworkService, GossipService, GossipMessa use polkadot_validation::{SharedTable, Network}; use polkadot_primitives::{Block, BlockNumber, Hash, Header, BlockId}; use polkadot_primitives::parachain::{ - Id as ParaId, Chain, DutyRoster, ParachainHost, ValidatorId, Status, - FeeSchedule, HeadData, Retriable, CollatorId, ErasureChunk, CandidateReceipt, + Id as ParaId, Chain, DutyRoster, ParachainHost, ValidatorId, + FeeSchedule, HeadData, Retriable, CollatorId, ErasureChunk, AbridgedCandidateReceipt, + GlobalValidationSchedule, LocalValidationData, }; use parking_lot::Mutex; use sp_blockchain::Result as ClientResult; @@ -270,23 +271,6 @@ impl ParachainHost for RuntimeApi { Ok(NativeOrEncoded::Native(self.data.lock().active_parachains.clone())) } - fn ParachainHost_parachain_status_runtime_api_impl( - &self, - _at: &BlockId, - _: ExecutionContext, - _: Option, - _: Vec, - ) -> ClientResult>> { - Ok(NativeOrEncoded::Native(Some(Status { - head_data: HeadData(Vec::new()), - balance: 0, - fee_schedule: FeeSchedule { - base: 0, - per_byte: 0, - } - }))) - } - fn ParachainHost_parachain_code_runtime_api_impl( &self, _at: &BlockId, @@ -297,13 +281,33 @@ impl ParachainHost for RuntimeApi { Ok(NativeOrEncoded::Native(Some(Vec::new()))) } + fn ParachainHost_global_validation_schedule_runtime_api_impl( + &self, + _at: &BlockId, + _: ExecutionContext, + _: Option<()>, + _: Vec, + ) -> ClientResult> { + Ok(NativeOrEncoded::Native(Default::default())) + } + + fn ParachainHost_local_validation_data_runtime_api_impl( + &self, + _at: &BlockId, + _: ExecutionContext, + _: Option, + _: Vec, + ) -> ClientResult>> { + Ok(NativeOrEncoded::Native(Some(Default::default()))) + } + fn ParachainHost_get_heads_runtime_api_impl( &self, _at: &BlockId, _: ExecutionContext, _extrinsics: Option::Extrinsic>>, _: Vec, - ) -> ClientResult>>> { + ) -> ClientResult>>> { Ok(NativeOrEncoded::Native(Some(Vec::new()))) } } diff --git a/polkadot/network/src/legacy/validation.rs b/polkadot/network/src/legacy/validation.rs index c64f21428d..3ab7ede6cf 100644 --- a/polkadot/network/src/legacy/validation.rs +++ b/polkadot/network/src/legacy/validation.rs @@ -25,7 +25,7 @@ use polkadot_validation::{ }; use polkadot_primitives::{Block, Hash}; use polkadot_primitives::parachain::{ - Id as ParaId, Collation, ParachainHost, CandidateReceipt, CollatorId, + Id as ParaId, Collation, ParachainHost, AbridgedCandidateReceipt, CollatorId, ValidatorId, PoVBlock, }; use sp_api::ProvideRuntimeApi; @@ -557,7 +557,7 @@ impl + Send, T> LeafWorkDataFetcher where T: Clone + Executor + Send + 'static, { /// Fetch PoV block for the given candidate receipt. - pub fn fetch_pov_block(&self, candidate: &CandidateReceipt) + pub fn fetch_pov_block(&self, candidate: &AbridgedCandidateReceipt) -> Pin> + Send>> { let parent_hash = self.parent_hash; diff --git a/polkadot/network/src/protocol.rs b/polkadot/network/src/protocol.rs index acdee68307..526a9fbf76 100644 --- a/polkadot/network/src/protocol.rs +++ b/polkadot/network/src/protocol.rs @@ -31,7 +31,7 @@ use log::{debug, trace}; use polkadot_primitives::{ Hash, Block, parachain::{ - PoVBlock, ValidatorId, ValidatorIndex, Collation, CandidateReceipt, + PoVBlock, ValidatorId, ValidatorIndex, Collation, AbridgedCandidateReceipt, ErasureChunk, ParachainHost, Id as ParaId, CollatorId, }, }; @@ -72,13 +72,13 @@ enum ServiceToWorkerMsg { DropConsensusNetworking(Hash), LocalCollation( Hash, // relay-parent - Collation, - CandidateReceipt, + AbridgedCandidateReceipt, + PoVBlock, (ValidatorIndex, Vec), ), FetchPoVBlock( Hash, // relay-parent - CandidateReceipt, + AbridgedCandidateReceipt, oneshot::Sender, ), AwaitCollation( @@ -650,7 +650,7 @@ async fn worker_loop( ServiceToWorkerMsg::DropConsensusNetworking(relay_parent) => { consensus_instances.remove(&relay_parent); } - ServiceToWorkerMsg::LocalCollation(relay_parent, collation, receipt, chunks) => { + ServiceToWorkerMsg::LocalCollation(relay_parent, receipt, pov_block, chunks) => { let instance = match consensus_instances.get(&relay_parent) { None => continue, Some(instance) => instance, @@ -658,8 +658,8 @@ async fn worker_loop( distribute_local_collation( instance, - collation, receipt, + pov_block, chunks, &gossip_handle, ); @@ -686,7 +686,7 @@ async fn statement_import_loop( table: Arc, api: Arc, weak_router: Weak, - validator: RegisteredMessageValidator, + gossip_handle: RegisteredMessageValidator, mut exit: exit_future::Exit, executor: impl Spawn, ) where @@ -694,7 +694,7 @@ async fn statement_import_loop( Api::Api: ParachainHost, { let topic = crate::legacy::router::attestation_topic(relay_parent); - let mut checked_messages = validator.gossip_messages_for(topic) + let mut checked_messages = gossip_handle.gossip_messages_for(topic) .filter_map(|msg| match msg.0 { crate::legacy::gossip::GossipMessage::Statement(s) => future::ready(Some(s.signed_statement)), _ => future::ready(None), @@ -758,8 +758,10 @@ async fn statement_import_loop( if let Some(producer) = producer { trace!(target: "validation", "driving statement work to completion"); - let work = producer.prime(api.clone()).validate(); + let table = table.clone(); + let gossip_handle = gossip_handle.clone(); + let work = producer.prime(api.clone()).validate(); let work = future::select(work.boxed(), exit.clone()).map(drop); let _ = executor.spawn(work); } @@ -773,17 +775,17 @@ async fn statement_import_loop( // group. fn distribute_local_collation( instance: &ConsensusNetworkingInstance, - collation: Collation, - receipt: CandidateReceipt, + receipt: AbridgedCandidateReceipt, + pov_block: PoVBlock, chunks: (ValidatorIndex, Vec), gossip_handle: &RegisteredMessageValidator, ) { // produce a signed statement. let hash = receipt.hash(); - let erasure_root = receipt.erasure_root; + let erasure_root = receipt.commitments.erasure_root; let validated = Validated::collated_local( receipt, - collation.pov.clone(), + pov_block, ); let statement = crate::legacy::gossip::GossipStatement::new( @@ -906,14 +908,14 @@ impl TableRouter for Router { fn local_collation( &self, - collation: Collation, - receipt: CandidateReceipt, + receipt: AbridgedCandidateReceipt, + pov_block: PoVBlock, chunks: (ValidatorIndex, &[ErasureChunk]), ) -> Self::SendLocalCollation { let message = ServiceToWorkerMsg::LocalCollation( self.inner.relay_parent.clone(), - collation, receipt, + pov_block, (chunks.0, chunks.1.to_vec()), ); let mut sender = self.inner.sender.clone(); @@ -922,7 +924,7 @@ impl TableRouter for Router { }) } - fn fetch_pov_block(&self, candidate: &CandidateReceipt) -> Self::FetchValidationProof { + fn fetch_pov_block(&self, candidate: &AbridgedCandidateReceipt) -> Self::FetchValidationProof { let (tx, rx) = oneshot::channel(); let message = ServiceToWorkerMsg::FetchPoVBlock( self.inner.relay_parent.clone(), diff --git a/polkadot/parachain/src/wasm_executor/mod.rs b/polkadot/parachain/src/wasm_executor/mod.rs index 1b6f556ac6..cb1e9a261f 100644 --- a/polkadot/parachain/src/wasm_executor/mod.rs +++ b/polkadot/parachain/src/wasm_executor/mod.rs @@ -79,7 +79,7 @@ pub enum Error { #[display(fmt = "IO error: {}", _0)] Io(std::io::Error), #[display(fmt = "System error: {}", _0)] - System(Box), + System(Box), #[display(fmt = "WASM worker error: {}", _0)] External(String), #[display(fmt = "Shared memory error: {}", _0)] @@ -129,10 +129,14 @@ pub fn validate_candidate( }, #[cfg(target_os = "unknown")] ExecutionMode::Remote => - Err(Error::System("Remote validator not available".to_string().into())), + Err(Error::System(Box::::from( + "Remote validator not available".to_string() + ) as Box<_>)), #[cfg(target_os = "unknown")] ExecutionMode::RemoteTest => - Err(Error::System("Remote validator not available".to_string().into())), + Err(Error::System(Box::::from( + "Remote validator not available".to_string() + ) as Box<_>)), } } diff --git a/polkadot/primitives/src/parachain.rs b/polkadot/primitives/src/parachain.rs index 045f99d623..6b050b9f33 100644 --- a/polkadot/primitives/src/parachain.rs +++ b/polkadot/primitives/src/parachain.rs @@ -28,12 +28,10 @@ use serde::{Serialize, Deserialize}; #[cfg(feature = "std")] use primitives::bytes; use primitives::RuntimeDebug; +use runtime_primitives::traits::{Block as BlockT}; use inherents::InherentIdentifier; use application_crypto::KeyTypeId; -#[cfg(feature = "std")] -use trie::TrieConfiguration; - pub use polkadot_parachain::{ Id, ParachainDispatchOrigin, LOWEST_USER_ID, UpwardMessage, }; @@ -164,98 +162,143 @@ pub struct DutyRoster { pub validator_duty: Vec, } -/// Compute a trie root for a set of messages, given the raw message data. -#[cfg(feature = "std")] -pub fn message_queue_root>(messages: I) -> Hash - where A: AsRef<[u8]> -{ - trie::trie_types::Layout::::ordered_trie_root(messages) -} - -/// Candidate receipt type. -#[derive(PartialEq, Eq, Clone, Encode, Decode)] -#[cfg_attr(feature = "std", derive(Debug))] -pub struct CollationInfo { - /// The ID of the parachain this is a candidate for. - pub parachain_index: Id, - /// The collator's relay-chain account ID - pub collator: CollatorId, - /// Signature on blake2-256 of the block data by collator. - pub signature: CollatorSignature, - /// The head-data - pub head_data: HeadData, - /// blake2-256 Hash of block data. - pub block_data_hash: Hash, - /// Messages destined to be interpreted by the Relay chain itself. - pub upward_messages: Vec, -} - -impl From for CollationInfo { - fn from(receipt: CandidateReceipt) -> Self { - CollationInfo { - parachain_index: receipt.parachain_index, - collator: receipt.collator, - signature: receipt.signature, - head_data: receipt.head_data, - block_data_hash: receipt.block_data_hash, - upward_messages: receipt.upward_messages, - } - } -} - -impl CollationInfo { - /// Check integrity vs. provided block data. - pub fn check_signature(&self) -> Result<(), ()> { - use runtime_primitives::traits::AppVerify; - - if self.signature.verify(self.block_data_hash.as_ref(), &self.collator) { - Ok(()) - } else { - Err(()) - } - } -} - -/// Candidate receipt type. +/// Extra data which is needed along with the other fields in a `CandidateReceipt` +/// to fully validate the candidate. +/// +/// These are global parameters that apply to all parachain candidates in a block. #[derive(PartialEq, Eq, Clone, Encode, Decode)] #[cfg_attr(feature = "std", derive(Debug, Default))] -pub struct CandidateReceipt { - /// The ID of the parachain this is a candidate for. - pub parachain_index: Id, - /// The collator's relay-chain account ID - pub collator: CollatorId, - /// Signature on blake2-256 of the block data by collator. - pub signature: CollatorSignature, - /// The head-data - pub head_data: HeadData, +pub struct GlobalValidationSchedule { + /// The maximum code size permitted, in bytes. + pub max_code_size: u32, + /// The maximum head-data size permitted, in bytes. + pub max_head_data_size: u32, +} + +/// Extra data which is needed along with the other fields in a `CandidateReceipt` +/// to fully validate the candidate. These fields are parachain-specific. +#[derive(PartialEq, Eq, Clone, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug, Default))] +pub struct LocalValidationData { /// The parent head-data. pub parent_head: HeadData, + /// The balance of the parachain at the moment of validation. + pub balance: Balance, +} + +/// Commitments made in a `CandidateReceipt`. Many of these are outputs of validation. +#[derive(PartialEq, Eq, Clone, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug, Default))] +pub struct CandidateCommitments { /// Fees paid from the chain to the relay chain validators pub fees: Balance, - /// blake2-256 Hash of block data. - pub block_data_hash: Hash, /// Messages destined to be interpreted by the Relay chain itself. pub upward_messages: Vec, /// The root of a block's erasure encoding Merkle tree. pub erasure_root: Hash, } -impl CandidateReceipt { - /// Get the blake2_256 hash - pub fn hash(&self) -> Hash { - use runtime_primitives::traits::{BlakeTwo256, Hash}; - BlakeTwo256::hash_of(self) - } +/// Get a collator signature payload on a relay-parent, block-data combo. +pub fn collator_signature_payload( + relay_parent: &Hash, + parachain_index: &Id, + pov_block_hash: &Hash, +) -> [u8; 68] { + // 32-byte hash length is protected in a test below. + let mut payload = [0u8; 68]; + payload[0..32].copy_from_slice(relay_parent.as_ref()); + u32::from(*parachain_index).using_encoded(|s| payload[32..32 + s.len()].copy_from_slice(s)); + payload[36..68].copy_from_slice(pov_block_hash.as_ref()); + + payload +} + +fn check_collator_signature( + relay_parent: &Hash, + parachain_index: &Id, + pov_block_hash: &Hash, + collator: &CollatorId, + signature: &CollatorSignature, +) -> Result<(),()> { + use runtime_primitives::traits::AppVerify; + + let payload = collator_signature_payload(relay_parent, parachain_index, pov_block_hash); + if signature.verify(&payload[..], collator) { + Ok(()) + } else { + Err(()) + } +} + +/// All data pertaining to the execution of a parachain candidate. +#[derive(PartialEq, Eq, Clone, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug, Default))] +pub struct CandidateReceipt { + /// The ID of the parachain this is a candidate for. + pub parachain_index: Id, + /// The hash of the relay-chain block this should be executed in + /// the context of. + pub relay_parent: Hash, + /// The head-data + pub head_data: HeadData, + /// The collator's relay-chain account ID + pub collator: CollatorId, + /// Signature on blake2-256 of the block data by collator. + pub signature: CollatorSignature, + /// The hash of the PoV-block. + pub pov_block_hash: Hash, + /// The global validation schedule. + pub global_validation: GlobalValidationSchedule, + /// The local validation data. + pub local_validation: LocalValidationData, + /// Commitments made as a result of validation. + pub commitments: CandidateCommitments, +} + +impl CandidateReceipt { /// Check integrity vs. provided block data. pub fn check_signature(&self) -> Result<(), ()> { - use runtime_primitives::traits::AppVerify; + check_collator_signature( + &self.relay_parent, + &self.parachain_index, + &self.pov_block_hash, + &self.collator, + &self.signature, + ) + } - if self.signature.verify(self.block_data_hash.as_ref(), &self.collator) { - Ok(()) - } else { - Err(()) - } + /// Abridge this `CandidateReceipt`, splitting it into an `AbridgedCandidateReceipt` + /// and its omitted component. + pub fn abridge(self) -> (AbridgedCandidateReceipt, OmittedValidationData) { + let CandidateReceipt { + parachain_index, + relay_parent, + head_data, + collator, + signature, + pov_block_hash, + global_validation, + local_validation, + commitments, + } = self; + + let abridged = AbridgedCandidateReceipt { + parachain_index, + relay_parent, + head_data, + collator, + signature, + pov_block_hash, + commitments, + }; + + let omitted = OmittedValidationData { + global_validation, + local_validation, + }; + + (abridged, omitted) } } @@ -265,17 +308,6 @@ impl PartialOrd for CandidateReceipt { } } -impl PartialEq for CandidateReceipt { - fn eq(&self, info: &CollationInfo) -> bool { - self.parachain_index == info.parachain_index && - self.collator == info.collator && - self.signature == info.signature && - self.head_data == info.head_data && - self.block_data_hash == info.block_data_hash && - self.upward_messages == info.upward_messages - } -} - impl Ord for CandidateReceipt { fn cmp(&self, other: &Self) -> Ordering { // TODO: compare signatures or something more sane @@ -285,6 +317,181 @@ impl Ord for CandidateReceipt { } } +/// All the data which is omitted in an `AbridgedCandidateReceipt`, but that +/// is necessary for validation of the parachain candidate. +#[derive(PartialEq, Eq, Clone, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug, Default))] +pub struct OmittedValidationData { + /// The global validation schedule. + pub global_validation: GlobalValidationSchedule, + /// The local validation data. + pub local_validation: LocalValidationData, +} + +/// An abridged candidate-receipt. +/// +/// Much info in a candidate-receipt is duplicated from the relay-chain state. +/// When submitting to the relay-chain, this data should be omitted as it can +/// be re-generated from relay-chain state. +#[derive(PartialEq, Eq, Clone, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug, Default))] +pub struct AbridgedCandidateReceipt { + /// The ID of the parachain this is a candidate for. + pub parachain_index: Id, + /// The hash of the relay-chain block this should be executed in + /// the context of. + // NOTE: the fact that the hash includes this value means that code depends + // on this for deduplication. Removing this field is likely to break things. + pub relay_parent: Hash, + /// The head-data + pub head_data: HeadData, + /// The collator's relay-chain account ID + pub collator: CollatorId, + /// Signature on blake2-256 of the block data by collator. + pub signature: CollatorSignature, + /// The hash of the pov-block. + pub pov_block_hash: Hash, + /// Commitments made as a result of validation. + pub commitments: CandidateCommitments, +} + +impl AbridgedCandidateReceipt { + /// Compute the hash of the abridged candidate receipt. + /// + /// This is often used as the canonical hash of the receipt, rather than + /// the hash of the full receipt. The reason being that all data in the full + /// receipt is comitted to in the abridged receipt; this receipt references + /// the relay-chain block in which context it should be executed, which implies + /// any blockchain state that must be referenced. + pub fn hash(&self) -> Hash { + use runtime_primitives::traits::{BlakeTwo256, Hash}; + BlakeTwo256::hash_of(self) + } + + /// Combine the abridged candidate receipt with the omitted data, + /// forming a full `CandidateReceipt`. + pub fn complete(self, omitted: OmittedValidationData) -> CandidateReceipt { + let AbridgedCandidateReceipt { + parachain_index, + relay_parent, + head_data, + collator, + signature, + pov_block_hash, + commitments, + } = self; + + let OmittedValidationData { + global_validation, + local_validation, + } = omitted; + + CandidateReceipt { + parachain_index, + relay_parent, + head_data, + collator, + signature, + pov_block_hash, + local_validation, + global_validation, + commitments, + } + } + + /// Clone the relevant portions of the `CandidateReceipt` to form a `CollationInfo`. + pub fn to_collation_info(&self) -> CollationInfo { + let AbridgedCandidateReceipt { + parachain_index, + relay_parent, + head_data, + collator, + signature, + pov_block_hash, + commitments: _commitments, + } = self; + + CollationInfo { + parachain_index: *parachain_index, + relay_parent: *relay_parent, + head_data: head_data.clone(), + collator: collator.clone(), + signature: signature.clone(), + pov_block_hash: *pov_block_hash, + } + } +} + + +impl PartialOrd for AbridgedCandidateReceipt { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for AbridgedCandidateReceipt { + fn cmp(&self, other: &Self) -> Ordering { + // TODO: compare signatures or something more sane + // https://github.com/paritytech/polkadot/issues/222 + self.parachain_index.cmp(&other.parachain_index) + .then_with(|| self.head_data.cmp(&other.head_data)) + } +} + +/// A collation sent by a collator. +#[derive(PartialEq, Eq, Clone, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug, Default))] +pub struct CollationInfo { + /// The ID of the parachain this is a candidate for. + pub parachain_index: Id, + /// The relay-chain block hash this block should execute in the + /// context of. + pub relay_parent: Hash, + /// The collator's relay-chain account ID + pub collator: CollatorId, + /// Signature on blake2-256 of the block data by collator. + pub signature: CollatorSignature, + /// The head-data + pub head_data: HeadData, + /// blake2-256 Hash of the pov-block + pub pov_block_hash: Hash, +} + +impl CollationInfo { + /// Check integrity vs. a pov-block. + pub fn check_signature(&self) -> Result<(), ()> { + check_collator_signature( + &self.relay_parent, + &self.parachain_index, + &self.pov_block_hash, + &self.collator, + &self.signature, + ) + } + + /// Turn this into an `AbridgedCandidateReceipt` by supplying a set of commitments. + pub fn into_receipt(self, commitments: CandidateCommitments) -> AbridgedCandidateReceipt { + let CollationInfo { + parachain_index, + relay_parent, + collator, + signature, + head_data, + pov_block_hash, + } = self; + + AbridgedCandidateReceipt { + parachain_index, + relay_parent, + collator, + signature, + head_data, + pov_block_hash, + commitments, + } + } +} + /// A full collation. #[derive(PartialEq, Eq, Clone)] #[cfg_attr(feature = "std", derive(Debug, Encode, Decode))] @@ -303,6 +510,27 @@ pub struct PoVBlock { pub block_data: BlockData, } +impl PoVBlock { + /// Compute hash of block data. + #[cfg(feature = "std")] + pub fn hash(&self) -> Hash { + use runtime_primitives::traits::{BlakeTwo256, Hash}; + BlakeTwo256::hash_of(&self) + } +} + +/// The data which is kept available about a particular parachain block. +#[derive(PartialEq, Eq, Clone)] +#[cfg_attr(feature = "std", derive(Debug, Encode, Decode))] +pub struct AvailableData { + /// The PoV block. + pub pov_block: PoVBlock, + /// Data which is omitted from an abridged candidate receipt + /// that is necessary for validation. + pub omitted_validation: OmittedValidationData, + // In the future, outgoing messages as well. +} + /// Parachain block data. /// /// contains everything required to validate para-block, may contain block and witness data @@ -350,13 +578,14 @@ pub struct ValidationCode(#[cfg_attr(feature = "std", serde(with="bytes"))] pub #[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))] pub struct Activity(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec); -/// Statements which can be made about parachain candidates. +/// Statements which can be made about parachain candidates. These are the +/// actual values which are signed by #[derive(Clone, PartialEq, Eq, Encode)] #[cfg_attr(feature = "std", derive(Debug))] pub enum Statement { /// Proposal of a parachain candidate. #[codec(index = "1")] - Candidate(CandidateReceipt), + Candidate(Hash), /// State that a parachain candidate is valid. #[codec(index = "2")] Valid(Hash), @@ -380,11 +609,12 @@ pub enum ValidityAttestation { Explicit(ValidatorSignature), } -/// An attested candidate. +/// An attested candidate. This is submitted to the relay chain by a block author. #[derive(Clone, PartialEq, Decode, Encode, RuntimeDebug)] pub struct AttestedCandidate { - /// The candidate data. - pub candidate: CandidateReceipt, + /// The candidate data. This is abridged, because the omitted data + /// is already present within the relay chain state. + pub candidate: AbridgedCandidateReceipt, /// Validity attestations. pub validity_votes: Vec, /// Indices of the corresponding validity votes. @@ -393,7 +623,7 @@ pub struct AttestedCandidate { impl AttestedCandidate { /// Get the candidate. - pub fn candidate(&self) -> &CandidateReceipt { + pub fn candidate(&self) -> &AbridgedCandidateReceipt { &self.candidate } @@ -424,20 +654,6 @@ impl FeeSchedule { } } -/// Current Status of a parachain. -#[derive(PartialEq, Eq, Clone, Encode, Decode)] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))] -pub struct Status { - /// The head of the parachain. - pub head_data: HeadData, - /// The current balance of the parachain. - pub balance: Balance, - /// The fee schedule for messages coming from this parachain. - pub fee_schedule: FeeSchedule, -} - -use runtime_primitives::traits::{Block as BlockT}; - sp_api::decl_runtime_apis! { /// The API for querying the state of parachains on-chain. #[api_version(2)] @@ -448,12 +664,16 @@ sp_api::decl_runtime_apis! { fn duty_roster() -> DutyRoster; /// Get the currently active parachains. fn active_parachains() -> Vec<(Id, Option<(CollatorId, Retriable)>)>; - /// Get the given parachain's status. - fn parachain_status(id: Id) -> Option; + /// Get the global validation schedule that all parachains should + /// be validated under. + fn global_validation_schedule() -> GlobalValidationSchedule; + /// Get the local validation data for a particular parachain. + fn local_validation_data(id: Id) -> Option; /// Get the given parachain's head code blob. fn parachain_code(id: Id) -> Option>; - /// Extract the heads that were set by this set of extrinsics. - fn get_heads(extrinsics: Vec<::Extrinsic>) -> Option>; + /// Extract the abridged head that was set in the extrinsics. + fn get_heads(extrinsics: Vec<::Extrinsic>) + -> Option>; } } @@ -476,4 +696,17 @@ mod tests { assert!(zero_b.leading_zeros() >= zero_u.leading_zeros()); } + + #[test] + fn collator_signature_payload_is_valid() { + // if this fails, collator signature verification code has to be updated. + let h = Hash::default(); + assert_eq!(h.as_ref().len(), 32); + + let _payload = collator_signature_payload( + &[1; 32].into(), + &5u32.into(), + &[2; 32].into(), + ); + } } diff --git a/polkadot/runtime/common/src/attestations.rs b/polkadot/runtime/common/src/attestations.rs index ca6ef02543..6ae0d08b86 100644 --- a/polkadot/runtime/common/src/attestations.rs +++ b/polkadot/runtime/common/src/attestations.rs @@ -25,7 +25,7 @@ use frame_support::{ decl_storage, decl_module, decl_error, ensure, dispatch::DispatchResult, traits::Get }; -use primitives::{Hash, parachain::{AttestedCandidate, CandidateReceipt, Id as ParaId}}; +use primitives::{Hash, parachain::{AttestedCandidate, AbridgedCandidateReceipt, Id as ParaId}}; use sp_runtime::RuntimeDebug; use sp_staking::SessionIndex; @@ -50,7 +50,7 @@ pub struct IncludedBlocks { /// Attestations kept over time on a parachain block. #[derive(Encode, Decode)] pub struct BlockAttestations { - receipt: CandidateReceipt, + receipt: AbridgedCandidateReceipt, valid: Vec, // stash account ID of voter. invalid: Vec, // stash account ID of voter. } diff --git a/polkadot/runtime/common/src/parachains.rs b/polkadot/runtime/common/src/parachains.rs index c420a5c8d8..504d056c06 100644 --- a/polkadot/runtime/common/src/parachains.rs +++ b/polkadot/runtime/common/src/parachains.rs @@ -18,7 +18,7 @@ use rstd::prelude::*; use rstd::result; -use codec::{Encode, Decode}; +use codec::{Decode, Encode}; use sp_runtime::traits::{ Hash as HashT, BlakeTwo256, Saturating, One, Dispatchable, @@ -29,8 +29,9 @@ use primitives::{ Balance, parachain::{ self, Id as ParaId, Chain, DutyRoster, AttestedCandidate, Statement, ParachainDispatchOrigin, - UpwardMessage, ValidatorId, ActiveParas, CollatorId, Retriable, - NEW_HEADS_IDENTIFIER, + UpwardMessage, ValidatorId, ActiveParas, CollatorId, Retriable, OmittedValidationData, + CandidateReceipt, GlobalValidationSchedule, AbridgedCandidateReceipt, + LocalValidationData, NEW_HEADS_IDENTIFIER, }, }; use frame_support::{ @@ -211,6 +212,10 @@ decl_error! { ParentMismatch, /// Head data was too large. HeadDataTooLarge, + /// Para does not have enough balance to pay fees. + CannotPayFees, + /// Unexpected relay-parent for a candidate receipt. + UnexpectedRelayParent, } } @@ -232,6 +237,11 @@ decl_module! { let mut proceeded = Vec::with_capacity(heads.len()); + let schedule = GlobalValidationSchedule { + max_code_size: T::MaxCodeSize::get(), + max_head_data_size: T::MaxHeadDataSize::get(), + }; + if !active_parachains.is_empty() { // perform integrity checks before writing to storage. { @@ -256,7 +266,7 @@ decl_module! { Self::check_upward_messages( id, - &head.candidate.upward_messages, + &head.candidate.commitments.upward_messages, MAX_QUEUE_COUNT, WATERMARK_QUEUE_SIZE, )?; @@ -267,7 +277,11 @@ decl_module! { } } - let para_blocks = Self::check_candidates(&heads, &active_parachains)?; + let para_blocks = Self::check_candidates( + &schedule, + &heads, + &active_parachains, + )?; >::note_included(&heads, para_blocks); @@ -275,6 +289,9 @@ decl_module! { &heads, ); + // note: we dispatch new messages _after_ the call to `check_candidates` + // which deducts any fees. if that were not the case, an upward message + // could be dispatched and spend money that invalidated a candidate. Self::dispatch_upward_messages( MAX_QUEUE_COUNT, WATERMARK_QUEUE_SIZE, @@ -397,7 +414,7 @@ impl Module { // Queue up upwards messages (from parachains to relay chain). Self::queue_upward_messages( id, - &head.candidate.upward_messages, + &head.candidate.commitments.upward_messages, &mut ordered_needs_dispatch, ); } @@ -433,7 +450,8 @@ impl Module { } } - /// Simple FIFO dispatcher. + /// Simple FIFO dispatcher. This must be called after parachain fees are checked, + /// as dispatched messages may spend parachain funds. fn dispatch_upward_messages( max_queue_count: usize, watermark_queue_size: usize, @@ -540,18 +558,19 @@ impl Module { (DutyRoster { validator_duty: roles_val, }, orig_seed) } - /// Get the parachain status necessary for validation. - pub fn parachain_status(id: ¶chain::Id) -> Option { - let balance = T::ParachainCurrency::free_balance(*id); - Self::parachain_head(id).map(|head_data| parachain::Status { - head_data: parachain::HeadData(head_data), - balance, - // TODO: https://github.com/paritytech/polkadot/issues/92 - // plug in some real values here. most likely governable. - fee_schedule: parachain::FeeSchedule { - base: 0, - per_byte: 0, - } + /// Get the global validation schedule for all parachains. + pub fn global_validation_schedule() -> GlobalValidationSchedule { + GlobalValidationSchedule { + max_code_size: T::MaxCodeSize::get(), + max_head_data_size: T::MaxHeadDataSize::get(), + } + } + + /// Get the local validation schedule for a particular parachain. + pub fn local_validation_data(id: ¶chain::Id) -> Option { + Self::parachain_head(id).map(|parent_head| LocalValidationData { + parent_head: primitives::parachain::HeadData(parent_head), + balance: T::ParachainCurrency::free_balance(*id), }) } @@ -563,6 +582,7 @@ impl Module { // check the attestations on these candidates. The candidates should have been checked // that each candidates' chain ID is valid. fn check_candidates( + schedule: &GlobalValidationSchedule, attested_candidates: &[AttestedCandidate], active_parachains: &[(ParaId, Option<(CollatorId, Retriable)>)] ) -> rstd::result::Result, sp_runtime::DispatchError> @@ -635,6 +655,29 @@ impl Module { sorted_duties }; + // computes the omitted validation data for a particular parachain. + let full_candidate = |abridged: &AbridgedCandidateReceipt| + -> rstd::result::Result + { + let para_id = abridged.parachain_index; + let parent_head = match Self::parachain_head(¶_id) + .map(primitives::parachain::HeadData) + { + Some(p) => p, + None => Err(Error::::ParentMismatch)?, + }; + + let omitted = OmittedValidationData { + global_validation: schedule.clone(), + local_validation: LocalValidationData { + parent_head, + balance: T::ParachainCurrency::free_balance(para_id), + }, + }; + + Ok(abridged.clone().complete(omitted)) + }; + let sorted_validators = make_sorted_duties(&duty_roster.validator_duty); let parent_hash = >::parent_hash(); @@ -644,18 +687,20 @@ impl Module { let mut para_block_hashes = Vec::new(); - let max_head_data_size = T::MaxHeadDataSize::get(); for candidate in attested_candidates { let para_id = candidate.parachain_index(); let validator_group = validator_groups.group_for(para_id) .ok_or(Error::::NoValidatorGroup)?; - let actual_head = Self::parachain_head(¶_id) - .map(primitives::parachain::HeadData); - + // NOTE: when changing this to allow older blocks, + // care must be taken in the availability store pruning to ensure that + // data is stored correctly. A block containing a candidate C can be + // orphaned before a block containing C is finalized. Care must be taken + // not to prune the data for C simply because an orphaned block contained + // it. ensure!( - actual_head.as_ref() == Some(&candidate.candidate.parent_head), - Error::::ParentMismatch, + candidate.candidate().relay_parent.as_ref() == parent_hash.as_ref(), + Error::::UnexpectedRelayParent, ); ensure!( @@ -669,14 +714,21 @@ impl Module { ); ensure!( - max_head_data_size >= candidate.candidate().head_data.0.len() as _, + schedule.max_head_data_size >= candidate.candidate().head_data.0.len() as _, Error::::HeadDataTooLarge, ); - let fees = candidate.candidate().fees; + let full_candidate = full_candidate(candidate.candidate())?; + let fees = full_candidate.commitments.fees; + + ensure!( + full_candidate.local_validation.balance >= full_candidate.commitments.fees, + Error::::CannotPayFees, + ); + T::ParachainCurrency::deduct(para_id, fees)?; - let mut candidate_hash = None; + let candidate_hash = candidate.candidate().hash(); let mut encoded_implicit = None; let mut encoded_explicit = None; @@ -702,18 +754,14 @@ impl Module { let (payload, sig) = match validity_attestation { ValidityAttestation::Implicit(sig) => { let payload = encoded_implicit.get_or_insert_with(|| localized_payload( - Statement::Candidate(candidate.candidate.clone()), + Statement::Candidate(candidate_hash), )); (payload, sig) } ValidityAttestation::Explicit(sig) => { - let hash = candidate_hash - .get_or_insert_with(|| candidate.candidate.hash()) - .clone(); - let payload = encoded_explicit.get_or_insert_with(|| localized_payload( - Statement::Valid(hash), + Statement::Valid(candidate_hash), )); (payload, sig) @@ -726,12 +774,12 @@ impl Module { ); } - para_block_hashes.push(candidate_hash.unwrap_or_else(|| candidate.candidate().hash())); - ensure!( candidate.validity_votes.len() == expected_votes_len, Error::::UntaggedVotes ); + + para_block_hashes.push(candidate_hash); } Ok(IncludedBlocks { @@ -833,7 +881,7 @@ mod tests { use primitives::{ parachain::{ CandidateReceipt, HeadData, ValidityAttestation, ValidatorId, Info as ParaInfo, - Scheduling, + Scheduling, LocalValidationData, CandidateCommitments, }, BlockNumber, }; @@ -981,7 +1029,7 @@ mod tests { type RewardRemainder = (); type CurrencyToVote = (); type Event = (); - type Currency = balances::Module; + type Currency = Balances; type Slash = (); type Reward = (); type SessionsPerEra = SessionsPerEra; @@ -1006,7 +1054,7 @@ mod tests { impl slots::Trait for Test { type Event = (); - type Currency = balances::Module; + type Currency = Balances; type Parachains = registrar::Module; type EndingPeriod = EndingPeriod; type LeasePeriod = LeasePeriod; @@ -1022,7 +1070,7 @@ mod tests { impl registrar::Trait for Test { type Event = (); type Origin = Origin; - type Currency = balances::Module; + type Currency = Balances; type ParathreadDeposit = ParathreadDeposit; type SwapAux = slots::Module; type QueueSize = QueueSize; @@ -1037,7 +1085,7 @@ mod tests { impl Trait for Test { type Origin = Origin; type Call = Call; - type ParachainCurrency = balances::Module; + type ParachainCurrency = Balances; type Randomness = RandomnessCollectiveFlip; type ActiveParachains = registrar::Module; type Registrar = registrar::Module; @@ -1046,6 +1094,7 @@ mod tests { } type Parachains = Module; + type Balances = balances::Module; type System = system::Module; type RandomnessCollectiveFlip = randomness_collective_flip::Module; type Registrar = registrar::Module; @@ -1125,6 +1174,38 @@ mod tests { ParachainsCall::set_heads(v) } + // creates a template candidate which pins to correct relay-chain state. + fn raw_candidate(para_id: ParaId) -> CandidateReceipt { + CandidateReceipt { + parachain_index: para_id, + relay_parent: System::parent_hash(), + head_data: Default::default(), + collator: Default::default(), + signature: Default::default(), + pov_block_hash: Default::default(), + global_validation: GlobalValidationSchedule { + max_code_size: ::MaxCodeSize::get(), + max_head_data_size: ::MaxHeadDataSize::get(), + }, + local_validation: LocalValidationData { + parent_head: HeadData(Parachains::parachain_head(¶_id).unwrap()), + balance: >::free_balance(para_id), + }, + commitments: CandidateCommitments::default(), + } + } + + // makes a blank attested candidate from a `CandidateReceipt`. + fn make_blank_attested(candidate: CandidateReceipt) -> AttestedCandidate { + let (candidate, _) = candidate.abridge(); + + AttestedCandidate { + validity_votes: vec![], + validator_indices: BitVec::new(), + candidate, + } + } + fn make_attestations(candidate: &mut AttestedCandidate) { let mut vote_implicit = false; let parent_hash = >::parent_hash(); @@ -1150,7 +1231,7 @@ mod tests { let key = extract_key(authorities[idx].clone()); let statement = if vote_implicit { - Statement::Candidate(candidate.candidate.clone()) + Statement::Candidate(candidate_hash.clone()) } else { Statement::Valid(candidate_hash.clone()) }; @@ -1176,23 +1257,12 @@ mod tests { id: u32, upward_messages: Vec<(ParachainDispatchOrigin, Vec)> ) -> AttestedCandidate { - AttestedCandidate { - validity_votes: vec![], - validator_indices: BitVec::new(), - candidate: CandidateReceipt { - parachain_index: id.into(), - collator: Default::default(), - signature: Default::default(), - head_data: HeadData(vec![1, 2, 3]), - parent_head: HeadData(vec![]), - fees: 0, - block_data_hash: Default::default(), - upward_messages: upward_messages.into_iter() - .map(|x| UpwardMessage { origin: x.0, data: x.1 }) - .collect(), - erasure_root: [1u8; 32].into(), - } - } + let mut raw_candidate = raw_candidate(id.into()); + raw_candidate.commitments.upward_messages = upward_messages.into_iter() + .map(|x| UpwardMessage { origin: x.0, data: x.1 }) + .collect(); + + make_blank_attested(raw_candidate) } fn init_block() { @@ -1586,23 +1656,7 @@ mod tests { new_test_ext(parachains.clone()).execute_with(|| { run_to_block(2); - let candidate = AttestedCandidate { - validity_votes: vec![], - validator_indices: BitVec::new(), - candidate: CandidateReceipt { - parachain_index: 0.into(), - collator: Default::default(), - signature: Default::default(), - head_data: HeadData(vec![1, 2, 3]), - parent_head: HeadData(vec![]), - fees: 0, - block_data_hash: Default::default(), - upward_messages: vec![], - erasure_root: [1u8; 32].into(), - }, - - }; - + let candidate = make_blank_attested(raw_candidate(0.into())); assert!(Parachains::dispatch(set_heads(vec![candidate]), Origin::NONE).is_err()); }) } @@ -1618,37 +1672,8 @@ mod tests { run_to_block(2); assert_eq!(Parachains::active_parachains().len(), 2); - let mut candidate_a = AttestedCandidate { - validity_votes: vec![], - validator_indices: BitVec::new(), - candidate: CandidateReceipt { - parachain_index: 0.into(), - collator: Default::default(), - signature: Default::default(), - head_data: HeadData(vec![1, 2, 3]), - parent_head: HeadData(vec![]), - fees: 0, - block_data_hash: Default::default(), - upward_messages: vec![], - erasure_root: [1u8; 32].into(), - } - }; - - let mut candidate_b = AttestedCandidate { - validity_votes: vec![], - validator_indices: BitVec::new(), - candidate: CandidateReceipt { - parachain_index: 1.into(), - collator: Default::default(), - signature: Default::default(), - head_data: HeadData(vec![2, 3, 4]), - parent_head: HeadData(vec![]), - fees: 0, - block_data_hash: Default::default(), - upward_messages: vec![], - erasure_root: [1u8; 32].into(), - } - }; + let mut candidate_a = make_blank_attested(raw_candidate(0.into())); + let mut candidate_b = make_blank_attested(raw_candidate(1.into())); make_attestations(&mut candidate_a); make_attestations(&mut candidate_b); @@ -1674,22 +1699,8 @@ mod tests { new_test_ext(parachains.clone()).execute_with(|| { run_to_block(2); - let mut candidate = AttestedCandidate { - validity_votes: vec![], - validator_indices: BitVec::new(), - candidate: CandidateReceipt { - parachain_index: 0.into(), - collator: Default::default(), - signature: Default::default(), - head_data: HeadData(vec![1, 2, 3]), - parent_head: HeadData(vec![]), - fees: 0, - block_data_hash: Default::default(), - upward_messages: vec![], - erasure_root: [1u8; 32].into(), - } - }; + let mut candidate = make_blank_attested(raw_candidate(0.into())); make_attestations(&mut candidate); let mut double_validity = candidate.clone(); @@ -1712,22 +1723,8 @@ mod tests { new_test_ext(parachains.clone()).execute_with(|| { run_to_block(2); - let mut candidate = AttestedCandidate { - validity_votes: vec![], - validator_indices: BitVec::new(), - candidate: CandidateReceipt { - parachain_index: 0.into(), - collator: Default::default(), - signature: Default::default(), - head_data: HeadData(vec![1, 2, 3]), - parent_head: HeadData(vec![]), - fees: 0, - block_data_hash: Default::default(), - upward_messages: vec![], - erasure_root: [1u8; 32].into(), - } - }; + let mut candidate = make_blank_attested(raw_candidate(0.into())); make_attestations(&mut candidate); // Change the last vote index to make it not corresponding to the assigned group. diff --git a/polkadot/runtime/common/src/registrar.rs b/polkadot/runtime/common/src/registrar.rs index 14930fca3d..227ec36098 100644 --- a/polkadot/runtime/common/src/registrar.rs +++ b/polkadot/runtime/common/src/registrar.rs @@ -651,7 +651,8 @@ mod tests { use primitives::{ parachain::{ ValidatorId, Info as ParaInfo, Scheduling, LOWEST_USER_ID, AttestedCandidate, - CandidateReceipt, HeadData, ValidityAttestation, Statement, Chain, CollatorPair, + CandidateReceipt, HeadData, ValidityAttestation, Statement, Chain, + CollatorPair, CandidateCommitments, GlobalValidationSchedule, LocalValidationData, }, Balance, BlockNumber, }; @@ -894,20 +895,33 @@ mod tests { LOWEST_USER_ID + i } - fn attest(id: ParaId, collator: &CollatorPair, head_data: &[u8], parent_head: &[u8], block_data: &[u8]) -> AttestedCandidate { - let block_data_hash = BlakeTwo256::hash(block_data); + fn attest(id: ParaId, collator: &CollatorPair, head_data: &[u8], block_data: &[u8]) -> AttestedCandidate { + let pov_block_hash = BlakeTwo256::hash(block_data); + let relay_parent = System::parent_hash(); let candidate = CandidateReceipt { parachain_index: id, - collator: collator.public(), - signature: block_data_hash.using_encoded(|d| collator.sign(d)), + relay_parent, head_data: HeadData(head_data.to_vec()), - parent_head: HeadData(parent_head.to_vec()), - fees: 0, - block_data_hash, - upward_messages: vec![], - erasure_root: [1; 32].into(), + collator: collator.public(), + signature: pov_block_hash.using_encoded(|d| collator.sign(d)), + pov_block_hash, + global_validation: GlobalValidationSchedule { + max_code_size: ::MaxCodeSize::get(), + max_head_data_size: ::MaxHeadDataSize::get(), + }, + local_validation: LocalValidationData { + balance: Balances::free_balance(&id.into_account()), + parent_head: HeadData(Parachains::parachain_head(&id).unwrap()), + }, + commitments: CandidateCommitments { + fees: 0, + upward_messages: vec![], + erasure_root: [1; 32].into(), + }, }; - let payload = (Statement::Valid(candidate.hash()), System::parent_hash()).encode(); + let (candidate, _) = candidate.abridge(); + let candidate_hash = candidate.hash(); + let payload = (Statement::Valid(candidate_hash), System::parent_hash()).encode(); let roster = Parachains::calculate_duty_roster().0.validator_duty; AttestedCandidate { candidate, @@ -1175,7 +1189,7 @@ mod tests { (user_id(0), Some((col.clone(), Retriable::WithRetries(0)))) ]); assert_ok!(Parachains::set_heads(Origin::NONE, vec![ - attest(user_id(0), &Sr25519Keyring::One.pair().into(), &[3; 3], &[3; 3], &[0; 0]) + attest(user_id(0), &Sr25519Keyring::One.pair().into(), &[3; 3], &[0; 0]) ])); run_to_block(6); diff --git a/polkadot/runtime/kusama/src/lib.rs b/polkadot/runtime/kusama/src/lib.rs index 707ce1ace4..e4b6640a43 100644 --- a/polkadot/runtime/kusama/src/lib.rs +++ b/polkadot/runtime/kusama/src/lib.rs @@ -25,7 +25,7 @@ use sp_core::u32_trait::{_1, _2, _3, _4, _5}; use codec::{Encode, Decode}; use primitives::{ AccountId, AccountIndex, Balance, BlockNumber, Hash, Nonce, Signature, Moment, - parachain::{self, ActiveParas, CandidateReceipt}, ValidityError, + parachain::{self, ActiveParas, AbridgedCandidateReceipt}, ValidityError, }; use runtime_common::{attestations, claims, parachains, registrar, slots, impls::{CurrencyToVoteHandler, TargetedFeeAdjustment, ToAuthor}, @@ -766,13 +766,18 @@ sp_api::impl_runtime_apis! { fn active_parachains() -> Vec<(parachain::Id, Option<(parachain::CollatorId, parachain::Retriable)>)> { Registrar::active_paras() } - fn parachain_status(id: parachain::Id) -> Option { - Parachains::parachain_status(&id) + fn global_validation_schedule() -> parachain::GlobalValidationSchedule { + Parachains::global_validation_schedule() + } + fn local_validation_data(id: parachain::Id) -> Option { + Parachains::local_validation_data(&id) } fn parachain_code(id: parachain::Id) -> Option> { Parachains::parachain_code(&id) } - fn get_heads(extrinsics: Vec<::Extrinsic>) -> Option> { + fn get_heads(extrinsics: Vec<::Extrinsic>) + -> Option> + { extrinsics .into_iter() .find_map(|ex| match UncheckedExtrinsic::decode(&mut ex.encode().as_slice()) { diff --git a/polkadot/runtime/polkadot/src/lib.rs b/polkadot/runtime/polkadot/src/lib.rs index 1ce31e88bd..3b38999b1c 100644 --- a/polkadot/runtime/polkadot/src/lib.rs +++ b/polkadot/runtime/polkadot/src/lib.rs @@ -31,7 +31,7 @@ use sp_core::u32_trait::{_1, _2, _3, _4, _5}; use codec::{Encode, Decode}; use primitives::{ AccountId, AccountIndex, Balance, BlockNumber, Hash, Nonce, Signature, Moment, - parachain::{self, ActiveParas, CandidateReceipt}, ValidityError, + parachain::{self, ActiveParas, AbridgedCandidateReceipt}, ValidityError, }; use sp_runtime::{ create_runtime_str, generic, impl_opaque_keys, @@ -688,13 +688,18 @@ sp_api::impl_runtime_apis! { fn active_parachains() -> Vec<(parachain::Id, Option<(parachain::CollatorId, parachain::Retriable)>)> { Registrar::active_paras() } - fn parachain_status(id: parachain::Id) -> Option { - Parachains::parachain_status(&id) + fn global_validation_schedule() -> parachain::GlobalValidationSchedule { + Parachains::global_validation_schedule() + } + fn local_validation_data(id: parachain::Id) -> Option { + Parachains::local_validation_data(&id) } fn parachain_code(id: parachain::Id) -> Option> { Parachains::parachain_code(&id) } - fn get_heads(extrinsics: Vec<::Extrinsic>) -> Option> { + fn get_heads(extrinsics: Vec<::Extrinsic>) + -> Option> + { extrinsics .into_iter() .find_map(|ex| match UncheckedExtrinsic::decode(&mut ex.encode().as_slice()) { diff --git a/polkadot/statement-table/src/lib.rs b/polkadot/statement-table/src/lib.rs index 397bfac0e5..16e8d9edfb 100644 --- a/polkadot/statement-table/src/lib.rs +++ b/polkadot/statement-table/src/lib.rs @@ -19,18 +19,28 @@ pub mod generic; pub use generic::Table; use primitives::parachain::{ - Id, CandidateReceipt, Statement as PrimitiveStatement, ValidatorSignature, ValidatorIndex, + Id, AbridgedCandidateReceipt, Statement as PrimitiveStatement, ValidatorSignature, ValidatorIndex, }; use primitives::Hash; /// Statements about candidates on the network. -pub type Statement = generic::Statement; +pub type Statement = generic::Statement; /// Signed statements about candidates. -pub type SignedStatement = generic::SignedStatement; +pub type SignedStatement = generic::SignedStatement< + AbridgedCandidateReceipt, + Hash, + ValidatorIndex, + ValidatorSignature, +>; /// Kinds of misbehavior, along with proof. -pub type Misbehavior = generic::Misbehavior; +pub type Misbehavior = generic::Misbehavior< + AbridgedCandidateReceipt, + Hash, + ValidatorIndex, + ValidatorSignature, +>; /// A summary of import of a statement. pub type Summary = generic::Summary; @@ -50,13 +60,13 @@ impl generic::Context for C { type Digest = Hash; type GroupId = Id; type Signature = ValidatorSignature; - type Candidate = CandidateReceipt; + type Candidate = AbridgedCandidateReceipt; - fn candidate_digest(candidate: &CandidateReceipt) -> Hash { + fn candidate_digest(candidate: &AbridgedCandidateReceipt) -> Hash { candidate.hash() } - fn candidate_group(candidate: &CandidateReceipt) -> Id { + fn candidate_group(candidate: &AbridgedCandidateReceipt) -> Id { candidate.parachain_index.clone() } @@ -69,12 +79,12 @@ impl generic::Context for C { } } -impl From for PrimitiveStatement { - fn from(s: Statement) -> PrimitiveStatement { - match s { +impl<'a> From<&'a Statement> for PrimitiveStatement { + fn from(s: &'a Statement) -> PrimitiveStatement { + match *s { generic::Statement::Valid(s) => PrimitiveStatement::Valid(s), generic::Statement::Invalid(s) => PrimitiveStatement::Invalid(s), - generic::Statement::Candidate(s) => PrimitiveStatement::Candidate(s), + generic::Statement::Candidate(ref s) => PrimitiveStatement::Candidate(s.hash()), } } } diff --git a/polkadot/test-parachains/adder/collator/src/main.rs b/polkadot/test-parachains/adder/collator/src/main.rs index 35e45b308d..dbf0de7b1b 100644 --- a/polkadot/test-parachains/adder/collator/src/main.rs +++ b/polkadot/test-parachains/adder/collator/src/main.rs @@ -24,7 +24,7 @@ use sp_core::Pair; use codec::{Encode, Decode}; use primitives::{ Hash, - parachain::{HeadData, BlockData, Id as ParaId, Status as ParachainStatus}, + parachain::{HeadData, BlockData, Id as ParaId, LocalValidationData}, }; use collator::{ InvalidHead, ParachainContext, Network, BuildParachainContext, load_spec, Configuration, @@ -60,10 +60,10 @@ impl ParachainContext for AdderContext { fn produce_candidate( &mut self, _relay_parent: Hash, - status: ParachainStatus, + local_validation: LocalValidationData, ) -> Self::ProduceCandidate { - let adder_head = match AdderHead::decode(&mut &status.head_data.0[..]) { + let adder_head = match AdderHead::decode(&mut &local_validation.parent_head.0[..]) { Ok(adder_head) => adder_head, Err(_) => return err(InvalidHead) }; diff --git a/polkadot/validation/src/block_production.rs b/polkadot/validation/src/block_production.rs index d548ff0659..4fd5127523 100644 --- a/polkadot/validation/src/block_production.rs +++ b/polkadot/validation/src/block_production.rs @@ -29,7 +29,7 @@ use sp_blockchain::HeaderBackend; use block_builder::BlockBuilderApi; use codec::Encode; use consensus::{Proposal, RecordProof}; -use polkadot_primitives::{Hash, Block, BlockId, BlockNumber, Header}; +use polkadot_primitives::{Hash, Block, BlockId, Header}; use polkadot_primitives::parachain::{ ParachainHost, AttestedCandidate, NEW_HEADS_IDENTIFIER, }; @@ -47,6 +47,9 @@ use crate::validation_service::ServiceHandle; use crate::dynamic_inclusion::DynamicInclusion; use crate::Error; +// block size limit. +pub(crate) const MAX_TRANSACTIONS_SIZE: usize = 4 * 1024 * 1024; + // Polkadot proposer factory. pub struct ProposerFactory { client: Arc, @@ -100,7 +103,6 @@ where parent_header: &Header, ) -> Self::CreateProposer { let parent_hash = parent_header.hash(); - let parent_number = parent_header.number; let parent_id = BlockId::hash(parent_hash); let client = self.client.clone(); @@ -114,9 +116,7 @@ where .and_then(move |tracker| future::ready(Ok(Proposer { client, tracker, - parent_hash, parent_id, - parent_number, transaction_pool, slot_duration, backend, @@ -129,9 +129,7 @@ where /// The Polkadot proposer logic. pub struct Proposer { client: Arc, - parent_hash: Hash, parent_id: BlockId, - parent_number: BlockNumber, tracker: crate::validation_service::ValidationInstanceHandle, transaction_pool: Arc, slot_duration: u64, @@ -172,8 +170,6 @@ impl consensus::Proposer for Proposer consensus::Proposer for Proposer u64 { /// Inner data of the create proposal. struct CreateProposalData { - parent_hash: Hash, - parent_number: BlockNumber, parent_id: BlockId, client: Arc, transaction_pool: Arc, table: Arc, - believed_minimum_timestamp: u64, inherent_data: Option, inherent_digests: DigestFor, deadline: Instant, @@ -298,7 +288,7 @@ impl CreateProposalData where let ready_iter = self.transaction_pool.ready(); for ready in ready_iter.take(MAX_TRANSACTIONS) { let encoded_size = ready.data().encode().len(); - if pending_size + encoded_size >= crate::evaluation::MAX_TRANSACTIONS_SIZE { + if pending_size + encoded_size >= MAX_TRANSACTIONS_SIZE { break; } if Instant::now() > self.deadline { @@ -339,16 +329,6 @@ impl CreateProposalData where .join(", ") ); - // TODO: full re-evaluation (https://github.com/paritytech/polkadot/issues/216) - let active_parachains = runtime_api.active_parachains(&self.parent_id)?; - assert!(crate::evaluation::evaluate_initial( - &new_block, - self.believed_minimum_timestamp, - &self.parent_hash, - self.parent_number, - &active_parachains[..], - ).is_ok()); - Ok(Proposal { block: new_block, storage_changes, proof }) } } diff --git a/polkadot/validation/src/collation.rs b/polkadot/validation/src/collation.rs index 6812dda586..b7cdc64461 100644 --- a/polkadot/validation/src/collation.rs +++ b/polkadot/validation/src/collation.rs @@ -22,22 +22,15 @@ use std::sync::Arc; use polkadot_primitives::{ - BlakeTwo256, Block, Hash, HashT, BlockId, Balance, + BlakeTwo256, Block, Hash, HashT, parachain::{ - CollatorId, CandidateReceipt, CollationInfo, - ParachainHost, Id as ParaId, Collation, FeeSchedule, ErasureChunk, - HeadData, PoVBlock, + CollatorId, ParachainHost, Id as ParaId, Collation, ErasureChunk, CollationInfo, }, }; use polkadot_erasure_coding as erasure; use sp_api::ProvideRuntimeApi; -use parachain::{ - wasm_executor::{self, ExecutionMode}, UpwardMessage, -}; -use trie::TrieConfiguration; use futures::prelude::*; use log::debug; -use parking_lot::Mutex; /// Encapsulates connections to collators and allows collation on any parachain. /// @@ -67,530 +60,55 @@ pub trait Collators: Clone { /// A future which resolves when a collation is available. pub async fn collation_fetch( parachain: ParaId, - relay_parent_hash: Hash, + relay_parent: Hash, collators: C, client: Arc

, max_block_data_size: Option, -) -> Result<(Collation, HeadData, Balance),C::Error> + n_validators: usize, +) -> Result<(CollationInfo, crate::pipeline::FullOutput), C::Error> where P::Api: ParachainHost, C: Collators + Unpin, P: ProvideRuntimeApi, ::Collation: Unpin, { - let relay_parent = BlockId::hash(relay_parent_hash); - loop { - let collation = collators.collate(parachain, relay_parent_hash) - .await?; - - let res = validate_collation( + let collation = collators.collate(parachain, relay_parent).await?; + let Collation { info, pov } = collation; + let res = crate::pipeline::full_output_validation_with_api( &*client, + &info, + &pov, &relay_parent, - &collation, max_block_data_size, + n_validators, ); match res { - Ok((parent_head, fees)) => { - return Ok((collation, parent_head, fees)) + Ok(full_output) => { + return Ok((info, full_output)) } Err(e) => { debug!("Failed to validate parachain due to API error: {}", e); // just continue if we got a bad collation or failed to validate - collators.note_bad_collator(collation.info.collator) + collators.note_bad_collator(info.collator) } } } } -// Errors that can occur when validating a parachain. -#[derive(Debug, derive_more::Display, derive_more::From)] -pub enum Error { - /// Client error - Client(sp_blockchain::Error), - /// Wasm validation error - WasmValidation(wasm_executor::Error), - /// Erasure-encoding error. - Erasure(erasure::Error), - /// Collated for inactive parachain - #[display(fmt = "Collated for inactive parachain: {:?}", _0)] - InactiveParachain(ParaId), - /// Unexpected egress root - #[display(fmt = "Got unexpected egress root to {:?}. (expected: {:?}, got {:?})", id, expected, got)] - EgressRootMismatch { id: ParaId, expected: Hash, got: Hash }, - /// Unexpected ingress root - #[display(fmt = "Got unexpected ingress root to {:?}. (expected: {:?}, got {:?})", id, expected, got)] - IngressRootMismatch { id: ParaId, expected: Hash, got: Hash }, - /// Ingress from wrong chain - #[display(fmt = "Got ingress from wrong chain. (expected: {:?}, got {:?})", expected, got)] - IngressChainMismatch { expected: ParaId, got: ParaId }, - /// Ingress canonicality mismatch - #[display(fmt = "Got data for {} roots, expected {}", expected, got)] - IngressCanonicalityMismatch { expected: usize, got: usize }, - /// Missing or extra egress root - #[display(fmt = "Missing or extra egress root. (expected: {:?}, got {:?})", expected, got)] - MissingEgressRoot { expected: Option, got: Option, }, - /// Parachain validation produced wrong head data - #[display(fmt = "Parachain validation produced wrong head data (expected: {:?}, got {:?})", expected, got)] - WrongHeadData { expected: Vec, got: Vec }, - /// Block data is too big - #[display(fmt = "Block data is too big (maximum allowed size: {}, actual size: {})", size, max_size)] - BlockDataTooBig { size: u64, max_size: u64 }, - /// Parachain validation produced wrong relay-chain messages - #[display(fmt = "Parachain validation produced wrong relay-chain messages (expected: {:?}, got {:?})", expected, got)] - UpwardMessagesInvalid { expected: Vec, got: Vec }, - /// Parachain validation produced wrong fees to charge to parachain. - #[display(fmt = "Parachain validation produced wrong relay-chain fees (expected: {:?}, got {:?})", expected, got)] - FeesChargedInvalid { expected: Balance, got: Balance }, - /// Candidate block has an erasure-encoded root that mismatches the actual - /// erasure-encoded root of block data and extrinsics. - #[display(fmt = "Got unexpected erasure root (expected: {:?}, got {:?})", expected, got)] - ErasureRootMismatch { expected: Hash, got: Hash }, - /// Candidate block collation info doesn't match candidate receipt. - #[display(fmt = "Got receipt mismatch for candidate {:?}", candidate)] - CandidateReceiptMismatch { candidate: Hash }, - /// The parent header given in the candidate did not match current relay-chain - /// state. - #[display(fmt = "Got unexpected parachain parent.")] - ParentMismatch { expected: HeadData, got: HeadData }, -} - -impl std::error::Error for Error { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match self { - Error::Client(ref err) => Some(err), - Error::WasmValidation(ref err) => Some(err), - _ => None, - } - } -} - -/// Compute a trie root for a set of messages, given the raw message data. -pub fn message_queue_root>(messages: I) -> Hash - where A: AsRef<[u8]> -{ - trie::trie_types::Layout::::ordered_trie_root(messages) -} - -struct ExternalitiesInner { - upward: Vec, - fees_charged: Balance, - free_balance: Balance, - fee_schedule: FeeSchedule, -} - -impl wasm_executor::Externalities for ExternalitiesInner { - fn post_upward_message(&mut self, message: UpwardMessage) -> Result<(), String> { - self.apply_message_fee(message.data.len())?; - - self.upward.push(message); - - Ok(()) - } -} - -impl ExternalitiesInner { - fn new(free_balance: Balance, fee_schedule: FeeSchedule) -> Self { - Self { - free_balance, - fee_schedule, - fees_charged: 0, - upward: Vec::new(), - } - } - - fn apply_message_fee(&mut self, message_len: usize) -> Result<(), String> { - let fee = self.fee_schedule.compute_fee(message_len); - let new_fees_charged = self.fees_charged.saturating_add(fee); - if new_fees_charged > self.free_balance { - Err("could not cover fee.".into()) - } else { - self.fees_charged = new_fees_charged; - Ok(()) - } - } - - // Performs final checks of validity, producing the outgoing message data. - fn final_checks( - &mut self, - upward_messages: &[UpwardMessage], - fees_charged: Option, - ) -> Result { - if self.upward != upward_messages { - return Err(Error::UpwardMessagesInvalid { - expected: upward_messages.to_vec(), - got: self.upward.clone(), - }); - } - - if let Some(fees_charged) = fees_charged { - if self.fees_charged != fees_charged { - return Err(Error::FeesChargedInvalid { - expected: fees_charged.clone(), - got: self.fees_charged.clone(), - }); - } - } - - - Ok(self.fees_charged) - } -} - -#[derive(Clone)] -struct Externalities(Arc>); - -impl Externalities { - fn new(free_balance: Balance, fee_schedule: FeeSchedule) -> Self { - Self(Arc::new(Mutex::new( - ExternalitiesInner::new(free_balance, fee_schedule) - ))) - } -} - -impl wasm_executor::Externalities for Externalities { - fn post_upward_message(&mut self, message: UpwardMessage) -> Result<(), String> { - self.0.lock().post_upward_message(message) - } -} - /// Validate an erasure chunk against an expected root. pub fn validate_chunk( root: &Hash, chunk: &ErasureChunk, -) -> Result<(), Error> { - let expected = erasure::branch_hash(root, &chunk.proof, chunk.index as usize)?; +) -> Result<(), ()> { + let expected = erasure::branch_hash(root, &chunk.proof, chunk.index as usize).map_err(|_| ())?; let got = BlakeTwo256::hash(&chunk.chunk); if expected != got { - return Err(Error::ErasureRootMismatch { - expected, - got, - }) + return Err(()) } Ok(()) } - -// A utility function that implements most of the collation validation logic. -// -// Reused by `validate_collation` and `validate_receipt`. -// Returns outgoing messages, parent nead data, and fees charged for later reuse. -fn do_validation

( - client: &P, - relay_parent: &BlockId, - pov_block: &PoVBlock, - para_id: ParaId, - max_block_data_size: Option, - fees_charged: Option, - head_data: &HeadData, - upward_messages: &Vec, -) -> Result<(HeadData, Balance), Error> where - P: ProvideRuntimeApi, - P::Api: ParachainHost, -{ - use parachain::ValidationParams; - - if let Some(max_size) = max_block_data_size { - let block_data_size = pov_block.block_data.0.len() as u64; - if block_data_size > max_size { - return Err(Error::BlockDataTooBig { size: block_data_size, max_size }); - } - } - - let api = client.runtime_api(); - let validation_code = api.parachain_code(relay_parent, para_id)? - .ok_or_else(|| Error::InactiveParachain(para_id))?; - - let chain_status = api.parachain_status(relay_parent, para_id)? - .ok_or_else(|| Error::InactiveParachain(para_id))?; - - - let params = ValidationParams { - parent_head: chain_status.head_data.0.clone(), - block_data: pov_block.block_data.0.clone(), - }; - - let ext = Externalities::new(chain_status.balance, chain_status.fee_schedule); - - match wasm_executor::validate_candidate( - &validation_code, - params, - ext.clone(), - ExecutionMode::Remote, - ) { - Ok(result) => { - if result.head_data == head_data.0 { - let fees = ext.0.lock().final_checks( - upward_messages, - fees_charged - )?; - - Ok((chain_status.head_data, fees)) - } else { - Err(Error::WrongHeadData { - expected: head_data.0.clone(), - got: result.head_data - }) - } - } - Err(e) => Err(e.into()) - } -} - -/// Produce a `CandidateReceipt` and erasure encoding chunks with a given collation. -/// -/// To produce a `CandidateReceipt` among other things the root of erasure encoding of -/// the block data and messages needs to be known. To avoid redundant re-computations -/// of erasure encoding this method creates an encoding and produces a candidate with -/// encoding's root returning both for re-use. -pub fn produce_receipt_and_chunks( - n_validators: usize, - parent_head: HeadData, - pov: &PoVBlock, - fees: Balance, - info: &CollationInfo, -) -> Result<(CandidateReceipt, Vec), Error> -{ - let erasure_chunks = erasure::obtain_chunks( - n_validators, - &pov.block_data, - )?; - - let branches = erasure::branches(erasure_chunks.as_ref()); - let erasure_root = branches.root(); - - let chunks: Vec<_> = erasure_chunks - .iter() - .zip(branches.map(|(proof, _)| proof)) - .enumerate() - .map(|(index, (chunk, proof))| ErasureChunk { - // branches borrows the original chunks, but this clone could probably be dodged. - chunk: chunk.clone(), - index: index as u32, - proof, - }) - .collect(); - - let receipt = CandidateReceipt { - parachain_index: info.parachain_index, - collator: info.collator.clone(), - signature: info.signature.clone(), - head_data: info.head_data.clone(), - parent_head, - fees, - block_data_hash: info.block_data_hash.clone(), - upward_messages: info.upward_messages.clone(), - erasure_root, - }; - - Ok((receipt, chunks)) -} - -/// Check if a given candidate receipt is valid with a given collation. -/// -/// This assumes that basic validity checks have been done: -/// - Block data hash is the same as linked in collation info and a receipt. -pub fn validate_receipt

( - client: &P, - relay_parent: &BlockId, - pov_block: &PoVBlock, - receipt: &CandidateReceipt, - max_block_data_size: Option, -) -> Result, Error> where - P: ProvideRuntimeApi, - P::Api: ParachainHost, -{ - let (parent_head, _fees) = do_validation( - client, - relay_parent, - pov_block, - receipt.parachain_index, - max_block_data_size, - Some(receipt.fees), - &receipt.head_data, - &receipt.upward_messages, - )?; - - if parent_head != receipt.parent_head { - return Err(Error::ParentMismatch { - expected: receipt.parent_head.clone(), - got: parent_head, - }); - } - - let api = client.runtime_api(); - let validators = api.validators(&relay_parent)?; - let n_validators = validators.len(); - - let (validated_receipt, chunks) = produce_receipt_and_chunks( - n_validators, - parent_head, - pov_block, - receipt.fees, - &receipt.clone().into(), - )?; - - if validated_receipt.erasure_root != receipt.erasure_root { - return Err(Error::ErasureRootMismatch { - expected: validated_receipt.erasure_root, - got: receipt.erasure_root, - }); - } - - Ok(chunks) -} - -/// Check whether a given collation is valid. Returns `Ok` on success, error otherwise. -/// Returns outgoing messages, parent head-data, and fees. -/// -/// This assumes that basic validity checks have been done: -/// - Block data hash is the same as linked in collation info. -pub fn validate_collation

( - client: &P, - relay_parent: &BlockId, - collation: &Collation, - max_block_data_size: Option, -) -> Result<(HeadData, Balance), Error> where - P: ProvideRuntimeApi, - P::Api: ParachainHost, -{ - let para_id = collation.info.parachain_index; - - debug!("Validating collation for parachain {} at relay parent: {}", para_id, relay_parent); - - do_validation( - client, - relay_parent, - &collation.pov, - para_id, - max_block_data_size, - None, - &collation.info.head_data, - &collation.info.upward_messages, - ) -} - -#[cfg(test)] -mod tests { - use super::*; - use parachain::wasm_executor::Externalities as ExternalitiesTrait; - use parachain::ParachainDispatchOrigin; - use polkadot_primitives::parachain::{CandidateReceipt, HeadData}; - - #[test] - fn ext_checks_upward_messages() { - let ext = || ExternalitiesInner { - upward: vec![ - UpwardMessage{ data: vec![42], origin: ParachainDispatchOrigin::Parachain }, - ], - fees_charged: 0, - free_balance: 1_000_000, - fee_schedule: FeeSchedule { - base: 1000, - per_byte: 10, - }, - }; - let receipt = CandidateReceipt { - parachain_index: 5.into(), - collator: Default::default(), - signature: Default::default(), - head_data: HeadData(Vec::new()), - parent_head: HeadData(Vec::new()), - fees: 0, - block_data_hash: Default::default(), - upward_messages: vec![ - UpwardMessage{ data: vec![42], origin: ParachainDispatchOrigin::Signed }, - UpwardMessage{ data: vec![69], origin: ParachainDispatchOrigin::Parachain }, - ], - erasure_root: [1u8; 32].into(), - }; - assert!(ext().final_checks( - &receipt.upward_messages, - Some(receipt.fees), - ).is_err()); - let receipt = CandidateReceipt { - parachain_index: 5.into(), - collator: Default::default(), - signature: Default::default(), - head_data: HeadData(Vec::new()), - parent_head: HeadData(Vec::new()), - fees: 0, - block_data_hash: Default::default(), - upward_messages: vec![ - UpwardMessage{ data: vec![42], origin: ParachainDispatchOrigin::Signed }, - ], - erasure_root: [1u8; 32].into(), - }; - assert!(ext().final_checks( - &receipt.upward_messages, - Some(receipt.fees), - ).is_err()); - let receipt = CandidateReceipt { - parachain_index: 5.into(), - collator: Default::default(), - signature: Default::default(), - head_data: HeadData(Vec::new()), - parent_head: HeadData(Vec::new()), - fees: 0, - block_data_hash: Default::default(), - upward_messages: vec![ - UpwardMessage{ data: vec![69], origin: ParachainDispatchOrigin::Parachain }, - ], - erasure_root: [1u8; 32].into(), - }; - assert!(ext().final_checks( - &receipt.upward_messages, - Some(receipt.fees), - ).is_err()); - let receipt = CandidateReceipt { - parachain_index: 5.into(), - collator: Default::default(), - signature: Default::default(), - head_data: HeadData(Vec::new()), - parent_head: HeadData(Vec::new()), - fees: 0, - block_data_hash: Default::default(), - upward_messages: vec![ - UpwardMessage{ data: vec![42], origin: ParachainDispatchOrigin::Parachain }, - ], - erasure_root: [1u8; 32].into(), - }; - assert!(ext().final_checks( - &receipt.upward_messages, - Some(receipt.fees), - ).is_ok()); - } - - #[test] - fn ext_checks_fees_and_updates_correctly() { - let mut ext = ExternalitiesInner { - upward: vec![ - UpwardMessage{ data: vec![42], origin: ParachainDispatchOrigin::Parachain }, - ], - fees_charged: 0, - free_balance: 1_000_000, - fee_schedule: FeeSchedule { - base: 1000, - per_byte: 10, - }, - }; - - ext.apply_message_fee(100).unwrap(); - assert_eq!(ext.fees_charged, 2000); - - ext.post_upward_message(UpwardMessage { - origin: ParachainDispatchOrigin::Signed, - data: vec![0u8; 100], - }).unwrap(); - assert_eq!(ext.fees_charged, 4000); - - ext.apply_message_fee((1_000_000 - 4000 - 1000) / 10).unwrap(); - assert_eq!(ext.fees_charged, 1_000_000); - - // cannot pay fee. - assert!(ext.apply_message_fee(1).is_err()); - } -} diff --git a/polkadot/validation/src/error.rs b/polkadot/validation/src/error.rs index 9ba4922eeb..d3b83176d9 100644 --- a/polkadot/validation/src/error.rs +++ b/polkadot/validation/src/error.rs @@ -16,7 +16,7 @@ //! Errors that can occur during the validation process. -use polkadot_primitives::parachain::ValidatorId; +use polkadot_primitives::{parachain::ValidatorId, Hash}; /// Error type for validation #[derive(Debug, derive_more::Display, derive_more::From)] @@ -25,6 +25,12 @@ pub enum Error { Client(sp_blockchain::Error), /// Consensus error Consensus(consensus::error::Error), + /// A wasm-validation error. + WasmValidation(parachain::wasm_executor::Error), + /// An I/O error. + Io(std::io::Error), + /// An error in the availability erasure-coding. + ErasureCoding(polkadot_erasure_coding::Error), #[display(fmt = "Invalid duty roster length: expected {}, got {}", expected, got)] InvalidDutyRosterLength { /// Expected roster length @@ -48,6 +54,30 @@ pub enum Error { DeadlineComputeFailure(std::time::Duration), #[display(fmt = "Validation service is down.")] ValidationServiceDown, + /// PoV-block in collation doesn't match provided. + #[display(fmt = "PoV hash mismatch. Expected {:?}, got {:?}", _0, _1)] + PoVHashMismatch(Hash, Hash), + /// Collator signature is invalid. + #[display(fmt = "Invalid collator signature on collation")] + InvalidCollatorSignature, + /// Head-data too large. + #[display(fmt = "Head data size of {} exceeded maximum of {}", _0, _1)] + HeadDataTooLarge(usize, usize), + /// Head-data mismatch after validation. + #[display(fmt = "Validation produced a different parachain header")] + HeadDataMismatch, + /// Relay parent of candidate not allowed. + #[display(fmt = "Relay parent {} of candidate not allowed in this context.", _0)] + DisallowedRelayParent(Hash), + /// Commitments in candidate match commitments produced by validation. + #[display(fmt = "Commitments in candidate receipt do not match those produced by validation")] + CommitmentsMismatch, + /// The parachain for which validation work is being done is not active. + #[display(fmt = "Parachain {:?} is not active", _0)] + InactiveParachain(polkadot_primitives::parachain::Id), + /// Block data is too big + #[display(fmt = "Block data is too big (maximum allowed size: {}, actual size: {})", size, max_size)] + BlockDataTooBig { size: u64, max_size: u64 }, Join(tokio::task::JoinError) } @@ -56,6 +86,9 @@ impl std::error::Error for Error { match self { Error::Client(ref err) => Some(err), Error::Consensus(ref err) => Some(err), + Error::WasmValidation(ref err) => Some(err), + Error::ErasureCoding(ref err) => Some(err), + Error::Io(ref err) => Some(err), _ => None, } } diff --git a/polkadot/validation/src/evaluation.rs b/polkadot/validation/src/evaluation.rs deleted file mode 100644 index 76e7e9f8fb..0000000000 --- a/polkadot/validation/src/evaluation.rs +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2017-2020 Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! Polkadot block evaluation and evaluation errors. - -// block size limit. -pub(crate) const MAX_TRANSACTIONS_SIZE: usize = 4 * 1024 * 1024; - -use codec::Encode; -use polkadot_primitives::{Block, Hash, BlockNumber}; -use polkadot_primitives::parachain::{Id as ParaId, CollatorId, Retriable}; - -/// Result type alias for block evaluation -pub type Result = std::result::Result; - -/// Error type for block evaluation -#[derive(Debug, derive_more::Display, derive_more::From)] -pub enum Error { - /// Client error - Client(sp_blockchain::Error), - /// Too many parachain candidates in proposal - #[display(fmt = "Proposal included {} candidates for {} parachains", expected, got)] - TooManyCandidates { expected: usize, got: usize }, - /// Proposal included unregistered parachain - #[display(fmt = "Proposal included unregistered parachain {:?}", _0)] - UnknownParachain(ParaId), - /// Proposal had wrong parent hash - #[display(fmt = "Proposal had wrong parent hash. Expected {:?}, got {:?}", expected, got)] - WrongParentHash { expected: Hash, got: Hash }, - /// Proposal had wrong number - #[display(fmt = "Proposal had wrong number. Expected {:?}, got {:?}", expected, got)] - WrongNumber { expected: BlockNumber, got: BlockNumber }, - /// Proposal exceeded the maximum size - #[display( - fmt = "Proposal exceeded the maximum size of {} by {} bytes.", - MAX_TRANSACTIONS_SIZE, MAX_TRANSACTIONS_SIZE.saturating_sub(*_0) - )] - ProposalTooLarge(usize), -} - -impl std::error::Error for Error { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match self { - Error::Client(ref err) => Some(err), - _ => None, - } - } -} - -/// Attempt to evaluate a substrate block as a polkadot block, returning error -/// upon any initial validity checks failing. -pub fn evaluate_initial( - proposal: &Block, - _now: u64, - parent_hash: &Hash, - parent_number: BlockNumber, - _active_parachains: &[(ParaId, Option<(CollatorId, Retriable)>)], -) -> Result<()> { - let transactions_size = proposal.extrinsics.iter().fold(0, |a, tx| { - a + Encode::encode(tx).len() - }); - - if transactions_size > MAX_TRANSACTIONS_SIZE { - return Err(Error::ProposalTooLarge(transactions_size)) - } - - if proposal.header.parent_hash != *parent_hash { - return Err(Error::WrongParentHash { - expected: *parent_hash, - got: proposal.header.parent_hash - }); - } - - if proposal.header.number != parent_number + 1 { - return Err(Error::WrongNumber { - expected: parent_number + 1, - got: proposal.header.number - }); - } - - Ok(()) -} diff --git a/polkadot/validation/src/lib.rs b/polkadot/validation/src/lib.rs index 82041eb32d..07d4bd905c 100644 --- a/polkadot/validation/src/lib.rs +++ b/polkadot/validation/src/lib.rs @@ -33,13 +33,12 @@ use std::{ collections::{HashMap, HashSet}, sync::Arc, }; - use codec::Encode; use polkadot_primitives::Hash; use polkadot_primitives::parachain::{ - Id as ParaId, Chain, DutyRoster, CandidateReceipt, + Id as ParaId, Chain, DutyRoster, AbridgedCandidateReceipt, Statement as PrimitiveStatement, - Collation, PoVBlock, ErasureChunk, ValidatorSignature, ValidatorIndex, + PoVBlock, ErasureChunk, ValidatorSignature, ValidatorIndex, ValidatorPair, ValidatorId, }; use primitives::Pair; @@ -47,10 +46,7 @@ use primitives::Pair; use futures::prelude::*; pub use self::block_production::ProposerFactory; -pub use self::collation::{ - validate_collation, message_queue_root, Collators, - produce_receipt_and_chunks, -}; +pub use self::collation::Collators; pub use self::error::Error; pub use self::shared_table::{ SharedTable, ParachainWork, PrimedParachainWork, Validated, Statement, SignedStatement, @@ -62,13 +58,13 @@ pub use self::validation_service::{ServiceHandle, ServiceBuilder}; pub use parachain::wasm_executor::{run_worker as run_validation_worker}; mod dynamic_inclusion; -mod evaluation; mod error; mod shared_table; -pub mod collation; -pub mod validation_service; pub mod block_production; +pub mod collation; +pub mod pipeline; +pub mod validation_service; /// A handle to a statement table router. /// @@ -83,12 +79,11 @@ pub trait TableRouter: Clone { /// Future that resolves when candidate data is fetched. type FetchValidationProof: Future>; - /// Call with local candidate data. This will make the data available on the network, - /// and sign, import, and broadcast a statement about the candidate. + /// Call with local candidate data. This will sign, import, and broadcast a statement about the candidate. fn local_collation( &self, - collation: Collation, - receipt: CandidateReceipt, + receipt: AbridgedCandidateReceipt, + pov_block: PoVBlock, chunks: (ValidatorIndex, &[ErasureChunk]), ) -> Self::SendLocalCollation; @@ -96,7 +91,7 @@ pub trait TableRouter: Clone { /// /// This future must conclude once all `Clone`s of this `TableRouter` have /// been cleaned up. - fn fetch_pov_block(&self, candidate: &CandidateReceipt) -> Self::FetchValidationProof; + fn fetch_pov_block(&self, candidate: &AbridgedCandidateReceipt) -> Self::FetchValidationProof; } /// A long-lived network which can create parachain statement and BFT message routing processes on demand. @@ -141,10 +136,7 @@ pub struct GroupInfo { /// The actual message signed is the encoded statement concatenated with the /// parent hash. pub fn sign_table_statement(statement: &Statement, key: &ValidatorPair, parent_hash: &Hash) -> ValidatorSignature { - // we sign using the primitive statement type because that's what the runtime - // expects. These types probably encode the same way so this clone could be optimized - // out in the future. - let mut encoded = PrimitiveStatement::from(statement.clone()).encode(); + let mut encoded = PrimitiveStatement::from(statement).encode(); encoded.extend(parent_hash.as_ref()); key.sign(&encoded) @@ -159,7 +151,7 @@ pub fn check_statement( ) -> bool { use runtime_primitives::traits::AppVerify; - let mut encoded = PrimitiveStatement::from(statement.clone()).encode(); + let mut encoded = PrimitiveStatement::from(statement).encode(); encoded.extend(parent_hash.as_ref()); signature.verify(&encoded[..], &signer) @@ -211,7 +203,6 @@ pub fn make_group_info( }); Ok((map, local_duty)) - } #[cfg(test)] diff --git a/polkadot/validation/src/pipeline.rs b/polkadot/validation/src/pipeline.rs new file mode 100644 index 0000000000..0b696af1fb --- /dev/null +++ b/polkadot/validation/src/pipeline.rs @@ -0,0 +1,375 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! The pipeline of validation functions a parachain block must pass through before +//! it can be voted for. + +use std::sync::Arc; + +use codec::Encode; +use polkadot_erasure_coding as erasure; +use polkadot_primitives::parachain::{ + CollationInfo, PoVBlock, LocalValidationData, GlobalValidationSchedule, OmittedValidationData, + AvailableData, FeeSchedule, CandidateCommitments, ErasureChunk, HeadData, ParachainHost, + Id as ParaId, AbridgedCandidateReceipt, +}; +use polkadot_primitives::{Block, BlockId, Balance, Hash}; +use parachain::{ + wasm_executor::{self, ExecutionMode}, + UpwardMessage, ValidationParams, +}; +use runtime_primitives::traits::{BlakeTwo256, Hash as HashT}; +use sp_api::ProvideRuntimeApi; +use parking_lot::Mutex; +use crate::Error; + +/// Does basic checks of a collation. Provide the encoded PoV-block. +pub fn basic_checks( + collation: &CollationInfo, + expected_relay_parent: &Hash, + max_block_data_size: Option, + encoded_pov: &[u8], +) -> Result<(), Error> { + if &collation.relay_parent != expected_relay_parent { + return Err(Error::DisallowedRelayParent(collation.relay_parent)); + } + + if let Some(max_size) = max_block_data_size { + if encoded_pov.len() as u64 > max_size { + return Err(Error::BlockDataTooBig { size: encoded_pov.len() as _, max_size }); + } + } + + let hash = BlakeTwo256::hash(encoded_pov); + if hash != collation.pov_block_hash { + return Err(Error::PoVHashMismatch(collation.pov_block_hash, hash)); + } + + if let Err(()) = collation.check_signature() { + return Err(Error::InvalidCollatorSignature); + } + + Ok(()) +} + +struct ExternalitiesInner { + upward: Vec, + fees_charged: Balance, + free_balance: Balance, + fee_schedule: FeeSchedule, +} + +impl wasm_executor::Externalities for ExternalitiesInner { + fn post_upward_message(&mut self, message: UpwardMessage) -> Result<(), String> { + self.apply_message_fee(message.data.len())?; + + self.upward.push(message); + + Ok(()) + } +} + +impl ExternalitiesInner { + fn new(free_balance: Balance, fee_schedule: FeeSchedule) -> Self { + Self { + free_balance, + fee_schedule, + fees_charged: 0, + upward: Vec::new(), + } + } + + fn apply_message_fee(&mut self, message_len: usize) -> Result<(), String> { + let fee = self.fee_schedule.compute_fee(message_len); + let new_fees_charged = self.fees_charged.saturating_add(fee); + if new_fees_charged > self.free_balance { + Err("could not cover fee.".into()) + } else { + self.fees_charged = new_fees_charged; + Ok(()) + } + } + + // Returns the noted outputs of execution so far - upward messages and balances. + fn outputs(self) -> (Vec, Balance) { + (self.upward, self.fees_charged) + } +} + +#[derive(Clone)] +struct Externalities(Arc>); + +impl Externalities { + fn new(free_balance: Balance, fee_schedule: FeeSchedule) -> Self { + Self(Arc::new(Mutex::new( + ExternalitiesInner::new(free_balance, fee_schedule) + ))) + } +} + +impl wasm_executor::Externalities for Externalities { + fn post_upward_message(&mut self, message: UpwardMessage) -> Result<(), String> { + self.0.lock().post_upward_message(message) + } +} + +/// Data from a fully-outputted validation of a parachain candidate. This contains +/// all outputs and commitments of the validation as well as all additional data to make available. +pub struct FullOutput { + /// Data about the candidate to keep available in the network. + pub available_data: AvailableData, + /// Commitments issued alongside the candidate to be placed on-chain. + pub commitments: CandidateCommitments, + /// All erasure-chunks associated with the available data. Each validator + /// should keep their chunk (by index). Other chunks do not need to be + /// kept available long-term, but should be distributed to other validators. + pub erasure_chunks: Vec, +} + +impl FullOutput { + /// Check consistency of the outputs produced by the validation pipeline against + /// data contained within a candidate receipt. + pub fn check_consistency(&self, receipt: &AbridgedCandidateReceipt) -> Result<(), Error> { + if self.commitments != receipt.commitments { + Err(Error::CommitmentsMismatch) + } else { + Ok(()) + } + } +} + +/// The successful result of validating a collation. If the full commitments of the +/// validation are needed, call `full_output`. Otherwise, safely drop this value. +pub struct ValidatedCandidate<'a> { + pov_block: &'a PoVBlock, + global_validation: &'a GlobalValidationSchedule, + parent_head: &'a HeadData, + balance: Balance, + upward_messages: Vec, + fees: Balance, +} + +impl<'a> ValidatedCandidate<'a> { + /// Fully-compute the commitments and outputs of the candidate. Provide the number + /// of validators. This computes the erasure-coding. + pub fn full_output(self, n_validators: usize) -> Result { + let ValidatedCandidate { + pov_block, + global_validation, + parent_head, + balance, + upward_messages, + fees, + } = self; + + let omitted_validation = OmittedValidationData { + global_validation: global_validation.clone(), + local_validation: LocalValidationData { + parent_head: parent_head.clone(), + balance, + }, + }; + + let available_data = AvailableData { + pov_block: pov_block.clone(), + omitted_validation, + }; + + let erasure_chunks = erasure::obtain_chunks( + n_validators, + &available_data, + )?; + + let branches = erasure::branches(erasure_chunks.as_ref()); + let erasure_root = branches.root(); + + let chunks: Vec<_> = erasure_chunks + .iter() + .zip(branches.map(|(proof, _)| proof)) + .enumerate() + .map(|(index, (chunk, proof))| ErasureChunk { + // branches borrows the original chunks, but this clone could probably be dodged. + chunk: chunk.clone(), + index: index as u32, + proof, + }) + .collect(); + + let commitments = CandidateCommitments { + upward_messages, + fees, + erasure_root, + }; + + Ok(FullOutput { + available_data, + commitments, + erasure_chunks: chunks, + }) + } +} + +/// Does full checks of a collation, with provided PoV-block and contextual data. +pub fn validate<'a>( + collation: &'a CollationInfo, + pov_block: &'a PoVBlock, + local_validation: &'a LocalValidationData, + global_validation: &'a GlobalValidationSchedule, + validation_code: &[u8], +) -> Result, Error> { + if collation.head_data.0.len() > global_validation.max_head_data_size as _ { + return Err(Error::HeadDataTooLarge( + collation.head_data.0.len(), + global_validation.max_head_data_size as _, + )); + } + + let params = ValidationParams { + parent_head: collation.head_data.0.clone(), + block_data: pov_block.block_data.0.clone(), + }; + + // TODO: remove when ext does not do this. + let fee_schedule = FeeSchedule { + base: 0, + per_byte: 0, + }; + + let ext = Externalities::new(local_validation.balance, fee_schedule); + match wasm_executor::validate_candidate( + &validation_code, + params, + ext.clone(), + ExecutionMode::Remote, + ) { + Ok(result) => { + if result.head_data == collation.head_data.0 { + let (upward_messages, fees) = Arc::try_unwrap(ext.0) + .map_err(|_| "") + .expect("Wasm executor drops passed externalities on completion; \ + call has concluded; qed") + .into_inner() + .outputs(); + + Ok(ValidatedCandidate { + pov_block, + global_validation, + parent_head: &local_validation.parent_head, + balance: local_validation.balance, + upward_messages, + fees, + }) + } else { + Err(Error::HeadDataMismatch) + } + } + Err(e) => Err(e.into()), + } +} + +/// Extracts validation parameters from a Polkadot runtime API for a specific parachain. +pub fn validation_params

(api: &P, relay_parent: Hash, para_id: ParaId) + -> Result<(LocalValidationData, GlobalValidationSchedule, Vec), Error> +where + P: ProvideRuntimeApi, + P::Api: ParachainHost, +{ + let api = api.runtime_api(); + let relay_parent = BlockId::hash(relay_parent); + + // fetch all necessary data from runtime. + let local_validation = api.local_validation_data(&relay_parent, para_id)? + .ok_or_else(|| Error::InactiveParachain(para_id))?; + + let global_validation = api.global_validation_schedule(&relay_parent)?; + let validation_code = api.parachain_code(&relay_parent, para_id)? + .ok_or_else(|| Error::InactiveParachain(para_id))?; + + Ok((local_validation, global_validation, validation_code)) +} + +/// Does full-pipeline validation of a collation with provided contextual parameters. +pub fn full_output_validation_with_api

( + api: &P, + collation: &CollationInfo, + pov_block: &PoVBlock, + expected_relay_parent: &Hash, + max_block_data_size: Option, + n_validators: usize, +) -> Result where + P: ProvideRuntimeApi, + P::Api: ParachainHost, +{ + let para_id = collation.parachain_index; + let (local_validation, global_validation, validation_code) + = validation_params(&*api, collation.relay_parent, para_id)?; + + // put the parameters through the validation pipeline, producing + // erasure chunks. + let encoded_pov = pov_block.encode(); + basic_checks( + &collation, + &expected_relay_parent, + max_block_data_size, + &encoded_pov, + ) + .and_then(|()| validate( + &collation, + &pov_block, + &local_validation, + &global_validation, + &validation_code, + )) + .and_then(|validated| validated.full_output(n_validators)) +} + +#[cfg(test)] +mod tests { + use super::*; + use parachain::wasm_executor::Externalities as ExternalitiesTrait; + use parachain::ParachainDispatchOrigin; + + #[test] + fn ext_checks_fees_and_updates_correctly() { + let mut ext = ExternalitiesInner { + upward: vec![ + UpwardMessage{ data: vec![42], origin: ParachainDispatchOrigin::Parachain }, + ], + fees_charged: 0, + free_balance: 1_000_000, + fee_schedule: FeeSchedule { + base: 1000, + per_byte: 10, + }, + }; + + ext.apply_message_fee(100).unwrap(); + assert_eq!(ext.fees_charged, 2000); + + ext.post_upward_message(UpwardMessage { + origin: ParachainDispatchOrigin::Signed, + data: vec![0u8; 100], + }).unwrap(); + assert_eq!(ext.fees_charged, 4000); + + + ext.apply_message_fee((1_000_000 - 4000 - 1000) / 10).unwrap(); + assert_eq!(ext.fees_charged, 1_000_000); + + // cannot pay fee. + assert!(ext.apply_message_fee(1).is_err()); + } +} diff --git a/polkadot/validation/src/shared_table/mod.rs b/polkadot/validation/src/shared_table/mod.rs index 77ab0a78ac..5391093d97 100644 --- a/polkadot/validation/src/shared_table/mod.rs +++ b/polkadot/validation/src/shared_table/mod.rs @@ -22,10 +22,10 @@ use std::sync::Arc; use availability_store::{Store as AvailabilityStore}; use table::{self, Table, Context as TableContextTrait}; -use polkadot_primitives::{Block, BlockId, Hash}; +use polkadot_primitives::{Block, Hash}; use polkadot_primitives::parachain::{ - Id as ParaId, CandidateReceipt, ValidatorPair, ValidatorId, - AttestedCandidate, ParachainHost, PoVBlock, ValidatorIndex, ErasureChunk, + Id as ParaId, AbridgedCandidateReceipt, ValidatorPair, ValidatorId, + AttestedCandidate, ParachainHost, PoVBlock, ValidatorIndex, }; use parking_lot::Mutex; @@ -39,6 +39,9 @@ use self::includable::IncludabilitySender; use primitives::Pair; use sp_api::ProvideRuntimeApi; +use crate::pipeline::FullOutput; +use crate::Error; + mod includable; pub use table::{SignedStatement, Statement}; @@ -188,8 +191,8 @@ impl SharedTableInner { availability_store: self.availability_store.clone(), relay_parent: context.parent_hash.clone(), work, - local_index: local_index as usize, max_block_data_size, + n_validators: context.validators.len(), }) } @@ -232,7 +235,7 @@ impl Validated { /// Note that we've collated a candidate. /// outgoing message required. pub fn collated_local( - receipt: CandidateReceipt, + receipt: AbridgedCandidateReceipt, collation: PoVBlock, ) -> Self { Validated { @@ -253,9 +256,9 @@ impl Validated { pub struct ParachainWork { work: Work, relay_parent: Hash, - local_index: usize, availability_store: AvailabilityStore, max_block_data_size: Option, + n_validators: usize, } impl ParachainWork { @@ -264,48 +267,47 @@ impl ParachainWork { pub fn prime>(self, api: Arc

) -> PrimedParachainWork< Fetch, - impl Send + FnMut(&BlockId, &PoVBlock, &CandidateReceipt) -> Result + Unpin, + impl Send + FnMut(&PoVBlock, &AbridgedCandidateReceipt) + -> Result + Unpin, > where P: Send + Sync + 'static, P::Api: ParachainHost, { let max_block_data_size = self.max_block_data_size; - let local_index = self.local_index; + let n_validators = self.n_validators; + let expected_relay_parent = self.relay_parent; - let validate = move |id: &_, pov_block: &_, receipt: &_| { - let res = crate::collation::validate_receipt( + let validate = move |pov_block: &PoVBlock, candidate: &AbridgedCandidateReceipt| { + let collation_info = candidate.to_collation_info(); + let full_output = crate::pipeline::full_output_validation_with_api( &*api, - id, + &collation_info, pov_block, - receipt, + &expected_relay_parent, max_block_data_size, - ); + n_validators, + )?; - match res { - Ok(mut chunks) => { - Ok(chunks.swap_remove(local_index)) - } - Err(e) => { - debug!(target: "validation", "Encountered bad collation: {}", e); - Err(()) - } - } + full_output.check_consistency(candidate)?; + Ok(full_output) }; PrimedParachainWork { inner: self, validate } } /// Prime the parachain work with a custom validation function. + #[cfg(test)] pub fn prime_with(self, validate: F) -> PrimedParachainWork - where F: FnMut(&BlockId, &PoVBlock, &CandidateReceipt) -> Result + where F: FnMut(&PoVBlock, &AbridgedCandidateReceipt) + -> Result { PrimedParachainWork { inner: self, validate } } } struct Work { - candidate_receipt: CandidateReceipt, + candidate_receipt: AbridgedCandidateReceipt, fetch: Fetch } @@ -318,46 +320,62 @@ pub struct PrimedParachainWork { impl PrimedParachainWork where Fetch: Future> + Unpin, - F: FnMut(&BlockId, &PoVBlock, &CandidateReceipt) -> Result + Unpin, - Err: From<::std::io::Error>, + F: FnMut(&PoVBlock, &AbridgedCandidateReceipt) -> Result + Unpin, + Err: From, { - pub async fn validate(mut self) -> Result<(Validated, Option), Err> { - let candidate = &self.inner.work.candidate_receipt; + pub async fn validate(self) -> Result { + let candidate = self.inner.work.candidate_receipt; let pov_block = self.inner.work.fetch.await?; - let validation_res = (self.validate)( - &BlockId::hash(self.inner.relay_parent), - &pov_block, - &candidate, - ); + let mut validate = self.validate; + let relay_parent = self.inner.relay_parent; + // create a wrapper around the custom validation function that does + // some more general pre and post checks. + let mut validate = move |pov_block: &_, candidate: &AbridgedCandidateReceipt| { + if candidate.relay_parent != relay_parent { + return Err(Error::DisallowedRelayParent(candidate.relay_parent)); + } + + let full_output = validate(pov_block, candidate)?; + + if full_output.commitments != candidate.commitments { + return Err(Error::CommitmentsMismatch); + } + + Ok(full_output) + }; + + let validation_res = (validate)(&pov_block, &candidate); let candidate_hash = candidate.hash(); debug!(target: "validation", "Making validity statement about candidate {}: is_good? {:?}", candidate_hash, validation_res.is_ok()); match validation_res { - Err(()) => Ok(( - Validated { + Err(err) => { + debug!(target: "validation", "candidate is invalid: {}", err); + Ok(Validated { statement: GenericStatement::Invalid(candidate_hash), result: Validation::Invalid(pov_block), - }, - None, - )), - Ok(our_chunk) => { - self.inner.availability_store.add_erasure_chunk( - self.inner.relay_parent, - candidate.clone(), - our_chunk.clone(), + }) + } + Ok(full_output) => { + // make data and all erasure chunks available. The chunk + // must be fully available before we add the chunks. + self.inner.availability_store.make_available( + candidate_hash, + full_output.available_data, + ).await?; + self.inner.availability_store.add_erasure_chunks( + candidate, + full_output.erasure_chunks, ).await?; - Ok(( - Validated { - statement: GenericStatement::Valid(candidate_hash), - result: Validation::Valid(pov_block), - }, - Some(our_chunk), - )) + Ok(Validated { + statement: GenericStatement::Valid(candidate_hash), + result: Validation::Valid(pov_block), + }) } } } @@ -461,7 +479,8 @@ impl SharedTable { } /// Sign and import the result of candidate validation. Returns `None` if the table - /// was instantiated without a local key. + /// was instantiated without a local key. Otherwise, returns a copy of the signed + /// statement. pub fn import_validated(&self, validated: Validated) -> Option { @@ -485,7 +504,7 @@ impl SharedTable { /// /// Deadlocks if called recursively. pub fn with_candidate(&self, digest: &Hash, f: F) -> U - where F: FnOnce(Option<&CandidateReceipt>) -> U + where F: FnOnce(Option<&AbridgedCandidateReceipt>) -> U { let inner = self.inner.lock(); f(inner.table.get_candidate(digest)) @@ -570,8 +589,9 @@ impl SharedTable { mod tests { use super::*; use sp_keyring::Sr25519Keyring; - use primitives::crypto::UncheckedInto; - use polkadot_primitives::parachain::{BlockData, Collation, HeadData}; + use polkadot_primitives::parachain::{ + BlockData, ErasureChunk, AvailableData, + }; use polkadot_erasure_coding::{self as erasure}; use availability_store::ProvideGossipMessages; use futures::future; @@ -613,12 +633,12 @@ mod tests { fn local_collation( &self, - _collation: Collation, - _candidate: CandidateReceipt, + _candidate: AbridgedCandidateReceipt, + _pov_block: PoVBlock, _chunks: (ValidatorIndex, &[ErasureChunk]) ) -> Self::SendLocalCollation { future::ready(Ok(())) } - fn fetch_pov_block(&self, _candidate: &CandidateReceipt) -> Self::FetchValidationProof { + fn fetch_pov_block(&self, _candidate: &AbridgedCandidateReceipt) -> Self::FetchValidationProof { future::ok(pov_block_with_data(vec![1, 2, 3, 4, 5])) } } @@ -652,17 +672,9 @@ mod tests { None, ); - let candidate = CandidateReceipt { - parachain_index: para_id, - collator: [1; 32].unchecked_into(), - signature: Default::default(), - head_data: HeadData(vec![1, 2, 3, 4]), - parent_head: HeadData(vec![]), - fees: 1_000_000, - block_data_hash: [2; 32].into(), - upward_messages: Vec::new(), - erasure_root: [1u8; 32].into(), - }; + let mut candidate = AbridgedCandidateReceipt::default(); + candidate.parachain_index = para_id; + candidate.relay_parent = parent_hash; let candidate_statement = GenericStatement::Candidate(candidate); @@ -708,17 +720,9 @@ mod tests { None, ); - let candidate = CandidateReceipt { - parachain_index: para_id, - collator: [1; 32].unchecked_into(), - signature: Default::default(), - head_data: HeadData(vec![1, 2, 3, 4]), - parent_head: HeadData(vec![]), - fees: 1_000_000, - block_data_hash: [2; 32].into(), - upward_messages: Vec::new(), - erasure_root: [1u8; 32].into(), - }; + let mut candidate = AbridgedCandidateReceipt::default(); + candidate.parachain_index = para_id; + candidate.relay_parent = parent_hash; let candidate_statement = GenericStatement::Candidate(candidate); @@ -741,25 +745,18 @@ mod tests { let relay_parent = [0; 32].into(); let para_id = 5.into(); let pov_block = pov_block_with_data(vec![1, 2, 3]); - let block_data_hash = [2; 32].into(); + let pov_block_hash = pov_block.hash(); let local_index = 0; let n_validators = 2; - let candidate = CandidateReceipt { - parachain_index: para_id, - collator: [1; 32].unchecked_into(), - signature: Default::default(), - head_data: HeadData(vec![1, 2, 3, 4]), - parent_head: HeadData(vec![]), - fees: 1_000_000, - block_data_hash, - upward_messages: Vec::new(), - erasure_root: [1u8; 32].into(), - }; + let mut candidate = AbridgedCandidateReceipt::default(); + candidate.parachain_index = para_id; + candidate.relay_parent = relay_parent; + candidate.pov_block_hash = pov_block_hash; - let hash = candidate.hash(); + let candidate_hash = candidate.hash(); - store.add_validator_index_and_n_validators( + store.note_validator_index_and_n_validators( &relay_parent, local_index as u32, n_validators as u32, @@ -770,25 +767,37 @@ mod tests { candidate_receipt: candidate, fetch: future::ok(pov_block.clone()), }, - local_index, relay_parent, availability_store: store.clone(), max_block_data_size: None, + n_validators, }; - let validated = block_on(producer.prime_with(|_, _, _| Ok( - ErasureChunk { - chunk: vec![1, 2, 3], - index: local_index as u32, - proof: vec![], + for i in 0..n_validators { + assert!(store.get_erasure_chunk(&candidate_hash, i).is_none()); + } + + let validated = block_on(producer.prime_with(|_, _| Ok( + FullOutput { + available_data: AvailableData { + pov_block: pov_block.clone(), + omitted_validation: Default::default(), + }, + erasure_chunks: (0..n_validators).map(|i| ErasureChunk { + chunk: vec![1, 2, 3], + index: i as u32, + proof: vec![], + }).collect(), + commitments: Default::default(), } )).validate()).unwrap(); - assert_eq!(validated.0.pov_block(), &pov_block); - assert_eq!(validated.0.statement, GenericStatement::Valid(hash)); + assert_eq!(validated.pov_block(), &pov_block); + assert_eq!(validated.statement, GenericStatement::Valid(candidate_hash)); - assert!(store.get_erasure_chunk(&relay_parent, block_data_hash, local_index).is_some()); - assert!(store.get_erasure_chunk(&relay_parent, block_data_hash, local_index + 1).is_none()); + for i in 0..n_validators { + assert!(store.get_erasure_chunk(&candidate_hash, i).is_some()); + } } #[test] @@ -797,25 +806,25 @@ mod tests { let relay_parent = [0; 32].into(); let para_id = 5.into(); let pov_block = pov_block_with_data(vec![1, 2, 3]); - let block_data_hash = pov_block.block_data.hash(); + let pov_block_hash = pov_block.hash(); let local_index = 0; let n_validators = 2; - let candidate = CandidateReceipt { - parachain_index: para_id, - collator: [1; 32].unchecked_into(), - signature: Default::default(), - head_data: HeadData(vec![1, 2, 3, 4]), - parent_head: HeadData(vec![]), - fees: 1_000_000, - block_data_hash: [2; 32].into(), - upward_messages: Vec::new(), - erasure_root: [1u8; 32].into(), + let mut candidate = AbridgedCandidateReceipt::default(); + candidate.parachain_index = para_id; + candidate.pov_block_hash = pov_block_hash; + candidate.relay_parent = relay_parent; + + let candidate_hash = candidate.hash(); + + let available_data = AvailableData { + pov_block: pov_block.clone(), + omitted_validation: Default::default(), }; - let chunks = erasure::obtain_chunks(n_validators, &pov_block.block_data).unwrap(); + let chunks = erasure::obtain_chunks(n_validators, &available_data).unwrap(); - store.add_validator_index_and_n_validators( + store.note_validator_index_and_n_validators( &relay_parent, local_index as u32, n_validators as u32, @@ -826,26 +835,30 @@ mod tests { candidate_receipt: candidate, fetch: future::ok::<_, ::std::io::Error>(pov_block.clone()), }, - local_index, relay_parent, availability_store: store.clone(), max_block_data_size: None, + n_validators, }; - let validated = block_on(producer.prime_with(|_, _, _| Ok( - ErasureChunk { - chunk: chunks[local_index].clone(), - index: local_index as u32, - proof: vec![], + let validated = block_on(producer.prime_with(|_, _| Ok( + FullOutput { + available_data: AvailableData { + pov_block: pov_block.clone(), + omitted_validation: Default::default(), }, - )).validate()).unwrap(); + erasure_chunks: (0..n_validators).map(|i| ErasureChunk { + chunk: chunks[i].clone(), + index: i as u32, + proof: vec![], + }).collect(), + commitments: Default::default(), + } + )).validate()).unwrap(); - assert_eq!(validated.0.pov_block(), &pov_block); + assert_eq!(validated.pov_block(), &pov_block); - // This works since there are only two validators and one erasure chunk should be - // enough to reconstruct the block data. - assert_eq!(store.block_data(relay_parent, block_data_hash).unwrap(), pov_block.block_data); - // TODO: check that a message queue is included by root. + assert_eq!(store.execution_data(&candidate_hash).unwrap().pov_block, pov_block); } #[test] @@ -877,19 +890,11 @@ mod tests { None, ); - let candidate = CandidateReceipt { - parachain_index: para_id, - collator: [1; 32].unchecked_into(), - signature: Default::default(), - head_data: HeadData(vec![1, 2, 3, 4]), - parent_head: HeadData(vec![]), - fees: 1_000_000, - block_data_hash: [2; 32].into(), - upward_messages: Vec::new(), - erasure_root: [1u8; 32].into(), - }; + let mut candidate = AbridgedCandidateReceipt::default(); + candidate.parachain_index = para_id; + candidate.relay_parent = parent_hash; - let hash = candidate.hash(); + let candidate_hash = candidate.hash(); let candidate_statement = GenericStatement::Candidate(candidate); let signature = crate::sign_table_statement(&candidate_statement, &validity_other_key.into(), &parent_hash); @@ -904,7 +909,8 @@ mod tests { signed_statement.clone(), ).expect("should produce work"); - assert!(shared_table.inner.lock().validated.get(&hash).expect("validation has started").is_in_progress()); + assert!(shared_table.inner.lock().validated.get(&candidate_hash) + .expect("validation has started").is_in_progress()); let b = shared_table.import_remote_statement( &DummyRouter, @@ -943,25 +949,18 @@ mod tests { None, ); - let candidate = CandidateReceipt { - parachain_index: para_id, - collator: [1; 32].unchecked_into(), - signature: Default::default(), - head_data: HeadData(vec![1, 2, 3, 4]), - parent_head: HeadData(vec![]), - fees: 1_000_000, - block_data_hash: [2; 32].into(), - upward_messages: Vec::new(), - erasure_root: [1u8; 32].into(), - }; + let mut candidate = AbridgedCandidateReceipt::default(); + candidate.parachain_index = para_id; + candidate.relay_parent = parent_hash; - let hash = candidate.hash(); + let candidate_hash = candidate.hash(); let signed_statement = shared_table.import_validated(Validated::collated_local( candidate, pov_block, )).unwrap(); - assert!(shared_table.inner.lock().validated.get(&hash).expect("validation has started").is_done()); + assert!(shared_table.inner.lock().validated.get(&candidate_hash) + .expect("validation has started").is_done()); let a = shared_table.import_remote_statement( &DummyRouter, diff --git a/polkadot/validation/src/validation_service/mod.rs b/polkadot/validation/src/validation_service/mod.rs index 3c4f75f5c9..a83777239f 100644 --- a/polkadot/validation/src/validation_service/mod.rs +++ b/polkadot/validation/src/validation_service/mod.rs @@ -322,7 +322,7 @@ impl ParachainValidationInstances where // If we are a validator, we need to store our index in this round in availability store. // This will tell which erasure chunk we should store. if let Some(ref local_duty) = local_duty { - if let Err(e) = self.availability_store.add_validator_index_and_n_validators( + if let Err(e) = self.availability_store.note_validator_index_and_n_validators( &parent_hash, local_duty.index, validators.len() as u32, @@ -374,7 +374,7 @@ impl ParachainValidationInstances where validation_para: ParaId, build_router: N::BuildTableRouter, max_block_data_size: Option, - authorities_num: usize, + n_validators: usize, local_id: ValidatorIndex, ) { let (collators, client) = (self.collators.clone(), self.client.clone()); @@ -388,49 +388,54 @@ impl ParachainValidationInstances where collators, client.clone(), max_block_data_size, + n_validators, ); collation_work.then(move |result| match result { - Ok((collation, parent_head, fees_charged)) => { - match crate::collation::produce_receipt_and_chunks( - authorities_num, - parent_head, - &collation.pov, - fees_charged, - &collation.info, - ) { - Ok((receipt, chunks)) => { - // Apparently the `async move` block is the only way to convince - // the compiler that we are not moving values out of borrowed context. - let av_clone = availability_store.clone(); - let chunks_clone = chunks.clone(); - let receipt_clone = receipt.clone(); + Ok((collation_info, full_output)) => { + let crate::pipeline::FullOutput { + commitments, + erasure_chunks, + available_data, + } = full_output; - let res = async move { - if let Err(e) = av_clone.clone().add_erasure_chunks( - relay_parent.clone(), - receipt_clone, - chunks_clone, - ).await { - warn!(target: "validation", "Failed to add erasure chunks: {}", e); - } - } - .unit_error() - .then(move |_| { - router.local_collation( - collation, - receipt, - (local_id, &chunks), - ).map_err(|e| warn!(target: "validation", "Failed to send local collation: {:?}", e)) - }); + let receipt = collation_info.into_receipt(commitments); - res.boxed() + // Apparently the `async move` block is the only way to convince + // the compiler that we are not moving values out of borrowed context. + let av_clone = availability_store.clone(); + let receipt_clone = receipt.clone(); + let erasure_chunks_clone = erasure_chunks.clone(); + let pov_block = available_data.pov_block.clone(); + + let res = async move { + if let Err(e) = av_clone.make_available( + receipt_clone.hash(), + available_data, + ).await { + warn!( + target: "validation", + "Failed to make parachain block data available: {}", + e, + ); } - Err(e) => { - warn!(target: "validation", "Failed to produce a receipt: {:?}", e); - Box::pin(ready(Ok(()))) + if let Err(e) = av_clone.clone().add_erasure_chunks( + receipt_clone, + erasure_chunks_clone, + ).await { + warn!(target: "validation", "Failed to add erasure chunks: {}", e); } } + .unit_error() + .then(move |_| { + router.local_collation( + receipt, + pov_block, + (local_id, &erasure_chunks), + ).map_err(|e| warn!(target: "validation", "Failed to send local collation: {:?}", e)) + }); + + res.boxed() } Err(e) => { warn!(target: "validation", "Failed to collate candidate: {:?}", e);