Minimal parachain framework part 1 (#113)

* dynamic inclusion threshold calculator

* collators interface

* collation helpers

* initial proposal-creation future

* create proposer when asked to propose

* remove local_availability duty

* statement table tracks includable parachain count

* beginnings of timing future

* finish proposal logic

* remove stray println

* extract shared table to separate module

* change ordering

* includability tracking

* fix doc

* initial changes to parachains module

* initialise dummy block before API calls

* give polkadot control over round proposer based on random seed

* propose only after enough candidates

* flesh out parachains module a bit more

* set_heads

* actually introduce set_heads to runtime

* update block_builder to accept parachains

* split block validity errors from real errors in evaluation

* update WASM runtimes

* polkadot-api methods for parachains additions

* delay evaluation until candidates are ready

* comments

* fix dynamic inclusion with zero initial

* test for includability tracker

* wasm validation of parachain candidates

* move primitives to primitives crate

* remove runtime-std dependency from codec

* adjust doc

* polkadot-parachain-primitives

* kill legacy polkadot-validator crate

* basic-add test chain

* test for basic_add parachain

* move to test-chains dir

* use wasm-build

* new wasm directory layout

* reorganize a bit more

* Fix for rh-minimal-parachain (#141)

* Remove extern "C"

We already encountered such behavior (bug?) in pwasm-std, I believe.

* Fix `panic_fmt` signature by adding `_col`

Wrong `panic_fmt` signature can inhibit some optimizations in LTO mode.

* Add linker flags and use wasm-gc in build script

Pass --import-memory to LLD to emit wasm binary with imported memory.

Also use wasm-gc instead of wasm-build.

* Fix effective_max.

I'm not sure why it was the way it was actually.

* Recompile wasm.

* Fix indent

* more basic_add tests

* validate parachain WASM

* produce statements on receiving statements

* tests for reactive statement production

* fix build

* add OOM lang item to runtime-io

* use dynamic_inclusion when evaluating as well

* fix update_includable_count

* remove dead code

* grumbles

* actually defer round_proposer logic

* update wasm

* address a few more grumbles

* grumbles

* update WASM checkins

* remove dependency on tokio-timer
This commit is contained in:
Robert Habermeier
2018-05-25 16:16:01 +02:00
committed by GitHub
parent 24d7d38c62
commit 27aafb0a04
63 changed files with 2825 additions and 921 deletions
@@ -0,0 +1,137 @@
// Copyright 2017 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Implements a future which resolves when all of the candidates referenced are includable.
use std::collections::HashMap;
use futures::prelude::*;
use futures::sync::oneshot;
use polkadot_primitives::Hash;
/// Track includability of a set of candidates,
pub(super) fn track<I: IntoIterator<Item=(Hash, bool)>>(candidates: I) -> (IncludabilitySender, Includable) {
let (tx, rx) = oneshot::channel();
let tracking: HashMap<_, _> = candidates.into_iter().collect();
let includable_count = tracking.values().filter(|x| **x).count();
let mut sender = IncludabilitySender {
tracking,
includable_count,
sender: Some(tx),
};
sender.try_complete();
(
sender,
Includable(rx),
)
}
/// The sending end of the includability sender.
pub(super) struct IncludabilitySender {
tracking: HashMap<Hash, bool>,
includable_count: usize,
sender: Option<oneshot::Sender<()>>,
}
impl IncludabilitySender {
/// update the inner candidate. wakes up the task as necessary.
/// returns `Err(Canceled)` if the other end has hung up.
///
/// returns `true` when this is completed and should be destroyed.
pub fn update_candidate(&mut self, candidate: Hash, includable: bool) -> bool {
use std::collections::hash_map::Entry;
match self.tracking.entry(candidate) {
Entry::Vacant(_) => {}
Entry::Occupied(mut entry) => {
let old = entry.insert(includable);
if !old && includable {
self.includable_count += 1;
} else if old && !includable {
self.includable_count -= 1;
}
}
}
self.try_complete()
}
/// whether the sender is completed.
pub fn is_complete(&self) -> bool {
self.sender.is_none()
}
fn try_complete(&mut self) -> bool {
if self.includable_count == self.tracking.len() {
if let Some(sender) = self.sender.take() {
let _ = sender.send(());
}
true
} else {
false
}
}
}
/// Future that resolves when all the candidates within are includable.
pub struct Includable(oneshot::Receiver<()>);
impl Future for Includable {
type Item = ();
type Error = oneshot::Canceled;
fn poll(&mut self) -> Poll<(), oneshot::Canceled> {
self.0.poll()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn it_works() {
let hash1 = [1; 32].into();
let hash2 = [2; 32].into();
let hash3 = [3; 32].into();
let (mut sender, recv) = track([
(hash1, true),
(hash2, true),
(hash2, false), // overwrite should favor latter.
(hash3, true),
].iter().cloned());
assert!(!sender.is_complete());
// true -> false transition is possible and should be handled.
sender.update_candidate(hash1, false);
assert!(!sender.is_complete());
sender.update_candidate(hash2, true);
assert!(!sender.is_complete());
sender.update_candidate(hash1, true);
assert!(sender.is_complete());
recv.wait().unwrap();
}
}
@@ -0,0 +1,566 @@
// Copyright 2017 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// Polkadot is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
//! Parachain statement table meant to to shared with a message router
//! and a consensus proposer.
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use table::{self, Table, Context as TableContextTrait};
use table::generic::Statement as GenericStatement;
use collation::Collation;
use polkadot_primitives::Hash;
use polkadot_primitives::parachain::{Id as ParaId, BlockData, Extrinsic, CandidateReceipt};
use primitives::AuthorityId;
use parking_lot::Mutex;
use futures::{future, prelude::*};
use super::{GroupInfo, TableRouter};
use self::includable::IncludabilitySender;
mod includable;
pub use self::includable::Includable;
struct TableContext {
parent_hash: Hash,
key: Arc<::ed25519::Pair>,
groups: HashMap<ParaId, GroupInfo>,
}
impl table::Context for TableContext {
fn is_member_of(&self, authority: &AuthorityId, group: &ParaId) -> bool {
self.groups.get(group).map_or(false, |g| g.validity_guarantors.contains(authority))
}
fn is_availability_guarantor_of(&self, authority: &AuthorityId, group: &ParaId) -> bool {
self.groups.get(group).map_or(false, |g| g.availability_guarantors.contains(authority))
}
fn requisite_votes(&self, group: &ParaId) -> (usize, usize) {
self.groups.get(group).map_or(
(usize::max_value(), usize::max_value()),
|g| (g.needed_validity, g.needed_availability),
)
}
}
impl TableContext {
fn local_id(&self) -> AuthorityId {
self.key.public().0
}
fn sign_statement(&self, statement: table::Statement) -> table::SignedStatement {
let signature = ::sign_table_statement(&statement, &self.key, &self.parent_hash).into();
let local_id = self.key.public().0;
table::SignedStatement {
statement,
signature,
sender: local_id,
}
}
}
/// Source of statements
pub enum StatementSource {
/// Locally produced statement.
Local,
/// Received statement from remote source, with optional sender.
Remote(Option<AuthorityId>),
}
// A shared table object.
struct SharedTableInner {
table: Table<TableContext>,
proposed_digest: Option<Hash>,
checked_validity: HashSet<Hash>,
checked_availability: HashSet<Hash>,
trackers: Vec<IncludabilitySender>,
}
impl SharedTableInner {
// Import a single statement. Provide a handle to a table router and a function
// used to determine if a referenced candidate is valid.
fn import_statement<R: TableRouter, C: FnMut(Collation) -> bool>(
&mut self,
context: &TableContext,
router: &R,
statement: table::SignedStatement,
statement_source: StatementSource,
check_candidate: C,
) -> StatementProducer<
<R::FetchCandidate as IntoFuture>::Future,
<R::FetchExtrinsic as IntoFuture>::Future,
C,
> {
// this blank producer does nothing until we attach some futures
// and set a candidate digest.
let received_from = match statement_source {
StatementSource::Local => return Default::default(),
StatementSource::Remote(from) => from,
};
let summary = match self.table.import_statement(context, statement, received_from) {
Some(summary) => summary,
None => return Default::default(),
};
self.update_trackers(&summary.candidate, context);
let local_id = context.local_id();
let is_validity_member = context.is_member_of(&local_id, &summary.group_id);
let is_availability_member =
context.is_availability_guarantor_of(&local_id, &summary.group_id);
let digest = &summary.candidate;
// TODO: consider a strategy based on the number of candidate votes as well.
// only check validity if this wasn't locally proposed.
let checking_validity = is_validity_member
&& self.proposed_digest.as_ref().map_or(true, |d| d != digest)
&& self.checked_validity.insert(digest.clone());
let checking_availability = is_availability_member
&& self.checked_availability.insert(digest.clone());
let work = if checking_validity || checking_availability {
match self.table.get_candidate(&digest) {
None => None, // TODO: handle table inconsistency somehow?
Some(candidate) => {
let fetch_block_data =
router.fetch_block_data(candidate).into_future().fuse();
let fetch_extrinsic = if checking_availability {
Some(
router.fetch_extrinsic_data(candidate).into_future().fuse()
)
} else {
None
};
Some(Work {
candidate_receipt: candidate.clone(),
fetch_block_data,
fetch_extrinsic,
evaluate: checking_validity,
check_candidate,
})
}
}
} else {
None
};
StatementProducer {
produced_statements: Default::default(),
work,
}
}
fn update_trackers(&mut self, candidate: &Hash, context: &TableContext) {
let includable = self.table.candidate_includable(candidate, context);
for i in (0..self.trackers.len()).rev() {
if self.trackers[i].update_candidate(candidate.clone(), includable) {
self.trackers.swap_remove(i);
}
}
}
}
/// Produced statements about a specific candidate.
/// Both may be `None`.
#[derive(Default)]
pub struct ProducedStatements {
/// A statement about the validity of the candidate.
pub validity: Option<table::Statement>,
/// A statement about availability of data. If this is `Some`,
/// then `block_data` and `extrinsic` should be `Some` as well.
pub availability: Option<table::Statement>,
/// Block data to ensure availability of.
pub block_data: Option<BlockData>,
/// Extrinsic data to ensure availability of.
pub extrinsic: Option<Extrinsic>,
}
/// Future that produces statements about a specific candidate.
pub struct StatementProducer<D: Future, E: Future, C> {
produced_statements: ProducedStatements,
work: Option<Work<D, E, C>>,
}
struct Work<D: Future, E: Future, C> {
candidate_receipt: CandidateReceipt,
fetch_block_data: future::Fuse<D>,
fetch_extrinsic: Option<future::Fuse<E>>,
evaluate: bool,
check_candidate: C
}
impl<D: Future, E: Future, C> Default for StatementProducer<D, E, C> {
fn default() -> Self {
StatementProducer {
produced_statements: Default::default(),
work: None,
}
}
}
impl<D, E, C, Err> Future for StatementProducer<D, E, C>
where
D: Future<Item=BlockData,Error=Err>,
E: Future<Item=Extrinsic,Error=Err>,
C: FnMut(Collation) -> bool,
{
type Item = ProducedStatements;
type Error = Err;
fn poll(&mut self) -> Poll<ProducedStatements, Err> {
let work = match self.work {
Some(ref mut work) => work,
None => return Ok(Async::Ready(::std::mem::replace(&mut self.produced_statements, Default::default()))),
};
if let Async::Ready(block_data) = work.fetch_block_data.poll()? {
self.produced_statements.block_data = Some(block_data.clone());
if work.evaluate {
let is_good = (work.check_candidate)(Collation {
block_data,
receipt: work.candidate_receipt.clone(),
});
let hash = work.candidate_receipt.hash();
self.produced_statements.validity = Some(if is_good {
GenericStatement::Valid(hash)
} else {
GenericStatement::Invalid(hash)
});
}
}
if let Some(ref mut fetch_extrinsic) = work.fetch_extrinsic {
if let Async::Ready(extrinsic) = fetch_extrinsic.poll()? {
self.produced_statements.extrinsic = Some(extrinsic);
}
}
let done = self.produced_statements.block_data.is_some() && {
if work.evaluate {
true
} else if self.produced_statements.extrinsic.is_some() {
self.produced_statements.availability =
Some(GenericStatement::Available(work.candidate_receipt.hash()));
true
} else {
false
}
};
if done {
Ok(Async::Ready(::std::mem::replace(&mut self.produced_statements, Default::default())))
} else {
Ok(Async::NotReady)
}
}
}
/// A shared table object.
pub struct SharedTable {
context: Arc<TableContext>,
inner: Arc<Mutex<SharedTableInner>>,
}
impl Clone for SharedTable {
fn clone(&self) -> Self {
SharedTable {
context: self.context.clone(),
inner: self.inner.clone(),
}
}
}
impl SharedTable {
/// Create a new shared table.
///
/// Provide the key to sign with, and the parent hash of the relay chain
/// block being built.
pub fn new(groups: HashMap<ParaId, GroupInfo>, key: Arc<::ed25519::Pair>, parent_hash: Hash) -> Self {
SharedTable {
context: Arc::new(TableContext { groups, key, parent_hash }),
inner: Arc::new(Mutex::new(SharedTableInner {
table: Table::default(),
proposed_digest: None,
checked_validity: HashSet::new(),
checked_availability: HashSet::new(),
trackers: Vec::new(),
}))
}
}
/// Get group info.
pub fn group_info(&self) -> &HashMap<ParaId, GroupInfo> {
&self.context.groups
}
/// Import a single statement. Provide a handle to a table router
/// for dispatching any other requests which come up.
pub fn import_statement<R: TableRouter, C: FnMut(Collation) -> bool>(
&self,
router: &R,
statement: table::SignedStatement,
received_from: StatementSource,
check_candidate: C,
) -> StatementProducer<<R::FetchCandidate as IntoFuture>::Future, <R::FetchExtrinsic as IntoFuture>::Future, C> {
self.inner.lock().import_statement(&*self.context, router, statement, received_from, check_candidate)
}
/// Sign and import a local statement.
pub fn sign_and_import<R: TableRouter>(
&self,
router: &R,
statement: table::Statement,
) {
let proposed_digest = match statement {
GenericStatement::Candidate(ref c) => Some(c.hash()),
_ => None,
};
let signed_statement = self.context.sign_statement(statement);
let mut inner = self.inner.lock();
if proposed_digest.is_some() {
inner.proposed_digest = proposed_digest;
}
let producer = inner.import_statement(
&*self.context,
router,
signed_statement,
StatementSource::Local,
|_| true,
);
assert!(producer.work.is_none(), "local statement import never leads to additional work; qed");
}
/// Import many statements at once.
///
/// Provide an iterator yielding pairs of (statement, statement_source).
pub fn import_statements<R, I, C, U>(&self, router: &R, iterable: I) -> U
where
R: TableRouter,
I: IntoIterator<Item=(table::SignedStatement, StatementSource, C)>,
C: FnMut(Collation) -> bool,
U: ::std::iter::FromIterator<StatementProducer<
<R::FetchCandidate as IntoFuture>::Future,
<R::FetchExtrinsic as IntoFuture>::Future,
C,
>>,
{
let mut inner = self.inner.lock();
iterable.into_iter().map(move |(statement, statement_source, check_candidate)| {
inner.import_statement(&*self.context, router, statement, statement_source, check_candidate)
}).collect()
}
/// Execute a closure using a specific candidate.
///
/// Deadlocks if called recursively.
pub fn with_candidate<F, U>(&self, digest: &Hash, f: F) -> U
where F: FnOnce(Option<&CandidateReceipt>) -> U
{
let inner = self.inner.lock();
f(inner.table.get_candidate(digest))
}
/// Execute a closure using the current proposed set.
///
/// Deadlocks if called recursively.
pub fn with_proposal<F, U>(&self, f: F) -> U
where F: FnOnce(Vec<&CandidateReceipt>) -> U
{
let inner = self.inner.lock();
f(inner.table.proposed_candidates(&*self.context))
}
/// Get the number of parachains which have available candidates.
pub fn includable_count(&self) -> usize {
self.inner.lock().table.includable_count()
}
/// Get all witnessed misbehavior.
pub fn get_misbehavior(&self) -> HashMap<AuthorityId, table::Misbehavior> {
self.inner.lock().table.get_misbehavior().clone()
}
/// Fill a statement batch.
pub fn fill_batch<B: table::StatementBatch>(&self, batch: &mut B) {
self.inner.lock().table.fill_batch(batch);
}
/// Track includability of a given set of candidate hashes.
pub fn track_includability<I>(&self, iterable: I) -> Includable
where I: IntoIterator<Item=Hash>
{
let mut inner = self.inner.lock();
let (tx, rx) = includable::track(iterable.into_iter().map(|x| {
let includable = inner.table.candidate_includable(&x, &*self.context);
(x, includable)
}));
if !tx.is_complete() {
inner.trackers.push(tx);
}
rx
}
}
#[cfg(test)]
mod tests {
use super::*;
use substrate_keyring::Keyring;
#[derive(Clone)]
struct DummyRouter;
impl TableRouter for DummyRouter {
type Error = ();
type FetchCandidate = ::futures::future::Empty<BlockData,()>;
type FetchExtrinsic = ::futures::future::Empty<Extrinsic,()>;
/// Note local candidate data, making it available on the network to other validators.
fn local_candidate_data(&self, _hash: Hash, _block_data: BlockData, _extrinsic: Extrinsic) {
}
/// Fetch block data for a specific candidate.
fn fetch_block_data(&self, _candidate: &CandidateReceipt) -> Self::FetchCandidate {
::futures::future::empty()
}
/// Fetch extrinsic data for a specific candidate.
fn fetch_extrinsic_data(&self, _candidate: &CandidateReceipt) -> Self::FetchExtrinsic {
::futures::future::empty()
}
}
#[test]
fn statement_triggers_fetch_and_evaluate() {
let mut groups = HashMap::new();
let para_id = ParaId::from(1);
let local_id = Keyring::Alice.to_raw_public();
let local_key = Arc::new(Keyring::Alice.pair());
let validity_other = Keyring::Bob.to_raw_public();
let validity_other_key = Keyring::Bob.pair();
let parent_hash = Default::default();
groups.insert(para_id, GroupInfo {
validity_guarantors: [local_id, validity_other].iter().cloned().collect(),
availability_guarantors: Default::default(),
needed_validity: 2,
needed_availability: 0,
});
let shared_table = SharedTable::new(groups, local_key.clone(), parent_hash);
let candidate = CandidateReceipt {
parachain_index: para_id,
collator: [1; 32],
head_data: ::polkadot_primitives::parachain::HeadData(vec![1, 2, 3, 4]),
balance_uploads: Vec::new(),
egress_queue_roots: Vec::new(),
fees: 1_000_000,
};
let candidate_statement = GenericStatement::Candidate(candidate);
let signature = ::sign_table_statement(&candidate_statement, &validity_other_key, &parent_hash);
let signed_statement = ::table::generic::SignedStatement {
statement: candidate_statement,
signature: signature.into(),
sender: validity_other,
};
let producer = shared_table.import_statement(
&DummyRouter,
signed_statement,
StatementSource::Remote(None),
|_| true,
);
assert!(producer.work.is_some(), "candidate and local validity group are same");
assert!(producer.work.as_ref().unwrap().evaluate, "should evaluate validity");
}
#[test]
fn statement_triggers_fetch_and_availability() {
let mut groups = HashMap::new();
let para_id = ParaId::from(1);
let local_id = Keyring::Alice.to_raw_public();
let local_key = Arc::new(Keyring::Alice.pair());
let validity_other = Keyring::Bob.to_raw_public();
let validity_other_key = Keyring::Bob.pair();
let parent_hash = Default::default();
groups.insert(para_id, GroupInfo {
validity_guarantors: [validity_other].iter().cloned().collect(),
availability_guarantors: [local_id].iter().cloned().collect(),
needed_validity: 1,
needed_availability: 1,
});
let shared_table = SharedTable::new(groups, local_key.clone(), parent_hash);
let candidate = CandidateReceipt {
parachain_index: para_id,
collator: [1; 32],
head_data: ::polkadot_primitives::parachain::HeadData(vec![1, 2, 3, 4]),
balance_uploads: Vec::new(),
egress_queue_roots: Vec::new(),
fees: 1_000_000,
};
let candidate_statement = GenericStatement::Candidate(candidate);
let signature = ::sign_table_statement(&candidate_statement, &validity_other_key, &parent_hash);
let signed_statement = ::table::generic::SignedStatement {
statement: candidate_statement,
signature: signature.into(),
sender: validity_other,
};
let producer = shared_table.import_statement(
&DummyRouter,
signed_statement,
StatementSource::Remote(None),
|_| true,
);
assert!(producer.work.is_some(), "candidate and local availability group are same");
assert!(producer.work.as_ref().unwrap().fetch_extrinsic.is_some(), "should fetch extrinsic when guaranteeing availability");
assert!(!producer.work.as_ref().unwrap().evaluate, "should not evaluate validity");
}
}