update substrate reference (#244)

* port polkadot_runtime and polkadot_validation

* update storages build (#245)

* all tests pass

* rebuild wasm

polkadot/cli/src/lib.rs

@@ -86,7 +86,7 @@ pub fn run<I, T, W>(args: I, worker: W, version: cli::VersionInfo) -> error::Res
 {
 	cli::parse_and_execute::<service::Factory, NoCustom, NoCustom, _, _, _, _, _>(
 		load_spec, &version, "parity-polkadot", args, worker,
-		|worker, _custom_args, mut config| {
+		|worker, _cli_args, _custom_args, mut config| {
 			info!("{}", version.name);
 			info!("  version {}", config.full_version());
 			info!("  by {}, 2017-2019", version.author);

polkadot/network/src/gossip.rs

@@ -16,11 +16,13 @@
 
 //! Gossip messages and the message validator
 
+use substrate_network::PeerId;
 use substrate_network::consensus_gossip::{
 	self as network_gossip, ValidationResult as GossipValidationResult,
+	ValidatorContext,
 };
 use polkadot_validation::SignedStatement;
-use polkadot_primitives::{Hash, SessionKey};
+use polkadot_primitives::{Block, Hash, SessionKey};
 use codec::Decode;
 
 use std::collections::HashMap;
@@ -80,7 +82,9 @@ pub fn register_validator<O: KnownOracle + 'static>(
 	});
 
 	let gossip_side = validator.clone();
-	service.with_gossip(|gossip, _| gossip.register_validator(POLKADOT_ENGINE_ID, gossip_side));
+	service.with_gossip(|gossip, ctx|
+		gossip.register_validator(ctx, POLKADOT_ENGINE_ID, gossip_side)
+	);
 
 	RegisteredMessageValidator { inner: validator as _ }
 }
@@ -140,29 +144,34 @@ pub struct MessageValidator<O: ?Sized> {
 	oracle: O,
 }
 
-impl<O: KnownOracle + ?Sized> network_gossip::Validator<Hash> for MessageValidator<O> {
+impl<O: KnownOracle + ?Sized> network_gossip::Validator<Block> for MessageValidator<O> {
-	fn validate(&self, mut data: &[u8]) -> GossipValidationResult<Hash> {
+	fn validate(&self, context: &mut ValidatorContext<Block>, _sender: &PeerId, mut data: &[u8])
+		-> GossipValidationResult<Hash>
+	{
+		let orig_data = data;
 		match GossipMessage::decode(&mut data) {
 			Some(GossipMessage { relay_parent, statement }) => {
 				let live = self.live_session.read();
 				let topic = || ::router::attestation_topic(relay_parent.clone());
 				if let Some(validation) = live.get(&relay_parent) {
 					if validation.check_statement(&relay_parent, &statement) {
-						GossipValidationResult::Valid(topic())
+						// repropagate
+						let topic = topic();
+						context.broadcast_message(topic, orig_data.to_owned(), false);
+						GossipValidationResult::ProcessAndKeep(topic)
 					} else {
-						GossipValidationResult::Invalid
+						GossipValidationResult::Discard
 					}
 				} else {
 					match self.oracle.is_known(&relay_parent) {
-						None | Some(Known::Leaf) => GossipValidationResult::Future(topic()),
+						None | Some(Known::Leaf) => GossipValidationResult::ProcessAndKeep(topic()),
-						Some(Known::Old) => GossipValidationResult::Expired,
+						Some(Known::Old) | Some(Known::Bad) => GossipValidationResult::Discard,
-						Some(Known::Bad) => GossipValidationResult::Invalid,
 					}
 				}
 			}
 			None => {
 				debug!(target: "validation", "Error decoding gossip message");
-				GossipValidationResult::Invalid
+				GossipValidationResult::Discard
 			}
 		}
 	}

polkadot/network/src/lib.rs

@@ -73,9 +73,6 @@ use std::collections::{HashMap, HashSet};
 #[cfg(test)]
 mod tests;
 
-/// Polkadot protocol id.
-pub const DOT_PROTOCOL_ID: ::substrate_network::ProtocolId = *b"dot";
-
 type FullStatus = GenericFullStatus<Block>;
 
 /// Specialization of the network service for the polkadot protocol.
@@ -188,7 +185,7 @@ pub enum Message {
 fn send_polkadot_message(ctx: &mut Context<Block>, to: PeerId, message: Message) {
 	trace!(target: "p_net", "Sending polkadot message to {}: {:?}", to, message);
 	let encoded = message.encode();
-	ctx.send_message(to, generic_message::Message::ChainSpecific(encoded))
+	ctx.send_chain_specific(to, encoded)
 }
 
 /// Polkadot protocol attachment for substrate.

polkadot/network/src/router.rs

@@ -88,7 +88,7 @@ impl<P, E, N: NetworkService, T> Router<P, E, N, T> {
 		self.network().gossip_messages_for(self.attestation_topic)
 			.filter_map(|msg| {
 				debug!(target: "validation", "Processing statement for live validation session");
-				crate::gossip::GossipMessage::decode(&mut &msg[..])
+				crate::gossip::GossipMessage::decode(&mut &msg.message[..])
 			})
 			.map(|msg| msg.statement)
 	}

polkadot/network/src/tests/mod.rs

@@ -29,8 +29,8 @@ use substrate_primitives::crypto::UncheckedInto;
 use codec::Encode;
 use substrate_network::{
 	Severity, PeerId, PeerInfo, ClientHandle, Context, config::Roles,
-	message::Message as SubstrateMessage, specialization::NetworkSpecialization,
+	message::{BlockRequest, generic::ConsensusMessage},
-	generic_message::Message as GenericMessage
+	specialization::NetworkSpecialization, generic_message::Message as GenericMessage
 };
 
 use futures::Future;
@@ -41,7 +41,7 @@ mod validation;
 struct TestContext {
 	disabled: Vec<PeerId>,
 	disconnected: Vec<PeerId>,
-	messages: Vec<(PeerId, SubstrateMessage<Block>)>,
+	messages: Vec<(PeerId, Vec<u8>)>,
 }
 
 impl Context<Block> for TestContext {
@@ -60,20 +60,25 @@ impl Context<Block> for TestContext {
 		unimplemented!()
 	}
 
-	fn send_message(&mut self, who: PeerId, data: SubstrateMessage<Block>) {
+	fn send_block_request(&mut self, _who: PeerId, _request: BlockRequest<Block>) {
-		self.messages.push((who, data))
+		unimplemented!()
+	}
+
+	fn send_consensus(&mut self, _who: PeerId, _consensus: ConsensusMessage) {
+		unimplemented!()
+	}
+
+	fn send_chain_specific(&mut self, who: PeerId, message: Vec<u8>){
+		self.messages.push((who, message))
 	}
 }
 
 impl TestContext {
 	fn has_message(&self, to: PeerId, message: Message) -> bool {
-		use substrate_network::generic_message::Message as GenericMessage;
-
 		let encoded = message.encode();
-		self.messages.iter().any(|&(ref peer, ref msg)| match msg {
+		self.messages.iter().any(|&(ref peer, ref data)|
-			GenericMessage::ChainSpecific(ref data) => peer == &to && data == &encoded,
+			peer == &to && data == &encoded
-			_ => false,
+		)
-		})
 	}
 }
 

polkadot/network/src/tests/validation.rs

@@ -18,6 +18,7 @@
 
 use validation::NetworkService;
 use substrate_network::Context as NetContext;
+use substrate_network::consensus_gossip::TopicNotification;
 use substrate_primitives::{NativeOrEncoded, ExecutionContext};
 use substrate_keyring::AuthorityKeyring;
 use {PolkadotProtocol};
@@ -52,25 +53,32 @@ impl Future for NeverExit {
 	}
 }
 
+fn clone_gossip(n: &TopicNotification) -> TopicNotification {
+	TopicNotification {
+		message: n.message.clone(),
+		sender: n.sender.clone(),
+	}
+}
+
 struct GossipRouter {
-	incoming_messages: mpsc::UnboundedReceiver<(Hash, Vec<u8>)>,
+	incoming_messages: mpsc::UnboundedReceiver<(Hash, TopicNotification)>,
-	incoming_streams: mpsc::UnboundedReceiver<(Hash, mpsc::UnboundedSender<Vec<u8>>)>,
+	incoming_streams: mpsc::UnboundedReceiver<(Hash, mpsc::UnboundedSender<TopicNotification>)>,
-	outgoing: Vec<(Hash, mpsc::UnboundedSender<Vec<u8>>)>,
+	outgoing: Vec<(Hash, mpsc::UnboundedSender<TopicNotification>)>,
-	messages: Vec<(Hash, Vec<u8>)>,
+	messages: Vec<(Hash, TopicNotification)>,
 }
 
 impl GossipRouter {
-	fn add_message(&mut self, topic: Hash, message: Vec<u8>) {
+	fn add_message(&mut self, topic: Hash, message: TopicNotification) {
 		self.outgoing.retain(|&(ref o_topic, ref sender)| {
-			o_topic != &topic || sender.unbounded_send(message.clone()).is_ok()
+			o_topic != &topic || sender.unbounded_send(clone_gossip(&message)).is_ok()
 		});
 		self.messages.push((topic, message));
 	}
 
-	fn add_outgoing(&mut self, topic: Hash, sender: mpsc::UnboundedSender<Vec<u8>>) {
+	fn add_outgoing(&mut self, topic: Hash, sender: mpsc::UnboundedSender<TopicNotification>) {
 		for message in self.messages.iter()
 			.filter(|&&(ref t, _)| t == &topic)
-			.map(|&(_, ref msg)| msg.clone())
+			.map(|&(_, ref msg)| clone_gossip(msg))
 		{
 			if let Err(_) = sender.unbounded_send(message) { return }
 		}
@@ -107,8 +115,8 @@ impl Future for GossipRouter {
 
 #[derive(Clone)]
 struct GossipHandle {
-	send_message: mpsc::UnboundedSender<(Hash, Vec<u8>)>,
+	send_message: mpsc::UnboundedSender<(Hash, TopicNotification)>,
-	send_listener: mpsc::UnboundedSender<(Hash, mpsc::UnboundedSender<Vec<u8>>)>,
+	send_listener: mpsc::UnboundedSender<(Hash, mpsc::UnboundedSender<TopicNotification>)>,
 }
 
 fn make_gossip() -> (GossipRouter, GossipHandle) {
@@ -132,14 +140,15 @@ struct TestNetwork {
 }
 
 impl NetworkService for TestNetwork {
-	fn gossip_messages_for(&self, topic: Hash) -> mpsc::UnboundedReceiver<Vec<u8>> {
+	fn gossip_messages_for(&self, topic: Hash) -> mpsc::UnboundedReceiver<TopicNotification> {
 		let (tx, rx) = mpsc::unbounded();
 		let _  = self.gossip.send_listener.unbounded_send((topic, tx));
 		rx
 	}
 
 	fn gossip_message(&self, topic: Hash, message: Vec<u8>) {
-		let _ = self.gossip.send_message.unbounded_send((topic, message));
+		let notification = TopicNotification { message, sender: None };
+		let _ = self.gossip.send_message.unbounded_send((topic, notification));
 	}
 
 	fn drop_gossip(&self, _topic: Hash) {}
@@ -233,6 +242,12 @@ impl ApiExt<Block> for RuntimeApi {
 	fn runtime_version_at(&self, _: &BlockId) -> ClientResult<RuntimeVersion> {
 		unimplemented!("Not required for testing!")
 	}
+
+	fn record_proof(&mut self) { }
+
+	fn extract_proof(&mut self) -> Option<Vec<Vec<u8>>> {
+		None
+	}
 }
 
 impl ParachainHost<Block> for RuntimeApi {

polkadot/network/src/validation.rs

@@ -21,6 +21,7 @@
 
 use sr_primitives::traits::{BlakeTwo256, ProvideRuntimeApi, Hash as HashT};
 use substrate_network::Context as NetContext;
+use substrate_network::consensus_gossip::{TopicNotification, MessageRecipient as GossipMessageRecipient};
 use polkadot_validation::{Network as ParachainNetwork, SharedTable, Collators, Statement, GenericStatement};
 use polkadot_primitives::{Block, BlockId, Hash, SessionKey};
 use polkadot_primitives::parachain::{
@@ -76,7 +77,7 @@ impl Executor for TaskExecutor {
 /// Basic functionality that a network has to fulfill.
 pub trait NetworkService: Send + Sync + 'static {
 	/// Get a stream of gossip messages for a given hash.
-	fn gossip_messages_for(&self, topic: Hash) -> mpsc::UnboundedReceiver<Vec<u8>>;
+	fn gossip_messages_for(&self, topic: Hash) -> mpsc::UnboundedReceiver<TopicNotification>;
 
 	/// Gossip a message on given topic.
 	fn gossip_message(&self, topic: Hash, message: Vec<u8>);
@@ -90,7 +91,7 @@ pub trait NetworkService: Send + Sync + 'static {
 }
 
 impl NetworkService for super::NetworkService {
-	fn gossip_messages_for(&self, topic: Hash) -> mpsc::UnboundedReceiver<Vec<u8>> {
+	fn gossip_messages_for(&self, topic: Hash) -> mpsc::UnboundedReceiver<TopicNotification> {
 		let (tx, rx) = std::sync::mpsc::channel();
 
 		self.with_gossip(move |gossip, _| {
@@ -105,7 +106,12 @@ impl NetworkService for super::NetworkService {
 	}
 
 	fn gossip_message(&self, topic: Hash, message: Vec<u8>) {
-		self.gossip_consensus_message(topic, POLKADOT_ENGINE_ID, message, false);
+		self.gossip_consensus_message(
+			topic,
+			POLKADOT_ENGINE_ID,
+			message,
+			GossipMessageRecipient::BroadcastToAll,
+		);
 	}
 
 	fn drop_gossip(&self, _topic: Hash) { }
@@ -781,7 +787,7 @@ impl<P: ProvideRuntimeApi + Send, E, N, T> SessionDataFetcher<P, E, N, T> where
 
 			let gossip_messages = self.network().gossip_messages_for(topic)
 				.map_err(|()| panic!("unbounded receivers do not throw errors; qed"))
-				.filter_map(|msg| IngressPair::decode(&mut msg.as_slice()));
+				.filter_map(|msg| IngressPair::decode(&mut msg.message.as_slice()));
 
 			let canon_roots = self.api.runtime_api().ingress(&BlockId::hash(parent_hash), parachain)
 				.map_err(|e| format!("Cannot fetch ingress for parachain {:?} at {:?}: {:?}",

polkadot/runtime/src/lib.rs

@@ -90,7 +90,7 @@ use client::{
 use sr_primitives::{
 	ApplyResult, generic, transaction_validity::TransactionValidity,
 	traits::{
-		BlakeTwo256, Block as BlockT, DigestFor, StaticLookup, CurrencyToVoteHandler, AuthorityIdFor
+		BlakeTwo256, Block as BlockT, DigestFor, StaticLookup, Convert, AuthorityIdFor
 	}
 };
 use version::RuntimeVersion;
@@ -188,6 +188,22 @@ impl session::Trait for Runtime {
 	type Event = Event;
 }
 
+/// Converter for currencies to votes.
+pub struct CurrencyToVoteHandler;
+
+impl CurrencyToVoteHandler {
+	fn factor() -> u128 { (Balances::total_issuance() / u64::max_value() as u128).max(1) }
+}
+
+impl Convert<u128, u64> for CurrencyToVoteHandler {
+	fn convert(x: u128) -> u64 { (x / Self::factor()) as u64 }
+}
+
+impl Convert<u128, u128> for CurrencyToVoteHandler {
+	fn convert(x: u128) -> u128 { x * Self::factor() }
+}
+
+
 impl staking::Trait for Runtime {
 	type OnRewardMinted = Treasury;
 	type CurrencyToVote = CurrencyToVoteHandler;

polkadot/runtime/src/parachains.rs

@@ -25,7 +25,7 @@ use primitives::Hash;
 use primitives::parachain::{Id as ParaId, Chain, DutyRoster, AttestedCandidate, Statement};
 use {system, session};
 
-use srml_support::{StorageValue, StorageMap};
+use srml_support::{StorageValue, StorageMap, storage::hashed::generator};
 use srml_support::dispatch::Result;
 
 use inherents::{ProvideInherent, InherentData, RuntimeString, MakeFatalError, InherentIdentifier};
@@ -64,7 +64,7 @@ decl_storage! {
 		config(parachains): Vec<(ParaId, Vec<u8>, Vec<u8>)>;
 		config(_phdata): PhantomData<T>;
 		build(|storage: &mut StorageOverlay, _: &mut ChildrenStorageOverlay, config: &GenesisConfig<T>| {
-			use codec::Encode;
+			let storage = std::cell::RefCell::new(storage);
 
 			let mut p = config.parachains.clone();
 			p.sort_unstable_by_key(|&(ref id, _, _)| id.clone());
@@ -72,15 +72,12 @@ decl_storage! {
 
 			let only_ids: Vec<_> = p.iter().map(|&(ref id, _, _)| id).cloned().collect();
 
-			storage.insert(Self::hash(<Parachains<T>>::key()).to_vec(), only_ids.encode());
+			<Parachains<T> as generator::StorageValue<_>>::put(&only_ids, &storage);
 
 			for (id, code, genesis) in p {
-				let code_key = Self::hash(&<Code<T>>::key_for(&id)).to_vec();
-				let head_key = Self::hash(&<Heads<T>>::key_for(&id)).to_vec();
 				// no ingress -- a chain cannot be routed to until it is live.
-
+				<Code<T> as generator::StorageMap<_, _>>::insert(&id, &code, &storage);
-				storage.insert(code_key, code.encode());
+				<Heads<T> as generator::StorageMap<_, _>>::insert(&id, &genesis, &storage);
-				storage.insert(head_key, genesis.encode());
 			}
 		});
 	}

polkadot/service/src/lib.rs

@@ -183,22 +183,43 @@ construct_service_factory! {
 						key.clone()
 					};
 
-					let voter = grandpa::run_grandpa(
+					let config = grandpa::Config {
-						grandpa::Config {
+						local_key,
-							// TODO: make gossip_duration available through chainspec
+						// FIXME #1578 make this available through chainspec
-							// https://github.com/paritytech/substrate/issues/1578
+						gossip_duration: Duration::from_millis(333),
-							gossip_duration: Duration::new(4, 0),
+						justification_period: 4096,
-							local_key,
+						name: Some(service.config.name.clone())
-							justification_period: 4096,
+					};
-							name: Some(service.config.name.clone()),
-						},
-						link_half,
-						grandpa::NetworkBridge::new(service.network()),
-						service.config.custom.inherent_data_providers.clone(),
-						service.on_exit(),
-					)?;
 
-					executor.spawn(voter);
+					match config.local_key {
+						None => {
+							executor.spawn(grandpa::run_grandpa_observer(
+								config,
+								link_half,
+								service.network(),
+								service.on_exit(),
+							)?);
+						},
+						Some(_) => {
+							use service::TelemetryOnConnect;
+
+							let telemetry_on_connect = TelemetryOnConnect {
+								on_exit: Box::new(service.on_exit()),
+								telemetry_connection_sinks: service.telemetry_on_connect_stream(),
+								executor: &executor,
+							};
+
+							let grandpa_config = grandpa::GrandpaParams {
+								config: config,
+								link: link_half,
+								network: service.network(),
+								inherent_data_providers: service.config.custom.inherent_data_providers.clone(),
+								on_exit: service.on_exit(),
+								telemetry_on_connect: Some(telemetry_on_connect),
+							};
+							executor.spawn(grandpa::run_grandpa_voter(grandpa_config)?);
+						},
+					}
 				}
 
 				let extrinsic_store = {

polkadot/validation/src/collation.rs

@@ -135,8 +135,11 @@ impl<C: Collators, P: ProvideRuntimeApi> Future for CollationFetch<C, P>
 error_chain! {
 	types { Error, ErrorKind, ResultExt; }
 
+	foreign_links {
+		Client(::client::error::Error);
+	}
+
 	links {
-		Client(::client::error::Error, ::client::error::ErrorKind);
 		WasmValidation(wasm_executor::Error, wasm_executor::ErrorKind);
 	}
 

polkadot/validation/src/error.rs

@@ -21,8 +21,11 @@ use runtime_primitives::RuntimeString;
 use primitives::ed25519::Public as AuthorityId;
 
 error_chain! {
+	foreign_links {
+		Client(::client::error::Error);
+	}
+
 	links {
-		Client(::client::error::Error, ::client::error::ErrorKind);
 		Consensus(::consensus::error::Error, ::consensus::error::ErrorKind);
 	}
 
@@ -53,9 +56,3 @@ error_chain! {
 		}
 	}
 }
-
-// impl From<::bft::InputStreamConcluded> for Error {
-// 	fn from(err: ::bft::InputStreamConcluded) -> Self {
-// 		::bft::Error::from(err).into()
-// 	}
-// }

polkadot/validation/src/evaluation.rs

@@ -23,8 +23,8 @@ use polkadot_primitives::{Block, Hash, BlockNumber};
 use polkadot_primitives::parachain::Id as ParaId;
 
 error_chain! {
-	links {
+	foreign_links {
-		Client(::client::error::Error, ::client::error::ErrorKind);
+		Client(::client::error::Error);
 	}
 
 	errors {

polkadot/validation/src/lib.rs

@@ -547,7 +547,7 @@ impl<C, N, P, TxApi> consensus::Environment<Block> for ProposerFactory<C, N, P,
 			parent_id,
 			parent_number: parent_header.number,
 			transaction_pool: self.transaction_pool.clone(),
-			slot_duration: self.aura_slot_duration,
+			slot_duration: self.aura_slot_duration.clone(),
 		})
 	}
 }
@@ -720,7 +720,7 @@ impl<C, TxApi> CreateProposal<C, TxApi> where
 
 		let runtime_api = self.client.runtime_api();
 
-		let mut block_builder = BlockBuilder::at_block(&self.parent_id, &*self.client)?;
+		let mut block_builder = BlockBuilder::at_block(&self.parent_id, &*self.client, false)?;
 
 		{
 			let inherents = runtime_api.inherent_extrinsics(&self.parent_id, inherent_data)?;
@@ -747,7 +747,7 @@ impl<C, TxApi> CreateProposal<C, TxApi> where
 						debug!("[{:?}] Pushed to the block.", ready.hash);
 						pending_size += encoded_size;
 					}
-					Err(client::error::Error(client::error::ErrorKind::ApplyExtrinsicFailed(ApplyError::FullBlock), _)) => {
+					Err(client::error::Error::ApplyExtrinsicFailed(ApplyError::FullBlock)) => {
 						debug!("Block is full, proceed with proposing.");
 						break;
 					}