// Copyright 2019 Parity Technologies (UK) Ltd. // This file is part of Substrate. // Substrate is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // Substrate is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . #![warn(missing_docs)] //! Substrate authority discovery. //! //! This crate enables Substrate authorities to directly connect to other authorities. [`AuthorityDiscovery`] implements //! the Future trait. By polling [`AuthorityDiscovery`] an authority: //! //! //! 1. **Makes itself discoverable** //! //! 1. Retrieves its external addresses. //! //! 2. Adds its network peer id to the addresses. //! //! 3. Signs the above. //! //! 4. Puts the signature and the addresses on the libp2p Kademlia DHT. //! //! //! 2. **Discovers other authorities** //! //! 1. Retrieves the current set of authorities. //! //! 2. Starts DHT queries for the ids of the authorities. //! //! 3. Validates the signatures of the retrieved key value pairs. //! //! 4. Adds the retrieved external addresses as priority nodes to the peerset. use std::collections::{HashMap, HashSet}; use std::convert::TryInto; use std::iter::FromIterator; use std::marker::PhantomData; use std::pin::Pin; use std::sync::Arc; use std::time::{Duration, Instant}; use futures::task::{Context, Poll}; use futures::{Future, FutureExt, Stream, StreamExt}; use futures_timer::Delay; use authority_discovery_primitives::{AuthorityDiscoveryApi, AuthorityId, AuthoritySignature, AuthorityPair}; use client_api::blockchain::HeaderBackend; use codec::{Decode, Encode}; use error::{Error, Result}; use log::{debug, error, log_enabled, warn}; use network::specialization::NetworkSpecialization; use network::{DhtEvent, ExHashT}; use primitives::crypto::{key_types, Pair}; use primitives::traits::BareCryptoStorePtr; use prost::Message; use sp_runtime::generic::BlockId; use sp_runtime::traits::{Block as BlockT, ProvideRuntimeApi}; type Interval = Box + Unpin + Send + Sync>; mod error; /// Dht payload schemas generated from Protobuf definitions via Prost crate in build.rs. mod schema { include!(concat!(env!("OUT_DIR"), "/authority_discovery.rs")); } /// Upper bound estimation on how long one should wait before accessing the Kademlia DHT. const LIBP2P_KADEMLIA_BOOTSTRAP_TIME: Duration = Duration::from_secs(30); /// Name of the Substrate peerset priority group for authorities discovered through the authority discovery module. const AUTHORITIES_PRIORITY_GROUP_NAME: &'static str = "authorities"; /// An `AuthorityDiscovery` makes a given authority discoverable and discovers other authorities. pub struct AuthorityDiscovery where Block: BlockT + 'static, Network: NetworkProvider, Client: ProvideRuntimeApi + Send + Sync + 'static + HeaderBackend, ::Api: AuthorityDiscoveryApi, { client: Arc, network: Arc, /// Channel we receive Dht events on. dht_event_rx: Pin + Send>>, key_store: BareCryptoStorePtr, /// Interval to be proactive, publishing own addresses. publish_interval: Interval, /// Interval on which to query for addresses of other authorities. query_interval: Interval, /// The network peerset interface for priority groups lets us only set an entire group, but we retrieve the /// addresses of other authorities one by one from the network. To use the peerset interface we need to cache the /// addresses and always overwrite the entire peerset priority group. To ensure this map doesn't grow indefinitely /// `purge_old_authorities_from_cache` function is called each time we add a new entry. address_cache: HashMap>, phantom: PhantomData, } impl AuthorityDiscovery where Block: BlockT + Unpin + 'static, Network: NetworkProvider, Client: ProvideRuntimeApi + Send + Sync + 'static + HeaderBackend, ::Api: AuthorityDiscoveryApi, Self: Future, { /// Return a new authority discovery. pub fn new( client: Arc, network: Arc, key_store: BareCryptoStorePtr, dht_event_rx: Pin + Send>>, ) -> Self { // Kademlia's default time-to-live for Dht records is 36h, republishing records every 24h. Given that a node // could restart at any point in time, one can not depend on the republishing process, thus publishing own // external addresses should happen on an interval < 36h. let publish_interval = interval_at( Instant::now() + LIBP2P_KADEMLIA_BOOTSTRAP_TIME, Duration::from_secs(12 * 60 * 60), ); // External addresses of other authorities can change at any given point in time. The interval on which to query // for external addresses of other authorities is a trade off between efficiency and performance. let query_interval = interval_at( Instant::now() + LIBP2P_KADEMLIA_BOOTSTRAP_TIME, Duration::from_secs(10 * 60), ); let address_cache = HashMap::new(); AuthorityDiscovery { client, network, dht_event_rx, key_store, publish_interval, query_interval, address_cache, phantom: PhantomData, } } fn publish_own_ext_addresses(&mut self) -> Result<()> { let addresses = self .network .external_addresses() .into_iter() .map(|a| { a.with(libp2p::core::multiaddr::Protocol::P2p( self.network.local_peer_id().into(), )) }) .map(|a| a.to_vec()) .collect(); let mut serialized_addresses = vec![]; schema::AuthorityAddresses { addresses } .encode(&mut serialized_addresses) .map_err(Error::EncodingProto)?; for key in self.get_priv_keys_within_authority_set()?.into_iter() { let signature = key.sign(&serialized_addresses); let mut signed_addresses = vec![]; schema::SignedAuthorityAddresses { addresses: serialized_addresses.clone(), signature: signature.encode(), } .encode(&mut signed_addresses) .map_err(Error::EncodingProto)?; self.network.put_value( hash_authority_id(key.public().as_ref())?, signed_addresses, ); } Ok(()) } fn request_addresses_of_others(&mut self) -> Result<()> { let id = BlockId::hash(self.client.info().best_hash); let authorities = self .client .runtime_api() .authorities(&id) .map_err(Error::CallingRuntime)?; for authority_id in authorities.iter() { self.network .get_value(&hash_authority_id(authority_id.as_ref())?); } Ok(()) } fn handle_dht_events(&mut self, cx: &mut Context) -> Result<()> { while let Poll::Ready(Some(event)) = self.dht_event_rx.poll_next_unpin(cx) { match event { DhtEvent::ValueFound(v) => { if log_enabled!(log::Level::Debug) { let hashes = v.iter().map(|(hash, _value)| hash.clone()); debug!(target: "sub-authority-discovery", "Value for hash '{:?}' found on Dht.", hashes); } self.handle_dht_value_found_event(v)?; } DhtEvent::ValueNotFound(hash) => warn!( target: "sub-authority-discovery", "Value for hash '{:?}' not found on Dht.", hash ), DhtEvent::ValuePut(hash) => debug!( target: "sub-authority-discovery", "Successfully put hash '{:?}' on Dht.", hash), DhtEvent::ValuePutFailed(hash) => warn!( target: "sub-authority-discovery", "Failed to put hash '{:?}' on Dht.", hash ), } } Ok(()) } fn handle_dht_value_found_event( &mut self, values: Vec<(libp2p::kad::record::Key, Vec)>, ) -> Result<()> { debug!(target: "sub-authority-discovery", "Got Dht value from network."); let block_id = BlockId::hash(self.client.info().best_hash); // From the Dht we only get the hashed authority id. In order to retrieve the actual authority id and to ensure // it is actually an authority, we match the hash against the hash of the authority id of all other authorities. let authorities = self.client.runtime_api().authorities(&block_id)?; self.purge_old_authorities_from_cache(&authorities); let authorities = authorities .into_iter() .map(|id| hash_authority_id(id.as_ref()).map(|h| (h, id))) .collect::>>()?; for (key, value) in values.iter() { // Check if the event origins from an authority in the current authority set. let authority_id: &AuthorityId = authorities .get(key) .ok_or(Error::MatchingHashedAuthorityIdWithAuthorityId)?; let schema::SignedAuthorityAddresses { signature, addresses, } = schema::SignedAuthorityAddresses::decode(value).map_err(Error::DecodingProto)?; let signature = AuthoritySignature::decode(&mut &signature[..]).map_err(Error::EncodingDecodingScale)?; if !AuthorityPair::verify(&signature, &addresses, authority_id) { return Err(Error::VerifyingDhtPayload); } let addresses: Vec = schema::AuthorityAddresses::decode(addresses) .map(|a| a.addresses) .map_err(Error::DecodingProto)? .into_iter() .map(|a| a.try_into()) .collect::>() .map_err(Error::ParsingMultiaddress)?; self.address_cache.insert(authority_id.clone(), addresses); } // Let's update the peerset priority group with all the addresses we have in our cache. let addresses = HashSet::from_iter( self.address_cache .iter() .map(|(_peer_id, addresses)| addresses.clone()) .flatten(), ); debug!(target: "sub-authority-discovery", "Applying priority group {:#?} to peerset.", addresses); self.network .set_priority_group(AUTHORITIES_PRIORITY_GROUP_NAME.to_string(), addresses) .map_err(Error::SettingPeersetPriorityGroup)?; Ok(()) } fn purge_old_authorities_from_cache(&mut self, current_authorities: &Vec) { self.address_cache .retain(|peer_id, _addresses| current_authorities.contains(peer_id)) } /// Retrieve all local authority discovery private keys that are within the current authority /// set. fn get_priv_keys_within_authority_set(&mut self) -> Result> { let keys = self.get_own_public_keys_within_authority_set()? .into_iter() .map(std::convert::Into::into) .filter_map(|pub_key| { self.key_store.read().sr25519_key_pair(key_types::AUTHORITY_DISCOVERY, &pub_key) }) .map(std::convert::Into::into) .collect(); Ok(keys) } /// Retrieve our public keys within the current authority set. // // A node might have multiple authority discovery keys within its keystore, e.g. an old one and // one for the upcoming session. In addition it could be participating in the current authority // set with two keys. The function does not return all of the local authority discovery public // keys, but only the ones intersecting with the current authority set. fn get_own_public_keys_within_authority_set(&mut self) -> Result> { let local_pub_keys = self.key_store .read() .sr25519_public_keys(key_types::AUTHORITY_DISCOVERY) .into_iter() .collect::>(); let id = BlockId::hash(self.client.info().best_hash); let current_authorities = self .client .runtime_api() .authorities(&id) .map_err(Error::CallingRuntime)? .into_iter() .map(std::convert::Into::into) .collect::>(); let intersection = local_pub_keys.intersection(¤t_authorities) .cloned() .map(std::convert::Into::into) .collect(); Ok(intersection) } } impl Future for AuthorityDiscovery where Block: BlockT + Unpin + 'static, Network: NetworkProvider, Client: ProvideRuntimeApi + Send + Sync + 'static + HeaderBackend, ::Api: AuthorityDiscoveryApi, { type Output = (); fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll { let mut inner = || -> Result<()> { // Process incoming events before triggering new ones. self.handle_dht_events(cx)?; if let Poll::Ready(_) = self.publish_interval.poll_next_unpin(cx) { // Make sure to call interval.poll until it returns Async::NotReady once. Otherwise, in case one of the // function calls within this block do a `return`, we don't call `interval.poll` again and thereby the // underlying Tokio task is never registered with Tokio's Reactor to be woken up on the next interval // tick. while let Poll::Ready(_) = self.publish_interval.poll_next_unpin(cx) {} self.publish_own_ext_addresses()?; } if let Poll::Ready(_) = self.query_interval.poll_next_unpin(cx) { // Make sure to call interval.poll until it returns Async::NotReady once. Otherwise, in case one of the // function calls within this block do a `return`, we don't call `interval.poll` again and thereby the // underlying Tokio task is never registered with Tokio's Reactor to be woken up on the next interval // tick. while let Poll::Ready(_) = self.query_interval.poll_next_unpin(cx) {} self.request_addresses_of_others()?; } Ok(()) }; match inner() { Ok(()) => {} Err(e) => error!(target: "sub-authority-discovery", "Poll failure: {:?}", e), }; // Make sure to always return NotReady as this is a long running task with the same lifetime as the node itself. Poll::Pending } } /// NetworkProvider provides AuthorityDiscovery with all necessary hooks into the underlying Substrate networking. Using /// this trait abstraction instead of NetworkService directly is necessary to unit test AuthorityDiscovery. pub trait NetworkProvider { /// Returns the local external addresses. fn external_addresses(&self) -> Vec; /// Returns the network identity of the node. fn local_peer_id(&self) -> libp2p::PeerId; /// Modify a peerset priority group. fn set_priority_group( &self, group_id: String, peers: HashSet, ) -> std::result::Result<(), String>; /// Start putting a value in the Dht. fn put_value(&self, key: libp2p::kad::record::Key, value: Vec); /// Start getting a value from the Dht. fn get_value(&self, key: &libp2p::kad::record::Key); } impl NetworkProvider for network::NetworkService where B: BlockT + 'static, S: NetworkSpecialization, H: ExHashT, { fn external_addresses(&self) -> Vec { self.external_addresses() } fn local_peer_id(&self) -> libp2p::PeerId { self.local_peer_id() } fn set_priority_group( &self, group_id: String, peers: HashSet, ) -> std::result::Result<(), String> { self.set_priority_group(group_id, peers) } fn put_value(&self, key: libp2p::kad::record::Key, value: Vec) { self.put_value(key, value) } fn get_value(&self, key: &libp2p::kad::record::Key) { self.get_value(key) } } fn hash_authority_id(id: &[u8]) -> Result { libp2p::multihash::encode(libp2p::multihash::Hash::SHA2256, id) .map(|k| libp2p::kad::record::Key::new(&k)) .map_err(Error::HashingAuthorityId) } fn interval_at(start: Instant, duration: Duration) -> Interval { let stream = futures::stream::unfold((), move |_| { let wait_time = start.saturating_duration_since(Instant::now()); futures::future::join( Delay::new(wait_time), Delay::new(duration) ).map(|_| Some(((), ()))) }).map(drop); Box::new(stream) } #[cfg(test)] mod tests { use super::*; use sp_api::{ApiExt, Core, RuntimeVersion, StorageProof}; use futures::channel::mpsc::channel; use futures::executor::block_on; use futures::future::poll_fn; use primitives::{ExecutionContext, NativeOrEncoded, testing::KeyStore}; use sp_runtime::traits::Zero; use sp_runtime::traits::{ApiRef, Block as BlockT, NumberFor, ProvideRuntimeApi}; use std::sync::{Arc, Mutex}; use test_client::runtime::Block; #[derive(Clone)] struct TestApi { authorities: Vec, } impl ProvideRuntimeApi for TestApi { type Api = RuntimeApi; fn runtime_api<'a>(&'a self) -> ApiRef<'a, Self::Api> { RuntimeApi{authorities: self.authorities.clone()}.into() } } /// Blockchain database header backend. Does not perform any validation. impl HeaderBackend for TestApi { fn header( &self, _id: BlockId, ) -> std::result::Result, sp_blockchain::Error> { Ok(None) } fn info(&self) -> client_api::blockchain::Info { client_api::blockchain::Info { best_hash: Default::default(), best_number: Zero::zero(), finalized_hash: Default::default(), finalized_number: Zero::zero(), genesis_hash: Default::default(), } } fn status( &self, _id: BlockId, ) -> std::result::Result { Ok(client_api::blockchain::BlockStatus::Unknown) } fn number( &self, _hash: Block::Hash, ) -> std::result::Result>, sp_blockchain::Error> { Ok(None) } fn hash( &self, _number: NumberFor, ) -> std::result::Result, sp_blockchain::Error> { Ok(None) } } struct RuntimeApi { authorities: Vec, } impl Core for RuntimeApi { fn Core_version_runtime_api_impl( &self, _: &BlockId, _: ExecutionContext, _: Option<()>, _: Vec, ) -> std::result::Result, sp_blockchain::Error> { unimplemented!("Not required for testing!") } fn Core_execute_block_runtime_api_impl( &self, _: &BlockId, _: ExecutionContext, _: Option<(Block)>, _: Vec, ) -> std::result::Result, sp_blockchain::Error> { unimplemented!("Not required for testing!") } fn Core_initialize_block_runtime_api_impl( &self, _: &BlockId, _: ExecutionContext, _: Option<&::Header>, _: Vec, ) -> std::result::Result, sp_blockchain::Error> { unimplemented!("Not required for testing!") } } impl ApiExt for RuntimeApi { type Error = sp_blockchain::Error; fn map_api_result std::result::Result, R, E>( &self, _: F, ) -> std::result::Result { unimplemented!("Not required for testing!") } fn runtime_version_at( &self, _: &BlockId, ) -> std::result::Result { unimplemented!("Not required for testing!") } fn record_proof(&mut self) { unimplemented!("Not required for testing!") } fn extract_proof(&mut self) -> Option { unimplemented!("Not required for testing!") } } impl AuthorityDiscoveryApi for RuntimeApi { fn AuthorityDiscoveryApi_authorities_runtime_api_impl( &self, _: &BlockId, _: ExecutionContext, _: Option<()>, _: Vec, ) -> std::result::Result>, sp_blockchain::Error> { return Ok(NativeOrEncoded::Native(self.authorities.clone())); } } #[derive(Default)] struct TestNetwork { // Whenever functions on `TestNetwork` are called, the function arguments are added to the vectors below. pub put_value_call: Arc)>>>, pub get_value_call: Arc>>, pub set_priority_group_call: Arc)>>>, } impl NetworkProvider for TestNetwork { fn external_addresses(&self) -> Vec { vec![] } fn local_peer_id(&self) -> libp2p::PeerId { libp2p::PeerId::random() } fn set_priority_group( &self, group_id: String, peers: HashSet, ) -> std::result::Result<(), String> { self.set_priority_group_call .lock() .unwrap() .push((group_id, peers)); Ok(()) } fn put_value(&self, key: libp2p::kad::record::Key, value: Vec) { self.put_value_call.lock().unwrap().push((key, value)); } fn get_value(&self, key: &libp2p::kad::record::Key) { self.get_value_call.lock().unwrap().push(key.clone()); } } #[test] fn publish_own_ext_addresses_puts_record_on_dht() { let (_dht_event_tx, dht_event_rx) = channel(1000); let network: Arc = Arc::new(Default::default()); let key_store = KeyStore::new(); let public = key_store.write().sr25519_generate_new(key_types::AUTHORITY_DISCOVERY, None).unwrap(); let test_api = Arc::new(TestApi {authorities: vec![public.into()]}); let mut authority_discovery = AuthorityDiscovery::new(test_api, network.clone(), key_store, dht_event_rx.boxed()); authority_discovery.publish_own_ext_addresses().unwrap(); // Expect authority discovery to put a new record onto the dht. assert_eq!(network.put_value_call.lock().unwrap().len(), 1); } #[test] fn request_addresses_of_others_triggers_dht_get_query() { let (_dht_event_tx, dht_event_rx) = channel(1000); // Generate authority keys let authority_1_key_pair = AuthorityPair::from_seed_slice(&[1; 32]).unwrap(); let authority_2_key_pair = AuthorityPair::from_seed_slice(&[2; 32]).unwrap(); let test_api = Arc::new(TestApi { authorities: vec![authority_1_key_pair.public(), authority_2_key_pair.public()], }); let network: Arc = Arc::new(Default::default()); let key_store = KeyStore::new(); let mut authority_discovery = AuthorityDiscovery::new(test_api, network.clone(), key_store, dht_event_rx.boxed()); authority_discovery.request_addresses_of_others().unwrap(); // Expect authority discovery to request new records from the dht. assert_eq!(network.get_value_call.lock().unwrap().len(), 2); } #[test] fn handle_dht_events_with_value_found_should_call_set_priority_group() { // Create authority discovery. let (mut dht_event_tx, dht_event_rx) = channel(1000); let key_pair = AuthorityPair::from_seed_slice(&[1; 32]).unwrap(); let test_api = Arc::new(TestApi {authorities: vec![key_pair.public()]}); let network: Arc = Arc::new(Default::default()); let key_store = KeyStore::new(); let mut authority_discovery = AuthorityDiscovery::new(test_api, network.clone(), key_store, dht_event_rx.boxed()); // Create sample dht event. let authority_id_1 = hash_authority_id(key_pair.public().as_ref()).unwrap(); let address_1: libp2p::Multiaddr = "/ip6/2001:db8::".parse().unwrap(); let mut serialized_addresses = vec![]; schema::AuthorityAddresses { addresses: vec![address_1.to_vec()], } .encode(&mut serialized_addresses) .unwrap(); let signature = key_pair.sign(serialized_addresses.as_ref()).encode(); let mut signed_addresses = vec![]; schema::SignedAuthorityAddresses { addresses: serialized_addresses, signature: signature, } .encode(&mut signed_addresses) .unwrap(); let dht_event = network::DhtEvent::ValueFound(vec![(authority_id_1, signed_addresses)]); dht_event_tx.try_send(dht_event).unwrap(); // Make authority discovery handle the event. let f = |cx: &mut Context<'_>| -> Poll<()> { authority_discovery.handle_dht_events(cx).unwrap(); // Expect authority discovery to set the priority set. assert_eq!(network.set_priority_group_call.lock().unwrap().len(), 1); assert_eq!( network.set_priority_group_call.lock().unwrap()[0], ( "authorities".to_string(), HashSet::from_iter(vec![address_1.clone()].into_iter()) ) ); Poll::Ready(()) }; let _ = block_on(poll_fn(f)); } }