Update futures and tokio for browser light client (#673)

* Make availability-store compile for WASM

* Use --manifest-path instead

* Make validation work on wasm!

* Switch to Spawn trait

* Migrate validation to std futures

* Migrate network to std futures

* Final changes to validation

* Tidy up network

* Tidy up validation

* Switch branch

* Migrate service

* Get polkadot to compile via wasm!

* Add browser-demo

* Add initial browser file

* Add browser-demo

* Tidy

* Temp switch back to substrate/master

* tidy

* Fix wasm build

* Re-add release flag

* Switch to polkadot-master

* Revert cli tokio version to avoid libp2p panic

* Update tokio version

* Fix availability store tests

* Fix validation tests

* Remove futures01 from availability-store

* Fix network tests

* Small changes

* Fix collator

* Fix typo

* Revert removal of tokio_executor that causes tokio version mismatch panic

* Fix adder test parachain

* Revert "Revert removal of tokio_executor that causes tokio version mismatch panic"

This reverts commit cfeb50c01d8df5e209483406a711e64761b44ae9.

* Update availability-store/src/worker.rs

Co-Authored-By: Pierre Krieger <pierre.krieger1708@gmail.com>

* Update network/src/lib.rs

Co-Authored-By: Pierre Krieger <pierre.krieger1708@gmail.com>

* Update network/src/lib.rs

Co-Authored-By: Pierre Krieger <pierre.krieger1708@gmail.com>

* Box pin changes

* Asyncify network functions

* Clean up browser validation worker error

* Fix av store test

* Nits

* Fix validation test

* Switch favicon

* Fix validation test again

* Revert "Asyncify network functions"

This reverts commit f20ae6548dc482cb1e75bc80641cfe55c6131a53.

* Add async blocks back in

polkadot/availability-store/Cargo.toml

@@ -12,9 +12,8 @@ polkadot-runtime = { path = "../runtime" }
 parking_lot = "0.9.0"
 derive_more = "0.99"
 log = "0.4.8"
-futures01 = "0.1.17"
-futures = { package = "futures", version = "0.3.1", features = ["compat"] }
-tokio = "0.1.7"
+futures = "0.3.1"
+tokio = { version = "0.2.4", features = ["rt-core"] }
 exit-future = "0.2.0"
 codec = { package = "parity-scale-codec", version = "1.1.0", default-features = false, features = ["derive"] }
 sc-network = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }

polkadot/availability-store/src/lib.rs

@@ -23,7 +23,7 @@
 #![warn(missing_docs)]
 
 use futures::prelude::*;
-use futures::channel::{mpsc, oneshot};
+use futures::{channel::{mpsc, oneshot}, task::Spawn};
 use keystore::KeyStorePtr;
 use polkadot_primitives::{
 	Hash, Block,
@@ -38,7 +38,7 @@ use client::{
 	BlockchainEvents, BlockBody,
 };
 use sp_api::ApiExt;
-
+use std::pin::Pin;
 use log::warn;
 
 use std::sync::Arc;
@@ -58,10 +58,7 @@ use worker::{
 use store::{Store as InnerStore};
 
 /// Abstraction over an executor that lets you spawn tasks in the background.
-pub(crate) type TaskExecutor =
-	Arc<dyn futures01::future::Executor<
-		Box<dyn futures01::Future<Item = (), Error = ()> + Send>
-	> + Send + Sync>;
+pub(crate) type TaskExecutor = Arc<dyn Spawn + Send + Sync>;
 
 const LOG_TARGET: &str = "availability";
 
@@ -110,7 +107,7 @@ pub trait ProvideGossipMessages {
 	fn gossip_messages_for(
 		&self,
 		topic: Hash,
-	) -> Box<dyn Stream<Item = (Hash, Hash, ErasureChunk)> + Send + Unpin>;
+	) -> Pin<Box<dyn Stream<Item = (Hash, Hash, ErasureChunk)> + Send>>;
 
 	/// Gossip an erasure chunk message.
 	fn gossip_erasure_chunk(
@@ -155,6 +152,7 @@ impl Store {
 	///
 	/// Creating a store among other things starts a background worker thread which
 	/// handles most of the write operations to the storage.
+	#[cfg(not(target_os = "unknown"))]
 	pub fn new<PGM>(config: Config, gossip: PGM) -> io::Result<Self>
 		where PGM: ProvideGossipMessages + Send + Sync + Clone + 'static
 	{

polkadot/availability-store/src/store.rs

@@ -14,6 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
+#[cfg(not(target_os = "unknown"))]
 use kvdb_rocksdb::{Database, DatabaseConfig};
 use kvdb::{KeyValueDB, DBTransaction};
 use codec::{Encode, Decode};
@@ -82,6 +83,7 @@ fn erasure_roots_in_relay_chain_block_key(relay_block: &Hash) -> Vec<u8> {
 
 impl Store {
 	/// Create a new `Store` with given condig on disk.
+	#[cfg(not(target_os = "unknown"))]
 	pub(super) fn new(config: Config) -> io::Result<Self> {
 		let mut db_config = DatabaseConfig::with_columns(Some(columns::NUM_COLUMNS));
 

polkadot/availability-store/src/worker.rs

@@ -37,11 +37,10 @@ use polkadot_primitives::parachain::{
 	CandidateReceipt, ParachainHost, ValidatorId,
 	ValidatorPair, AvailableMessages, BlockData, ErasureChunk,
 };
-use futures::channel::{mpsc, oneshot};
-use futures::{FutureExt, Sink, SinkExt, TryFutureExt, StreamExt, future::select};
+use futures::{prelude::*, future::select, channel::{mpsc, oneshot}, task::SpawnExt};
 use keystore::KeyStorePtr;
 
-use tokio::runtime::current_thread::{Handle, Runtime as LocalRuntime};
+use tokio::runtime::{Handle, Runtime as LocalRuntime};
 
 use crate::{LOG_TARGET, Data, TaskExecutor, ProvideGossipMessages, erasure_coding_topic};
 use crate::store::Store;
@@ -308,7 +307,7 @@ where
 	// Called on startup of the worker to register listeners for all awaited chunks.
 	fn register_listeners(
 		&mut self,
-		runtime_handle: &mut Handle,
+		runtime_handle: &Handle,
 		sender: &mut mpsc::UnboundedSender<WorkerMsg>,
 	) {
 		if let Some(awaited_chunks) = self.availability_store.awaited_chunks() {
@@ -327,7 +326,7 @@ where
 
 	fn register_chunks_listener(
 		&mut self,
-		runtime_handle: &mut Handle,
+		runtime_handle: &Handle,
 		sender: &mut mpsc::UnboundedSender<WorkerMsg>,
 		relay_parent: Hash,
 		erasure_root: Hash,
@@ -354,18 +353,14 @@ where
 
 		self.registered_gossip_streams.insert(topic, signal);
 
-		let _ = runtime_handle.spawn(
-			select(fut.boxed(), exit)
-				.map(|_| Ok(()))
-				.compat()
-		);
+		let _ = runtime_handle.spawn(select(fut.boxed(), exit).map(drop));
 
 		Ok(())
 	}
 
 	fn on_parachain_blocks_received(
 		&mut self,
-		runtime_handle: &mut Handle,
+		runtime_handle: &Handle,
 		sender: &mut mpsc::UnboundedSender<WorkerMsg>,
 		relay_parent: Hash,
 		blocks: Vec<(CandidateReceipt, Option<(BlockData, AvailableMessages)>)>,
@@ -450,7 +445,7 @@ where
 	// we don't have that piece, and then it registers a listener.
 	fn on_listen_for_chunks_received(
 		&mut self,
-		runtime_handle: &mut Handle,
+		runtime_handle: &Handle,
 		sender: &mut mpsc::UnboundedSender<WorkerMsg>,
 		relay_parent: Hash,
 		candidate_hash: Hash,
@@ -496,11 +491,11 @@ where
 			let mut runtime = LocalRuntime::new()?;
 			let mut sender = worker.sender.clone();
 
-			let mut runtime_handle = runtime.handle();
+			let runtime_handle = runtime.handle().clone();
 
 			// On startup, registers listeners (gossip streams) for all
 			// (relay_parent, erasure-root, i) in the awaited frontier.
-			worker.register_listeners(&mut runtime_handle, &mut sender);
+			worker.register_listeners(runtime.handle(), &mut sender);
 
 			let process_notification = async move {
 				while let Some(msg) = receiver.next().await {
@@ -525,7 +520,7 @@ where
 							} = msg;
 
 							let res = worker.on_listen_for_chunks_received(
-								&mut runtime_handle,
+								&runtime_handle,
 								&mut sender,
 								relay_parent,
 								candidate_hash,
@@ -545,7 +540,7 @@ where
 							} = msg;
 
 							let res = worker.on_parachain_blocks_received(
-								&mut runtime_handle,
+								&runtime_handle,
 								&mut sender,
 								relay_parent,
 								blocks,
@@ -589,15 +584,9 @@ where
 
 			};
 
-			runtime.spawn(
-				futures::future::select(process_notification.boxed(), exit.clone())
-					.map(|_| Ok(()))
-					.compat()
-			);
+			runtime.spawn(select(process_notification.boxed(), exit.clone()).map(drop));
 
-			if let Err(e) = runtime.block_on(exit.unit_error().compat()) {
-				warn!(target: LOG_TARGET, "Availability worker error {:?}", e);
-			}
+			runtime.block_on(exit);
 
 			info!(target: LOG_TARGET, "Availability worker exiting");
 
@@ -771,9 +760,9 @@ impl<I, P> AvailabilityBlockImport<I, P> {
 		let prune_available = select(
 			prune_unneeded_availability(client.clone(), to_worker.clone()).boxed(),
 			exit.clone()
-		).map(|_| Ok(())).compat();
+		).map(drop);
 
-		if let Err(_) = thread_pool.execute(Box::new(prune_available)) {
+		if let Err(_) = thread_pool.spawn(Box::new(prune_available)) {
 			error!(target: LOG_TARGET, "Failed to spawn availability pruning task");
 			exit_signal = None;
 		}
@@ -806,6 +795,7 @@ mod tests {
 	use std::time::Duration;
 	use futures::{stream, channel::mpsc, Stream};
 	use std::sync::{Arc, Mutex};
+	use std::pin::Pin;
 	use tokio::runtime::Runtime;
 
 	// Just contains topic->channel mapping to give to outer code on `gossip_messages_for` calls.
@@ -815,11 +805,11 @@ mod tests {
 
 	impl ProvideGossipMessages for TestGossipMessages {
 		fn gossip_messages_for(&self, topic: Hash)
-			-> Box<dyn Stream<Item = (Hash, Hash, ErasureChunk)> + Send + Unpin>
+			-> Pin<Box<dyn Stream<Item = (Hash, Hash, ErasureChunk)> + Send>>
 		{
 			match self.messages.lock().unwrap().remove(&topic) {
-				Some(receiver) => Box::new(receiver),
-				None => Box::new(stream::iter(vec![])),
+				Some(receiver) => receiver.boxed(),
+				None => stream::iter(vec![]).boxed(),
 			}
 		}
 
@@ -890,7 +880,7 @@ mod tests {
 		// chunk topics.
 		handle.sender.unbounded_send(msg).unwrap();
 
-		runtime.block_on(r.unit_error().boxed().compat()).unwrap().unwrap().unwrap();
+		runtime.block_on(r).unwrap().unwrap();
 
 		// Make sure that at this point we are waiting for the appropriate chunk.
 		assert_eq!(
@@ -992,7 +982,7 @@ mod tests {
 
 		handle.sender.unbounded_send(listen_msg_2).unwrap();
 
-		runtime.block_on(r2.unit_error().boxed().compat()).unwrap().unwrap().unwrap();
+		runtime.block_on(r2).unwrap().unwrap();
 		// The gossip sender for this topic left intact => listener not registered.
 		assert!(messages.messages.lock().unwrap().contains_key(&topic_2));
 
@@ -1008,7 +998,7 @@ mod tests {
 		});
 
 		handle.sender.unbounded_send(listen_msg_1).unwrap();
-		runtime.block_on(r1.unit_error().boxed().compat()).unwrap().unwrap().unwrap();
+		runtime.block_on(r1).unwrap().unwrap();
 
 		// The gossip sender taken => listener registered.
 		assert!(!messages.messages.lock().unwrap().contains_key(&topic_1));