Update futures and tokio for browser light client (#673)

* Make availability-store compile for WASM * Use --manifest-path instead * Make validation work on wasm! * Switch to Spawn trait * Migrate validation to std futures * Migrate network to std futures * Final changes to validation * Tidy up network * Tidy up validation * Switch branch * Migrate service * Get polkadot to compile via wasm! * Add browser-demo * Add initial browser file * Add browser-demo * Tidy * Temp switch back to substrate/master * tidy * Fix wasm build * Re-add release flag * Switch to polkadot-master * Revert cli tokio version to avoid libp2p panic * Update tokio version * Fix availability store tests * Fix validation tests * Remove futures01 from availability-store * Fix network tests * Small changes * Fix collator * Fix typo * Revert removal of tokio_executor that causes tokio version mismatch panic * Fix adder test parachain * Revert "Revert removal of tokio_executor that causes tokio version mismatch panic" This reverts commit cfeb50c01d8df5e209483406a711e64761b44ae9. * Update availability-store/src/worker.rs Co-Authored-By: Pierre Krieger <pierre.krieger1708@gmail.com> * Update network/src/lib.rs Co-Authored-By: Pierre Krieger <pierre.krieger1708@gmail.com> * Update network/src/lib.rs Co-Authored-By: Pierre Krieger <pierre.krieger1708@gmail.com> * Box pin changes * Asyncify network functions * Clean up browser validation worker error * Fix av store test * Nits * Fix validation test * Switch favicon * Fix validation test again * Revert "Asyncify network functions" This reverts commit f20ae6548dc482cb1e75bc80641cfe55c6131a53. * Add async blocks back in
2026-07-20 22:05:42 +00:00 · 2019-12-10 11:58:22 +01:00
parent df3ea965e7
commit 25aa988f9b
34 changed files with 826 additions and 444 deletions
@@ -5,6 +5,9 @@ authors = ["Parity Technologies <admin@parity.io>"]
 description = "Polkadot node implementation in Rust."
 edition = "2018"

+[lib]
+crate-type = ["cdylib", "rlib"]
+
 [dependencies]
 log = "0.4.8"
 tokio = "0.1.22"
@@ -14,6 +17,28 @@ structopt = "0.3.4"
 cli = { package = "sc-cli", git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 service = { package = "polkadot-service", path = "../service" }

+libp2p = { version = "0.13.0", default-features = false, optional = true }
+wasm-bindgen = { version = "0.2.45", optional = true }
+wasm-bindgen-futures = { version = "0.3.22", optional = true }
+console_log = { version = "0.1.2", optional = true }
+console_error_panic_hook = { version = "0.1.1", optional = true }
+js-sys = { version = "0.3.22", optional = true }
+kvdb-memorydb = { version = "0.1.1", optional = true }
+substrate-service = { package = "sc-service", git = "https://github.com/paritytech/substrate", branch = "polkadot-master", optional = true, default-features = false }
+substrate-network = { package = "sc-network", git = "https://github.com/paritytech/substrate", branch = "polkadot-master", optional = true }
+
 [features]
-default = [ "wasmtime" ]
+default = [ "wasmtime", "rocksdb" ]
 wasmtime = [ "cli/wasmtime" ]
+rocksdb = [ "service/rocksdb" ]
+browser = [
+	"libp2p",
+	"wasm-bindgen",
+	"console_error_panic_hook",
+	"wasm-bindgen-futures",
+	"console_log",
+	"js-sys",
+	"kvdb-memorydb",
+	"substrate-service",
+	"substrate-network"
+]
@@ -0,0 +1 @@
+pkg
@@ -0,0 +1,9 @@
+# How to run this demo
+
+```sh
+cargo install wasm-pack		# If necessary
+
+wasm-pack build --target web --out-dir ./browser-demo/pkg --no-typescript --release ./.. -- --no-default-features --features "browser"
+
+xdg-open index.html
+```
@@ -0,0 +1,3 @@
+#!/usr/bin/env sh
+wasm-pack build --target web --out-dir ./browser-demo/pkg --no-typescript --release ./.. -- --no-default-features --features "browser"
+python -m http.server 8000
@@ -0,0 +1,39 @@
+<!DOCTYPE html>
+<html>
+  <head>
+	<meta http-equiv="Content-type" content="text/html; charset=utf-8"/>
+	<title>Polkadot node</title>
+	<link rel="shortcut icon" href="/favicon.png" />
+	<script type="module">
+import { start_client, default as init } from './pkg/polkadot_cli.js';
+import ws from './ws.js';
+
+function log(msg) {
+	document.getElementsByTagName('body')[0].innerHTML += msg + '\n';
+}
+
+async function start() {
+	log('Loading WASM');
+	await init('./pkg/polkadot_cli_bg.wasm');
+	log('Successfully loaded WASM');
+
+	// Build our client.
+	log('Starting client');
+	let client = start_client(ws());
+	log('Client started');
+
+	client.rpcSubscribe('{"method":"chain_subscribeNewHead","params":[],"id":1,"jsonrpc":"2.0"}',
+		(r) => log("New chain head: " + r));
+
+	setInterval(() => {
+		client
+			.rpcSend('{"method":"system_networkState","params":[],"id":1,"jsonrpc":"2.0"}')
+			.then((r) => log("Network state: " + r));
+	}, 1000);
+}
+
+start();
+	</script>
+  </head>
+  <body style="white-space: pre"></body>
+</html>
@@ -0,0 +1,148 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+export default () => {
+	return {
+		dial: dial,
+		listen_on: (addr) => {
+			let err = new Error("Listening on WebSockets is not possible from within a browser");
+			err.name = "NotSupportedError";
+			throw err;
+		},
+	};
+}
+
+/// Turns a string multiaddress into a WebSockets string URL.
+// TODO: support dns addresses as well
+const multiaddr_to_ws = (addr) => {
+	let parsed = addr.match(/^\/(ip4|ip6|dns4|dns6)\/(.*?)\/tcp\/(.*?)\/(ws|wss|x-parity-ws\/(.*)|x-parity-wss\/(.*))$/);
+	let proto = 'wss';
+	if (parsed[4] == 'ws' || parsed[4] == 'x-parity-ws') {
+		proto = 'ws';
+	}
+	let url = decodeURIComponent(parsed[5] || parsed[6] || '');
+	if (parsed != null) {
+		if (parsed[1] == 'ip6') {
+			return proto + "://[" + parsed[2] + "]:" + parsed[3] + url;
+		} else {
+			return proto + "://" + parsed[2] + ":" + parsed[3] + url;
+		}
+	}
+
+	let err = new Error("Address not supported: " + addr);
+	err.name = "NotSupportedError";
+	throw err;
+}
+
+// Attempt to dial a multiaddress.
+const dial = (addr) => {
+	let ws = new WebSocket(multiaddr_to_ws(addr));
+	let reader = read_queue();
+
+	return new Promise((resolve, reject) => {
+		// TODO: handle ws.onerror properly after dialing has happened
+		ws.onerror = (ev) => reject(ev);
+		ws.onmessage = (ev) => reader.inject_blob(ev.data);
+		ws.onclose = () => reader.inject_eof();
+		ws.onopen = () => resolve({
+			read: (function*() { while(ws.readyState == 1) { yield reader.next(); } })(),
+			write: (data) => {
+				if (ws.readyState == 1) {
+					ws.send(data);
+					return promise_when_ws_finished(ws);
+				} else {
+					return Promise.reject("WebSocket is closed");
+				}
+			},
+			shutdown: () => {},
+			close: () => ws.close()
+		});
+	});
+}
+
+// Takes a WebSocket object and returns a Promise that resolves when bufferedAmount is 0.
+const promise_when_ws_finished = (ws) => {
+	if (ws.bufferedAmount == 0) {
+		return Promise.resolve();
+	}
+
+	return new Promise((resolve, reject) => {
+		setTimeout(function check() {
+			if (ws.bufferedAmount == 0) {
+				resolve();
+			} else {
+				setTimeout(check, 100);
+			}
+		}, 2);
+	})
+}
+
+// Creates a queue reading system.
+const read_queue = () => {
+	// State of the queue.
+	let state = {
+		// Array of promises resolving to `ArrayBuffer`s, that haven't been transmitted back with
+		// `next` yet.
+		queue: new Array(),
+		// If `resolve` isn't null, it is a "resolve" function of a promise that has already been
+		// returned by `next`. It should be called with some data.
+		resolve: null,
+	};
+
+	return {
+		// Inserts a new Blob in the queue.
+		inject_blob: (blob) => {
+			if (state.resolve != null) {
+				var resolve = state.resolve;
+				state.resolve = null;
+
+				var reader = new FileReader();
+				reader.addEventListener("loadend", () => resolve(reader.result));
+				reader.readAsArrayBuffer(blob);
+			} else {
+				state.queue.push(new Promise((resolve, reject) => {
+					var reader = new FileReader();
+					reader.addEventListener("loadend", () => resolve(reader.result));
+					reader.readAsArrayBuffer(blob);
+				}));
+			}
+		},
+
+		// Inserts an EOF message in the queue.
+		inject_eof: () => {
+			if (state.resolve != null) {
+				var resolve = state.resolve;
+				state.resolve = null;
+				resolve(null);
+			} else {
+				state.queue.push(Promise.resolve(null));
+			}
+		},
+
+		// Returns a Promise that yields the next entry as an ArrayBuffer.
+		next: () => {
+			if (state.queue.length != 0) {
+				return state.queue.shift(0);
+			} else {
+				if (state.resolve !== null)
+					throw "Internal error: already have a pending promise";
+				return new Promise((resolve, reject) => {
+					state.resolve = resolve;
+				});
+			}
+		}
+	};
+};
@@ -0,0 +1,169 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::ChainSpec;
+use futures01::{prelude::*, sync::oneshot, sync::mpsc};
+use libp2p::wasm_ext;
+use log::{debug, info};
+use std::sync::Arc;
+use service::{AbstractService, Roles as ServiceRoles};
+use substrate_service::{RpcSession, Configuration, config::DatabaseConfig};
+use wasm_bindgen::prelude::*;
+
+/// Starts the client.
+///
+/// You must pass a libp2p transport that supports .
+#[wasm_bindgen]
+pub fn start_client(wasm_ext: wasm_ext::ffi::Transport) -> Result<Client, JsValue> {
+	start_inner(wasm_ext)
+		.map_err(|err| JsValue::from_str(&err.to_string()))
+}
+
+fn start_inner(wasm_ext: wasm_ext::ffi::Transport) -> Result<Client, Box<dyn std::error::Error>> {
+	console_error_panic_hook::set_once();
+	console_log::init_with_level(log::Level::Info);
+
+	// Build the configuration to pass to the service.
+	let config = {
+		let wasm_ext = wasm_ext::ExtTransport::new(wasm_ext);
+		let chain_spec = ChainSpec::Kusama.load().map_err(|e| format!("{:?}", e))?;
+		let mut config = Configuration::<service::CustomConfiguration, _, _>::default_with_spec_and_base_path(chain_spec, None);
+		config.network.transport = substrate_network::config::TransportConfig::Normal {
+			wasm_external_transport: Some(wasm_ext.clone()),
+			allow_private_ipv4: true,
+			enable_mdns: false,
+		};
+		config.telemetry_external_transport = Some(wasm_ext);
+		config.roles = ServiceRoles::LIGHT;
+		config.name = "Browser node".to_string();
+		config.database = {
+			let db = Arc::new(kvdb_memorydb::create(10));
+			DatabaseConfig::Custom(db)
+		};
+		config.keystore_path = Some(std::path::PathBuf::from("/"));
+		config
+	};
+
+	info!("Polkadot browser node");
+	info!("  version {}", config.full_version());
+	info!("  by Parity Technologies, 2017-2019");
+	info!("Chain specification: {}", config.chain_spec.name());
+	if config.chain_spec.name().starts_with("Kusama") {
+		info!("----------------------------");
+		info!("This chain is not in any way");
+		info!("      endorsed by the       ");
+		info!("     KUSAMA FOUNDATION      ");
+		info!("----------------------------");
+	}
+	info!("Node name: {}", config.name);
+	info!("Roles: {:?}", config.roles);
+
+	// Create the service. This is the most heavy initialization step.
+	let mut service = service::new_light(config).map_err(|e| format!("{:?}", e))?;
+
+	// We now dispatch a background task responsible for processing the service.
+	//
+	// The main action performed by the code below consists in polling the service with
+	// `service.poll()`.
+	// The rest consists in handling RPC requests.
+	let (rpc_send_tx, mut rpc_send_rx) = mpsc::unbounded::<RpcMessage>();
+	wasm_bindgen_futures::spawn_local(futures01::future::poll_fn(move || {
+		loop {
+			match rpc_send_rx.poll() {
+				Ok(Async::Ready(Some(message))) => {
+					let fut = service.rpc_query(&message.session, &message.rpc_json);
+					let _ = message.send_back.send(Box::new(fut));
+				},
+				Ok(Async::NotReady) => break,
+				Err(_) | Ok(Async::Ready(None)) => return Ok(Async::Ready(())),
+			}
+		}
+
+		loop {
+			match service.poll().map_err(|_| ())? {
+				Async::Ready(()) => return Ok(Async::Ready(())),
+				Async::NotReady => break
+			}
+		}
+
+		Ok(Async::NotReady)
+	}));
+
+	Ok(Client {
+		rpc_send_tx,
+	})
+}
+
+/// A running client.
+#[wasm_bindgen]
+pub struct Client {
+	rpc_send_tx: mpsc::UnboundedSender<RpcMessage>,
+}
+
+struct RpcMessage {
+	rpc_json: String,
+	session: RpcSession,
+	send_back: oneshot::Sender<Box<dyn Future<Item = Option<String>, Error = ()>>>,
+}
+
+#[wasm_bindgen]
+impl Client {
+	/// Allows starting an RPC request. Returns a `Promise` containing the result of that request.
+	#[wasm_bindgen(js_name = "rpcSend")]
+	pub fn rpc_send(&mut self, rpc: &str) -> js_sys::Promise {
+		let rpc_session = RpcSession::new(mpsc::channel(1).0);
+		let (tx, rx) = oneshot::channel();
+		let _ = self.rpc_send_tx.unbounded_send(RpcMessage {
+			rpc_json: rpc.to_owned(),
+			session: rpc_session,
+			send_back: tx,
+		});
+		let fut = rx
+			.map_err(|_| ())
+			.and_then(|fut| fut)
+			.map(|s| JsValue::from_str(&s.unwrap_or(String::new())))
+			.map_err(|_| JsValue::NULL);
+		wasm_bindgen_futures::future_to_promise(fut)
+	}
+
+	/// Subscribes to an RPC pubsub endpoint.
+	#[wasm_bindgen(js_name = "rpcSubscribe")]
+	pub fn rpc_subscribe(&mut self, rpc: &str, callback: js_sys::Function) {
+		let (tx, rx) = mpsc::channel(4);
+		let rpc_session = RpcSession::new(tx);
+		let (fut_tx, fut_rx) = oneshot::channel();
+		let _ = self.rpc_send_tx.unbounded_send(RpcMessage {
+			rpc_json: rpc.to_owned(),
+			session: rpc_session.clone(),
+			send_back: fut_tx,
+		});
+		let fut_rx = fut_rx
+			.map_err(|_| ())
+			.and_then(|fut| fut);
+		wasm_bindgen_futures::spawn_local(fut_rx.then(|_| Ok(())));
+		wasm_bindgen_futures::spawn_local(rx.for_each(move |s| {
+			match callback.call1(&callback, &JsValue::from_str(&s)) {
+				Ok(_) => Ok(()),
+				Err(_) => Err(()),
+			}
+		}).then(move |v| {
+			// We need to keep `rpc_session` alive.
+			debug!("RPC subscription has ended");
+			drop(rpc_session);
+			v
+		}));
+	}
+}
@@ -20,26 +20,27 @@
 #![warn(unused_extern_crates)]

 mod chain_spec;
+#[cfg(feature = "browser")]
+mod browser;

 use chain_spec::ChainSpec;
-use futures::{Future, FutureExt, TryFutureExt, future::select, channel::oneshot, compat::Future01CompatExt};
+use futures::{
+	Future, FutureExt, TryFutureExt, future::select, channel::oneshot, compat::Future01CompatExt,
+	task::Spawn
+};
 use tokio::runtime::Runtime;
-use std::sync::Arc;
 use log::{info, error};
 use structopt::StructOpt;

 pub use service::{
 	AbstractService, CustomConfiguration,
 	ProvideRuntimeApi, CoreApi, ParachainHost,
+	WrappedExecutor
 };

 pub use cli::{VersionInfo, IntoExit, NoCustom};
 pub use cli::{display_role, error};

-type BoxedFuture = Box<dyn futures01::Future<Item = (), Error = ()> + Send>;
-/// Abstraction over an executor that lets you spawn tasks in the background.
-pub type TaskExecutor = Arc<dyn futures01::future::Executor<BoxedFuture> + Send + Sync>;
-
 fn load_spec(id: &str) -> Result<Option<service::ChainSpec>, String> {
 	Ok(match ChainSpec::from(id) {
 		Some(spec) => Some(spec.load()?),
@@ -62,13 +63,14 @@ pub trait Worker: IntoExit {
 	fn configuration(&self) -> service::CustomConfiguration { Default::default() }

 	/// Do work and schedule exit.
-	fn work<S, SC, B, CE>(self, service: &S, executor: TaskExecutor) -> Self::Work
+	fn work<S, SC, B, CE, SP>(self, service: &S, spawner: SP) -> Self::Work
 	where S: AbstractService<Block = service::Block, RuntimeApi = service::RuntimeApi,
 		Backend = B, SelectChain = SC,
 		NetworkSpecialization = service::PolkadotProtocol, CallExecutor = CE>,
 		SC: service::SelectChain<service::Block> + 'static,
 		B: service::Backend<service::Block, service::Blake2Hasher> + 'static,
-		CE: service::CallExecutor<service::Block, service::Blake2Hasher> + Clone + Send + Sync + 'static;
+		CE: service::CallExecutor<service::Block, service::Blake2Hasher> + Clone + Send + Sync + 'static,
+		SP: Spawn + Clone + Send + Sync + 'static;
 }

 #[derive(Debug, StructOpt, Clone)]
@@ -147,8 +149,13 @@ pub fn run<W>(worker: W, version: cli::VersionInfo) -> error::Result<()> where
 		cli::ParseAndPrepare::RevertChain(cmd) => cmd.run_with_builder::<(), _, _, _, _, _>(|config|
 			Ok(service::new_chain_ops(config)?), load_spec),
 		cli::ParseAndPrepare::CustomCommand(PolkadotSubCommands::ValidationWorker(args)) => {
-			service::run_validation_worker(&args.mem_id)?;
-			Ok(())
+			if cfg!(feature = "browser") {
+				Err(error::Error::Input("Cannot run validation worker in browser".into()))
+			} else {
+				#[cfg(not(feature = "browser"))]
+				service::run_validation_worker(&args.mem_id)?;
+				Ok(())
+			}
 		}
 	}
 }
@@ -180,7 +187,7 @@ fn run_until_exit<T, SC, B, CE, W>(
 	// but we need to keep holding a reference to the global telemetry guard
 	let _telemetry = service.telemetry();

-	let work = worker.work(&service, Arc::new(executor));
+	let work = worker.work(&service, WrappedExecutor(executor));
 	let service = service
 		.map_err(|err| error!("Error while running Service: {}", err))
 		.compat();