Add Subscription RPC for Grandpa Finality (#5732)

* Rough skeleton for what I think the RPC should look like

* Create channel for sending justifications

Sends finalized header and justification from Grandpa to the
client. This lays the groundwork for hooking into the RPC module.

* WIP: Add subscribers for justifications to Grandpa

Adds the Sender end of a channel into Grandpa, through which notifications
about block finality events can be sent.

* WIP: Add a struct for managing subscriptions

Slightly different approach from the last commit, but same
basic idea. Still a rough sketch, very much doesn't compile yet.

* Make naming more clear and lock data in Arc

* Rough idea of what RPC would look like

* Remove code from previous approach

* Missed some things

* Update client/rpc-api/src/chain/mod.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Update client/rpc-api/src/chain/mod.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Split justification subscription into sender and receiver halves

* Replace RwLock with a Mutex

* Add sample usage from the Service's point of view

* Remove code that referred to "chain_" RPC

* Use the Justification sender/receivers from Grandpa LinkHalf

* Add some PubSub boilerplate

* Add guiding comments

* TMP: comment out to fix compilation

* Return MetaIoHandler from PubSubHandler in create_full

* Uncomment pubsub methods in rpc handler (fails to build)

* node/rpc: make Metadata concrete in create_full to fix compilation

* node: pass in SubscriptionManger to grandpa rpc handler

* grandpa-rpc: use SubscriptionManger to add subscriber

* grandpa-rpc: attempt at setting up the justification stream (fails to build)

* grandpa-rpc: fix compilation of connecting stream to sink

* grandpa-rpc: implement unsubscribe

* grandpa-rpc: update older tests

* grandpa-rpc: add full prefix to avoid confusing rust-analyzer

* grandpa-rpc: add test for pubsub not available

* grandpa-rpc: tidy up leftover code

* grandpa-rpc: add test for sub and unsub of justifications

* grandpa-rpc: minor stylistic changes

* grandpa-rpc: split unit test

* grandpa-rpc: minor stylistic changes in test

* grandpa-rpc: skip returning future when cancelling

* grandpa-rpc: reuse testing executor from sc-rpc

* grandpa-rpc: don't need to use PubSubHandler in tests

* node-rpc: use MetaIoHandler rather than PubSubHandler

* grandpa: log if getting header failed

* grandpa: move justification channel creation into factory function

* grandpa: make the justification sender optional

* grandpa: fix compilation warnings

* grandpa: move justification notification types to new file

* grandpa-rpc: move JustificationNotification to grandpa-rpc

* grandpa-rpc: move JustificationNotification to its own file

* grandpa: rename justification channel pairs

* grandpa: rename notifier types

* grandpa: pass justification as GrandpaJustification to the rpc module

* Move Metadata to sc-rpc-api

* grandpa-rpc: remove unsed error code

* grandpa: fix bug for checking if channel is closed before sendind

* grandpa-rpc: unit test for sending justifications

* grandpa-rpc: update comments for the pubsub test

* grandpa-rpc: update pubsub tests with more steps

* grandpa-rpc: fix pubsub test

* grandpa-rpc: minor indendation

* grandpa-rpc: decode instead of encode in test

* grandpa: fix review comments

* grandpa: remove unused serde dependency

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>
Co-authored-by: Jon Häggblad <jon.haggblad@gmail.com>
Co-authored-by: Tomasz Drwięga <tomasz@parity.io>

substrate/client/rpc/src/author/mod.rs

@@ -94,7 +94,7 @@ impl<P, Client> AuthorApi<TxHash<P>, BlockHash<P>> for Author<P, Client>
 		Client: HeaderBackend<P::Block> + ProvideRuntimeApi<P::Block> + Send + Sync + 'static,
 		Client::Api: SessionKeys<P::Block, Error = ClientError>,
 {
-	type Metadata = crate::metadata::Metadata;
+	type Metadata = crate::Metadata;
 
 	fn insert_key(
 		&self,

substrate/client/rpc/src/chain/mod.rs

@@ -106,7 +106,7 @@ trait ChainBackend<Client, Block: BlockT>: Send + Sync + 'static
 	/// All new head subscription
 	fn subscribe_all_heads(
 		&self,
-		_metadata: crate::metadata::Metadata,
+		_metadata: crate::Metadata,
 		subscriber: Subscriber<Block::Header>,
 	) {
 		subscribe_headers(
@@ -123,7 +123,7 @@ trait ChainBackend<Client, Block: BlockT>: Send + Sync + 'static
 	/// Unsubscribe from all head subscription.
 	fn unsubscribe_all_heads(
 		&self,
-		_metadata: Option<crate::metadata::Metadata>,
+		_metadata: Option<crate::Metadata>,
 		id: SubscriptionId,
 	) -> RpcResult<bool> {
 		Ok(self.subscriptions().cancel(id))
@@ -132,7 +132,7 @@ trait ChainBackend<Client, Block: BlockT>: Send + Sync + 'static
 	/// New best head subscription
 	fn subscribe_new_heads(
 		&self,
-		_metadata: crate::metadata::Metadata,
+		_metadata: crate::Metadata,
 		subscriber: Subscriber<Block::Header>,
 	) {
 		subscribe_headers(
@@ -150,7 +150,7 @@ trait ChainBackend<Client, Block: BlockT>: Send + Sync + 'static
 	/// Unsubscribe from new best head subscription.
 	fn unsubscribe_new_heads(
 		&self,
-		_metadata: Option<crate::metadata::Metadata>,
+		_metadata: Option<crate::Metadata>,
 		id: SubscriptionId,
 	) -> RpcResult<bool> {
 		Ok(self.subscriptions().cancel(id))
@@ -159,7 +159,7 @@ trait ChainBackend<Client, Block: BlockT>: Send + Sync + 'static
 	/// Finalized head subscription
 	fn subscribe_finalized_heads(
 		&self,
-		_metadata: crate::metadata::Metadata,
+		_metadata: crate::Metadata,
 		subscriber: Subscriber<Block::Header>,
 	) {
 		subscribe_headers(
@@ -176,7 +176,7 @@ trait ChainBackend<Client, Block: BlockT>: Send + Sync + 'static
 	/// Unsubscribe from finalized head subscription.
 	fn unsubscribe_finalized_heads(
 		&self,
-		_metadata: Option<crate::metadata::Metadata>,
+		_metadata: Option<crate::Metadata>,
 		id: SubscriptionId,
 	) -> RpcResult<bool> {
 		Ok(self.subscriptions().cancel(id))
@@ -230,7 +230,7 @@ impl<Block, Client> ChainApi<NumberFor<Block>, Block::Hash, Block::Header, Signe
 			Block: BlockT + 'static,
 			Client: HeaderBackend<Block> + BlockchainEvents<Block> + 'static,
 {
-	type Metadata = crate::metadata::Metadata;
+	type Metadata = crate::Metadata;
 
 	fn header(&self, hash: Option<Block::Hash>) -> FutureResult<Option<Block::Header>> {
 		self.backend.header(hash)

substrate/client/rpc/src/lib.rs

@@ -27,10 +27,7 @@ use rpc::futures::future::{Executor, ExecuteError, Future};
 use sp_core::traits::SpawnNamed;
 use std::sync::Arc;
 
-mod metadata;
-
-pub use sc_rpc_api::DenyUnsafe;
-pub use self::metadata::Metadata;
+pub use sc_rpc_api::{DenyUnsafe, Metadata};
 pub use rpc::IoHandlerExtension as RpcExtension;
 
 pub mod author;
@@ -38,8 +35,9 @@ pub mod chain;
 pub mod offchain;
 pub mod state;
 pub mod system;
-#[cfg(test)]
-mod testing;
+
+#[cfg(any(test, feature = "test-helpers"))]
+pub mod testing;
 
 /// Task executor that is being used by RPC subscriptions.
 #[derive(Clone)]

substrate/client/rpc/src/metadata.rs

@@ -1,62 +0,0 @@
-// This file is part of Substrate.
-
-// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd.
-// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-//! RPC Metadata
-use std::sync::Arc;
-
-use jsonrpc_pubsub::{Session, PubSubMetadata};
-use rpc::futures::sync::mpsc;
-
-/// RPC Metadata.
-///
-/// Manages persistent session for transports that support it
-/// and may contain some additional info extracted from specific transports
-/// (like remote client IP address, request headers, etc)
-#[derive(Default, Clone)]
-pub struct Metadata {
-	session: Option<Arc<Session>>,
-}
-
-impl rpc::Metadata for Metadata {}
-impl PubSubMetadata for Metadata {
-	fn session(&self) -> Option<Arc<Session>> {
-		self.session.clone()
-	}
-}
-
-impl Metadata {
-	/// Create new `Metadata` with session (Pub/Sub) support.
-	pub fn new(transport: mpsc::Sender<String>) -> Self {
-		Metadata {
-			session: Some(Arc::new(Session::new(transport))),
-		}
-	}
-
-	/// Create new `Metadata` for tests.
-	#[cfg(test)]
-	pub fn new_test() -> (mpsc::Receiver<String>, Self) {
-		let (tx, rx) = mpsc::channel(1);
-		(rx, Self::new(tx))
-	}
-}
-
-impl From<mpsc::Sender<String>> for Metadata {
-	fn from(sender: mpsc::Sender<String>) -> Self {
-		Self::new(sender)
-	}
-}

substrate/client/rpc/src/state/mod.rs

@@ -140,21 +140,21 @@ pub trait StateBackend<Block: BlockT, Client>: Send + Sync + 'static
 	/// New runtime version subscription
 	fn subscribe_runtime_version(
 		&self,
-		_meta: crate::metadata::Metadata,
+		_meta: crate::Metadata,
 		subscriber: Subscriber<RuntimeVersion>,
 	);
 
 	/// Unsubscribe from runtime version subscription
 	fn unsubscribe_runtime_version(
 		&self,
-		_meta: Option<crate::metadata::Metadata>,
+		_meta: Option<crate::Metadata>,
 		id: SubscriptionId,
 	) -> RpcResult<bool>;
 
 	/// New storage subscription
 	fn subscribe_storage(
 		&self,
-		_meta: crate::metadata::Metadata,
+		_meta: crate::Metadata,
 		subscriber: Subscriber<StorageChangeSet<Block::Hash>>,
 		keys: Option<Vec<StorageKey>>,
 	);
@@ -162,7 +162,7 @@ pub trait StateBackend<Block: BlockT, Client>: Send + Sync + 'static
 	/// Unsubscribe from storage subscription
 	fn unsubscribe_storage(
 		&self,
-		_meta: Option<crate::metadata::Metadata>,
+		_meta: Option<crate::Metadata>,
 		id: SubscriptionId,
 	) -> RpcResult<bool>;
 }
@@ -230,7 +230,7 @@ impl<Block, Client> StateApi<Block::Hash> for State<Block, Client>
 		Block: BlockT + 'static,
 		Client: Send + Sync + 'static,
 {
-	type Metadata = crate::metadata::Metadata;
+	type Metadata = crate::Metadata;
 
 	fn call(&self, method: String, data: Bytes, block: Option<Block::Hash>) -> FutureResult<Bytes> {
 		self.backend.call(block, method, data)
@@ -390,7 +390,7 @@ impl<Block, Client> ChildStateApi<Block::Hash> for ChildState<Block, Client>
 		Block: BlockT + 'static,
 		Client: Send + Sync + 'static,
 {
-	type Metadata = crate::metadata::Metadata;
+	type Metadata = crate::Metadata;
 
 	fn storage(
 		&self,

substrate/client/rpc/src/state/state_full.rs

@@ -373,7 +373,7 @@ impl<BE, Block, Client> StateBackend<Block, Client> for FullState<BE, Block, Cli
 
 	fn subscribe_runtime_version(
 		&self,
-		_meta: crate::metadata::Metadata,
+		_meta: crate::Metadata,
 		subscriber: Subscriber<RuntimeVersion>,
 	) {
 		let stream = match self.client.storage_changes_notification_stream(
@@ -424,7 +424,7 @@ impl<BE, Block, Client> StateBackend<Block, Client> for FullState<BE, Block, Cli
 
 	fn unsubscribe_runtime_version(
 		&self,
-		_meta: Option<crate::metadata::Metadata>,
+		_meta: Option<crate::Metadata>,
 		id: SubscriptionId,
 	) -> RpcResult<bool> {
 		Ok(self.subscriptions.cancel(id))
@@ -432,7 +432,7 @@ impl<BE, Block, Client> StateBackend<Block, Client> for FullState<BE, Block, Cli
 
 	fn subscribe_storage(
 		&self,
-		_meta: crate::metadata::Metadata,
+		_meta: crate::Metadata,
 		subscriber: Subscriber<StorageChangeSet<Block::Hash>>,
 		keys: Option<Vec<StorageKey>>,
 	) {
@@ -484,7 +484,7 @@ impl<BE, Block, Client> StateBackend<Block, Client> for FullState<BE, Block, Cli
 
 	fn unsubscribe_storage(
 		&self,
-		_meta: Option<crate::metadata::Metadata>,
+		_meta: Option<crate::Metadata>,
 		id: SubscriptionId,
 	) -> RpcResult<bool> {
 		Ok(self.subscriptions.cancel(id))

substrate/client/rpc/src/state/state_light.rs

@@ -289,7 +289,7 @@ impl<Block, F, Client> StateBackend<Block, Client> for LightState<Block, F, Clie
 
 	fn subscribe_storage(
 		&self,
-		_meta: crate::metadata::Metadata,
+		_meta: crate::Metadata,
 		subscriber: Subscriber<StorageChangeSet<Block::Hash>>,
 		keys: Option<Vec<StorageKey>>
 	) {
@@ -384,7 +384,7 @@ impl<Block, F, Client> StateBackend<Block, Client> for LightState<Block, F, Clie
 
 	fn unsubscribe_storage(
 		&self,
-		_meta: Option<crate::metadata::Metadata>,
+		_meta: Option<crate::Metadata>,
 		id: SubscriptionId,
 	) -> RpcResult<bool> {
 		if !self.subscriptions.cancel(id.clone()) {
@@ -412,7 +412,7 @@ impl<Block, F, Client> StateBackend<Block, Client> for LightState<Block, F, Clie
 
 	fn subscribe_runtime_version(
 		&self,
-		_meta: crate::metadata::Metadata,
+		_meta: crate::Metadata,
 		subscriber: Subscriber<RuntimeVersion>,
 	) {
 		self.subscriptions.add(subscriber, move |sink| {
@@ -459,7 +459,7 @@ impl<Block, F, Client> StateBackend<Block, Client> for LightState<Block, F, Clie
 
 	fn unsubscribe_runtime_version(
 		&self,
-		_meta: Option<crate::metadata::Metadata>,
+		_meta: Option<crate::Metadata>,
 		id: SubscriptionId,
 	) -> RpcResult<bool> {
 		Ok(self.subscriptions.cancel(id))

substrate/client/rpc/src/testing.rs

@@ -32,6 +32,7 @@ lazy_static::lazy_static! {
 
 type Boxed01Future01 = Box<dyn future01::Future<Item = (), Error = ()> + Send + 'static>;
 
+/// Executor for use in testing
 pub struct TaskExecutor;
 impl future01::Executor<Boxed01Future01> for TaskExecutor {
 	fn execute(