Extrinsic pool (#182)

* Use latest version of txpool. * Initial version of the pool. * Fix abstraction. * Implement watchers and notifications. * Return hash from RPC. * Remove commented code. * Remove client dep. * Fix tests.
2026-07-14 22:35:43 +00:00 · 2018-05-31 22:49:17 +02:00
parent 44eaa4a180
commit 1dada4f7a0
27 changed files with 770 additions and 315 deletions
@@ -0,0 +1,12 @@
+[package]
+name = "substrate-extrinsic-pool"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+
+[dependencies]
+error-chain = "0.11"
+futures = "0.1"
+log = "0.3"
+parking_lot = "0.4"
+substrate-primitives = { path = "../primitives" }
+transaction-pool = "1.12"
@@ -0,0 +1,64 @@
+// Copyright 2018 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! External API for extrinsic pool.
+
+use std::ops::Deref;
+use txpool::{self, VerifiedTransaction};
+use primitives::{
+	Hash,
+	block::{Extrinsic, ExtrinsicHash},
+};
+
+/// Extrinsic pool error.
+pub trait Error: ::std::error::Error + Send + Sized {
+	/// Try to extract original `txpool::Error`
+	///
+	/// This implementation is optional and used only to
+	/// provide more descriptive error messages for end users
+	/// of RPC API.
+	fn into_pool_error(self) -> Result<txpool::Error, Self> { Err(self) }
+}
+
+impl Error for txpool::Error {
+	fn into_pool_error(self) -> Result<txpool::Error, Self> { Ok(self) }
+}
+
+/// Extrinsic pool.
+pub trait ExtrinsicPool: Send + Sync + 'static {
+	/// Error type
+	type Error: Error;
+
+	/// Submit a collection of extrinsics to the pool.
+	fn submit(&self, xt: Vec<Extrinsic>) -> Result<Vec<ExtrinsicHash>, Self::Error>;
+}
+
+// Blanket implementation for anything that `Derefs` to the pool.
+impl<V, S, E, T> ExtrinsicPool for T where
+	T: Deref<Target=super::Pool<V, S, E>> + Send + Sync + 'static,
+	V: txpool::Verifier<Extrinsic>,
+	S: txpool::Scoring<V::VerifiedTransaction>,
+	V::VerifiedTransaction: txpool::VerifiedTransaction<Hash=Hash>,
+	E: From<V::Error>,
+	E: From<txpool::Error>,
+	E: Error,
+{
+	type Error = E;
+
+	fn submit(&self, xt: Vec<Extrinsic>) -> Result<Vec<ExtrinsicHash>, Self::Error> {
+		self.deref().submit(xt).map(|result| result.into_iter().map(|xt| *xt.hash()).collect())
+	}
+}
@@ -0,0 +1,37 @@
+// Copyright 2018 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+#![warn(missing_docs)]
+
+//! Generic extrinsic pool.
+
+extern crate futures;
+extern crate parking_lot;
+extern crate substrate_primitives as primitives;
+
+#[macro_use]
+extern crate log;
+
+pub extern crate transaction_pool as txpool;
+
+pub mod api;
+
+mod listener;
+mod pool;
+mod watcher;
+
+pub use self::pool::Pool;
+pub use self::watcher::Watcher;
@@ -0,0 +1,89 @@
+// Copyright 2018 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+	sync::Arc,
+	collections::HashMap,
+};
+use primitives::Hash;
+use txpool;
+
+use watcher;
+
+#[derive(Default)]
+pub struct Listener {
+	watchers: HashMap<Hash, watcher::Sender>
+}
+
+impl Listener {
+	pub fn create_watcher<T: txpool::VerifiedTransaction<Hash=Hash>>(&mut self, xt: Arc<T>) -> watcher::Watcher {
+		let sender = self.watchers.entry(*xt.hash()).or_insert_with(watcher::Sender::default);
+		sender.new_watcher()
+	}
+
+	pub fn broadcasted(&mut self, hash: &Hash, peers: Vec<String>) {
+		self.fire(hash, |watcher| watcher.broadcast(peers));
+	}
+
+	fn fire<F: FnOnce(&mut watcher::Sender)>(&mut self, hash: &Hash, fun: F) {
+		let clean = if let Some(h) = self.watchers.get_mut(hash) {
+			fun(h);
+			h.is_done()
+		} else {
+			false
+		};
+
+		if clean {
+			self.watchers.remove(hash);
+		}
+	}
+}
+
+impl<T: txpool::VerifiedTransaction<Hash=Hash>> txpool::Listener<T> for Listener {
+	fn added(&mut self, tx: &Arc<T>, old: Option<&Arc<T>>) {
+		if let Some(old) = old {
+			let hash = tx.hash();
+			self.fire(old.hash(), |watcher| watcher.usurped(*hash));
+		}
+	}
+
+	fn dropped(&mut self, tx: &Arc<T>, by: Option<&T>) {
+		self.fire(tx.hash(), |watcher| match by {
+			Some(t) => watcher.usurped(*t.hash()),
+			None => watcher.dropped(),
+		})
+	}
+
+	fn rejected(&mut self, tx: &Arc<T>, reason: &txpool::ErrorKind) {
+		warn!("Extrinsic rejected ({}): {:?}", reason, tx);
+	}
+
+	fn invalid(&mut self, tx: &Arc<T>) {
+		warn!("Extrinsic invalid: {:?}", tx);
+	}
+
+	fn canceled(&mut self, tx: &Arc<T>) {
+		warn!("Extrinsic canceled: {:?}", tx);
+	}
+
+	fn mined(&mut self, tx: &Arc<T>) {
+		// TODO [ToDr] latest block number?
+		let header_hash = 1.into();
+		self.fire(tx.hash(), |watcher| watcher.finalised(header_hash))
+	}
+}
+
+
@@ -0,0 +1,141 @@
+// Copyright 2018 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+	collections::HashMap,
+	marker::PhantomData,
+	sync::{Arc, Weak},
+};
+
+use futures::sync::mpsc;
+use parking_lot::{RwLock, Mutex};
+use txpool;
+use primitives::{Hash, block::Extrinsic};
+
+use listener::Listener;
+use watcher::Watcher;
+
+/// Extrinsics pool.
+pub struct Pool<V, S, E> where
+	V: txpool::Verifier<Extrinsic>,
+	S: txpool::Scoring<V::VerifiedTransaction>,
+{
+	_error: Mutex<PhantomData<E>>,
+	pool: RwLock<txpool::Pool<
+		V::VerifiedTransaction,
+		S,
+		Listener,
+	>>,
+	verifier: V,
+	import_notification_sinks: Mutex<Vec<mpsc::UnboundedSender<Weak<V::VerifiedTransaction>>>>,
+}
+
+impl<V, S, E> Pool<V, S, E> where
+	V: txpool::Verifier<Extrinsic>,
+	S: txpool::Scoring<V::VerifiedTransaction>,
+	V::VerifiedTransaction: txpool::VerifiedTransaction<Hash=Hash>,
+	E: From<V::Error>,
+	E: From<txpool::Error>,
+{
+	/// Create a new transaction pool.
+	pub fn new(options: txpool::Options, verifier: V, scoring: S) -> Self {
+		Pool {
+			_error: Default::default(),
+			pool: RwLock::new(txpool::Pool::new(Listener::default(), scoring, options)),
+			verifier,
+			import_notification_sinks: Default::default(),
+		}
+	}
+
+	/// Imports a pre-verified extrinsic to the pool.
+	pub fn import(&self, xt: V::VerifiedTransaction) -> Result<Arc<V::VerifiedTransaction>, E> {
+		let result = self.pool.write().import(xt)?;
+
+		let weak = Arc::downgrade(&result);
+		self.import_notification_sinks.lock()
+			.retain(|sink| sink.unbounded_send(weak.clone()).is_ok());
+
+		Ok(result)
+	}
+
+	/// Return an event stream of transactions imported to the pool.
+	pub fn import_notification_stream(&self) -> mpsc::UnboundedReceiver<Weak<V::VerifiedTransaction>> {
+		let (sink, stream) = mpsc::unbounded();
+		self.import_notification_sinks.lock().push(sink);
+		stream
+	}
+
+	/// Invoked when extrinsics are broadcasted.
+	pub fn on_broadcasted(&self, propagated: HashMap<Hash, Vec<String>>) {
+		for (hash, peers) in propagated.into_iter() {
+			self.pool.write().listener_mut().broadcasted(&hash, peers);
+		}
+	}
+
+	/// Imports a bunch of extrinsics to the pool
+	pub fn submit(&self, xts: Vec<Extrinsic>) -> Result<Vec<Arc<V::VerifiedTransaction>>, E> {
+		xts
+			.into_iter()
+			.map(|xt| self.verifier.verify_transaction(xt))
+			.map(|xt| {
+				Ok(self.pool.write().import(xt?)?)
+			})
+			.collect()
+	}
+
+	/// Import a single extrinsic and starts to watch their progress in the pool.
+	pub fn submit_and_watch(&self, xt: Extrinsic) -> Result<Watcher, E> {
+		let xt = self.submit(vec![xt])?.pop().expect("One extrinsic passed; one result returned; qed");
+		Ok(self.pool.write().listener_mut().create_watcher(xt))
+	}
+
+	/// Remove from the pool.
+	pub fn remove(&self, hashes: &[Hash], is_valid: bool) -> Vec<Option<Arc<V::VerifiedTransaction>>> {
+		let mut pool = self.pool.write();
+		let mut results = Vec::with_capacity(hashes.len());
+		for hash in hashes {
+			results.push(pool.remove(hash, is_valid));
+		}
+		results
+	}
+
+	/// Cull transactions from the queue.
+	pub fn cull<R>(&self, senders: Option<&[<V::VerifiedTransaction as txpool::VerifiedTransaction>::Sender]>, ready: R) -> usize where
+		R: txpool::Ready<V::VerifiedTransaction>,
+	{
+		self.pool.write().cull(senders, ready)
+	}
+
+	/// Cull transactions from the queue and then compute the pending set.
+	pub fn cull_and_get_pending<R, F, T>(&self, ready: R, f: F) -> T where
+		R: txpool::Ready<V::VerifiedTransaction> + Clone,
+		F: FnOnce(txpool::PendingIterator<V::VerifiedTransaction, R, S, Listener>) -> T,
+	{
+		let mut pool = self.pool.write();
+		pool.cull(None, ready.clone());
+		f(pool.pending(ready))
+	}
+
+	/// Get the full status of the queue (including readiness)
+	pub fn status<R: txpool::Ready<V::VerifiedTransaction>>(&self, ready: R) -> txpool::Status {
+		self.pool.read().status(ready)
+	}
+
+	/// Returns light status of the pool.
+	pub fn light_status(&self) -> txpool::LightStatus {
+		self.pool.read().light_status()
+	}
+}
@@ -0,0 +1,88 @@
+// Copyright 2018 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use futures::sync::mpsc;
+use primitives::{
+	block::{HeaderHash, ExtrinsicHash}
+};
+
+/// Possible extrinsic status events
+#[derive(Debug, Clone)]
+pub enum Status {
+	/// Extrinsic has been finalised in block with given hash.
+	Finalised(HeaderHash),
+	/// Some state change (perhaps another extrinsic was included) rendered this extrinsic invalid.
+	Usurped(ExtrinsicHash),
+	/// The extrinsic has been broadcast to the given peers.
+	Broadcast(Vec<String>),
+	/// Extrinsic has been dropped from the pool because of the limit.
+	Dropped,
+}
+
+/// Extrinsic watcher.
+///
+/// Represents a stream of status updates for particular extrinsic.
+#[derive(Debug)]
+pub struct Watcher {
+	receiver: mpsc::UnboundedReceiver<Status>,
+}
+
+#[derive(Debug, Default)]
+pub(crate) struct Sender {
+	receivers: Vec<mpsc::UnboundedSender<Status>>,
+	finalised: bool,
+}
+
+impl Sender {
+	/// Add a new watcher to this sender object.
+	pub fn new_watcher(&mut self) -> Watcher {
+		let (tx, receiver) = mpsc::unbounded();
+		self.receivers.push(tx);
+		Watcher {
+			receiver,
+		}
+	}
+
+	/// Some state change (perhaps another extrinsic was included) rendered this extrinsic invalid.
+	pub fn usurped(&mut self, hash: ExtrinsicHash) {
+		self.send(Status::Usurped(hash))
+	}
+
+	/// Extrinsic has been finalised in block with given hash.
+	pub fn finalised(&mut self, hash: HeaderHash) {
+		self.send(Status::Finalised(hash));
+		self.finalised = true;
+	}
+
+	/// Transaction has been dropped from the pool because of the limit.
+	pub fn dropped(&mut self) {
+		self.send(Status::Dropped);
+	}
+
+	/// The extrinsic has been broadcast to the given peers.
+	pub fn broadcast(&mut self, peers: Vec<String>) {
+		self.send(Status::Broadcast(peers))
+	}
+
+	/// Returns true if the are no more listeners for this extrinsic or it was finalised.
+	pub fn is_done(&self) -> bool {
+		self.finalised || self.receivers.is_empty()
+	}
+
+	fn send(&mut self, status: Status) {
+		self.receivers.retain(|sender| sender.unbounded_send(status.clone()).is_ok())
+	}
+}