Custom RPC for Merkle Mountain Range pallet (#8137)

* Add MMR custom RPC. * Change RuntimeApi to avoid hardcoding leaf type. * Properly implement the new RuntimeAPI and wire up RPC. * Extract Offchain DB as separate execution extension. * Enable offchain DB access for offchain calls. * Fix offchain_election tests. * Skip block initialisation for proof generation. * Fix integration test setup. * Fix offchain tests. Not sure how I missed them earlier 🤷. * Fix long line. * One more test missing. * Update mock for multi-phase. * Address review grumbbles. * Address review grumbles. * Fix line width of a comment
2026-06-09 20:11:09 +00:00 · 2021-03-10 16:28:56 +01:00
parent 9637faae0c
commit f3d4355a20
37 changed files with 837 additions and 350 deletions
@@ -26,12 +26,11 @@ use std::{

 use crate::NetworkProvider;
 use futures::Future;
-use log::error;
 use sc_network::{PeerId, Multiaddr};
 use codec::{Encode, Decode};
 use sp_core::OpaquePeerId;
 use sp_core::offchain::{
-	Externalities as OffchainExt, HttpRequestId, Timestamp, HttpRequestStatus, HttpError,
+	self, HttpRequestId, Timestamp, HttpRequestStatus, HttpError,
 	OffchainStorage, OpaqueNetworkState, OpaqueMultiaddr, StorageKind,
 };
 pub use sp_offchain::STORAGE_PREFIX;
@@ -47,22 +46,9 @@ mod http_dummy;

 mod timestamp;

-/// Asynchronous offchain API.
-///
-/// NOTE this is done to prevent recursive calls into the runtime (which are not supported currently).
-pub(crate) struct Api<Storage> {
-	/// Offchain Workers database.
-	db: Storage,
-	/// A provider for substrate networking.
-	network_provider: Arc<dyn NetworkProvider + Send + Sync>,
-	/// Is this node a potential validator?
-	is_validator: bool,
-	/// Everything HTTP-related is handled by a different struct.
-	http: http::HttpApi,
-}
-
 fn unavailable_yet<R: Default>(name: &str) -> R {
-	error!(
+	log::error!(
+		target: "sc_offchain",
 		"The {:?} API is not available for offchain workers yet. Follow \
 		https://github.com/paritytech/substrate/issues/1458 for details", name
 	);
@@ -71,7 +57,109 @@ fn unavailable_yet<R: Default>(name: &str) -> R {

 const LOCAL_DB: &str = "LOCAL (fork-aware) DB";

-impl<Storage: OffchainStorage> OffchainExt for Api<Storage> {
+/// Offchain DB reference.
+#[derive(Debug, Clone)]
+pub struct Db<Storage> {
+	/// Persistent storage database.
+	persistent: Storage,
+}
+
+impl<Storage: OffchainStorage> Db<Storage> {
+	/// Create new instance of Offchain DB.
+	pub fn new(persistent: Storage) -> Self {
+		Self { persistent }
+	}
+
+	/// Create new instance of Offchain DB, backed by given backend.
+	pub fn factory_from_backend<Backend, Block>(backend: &Backend) -> Option<
+		Box<dyn sc_client_api::execution_extensions::DbExternalitiesFactory>
+	> where
+		Backend: sc_client_api::Backend<Block, OffchainStorage = Storage>,
+		Block: sp_runtime::traits::Block,
+		Storage: 'static,
+	{
+		sc_client_api::Backend::offchain_storage(backend).map(|db|
+			Box::new(Self::new(db)) as _
+		)
+	}
+}
+
+impl<Storage: OffchainStorage> offchain::DbExternalities for Db<Storage> {
+	fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) {
+		log::debug!(
+			target: "sc_offchain",
+			"{:?}: Write: {:?} <= {:?}", kind, hex::encode(key), hex::encode(value)
+		);
+		match kind {
+			StorageKind::PERSISTENT => self.persistent.set(STORAGE_PREFIX, key, value),
+			StorageKind::LOCAL => unavailable_yet(LOCAL_DB),
+		}
+	}
+
+	fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]) {
+		log::debug!(
+			target: "sc_offchain",
+			"{:?}: Clear: {:?}", kind, hex::encode(key)
+		);
+		match kind {
+			StorageKind::PERSISTENT => self.persistent.remove(STORAGE_PREFIX, key),
+			StorageKind::LOCAL => unavailable_yet(LOCAL_DB),
+		}
+	}
+
+	fn local_storage_compare_and_set(
+		&mut self,
+		kind: StorageKind,
+		key: &[u8],
+		old_value: Option<&[u8]>,
+		new_value: &[u8],
+	) -> bool {
+		log::debug!(
+			target: "sc_offchain",
+			"{:?}: CAS: {:?} <= {:?} vs {:?}",
+			kind,
+			hex::encode(key),
+			hex::encode(new_value),
+			old_value.as_ref().map(hex::encode),
+		);
+		match kind {
+			StorageKind::PERSISTENT => {
+				self.persistent.compare_and_set(STORAGE_PREFIX, key, old_value, new_value)
+			},
+			StorageKind::LOCAL => unavailable_yet(LOCAL_DB),
+		}
+	}
+
+	fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option<Vec<u8>> {
+		let result = match kind {
+			StorageKind::PERSISTENT => self.persistent.get(STORAGE_PREFIX, key),
+			StorageKind::LOCAL => unavailable_yet(LOCAL_DB),
+		};
+		log::debug!(
+			target: "sc_offchain",
+			"{:?}: Read: {:?} => {:?}",
+			kind,
+			hex::encode(key),
+			result.as_ref().map(hex::encode)
+		);
+		result
+	}
+}
+
+/// Asynchronous offchain API.
+///
+/// NOTE this is done to prevent recursive calls into the runtime
+/// (which are not supported currently).
+pub(crate) struct Api {
+	/// A provider for substrate networking.
+	network_provider: Arc<dyn NetworkProvider + Send + Sync>,
+	/// Is this node a potential validator?
+	is_validator: bool,
+	/// Everything HTTP-related is handled by a different struct.
+	http: http::HttpApi,
+}
+
+impl offchain::Externalities for Api {
 	fn is_validator(&self) -> bool {
 		self.is_validator
 	}
@@ -98,42 +186,6 @@ impl<Storage: OffchainStorage> OffchainExt for Api<Storage> {
 		rand::random()
 	}

-	fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) {
-		match kind {
-			StorageKind::PERSISTENT => self.db.set(STORAGE_PREFIX, key, value),
-			StorageKind::LOCAL => unavailable_yet(LOCAL_DB),
-		}
-	}
-
-	fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]) {
-		match kind {
-			StorageKind::PERSISTENT => self.db.remove(STORAGE_PREFIX, key),
-			StorageKind::LOCAL => unavailable_yet(LOCAL_DB),
-		}
-	}
-
-	fn local_storage_compare_and_set(
-		&mut self,
-		kind: StorageKind,
-		key: &[u8],
-		old_value: Option<&[u8]>,
-		new_value: &[u8],
-	) -> bool {
-		match kind {
-			StorageKind::PERSISTENT => {
-				self.db.compare_and_set(STORAGE_PREFIX, key, old_value, new_value)
-			},
-			StorageKind::LOCAL => unavailable_yet(LOCAL_DB),
-		}
-	}
-
-	fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option<Vec<u8>> {
-		match kind {
-			StorageKind::PERSISTENT => self.db.get(STORAGE_PREFIX, key),
-			StorageKind::LOCAL => unavailable_yet(LOCAL_DB),
-		}
-	}
-
 	fn http_request_start(
 		&mut self,
 		method: &str,
@@ -270,16 +322,14 @@ pub(crate) struct AsyncApi {

 impl AsyncApi {
 	/// Creates new Offchain extensions API implementation an the asynchronous processing part.
-	pub fn new<S: OffchainStorage>(
-		db: S,
+	pub fn new(
 		network_provider: Arc<dyn NetworkProvider + Send + Sync>,
 		is_validator: bool,
 		shared_client: SharedClient,
-	) -> (Api<S>, Self) {
+	) -> (Api, Self) {
 		let (http_api, http_worker) = http::http(shared_client);

 		let api = Api {
-			db,
 			network_provider,
 			is_validator,
 			http: http_api,
@@ -303,9 +353,10 @@ impl AsyncApi {
 #[cfg(test)]
 mod tests {
 	use super::*;
-	use std::{convert::{TryFrom, TryInto}, time::SystemTime};
 	use sc_client_db::offchain::LocalStorage;
 	use sc_network::{NetworkStateInfo, PeerId};
+	use sp_core::offchain::{Externalities, DbExternalities};
+	use std::{convert::{TryFrom, TryInto}, time::SystemTime};

 	struct TestNetwork();

@@ -329,20 +380,22 @@ mod tests {
 		}
 	}

-	fn offchain_api() -> (Api<LocalStorage>, AsyncApi) {
+	fn offchain_api() -> (Api, AsyncApi) {
 		sp_tracing::try_init_simple();
-		let db = LocalStorage::new_test();
 		let mock = Arc::new(TestNetwork());
 		let shared_client = SharedClient::new();

 		AsyncApi::new(
-			db,
 			mock,
 			false,
 			shared_client,
 		)
 	}

+	fn offchain_db() -> Db<LocalStorage> {
+		Db::new(LocalStorage::new_test())
+	}
+
 	#[test]
 	fn should_get_timestamp() {
 		let mut api = offchain_api().0;
@@ -381,7 +434,7 @@ mod tests {
 	fn should_set_and_get_local_storage() {
 		// given
 		let kind = StorageKind::PERSISTENT;
-		let mut api = offchain_api().0;
+		let mut api = offchain_db();
 		let key = b"test";

 		// when
@@ -396,7 +449,7 @@ mod tests {
 	fn should_compare_and_set_local_storage() {
 		// given
 		let kind = StorageKind::PERSISTENT;
-		let mut api = offchain_api().0;
+		let mut api = offchain_db();
 		let key = b"test";
 		api.local_storage_set(kind, key, b"value");

@@ -413,7 +466,7 @@ mod tests {
 	fn should_compare_and_set_local_storage_with_none() {
 		// given
 		let kind = StorageKind::PERSISTENT;
-		let mut api = offchain_api().0;
+		let mut api = offchain_db();
 		let key = b"test";

 		// when
@@ -46,13 +46,13 @@ use sp_api::{ApiExt, ProvideRuntimeApi};
 use futures::future::Future;
 use log::{debug, warn};
 use sc_network::{ExHashT, NetworkService, NetworkStateInfo, PeerId};
-use sp_core::{offchain::{self, OffchainStorage}, ExecutionContext, traits::SpawnNamed};
+use sp_core::{offchain, ExecutionContext, traits::SpawnNamed};
 use sp_runtime::{generic::BlockId, traits::{self, Header}};
 use futures::{prelude::*, future::ready};

 mod api;
-use api::SharedClient;

+pub use api::Db as OffchainDb;
 pub use sp_offchain::{OffchainWorkerApi, STORAGE_PREFIX};

 /// NetworkProvider provides [`OffchainWorkers`] with all necessary hooks into the
@@ -80,21 +80,19 @@ where
 }

 /// An offchain workers manager.
-pub struct OffchainWorkers<Client, Storage, Block: traits::Block> {
+pub struct OffchainWorkers<Client, Block: traits::Block> {
 	client: Arc<Client>,
-	db: Storage,
 	_block: PhantomData<Block>,
 	thread_pool: Mutex<ThreadPool>,
-	shared_client: SharedClient,
+	shared_client: api::SharedClient,
 }

-impl<Client, Storage, Block: traits::Block> OffchainWorkers<Client, Storage, Block> {
+impl<Client, Block: traits::Block> OffchainWorkers<Client, Block> {
 	/// Creates new `OffchainWorkers`.
-	pub fn new(client: Arc<Client>, db: Storage) -> Self {
-		let shared_client = SharedClient::new();
+	pub fn new(client: Arc<Client>) -> Self {
+		let shared_client = api::SharedClient::new();
 		Self {
 			client,
-			db,
 			_block: PhantomData,
 			thread_pool: Mutex::new(ThreadPool::new(num_cpus::get())),
 			shared_client,
@@ -102,9 +100,8 @@ impl<Client, Storage, Block: traits::Block> OffchainWorkers<Client, Storage, Blo
 	}
 }

-impl<Client, Storage, Block: traits::Block> fmt::Debug for OffchainWorkers<
+impl<Client, Block: traits::Block> fmt::Debug for OffchainWorkers<
 	Client,
-	Storage,
 	Block,
 > {
 	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
@@ -112,15 +109,13 @@ impl<Client, Storage, Block: traits::Block> fmt::Debug for OffchainWorkers<
 	}
 }

-impl<Client, Storage, Block> OffchainWorkers<
+impl<Client, Block> OffchainWorkers<
 	Client,
-	Storage,
 	Block,
 > where
 	Block: traits::Block,
 	Client: ProvideRuntimeApi<Block> + Send + Sync + 'static,
 	Client::Api: OffchainWorkerApi<Block>,
-	Storage: OffchainStorage + 'static,
 {
 	/// Start the offchain workers after given block.
 	#[must_use]
@@ -150,7 +145,6 @@ impl<Client, Storage, Block> OffchainWorkers<
 		debug!("Checking offchain workers at {:?}: version:{}", at, version);
 		if version > 0 {
 			let (api, runner) = api::AsyncApi::new(
-				self.db.clone(),
 				network_provider,
 				is_validator,
 				self.shared_client.clone(),
@@ -197,10 +191,10 @@ impl<Client, Storage, Block> OffchainWorkers<
 }

 /// Inform the offchain worker about new imported blocks
-pub async fn notification_future<Client, Storage, Block, Spawner>(
+pub async fn notification_future<Client, Block, Spawner>(
 	is_validator: bool,
 	client: Arc<Client>,
-	offchain: Arc<OffchainWorkers<Client, Storage, Block>>,
+	offchain: Arc<OffchainWorkers<Client, Block>>,
 	spawner: Spawner,
 	network_provider: Arc<dyn NetworkProvider + Send + Sync>,
 )
@@ -208,7 +202,6 @@ pub async fn notification_future<Client, Storage, Block, Spawner>(
 		Block: traits::Block,
 		Client: ProvideRuntimeApi<Block> + sc_client_api::BlockchainEvents<Block> + Send + Sync + 'static,
 		Client::Api: OffchainWorkerApi<Block>,
-		Storage: OffchainStorage + 'static,
 		Spawner: SpawnNamed
 {
 	client.import_notification_stream().for_each(move |n| {
@@ -300,12 +293,11 @@ mod tests {
 			spawner,
 			client.clone(),
 		));
-		let db = sc_client_db::offchain::LocalStorage::new_test();
 		let network = Arc::new(TestNetwork());
 		let header = client.header(&BlockId::number(0)).unwrap().unwrap();

 		// when
-		let offchain = OffchainWorkers::new(client, db);
+		let offchain = OffchainWorkers::new(client);
 		futures::executor::block_on(
 			offchain.on_block_imported(&header, network, false)
 		);
@@ -317,6 +309,8 @@ mod tests {

 	#[test]
 	fn offchain_index_set_and_clear_works() {
+		use sp_core::offchain::OffchainStorage;
+
 		sp_tracing::try_init_simple();

 		let (client, backend) =