Add dev_getBlockStats RPC (#10939)

* Add chain_getBlockStats rpc

* Fix broken doc link

* Apply suggestions from code review

Co-authored-by: Niklas Adolfsson <niklasadolfsson1@gmail.com>

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* fmt

* Fix compilation

* Move Blockstats

* Apply suggestions from code review

Co-authored-by: David <dvdplm@gmail.com>

* fmt

Co-authored-by: ascjones <ascjones@gmail.com>
Co-authored-by: Niklas Adolfsson <niklasadolfsson1@gmail.com>
Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
Co-authored-by: David <dvdplm@gmail.com>

substrate/Cargo.lock

@@ -8864,6 +8864,7 @@ dependencies = [
  "parking_lot 0.12.0",
  "sc-chain-spec",
  "sc-transaction-pool-api",
+ "scale-info",
  "serde",
  "serde_json",
  "sp-core",

substrate/bin/node/rpc/src/lib.rs

@@ -103,6 +103,7 @@ pub fn create_full<C, P, SC, B>(
 ) -> Result<jsonrpc_core::IoHandler<sc_rpc_api::Metadata>, Box<dyn std::error::Error + Send + Sync>>
 where
 	C: ProvideRuntimeApi<Block>
+		+ sc_client_api::BlockBackend<Block>
 		+ HeaderBackend<Block>
 		+ AuxStore
 		+ HeaderMetadata<Block, Error = BlockChainError>
@@ -123,6 +124,7 @@ where
 	use pallet_contracts_rpc::{Contracts, ContractsApi};
 	use pallet_mmr_rpc::{Mmr, MmrApi};
 	use pallet_transaction_payment_rpc::{TransactionPayment, TransactionPaymentApi};
+	use sc_rpc::dev::{Dev, DevApi};
 	use substrate_frame_rpc_system::{FullSystem, SystemApi};
 
 	let mut io = jsonrpc_core::IoHandler::default();
@@ -159,19 +161,18 @@ where
 		subscription_executor,
 		finality_provider,
 	)));
-
 	io.extend_with(substrate_state_trie_migration_rpc::StateMigrationApi::to_delegate(
 		substrate_state_trie_migration_rpc::MigrationRpc::new(client.clone(), backend, deny_unsafe),
 	));
-
 	io.extend_with(sc_sync_state_rpc::SyncStateRpcApi::to_delegate(
 		sc_sync_state_rpc::SyncStateRpcHandler::new(
 			chain_spec,
-			client,
+			client.clone(),
 			shared_authority_set,
 			shared_epoch_changes,
 		)?,
 	));
+	io.extend_with(DevApi::to_delegate(Dev::new(client, deny_unsafe)));
 
 	Ok(io)
 }

substrate/client/rpc-api/Cargo.toml

@@ -23,6 +23,7 @@ log = "0.4.8"
 parking_lot = "0.12.0"
 thiserror = "1.0"
 
+scale-info = { version = "2.0.1", default-features = false, features = ["derive"] }
 sp-core = { version = "6.0.0", path = "../../primitives/core" }
 sp-version = { version = "5.0.0", path = "../../primitives/version" }
 sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" }

substrate/client/rpc-api/src/dev/error.rs

@@ -0,0 +1,71 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Error helpers for Dev RPC module.
+
+use crate::errors;
+use jsonrpc_core as rpc;
+
+/// Dev RPC Result type.
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// Dev RPC future Result type.
+pub type FutureResult<T> = jsonrpc_core::BoxFuture<Result<T>>;
+
+/// Dev RPC errors.
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+	/// Failed to query specified block or its parent: Probably an invalid hash.
+	#[error("Error while querying block: {0}")]
+	BlockQueryError(Box<dyn std::error::Error + Send>),
+	/// The re-execution of the specified block failed.
+	#[error("Failed to re-execute the specified block")]
+	BlockExecutionFailed,
+	/// The witness compaction failed.
+	#[error("Failed to create to compact the witness")]
+	WitnessCompactionFailed,
+	/// The method is marked as unsafe but unsafe flag wasn't supplied on the CLI.
+	#[error(transparent)]
+	UnsafeRpcCalled(#[from] crate::policy::UnsafeRpcError),
+}
+
+/// Base error code for all dev errors.
+const BASE_ERROR: i64 = 6000;
+
+impl From<Error> for rpc::Error {
+	fn from(e: Error) -> Self {
+		match e {
+			Error::BlockQueryError(_) => rpc::Error {
+				code: rpc::ErrorCode::ServerError(BASE_ERROR + 1),
+				message: e.to_string(),
+				data: None,
+			},
+			Error::BlockExecutionFailed => rpc::Error {
+				code: rpc::ErrorCode::ServerError(BASE_ERROR + 3),
+				message: e.to_string(),
+				data: None,
+			},
+			Error::WitnessCompactionFailed => rpc::Error {
+				code: rpc::ErrorCode::ServerError(BASE_ERROR + 4),
+				message: e.to_string(),
+				data: None,
+			},
+			e => errors::internal(e),
+		}
+	}
+}

substrate/client/rpc-api/src/dev/mod.rs

@@ -0,0 +1,64 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Substrate dev API containing RPCs that are mainly meant for debugging and stats collection for
+//! developers. The endpoints in this RPC module are not meant to be available to non-local users
+//! and are all marked `unsafe`.
+
+pub mod error;
+
+use self::error::Result;
+use codec::{Decode, Encode};
+use jsonrpc_derive::rpc;
+use scale_info::TypeInfo;
+use serde::{Deserialize, Serialize};
+
+/// Statistics of a block returned by the `dev_getBlockStats` RPC.
+#[derive(Eq, PartialEq, Clone, Copy, Encode, Decode, Debug, TypeInfo, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct BlockStats {
+	/// The length in bytes of the storage proof produced by executing the block.
+	pub witness_len: u64,
+	/// The length in bytes of the storage proof after compaction.
+	pub witness_compact_len: u64,
+	/// Length of the block in bytes.
+	///
+	/// This information can also be acquired by downloading the whole block. This merely
+	/// saves some complexity on the client side.
+	pub block_len: u64,
+	/// Number of extrinsics in the block.
+	///
+	/// This information can also be acquired by downloading the whole block. This merely
+	/// saves some complexity on the client side.
+	pub num_extrinsics: u64,
+}
+
+/// Substrate dev API.
+///
+/// This API contains unstable and unsafe methods only meant for development nodes. They
+/// are all flagged as unsafe for this reason.
+#[rpc]
+pub trait DevApi<Hash> {
+	/// Reexecute the specified `block_hash` and gather statistics while doing so.
+	///
+	/// This function requires the specified block and its parent to be available
+	/// at the queried node. If either the specified block or the parent is pruned,
+	/// this function will return `None`.
+	#[rpc(name = "dev_getBlockStats")]
+	fn block_stats(&self, block_hash: Hash) -> Result<Option<BlockStats>>;
+}

substrate/client/rpc-api/src/lib.rs

@@ -35,6 +35,7 @@ pub use policy::{DenyUnsafe, UnsafeRpcError};
 pub mod author;
 pub mod chain;
 pub mod child_state;
+pub mod dev;
 pub mod offchain;
 pub mod state;
 pub mod system;

substrate/client/rpc/src/dev/mod.rs

@@ -0,0 +1,118 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Implementation of the [`DevApi`] trait providing debug utilities for Substrate based
+//! blockchains.
+
+#[cfg(test)]
+mod tests;
+
+pub use sc_rpc_api::dev::{BlockStats, DevApi};
+
+use sc_client_api::{BlockBackend, HeaderBackend};
+use sc_rpc_api::{
+	dev::error::{Error, Result},
+	DenyUnsafe,
+};
+use sp_api::{ApiExt, Core, ProvideRuntimeApi};
+use sp_core::Encode;
+use sp_runtime::{
+	generic::{BlockId, DigestItem},
+	traits::{Block as BlockT, Header},
+};
+use std::{
+	marker::{PhantomData, Send, Sync},
+	sync::Arc,
+};
+
+type HasherOf<Block> = <<Block as BlockT>::Header as Header>::Hashing;
+
+/// The Dev API. All methods are unsafe.
+pub struct Dev<Block: BlockT, Client> {
+	client: Arc<Client>,
+	deny_unsafe: DenyUnsafe,
+	_phantom: PhantomData<Block>,
+}
+
+impl<Block: BlockT, Client> Dev<Block, Client> {
+	/// Create a new Dev API.
+	pub fn new(client: Arc<Client>, deny_unsafe: DenyUnsafe) -> Self {
+		Self { client, deny_unsafe, _phantom: PhantomData::default() }
+	}
+}
+
+impl<Block, Client> DevApi<Block::Hash> for Dev<Block, Client>
+where
+	Block: BlockT + 'static,
+	Client: BlockBackend<Block>
+		+ HeaderBackend<Block>
+		+ ProvideRuntimeApi<Block>
+		+ Send
+		+ Sync
+		+ 'static,
+	Client::Api: Core<Block>,
+{
+	fn block_stats(&self, hash: Block::Hash) -> Result<Option<BlockStats>> {
+		self.deny_unsafe.check_if_safe()?;
+
+		let block = {
+			let block = self
+				.client
+				.block(&BlockId::Hash(hash))
+				.map_err(|e| Error::BlockQueryError(Box::new(e)))?;
+			if let Some(block) = block {
+				let (mut header, body) = block.block.deconstruct();
+				// Remove the `Seal` to ensure we have the number of digests as expected by the
+				// runtime.
+				header.digest_mut().logs.retain(|item| !matches!(item, DigestItem::Seal(_, _)));
+				Block::new(header, body)
+			} else {
+				return Ok(None)
+			}
+		};
+		let parent_header = {
+			let parent_hash = *block.header().parent_hash();
+			let parent_header = self
+				.client
+				.header(BlockId::Hash(parent_hash))
+				.map_err(|e| Error::BlockQueryError(Box::new(e)))?;
+			if let Some(header) = parent_header {
+				header
+			} else {
+				return Ok(None)
+			}
+		};
+		let block_len = block.encoded_size() as u64;
+		let num_extrinsics = block.extrinsics().len() as u64;
+		let pre_root = *parent_header.state_root();
+		let mut runtime_api = self.client.runtime_api();
+		runtime_api.record_proof();
+		runtime_api
+			.execute_block(&BlockId::Hash(parent_header.hash()), block)
+			.map_err(|_| Error::BlockExecutionFailed)?;
+		let witness = runtime_api
+			.extract_proof()
+			.expect("We enabled proof recording. A proof must be available; qed");
+		let witness_len = witness.encoded_size() as u64;
+		let witness_compact_len = witness
+			.into_compact_proof::<HasherOf<Block>>(pre_root)
+			.map_err(|_| Error::WitnessCompactionFailed)?
+			.encoded_size() as u64;
+		Ok(Some(BlockStats { witness_len, witness_compact_len, block_len, num_extrinsics }))
+	}
+}

substrate/client/rpc/src/dev/tests.rs

@@ -0,0 +1,58 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use super::*;
+use assert_matches::assert_matches;
+use futures::executor;
+use sc_block_builder::BlockBuilderProvider;
+use sp_blockchain::HeaderBackend;
+use sp_consensus::BlockOrigin;
+use substrate_test_runtime_client::{prelude::*, runtime::Block};
+
+#[test]
+fn block_stats_work() {
+	let mut client = Arc::new(substrate_test_runtime_client::new());
+	let api = <Dev<Block, _>>::new(client.clone(), DenyUnsafe::No);
+
+	let block = client.new_block(Default::default()).unwrap().build().unwrap().block;
+	executor::block_on(client.import(BlockOrigin::Own, block)).unwrap();
+
+	// Can't gather stats for a block without a parent.
+	assert_eq!(api.block_stats(client.genesis_hash()).unwrap(), None);
+
+	assert_eq!(
+		api.block_stats(client.info().best_hash).unwrap(),
+		Some(BlockStats {
+			witness_len: 597,
+			witness_compact_len: 500,
+			block_len: 99,
+			num_extrinsics: 0,
+		}),
+	);
+}
+
+#[test]
+fn deny_unsafe_works() {
+	let mut client = Arc::new(substrate_test_runtime_client::new());
+	let api = <Dev<Block, _>>::new(client.clone(), DenyUnsafe::Yes);
+
+	let block = client.new_block(Default::default()).unwrap().build().unwrap().block;
+	executor::block_on(client.import(BlockOrigin::Own, block)).unwrap();
+
+	assert_matches!(api.block_stats(client.info().best_hash), Err(Error::UnsafeRpcCalled(_)));
+}

substrate/client/rpc/src/lib.rs

@@ -34,6 +34,7 @@ pub use sc_rpc_api::{DenyUnsafe, Metadata};
 
 pub mod author;
 pub mod chain;
+pub mod dev;
 pub mod offchain;
 pub mod state;
 pub mod system;