archive: Implement archive_unstable_storage (#1846)

This PR implements the `archive_unstable_storage` method that offers support for: - fetching values - fetching hashes - iterating over keys and values - iterating over keys and hashes - fetching merkle values from the trie-db A common component dedicated to RPC-V2 storage queries is created to bridge the gap between `chainHead/storage` and `archive/storage`. Query pagination is supported by `paginationStartKey`, similar to the old APIs. Similarly to the `chainHead/storage`, the `archive/storage` method accepts a maximum number of queried items. The design builds upon: https://github.com/paritytech/json-rpc-interface-spec/pull/94. Closes https://github.com/paritytech/polkadot-sdk/issues/1512. cc @paritytech/subxt-team --------- Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> Co-authored-by: Niklas Adolfsson <niklasadolfsson1@gmail.com>
2026-06-19 00:41:03 +00:00 · 2024-01-15 16:03:32 +02:00
parent 46090ff114
commit 53bcbb15f1
15 changed files with 1278 additions and 363 deletions
@@ -0,0 +1,273 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Common events for RPC-V2 spec.
+
+use serde::{Deserialize, Serialize};
+
+/// The storage item to query.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct StorageQuery<Key> {
+	/// The provided key.
+	pub key: Key,
+	/// The type of the storage query.
+	#[serde(rename = "type")]
+	pub query_type: StorageQueryType,
+}
+
+/// The storage item to query with pagination.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct PaginatedStorageQuery<Key> {
+	/// The provided key.
+	pub key: Key,
+	/// The type of the storage query.
+	#[serde(rename = "type")]
+	pub query_type: StorageQueryType,
+	/// The pagination key from which the iteration should resume.
+	#[serde(skip_serializing_if = "Option::is_none")]
+	#[serde(default)]
+	pub pagination_start_key: Option<Key>,
+}
+
+/// The type of the storage query.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum StorageQueryType {
+	/// Fetch the value of the provided key.
+	Value,
+	/// Fetch the hash of the value of the provided key.
+	Hash,
+	/// Fetch the closest descendant merkle value.
+	ClosestDescendantMerkleValue,
+	/// Fetch the values of all descendants of they provided key.
+	DescendantsValues,
+	/// Fetch the hashes of the values of all descendants of they provided key.
+	DescendantsHashes,
+}
+
+impl StorageQueryType {
+	/// Returns `true` if the query is a descendant query.
+	pub fn is_descendant_query(&self) -> bool {
+		matches!(self, Self::DescendantsValues | Self::DescendantsHashes)
+	}
+}
+
+/// The storage result.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct StorageResult {
+	/// The hex-encoded key of the result.
+	pub key: String,
+	/// The result of the query.
+	#[serde(flatten)]
+	pub result: StorageResultType,
+}
+
+/// The type of the storage query.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum StorageResultType {
+	/// Fetch the value of the provided key.
+	Value(String),
+	/// Fetch the hash of the value of the provided key.
+	Hash(String),
+	/// Fetch the closest descendant merkle value.
+	ClosestDescendantMerkleValue(String),
+}
+
+/// The error of a storage call.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct StorageResultErr {
+	/// The hex-encoded key of the result.
+	pub key: String,
+	/// The result of the query.
+	#[serde(flatten)]
+	pub error: StorageResultType,
+}
+
+/// The result of a storage call.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum ArchiveStorageResult {
+	/// Query generated a result.
+	Ok(ArchiveStorageMethodOk),
+	/// Query encountered an error.
+	Err(ArchiveStorageMethodErr),
+}
+
+impl ArchiveStorageResult {
+	/// Create a new `ArchiveStorageResult::Ok` result.
+	pub fn ok(result: Vec<StorageResult>, discarded_items: usize) -> Self {
+		Self::Ok(ArchiveStorageMethodOk { result, discarded_items })
+	}
+
+	/// Create a new `ArchiveStorageResult::Err` result.
+	pub fn err(error: String) -> Self {
+		Self::Err(ArchiveStorageMethodErr { error })
+	}
+}
+
+/// The result of a storage call.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ArchiveStorageMethodOk {
+	/// Reported results.
+	pub result: Vec<StorageResult>,
+	/// Number of discarded items.
+	pub discarded_items: usize,
+}
+
+/// The error of a storage call.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ArchiveStorageMethodErr {
+	/// Reported error.
+	pub error: String,
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn storage_result() {
+		// Item with Value.
+		let item =
+			StorageResult { key: "0x1".into(), result: StorageResultType::Value("res".into()) };
+		// Encode
+		let ser = serde_json::to_string(&item).unwrap();
+		let exp = r#"{"key":"0x1","value":"res"}"#;
+		assert_eq!(ser, exp);
+		// Decode
+		let dec: StorageResult = serde_json::from_str(exp).unwrap();
+		assert_eq!(dec, item);
+
+		// Item with Hash.
+		let item =
+			StorageResult { key: "0x1".into(), result: StorageResultType::Hash("res".into()) };
+		// Encode
+		let ser = serde_json::to_string(&item).unwrap();
+		let exp = r#"{"key":"0x1","hash":"res"}"#;
+		assert_eq!(ser, exp);
+		// Decode
+		let dec: StorageResult = serde_json::from_str(exp).unwrap();
+		assert_eq!(dec, item);
+
+		// Item with DescendantsValues.
+		let item = StorageResult {
+			key: "0x1".into(),
+			result: StorageResultType::ClosestDescendantMerkleValue("res".into()),
+		};
+		// Encode
+		let ser = serde_json::to_string(&item).unwrap();
+		let exp = r#"{"key":"0x1","closestDescendantMerkleValue":"res"}"#;
+		assert_eq!(ser, exp);
+		// Decode
+		let dec: StorageResult = serde_json::from_str(exp).unwrap();
+		assert_eq!(dec, item);
+	}
+
+	#[test]
+	fn storage_query() {
+		// Item with Value.
+		let item = StorageQuery { key: "0x1", query_type: StorageQueryType::Value };
+		// Encode
+		let ser = serde_json::to_string(&item).unwrap();
+		let exp = r#"{"key":"0x1","type":"value"}"#;
+		assert_eq!(ser, exp);
+		// Decode
+		let dec: StorageQuery<&str> = serde_json::from_str(exp).unwrap();
+		assert_eq!(dec, item);
+
+		// Item with Hash.
+		let item = StorageQuery { key: "0x1", query_type: StorageQueryType::Hash };
+		// Encode
+		let ser = serde_json::to_string(&item).unwrap();
+		let exp = r#"{"key":"0x1","type":"hash"}"#;
+		assert_eq!(ser, exp);
+		// Decode
+		let dec: StorageQuery<&str> = serde_json::from_str(exp).unwrap();
+		assert_eq!(dec, item);
+
+		// Item with DescendantsValues.
+		let item = StorageQuery { key: "0x1", query_type: StorageQueryType::DescendantsValues };
+		// Encode
+		let ser = serde_json::to_string(&item).unwrap();
+		let exp = r#"{"key":"0x1","type":"descendantsValues"}"#;
+		assert_eq!(ser, exp);
+		// Decode
+		let dec: StorageQuery<&str> = serde_json::from_str(exp).unwrap();
+		assert_eq!(dec, item);
+
+		// Item with DescendantsHashes.
+		let item = StorageQuery { key: "0x1", query_type: StorageQueryType::DescendantsHashes };
+		// Encode
+		let ser = serde_json::to_string(&item).unwrap();
+		let exp = r#"{"key":"0x1","type":"descendantsHashes"}"#;
+		assert_eq!(ser, exp);
+		// Decode
+		let dec: StorageQuery<&str> = serde_json::from_str(exp).unwrap();
+		assert_eq!(dec, item);
+
+		// Item with Merkle.
+		let item =
+			StorageQuery { key: "0x1", query_type: StorageQueryType::ClosestDescendantMerkleValue };
+		// Encode
+		let ser = serde_json::to_string(&item).unwrap();
+		let exp = r#"{"key":"0x1","type":"closestDescendantMerkleValue"}"#;
+		assert_eq!(ser, exp);
+		// Decode
+		let dec: StorageQuery<&str> = serde_json::from_str(exp).unwrap();
+		assert_eq!(dec, item);
+	}
+
+	#[test]
+	fn storage_query_paginated() {
+		let item = PaginatedStorageQuery {
+			key: "0x1",
+			query_type: StorageQueryType::Value,
+			pagination_start_key: None,
+		};
+		// Encode
+		let ser = serde_json::to_string(&item).unwrap();
+		let exp = r#"{"key":"0x1","type":"value"}"#;
+		assert_eq!(ser, exp);
+		// Decode
+		let dec: StorageQuery<&str> = serde_json::from_str(exp).unwrap();
+		assert_eq!(dec.key, item.key);
+		assert_eq!(dec.query_type, item.query_type);
+		let dec: PaginatedStorageQuery<&str> = serde_json::from_str(exp).unwrap();
+		assert_eq!(dec, item);
+
+		let item = PaginatedStorageQuery {
+			key: "0x1",
+			query_type: StorageQueryType::Value,
+			pagination_start_key: Some("0x2"),
+		};
+		// Encode
+		let ser = serde_json::to_string(&item).unwrap();
+		let exp = r#"{"key":"0x1","type":"value","paginationStartKey":"0x2"}"#;
+		assert_eq!(ser, exp);
+		// Decode
+		let dec: PaginatedStorageQuery<&str> = serde_json::from_str(exp).unwrap();
+		assert_eq!(dec, item);
+	}
+}
@@ -0,0 +1,17 @@
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Common types and functionality for the RPC-V2 spec.
+
+pub mod events;
+pub mod storage;
@@ -0,0 +1,198 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Storage queries for the RPC-V2 spec.
+
+use std::{marker::PhantomData, sync::Arc};
+
+use sc_client_api::{Backend, ChildInfo, StorageKey, StorageProvider};
+use sp_runtime::traits::Block as BlockT;
+
+use super::events::{StorageResult, StorageResultType};
+use crate::hex_string;
+
+/// Call into the storage of blocks.
+pub struct Storage<Client, Block, BE> {
+	/// Substrate client.
+	client: Arc<Client>,
+	_phandom: PhantomData<(BE, Block)>,
+}
+
+impl<Client, Block, BE> Storage<Client, Block, BE> {
+	/// Constructs a new [`Storage`].
+	pub fn new(client: Arc<Client>) -> Self {
+		Self { client, _phandom: PhantomData }
+	}
+}
+
+/// Query to iterate over storage.
+pub struct QueryIter {
+	/// The key from which the iteration was started.
+	pub query_key: StorageKey,
+	/// The key after which pagination should resume.
+	pub pagination_start_key: Option<StorageKey>,
+	/// The type of the query (either value or hash).
+	pub ty: IterQueryType,
+}
+
+/// The query type of an iteration.
+pub enum IterQueryType {
+	/// Iterating over (key, value) pairs.
+	Value,
+	/// Iterating over (key, hash) pairs.
+	Hash,
+}
+
+/// The result of making a query call.
+pub type QueryResult = Result<Option<StorageResult>, String>;
+
+/// The result of iterating over keys.
+pub type QueryIterResult = Result<(Vec<StorageResult>, Option<QueryIter>), String>;
+
+impl<Client, Block, BE> Storage<Client, Block, BE>
+where
+	Block: BlockT + 'static,
+	BE: Backend<Block> + 'static,
+	Client: StorageProvider<Block, BE> + 'static,
+{
+	/// Fetch the value from storage.
+	pub fn query_value(
+		&self,
+		hash: Block::Hash,
+		key: &StorageKey,
+		child_key: Option<&ChildInfo>,
+	) -> QueryResult {
+		let result = if let Some(child_key) = child_key {
+			self.client.child_storage(hash, child_key, key)
+		} else {
+			self.client.storage(hash, key)
+		};
+
+		result
+			.map(|opt| {
+				QueryResult::Ok(opt.map(|storage_data| StorageResult {
+					key: hex_string(&key.0),
+					result: StorageResultType::Value(hex_string(&storage_data.0)),
+				}))
+			})
+			.unwrap_or_else(|error| QueryResult::Err(error.to_string()))
+	}
+
+	/// Fetch the hash of a value from storage.
+	pub fn query_hash(
+		&self,
+		hash: Block::Hash,
+		key: &StorageKey,
+		child_key: Option<&ChildInfo>,
+	) -> QueryResult {
+		let result = if let Some(child_key) = child_key {
+			self.client.child_storage_hash(hash, child_key, key)
+		} else {
+			self.client.storage_hash(hash, key)
+		};
+
+		result
+			.map(|opt| {
+				QueryResult::Ok(opt.map(|storage_data| StorageResult {
+					key: hex_string(&key.0),
+					result: StorageResultType::Hash(hex_string(&storage_data.as_ref())),
+				}))
+			})
+			.unwrap_or_else(|error| QueryResult::Err(error.to_string()))
+	}
+
+	/// Fetch the closest merkle value.
+	pub fn query_merkle_value(
+		&self,
+		hash: Block::Hash,
+		key: &StorageKey,
+		child_key: Option<&ChildInfo>,
+	) -> QueryResult {
+		let result = if let Some(child_key) = child_key {
+			self.client.child_closest_merkle_value(hash, child_key, key)
+		} else {
+			self.client.closest_merkle_value(hash, key)
+		};
+
+		result
+			.map(|opt| {
+				QueryResult::Ok(opt.map(|storage_data| {
+					let result = match &storage_data {
+						sc_client_api::MerkleValue::Node(data) => hex_string(&data.as_slice()),
+						sc_client_api::MerkleValue::Hash(hash) => hex_string(&hash.as_ref()),
+					};
+
+					StorageResult {
+						key: hex_string(&key.0),
+						result: StorageResultType::ClosestDescendantMerkleValue(result),
+					}
+				}))
+			})
+			.unwrap_or_else(|error| QueryResult::Err(error.to_string()))
+	}
+
+	/// Iterate over at most the provided number of keys.
+	///
+	/// Returns the storage result with a potential next key to resume iteration.
+	pub fn query_iter_pagination(
+		&self,
+		query: QueryIter,
+		hash: Block::Hash,
+		child_key: Option<&ChildInfo>,
+		count: usize,
+	) -> QueryIterResult {
+		let QueryIter { ty, query_key, pagination_start_key } = query;
+
+		let mut keys_iter = if let Some(child_key) = child_key {
+			self.client.child_storage_keys(
+				hash,
+				child_key.to_owned(),
+				Some(&query_key),
+				pagination_start_key.as_ref(),
+			)
+		} else {
+			self.client.storage_keys(hash, Some(&query_key), pagination_start_key.as_ref())
+		}
+		.map_err(|err| err.to_string())?;
+
+		let mut ret = Vec::with_capacity(count);
+		let mut next_pagination_key = None;
+		for _ in 0..count {
+			let Some(key) = keys_iter.next() else { break };
+
+			next_pagination_key = Some(key.clone());
+
+			let result = match ty {
+				IterQueryType::Value => self.query_value(hash, &key, child_key),
+				IterQueryType::Hash => self.query_hash(hash, &key, child_key),
+			}?;
+
+			if let Some(value) = result {
+				ret.push(value);
+			}
+		}
+
+		// Save the next key if any to continue the iteration.
+		let maybe_next_query = keys_iter.next().map(|_| QueryIter {
+			ty,
+			query_key,
+			pagination_start_key: next_pagination_key,
+		});
+		Ok((ret, maybe_next_query))
+	}
+}