Storage chain: Runtime module (#8624)

* Transaction storage runtime module

* WIP: Tests

* Tests, benchmarks  and docs

* Made check_proof mandatory

* Typo

* Renamed a crate

* Apply suggestions from code review

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* Added weight for on_finalize

* Fixed counter mutations

* Reorganized tests

* Fixed build

* Update for the new inherent API

* Reworked for the new inherents API

* Apply suggestions from code review

Co-authored-by: cheme <emericchevalier.pro@gmail.com>
Co-authored-by: Alexander Popiak <alexander.popiak@parity.io>
Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>

* Store transactions in a Vec

* Added FeeDestination

* Get rid of constants

* Fixed node runtime build

* Fixed benches

* Update frame/transaction-storage/src/lib.rs

Co-authored-by: cheme <emericchevalier.pro@gmail.com>

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
Co-authored-by: cheme <emericchevalier.pro@gmail.com>
Co-authored-by: Alexander Popiak <alexander.popiak@parity.io>
Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>

substrate/primitives/blockchain/src/backend.rs

@@ -227,6 +227,8 @@ pub trait Backend<Block: BlockT>: HeaderBackend<Block> + HeaderMetadata<Block, E
 	fn has_indexed_transaction(&self, hash: &Block::Hash) -> Result<bool> {
 		Ok(self.indexed_transaction(hash)?.is_some())
 	}
+
+	fn block_indexed_body(&self, id: BlockId<Block>) -> Result<Option<Vec<Vec<u8>>>>;
 }
 
 /// Provides access to the optional cache.

substrate/primitives/externalities/src/lib.rs

@@ -229,12 +229,12 @@ pub trait Externalities: ExtensionStore {
 	fn storage_commit_transaction(&mut self) -> Result<(), ()>;
 
 	/// Index specified transaction slice and store it.
-	fn storage_index_transaction(&mut self, _index: u32, _offset: u32) {
+	fn storage_index_transaction(&mut self, _index: u32, _hash: &[u8], _size: u32) {
 		unimplemented!("storage_index_transaction");
 	}
 
 	/// Renew existing piece of transaction storage.
-	fn storage_renew_transaction_index(&mut self, _index: u32, _hash: &[u8], _size: u32) {
+	fn storage_renew_transaction_index(&mut self, _index: u32, _hash: &[u8]) {
 		unimplemented!("storage_renew_transaction_index");
 	}
 

substrate/primitives/io/src/lib.rs

@@ -429,6 +429,24 @@ pub trait Trie {
 	fn keccak_256_ordered_root(input: Vec<Vec<u8>>) -> H256 {
 		Layout::<sp_core::KeccakHasher>::ordered_trie_root(input)
 	}
+
+	/// Verify trie proof
+	fn blake2_256_verify_proof(root: H256, proof: &[Vec<u8>], key: &[u8], value: &[u8]) -> bool {
+		sp_trie::verify_trie_proof::<Layout<sp_core::Blake2Hasher>, _, _, _>(
+			&root,
+			proof,
+			&[(key, Some(value))],
+		).is_ok()
+	}
+
+	/// Verify trie proof
+	fn keccak_256_verify_proof(root: H256, proof: &[Vec<u8>], key: &[u8], value: &[u8]) -> bool {
+		sp_trie::verify_trie_proof::<Layout<sp_core::KeccakHasher>, _, _, _>(
+			&root,
+			proof,
+			&[(key, Some(value))],
+		).is_ok()
+	}
 }
 
 /// Interface that provides miscellaneous functions for communicating between the runtime and the node.
@@ -824,6 +842,20 @@ pub trait Hashing {
 	}
 }
 
+/// Interface that provides transaction indexing API.
+#[runtime_interface]
+pub trait TransactionIndex {
+	/// Add transaction index. Returns indexed content hash.
+	fn index(&mut self, extrinsic: u32, size: u32, context_hash: [u8; 32]) {
+		self.storage_index_transaction(extrinsic, &context_hash, size);
+	}
+
+	/// Conduct a 512-bit Keccak hash.
+	fn renew(&mut self, extrinsic: u32, context_hash: [u8; 32]) {
+		self.storage_renew_transaction_index(extrinsic, &context_hash);
+	}
+}
+
 /// Interface that provides functions to access the Offchain DB.
 #[runtime_interface]
 pub trait OffchainIndex {
@@ -1434,6 +1466,7 @@ pub type SubstrateHostFunctions = (
 	crate::trie::HostFunctions,
 	offchain_index::HostFunctions,
 	runtime_tasks::HostFunctions,
+	transaction_index::HostFunctions,
 );
 
 #[cfg(test)]

substrate/primitives/state-machine/src/ext.rs

@@ -629,33 +629,34 @@ where
 		}
 	}
 
-	fn storage_index_transaction(&mut self, index: u32, offset: u32) {
+	fn storage_index_transaction(&mut self, index: u32, hash: &[u8], size: u32) {
 		trace!(
 			target: "state",
-			"{:04x}: IndexTransaction ({}): [{}..]",
+			"{:04x}: IndexTransaction ({}): {}, {} bytes",
 			self.id,
 			index,
-			offset,
+			HexDisplay::from(&hash),
+			size,
 		);
 		self.overlay.add_transaction_index(IndexOperation::Insert {
 			extrinsic: index,
-			offset,
+			hash: hash.to_vec(),
+			size,
 		});
 	}
 
 	/// Renew existing piece of data storage.
-	fn storage_renew_transaction_index(&mut self, index: u32, hash: &[u8], size: u32) {
+	fn storage_renew_transaction_index(&mut self, index: u32, hash: &[u8]) {
 		trace!(
 			target: "state",
-			"{:04x}: RenewTransactionIndex ({}) {} bytes",
+			"{:04x}: RenewTransactionIndex ({}): {}",
 			self.id,
+			index,
 			HexDisplay::from(&hash),
-			size,
 		);
 		self.overlay.add_transaction_index(IndexOperation::Renew {
 			extrinsic: index,
 			hash: hash.to_vec(),
-			size
 		});
 	}
 

substrate/primitives/state-machine/src/overlayed_changes/mod.rs

@@ -118,8 +118,10 @@ pub enum IndexOperation {
 	Insert {
 		/// Extrinsic index in the current block.
 		extrinsic: u32,
-		/// Data offset in the extrinsic.
-		offset: u32,
+		/// Data content hash.
+		hash: Vec<u8>,
+		/// Indexed data size.
+		size: u32,
 	},
 	/// Renew existing transaction storage.
 	Renew {
@@ -127,8 +129,6 @@ pub enum IndexOperation {
 		extrinsic: u32,
 		/// Referenced index hash.
 		hash: Vec<u8>,
-		/// Expected data size.
-		size: u32,
 	}
 }
 
@@ -520,6 +520,11 @@ impl OverlayedChanges {
 		self.children.get(key).map(|(overlay, info)| (overlay.changes(), info))
 	}
 
+	/// Get an list of all index operations.
+	pub fn transaction_index_ops(&self) -> &[IndexOperation] {
+		&self.transaction_index_ops
+	}
+
 	/// Convert this instance with all changes into a [`StorageChanges`] instance.
 	#[cfg(feature = "std")]
 	pub fn into_storage_changes<

substrate/primitives/transaction-storage-proof/Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+name = "sp-transaction-storage-proof"
+version = "3.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "Transaction storage proof primitives"
+edition = "2018"
+license = "Apache-2.0"
+homepage = "https://substrate.dev"
+repository = "https://github.com/paritytech/substrate/"
+readme = "README.md"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[dependencies]
+sp-inherents = { version = "3.0.0", default-features = false, path = "../inherents" }
+sp-runtime = { version = "3.0.0", default-features = false, path = "../runtime" }
+sp-std = { version = "3.0.0", default-features = false, path = "../std" }
+sp-trie = { version = "3.0.0", optional = true, path = "../trie" }
+sp-core = { version = "3.0.0", path = "../core", optional = true }
+codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] }
+log = { version = "0.4.8", optional = true }
+async-trait = { version = "0.1.48", optional = true }
+
+[features]
+default = [ "std" ]
+std = [
+	"codec/std",
+	"sp-std/std",
+	"sp-inherents/std",
+	"sp-runtime/std",
+	"sp-trie/std",
+	"sp-core",
+	"log",
+	"async-trait",
+]

substrate/primitives/transaction-storage-proof/README.md

@@ -0,0 +1,3 @@
+Authorship Primitives
+
+License: Apache-2.0

substrate/primitives/transaction-storage-proof/src/lib.rs

@@ -0,0 +1,240 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Storge proof primitives. Constains types and basic code to extract storage
+//! proofs for indexed transactions.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+
+use sp_std::{result::Result, prelude::*};
+
+use codec::{Encode, Decode};
+use sp_inherents::{InherentIdentifier, InherentData, IsFatalError};
+use sp_runtime::{traits::{Block as BlockT, NumberFor}};
+
+pub use sp_inherents::Error;
+
+/// The identifier for the proof inherent.
+pub const INHERENT_IDENTIFIER: InherentIdentifier = *b"tx_proof";
+/// Storage period for data.
+pub const DEFAULT_STORAGE_PERIOD: u32 = 100800;
+/// Proof trie value size.
+pub const CHUNK_SIZE: usize = 256;
+
+/// Errors that can occur while checking the storage proof.
+#[derive(Encode, sp_runtime::RuntimeDebug)]
+#[cfg_attr(feature = "std", derive(Decode))]
+pub enum InherentError {
+	InvalidProof,
+	TrieError
+}
+
+impl IsFatalError for InherentError {
+	fn is_fatal_error(&self) -> bool {
+		true
+	}
+}
+
+#[derive(Encode, Decode, Clone, PartialEq, Debug)]
+pub struct TransactionStorageProof {
+	/// Data chunk that is proved to exist.
+	pub chunk: Vec<u8>,
+	/// Trie nodes that compose the proof.
+	pub proof: Vec<Vec<u8>>,
+}
+
+/// Auxiliary trait to extract storage proof.
+pub trait TransactionStorageProofInherentData {
+	/// Get the proof.
+	fn storage_proof(&self) -> Result<Option<TransactionStorageProof>, Error>;
+}
+
+impl TransactionStorageProofInherentData for InherentData {
+	fn storage_proof(&self) -> Result<Option<TransactionStorageProof>, Error> {
+		Ok(self.get_data(&INHERENT_IDENTIFIER)?)
+	}
+}
+
+/// Provider for inherent data.
+#[cfg(feature = "std")]
+pub struct InherentDataProvider {
+	proof: Option<TransactionStorageProof>,
+}
+
+#[cfg(feature = "std")]
+impl InherentDataProvider {
+	pub fn new(proof: Option<TransactionStorageProof>) -> Self {
+		InherentDataProvider { proof }
+	}
+}
+
+#[cfg(feature = "std")]
+#[async_trait::async_trait]
+impl sp_inherents::InherentDataProvider for InherentDataProvider {
+	fn provide_inherent_data(&self, inherent_data: &mut InherentData) -> Result<(), Error> {
+		if let Some(proof) = &self.proof {
+			inherent_data.put_data(INHERENT_IDENTIFIER, proof)
+		} else {
+			Ok(())
+		}
+	}
+
+	async fn try_handle_error(
+		&self,
+		identifier: &InherentIdentifier,
+		error: &[u8],
+	) -> Option<Result<(), Error>> {
+		if *identifier != INHERENT_IDENTIFIER {
+			return None
+		}
+
+		let error = InherentError::decode(&mut &error[..]).ok()?;
+
+		Some(Err(Error::Application(Box::from(format!("{:?}", error)))))
+	}
+}
+
+/// An utility function to extract chunk index from the source of randomness.
+pub fn random_chunk(random_hash: &[u8], total_chunks: u32) -> u32 {
+	let mut buf = [0u8; 8];
+	buf.copy_from_slice(&random_hash[0..8]);
+	let random_u64 = u64::from_be_bytes(buf);
+	(random_u64 % total_chunks as u64) as u32
+}
+
+/// An utility function to enocde transaction index as trie key.
+pub fn encode_index(input: u32) -> Vec<u8> {
+	codec::Encode::encode(&codec::Compact(input))
+}
+
+/// An interface to request indexed data from the client.
+pub trait IndexedBody<B: BlockT> {
+	fn block_indexed_body(
+		&self,
+		number: NumberFor<B>,
+	) -> Result<Option<Vec<Vec<u8>>>, Error>;
+
+	fn number(
+		&self,
+		hash: B::Hash,
+	) -> Result<Option<NumberFor<B>>, Error>;
+}
+
+#[cfg(feature = "std")]
+pub mod registration {
+	use sp_runtime::{traits::{Block as BlockT, Saturating, Zero, One}};
+	use sp_trie::TrieMut;
+	use super::*;
+
+	type Hasher = sp_core::Blake2Hasher;
+	type TrieLayout = sp_trie::Layout::<Hasher>;
+
+	/// Create a new inherent data provider instance for a given parent block hash.
+	pub fn new_data_provider<B, C>(
+		client: &C,
+		parent: &B::Hash,
+	) -> Result<InherentDataProvider, Error>
+	where
+		B: BlockT,
+		C: IndexedBody<B>,
+	{
+		let parent_number = client.number(parent.clone())?.unwrap_or(Zero::zero());
+		let number = parent_number
+			.saturating_add(One::one())
+			.saturating_sub(DEFAULT_STORAGE_PERIOD.into());
+		if number.is_zero() {
+			// Too early to collect proofs.
+			return Ok(InherentDataProvider::new(None));
+		}
+
+		let proof = match client.block_indexed_body(number)? {
+			Some(transactions) => {
+				Some(build_proof(parent.as_ref(), transactions)?)
+			},
+			None => {
+				// Nothing was indexed in that block.
+				None
+			}
+		};
+		Ok(InherentDataProvider::new(proof))
+	}
+
+	/// Build a proof for a given source of randomness and indexed transactions.
+	pub fn build_proof(random_hash: &[u8], transactions: Vec<Vec<u8>>)
+		-> Result<TransactionStorageProof, Error>
+	{
+		let mut db = sp_trie::MemoryDB::<Hasher>::default();
+
+		let mut target_chunk = None;
+		let mut target_root = Default::default();
+		let mut target_chunk_key = Default::default();
+		let mut chunk_proof = Default::default();
+
+		let total_chunks: u64 = transactions.iter().map(|t| ((t.len() + CHUNK_SIZE - 1) / CHUNK_SIZE) as u64).sum();
+		let mut buf = [0u8; 8];
+		buf.copy_from_slice(&random_hash[0..8]);
+		let random_u64 = u64::from_be_bytes(buf);
+		let target_chunk_index = random_u64 % total_chunks;
+		//Generate tries for each transaction.
+		let mut chunk_index = 0;
+		for transaction in transactions {
+			let mut transaction_root = sp_trie::empty_trie_root::<TrieLayout>();
+			{
+				let mut trie = sp_trie::TrieDBMut::<TrieLayout>::new(&mut db, &mut transaction_root);
+				let chunks = transaction.chunks(CHUNK_SIZE).map(|c| c.to_vec());
+				for (index, chunk) in chunks.enumerate() {
+					let index = encode_index(index as u32);
+					trie.insert(&index, &chunk)
+						.map_err(|e| Error::Application(Box::new(e)))?;
+					if chunk_index == target_chunk_index {
+						target_chunk = Some(chunk);
+						target_chunk_key = index;
+					}
+					chunk_index += 1;
+				}
+				trie.commit();
+			}
+			if target_chunk.is_some() && target_root == Default::default() {
+				target_root = transaction_root.clone();
+				chunk_proof = sp_trie::generate_trie_proof::<TrieLayout, _, _, _>(
+					&db,
+					transaction_root.clone(),
+					&[target_chunk_key.clone()]
+				).map_err(|e| Error::Application(Box::new(e)))?;
+			}
+		};
+
+		Ok(TransactionStorageProof {
+			proof: chunk_proof,
+			chunk: target_chunk.unwrap(),
+		})
+	}
+
+	#[test]
+	fn build_proof_check() {
+		use std::str::FromStr;
+		let random = [0u8; 32];
+		let proof = build_proof(&random, vec![vec![42]]).unwrap();
+		let root = sp_core::H256::from_str("0xff8611a4d212fc161dae19dd57f0f1ba9309f45d6207da13f2d3eab4c6839e91").unwrap();
+		sp_trie::verify_trie_proof::<TrieLayout, _, _, _>(
+			&root,
+			&proof.proof,
+			&[(encode_index(0), Some(proof.chunk))],
+		).unwrap();
+	}
+}
+

substrate/primitives/trie/src/storage_proof.rs

@@ -58,6 +58,10 @@ impl StorageProof {
 		StorageProofNodeIterator::new(self)
 	}
 
+	/// Convert into plain node vector.
+	pub fn into_nodes(self) -> Vec<Vec<u8>> {
+		self.trie_nodes
+	}
 	/// Creates a `MemoryDB` from `Self`.
 	pub fn into_memory_db<H: Hasher>(self) -> crate::MemoryDB<H> {
 		self.into()