Storage chain: Runtime module (#8624)

* Transaction storage runtime module

* WIP: Tests

* Tests, benchmarks  and docs

* Made check_proof mandatory

* Typo

* Renamed a crate

* Apply suggestions from code review

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* Added weight for on_finalize

* Fixed counter mutations

* Reorganized tests

* Fixed build

* Update for the new inherent API

* Reworked for the new inherents API

* Apply suggestions from code review

Co-authored-by: cheme <emericchevalier.pro@gmail.com>
Co-authored-by: Alexander Popiak <alexander.popiak@parity.io>
Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>

* Store transactions in a Vec

* Added FeeDestination

* Get rid of constants

* Fixed node runtime build

* Fixed benches

* Update frame/transaction-storage/src/lib.rs

Co-authored-by: cheme <emericchevalier.pro@gmail.com>

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
Co-authored-by: cheme <emericchevalier.pro@gmail.com>
Co-authored-by: Alexander Popiak <alexander.popiak@parity.io>
Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>

substrate/Cargo.lock

@@ -4377,6 +4377,7 @@ dependencies = [
  "pallet-tips",
  "pallet-transaction-payment",
  "pallet-transaction-payment-rpc-runtime-api",
+ "pallet-transaction-storage",
  "pallet-treasury",
  "pallet-uniques",
  "pallet-utility",
@@ -5602,6 +5603,26 @@ dependencies = [
  "sp-runtime",
 ]
 
+[[package]]
+name = "pallet-transaction-storage"
+version = "3.0.0"
+dependencies = [
+ "frame-benchmarking",
+ "frame-support",
+ "frame-support-test",
+ "frame-system",
+ "hex-literal",
+ "pallet-balances",
+ "parity-scale-codec",
+ "serde",
+ "sp-core",
+ "sp-inherents",
+ "sp-io",
+ "sp-runtime",
+ "sp-std",
+ "sp-transaction-storage-proof",
+]
+
 [[package]]
 name = "pallet-treasury"
 version = "3.0.0"
@@ -7953,6 +7974,7 @@ dependencies = [
  "sp-state-machine",
  "sp-tracing",
  "sp-transaction-pool",
+ "sp-transaction-storage-proof",
  "sp-trie",
  "sp-utils",
  "sp-version",
@@ -9314,6 +9336,20 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "sp-transaction-storage-proof"
+version = "3.0.0"
+dependencies = [
+ "async-trait",
+ "log",
+ "parity-scale-codec",
+ "sp-core",
+ "sp-inherents",
+ "sp-runtime",
+ "sp-std",
+ "sp-trie",
+]
+
 [[package]]
 name = "sp-trie"
 version = "3.0.0"

substrate/Cargo.toml

@@ -123,6 +123,7 @@ members = [
 	"frame/transaction-payment",
 	"frame/transaction-payment/rpc",
 	"frame/transaction-payment/rpc/runtime-api",
+	"frame/transaction-storage",
 	"frame/treasury",
 	"frame/tips",
 	"frame/uniques",
@@ -180,6 +181,7 @@ members = [
 	"primitives/timestamp",
 	"primitives/tracing",
 	"primitives/transaction-pool",
+	"primitives/transaction-storage-proof",
 	"primitives/trie",
 	"primitives/utils",
 	"primitives/version",

substrate/bin/node/cli/src/chain_spec.rs

@@ -335,6 +335,7 @@ pub fn testnet_genesis(
 		},
 		pallet_vesting: Default::default(),
 		pallet_gilt: Default::default(),
+		pallet_transaction_storage: Default::default(),
 	}
 }
 

substrate/bin/node/runtime/Cargo.toml

@@ -85,6 +85,7 @@ pallet-treasury = { version = "3.0.0", default-features = false, path = "../../.
 pallet-utility = { version = "3.0.0", default-features = false, path = "../../../frame/utility" }
 pallet-transaction-payment = { version = "3.0.0", default-features = false, path = "../../../frame/transaction-payment" }
 pallet-transaction-payment-rpc-runtime-api = { version = "3.0.0", default-features = false, path = "../../../frame/transaction-payment/rpc/runtime-api/" }
+pallet-transaction-storage = { version = "3.0.0", default-features = false, path = "../../../frame/transaction-storage" }
 pallet-uniques = { version = "3.0.0", default-features = false, path = "../../../frame/uniques" }
 pallet-vesting = { version = "3.0.0", default-features = false, path = "../../../frame/vesting" }
 
@@ -152,6 +153,7 @@ std = [
 	"pallet-tips/std",
 	"pallet-transaction-payment-rpc-runtime-api/std",
 	"pallet-transaction-payment/std",
+	"pallet-transaction-storage/std",
 	"pallet-treasury/std",
 	"sp-transaction-pool/std",
 	"pallet-utility/std",
@@ -194,6 +196,7 @@ runtime-benchmarks = [
 	"pallet-staking/runtime-benchmarks",
 	"pallet-timestamp/runtime-benchmarks",
 	"pallet-tips/runtime-benchmarks",
+	"pallet-transaction-storage/runtime-benchmarks",
 	"pallet-treasury/runtime-benchmarks",
 	"pallet-utility/runtime-benchmarks",
 	"pallet-uniques/runtime-benchmarks",

substrate/bin/node/runtime/src/lib.rs

@@ -1114,6 +1114,14 @@ impl pallet_uniques::Config for Runtime {
 	type WeightInfo = pallet_uniques::weights::SubstrateWeight<Runtime>;
 }
 
+impl pallet_transaction_storage::Config for Runtime {
+	type Event = Event;
+	type Currency = Balances;
+	type Call = Call;
+	type FeeDestination = ();
+	type WeightInfo = pallet_transaction_storage::weights::SubstrateWeight<Runtime>;
+}
+
 construct_runtime!(
 	pub enum Runtime where
 		Block = Block,
@@ -1159,6 +1167,7 @@ construct_runtime!(
 		Lottery: pallet_lottery::{Pallet, Call, Storage, Event<T>},
 		Gilt: pallet_gilt::{Pallet, Call, Storage, Event<T>, Config},
 		Uniques: pallet_uniques::{Pallet, Call, Storage, Event<T>},
+		TransactionStorage: pallet_transaction_storage::{Pallet, Call, Storage, Inherent, Config<T>, Event<T>},
 	}
 );
 
@@ -1532,6 +1541,7 @@ impl_runtime_apis! {
 			add_benchmark!(params, batches, frame_system, SystemBench::<Runtime>);
 			add_benchmark!(params, batches, pallet_timestamp, Timestamp);
 			add_benchmark!(params, batches, pallet_tips, Tips);
+			add_benchmark!(params, batches, pallet_transaction_storage, TransactionStorage);
 			add_benchmark!(params, batches, pallet_treasury, Treasury);
 			add_benchmark!(params, batches, pallet_uniques, Uniques);
 			add_benchmark!(params, batches, pallet_utility, Utility);

substrate/bin/node/testing/src/genesis.rs

@@ -120,5 +120,6 @@ pub fn config_endowed(
 		},
 		pallet_vesting: Default::default(),
 		pallet_gilt: Default::default(),
+		pallet_transaction_storage: Default::default(),
 	}
 }

substrate/client/api/src/client.rs

@@ -84,6 +84,16 @@ pub trait BlockBackend<Block: BlockT> {
 		id: &BlockId<Block>
 	) -> sp_blockchain::Result<Option<Vec<<Block as BlockT>::Extrinsic>>>;
 
+	/// Get all indexed transactions for a block,
+	/// including renewed transactions.
+	///
+	/// Note that this will only fetch transactions
+	/// that are indexed by the runtime with `storage_index_transaction`.
+	fn block_indexed_body(
+		&self,
+		id: &BlockId<Block>,
+	) -> sp_blockchain::Result<Option<Vec<Vec<u8>>>>;
+
 	/// Get full block by id.
 	fn block(&self, id: &BlockId<Block>) -> sp_blockchain::Result<Option<SignedBlock<Block>>>;
 

substrate/client/api/src/in_mem.rs

@@ -419,6 +419,13 @@ impl<Block: BlockT> blockchain::Backend<Block> for Blockchain<Block> {
 	) -> sp_blockchain::Result<Option<Vec<u8>>> {
 		unimplemented!("Not supported by the in-mem backend.")
 	}
+
+	fn block_indexed_body(
+		&self,
+		_id: BlockId<Block>
+	) -> sp_blockchain::Result<Option<Vec<Vec<u8>>>> {
+		unimplemented!("Not supported by the in-mem backend.")
+	}
 }
 
 impl<Block: BlockT> blockchain::ProvideCache<Block> for Blockchain<Block> {

substrate/client/api/src/lib.rs

@@ -38,6 +38,7 @@ pub use client::*;
 pub use light::*;
 pub use notifications::*;
 pub use proof_provider::*;
+pub use sp_blockchain::HeaderBackend;
 
 pub use sp_state_machine::{StorageProof, ExecutionStrategy};
 

substrate/client/db/src/lib.rs

@@ -67,7 +67,7 @@ use codec::{Decode, Encode};
 use hash_db::Prefix;
 use sp_trie::{MemoryDB, PrefixedMemoryDB, prefixed_key};
 use sp_database::Transaction;
-use sp_core::{Hasher, ChangesTrieConfiguration};
+use sp_core::ChangesTrieConfiguration;
 use sp_core::offchain::OffchainOverlayedChange;
 use sp_core::storage::{well_known_keys, ChildInfo};
 use sp_arithmetic::traits::Saturating;
@@ -591,6 +591,37 @@ impl<Block: BlockT> sc_client_api::blockchain::Backend<Block> for BlockchainDb<B
 	fn has_indexed_transaction(&self, hash: &Block::Hash) -> ClientResult<bool> {
 		Ok(self.db.contains(columns::TRANSACTION, hash.as_ref()))
 	}
+
+	fn block_indexed_body(&self, id: BlockId<Block>) -> ClientResult<Option<Vec<Vec<u8>>>> {
+		match self.transaction_storage {
+			TransactionStorageMode::BlockBody => Ok(None),
+			TransactionStorageMode::StorageChain => {
+				let body = match read_db(&*self.db, columns::KEY_LOOKUP, columns::BODY, id)? {
+					Some(body) => body,
+					None => return Ok(None),
+				};
+				match Vec::<ExtrinsicHeader>::decode(&mut &body[..]) {
+					Ok(index) => {
+						let mut transactions = Vec::new();
+						for ExtrinsicHeader { indexed_hash, .. } in index.into_iter() {
+							if indexed_hash != Default::default() {
+								match self.db.get(columns::TRANSACTION, indexed_hash.as_ref()) {
+									Some(t) => transactions.push(t),
+									None => return Err(sp_blockchain::Error::Backend(
+										format!("Missing indexed transaction {:?}", indexed_hash))
+									)
+								}
+							}
+						}
+						Ok(Some(transactions))
+					}
+					Err(err) => return Err(sp_blockchain::Error::Backend(
+							format!("Error decoding body list: {}", err)
+					)),
+				}
+			}
+		}
+	}
 }
 
 impl<Block: BlockT> sc_client_api::blockchain::ProvideCache<Block> for BlockchainDb<Block> {
@@ -1624,10 +1655,10 @@ fn apply_index_ops<Block: BlockT>(
 	let mut renewed_map = HashMap::new();
 	for op in ops {
 		match op {
-			IndexOperation::Insert { extrinsic, offset } => {
-				index_map.insert(extrinsic, offset);
+			IndexOperation::Insert { extrinsic, hash, size } => {
+				index_map.insert(extrinsic, (hash, size));
 			}
-			IndexOperation::Renew { extrinsic, hash, .. } => {
+			IndexOperation::Renew { extrinsic, hash } => {
 				renewed_map.insert(extrinsic, DbHash::from_slice(hash.as_ref()));
 			}
 		}
@@ -1643,9 +1674,8 @@ fn apply_index_ops<Block: BlockT>(
 			}
 		} else {
 			match index_map.get(&(index as u32)) {
-				Some(offset) if *offset as usize <= extrinsic.len() => {
-					let offset = *offset as usize;
-					let hash = HashFor::<Block>::hash(&extrinsic[offset..]);
+				Some((hash, size)) if *size as usize <= extrinsic.len() => {
+					let offset = extrinsic.len() - *size as usize;
 					transaction.store(
 						columns::TRANSACTION,
 						DbHash::from_slice(hash.as_ref()),
@@ -3024,13 +3054,16 @@ pub(crate) mod tests {
 		for i in 0 .. 10 {
 			let mut index = Vec::new();
 			if i == 0 {
-				index.push(IndexOperation::Insert { extrinsic: 0, offset: 1 });
+				index.push(IndexOperation::Insert {
+					extrinsic: 0,
+					hash: x1_hash.as_ref().to_vec(),
+					size: (x1.len() - 1) as u32,
+				});
 			} else if i < 5 {
 				// keep renewing 1st
 				index.push(IndexOperation::Renew {
 					extrinsic: 0,
 					hash: x1_hash.as_ref().to_vec(),
-					size: (x1.len() - 1) as u32,
 				});
 			} // else stop renewing
 			let hash = insert_block(

substrate/client/light/src/blockchain.rs

@@ -135,6 +135,13 @@ impl<S, Block> BlockchainBackend<Block> for Blockchain<S> where Block: BlockT, S
 	) -> ClientResult<Option<Vec<u8>>> {
 		Err(ClientError::NotAvailableOnLightClient)
 	}
+
+	fn block_indexed_body(
+		&self,
+		_id: BlockId<Block>
+	) -> sp_blockchain::Result<Option<Vec<Vec<u8>>>> {
+		Err(ClientError::NotAvailableOnLightClient)
+	}
 }
 
 impl<S: Storage<Block>, Block: BlockT> ProvideCache<Block> for Blockchain<S> {

substrate/client/service/Cargo.toml

@@ -65,6 +65,7 @@ codec = { package = "parity-scale-codec", version = "2.0.0" }
 sc-executor = { version = "0.9.0", path = "../executor" }
 sc-transaction-pool = { version = "3.0.0", path = "../transaction-pool" }
 sp-transaction-pool = { version = "3.0.0", path = "../../primitives/transaction-pool" }
+sp-transaction-storage-proof = { version = "3.0.0", path = "../../primitives/transaction-storage-proof" }
 sc-rpc-server = { version = "3.0.0", path = "../rpc-servers" }
 sc-rpc = { version = "3.0.0", path = "../rpc" }
 sc-block-builder = { version = "0.9.0", path = "../block-builder" }

substrate/client/service/src/client/client.rs

@@ -1982,6 +1982,13 @@ impl<B, E, Block, RA> BlockBackend<Block> for Client<B, E, Block, RA>
 	fn has_indexed_transaction(&self, hash: &Block::Hash) -> sp_blockchain::Result<bool> {
 		self.backend.blockchain().has_indexed_transaction(hash)
 	}
+
+	fn block_indexed_body(
+		&self,
+		id: &BlockId<Block>
+	) -> sp_blockchain::Result<Option<Vec<Vec<u8>>>> {
+		self.backend.blockchain().block_indexed_body(*id)
+	}
 }
 
 impl<B, E, Block, RA> backend::AuxStore for Client<B, E, Block, RA>
@@ -2050,3 +2057,26 @@ impl<BE, E, B, RA> sp_consensus::block_validation::Chain<B> for Client<BE, E, B,
 		Client::block_status(self, id).map_err(|e| Box::new(e) as Box<_>)
 	}
 }
+
+impl<BE, E, B, RA> sp_transaction_storage_proof::IndexedBody<B> for Client<BE, E, B, RA>
+where
+	BE: backend::Backend<B>,
+	E: CallExecutor<B>,
+	B: BlockT,
+{
+	fn block_indexed_body(
+		&self,
+		number: NumberFor<B>,
+	) ->Result<Option<Vec<Vec<u8>>>, sp_transaction_storage_proof::Error> {
+		self.backend.blockchain().block_indexed_body(BlockId::number(number))
+			.map_err(|e| sp_transaction_storage_proof::Error::Application(Box::new(e)))
+	}
+
+	fn number(
+		&self,
+		hash: B::Hash,
+	) -> Result<Option<NumberFor<B>>, sp_transaction_storage_proof::Error> {
+		self.backend.blockchain().number(hash)
+			.map_err(|e| sp_transaction_storage_proof::Error::Application(Box::new(e)))
+	}
+}

substrate/frame/benchmarking/src/lib.rs

@@ -736,17 +736,20 @@ macro_rules! impl_benchmark {
 					SelectedBenchmark as $crate::BenchmarkingSetup<T $(, $instance)?>
 				>::components(&selected_benchmark);
 
+				let mut progress = $crate::benchmarking::current_time();
 				// Default number of steps for a component.
 				let mut prev_steps = 10;
 
-				let repeat_benchmark = |
+				let mut repeat_benchmark = |
 					repeat: u32,
 					c: &[($crate::BenchmarkParameter, u32)],
 					results: &mut $crate::Vec<$crate::BenchmarkResults>,
 					verify: bool,
+					step: u32,
+					num_steps: u32,
 				| -> Result<(), &'static str> {
 					// Run the benchmark `repeat` times.
-					for _ in 0..repeat {
+					for r in 0..repeat {
 						// Set up the externalities environment for the setup we want to
 						// benchmark.
 						let closure_to_benchmark = <
@@ -801,6 +804,20 @@ macro_rules! impl_benchmark {
 								"Read/Write Count {:?}", read_write_count
 							);
 
+							let time = $crate::benchmarking::current_time();
+							if time.saturating_sub(progress) > 5000000000 {
+								progress = $crate::benchmarking::current_time();
+								$crate::log::info!(
+									target: "benchmark",
+									"Benchmarking {} {}/{}, run {}/{}",
+									extrinsic,
+									step,
+									num_steps,
+									r,
+									repeat,
+								);
+							}
+
 							// Time the storage root recalculation.
 							let start_storage_root = $crate::benchmarking::current_time();
 							$crate::storage_root();
@@ -829,9 +846,9 @@ macro_rules! impl_benchmark {
 				if components.is_empty() {
 					if verify {
 						// If `--verify` is used, run the benchmark once to verify it would complete.
-						repeat_benchmark(1, Default::default(), &mut $crate::Vec::new(), true)?;
+						repeat_benchmark(1, Default::default(), &mut $crate::Vec::new(), true, 1, 1)?;
 					}
-					repeat_benchmark(repeat, Default::default(), &mut results, false)?;
+					repeat_benchmark(repeat, Default::default(), &mut results, false, 1, 1)?;
 				} else {
 					// Select the component we will be benchmarking. Each component will be benchmarked.
 					for (idx, (name, low, high)) in components.iter().enumerate() {
@@ -869,9 +886,9 @@ macro_rules! impl_benchmark {
 
 							if verify {
 								// If `--verify` is used, run the benchmark once to verify it would complete.
-								repeat_benchmark(1, &c, &mut $crate::Vec::new(), true)?;
+								repeat_benchmark(1, &c, &mut $crate::Vec::new(), true, s, num_of_steps)?;
 							}
-							repeat_benchmark(repeat, &c, &mut results, false)?;
+							repeat_benchmark(repeat, &c, &mut results, false, s, num_of_steps)?;
 						}
 					}
 				}

substrate/frame/transaction-storage/Cargo.toml

@@ -0,0 +1,50 @@
+[package]
+name = "pallet-transaction-storage"
+version = "3.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+license = "Unlicense"
+homepage = "https://substrate.dev"
+repository = "https://github.com/paritytech/substrate/"
+description = "Storage chain pallet"
+readme = "README.md"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[dependencies]
+serde = { version = "1.0.101", optional = true }
+hex-literal = { version = "0.3.1", optional = true }
+codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false }
+frame-support = { version = "3.0.0", default-features = false, path = "../support" }
+frame-system = { version = "3.0.0", default-features = false, path = "../system" }
+pallet-balances = { version = "3.0.0", default-features = false, path = "../balances" }
+sp-runtime = { version = "3.0.0", default-features = false, path = "../../primitives/runtime" }
+sp-std = { version = "3.0.0", default-features = false, path = "../../primitives/std" }
+sp-io = { version = "3.0.0", default-features = false, path = "../../primitives/io" }
+sp-inherents = { version = "3.0.0", default-features = false, path = "../../primitives/inherents" }
+sp-transaction-storage-proof = { version = "3.0.0", default-features = false, path = "../../primitives/transaction-storage-proof" }
+frame-benchmarking = { version = "3.1.0", default-features = false, path = "../benchmarking", optional = true }
+
+[dev-dependencies]
+frame-support-test = { version = "3.0.0", path = "../support/test" }
+sp-transaction-storage-proof = { version = "3.0.0", default-features = true, path = "../../primitives/transaction-storage-proof" }
+sp-core = { version = "3.0.0", path = "../../primitives/core", default-features = false }
+
+[features]
+default = ["std"]
+runtime-benchmarks = [
+	"frame-benchmarking",
+	"hex-literal",
+]
+std = [
+	"serde",
+	"codec/std",
+	"sp-runtime/std",
+	"frame-support/std",
+	"frame-system/std",
+	"pallet-balances/std",
+	"sp-io/std",
+	"sp-std/std",
+	"sp-inherents/std",
+]

substrate/frame/transaction-storage/README.md

@@ -0,0 +1,8 @@
+# Transaction Storage Pallet
+
+Indexes transactions and manages storage proofs.
+# Transaction Storage Pallet
+
+Indexes transactions and manages storage proofs.
+
+License: Apache-2.0

substrate/frame/transaction-storage/src/benchmarking.rs

@@ -0,0 +1,147 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Benchmarks for transaction-storage Pallet
+
+#![cfg(feature = "runtime-benchmarks")]
+
+use sp_std::*;
+use super::*;
+use sp_runtime::traits::{Zero, One, Bounded};
+use sp_transaction_storage_proof::TransactionStorageProof;
+use frame_system::{RawOrigin, Pallet as System, EventRecord};
+use frame_benchmarking::{benchmarks, whitelisted_caller, impl_benchmark_test_suite};
+use frame_support::{traits::{Currency, OnFinalize, OnInitialize}};
+
+use crate::Pallet as TransactionStorage;
+
+const PROOF: &[u8] = &hex_literal::hex!("
+	0104000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+	0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+	0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+	0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+	0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+	000000000000000000000000000000014cd0780ffff80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe8
+	7d12a3662c4c0080e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb
+	13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2
+	f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f
+	1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f
+	3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a47
+	8e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cf
+	f93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e31
+	6a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f
+	53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c8
+	0e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4cbd05807777809a5d7a720ce5f9d9a012
+	fbf25e92c30e732dadba8f312b05e02976313ea64d9f807d43bcbf8a3dc2f6b9e957d129e610c06d411e11743062dc1cf
+	3ac289390ae4c8008592aa2d915f52941036afbe72bac4ebe7ce186c4ddc53f118e0ddd4decd8cc809a5d7a720ce5f9d9
+	a012fbf25e92c30e732dadba8f312b05e02976313ea64d9f807d43bcbf8a3dc2f6b9e957d129e610c06d411e11743062d
+	c1cf3ac289390ae4c00809a5d7a720ce5f9d9a012fbf25e92c30e732dadba8f312b05e02976313ea64d9f807d43bcbf8a
+	3dc2f6b9e957d129e610c06d411e11743062dc1cf3ac289390ae4c8008592aa2d915f52941036afbe72bac4ebe7ce186c
+	4ddc53f118e0ddd4decd8cc809a5d7a720ce5f9d9a012fbf25e92c30e732dadba8f312b05e02976313ea64d9f807d43bc
+	bf8a3dc2f6b9e957d129e610c06d411e11743062dc1cf3ac289390ae4c8008592aa2d915f52941036afbe72bac4ebe7ce
+	186c4ddc53f118e0ddd4decd8cccd0780ffff8081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb0
+	3bdb31008081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253
+	515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa139
+	8e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5
+	f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3a
+	a1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2b
+	a8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f32
+	2d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa
+	9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f0
+	2f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b82
+	5bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb31cd0780ffff80b4f23ac50c8e67d9b280f2b31a
+	5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd1885
+	44c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2
+	b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd
+	188544c5f9b0080b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9
+	b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84
+	d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e
+	67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977aca
+	ac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac5
+	0c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b89297
+	7acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b104401
+	0000
+");
+
+type BalanceOf<T> = <<T as Config>::Currency as Currency<<T as frame_system::Config>::AccountId>>::Balance;
+
+fn assert_last_event<T: Config>(generic_event: <T as Config>::Event) {
+	let events = System::<T>::events();
+	let system_event: <T as frame_system::Config>::Event = generic_event.into();
+	let EventRecord { event, .. } = &events[events.len() - 1];
+	assert_eq!(event, &system_event);
+}
+
+pub fn run_to_block<T: Config>(n: T::BlockNumber) {
+	while frame_system::Pallet::<T>::block_number() < n {
+		crate::Pallet::<T>::on_finalize(frame_system::Pallet::<T>::block_number());
+		frame_system::Pallet::<T>::on_finalize(frame_system::Pallet::<T>::block_number());
+		frame_system::Pallet::<T>::set_block_number(frame_system::Pallet::<T>::block_number() + One::one());
+		frame_system::Pallet::<T>::on_initialize(frame_system::Pallet::<T>::block_number());
+		crate::Pallet::<T>::on_initialize(frame_system::Pallet::<T>::block_number());
+	}
+}
+
+benchmarks! {
+	store {
+		let l in 1 .. MaxTransactionSize::<T>::get();
+		let caller: T::AccountId = whitelisted_caller();
+		T::Currency::make_free_balance_be(&caller, BalanceOf::<T>::max_value());
+	}: _(RawOrigin::Signed(caller.clone()), vec![0u8; l as usize])
+	verify {
+		assert!(!BlockTransactions::<T>::get().is_empty());
+		assert_last_event::<T>(Event::Stored(0).into());
+	}
+
+	renew {
+		let caller: T::AccountId = whitelisted_caller();
+		T::Currency::make_free_balance_be(&caller, BalanceOf::<T>::max_value());
+		TransactionStorage::<T>::store(
+			RawOrigin::Signed(caller.clone()).into(),
+			vec![0u8; MaxTransactionSize::<T>::get() as usize],
+		)?;
+		run_to_block::<T>(1u32.into());
+	}: _(RawOrigin::Signed(caller.clone()), T::BlockNumber::zero(), 0)
+	verify {
+		assert_last_event::<T>(Event::Renewed(0).into());
+	}
+
+	check_proof_max {
+		run_to_block::<T>(1u32.into());
+		let caller: T::AccountId = whitelisted_caller();
+		T::Currency::make_free_balance_be(&caller, BalanceOf::<T>::max_value());
+		for _ in 0 .. MaxBlockTransactions::<T>::get() {
+			TransactionStorage::<T>::store(
+				RawOrigin::Signed(caller.clone()).into(),
+				vec![0u8; MaxTransactionSize::<T>::get() as usize],
+			)?;
+		}
+		run_to_block::<T>(StoragePeriod::<T>::get() + T::BlockNumber::one());
+		let random_hash = [0u8];
+		let mut encoded_proof = PROOF;
+		let proof = TransactionStorageProof::decode(&mut encoded_proof).unwrap();
+	}: check_proof(RawOrigin::None, proof)
+	verify {
+		assert_last_event::<T>(Event::ProofChecked.into());
+	}
+}
+
+impl_benchmark_test_suite!(
+	TransactionStorage,
+	crate::mock::new_test_ext(),
+	crate::mock::Test,
+);

substrate/frame/transaction-storage/src/lib.rs

@@ -0,0 +1,436 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Transaction storage pallet. Indexes transactions and manages storage proofs.
+
+// Ensure we're `no_std` when compiling for Wasm.
+#![cfg_attr(not(feature = "std"), no_std)]
+
+mod benchmarking;
+pub mod weights;
+
+#[cfg(test)]
+mod mock;
+#[cfg(test)]
+mod tests;
+
+use frame_support::{
+	traits::{ReservableCurrency, Currency, OnUnbalanced},
+	dispatch::{Dispatchable, GetDispatchInfo},
+};
+use sp_std::prelude::*;
+use sp_std::{result};
+use codec::{Encode, Decode};
+use sp_runtime::traits::{Saturating, BlakeTwo256, Hash, Zero, One};
+use sp_transaction_storage_proof::{
+	TransactionStorageProof, InherentError,
+	random_chunk, encode_index,
+	CHUNK_SIZE, INHERENT_IDENTIFIER, DEFAULT_STORAGE_PERIOD,
+};
+
+/// A type alias for the balance type from this pallet's point of view.
+type BalanceOf<T> = <<T as Config>::Currency as Currency<<T as frame_system::Config>::AccountId>>::Balance;
+type NegativeImbalanceOf<T> = <<T as Config>::Currency as Currency<<T as frame_system::Config>::AccountId>>
+	::NegativeImbalance;
+
+// Re-export pallet items so that they can be accessed from the crate namespace.
+pub use pallet::*;
+pub use weights::WeightInfo;
+
+/// Maximum bytes that can be stored in one transaction.
+// Setting higher limit also requires raising the allocator limit.
+pub const DEFAULT_MAX_TRANSACTION_SIZE: u32 = 8 * 1024 * 1024;
+pub const DEFAULT_MAX_BLOCK_TRANSACTIONS: u32 = 512;
+
+/// State data for a stored transaction.
+#[derive(Encode, Decode, Clone, sp_runtime::RuntimeDebug, PartialEq, Eq)]
+pub struct TransactionInfo {
+	/// Chunk trie root.
+	chunk_root: <BlakeTwo256 as Hash>::Output,
+	/// Plain hash of indexed data.
+	content_hash: <BlakeTwo256 as Hash>::Output,
+	/// Size of indexed data in bytes.
+	size: u32,
+	/// Total number of chunks added in the block with this transaction. This
+	/// is used find transaction info by block chunk index using binary search.
+	block_chunks: u32,
+}
+
+fn num_chunks(bytes: u32) -> u32 {
+	((bytes as u64 + CHUNK_SIZE as u64 - 1) / CHUNK_SIZE as u64) as u32
+}
+
+#[frame_support::pallet]
+pub mod pallet {
+	use frame_support::pallet_prelude::*;
+	use frame_system::pallet_prelude::*;
+	use super::*;
+
+	#[pallet::config]
+	pub trait Config: frame_system::Config {
+		/// The overarching event type.
+		type Event: From<Event<Self>> + IsType<<Self as frame_system::Config>::Event>;
+		/// A dispatchable call.
+		type Call: Parameter + Dispatchable<Origin=Self::Origin> + GetDispatchInfo + From<frame_system::Call<Self>>;
+		/// The currency trait.
+		type Currency: ReservableCurrency<Self::AccountId>;
+		/// Handler for the unbalanced decrease when fees are burned.
+		type FeeDestination: OnUnbalanced<NegativeImbalanceOf<Self>>;
+		/// Weight information for extrinsics in this pallet.
+		type WeightInfo: WeightInfo;
+	}
+
+	#[pallet::error]
+	pub enum Error<T> {
+		/// Insufficient account balance.
+		InsufficientFunds,
+		/// Invalid configuration.
+		NotConfigured,
+		/// Renewed extrinsic is not found.
+		RenewedNotFound,
+		/// Attempting to store empty transaction
+		EmptyTransaction,
+		/// Proof was not expected in this block.
+		UnexpectedProof,
+		/// Proof failed verification.
+		InvalidProof,
+		/// Missing storage proof.
+		MissingProof,
+		/// Unable to verify proof becasue state data is missing.
+		MissingStateData,
+		/// Double proof check in the block.
+		DoubleCheck,
+		/// Storage proof was not checked in the block.
+		ProofNotChecked,
+		/// Transaction is too large.
+		TransactionTooLarge,
+		/// Too many transactions in the block.
+		TooManyTransactions,
+		/// Attempted to call `store` outside of block execution.
+		BadContext,
+	}
+
+	#[pallet::pallet]
+	#[pallet::generate_store(pub(super) trait Store)]
+	pub struct Pallet<T>(_);
+
+	#[pallet::hooks]
+	impl<T: Config> Hooks<BlockNumberFor<T>> for Pallet<T> {
+		fn on_initialize(n: T::BlockNumber) -> Weight {
+			// Drop obsolete roots. The proof for `obsolete` will be checked later
+			// in this block, so we drop `obsolete` - 1.
+			let period = <StoragePeriod<T>>::get();
+			let obsolete = n.saturating_sub(period.saturating_add(One::one()));
+			if obsolete > Zero::zero() {
+				<Transactions<T>>::remove(obsolete);
+				<ChunkCount<T>>::remove(obsolete);
+			}
+			// 2 writes in `on_initialize` and 2 writes + 2 reads in `on_finalize`
+			T::DbWeight::get().reads_writes(2, 4)
+		}
+
+		fn on_finalize(n: T::BlockNumber) {
+			assert!(
+				<ProofChecked<T>>::take()
+				|| {
+					// Proof is not required for early or empty blocks.
+					let number = <frame_system::Pallet<T>>::block_number();
+					let period = <StoragePeriod<T>>::get();
+					let target_number = number.saturating_sub(period);
+					target_number.is_zero() || <ChunkCount<T>>::get(target_number) == 0
+				},
+				"Storage proof must be checked once in the block"
+			);
+			// Insert new transactions
+			let transactions = <BlockTransactions<T>>::take();
+			let total_chunks = transactions.last().map_or(0, |t| t.block_chunks);
+			if total_chunks != 0 {
+				<ChunkCount<T>>::insert(n, total_chunks);
+				<Transactions<T>>::insert(n, transactions);
+			}
+		}
+	}
+
+	#[pallet::call]
+	impl<T: Config> Pallet<T> {
+		/// Index and store data on chain. Minimum data size is 1 bytes, maximum is `MaxTransactionSize`.
+		/// Data will be removed after `STORAGE_PERIOD` blocks, unless `renew` is called.
+		/// # <weight>
+		/// - n*log(n) of data size, as all data is pushed to an in-memory trie.
+		/// Additionally contains a DB write.
+		/// # </weight>
+		#[pallet::weight(T::WeightInfo::store(data.len() as u32))]
+		pub(super) fn store(
+			origin: OriginFor<T>,
+			data: Vec<u8>,
+		) -> DispatchResult {
+			ensure!(data.len() > 0, Error::<T>::EmptyTransaction);
+			ensure!(data.len() <= MaxTransactionSize::<T>::get() as usize, Error::<T>::TransactionTooLarge);
+			let sender = ensure_signed(origin)?;
+			Self::apply_fee(sender, data.len() as u32)?;
+
+			// Chunk data and compute storage root
+			let chunk_count = num_chunks(data.len() as u32);
+			let chunks = data.chunks(CHUNK_SIZE).map(|c| c.to_vec()).collect();
+			let root = sp_io::trie::blake2_256_ordered_root(chunks);
+
+			let content_hash = sp_io::hashing::blake2_256(&data);
+			let extrinsic_index = <frame_system::Pallet<T>>::extrinsic_index().ok_or_else(
+				|| Error::<T>::BadContext)?;
+			sp_io::transaction_index::index(extrinsic_index, data.len() as u32, content_hash);
+
+			let mut index = 0;
+			<BlockTransactions<T>>::mutate(|transactions| {
+				if transactions.len() + 1 > MaxBlockTransactions::<T>::get() as usize {
+					return Err(Error::<T>::TooManyTransactions)
+				}
+				let total_chunks = transactions.last().map_or(0, |t| t.block_chunks) + chunk_count;
+				index = transactions.len() as u32;
+				transactions.push(TransactionInfo {
+					chunk_root: root,
+					size: data.len() as u32,
+					content_hash: content_hash.into(),
+					block_chunks: total_chunks,
+				});
+				Ok(())
+			})?;
+			Self::deposit_event(Event::Stored(index));
+			Ok(())
+		}
+
+		/// Renew previously stored data. Parameters are the block number that contains
+		/// previous `store` or `renew` call and transaction index within that block.
+		/// Transaction index is emitted in the `Stored` or `Renewed` event.
+		/// Applies same fees as `store`.
+		/// # <weight>
+		/// - Constant.
+		/// # </weight>
+		#[pallet::weight(T::WeightInfo::renew())]
+		pub(super) fn renew(
+			origin: OriginFor<T>,
+			block: T::BlockNumber,
+			index: u32,
+		) -> DispatchResultWithPostInfo {
+			let sender = ensure_signed(origin)?;
+			let transactions = <Transactions<T>>::get(block).ok_or(Error::<T>::RenewedNotFound)?;
+			let info = transactions.get(index as usize).ok_or(Error::<T>::RenewedNotFound)?;
+			Self::apply_fee(sender, info.size)?;
+
+			let extrinsic_index = <frame_system::Pallet<T>>::extrinsic_index().unwrap();
+			sp_io::transaction_index::renew(extrinsic_index, info.content_hash.into());
+
+			let mut index = 0;
+			<BlockTransactions<T>>::mutate(|transactions| {
+				if transactions.len() + 1 > MaxBlockTransactions::<T>::get() as usize {
+					return Err(Error::<T>::TooManyTransactions)
+				}
+				let chunks = num_chunks(info.size);
+				let total_chunks = transactions.last().map_or(0, |t| t.block_chunks) + chunks;
+				index = transactions.len() as u32;
+				transactions.push(TransactionInfo {
+					chunk_root: info.chunk_root,
+					size: info.size,
+					content_hash: info.content_hash,
+					block_chunks: total_chunks,
+				});
+				Ok(())
+			})?;
+			Self::deposit_event(Event::Renewed(index));
+			Ok(().into())
+		}
+
+		/// Check storage proof for block number `block_number() - StoragePeriod`.
+		/// If such block does not exist the proof is expected to be `None`.
+		/// # <weight>
+		/// - Linear w.r.t the number of indexed transactions in the proved block for random probing.
+		/// There's a DB read for each transaction.
+		/// Here we assume a maximum of 100 probed transactions.
+		/// # </weight>
+		#[pallet::weight((T::WeightInfo::check_proof_max(), DispatchClass::Mandatory))]
+		pub(super) fn check_proof(
+			origin: OriginFor<T>,
+			proof: TransactionStorageProof,
+		) -> DispatchResultWithPostInfo {
+			ensure_none(origin)?;
+			ensure!(!ProofChecked::<T>::get(), Error::<T>::DoubleCheck);
+			let number = <frame_system::Pallet<T>>::block_number();
+			let period = <StoragePeriod<T>>::get();
+			let target_number = number.saturating_sub(period);
+			ensure!(!target_number.is_zero(), Error::<T>::UnexpectedProof);
+			let total_chunks = <ChunkCount<T>>::get(target_number);
+			ensure!(total_chunks != 0, Error::<T>::UnexpectedProof);
+			let parent_hash = <frame_system::Pallet<T>>::parent_hash();
+			let selected_chunk_index = random_chunk(parent_hash.as_ref(), total_chunks);
+			let (info, chunk_index) = match <Transactions<T>>::get(target_number) {
+				Some(infos) => {
+					let index = match infos.binary_search_by_key(&selected_chunk_index, |info| info.block_chunks) {
+						Ok(index) => index,
+						Err(index) => index,
+					};
+					let info = infos.get(index).ok_or_else(|| Error::<T>::MissingStateData)?.clone();
+					let chunks = num_chunks(info.size);
+					let prev_chunks = info.block_chunks - chunks;
+					(info, selected_chunk_index - prev_chunks)
+				},
+				None => Err(Error::<T>::MissingStateData)?,
+			};
+			ensure!(
+				sp_io::trie::blake2_256_verify_proof(
+					info.chunk_root,
+					&proof.proof,
+					&encode_index(chunk_index),
+					&proof.chunk,
+				),
+				Error::<T>::InvalidProof
+			);
+			ProofChecked::<T>::put(true);
+			Self::deposit_event(Event::ProofChecked);
+			Ok(().into())
+		}
+	}
+
+	#[pallet::event]
+	#[pallet::generate_deposit(pub(super) fn deposit_event)]
+	pub enum Event<T: Config> {
+		/// Stored data under specified index.
+		Stored(u32),
+		/// Renewed data under specified index.
+		Renewed(u32),
+		/// Storage proof was successfully checked.
+		ProofChecked,
+	}
+
+	/// Collection of transaction metadata by block number.
+	#[pallet::storage]
+	#[pallet::getter(fn transaction_roots)]
+	pub(super) type Transactions<T: Config> = StorageMap<
+		_,
+		Blake2_128Concat,
+		T::BlockNumber,
+		Vec<TransactionInfo>,
+		OptionQuery,
+	>;
+
+	/// Count indexed chunks for each block.
+	#[pallet::storage]
+	pub(super) type ChunkCount<T: Config> = StorageMap<
+		_,
+		Blake2_128Concat,
+		T::BlockNumber,
+		u32,
+		ValueQuery,
+	>;
+
+	#[pallet::storage]
+	#[pallet::getter(fn byte_fee)]
+	/// Storage fee per byte.
+	pub(super) type ByteFee<T: Config> = StorageValue<_, BalanceOf<T>>;
+
+	#[pallet::storage]
+	#[pallet::getter(fn entry_fee)]
+	/// Storage fee per transaction.
+	pub(super) type EntryFee<T: Config> = StorageValue<_, BalanceOf<T>>;
+
+	#[pallet::storage]
+	#[pallet::getter(fn max_transaction_size)]
+	/// Maximum data set in a single transaction in bytes.
+	pub(super) type MaxTransactionSize<T: Config> = StorageValue<_, u32, ValueQuery>;
+
+	#[pallet::storage]
+	#[pallet::getter(fn max_block_transactions)]
+	/// Maximum number of indexed transactions in the block.
+	pub(super) type MaxBlockTransactions<T: Config> = StorageValue<_, u32, ValueQuery>;
+
+	/// Storage period for data in blocks. Should match `sp_storage_proof::DEFAULT_STORAGE_PERIOD`
+	/// for block authoring.
+	#[pallet::storage]
+	pub(super) type StoragePeriod<T: Config> = StorageValue<_, T::BlockNumber, ValueQuery>;
+
+	// Intermediates
+	#[pallet::storage]
+	pub(super) type BlockTransactions<T: Config> = StorageValue<_, Vec<TransactionInfo>, ValueQuery>;
+
+	/// Was the proof checked in this block?
+	#[pallet::storage]
+	pub(super) type ProofChecked<T: Config> = StorageValue<_, bool, ValueQuery>;
+
+
+	#[pallet::genesis_config]
+	pub struct GenesisConfig<T: Config> {
+		pub byte_fee: BalanceOf<T>,
+		pub entry_fee: BalanceOf<T>,
+		pub storage_period: T::BlockNumber,
+		pub max_block_transactions: u32,
+		pub max_transaction_size: u32,
+	}
+
+	#[cfg(feature = "std")]
+	impl<T: Config> Default for GenesisConfig<T> {
+		fn default() -> Self {
+			Self {
+				byte_fee: 10u32.into(),
+				entry_fee: 1000u32.into(),
+				storage_period: DEFAULT_STORAGE_PERIOD.into(),
+				max_block_transactions: DEFAULT_MAX_BLOCK_TRANSACTIONS,
+				max_transaction_size: DEFAULT_MAX_TRANSACTION_SIZE,
+			}
+		}
+	}
+
+	#[pallet::genesis_build]
+	impl<T: Config> GenesisBuild<T> for GenesisConfig<T> {
+		fn build(&self) {
+			<ByteFee<T>>::put(&self.byte_fee);
+			<EntryFee<T>>::put(&self.entry_fee);
+			<MaxTransactionSize<T>>::put(&self.max_transaction_size);
+			<MaxBlockTransactions<T>>::put(&self.max_block_transactions);
+			<StoragePeriod<T>>::put(&self.storage_period);
+		}
+	}
+
+	#[pallet::inherent]
+	impl<T: Config> ProvideInherent for Pallet<T> {
+		type Call = Call<T>;
+		type Error = InherentError;
+		const INHERENT_IDENTIFIER: InherentIdentifier = INHERENT_IDENTIFIER;
+
+		fn create_inherent(data: &InherentData) -> Option<Self::Call> {
+			let proof = data.get_data::<TransactionStorageProof>(&Self::INHERENT_IDENTIFIER).unwrap_or(None);
+			proof.map(Call::check_proof)
+		}
+
+		fn check_inherent(_call: &Self::Call, _data: &InherentData) -> result::Result<(), Self::Error> {
+			Ok(())
+		}
+
+		fn is_inherent(call: &Self::Call) -> bool {
+			matches!(call, Call::check_proof(_))
+		}
+	}
+
+	impl<T: Config> Pallet<T> {
+		fn apply_fee(sender: T::AccountId, size: u32) -> DispatchResult {
+			let byte_fee = ByteFee::<T>::get().ok_or(Error::<T>::NotConfigured)?;
+			let entry_fee = EntryFee::<T>::get().ok_or(Error::<T>::NotConfigured)?;
+			let fee = byte_fee.saturating_mul(size.into()).saturating_add(entry_fee);
+			ensure!(T::Currency::can_slash(&sender, fee), Error::<T>::InsufficientFunds);
+			let (credit, _) = T::Currency::slash(&sender, fee);
+			T::FeeDestination::on_unbalanced(credit);
+			Ok(())
+		}
+	}
+}

substrate/frame/transaction-storage/src/mock.rs

@@ -0,0 +1,129 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Test environment for transaction-storage pallet.
+
+use crate as pallet_transaction_storage;
+use crate::TransactionStorageProof;
+use sp_core::H256;
+use sp_runtime::{traits::{BlakeTwo256, IdentityLookup}, testing::Header, BuildStorage};
+use frame_support::{
+	parameter_types,
+	traits::{OnInitialize, OnFinalize},
+};
+
+
+type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic<Test>;
+pub type Block = frame_system::mocking::MockBlock<Test>;
+
+// Configure a mock runtime to test the pallet.
+frame_support::construct_runtime!(
+	pub enum Test where
+		Block = Block,
+		NodeBlock = Block,
+		UncheckedExtrinsic = UncheckedExtrinsic,
+	{
+		System: frame_system::{Pallet, Call, Config, Storage, Event<T>},
+		Balances: pallet_balances::{Pallet, Call, Config<T>, Storage, Event<T>},
+		TransactionStorage: pallet_transaction_storage::{
+			Pallet, Call, Storage, Config<T>, Inherent, Event<T>
+		},
+	}
+);
+
+parameter_types! {
+	pub const BlockHashCount: u64 = 250;
+	pub const SS58Prefix: u8 = 42;
+}
+
+impl frame_system::Config for Test {
+	type BaseCallFilter = ();
+	type BlockWeights = ();
+	type BlockLength = ();
+	type Origin = Origin;
+	type Call = Call;
+	type Index = u64;
+	type BlockNumber = u64;
+	type Hash = H256;
+	type Hashing = BlakeTwo256;
+	type AccountId = u64;
+	type Lookup = IdentityLookup<Self::AccountId>;
+	type Header = Header;
+	type Event = Event;
+	type BlockHashCount = BlockHashCount;
+	type DbWeight = ();
+	type Version = ();
+	type PalletInfo = PalletInfo;
+	type AccountData = pallet_balances::AccountData<u64>;
+	type OnNewAccount = ();
+	type OnKilledAccount = ();
+	type SystemWeightInfo = ();
+	type SS58Prefix = SS58Prefix;
+	type OnSetCode = ();
+}
+
+parameter_types! {
+	pub const ExistentialDeposit: u64 = 1;
+}
+
+impl pallet_balances::Config for Test {
+	type Balance = u64;
+	type DustRemoval = ();
+	type Event = Event;
+	type ExistentialDeposit = ExistentialDeposit;
+	type AccountStore = System;
+	type WeightInfo = ();
+	type MaxLocks = ();
+}
+
+impl pallet_transaction_storage::Config for Test {
+	type Event = Event;
+	type Call = Call;
+	type Currency = Balances;
+	type FeeDestination = ();
+	type WeightInfo = ();
+}
+
+pub fn new_test_ext() -> sp_io::TestExternalities {
+	let t = GenesisConfig {
+		frame_system: Default::default(),
+		pallet_balances: pallet_balances::GenesisConfig::<Test> {
+			balances: vec![(1, 1000000000), (2, 100), (3, 100), (4, 100)]
+		},
+		pallet_transaction_storage: pallet_transaction_storage::GenesisConfig::<Test> {
+			storage_period: 10,
+			byte_fee: 2,
+			entry_fee: 200,
+			max_block_transactions: crate::DEFAULT_MAX_BLOCK_TRANSACTIONS,
+			max_transaction_size: crate::DEFAULT_MAX_TRANSACTION_SIZE,
+		},
+	}.build_storage().unwrap();
+	t.into()
+}
+
+pub fn run_to_block(n: u64, f: impl Fn() -> Option<TransactionStorageProof>) {
+	while System::block_number() < n {
+		if let Some(proof) = f() {
+			TransactionStorage::check_proof(Origin::none(), proof).unwrap();
+		}
+		TransactionStorage::on_finalize(System::block_number());
+		System::on_finalize(System::block_number());
+		System::set_block_number(System::block_number() + 1);
+		System::on_initialize(System::block_number());
+		TransactionStorage::on_initialize(System::block_number());
+	}
+}

substrate/frame/transaction-storage/src/tests.rs

@@ -0,0 +1,157 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Tests for transction-storage pallet.
+
+use super::*;
+use crate::mock::*;
+use super::Pallet as TransactionStorage;
+use frame_support::{assert_ok, assert_noop};
+use frame_system::RawOrigin;
+use sp_transaction_storage_proof::registration::build_proof;
+
+const MAX_DATA_SIZE: u32 = DEFAULT_MAX_TRANSACTION_SIZE;
+
+#[test]
+fn discards_data() {
+	new_test_ext().execute_with(|| {
+		run_to_block(1, || None);
+		let caller = 1;
+		assert_ok!(TransactionStorage::<Test>::store(
+			RawOrigin::Signed(caller.clone()).into(),
+			vec![0u8; 2000 as usize]
+		));
+		assert_ok!(TransactionStorage::<Test>::store(
+			RawOrigin::Signed(caller.clone()).into(),
+			vec![0u8; 2000 as usize]
+		));
+		let proof_provider = || {
+			let block_num = <frame_system::Pallet<Test>>::block_number();
+			if block_num == 11  {
+				let parent_hash = <frame_system::Pallet<Test>>::parent_hash();
+				Some(build_proof(parent_hash.as_ref(), vec![vec![0u8; 2000], vec![0u8; 2000]]).unwrap())
+			} else {
+				None
+			}
+		};
+		run_to_block(11, proof_provider);
+		assert!(Transactions::<Test>::get(1).is_some());
+		let transctions = Transactions::<Test>::get(1).unwrap();
+		assert_eq!(transctions.len(), 2);
+		assert_eq!(ChunkCount::<Test>::get(1), 16);
+		run_to_block(12, proof_provider);
+		assert!(Transactions::<Test>::get(1).is_none());
+		assert_eq!(ChunkCount::<Test>::get(1), 0);
+	});
+}
+
+#[test]
+fn burns_fee() {
+	new_test_ext().execute_with(|| {
+		run_to_block(1, || None);
+		let caller = 1;
+		assert_noop!(TransactionStorage::<Test>::store(
+				RawOrigin::Signed(5).into(),
+				vec![0u8; 2000 as usize]
+			),
+			Error::<Test>::InsufficientFunds,
+		);
+		assert_ok!(TransactionStorage::<Test>::store(
+				RawOrigin::Signed(caller.clone()).into(),
+				vec![0u8; 2000 as usize]
+		));
+		assert_eq!(Balances::free_balance(1), 1_000_000_000 - 2000 * 2 - 200);
+	});
+}
+
+#[test]
+fn checks_proof() {
+	new_test_ext().execute_with(|| {
+		run_to_block(1, || None);
+		let caller = 1;
+		assert_ok!(TransactionStorage::<Test>::store(
+			RawOrigin::Signed(caller.clone()).into(),
+			vec![0u8; MAX_DATA_SIZE as usize]
+		));
+		run_to_block(10, || None);
+		let parent_hash = <frame_system::Pallet<Test>>::parent_hash();
+		let proof = build_proof(
+			parent_hash.as_ref(),
+			vec![vec![0u8; MAX_DATA_SIZE as usize]]
+		).unwrap();
+		assert_noop!(TransactionStorage::<Test>::check_proof(
+				Origin::none(),
+				proof,
+			),
+			Error::<Test>::UnexpectedProof,
+		);
+		run_to_block(11, || None);
+		let parent_hash = <frame_system::Pallet<Test>>::parent_hash();
+
+		let invalid_proof = build_proof(
+			parent_hash.as_ref(),
+			vec![vec![0u8; 1000]]
+		).unwrap();
+		assert_noop!(TransactionStorage::<Test>::check_proof(
+				Origin::none(),
+				invalid_proof,
+		),
+		Error::<Test>::InvalidProof,
+		);
+
+		let proof = build_proof(
+			parent_hash.as_ref(),
+			vec![vec![0u8; MAX_DATA_SIZE as usize]]
+		).unwrap();
+		assert_ok!(TransactionStorage::<Test>::check_proof(Origin::none(), proof));
+	});
+}
+
+#[test]
+fn renews_data() {
+	new_test_ext().execute_with(|| {
+		run_to_block(1, || None);
+		let caller = 1;
+		assert_ok!(TransactionStorage::<Test>::store(
+				RawOrigin::Signed(caller.clone()).into(),
+				vec![0u8; 2000]
+		));
+		let info = BlockTransactions::<Test>::get().last().unwrap().clone();
+		run_to_block(6, || None);
+		assert_ok!(TransactionStorage::<Test>::renew(
+				RawOrigin::Signed(caller.clone()).into(),
+				1, // block
+				0, // transaction
+		));
+		assert_eq!(Balances::free_balance(1), 1_000_000_000 - 4000 * 2 - 200 * 2);
+		let proof_provider = || {
+			let block_num = <frame_system::Pallet<Test>>::block_number();
+			if block_num == 11 || block_num == 16  {
+				let parent_hash = <frame_system::Pallet<Test>>::parent_hash();
+				Some(build_proof(parent_hash.as_ref(), vec![vec![0u8; 2000]]).unwrap())
+			} else {
+				None
+			}
+		};
+		run_to_block(16, proof_provider);
+		assert!(Transactions::<Test>::get(1).is_none());
+		assert_eq!(Transactions::<Test>::get(6).unwrap().get(0), Some(info).as_ref());
+		run_to_block(17, proof_provider);
+		assert!(Transactions::<Test>::get(6).is_none());
+	});
+}
+

substrate/frame/transaction-storage/src/weights.rs

@@ -0,0 +1,95 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Autogenerated weights for pallet_transaction_storage
+//!
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 3.0.0
+//! DATE: 2021-06-03, STEPS: `[20, ]`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]`
+//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128
+
+// Executed Command:
+// ./target/release/substrate
+// benchmark
+// --chain
+// dev
+// --steps
+// 20
+// --repeat=20
+// --pallet=pallet_transaction_storage
+// --extrinsic=*
+// --execution=wasm
+// --wasm-execution=compiled
+// --heap-pages=4096
+// --output=./frame/transaction-storage/src/weights.rs
+// --template=./.maintain/frame-weight-template.hbs
+
+
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::{traits::Get, weights::{Weight, constants::RocksDbWeight}};
+use sp_std::marker::PhantomData;
+
+/// Weight functions needed for pallet_transaction_storage.
+pub trait WeightInfo {
+	fn store(l: u32, ) -> Weight;
+	fn renew() -> Weight;
+	fn check_proof_max() -> Weight;
+}
+
+/// Weights for pallet_transaction_storage using the Substrate node and recommended hardware.
+pub struct SubstrateWeight<T>(PhantomData<T>);
+impl<T: frame_system::Config> WeightInfo for SubstrateWeight<T> {
+	fn store(l: u32, ) -> Weight {
+		(0 as Weight)
+			// Standard Error: 0
+			.saturating_add((10_000 as Weight).saturating_mul(l as Weight))
+			.saturating_add(T::DbWeight::get().reads(6 as Weight))
+			.saturating_add(T::DbWeight::get().writes(1 as Weight))
+	}
+	fn renew() -> Weight {
+		(97_000_000 as Weight)
+			.saturating_add(T::DbWeight::get().reads(6 as Weight))
+			.saturating_add(T::DbWeight::get().writes(1 as Weight))
+	}
+	fn check_proof_max() -> Weight {
+		(99_000_000 as Weight)
+			.saturating_add(T::DbWeight::get().reads(5 as Weight))
+			.saturating_add(T::DbWeight::get().writes(1 as Weight))
+	}
+}
+
+// For backwards compatibility and tests
+impl WeightInfo for () {
+	fn store(l: u32, ) -> Weight {
+		(0 as Weight)
+			// Standard Error: 0
+			.saturating_add((10_000 as Weight).saturating_mul(l as Weight))
+			.saturating_add(RocksDbWeight::get().reads(6 as Weight))
+			.saturating_add(RocksDbWeight::get().writes(1 as Weight))
+	}
+	fn renew() -> Weight {
+		(97_000_000 as Weight)
+			.saturating_add(RocksDbWeight::get().reads(6 as Weight))
+			.saturating_add(RocksDbWeight::get().writes(1 as Weight))
+	}
+	fn check_proof_max() -> Weight {
+		(99_000_000 as Weight)
+			.saturating_add(RocksDbWeight::get().reads(5 as Weight))
+			.saturating_add(RocksDbWeight::get().writes(1 as Weight))
+	}
+}

substrate/primitives/blockchain/src/backend.rs

@@ -227,6 +227,8 @@ pub trait Backend<Block: BlockT>: HeaderBackend<Block> + HeaderMetadata<Block, E
 	fn has_indexed_transaction(&self, hash: &Block::Hash) -> Result<bool> {
 		Ok(self.indexed_transaction(hash)?.is_some())
 	}
+
+	fn block_indexed_body(&self, id: BlockId<Block>) -> Result<Option<Vec<Vec<u8>>>>;
 }
 
 /// Provides access to the optional cache.

substrate/primitives/externalities/src/lib.rs

@@ -229,12 +229,12 @@ pub trait Externalities: ExtensionStore {
 	fn storage_commit_transaction(&mut self) -> Result<(), ()>;
 
 	/// Index specified transaction slice and store it.
-	fn storage_index_transaction(&mut self, _index: u32, _offset: u32) {
+	fn storage_index_transaction(&mut self, _index: u32, _hash: &[u8], _size: u32) {
 		unimplemented!("storage_index_transaction");
 	}
 
 	/// Renew existing piece of transaction storage.
-	fn storage_renew_transaction_index(&mut self, _index: u32, _hash: &[u8], _size: u32) {
+	fn storage_renew_transaction_index(&mut self, _index: u32, _hash: &[u8]) {
 		unimplemented!("storage_renew_transaction_index");
 	}
 

substrate/primitives/io/src/lib.rs

@@ -429,6 +429,24 @@ pub trait Trie {
 	fn keccak_256_ordered_root(input: Vec<Vec<u8>>) -> H256 {
 		Layout::<sp_core::KeccakHasher>::ordered_trie_root(input)
 	}
+
+	/// Verify trie proof
+	fn blake2_256_verify_proof(root: H256, proof: &[Vec<u8>], key: &[u8], value: &[u8]) -> bool {
+		sp_trie::verify_trie_proof::<Layout<sp_core::Blake2Hasher>, _, _, _>(
+			&root,
+			proof,
+			&[(key, Some(value))],
+		).is_ok()
+	}
+
+	/// Verify trie proof
+	fn keccak_256_verify_proof(root: H256, proof: &[Vec<u8>], key: &[u8], value: &[u8]) -> bool {
+		sp_trie::verify_trie_proof::<Layout<sp_core::KeccakHasher>, _, _, _>(
+			&root,
+			proof,
+			&[(key, Some(value))],
+		).is_ok()
+	}
 }
 
 /// Interface that provides miscellaneous functions for communicating between the runtime and the node.
@@ -824,6 +842,20 @@ pub trait Hashing {
 	}
 }
 
+/// Interface that provides transaction indexing API.
+#[runtime_interface]
+pub trait TransactionIndex {
+	/// Add transaction index. Returns indexed content hash.
+	fn index(&mut self, extrinsic: u32, size: u32, context_hash: [u8; 32]) {
+		self.storage_index_transaction(extrinsic, &context_hash, size);
+	}
+
+	/// Conduct a 512-bit Keccak hash.
+	fn renew(&mut self, extrinsic: u32, context_hash: [u8; 32]) {
+		self.storage_renew_transaction_index(extrinsic, &context_hash);
+	}
+}
+
 /// Interface that provides functions to access the Offchain DB.
 #[runtime_interface]
 pub trait OffchainIndex {
@@ -1434,6 +1466,7 @@ pub type SubstrateHostFunctions = (
 	crate::trie::HostFunctions,
 	offchain_index::HostFunctions,
 	runtime_tasks::HostFunctions,
+	transaction_index::HostFunctions,
 );
 
 #[cfg(test)]

substrate/primitives/state-machine/src/ext.rs

@@ -629,33 +629,34 @@ where
 		}
 	}
 
-	fn storage_index_transaction(&mut self, index: u32, offset: u32) {
+	fn storage_index_transaction(&mut self, index: u32, hash: &[u8], size: u32) {
 		trace!(
 			target: "state",
-			"{:04x}: IndexTransaction ({}): [{}..]",
+			"{:04x}: IndexTransaction ({}): {}, {} bytes",
 			self.id,
 			index,
-			offset,
+			HexDisplay::from(&hash),
+			size,
 		);
 		self.overlay.add_transaction_index(IndexOperation::Insert {
 			extrinsic: index,
-			offset,
+			hash: hash.to_vec(),
+			size,
 		});
 	}
 
 	/// Renew existing piece of data storage.
-	fn storage_renew_transaction_index(&mut self, index: u32, hash: &[u8], size: u32) {
+	fn storage_renew_transaction_index(&mut self, index: u32, hash: &[u8]) {
 		trace!(
 			target: "state",
-			"{:04x}: RenewTransactionIndex ({}) {} bytes",
+			"{:04x}: RenewTransactionIndex ({}): {}",
 			self.id,
+			index,
 			HexDisplay::from(&hash),
-			size,
 		);
 		self.overlay.add_transaction_index(IndexOperation::Renew {
 			extrinsic: index,
 			hash: hash.to_vec(),
-			size
 		});
 	}
 

substrate/primitives/state-machine/src/overlayed_changes/mod.rs

@@ -118,8 +118,10 @@ pub enum IndexOperation {
 	Insert {
 		/// Extrinsic index in the current block.
 		extrinsic: u32,
-		/// Data offset in the extrinsic.
-		offset: u32,
+		/// Data content hash.
+		hash: Vec<u8>,
+		/// Indexed data size.
+		size: u32,
 	},
 	/// Renew existing transaction storage.
 	Renew {
@@ -127,8 +129,6 @@ pub enum IndexOperation {
 		extrinsic: u32,
 		/// Referenced index hash.
 		hash: Vec<u8>,
-		/// Expected data size.
-		size: u32,
 	}
 }
 
@@ -520,6 +520,11 @@ impl OverlayedChanges {
 		self.children.get(key).map(|(overlay, info)| (overlay.changes(), info))
 	}
 
+	/// Get an list of all index operations.
+	pub fn transaction_index_ops(&self) -> &[IndexOperation] {
+		&self.transaction_index_ops
+	}
+
 	/// Convert this instance with all changes into a [`StorageChanges`] instance.
 	#[cfg(feature = "std")]
 	pub fn into_storage_changes<

substrate/primitives/transaction-storage-proof/Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+name = "sp-transaction-storage-proof"
+version = "3.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "Transaction storage proof primitives"
+edition = "2018"
+license = "Apache-2.0"
+homepage = "https://substrate.dev"
+repository = "https://github.com/paritytech/substrate/"
+readme = "README.md"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[dependencies]
+sp-inherents = { version = "3.0.0", default-features = false, path = "../inherents" }
+sp-runtime = { version = "3.0.0", default-features = false, path = "../runtime" }
+sp-std = { version = "3.0.0", default-features = false, path = "../std" }
+sp-trie = { version = "3.0.0", optional = true, path = "../trie" }
+sp-core = { version = "3.0.0", path = "../core", optional = true }
+codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] }
+log = { version = "0.4.8", optional = true }
+async-trait = { version = "0.1.48", optional = true }
+
+[features]
+default = [ "std" ]
+std = [
+	"codec/std",
+	"sp-std/std",
+	"sp-inherents/std",
+	"sp-runtime/std",
+	"sp-trie/std",
+	"sp-core",
+	"log",
+	"async-trait",
+]

substrate/primitives/transaction-storage-proof/README.md

@@ -0,0 +1,3 @@
+Authorship Primitives
+
+License: Apache-2.0

substrate/primitives/transaction-storage-proof/src/lib.rs

@@ -0,0 +1,240 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Storge proof primitives. Constains types and basic code to extract storage
+//! proofs for indexed transactions.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+
+use sp_std::{result::Result, prelude::*};
+
+use codec::{Encode, Decode};
+use sp_inherents::{InherentIdentifier, InherentData, IsFatalError};
+use sp_runtime::{traits::{Block as BlockT, NumberFor}};
+
+pub use sp_inherents::Error;
+
+/// The identifier for the proof inherent.
+pub const INHERENT_IDENTIFIER: InherentIdentifier = *b"tx_proof";
+/// Storage period for data.
+pub const DEFAULT_STORAGE_PERIOD: u32 = 100800;
+/// Proof trie value size.
+pub const CHUNK_SIZE: usize = 256;
+
+/// Errors that can occur while checking the storage proof.
+#[derive(Encode, sp_runtime::RuntimeDebug)]
+#[cfg_attr(feature = "std", derive(Decode))]
+pub enum InherentError {
+	InvalidProof,
+	TrieError
+}
+
+impl IsFatalError for InherentError {
+	fn is_fatal_error(&self) -> bool {
+		true
+	}
+}
+
+#[derive(Encode, Decode, Clone, PartialEq, Debug)]
+pub struct TransactionStorageProof {
+	/// Data chunk that is proved to exist.
+	pub chunk: Vec<u8>,
+	/// Trie nodes that compose the proof.
+	pub proof: Vec<Vec<u8>>,
+}
+
+/// Auxiliary trait to extract storage proof.
+pub trait TransactionStorageProofInherentData {
+	/// Get the proof.
+	fn storage_proof(&self) -> Result<Option<TransactionStorageProof>, Error>;
+}
+
+impl TransactionStorageProofInherentData for InherentData {
+	fn storage_proof(&self) -> Result<Option<TransactionStorageProof>, Error> {
+		Ok(self.get_data(&INHERENT_IDENTIFIER)?)
+	}
+}
+
+/// Provider for inherent data.
+#[cfg(feature = "std")]
+pub struct InherentDataProvider {
+	proof: Option<TransactionStorageProof>,
+}
+
+#[cfg(feature = "std")]
+impl InherentDataProvider {
+	pub fn new(proof: Option<TransactionStorageProof>) -> Self {
+		InherentDataProvider { proof }
+	}
+}
+
+#[cfg(feature = "std")]
+#[async_trait::async_trait]
+impl sp_inherents::InherentDataProvider for InherentDataProvider {
+	fn provide_inherent_data(&self, inherent_data: &mut InherentData) -> Result<(), Error> {
+		if let Some(proof) = &self.proof {
+			inherent_data.put_data(INHERENT_IDENTIFIER, proof)
+		} else {
+			Ok(())
+		}
+	}
+
+	async fn try_handle_error(
+		&self,
+		identifier: &InherentIdentifier,
+		error: &[u8],
+	) -> Option<Result<(), Error>> {
+		if *identifier != INHERENT_IDENTIFIER {
+			return None
+		}
+
+		let error = InherentError::decode(&mut &error[..]).ok()?;
+
+		Some(Err(Error::Application(Box::from(format!("{:?}", error)))))
+	}
+}
+
+/// An utility function to extract chunk index from the source of randomness.
+pub fn random_chunk(random_hash: &[u8], total_chunks: u32) -> u32 {
+	let mut buf = [0u8; 8];
+	buf.copy_from_slice(&random_hash[0..8]);
+	let random_u64 = u64::from_be_bytes(buf);
+	(random_u64 % total_chunks as u64) as u32
+}
+
+/// An utility function to enocde transaction index as trie key.
+pub fn encode_index(input: u32) -> Vec<u8> {
+	codec::Encode::encode(&codec::Compact(input))
+}
+
+/// An interface to request indexed data from the client.
+pub trait IndexedBody<B: BlockT> {
+	fn block_indexed_body(
+		&self,
+		number: NumberFor<B>,
+	) -> Result<Option<Vec<Vec<u8>>>, Error>;
+
+	fn number(
+		&self,
+		hash: B::Hash,
+	) -> Result<Option<NumberFor<B>>, Error>;
+}
+
+#[cfg(feature = "std")]
+pub mod registration {
+	use sp_runtime::{traits::{Block as BlockT, Saturating, Zero, One}};
+	use sp_trie::TrieMut;
+	use super::*;
+
+	type Hasher = sp_core::Blake2Hasher;
+	type TrieLayout = sp_trie::Layout::<Hasher>;
+
+	/// Create a new inherent data provider instance for a given parent block hash.
+	pub fn new_data_provider<B, C>(
+		client: &C,
+		parent: &B::Hash,
+	) -> Result<InherentDataProvider, Error>
+	where
+		B: BlockT,
+		C: IndexedBody<B>,
+	{
+		let parent_number = client.number(parent.clone())?.unwrap_or(Zero::zero());
+		let number = parent_number
+			.saturating_add(One::one())
+			.saturating_sub(DEFAULT_STORAGE_PERIOD.into());
+		if number.is_zero() {
+			// Too early to collect proofs.
+			return Ok(InherentDataProvider::new(None));
+		}
+
+		let proof = match client.block_indexed_body(number)? {
+			Some(transactions) => {
+				Some(build_proof(parent.as_ref(), transactions)?)
+			},
+			None => {
+				// Nothing was indexed in that block.
+				None
+			}
+		};
+		Ok(InherentDataProvider::new(proof))
+	}
+
+	/// Build a proof for a given source of randomness and indexed transactions.
+	pub fn build_proof(random_hash: &[u8], transactions: Vec<Vec<u8>>)
+		-> Result<TransactionStorageProof, Error>
+	{
+		let mut db = sp_trie::MemoryDB::<Hasher>::default();
+
+		let mut target_chunk = None;
+		let mut target_root = Default::default();
+		let mut target_chunk_key = Default::default();
+		let mut chunk_proof = Default::default();
+
+		let total_chunks: u64 = transactions.iter().map(|t| ((t.len() + CHUNK_SIZE - 1) / CHUNK_SIZE) as u64).sum();
+		let mut buf = [0u8; 8];
+		buf.copy_from_slice(&random_hash[0..8]);
+		let random_u64 = u64::from_be_bytes(buf);
+		let target_chunk_index = random_u64 % total_chunks;
+		//Generate tries for each transaction.
+		let mut chunk_index = 0;
+		for transaction in transactions {
+			let mut transaction_root = sp_trie::empty_trie_root::<TrieLayout>();
+			{
+				let mut trie = sp_trie::TrieDBMut::<TrieLayout>::new(&mut db, &mut transaction_root);
+				let chunks = transaction.chunks(CHUNK_SIZE).map(|c| c.to_vec());
+				for (index, chunk) in chunks.enumerate() {
+					let index = encode_index(index as u32);
+					trie.insert(&index, &chunk)
+						.map_err(|e| Error::Application(Box::new(e)))?;
+					if chunk_index == target_chunk_index {
+						target_chunk = Some(chunk);
+						target_chunk_key = index;
+					}
+					chunk_index += 1;
+				}
+				trie.commit();
+			}
+			if target_chunk.is_some() && target_root == Default::default() {
+				target_root = transaction_root.clone();
+				chunk_proof = sp_trie::generate_trie_proof::<TrieLayout, _, _, _>(
+					&db,
+					transaction_root.clone(),
+					&[target_chunk_key.clone()]
+				).map_err(|e| Error::Application(Box::new(e)))?;
+			}
+		};
+
+		Ok(TransactionStorageProof {
+			proof: chunk_proof,
+			chunk: target_chunk.unwrap(),
+		})
+	}
+
+	#[test]
+	fn build_proof_check() {
+		use std::str::FromStr;
+		let random = [0u8; 32];
+		let proof = build_proof(&random, vec![vec![42]]).unwrap();
+		let root = sp_core::H256::from_str("0xff8611a4d212fc161dae19dd57f0f1ba9309f45d6207da13f2d3eab4c6839e91").unwrap();
+		sp_trie::verify_trie_proof::<TrieLayout, _, _, _>(
+			&root,
+			&proof.proof,
+			&[(encode_index(0), Some(proof.chunk))],
+		).unwrap();
+	}
+}
+

substrate/primitives/trie/src/storage_proof.rs

@@ -58,6 +58,10 @@ impl StorageProof {
 		StorageProofNodeIterator::new(self)
 	}
 
+	/// Convert into plain node vector.
+	pub fn into_nodes(self) -> Vec<Vec<u8>> {
+		self.trie_nodes
+	}
 	/// Creates a `MemoryDB` from `Self`.
 	pub fn into_memory_db<H: Hasher>(self) -> crate::MemoryDB<H> {
 		self.into()