diff --git a/substrate/Cargo.lock b/substrate/Cargo.lock index e277d1d24a..6a92f8e35f 100644 --- a/substrate/Cargo.lock +++ b/substrate/Cargo.lock @@ -419,7 +419,7 @@ dependencies = [ "cfg-if 0.1.10", "clang-sys", "clap", - "env_logger", + "env_logger 0.7.1", "lazy_static", "lazycell", "log", @@ -716,6 +716,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "ckb-merkle-mountain-range" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e486fe53bb9f2ca0f58cb60e8679a5354fd6687a839942ef0a75967250289ca6" +dependencies = [ + "cfg-if 0.1.10", +] + [[package]] name = "clang-sys" version = "0.29.3" @@ -1328,6 +1337,19 @@ dependencies = [ "syn", ] +[[package]] +name = "env_logger" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b0a4d2e39f8420210be8b27eeda28029729e2fd4291019455016c348240c38" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + [[package]] name = "env_logger" version = "0.7.1" @@ -1450,7 +1472,7 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fdbe0d94371f9ce939b555dd342d0686cc4c0cadbcd4b61d70af5ff97eb4126" dependencies = [ - "env_logger", + "env_logger 0.7.1", "log", ] @@ -3911,6 +3933,7 @@ dependencies = [ "pallet-im-online", "pallet-indices", "pallet-membership", + "pallet-mmr", "pallet-multisig", "pallet-offences", "pallet-offences-benchmarking", @@ -4651,6 +4674,24 @@ dependencies = [ "sp-std", ] +[[package]] +name = "pallet-mmr" +version = "2.0.0" +dependencies = [ + "ckb-merkle-mountain-range", + "env_logger 0.5.13", + "frame-benchmarking", + "frame-support", + "frame-system", + "hex-literal", + "parity-scale-codec", + "serde", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std", +] + [[package]] name = "pallet-multisig" version = "2.0.0" @@ -5754,7 +5795,7 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44883e74aa97ad63db83c4bf8ca490f02b2fc02f92575e720c8551e843c945f" dependencies = [ - "env_logger", + "env_logger 0.7.1", "log", "rand 0.7.3", "rand_core 0.5.1", diff --git a/substrate/Cargo.toml b/substrate/Cargo.toml index 6a007a209f..206673c0ef 100644 --- a/substrate/Cargo.toml +++ b/substrate/Cargo.toml @@ -1,19 +1,19 @@ [workspace] members = [ "bin/node-template/node", - "bin/node-template/runtime", "bin/node-template/pallets/template", + "bin/node-template/runtime", "bin/node/bench", "bin/node/browser-testing", "bin/node/cli", "bin/node/executor", "bin/node/primitives", - "bin/node/rpc-client", "bin/node/rpc", + "bin/node/rpc-client", "bin/node/runtime", "bin/node/testing", - "bin/utils/subkey", "bin/utils/chain-spec-builder", + "bin/utils/subkey", "client/api", "client/authority-discovery", "client/basic-authorship", @@ -26,42 +26,41 @@ members = [ "client/consensus/babe", "client/consensus/babe/rpc", "client/consensus/common", + "client/consensus/epochs", "client/consensus/manual-seal", "client/consensus/pow", - "client/consensus/uncles", "client/consensus/slots", - "client/consensus/epochs", + "client/consensus/uncles", "client/db", "client/executor", "client/executor/common", + "client/executor/runtime-test", "client/executor/wasmi", "client/executor/wasmtime", - "client/executor/runtime-test", "client/finality-grandpa", "client/informant", - "client/light", - "client/tracing", "client/keystore", + "client/light", "client/network", - "client/network/test", "client/network-gossip", + "client/network/test", "client/offchain", "client/peerset", "client/proposer-metrics", - "client/rpc-servers", "client/rpc", "client/rpc-api", + "client/rpc-servers", "client/service", "client/service/test", "client/state-db", "client/sync-state-rpc", "client/telemetry", + "client/tracing", "client/transaction-pool", "client/transaction-pool/graph", - "utils/prometheus", "frame/assets", - "frame/aura", "frame/atomic-swap", + "frame/aura", "frame/authority-discovery", "frame/authorship", "frame/babe", @@ -72,7 +71,6 @@ members = [ "frame/contracts/rpc", "frame/contracts/rpc/runtime-api", "frame/democracy", - "frame/elections-phragmen", "frame/elections", "frame/example", "frame/example-offchain-worker", @@ -83,6 +81,7 @@ members = [ "frame/im-online", "frame/indices", "frame/membership", + "frame/merkle-mountain-range", "frame/metadata", "frame/multisig", "frame/nicks", @@ -97,8 +96,8 @@ members = [ "frame/session/benchmarking", "frame/society", "frame/staking", - "frame/staking/reward-curve", "frame/staking/fuzzer", + "frame/staking/reward-curve", "frame/sudo", "frame/support", "frame/support/procedural", @@ -116,59 +115,59 @@ members = [ "frame/utility", "frame/vesting", "primitives/allocator", + "primitives/api", + "primitives/api/proc-macro", + "primitives/api/test", "primitives/application-crypto", "primitives/application-crypto/test", + "primitives/arithmetic", + "primitives/arithmetic/fuzzer", "primitives/authority-discovery", "primitives/authorship", "primitives/block-builder", "primitives/blockchain", + "primitives/chain-spec", "primitives/consensus/aura", "primitives/consensus/babe", "primitives/consensus/common", "primitives/consensus/pow", "primitives/consensus/vrf", "primitives/core", - "primitives/chain-spec", "primitives/database", "primitives/debug-derive", - "primitives/storage", "primitives/externalities", "primitives/finality-grandpa", "primitives/inherents", + "primitives/io", "primitives/keyring", "primitives/keystore", + "primitives/npos-elections", + "primitives/npos-elections/compact", + "primitives/npos-elections/fuzzer", "primitives/offchain", "primitives/panic-handler", - "primitives/npos-elections", - "primitives/npos-elections/fuzzer", - "primitives/npos-elections/compact", "primitives/rpc", + "primitives/runtime", "primitives/runtime-interface", "primitives/runtime-interface/proc-macro", + "primitives/runtime-interface/test", "primitives/runtime-interface/test-wasm", "primitives/runtime-interface/test-wasm-deprecated", - "primitives/runtime-interface/test", + "primitives/sandbox", "primitives/serializer", "primitives/session", - "primitives/api", - "primitives/api/proc-macro", - "primitives/api/test", - "primitives/arithmetic", - "primitives/arithmetic/fuzzer", - "primitives/io", - "primitives/runtime", - "primitives/sandbox", "primitives/staking", - "primitives/std", - "primitives/version", "primitives/state-machine", + "primitives/std", + "primitives/storage", "primitives/tasks", - "primitives/timestamp", "primitives/test-primitives", - "primitives/transaction-pool", + "primitives/timestamp", "primitives/tracing", + "primitives/transaction-pool", "primitives/trie", "primitives/utils", + "primitives/version", "primitives/wasm-interface", "test-utils/client", "test-utils/derive", @@ -183,6 +182,7 @@ members = [ "utils/frame/frame-utilities-cli", "utils/frame/rpc/support", "utils/frame/rpc/system", + "utils/prometheus", "utils/wasm-builder", ] diff --git a/substrate/bin/node/runtime/Cargo.toml b/substrate/bin/node/runtime/Cargo.toml index 7d4cf5588e..ad0f6b35a2 100644 --- a/substrate/bin/node/runtime/Cargo.toml +++ b/substrate/bin/node/runtime/Cargo.toml @@ -59,6 +59,7 @@ pallet-im-online = { version = "2.0.0", default-features = false, path = "../../ pallet-indices = { version = "2.0.0", default-features = false, path = "../../../frame/indices" } pallet-identity = { version = "2.0.0", default-features = false, path = "../../../frame/identity" } pallet-membership = { version = "2.0.0", default-features = false, path = "../../../frame/membership" } +pallet-mmr = { version = "2.0.0", default-features = false, path = "../../../frame/merkle-mountain-range" } pallet-multisig = { version = "2.0.0", default-features = false, path = "../../../frame/multisig" } pallet-offences = { version = "2.0.0", default-features = false, path = "../../../frame/offences" } pallet-offences-benchmarking = { version = "2.0.0", path = "../../../frame/offences/benchmarking", default-features = false, optional = true } @@ -110,6 +111,7 @@ std = [ "pallet-indices/std", "sp-inherents/std", "pallet-membership/std", + "pallet-mmr/std", "pallet-multisig/std", "pallet-identity/std", "pallet-scheduler/std", @@ -160,6 +162,7 @@ runtime-benchmarks = [ "pallet-identity/runtime-benchmarks", "pallet-im-online/runtime-benchmarks", "pallet-indices/runtime-benchmarks", + "pallet-mmr/runtime-benchmarks", "pallet-multisig/runtime-benchmarks", "pallet-proxy/runtime-benchmarks", "pallet-scheduler/runtime-benchmarks", diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs index ea9921beee..2962ef1066 100644 --- a/substrate/bin/node/runtime/src/lib.rs +++ b/substrate/bin/node/runtime/src/lib.rs @@ -915,6 +915,15 @@ impl pallet_vesting::Config for Runtime { type WeightInfo = pallet_vesting::weights::SubstrateWeight; } +impl pallet_mmr::Config for Runtime { + const INDEXING_PREFIX: &'static [u8] = b"mmr"; + type Hashing = ::Hashing; + type Hash = ::Hash; + type LeafData = frame_system::Module; + type OnNewRoot = (); + type WeightInfo = (); +} + parameter_types! { pub const AssetDepositBase: Balance = 100 * DOLLARS; pub const AssetDepositPerZombie: Balance = 1 * DOLLARS; @@ -969,6 +978,7 @@ construct_runtime!( Proxy: pallet_proxy::{Module, Call, Storage, Event}, Multisig: pallet_multisig::{Module, Call, Storage, Event}, Assets: pallet_assets::{Module, Call, Storage, Event}, + Mmr: pallet_mmr::{Module, Storage}, } ); @@ -1246,6 +1256,7 @@ impl_runtime_apis! { add_benchmark!(params, batches, pallet_identity, Identity); add_benchmark!(params, batches, pallet_im_online, ImOnline); add_benchmark!(params, batches, pallet_indices, Indices); + add_benchmark!(params, batches, pallet_mmr, Mmr); add_benchmark!(params, batches, pallet_multisig, Multisig); add_benchmark!(params, batches, pallet_offences, OffencesBench::); add_benchmark!(params, batches, pallet_proxy, Proxy); diff --git a/substrate/client/executor/src/integration_tests/mod.rs b/substrate/client/executor/src/integration_tests/mod.rs index d41784f5aa..0a00375145 100644 --- a/substrate/client/executor/src/integration_tests/mod.rs +++ b/substrate/client/executor/src/integration_tests/mod.rs @@ -475,8 +475,6 @@ fn offchain_index(wasm_method: WasmExecutionMethod) { #[test_case(WasmExecutionMethod::Interpreted)] #[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))] fn offchain_local_storage_should_work(wasm_method: WasmExecutionMethod) { - use sp_core::offchain::OffchainStorage; - let mut ext = TestExternalities::default(); let (offchain, state) = testing::TestOffchainExt::new(); ext.register_extension(OffchainExt::new(offchain)); @@ -489,7 +487,7 @@ fn offchain_local_storage_should_work(wasm_method: WasmExecutionMethod) { ).unwrap(), true.encode(), ); - assert_eq!(state.read().persistent_storage.get(b"", b"test"), Some(vec![])); + assert_eq!(state.read().persistent_storage.get(b"test"), Some(vec![])); } #[test_case(WasmExecutionMethod::Interpreted)] diff --git a/substrate/frame/merkle-mountain-range/Cargo.toml b/substrate/frame/merkle-mountain-range/Cargo.toml new file mode 100644 index 0000000000..b46f42cacf --- /dev/null +++ b/substrate/frame/merkle-mountain-range/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "pallet-mmr" +version = "2.0.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME Merkle Mountain Range pallet." + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +codec = { package = "parity-scale-codec", version = "1.3.4", default-features = false } +frame-benchmarking = { version = "2.0.0", default-features = false, path = "../benchmarking", optional = true } +frame-support = { version = "2.0.0", default-features = false, path = "../support" } +frame-system = { version = "2.0.0", default-features = false, path = "../system" } +mmr-lib = { package = "ckb-merkle-mountain-range", default-features = false, version = "0.3.1" } +serde = { version = "1.0.101", optional = true } +sp-core = { version = "2.0.0", default-features = false, path = "../../primitives/core" } +sp-io = { version = "2.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } + +[dev-dependencies] +env_logger = "0.5" +hex-literal = "0.3" + +[features] +default = ["std"] +std = [ + "codec/std", + "frame-benchmarking/std", + "frame-support/std", + "frame-system/std", + "mmr-lib/std", + "serde", + "sp-core/std", + "sp-io/std", + "sp-runtime/std", + "sp-std/std", +] +runtime-benchmarks = ["frame-benchmarking"] diff --git a/substrate/frame/merkle-mountain-range/src/benchmarking.rs b/substrate/frame/merkle-mountain-range/src/benchmarking.rs new file mode 100644 index 0000000000..af634e1882 --- /dev/null +++ b/substrate/frame/merkle-mountain-range/src/benchmarking.rs @@ -0,0 +1,56 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Benchmarks for the MMR pallet. + +#![cfg_attr(not(feature = "std"), no_std)] + +use crate::*; +use frame_support::traits::OnInitialize; +use frame_benchmarking::benchmarks; +use sp_std::prelude::*; + +benchmarks! { + _ { } + + on_initialize { + let x in 1 .. 1_000; + + let leaves = x as u64; + }: { + for b in 0..leaves { + Module::::on_initialize((b as u32).into()); + } + } verify { + assert_eq!(crate::NumberOfLeaves::::get(), leaves); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mock::*; + use crate::tests::new_test_ext; + use frame_support::assert_ok; + + #[test] + fn test_benchmarks() { + new_test_ext().execute_with(|| { + assert_ok!(test_benchmark_on_initialize::()); + }) + } +} diff --git a/substrate/frame/merkle-mountain-range/src/default_weights.rs b/substrate/frame/merkle-mountain-range/src/default_weights.rs new file mode 100644 index 0000000000..0b31698545 --- /dev/null +++ b/substrate/frame/merkle-mountain-range/src/default_weights.rs @@ -0,0 +1,42 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Default weights for the MMR Pallet +//! This file was not auto-generated. + +use frame_support::weights::{ + Weight, constants::{WEIGHT_PER_NANOS, RocksDbWeight as DbWeight}, +}; + +impl crate::WeightInfo for () { + fn on_initialize(peaks: u64) -> Weight { + // Reading the parent hash. + let leaf_weight = DbWeight::get().reads(1); + // Blake2 hash cost. + let hash_weight = 2 * WEIGHT_PER_NANOS; + // No-op hook. + let hook_weight = 0; + + leaf_weight + .saturating_add(hash_weight) + .saturating_add(hook_weight) + .saturating_add(DbWeight::get().reads_writes( + 2 + peaks, + 2 + peaks, + )) + } +} diff --git a/substrate/frame/merkle-mountain-range/src/lib.rs b/substrate/frame/merkle-mountain-range/src/lib.rs new file mode 100644 index 0000000000..afa6c9083c --- /dev/null +++ b/substrate/frame/merkle-mountain-range/src/lib.rs @@ -0,0 +1,231 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Merkle Mountain Range +//! +//! ## Overview +//! +//! Details on Merkle Mountain Ranges (MMRs) can be found here: +//! https://github.com/mimblewimble/grin/blob/master/doc/mmr.md +//! +//! The MMR pallet constructs a MMR from leaf data obtained on every block from +//! `LeafDataProvider`. MMR nodes are stored both in: +//! - on-chain storage - hashes only; not full leaf content) +//! - off-chain storage - via Indexing API we push full leaf content (and all internal nodes as +//! well) to the Off-chain DB, so that the data is available for Off-chain workers. +//! Hashing used for MMR is configurable independently from the rest of the runtime (i.e. not using +//! `frame_system::Hashing`) so something compatible with external chains can be used (like +//! Keccak256 for Ethereum compatibility). +//! +//! Depending on the usage context (off-chain vs on-chain) the pallet is able to: +//! - verify MMR leaf proofs (on-chain) +//! - generate leaf proofs (off-chain) +//! +//! See [primitives::Compact] documentation for how you can optimize proof size for leafs that are +//! composed from multiple elements. +//! +//! ## What for? +//! +//! Primary use case for this pallet is to generate MMR root hashes, that can latter on be used by +//! BEEFY protocol (see https://github.com/paritytech/grandpa-bridge-gadget). +//! MMR root hashes along with BEEFY will make it possible to build Super Light Clients (SLC) of +//! Substrate-based chains. The SLC will be able to follow finality and can be shown proofs of more +//! details that happened on the source chain. +//! In that case the chain which contains the pallet generates the Root Hashes and Proofs, which +//! are then presented to another chain acting as a light client which can verify them. +//! +//! Secondary use case is to archive historical data, but still be able to retrieve them on-demand +//! if needed. For instance if parent block hashes are stored in the MMR it's possible at any point +//! in time to provide a MMR proof about some past block hash, while this data can be safely pruned +//! from on-chain storage. +//! +//! NOTE This pallet is experimental and not proven to work in production. +//! +#![cfg_attr(not(feature = "std"), no_std)] + +use codec::Encode; +use frame_support::{ + decl_module, decl_storage, + weights::Weight, +}; +use sp_runtime::traits; + +mod default_weights; +mod mmr; +#[cfg(any(feature = "runtime-benchmarks", test))] +mod benchmarking; +#[cfg(test)] +mod mock; +#[cfg(test)] +mod tests; + +pub mod primitives; + +pub trait WeightInfo { + fn on_initialize(peaks: u64) -> Weight; +} + +/// This pallet's configuration trait +pub trait Config: frame_system::Config { + /// Prefix for elements stored in the Off-chain DB via Indexing API. + /// + /// Each node of the MMR is inserted both on-chain and off-chain via Indexing API. + /// The former does not store full leaf content, just it's compact version (hash), + /// and some of the inner mmr nodes might be pruned from on-chain storage. + /// The later will contain all the entries in their full form. + /// + /// Each node is stored in the Off-chain DB under key derived from the [INDEXING_PREFIX] and + /// it's in-tree index (MMR position). + const INDEXING_PREFIX: &'static [u8]; + + /// A hasher type for MMR. + /// + /// To construct trie nodes that result in merging (bagging) two peaks, depending on the node + /// kind we take either: + /// - The node (hash) itself if it's an inner node. + /// - The hash of SCALE-encoding of the leaf data if it's a leaf node. + /// + /// Then we create a tuple of these two hashes, SCALE-encode it (concatenate) and + /// hash, to obtain a new MMR inner node - the new peak. + type Hashing: traits::Hash>::Hash>; + + /// The hashing output type. + /// + /// This type is actually going to be stored in the MMR. + /// Required to be provided again, to satisfy trait bounds for storage items. + type Hash: traits::Member + traits::MaybeSerializeDeserialize + sp_std::fmt::Debug + + sp_std::hash::Hash + AsRef<[u8]> + AsMut<[u8]> + Copy + Default + codec::Codec + + codec::EncodeLike; + + /// Data stored in the leaf nodes. + /// + /// The [LeafData](primitives::LeafDataProvider) is responsible for returning the entire leaf + /// data that will be inserted to the MMR. + /// [LeafDataProvider](primitives::LeafDataProvider)s can be composed into tuples to put + /// multiple elements into the tree. In such a case it might be worth using [primitives::Compact] + /// to make MMR proof for one element of the tuple leaner. + type LeafData: primitives::LeafDataProvider; + + /// A hook to act on the new MMR root. + /// + /// For some applications it might be beneficial to make the MMR root available externally + /// apart from having it in the storage. For instance you might output it in the header digest + /// (see [frame_system::Module::deposit_log]) to make it available for Light Clients. + /// Hook complexity should be `O(1)`. + type OnNewRoot: primitives::OnNewRoot<>::Hash>; + + /// Weights for this pallet. + type WeightInfo: WeightInfo; +} + +decl_storage! { + trait Store for Module, I: Instance = DefaultInstance> as MerkleMountainRange { + /// Latest MMR Root hash. + pub RootHash get(fn mmr_root_hash): >::Hash; + + /// Current size of the MMR (number of leaves). + pub NumberOfLeaves get(fn mmr_leaves): u64; + + /// Hashes of the nodes in the MMR. + /// + /// Note this collection only contains MMR peaks, the inner nodes (and leaves) + /// are pruned and only stored in the Offchain DB. + pub Nodes get(fn mmr_peak): map hasher(identity) u64 => Option<>::Hash>; + } +} + +decl_module! { + /// A public part of the pallet. + pub struct Module, I: Instance = DefaultInstance> for enum Call where origin: T::Origin { + fn on_initialize(n: T::BlockNumber) -> Weight { + use primitives::LeafDataProvider; + let leaves = Self::mmr_leaves(); + let peaks_before = mmr::utils::NodesUtils::new(leaves).number_of_peaks(); + let data = T::LeafData::leaf_data(); + // append new leaf to MMR + let mut mmr: ModuleMmr = mmr::Mmr::new(leaves); + mmr.push(data).expect("MMR push never fails."); + + // update the size + let (leaves, root) = mmr.finalize().expect("MMR finalize never fails."); + >::on_new_root(&root); + + ::put(leaves); + >::put(root); + + let peaks_after = mmr::utils::NodesUtils::new(leaves).number_of_peaks(); + T::WeightInfo::on_initialize(peaks_before.max(peaks_after)) + } + } +} + +/// A MMR specific to the pallet. +type ModuleMmr = mmr::Mmr>; + +/// Leaf data. +type LeafOf = <>::LeafData as primitives::LeafDataProvider>::LeafData; + +/// Hashing used for the pallet. +pub(crate) type HashingOf = >::Hashing; + +impl, I: Instance> Module { + fn offchain_key(pos: u64) -> sp_std::prelude::Vec { + (T::INDEXING_PREFIX, pos).encode() + } + + /// Generate a MMR proof for the given `leaf_index`. + /// + /// Note this method can only be used from an off-chain context + /// (Offchain Worker or Runtime API call), since it requires + /// all the leaves to be present. + /// It may return an error or panic if used incorrectly. + pub fn generate_proof(leaf_index: u64) -> Result< + (LeafOf, primitives::Proof<>::Hash>), + mmr::Error, + > { + let mmr: ModuleMmr = mmr::Mmr::new(Self::mmr_leaves()); + mmr.generate_proof(leaf_index) + } + + /// Verify MMR proof for given `leaf`. + /// + /// This method is safe to use within the runtime code. + /// It will return `Ok(())` if the proof is valid + /// and an `Err(..)` if MMR is inconsistent (some leaves are missing) + /// or the proof is invalid. + pub fn verify_leaf( + leaf: LeafOf, + proof: primitives::Proof<>::Hash>, + ) -> Result<(), mmr::Error> { + if proof.leaf_count > Self::mmr_leaves() + || proof.leaf_count == 0 + || proof.items.len() as u32 > mmr::utils::NodesUtils::new(proof.leaf_count).depth() + { + return Err(mmr::Error::Verify.log_debug( + "The proof has incorrect number of leaves or proof items." + )); + } + + let mmr: ModuleMmr = mmr::Mmr::new(proof.leaf_count); + let is_valid = mmr.verify_leaf_proof(leaf, proof)?; + if is_valid { + Ok(()) + } else { + Err(mmr::Error::Verify.log_debug("The proof is incorrect.")) + } + } +} diff --git a/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs new file mode 100644 index 0000000000..ee27163ae4 --- /dev/null +++ b/substrate/frame/merkle-mountain-range/src/mmr/mmr.rs @@ -0,0 +1,186 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + Config, HashingOf, Instance, + mmr::{ + Node, NodeOf, Hasher, + storage::{Storage, OffchainStorage, RuntimeStorage}, + utils::NodesUtils, + }, + primitives, +}; +use frame_support::{debug, RuntimeDebug}; +use sp_std::fmt; +#[cfg(not(feature = "std"))] +use sp_std::{vec, prelude::Vec}; + +/// A wrapper around a MMR library to expose limited functionality. +/// +/// Available functions depend on the storage kind ([Runtime](crate::mmr::storage::RuntimeStorage) +/// vs [Off-chain](crate::mmr::storage::OffchainStorage)). +pub struct Mmr where + T: Config, + I: Instance, + L: primitives::FullLeaf, + Storage: mmr_lib::MMRStore>, +{ + mmr: mmr_lib::MMR< + NodeOf, + Hasher, L>, + Storage + >, + leaves: u64, +} + +impl Mmr where + T: Config, + I: Instance, + L: primitives::FullLeaf, + Storage: mmr_lib::MMRStore>, +{ + /// Create a pointer to an existing MMR with given number of leaves. + pub fn new(leaves: u64) -> Self { + let size = NodesUtils::new(leaves).size(); + Self { + mmr: mmr_lib::MMR::new(size, Default::default()), + leaves, + } + } + + /// Verify proof of a single leaf. + pub fn verify_leaf_proof( + &self, + leaf: L, + proof: primitives::Proof<>::Hash>, + ) -> Result { + let p = mmr_lib::MerkleProof::< + NodeOf, + Hasher, L>, + >::new( + self.mmr.mmr_size(), + proof.items.into_iter().map(Node::Hash).collect(), + ); + let position = mmr_lib::leaf_index_to_pos(proof.leaf_index); + let root = self.mmr.get_root().map_err(|e| Error::GetRoot.log_error(e))?; + p.verify( + root, + vec![(position, Node::Data(leaf))], + ).map_err(|e| Error::Verify.log_debug(e)) + } + + /// Return the internal size of the MMR (number of nodes). + #[cfg(test)] + pub fn size(&self) -> u64 { + self.mmr.mmr_size() + } +} + +/// Runtime specific MMR functions. +impl Mmr where + T: Config, + I: Instance, + L: primitives::FullLeaf, +{ + + /// Push another item to the MMR. + /// + /// Returns element position (index) in the MMR. + pub fn push(&mut self, leaf: L) -> Option { + let position = self.mmr.push(Node::Data(leaf)) + .map_err(|e| Error::Push.log_error(e)) + .ok()?; + + self.leaves += 1; + + Some(position) + } + + /// Commit the changes to underlying storage, return current number of leaves and + /// calculate the new MMR's root hash. + pub fn finalize(self) -> Result<(u64, >::Hash), Error> { + let root = self.mmr.get_root().map_err(|e| Error::GetRoot.log_error(e))?; + self.mmr.commit().map_err(|e| Error::Commit.log_error(e))?; + Ok((self.leaves, root.hash())) + } +} + +/// Off-chain specific MMR functions. +impl Mmr where + T: Config, + I: Instance, + L: primitives::FullLeaf, +{ + /// Generate a proof for given leaf index. + /// + /// Proof generation requires all the nodes (or their hashes) to be available in the storage. + /// (i.e. you can't run the function in the pruned storage). + pub fn generate_proof(&self, leaf_index: u64) -> Result< + (L, primitives::Proof<>::Hash>), + Error + > { + let position = mmr_lib::leaf_index_to_pos(leaf_index); + let store = >::default(); + let leaf = match mmr_lib::MMRStore::get_elem(&store, position) { + Ok(Some(Node::Data(leaf))) => leaf, + e => return Err(Error::LeafNotFound.log_debug(e)), + }; + let leaf_count = self.leaves; + self.mmr.gen_proof(vec![position]) + .map_err(|e| Error::GenerateProof.log_error(e)) + .map(|p| primitives::Proof { + leaf_index, + leaf_count, + items: p.proof_items().iter().map(|x| x.hash()).collect(), + }) + .map(|p| (leaf, p)) + } +} + +/// Merkle Mountain Range operation error. +#[derive(RuntimeDebug)] +#[cfg_attr(test, derive(PartialEq, Eq))] +pub enum Error { + /// Error while pushing new node. + Push, + /// Error getting the new root. + GetRoot, + /// Error commiting changes. + Commit, + /// Error during proof generation. + GenerateProof, + /// Proof verification error. + Verify, + /// Leaf not found in the storage. + LeafNotFound, +} + +impl Error { + /// Consume given error `e` with `self` and generate a native log entry with error details. + pub(crate) fn log_error(self, e: impl fmt::Debug) -> Self { + debug::native::error!("[{:?}] MMR error: {:?}", self, e); + self + } + + /// Consume given error `e` with `self` and generate a native log entry with error details. + pub(crate) fn log_debug(self, e: impl fmt::Debug) -> Self { + debug::native::debug!("[{:?}] MMR error: {:?}", self, e); + self + } + +} + diff --git a/substrate/frame/merkle-mountain-range/src/mmr/mod.rs b/substrate/frame/merkle-mountain-range/src/mmr/mod.rs new file mode 100644 index 0000000000..7fd8f5ae1b --- /dev/null +++ b/substrate/frame/merkle-mountain-range/src/mmr/mod.rs @@ -0,0 +1,45 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod storage; +pub mod utils; +mod mmr; + +use crate::primitives::FullLeaf; +use sp_runtime::traits; + +pub use self::mmr::{Mmr, Error}; + +/// Node type for runtime `T`. +pub type NodeOf = Node<>::Hashing, L>; + +/// A node stored in the MMR. +pub type Node = crate::primitives::DataOrHash; + +/// Default Merging & Hashing behavior for MMR. +pub struct Hasher(sp_std::marker::PhantomData<(H, L)>); + +impl mmr_lib::Merge for Hasher { + type Item = Node; + + fn merge(left: &Self::Item, right: &Self::Item) -> Self::Item { + let mut concat = left.hash().as_ref().to_vec(); + concat.extend_from_slice(right.hash().as_ref()); + + Node::Hash(::hash(&concat)) + } +} diff --git a/substrate/frame/merkle-mountain-range/src/mmr/storage.rs b/substrate/frame/merkle-mountain-range/src/mmr/storage.rs new file mode 100644 index 0000000000..a1aa57087a --- /dev/null +++ b/substrate/frame/merkle-mountain-range/src/mmr/storage.rs @@ -0,0 +1,112 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! A MMR storage implementations. + +use codec::Encode; +use crate::mmr::{NodeOf, Node}; +use crate::{NumberOfLeaves, Nodes, Module, Config, Instance, primitives}; +use frame_support::{StorageMap, StorageValue}; +#[cfg(not(feature = "std"))] +use sp_std::prelude::Vec; + +/// A marker type for runtime-specific storage implementation. +/// +/// Allows appending new items to the MMR and proof verification. +/// MMR nodes are appended to two different storages: +/// 1. We add nodes (leaves) hashes to the on-chain storge (see [crate::Nodes]). +/// 2. We add full leaves (and all inner nodes as well) into the `IndexingAPI` during block +/// processing, so the values end up in the Offchain DB if indexing is enabled. +pub struct RuntimeStorage; + +/// A marker type for offchain-specific storage implementation. +/// +/// Allows proof generation and verification, but does not support appending new items. +/// MMR nodes are assumed to be stored in the Off-Chain DB. Note this storage type +/// DOES NOT support adding new items to the MMR. +pub struct OffchainStorage; + +/// A storage layer for MMR. +/// +/// There are two different implementations depending on the use case. +/// See docs for [RuntimeStorage] and [OffchainStorage]. +pub struct Storage( + sp_std::marker::PhantomData<(StorageType, T, I, L)> +); + +impl Default for Storage { + fn default() -> Self { + Self(Default::default()) + } +} + +impl mmr_lib::MMRStore> for Storage where + T: Config, + I: Instance, + L: primitives::FullLeaf, +{ + fn get_elem(&self, pos: u64) -> mmr_lib::Result>> { + let key = Module::::offchain_key(pos); + // Retrieve the element from Off-chain DB. + Ok( + sp_io::offchain ::local_storage_get(sp_core::offchain::StorageKind::PERSISTENT, &key) + .and_then(|v| codec::Decode::decode(&mut &*v).ok()) + ) + } + + fn append(&mut self, _: u64, _: Vec>) -> mmr_lib::Result<()> { + panic!("MMR must not be altered in the off-chain context.") + } +} + +impl mmr_lib::MMRStore> for Storage where + T: Config, + I: Instance, + L: primitives::FullLeaf, +{ + fn get_elem(&self, pos: u64) -> mmr_lib::Result>> { + Ok(>::get(pos) + .map(Node::Hash) + ) + } + + fn append(&mut self, pos: u64, elems: Vec>) -> mmr_lib::Result<()> { + let mut leaves = crate::NumberOfLeaves::::get(); + let mut size = crate::mmr::utils::NodesUtils::new(leaves).size(); + if pos != size { + return Err(mmr_lib::Error::InconsistentStore); + } + + for elem in elems { + // on-chain we only store the hash (even if it's a leaf) + >::insert(size, elem.hash()); + // Indexing API is used to store the full leaf content. + elem.using_encoded(|elem| { + sp_io::offchain_index::set(&Module::::offchain_key(size), elem) + }); + size += 1; + + if let Node::Data(..) = elem { + leaves += 1; + } + } + + NumberOfLeaves::::put(leaves); + + Ok(()) + } +} diff --git a/substrate/frame/merkle-mountain-range/src/mmr/utils.rs b/substrate/frame/merkle-mountain-range/src/mmr/utils.rs new file mode 100644 index 0000000000..7a55605a64 --- /dev/null +++ b/substrate/frame/merkle-mountain-range/src/mmr/utils.rs @@ -0,0 +1,131 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Merkle Mountain Range utilities. + +/// MMR nodes & size -related utilities. +pub struct NodesUtils { + no_of_leaves: u64, +} + +impl NodesUtils { + /// Create new instance of MMR nodes utilities for given number of leaves. + pub fn new(no_of_leaves: u64) -> Self { + Self { no_of_leaves } + } + + /// Calculate number of peaks in the MMR. + pub fn number_of_peaks(&self) -> u64 { + self.number_of_leaves().count_ones() as u64 + } + + /// Return the number of leaves in the MMR. + pub fn number_of_leaves(&self) -> u64 { + self.no_of_leaves + } + + /// Calculate the total size of MMR (number of nodes). + pub fn size(&self) -> u64 { + 2 * self.no_of_leaves - self.number_of_peaks() + } + + /// Calculate maximal depth of the MMR. + pub fn depth(&self) -> u32 { + if self.no_of_leaves == 0 { + return 0 + } + + 64 - self.no_of_leaves + .next_power_of_two() + .leading_zeros() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn should_calculate_number_of_leaves_correctly() { + assert_eq!( + vec![0, 1, 2, 3, 4, 9, 15, 21] + .into_iter() + .map(|n| NodesUtils::new(n).depth()) + .collect::>(), + vec![0, 1, 2, 3, 3, 5, 5, 6] + ); + } + + #[test] + fn should_calculate_depth_correclty() { + assert_eq!( + vec![0, 1, 2, 3, 4, 9, 15, 21] + .into_iter() + .map(|n| NodesUtils::new(n).number_of_leaves()) + .collect::>(), + vec![0, 1, 2, 3, 4, 9, 15, 21] + ); + } + + #[test] + fn should_calculate_number_of_peaks_correctly() { + assert_eq!( + vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 21] + .into_iter() + .map(|n| NodesUtils::new(n).number_of_peaks()) + .collect::>(), + vec![0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 3] + ); + } + + #[test] + fn should_calculate_the_size_correctly() { + let _ = env_logger::try_init(); + + let leaves = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 21]; + let sizes = vec![0, 1, 3, 4, 7, 8, 10, 11, 15, 16, 18, 19, 22, 23, 25, 26, 39]; + assert_eq!( + leaves + .clone() + .into_iter() + .map(|n| NodesUtils::new(n).size()) + .collect::>(), + sizes.clone() + ); + + // size cross-check + let mut actual_sizes = vec![]; + for s in &leaves[1..] { + crate::tests::new_test_ext().execute_with(|| { + let mut mmr = crate::mmr::Mmr::< + crate::mmr::storage::RuntimeStorage, + crate::mock::Test, + crate::DefaultInstance, + _, + >::new(0); + for i in 0..*s { + mmr.push(i); + } + actual_sizes.push(mmr.size()); + }) + } + assert_eq!( + sizes[1..], + actual_sizes[..], + ); + } +} diff --git a/substrate/frame/merkle-mountain-range/src/mock.rs b/substrate/frame/merkle-mountain-range/src/mock.rs new file mode 100644 index 0000000000..c311d53446 --- /dev/null +++ b/substrate/frame/merkle-mountain-range/src/mock.rs @@ -0,0 +1,105 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::*; +use crate::primitives::{LeafDataProvider, Compact}; + +use codec::{Encode, Decode}; +use frame_support::{ + impl_outer_origin, parameter_types, +}; +use sp_core::H256; +use sp_runtime::{ + testing::Header, + traits::{ + BlakeTwo256, Keccak256, IdentityLookup, + }, +}; +use sp_std::cell::RefCell; +use sp_std::prelude::*; + +impl_outer_origin! { + pub enum Origin for Test where system = frame_system {} +} + +#[derive(Clone, Eq, PartialEq, Encode, Decode)] +pub struct Test; +parameter_types! { + pub const BlockHashCount: u64 = 250; +} +impl frame_system::Config for Test { + type BaseCallFilter = (); + type Origin = Origin; + type Call = (); + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = sp_core::sr25519::Public; + type Lookup = IdentityLookup; + type Header = Header; + type Event = (); + type BlockHashCount = BlockHashCount; + type DbWeight = (); + type BlockWeights = (); + type BlockLength = (); + type Version = (); + type PalletInfo = (); + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); +} + +impl Config for Test { + const INDEXING_PREFIX: &'static [u8] = b"mmr-"; + + type Hashing = Keccak256; + type Hash = H256; + type LeafData = Compact, LeafData)>; + type OnNewRoot = (); + type WeightInfo = (); +} + +#[derive(Encode, Decode, Clone, Default, Eq, PartialEq, Debug)] +pub struct LeafData { + pub a: u64, + pub b: Vec, +} + +impl LeafData { + pub fn new(a: u64) -> Self { + Self { + a, + b: Default::default(), + } + } +} + +thread_local! { + pub static LEAF_DATA: RefCell = RefCell::new(Default::default()); +} + +impl LeafDataProvider for LeafData { + type LeafData = Self; + + fn leaf_data() -> Self::LeafData { + LEAF_DATA.with(|r| r.borrow().clone()) + } +} + +pub(crate) type MMR = Module; diff --git a/substrate/frame/merkle-mountain-range/src/primitives.rs b/substrate/frame/merkle-mountain-range/src/primitives.rs new file mode 100644 index 0000000000..cab4b6a0dc --- /dev/null +++ b/substrate/frame/merkle-mountain-range/src/primitives.rs @@ -0,0 +1,415 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Merkle Mountain Range primitive types. + +use frame_support::RuntimeDebug; +use sp_runtime::traits; +use sp_std::fmt; +#[cfg(not(feature = "std"))] +use sp_std::prelude::Vec; + +/// A provider of the MMR's leaf data. +pub trait LeafDataProvider { + /// A type that should end up in the leaf of MMR. + type LeafData: FullLeaf; + + /// The method to return leaf data that should be placed + /// in the leaf node appended MMR at this block. + /// + /// This is being called by the `on_initialize` method of + /// this pallet at the very beginning of each block. + fn leaf_data() -> Self::LeafData; +} + +impl LeafDataProvider for () { + type LeafData = (); + + fn leaf_data() -> Self::LeafData { + () + } +} + +/// The most common use case for MMRs is to store historical block hashes, +/// so that any point in time in the future we can receive a proof about some past +/// blocks without using excessive on-chain storage. +/// Hence we implement the [LeafDataProvider] for [frame_system::Module], since the +/// current block hash is not available (since the block is not finished yet), +/// we use the `parent_hash` here. +impl LeafDataProvider for frame_system::Module { + type LeafData = ::Hash; + + fn leaf_data() -> Self::LeafData { + Self::parent_hash() + } +} + +/// New MMR root notification hook. +pub trait OnNewRoot { + /// Function called by the pallet in case new MMR root has been computed. + fn on_new_root(root: &Hash); +} + +/// No-op implementation of [OnNewRoot]. +impl OnNewRoot for () { + fn on_new_root(_root: &Hash) {} +} + +/// A full leaf content stored in the offchain-db. +pub trait FullLeaf: Clone + PartialEq + fmt::Debug + codec::Decode { + /// Encode the leaf either in it's full or compact form. + /// + /// NOTE the encoding returned here MUST be `Decode`able into `FullLeaf`. + fn using_encoded R>(&self, f: F, compact: bool) -> R; +} + +impl FullLeaf for T { + fn using_encoded R>(&self, f: F, _compact: bool) -> R { + codec::Encode::using_encoded(self, f) + } +} + +/// An element representing either full data or it's hash. +/// +/// See [Compact] to see how it may be used in practice to reduce the size +/// of proofs in case multiple [LeafDataProvider]s are composed together. +/// This is also used internally by the MMR to differentiate leaf nodes (data) +/// and inner nodes (hashes). +/// +/// [DataOrHash::hash] method calculates the hash of this element in it's compact form, +/// so should be used instead of hashing the encoded form (which will always be non-compact). +#[derive(RuntimeDebug, Clone, PartialEq)] +pub enum DataOrHash { + /// Arbitrary data in it's full form. + Data(L), + /// A hash of some data. + Hash(H::Output), +} + +impl From for DataOrHash { + fn from(l: L) -> Self { + Self::Data(l) + } +} + +mod encoding { + use super::*; + + /// A helper type to implement [codec::Codec] for [DataOrHash]. + #[derive(codec::Encode, codec::Decode)] + enum Either { + Left(A), + Right(B), + } + + impl codec::Encode for DataOrHash { + fn encode_to(&self, dest: &mut T) { + match self { + Self::Data(l) => l.using_encoded( + |data| Either::<&[u8], &H::Output>::Left(data).encode_to(dest), false + ), + Self::Hash(h) => Either::<&[u8], &H::Output>::Right(h).encode_to(dest), + } + } + } + + impl codec::Decode for DataOrHash { + fn decode(value: &mut I) -> Result { + let decoded: Either, H::Output> = Either::decode(value)?; + Ok(match decoded { + Either::Left(l) => DataOrHash::Data(L::decode(&mut &*l)?), + Either::Right(r) => DataOrHash::Hash(r), + }) + } + } +} + +impl DataOrHash { + /// Retrieve a hash of this item. + /// + /// Depending on the node type it's going to either be a contained value for [DataOrHash::Hash] + /// node, or a hash of SCALE-encoded [DataOrHash::Data] data. + pub fn hash(&self) -> H::Output { + match *self { + Self::Data(ref leaf) => leaf.using_encoded(::hash, true), + Self::Hash(ref hash) => hash.clone(), + } + } +} + +/// A composition of multiple leaf elements with compact form representation. +/// +/// When composing together multiple [LeafDataProvider]s you will end up with +/// a tuple of `LeafData` that each element provides. +/// +/// However this will cause the leaves to have significant size, while for some +/// use cases it will be enough to prove only one element of the tuple. +/// That's the rationale for [Compact] struct. We wrap each element of the tuple +/// into [DataOrHash] and each tuple element is hashed first before constructing +/// the final hash of the entire tuple. This allows you to replace tuple elements +/// you don't care about with their hashes. +#[derive(RuntimeDebug, Clone, PartialEq)] +pub struct Compact { + pub tuple: T, + _hash: sp_std::marker::PhantomData, +} + +impl sp_std::ops::Deref for Compact { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.tuple + } +} + +impl Compact { + pub fn new(tuple: T) -> Self { + Self { tuple, _hash: Default::default() } + } +} + +impl codec::Decode for Compact { + fn decode(value: &mut I) -> Result { + T::decode(value).map(Compact::new) + } +} + +macro_rules! impl_leaf_data_for_tuple { + ( $( $name:ident : $id:tt ),+ ) => { + /// [FullLeaf] implementation for `Compact, ...)>` + impl FullLeaf for Compact, )+ )> where + H: traits::Hash, + $( $name: FullLeaf ),+ + { + fn using_encoded R>(&self, f: F, compact: bool) -> R { + if compact { + codec::Encode::using_encoded(&( + $( DataOrHash::::Hash(self.tuple.$id.hash()), )+ + ), f) + } else { + codec::Encode::using_encoded(&self.tuple, f) + } + } + } + + /// [LeafDataProvider] implementation for `Compact, ...)>` + /// + /// This provides a compact-form encoding for tuples wrapped in [Compact]. + impl LeafDataProvider for Compact where + H: traits::Hash, + $( $name: LeafDataProvider ),+ + { + type LeafData = Compact< + H, + ( $( DataOrHash, )+ ), + >; + + fn leaf_data() -> Self::LeafData { + let tuple = ( + $( DataOrHash::Data($name::leaf_data()), )+ + ); + Compact::new(tuple) + } + } + + /// [LeafDataProvider] implementation for `(Tuple, ...)` + /// + /// This provides regular (non-compactable) composition of [LeafDataProvider]s. + impl<$( $name ),+> LeafDataProvider for ( $( $name, )+ ) where + ( $( $name::LeafData, )+ ): FullLeaf, + $( $name: LeafDataProvider ),+ + { + type LeafData = ( $( $name::LeafData, )+ ); + + fn leaf_data() -> Self::LeafData { + ( + $( $name::leaf_data(), )+ + ) + } + } + } +} + +/// Test functions implementation for `Compact, ...)>` +#[cfg(test)] +impl Compact, DataOrHash)> where + H: traits::Hash, + A: FullLeaf, + B: FullLeaf, +{ + /// Retrieve a hash of this item in it's compact form. + pub fn hash(&self) -> H::Output { + self.using_encoded(::hash, true) + } +} + +impl_leaf_data_for_tuple!(A:0); +impl_leaf_data_for_tuple!(A:0, B:1); +impl_leaf_data_for_tuple!(A:0, B:1, C:2); +impl_leaf_data_for_tuple!(A:0, B:1, C:2, D:3); +impl_leaf_data_for_tuple!(A:0, B:1, C:2, D:3, E:4); + +/// A MMR proof data for one of the leaves. +#[derive(codec::Encode, codec::Decode, RuntimeDebug, Clone, PartialEq, Eq)] +pub struct Proof { + /// The index of the leaf the proof is for. + pub leaf_index: u64, + /// Number of leaves in MMR, when the proof was generated. + pub leaf_count: u64, + /// Proof elements (hashes of siblings of inner nodes on the path to the leaf). + pub items: Vec, +} + + +#[cfg(test)] +mod tests { + use super::*; + + use codec::Decode; + use crate::tests::hex; + use sp_runtime::traits::Keccak256; + + type Test = DataOrHash; + type TestCompact = Compact; + type TestProof = Proof<::Output>; + + #[test] + fn should_encode_decode_proof() { + // given + let proof: TestProof = Proof { + leaf_index: 5, + leaf_count: 10, + items: vec![ + hex("c3e7ba6b511162fead58f2c8b5764ce869ed1118011ac37392522ed16720bbcd"), + hex("d3e7ba6b511162fead58f2c8b5764ce869ed1118011ac37392522ed16720bbcd"), + hex("e3e7ba6b511162fead58f2c8b5764ce869ed1118011ac37392522ed16720bbcd"), + ], + }; + + // when + let encoded = codec::Encode::encode(&proof); + let decoded = TestProof::decode(&mut &*encoded); + + // then + assert_eq!(decoded, Ok(proof)); + } + + #[test] + fn should_encode_decode_correctly_if_no_compact() { + // given + let cases = vec![ + Test::Data("Hello World!".into()), + Test::Hash(hex("c3e7ba6b511162fead58f2c8b5764ce869ed1118011ac37392522ed16720bbcd")), + Test::Data("".into()), + Test::Data("3e48d6bcd417fb22e044747242451e2c0f3e602d1bcad2767c34808621956417".into()), + ]; + + // when + let encoded = cases + .iter() + .map(codec::Encode::encode) + .collect::>(); + + let decoded = encoded + .iter() + .map(|x| Test::decode(&mut &**x)) + .collect::>(); + + // then + assert_eq!(decoded, cases.into_iter().map(Result::<_, codec::Error>::Ok).collect::>()); + // check encoding correctness + assert_eq!(&encoded[0], &hex_literal::hex!("00343048656c6c6f20576f726c6421")); + assert_eq!( + encoded[1].as_slice(), + hex_literal::hex!( + "01c3e7ba6b511162fead58f2c8b5764ce869ed1118011ac37392522ed16720bbcd" + ).as_ref() + ); + } + + #[test] + fn should_return_the_hash_correctly() { + // given + let a = Test::Data("Hello World!".into()); + let b = Test::Hash(hex("c3e7ba6b511162fead58f2c8b5764ce869ed1118011ac37392522ed16720bbcd")); + + // when + let a = a.hash(); + let b = b.hash(); + + // then + assert_eq!(a, hex("a9c321be8c24ba4dc2bd73f5300bde67dc57228ab8b68b607bb4c39c5374fac9")); + assert_eq!(b, hex("c3e7ba6b511162fead58f2c8b5764ce869ed1118011ac37392522ed16720bbcd")); + } + + #[test] + fn compact_should_work() { + // given + let a = Test::Data("Hello World!".into()); + let b = Test::Data("".into()); + + // when + let c: TestCompact = Compact::new((a.clone(), b.clone())); + let d: TestCompact = Compact::new(( + Test::Hash(a.hash()), + Test::Hash(b.hash()), + )); + + // then + assert_eq!(c.hash(), d.hash()); + } + + #[test] + fn compact_should_encode_decode_correctly() { + // given + let a = Test::Data("Hello World!".into()); + let b = Test::Data("".into()); + + let c: TestCompact = Compact::new((a.clone(), b.clone())); + let d: TestCompact = Compact::new(( + Test::Hash(a.hash()), + Test::Hash(b.hash()), + )); + let cases = vec![c, d.clone()]; + + // when + let encoded_compact = cases + .iter() + .map(|c| c.using_encoded(|x| x.to_vec(), true)) + .collect::>(); + + let encoded = cases + .iter() + .map(|c| c.using_encoded(|x| x.to_vec(), false)) + .collect::>(); + + let decoded_compact = encoded_compact + .iter() + .map(|x| TestCompact::decode(&mut &**x)) + .collect::>(); + + let decoded = encoded + .iter() + .map(|x| TestCompact::decode(&mut &**x)) + .collect::>(); + + // then + assert_eq!(decoded, cases.into_iter().map(Result::<_, codec::Error>::Ok).collect::>()); + + assert_eq!(decoded_compact, vec![Ok(d.clone()), Ok(d.clone())]); + } +} diff --git a/substrate/frame/merkle-mountain-range/src/tests.rs b/substrate/frame/merkle-mountain-range/src/tests.rs new file mode 100644 index 0000000000..059ff6612f --- /dev/null +++ b/substrate/frame/merkle-mountain-range/src/tests.rs @@ -0,0 +1,275 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::*; +use crate::mock::*; +use crate::primitives::{Proof, Compact}; + +use frame_support::traits::OnInitialize; +use sp_core::{ + H256, + offchain::{ + testing::TestOffchainExt, + OffchainExt, + }, +}; + +pub(crate) fn new_test_ext() -> sp_io::TestExternalities { + frame_system::GenesisConfig::default().build_storage::().unwrap().into() +} + +fn register_offchain_ext(ext: &mut sp_io::TestExternalities) { + let (offchain, _offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db()); + ext.register_extension(OffchainExt::new(offchain)); +} + +fn new_block() -> u64 { + let number = frame_system::Module::::block_number() + 1; + let hash = H256::repeat_byte(number as u8); + LEAF_DATA.with(|r| r.borrow_mut().a = number); + + frame_system::Module::::initialize( + &number, + &hash, + &Default::default(), + &Default::default(), + frame_system::InitKind::Full, + ); + MMR::on_initialize(number) +} + +pub(crate) fn hex(s: &str) -> H256 { + s.parse().unwrap() +} + +fn decode_node(v: Vec) -> mmr::Node< + ::Hashing, + (H256, LeafData), +> { + use crate::primitives::DataOrHash; + type A = DataOrHash::<::Hashing, H256>; + type B = DataOrHash::<::Hashing, LeafData>; + type Node = mmr::Node<::Hashing, (A, B)>; + let tuple: Node = codec::Decode::decode(&mut &v[..]).unwrap(); + + match tuple { + mmr::Node::Data((DataOrHash::Data(a), DataOrHash::Data(b))) => mmr::Node::Data((a, b)), + mmr::Node::Hash(hash) => mmr::Node::Hash(hash), + _ => unreachable!(), + } +} + +fn init_chain(blocks: usize) { + // given + for _ in 0..blocks { + new_block(); + } +} + +#[test] +fn should_start_empty() { + let _ = env_logger::try_init(); + new_test_ext().execute_with(|| { + // given + assert_eq!( + crate::RootHash::::get(), + "0000000000000000000000000000000000000000000000000000000000000000".parse().unwrap() + ); + assert_eq!(crate::NumberOfLeaves::::get(), 0); + assert_eq!(crate::Nodes::::get(0), None); + + // when + let weight = new_block(); + + // then + assert_eq!(crate::NumberOfLeaves::::get(), 1); + assert_eq!(crate::Nodes::::get(0), + Some(hex("da5e6d0616e05c6a6348605a37ca33493fc1a15ad1e6a405ee05c17843fdafed"))); + assert_eq!( + crate::RootHash::::get(), + hex("da5e6d0616e05c6a6348605a37ca33493fc1a15ad1e6a405ee05c17843fdafed") + ); + assert!(weight != 0); + }); +} + +#[test] +fn should_append_to_mmr_when_on_initialize_is_called() { + let _ = env_logger::try_init(); + let mut ext = new_test_ext(); + ext.execute_with(|| { + // when + new_block(); + new_block(); + + // then + assert_eq!(crate::NumberOfLeaves::::get(), 2); + assert_eq!(crate::Nodes::::get(0), + Some(hex("da5e6d0616e05c6a6348605a37ca33493fc1a15ad1e6a405ee05c17843fdafed"))); + assert_eq!(crate::Nodes::::get(1), + Some(hex("ff5d891b28463a3440e1b650984685efdf260e482cb3807d53c49090841e755f"))); + assert_eq!(crate::Nodes::::get(2), + Some(hex("bc54778fab79f586f007bd408dca2c4aa07959b27d1f2c8f4f2549d1fcfac8f8"))); + assert_eq!(crate::Nodes::::get(3), None); + assert_eq!( + crate::RootHash::::get(), + hex("bc54778fab79f586f007bd408dca2c4aa07959b27d1f2c8f4f2549d1fcfac8f8") + ); + }); + + // make sure the leaves end up in the offchain DB + ext.persist_offchain_overlay(); + let offchain_db = ext.offchain_db(); + assert_eq!(offchain_db.get(&MMR::offchain_key(0)).map(decode_node), Some(mmr::Node::Data(( + H256::repeat_byte(1), + LeafData::new(1), + )))); + assert_eq!(offchain_db.get(&MMR::offchain_key(1)).map(decode_node), Some(mmr::Node::Data(( + H256::repeat_byte(2), + LeafData::new(2), + )))); + assert_eq!(offchain_db.get(&MMR::offchain_key(2)).map(decode_node), Some(mmr::Node::Hash( + hex("bc54778fab79f586f007bd408dca2c4aa07959b27d1f2c8f4f2549d1fcfac8f8") + ))); + assert_eq!(offchain_db.get(&MMR::offchain_key(3)), None); +} + +#[test] +fn should_construct_larger_mmr_correctly() { + let _ = env_logger::try_init(); + new_test_ext().execute_with(|| { + // when + init_chain(7); + + // then + assert_eq!(crate::NumberOfLeaves::::get(), 7); + assert_eq!(crate::Nodes::::get(0), + Some(hex("da5e6d0616e05c6a6348605a37ca33493fc1a15ad1e6a405ee05c17843fdafed"))); + assert_eq!(crate::Nodes::::get(10), + Some(hex("af3327deed0515c8d1902c9b5cd375942d42f388f3bfe3d1cd6e1b86f9cc456c"))); + assert_eq!( + crate::RootHash::::get(), + hex("fc4f9042bd2f73feb26f3fc42db834c5f1943fa20070ddf106c486a478a0d561") + ); + }); +} + +#[test] +fn should_generate_proofs_correctly() { + let _ = env_logger::try_init(); + let mut ext = new_test_ext(); + // given + ext.execute_with(|| init_chain(7)); + ext.persist_offchain_overlay(); + + // Try to generate proofs now. This requires the offchain extensions to be present + // to retrieve full leaf data. + register_offchain_ext(&mut ext); + ext.execute_with(|| { + // when generate proofs for all leaves + let proofs = (0_u64..crate::NumberOfLeaves::::get()) + .into_iter() + .map(|leaf_index| crate::Module::::generate_proof(leaf_index).unwrap()) + .collect::>(); + + // then + assert_eq!(proofs[0], (Compact::new(( + H256::repeat_byte(1).into(), + LeafData::new(1).into(), + )), Proof { + leaf_index: 0, + leaf_count: 7, + items: vec![ + hex("ff5d891b28463a3440e1b650984685efdf260e482cb3807d53c49090841e755f"), + hex("00b0046bd2d63fcb760cf50a262448bb2bbf9a264b0b0950d8744044edf00dc3"), + hex("16de0900b57bf359a0733674ebfbba0f494e95a8391b4bfeae850019399f3ec0"), + ], + })); + assert_eq!(proofs[4], (Compact::new(( + H256::repeat_byte(5).into(), + LeafData::new(5).into(), + )), Proof { + leaf_index: 4, + leaf_count: 7, + items: vec![ + hex("e53ee36ba6c068b1a6cfef7862fed5005df55615e1c9fa6eeefe08329ac4b94b"), + hex("c09d4a008a0f1ef37860bef33ec3088ccd94268c0bfba7ff1b3c2a1075b0eb92"), + hex("af3327deed0515c8d1902c9b5cd375942d42f388f3bfe3d1cd6e1b86f9cc456c"), + ], + })); + assert_eq!(proofs[6], (Compact::new(( + H256::repeat_byte(7).into(), + LeafData::new(7).into(), + )), Proof { + leaf_index: 6, + leaf_count: 7, + items: vec![ + hex("e53ee36ba6c068b1a6cfef7862fed5005df55615e1c9fa6eeefe08329ac4b94b"), + hex("dad09f50b41822fc5ecadc25b08c3a61531d4d60e962a5aa0b6998fad5c37c5e"), + ], + })); + }); +} + +#[test] +fn should_verify() { + let _ = env_logger::try_init(); + + // Start off with chain initialisation and storing indexing data off-chain + // (MMR Leafs) + let mut ext = new_test_ext(); + ext.execute_with(|| init_chain(7)); + ext.persist_offchain_overlay(); + + // Try to generate proof now. This requires the offchain extensions to be present + // to retrieve full leaf data. + register_offchain_ext(&mut ext); + let (leaf, proof5) = ext.execute_with(|| { + // when + crate::Module::::generate_proof(5).unwrap() + }); + + // Now to verify the proof, we really shouldn't require offchain storage or extension. + // Hence we initialize the storage once again, using different externalities and then + // verify. + let mut ext2 = new_test_ext(); + ext2.execute_with(|| { + init_chain(7); + // then + assert_eq!(crate::Module::::verify_leaf(leaf, proof5), Ok(())); + }); +} + +#[test] +fn should_verify_on_the_next_block_since_there_is_no_pruning_yet() { + let _ = env_logger::try_init(); + let mut ext = new_test_ext(); + // given + ext.execute_with(|| init_chain(7)); + + ext.persist_offchain_overlay(); + register_offchain_ext(&mut ext); + + ext.execute_with(|| { + // when + let (leaf, proof5) = crate::Module::::generate_proof(5).unwrap(); + new_block(); + + // then + assert_eq!(crate::Module::::verify_leaf(leaf, proof5), Ok(())); + }); +} diff --git a/substrate/primitives/core/src/offchain/testing.rs b/substrate/primitives/core/src/offchain/testing.rs index 3fe34cc0cf..5256f41771 100644 --- a/substrate/primitives/core/src/offchain/testing.rs +++ b/substrate/primitives/core/src/offchain/testing.rs @@ -70,6 +70,8 @@ pub struct TestPersistentOffchainDB { } impl TestPersistentOffchainDB { + const PREFIX: &'static [u8] = b""; + /// Create a new and empty offchain storage db for persistent items pub fn new() -> Self { Self { @@ -82,11 +84,16 @@ impl TestPersistentOffchainDB { let mut me = self.persistent.write(); for ((_prefix, key), value_operation) in changes.drain() { match value_operation { - OffchainOverlayedChange::SetValue(val) => me.set(b"", key.as_slice(), val.as_slice()), - OffchainOverlayedChange::Remove => me.remove(b"", key.as_slice()), + OffchainOverlayedChange::SetValue(val) => me.set(Self::PREFIX, key.as_slice(), val.as_slice()), + OffchainOverlayedChange::Remove => me.remove(Self::PREFIX, key.as_slice()), } } } + + /// Retrieve a key from the test backend. + pub fn get(&self, key: &[u8]) -> Option> { + OffchainStorage::get(self, Self::PREFIX, key) + } } impl OffchainStorage for TestPersistentOffchainDB { @@ -266,8 +273,8 @@ impl offchain::Externalities for TestOffchainExt { fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option> { let state = self.0.read(); match kind { - StorageKind::LOCAL => state.local_storage.get(b"", key), - StorageKind::PERSISTENT => state.persistent_storage.get(b"", key), + StorageKind::LOCAL => state.local_storage.get(TestPersistentOffchainDB::PREFIX, key), + StorageKind::PERSISTENT => state.persistent_storage.get(key), } } diff --git a/substrate/primitives/runtime/src/offchain/storage.rs b/substrate/primitives/runtime/src/offchain/storage.rs index 2f62d400c0..e39514686e 100644 --- a/substrate/primitives/runtime/src/offchain/storage.rs +++ b/substrate/primitives/runtime/src/offchain/storage.rs @@ -105,7 +105,6 @@ mod tests { use sp_io::TestExternalities; use sp_core::offchain::{ OffchainExt, - OffchainStorage, testing, }; @@ -125,7 +124,7 @@ mod tests { assert_eq!(val.get::(), Some(Some(15_u32))); assert_eq!(val.get::>(), Some(None)); assert_eq!( - state.read().persistent_storage.get(b"", b"testval"), + state.read().persistent_storage.get(b"testval"), Some(vec![15_u8, 0, 0, 0]) ); }) @@ -148,7 +147,7 @@ mod tests { assert_eq!(result, Ok(Ok(16_u32))); assert_eq!(val.get::(), Some(Some(16_u32))); assert_eq!( - state.read().persistent_storage.get(b"", b"testval"), + state.read().persistent_storage.get(b"testval"), Some(vec![16_u8, 0, 0, 0]) ); diff --git a/substrate/primitives/runtime/src/offchain/storage_lock.rs b/substrate/primitives/runtime/src/offchain/storage_lock.rs index 451753931e..0d9cf835c1 100644 --- a/substrate/primitives/runtime/src/offchain/storage_lock.rs +++ b/substrate/primitives/runtime/src/offchain/storage_lock.rs @@ -452,7 +452,7 @@ pub trait BlockNumberProvider { #[cfg(test)] mod tests { use super::*; - use sp_core::offchain::{testing, OffchainExt, OffchainStorage}; + use sp_core::offchain::{testing, OffchainExt}; use sp_io::TestExternalities; const VAL_1: u32 = 0u32; @@ -485,7 +485,7 @@ mod tests { } }); // lock must have been cleared at this point - assert_eq!(state.read().persistent_storage.get(b"", b"lock_1"), None); + assert_eq!(state.read().persistent_storage.get(b"lock_1"), None); } #[test] @@ -508,7 +508,7 @@ mod tests { guard.forget(); }); // lock must have been cleared at this point - let opt = state.read().persistent_storage.get(b"", b"lock_2"); + let opt = state.read().persistent_storage.get(b"lock_2"); assert!(opt.is_some()); } @@ -540,7 +540,7 @@ mod tests { }); // lock must have been cleared at this point - let opt = state.read().persistent_storage.get(b"", b"lock_3"); + let opt = state.read().persistent_storage.get(b"lock_3"); assert!(opt.is_some()); } @@ -587,7 +587,7 @@ mod tests { }); // lock must have been cleared at this point - let opt = state.read().persistent_storage.get(b"", b"lock_4"); + let opt = state.read().persistent_storage.get(b"lock_4"); assert_eq!(opt.unwrap(), vec![132_u8, 3u8, 0, 0, 0, 0, 0, 0]); // 132 + 256 * 3 = 900 } }