Compact proof utilities in sp_trie. (#8574)

* validation extension in sp_io

* need paths

* arc impl

* missing host function in executor

* io to pkdot

* decode function.

* encode primitive.

* trailing tab

* multiple patch

* fix child trie logic

* restore master versionning

* bench compact proof size

* trie-db 22.3 is needed

* line width

* split line

* fixes for bench (additional root may not be needed as original issue was
with empty proof).

* revert compact from block size calculation.

* New error type for compression.

* Adding test (incomplete (failing)).
Also lacking real proof checking (no good primitives in sp-trie crate).

* There is currently no proof recording utility in sp_trie, removing
test.

* small test of child root in proof without a child proof.

* remove empty test.

* remove non compact proof size

* Missing revert.

* proof method to encode decode.

substrate/client/db/src/bench.rs

@@ -117,6 +117,7 @@ pub struct BenchmarkingState<B: BlockT> {
 	read_write_tracker: RefCell<ReadWriteTracker>,
 	whitelist: RefCell<Vec<TrackedStorageKey>>,
 	proof_recorder: Option<ProofRecorder<B::Hash>>,
+	proof_recorder_root: Cell<B::Hash>,
 }
 
 impl<B: BlockT> BenchmarkingState<B> {
@@ -129,7 +130,7 @@ impl<B: BlockT> BenchmarkingState<B> {
 		let mut state = BenchmarkingState {
 			state: RefCell::new(None),
 			db: Cell::new(None),
-			root: Cell::new(root),
+			root: Cell::new(root.clone()),
 			genesis: Default::default(),
 			genesis_root: Default::default(),
 			record: Default::default(),
@@ -139,6 +140,7 @@ impl<B: BlockT> BenchmarkingState<B> {
 			read_write_tracker: Default::default(),
 			whitelist: Default::default(),
 			proof_recorder: record_proof.then(Default::default),
+			proof_recorder_root: Cell::new(root.clone()),
 		};
 
 		state.add_whitelist_to_tracker();
@@ -166,7 +168,10 @@ impl<B: BlockT> BenchmarkingState<B> {
 			None => Arc::new(kvdb_memorydb::create(1)),
 		};
 		self.db.set(Some(db.clone()));
-		self.proof_recorder.as_ref().map(|r| r.reset());
+		if let Some(recorder) = &self.proof_recorder {
+			recorder.reset();
+			self.proof_recorder_root.set(self.root.get());
+		}
 		let storage_db = Arc::new(StorageDb::<B> {
 			db,
 			proof_recorder: self.proof_recorder.clone(),
@@ -516,7 +521,27 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
 	}
 
 	fn proof_size(&self) -> Option<u32> {
-		self.proof_recorder.as_ref().map(|recorder| recorder.estimate_encoded_size() as u32)
+		self.proof_recorder.as_ref().map(|recorder| {
+			let proof_size = recorder.estimate_encoded_size() as u32;
+			let proof = recorder.to_storage_proof();
+			let proof_recorder_root = self.proof_recorder_root.get();
+			if proof_recorder_root == Default::default() || proof_size == 1 {
+				// empty trie
+				proof_size
+			} else {
+				if let Some(size) = proof.encoded_compact_size::<HashFor<B>>(proof_recorder_root) {
+					size as u32
+				} else {
+					panic!(
+						"proof rec root {:?}, root {:?}, genesis {:?}, rec_len {:?}",
+						self.proof_recorder_root.get(),
+						self.root.get(),
+						self.genesis_root,
+						proof_size,
+					);
+				}
+			}
+		})
 	}
 }
 

substrate/primitives/state-machine/src/lib.rs

@@ -1402,14 +1402,22 @@ mod tests {
 		}
 	}
 
+	fn test_compact(remote_proof: StorageProof, remote_root: &sp_core::H256) -> StorageProof {
+		let compact_remote_proof = remote_proof.into_compact_proof::<BlakeTwo256>(
+			remote_root.clone(),
+		).unwrap();
+		compact_remote_proof.to_storage_proof::<BlakeTwo256>(Some(remote_root)).unwrap().0
+	}
+
 	#[test]
 	fn prove_read_and_proof_check_works() {
 		let child_info = ChildInfo::new_default(b"sub1");
 		let child_info = &child_info;
 		// fetch read proof from 'remote' full node
 		let remote_backend = trie_backend::tests::test_trie();
-		let remote_root = remote_backend.storage_root(::std::iter::empty()).0;
+		let remote_root = remote_backend.storage_root(std::iter::empty()).0;
 		let remote_proof = prove_read(remote_backend, &[b"value2"]).unwrap();
+		let remote_proof = test_compact(remote_proof, &remote_root);
  		// check proof locally
 		let local_result1 = read_proof_check::<BlakeTwo256, _>(
 			remote_root,
@@ -1429,12 +1437,13 @@ mod tests {
 		assert_eq!(local_result2, false);
 		// on child trie
 		let remote_backend = trie_backend::tests::test_trie();
-		let remote_root = remote_backend.storage_root(::std::iter::empty()).0;
+		let remote_root = remote_backend.storage_root(std::iter::empty()).0;
 		let remote_proof = prove_child_read(
 			remote_backend,
 			child_info,
 			&[b"value3"],
 		).unwrap();
+		let remote_proof = test_compact(remote_proof, &remote_root);
 		let local_result1 = read_child_proof_check::<BlakeTwo256, _>(
 			remote_root,
 			remote_proof.clone(),
@@ -1457,6 +1466,50 @@ mod tests {
 		);
 	}
 
+	#[test]
+	fn compact_multiple_child_trie() {
+		// this root will be queried
+		let child_info1 = ChildInfo::new_default(b"sub1");
+		// this root will not be include in proof
+		let child_info2 = ChildInfo::new_default(b"sub2");
+		// this root will be include in proof
+		let child_info3 = ChildInfo::new_default(b"sub");
+		let mut remote_backend = trie_backend::tests::test_trie();
+		let (remote_root, transaction) = remote_backend.full_storage_root(
+			std::iter::empty(),
+			vec![
+				(&child_info1, vec![
+					(&b"key1"[..], Some(&b"val2"[..])),
+					(&b"key2"[..], Some(&b"val3"[..])),
+				].into_iter()),
+				(&child_info2, vec![
+					(&b"key3"[..], Some(&b"val4"[..])),
+					(&b"key4"[..], Some(&b"val5"[..])),
+				].into_iter()),
+				(&child_info3, vec![
+					(&b"key5"[..], Some(&b"val6"[..])),
+					(&b"key6"[..], Some(&b"val7"[..])),
+				].into_iter()),
+			].into_iter(),
+		);
+		remote_backend.backend_storage_mut().consolidate(transaction);
+		remote_backend.essence.set_root(remote_root.clone());
+		let remote_proof = prove_child_read(
+			remote_backend,
+			&child_info1,
+			&[b"key1"],
+		).unwrap();
+		let remote_proof = test_compact(remote_proof, &remote_root);
+		let local_result1 = read_child_proof_check::<BlakeTwo256, _>(
+			remote_root,
+			remote_proof.clone(),
+			&child_info1,
+			&[b"key1"],
+		).unwrap();
+		assert_eq!(local_result1.len(), 1);
+		assert_eq!(local_result1.get(&b"key1"[..]), Some(&Some(b"val2".to_vec())));
+	}
+
 	#[test]
 	fn child_storage_uuid() {
 

substrate/primitives/trie/Cargo.toml

@@ -21,7 +21,7 @@ harness = false
 codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false }
 sp-std = { version = "3.0.0", default-features = false, path = "../std" }
 hash-db = { version = "0.15.2", default-features = false }
-trie-db = { version = "0.22.2", default-features = false }
+trie-db = { version = "0.22.3", default-features = false }
 trie-root = { version = "0.16.0", default-features = false }
 memory-db = { version = "0.26.0", default-features = false }
 sp-core = { version = "3.0.0", default-features = false, path = "../core" }

substrate/primitives/trie/src/error.rs

@@ -26,7 +26,7 @@ pub enum Error {
 	/// Bad format.
 	BadFormat,
 	/// Decoding error.
-	Decode(codec::Error)
+	Decode(codec::Error),
 }
 
 impl From<codec::Error> for Error {

substrate/primitives/trie/src/lib.rs

@@ -23,6 +23,7 @@ mod error;
 mod node_header;
 mod node_codec;
 mod storage_proof;
+mod trie_codec;
 mod trie_stream;
 
 use sp_std::{boxed::Box, marker::PhantomData, vec::Vec, borrow::Borrow};
@@ -35,7 +36,7 @@ pub use error::Error;
 pub use trie_stream::TrieStream;
 /// The Substrate format implementation of `NodeCodec`.
 pub use node_codec::NodeCodec;
-pub use storage_proof::StorageProof;
+pub use storage_proof::{StorageProof, CompactProof};
 /// Various re-exports from the `trie-db` crate.
 pub use trie_db::{
 	Trie, TrieMut, DBValue, Recorder, CError, Query, TrieLayout, TrieConfiguration, nibble_ops, TrieDBIterator,
@@ -45,6 +46,9 @@ pub use memory_db::KeyFunction;
 pub use memory_db::prefixed_key;
 /// Various re-exports from the `hash-db` crate.
 pub use hash_db::{HashDB as HashDBT, EMPTY_PREFIX};
+/// Trie codec reexport, mainly child trie support
+/// for trie compact proof.
+pub use trie_codec::{decode_compact, encode_compact, Error as CompactProofError};
 
 #[derive(Default)]
 /// substrate trie layout

substrate/primitives/trie/src/storage_proof.rs

@@ -31,6 +31,12 @@ pub struct StorageProof {
 	trie_nodes: Vec<Vec<u8>>,
 }
 
+/// Storage proof in compact form.
+#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
+pub struct CompactProof {
+	pub encoded_nodes: Vec<Vec<u8>>,
+}
+
 impl StorageProof {
 	/// Constructs a storage proof from a subset of encoded trie nodes in a storage backend.
 	pub fn new(trie_nodes: Vec<Vec<u8>>) -> Self {
@@ -79,6 +85,56 @@ impl StorageProof {
 
 		Self { trie_nodes }
 	}
+
+	/// Encode as a compact proof with default
+	/// trie layout.
+	pub fn into_compact_proof<H: Hasher>(
+		self,
+		root: H::Out,
+	) -> Result<CompactProof, crate::CompactProofError<crate::Layout<H>>> {
+		crate::encode_compact::<crate::Layout<H>>(self, root)
+	}
+	
+	/// Returns the estimated encoded size of the compact proof.
+	///
+	/// Runing this operation is a slow operation (build the whole compact proof) and should only be
+	/// in non sensitive path.
+	/// Return `None` on error.
+	pub fn encoded_compact_size<H: Hasher>(self, root: H::Out) -> Option<usize> {
+		let compact_proof = self.into_compact_proof::<H>(root);
+		compact_proof.ok().map(|p| p.encoded_size())
+	}
+
+}
+
+impl CompactProof {
+	/// Return an iterator on the compact encoded nodes.
+	pub fn iter_compact_encoded_nodes(&self) -> impl Iterator<Item = &[u8]> {
+		self.encoded_nodes.iter().map(Vec::as_slice)
+	}
+
+	/// Decode to a full storage_proof.
+	///
+	/// Method use a temporary `HashDB`, and `sp_trie::decode_compact`
+	/// is often better.
+	pub fn to_storage_proof<H: Hasher>(
+		&self,
+		expected_root: Option<&H::Out>,
+	) -> Result<(StorageProof, H::Out), crate::CompactProofError<crate::Layout<H>>> {
+		let mut db = crate::MemoryDB::<H>::new(&[]);
+		let root = crate::decode_compact::<crate::Layout<H>, _, _>(
+			&mut db,
+			self.iter_compact_encoded_nodes(),
+			expected_root,
+		)?;
+		Ok((StorageProof::new(db.drain().into_iter().filter_map(|kv|
+			if (kv.1).1 > 0 {
+				Some((kv.1).0)
+			} else {
+				None
+			}
+		).collect()), root))
+	}
 }
 
 /// An iterator over trie nodes constructed from a storage proof. The nodes are not guaranteed to

substrate/primitives/trie/src/trie_codec.rs

@@ -0,0 +1,259 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2021-2021 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Compact proof support.
+//!
+//! This uses compact proof from trie crate and extends
+//! it to substrate specific layout and child trie system.
+
+use crate::{
+	EMPTY_PREFIX, HashDBT, TrieHash, TrieError, TrieConfiguration,
+	CompactProof, StorageProof,
+};
+use sp_std::boxed::Box;
+use sp_std::vec::Vec;
+use trie_db::Trie;
+#[cfg(feature="std")]
+use std::fmt;
+#[cfg(feature="std")]
+use std::error::Error as StdError;
+
+
+/// Error for trie node decoding.
+pub enum Error<L: TrieConfiguration> {
+	/// Verification failed due to root mismatch.
+	RootMismatch(TrieHash<L>, TrieHash<L>),
+	/// Missing nodes in proof.
+	IncompleteProof,
+	/// Compact node is not needed.
+	ExtraneousChildNode,
+	/// Child content with root not in proof.
+	ExtraneousChildProof(TrieHash<L>),
+	/// Bad child trie root.
+	InvalidChildRoot(Vec<u8>, Vec<u8>),
+	/// Errors from trie crate.
+	TrieError(Box<TrieError<L>>),
+}
+
+impl<L: TrieConfiguration> From<Box<TrieError<L>>> for Error<L> {
+	fn from(error: Box<TrieError<L>>) -> Self {
+		Error::TrieError(error)
+	}
+}
+
+#[cfg(feature="std")]
+impl<L: TrieConfiguration> StdError for Error<L> {
+	fn description(&self) -> &str {
+		match self {
+			Error::InvalidChildRoot(..) => "Invalid child root error",
+			Error::TrieError(..) => "Trie db error",
+			Error::RootMismatch(..) => "Trie db error",
+			Error::IncompleteProof => "Incomplete proof",
+			Error::ExtraneousChildNode => "Extraneous child node",
+			Error::ExtraneousChildProof(..) => "Extraneous child proof",
+		}
+	}
+}
+
+#[cfg(feature="std")]
+impl<L: TrieConfiguration> fmt::Debug for Error<L> {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		<Self as fmt::Display>::fmt(&self, f)
+	}
+}
+
+#[cfg(feature="std")]
+impl<L: TrieConfiguration> fmt::Display for Error<L> {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		match self {
+			Error::InvalidChildRoot(k, v) => write!(f, "InvalidChildRoot at {:x?}: {:x?}", k, v),
+			Error::TrieError(e) => write!(f, "Trie error: {}", e),
+			Error::IncompleteProof => write!(f, "Incomplete proof"),
+			Error::ExtraneousChildNode => write!(f, "Child node content with no root in proof"),
+			Error::ExtraneousChildProof(root) => write!(f, "Proof of child trie {:x?} not in parent proof", root.as_ref()),
+			Error::RootMismatch(root, expected) => write!(
+				f,
+				"Verification error, root is {:x?}, expected: {:x?}",
+				root.as_ref(),
+				expected.as_ref(),
+			),
+		}
+	}
+}
+
+/// Decode a compact proof.
+///
+/// Takes as input a destination `db` for decoded node and `encoded`
+/// an iterator of compact encoded nodes.
+///
+/// Child trie are decoded in order of child trie root present
+/// in the top trie.
+pub fn decode_compact<'a, L, DB, I>(
+	db: &mut DB,
+	encoded: I,
+	expected_root: Option<&TrieHash<L>>,
+) -> Result<TrieHash<L>, Error<L>>
+	where
+		L: TrieConfiguration,
+		DB: HashDBT<L::Hash, trie_db::DBValue> + hash_db::HashDBRef<L::Hash, trie_db::DBValue>,
+		I: IntoIterator<Item = &'a [u8]>,
+{
+	let mut nodes_iter = encoded.into_iter();
+	let (top_root, _nb_used) = trie_db::decode_compact_from_iter::<L, _, _, _>(
+		db,
+		&mut nodes_iter,
+	)?;
+
+	// Only check root if expected root is passed as argument.
+	if let Some(expected_root) = expected_root {
+		if expected_root != &top_root {
+			return Err(Error::RootMismatch(top_root.clone(), expected_root.clone()));
+		}
+	}
+
+	let mut child_tries = Vec::new();
+	{
+		// fetch child trie roots
+		let trie = crate::TrieDB::<L>::new(db, &top_root)?;
+
+		let mut iter = trie.iter()?;
+
+		let childtrie_roots = sp_core::storage::well_known_keys::DEFAULT_CHILD_STORAGE_KEY_PREFIX;
+		if iter.seek(childtrie_roots).is_ok() {
+			loop {
+				match iter.next() {
+					Some(Ok((key, value))) if key.starts_with(childtrie_roots) => {
+						// we expect all default child trie root to be correctly encoded.
+						// see other child trie functions.
+						let mut root = TrieHash::<L>::default();
+						// still in a proof so prevent panic
+						if root.as_mut().len() != value.as_slice().len() {
+							return Err(Error::InvalidChildRoot(key, value));
+						}
+						root.as_mut().copy_from_slice(value.as_ref());
+						child_tries.push(root);
+					},
+					// allow incomplete database error: we only
+					// require access to data in the proof.
+					Some(Err(error)) => match *error {
+						trie_db::TrieError::IncompleteDatabase(..) => (),
+						e => return Err(Box::new(e).into()),
+					},
+					_ => break,
+				}
+			}
+		}
+	}
+
+	if !HashDBT::<L::Hash, _>::contains(db, &top_root, EMPTY_PREFIX) {
+		return Err(Error::IncompleteProof);
+	}
+
+	let mut previous_extracted_child_trie = None;
+	for child_root in child_tries.into_iter() {
+		if previous_extracted_child_trie.is_none() {
+			let (top_root, _) = trie_db::decode_compact_from_iter::<L, _, _, _>(
+				db,
+				&mut nodes_iter,
+			)?;
+			previous_extracted_child_trie = Some(top_root);
+		}
+
+		// we do not early exit on root mismatch but try the
+		// other read from proof (some child root may be
+		// in proof without actual child content).
+		if Some(child_root) == previous_extracted_child_trie {
+			previous_extracted_child_trie = None;
+		}
+	}
+
+	if let Some(child_root) = previous_extracted_child_trie {
+		// A child root was read from proof but is not present
+		// in top trie.
+		return Err(Error::ExtraneousChildProof(child_root));
+	}
+
+	if nodes_iter.next().is_some() {
+		return Err(Error::ExtraneousChildNode);
+	}
+
+	Ok(top_root)
+}
+
+/// Encode a compact proof.
+///
+/// Takes as input all full encoded node from the proof, and
+/// the root.
+/// Then parse all child trie root and compress main trie content first
+/// then all child trie contents.
+/// Child trie are ordered by the order of their roots in the top trie.
+pub fn encode_compact<L>(
+	proof: StorageProof,
+	root: TrieHash<L>,
+) -> Result<CompactProof, Error<L>>
+	where
+		L: TrieConfiguration,
+{
+	let mut child_tries = Vec::new();
+	let partial_db = proof.into_memory_db();
+	let mut compact_proof = {
+		let trie = crate::TrieDB::<L>::new(&partial_db, &root)?;
+
+		let mut iter = trie.iter()?;
+
+		let childtrie_roots = sp_core::storage::well_known_keys::DEFAULT_CHILD_STORAGE_KEY_PREFIX;
+		if iter.seek(childtrie_roots).is_ok() {
+			loop {
+				match iter.next() {
+					Some(Ok((key, value))) if key.starts_with(childtrie_roots) => {
+						let mut root = TrieHash::<L>::default();
+						if root.as_mut().len() != value.as_slice().len() {
+							// some child trie root in top trie are not an encoded hash.
+							return Err(Error::InvalidChildRoot(key.to_vec(), value.to_vec()));
+						}
+						root.as_mut().copy_from_slice(value.as_ref());
+						child_tries.push(root);
+					},
+					// allow incomplete database error: we only
+					// require access to data in the proof.
+					Some(Err(error)) => match *error {
+						trie_db::TrieError::IncompleteDatabase(..) => (),
+						e => return Err(Box::new(e).into()),
+					},
+					_ => break,
+				}
+			}
+		}
+
+		trie_db::encode_compact::<L>(&trie)?
+	};
+
+	for child_root in child_tries {
+		if !HashDBT::<L::Hash, _>::contains(&partial_db, &child_root, EMPTY_PREFIX) {
+			// child proof are allowed to be missing (unused root can be included
+			// due to trie structure modification).
+			continue;
+		}
+
+		let trie = crate::TrieDB::<L>::new(&partial_db, &child_root)?;
+		let child_proof = trie_db::encode_compact::<L>(&trie)?;
+
+		compact_proof.extend(child_proof);
+	}
+
+	Ok(CompactProof { encoded_nodes: compact_proof })
+}

substrate/utils/wasm-builder/src/wasm_project.rs

@@ -232,7 +232,8 @@ fn create_project_cargo_toml(
 	wasm_workspace_toml.insert("profile".into(), profile.into());
 
 	// Add patch section from the project root `Cargo.toml`
-	if let Some(mut patch) = workspace_toml.remove("patch").and_then(|p| p.try_into::<Table>().ok()) {
+	while let Some(mut patch) = workspace_toml.remove("patch")
+		.and_then(|p| p.try_into::<Table>().ok()) {
 		// Iterate over all patches and make the patch path absolute from the workspace root path.
 		patch.iter_mut()
 			.filter_map(|p|