Light friendly storage tracking: changes trie + extending over ranges (#628)

* changes_trie

* changs_trie: continue

* changes_trie: adding tests

* fixed TODO

* removed obsolete ExtrinsicChanges

* encodable ChangesTrieConfiguration

* removed polkadot fle

* fixed grumbles

* ext_storage_changes_root returns u32

* moved changes trie root to digest

* removed commented code

* read storage values from native code

* fixed grumbles

* fixed grumbles

* missing comma

substrate/core/sr-primitives/src/generic/digest.rs

@@ -36,6 +36,7 @@ impl<Item> Default for Digest<Item> {
 impl<Item> traits::Digest for Digest<Item> where
 	Item: DigestItemT + Codec
 {
+	type Hash = Item::Hash;
 	type Item = Item;
 
 	fn logs(&self) -> &[Self::Item] {
@@ -51,10 +52,14 @@ impl<Item> traits::Digest for Digest<Item> where
 /// provide opaque access to other items.
 #[derive(PartialEq, Eq, Clone)]
 #[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))]
-pub enum DigestItem<AuthorityId> {
+pub enum DigestItem<Hash, AuthorityId> {
 	/// System digest item announcing that authorities set has been changed
 	/// in the block. Contains the new set of authorities.
 	AuthoritiesChange(Vec<AuthorityId>),
+	/// System digest item that contains the root of changes trie at given
+	/// block. It is created for every block iff runtime supports changes
+	/// trie creation.
+	ChangesTrieRoot(Hash),
 	/// Any 'non-system' digest item, opaque to the native code.
 	Other(Vec<u8>),
 }
@@ -63,9 +68,11 @@ pub enum DigestItem<AuthorityId> {
 /// final runtime implementations for encoding/decoding its log items.
 #[derive(PartialEq, Eq, Clone)]
 #[cfg_attr(feature = "std", derive(Debug))]
-pub enum DigestItemRef<'a, AuthorityId: 'a> {
+pub enum DigestItemRef<'a, Hash: 'a, AuthorityId: 'a> {
 	/// Reference to `DigestItem::AuthoritiesChange`.
 	AuthoritiesChange(&'a [AuthorityId]),
+	/// Reference to `DigestItem::ChangesTrieRoot`.
+	ChangesTrieRoot(&'a Hash),
 	/// Reference to `DigestItem::Other`.
 	Other(&'a Vec<u8>),
 }
@@ -79,9 +86,10 @@ pub enum DigestItemRef<'a, AuthorityId: 'a> {
 enum DigestItemType {
 	Other = 0,
 	AuthoritiesChange,
+	ChangesTrieRoot,
 }
 
-impl<AuthorityId> DigestItem<AuthorityId> {
+impl<Hash, AuthorityId> DigestItem<Hash, AuthorityId> {
 	/// Returns Some if `self` is a `DigestItem::Other`.
 	pub fn as_other(&self) -> Option<&Vec<u8>> {
 		match *self {
@@ -91,15 +99,17 @@ impl<AuthorityId> DigestItem<AuthorityId> {
 	}
 
 	/// Returns a 'referencing view' for this digest item.
-	fn dref<'a>(&'a self) -> DigestItemRef<'a, AuthorityId> {
+	fn dref<'a>(&'a self) -> DigestItemRef<'a, Hash, AuthorityId> {
 		match *self {
 			DigestItem::AuthoritiesChange(ref v) => DigestItemRef::AuthoritiesChange(v),
+			DigestItem::ChangesTrieRoot(ref v) => DigestItemRef::ChangesTrieRoot(v),
 			DigestItem::Other(ref v) => DigestItemRef::Other(v),
 		}
 	}
 }
 
-impl<AuthorityId: Member> traits::DigestItem for DigestItem<AuthorityId> {
+impl<Hash: Member, AuthorityId: Member> traits::DigestItem for DigestItem<Hash, AuthorityId> {
+	type Hash = Hash;
 	type AuthorityId = AuthorityId;
 
 	fn as_authorities_change(&self) -> Option<&[Self::AuthorityId]> {
@@ -108,21 +118,31 @@ impl<AuthorityId: Member> traits::DigestItem for DigestItem<AuthorityId> {
 			_ => None,
 		}
 	}
+
+	fn as_changes_trie_root(&self) -> Option<&Hash> {
+		match *self {
+			DigestItem::ChangesTrieRoot(ref changes_trie_root) => Some(changes_trie_root),
+			_ => None,
+		}
+	}
 }
 
-impl<AuthorityId: Encode> Encode for DigestItem<AuthorityId> {
+impl<Hash: Encode, AuthorityId: Encode> Encode for DigestItem<Hash, AuthorityId> {
 	fn encode(&self) -> Vec<u8> {
 		self.dref().encode()
 	}
 }
 
-impl<AuthorityId: Decode> Decode for DigestItem<AuthorityId> {
+impl<Hash: Decode, AuthorityId: Decode> Decode for DigestItem<Hash, AuthorityId> {
 	fn decode<I: Input>(input: &mut I) -> Option<Self> {
 		let item_type: DigestItemType = Decode::decode(input)?;
 		match item_type {
 			DigestItemType::AuthoritiesChange => Some(DigestItem::AuthoritiesChange(
 				Decode::decode(input)?,
 			)),
+			DigestItemType::ChangesTrieRoot => Some(DigestItem::ChangesTrieRoot(
+				Decode::decode(input)?,
+			)),
 			DigestItemType::Other => Some(DigestItem::Other(
 				Decode::decode(input)?,
 			)),
@@ -130,7 +150,7 @@ impl<AuthorityId: Decode> Decode for DigestItem<AuthorityId> {
 	}
 }
 
-impl<'a, AuthorityId: Encode> Encode for DigestItemRef<'a, AuthorityId> {
+impl<'a, Hash: Encode, AuthorityId: Encode> Encode for DigestItemRef<'a, Hash, AuthorityId> {
 	fn encode(&self) -> Vec<u8> {
 		let mut v = Vec::new();
 
@@ -139,6 +159,10 @@ impl<'a, AuthorityId: Encode> Encode for DigestItemRef<'a, AuthorityId> {
 				DigestItemType::AuthoritiesChange.encode_to(&mut v);
 				authorities.encode_to(&mut v);
 			},
+			DigestItemRef::ChangesTrieRoot(changes_trie_root) => {
+				DigestItemType::ChangesTrieRoot.encode_to(&mut v);
+				changes_trie_root.encode_to(&mut v);
+			},
 			DigestItemRef::Other(val) => {
 				DigestItemType::Other.encode_to(&mut v);
 				val.encode_to(&mut v);

substrate/core/sr-primitives/src/generic/header.rs

@@ -117,7 +117,7 @@ impl<Number, Hash, DigestItem> Encode for Header<Number, Hash, DigestItem> where
 impl<Number, Hash, DigestItem> traits::Header for Header<Number, Hash, DigestItem> where
 	Number: Member + ::rstd::hash::Hash + Copy + Codec + MaybeDisplay + SimpleArithmetic + Codec,
 	Hash: HashT,
-	DigestItem: DigestItemT + Codec,
+	DigestItem: DigestItemT<Hash = Hash::Output> + Codec,
 	Hash::Output: Default + ::rstd::hash::Hash + Copy + Member + MaybeDisplay + SimpleBitOps + Codec,
  {
 	type Number = Number;
@@ -148,7 +148,11 @@ impl<Number, Hash, DigestItem> traits::Header for Header<Number, Hash, DigestIte
 		digest: Self::Digest
 	) -> Self {
 		Header {
-			number, extrinsics_root: extrinsics_root, state_root, parent_hash, digest
+			number,
+			extrinsics_root,
+			state_root,
+			parent_hash,
+			digest
 		}
 	}
 }

substrate/core/sr-primitives/src/generic/tests.rs

@@ -21,7 +21,7 @@ use substrate_primitives::{H256, H512};
 use super::{Digest, Header, DigestItem, UncheckedExtrinsic};
 
 type Block = super::Block<
-	Header<u64, ::traits::BlakeTwo256, DigestItem<u32>>,
+	Header<u64, ::traits::BlakeTwo256, DigestItem<H256, u32>>,
 	UncheckedExtrinsic<H256, u64, u64, ::Ed25519Signature>,
 >;
 
@@ -34,8 +34,8 @@ fn block_roundtrip_serialization() {
 			state_root: [1u8; 32].into(),
 			extrinsics_root: [2u8; 32].into(),
 			digest: Digest { logs: vec![
-				DigestItem::Other::<u32>(vec![1, 2, 3]),
-				DigestItem::Other::<u32>(vec![4, 5, 6]),
+				DigestItem::Other::<H256, u32>(vec![1, 2, 3]),
+				DigestItem::Other::<H256, u32>(vec![4, 5, 6]),
 			] },
 		},
 		extrinsics: vec![
@@ -70,7 +70,7 @@ fn block_roundtrip_serialization() {
 
 #[test]
 fn system_digest_item_encoding() {
-	let item = DigestItem::AuthoritiesChange::<u32>(vec![10, 20, 30]);
+	let item = DigestItem::AuthoritiesChange::<H256, u32>(vec![10, 20, 30]);
 	let encoded = item.encode();
 	assert_eq!(encoded, vec![
 		// type = DigestItemType::AuthoritiesChange
@@ -83,13 +83,13 @@ fn system_digest_item_encoding() {
 		30, 0, 0, 0,
 	]);
 
-	let decoded: DigestItem<u32> = Decode::decode(&mut &encoded[..]).unwrap();
+	let decoded: DigestItem<H256, u32> = Decode::decode(&mut &encoded[..]).unwrap();
 	assert_eq!(item, decoded);
 }
 
 #[test]
 fn non_system_digest_item_encoding() {
-	let item = DigestItem::Other::<u32>(vec![10, 20, 30]);
+	let item = DigestItem::Other::<H256, u32>(vec![10, 20, 30]);
 	let encoded = item.encode();
 	assert_eq!(encoded, vec![
 		// type = DigestItemType::Other
@@ -100,6 +100,6 @@ fn non_system_digest_item_encoding() {
 		10, 20, 30,
 	]);
 
-	let decoded: DigestItem<u32> = Decode::decode(&mut &encoded[..]).unwrap();
+	let decoded: DigestItem<H256, u32> = Decode::decode(&mut &encoded[..]).unwrap();
 	assert_eq!(item, decoded);
 }

substrate/core/sr-primitives/src/lib.rs

@@ -366,6 +366,7 @@ macro_rules! impl_outer_log {
 
 #[cfg(test)]
 mod tests {
+	use substrate_primitives::hash::H256;
 	use codec::{Encode, Decode, Input};
 
 	pub trait RuntimeT {
@@ -400,7 +401,7 @@ mod tests {
 
 		// TODO try to avoid redundant brackets: a(AuthoritiesChange), b
 		impl_outer_log! {
-			pub enum Log(InternalLog: DigestItem<u64>) for Runtime {
+			pub enum Log(InternalLog: DigestItem<H256, u64>) for Runtime {
 				a(AuthoritiesChange), b()
 			}
 		}
@@ -418,16 +419,16 @@ mod tests {
 		assert_eq!(auth_change, decoded_auth_change);
 
 		// interpret regular item using `generic::DigestItem`
-		let generic_b1: generic::DigestItem<u64> = Decode::decode(&mut &encoded_b1[..]).unwrap();
+		let generic_b1: generic::DigestItem<H256, u64> = Decode::decode(&mut &encoded_b1[..]).unwrap();
 		match generic_b1 {
 			generic::DigestItem::Other(_) => (),
 			_ => panic!("unexpected generic_b1: {:?}", generic_b1),
 		}
 
 		// interpret system item using `generic::DigestItem`
-		let generic_auth_change: generic::DigestItem<u64> = Decode::decode(&mut &encoded_auth_change[..]).unwrap();
+		let generic_auth_change: generic::DigestItem<H256, u64> = Decode::decode(&mut &encoded_auth_change[..]).unwrap();
 		match generic_auth_change {
-			generic::DigestItem::AuthoritiesChange(authorities) => assert_eq!(authorities, vec![100, 200, 300]),
+			generic::DigestItem::AuthoritiesChange::<H256, u64>(authorities) => assert_eq!(authorities, vec![100, 200, 300]),
 			_ => panic!("unexpected generic_auth_change: {:?}", generic_auth_change),
 		}
 	}

substrate/core/sr-primitives/src/testing.rs

@@ -20,16 +20,20 @@ use serde::{Serialize, de::DeserializeOwned};
 use std::fmt::Debug;
 use codec::Codec;
 use traits::{self, Checkable, Applyable, BlakeTwo256};
+use generic::DigestItem as GenDigestItem;
 
 pub use substrate_primitives::H256;
 
+pub type DigestItem = GenDigestItem<H256, u64>;
+
 #[derive(Default, PartialEq, Eq, Clone, Serialize, Deserialize, Debug, Encode, Decode)]
 pub struct Digest {
-	pub logs: Vec<u64>,
+	pub logs: Vec<DigestItem>,
 }
 
 impl traits::Digest for Digest {
-	type Item = u64;
+	type Hash = H256;
+	type Item = DigestItem;
 
 	fn logs(&self) -> &[Self::Item] {
 		&self.logs
@@ -40,14 +44,6 @@ impl traits::Digest for Digest {
 	}
 }
 
-impl traits::DigestItem for () {
-	type AuthorityId = ();
-}
-
-impl traits::DigestItem for u64 {
-	type AuthorityId = ();
-}
-
 #[derive(PartialEq, Eq, Clone, Serialize, Deserialize, Debug, Encode, Decode)]
 #[serde(rename_all = "camelCase")]
 #[serde(deny_unknown_fields)]
@@ -88,7 +84,11 @@ impl traits::Header for Header {
 		digest: Self::Digest
 	) -> Self {
 		Header {
-			number, extrinsics_root: extrinsics_root, state_root, parent_hash, digest
+			number,
+			extrinsics_root: extrinsics_root,
+			state_root,
+			parent_hash,
+			digest
 		}
 	}
 }

substrate/core/sr-primitives/src/traits.rs

@@ -244,6 +244,9 @@ pub trait Hash: 'static + MaybeSerializeDebug + Clone + Eq + PartialEq {	// Stup
 
 	/// Acquire the global storage root.
 	fn storage_root() -> Self::Output;
+
+	/// Acquire the global storage changes root.
+	fn storage_changes_root(block: u64) -> Option<Self::Output>;
 }
 
 /// Blake2-256 Hash implementation.
@@ -275,6 +278,9 @@ impl Hash for BlakeTwo256 {
 	fn storage_root() -> Self::Output {
 		runtime_io::storage_root().into()
 	}
+	fn storage_changes_root(block: u64) -> Option<Self::Output> {
+		runtime_io::storage_changes_root(block).map(Into::into)
+	}
 }
 
 /// Something that can be checked for equality and printed out to a debug channel if bad.
@@ -343,7 +349,7 @@ pub trait Header: Clone + Send + Sync + Codec + Eq + MaybeSerializeDebug + 'stat
 	type Number: Member + ::rstd::hash::Hash + Copy + MaybeDisplay + SimpleArithmetic + Codec;
 	type Hash: Member + ::rstd::hash::Hash + Copy + MaybeDisplay + Default + SimpleBitOps + Codec + AsRef<[u8]>;
 	type Hashing: Hash<Output = Self::Hash>;
-	type Digest: Digest;
+	type Digest: Digest<Hash = Self::Hash>;
 
 	fn new(
 		number: Self::Number,
@@ -444,8 +450,12 @@ pub trait Applyable: Sized + Send + Sync {
 /// Something that acts like a `Digest` - it can have `Log`s `push`ed onto it and these `Log`s are
 /// each `Codec`.
 pub trait Digest: Member + Default {
-	type Item: DigestItem;
+	type Hash: Member;
+	type Item: DigestItem<Hash = Self::Hash>;
+
+	/// Get reference to all digest items.
 	fn logs(&self) -> &[Self::Item];
+	/// Push new digest item.
 	fn push(&mut self, item: Self::Item);
 }
 
@@ -454,10 +464,16 @@ pub trait Digest: Member + Default {
 ///
 /// If the runtime does not supports some 'system' items, use `()` as a stub.
 pub trait DigestItem: Member {
-	type AuthorityId;
+	type Hash: Member;
+	type AuthorityId: Member;
 
- 	/// Returns Some if the entry is the `AuthoritiesChange` entry.
+	/// Returns Some if the entry is the `AuthoritiesChange` entry.
 	fn as_authorities_change(&self) -> Option<&[Self::AuthorityId]> {
 		None
 	}
+
+	/// Returns Some if the entry is the `ChangesTrieRoot` entry.
+	fn as_changes_trie_root(&self) -> Option<&Self::Hash> {
+		None
+	}
 }