Cumulus changes version 2 (#2313)

* ensure imbalances are properly accounted for (#2183)

* ensure imbalances are properly accounted for

* bump runtime version

* Update node/runtime/src/lib.rs

* implement contract events (#2161)

* implement contract events

* update runtime

* renaming

* update test code hash

* improve complexity details

* add deposit event base cost

* add test

* Revert "add deposit event base cost"

This reverts commit 58ec010c0f4f4f0e16935ad41da32aedd17a8c57.

* update test

* Revert "update test"

This reverts commit 6fe61a593ccf0d41f09a0b97472b28ed8751a999.

* Revert "Revert "add deposit event base cost""

This reverts commit 145e8a9bac15313a4c380aa66b94fd4d36fa3f6d.

* Fix format a bit

*  Replace Vec<u8> with [u8; 32] for contract storage key (#2184)

* Replace Vec<u8> with [u8; 32] for contract storage key

* Read storage keys from sandbox memory into fixed size buffer

* Increment `impl_version`

* Remove redundant Ok(()) and explicitly specify StorageKey buffer type (#2188)

* Switch to `derive(Encode, Decode)` for `Call` (#2178)

* Add some tests

* More tests

* Switch to `derive(Encode, Decode)` for `Call`

* Update lock files

* Simplify the macro cases

* Cache changes trie config in db storage (#2170)

* cache changes trie config in db storage

* Update core/client/db/src/lib.rs

Co-Authored-By: svyatonik <svyatonik@gmail.com>

* Update core/client/db/src/lib.rs

Co-Authored-By: svyatonik <svyatonik@gmail.com>

* Fix version check for renamed runtime api methods (#2190)

* Add feature to disable including the test-runtime wasm blob

* Enable `std` feature for `consensus_authorities`

* Implement `skip_initialize_block` and `initialize_block` for runtime api

* Add test and fixes bug

* Begin to implement support for passing the `ProofRecorder`

* Make sure proof generation works as intended

* Fixes tests

* Make `BlockBuilder` generate proofs on request.

* Adds `TestClientBuilder` to simplify creating a test client

* Add `include-wasm-blob` to `test-client` as well

* Make `test-client` compile without including the wasm file

* Disable more stuff in test-client without wasm

* Reorganize the re-exports

* Use correct bounds

* Update docs

* Update core/client/src/block_builder/block_builder.rs

Co-Authored-By: bkchr <bkchr@users.noreply.github.com>

* Extend test to actually generated proof

* Switch to enum for `skip_initialize_block`

* Some wasm files updates

substrate/core/sr-api-macros/Cargo.toml

@@ -22,6 +22,8 @@ runtime_primitives = { package = "sr-primitives", path = "../sr-primitives" }
 sr-version = { path = "../sr-version" }
 substrate-primitives = { path = "../primitives" }
 criterion = "0.2"
+consensus_common = { package = "substrate-consensus-common", path = "../consensus/common" }
+codec = { package = "parity-codec", version = "3.5.1" }
 
 [[bench]]
 name = "bench"

substrate/core/sr-api-macros/src/decl_runtime_apis.rs

@@ -57,9 +57,21 @@ const CHANGED_IN_ATTRIBUTE: &str = "changed_in";
 ///
 /// Is used when a trait method was renamed.
 const RENAMED_ATTRIBUTE: &str = "renamed";
+/// The `skip_initialize_block` attribute.
+///
+/// Is used when a trait method does not require that the block is initialized
+/// before being called.
+const SKIP_INITIALIZE_BLOCK_ATTRIBUTE: &str = "skip_initialize_block";
+/// The `initialize_block` attribute.
+///
+/// A trait method tagged with this attribute, initializes the runtime at
+/// certain block.
+const INITIALIZE_BLOCK_ATTRIBUTE: &str = "initialize_block";
 /// All attributes that we support in the declaration of a runtime api trait.
 const SUPPORTED_ATTRIBUTE_NAMES: &[&str] = &[
-	CORE_TRAIT_ATTRIBUTE, API_VERSION_ATTRIBUTE, CHANGED_IN_ATTRIBUTE, RENAMED_ATTRIBUTE
+	CORE_TRAIT_ATTRIBUTE, API_VERSION_ATTRIBUTE, CHANGED_IN_ATTRIBUTE,
+	RENAMED_ATTRIBUTE, SKIP_INITIALIZE_BLOCK_ATTRIBUTE,
+	INITIALIZE_BLOCK_ATTRIBUTE,
 ];
 
 /// The structure used for parsing the runtime api declarations.
@@ -338,7 +350,12 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 
 		if attrs.contains_key(RENAMED_ATTRIBUTE) && attrs.contains_key(CHANGED_IN_ATTRIBUTE) {
 			return Err(Error::new(
-				fn_.span(), format!("`{}` and `{}` are not supported at once.", RENAMED_ATTRIBUTE, CHANGED_IN_ATTRIBUTE)
+				fn_.span(),
+				format!(
+					"`{}` and `{}` are not supported at once.",
+					RENAMED_ATTRIBUTE,
+					CHANGED_IN_ATTRIBUTE
+				)
 			));
 		}
 
@@ -347,6 +364,15 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 			continue;
 		}
 
+		let skip_initialize_block = attrs.contains_key(SKIP_INITIALIZE_BLOCK_ATTRIBUTE);
+		let update_initialized_block = if attrs.contains_key(INITIALIZE_BLOCK_ATTRIBUTE) {
+			quote!(
+				|| *initialized_block.borrow_mut() = Some(*at)
+			)
+		} else {
+			quote!(|| ())
+		};
+
 		// Parse the renamed attributes.
 		let mut renames = Vec::new();
 		if let Some((_, a)) = attrs
@@ -375,43 +401,63 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 				NC: FnOnce() -> ::std::result::Result<R, &'static str> + ::std::panic::UnwindSafe,
 				Block: #crate_::runtime_api::BlockT,
 				T: #crate_::runtime_api::CallRuntimeAt<Block>,
+				C: #crate_::runtime_api::Core<Block>,
 			>(
 				call_runtime_at: &T,
+				core_api: &C,
 				at: &#crate_::runtime_api::BlockId<Block>,
 				args: Vec<u8>,
-				changes: &mut #crate_::runtime_api::OverlayedChanges,
-				initialized_block: &mut Option<#crate_::runtime_api::BlockId<Block>>,
+				changes: &std::cell::RefCell<#crate_::runtime_api::OverlayedChanges>,
+				initialized_block: &std::cell::RefCell<Option<#crate_::runtime_api::BlockId<Block>>>,
 				native_call: Option<NC>,
 				context: #crate_::runtime_api::ExecutionContext,
+				recorder: &Option<std::rc::Rc<std::cell::RefCell<#crate_::runtime_api::ProofRecorder<Block>>>>,
 			) -> #crate_::error::Result<#crate_::runtime_api::NativeOrEncoded<R>> {
 				let version = call_runtime_at.runtime_version_at(at)?;
+				use #crate_::runtime_api::InitializeBlock;
+				let initialize_block = if #skip_initialize_block {
+					InitializeBlock::Skip
+				} else {
+					InitializeBlock::Do(&initialized_block)
+				};
+				let update_initialized_block = #update_initialized_block;
 
 				#(
 					// Check if we need to call the function by an old name.
 					if version.apis.iter().any(|(s, v)| {
 						s == &ID && *v < #versions
 					}) {
-						return call_runtime_at.call_api_at::<R, fn() -> _>(
+						let ret = call_runtime_at.call_api_at::<R, fn() -> _, _>(
+							core_api,
 							at,
 							#old_names,
 							args,
 							changes,
-							initialized_block,
+							initialize_block,
 							None,
-							context
-						);
+							context,
+							recorder,
+						)?;
+
+						update_initialized_block();
+						return Ok(ret);
 					}
 				)*
 
-				call_runtime_at.call_api_at(
+				let ret = call_runtime_at.call_api_at(
+					core_api,
 					at,
 					#trait_fn_name,
 					args,
 					changes,
-					initialized_block,
+					initialize_block,
 					native_call,
-					context
-				)
+					context,
+					recorder,
+				)?;
+
+				update_initialized_block();
+				Ok(ret)
 			}
 		));
 	}

substrate/core/sr-api-macros/src/impl_runtime_apis.rs

@@ -263,9 +263,10 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result<TokenStrea
 		#[cfg(any(feature = "std", test))]
 		pub struct RuntimeApiImpl<C: #crate_::runtime_api::CallRuntimeAt<#block> + 'static> {
 			call: &'static C,
-			commit_on_success: ::std::cell::RefCell<bool>,
-			initialized_block: ::std::cell::RefCell<Option<#block_id>>,
-			changes: ::std::cell::RefCell<#crate_::runtime_api::OverlayedChanges>,
+			commit_on_success: std::cell::RefCell<bool>,
+			initialized_block: std::cell::RefCell<Option<#block_id>>,
+			changes: std::cell::RefCell<#crate_::runtime_api::OverlayedChanges>,
+			recorder: Option<std::rc::Rc<std::cell::RefCell<#crate_::runtime_api::ProofRecorder<#block>>>>,
 		}
 
 		// `RuntimeApi` itself is not threadsafe. However, an instance is only available in a
@@ -299,6 +300,22 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result<TokenStrea
 			) -> #crate_::error::Result<#crate_::runtime_api::RuntimeVersion> {
 				self.call.runtime_version_at(at)
 			}
+
+			fn record_proof(&mut self) {
+				self.recorder = Some(Default::default());
+			}
+
+			fn extract_proof(&mut self) -> Option<Vec<Vec<u8>>> {
+				self.recorder
+					.take()
+					.map(|r| {
+						r.borrow_mut()
+							.drain()
+							.into_iter()
+							.map(|n| n.data.to_vec())
+							.collect()
+					})
+			}
 		}
 
 		#[cfg(any(feature = "std", test))]
@@ -315,6 +332,7 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result<TokenStrea
 					commit_on_success: true.into(),
 					initialized_block: None.into(),
 					changes: Default::default(),
+					recorder: Default::default(),
 				}.into()
 			}
 		}
@@ -325,9 +343,11 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result<TokenStrea
 				R: #crate_::runtime_api::Encode + #crate_::runtime_api::Decode + PartialEq,
 				F: FnOnce(
 					&C,
-					&mut #crate_::runtime_api::OverlayedChanges,
-					&mut Option<#crate_::runtime_api::BlockId<#block>>,
-				) -> #crate_::error::Result<#crate_::runtime_api::NativeOrEncoded<R>>
+					&Self,
+					&std::cell::RefCell<#crate_::runtime_api::OverlayedChanges>,
+					&std::cell::RefCell<Option<#crate_::runtime_api::BlockId<#block>>>,
+					&Option<std::rc::Rc<std::cell::RefCell<#crate_::runtime_api::ProofRecorder<#block>>>>,
+				) -> #crate_::error::Result<#crate_::runtime_api::NativeOrEncoded<R>>,
 			>(
 				&self,
 				call_api_at: F,
@@ -335,8 +355,10 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result<TokenStrea
 				let res = unsafe {
 					call_api_at(
 						&self.call,
-						&mut *self.changes.borrow_mut(),
-						&mut *self.initialized_block.borrow_mut()
+						self,
+						&self.changes,
+						&self.initialized_block,
+						&self.recorder,
 					)
 				};
 
@@ -479,9 +501,10 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 					#( #error )*
 
 					self.call_api_at(
-						|call_runtime_at, changes, initialized_block| {
+						|call_runtime_at, core_api, changes, initialized_block, recorder| {
 							#runtime_mod_path #call_api_at_call(
 								call_runtime_at,
+								core_api,
 								at,
 								params_encoded,
 								changes,
@@ -493,6 +516,7 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 									)
 								}),
 								context,
+								recorder,
 							)
 						}
 					)

substrate/core/sr-api-macros/tests/runtime_calls.rs

@@ -14,9 +14,20 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
-use test_client::runtime::{TestAPI, DecodeFails};
-use runtime_primitives::{generic::BlockId, traits::ProvideRuntimeApi};
-use state_machine::ExecutionStrategy;
+use test_client::{
+	AccountKeyring, runtime::{TestAPI, DecodeFails, Transfer, Header},
+	NativeExecutor, LocalExecutor,
+};
+use runtime_primitives::{
+	generic::BlockId,
+	traits::{ProvideRuntimeApi, Header as HeaderT, Hash as HashT},
+};
+use state_machine::{
+	ExecutionStrategy, create_proof_check_backend,
+	execution_proof_check_on_trie_backend,
+};
+
+use codec::Encode;
 
 fn calling_function_with_strat(strat: ExecutionStrategy) {
 	let client = test_client::new_with_execution_strategy(strat);
@@ -114,3 +125,66 @@ fn use_trie_function() {
 	let block_id = BlockId::Number(client.info().unwrap().chain.best_number);
 	assert_eq!(runtime_api.use_trie(&block_id).unwrap(), 2);
 }
+
+#[test]
+fn initialize_block_works() {
+	let client = test_client::new_with_execution_strategy(ExecutionStrategy::Both);
+	let runtime_api = client.runtime_api();
+	let block_id = BlockId::Number(client.info().unwrap().chain.best_number);
+	assert_eq!(runtime_api.get_block_number(&block_id).unwrap(), 1);
+}
+
+#[test]
+fn initialize_block_is_called_only_once() {
+	let client = test_client::new_with_execution_strategy(ExecutionStrategy::Both);
+	let runtime_api = client.runtime_api();
+	let block_id = BlockId::Number(client.info().unwrap().chain.best_number);
+	assert_eq!(runtime_api.take_block_number(&block_id).unwrap(), Some(1));
+	assert_eq!(runtime_api.take_block_number(&block_id).unwrap(), None);
+}
+
+#[test]
+fn initialize_block_is_skipped() {
+	let client = test_client::new_with_execution_strategy(ExecutionStrategy::Both);
+	let runtime_api = client.runtime_api();
+	let block_id = BlockId::Number(client.info().unwrap().chain.best_number);
+	assert!(runtime_api.without_initialize_block(&block_id).unwrap());
+}
+
+#[test]
+fn record_proof_works() {
+	let client = test_client::new_with_execution_strategy(ExecutionStrategy::Both);
+
+	let block_id = BlockId::Number(client.info().unwrap().chain.best_number);
+	let storage_root = client.best_block_header().unwrap().state_root().clone();
+
+	let transaction = Transfer {
+		amount: 1000,
+		nonce: 0,
+		from: AccountKeyring::Alice.into(),
+		to: Default::default(),
+	}.into_signed_tx();
+
+	// Build the block and record proof
+	let mut builder = client
+		.new_block_at_with_proof_recording(&block_id)
+		.expect("Creates block builder");
+	builder.push(transaction.clone()).unwrap();
+	let (block, proof) = builder.bake_and_extract_proof().expect("Bake block");
+
+	let backend = create_proof_check_backend::<<<Header as HeaderT>::Hashing as HashT>::Hasher>(
+		storage_root,
+		proof.expect("Proof was generated"),
+	).expect("Creates proof backend.");
+
+	// Use the proof backend to execute `execute_block`.
+	let mut overlay = Default::default();
+	let executor = NativeExecutor::<LocalExecutor>::new(None);
+	execution_proof_check_on_trie_backend(
+		&backend,
+		&mut overlay,
+		&executor,
+		"Core_execute_block",
+		&block.encode(),
+	).expect("Executes block while using the proof backend");
+}