Transaction versioning in the RuntimeVersion (#5582)

* Add transaction_version

* Semantic versioning for runtimes

* Move new field to bottom

* Versioning

* Runtime versioning stuff.

* Fix test

* Adds tests and fixes bugs

* Bump runtime

Co-authored-by: Bastian Köcher <git@kchr.de>

substrate/Cargo.lock

@@ -6218,6 +6218,7 @@ dependencies = [
  "sc-executor-wasmi",
  "sc-executor-wasmtime",
  "sc-runtime-test",
+ "sp-api",
  "sp-core",
  "sp-externalities",
  "sp-io",

substrate/bin/node-template/runtime/src/lib.rs

@@ -98,6 +98,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
 	spec_version: 1,
 	impl_version: 1,
 	apis: RUNTIME_API_VERSIONS,
+	transaction_version: 1,
 };
 
 pub const MILLISECS_PER_BLOCK: u64 = 6000;

substrate/bin/node/runtime/src/lib.rs

@@ -130,6 +130,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
 	spec_version: 243,
 	impl_version: 0,
 	apis: RUNTIME_API_VERSIONS,
+	transaction_version: 1,
 };
 
 /// Native version.

substrate/client/executor/Cargo.toml

@@ -24,6 +24,7 @@ sp-panic-handler = { version = "2.0.0-dev", path = "../../primitives/panic-handl
 wasmi = "0.6.2"
 parity-wasm = "0.41.0"
 lazy_static = "1.4.0"
+sp-api = { version = "2.0.0-dev", path = "../../primitives/api" }
 sp-wasm-interface = { version = "2.0.0-dev", path = "../../primitives/wasm-interface" }
 sp-runtime-interface = { version = "2.0.0-dev", path = "../../primitives/runtime-interface" }
 sp-externalities = { version = "0.8.0-dev", path = "../../primitives/externalities" }

substrate/client/executor/src/wasm_runtime.rs

@@ -287,6 +287,25 @@ pub fn create_wasm_runtime_with_code(
 	}
 }
 
+fn decode_version(version: &[u8]) -> Result<RuntimeVersion, WasmError> {
+	let v: RuntimeVersion = sp_api::OldRuntimeVersion::decode(&mut &version[..])
+		.map_err(|_|
+				 WasmError::Instantiation(
+					 "failed to decode \"Core_version\" result using old runtime version".into(),
+				 )
+		)?.into();
+
+	let core_api_id = sp_core::hashing::blake2_64(b"Core");
+	if v.has_api_with(&core_api_id, |v| v >= 3) {
+		sp_api::RuntimeVersion::decode(&mut &version[..])
+			.map_err(|_|
+				WasmError::Instantiation("failed to decode \"Core_version\" result".into())
+			)
+	} else {
+		Ok(v)
+	}
+}
+
 fn create_versioned_wasm_runtime(
 	code: &[u8],
 	code_hash: Vec<u8>,
@@ -321,10 +340,7 @@ fn create_versioned_wasm_runtime(
 		).map_err(|_| WasmError::Instantiation("panic in call to get runtime version".into()))?
 	};
 	let version = match version_result {
-		Ok(version) => Some(RuntimeVersion::decode(&mut version.as_slice())
+		Ok(version) => Some(decode_version(&version)?),
-			.map_err(|_|
-				WasmError::Instantiation("failed to decode \"Core_version\" result".into())
-			)?),
 		Err(_) => None,
 	};
 	#[cfg(not(target_os = "unknown"))]
@@ -350,7 +366,11 @@ fn create_versioned_wasm_runtime(
 
 #[cfg(test)]
 mod tests {
+	use super::*;
 	use sp_wasm_interface::HostFunctions;
+	use sp_api::{Core, RuntimeApiInfo};
+	use substrate_test_runtime::Block;
+	use codec::Encode;
 
 	#[test]
 	fn host_functions_are_equal() {
@@ -359,4 +379,49 @@ mod tests {
 		let equal = &host_functions[..] == &host_functions[..];
 		assert!(equal, "Host functions are not equal");
 	}
+
+	#[test]
+	fn old_runtime_version_decodes() {
+		let old_runtime_version = sp_api::OldRuntimeVersion {
+			spec_name: "test".into(),
+			impl_name: "test".into(),
+			authoring_version: 1,
+			spec_version: 1,
+			impl_version: 1,
+			apis: sp_api::create_apis_vec!([(Core::<Block, Error = ()>::ID, 1)]),
+		};
+
+		let version = decode_version(&old_runtime_version.encode()).unwrap();
+		assert_eq!(1, version.transaction_version);
+	}
+
+	#[test]
+	fn old_runtime_version_decodes_fails_with_version_3() {
+		let old_runtime_version = sp_api::OldRuntimeVersion {
+			spec_name: "test".into(),
+			impl_name: "test".into(),
+			authoring_version: 1,
+			spec_version: 1,
+			impl_version: 1,
+			apis: sp_api::create_apis_vec!([(Core::<Block, Error = ()>::ID, 3)]),
+		};
+
+		decode_version(&old_runtime_version.encode()).unwrap_err();
+	}
+
+	#[test]
+	fn new_runtime_version_decodes() {
+		let old_runtime_version = sp_api::RuntimeVersion {
+			spec_name: "test".into(),
+			impl_name: "test".into(),
+			authoring_version: 1,
+			spec_version: 1,
+			impl_version: 1,
+			apis: sp_api::create_apis_vec!([(Core::<Block, Error = ()>::ID, 3)]),
+			transaction_version: 3,
+		};
+
+		let version = decode_version(&old_runtime_version.encode()).unwrap();
+		assert_eq!(3, version.transaction_version);
+	}
 }

substrate/client/rpc/src/state/tests.rs

@@ -437,10 +437,11 @@ fn should_return_runtime_version() {
 	let api = new_full(client.clone(), Subscriptions::new(Arc::new(core.executor())));
 
 	let result = "{\"specName\":\"test\",\"implName\":\"parity-test\",\"authoringVersion\":1,\
-		\"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",2],\
+		\"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",3],\
 		[\"0x37e397fc7c91f5e4\",1],[\"0xd2bc9897eed08f15\",2],[\"0x40fe3ad401f8959a\",4],\
 		[\"0xc6e9a76309f39b09\",1],[\"0xdd718d5cc53262d4\",1],[\"0xcbca25e39f142387\",1],\
-		[\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],[\"0xbc9d89904f5b923f\",1]]}";
+		[\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],[\"0xbc9d89904f5b923f\",1]],\
+		\"transactionVersion\":1}";
 
 	let runtime_version = api.runtime_version(None.into()).wait().unwrap();
 	let serialized = serde_json::to_string(&runtime_version).unwrap();

substrate/frame/system/src/lib.rs

@@ -1617,6 +1617,7 @@ mod tests {
 			spec_version: 1,
 			impl_version: 1,
 			apis: sp_version::create_apis_vec!([]),
+			transaction_version: 1,
 		};
 	}
 

substrate/primitives/api/src/lib.rs

@@ -53,7 +53,7 @@ pub use sp_runtime::{
 		Block as BlockT, GetNodeBlockType, GetRuntimeBlockType, HashFor, NumberFor,
 		Header as HeaderT, Hash as HashT,
 	},
-	generic::BlockId, transaction_validity::TransactionValidity,
+	generic::BlockId, transaction_validity::TransactionValidity, RuntimeString,
 };
 #[doc(hidden)]
 pub use sp_core::{offchain, ExecutionContext};
@@ -224,6 +224,7 @@ pub use sp_api_proc_macro::decl_runtime_apis;
 ///     impl_version: 0,
 ///     // Here we are exposing the runtime api versions.
 ///     apis: RUNTIME_API_VERSIONS,
+///     transaction_version: 1,
 /// };
 ///
 /// # fn main() {}
@@ -520,13 +521,53 @@ pub trait RuntimeApiInfo {
 #[cfg(feature = "std")]
 pub type ApiErrorFor<T, Block> = <<T as ProvideRuntimeApi<Block>>::Api as ApiErrorExt>::Error;
 
+#[derive(codec::Encode, codec::Decode)]
+pub struct OldRuntimeVersion {
+	pub spec_name: RuntimeString,
+	pub impl_name: RuntimeString,
+	pub authoring_version: u32,
+	pub spec_version: u32,
+	pub impl_version: u32,
+	pub apis: ApisVec,
+}
+
+impl From<OldRuntimeVersion> for RuntimeVersion {
+	fn from(x: OldRuntimeVersion) -> Self {
+		Self {
+			spec_name: x.spec_name,
+			impl_name: x.impl_name,
+			authoring_version: x.authoring_version,
+			spec_version: x.spec_version,
+			impl_version: x.impl_version,
+			apis: x.apis,
+			transaction_version: 1,
+		}
+	}
+}
+
+impl From<RuntimeVersion> for OldRuntimeVersion {
+	fn from(x: RuntimeVersion) -> Self {
+		Self {
+			spec_name: x.spec_name,
+			impl_name: x.impl_name,
+			authoring_version: x.authoring_version,
+			spec_version: x.spec_version,
+			impl_version: x.impl_version,
+			apis: x.apis,
+		}
+	}
+}
+
 decl_runtime_apis! {
 	/// The `Core` runtime api that every Substrate runtime needs to implement.
 	#[core_trait]
-	#[api_version(2)]
+	#[api_version(3)]
 	pub trait Core {
 		/// Returns the version of the runtime.
 		fn version() -> RuntimeVersion;
+		/// Returns the version of the runtime.
+		#[changed_in(3)]
+		fn version() -> OldRuntimeVersion;
 		/// Execute the given block.
 		#[skip_initialize_block]
 		fn execute_block(block: Block);

substrate/primitives/core/src/hashing.rs

@@ -57,6 +57,18 @@ pub fn blake2_128(data: &[u8]) -> [u8; 16] {
 	r
 }
 
+/// Do a Blake2 64-bit hash and place result in `dest`.
+pub fn blake2_64_into(data: &[u8], dest: &mut [u8; 8]) {
+	dest.copy_from_slice(blake2_rfc::blake2b::blake2b(8, &[], data).as_bytes());
+}
+
+/// Do a Blake2 64-bit hash and return result.
+pub fn blake2_64(data: &[u8]) -> [u8; 8] {
+	let mut r = [0; 8];
+	blake2_64_into(data, &mut r);
+	r
+}
+
 /// Do a XX 64-bit hash and place result in `dest`.
 pub fn twox_64_into(data: &[u8], dest: &mut [u8; 8]) {
 	use ::core::hash::Hasher;

substrate/primitives/sr-api/proc-macro/src/lib.rs

@@ -103,6 +103,7 @@ mod utils;
 ///     impl_version: 0,
 ///     // Here we are exposing the runtime api versions.
 ///     apis: RUNTIME_API_VERSIONS,
+///     transaction_version: 1,
 /// };
 ///
 /// # fn main() {}

substrate/primitives/version/src/lib.rs

@@ -93,17 +93,29 @@ pub struct RuntimeVersion {
 		)
 	)]
 	pub apis: ApisVec,
+
+	/// All existing dispatches are fully compatible when this number doesn't change. If this
+	/// number changes, then `spec_version` must change, also.
+	///
+	/// This number must change when an existing dispatchable (module ID, dispatch ID) is changed,
+	/// either through an alteration in its user-level semantics, a parameter added/removed/changed,
+	/// a dispatchable being removed, a module being removed, or a dispatchable/module changing its
+	/// index.
+	///
+	/// It need *not* change when a new module is added or when a dispatchable is added.
+	pub transaction_version: u32,
 }
 
 #[cfg(feature = "std")]
 impl fmt::Display for RuntimeVersion {
 	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-		write!(f, "{}-{}:{}({}-{})",
+		write!(f, "{}-{} ({}-{}.tx{}.au{})",
 			self.spec_name,
 			self.spec_version,
-			self.authoring_version,
 			self.impl_name,
-			self.impl_version
+			self.impl_version,
+			self.transaction_version,
+			self.authoring_version,
 		)
 	}
 }

substrate/test-utils/runtime/src/lib.rs

@@ -68,6 +68,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
 	spec_version: 2,
 	impl_version: 2,
 	apis: RUNTIME_API_VERSIONS,
+	transaction_version: 1,
 };
 
 fn version() -> RuntimeVersion {