Use wasm-builder from git (#3354)

* Use wasm-builder from git

This brings new features like compressed runtimes out of the box.

* chore: update wasm builder dep. from 3.0.0 to master

* Fix tests

* Update node/core/pvf/tests/it/main.rs

Co-authored-by: Sergei Shulepov <sergei@parity.io>

Co-authored-by: chevdor <chevdor@gmail.com>
Co-authored-by: Sergei Shulepov <sergei@parity.io>

polkadot/Cargo.lock

@@ -891,6 +891,15 @@ version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba"
 
+[[package]]
+name = "camino"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4648c6d00a709aa069a236adcaae4f605a6241c72bf5bee79331a4b625921a9"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "cargo-platform"
 version = "0.1.1"
@@ -902,10 +911,11 @@ dependencies = [
 
 [[package]]
 name = "cargo_metadata"
-version = "0.12.3"
+version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7714a157da7991e23d90686b9524b9e12e0407a108647f52e9328f4b3d51ac7f"
+checksum = "081e3f0755c1f380c2d010481b6fa2e02973586d5f2b24eebb7a2a1d98b143d8"
 dependencies = [
+ "camino",
  "cargo-platform",
  "semver 0.11.0",
  "semver-parser 0.10.2",
@@ -6217,6 +6227,7 @@ dependencies = [
  "sp-core",
  "sp-externalities",
  "sp-io",
+ "sp-maybe-compressed-blob",
  "sp-wasm-interface",
  "tempfile",
  "test-parachain-adder",
@@ -10226,14 +10237,14 @@ dependencies = [
 
 [[package]]
 name = "substrate-wasm-builder"
-version = "3.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79091baab813855ddf65b191de9fe53e656b6b67c1e9bd23fdcbff8788164684"
+version = "4.0.0"
+source = "git+https://github.com/paritytech/substrate?branch=master#01ff4cef6626448998a3bcbc5be401dc15a394cf"
 dependencies = [
  "ansi_term 0.12.1",
  "atty",
  "build-helper",
  "cargo_metadata",
+ "sp-maybe-compressed-blob",
  "tempfile",
  "toml",
  "walkdir",

polkadot/node/core/pvf/Cargo.toml

@@ -30,6 +30,7 @@ sp-externalities = { git = "https://github.com/paritytech/substrate", branch = "
 sp-io = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-wasm-interface = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-maybe-compressed-blob = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
 [dev-dependencies]
 adder = { package = "test-parachain-adder", path = "../../../parachain/test-parachains/adder" }

polkadot/node/core/pvf/src/testing.rs

@@ -31,7 +31,9 @@ pub fn validate_candidate(
 ) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
 	use crate::executor_intf::{prevalidate, prepare, execute, TaskExecutor};
 
-	let blob = prevalidate(code)?;
+	let code = sp_maybe_compressed_blob::decompress(code, 10 * 1024 * 1024).expect("Decompressing code failed");
+
+	let blob = prevalidate(&*code)?;
 	let artifact = prepare(blob)?;
 	let executor = TaskExecutor::new()?;
 	let result = unsafe {

polkadot/node/core/pvf/tests/it/main.rs

@@ -15,9 +15,7 @@
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
 use polkadot_node_core_pvf::{Pvf, ValidationHost, start, Config, InvalidCandidate, ValidationError};
-use polkadot_parachain::{
-	primitives::{BlockData, ValidationParams, ValidationResult},
-};
+use polkadot_parachain::primitives::{BlockData, ValidationParams, ValidationResult};
 use parity_scale_codec::Encode as _;
 use async_std::sync::Mutex;
 
@@ -58,11 +56,14 @@ impl TestHost {
 		params: ValidationParams,
 	) -> Result<ValidationResult, ValidationError> {
 		let (result_tx, result_rx) = futures::channel::oneshot::channel();
+
+		let code = sp_maybe_compressed_blob::decompress(code, 16 * 1024 * 1024).expect("Compression works");
+
 		self.host
 			.lock()
 			.await
 			.execute_pvf(
-				Pvf::from_code(code.to_vec()),
+				Pvf::from_code(code.into()),
 				params.encode(),
 				polkadot_node_core_pvf::Priority::Normal,
 				result_tx,

polkadot/parachain/test-parachains/adder/Cargo.toml

@@ -17,7 +17,7 @@ dlmalloc = { version = "0.2.1", features = [ "global" ] }
 sp-io = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, features = [ "disable_allocator" ] }
 
 [build-dependencies]
-substrate-wasm-builder = "3.0.0"
+substrate-wasm-builder = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
 [features]
 default = [ "std" ]

polkadot/parachain/test-parachains/halt/Cargo.toml

@@ -9,7 +9,7 @@ build = "build.rs"
 [dependencies]
 
 [build-dependencies]
-substrate-wasm-builder = "3.0.0"
+substrate-wasm-builder = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
 [features]
 default = [ "std" ]

polkadot/runtime/kusama/Cargo.toml

@@ -100,7 +100,7 @@ separator = "0.4.1"
 serde_json = "1.0.61"
 
 [build-dependencies]
-substrate-wasm-builder = "3.0.0"
+substrate-wasm-builder = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
 [features]
 default = ["std"]

polkadot/runtime/polkadot/Cargo.toml

@@ -91,7 +91,7 @@ serde_json = "1.0.61"
 separator = "0.4.1"
 
 [build-dependencies]
-substrate-wasm-builder = "3.0.0"
+substrate-wasm-builder = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
 [features]
 default = ["std"]

polkadot/runtime/rococo/Cargo.toml

@@ -76,7 +76,7 @@ pallet-bridge-grandpa = { path = "../../bridges/modules/grandpa", default-featur
 max-encoded-len = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 
 [build-dependencies]
-substrate-wasm-builder = "3.0.0"
+substrate-wasm-builder = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
 [features]
 default = ["std"]

polkadot/runtime/test-runtime/Cargo.toml

@@ -67,7 +67,7 @@ sp-trie = { git = "https://github.com/paritytech/substrate", branch = "master" }
 serde_json = "1.0.61"
 
 [build-dependencies]
-substrate-wasm-builder = "3.0.0"
+substrate-wasm-builder = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
 [features]
 default = ["std"]

polkadot/runtime/westend/Cargo.toml

@@ -97,7 +97,7 @@ sp-trie = { git = "https://github.com/paritytech/substrate", branch = "master" }
 serde_json = "1.0.61"
 
 [build-dependencies]
-substrate-wasm-builder = "3.0.0"
+substrate-wasm-builder = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
 [features]
 default = ["std"]