Build WASM binaries as part of cargo build (#2868)

* Introduce `wasm-builder` and `wasm-builder-runner` to retire `build.sh`

Make use of `wasm-builder` in `test-runtime`.

* Add build script and remove the wasm project

* Port `node-runtime` to new wasm-builder

* Make `substrate-executor` tests work with `wasm-builder`

* Move `node-template` to `wasm-builder`

* Remove `build.sh` :)

* Remove the last include_bytes

* Adds the missing build.rs files

* Remove `build.sh` from CI

* Debug CI

* Make it work in CI

* CI attempt 3

* Make `substrate-runtime-test` compile on stable

* Ahhh, some missed `include_bytes!`

* AHH

* Add suggestions

* Improve search for `Cargo.lock` and don't panic if it is not found

* Searching from manifest path was no good idea

* Make the `wasm-builder` source better configurable

* Expose the bloaty wasm binary as well

* Make sure to rerun WASM recompilation on changes in dependencies

* Introduce new `WASM_BUILD_TYPE` env and make sure to call `build.rs` on
changes to env variables

* Remove `build.sh` from READMEs

* Rename the projects

* Fixes CI

* Update lock file

* Fixes merge-conflict

* Apply suggestions from code review

Co-Authored-By: TriplEight <denis.pisarev@parity.io>

* Try to make windows happy

* Replace all back slashes in paths with slashes

* Apply suggestions from code review

Co-Authored-By: Pierre Krieger <pierre.krieger1708@gmail.com>

* Use cargo from `CARGO` env variable

* Fix compilation

* Use `rustup` for running the nightly build

* Make individual projects skipable

* Fix compilation

* Fixes compilation

* Build all WASM projects in one workspace

* Replace more back slashes!

* Remove `inlcude_bytes!`

* Adds some documentation

* Apply suggestions from code review

Co-Authored-By: Shawn Tabrizi <shawntabrizi@gmail.com>

* Apply suggestions from code review

Co-Authored-By: Shawn Tabrizi <shawntabrizi@gmail.com>

* More review comments

* Update `Cargo.lock`

* Set license

* Apply suggestions from code review

Co-Authored-By: joe petrowski <25483142+joepetrowski@users.noreply.github.com>

* More review comments + adds `TRIGGER_WASM_BUILD` env

* Fix doc tests

* Increase version + update README

* Switch crates.io version of `wasm-builder`

* Update README

* Switch to released version of `wasm-builder-runner`

substrate/core/test-runtime/Cargo.toml

@@ -3,6 +3,7 @@ name = "substrate-test-runtime"
 version = "2.0.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 edition = "2018"
+build = "build.rs"
 
 [dependencies]
 log = { version = "0.4", optional = true }
@@ -29,11 +30,14 @@ cfg-if = "0.1.6"
 substrate-executor = { path = "../executor" }
 substrate-test-runtime-client = { path = "./client" }
 
+[build-dependencies]
+wasm-builder-runner = { package = "substrate-wasm-builder-runner", version = "1.0.0" }
+
 [features]
 default = [
 	"std",
-	"include-wasm-blob"
 ]
+no_std = []
 std = [
 	"log",
 	"serde",
@@ -55,5 +59,3 @@ std = [
 	"offchain-primitives/std",
 	"executive/std",
 ]
-# If enabled, the WASM blob is added to the `GenesisConfig`.
-include-wasm-blob = []

substrate/core/test-runtime/build.rs

@@ -1,4 +1,4 @@
-// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// Copyright 2019 Parity Technologies (UK) Ltd.
 // This file is part of Substrate.
 
 // Substrate is free software: you can redistribute it and/or modify
@@ -14,8 +14,8 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
-//! The Substrate test runtime reexported for WebAssembly compile.
+use wasm_builder_runner::{build_current_project, WasmBuilderSource};
 
-#![cfg_attr(not(feature = "std"), no_std)]
-
-pub use substrate_test_runtime::*;
+fn main() {
+	build_current_project("wasm_binary.rs", WasmBuilderSource::Crates("1.0.1"));
+}

substrate/core/test-runtime/client/Cargo.toml

@@ -13,7 +13,6 @@ runtime_primitives = { package = "sr-primitives", path = "../../sr-primitives" }
 [features]
 default = [
 	"std",
-	"runtime/include-wasm-blob",
 ]
 std = [
 	"runtime/std",

substrate/core/test-runtime/client/src/lib.rs

@@ -51,7 +51,7 @@ mod local_executor {
 		pub LocalExecutor,
 		runtime::api::dispatch,
 		runtime::native_version,
-		include_bytes!("../../wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.compact.wasm")
+		runtime::WASM_BINARY
 	);
 }
 

substrate/core/test-runtime/src/genesismap.rs

@@ -18,7 +18,7 @@
 
 use std::collections::HashMap;
 use runtime_io::{blake2_256, twox_128};
-use super::{AuthorityId, AccountId};
+use super::{AuthorityId, AccountId, WASM_BINARY};
 use parity_codec::{Encode, KeyedVec, Joiner};
 use primitives::{ChangesTrieConfiguration, map, storage::well_known_keys};
 use runtime_primitives::traits::Block;
@@ -48,13 +48,11 @@ impl GenesisConfig {
 	}
 
 	pub fn genesis_map(&self) -> HashMap<Vec<u8>, Vec<u8>> {
-		#[cfg(feature = "include-wasm-blob")]
-		let wasm_runtime = include_bytes!("../wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.compact.wasm").to_vec();
+		let wasm_runtime = WASM_BINARY.to_vec();
 		let mut map: HashMap<Vec<u8>, Vec<u8>> = self.balances.iter()
 			.map(|&(ref account, balance)| (account.to_keyed_vec(b"balance:"), vec![].and(&balance)))
 			.map(|(k, v)| (blake2_256(&k[..])[..].to_vec(), v.to_vec()))
 			.chain(vec![
-				#[cfg(feature = "include-wasm-blob")]
 				(well_known_keys::CODE.into(), wasm_runtime),
 				(well_known_keys::HEAP_PAGES.into(), vec![].and(&(16 as u64))),
 			].into_iter())

substrate/core/test-runtime/src/lib.rs

@@ -56,6 +56,10 @@ pub type AuraId = AuthorityId;
 // Ensure Babe and Aura use the same crypto to simplify things a bit.
 pub type BabeId = AuthorityId;
 
+// Inlucde the WASM binary
+#[cfg(feature = "std")]
+include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs"));
+
 /// Test runtime version.
 pub const VERSION: RuntimeVersion = RuntimeVersion {
 	spec_name: create_runtime_str!("test"),

substrate/core/test-runtime/src/system.rs

@@ -313,13 +313,10 @@ mod tests {
 
 	use runtime_io::{with_externalities, TestExternalities};
 	use substrate_test_runtime_client::{AuthorityKeyring, AccountKeyring};
-	use crate::{Header, Transfer};
+	use crate::{Header, Transfer, WASM_BINARY};
 	use primitives::{Blake2Hasher, map};
 	use substrate_executor::WasmExecutor;
 
-	const WASM_CODE: &'static [u8] =
-			include_bytes!("../wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.compact.wasm");
-
 	fn new_test_ext() -> TestExternalities<Blake2Hasher> {
 		let authorities = vec![
 			AuthorityKeyring::Alice.to_raw_public(),
@@ -365,7 +362,7 @@ mod tests {
 	#[test]
 	fn block_import_works_wasm() {
 		block_import_works(|b, ext| {
-			WasmExecutor::new().call(ext, 8, &WASM_CODE, "Core_execute_block", &b.encode()).unwrap();
+			WasmExecutor::new().call(ext, 8, &WASM_BINARY, "Core_execute_block", &b.encode()).unwrap();
 		})
 	}
 
@@ -453,7 +450,7 @@ mod tests {
 	#[test]
 	fn block_import_with_transaction_works_wasm() {
 		block_import_with_transaction_works(|b, ext| {
-			WasmExecutor::new().call(ext, 8, &WASM_CODE, "Core_execute_block", &b.encode()).unwrap();
+			WasmExecutor::new().call(ext, 8, &WASM_BINARY, "Core_execute_block", &b.encode()).unwrap();
 		})
 	}
 }

substrate/core/test-runtime/wasm/Cargo.toml

@@ -1,25 +0,0 @@
-[package]
-name = "substrate-test-runtime-wasm"
-version = "2.0.0"
-authors = ["Parity Technologies <admin@parity.io>"]
-edition = "2018"
-
-[lib]
-name = "substrate_test_runtime"
-crate-type = ["cdylib"]
-
-[dependencies]
-substrate-test-runtime = { path = "..", default-features = false }
-
-[features]
-default = []
-std = [
-	"substrate-test-runtime/std",
-]
-
-[profile.release]
-panic = "abort"
-lto = true
-
-[workspace]
-members = []

substrate/core/test-runtime/wasm/build.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-if cargo --version | grep -q "nightly"; then
-	CARGO_CMD="cargo"
-else
-	CARGO_CMD="cargo +nightly"
-fi
-CARGO_INCREMENTAL=0 RUSTFLAGS="-C link-arg=--export-table" $CARGO_CMD build --target=wasm32-unknown-unknown --release "$@"
-for i in substrate_test_runtime
-do
-	wasm-gc "target/wasm32-unknown-unknown/release/$i.wasm" "target/wasm32-unknown-unknown/release/$i.compact.wasm"
-done