Build WASM binaries as part of cargo build (#2868)

* Introduce `wasm-builder` and `wasm-builder-runner` to retire `build.sh`

Make use of `wasm-builder` in `test-runtime`.

* Add build script and remove the wasm project

* Port `node-runtime` to new wasm-builder

* Make `substrate-executor` tests work with `wasm-builder`

* Move `node-template` to `wasm-builder`

* Remove `build.sh` :)

* Remove the last include_bytes

* Adds the missing build.rs files

* Remove `build.sh` from CI

* Debug CI

* Make it work in CI

* CI attempt 3

* Make `substrate-runtime-test` compile on stable

* Ahhh, some missed `include_bytes!`

* AHH

* Add suggestions

* Improve search for `Cargo.lock` and don't panic if it is not found

* Searching from manifest path was no good idea

* Make the `wasm-builder` source better configurable

* Expose the bloaty wasm binary as well

* Make sure to rerun WASM recompilation on changes in dependencies

* Introduce new `WASM_BUILD_TYPE` env and make sure to call `build.rs` on
changes to env variables

* Remove `build.sh` from READMEs

* Rename the projects

* Fixes CI

* Update lock file

* Fixes merge-conflict

* Apply suggestions from code review

Co-Authored-By: TriplEight <denis.pisarev@parity.io>

* Try to make windows happy

* Replace all back slashes in paths with slashes

* Apply suggestions from code review

Co-Authored-By: Pierre Krieger <pierre.krieger1708@gmail.com>

* Use cargo from `CARGO` env variable

* Fix compilation

* Use `rustup` for running the nightly build

* Make individual projects skipable

* Fix compilation

* Fixes compilation

* Build all WASM projects in one workspace

* Replace more back slashes!

* Remove `inlcude_bytes!`

* Adds some documentation

* Apply suggestions from code review

Co-Authored-By: Shawn Tabrizi <shawntabrizi@gmail.com>

* Apply suggestions from code review

Co-Authored-By: Shawn Tabrizi <shawntabrizi@gmail.com>

* More review comments

* Update `Cargo.lock`

* Set license

* Apply suggestions from code review

Co-Authored-By: joe petrowski <25483142+joepetrowski@users.noreply.github.com>

* More review comments + adds `TRIGGER_WASM_BUILD` env

* Fix doc tests

* Increase version + update README

* Switch crates.io version of `wasm-builder`

* Update README

* Switch to released version of `wasm-builder-runner`

substrate/node-template/README.md

@@ -16,12 +16,6 @@ Install required tools:
 ./scripts/init.sh
 ```
 
-Build the WebAssembly binary:
-
-```bash
-./scripts/build.sh
-```
-
 Build all native code:
 
 ```bash

substrate/node-template/runtime/Cargo.toml

@@ -25,6 +25,9 @@ client = { package = "substrate-client", path = "../../core/client", default_fea
 consensus-aura = { package = "substrate-consensus-aura-primitives", path = "../../core/consensus/aura/primitives", default_features = false }
 offchain-primitives = { package = "substrate-offchain-primitives", path = "../../core/offchain/primitives", default-features = false }
 
+[build-dependencies]
+wasm-builder-runner = { package = "substrate-wasm-builder-runner", version = "1.0.0" }
+
 [features]
 default = ["std"]
 std = [
@@ -48,3 +51,4 @@ std = [
 	"consensus-aura/std",
 	"offchain-primitives/std",
 ]
+no_std = []

substrate/node-template/runtime/build.rs

@@ -0,0 +1,21 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use wasm_builder_runner::{build_current_project, WasmBuilderSource};
+
+fn main() {
+	build_current_project("wasm_binary.rs", WasmBuilderSource::Crates("1.0.1"));
+}

substrate/node-template/runtime/src/lib.rs

@@ -4,6 +4,10 @@
 // `construct_runtime!` does a lot of recursion and requires us to increase the limit to 256.
 #![recursion_limit="256"]
 
+// Make the WASM binary available.
+#[cfg(feature = "std")]
+include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs"));
+
 #[cfg(feature = "std")]
 use serde::{Serialize, Deserialize};
 use parity_codec::{Encode, Decode};

substrate/node-template/runtime/wasm/Cargo.toml

@@ -1,24 +0,0 @@
-[package]
-name = "node-template-runtime-wasm"
-version = "2.0.0"
-authors = ["Anonymous"]
-edition = "2018"
-
-[lib]
-crate-type = ["cdylib"]
-
-[dependencies]
-node-template-runtime = { path = "..", default-features = false }
-
-[features]
-default = []
-std = [
-	"node-template-runtime/std",
-]
-
-[profile.release]
-panic = "abort"
-lto = true
-
-[workspace]
-members = []

substrate/node-template/runtime/wasm/build.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-if cargo --version | grep -q "nightly"; then
-	CARGO_CMD="cargo"
-else
-	CARGO_CMD="cargo +nightly"
-fi
-CARGO_INCREMENTAL=0 RUSTFLAGS="-C link-arg=--export-table" $CARGO_CMD build --target=wasm32-unknown-unknown --release "$@"
-for i in node_template_runtime_wasm
-do
-	wasm-gc target/wasm32-unknown-unknown/release/$i.wasm target/wasm32-unknown-unknown/release/$i.compact.wasm
-done

substrate/node-template/runtime/wasm/src/lib.rs

@@ -1,5 +0,0 @@
-//! The Substrate node template runtime reexported for WebAssembly compile.
-
-#![cfg_attr(not(feature = "std"), no_std)]
-
-pub use node_template_runtime::*;

substrate/node-template/scripts/build.sh

@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-PROJECT_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." >/dev/null && pwd )"
-
-export CARGO_INCREMENTAL=0
-
-bold=$(tput bold)
-normal=$(tput sgr0)
-
-# Save current directory.
-pushd . >/dev/null
-
-for SRC in runtime/wasm
-do
-  echo "${bold}Building webassembly binary in $SRC...${normal}"
-  cd "$PROJECT_ROOT/$SRC"
-
-  ./build.sh "$@"
-
-  cd - >> /dev/null
-done
-
-# Restore initial directory.
-popd >/dev/null

substrate/node-template/src/chain_spec.rs

@@ -1,7 +1,7 @@
 use primitives::{ed25519, sr25519, Pair};
 use node_template_runtime::{
-	AccountId, AuraId as AuthorityId, GenesisConfig, AuraConfig, TimestampConfig, BalancesConfig,
-	SudoConfig, IndicesConfig, SystemConfig
+	AccountId, GenesisConfig, AuraConfig, TimestampConfig, BalancesConfig,
+	SudoConfig, IndicesConfig, SystemConfig, WASM_BINARY, AuraId
 };
 use substrate_service;
 
@@ -22,7 +22,7 @@ pub enum Alternative {
 	LocalTestnet,
 }
 
-fn authority_key(s: &str) -> AuthorityId {
+fn authority_key(s: &str) -> AuraId {
 	ed25519::Pair::from_string(&format!("//{}", s), None)
 		.expect("static values are valid; qed")
 		.public()
@@ -88,10 +88,10 @@ impl Alternative {
 	}
 }
 
-fn testnet_genesis(initial_authorities: Vec<AuthorityId>, endowed_accounts: Vec<AccountId>, root_key: AccountId) -> GenesisConfig {
+fn testnet_genesis(initial_authorities: Vec<AuraId>, endowed_accounts: Vec<AccountId>, root_key: AccountId) -> GenesisConfig {
 	GenesisConfig {
 		system: Some(SystemConfig {
-			code: include_bytes!("../runtime/wasm/target/wasm32-unknown-unknown/release/node_template_runtime_wasm.compact.wasm").to_vec(),
+			code: WASM_BINARY.to_vec(),
 			changes_trie_config: Default::default(),
 		}),
 		aura: Some(AuraConfig {

substrate/node-template/src/service.rs

@@ -5,7 +5,7 @@
 use std::sync::Arc;
 use log::info;
 use transaction_pool::{self, txpool::{Pool as TransactionPool}};
-use node_template_runtime::{self, GenesisConfig, opaque::Block, RuntimeApi};
+use node_template_runtime::{self, GenesisConfig, opaque::Block, RuntimeApi, WASM_BINARY};
 use substrate_service::{
 	FactoryFullConfiguration, LightComponents, FullComponents, FullBackend,
 	FullClient, LightClient, LightBackend, FullExecutor, LightExecutor,
@@ -27,7 +27,7 @@ native_executor_instance!(
 	pub Executor,
 	node_template_runtime::api::dispatch,
 	node_template_runtime::native_version,
-	include_bytes!("../runtime/wasm/target/wasm32-unknown-unknown/release/node_template_runtime_wasm.compact.wasm")
+	WASM_BINARY
 );
 
 #[derive(Default)]