Make it easier to just rebuild the WASM files (#3510)

* Adds `WASM_TARGET_DIRECTORY` env variable to `wasm-builder`

* Create the `wasm-builder-runner`'s in a common workspace

* Make `wasm-builder` trigger less rebuilds

* Version up

* Adds script for building only the WASM files

* Apply suggestions from code review

Co-Authored-By: André Silva <andre.beat@gmail.com>
Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>
This commit is contained in:
Bastian Köcher
2019-08-29 09:06:51 +02:00
committed by GitHub
parent c26b7163a3
commit 4ff97bd856
12 changed files with 182 additions and 58 deletions
+11 -11
View File
@@ -2467,7 +2467,7 @@ dependencies = [
"substrate-offchain-primitives 2.0.0",
"substrate-primitives 2.0.0",
"substrate-session 2.0.0",
"substrate-wasm-builder-runner 1.0.2",
"substrate-wasm-builder-runner 1.0.3",
]
[[package]]
@@ -2976,7 +2976,7 @@ name = "proc-macro-crate"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"toml 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"toml 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@@ -5020,7 +5020,7 @@ dependencies = [
"sr-sandbox 2.0.0",
"sr-std 2.0.0",
"substrate-primitives 2.0.0",
"substrate-wasm-builder-runner 1.0.2",
"substrate-wasm-builder-runner 1.0.3",
]
[[package]]
@@ -5201,7 +5201,7 @@ dependencies = [
"substrate-state-machine 2.0.0",
"substrate-test-runtime-client 2.0.0",
"substrate-trie 2.0.0",
"substrate-wasm-builder-runner 1.0.2",
"substrate-wasm-builder-runner 1.0.3",
"trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
@@ -5269,24 +5269,24 @@ dependencies = [
[[package]]
name = "substrate-wasm-builder"
version = "1.0.4"
version = "1.0.5"
dependencies = [
"build-helper 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"cargo_metadata 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"fs2 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
"tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"toml 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"toml 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
"walkdir 2.2.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "substrate-wasm-builder-runner"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "substrate-wasm-builder-runner"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
version = "1.0.3"
[[package]]
name = "subtle"
@@ -5643,7 +5643,7 @@ dependencies = [
[[package]]
name = "toml"
version = "0.5.1"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"serde 1.0.97 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -5727,7 +5727,7 @@ dependencies = [
"serde 1.0.97 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)",
"termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
"toml 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"toml 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@@ -6641,7 +6641,7 @@ dependencies = [
"checksum tokio-tls 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "354b8cd83825b3c20217a9dc174d6a0c67441a2fae5c41bcb1ea6679f6ae0f7c"
"checksum tokio-udp 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "66268575b80f4a4a710ef83d087fdfeeabdce9b74c797535fbac18a2cb906e92"
"checksum tokio-uds 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "037ffc3ba0e12a0ab4aca92e5234e0dedeb48fddf6ccd260f1f150a36a9f2445"
"checksum toml 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b8c96d7873fa7ef8bdeb3a9cda3ac48389b4154f32b9803b4bc26220b677b039"
"checksum toml 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c7aabe75941d914b72bf3e5d3932ed92ce0664d49d8432305a8b547c37227724"
"checksum traitobject 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "efd1f82c56340fdf16f2a953d7bda4f8fdffba13d93b00844c25572110b26079"
"checksum trie-bench 0.16.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3073600c543ed001319d7e092c46dfd8c245af1a218ec5c75eb01582660a2b3e"
"checksum trie-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d0b62d27e8aa1c07414549ac872480ac82380bab39e730242ab08d82d7cc098a"
+2
View File
@@ -321,6 +321,8 @@ we support multiple environment variables:
* `TRIGGER_WASM_BUILD` - Can be set to trigger a WASM build. On subsequent calls the value of the variable
needs to change. As WASM builder instructs `cargo` to watch for file changes
this environment variable should only be required in certain circumstances.
* `WASM_TARGET_DIRECTORY` - Will copy any build WASM binary to the given directory. The path needs
to be absolute.
Each project can be skipped individually by using the environment variable `SKIP_PROJECT_NAME_WASM_BUILD`.
Where `PROJECT_NAME` needs to be replaced by the name of the cargo project, e.g. `node-runtime` will
@@ -21,7 +21,7 @@ fn main() {
"wasm_binary.rs",
WasmBuilderSource::CratesOrPath {
path: "../../utils/wasm-builder",
version: "1.0.4",
version: "1.0.5",
},
// This instructs LLD to export __heap_base as a global variable, which is used by the
// external memory allocator.
+1 -1
View File
@@ -21,7 +21,7 @@ fn main() {
"wasm_binary.rs",
WasmBuilderSource::CratesOrPath {
path: "../utils/wasm-builder",
version: "1.0.4",
version: "1.0.5",
},
// Note that we set the stack-size to 1MB explicitly even though it is set
// to this value by default. This is because some of our tests (`restoration_of_globals`)
@@ -1,6 +1,6 @@
[package]
name = "substrate-wasm-builder-runner"
version = "1.0.2"
version = "1.0.3"
authors = ["Parity Technologies <admin@parity.io>"]
description = "Runner for substrate-wasm-builder"
edition = "2018"
@@ -30,9 +30,6 @@ use std::{env, process::{Command, self}, fs, path::{PathBuf, Path}};
/// Environment variable that tells us to skip building the WASM binary.
const SKIP_BUILD_ENV: &str = "SKIP_WASM_BUILD";
/// Environment variable to extend the `RUSTFLAGS` variable given to the WASM build.
const WASM_BUILD_RUSTFLAGS_ENV: &str = "WASM_BUILD_RUSTFLAGS";
/// Environment variable that tells us to create a dummy WASM binary.
///
/// This is useful for `cargo check` to speed-up the compilation.
@@ -102,22 +99,8 @@ impl WasmBuilderSource {
pub fn build_current_project_with_rustflags(
file_name: &str,
wasm_builder_source: WasmBuilderSource,
rustflags: &str,
default_rustflags: &str,
) {
let given_rustflags = env::var(WASM_BUILD_RUSTFLAGS_ENV).unwrap_or_default();
env::set_var(WASM_BUILD_RUSTFLAGS_ENV, format!("{} {}", given_rustflags, rustflags));
build_current_project(file_name, wasm_builder_source)
}
/// Build the currently built project as WASM binary.
///
/// The current project is determined using the `CARGO_MANIFEST_DIR` environment variable.
///
/// `file_name` - The name of the file being generated in the `OUT_DIR`. The file contains the
/// constant `WASM_BINARY` which contains the build wasm binary.
/// `wasm_builder_path` - Path to the wasm-builder project, relative to `CARGO_MANIFEST_DIR`.
pub fn build_current_project(file_name: &str, wasm_builder_source: WasmBuilderSource) {
if check_skip_build() {
// If we skip the build, we still want to make sure to be called when an env variable changes
generate_rerun_if_changed_instructions();
@@ -133,12 +116,20 @@ pub fn build_current_project(file_name: &str, wasm_builder_source: WasmBuilderSo
let cargo_toml_path = manifest_dir.join("Cargo.toml");
let out_dir = PathBuf::from(env::var("OUT_DIR").expect("`OUT_DIR` is set by cargo!"));
let file_path = out_dir.join(file_name);
let project_folder = out_dir.join("wasm_build_runner");
let project_name = env::var("CARGO_PKG_NAME").expect("`CARGO_PKG_NAME` is set by cargo!");
let project_folder = get_workspace_root().join(project_name);
if check_provide_dummy_wasm_binary() {
provide_dummy_wasm_binary(&file_path);
} else {
create_project(&project_folder, &file_path, &manifest_dir, wasm_builder_source, &cargo_toml_path);
create_project(
&project_folder,
&file_path,
&manifest_dir,
wasm_builder_source,
&cargo_toml_path,
default_rustflags,
);
run_project(&project_folder);
}
@@ -147,12 +138,43 @@ pub fn build_current_project(file_name: &str, wasm_builder_source: WasmBuilderSo
generate_rerun_if_changed_instructions();
}
/// Build the currently built project as WASM binary.
///
/// The current project is determined using the `CARGO_MANIFEST_DIR` environment variable.
///
/// `file_name` - The name of the file being generated in the `OUT_DIR`. The file contains the
/// constant `WASM_BINARY` which contains the build wasm binary.
/// `wasm_builder_path` - Path to the wasm-builder project, relative to `CARGO_MANIFEST_DIR`.
pub fn build_current_project(file_name: &str, wasm_builder_source: WasmBuilderSource) {
build_current_project_with_rustflags(file_name, wasm_builder_source, "");
}
/// Returns the root path of the wasm-builder-runner workspace.
///
/// The wasm-builder-runner workspace contains all wasm-builder-runner's projects.
fn get_workspace_root() -> PathBuf {
let out_dir_env = env::var("OUT_DIR").expect("`OUT_DIR` is set by cargo!");
let mut out_dir = PathBuf::from(&out_dir_env);
loop {
match out_dir.parent() {
Some(parent) if out_dir.ends_with("build") => return parent.join("wbuild-runner"),
_ => if !out_dir.pop() {
break;
}
}
}
panic!("Could not find target dir in: {}", out_dir_env)
}
fn create_project(
project_folder: &Path,
file_path: &Path,
manifest_dir: &Path,
wasm_builder_source: WasmBuilderSource,
cargo_toml_path: &Path,
default_rustflags: &str,
) {
fs::create_dir_all(project_folder.join("src"))
.expect("WASM build runner dir create can not fail; qed");
@@ -179,12 +201,19 @@ fn create_project(
project_folder.join("src/main.rs"),
format!(
r#"
use substrate_wasm_builder::build_project_with_default_rustflags;
fn main() {{
substrate_wasm_builder::build_project("{file_path}", "{cargo_toml_path}")
build_project_with_default_rustflags(
"{file_path}",
"{cargo_toml_path}",
"{default_rustflags}",
)
}}
"#,
file_path = replace_back_slashes(file_path.display()),
cargo_toml_path = replace_back_slashes(cargo_toml_path.display()),
default_rustflags = default_rustflags,
)
).expect("WASM build runner `main.rs` writing can not fail; qed");
}
+3 -3
View File
@@ -1,6 +1,6 @@
[package]
name = "substrate-wasm-builder"
version = "1.0.4"
version = "1.0.5"
authors = ["Parity Technologies <admin@parity.io>"]
description = "Utility for building WASM binaries"
edition = "2018"
@@ -12,6 +12,6 @@ license = "GPL-3.0"
build-helper = "0.1.1"
cargo_metadata = "0.8"
tempfile = "3.1.0"
toml = "0.5.1"
walkdir = "2.2.8"
toml = "0.5.3"
walkdir = "2.2.9"
fs2 = "0.4.3"
+34 -8
View File
@@ -66,6 +66,8 @@
//! needs to change. As WASM builder instructs `cargo` to watch for file changes
//! this environment variable should only be required in certain circumstances.
//! - `WASM_BUILD_RUSTFLAGS` - Extend `RUSTFLAGS` given to `cargo build` while building the WASM binary.
//! - `WASM_TARGET_DIRECTORY` - Will copy any build WASM binary to the given directory. The path needs
//! to be absolute.
//!
//! Each project can be skipped individually by using the environment variable `SKIP_PROJECT_NAME_WASM_BUILD`.
//! Where `PROJECT_NAME` needs to be replaced by the name of the cargo project, e.g. `node-runtime` will
@@ -96,6 +98,11 @@ const WASM_BUILD_TYPE_ENV: &str = "WASM_BUILD_TYPE";
/// Environment variable to extend the `RUSTFLAGS` variable given to the WASM build.
const WASM_BUILD_RUSTFLAGS_ENV: &str = "WASM_BUILD_RUSTFLAGS";
/// Environment variable to set the target directory to copy the final WASM binary.
///
/// The directory needs to be an absolute path.
const WASM_TARGET_DIRECTORY: &str = "WASM_TARGET_DIRECTORY";
/// Build the currently built project as WASM binary.
///
/// The current project is determined by using the `CARGO_MANIFEST_DIR` environment variable.
@@ -104,6 +111,22 @@ const WASM_BUILD_RUSTFLAGS_ENV: &str = "WASM_BUILD_RUSTFLAGS";
/// constant `WASM_BINARY`, which contains the built WASM binary.
/// `cargo_manifest` - The path to the `Cargo.toml` of the project that should be built.
pub fn build_project(file_name: &str, cargo_manifest: &str) {
build_project_with_default_rustflags(file_name, cargo_manifest, "");
}
/// Build the currently built project as WASM binary.
///
/// The current project is determined by using the `CARGO_MANIFEST_DIR` environment variable.
///
/// `file_name` - The name + path of the file being generated. The file contains the
/// constant `WASM_BINARY`, which contains the built WASM binary.
/// `cargo_manifest` - The path to the `Cargo.toml` of the project that should be built.
/// `default_rustflags` - Default `RUSTFLAGS` that will always be set for the build.
pub fn build_project_with_default_rustflags(
file_name: &str,
cargo_manifest: &str,
default_rustflags: &str,
) {
if check_skip_build() {
return;
}
@@ -123,10 +146,13 @@ pub fn build_project(file_name: &str, cargo_manifest: &str) {
process::exit(1);
}
let (wasm_binary, bloaty) = wasm_project::create_and_compile(&cargo_manifest);
let (wasm_binary, bloaty) = wasm_project::create_and_compile(
&cargo_manifest,
default_rustflags,
);
create_out_file(
file_name,
write_file_if_changed(
file_name.into(),
format!(
r#"
pub const WASM_BINARY: &[u8] = include_bytes!("{wasm_binary}");
@@ -143,11 +169,11 @@ fn check_skip_build() -> bool {
env::var(SKIP_BUILD_ENV).is_ok()
}
fn create_out_file(file_name: &str, content: String) {
fs::write(
file_name,
content
).expect("Creating and writing can not fail; qed");
/// Write to the given `file` if the `content` is different.
fn write_file_if_changed(file: PathBuf, content: String) {
if fs::read_to_string(&file).ok().as_ref() != Some(&content) {
fs::write(&file, content).expect(&format!("Writing `{}` can not fail!", file.display()));
}
}
/// Get a cargo command that compiles with nightly
@@ -14,6 +14,8 @@
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
use crate::write_file_if_changed;
use std::{fs, path::{Path, PathBuf}, borrow::ToOwned, process::{Command, self}, env};
use toml::value::Table;
@@ -75,7 +77,10 @@ impl Drop for WorkspaceLock {
///
/// # Returns
/// The path to the compact WASM binary and the bloaty WASM binary.
pub fn create_and_compile(cargo_manifest: &Path) -> (WasmBinary, WasmBinaryBloaty) {
pub fn create_and_compile(
cargo_manifest: &Path,
default_rustflags: &str,
) -> (WasmBinary, WasmBinaryBloaty) {
let wasm_workspace_root = get_wasm_workspace_root();
let wasm_workspace = wasm_workspace_root.join("wbuild");
@@ -85,8 +90,14 @@ pub fn create_and_compile(cargo_manifest: &Path) -> (WasmBinary, WasmBinaryBloat
let project = create_project(cargo_manifest, &wasm_workspace);
create_wasm_workspace_project(&wasm_workspace);
build_project(&project);
let (wasm_binary, bloaty) = compact_wasm_file(&project, cargo_manifest, &wasm_workspace);
build_project(&project, default_rustflags);
let (wasm_binary, bloaty) = compact_wasm_file(
&project,
cargo_manifest,
&wasm_workspace,
);
copy_wasm_to_target_directory(cargo_manifest, &wasm_binary);
generate_rerun_if_changed_instructions(cargo_manifest, &project, &wasm_workspace);
@@ -122,6 +133,7 @@ fn find_cargo_lock(cargo_manifest: &Path) -> Option<PathBuf> {
cargo_manifest.display(),
build_helper::out_dir().display()
);
None
}
@@ -203,7 +215,7 @@ fn create_project(cargo_manifest: &Path, wasm_workspace: &Path) -> PathBuf {
fs::create_dir_all(project_folder.join("src")).expect("Wasm project dir create can not fail; qed");
fs::write(
write_file_if_changed(
project_folder.join("Cargo.toml"),
format!(
r#"
@@ -223,12 +235,12 @@ fn create_project(cargo_manifest: &Path, wasm_workspace: &Path) -> PathBuf {
crate_path = crate_path.display(),
wasm_binary = wasm_binary,
)
).expect("Project `Cargo.toml` writing can not fail; qed");
);
fs::write(
write_file_if_changed(
project_folder.join("src/lib.rs"),
"#![no_std] pub use wasm_project::*;",
).expect("Project `lib.rs` writing can not fail; qed");
"#![no_std] pub use wasm_project::*;".into(),
);
if let Some(crate_lock_file) = find_cargo_lock(cargo_manifest) {
// Use the `Cargo.lock` of the main project.
@@ -257,12 +269,13 @@ fn is_release_build() -> bool {
}
/// Build the project to create the WASM binary.
fn build_project(project: &Path) {
fn build_project(project: &Path, default_rustflags: &str) {
let manifest_path = project.join("Cargo.toml");
let mut build_cmd = crate::get_nightly_cargo().command();
let rustflags = format!(
"-C link-arg=--export-table {}",
"-C link-arg=--export-table {} {}",
default_rustflags,
env::var(crate::WASM_BUILD_RUSTFLAGS_ENV).unwrap_or_default(),
);
@@ -349,4 +362,29 @@ fn generate_rerun_if_changed_instructions(
println!("cargo:rerun-if-env-changed={}", crate::SKIP_BUILD_ENV);
println!("cargo:rerun-if-env-changed={}", crate::WASM_BUILD_TYPE_ENV);
println!("cargo:rerun-if-env-changed={}", crate::WASM_BUILD_RUSTFLAGS_ENV);
println!("cargo:rerun-if-env-changed={}", crate::WASM_TARGET_DIRECTORY);
}
/// Copy the WASM binary to the target directory set in `WASM_TARGET_DIRECTORY` environment variable.
/// If the variable is not set, this is a no-op.
fn copy_wasm_to_target_directory(cargo_manifest: &Path, wasm_binary: &WasmBinary) {
let target_dir = match env::var(crate::WASM_TARGET_DIRECTORY) {
Ok(path) => PathBuf::from(path),
Err(_) => return,
};
if !target_dir.is_absolute() {
panic!(
"Environment variable `{}` with `{}` is not an absolute path!",
crate::WASM_TARGET_DIRECTORY,
target_dir.display(),
);
}
fs::create_dir_all(&target_dir).expect("Creates `WASM_TARGET_DIRECTORY`.");
fs::copy(
wasm_binary.wasm_binary_path(),
target_dir.join(format!("{}.wasm", get_wasm_binary_name(cargo_manifest))),
).expect("Copies WASM binary to `WASM_TARGET_DIRECTORY`.");
}
+1 -1
View File
@@ -19,7 +19,7 @@ use wasm_builder_runner::{build_current_project_with_rustflags, WasmBuilderSourc
fn main() {
build_current_project_with_rustflags(
"wasm_binary.rs",
WasmBuilderSource::Crates("1.0.4"),
WasmBuilderSource::Crates("1.0.5"),
// This instructs LLD to export __heap_base as a global variable, which is used by the
// external memory allocator.
"-Clink-arg=--export=__heap_base",
+1 -1
View File
@@ -21,7 +21,7 @@ fn main() {
"wasm_binary.rs",
WasmBuilderSource::CratesOrPath {
path: "../../core/utils/wasm-builder",
version: "1.0.4",
version: "1.0.5",
},
// This instructs LLD to export __heap_base as a global variable, which is used by the
// external memory allocator.
+29
View File
@@ -0,0 +1,29 @@
#!/usr/bin/env sh
# Script for building only the WASM binary of the given project.
set -e
PROJECT_ROOT=`git rev-parse --show-toplevel`
if [ "$#" -lt 1 ]; then
echo "You need to pass the name of the crate you want to compile!"
exit 1
fi
WASM_BUILDER_RUNNER="$PROJECT_ROOT/target/release/wbuild-runner/$1"
if [ -z "$2" ]; then
export WASM_TARGET_DIRECTORY=$(pwd)
else
export WASM_TARGET_DIRECTORY=$2
fi
if [ -d $WASM_BUILDER_RUNNER ]; then
export DEBUG=false
export OUT_DIR="$PROJECT_ROOT/target/release/build"
cargo run --release --manifest-path="$WASM_BUILDER_RUNNER/Cargo.toml" \
| grep -vE "cargo:rerun-if-|Executing build command" \
else
cargo build --release -p $1
fi