Initial support for building RISC-V runtimes targeting PolkaVM (#3179)

This PR adds initial support for building RISC-V runtimes targeting
PolkaVM.

- Setting the `SUBSTRATE_RUNTIME_TARGET=riscv` environment variable will
now build a RISC-V runtime instead of a WASM runtime.
- This only adds support for *building* runtimes; running them will need
a PolkaVM-based executor, which I will add in a future PR.
- Only building the minimal runtime is supported (building the Polkadot
runtime doesn't work *yet* due to one of the dependencies).
- The builder now sets a `substrate_runtime` cfg flag when building the
runtimes, with the idea being that instead of doing `#[cfg(not(feature =
"std"))]` or `#[cfg(target_arch = "wasm32")]` to detect that we're
building a runtime you'll do `#[cfg(substrate_runtime)]`. (Switching the
whole codebase to use this will be done in a future PR; I deliberately
didn't do this here to keep this PR minimal and reviewable.)
- Further renaming of things (e.g. types, environment variables and proc
macro attributes having "wasm" in their name) to be target-agnostic will
also be done in a future refactoring PR (while keeping backwards
compatibility where it makes sense; I don't intend to break anyone's
workflow or create unnecessary churn).
- This PR also fixes two bugs in the `wasm-builder` crate:
* The `RUSTC` environment variable is now removed when invoking the
compiler. This prevents the toolchain version from being overridden when
called from a `build.rs` script.
* When parsing the `rustup toolchain list` output the `(default)` is now
properly stripped and not treated as part of the version.
- I've also added a minimal CI job that makes sure this doesn't break in
the future. (cc @paritytech/ci)

cc @athei

------

Also, just a fun little tidbit: quickly comparing the size of the built
runtimes it seems that the PolkaVM runtime is slightly smaller than the
WASM one. (`production` build, with the `names` section substracted from
the WASM's size to keep things fair, since for the PolkaVM runtime we're
currently stripping out everything)

- `.wasm`: 625505 bytes
- `.wasm` (after wasm-opt -O3): 563205 bytes
- `.wasm` (after wasm-opt -Os): 562987 bytes
- `.wasm` (after wasm-opt -Oz): 536852 bytes
- `.polkavm`: ~~580338 bytes~~ 550476 bytes (after enabling extra target
features; I'll add those in another PR once we have an executor working)

---------

Co-authored-by: Bastian Köcher <git@kchr.de>

substrate/utils/wasm-builder/src/lib.rs

@@ -113,6 +113,7 @@
 //! wasm32-unknown-unknown --toolchain nightly-2020-02-20`.
 
 use std::{
+	collections::BTreeSet,
 	env, fs,
 	io::BufRead,
 	path::{Path, PathBuf},
@@ -164,6 +165,9 @@ const WASM_BUILD_WORKSPACE_HINT: &str = "WASM_BUILD_WORKSPACE_HINT";
 /// Environment variable to set whether we'll build `core`/`std`.
 const WASM_BUILD_STD: &str = "WASM_BUILD_STD";
 
+/// The target to use for the runtime. Valid values are `wasm` (default) or `riscv`.
+const RUNTIME_TARGET: &str = "SUBSTRATE_RUNTIME_TARGET";
+
 /// Write to the given `file` if the `content` is different.
 fn write_file_if_changed(file: impl AsRef<Path>, content: impl AsRef<str>) {
 	if fs::read_to_string(file.as_ref()).ok().as_deref() != Some(content.as_ref()) {
@@ -185,7 +189,7 @@ fn copy_file_if_changed(src: PathBuf, dst: PathBuf) {
 }
 
 /// Get a cargo command that should be used to invoke the compilation.
-fn get_cargo_command() -> CargoCommand {
+fn get_cargo_command(target: RuntimeTarget) -> CargoCommand {
 	let env_cargo =
 		CargoCommand::new(&env::var("CARGO").expect("`CARGO` env variable is always set by cargo"));
 	let default_cargo = CargoCommand::new("cargo");
@@ -196,35 +200,33 @@ fn get_cargo_command() -> CargoCommand {
 		wasm_toolchain.map(|t| CargoCommand::new_with_args("rustup", &["run", &t, "cargo"]))
 	{
 		cmd
-	} else if env_cargo.supports_substrate_wasm_env() {
+	} else if env_cargo.supports_substrate_runtime_env(target) {
 		env_cargo
-	} else if default_cargo.supports_substrate_wasm_env() {
+	} else if default_cargo.supports_substrate_runtime_env(target) {
 		default_cargo
 	} else {
 		// If no command before provided us with a cargo that supports our Substrate wasm env, we
 		// try to search one with rustup. If that fails as well, we return the default cargo and let
 		// the prequisities check fail.
-		get_rustup_command().unwrap_or(default_cargo)
+		get_rustup_command(target).unwrap_or(default_cargo)
 	}
 }
 
-/// Get the newest rustup command that supports our Substrate wasm env.
+/// Get the newest rustup command that supports compiling a runtime.
 ///
 /// Stable versions are always favored over nightly versions even if the nightly versions are
 /// newer.
-fn get_rustup_command() -> Option<CargoCommand> {
-	let host = format!("-{}", env::var("HOST").expect("`HOST` is always set by cargo"));
-
+fn get_rustup_command(target: RuntimeTarget) -> Option<CargoCommand> {
 	let output = Command::new("rustup").args(&["toolchain", "list"]).output().ok()?.stdout;
 	let lines = output.as_slice().lines();
 
 	let mut versions = Vec::new();
 	for line in lines.filter_map(|l| l.ok()) {
-		let rustup_version = line.trim_end_matches(&host);
-
+		// Split by a space to get rid of e.g. " (default)" at the end.
+		let rustup_version = line.split(" ").next().unwrap();
 		let cmd = CargoCommand::new_with_args("rustup", &["run", &rustup_version, "cargo"]);
 
-		if !cmd.supports_substrate_wasm_env() {
+		if !cmd.supports_substrate_runtime_env(target) {
 			continue
 		}
 
@@ -247,22 +249,26 @@ struct CargoCommand {
 	program: String,
 	args: Vec<String>,
 	version: Option<Version>,
+	target_list: Option<BTreeSet<String>>,
 }
 
 impl CargoCommand {
 	fn new(program: &str) -> Self {
 		let version = Self::extract_version(program, &[]);
+		let target_list = Self::extract_target_list(program, &[]);
 
-		CargoCommand { program: program.into(), args: Vec::new(), version }
+		CargoCommand { program: program.into(), args: Vec::new(), version, target_list }
 	}
 
 	fn new_with_args(program: &str, args: &[&str]) -> Self {
 		let version = Self::extract_version(program, args);
+		let target_list = Self::extract_target_list(program, args);
 
 		CargoCommand {
 			program: program.into(),
 			args: args.iter().map(ToString::to_string).collect(),
 			version,
+			target_list,
 		}
 	}
 
@@ -283,6 +289,23 @@ impl CargoCommand {
 		Version::extract(&version)
 	}
 
+	fn extract_target_list(program: &str, args: &[&str]) -> Option<BTreeSet<String>> {
+		// This is technically an unstable option, but we don't care because we only need this
+		// to build RISC-V runtimes, and those currently require a specific nightly toolchain
+		// anyway, so it's totally fine for this to fail in other cases.
+		let list = Command::new(program)
+			.args(args)
+			.args(&["rustc", "-Z", "unstable-options", "--print", "target-list"])
+			// Make sure if we're called from within a `build.rs` the host toolchain won't override
+			// a rustup toolchain we've picked.
+			.env_remove("RUSTC")
+			.output()
+			.ok()
+			.and_then(|o| String::from_utf8(o.stdout).ok())?;
+
+		Some(list.trim().split("\n").map(ToString::to_string).collect())
+	}
+
 	/// Returns the version of this cargo command or `None` if it failed to extract the version.
 	fn version(&self) -> Option<Version> {
 		self.version
@@ -294,12 +317,29 @@ impl CargoCommand {
 			env::var("RUSTC_BOOTSTRAP").is_ok()
 	}
 
+	/// Check if the supplied cargo command supports our runtime environment.
+	fn supports_substrate_runtime_env(&self, target: RuntimeTarget) -> bool {
+		match target {
+			RuntimeTarget::Wasm => self.supports_substrate_runtime_env_wasm(),
+			RuntimeTarget::Riscv => self.supports_substrate_runtime_env_riscv(),
+		}
+	}
+
+	/// Check if the supplied cargo command supports our RISC-V runtime environment.
+	fn supports_substrate_runtime_env_riscv(&self) -> bool {
+		let Some(target_list) = self.target_list.as_ref() else { return false };
+		// This is our custom target which currently doesn't exist on any upstream toolchain,
+		// so if it exists it's guaranteed to be our custom toolchain and have have everything
+		// we need, so any further version checks are unnecessary at this point.
+		target_list.contains("riscv32ema-unknown-none-elf")
+	}
+
 	/// Check if the supplied cargo command supports our Substrate wasm environment.
 	///
 	/// This means that either the cargo version is at minimum 1.68.0 or this is a nightly cargo.
 	///
 	/// Assumes that cargo version matches the rustc version.
-	fn supports_substrate_wasm_env(&self) -> bool {
+	fn supports_substrate_runtime_env_wasm(&self) -> bool {
 		// `RUSTC_BOOTSTRAP` tells a stable compiler to behave like a nightly. So, when this env
 		// variable is set, we can assume that whatever rust compiler we have, it is a nightly
 		// compiler. For "more" information, see:
@@ -365,5 +405,48 @@ fn get_bool_environment_variable(name: &str) -> Option<bool> {
 
 /// Returns whether we need to also compile the standard library when compiling the runtime.
 fn build_std_required() -> bool {
-	crate::get_bool_environment_variable(crate::WASM_BUILD_STD).unwrap_or(true)
+	let default = runtime_target() == RuntimeTarget::Wasm;
+
+	crate::get_bool_environment_variable(crate::WASM_BUILD_STD).unwrap_or(default)
+}
+
+#[derive(Copy, Clone, PartialEq, Eq)]
+enum RuntimeTarget {
+	Wasm,
+	Riscv,
+}
+
+impl RuntimeTarget {
+	fn rustc_target(self) -> &'static str {
+		match self {
+			RuntimeTarget::Wasm => "wasm32-unknown-unknown",
+			RuntimeTarget::Riscv => "riscv32ema-unknown-none-elf",
+		}
+	}
+
+	fn build_subdirectory(self) -> &'static str {
+		// Keep the build directories separate so that when switching between
+		// the targets we won't trigger unnecessary rebuilds.
+		match self {
+			RuntimeTarget::Wasm => "wbuild",
+			RuntimeTarget::Riscv => "rbuild",
+		}
+	}
+}
+
+fn runtime_target() -> RuntimeTarget {
+	let Some(value) = env::var_os(RUNTIME_TARGET) else {
+		return RuntimeTarget::Wasm;
+	};
+
+	if value == "wasm" {
+		RuntimeTarget::Wasm
+	} else if value == "riscv" {
+		RuntimeTarget::Riscv
+	} else {
+		build_helper::warning!(
+			"the '{RUNTIME_TARGET}' environment variable has an invalid value; it must be either 'wasm' or 'riscv'"
+		);
+		std::process::exit(1);
+	}
 }