Fixes wasm-builder rerun if changed logic (#6144)

There was a bug which related in required files not being tracked of
being modified. This pr fixes this bug by making sure we ignore version
requirements for path dependencies and git dependencies. This also
ensures that we only track `.rs` or `.toml` files. Another improvement
is that we only include paths which don't contain a `Cargo.toml` if this
`Cargo.toml` does not belongs to the package being processed. This
prevents that sub-crates are added to the tracked files, while not being
part of the dependencies.

substrate/Cargo.lock

@@ -520,9 +520,9 @@ checksum = "4964518bd3b4a8190e832886cdc0da9794f12e8e6c1613a9e90ff331c4c8724b"
 
 [[package]]
 name = "cargo_metadata"
-version = "0.9.1"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46e3374c604fb39d1a2f35ed5e4a4e30e60d01fab49446e08f1b3e9a90aef202"
+checksum = "b8de60b887edf6d74370fc8eb177040da4847d971d6234c7b13a6da324ef0caf"
 dependencies = [
  "semver 0.9.0",
  "serde",
@@ -7021,18 +7021,18 @@ checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0"
 
 [[package]]
 name = "serde"
-version = "1.0.106"
+version = "1.0.110"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36df6ac6412072f67cf767ebbde4133a5b2e88e76dc6187fa7104cd16f783399"
+checksum = "99e7b308464d16b56eba9964e4972a3eee817760ab60d88c3f86e1fecb08204c"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.106"
+version = "1.0.110"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e549e3abf4fb8621bd1609f11dfc9f5e50320802273b12f3811a67e6716ea6c"
+checksum = "818fbf6bfa9a42d3bfcaca148547aa00c7b915bec71d1757aa2d44ca68771984"
 dependencies = [
  "proc-macro2",
  "quote 1.0.3",

substrate/utils/wasm-builder/Cargo.toml

@@ -14,10 +14,10 @@ targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
 build-helper = "0.1.1"
-cargo_metadata = "0.9.0"
+cargo_metadata = "0.10.0"
 tempfile = "3.1.0"
 toml = "0.5.4"
-walkdir = "2.2.9"
+walkdir = "2.3.1"
 fs2 = "0.4.3"
 wasm-gc-api = "0.1.11"
 atty = "0.2.13"

substrate/utils/wasm-builder/src/wasm_project.rs

@@ -26,7 +26,7 @@ use toml::value::Table;
 
 use build_helper::rerun_if_changed;
 
-use cargo_metadata::MetadataCommand;
+use cargo_metadata::{MetadataCommand, Metadata};
 
 use walkdir::WalkDir;
 
@@ -342,11 +342,7 @@ fn project_enabled_features(
 ///
 /// # Returns
 /// The path to the created project.
-fn create_project(
+fn create_project(cargo_manifest: &Path, wasm_workspace: &Path, crate_metadata: &Metadata) -> PathBuf {
-	cargo_manifest: &Path,
-	wasm_workspace: &Path,
-	crate_metadata: &cargo_metadata::Metadata,
-) -> PathBuf {
 	let crate_name = get_crate_name(cargo_manifest);
 	let crate_path = cargo_manifest.parent().expect("Parent path exists; qed");
 	let wasm_binary = get_wasm_binary_name(cargo_manifest);
@@ -519,22 +515,33 @@ fn generate_rerun_if_changed_instructions(
 		.exec()
 		.expect("`cargo metadata` can not fail!");
 
-	// Start with the dependencies of the crate we want to compile for wasm.
+	let package = metadata.packages
-	let mut dependencies = metadata.packages
 		.iter()
 		.find(|p| p.manifest_path == cargo_manifest)
-		.expect("The crate package is contained in its own metadata; qed")
+		.expect("The crate package is contained in its own metadata; qed");
-		.dependencies
+
-		.iter()
+	// Start with the dependencies of the crate we want to compile for wasm.
-		.collect::<Vec<_>>();
+	let mut dependencies = package.dependencies.iter().collect::<Vec<_>>();
 
 	// Collect all packages by follow the dependencies of all packages we find.
 	let mut packages = HashSet::new();
+	packages.insert(DeduplicatePackage::from(package));
+
 	while let Some(dependency) = dependencies.pop() {
+		let path_or_git_dep = dependency.source
+			.as_ref()
+			.map(|s| s.starts_with("git+"))
+			.unwrap_or(true);
+
 		let package = metadata.packages
 			.iter()
 			.filter(|p| !p.manifest_path.starts_with(wasm_workspace))
-			.find(|p| dependency.req.matches(&p.version) && dependency.name == p.name);
+			.find(|p| {
+				// Check that the name matches and that the version matches or this is
+				// a git or path dep. A git or path dependency can only occur once, so we don't
+				// need to check the version.
+				(path_or_git_dep || dependency.req.matches(&p.version)) && dependency.name == p.name
+			});
 
 		if let Some(package) = package {
 			if packages.insert(DeduplicatePackage::from(package)) {
@@ -544,21 +551,7 @@ fn generate_rerun_if_changed_instructions(
 	}
 
 	// Make sure that if any file/folder of a dependency change, we need to rerun the `build.rs`
-	packages.into_iter()
+	packages.iter().for_each(package_rerun_if_changed);
-		.filter(|p| !p.manifest_path.starts_with(wasm_workspace))
-		.for_each(|package| {
-			let mut manifest_path = package.manifest_path.clone();
-			if manifest_path.ends_with("Cargo.toml") {
-				manifest_path.pop();
-			}
-
-			rerun_if_changed(&manifest_path);
-
-			WalkDir::new(manifest_path)
-				.into_iter()
-				.filter_map(|p| p.ok())
-				.for_each(|p| rerun_if_changed(p.path()));
-		});
 
 	// Register our env variables
 	println!("cargo:rerun-if-env-changed={}", crate::SKIP_BUILD_ENV);
@@ -568,8 +561,32 @@ fn generate_rerun_if_changed_instructions(
 	println!("cargo:rerun-if-env-changed={}", crate::WASM_BUILD_TOOLCHAIN);
 }
 
-/// Copy the WASM binary to the target directory set in `WASM_TARGET_DIRECTORY` environment variable.
+/// Track files and paths related to the given package to rerun `build.rs` on any relevant change.
-/// If the variable is not set, this is a no-op.
+fn package_rerun_if_changed(package: &DeduplicatePackage) {
+	let mut manifest_path = package.manifest_path.clone();
+	if manifest_path.ends_with("Cargo.toml") {
+		manifest_path.pop();
+	}
+
+	WalkDir::new(&manifest_path)
+		.into_iter()
+		.filter_entry(|p| {
+			// Ignore this entry if it is a directory that contains a `Cargo.toml` that is not the
+			// `Cargo.toml` related to the current package. This is done to ignore sub-crates of a crate.
+			// If such a sub-crate is a dependency, it will be processed independently anyway.
+			p.path() == manifest_path
+				|| !p.path().is_dir()
+				|| !p.path().join("Cargo.toml").exists()
+		})
+		.filter_map(|p| p.ok().map(|p| p.into_path()))
+		.filter(|p| {
+			p.is_dir() || p.extension().map(|e| e == "rs" || e == "toml").unwrap_or_default()
+		})
+		.for_each(|p| rerun_if_changed(p));
+}
+
+/// Copy the WASM binary to the target directory set in `WASM_TARGET_DIRECTORY` environment
+/// variable. If the variable is not set, this is a no-op.
 fn copy_wasm_to_target_directory(cargo_manifest: &Path, wasm_binary: &WasmBinary) {
 	let target_dir = match env::var(crate::WASM_TARGET_DIRECTORY) {
 		Ok(path) => PathBuf::from(path),