introduce expander (#4974)

* introduce expander

* unify towards expand

* bubble up expand

* spelling

Co-authored-by: Andrei Sandu <54316454+sandreim@users.noreply.github.com>

* add newlines

Co-authored-by: Andrei Sandu <54316454+sandreim@users.noreply.github.com>

polkadot/Cargo.lock

@@ -626,6 +626,20 @@ dependencies = [
  "digest 0.9.0",
 ]
 
+[[package]]
+name = "blake3"
+version = "1.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a08e53fc5a564bb15bfe6fae56bd71522205f1f91893f9c0116edad6496c183f"
+dependencies = [
+ "arrayref",
+ "arrayvec 0.7.2",
+ "cc",
+ "cfg-if 1.0.0",
+ "constant_time_eq",
+ "digest 0.10.2",
+]
+
 [[package]]
 name = "block-buffer"
 version = "0.7.3"
@@ -1761,6 +1775,18 @@ dependencies = [
  "futures 0.3.21",
 ]
 
+[[package]]
+name = "expander"
+version = "0.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a718c0675c555c5f976fff4ea9e2c150fa06cefa201cadef87cfbf9324075881"
+dependencies = [
+ "blake3 1.3.1",
+ "fs-err",
+ "proc-macro2",
+ "quote",
+]
+
 [[package]]
 name = "eyre"
 version = "0.6.5"
@@ -4392,7 +4418,7 @@ checksum = "4dac63698b887d2d929306ea48b63760431ff8a24fac40ddb22f9c7f49fb7cab"
 dependencies = [
  "blake2b_simd",
  "blake2s_simd",
- "blake3",
+ "blake3 0.3.8",
  "digest 0.9.0",
  "generic-array 0.14.4",
  "multihash-derive",
@@ -6846,6 +6872,7 @@ name = "polkadot-overseer-gen-proc-macro"
 version = "0.9.17"
 dependencies = [
  "assert_matches",
+ "expander",
  "proc-macro-crate 1.1.2",
  "proc-macro2",
  "quote",

polkadot/node/overseer/Cargo.toml

@@ -30,3 +30,4 @@ test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../pri
 
 [features]
 default = []
+expand = ["polkadot-overseer-gen/expand"]

polkadot/node/overseer/overseer-gen/Cargo.toml

@@ -20,3 +20,7 @@ pin-project = "1.0"
 
 [dev-dependencies]
 trybuild = "1.0.53"
+
+[features]
+default = []
+expand = ["polkadot-overseer-gen-proc-macro/expand"]

polkadot/node/overseer/overseer-gen/README.md

@@ -76,3 +76,12 @@ subsystems must be initialized only once (another compile time check) or be _rep
 a special setter like method `replace_<subsystem>`.
 
 A task spawner and subsystem context are required to be defined with `SpawnNamed` and respectively `SubsystemContext` implemented.
+
+## Debugging
+
+As always, debugging is notoriously annoying with bugged proc-macros.
+
+Therefore [`expander`](https://github.com/drahnr/expander) is employed to yield better
+error messages. Enable with `--feature=polkadot-overseer-gen/expand` or
+`--feature=polkadot-overseer/expand` from the root of the project or
+make `"expand"` part of the default feature set.

polkadot/node/overseer/overseer-gen/proc-macro/Cargo.toml

@@ -16,12 +16,13 @@ syn = { version = "1.0.86", features = ["full", "extra-traits"] }
 quote = "1.0.15"
 proc-macro2 = "1.0.36"
 proc-macro-crate = "1.1.2"
+expander = "0.0.4"
 
 [dev-dependencies]
 assert_matches = "1.5.0"
 
 [features]
 default = []
-# write the expanded version to a `overlord-expansion.rs`
+# write the expanded version to a `overlord-expansion.[a-f0-9]{10}.rs`
-# in the `OUT_DIR` as defined by `cargo`.
+# in the `OUT_DIR` as defined by `cargo` for the `expander` crate.
-expansion = []
+expand = []

polkadot/node/overseer/overseer-gen/proc-macro/build.rs

@@ -1,2 +0,0 @@
-/// A dummy build script, so `OUT_DIR` is set.
-fn main() {}

polkadot/node/overseer/overseer-gen/proc-macro/src/lib.rs

@@ -98,36 +98,13 @@ pub(crate) fn impl_overseer_gen(
 	additive.extend(impl_message_wrapper_enum(&info)?);
 	additive.extend(impl_dispatch(&info));
 
-	// Write to a file for expansion, and then use it via `include!()`
+	let ts = expander::Expander::new("overlord-expansion")
-	// in order to obtain better compiler errors when modifying `overlord`.
+		.add_comment("Generated overseer code by `#[overlord(..)]`".to_owned())
-	if cfg!(feature = "expansion") {
+		.dry(!cfg!(feature = "expand"))
-		use std::io::Write;
+		.verbose(false)
+		.fmt(expander::Edition::_2018)
+		.write_to_out_dir(additive)
+		.expect("Expander does not fail due to IO in OUT_DIR. qed");
 
-		let out = env!("OUT_DIR");
+	Ok(ts)
-		let out = std::path::PathBuf::from(out);
-		let path = out.join("overlord-expansion.rs");
-		let mut f = std::fs::OpenOptions::new()
-			.write(true)
-			.create(true)
-			.truncate(true)
-			.open(&path)
-			.expect("File exists. qed");
-		f.write_all(
-			&mut format!("// {:?} \n{}", std::time::SystemTime::now(), additive).as_bytes(),
-		)
-		.expect("Got permissions to write to file. qed");
-		std::process::Command::new("rustfmt")
-			.arg("--edition=2018")
-			.arg(&path)
-			.current_dir(out)
-			.spawn()
-			.expect("Running rustfmt works. qed");
-
-		let path = path.display().to_string();
-		Ok(quote! {
-			include!( #path );
-		})
-	} else {
-		Ok(additive)
-	}
 }