Improve XCM fuzzer (#6190)

* Add improved XCM fuzzer

* Add command for running a single input

* Add installation command

* Fix @m-cat's nit

* Add newline

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* Add info about current directory in fuzzing README

* Update Cargo.lock

---------

Co-authored-by: Keith Yeung <kungfukeith11@gmail.com>
Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

polkadot/Cargo.lock

@@ -196,9 +196,9 @@ dependencies = [
 
 [[package]]
 name = "arbitrary"
-version = "1.0.3"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "510c76ecefdceada737ea728f4f9a84bd2e1ef29f1ba555e560940fe279954de"
+checksum = "29d47fbf90d5149a107494b15a7dc8d69b351be2db3bb9691740e88ec17fd880"
 
 [[package]]
 name = "arc-swap"
@@ -13846,6 +13846,7 @@ dependencies = [
 name = "xcm-simulator-fuzzer"
 version = "0.9.37"
 dependencies = [
+ "arbitrary",
  "frame-support",
  "frame-system",
  "honggfuzz",

polkadot/xcm/xcm-simulator/fuzzer/.gitignore

@@ -0,0 +1,5 @@
+hfuzz_target
+hfuzz_workspace
+cargo
+coverage
+ccov.zip

polkadot/xcm/xcm-simulator/fuzzer/Cargo.toml

@@ -8,6 +8,7 @@ edition.workspace = true
 [dependencies]
 codec = { package = "parity-scale-codec", version = "3.3.0" }
 honggfuzz = "0.5.55"
+arbitrary = "1.2.0"
 scale-info = { version = "2.1.2", features = ["derive"] }
 
 frame-system = { git = "https://github.com/paritytech/substrate", branch = "master" }

polkadot/xcm/xcm-simulator/fuzzer/README.md

@@ -0,0 +1,38 @@
+# XCM Simulator Fuzzer
+
+This project will fuzz-test the XCM simulator. It can catch reachable panics, timeouts as well as integer overflows and underflows.
+
+## Install dependencies
+
+```
+cargo install honggfuzz
+```
+
+## Run the fuzzer
+
+In this directory, run this command:
+
+```
+cargo hfuzz run xcm-fuzzer
+```
+
+## Run a single input
+
+In this directory, run this command:
+
+```
+cargo hfuzz run-debug xcm-fuzzer hfuzz_workspace/xcm-fuzzer/fuzzer_input_file
+```
+
+## Generate coverage
+
+In this directory, run these four commands:
+
+```
+RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" CARGO_INCREMENTAL=0 SKIP_WASM_BUILD=1 CARGO_HOME=./cargo cargo build
+../../../target/debug/xcm-fuzzer hfuzz_workspace/xcm-fuzzer/input/
+zip -0 ccov.zip `find ../../../target/ \( -name "*.gc*" -o -name "test-*.gc*" \) -print`
+grcov ccov.zip -s ../../../ -t html --llvm --branch --ignore-not-existing -o ./coverage
+```
+
+The code coverage will be in `./coverage/index.html`.

polkadot/xcm/xcm-simulator/fuzzer/src/fuzz.rs

@@ -18,6 +18,7 @@ mod parachain;
 mod relay_chain;
 
 use codec::DecodeLimit;
+use polkadot_core_primitives::AccountId;
 use polkadot_parachain::primitives::Id as ParaId;
 use sp_runtime::traits::AccountIdConversion;
 use xcm_simulator::{decl_test_network, decl_test_parachain, decl_test_relay_chain, TestExt};
@@ -25,7 +26,8 @@ use xcm_simulator::{decl_test_network, decl_test_parachain, decl_test_relay_chai
 use frame_support::assert_ok;
 use xcm::{latest::prelude::*, MAX_XCM_DECODE_DEPTH};
 
-pub const ALICE: sp_runtime::AccountId32 = sp_runtime::AccountId32::new([0u8; 32]);
+use arbitrary::{Arbitrary, Error, Unstructured};
+
 pub const INITIAL_BALANCE: u128 = 1_000_000_000;
 
 decl_test_parachain! {
@@ -46,6 +48,15 @@ decl_test_parachain! {
 	}
 }
 
+decl_test_parachain! {
+	pub struct ParaC {
+		Runtime = parachain::Runtime,
+		XcmpMessageHandler = parachain::MsgQueue,
+		DmpMessageHandler = parachain::MsgQueue,
+		new_ext = para_ext(3),
+	}
+}
+
 decl_test_relay_chain! {
 	pub struct Relay {
 		Runtime = relay_chain::Runtime,
@@ -60,10 +71,35 @@ decl_test_network! {
 		parachains = vec![
 			(1, ParaA),
 			(2, ParaB),
+			(3, ParaC),
 		],
 	}
 }
 
+// An XCM message that will be generated by the fuzzer through the Arbitrary trait
+struct XcmMessage {
+	// Source chain
+	source: u32,
+	// Destination chain
+	destination: u32,
+	// XCM message
+	message: Xcm<()>,
+}
+
+impl<'a> Arbitrary<'a> for XcmMessage {
+	fn arbitrary(u: &mut Unstructured<'a>) -> Result<Self, Error> {
+		let source: u32 = u.arbitrary()?;
+		let destination: u32 = u.arbitrary()?;
+		let mut encoded_message: &[u8] = u.arbitrary()?;
+		if let Ok(message) =
+			DecodeLimit::decode_with_depth_limit(MAX_XCM_DECODE_DEPTH, &mut encoded_message)
+		{
+			return Ok(XcmMessage { source, destination, message })
+		}
+		Err(Error::IncorrectFormat)
+	}
+}
+
 pub fn para_account_id(id: u32) -> relay_chain::AccountId {
 	ParaId::from(id).into_account_truncating()
 }
@@ -73,9 +109,11 @@ pub fn para_ext(para_id: u32) -> sp_io::TestExternalities {
 
 	let mut t = frame_system::GenesisConfig::default().build_storage::<Runtime>().unwrap();
 
-	pallet_balances::GenesisConfig::<Runtime> { balances: vec![(ALICE, INITIAL_BALANCE)] }
-		.assimilate_storage(&mut t)
-		.unwrap();
+	pallet_balances::GenesisConfig::<Runtime> {
+		balances: (0..6).map(|i| ([i; 32].into(), INITIAL_BALANCE)).collect(),
+	}
+	.assimilate_storage(&mut t)
+	.unwrap();
 
 	let mut ext = sp_io::TestExternalities::new(t);
 	ext.execute_with(|| {
@@ -90,11 +128,13 @@ pub fn relay_ext() -> sp_io::TestExternalities {
 
 	let mut t = frame_system::GenesisConfig::default().build_storage::<Runtime>().unwrap();
 
-	pallet_balances::GenesisConfig::<Runtime> {
-		balances: vec![(ALICE, INITIAL_BALANCE), (para_account_id(1), INITIAL_BALANCE)],
-	}
-	.assimilate_storage(&mut t)
-	.unwrap();
+	let mut balances: Vec<(AccountId, u128)> = vec![];
+	balances.append(&mut (1..=3).map(|i| (para_account_id(i), INITIAL_BALANCE)).collect());
+	balances.append(&mut (0..6).map(|i| ([i; 32].into(), INITIAL_BALANCE)).collect());
+
+	pallet_balances::GenesisConfig::<Runtime> { balances }
+		.assimilate_storage(&mut t)
+		.unwrap();
 
 	let mut ext = sp_io::TestExternalities::new(t);
 	ext.execute_with(|| System::set_block_number(1));
@@ -104,46 +144,70 @@ pub fn relay_ext() -> sp_io::TestExternalities {
 pub type RelayChainPalletXcm = pallet_xcm::Pallet<relay_chain::Runtime>;
 pub type ParachainPalletXcm = pallet_xcm::Pallet<parachain::Runtime>;
 
-fn run_one_input(mut data: &[u8]) {
+fn run_input(xcm_messages: [XcmMessage; 5]) {
 	MockNet::reset();
-	if let Ok(m) = Xcm::decode_all_with_depth_limit(MAX_XCM_DECODE_DEPTH, &mut data) {
-		#[cfg(not(fuzzing))]
-		{
-			println!("Executing message {:?}", m);
+
+	#[cfg(not(fuzzing))]
+	println!();
+
+	for xcm_message in xcm_messages {
+		if xcm_message.source % 4 == 0 {
+			// We get the destination for the message
+			let parachain_id = (xcm_message.destination % 3) + 1;
+			let destination: MultiLocation = Parachain(parachain_id).into();
+			#[cfg(not(fuzzing))]
+			{
+				println!("  source:      Relay Chain");
+				println!("  destination: Parachain {parachain_id}");
+				println!("  message:     {:?}", xcm_message.message);
+			}
+			Relay::execute_with(|| {
+				assert_ok!(RelayChainPalletXcm::send_xcm(Here, destination, xcm_message.message));
+			})
+		} else {
+			// We get the source's execution method
+			let execute_with = match xcm_message.source % 4 {
+				1 => ParaA::execute_with,
+				2 => ParaB::execute_with,
+				_ => ParaC::execute_with,
+			};
+			// We get the destination for the message
+			let destination: MultiLocation = match xcm_message.destination % 4 {
+				n @ 1..=3 => (Parent, Parachain(n)).into(),
+				_ => Parent.into(),
+			};
+			#[cfg(not(fuzzing))]
+			{
+				let destination_str = match xcm_message.destination % 4 {
+					n @ 1..=3 => format!("Parachain {n}"),
+					_ => "Relay Chain".to_string(),
+				};
+				println!("  source:      Parachain {}", xcm_message.source % 4);
+				println!("  destination: {}", destination_str);
+				println!("  message:     {:?}", xcm_message.message);
+			}
+			// We execute the message with the appropriate source and destination
+			execute_with(|| {
+				assert_ok!(ParachainPalletXcm::send_xcm(Here, destination, xcm_message.message));
+			});
 		}
-		ParaA::execute_with(|| {
-			assert_ok!(ParachainPalletXcm::send_xcm(Here, Parent, m));
-		});
-		Relay::execute_with(|| {});
+		#[cfg(not(fuzzing))]
+		println!();
 	}
+	Relay::execute_with(|| {});
 }
 
 fn main() {
 	#[cfg(fuzzing)]
 	{
 		loop {
-			honggfuzz::fuzz!(|data: &[u8]| {
-				run_one_input(data);
-			});
+			honggfuzz::fuzz!(|xcm_messages: [XcmMessage; 5]| {
+				run_input(xcm_messages);
+			})
 		}
 	}
 	#[cfg(not(fuzzing))]
 	{
-		//This code path can be used to generate a line-code coverage report in HTML
-		//that depicts which lines are executed by at least one input in the current fuzzing queue.
-		//To generate this code coverage report, run the following commands:
-		/*
-		```
-			export CARGO_INCREMENTAL=0
-			export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort"
-			export RUSTDOCFLAGS="-Cpanic=abort"
-			rustup override set nightly
-			SKIP_WASM_BUILD=1 cargo build
-			./xcm/xcm-simulator/fuzzer/target/debug/xcm-fuzzer hfuzz_workspace/xcm-fuzzer/input
-			zip -0 ccov.zip `find ../../target/debug \( -name "*.gc*" -o -name "test-*.gc*" \) -print`
-			grcov ccov.zip -s / -t html --llvm --branch --ignore-not-existing -o ../../target/debug/coverage/
-		```
-		*/
 		use std::{env, fs, fs::File, io::Read};
 		let args: Vec<_> = env::args().collect();
 		let md = fs::metadata(&args[1]).unwrap();
@@ -152,7 +216,7 @@ fn main() {
 				.unwrap()
 				.map(|x| x.unwrap().path().to_str().unwrap().to_string())
 				.collect::<Vec<String>>(),
-			false => (&args[1..]).to_vec(),
+			false => (args[1..]).to_vec(),
 		};
 		println!("All_files {:?}", all_files);
 		for argument in all_files {
@@ -160,7 +224,10 @@ fn main() {
 			let mut buffer: Vec<u8> = Vec::new();
 			let mut f = File::open(argument).unwrap();
 			f.read_to_end(&mut buffer).unwrap();
-			run_one_input(&buffer.as_slice());
+			let mut unstructured = Unstructured::new(&buffer);
+			if let Ok(xcm_messages) = unstructured.arbitrary() {
+				run_input(xcm_messages);
+			}
 		}
 	}
 }