Add async test helper to timeout and provide a task_executor automatically (#6651)

* Initial commit Forked at: 74655edebc Parent branch: origin/master * Add async test helper to timeout and provide a task_executor automatically * simplify error message to avoid difference between CI and locally * forgot env var * Use runtime env var instead of build env var * Rename variable to SUBSTRATE_TEST_TIMEOUT * CLEANUP Forked at: 74655edebc Parent branch: origin/master * Apply suggestions from code review Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com> * Re-export from test-utils * Default value to 120 * fix wrong crate in ci * Revert "Default value to 120" This reverts commit 8e458717078b242ffce7d3c4f66241d76f075125. * Fix version * WIP Forked at: 74655edebc Parent branch: origin/master * WIP Forked at: 74655edebc Parent branch: origin/master * WIP Forked at: 74655edebc Parent branch: origin/master * remove feature flag * fix missing dependency * CLEANUP Forked at: 74655edebc Parent branch: origin/master * fix test * Removed autotests=false * Some doc... * Apply suggestions from code review Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com> * WIP Forked at: 74655edebc Parent branch: origin/master * WIP Forked at: 74655edebc Parent branch: origin/master * Update test-utils/src/lib.rs Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
2026-06-13 22:11:06 +00:00 · 2020-08-12 16:07:11 +02:00
parent 9442c19b48
commit 93612c11f4
15 changed files with 375 additions and 1 deletions
@@ -263,6 +263,7 @@ test-linux-stable:                 &test-linux
  script:
    # this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests
    - time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path bin/node/cli/Cargo.toml
+    - WASM_BUILD_NO_COLOR=1 SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout
    - sccache -s

 unleash-check:
@@ -727,7 +728,7 @@ deploy-kubernetes-alerting-rules:
    RULES:                         .maintain/monitoring/alerting-rules/alerting-rules.yaml
  script:
    - echo "deploying prometheus alerting rules"
-    - kubectl -n ${NAMESPACE} patch prometheusrule ${PROMETHEUSRULE} 
+    - kubectl -n ${NAMESPACE} patch prometheusrule ${PROMETHEUSRULE}
        --type=merge --patch "$(sed 's/^/  /;1s/^/spec:\n/' ${RULES})"
  only:
    refs:
@@ -1186,6 +1186,12 @@ dependencies = [
 "winapi 0.3.8",
 ]

+[[package]]
+name = "dissimilar"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc4b29f4b9bb94bf267d57269fd0706d343a160937108e9619fe380645428abb"
+
 [[package]]
 name = "dns-parser"
 version = "0.8.0"
@@ -8605,6 +8611,31 @@ dependencies = [
 [[package]]
 name = "substrate-test-utils"
 version = "2.0.0-rc5"
+dependencies = [
+ "futures 0.3.5",
+ "sc-service",
+ "substrate-test-utils-derive",
+ "tokio 0.2.21",
+ "trybuild",
+]
+
+[[package]]
+name = "substrate-test-utils-derive"
+version = "0.8.0-rc5"
+dependencies = [
+ "proc-macro-crate",
+ "quote 1.0.6",
+ "syn 1.0.33",
+]
+
+[[package]]
+name = "substrate-test-utils-test-crate"
+version = "0.1.0"
+dependencies = [
+ "sc-service",
+ "substrate-test-utils",
+ "tokio 0.2.21",
+]

 [[package]]
 name = "substrate-wasm-builder"
@@ -9271,6 +9302,7 @@ version = "1.0.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "459186ab1afd6d93bd23c2269125f4f7694f8771fe0e64434b4bdc212b94034d"
 dependencies = [
+ "dissimilar",
 "glob 0.3.0",
 "lazy_static",
 "serde",
@@ -171,9 +171,11 @@ members = [
 	"primitives/utils",
 	"primitives/wasm-interface",
 	"test-utils/client",
+	"test-utils/derive",
 	"test-utils/runtime",
 	"test-utils/runtime/client",
 	"test-utils/runtime/transaction-pool",
+	"test-utils/test-crate",
 	"utils/browser",
 	"utils/build-script-utils",
 	"utils/fork-tree",
@@ -9,3 +9,12 @@ repository = "https://github.com/paritytech/substrate/"

 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
+
+[dependencies]
+futures = { version = "0.3.1", features = ["compat"] }
+substrate-test-utils-derive = { path = "./derive" }
+tokio = { version = "0.2.13", features = ["macros"] }
+
+[dev-dependencies]
+sc-service = { path = "../client/service" }
+trybuild = { version = "1.0", features = ["diff"] }
@@ -0,0 +1,16 @@
+[package]
+name = "substrate-test-utils-derive"
+version = "0.8.0-rc5"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+license = "Apache-2.0"
+homepage = "https://substrate.dev"
+repository = "https://github.com/paritytech/substrate/"
+
+[dependencies]
+quote = "1.0.6"
+syn = { version = "1.0.33", features = ["full"] }
+proc-macro-crate = "0.1.4"
+
+[lib]
+proc-macro = true
@@ -0,0 +1,107 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use proc_macro::{Span, TokenStream};
+use proc_macro_crate::crate_name;
+use quote::quote;
+use std::env;
+
+#[proc_macro_attribute]
+pub fn test(args: TokenStream, item: TokenStream) -> TokenStream {
+	impl_test(args, item)
+}
+
+fn impl_test(args: TokenStream, item: TokenStream) -> TokenStream {
+	let input = syn::parse_macro_input!(item as syn::ItemFn);
+	let args = syn::parse_macro_input!(args as syn::AttributeArgs);
+
+	parse_knobs(input, args).unwrap_or_else(|e| e.to_compile_error().into())
+}
+
+fn parse_knobs(
+	mut input: syn::ItemFn,
+	args: syn::AttributeArgs,
+) -> Result<TokenStream, syn::Error> {
+	let sig = &mut input.sig;
+	let body = &input.block;
+	let attrs = &input.attrs;
+	let vis = input.vis;
+
+	if sig.inputs.len() != 1 {
+		let msg = "the test function accepts only one argument of type sc_service::TaskExecutor";
+		return Err(syn::Error::new_spanned(&sig, msg));
+	}
+	let (task_executor_name, task_executor_type) = match sig.inputs.pop().map(|x| x.into_value()) {
+		Some(syn::FnArg::Typed(x)) => (x.pat, x.ty),
+		_ => {
+			let msg =
+				"the test function accepts only one argument of type sc_service::TaskExecutor";
+			return Err(syn::Error::new_spanned(&sig, msg));
+		}
+	};
+
+	let crate_name = if env::var("CARGO_PKG_NAME").unwrap() == "substrate-test-utils" {
+		syn::Ident::new("substrate_test_utils", Span::call_site().into())
+	} else {
+		let crate_name = crate_name("substrate-test-utils")
+			.map_err(|e| syn::Error::new_spanned(&sig, e))?;
+
+		syn::Ident::new(&crate_name, Span::call_site().into())
+	};
+
+	let header = {
+		quote! {
+			#[#crate_name::tokio::test(#(#args)*)]
+		}
+	};
+
+	let result = quote! {
+		#header
+		#(#attrs)*
+		#vis #sig {
+			use #crate_name::futures::future::FutureExt;
+
+			let #task_executor_name: #task_executor_type = (|fut, _| {
+				#crate_name::tokio::spawn(fut).map(drop)
+			})
+			.into();
+			let timeout_task = #crate_name::tokio::time::delay_for(
+				std::time::Duration::from_secs(
+					std::env::var("SUBSTRATE_TEST_TIMEOUT")
+						.ok()
+						.and_then(|x| x.parse().ok())
+						.unwrap_or(600))
+			).fuse();
+			let actual_test_task = async move {
+				#body
+			}
+			.fuse();
+
+			#crate_name::futures::pin_mut!(timeout_task, actual_test_task);
+
+			#crate_name::futures::select! {
+				_ = timeout_task => {
+					panic!("The test took too long!");
+				},
+				_ = actual_test_task => {},
+			}
+		}
+	};
+
+	Ok(result.into())
+}
@@ -17,6 +17,29 @@

 //! Test utils

+#[doc(hidden)]
+pub use futures;
+/// Marks async function to be executed by an async runtime and provide a `TaskExecutor`, suitable
+/// to test environment.
+///
+/// # Requirements
+///
+///	You must have tokio in the `[dev-dependencies]` of your crate to use this macro.
+///
+/// # Example
+///
+/// ```
+/// #[substrate_test_utils::test]
+/// async fn basic_test(task_executor: TaskExecutor) {
+///     assert!(true);
+///     // create your node in here and use task_executor
+///     // then don't forget to gracefully shutdown your node before exit
+/// }
+/// ```
+pub use substrate_test_utils_derive::test;
+#[doc(hidden)]
+pub use tokio;
+
 /// Panic when the vectors are different, without taking the order into account.
 ///
 /// # Examples
@@ -0,0 +1,16 @@
+[package]
+name = "substrate-test-utils-test-crate"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+license = "Apache-2.0"
+homepage = "https://substrate.dev"
+repository = "https://github.com/paritytech/substrate/"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[dev-dependencies]
+tokio = { version = "0.2.13", features = ["macros"] }
+test-utils = { path = "..", package = "substrate-test-utils" }
+sc-service = { path = "../../client/service" }
@@ -0,0 +1,25 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+#[cfg(test)]
+#[test_utils::test]
+async fn basic_test(_: sc_service::TaskExecutor) {
+	assert!(true);
+}
+
+fn main() {}
@@ -0,0 +1,58 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use sc_service::{TaskExecutor, TaskType};
+
+#[substrate_test_utils::test]
+async fn basic_test(_: TaskExecutor) {
+	assert!(true);
+}
+
+#[substrate_test_utils::test]
+#[should_panic(expected = "boo!")]
+async fn panicking_test(_: TaskExecutor) {
+	panic!("boo!");
+}
+
+#[substrate_test_utils::test(max_threads = 2)]
+async fn basic_test_with_args(_: TaskExecutor) {
+	assert!(true);
+}
+
+#[substrate_test_utils::test]
+async fn rename_argument(ex: TaskExecutor) {
+	let ex2 = ex.clone();
+	ex2.spawn(Box::pin(async { () }), TaskType::Blocking);
+	assert!(true);
+}
+
+#[substrate_test_utils::test]
+#[should_panic(expected = "test took too long")]
+// NOTE: enable this test only after setting SUBSTRATE_TEST_TIMEOUT to a smaller value
+//
+// SUBSTRATE_TEST_TIMEOUT=1 cargo test -- --ignored timeout
+#[ignore]
+async fn timeout(_: TaskExecutor) {
+	tokio::time::delay_for(std::time::Duration::from_secs(
+		std::env::var("SUBSTRATE_TEST_TIMEOUT")
+			.expect("env var SUBSTRATE_TEST_TIMEOUT has been provided by the user")
+			.parse::<u64>()
+			.unwrap() + 1,
+	))
+	.await;
+}
@@ -0,0 +1,24 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+#[test]
+fn substrate_test_utils_derive_trybuild() {
+	let t = trybuild::TestCases::new();
+	t.compile_fail("tests/ui/missing-func-parameter.rs");
+	t.compile_fail("tests/ui/too-many-func-parameters.rs");
+}
@@ -0,0 +1,24 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+#[substrate_test_utils::test]
+async fn missing_func_parameter() {
+	assert!(true);
+}
+
+fn main() {}
@@ -0,0 +1,5 @@
+error: the test function accepts only one argument of type sc_service::TaskExecutor
+  --> $DIR/missing-func-parameter.rs:20:1
+   |
+20 | async fn missing_func_parameter() {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -0,0 +1,27 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+#[allow(unused_imports)]
+use sc_service::TaskExecutor;
+
+#[substrate_test_utils::test]
+async fn too_many_func_parameters(task_executor_1: TaskExecutor, task_executor_2: TaskExecutor) {
+	assert!(true);
+}
+
+fn main() {}
@@ -0,0 +1,5 @@
+error: the test function accepts only one argument of type sc_service::TaskExecutor
+  --> $DIR/too-many-func-parameters.rs:23:1
+   |
+23 | async fn too_many_func_parameters(task_executor_1: TaskExecutor, task_executor_2: TaskExecutor) {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^