Run timeout-based tests only in CI (#7390)

* Introduce `ci-only-tests` feature

* Add comment

polkadot/Cargo.lock

@@ -6824,6 +6824,7 @@ dependencies = [
  "nix 0.26.2",
  "polkadot-cli",
  "polkadot-core-primitives",
+ "polkadot-node-core-pvf",
  "polkadot-node-core-pvf-prepare-worker",
  "polkadot-overseer",
  "substrate-rpc-client",

polkadot/Cargo.toml

@@ -24,6 +24,7 @@ tikv-jemallocator = "0.5.0"
 
 # Crates in our workspace, defined as dependencies so we can pass them feature flags.
 polkadot-cli = { path = "cli", features = [ "polkadot-native", "kusama-native", "westend-native", "rococo-native" ]  }
+polkadot-node-core-pvf = { path = "node/core/pvf" }
 polkadot-node-core-pvf-prepare-worker = { path = "node/core/pvf/prepare-worker" }
 polkadot-overseer = { path = "node/overseer" }
 
@@ -210,6 +211,10 @@ runtime-metrics = [ "polkadot-cli/runtime-metrics" ]
 pyroscope = ["polkadot-cli/pyroscope"]
 jemalloc-allocator = ["polkadot-node-core-pvf-prepare-worker/jemalloc-allocator", "polkadot-overseer/jemalloc-allocator"]
 
+# Enables timeout-based tests supposed to be run only in CI environment as they may be flaky
+# when run locally depending on system load
+ci-only-tests = ["polkadot-node-core-pvf/ci-only-tests"]
+
 # Configuration for building a .deb package - for use with `cargo-deb`
 [package.metadata.deb]
 name = "polkadot"

polkadot/node/core/pvf/Cargo.toml

@@ -45,3 +45,6 @@ hex-literal = "0.3.4"
 
 adder = { package = "test-parachain-adder", path = "../../../parachain/test-parachains/adder" }
 halt = { package = "test-parachain-halt", path = "../../../parachain/test-parachains/halt" }
+
+[features]
+ci-only-tests = []

polkadot/node/core/pvf/tests/it/main.rs

@@ -14,6 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
+#[cfg(feature = "ci-only-tests")]
 use assert_matches::assert_matches;
 use parity_scale_codec::Encode as _;
 use polkadot_node_core_pvf::{
@@ -21,7 +22,11 @@ use polkadot_node_core_pvf::{
 	ValidationHost, JOB_TIMEOUT_WALL_CLOCK_FACTOR,
 };
 use polkadot_parachain::primitives::{BlockData, ValidationParams, ValidationResult};
-use polkadot_primitives::{ExecutorParam, ExecutorParams};
+use polkadot_primitives::ExecutorParams;
+
+#[cfg(feature = "ci-only-tests")]
+use polkadot_primitives::ExecutorParam;
+
 use std::time::Duration;
 use tokio::sync::Mutex;
 
@@ -115,6 +120,7 @@ async fn terminates_on_timeout() {
 	assert!(duration < TEST_EXECUTION_TIMEOUT * JOB_TIMEOUT_WALL_CLOCK_FACTOR);
 }
 
+#[cfg(feature = "ci-only-tests")]
 #[tokio::test]
 async fn ensure_parallel_execution() {
 	// Run some jobs that do not complete, thus timing out.
@@ -197,6 +203,7 @@ async fn execute_queue_doesnt_stall_if_workers_died() {
 	);
 }
 
+#[cfg(feature = "ci-only-tests")]
 #[tokio::test]
 async fn execute_queue_doesnt_stall_with_varying_executor_params() {
 	let host = TestHost::new_with_config(|cfg| {

polkadot/scripts/ci/gitlab/pipeline/test.yml

@@ -41,7 +41,7 @@ test-linux-stable:
     # but still want to have debug assertions.
     RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
   script:
-    - time cargo test --workspace --profile testnet --verbose --locked --features=runtime-benchmarks,runtime-metrics,try-runtime
+    - time cargo test --workspace --profile testnet --verbose --locked --features=runtime-benchmarks,runtime-metrics,try-runtime,ci-only-tests
 
 test-linux-oldkernel-stable:
   extends: test-linux-stable