Increase substrate's timeout

* Add support for profiles

* Set a default workdir for debug profile

crates/config/src/lib.rs

@@ -47,6 +47,15 @@ impl Context {
     pub fn report_configuration(&self) -> &ReportConfiguration {
         self.as_ref()
     }
+
+    pub fn update_for_profile(&mut self) {
+        match self {
+            Context::Test(ctx) => ctx.update_for_profile(),
+            Context::Benchmark(ctx) => ctx.update_for_profile(),
+            Context::ExportJsonSchema => {}
+            Context::ExportGenesis(..) => {}
+        }
+    }
 }
 
 impl AsRef<WorkingDirectoryConfiguration> for Context {
@@ -220,6 +229,11 @@ impl AsRef<IgnoreSuccessConfiguration> for Context {
 
 #[derive(Clone, Debug, Parser, Serialize, Deserialize)]
 pub struct TestExecutionContext {
+    /// The commandline profile to use. Different profiles change the defaults of the various cli
+    /// arguments.
+    #[arg(long = "profile", default_value_t = Profile::Default)]
+    pub profile: Profile,
+
     /// The set of platforms that the differential tests should run on.
     #[arg(
         short = 'p',
@@ -306,8 +320,49 @@ pub struct TestExecutionContext {
     pub ignore_success_configuration: IgnoreSuccessConfiguration,
 }
 
+impl TestExecutionContext {
+    pub fn update_for_profile(&mut self) {
+        match self.profile {
+            Profile::Default => {}
+            Profile::Debug => {
+                let default_concurrency_config =
+                    ConcurrencyConfiguration::parse_from(["concurrency-configuration"]);
+                let working_directory_config = WorkingDirectoryConfiguration::default();
+
+                if self.concurrency_configuration.number_of_nodes
+                    == default_concurrency_config.number_of_nodes
+                {
+                    self.concurrency_configuration.number_of_nodes = 1;
+                }
+                if self.concurrency_configuration.number_of_threads
+                    == default_concurrency_config.number_of_threads
+                {
+                    self.concurrency_configuration.number_of_threads = 5;
+                }
+                if self.concurrency_configuration.number_concurrent_tasks
+                    == default_concurrency_config.number_concurrent_tasks
+                {
+                    self.concurrency_configuration.number_concurrent_tasks = 1;
+                }
+
+                if working_directory_config == self.working_directory {
+                    let home_directory =
+                        PathBuf::from(std::env::var("HOME").expect("Home dir not found"));
+                    let working_directory = home_directory.join(".retester-workdir");
+                    self.working_directory = WorkingDirectoryConfiguration::Path(working_directory)
+                }
+            }
+        }
+    }
+}
+
 #[derive(Clone, Debug, Parser, Serialize, Deserialize)]
 pub struct BenchmarkingContext {
+    /// The commandline profile to use. Different profiles change the defaults of the various cli
+    /// arguments.
+    #[arg(long = "profile", default_value_t = Profile::Default)]
+    pub profile: Profile,
+
     /// The working directory that the program will use for all of the temporary artifacts needed at
     /// runtime.
     ///
@@ -387,6 +442,42 @@ pub struct BenchmarkingContext {
     pub report_configuration: ReportConfiguration,
 }
 
+impl BenchmarkingContext {
+    pub fn update_for_profile(&mut self) {
+        match self.profile {
+            Profile::Default => {}
+            Profile::Debug => {
+                let default_concurrency_config =
+                    ConcurrencyConfiguration::parse_from(["concurrency-configuration"]);
+                let working_directory_config = WorkingDirectoryConfiguration::default();
+
+                if self.concurrency_configuration.number_of_nodes
+                    == default_concurrency_config.number_of_nodes
+                {
+                    self.concurrency_configuration.number_of_nodes = 1;
+                }
+                if self.concurrency_configuration.number_of_threads
+                    == default_concurrency_config.number_of_threads
+                {
+                    self.concurrency_configuration.number_of_threads = 5;
+                }
+                if self.concurrency_configuration.number_concurrent_tasks
+                    == default_concurrency_config.number_concurrent_tasks
+                {
+                    self.concurrency_configuration.number_concurrent_tasks = 1;
+                }
+
+                if working_directory_config == self.working_directory {
+                    let home_directory =
+                        PathBuf::from(std::env::var("HOME").expect("Home dir not found"));
+                    let working_directory = home_directory.join(".retester-workdir");
+                    self.working_directory = WorkingDirectoryConfiguration::Path(working_directory)
+                }
+            }
+        }
+    }
+}
+
 #[derive(Clone, Debug, Parser, Serialize, Deserialize)]
 pub struct ExportGenesisContext {
     /// The platform of choice to export the genesis for.
@@ -989,7 +1080,7 @@ pub struct IgnoreSuccessConfiguration {
 }
 
 /// Represents the working directory that the program uses.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub enum WorkingDirectoryConfiguration {
     /// A temporary directory is used as the working directory. This will be removed when dropped.
     TemporaryDirectory(Arc<TempDir>),
@@ -1125,3 +1216,39 @@ pub enum OutputFormat {
     /// An output format that looks heavily resembles the output from `cargo test`.
     CargoTestLike,
 }
+
+/// Command line profiles used to override the default values provided for the commands.
+#[derive(
+    Clone,
+    Copy,
+    Debug,
+    Default,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Hash,
+    Serialize,
+    Deserialize,
+    ValueEnum,
+    EnumString,
+    Display,
+    AsRefStr,
+    IntoStaticStr,
+)]
+#[strum(serialize_all = "kebab-case")]
+pub enum Profile {
+    /// The default profile used by the framework. This profile is optimized to make the test
+    /// and workload execution happen as fast as possible.
+    #[default]
+    Default,
+
+    /// A debug profile optimized for use cases when certain tests are being debugged. This profile
+    /// sets up the framework with the following:
+    ///
+    /// * `concurrency.number-of-nodes` set to 1 node.
+    /// * `concurrency.number-of-concurrent-tasks` set to 1 such that tests execute sequentially.
+    /// * `concurrency.number-of-threads` set to 5.
+    /// * `working-directory` set to ~/.retester-workdir
+    Debug,
+}

crates/core/src/differential_tests/entry_point.rs

@@ -205,6 +205,7 @@ pub async fn handle_differential_tests(
                 ?remaining_tasks,
                 "Remaining Tests"
             );
+            drop(remaining_tasks);
             tokio::time::sleep(Duration::from_secs(10)).await
         }
     });

crates/core/src/main.rs

@@ -6,7 +6,7 @@ use anyhow::Context as _;
 use clap::Parser;
 use revive_dt_report::ReportAggregator;
 use schemars::schema_for;
-use tracing::info;
+use tracing::{info, level_filters::LevelFilter};
 use tracing_subscriber::{EnvFilter, FmtSubscriber};
 
 use revive_dt_config::Context;
@@ -31,14 +31,20 @@ fn main() -> anyhow::Result<()> {
         .with_writer(writer)
         .with_thread_ids(false)
         .with_thread_names(false)
-        .with_env_filter(EnvFilter::from_default_env())
+        .with_env_filter(
+            EnvFilter::builder()
+                .with_default_directive(LevelFilter::OFF.into())
+                .from_env_lossy(),
+        )
         .with_ansi(false)
         .pretty()
         .finish();
     tracing::subscriber::set_global_default(subscriber)?;
     info!("Differential testing tool is starting");
 
-    let context = Context::try_parse()?;
+    let mut context = Context::try_parse()?;
+    context.update_for_profile();
+
     let (reporter, report_aggregator_task) = ReportAggregator::new(context.clone()).into_task();
 
     match context {

crates/node/src/node_implementations/substrate.rs

@@ -218,7 +218,7 @@ impl SubstrateNode {
                 }
             },
             ProcessReadinessWaitBehavior::TimeBoundedWaitFunction {
-                max_wait_duration: Duration::from_secs(30),
+                max_wait_duration: Duration::from_secs(90),
                 check_function: Box::new(|_, stderr_line| match stderr_line {
                     Some(line) => Ok(line.contains(Self::SUBSTRATE_READY_MARKER)),
                     None => Ok(false),