Generate schema for the metadata file

crates/format/src/case.rs

@@ -1,3 +1,4 @@
+use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 
 use revive_dt_common::{macros::define_wrapper_type, types::Mode};
@@ -7,26 +8,48 @@ use crate::{
     mode::ParsedMode,
 };
 
-#[derive(Debug, Default, Serialize, Deserialize, Clone, Eq, PartialEq)]
+#[derive(Debug, Default, Serialize, Deserialize, Clone, Eq, PartialEq, JsonSchema)]
 pub struct Case {
+    /// An optional name of the test case.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub name: Option<String>,
 
+    /// An optional comment on the case which has no impact on the execution in any way.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub comment: Option<String>,
 
+    /// This represents a mode that has been parsed from test metadata.
+    ///
+    /// Mode strings can take the following form (in pseudo-regex):
+    ///
+    /// ```text
+    /// [YEILV][+-]? (M[0123sz])? <semver>?
+    /// ```
+    ///
+    /// If this is provided then it takes higher priority than the modes specified in the metadata
+    /// file.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub modes: Option<Vec<ParsedMode>>,
 
+    /// The set of steps to run as part of this test case.
     #[serde(rename = "inputs")]
     pub steps: Vec<Step>,
 
+    /// An optional name of the group of tests that this test belongs to.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub group: Option<String>,
 
+    /// An optional set of expectations and assertions to make about the transaction after it ran.
+    ///
+    /// If this is not specified then the only assertion that will be ran is that the transaction
+    /// was successful.
+    ///
+    /// This expectation that's on the case itself will be attached to the final step of the case.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub expected: Option<Expected>,
 
+    /// An optional boolean which defines if the case as a whole should be ignored. If null then the
+    /// case will not be ignored.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub ignore: Option<bool>,
 }

crates/format/src/input.rs

@@ -10,6 +10,7 @@ use alloy::{
 use alloy_primitives::{FixedBytes, utils::parse_units};
 use anyhow::Context as _;
 use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt, stream};
+use schemars::JsonSchema;
 use semver::VersionReq;
 use serde::{Deserialize, Serialize};
 
@@ -23,7 +24,7 @@ use crate::{metadata::ContractInstance, traits::ResolutionContext};
 ///
 /// A test step can be anything. It could be an invocation to a function, an assertion, or any other
 /// action that needs to be run or executed on the nodes used in the tests.
-#[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq)]
+#[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq, JsonSchema)]
 #[serde(untagged)]
 pub enum Step {
     /// A function call or an invocation to some function on some smart contract.
@@ -39,36 +40,51 @@ define_wrapper_type!(
     pub struct StepIdx(usize) impl Display;
 );
 
-#[derive(Clone, Debug, Default, Serialize, Deserialize, Eq, PartialEq)]
+/// This is an input step which is a transaction description that the framework translates into a
+/// transaction and executes on the nodes.
+#[derive(Clone, Debug, Default, Serialize, Deserialize, Eq, PartialEq, JsonSchema)]
 pub struct Input {
+    /// The address of the account performing the call and paying the fees for it.
     #[serde(default = "Input::default_caller")]
+    #[schemars(with = "String")]
     pub caller: Address,
 
+    /// An optional comment on the step which has no impact on the execution in any way.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub comment: Option<String>,
 
+    /// The contract instance that's being called in this transaction step.
     #[serde(default = "Input::default_instance")]
     pub instance: ContractInstance,
 
+    /// The method that's being called in this step.
     pub method: Method,
 
+    /// The calldata that the function should be invoked with.
     #[serde(default)]
     pub calldata: Calldata,
 
+    /// A set of assertions and expectations to have for the transaction.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub expected: Option<Expected>,
 
+    /// An optional value to provide as part of the transaction.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub value: Option<EtherValue>,
 
     #[serde(skip_serializing_if = "Option::is_none")]
+    #[schemars(skip)]
     pub storage: Option<HashMap<String, Calldata>>,
 
+    /// Variable assignment to perform in the framework allowing us to reference them again later on
+    /// during the execution.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub variable_assignments: Option<VariableAssignments>,
 }
 
-#[derive(Clone, Debug, Default, Serialize, Deserialize, Eq, PartialEq)]
+/// This represents a balance assertion step where the framework needs to query the balance of some
+/// account or contract and assert that it's some amount.
+#[derive(Clone, Debug, Default, Serialize, Deserialize, Eq, PartialEq, JsonSchema)]
 pub struct BalanceAssertion {
     /// An optional comment on the balance assertion.
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -82,11 +98,13 @@ pub struct BalanceAssertion {
     /// followed in the calldata.
     pub address: String,
 
-    /// The amount of balance to assert that the account or contract has.
+    /// The amount of balance to assert that the account or contract has. This is a 256 bit string
+    /// that's serialized and deserialized into a decimal string.
+    #[schemars(with = "String")]
     pub expected_balance: U256,
 }
 
-#[derive(Clone, Debug, Default, Serialize, Deserialize, Eq, PartialEq)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize, Eq, PartialEq, JsonSchema)]
 pub struct StorageEmptyAssertion {
     /// An optional comment on the storage empty assertion.
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -104,31 +122,52 @@ pub struct StorageEmptyAssertion {
     pub is_storage_empty: bool,
 }
 
-#[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq)]
+/// A set of expectations and assertions to make about the transaction after it ran.
+///
+/// If this is not specified then the only assertion that will be ran is that the transaction
+/// was successful.
+#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Eq, PartialEq)]
 #[serde(untagged)]
 pub enum Expected {
+    /// An assertion that the transaction succeeded and returned the provided set of data.
     Calldata(Calldata),
+    /// A more complex assertion.
     Expected(ExpectedOutput),
+    /// A set of assertions.
     ExpectedMany(Vec<ExpectedOutput>),
 }
 
-#[derive(Clone, Debug, Default, Serialize, Deserialize, Eq, PartialEq)]
+/// A set of assertions to run on the transaction.
+#[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema, Eq, PartialEq)]
 pub struct ExpectedOutput {
+    /// An optional compiler version that's required in order for this assertion to run.
     #[serde(skip_serializing_if = "Option::is_none")]
+    #[schemars(with = "Option<String>")]
     pub compiler_version: Option<VersionReq>,
+
+    /// An optional field of the expected returns from the invocation.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub return_data: Option<Calldata>,
+
+    /// An optional set of assertions to run on the emitted events from the transaction.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub events: Option<Vec<Event>>,
+
+    /// A boolean which defines whether we expect the transaction to succeed or fail.
     #[serde(default)]
     pub exception: bool,
 }
 
-#[derive(Clone, Debug, Default, Serialize, Deserialize, Eq, PartialEq)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema, Eq, PartialEq)]
 pub struct Event {
+    /// An optional field of the address of the emitter of the event.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub address: Option<String>,
+
+    /// The set of topics to expect the event to have.
     pub topics: Vec<String>,
+
+    /// The set of values to expect the event to have.
     pub values: Calldata,
 }
 
@@ -183,16 +222,17 @@ pub struct Event {
 /// [`Single`]: Calldata::Single
 /// [`Compound`]: Calldata::Compound
 /// [reverse polish notation]: https://en.wikipedia.org/wiki/Reverse_Polish_notation
-#[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq)]
+#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Eq, PartialEq)]
 #[serde(untagged)]
 pub enum Calldata {
-    Single(Bytes),
+    Single(#[schemars(with = "String")] Bytes),
     Compound(Vec<CalldataItem>),
 }
 
 define_wrapper_type! {
-    /// This represents an item in the [`Calldata::Compound`] variant.
-    #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+    /// This represents an item in the [`Calldata::Compound`] variant. Each item will be resolved
+    /// according to the resolution rules of the tool.
+    #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema)]
     #[serde(transparent)]
     pub struct CalldataItem(String) impl Display;
 }
@@ -217,7 +257,7 @@ enum Operation {
 }
 
 /// Specify how the contract is called.
-#[derive(Debug, Default, Serialize, Deserialize, Clone, Eq, PartialEq)]
+#[derive(Debug, Default, Serialize, Deserialize, JsonSchema, Clone, Eq, PartialEq)]
 pub enum Method {
     /// Initiate a deploy transaction, calling contracts constructor.
     ///
@@ -238,11 +278,16 @@ pub enum Method {
 }
 
 define_wrapper_type!(
-    #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
+    /// Defines an Ether value.
+    ///
+    /// This is an unsigned 256 bit integer that's followed by some denomination which can either be
+    /// eth, ether, gwei, or wei.
+    #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash, JsonSchema)]
+    #[schemars(with = "String")]
     pub struct EtherValue(U256) impl Display;
 );
 
-#[derive(Clone, Debug, Default, Serialize, Deserialize, Eq, PartialEq)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize, Eq, PartialEq, JsonSchema)]
 pub struct VariableAssignments {
     /// A vector of the variable names to assign to the return data.
     ///

crates/format/src/metadata.rs

@@ -8,6 +8,7 @@ use std::{
     str::FromStr,
 };
 
+use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 
 use revive_common::EVMVersion;
@@ -56,30 +57,62 @@ impl Deref for MetadataFile {
     }
 }
 
-#[derive(Debug, Default, Serialize, Deserialize, Clone, Eq, PartialEq)]
+/// A MatterLabs metadata file.
+///
+/// This defines the structure that the MatterLabs metadata files follow for defining the tests or
+/// the workloads.
+///
+/// Each metadata file is composed of multiple test cases where each test case is isolated from the
+/// others and runs in a completely different address space. Each test case is composed of a number
+/// of steps and assertions that should be performed as part of the test case.
+#[derive(Debug, Default, Serialize, Deserialize, JsonSchema, Clone, Eq, PartialEq)]
 pub struct Metadata {
-    /// A comment on the test case that's added for human-readability.
+    /// This is an optional comment on the metadata file which has no impact on the execution in any
+    /// way.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub comment: Option<String>,
 
+    /// An optional boolean which defines if the metadata file as a whole should be ignored. If null
+    /// then the metadata file will not be ignored.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub ignore: Option<bool>,
 
+    /// An optional vector of targets that this Metadata file's cases can be executed on. As an
+    /// example, if we wish for the metadata file's cases to only be run on PolkaVM then we'd
+    /// specify a target of "PolkaVM" in here.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub targets: Option<Vec<String>>,
 
+    /// A vector of the test cases and workloads contained within the metadata file. This is their
+    /// primary description.
     pub cases: Vec<Case>,
 
+    /// A map of all of the contracts that the test requires to run.
+    ///
+    /// This is a map where the key is the name of the contract instance and the value is the
+    /// contract's path and ident in the file.
+    ///
+    /// If any contract is to be used by the test then it must be included in here first so that the
+    /// framework is aware of its path, compiles it, and prepares it.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub contracts: Option<BTreeMap<ContractInstance, ContractPathAndIdent>>,
 
+    /// The set of libraries that this metadata file requires.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub libraries: Option<BTreeMap<PathBuf, BTreeMap<ContractIdent, ContractInstance>>>,
 
+    /// This represents a mode that has been parsed from test metadata.
+    ///
+    /// Mode strings can take the following form (in pseudo-regex):
+    ///
+    /// ```text
+    /// [YEILV][+-]? (M[0123sz])? <semver>?
+    /// ```
     #[serde(skip_serializing_if = "Option::is_none")]
     pub modes: Option<Vec<ParsedMode>>,
 
     #[serde(skip_serializing_if = "Option::is_none")]
+    #[schemars(skip)]
     pub file_path: Option<PathBuf>,
 
     /// This field specifies an EVM version requirement that the test case has where the test might
@@ -87,9 +120,9 @@ pub struct Metadata {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub required_evm_version: Option<EvmVersionRequirement>,
 
-    /// A set of compilation directives that will be passed to the compiler whenever the contracts for
-    /// the test are being compiled. Note that this differs from the [`Mode`]s in that a [`Mode`] is
-    /// just a filter for when a test can run whereas this is an instruction to the compiler.
+    /// A set of compilation directives that will be passed to the compiler whenever the contracts
+    /// for the test are being compiled. Note that this differs from the [`Mode`]s in that a [`Mode`]
+    /// is just a filter for when a test can run whereas this is an instruction to the compiler.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub compiler_directives: Option<CompilationDirectives>,
 }
@@ -262,7 +295,7 @@ define_wrapper_type!(
     ///
     /// Typically, this is used as the key to the "contracts" field of metadata files.
     #[derive(
-        Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize,
+        Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema
     )]
     #[serde(transparent)]
     pub struct ContractInstance(String) impl Display;
@@ -273,7 +306,7 @@ define_wrapper_type!(
     ///
     /// A contract identifier is the name of the contract in the source code.
     #[derive(
-        Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize,
+        Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema
     )]
     #[serde(transparent)]
     pub struct ContractIdent(String) impl Display;
@@ -286,7 +319,9 @@ define_wrapper_type!(
 /// ```text
 /// ${path}:${contract_ident}
 /// ```
-#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+#[derive(
+    Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema,
+)]
 #[serde(try_from = "String", into = "String")]
 pub struct ContractPathAndIdent {
     /// The path of the contract source code relative to the directory containing the metadata file.
@@ -363,9 +398,15 @@ impl From<ContractPathAndIdent> for String {
     }
 }
 
-/// An EVM version requirement that the test case has. This gets serialized and
-/// deserialized from and into [`String`].
-#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+/// An EVM version requirement that the test case has. This gets serialized and deserialized from
+/// and into [`String`]. This follows a simple format of (>=|<=|=|>|<) followed by a string of the
+/// EVM version.
+///
+/// When specified, the framework will only run the test if the node's EVM version matches that
+/// required by the metadata file.
+#[derive(
+    Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema,
+)]
 #[serde(try_from = "String", into = "String")]
 pub struct EvmVersionRequirement {
     ordering: Ordering,
@@ -493,7 +534,18 @@ impl From<EvmVersionRequirement> for String {
 /// just a filter for when a test can run whereas this is an instruction to the compiler.
 /// Defines how the compiler should handle revert strings.
 #[derive(
-    Clone, Debug, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default, Serialize, Deserialize,
+    Clone,
+    Debug,
+    Copy,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Hash,
+    Default,
+    Serialize,
+    Deserialize,
+    JsonSchema,
 )]
 pub struct CompilationDirectives {
     /// Defines how the revert strings should be handled.
@@ -502,14 +554,29 @@ pub struct CompilationDirectives {
 
 /// Defines how the compiler should handle revert strings.
 #[derive(
-    Clone, Debug, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default, Serialize, Deserialize,
+    Clone,
+    Debug,
+    Copy,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Hash,
+    Default,
+    Serialize,
+    Deserialize,
+    JsonSchema,
 )]
 #[serde(rename_all = "camelCase")]
 pub enum RevertString {
+    /// The default handling of the revert strings.
     #[default]
     Default,
+    /// The debug handling of the revert strings.
     Debug,
+    /// Strip the revert strings.
     Strip,
+    /// Provide verbose debug strings for the revert string.
     VerboseDebug,
 }
 

crates/format/src/mode.rs

@@ -2,6 +2,7 @@ use anyhow::Context as _;
 use regex::Regex;
 use revive_dt_common::iterators::EitherIter;
 use revive_dt_common::types::{Mode, ModeOptimizerSetting, ModePipeline};
+use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use std::collections::HashSet;
 use std::fmt::Display;
@@ -17,7 +18,7 @@ use std::sync::LazyLock;
 /// ```
 ///
 /// We can parse valid mode strings into [`ParsedMode`] using [`ParsedMode::from_str`].
-#[derive(Clone, Debug, PartialEq, Eq, Hash, Deserialize, Serialize)]
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Deserialize, Serialize, JsonSchema)]
 #[serde(try_from = "String", into = "String")]
 pub struct ParsedMode {
     pub pipeline: Option<ModePipeline>,