Add expectations tests to the CI action

.github/actions/run-differential-tests/action.yml

@@ -41,6 +41,10 @@ inputs:
     description: "The id of the parachain to spawn with the polkadot-omni-node. This is only required if the polkadot-omni-node is one of the selected platforms."
     type: number
     required: false
+  expectations-file-path:
+    description: "Path to the expectations file to use to compare against."
+    type: string
+    required: false
 
 runs:
   using: "composite"
@@ -79,6 +83,12 @@ runs:
       run: |
         ${{ inputs['cargo-command'] }} build --locked --profile release -p pallet-revive-eth-rpc -p revive-dev-node --manifest-path ${{ inputs['polkadot-sdk-path'] }}/Cargo.toml
         ${{ inputs['cargo-command'] }} build --locked --profile release --bin polkadot-omni-node --manifest-path ${{ inputs['polkadot-sdk-path'] }}/Cargo.toml
+    - name: Installing retester
+      shell: bash
+      run: ${{ inputs['cargo-command'] }} install --path ./revive-differential-tests/crates/core
+    - name: Installing report-processor
+      shell: bash
+      run: ${{ inputs['cargo-command'] }} install --path ./revive-differential-tests/crates/report-processor
     - name: Running the Differential Tests
       shell: bash
       run: |
@@ -96,7 +106,7 @@ runs:
           )
         fi
 
-        ${{ inputs['cargo-command'] }} run --locked --manifest-path revive-differential-tests/Cargo.toml -- test \
+        retester test \
           --test ./revive-differential-tests/resolc-compiler-tests/fixtures/solidity/simple \
           --test ./revive-differential-tests/resolc-compiler-tests/fixtures/solidity/complex \
           --test ./revive-differential-tests/resolc-compiler-tests/fixtures/solidity/translated_semantic_tests \
@@ -110,22 +120,22 @@ runs:
           --eth-rpc.path ${{ inputs['polkadot-sdk-path'] }}/target/release/eth-rpc \
           --polkadot-omni-node.path ${{ inputs['polkadot-sdk-path'] }}/target/release/polkadot-omni-node \
           --resolc.path ./resolc \
-          "${OMNI_ARGS[@]}"
+          --report.file-name ${{ inputs['platform'] }}.json \
-    - name: Creating a markdown report of the test execution
+          "${OMNI_ARGS[@]}" || true
+    - name: Generate the expectation file
       shell: bash
-      if: ${{ always() }}
+      run: report-processor --report-path ./revive-differential-tests/workdir/${{ inputs['platform'] }}.json --output-path ${{ inputs['platform'] }}.json --remove-prefix ./revive-differential-tests/resolc-compiler-tests
-      run: |
-        mv ./workdir/*.json report.json
-        python3 revive-differential-tests/scripts/process-differential-tests-report.py report.json ${{ inputs['platform'] }}
     - name: Upload the Report to the CI
       uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
-      if: ${{ always() }}
       with:
-        name: report-${{ inputs['platform'] }}.md
+        name: ${{ inputs['platform'] }}-report.json
-        path: report.md
+        path: ./revive-differential-tests/workdir/${{ inputs['platform'] }}.json
-    - name: Posting the report as a comment on the PR
+    - name: Upload the Report to the CI
-      uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405
+      uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
-      if: ${{ always() }}
       with:
-        header: diff-tests-report-${{ inputs['platform'] }}
+        name: ${{ inputs['platform'] }}.json
-        path: report.md
+        path: ./${{ inputs['platform'] }}.json
+    - name: Check Expectations
+      shell: bash
+      if: ${{ inputs['expectations-file-path'] != '' }}
+      run: report-processor compare-expectation-files --base-expectation-path ${{ inputs['expectations-file-path'] }} --other-expectation-path ${{ inputs['platform'] }}.json

crates/config/src/lib.rs

@@ -1113,6 +1113,10 @@ pub struct ReportConfiguration {
     /// Controls if the compiler output is included in the final report.
     #[clap(long = "report.include-compiler-output")]
     pub include_compiler_output: bool,
+
+    /// The filename to use for the report.
+    #[clap(long = "report.file-name")]
+    pub file_name: Option<String>,
 }
 
 #[derive(Clone, Debug, Parser, Serialize, Deserialize)]

crates/report/src/aggregator.rs

@@ -36,6 +36,8 @@ pub struct ReportAggregator {
     runner_tx: Option<UnboundedSender<RunnerEvent>>,
     runner_rx: UnboundedReceiver<RunnerEvent>,
     listener_tx: Sender<ReporterEvent>,
+    /* Context */
+    file_name: Option<String>,
 }
 
 impl ReportAggregator {
@@ -43,6 +45,11 @@ impl ReportAggregator {
         let (runner_tx, runner_rx) = unbounded_channel::<RunnerEvent>();
         let (listener_tx, _) = channel::<ReporterEvent>(0xFFFF);
         Self {
+            file_name: match context {
+                Context::Test(ref context) => context.report_configuration.file_name.clone(),
+                Context::Benchmark(ref context) => context.report_configuration.file_name.clone(),
+                Context::ExportJsonSchema | Context::ExportGenesis(..) => None,
+            },
             report: Report::new(context),
             remaining_cases: Default::default(),
             runner_tx: Some(runner_tx),
@@ -121,7 +128,7 @@ impl ReportAggregator {
         self.handle_completion(CompletionEvent {});
         debug!("Report aggregation completed");
 
-        let file_name = {
+        let default_file_name = {
             let current_timestamp = SystemTime::now()
                 .duration_since(UNIX_EPOCH)
                 .context("System clock is before UNIX_EPOCH; cannot compute report timestamp")?
@@ -130,6 +137,7 @@ impl ReportAggregator {
             file_name.push_str(".json");
             file_name
         };
+        let file_name = self.file_name.unwrap_or(default_file_name);
         let file_path = self
             .report
             .context