Merge branch 'main' into make_stats

.github/workflows/ci_levm.yaml

@@ -41,7 +41,7 @@ jobs:
       - name: Run tests
         run: |
           cd crates/vm/levm
-          make run-evm-ef-tests
+          make run-evm-ef-tests-ci
   test:
     # "Test" is a required check, don't change the name
     name: Test

cmd/ef_tests/levm/parser.rs

@@ -2,6 +2,7 @@ use crate::{
     report::format_duration_as_mm_ss,
     runner::EFTestRunnerOptions,
     types::{EFTest, EFTests},
+    utils::{spinner_success_or_print, spinner_update_text_or_print},
 };
 use colored::Colorize;
 use spinoff::{spinners::Dots, Color, Spinner};
@@ -24,6 +25,9 @@ pub fn parse_ef_tests(opts: &EFTestRunnerOptions) -> Result<Vec<EFTest>, EFTestP
     let cargo_manifest_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
     let ef_general_state_tests_path = cargo_manifest_dir.join("vectors/GeneralStateTests");
     let mut spinner = Spinner::new(Dots, "Parsing EF Tests".bold().to_string(), Color::Cyan);
+    if opts.disable_spinner {
+        spinner.stop();
+    }
     let mut tests = Vec::new();
     for test_dir in std::fs::read_dir(ef_general_state_tests_path.clone())
         .map_err(|err| {
@@ -37,12 +41,13 @@ pub fn parse_ef_tests(opts: &EFTestRunnerOptions) -> Result<Vec<EFTest>, EFTestP
         let directory_tests = parse_ef_test_dir(test_dir, opts, &mut spinner)?;
         tests.extend(directory_tests);
     }
-    spinner.success(
-        &format!(
+    spinner_success_or_print(
+        &mut spinner,
+        format!(
             "Parsed EF Tests in {}",
             format_duration_as_mm_ss(parsing_time.elapsed())
-        )
-        .bold(),
+        ),
+        opts.disable_spinner,
     );
     Ok(tests)
 }
@@ -52,7 +57,11 @@ pub fn parse_ef_test_dir(
     opts: &EFTestRunnerOptions,
     directory_parsing_spinner: &mut Spinner,
 ) -> Result<Vec<EFTest>, EFTestParseError> {
-    directory_parsing_spinner.update_text(format!("Parsing directory {:?}", test_dir.file_name()));
+    spinner_update_text_or_print(
+        directory_parsing_spinner,
+        format!("Parsing directory {:?}", test_dir.file_name()),
+        opts.disable_spinner,
+    );
 
     let mut directory_tests = Vec::new();
     for test in std::fs::read_dir(test_dir.path())
@@ -78,31 +87,68 @@ pub fn parse_ef_test_dir(
         {
             continue;
         }
-        // Skip the ValueOverflowParis.json file.
+        // Skip the ValueOverflowParis.json file because of errors, and loopMul.json because it takes too long to run.
         if test
             .path()
             .file_name()
-            .is_some_and(|name| name == "ValueOverflowParis.json")
+            .is_some_and(|name| name == "ValueOverflowParis.json" || name == "loopMul.json")
         {
             continue;
         }
 
-        // Skip tests that are not in the list of tests to run.
-        if &test_dir.file_name().to_str().unwrap().to_owned() == "vmPerformance" {
-            return Ok(Vec::new());
-        }
-
         // Skip tests that are not in the list of tests to run.
         if !opts.tests.is_empty()
             && !opts
                 .tests
                 .contains(&test_dir.file_name().to_str().unwrap().to_owned())
+            && !opts.tests.contains(
+                &test
+                    .path()
+                    .file_name()
+                    .unwrap()
+                    .to_str()
+                    .unwrap()
+                    .to_owned(),
+            )
         {
-            directory_parsing_spinner.update_text(format!(
-                "Skipping test {:?} as it is not in the list of tests to run",
-                test.path().file_name()
-            ));
-            return Ok(Vec::new());
+            continue;
+        }
+
+        // Skips all tests in a particular directory.
+        if opts
+            .skip
+            .contains(&test_dir.file_name().to_str().unwrap().to_owned())
+        {
+            spinner_update_text_or_print(
+                directory_parsing_spinner,
+                format!(
+                    "Skipping test {:?} as it is in the folder of tests to skip",
+                    test.path().file_name().unwrap()
+                ),
+                opts.disable_spinner,
+            );
+            continue;
+        }
+
+        // Skip tests by name (with .json extension)
+        if opts.skip.contains(
+            &test
+                .path()
+                .file_name()
+                .unwrap()
+                .to_str()
+                .unwrap()
+                .to_owned(),
+        ) {
+            spinner_update_text_or_print(
+                directory_parsing_spinner,
+                format!(
+                    "Skipping test {:?} as it is in the list of tests to skip",
+                    test.path().file_name().unwrap()
+                ),
+                opts.disable_spinner,
+            );
+            continue;
         }
 
         let test_file = std::fs::File::open(test.path()).map_err(|err| {

cmd/ef_tests/levm/runner/mod.rs

@@ -1,6 +1,7 @@
 use crate::{
     report::{self, format_duration_as_mm_ss, EFTestReport, TestReRunReport},
     types::EFTest,
+    utils::{spinner_success_or_print, spinner_update_text_or_print},
 };
 use clap::Parser;
 use colored::Colorize;
@@ -42,10 +43,14 @@ pub enum InternalError {
 pub struct EFTestRunnerOptions {
     #[arg(short, long, value_name = "FORK", default_value = "Cancun")]
     pub fork: Vec<SpecId>,
-    #[arg(short, long, value_name = "TESTS")]
+    #[arg(short, long, value_name = "TESTS", use_value_delimiter = true)]
     pub tests: Vec<String>,
     #[arg(short, long, value_name = "SUMMARY", default_value = "false")]
     pub summary: bool,
+    #[arg(long, value_name = "SKIP", use_value_delimiter = true)]
+    pub skip: Vec<String>,
+    #[arg(long, value_name = "DISABLE_SPINNER", default_value = "false")]
+    pub disable_spinner: bool, // Replaces spinner for normal prints.
 }
 
 pub fn run_ef_tests(
@@ -59,7 +64,7 @@ pub fn run_ef_tests(
     if opts.summary {
         return Ok(());
     }
-    re_run_with_revm(&mut reports, &ef_tests)?;
+    re_run_with_revm(&mut reports, &ef_tests, opts)?;
     write_report(&reports)
 }
 
@@ -74,7 +79,13 @@ fn run_with_levm(
         report::progress(reports, levm_run_time.elapsed()),
         Color::Cyan,
     );
+    if opts.disable_spinner {
+        levm_run_spinner.stop();
+    }
     for test in ef_tests.iter() {
+        if opts.disable_spinner {
+            println!("Running test: {:?}", test.name);
+        }
         let ef_test_report = match levm_runner::run_ef_test(test) {
             Ok(ef_test_report) => ef_test_report,
             Err(EFTestRunnerError::Internal(err)) => return Err(EFTestRunnerError::Internal(err)),
@@ -85,17 +96,33 @@ fn run_with_levm(
             }
         };
         reports.push(ef_test_report);
-        levm_run_spinner.update_text(report::progress(reports, levm_run_time.elapsed()));
+        spinner_update_text_or_print(
+            &mut levm_run_spinner,
+            report::progress(reports, levm_run_time.elapsed()),
+            opts.disable_spinner,
+        );
     }
-    levm_run_spinner.success(&report::progress(reports, levm_run_time.elapsed()));
+    spinner_success_or_print(
+        &mut levm_run_spinner,
+        report::progress(reports, levm_run_time.elapsed()),
+        opts.disable_spinner,
+    );
 
     if opts.summary {
         report::write_summary_for_slack(reports)?;
         report::write_summary_for_github(reports)?;
     }
 
     let mut summary_spinner = Spinner::new(Dots, "Loading summary...".to_owned(), Color::Cyan);
-    summary_spinner.success(&report::summary_for_shell(reports));
+    if opts.disable_spinner {
+        summary_spinner.stop();
+    }
+    spinner_success_or_print(
+        &mut summary_spinner,
+        report::summary_for_shell(reports),
+        opts.disable_spinner,
+    );
+
     Ok(())
 }
 
@@ -104,21 +131,32 @@ fn run_with_levm(
 fn _run_with_revm(
     reports: &mut Vec<EFTestReport>,
     ef_tests: &[EFTest],
+    opts: &EFTestRunnerOptions,
 ) -> Result<(), EFTestRunnerError> {
     let revm_run_time = std::time::Instant::now();
     let mut revm_run_spinner = Spinner::new(
         Dots,
         "Running all tests with REVM...".to_owned(),
         Color::Cyan,
     );
+    if opts.disable_spinner {
+        revm_run_spinner.stop();
+    }
     for (idx, test) in ef_tests.iter().enumerate() {
+        if opts.disable_spinner {
+            println!("Running test: {:?}", test.name);
+        }
         let total_tests = ef_tests.len();
-        revm_run_spinner.update_text(format!(
-            "{} {}/{total_tests} - {}",
-            "Running all tests with REVM".bold(),
-            idx + 1,
-            format_duration_as_mm_ss(revm_run_time.elapsed())
-        ));
+        spinner_update_text_or_print(
+            &mut revm_run_spinner,
+            format!(
+                "{} {}/{total_tests} - {}",
+                "Running all tests with REVM".bold(),
+                idx + 1,
+                format_duration_as_mm_ss(revm_run_time.elapsed())
+            ),
+            opts.disable_spinner,
+        );
         let ef_test_report = match revm_runner::_run_ef_test_revm(test) {
             Ok(ef_test_report) => ef_test_report,
             Err(EFTestRunnerError::Internal(err)) => return Err(EFTestRunnerError::Internal(err)),
@@ -129,36 +167,59 @@ fn _run_with_revm(
             }
         };
         reports.push(ef_test_report);
-        revm_run_spinner.update_text(report::progress(reports, revm_run_time.elapsed()));
+        spinner_update_text_or_print(
+            &mut revm_run_spinner,
+            report::progress(reports, revm_run_time.elapsed()),
+            opts.disable_spinner,
+        );
     }
-    revm_run_spinner.success(&format!(
-        "Ran all tests with REVM in {}",
-        format_duration_as_mm_ss(revm_run_time.elapsed())
-    ));
+    spinner_success_or_print(
+        &mut revm_run_spinner,
+        format!(
+            "Ran all tests with REVM in {}",
+            format_duration_as_mm_ss(revm_run_time.elapsed())
+        ),
+        opts.disable_spinner,
+    );
     Ok(())
 }
 
 fn re_run_with_revm(
     reports: &mut [EFTestReport],
     ef_tests: &[EFTest],
+    opts: &EFTestRunnerOptions,
 ) -> Result<(), EFTestRunnerError> {
     let revm_run_time = std::time::Instant::now();
     let mut revm_run_spinner = Spinner::new(
         Dots,
         "Running failed tests with REVM...".to_owned(),
         Color::Cyan,
     );
+    if opts.disable_spinner {
+        revm_run_spinner.stop();
+    }
     let failed_tests = reports.iter().filter(|report| !report.passed()).count();
-    for (idx, failed_test_report) in reports.iter_mut().enumerate() {
-        if failed_test_report.passed() {
-            continue;
+
+    // Iterate only over failed tests
+    for (idx, failed_test_report) in reports
+        .iter_mut()
+        .filter(|report| !report.passed())
+        .enumerate()
+    {
+        if opts.disable_spinner {
+            println!("Running test: {:?}", failed_test_report.name);
         }
-        revm_run_spinner.update_text(format!(
-            "{} {}/{failed_tests} - {}",
-            "Re-running failed tests with REVM".bold(),
-            idx + 1,
-            format_duration_as_mm_ss(revm_run_time.elapsed())
-        ));
+        spinner_update_text_or_print(
+            &mut revm_run_spinner,
+            format!(
+                "{} {}/{failed_tests} - {}",
+                "Re-running failed tests with REVM".bold(),
+                idx + 1,
+                format_duration_as_mm_ss(revm_run_time.elapsed())
+            ),
+            opts.disable_spinner,
+        );
+
         match revm_runner::re_run_failed_ef_test(
             ef_tests
                 .iter()
@@ -184,10 +245,14 @@ fn re_run_with_revm(
             }
         }
     }
-    revm_run_spinner.success(&format!(
-        "Re-ran failed tests with REVM in {}",
-        format_duration_as_mm_ss(revm_run_time.elapsed())
-    ));
+    spinner_success_or_print(
+        &mut revm_run_spinner,
+        format!(
+            "Re-ran failed tests with REVM in {}",
+            format_duration_as_mm_ss(revm_run_time.elapsed())
+        ),
+        opts.disable_spinner,
+    );
     Ok(())
 }
 

cmd/ef_tests/levm/utils.rs

@@ -5,6 +5,7 @@ use crate::{
 use ethrex_core::{types::Genesis, H256, U256};
 use ethrex_storage::{EngineType, Store};
 use ethrex_vm::{evm_state, EvmState};
+use spinoff::Spinner;
 
 pub fn load_initial_state(test: &EFTest) -> (EvmState, H256) {
     let genesis = Genesis::from(test);
@@ -21,6 +22,22 @@ pub fn load_initial_state(test: &EFTest) -> (EvmState, H256) {
     )
 }
 
+pub fn spinner_update_text_or_print(spinner: &mut Spinner, text: String, no_spinner: bool) {
+    if no_spinner {
+        println!("{}", text);
+    } else {
+        spinner.update_text(text);
+    }
+}
+
+pub fn spinner_success_or_print(spinner: &mut Spinner, text: String, no_spinner: bool) {
+    if no_spinner {
+        println!("{}", text);
+    } else {
+        spinner.success(&text);
+    }
+}
+
 // If gas price is not provided, calculate it with current base fee and priority fee
 pub fn effective_gas_price(
     test: &EFTest,

crates/vm/levm/Makefile

@@ -39,6 +39,10 @@ run-evm-ef-tests: ## 🏃‍♂️ Run EF Tests
 		time cargo test -p ef_tests-levm --test ef_tests_levm;\
 	fi
 
+run-evm-ef-tests-ci: ## 🏃‍♂️ Run EF Tests only with LEVM and without spinner, for CI.
+	cd ../../../ && \
+	time cargo test -p ef_tests-levm --test ef_tests_levm -- --disable-spinner --summary
+
 generate-evm-ef-tests-report: ## 📊 Generate EF Tests Report
 	cd ../../../ && \
 	cargo test -p ef_tests-levm --test ef_tests_levm -- --summary