feat(levm): run hive tests with levm, generate and publish report (#1546

)

**Motivation**

<!-- Why does this pull request exist? What are its goals? -->

**Description**

<!-- A clear and concise general description of the changes this PR
introduces -->
- We now can execute multiple transactions in a block and commit them
after execution of them all.
- It refactors `get_state_transitions_levm`, now in `account_updates`
we'll have only the things that have changed from the account, so that
we don't have much redundant information

- We build the client image with LEVM, we run hive tests and then we
publish them
- Add to the Dockerfile the possibility to add build flags.
- Avoid panicking if setup wasn't completed successfully, as no tests
will run in that scenario.


We can run tests with hive but the problem is that these tests execute a
lot of blocks at the beginning and we usually fail at some block because
of a difference in state, I tried debugging some tests and I fixed some
things in other PRs but at some point I got stuck and decided that it
was a better idea to leave it as is and merge this to main.

.github/scripts/publish_levm_hive.sh

@@ -0,0 +1,22 @@
+curl -X POST $url \
+-H 'Content-Type: application/json; charset=utf-8' \
+--data @- <<EOF
+$(jq -n --arg text "$(cat results.md)" '{
+    "blocks": [
+        {
+            "type": "header",
+            "text": {
+                "type": "plain_text",
+                "text": "LEVM Hive Coverage Report"
+            }
+        },
+        {
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": $text
+            }             
+        }
+    ]
+}')
+EOF

.github/workflows/daily_reports_levm_hive.yaml

@@ -0,0 +1,92 @@
+name: Daily Reports Hive LEVM
+
+on:
+  schedule:
+    # Every day at UTC midnight
+    - cron: "0 0 * * 1,2,3,4,5"
+  workflow_dispatch:
+
+env:
+  RUST_VERSION: 1.81.0
+
+jobs:
+  run-hive:
+    name: Hive
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+
+      - name: Build Image with LEVM
+        run: cd crates/vm/levm && make build-image-levm
+
+      - name: Setup Hive
+        run: make setup-hive
+
+      - name: Run Hive RPC Simulation
+        run: cd hive && ./hive --sim ethereum/rpc-compat --client ethrex --sim.limit "*" --sim.parallelism 16
+        continue-on-error: true
+
+      - name: Run Hive Sync Simulation
+        run: cd hive && ./hive --sim ethereum/sync --client ethrex --sim.limit "*" --sim.parallelism 16
+        continue-on-error: true
+
+      - name: Run Hive Engine Simulation
+        run: cd hive && ./hive --sim ethereum/engine --client ethrex --sim.limit "*" --sim.parallelism 16
+        continue-on-error: true
+
+      - name: Run Hive P2P Simulation
+        run: cd hive && ./hive --sim devp2p --client ethrex --sim.limit "*" --sim.parallelism 16
+        continue-on-error: true
+
+      - name: Upload results
+        uses: actions/upload-artifact@v4
+        with:
+          name: testing_logs
+          path: hive/workspace/logs/*-*.json
+
+  hive-report:
+    name: Generate report and upload to summary and slack
+    needs: run-hive
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Rustup toolchain install
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: ${{ env.RUST_VERSION }}
+
+      - name: Download all results
+        uses: actions/download-artifact@v4
+        with:
+          path: hive/workspace/logs
+          pattern: "*_logs"
+          merge-multiple: true
+
+      - name: Caching
+        uses: Swatinem/rust-cache@v2
+
+      - name: Generate the hive report
+        run: cargo run -p hive_report > results.md
+
+      - name: Post results in summary
+        run: |
+          echo "# LEVM Hive coverage report" >> $GITHUB_STEP_SUMMARY
+          cat results.md >> $GITHUB_STEP_SUMMARY
+
+      - name: Post results to levm slack channel
+        env:
+          url: ${{ secrets.LEVM_SLACK_WEBHOOK }}
+        run: sh .github/scripts/publish_levm_hive.sh
+
+      # Note: Leave this commented, as it is for testing purposes.
+      # - name: Post results to test channel for debugging
+      #   env:
+      #     url: ${{ secrets.TEST_CHANNEL_SLACK }}
+      #   run: sh .github/scripts/publish_levm_hive.sh

.gitignore

@@ -48,6 +48,7 @@ levm_ef_tests_summary_slack.txt
 levm_ef_tests_summary_github.txt
 levm_ef_tests_summary.txt
 
+results.md
 loc_report.md
 loc_report_slack.txt
 loc_report_github.txt

Dockerfile

@@ -21,8 +21,10 @@ COPY --from=planner /ethrex/recipe.json recipe.json
 # Build dependencies only, these remained cached
 RUN cargo chef cook --release --recipe-path recipe.json
 
+# Optional build flags
+ARG BUILD_FLAGS=""
 COPY . .
-RUN cargo build --release
+RUN cargo build --release $BUILD_FLAGS
 
 FROM ubuntu:24.04
 WORKDIR /usr/local/bin

cmd/ethrex/Cargo.toml

@@ -45,4 +45,4 @@ dev = ["dep:ethrex-dev"]
 libmdbx = ["dep:libmdbx", "ethrex-storage/libmdbx"]
 redb = ["dep:redb", "ethrex-storage/redb"]
 l2 = ["dep:ethrex-l2", "ethrex-vm/l2"]
-levm = ["ethrex-vm/levm", "ethrex-blockchain/levm"]
+levm = ["default", "ethrex-vm/levm", "ethrex-blockchain/levm"]

cmd/ethrex/ethrex.rs

@@ -356,7 +356,15 @@ fn import_blocks(store: &Store, blocks: &Vec<Block>) {
     }
     if let Some(last_block) = blocks.last() {
         let hash = last_block.hash();
-        apply_fork_choice(store, hash, hash, hash).unwrap();
+        cfg_if::cfg_if! {
+            if #[cfg(feature = "levm")] {
+                // We are allowing this not to unwrap so that tests can run even if block execution results in the wrong root hash with LEVM.
+                let _ = apply_fork_choice(store, hash, hash, hash);
+            }
+            else {
+                apply_fork_choice(store, hash, hash, hash).unwrap();
+            }
+        }
     }
     info!("Added {} blocks to blockchain", size);
 }

crates/blockchain/Cargo.toml

@@ -28,10 +28,10 @@ path = "./blockchain.rs"
 
 [features]
 default = ["c-kzg"]
+levm = ["default", "ethrex-vm/levm"]
 libmdbx = [
     "ethrex-core/libmdbx",
     "ethrex-storage/default",
     "ethrex-vm/libmdbx",
 ]
-levm = ["ethrex-vm/levm"]
 c-kzg =["ethrex-core/c-kzg"]

crates/common/types/account.rs

@@ -37,7 +37,7 @@ pub struct Account {
     pub storage: HashMap<H256, U256>,
 }
 
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Eq)]
 pub struct AccountInfo {
     pub code_hash: H256,
     pub balance: U256,

crates/l2/Makefile

@@ -66,6 +66,17 @@ init-l1: ## 🚀 Initializes an L1 Lambda ethrex Client
 	--authrpc.port ${L1_AUTH_PORT} \
 	--datadir ${ethrex_L1_DEV_LIBMDBX}
 
+init-l1-levm: ## 🚀 Initializes an L1 Lambda ethrex Client with LEVM
+	cargo run --release \
+    --manifest-path ../../Cargo.toml \
+    --bin ethrex \
+    --features dev,ethrex-blockchain/levm,ethrex-vm/levm -- \
+    --network ${L1_GENESIS_FILE_PATH} \
+    --http.port ${L1_PORT} \
+    --http.addr 0.0.0.0 \
+    --authrpc.port ${L1_AUTH_PORT} \
+    --datadir ${ethrex_L1_DEV_LIBMDBX}
+
 down-local-l1: ## 🛑 Shuts down the L1 Lambda ethrex Client
 	docker compose -f ${ethrex_DEV_DOCKER_COMPOSE_PATH} down
 	docker compose -f docker-compose-l2.yaml down

crates/storage/store/storage.rs

@@ -44,7 +44,7 @@ pub enum EngineType {
     RedB,
 }
 
-#[derive(Default, Debug, Clone, Serialize, Deserialize)]
+#[derive(Default, Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct AccountUpdate {
     pub address: Address,
     pub removed: bool,

crates/vm/Cargo.toml

@@ -39,11 +39,11 @@ path = "./vm.rs"
 
 [features]
 default = ["c-kzg", "blst"]
+levm = ["default", "ethrex-levm"]
 l2 = []
 c-kzg = ["revm/c-kzg"]
 blst = ["revm/blst"]
 libmdbx = ["ethrex-storage/default", "ethrex-core/libmdbx"]
-levm = ["ethrex-levm"]
 
 [profile.test]
 opt-level = 3

crates/vm/levm/Makefile

@@ -76,3 +76,15 @@ build-revm-comparison:
 		--bin levm_factorial \
 		--bin revm_fibonacci \
 		--bin levm_fibonacci
+
+###### Build Client with LEVM ######
+
+FLAGS := "--features ethrex-blockchain/levm,ethrex-vm/levm,ethrex/levm"
+
+build-image-levm: ## 🐳 Build the Docker image with LEVM features
+	cd ../../../ && \
+	docker build -t ethrex --build-arg BUILD_FLAGS=$(FLAGS) .
+
+run-hive-debug-levm: build-image-levm 
+	$(MAKE) -C ../../../ setup-hive
+	cd ../../../hive && ./hive --sim $(SIMULATION) --client ethrex --sim.limit "$(TEST_PATTERN)" --docker.output