feat(ci): add native dependencies to the CI (dynamic docker) (starkwa…

…re-libs#726) * fix: ci * fix: return back old images in the CI * fix: add native deps to some CI jobs * chore: update feature contracts * fix: add LLVM deps to the Dockerfile * fix: add LLVM to `install_build_tools.sh` * fix: make larger runner * fix: make ubuntu latest runner * fix: update build tools, add env vars * fix: update build tools, add env vars * fix: add clean up of the native artifacts * fix: add clean up of the native artifacts * fix: native artifacts push CI * fix: native artifacts push CI * fix: native artifacts push CI * fix: add -y arg to apt * fix: add env to ignore interactive env * fix: increase size of the runners * fix: update runner version * fix: remove native and rust dependencies * fix: update sequencer-ci.Dockerfile * fix: update sequencer-ci.Dockerfile * fix: update sequencer-ci.Dockerfile * fix: increase runner size * fix: try to fix Dockerfile with adding musl-g++ * fix: try to fix Dockerfile with adding musl-g++ #2 * fix: try to fix Dockerfile with adding musl-g++ #2 * fix: try to fix Dockerfile with adding musl-g++ #4 * fix: try to fix Dockerfile with adding musl-g++ #4 * fix: try to fix Dockerfile * fix: try to fix Dockerfile finally? * fix: try to fix Dockerfile * fix: add hard option for sym link creation * fix: make Dockerfile dynamic * Merge remote-tracking branch 'origin/main' into fix/native-dynamic-ci * feat: add script to install native deps * feat: add script to install native deps * fix: add sudo for running the script * fix: add sudo for running the script * fix: add sudo for running the script * fix: fix llvm paths * fix: script * fix: script * fix: script * fix: script * fix: script * fix: script * fix: script * fix: script * fix: script * fix: verify_cairo_file_dependencies ci * fix: cairo native runtime library path * Merge branch 'main' into fix/native-dynamic-ci # Conflicts: # Cargo.lock # Cargo.toml * fix: conflicts after merge * fix: update feature contracts * fix: update gas in secp tests * fix: increase merge-gatekeeper timeout to 25 minutes (1500 seconds)
lambdaclass · Sep 17, 2024 · c1f680a · c1f680a
1 parent 1233182
commit c1f680a
Show file tree

Hide file tree

Showing 27 changed files with 4,571 additions and 1,445 deletions.
diff --git a/.github/actions/setup-native-deps/action.yml b/.github/actions/setup-native-deps/action.yml
@@ -0,0 +1,28 @@
+name: "Setup Cairo Native Dependencies"
+description: "Sets up LLVM and GMP libraries"
+
+outputs:
+  cairo-native-runtime-library:
+    description: "The path to the cairo native runtime library"
+    value: ${{ steps.set-env-vars.outputs.cairo-native-runtime-library }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Make script executable
+      shell: bash
+      run: chmod +x ./scripts/setup_native_deps.sh
+
+    - name: Install Cairo Native Runtime Dependencies
+      id: set-runtime-deps
+      shell: bash
+      run: |
+        sudo ./scripts/setup_native_deps.sh
+
+    - name: Set Environment Variables
+      id: set-env-vars
+      shell: bash
+      run: |
+        echo "MLIR_SYS_180_PREFIX=/usr/lib/llvm-18/" >> $GITHUB_ENV &&
+        echo "LLVM_SYS_181_PREFIX=/usr/lib/llvm-18/" >> $GITHUB_ENV &&
+        echo "TABLEGEN_180_PREFIX=/usr/lib/llvm-18/" >> $GITHUB_ENV
diff --git a/.github/workflows/blockifier_ci.yml b/.github/workflows/blockifier_ci.yml
@@ -46,19 +46,17 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           prefix-key: "v0-rust-ubuntu-20.04"
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - run: cargo build -p blockifier
       - run: cargo test -p blockifier
 
   native-blockifier-artifacts-push:
-    runs-on: starkware-ubuntu-20-04-medium
+    runs-on: starkware-ubuntu-latest-large
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
-
-      - uses: Swatinem/rust-cache@v2
-        with:
-          prefix-key: "v1-rust-ubuntu-20.04"
-
+
       - name: Build native blockifier
         run: ./build_native_in_docker.sh scripts/build_native_blockifier.sh
 

diff --git a/.github/workflows/blockifier_compiled_cairo.yml b/.github/workflows/blockifier_compiled_cairo.yml
@@ -40,6 +40,10 @@ jobs:
         with:
           prefix-key: "v0-rust-ubuntu-20.04"
 
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
+
       # Setup pypy and link to the location expected by .cargo/config.toml.
       - uses: actions/setup-python@v5
         id: setup-pypy

diff --git a/.github/workflows/blockifier_post-merge.yml b/.github/workflows/blockifier_post-merge.yml
@@ -21,6 +21,10 @@ jobs:
         with:
           prefix-key: "v0-rust-ubuntu-20.04"
 
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
+
       # Setup pypy and link to the location expected by .cargo/config.toml.
       - uses: actions/setup-python@v5
         id: setup-pypy

diff --git a/.github/workflows/committer_ci.yml b/.github/workflows/committer_ci.yml
@@ -76,7 +76,7 @@ jobs:
 
       # List the existing benchmarks.
       - run: |
-            cargo bench -p committer_cli -- --list | grep ': benchmark$' | sed -e "s/: benchmark$//" > benchmarks_list.txt
+          cargo bench -p committer_cli -- --list | grep ': benchmark$' | sed -e "s/: benchmark$//" > benchmarks_list.txt
 
       # Benchmark the old code.
       - run: cargo bench -p committer_cli
@@ -119,6 +119,7 @@ jobs:
               owner: context.repo.owner,
               repo: context.repo.repo,
               body: fs.readFileSync('bench_new.txt', 'utf8'),
+              path: 'Commits'
             })
 
   gcs-push:

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -66,6 +66,11 @@ jobs:
       - uses: ./.github/actions/install_rust
         with:
           components: clippy
+
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
+
       - uses: Swatinem/rust-cache@v2
         with:
           prefix-key: "v0-rust-ubuntu-20.04"
@@ -89,6 +94,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - uses: Swatinem/rust-cache@v2
       - uses: Noelware/[email protected]
         with:
@@ -100,17 +108,23 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - uses: Swatinem/rust-cache@v2
       - run: cargo test -p workspace_tests
 
   run-tests:
-    runs-on: starkware-ubuntu-20-04-medium
+    runs-on: starkware-ubuntu-latest-large
     steps:
       - uses: actions/checkout@v4
         with:
           # Fetch the entire history.
           fetch-depth: 0
       - uses: ./.github/actions/install_rust
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - uses: Noelware/[email protected]
       - uses: Swatinem/rust-cache@v2
         with:
@@ -175,6 +189,9 @@ jobs:
       - uses: Noelware/[email protected]
         with:
           version: ${{env.PROTOC_VERSION}}
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - run: cargo check --workspace -r --all-features
 
   merge-gatekeeper:
@@ -190,7 +207,7 @@ jobs:
         uses: upsidr/merge-gatekeeper@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          timeout: 1200
+          timeout: 1500
           interval: 30
           ignored: "code-review/reviewable"
 
@@ -200,7 +217,7 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           ref: ${{github.ref}}
-          timeout: 1200
+          timeout: 1500
           interval: 30
           ignored: "code-review/reviewable"
 
@@ -216,6 +233,9 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y clang llvm libudev-dev
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - uses: Noelware/[email protected]
         with:
           version: ${{env.PROTOC_VERSION}}

diff --git a/.github/workflows/papyrus_benchmark.yaml b/.github/workflows/papyrus_benchmark.yaml
@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
   # TODO: Uncomment and run this automatically when the storage benchmark is fixed.
   # push:
-    # branches: [main]
+  # branches: [main]
 
 jobs:
   storage-benchmark:

diff --git a/.github/workflows/papyrus_ci.yml b/.github/workflows/papyrus_ci.yml
@@ -2,7 +2,7 @@ name: Papyrus-CI
 
 on:
   push:
-    branches: [main]
+    branches: [ main ]
     paths:
       - '.github/workflows/papyrus_ci.yml'
       - 'Dockerfile'
@@ -31,7 +31,7 @@ on:
       - 'crates/starknet_client/**'
 
   merge_group:
-    types: [checks_requested]
+    types: [ checks_requested ]
 
 # On PR events, cancel existing CI runs on this same PR for this workflow.
 concurrency:
@@ -48,6 +48,9 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust
       - uses: Swatinem/rust-cache@v2
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - uses: Noelware/[email protected]
         with:
           version: ${{env.PROTOC_VERSION}}
@@ -67,6 +70,9 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust
       - uses: Swatinem/rust-cache@v2
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - uses: Noelware/[email protected]
         with:
           version: ${{env.PROTOC_VERSION}}
@@ -86,6 +92,9 @@ jobs:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - uses: Noelware/[email protected]
         with:
           version: ${{env.PROTOC_VERSION}}
@@ -107,6 +116,9 @@ jobs:
       - uses: Noelware/[email protected]
         with:
           version: ${{env.PROTOC_VERSION}}
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - run: >
           cargo test -r
           --test latency_histogram
@@ -122,6 +134,9 @@ jobs:
       - uses: ./.github/actions/install_rust
       - uses: Swatinem/rust-cache@v2
       - uses: Noelware/[email protected]
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
 
       - run: |
           cargo test -p papyrus_node --no-default-features
@@ -140,7 +155,9 @@ jobs:
       - uses: Noelware/[email protected]
         with:
           version: ${{env.PROTOC_VERSION}}
-
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@cargo-llvm-cov
       - uses: Swatinem/rust-cache@v2
@@ -197,6 +214,9 @@ jobs:
             target_directory:
               - 'crates/papyrus_storage/src/db/**'
       - uses: ./.github/actions/install_rust
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - uses: Swatinem/rust-cache@v2
       # repeat this job 32 times. this is a random test for part of the code that may cause a corrupted database.
       - run: for run in {1..32}; do cargo test -r -p papyrus_storage -- --include-ignored common_prefix_compare_with_simple_table_random; done

diff --git a/.github/workflows/papyrus_docker-publish.yml b/.github/workflows/papyrus_docker-publish.yml
@@ -3,8 +3,8 @@ name: Papyrus-Docker-Publish
 on:
   workflow_dispatch:
   push:
-    branches: [main]
-    tags: ["v*.*.*"]
+    branches: [ main ]
+    tags: [ "v*.*.*" ]
     paths:
       - '.github/workflows/papyrus_docker-publish.yml'
       - 'crates/papyrus**/**'
@@ -25,7 +25,7 @@ env:
 
 jobs:
   docker-build-push:
-    runs-on: starkware-ubuntu-latest-medium
+    runs-on: starkware-ubuntu-latest-large
 
     steps:
       - name: Checkout repository

diff --git a/.github/workflows/papyrus_nightly-tests-call.yml b/.github/workflows/papyrus_nightly-tests-call.yml
@@ -22,6 +22,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - uses: Swatinem/rust-cache@v2
       - run: sudo apt update; sudo apt -y install libclang-dev
         # Install libclang-dev that is not a part of the ubuntu vm in github actions.

diff --git a/.github/workflows/papyrus_nightly-tests.yml b/.github/workflows/papyrus_nightly-tests.yml
@@ -44,6 +44,10 @@ jobs:
 
       - run: brew install protobuf@$PROTOC_VERSION
 
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
+
       - name: Build node
         run: cargo build -r -p papyrus_node
 
@@ -58,6 +62,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - uses: Swatinem/rust-cache@v2
       - run: npm install -g [email protected]
 
@@ -75,6 +82,9 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust
       - uses: Swatinem/rust-cache@v2
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - run: cargo build -r -p papyrus_load_test
 
   integration-test:
@@ -84,6 +94,9 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust
       - uses: Swatinem/rust-cache@v2
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - run: >
           cargo test -r
           --test latency_histogram
@@ -100,4 +113,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust
       - uses: Swatinem/rust-cache@v2
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - run: for run in {1..100}; do cargo test -r -p papyrus_storage -- --include-ignored common_prefix_compare_with_simple_table_random; done
diff --git a/.github/workflows/verify-deps.yml b/.github/workflows/verify-deps.yml
@@ -12,6 +12,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - name: Update Dependencies
         run: cargo update --verbose
       - name: Build