Frame Omni Bencher integration

.github/workflows/README.md

@@ -8,22 +8,22 @@ Latest commit hash in PR branch will be built and used for benchmarking.
 ### How to execute
 Usage
 ```
-# [chain names] - Use "," for multiple runtimes. Available values are: astar-dev, shiden-dev, shibuya-dev, dev
+# [chain names] - Use "," for multiple runtimes. Available values are: astar, shiden, shibuya
 # [pallet names] - Use "," for multiple pallets, "all" for all pallets
 /bench [chain names] [pallet names]
 ```
 ```
 # benchmark a pallet
-/bench astar-dev pallet_balances
+/bench astar pallet_balances
 
 # benchmark multiple pallets
-/bench astar-dev pallet_balances,pallet_dapps_staking
+/bench astar pallet_balances,pallet_dapps_staking
 
 # benchmark all pallets
-/bench astar-dev all
+/bench astar all
 
 # benchmark multiple runtimes with multiple pallets
-/bench astar-dev,shibuya-dev pallet_balances,pallet_dapps_staking
+/bench astar,shibuya pallet_balances,pallet_dapps_staking
 ```
 
 

.github/workflows/benchmark.yml

@@ -109,6 +109,9 @@ jobs:
     - name: Check targets are installed correctly
       run: rustup target list --installed
 
+    - name: Install frame-omni bencher tool
+      run: cargo install frame-omni-bencher
+
     - name: Execute benchmarking
       run: |
         mkdir -p ./benchmark-results

scripts/run_benchmarks.sh

@@ -30,7 +30,7 @@ while getopts 'bc:fo:p:v' flag; do
       ;;
     c)
       chains=$(echo ${OPTARG} | tr '[:upper:]' '[:lower:]')
-      chains_default=("astar-dev" "shiden-dev" "shibuya-dev" "dev")
+      chains_default=("astar" "shiden" "shibuya")
       for chain in ${chains//,/ }; do
         if [[ ! " ${chains_default[*]} " =~ " ${chain} " ]]; then
           echo "Chain input is invalid. ${chain} not included in ${chains_default[*]}"
@@ -73,11 +73,12 @@ done
 if [ "$skip_build" != true ]
 then
   echo "[+] Compiling astar-collator benchmarks..."
-  CARGO_PROFILE_RELEASE_LTO=true RUSTFLAGS="-C codegen-units=1" cargo build --release --verbose --features=runtime-benchmarks
+  CARGO_PROFILE_RELEASE_LTO=true RUSTFLAGS="-C codegen-units=1" cargo build --release --verbose --features=runtime-benchmarks \
+  -p astar-runtime -p shiden-runtime -p shibuya-runtime
 fi
 
 # The executable to use.
-ASTAR_COLLATOR=./target/release/astar-collator
+BENCHMARK_TOOL="frame-omni-bencher v1"
 
 # Manually exclude some pallets.
 EXCLUDED_PALLETS=(
@@ -86,7 +87,7 @@ EXCLUDED_PALLETS=(
 
 # Load all pallet names in an array.
 ALL_PALLETS=($(
-  $ASTAR_COLLATOR benchmark pallet --list --chain=$chain |\
+  $BENCHMARK_TOOL benchmark pallet --list --runtime ./target/release/wbuild/${chain}-runtime/${chain}_runtime.compact.compressed.wasm |\
     tail -n+2 |\
     cut -d',' -f1 |\
     sort |\
@@ -120,8 +121,8 @@ for chain in ${chains//,/ }; do
       echo "[+] Benchmarking $PALLET with weight file $WEIGHT_FILE";
 
       OUTPUT=$(
-        $ASTAR_COLLATOR benchmark pallet \
-        --chain=$chain \
+        $BENCHMARK_TOOL benchmark pallet \
+        --runtime ./target/release/wbuild/${chain}-runtime/${chain}_runtime.compact.compressed.wasm \
         --steps=50 \
         --repeat=20 \
         --pallet="$PALLET" \
@@ -137,16 +138,19 @@ for chain in ${chains//,/ }; do
       fi
     done
 
-    echo "[+] Benchmarking the machine..."
-    OUTPUT=$(
-      $ASTAR_COLLATOR benchmark machine --chain=$chain 2>&1
-    )
-    if [ $? -ne 0 ]; then
-      echo "[-] Failed the machine benchmark"
-      echo "$OUTPUT" >> "$ERR_FILE"
-    else
-      echo "$OUTPUT" >> "$output_path/$chain/$chain-machine-bench.txt"
-    fi
+    # Disabled for now - command isn't available (yet).
+    # With latest changes we also benchmark the HW each time the client starts.
+    #
+    # echo "[+] Benchmarking the machine..."
+    # OUTPUT=$(
+    #   $BENCHMARK_TOOL benchmark machine --runtime ./target/release/wbuild/astar-runtime/${chain}_runtime.compact.compressed.wasm 2>&1
+    # )
+    # if [ $? -ne 0 ]; then
+    #   echo "[-] Failed the machine benchmark"
+    #   echo "$OUTPUT" >> "$ERR_FILE"
+    # else
+    #   echo "$OUTPUT" >> "$output_path/$chain/$chain-machine-bench.txt"
+    # fi
 
     # Check if the error file exists.
     if [ -f "$ERR_FILE" ]; then