diff --git a/.github/workflows/bench-hyperfine.yml b/.github/workflows/bench-hyperfine.yml index 94e6bfd83..012084b9d 100644 --- a/.github/workflows/bench-hyperfine.yml +++ b/.github/workflows/bench-hyperfine.yml @@ -86,3 +86,169 @@ jobs: issue-number: ${{ github.event.pull_request.number }} body-path: bench-hyperfine.md edit-mode: replace + + build-binaries: + env: + MLIR_SYS_180_PREFIX: /usr/lib/llvm-18/ + LLVM_SYS_181_PREFIX: /usr/lib/llvm-18/ + TABLEGEN_180_PREFIX: /usr/lib/llvm-18/ + RUST_LOG: cairo_native=debug,cairo_native_test=debug + strategy: + matrix: + branch: [ base, head ] + name: Build cairo-native-run for ${{ matrix.branch }} + runs-on: ubuntu-latest + steps: + - name: Cache binary + uses: actions/cache@v3 + id: cache-binary + with: + path: bin/cairo-native-run-${{ matrix.branch }} + key: binary-${{ github.event.pull_request[matrix.branch].sha }} + + - name: Cache runtime-library + uses: actions/cache@v3 + id: cache-library + with: + path: lib/libcairo-native-runtime-${{ matrix.branch }}.a + key: library-${{ github.event.pull_request[matrix.branch].sha }} + + - name: check and free hdd space left + if: ${{ steps.cache-binary.outputs.cache-hit != 'true' }} + run: | + echo "Listing 20 largest packages" + dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 20 + df -h + sudo apt-get update + sudo apt-get remove -y '^llvm-.*' + sudo apt-get remove -y 'php.*' + sudo apt-get remove -y '^dotnet-.*' + sudo apt-get remove -y '^temurin-.*' + sudo apt-get remove -y azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox powershell mono-devel + sudo apt-get autoremove -y + sudo apt-get clean + df -h + echo "Removing large directories" + # deleting 15GB + sudo rm -rf /usr/share/dotnet/ + sudo rm -rf /usr/local/lib/android + df -h + + - name: Install Rust + if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }} + uses: dtolnay/rust-toolchain@1.81.0 + + - name: add llvm deb repository + if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }} + uses: myci-actions/add-deb-repo@11 + with: + repo: deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main + repo-name: llvm-repo + keys-asc: https://apt.llvm.org/llvm-snapshot.gpg.key + + - name: Install LLVM + if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }} + run: sudo apt-get update && sudo apt-get install llvm-18 llvm-18-dev llvm-18-runtime clang-18 clang-tools-18 lld-18 libpolly-18-dev libmlir-18-dev mlir-18-tools + + - name: Checkout base commit + if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }} + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request[matrix.branch].sha }} + + - name: Fetch Rust cache + if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }} + uses: Swatinem/rust-cache@v2 + + - name: Build binary + if: ${{ steps.cache-binary.outputs.cache-hit != 'true' || steps.cache-library.outputs.cache-hit != 'true' }} + run: | + make build + mkdir lib bin + cp target/release/cairo-native-run bin/cairo-native-run-${{ matrix.branch }} + cp target/release/libcairo_native_runtime.a lib/libcairo-native-runtime-${{ matrix.branch }}.a + + hyperfine-prs: + name: Bench PR (linux, amd64) + needs: [ build-binaries ] + runs-on: ubuntu-latest + env: + PROGRAM: fib_2M + OUTPUT_DIR: bench-outputs + steps: + - uses: actions/checkout@v4 + + - name: Install Hyperfine + uses: taiki-e/install-action@v2 + with: + tool: hyperfine@1.16 + + - name: Fetch base binary + uses: actions/cache/restore@v3 + with: + path: bin/cairo-native-run-base + key: binary-${{ github.event.pull_request.base.sha }} + + - name: Fetch HEAD binary + uses: actions/cache/restore@v3 + with: + path: bin/cairo-native-run-head + key: binary-${{ github.event.pull_request.head.sha }} + + - name: Fetch base runtime-library + uses: actions/cache/restore@v3 + with: + path: lib/libcairo-native-runtime-base.a + key: library-${{ github.event.pull_request.base.sha }} + + - name: Fetch head runtime-library + uses: actions/cache/restore@v3 + with: + path: lib/libcairo-native-runtime-head.a + key: library-${{ github.event.pull_request.head.sha }} + + - name: Build Dependencies + run: make deps + + - name: Run Program + run: | + mkdir $OUTPUT_DIR + cp programs/benches/*.cairo $OUTPUT_DIR + for f in $OUTPUT_DIR/*.cairo; do + hyperfine \ + --export-markdown "${f%.*}.md" \ + --export-json "${f%.*}.json" \ + --warmup 3 \ + -n "head $(basename $f) (JIT)" "./bin/cairo-native-run-head --run-mode=jit -s $f --opt-level 3 --available-gas 18446744073709551615" \ + -n "base $(basename $f) (JIT)" "./bin/cairo-native-run-base --run-mode=jit -s $f --opt-level 3 --available-gas 18446744073709551615" \ + -n "head $(basename $f) (AOT)" "CAIRO_NATIVE_RUNTIME_LIBRARY=lib/libcairo-native-runtime-head.a ./bin/cairo-native-run-head --run-mode=aot -s $f --opt-level 3 --available-gas 18446744073709551615" \ + -n "base $(basename $f) (AOT)" "CAIRO_NATIVE_RUNTIME_LIBRARY=lib/libcairo-native-runtime-base.a ./bin/cairo-native-run-base --run-mode=aot -s $f --opt-level 3 --available-gas 18446744073709551615" \ + >> /dev/stderr + done + - name: Print tables + run: | + { + echo "Benchmark results Main vs HEAD" + for f in $OUTPUT_DIR/*.md; do + echo + cat $f + done + } | tee -a comment_body.md + + - name: Find Bench Comment + continue-on-error: true + uses: peter-evans/find-comment@v3 + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: "github-actions[bot]" + body-includes: Benchmarking + + - name: Create or update bench comment + continue-on-error: true + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body-path: comment_body.md + edit-mode: replace diff --git a/.gitignore b/.gitignore index af7823554..9c9a1c17e 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,5 @@ cairo-*.tar *.ipynb cairo-native-stress-logs.jsonl + +.DS_Store