rust.yml

name: Rust

# On Rust, GitHub Actions, and caching
# ===========
# Here's a list of things to keep in mind if you find yourself maintaining this
# CI:
#
# https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#matching-a-cache-key
#
# - Always install and select the desired Rust toolchain *before* running
#   `Swatinem/rust-cache`. This is because the active Rust toolchain is used as
#   a cache key.
# - You can use `rustup show` to install and select the right Rust toolchain if
#   you have a `rust-toolchain.toml` file:
#   https://github.com/rust-lang/rustup/issues/1397.
# - When caching Rust compilation artifacts, keep in mind that different `cargo`
#   commands will use different profiles
#   (https://doc.rust-lang.org/cargo/reference/profiles.html). Learn what you
#   can reuse between one job and another and don't assume two commands will
#   just share caches without conflicts.
# - Be extremely aware of cache thrashing a.k.a. churning. GitHub Actions' cache
#   allows for 10GiB of data which is easily exceeded if not careful.
#   Sometimes it's better not to cache than cache excessively.
#   Disabling cache writes for non-default branches altogether if cache churning
#   is unacceptably high is supposed to help with this.
# - Learn cache invalidation rules of `Swatinem/rust-cache` before making
#   changes, e.g. what happens when `rustc --version` changes or `Cargo.lock`
#   changes (or is missing).
# - The jobs dependency tree is the way it is to accommodate for sharing caches,
#   not necessarily because it makes logical sense to run one job after the
#   other. This is due to the fact that we can't share caches between jobs that
#   run in parallel.
# - `sccache` is a good alternative to `Swatinem/rust-cache`, but it behaves
#   poorly with GHA and often incurs into cache requests rate limits. We should
#   probably explore `sccache` with a different backend.
# - If a job makes good use of extra cores, consider give it a bigger machine.
#   GHA larger runners increase in cost linearly with the number of cores
#   (https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions),
#   so you're not wasting money unless several cores are sitting idle for long.

on:
  # Relevant docs:
  # - https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue#how-merge-queues-work
  # - https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#merge_group
  merge_group:
    types: ["checks_requested"]
  push:
    branches: ["nightly", "stable"]
  pull_request:
    branches: ["nightly", "stable"]

env:
  CARGO_TERM_COLOR: always
  RUSTFLAGS: -D warnings

# Automatically cancels a job if a new commit if pushed to the same PR, branch, or tag.
# Source: <https://stackoverflow.com/a/72408109/5148606>
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  # Except in `nightly` and `stable` branches! Any cancelled job will cause the
  # CI run to fail, and we want to keep a clean history for major branches.
  cancel-in-progress: ${{ (github.ref != 'refs/heads/nightly') && (github.ref != 'refs/heads/stable') }}

jobs:
  check-aggregate:
    # Follows the guide at <https://github.com/re-actors/alls-green>.
    runs-on: ubuntu-latest
    if: always()
    needs:
      # If you're modifying this workflow file and you're adding/removing a job
      # which should be required to pass before merging a PR, don't forget to
      # update this list!
      - check
      - check_no_std
      - hack
      - nextest
      - test
      - coverage
      - check-licenses
      - cargo-doc-artifact
      - deploy-github-pages
      - check-demo-rollup-table-of-contents
      - check-demo-rollup-bash-commands
      - validate-packages-to-publish-yml
    steps:
      - name: Compute whether the needed jobs succeeded or failed
        uses: re-actors/alls-green@release/v1
        with:
          allowed-skips: deploy-github-pages
          jobs: ${{ toJSON(needs) }}

  check:
    name: check
    runs-on: buildjet-4vcpu-ubuntu-2204
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v3
      - uses: rui314/setup-mold@v1
      - name: Install Protoc
        uses: arduino/setup-protoc@v2
        with:
          version: "23.2"
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install Rust
        run: rustup show && rustup install nightly && rustup component add rustfmt --toolchain nightly-x86_64-unknown-linux-gnu # Nightly is needed for our configuration of cargo fmt
      - name: Install cargo-risczero
        uses: taiki-e/install-action@v2
        with:
          tool: cargo-risczero@0.19
      - name: Install risc0-zkvm toolchain # Use the risc0 cargo extension to install the risc0 std library for the current toolchain
        run: cargo risczero install
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - uses: Swatinem/rust-cache@v2
        with:
          cache-provider: "buildjet"
          shared-key: cargo-check-cache
          save-if: ${{ github.ref == 'refs/heads/nightly' }}
          workspaces: |
            .
            fuzz
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Run lint
        run: |
          if ! make lint ; then
            echo "Linting or formatting errors detected, please run 'make lint-fix' to fix it";
            exit 1
          fi

  check_no_std:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Install Rust Bare Metal
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          target: thumbv6m-none-eabi
          override: true
      - name: Install Rust WASM
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          target: wasm32-unknown-unknown
          override: true
      - name: cargo install cargo-hack
        uses: taiki-e/install-action@cargo-hack
      - uses: Swatinem/rust-cache@v2
        with:
          save-if: ${{ github.ref == 'refs/heads/nightly' }}
      - name: Run check
        run: make check-no-std

  prover_bench_check:
    name: prover_bench_check
    needs: check
    runs-on: buildjet-8vcpu-ubuntu-2204
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v3
      - uses: rui314/setup-mold@v1
      - name: Install Protoc
        uses: arduino/setup-protoc@v2
        with:
          version: "23.2"
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install Rust
        run: rustup show && rustup install nightly && rustup component add rustfmt --toolchain nightly-x86_64-unknown-linux-gnu
      - name: Install cargo-risczero
        uses: taiki-e/install-action@v2
        with:
          tool: cargo-risczero@0.19
      - name: Install risc0-zkvm toolchain # Use the risc0 cargo extension to install the risc0 std library for the current toolchain
        run: cargo risczero install
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - uses: Swatinem/rust-cache@v2
        with:
          cache-provider: "buildjet"
          shared-key: cargo-check-cache
          save-if: ${{ github.ref == 'refs/heads/nightly' }}
          workspaces: |
            .
            fuzz
      - name: cargo prover bench check
        env:
          BLOCKS: 1
          TXNS_PER_BLOCK: 10
          NUM_PUB_KEYS: 100
        run: sh ci_prover_count_check.sh

  bench_check:
    name: bench_check
    needs: check
    runs-on: buildjet-8vcpu-ubuntu-2204
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v3
      - uses: rui314/setup-mold@v1
      - name: Install Protoc
        uses: arduino/setup-protoc@v2
        with:
          version: "23.2"
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install Rust
        run: rustup show && rustup install nightly && rustup component add rustfmt --toolchain nightly-x86_64-unknown-linux-gnu
      - name: Install cargo-risczero
        uses: taiki-e/install-action@v2
        with:
          tool: cargo-risczero@0.19
      - name: Install risc0-zkvm toolchain # Use the risc0 cargo extension to install the risc0 std library for the current toolchain
        run: cargo risczero install
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - uses: Swatinem/rust-cache@v2
        with:
          cache-provider: "buildjet"
          shared-key: cargo-check-cache
          save-if: ${{ github.ref == 'refs/heads/nightly' }}
          workspaces: |
            .
            fuzz
      - name: cargo bench check
        env:
          BLOCKS: 1
          TXNS_PER_BLOCK: 10
          NUM_PUB_KEYS: 100
        run: cargo bench

  rollup_bench_check:
    name: rollup_bench_check
    needs: check
    runs-on: buildjet-8vcpu-ubuntu-2204
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v3
      - uses: rui314/setup-mold@v1
      - name: Install Protoc
        uses: arduino/setup-protoc@v2
        with:
          version: "23.2"
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install Rust
        run: rustup show && rustup install nightly && rustup component add rustfmt --toolchain nightly-x86_64-unknown-linux-gnu
      - name: Install cargo-risczero
        uses: taiki-e/install-action@v2
        with:
          tool: cargo-risczero@0.19
      - name: Install risc0-zkvm toolchain # Use the risc0 cargo extension to install the risc0 std library for the current toolchain
        run: cargo risczero install
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - uses: Swatinem/rust-cache@v2
        with:
          cache-provider: "buildjet"
          shared-key: cargo-check-cache
          save-if: ${{ github.ref == 'refs/heads/nightly' }}
          workspaces: |
            .
            fuzz
      - name: rollup bench check
        env:
          BLOCKS: 10
          TXNS_PER_BLOCK: 10000
          NUM_PUB_KEYS: 100
        run: sh ci_rollup_count_check.sh

  # Check that every combination of features is working properly.
  hack:
    name: features
    # `cargo-hack` uses the same profile as `cargo check` and doesn't require
    # building dependencies, only checking them, so we can share caches
    # effectively.
    needs: check
    runs-on: buildjet-8vcpu-ubuntu-2204
    timeout-minutes: 120
    env:
      SKIP_GUEST_BUILD: "1"
    steps:
      - uses: actions/checkout@v3
      - uses: rui314/setup-mold@v1
      - name: Install Protoc
        uses: arduino/setup-protoc@v2
        with:
          version: "23.2"
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install Rust
        run: rustup show
      - name: cargo install cargo-hack
        uses: taiki-e/install-action@cargo-hack
      - uses: Swatinem/rust-cache@v2
        with:
          cache-provider: "buildjet"
          shared-key: cargo-check-cache
          save-if: ${{ github.ref == 'refs/heads/nightly' }}
          workspaces: |
            .
            fuzz
      # intentionally no target specifier; see https://github.com/jonhoo/rust-ci-conf/pull/4
      - name: cargo hack
        run: make check-features
  nextest:
    name: nextest
    runs-on: buildjet-4vcpu-ubuntu-2204
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v3
      - uses: rui314/setup-mold@v1
      - name: Install Protoc
        uses: arduino/setup-protoc@v2
        with:
          version: "23.2"
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install cargo-risczero
        uses: taiki-e/install-action@v2
        with:
          tool: cargo-risczero@0.19
      - name: Install risc0-zkvm toolchain # Use the risc0 cargo extension to install the risc0 std library for the current toolchain
        run: cargo risczero install
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        # `cargo-nextest` is much faster than standard `cargo test`.
      - uses: taiki-e/install-action@nextest
      - name: Install Rust
        run: rustup show
      - uses: Swatinem/rust-cache@v2
        with:
          cache-provider: "buildjet"
          save-if: ${{ github.ref == 'refs/heads/nightly' }}
      - run: cargo nextest run --workspace --all-features
  test:
    name: test
    runs-on: buildjet-4vcpu-ubuntu-2204
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v3
      - uses: rui314/setup-mold@v1
      - name: Install Protoc
        uses: arduino/setup-protoc@v2
        with:
          version: "23.2"
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install cargo-risczero
        uses: taiki-e/install-action@v2
        with:
          tool: cargo-risczero@0.19
      - name: Install risc0-zkvm toolchain # Use the risc0 cargo extension to install the risc0 std library for the current toolchain
        run: cargo risczero install
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Install Rust
        run: rustup show
      - uses: Swatinem/rust-cache@v2
        with:
          cache-provider: "buildjet"
          save-if: ${{ github.ref == 'refs/heads/nightly' }}
      # `cargo-nextest` does not support doctests (yet?), so we have to run them
      # separately.
      # TODO: https://github.com/nextest-rs/nextest/issues/16
      - run: cargo test --workspace --doc --all-features
  coverage:
    name: coverage
    runs-on: buildjet-8vcpu-ubuntu-2204
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: true
      - uses: rui314/setup-mold@v1
      - name: Install Protoc
        uses: arduino/setup-protoc@v2
        with:
          version: "23.2"
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install Rust
        run: rustup show
      - name: add llvm component
        run: rustup component add llvm-tools-preview
      - name: cargo install cargo-llvm-cov
        uses: taiki-e/install-action@cargo-llvm-cov
      - name: Install cargo-risczero
        uses: taiki-e/install-action@v2
        with:
          tool: cargo-risczero@0.19
      - name: Install risc0-zkvm toolchain
        run: cargo risczero install
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - uses: Swatinem/rust-cache@v2
        with:
          cache-provider: "buildjet"
          save-if: ${{ github.ref == 'refs/heads/nightly' }}
      - name: cargo generate-lockfile
        if: hashFiles('Cargo.lock') == ''
        run: cargo generate-lockfile
      - name: cargo llvm-cov
        run: make coverage
      - name: Upload to codecov.io
        uses: codecov/codecov-action@v3
        with:
          # When retry is implemented, we can enable it again: https://github.com/codecov/codecov-action/issues/926
          fail_ci_if_error: false
  check-licenses:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v3
      - uses: taiki-e/install-action@cargo-deny
      - run: cargo deny check licenses
  cargo-doc-artifact:
    runs-on: ubuntu-latest
    timeout-minutes: 90
    env:
      RUSTDOCFLAGS: "-D warnings"
      SKIP_GUEST_BUILD: "1"
    steps:
      - uses: actions/checkout@v3
        # Not sure installing `mold` is actually needed, but it's what the
        # `check` job, and we'd rather keep things as similar as possible.
      - uses: rui314/setup-mold@v1
      - name: Install Protoc
        uses: arduino/setup-protoc@v2
        with:
          version: "23.2"
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install Rust
        run: rustup show
      - uses: Swatinem/rust-cache@v2
        with:
          cache-provider: "buildjet"
          save-if: ${{ github.ref == 'refs/heads/nightly' }}
      # The docs' artifact tends to become quite large with all the
      # dependencies, so we don't include them.
      - run: cargo doc --no-deps --all-features
      - name: Add index.html
        # We're inside a Cargo workspace, so there's no index.html by default.
        # We must redirect to one of the workspace member crates' documentation.
        # <https://dev.to/deciduously/prepare-your-rust-api-docs-for-github-pages-2n5i>
        run: echo '<meta http-equiv="refresh" content="0; url=sov_rollup_interface/index.html">' > target/doc/index.html
      # https://github.com/actions/deploy-pages/issues/188
      - name: Fix assets' file permissions
        run: chmod -c -R +rX "target/doc"
      - name: Upload artifacts
        uses: actions/upload-pages-artifact@v1
        with:
          path: target/doc
  deploy-github-pages:
    needs: cargo-doc-artifact
    timeout-minutes: 5
    if: github.ref == 'refs/heads/stable'
    # Grant GITHUB_TOKEN the permissions required to make a Pages deployment
    permissions:
      pages: write # to deploy to Pages
      id-token: write # to verify the deployment originates from an appropriate source
    # Deploy to the github-pages environment
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    runs-on: ubuntu-latest
    steps:
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v2
  check-demo-rollup-table-of-contents:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-node@v3
      - run: npm install -g doctoc
      - name: Update table of contents for mock-da
        run: doctoc README.md --github --notitle
        working-directory: ./examples/demo-rollup
      - name: Update table of contents for celestia-da
        run: doctoc README_CELESTIA.md --github --notitle
        working-directory: ./examples/demo-rollup
      - name: Check table of contents
        # Exit status 0 means no changes were made, so the table of contents is
        # up to date.
        run: git diff --exit-code
  check-demo-rollup-bash-commands:
    runs-on: buildjet-4vcpu-ubuntu-2204
    needs: nextest
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v3
      - uses: rui314/setup-mold@v1
      - name: Install Protoc
        uses: arduino/setup-protoc@v2
        with:
          version: "23.2"
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install cargo-risczero
        uses: taiki-e/install-action@v2
        with:
          tool: cargo-risczero@0.19
      - name: Install risc0-zkvm toolchain # Use the risc0 cargo extension to install the risc0 std library for the current toolchain
        run: cargo risczero install
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Install Rust
        run: rustup show
      - uses: Swatinem/rust-cache@v2
        with:
          cache-provider: "buildjet"
          save-if: ${{ github.ref == 'refs/heads/nightly' }}
      - name: Compile README.md to Bash
        run: cargo run --bin bashtestmd -- --input examples/demo-rollup/README_CELESTIA.md --output demo-rollup-readme.sh --tag test-ci
      - run: cat demo-rollup-readme.sh
      - run: chmod +x demo-rollup-readme.sh && ./demo-rollup-readme.sh
  check-demo-rollup-bash-commands-mock-da:
    runs-on: buildjet-4vcpu-ubuntu-2204
    needs: nextest
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v3
      - uses: rui314/setup-mold@v1
      - name: Install Protoc
        uses: arduino/setup-protoc@v2
        with:
          version: "23.2"
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install cargo-risczero
        uses: taiki-e/install-action@v2
        with:
          tool: cargo-risczero@0.19
      - name: Install risc0-zkvm toolchain # Use the risc0 cargo extension to install the risc0 std library for the current toolchain
        run: cargo risczero install
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Install Rust
        run: rustup show
      - uses: Swatinem/rust-cache@v2
        with:
          cache-provider: "buildjet"
          save-if: ${{ github.ref == 'refs/heads/nightly' }}
      - name: Compile README.md to Bash
        run: cargo run --bin bashtestmd -- --input examples/demo-rollup/README.md --output demo-rollup-readme.sh --tag test-ci
      - run: cat demo-rollup-readme.sh
      - run: chmod +x demo-rollup-readme.sh && ./demo-rollup-readme.sh

  validate-packages-to-publish-yml:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - run: ./scripts/validate_packages_to_publish_yml.sh
        shell: bash
        working-directory: .