diff --git a/.github/workflows/build-enzyme-v0.0.130.yaml b/.github/workflows/build-enzyme-v0.0.130.yaml deleted file mode 100644 index 297dadaec7..0000000000 --- a/.github/workflows/build-enzyme-v0.0.130.yaml +++ /dev/null @@ -1,155 +0,0 @@ -name: Build Enzyme v0.0.130 - -on: - push: - branches: [ main ] - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - determine_runner: - if: github.event.pull_request.draft == false - name: Determine runner type to use - uses: ./.github/workflows/determine-workflow-runner.yml - with: - default_runner: ubuntu-22.04 - - constants: - name: "Set build matrix" - uses: ./.github/workflows/constants.yaml - needs: [determine_runner] - with: - multiple_compilers: ${{ github.trigger == 'push' && github.ref_name == 'main' }} - runs_on: ${{ needs.determine_runner.outputs.runner_group }} - - llvm: - name: LLVM Build - needs: [constants, determine_runner] - runs-on: ${{ needs.determine_runner.outputs.runner_group }} - strategy: - matrix: - compiler: ${{ fromJson(needs.constants.outputs.compilers) }} - - steps: - - name: Checkout Catalyst repo - uses: actions/checkout@v4 - - # Both the LLVM source and build folder are required for further dialect builds. - # Caching is significantly faster than git cloning since LLVM is such a large repository. - - - name: Cache LLVM Source - id: cache-llvm-source - uses: actions/cache@v4 - with: - path: mlir/llvm-project - key: llvm-${{ needs.constants.outputs.llvm_version }}-default-source - enableCrossOsArchive: true - - - name: Clone LLVM Submodule - if: steps.cache-llvm-source.outputs.cache-hit != 'true' - uses: actions/checkout@v4 - with: - repository: llvm/llvm-project - ref: ${{ needs.constants.outputs.llvm_version }} - path: mlir/llvm-project - - - name: Cache LLVM Build - id: cache-llvm-build - uses: actions/cache@v4 - with: - path: llvm-build - key: ${{ runner.os }}-llvm-${{ needs.constants.outputs.llvm_version }}-default-build-${{ matrix.compiler }} - - - name: Install Deps - if: steps.cache-llvm-build.outputs.cache-hit != 'true' - run: | - sudo apt-get update - sudo apt-get install -y python3 python3-pip cmake ninja-build clang lld - python3 --version | grep ${{ needs.constants.outputs.primary_python_version }} - python3 -m pip install numpy pybind11 - - - name: Build LLVM - if: steps.cache-llvm-build.outputs.cache-hit != 'true' - # Note: Disable instrumentation for the mlir runtime support library, - # as user programs aren't instrumented. - run: | - # echo 'target_compile_options(mlir_c_runner_utils PRIVATE "-fno-sanitize=all")' \ - # >> mlir/llvm-project/mlir/lib/ExecutionEngine/CMakeLists.txt - C_COMPILER=$(which ${{ needs.constants.outputs[format('c_compiler.{0}', matrix.compiler)] }}) \ - CXX_COMPILER=$(which ${{ needs.constants.outputs[format('cxx_compiler.{0}', matrix.compiler)] }}) \ - LLVM_BUILD_DIR="$(pwd)/llvm-build" \ - COMPILER_LAUNCHER="" \ - make llvm - - enzyme: - name: Enzyme Build - needs: [constants, llvm, determine_runner] - runs-on: ${{ needs.determine_runner.outputs.runner_group }} - strategy: - matrix: - compiler: ${{ fromJson(needs.constants.outputs.compilers) }} - - steps: - - name: Checkout Catalyst repo - uses: actions/checkout@v4 - - - name: Cache Enzyme Source - id: cache-enzyme-source - uses: actions/cache@v4 - with: - path: mlir/Enzyme - key: enzyme-v0.0.130-default-source - enableCrossOsArchive: true - - - name: Clone Enzyme Submodule - if: steps.cache-enzyme-build.outputs.cache-hit != 'true' - uses: actions/checkout@v4 - with: - repository: EnzymeAD/Enzyme - ref: v0.0.130 - path: mlir/Enzyme - - - name: Cache Enzyme Build - id: cache-enzyme-build - uses: actions/cache@v4 - with: - path: enzyme-build - key: ${{ runner.os }}-enzyme-${{ needs.constants.outputs.llvm_version }}-v0.0.130-default-build-${{ matrix.compiler }} - - - name: Get Cached LLVM Source - id: cache-llvm-source - if: steps.cache-enzyme-build.outputs.cache-hit != 'true' - uses: actions/cache@v4 - with: - path: mlir/llvm-project - key: llvm-${{ needs.constants.outputs.llvm_version }}-default-source - enableCrossOsArchive: true - fail-on-cache-miss: true - - - name: Get Cached LLVM Build - id: cache-llvm-build - if: steps.cache-enzyme-build.outputs.cache-hit != 'true' - uses: actions/cache@v4 - with: - path: llvm-build - key: ${{ runner.os }}-llvm-${{ needs.constants.outputs.llvm_version }}-default-build-${{ matrix.compiler }} - fail-on-cache-miss: true - - - name: Install Deps - if: steps.cache-enzyme-build.outputs.cache-hit != 'true' - run: | - sudo apt-get update - sudo apt-get install -y cmake ninja-build clang lld - - - name: Build Enzyme - if: steps.cache-enzyme-build.outputs.cache-hit != 'true' - run: | - C_COMPILER=$(which ${{ needs.constants.outputs[format('c_compiler.{0}', matrix.compiler)] }}) \ - CXX_COMPILER=$(which ${{ needs.constants.outputs[format('cxx_compiler.{0}', matrix.compiler)] }}) \ - LLVM_BUILD_DIR="$(pwd)/llvm-build" \ - ENZYME_BUILD_DIR="$(pwd)/enzyme-build" \ - COMPILER_LAUNCHER="" \ - make enzyme diff --git a/.github/workflows/build-wheel-linux-arm64.yaml b/.github/workflows/build-wheel-linux-arm64.yaml index fdf81109d7..52aca51cd0 100644 --- a/.github/workflows/build-wheel-linux-arm64.yaml +++ b/.github/workflows/build-wheel-linux-arm64.yaml @@ -62,7 +62,7 @@ jobs: id: cache-llvm-source uses: actions/cache@v4 with: - path: mlir/llvm-project + path: ${{ github.workspace }}/mlir/llvm-project key: llvm-${{ needs.constants.outputs.llvm_version }}-default-source enableCrossOsArchive: True @@ -70,7 +70,7 @@ jobs: id: cache-mhlo-source uses: actions/cache@v4 with: - path: mlir/mlir-hlo + path: ${{ github.workspace }}/mlir/mlir-hlo key: mhlo-${{ needs.constants.outputs.mhlo_version }}-default-source enableCrossOsArchive: True @@ -78,7 +78,7 @@ jobs: id: cache-enzyme-source uses: actions/cache@v4 with: - path: mlir/Enzyme + path: ${{ github.workspace }}/mlir/Enzyme key: enzyme-${{ needs.constants.outputs.enzyme_version }}-default-source enableCrossOsArchive: True @@ -88,7 +88,7 @@ jobs: with: repository: llvm/llvm-project ref: ${{ needs.constants.outputs.llvm_version }} - path: mlir/llvm-project + path: ${{ github.workspace }}/mlir/llvm-project - name: Clone MHLO Submodule if: steps.cache-mhlo-source.outputs.cache-hit != 'true' @@ -96,7 +96,7 @@ jobs: with: repository: tensorflow/mlir-hlo ref: ${{ needs.constants.outputs.mhlo_version }} - path: mlir/mlir-hlo + path: ${{ github.workspace }}/mlir/mlir-hlo - name: Clone Enzyme Submodule if: steps.cache-enzyme-source.outputs.cache-hit != 'true' @@ -104,21 +104,21 @@ jobs: with: repository: EnzymeAD/Enzyme ref: ${{ needs.constants.outputs.enzyme_version }} - path: mlir/Enzyme + path: ${{ github.workspace }}/mlir/Enzyme # Cache external project builds - name: Restore LLVM Build id: cache-llvm-build uses: actions/cache/restore@v4 with: - path: llvm-build + path: ${{ github.workspace }}/llvm-build key: ${{ matrix.container_name }}-llvm-${{ needs.constants.outputs.llvm_version }}-wheel-build - name: Restore MHLO Build id: cache-mhlo-build uses: actions/cache/restore@v4 with: - path: mhlo-build + path: ${{ github.workspace }}/mhlo-build key: ${{ matrix.container_name }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-wheel-build lookup-only: True @@ -126,22 +126,10 @@ jobs: id: cache-enzyme-build uses: actions/cache/restore@v4 with: - path: enzyme-build + path: ${{ github.workspace }}/enzyme-build key: ${{ matrix.container_name }}-enzyme-${{ needs.constants.outputs.llvm_version }}-${{ needs.constants.outputs.enzyme_version }}-wheel-build lookup-only: True - - name: Build LLD - if: steps.cache-llvm-build.outputs.cache-hit != 'true' - run: | - set -x - # With GCC 13, LLVM fails some tests, then we use GCC 12 instead (Copied from below) - export GCC_VERSION=12 - docker run --rm --platform linux/aarch64 \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v `pwd`:/catalyst \ - -i ${{ matrix.container_img }} \ - bash /catalyst/.github/workflows/scripts/linux_arm64/rh8/build_lld.sh $GCC_VERSION ${{ matrix.python_version.major_minor }} ${{ matrix.python_version.patch }} ${{ matrix.python_version.package }} - - name: Build LLVM / MLIR if: steps.cache-llvm-build.outputs.cache-hit != 'true' run: | @@ -159,7 +147,7 @@ jobs: if: steps.cache-llvm-build.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: - path: llvm-build + path: ${{ github.workspace }}/llvm-build key: ${{ matrix.container_name }}-llvm-${{ needs.constants.outputs.llvm_version }}-wheel-build - name: Build MHLO Dialect @@ -178,7 +166,7 @@ jobs: if: steps.cache-mhlo-build.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: - path: mhlo-build + path: ${{ github.workspace }}/mhlo-build key: ${{ matrix.container_name }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-wheel-build - name: Build Enzyme @@ -197,7 +185,7 @@ jobs: if: steps.cache-enzyme-build.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: - path: enzyme-build + path: ${{ github.workspace }}/enzyme-build key: ${{ matrix.container_name }}-enzyme-${{ needs.constants.outputs.llvm_version }}-${{ needs.constants.outputs.enzyme_version }}-wheel-build catalyst-linux-wheels-arm64: @@ -230,7 +218,7 @@ jobs: id: cache-llvm-source uses: actions/cache/restore@v4 with: - path: mlir/llvm-project + path: ${{ github.workspace }}/mlir/llvm-project key: llvm-${{ needs.constants.outputs.llvm_version }}-default-source enableCrossOsArchive: True fail-on-cache-miss: True @@ -239,7 +227,7 @@ jobs: id: cache-llvm-build uses: actions/cache/restore@v4 with: - path: llvm-build + path: ${{ github.workspace }}/llvm-build key: ${{ matrix.container_name }}-llvm-${{ needs.constants.outputs.llvm_version }}-wheel-build fail-on-cache-miss: True @@ -247,7 +235,7 @@ jobs: id: cache-mhlo-source uses: actions/cache/restore@v4 with: - path: mlir/mlir-hlo + path: ${{ github.workspace }}/mlir/mlir-hlo key: mhlo-${{ needs.constants.outputs.mhlo_version }}-default-source enableCrossOsArchive: True fail-on-cache-miss: True @@ -256,7 +244,7 @@ jobs: id: cache-mhlo-build uses: actions/cache/restore@v4 with: - path: mhlo-build + path: ${{ github.workspace }}/mhlo-build key: ${{ matrix.container_name }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-wheel-build fail-on-cache-miss: True @@ -264,7 +252,7 @@ jobs: id: cache-enzyme-source uses: actions/cache/restore@v4 with: - path: mlir/Enzyme + path: ${{ github.workspace }}/mlir/Enzyme key: enzyme-${{ needs.constants.outputs.enzyme_version }}-default-source enableCrossOsArchive: True fail-on-cache-miss: True @@ -273,7 +261,7 @@ jobs: id: cache-enzyme-build uses: actions/cache/restore@v4 with: - path: enzyme-build + path: ${{ github.workspace }}/enzyme-build key: ${{ matrix.container_name }}-enzyme-${{ needs.constants.outputs.llvm_version }}-${{ needs.constants.outputs.enzyme_version }}-wheel-build fail-on-cache-miss: True @@ -292,7 +280,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: catalyst-linux_arm64-wheel-py-${{ matrix.python_version.major_minor}}.zip - path: wheel/ + path: ${{ github.workspace }}/wheel/ retention-days: 14 test-wheels: @@ -326,16 +314,7 @@ jobs: uses: actions/download-artifact@v4 with: name: catalyst-linux_arm64-wheel-py-${{ matrix.python_version.major_minor }}.zip - path: dist - - # Needed for accessing llvm-symbolizer - - name: Get Cached LLVM Build - id: cache-llvm-build - uses: actions/cache@v4 - with: - path: llvm-build - key: ${{ matrix.container_name }}-llvm-${{ needs.constants.outputs.llvm_version }}-wheel-build - fail-on-cache-miss: True + path: ${{ github.workspace }}/dist - name: Run Python Pytest Tests run: | diff --git a/.github/workflows/build-wheel-linux-x86_64.yaml b/.github/workflows/build-wheel-linux-x86_64.yaml index b7104482fe..a831c9ebb3 100644 --- a/.github/workflows/build-wheel-linux-x86_64.yaml +++ b/.github/workflows/build-wheel-linux-x86_64.yaml @@ -71,7 +71,7 @@ jobs: id: cache-llvm-source uses: actions/cache@v4 with: - path: mlir/llvm-project + path: ${{ github.workspace }}/mlir/llvm-project key: llvm-${{ needs.constants.outputs.llvm_version }}-container-source enableCrossOsArchive: True @@ -79,7 +79,7 @@ jobs: id: cache-mhlo-source uses: actions/cache@v4 with: - path: mlir/mlir-hlo + path: ${{ github.workspace }}/mlir/mlir-hlo key: mhlo-${{ needs.constants.outputs.mhlo_version }}-container-source enableCrossOsArchive: True @@ -87,7 +87,7 @@ jobs: id: cache-enzyme-source uses: actions/cache@v4 with: - path: mlir/Enzyme + path: ${{ github.workspace }}/mlir/Enzyme key: enzyme-${{ needs.constants.outputs.enzyme_version }}-container-source enableCrossOsArchive: True @@ -97,7 +97,7 @@ jobs: with: repository: llvm/llvm-project ref: ${{ needs.constants.outputs.llvm_version }} - path: mlir/llvm-project + path: ${{ github.workspace }}/mlir/llvm-project - name: Clone MHLO Submodule if: steps.cache-mhlo-source.outputs.cache-hit != 'true' @@ -105,7 +105,7 @@ jobs: with: repository: tensorflow/mlir-hlo ref: ${{ needs.constants.outputs.mhlo_version }} - path: mlir/mlir-hlo + path: ${{ github.workspace }}/mlir/mlir-hlo - name: Clone Enzyme Submodule if: steps.cache-enzyme-source.outputs.cache-hit != 'true' @@ -113,21 +113,21 @@ jobs: with: repository: EnzymeAD/Enzyme ref: ${{ needs.constants.outputs.enzyme_version }} - path: mlir/Enzyme + path: ${{ github.workspace }}/mlir/Enzyme # Cache external project builds - name: Restore LLVM Build id: cache-llvm-build uses: actions/cache/restore@v4 with: - path: llvm-build + path: ${{ github.workspace }}/llvm-build key: ${{ matrix.container_img }}-llvm-${{ needs.constants.outputs.llvm_version }}-${{matrix.python_version}}-wheel-build - name: Restore MHLO Build id: cache-mhlo-build uses: actions/cache/restore@v4 with: - path: mhlo-build + path: ${{ github.workspace }}/mhlo-build key: ${{ matrix.container_img }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-wheel-build lookup-only: True @@ -135,7 +135,7 @@ jobs: id: cache-enzyme-build uses: actions/cache/restore@v4 with: - path: enzyme-build + path: ${{ github.workspace }}/enzyme-build key: ${{ matrix.container_img }}-enzyme-${{ needs.constants.outputs.llvm_version }}-${{ needs.constants.outputs.enzyme_version }}-wheel-build lookup-only: True @@ -158,49 +158,29 @@ jobs: PYTHON_BINS=$(find /opt/_internal/cpython-${{ matrix.python_version }}.*/bin -maxdepth 1 -type d | tr '\n' ':' | sed 's/:$//') echo $PYTHON_BINS >> $GITHUB_PATH - # Required for MHLO and building MLIR with protected symbols. + # LLD is required for MHLO builds. # (Don't forget to add the build directory to PATH in subsequent steps, so # other tools can find it, in particular collect2 invoked by gcc.) - - name: Build LLD - if: steps.cache-llvm-build.outputs.cache-hit != 'true' - run: | - cmake -S mlir/llvm-project/llvm -B llvm-build -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_TARGETS_TO_BUILD="host" \ - -DLLVM_ENABLE_PROJECTS="lld" - - cmake --build llvm-build --target lld - - name: Build LLVM / MLIR if: steps.cache-llvm-build.outputs.cache-hit != 'true' run: | export PATH=$GITHUB_WORKSPACE/llvm-build/bin:$PATH - cmake -S mlir/llvm-project/llvm -B llvm-build -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_BUILD_EXAMPLES=OFF \ - -DLLVM_TARGETS_TO_BUILD="host" \ - -DLLVM_ENABLE_PROJECTS="mlir" \ - -DLLVM_ENABLE_ASSERTIONS=ON \ - -DLLVM_INSTALL_UTILS=ON \ - -DLLVM_ENABLE_ZLIB=FORCE_ON \ - -DLLVM_ENABLE_ZSTD=OFF \ - -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ - -DPython3_EXECUTABLE=$(which python${{ matrix.python_version }}) \ - -DPython3_NumPy_INCLUDE_DIRS=$(python${{ matrix.python_version }} -c "import numpy as np; print(np.get_include())") \ - -DCMAKE_CXX_VISIBILITY_PRESET=default \ - -DLLVM_ENABLE_LLD=ON - - # TODO: when updating LLVM, test to see if mlir/unittests/Bytecode/BytecodeTest.cpp:55 is passing - # and remove filter - # This tests fails on CI/CD not locally. - LIT_FILTER_OUT="Bytecode" cmake --build llvm-build --target check-mlir + PYTHON=$(which python${{ matrix.python_version }}) \ + C_COMPILER=$(which gcc) \ + CXX_COMPILER=$(which g++) \ + LLVM_BUILD_DIR="$GITHUB_WORKSPACE/llvm-build" \ + LLVM_PROJECTS="lld;mlir" \ + LLVM_TARGETS="lld check-mlir" \ + ENABLE_ZLIB=FORCE_ON \ + ENABLE_LLD=OFF \ + make llvm - name: Save LLVM Build id: save-llvm-build if: steps.cache-llvm-build.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: - path: llvm-build + path: ${{ github.workspace }}/llvm-build key: ${{ matrix.container_img }}-llvm-${{ needs.constants.outputs.llvm_version }}-${{matrix.python_version}}-wheel-build - name: Build MHLO Dialect @@ -213,45 +193,45 @@ jobs: export PATCH_FILE=mlir/patches/mhlo-Add-PassesIncGen-in-transforms-CMakeList.patch if patch --dry-run -p1 -N $TARGET_FILE $PATCH_FILE > /dev/null 2>&1; then patch -p1 $TARGET_FILE $PATCH_FILE; fi - cmake -S mlir/mlir-hlo -B mhlo-build -G Ninja \ + cmake -S mlir/mlir-hlo -B $GITHUB_WORKSPACE/mhlo-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_ENABLE_ASSERTIONS=ON \ - -DMLIR_DIR=$GITHUB_WORKSPACE/llvm-build/lib/cmake/mlir \ + -DMLIR_DIR="$GITHUB_WORKSPACE/llvm-build/lib/cmake/mlir" \ -DPython3_EXECUTABLE=$(which python${{ matrix.python_version }}) \ -DLLVM_ENABLE_ZLIB=FORCE_ON \ -DLLVM_ENABLE_ZSTD=OFF \ -DCMAKE_CXX_VISIBILITY_PRESET=default \ -DLLVM_ENABLE_LLD=ON - LIT_FILTER_OUT="chlo_legalize_to_mhlo" cmake --build mhlo-build --target check-mlir-hlo + LIT_FILTER_OUT="chlo_legalize_to_mhlo" cmake --build $GITHUB_WORKSPACE/mhlo-build --target check-mlir-hlo - name: Save MHLO Build id: save-mhlo-build if: steps.cache-mhlo-build.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: - path: mhlo-build + path: ${{ github.workspace }}/mhlo-build key: ${{ matrix.container_img }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-wheel-build - name: Build Enzyme if: steps.cache-enzyme-build.outputs.cache-hit != 'true' run: | export PATH=$GITHUB_WORKSPACE/llvm-build/bin:$PATH - cmake -S mlir/Enzyme/enzyme -B enzyme-build -G Ninja \ + cmake -S mlir/Enzyme/enzyme -B $GITHUB_WORKSPACE/enzyme-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_DIR=$GITHUB_WORKSPACE/llvm-build/lib/cmake/llvm \ + -DLLVM_DIR="$GITHUB_WORKSPACE/llvm-build/lib/cmake/llvm" \ -DENZYME_STATIC_LIB=ON \ -DCMAKE_CXX_VISIBILITY_PRESET=default \ -DCMAKE_CXX_FLAGS="-fuse-ld=lld" - cmake --build enzyme-build --target EnzymeStatic-19 + cmake --build $GITHUB_WORKSPACE/enzyme-build --target EnzymeStatic-19 - name: Save Enzyme Build id: save-enzyme-build if: steps.cache-enzyme-build.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: - path: enzyme-build + path: ${{ github.workspace }}/enzyme-build key: ${{ matrix.container_img }}-enzyme-${{ needs.constants.outputs.llvm_version }}-${{ needs.constants.outputs.enzyme_version }}-wheel-build catalyst-linux-wheels-x86-64: @@ -289,7 +269,7 @@ jobs: id: cache-llvm-source uses: actions/cache/restore@v4 with: - path: mlir/llvm-project + path: ${{ github.workspace }}/mlir/llvm-project key: llvm-${{ needs.constants.outputs.llvm_version }}-container-source enableCrossOsArchive: True fail-on-cache-miss: True @@ -298,7 +278,7 @@ jobs: id: cache-llvm-build uses: actions/cache/restore@v4 with: - path: llvm-build + path: ${{ github.workspace }}/llvm-build key: ${{ matrix.container_img }}-llvm-${{ needs.constants.outputs.llvm_version }}-3.10-wheel-build fail-on-cache-miss: True @@ -306,7 +286,7 @@ jobs: id: cache-mhlo-source uses: actions/cache/restore@v4 with: - path: mlir/mlir-hlo + path: ${{ github.workspace }}/mlir/mlir-hlo key: mhlo-${{ needs.constants.outputs.mhlo_version }}-container-source enableCrossOsArchive: True fail-on-cache-miss: True @@ -315,7 +295,7 @@ jobs: id: cache-mhlo-build uses: actions/cache/restore@v4 with: - path: mhlo-build + path: ${{ github.workspace }}/mhlo-build key: ${{ matrix.container_img }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-wheel-build fail-on-cache-miss: True @@ -323,7 +303,7 @@ jobs: id: cache-enzyme-source uses: actions/cache/restore@v4 with: - path: mlir/Enzyme + path: ${{ github.workspace }}/mlir/Enzyme key: enzyme-${{ needs.constants.outputs.enzyme_version }}-container-source enableCrossOsArchive: True fail-on-cache-miss: True @@ -332,28 +312,28 @@ jobs: id: cache-enzyme-build uses: actions/cache/restore@v4 with: - path: enzyme-build + path: ${{ github.workspace }}/enzyme-build key: ${{ matrix.container_img }}-enzyme-${{ needs.constants.outputs.llvm_version }}-${{ needs.constants.outputs.enzyme_version }}-wheel-build fail-on-cache-miss: True # Build Catalyst-Runtime - name: Build Catalyst-Runtime run: | - cmake -S runtime -B runtime-build -G Ninja \ + cmake -S runtime -B $GITHUB_WORKSPACE/runtime-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_LIBRARY_OUTPUT_DIRECTORY=$GITHUB_WORKSPACE/runtime-build/lib \ + -DCMAKE_LIBRARY_OUTPUT_DIRECTORY="$GITHUB_WORKSPACE/runtime-build/lib" \ -DPython_EXECUTABLE=$(which python${{ matrix.python_version }}) \ -DENABLE_OPENQASM=ON - cmake --build runtime-build --target rt_capi rtd_openqasm rtd_null_qubit + cmake --build $GITHUB_WORKSPACE/runtime-build --target rt_capi rtd_openqasm rtd_null_qubit # Build OQC-Runtime - name: Build OQC-Runtime run: | C_COMPILER=$(which gcc) \ CXX_COMPILER=$(which g++) \ - OQC_BUILD_DIR=$GITHUB_WORKSPACE/oqc-build \ - RT_BUILD_DIR=$GITHUB_WORKSPACE/runtime-build \ + OQC_BUILD_DIR="$GITHUB_WORKSPACE/oqc-build" \ + RT_BUILD_DIR="$GITHUB_WORKSPACE/runtime-build" \ PYTHON=$(which python${{ matrix.python_version }}) \ make oqc @@ -362,8 +342,8 @@ jobs: run: | C_COMPILER=$(which gcc) \ CXX_COMPILER=$(which g++) \ - OQD_BUILD_DIR=$GITHUB_WORKSPACE/oqd-build \ - RT_BUILD_DIR=$GITHUB_WORKSPACE/runtime-build \ + OQD_BUILD_DIR="$GITHUB_WORKSPACE/oqd-build" \ + RT_BUILD_DIR="$GITHUB_WORKSPACE/runtime-build" \ PYTHON=$(which python${{ matrix.python_version }}) \ make oqd @@ -371,28 +351,27 @@ jobs: - name: Build MLIR Dialects run: | export PATH=$GITHUB_WORKSPACE/llvm-build/bin:$PATH - cmake -S mlir -B quantum-build -G Ninja \ + cmake -S mlir -B $GITHUB_WORKSPACE/quantum-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_ENABLE_ASSERTIONS=ON \ -DQUANTUM_ENABLE_BINDINGS_PYTHON=ON \ -DPython3_EXECUTABLE=$(which python${{ matrix.python_version }}) \ -DPython3_NumPy_INCLUDE_DIRS=$(python${{ matrix.python_version }} -c "import numpy as np; print(np.get_include())") \ - -DMLIR_DIR=$GITHUB_WORKSPACE/llvm-build/lib/cmake/mlir \ - -DMHLO_DIR=$GITHUB_WORKSPACE/mhlo-build/lib/cmake/mlir-hlo \ - -DMHLO_BINARY_DIR=$GITHUB_WORKSPACE/mhlo-build/bin \ - -DEnzyme_DIR=$GITHUB_WORKSPACE/enzyme-build \ - -DENZYME_SRC_DIR=$GITHUB_WORKSPACE/mlir/Enzyme \ + -DMLIR_DIR="$GITHUB_WORKSPACE/llvm-build/lib/cmake/mlir" \ + -DMHLO_DIR="$GITHUB_WORKSPACE/mhlo-build/lib/cmake/mlir-hlo" \ + -DMHLO_BINARY_DIR="$GITHUB_WORKSPACE/mhlo-build/bin" \ + -DEnzyme_DIR="$GITHUB_WORKSPACE/enzyme-build" \ + -DENZYME_SRC_DIR="$GITHUB_WORKSPACE/mlir/Enzyme" \ -DLLVM_ENABLE_ZLIB=FORCE_ON \ -DLLVM_ENABLE_ZSTD=OFF \ -DLLVM_ENABLE_LLD=ON - cmake --build quantum-build --target check-dialects catalyst-cli + cmake --build $GITHUB_WORKSPACE/quantum-build --target check-dialects catalyst-cli - name: Build Plugin wheel # Run only on Thursday at the given time if: github.event.schedule == '35 4 * * 4' run: | - CCACHE_DIR="$(pwd)/.ccache" \ MLIR_DIR="$GITHUB_WORKSPACE/llvm-build/lib/cmake/mlir" \ LLVM_BUILD_DIR="$GITHUB_WORKSPACE/llvm-build" \ make plugin-wheel @@ -400,13 +379,13 @@ jobs: - name: Build wheel run: | PYTHON=python${{ matrix.python_version }} \ - LLVM_BUILD_DIR=$GITHUB_WORKSPACE/llvm-build \ - MHLO_BUILD_DIR=$GITHUB_WORKSPACE/mhlo-build \ - DIALECTS_BUILD_DIR=$GITHUB_WORKSPACE/quantum-build \ - RT_BUILD_DIR=$GITHUB_WORKSPACE/runtime-build \ - OQC_BUILD_DIR=$GITHUB_WORKSPACE/oqc-build \ - OQD_BUILD_DIR=$GITHUB_WORKSPACE/oqd-build \ - ENZYME_BUILD_DIR=$GITHUB_WORKSPACE/enzyme-build \ + LLVM_BUILD_DIR="$GITHUB_WORKSPACE/llvm-build" \ + MHLO_BUILD_DIR="$GITHUB_WORKSPACE/mhlo-build" \ + DIALECTS_BUILD_DIR="$GITHUB_WORKSPACE/quantum-build" \ + RT_BUILD_DIR="$GITHUB_WORKSPACE/runtime-build" \ + OQC_BUILD_DIR="$GITHUB_WORKSPACE/oqc-build" \ + OQD_BUILD_DIR="$GITHUB_WORKSPACE/oqd-build" \ + ENZYME_BUILD_DIR="$GITHUB_WORKSPACE/enzyme-build" \ make wheel - name: Repair wheel using auditwheel @@ -418,7 +397,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: catalyst-manylinux_2_28_x86_64-wheel-py-${{ matrix.python_version }}.zip - path: wheel/ + path: ${{ github.workspace }}/wheel/ retention-days: 14 - name: Upload Standalone Plugin Wheel Artifact @@ -427,7 +406,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: standalone-plugin-manylinux_2_28_x86_64-wheel-py-${{ matrix.python_version }}.zip - path: standalone_plugin_wheel/dist + path: ${{ github.workspace }}/standalone_plugin_wheel/dist retention-days: 14 test-wheels: @@ -449,7 +428,7 @@ jobs: uses: actions/download-artifact@v4 with: name: catalyst-manylinux_2_28_x86_64-wheel-py-${{ matrix.python_version }}.zip - path: dist + path: ${{ github.workspace }}/dist - name: Download Standalone Plugin Wheel Artifact # Run only on Thursday at the given time @@ -457,7 +436,7 @@ jobs: uses: actions/download-artifact@v4 with: name: standalone-plugin-manylinux_2_28_x86_64-wheel-py-${{ matrix.python_version }}.zip - path: standalone_plugin_wheel/wheel + path: ${{ github.workspace }}/standalone_plugin_wheel/wheel - name: Set up Python ${{ matrix.python_version }} uses: actions/setup-python@v5 @@ -486,7 +465,7 @@ jobs: if: github.event.schedule == '35 4 * * 4' run: | python${{ matrix.python_version }} -m pip install standalone_plugin_wheel/wheel/*.whl --no-deps - + - name: Run Python Pytest Tests run: | python${{ matrix.python_version }} -m pytest frontend/test/pytest -n auto diff --git a/.github/workflows/build-wheel-macos-arm64.yaml b/.github/workflows/build-wheel-macos-arm64.yaml index b6a79be800..63fd776744 100644 --- a/.github/workflows/build-wheel-macos-arm64.yaml +++ b/.github/workflows/build-wheel-macos-arm64.yaml @@ -64,7 +64,7 @@ jobs: id: cache-llvm-source uses: actions/cache@v4 with: - path: mlir/llvm-project + path: ${{ github.workspace }}/mlir/llvm-project key: llvm-${{ needs.constants.outputs.llvm_version }}-default-source enableCrossOsArchive: True @@ -72,7 +72,7 @@ jobs: id: cache-mhlo-source uses: actions/cache@v4 with: - path: mlir/mlir-hlo + path: ${{ github.workspace }}/mlir/mlir-hlo key: mhlo-${{ needs.constants.outputs.mhlo_version }}-default-source enableCrossOsArchive: True @@ -80,7 +80,7 @@ jobs: id: cache-enzyme-source uses: actions/cache@v4 with: - path: mlir/Enzyme + path: ${{ github.workspace }}/mlir/Enzyme key: enzyme-${{ needs.constants.outputs.enzyme_version }}-default-source enableCrossOsArchive: True @@ -90,7 +90,7 @@ jobs: with: repository: llvm/llvm-project ref: ${{ needs.constants.outputs.llvm_version }} - path: mlir/llvm-project + path: ${{ github.workspace }}/mlir/llvm-project - name: Clone MHLO Submodule if: steps.cache-mhlo-source.outputs.cache-hit != 'true' @@ -98,7 +98,7 @@ jobs: with: repository: tensorflow/mlir-hlo ref: ${{ needs.constants.outputs.mhlo_version }} - path: mlir/mlir-hlo + path: ${{ github.workspace }}/mlir/mlir-hlo - name: Clone Enzyme Submodule if: steps.cache-enzyme-source.outputs.cache-hit != 'true' @@ -106,21 +106,21 @@ jobs: with: repository: EnzymeAD/Enzyme ref: ${{ needs.constants.outputs.enzyme_version }} - path: mlir/Enzyme + path: ${{ github.workspace }}/mlir/Enzyme # Cache external project builds - name: Restore LLVM Build id: cache-llvm-build uses: actions/cache/restore@v4 with: - path: llvm-build + path: ${{ github.workspace }}/llvm-build key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ needs.constants.outputs.llvm_version }}-${{matrix.python_version}}-wheel-build - name: Restore MHLO Build id: cache-mhlo-build uses: actions/cache/restore@v4 with: - path: mhlo-build + path: ${{ github.workspace }}/mhlo-build key: ${{ runner.os }}-${{ runner.arch }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-wheel-build lookup-only: True @@ -128,7 +128,7 @@ jobs: id: cache-enzyme-build uses: actions/cache/restore@v4 with: - path: enzyme-build + path: ${{ github.workspace }}/enzyme-build key: ${{ runner.os }}-${{ runner.arch }}-enzyme-${{ needs.constants.outputs.llvm_version }}-${{ needs.constants.outputs.enzyme_version }}-wheel-build lookup-only: True @@ -146,32 +146,18 @@ jobs: - name: Build LLVM / MLIR if: steps.cache-llvm-build.outputs.cache-hit != 'true' run: | - cmake -S mlir/llvm-project/llvm -B llvm-build -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_BUILD_EXAMPLES=OFF \ - -DLLVM_TARGETS_TO_BUILD="host" \ - -DLLVM_ENABLE_PROJECTS="mlir" \ - -DLLVM_ENABLE_ASSERTIONS=ON \ - -DLLVM_INSTALL_UTILS=ON \ - -DLLVM_ENABLE_ZLIB=FORCE_ON \ - -DLLVM_ENABLE_ZSTD=OFF \ - -DLLVM_ENABLE_LLD=OFF \ - -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ - -DPython3_EXECUTABLE=$(which python${{ matrix.python_version }}) \ - -DPython3_NumPy_INCLUDE_DIRS=$(python${{ matrix.python_version }} -c "import numpy as np; print(np.get_include())") \ - -DCMAKE_CXX_VISIBILITY_PRESET=default - - # TODO: when updating LLVM, test to see if mlir/unittests/Bytecode/BytecodeTest.cpp:55 is passing - # and remove filter - # This tests fails on CI/CD not locally. - LIT_FILTER_OUT="Bytecode" cmake --build llvm-build --target check-mlir + PYTHON=$(which python${{ matrix.python_version }}) \ + LLVM_BUILD_DIR="$GITHUB_WORKSPACE/llvm-build" \ + LLVM_TARGETS="check-mlir" \ + ENABLE_ZLIB=FORCE_ON \ + make llvm - name: Save LLVM Build id: save-llvm-build if: steps.cache-llvm-build.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: - path: llvm-build + path: ${{ github.workspace }}/llvm-build key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ needs.constants.outputs.llvm_version }}-${{matrix.python_version}}-wheel-build - name: Build MHLO Dialect @@ -183,43 +169,43 @@ jobs: export PATCH_FILE=mlir/patches/mhlo-Add-PassesIncGen-in-transforms-CMakeList.patch if patch --dry-run -p1 -N $TARGET_FILE $PATCH_FILE > /dev/null 2>&1; then patch -p1 $TARGET_FILE $PATCH_FILE; fi - cmake -S mlir/mlir-hlo -B mhlo-build -G Ninja \ + cmake -S mlir/mlir-hlo -B $GITHUB_WORKSPACE/mhlo-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_ENABLE_ASSERTIONS=ON \ - -DMLIR_DIR=$GITHUB_WORKSPACE/llvm-build/lib/cmake/mlir \ + -DMLIR_DIR="$GITHUB_WORKSPACE/llvm-build/lib/cmake/mlir" \ -DPython3_EXECUTABLE=$(which python${{ matrix.python_version }}) \ -DLLVM_ENABLE_LLD=OFF \ -DLLVM_ENABLE_ZLIB=FORCE_ON \ -DLLVM_ENABLE_ZSTD=OFF \ -DCMAKE_CXX_VISIBILITY_PRESET=default - cmake --build mhlo-build --target check-mlir-hlo + cmake --build $GITHUB_WORKSPACE/mhlo-build --target check-mlir-hlo - name: Save MHLO Build id: save-mhlo-build if: steps.cache-mhlo-build.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: - path: mhlo-build + path: ${{ github.workspace }}/mhlo-build key: ${{ runner.os }}-${{ runner.arch }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-wheel-build - name: Build Enzyme if: steps.cache-enzyme-build.outputs.cache-hit != 'true' run: | - cmake -S mlir/Enzyme/enzyme -B enzyme-build -G Ninja \ + cmake -S mlir/Enzyme/enzyme -B $GITHUB_WORKSPACE/enzyme-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_DIR=$GITHUB_WORKSPACE/llvm-build/lib/cmake/llvm \ + -DLLVM_DIR="$GITHUB_WORKSPACE/llvm-build/lib/cmake/llvm" \ -DENZYME_STATIC_LIB=ON \ -DCMAKE_CXX_VISIBILITY_PRESET=default - cmake --build enzyme-build --target EnzymeStatic-19 + cmake --build $GITHUB_WORKSPACE/enzyme-build --target EnzymeStatic-19 - name: Save Enzyme Build id: save-enzyme-build if: steps.cache-enzyme-build.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: - path: enzyme-build + path: ${{ github.workspace }}/enzyme-build key: ${{ runner.os }}-${{ runner.arch }}-enzyme-${{ needs.constants.outputs.llvm_version }}-${{ needs.constants.outputs.enzyme_version }}-wheel-build catalyst-macos-wheels-arm64: @@ -262,7 +248,7 @@ jobs: id: cache-llvm-source uses: actions/cache/restore@v4 with: - path: mlir/llvm-project + path: ${{ github.workspace }}/mlir/llvm-project key: llvm-${{ needs.constants.outputs.llvm_version }}-default-source enableCrossOsArchive: True fail-on-cache-miss: True @@ -271,7 +257,7 @@ jobs: id: cache-llvm-build uses: actions/cache/restore@v4 with: - path: llvm-build + path: ${{ github.workspace }}/llvm-build key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ needs.constants.outputs.llvm_version }}-3.10-wheel-build fail-on-cache-miss: True @@ -279,7 +265,7 @@ jobs: id: cache-mhlo-source uses: actions/cache/restore@v4 with: - path: mlir/mlir-hlo + path: ${{ github.workspace }}/mlir/mlir-hlo key: mhlo-${{ needs.constants.outputs.mhlo_version }}-default-source enableCrossOsArchive: True fail-on-cache-miss: True @@ -288,7 +274,7 @@ jobs: id: cache-mhlo-build uses: actions/cache/restore@v4 with: - path: mhlo-build + path: ${{ github.workspace }}/mhlo-build key: ${{ runner.os }}-${{ runner.arch }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-wheel-build fail-on-cache-miss: True @@ -296,7 +282,7 @@ jobs: id: cache-enzyme-source uses: actions/cache/restore@v4 with: - path: mlir/Enzyme + path: ${{ github.workspace }}/mlir/Enzyme key: enzyme-${{ needs.constants.outputs.enzyme_version }}-default-source enableCrossOsArchive: True fail-on-cache-miss: True @@ -305,7 +291,7 @@ jobs: id: cache-enzyme-build uses: actions/cache/restore@v4 with: - path: enzyme-build + path: ${{ github.workspace }}/enzyme-build key: ${{ runner.os }}-${{ runner.arch }}-enzyme-${{ needs.constants.outputs.llvm_version }}-${{ needs.constants.outputs.enzyme_version }}-wheel-build fail-on-cache-miss: True @@ -314,56 +300,56 @@ jobs: run: | # On GH images, gfortran is only available as a specific version. export FC=gfortran-14 - cmake -S runtime -B runtime-build -G Ninja \ + cmake -S runtime -B $GITHUB_WORKSPACE/runtime-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_LIBRARY_OUTPUT_DIRECTORY=$GITHUB_WORKSPACE/runtime-build/lib \ + -DCMAKE_LIBRARY_OUTPUT_DIRECTORY="$GITHUB_WORKSPACE/runtime-build/lib" \ -DPython_EXECUTABLE=$(which python${{ matrix.python_version }}) \ -DENABLE_OPENQASM=ON - cmake --build runtime-build --target rt_capi rtd_openqasm rtd_null_qubit + cmake --build $GITHUB_WORKSPACE/runtime-build --target rt_capi rtd_openqasm rtd_null_qubit - name: Test Catalyst-Runtime run: | python${{ matrix.python_version }} -m pip install 'amazon-braket-pennylane-plugin>1.27.1' - cmake --build runtime-build --target runner_tests_openqasm - ./runtime-build/tests/runner_tests_openqasm + cmake --build $GITHUB_WORKSPACE/runtime-build --target runner_tests_openqasm + $GITHUB_WORKSPACE/runtime-build/tests/runner_tests_openqasm # Build OQC-Runtime - name: Build OQC-Runtime run: | - OQC_BUILD_DIR=$GITHUB_WORKSPACE/oqc-build \ - RT_BUILD_DIR=$GITHUB_WORKSPACE/runtime-build \ + OQC_BUILD_DIR="$GITHUB_WORKSPACE/oqc-build" \ + RT_BUILD_DIR="$GITHUB_WORKSPACE/runtime-build" \ PYTHON=$(which python${{ matrix.python_version }}) \ make oqc # Build OQD-Runtime - name: Build OQD-Runtime run: | - OQD_BUILD_DIR=$GITHUB_WORKSPACE/oqd-build \ - RT_BUILD_DIR=$GITHUB_WORKSPACE/runtime-build \ + OQD_BUILD_DIR="$GITHUB_WORKSPACE/oqd-build" \ + RT_BUILD_DIR="$GITHUB_WORKSPACE/runtime-build" \ PYTHON=$(which python${{ matrix.python_version }}) \ make oqd # Build Quantum and Gradient Dialects - name: Build MLIR Dialects run: | - cmake -S mlir -B quantum-build -G Ninja \ + cmake -S mlir -B $GITHUB_WORKSPACE/quantum-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_ENABLE_ASSERTIONS=ON \ -DQUANTUM_ENABLE_BINDINGS_PYTHON=ON \ -DPython3_EXECUTABLE=$(which python${{ matrix.python_version }}) \ -DPython3_NumPy_INCLUDE_DIRS=$(python${{ matrix.python_version }} -c "import numpy as np; print(np.get_include())") \ - -DMLIR_DIR=$GITHUB_WORKSPACE/llvm-build/lib/cmake/mlir \ - -DMHLO_DIR=$GITHUB_WORKSPACE/mhlo-build/lib/cmake/mlir-hlo \ - -DMHLO_BINARY_DIR=$GITHUB_WORKSPACE/mhlo-build/bin \ - -DEnzyme_DIR=$GITHUB_WORKSPACE/enzyme-build \ - -DENZYME_SRC_DIR=$GITHUB_WORKSPACE/mlir/Enzyme \ + -DMLIR_DIR="$GITHUB_WORKSPACE/llvm-build/lib/cmake/mlir" \ + -DMHLO_DIR="$GITHUB_WORKSPACE/mhlo-build/lib/cmake/mlir-hlo" \ + -DMHLO_BINARY_DIR="$GITHUB_WORKSPACE/mhlo-build/bin" \ + -DEnzyme_DIR="$GITHUB_WORKSPACE/enzyme-build" \ + -DENZYME_SRC_DIR="$GITHUB_WORKSPACE/mlir/Enzyme" \ -DLLVM_ENABLE_ZLIB=FORCE_ON \ -DLLVM_ENABLE_ZSTD=OFF \ -DLLVM_ENABLE_LLD=OFF \ - -DLLVM_DIR=$GITHUB_WORKSPACE/llvm-build/lib/cmake/llvm + -DLLVM_DIR="$GITHUB_WORKSPACE/llvm-build/lib/cmake/llvm" - cmake --build quantum-build --target check-dialects catalyst-cli + cmake --build $GITHUB_WORKSPACE/quantum-build --target check-dialects catalyst-cli - name: Build Plugin wheel # Run only on Thursday at the given time @@ -376,13 +362,13 @@ jobs: - name: Build wheel run: | PYTHON=python${{ matrix.python_version }} \ - LLVM_BUILD_DIR=$GITHUB_WORKSPACE/llvm-build \ - MHLO_BUILD_DIR=$GITHUB_WORKSPACE/mhlo-build \ - DIALECTS_BUILD_DIR=$GITHUB_WORKSPACE/quantum-build \ - RT_BUILD_DIR=$GITHUB_WORKSPACE/runtime-build \ - OQC_BUILD_DIR=$GITHUB_WORKSPACE/oqc-build \ - OQD_BUILD_DIR=$GITHUB_WORKSPACE/oqd-build \ - ENZYME_BUILD_DIR=$GITHUB_WORKSPACE/enzyme-build \ + LLVM_BUILD_DIR="$GITHUB_WORKSPACE/llvm-build" \ + MHLO_BUILD_DIR="$GITHUB_WORKSPACE/mhlo-build" \ + DIALECTS_BUILD_DIR="$GITHUB_WORKSPACE/quantum-build" \ + RT_BUILD_DIR="$GITHUB_WORKSPACE/runtime-build" \ + OQC_BUILD_DIR="$GITHUB_WORKSPACE/oqc-build" \ + OQD_BUILD_DIR="$GITHUB_WORKSPACE/oqd-build" \ + ENZYME_BUILD_DIR="$GITHUB_WORKSPACE/enzyme-build" \ make wheel - name: Repair wheel using delocate-wheel @@ -394,7 +380,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: catalyst-macos_arm64-wheel-py-${{ matrix.python_version }}.zip - path: wheel/ + path: ${{ github.workspace }}/wheel/ retention-days: 14 - name: Upload Standalone Plugin Wheel Artifact @@ -403,7 +389,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: standalone-plugin-macos_arm64-wheel-py-${{ matrix.python_version }}.zip - path: standalone_plugin_wheel/dist + path: ${{ github.workspace }}/standalone_plugin_wheel/dist retention-days: 14 test-wheels: @@ -432,7 +418,7 @@ jobs: uses: actions/download-artifact@v4 with: name: catalyst-macos_arm64-wheel-py-${{ matrix.python_version }}.zip - path: dist + path: ${{ github.workspace }}/dist - name: Download Standalone Plugin Wheel Artifact # Run only on Thursday at the given time @@ -440,7 +426,7 @@ jobs: uses: actions/download-artifact@v4 with: name: standalone-plugin-macos_arm64-wheel-py-${{ matrix.python_version }}.zip - path: standalone_plugin_wheel/wheel + path: ${{ github.workspace }}/standalone_plugin_wheel/wheel - name: Setup Python version # There are multiple Python versions installed on the GitHub image, 3.10 - 3.12 is already diff --git a/.github/workflows/build-wheel-macos-x86_64.yaml b/.github/workflows/build-wheel-macos-x86_64.yaml index 9e99f3b6c3..19a3f341f2 100644 --- a/.github/workflows/build-wheel-macos-x86_64.yaml +++ b/.github/workflows/build-wheel-macos-x86_64.yaml @@ -56,7 +56,7 @@ jobs: id: cache-llvm-source uses: actions/cache@v4 with: - path: mlir/llvm-project + path: ${{ github.workspace }}/mlir/llvm-project key: llvm-${{ needs.constants.outputs.llvm_version }}-default-source enableCrossOsArchive: True @@ -64,7 +64,7 @@ jobs: id: cache-mhlo-source uses: actions/cache@v4 with: - path: mlir/mlir-hlo + path: ${{ github.workspace }}/mlir/mlir-hlo key: mhlo-${{ needs.constants.outputs.mhlo_version }}-default-source enableCrossOsArchive: True @@ -72,7 +72,7 @@ jobs: id: cache-enzyme-source uses: actions/cache@v4 with: - path: mlir/Enzyme + path: ${{ github.workspace }}/mlir/Enzyme key: enzyme-${{ needs.constants.outputs.enzyme_version }}-default-source enableCrossOsArchive: True @@ -82,7 +82,7 @@ jobs: with: repository: llvm/llvm-project ref: ${{ needs.constants.outputs.llvm_version }} - path: mlir/llvm-project + path: ${{ github.workspace }}/mlir/llvm-project - name: Clone MHLO Submodule if: steps.cache-mhlo-source.outputs.cache-hit != 'true' @@ -90,7 +90,7 @@ jobs: with: repository: tensorflow/mlir-hlo ref: ${{ needs.constants.outputs.mhlo_version }} - path: mlir/mlir-hlo + path: ${{ github.workspace }}/mlir/mlir-hlo - name: Clone Enzyme Submodule if: steps.cache-enzyme-source.outputs.cache-hit != 'true' @@ -98,21 +98,21 @@ jobs: with: repository: EnzymeAD/Enzyme ref: ${{ needs.constants.outputs.enzyme_version }} - path: mlir/Enzyme + path: ${{ github.workspace }}/mlir/Enzyme # Cache external project builds - name: Restore LLVM Build id: cache-llvm-build uses: actions/cache/restore@v4 with: - path: llvm-build + path: ${{ github.workspace }}/llvm-build key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ needs.constants.outputs.llvm_version }}-${{matrix.python_version}}-wheel-build - name: Restore MHLO Build id: cache-mhlo-build uses: actions/cache/restore@v4 with: - path: mhlo-build + path: ${{ github.workspace }}/mhlo-build key: ${{ runner.os }}-${{ runner.arch }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-wheel-build lookup-only: True @@ -120,7 +120,7 @@ jobs: id: cache-enzyme-build uses: actions/cache/restore@v4 with: - path: enzyme-build + path: ${{ github.workspace }}/enzyme-build key: ${{ runner.os }}-${{ runner.arch }}-enzyme-${{ needs.constants.outputs.llvm_version }}-${{ needs.constants.outputs.enzyme_version }}-wheel-build lookup-only: True @@ -136,32 +136,18 @@ jobs: - name: Build LLVM / MLIR if: steps.cache-llvm-build.outputs.cache-hit != 'true' run: | - cmake -S mlir/llvm-project/llvm -B llvm-build -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_BUILD_EXAMPLES=OFF \ - -DLLVM_TARGETS_TO_BUILD="host" \ - -DLLVM_ENABLE_PROJECTS="mlir" \ - -DLLVM_ENABLE_ASSERTIONS=ON \ - -DLLVM_INSTALL_UTILS=ON \ - -DLLVM_ENABLE_ZLIB=FORCE_ON \ - -DLLVM_ENABLE_ZSTD=OFF \ - -DLLVM_ENABLE_LLD=OFF \ - -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ - -DPython3_EXECUTABLE=$(which python${{ matrix.python_version }}) \ - -DPython3_NumPy_INCLUDE_DIRS=$(python${{ matrix.python_version }} -c "import numpy as np; print(np.get_include())") \ - -DCMAKE_CXX_VISIBILITY_PRESET=default - - # TODO: when updating LLVM, test to see if mlir/unittests/Bytecode/BytecodeTest.cpp:55 is passing - # and remove filter - # This tests fails on CI/CD not locally. - LIT_FILTER_OUT="Bytecode" cmake --build llvm-build --target check-mlir + PYTHON=$(which python${{ matrix.python_version }}) \ + LLVM_BUILD_DIR="$GITHUB_WORKSPACE/llvm-build" \ + LLVM_TARGETS="check-mlir" \ + ENABLE_ZLIB=FORCE_ON \ + make llvm - name: Save LLVM Build id: save-llvm-build if: steps.cache-llvm-build.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: - path: llvm-build + path: ${{ github.workspace }}/llvm-build key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ needs.constants.outputs.llvm_version }}-${{matrix.python_version}}-wheel-build - name: Build MHLO Dialect @@ -173,43 +159,43 @@ jobs: export PATCH_FILE=mlir/patches/mhlo-Add-PassesIncGen-in-transforms-CMakeList.patch if patch --dry-run -p1 -N $TARGET_FILE $PATCH_FILE > /dev/null 2>&1; then patch -p1 $TARGET_FILE $PATCH_FILE; fi - cmake -S mlir/mlir-hlo -B mhlo-build -G Ninja \ + cmake -S mlir/mlir-hlo -B $GITHUB_WORKSPACE/mhlo-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_ENABLE_ASSERTIONS=ON \ - -DMLIR_DIR=$GITHUB_WORKSPACE/llvm-build/lib/cmake/mlir \ + -DMLIR_DIR="$GITHUB_WORKSPACE/llvm-build/lib/cmake/mlir" \ -DPython3_EXECUTABLE=$(which python${{ matrix.python_version }}) \ -DLLVM_ENABLE_LLD=OFF \ -DLLVM_ENABLE_ZLIB=FORCE_ON \ -DLLVM_ENABLE_ZSTD=OFF \ -DCMAKE_CXX_VISIBILITY_PRESET=default - cmake --build mhlo-build --target check-mlir-hlo + cmake --build $GITHUB_WORKSPACE/mhlo-build --target check-mlir-hlo - name: Save MHLO Build id: save-mhlo-build if: steps.cache-mhlo-build.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: - path: mhlo-build + path: ${{ github.workspace }}/mhlo-build key: ${{ runner.os }}-${{ runner.arch }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-wheel-build - name: Build Enzyme if: steps.cache-enzyme-build.outputs.cache-hit != 'true' run: | - cmake -S mlir/Enzyme/enzyme -B enzyme-build -G Ninja \ + cmake -S mlir/Enzyme/enzyme -B $GITHUB_WORKSPACE/enzyme-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_DIR=$GITHUB_WORKSPACE/llvm-build/lib/cmake/llvm \ + -DLLVM_DIR="$GITHUB_WORKSPACE/llvm-build/lib/cmake/llvm" \ -DENZYME_STATIC_LIB=ON \ -DCMAKE_CXX_VISIBILITY_PRESET=default - cmake --build enzyme-build --target EnzymeStatic-19 + cmake --build $GITHUB_WORKSPACE/enzyme-build --target EnzymeStatic-19 - name: Save Enzyme Build id: save-enzyme-build if: steps.cache-enzyme-build.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: - path: enzyme-build + path: ${{ github.workspace }}/enzyme-build key: ${{ runner.os }}-${{ runner.arch }}-enzyme-${{ needs.constants.outputs.llvm_version }}-${{ needs.constants.outputs.enzyme_version }}-wheel-build catalyst-macos-wheels-x86-64: @@ -239,7 +225,7 @@ jobs: id: cache-llvm-source uses: actions/cache/restore@v4 with: - path: mlir/llvm-project + path: ${{ github.workspace }}/mlir/llvm-project key: llvm-${{ needs.constants.outputs.llvm_version }}-default-source enableCrossOsArchive: True fail-on-cache-miss: True @@ -248,7 +234,7 @@ jobs: id: cache-llvm-build uses: actions/cache/restore@v4 with: - path: llvm-build + path: ${{ github.workspace }}/llvm-build key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ needs.constants.outputs.llvm_version }}-3.10-wheel-build fail-on-cache-miss: True @@ -256,7 +242,7 @@ jobs: id: cache-mhlo-source uses: actions/cache/restore@v4 with: - path: mlir/mlir-hlo + path: ${{ github.workspace }}/mlir/mlir-hlo key: mhlo-${{ needs.constants.outputs.mhlo_version }}-default-source enableCrossOsArchive: True fail-on-cache-miss: True @@ -265,7 +251,7 @@ jobs: id: cache-mhlo-build uses: actions/cache/restore@v4 with: - path: mhlo-build + path: ${{ github.workspace }}/mhlo-build key: ${{ runner.os }}-${{ runner.arch }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-wheel-build fail-on-cache-miss: True @@ -273,7 +259,7 @@ jobs: id: cache-enzyme-source uses: actions/cache/restore@v4 with: - path: mlir/Enzyme + path: ${{ github.workspace }}/mlir/Enzyme key: enzyme-${{ needs.constants.outputs.enzyme_version }}-default-source enableCrossOsArchive: True fail-on-cache-miss: True @@ -282,7 +268,7 @@ jobs: id: cache-enzyme-build uses: actions/cache/restore@v4 with: - path: enzyme-build + path: ${{ github.workspace }}/enzyme-build key: ${{ runner.os }}-${{ runner.arch }}-enzyme-${{ needs.constants.outputs.llvm_version }}-${{ needs.constants.outputs.enzyme_version }}-wheel-build fail-on-cache-miss: True @@ -290,75 +276,74 @@ jobs: - name: Build Catalyst-Runtime run: | # Segfaults in computing Lightning's adjoint-jacobian when building with OMP - cmake -S runtime -B runtime-build -G Ninja \ + cmake -S runtime -B $GITHUB_WORKSPACE/runtime-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_LIBRARY_OUTPUT_DIRECTORY=$GITHUB_WORKSPACE/runtime-build/lib \ + -DCMAKE_LIBRARY_OUTPUT_DIRECTORY="$GITHUB_WORKSPACE/runtime-build/lib" \ -DPython_EXECUTABLE=$(which python${{ matrix.python_version }}) \ -DENABLE_OPENQASM=ON - cmake --build runtime-build --target rt_capi rtd_openqasm rtd_null_qubit + cmake --build $GITHUB_WORKSPACE/runtime-build --target rt_capi rtd_openqasm rtd_null_qubit # Build OQC-Runtime - name: Build OQC-Runtime run: | - OQC_BUILD_DIR="$(pwd)/oqc-build" \ - RT_BUILD_DIR="$(pwd)/runtime-build" \ + OQC_BUILD_DIR="$GITHUB_WORKSPACE/oqc-build" \ + RT_BUILD_DIR="$GITHUB_WORKSPACE/runtime-build" \ PYTHON=$(which python${{ matrix.python_version }}) \ make oqc # Build OQD-Runtime - name: Build OQD-Runtime run: | - OQD_BUILD_DIR="$(pwd)/oqd-build" \ - RT_BUILD_DIR="$(pwd)/runtime-build" \ + OQD_BUILD_DIR="$GITHUB_WORKSPACE/oqd-build" \ + RT_BUILD_DIR="$GITHUB_WORKSPACE/runtime-build" \ PYTHON=$(which python${{ matrix.python_version }}) \ make oqd - name: Test Catalyst-Runtime run: | python${{ matrix.python_version }} -m pip install 'amazon-braket-pennylane-plugin>1.27.1' - cmake --build runtime-build --target runner_tests_openqasm + cmake --build $GITHUB_WORKSPACE/runtime-build --target runner_tests_openqasm ./runtime-build/tests/runner_tests_openqasm # Build Quantum and Gradient Dialects - name: Build MLIR Dialects run: | - cmake -S mlir -B quantum-build -G Ninja \ + cmake -S mlir -B $GITHUB_WORKSPACE/quantum-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_ENABLE_ASSERTIONS=ON \ -DQUANTUM_ENABLE_BINDINGS_PYTHON=ON \ -DPython3_EXECUTABLE=$(which python${{ matrix.python_version }}) \ -DPython3_NumPy_INCLUDE_DIRS=$(python${{ matrix.python_version }} -c "import numpy as np; print(np.get_include())") \ - -DMLIR_DIR=$GITHUB_WORKSPACE/llvm-build/lib/cmake/mlir \ - -DMHLO_DIR=$GITHUB_WORKSPACE/mhlo-build/lib/cmake/mlir-hlo \ - -DMHLO_BINARY_DIR=$GITHUB_WORKSPACE/mhlo-build/bin \ - -DEnzyme_DIR=$GITHUB_WORKSPACE/enzyme-build \ - -DENZYME_SRC_DIR=$GITHUB_WORKSPACE/mlir/Enzyme \ + -DMLIR_DIR="$GITHUB_WORKSPACE/llvm-build/lib/cmake/mlir" \ + -DMHLO_DIR="$GITHUB_WORKSPACE/mhlo-build/lib/cmake/mlir-hlo" \ + -DMHLO_BINARY_DIR="$GITHUB_WORKSPACE/mhlo-build/bin" \ + -DEnzyme_DIR="$GITHUB_WORKSPACE/enzyme-build" \ + -DENZYME_SRC_DIR="$GITHUB_WORKSPACE/mlir/Enzyme" \ -DLLVM_ENABLE_ZLIB=FORCE_ON \ -DLLVM_ENABLE_ZSTD=OFF \ -DLLVM_ENABLE_LLD=OFF - cmake --build quantum-build --target check-dialects catalyst-cli + cmake --build $GITHUB_WORKSPACE/quantum-build --target check-dialects catalyst-cli - name: Build Plugin wheel # Run only on Thursday at the given time if: github.event.schedule == '35 4 * * 4' run: | - CCACHE_DIR="$(pwd)/.ccache" \ LLVM_BUILD_DIR="$GITHUB_WORKSPACE/llvm-build" \ - MLIR_DIR="$(pwd)/llvm-build/lib/cmake/mlir" \ + MLIR_DIR="$GITHUB_WORKSPACE/llvm-build/lib/cmake/mlir" \ make plugin-wheel - name: Build wheel run: | PYTHON=python${{ matrix.python_version }} \ - LLVM_BUILD_DIR=$GITHUB_WORKSPACE/llvm-build \ - MHLO_BUILD_DIR=$GITHUB_WORKSPACE/mhlo-build \ - DIALECTS_BUILD_DIR=$GITHUB_WORKSPACE/quantum-build \ - RT_BUILD_DIR=$GITHUB_WORKSPACE/runtime-build \ - OQC_BUILD_DIR=$GITHUB_WORKSPACE/oqc-build \ - OQD_BUILD_DIR=$GITHUB_WORKSPACE/oqd-build \ - ENZYME_BUILD_DIR=$GITHUB_WORKSPACE/enzyme-build \ + LLVM_BUILD_DIR="$GITHUB_WORKSPACE/llvm-build" \ + MHLO_BUILD_DIR="$GITHUB_WORKSPACE/mhlo-build" \ + DIALECTS_BUILD_DIR="$GITHUB_WORKSPACE/quantum-build" \ + RT_BUILD_DIR="$GITHUB_WORKSPACE/runtime-build" \ + OQC_BUILD_DIR="$GITHUB_WORKSPACE/oqc-build" \ + OQD_BUILD_DIR="$GITHUB_WORKSPACE/oqd-build" \ + ENZYME_BUILD_DIR="$GITHUB_WORKSPACE/enzyme-build" \ make wheel - name: Repair wheel using delocate-wheel @@ -370,7 +355,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: catalyst-macos_x86_64-wheel-py-${{ matrix.python_version }}.zip - path: wheel/ + path: ${{ github.workspace }}/wheel/ retention-days: 14 - name: Upload Standalone Plugin Wheel Artifact @@ -379,7 +364,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: standalone-plugin-macos_x86_64-wheel-py-${{ matrix.python_version }}.zip - path: standalone_plugin_wheel/dist + path: ${{ github.workspace }}/standalone_plugin_wheel/dist retention-days: 14 test-wheels: @@ -401,7 +386,7 @@ jobs: uses: actions/download-artifact@v4 with: name: catalyst-macos_x86_64-wheel-py-${{ matrix.python_version }}.zip - path: dist + path: ${{ github.workspace }}/dist - name: Download Standalone Plugin Wheel Artifact # Run only on Thursday at the given time @@ -409,7 +394,7 @@ jobs: uses: actions/download-artifact@v4 with: name: standalone-plugin-macos_x86_64-wheel-py-${{ matrix.python_version }}.zip - path: standalone_plugin_wheel/wheel + path: ${{ github.workspace }}/standalone_plugin_wheel/wheel - name: Set up Python ${{ matrix.python_version }} uses: actions/setup-python@v5 diff --git a/.github/workflows/scripts/linux_arm64/rh8/build_catalyst.sh b/.github/workflows/scripts/linux_arm64/rh8/build_catalyst.sh index 488cf37b4b..9014e2af81 100644 --- a/.github/workflows/scripts/linux_arm64/rh8/build_catalyst.sh +++ b/.github/workflows/scripts/linux_arm64/rh8/build_catalyst.sh @@ -55,7 +55,7 @@ export OQD_BUILD_DIR="/catalyst/oqd-build" make oqd # Build Catalyst dialects -cmake -S mlir -B quantum-build -G Ninja \ +cmake -S mlir -B /catalyst/quantum-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_ENABLE_ASSERTIONS=ON \ -DQUANTUM_ENABLE_BINDINGS_PYTHON=ON \ @@ -70,7 +70,7 @@ cmake -S mlir -B quantum-build -G Ninja \ -DLLVM_ENABLE_ZSTD=OFF \ -DLLVM_ENABLE_LLD=ON \ -DLLVM_DIR=/catalyst/llvm-build/lib/cmake/llvm -cmake --build quantum-build --target check-dialects catalyst-cli +cmake --build /catalyst/quantum-build --target check-dialects catalyst-cli # Copy files needed for the wheel where they are expected cp /catalyst/runtime-build/lib/*/*/*/*/librtd* /catalyst/runtime-build/lib diff --git a/.github/workflows/scripts/linux_arm64/rh8/build_lld.sh b/.github/workflows/scripts/linux_arm64/rh8/build_lld.sh deleted file mode 100644 index e6522c443b..0000000000 --- a/.github/workflows/scripts/linux_arm64/rh8/build_lld.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -set -e -x -cd /catalyst - -# Process args -export GCC_VERSION=$1 -export PYTHON_VERSION=$2 -export PYTHON_SUBVERSION=$3 -export PYTHON_PACKAGE=$4 - -# Install system dependencies -dnf update -y -dnf install -y libzstd-devel gcc-toolset-${GCC_VERSION} -if [ "$PYTHON_VERSION" != "3.10" ]; then - dnf install -y ${PYTHON_PACKAGE} ${PYTHON_PACKAGE}-devel -fi -dnf clean all -y - -# Make GCC the default compiler -source /opt/rh/gcc-toolset-${GCC_VERSION}/enable -y -export C_COMPILER=/opt/rh/gcc-toolset-${GCC_VERSION}/root/usr/bin/gcc -export CXX_COMPILER=/opt/rh/gcc-toolset-${GCC_VERSION}/root/usr/bin/g++ - -# Set the right Python interpreter -rm -rf /usr/bin/python3 -ln -s /opt/_internal/cpython-${PYTHON_VERSION}.${PYTHON_SUBVERSION}/bin/python3 /usr/bin/python3 -export PYTHON=/usr/bin/python3 - -# Add Python and GCC to the PATH env var -export PATH=/opt/_internal/cpython-${PYTHON_VERSION}.${PYTHON_SUBVERSION}/bin:/opt/rh/gcc-toolset-${GCC_VERSION}/root/usr/bin:$PATH - -# Install python dependencies -/usr/bin/python3 -m pip install pennylane pybind11 PyYAML cmake ninja - -cmake -S /catalyst/mlir/llvm-project/llvm -B llvm-build -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_TARGETS_TO_BUILD="host" \ - -DLLVM_ENABLE_PROJECTS="lld" - -cmake --build /catalyst/llvm-build --target lld diff --git a/.github/workflows/scripts/linux_arm64/rh8/build_llvm.sh b/.github/workflows/scripts/linux_arm64/rh8/build_llvm.sh index 67082dc4c4..804d4e8431 100644 --- a/.github/workflows/scripts/linux_arm64/rh8/build_llvm.sh +++ b/.github/workflows/scripts/linux_arm64/rh8/build_llvm.sh @@ -31,22 +31,12 @@ export PYTHON=/usr/bin/python3 export PATH=/opt/_internal/cpython-${PYTHON_VERSION}.${PYTHON_SUBVERSION}/bin:/opt/rh/gcc-toolset-${GCC_VERSION}/root/usr/bin:/catalyst/llvm-build/bin:$PATH # Install python dependencies -/usr/bin/python3 -m pip install pennylane pybind11 PyYAML cmake ninja +/usr/bin/python3 -m pip install numpy pybind11 PyYAML cmake ninja # Build LLVM -cmake -S /catalyst/mlir/llvm-project/llvm -B /catalyst/llvm-build -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_BUILD_EXAMPLES=OFF \ - -DLLVM_TARGETS_TO_BUILD="host" \ - -DLLVM_ENABLE_PROJECTS="mlir" \ - -DLLVM_ENABLE_ASSERTIONS=ON \ - -DLLVM_INSTALL_UTILS=ON \ - -DLLVM_ENABLE_ZLIB=FORCE_ON \ - -DLLVM_ENABLE_ZSTD=OFF \ - -DLLVM_ENABLE_LLD=ON \ - -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ - -DPython3_EXECUTABLE=/usr/bin/python3 \ - -DPython3_NumPy_INCLUDE_DIRS=/opt/_internal/cpython-${PYTHON_VERSION}.${PYTHON_SUBVERSION}/lib/python${PYTHON_VERSION}/site-packages/numpy/core/include \ - -DCMAKE_CXX_VISIBILITY_PRESET=default - -LIT_FILTER_OUT="Bytecode|tosa-to-tensor" cmake --build /catalyst/llvm-build --target check-mlir llvm-symbolizer +export LLVM_BUILD_DIR="/catalyst/llvm-build" +export LLVM_PROJECTS="lld;mlir" +export LLVM_TARGETS="lld check-mlir" +export ENABLE_LLD=OFF +export ENABLE_ZLIB=FORCE_ON +make llvm diff --git a/.github/workflows/scripts/linux_arm64/rh8/test_wheels.sh b/.github/workflows/scripts/linux_arm64/rh8/test_wheels.sh index 531dbcce1e..398d23e113 100644 --- a/.github/workflows/scripts/linux_arm64/rh8/test_wheels.sh +++ b/.github/workflows/scripts/linux_arm64/rh8/test_wheels.sh @@ -10,7 +10,7 @@ export PYTHON_PATCH=$3 export PYTHON_PACKAGE=$4 # Install system dependencies (gcc gives access to c99, which is needed by some tests) -dnf update -y +dnf update -y dnf install -y libzstd-devel gcc-toolset-${GCC_VERSION} gcc if [ "$PYTHON_MAJOR_MINOR" != "3.10" ]; then dnf install -y ${PYTHON_PACKAGE} @@ -18,8 +18,8 @@ fi dnf clean all -y # Make GCC the default compiler -source /opt/rh/gcc-toolset-${GCC_VERSION}/enable -y -export C_COMPILER=/opt/rh/gcc-toolset-${GCC_VERSION}/root/usr/bin/gcc +source /opt/rh/gcc-toolset-${GCC_VERSION}/enable -y +export C_COMPILER=/opt/rh/gcc-toolset-${GCC_VERSION}/root/usr/bin/gcc export CXX_COMPILER=/opt/rh/gcc-toolset-${GCC_VERSION}/root/usr/bin/g++ # Set the right Python interpreter @@ -27,10 +27,6 @@ rm -rf /usr/bin/python3 ln -s /opt/_internal/cpython-${PYTHON_MAJOR_MINOR}.${PYTHON_PATCH}/bin/python3 /usr/bin/python3 export PYTHON=/usr/bin/python3 -# Set llvm-symbolizer -ls -la /catalyst/llvm-build/bin/llvm-symbolizer -export LLVM_SYMBOLIZER_PATH=/catalyst/llvm-build/bin/llvm-symbolizer - # Add LLVM, Python and GCC to the PATH env var export PATH=/catalyst/llvm-build/bin:/opt/_internal/cpython-${PYTHON_MAJOR_MINOR}.${PYTHON_PATCH}/bin:/opt/rh/gcc-toolset-${GCC_VERSION}/root/usr/bin:$PATH diff --git a/mlir/Makefile b/mlir/Makefile index aebb37294c..5951415768 100644 --- a/mlir/Makefile +++ b/mlir/Makefile @@ -12,8 +12,6 @@ ENZYME_BUILD_DIR?=$(MK_DIR)/Enzyme/build RT_BUILD_DIR?=$(MK_DIR)/../runtime/build ENABLE_ASAN?=OFF BUILD_TYPE?=Release -TARGET_FILE=$(MK_DIR)/mlir-hlo/mhlo/transforms/CMakeLists.txt -PATCH_FILE=$(MK_DIR)/patches/mhlo-Add-PassesIncGen-in-transforms-CMakeList.patch LLVM_EXTERNAL_LIT ?= $(LLVM_BUILD_DIR)/bin/llvm-lit ifeq ($(shell uname), Darwin) @@ -36,6 +34,9 @@ USE_SANITIZER_NAMES="" USE_SANITIZER_FLAGS="" endif +LLVM_PROJECTS ?= mlir +LLVM_TARGETS ?= check-mlir llvm-symbolizer + .PHONY: help help: @echo "Please use \`make ' where is one of" @@ -53,13 +54,20 @@ help: all: llvm mhlo enzyme dialects plugin .PHONY: llvm +llvm: TARGET_FILE := $(MK_DIR)/llvm-project/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocation.cpp +llvm: PATCH_FILE := $(MK_DIR)/patches/mlir-buffer-deallocation.patch llvm: @echo "build LLVM and MLIR enabling Python bindings" + # Patch in MLIR buffer deallocation bugfix + # TODO: remove once https://github.com/llvm/llvm-project/pull/121582 is merged & the dep updated + @if patch --dry-run -p1 -N $(TARGET_FILE) $(PATCH_FILE) > /dev/null 2>&1; then \ + patch -p1 $(TARGET_FILE) $(PATCH_FILE); \ + fi cmake -G Ninja -S llvm-project/llvm -B $(LLVM_BUILD_DIR) \ -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \ -DLLVM_BUILD_EXAMPLES=OFF \ -DLLVM_TARGETS_TO_BUILD="host" \ - -DLLVM_ENABLE_PROJECTS="mlir" \ + -DLLVM_ENABLE_PROJECTS="$(LLVM_PROJECTS)" \ -DLLVM_ENABLE_ASSERTIONS=ON \ -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ -DPython3_EXECUTABLE=$(PYTHON) \ @@ -75,10 +83,12 @@ llvm: -DCMAKE_CXX_VISIBILITY_PRESET=$(SYMBOL_VISIBILITY) # TODO: when updating LLVM, test to see if mlir/unittests/Bytecode/BytecodeTest.cpp:55 is passing - # and remove filter - LIT_FILTER_OUT="Bytecode" cmake --build $(LLVM_BUILD_DIR) --target check-mlir llvm-symbolizer + # and remove filter. This tests fails on CI/CD not locally. + LIT_FILTER_OUT="Bytecode|tosa-to-tensor" cmake --build $(LLVM_BUILD_DIR) --target $(LLVM_TARGETS) .PHONY: mhlo +mhlo: TARGET_FILE := $(MK_DIR)/mlir-hlo/mhlo/transforms/CMakeLists.txt +mhlo: PATCH_FILE := $(MK_DIR)/patches/mhlo-Add-PassesIncGen-in-transforms-CMakeList.patch mhlo: @echo "build MLIR-HLO" # Patch MHLO cmake dependency diff --git a/mlir/patches/mlir-buffer-deallocation.patch b/mlir/patches/mlir-buffer-deallocation.patch new file mode 100644 index 0000000000..852e6e84c5 --- /dev/null +++ b/mlir/patches/mlir-buffer-deallocation.patch @@ -0,0 +1,14 @@ +diff --git a/mlir/llvm-project/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocation.cpp b/mlir/llvm-project/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocation.cpp +index a0a81d4add..7b7be9e577 100644 +--- a/mlir/llvm-project/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocation.cpp ++++ b/mlir/llvm-project/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocation.cpp +@@ -308,6 +308,9 @@ private: + + // Add new allocs and additional clone operations. + for (Value value : valuesToFree) { ++ if (!isa(value.getType())) { ++ continue; ++ } + if (failed(isa(value) + ? introduceBlockArgCopy(cast(value)) + : introduceValueCopyForRegionResult(value)))