Testing/gpu

.github/CHANGELOG.md

@@ -10,6 +10,9 @@
 
 ### Breaking changes
 
+* Add `tests_gpu.yml` workflow to test the Lightning-Kokkos backend with CUDA-12. 
+  [(#494)](https://github.com/PennyLaneAI/pennylane-lightning/pull/494)
+
 * Add workflows validating compatibility between PennyLane and Lightning's most recent stable releases and development (latest) versions.
   [(#507)](https://github.com/PennyLaneAI/pennylane-lightning/pull/507)
   [(#498)](https://github.com/PennyLaneAI/pennylane-lightning/pull/498)

.github/workflows/build_and_cache_Kokkos_linux.yml

@@ -60,10 +60,7 @@ jobs:
       - name: Clone Kokkos repository
         if: steps.kokkos-cache.outputs.cache-hit != 'true'
         run: |
-          git clone https://github.com/kokkos/kokkos.git
-          cd kokkos
-          git checkout ${{ matrix.kokkos_version }}
-          cd -
+          git clone --branch ${{ matrix.kokkos_version }} https://github.com/kokkos/kokkos.git
           pushd . &> /dev/null
 
       - name: Install dependencies (Ubuntu)

.github/workflows/tests_gpu.yml

@@ -0,0 +1,300 @@
+name: Testing (GPU)
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+
+env:
+  CI_CUDA_ARCH: 86
+  COVERAGE_FLAGS: "--cov=pennylane_lightning --cov-report=term-missing --cov-report=xml:./coverage.xml --no-flaky-report -p no:warnings --tb=native"
+  GCC_VERSION: 11
+  KOKKOS_VERSION: e542e989af99cb87881ee2a0d06916541ee76006
+  TF_VERSION: 2.10.0
+  TORCH_VERSION: 1.11.0+cpu
+
+concurrency:
+  group: tests_gpu-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  builddeps:
+    runs-on:
+      - self-hosted
+      - ubuntu-22.04
+      - gpu
+
+    strategy:
+      max-parallel: 1
+      matrix:
+        os: [ubuntu-22.04]
+        exec_model: ["CUDA"]
+        kokkos_version: ["4.1.00"]
+
+    steps:
+      - name: Validate GPU version and installed compiler
+        run: |
+          source /etc/profile.d/modules.sh
+          module use /opt/modules
+          module load cuda/12.2
+          echo "${PATH}" >> $GITHUB_PATH
+          echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+          nvcc --version
+          nvidia-smi
+
+      - name: Clone Kokkos repository
+        run: |
+          rm -rf kokkos_${{ matrix.exec_model }}
+          # git clone --branch fix-rtld-deepbind https://github.com/cz4rs/kokkos.git kokkos_${{ matrix.exec_model }}
+          # git clone --branch ${{ matrix.kokkos_version }} https://github.com/kokkos/kokkos.git kokkos_${{ matrix.exec_model }}
+          git clone --branch develop https://github.com/kokkos/kokkos.git kokkos_${{ matrix.exec_model }}
+          cd kokkos_${{ matrix.exec_model }}
+          git checkout $KOKKOS_VERSION
+          git log | head
+          cd -
+          pushd . &> /dev/null
+
+      - name: Build Kokkos core library (Ubuntu)
+        run: |
+          rm -rf ${{ github.workspace }}/Kokkos_install/${{ matrix.exec_model }}
+          mkdir -p ${{ github.workspace }}/Kokkos_install/${{ matrix.exec_model }}
+          cd kokkos_${{ matrix.exec_model }}
+          cmake -BBuild . -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/Kokkos_install/${{ matrix.exec_model }} \
+                          -DKokkos_ENABLE_COMPLEX_ALIGN=OFF \
+                          -DKokkos_ENABLE_SERIAL=ON \
+                          -DKokkos_ENABLE_${{ matrix.exec_model }}=ON \
+                          -DKokkos_ENABLE_DEPRECATION_WARNINGS=OFF \
+                          -DCMAKE_CXX_COMPILER="$(which g++-$GCC_VERSION)" \
+                          -DCMAKE_CXX_STANDARD=20 \
+                          -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+                          -G Ninja
+          cmake --build ./Build --verbose
+          cmake --install ./Build
+          cd -
+
+      - name: Test Kokkos core library (Ubuntu)
+        env: 
+          CMAKE_PREFIX_PATH: ${{ github.workspace }}/Kokkos_install/${{ matrix.exec_model }}
+        run: |
+          set -x
+          sudo apt-get update && sudo apt-get -y -q install gfortran-$GCC_VERSION
+          cd kokkos_${{ matrix.exec_model }}/example/build_cmake_installed
+          which -a gfortran || which -a gfortran-$GCC_VERSION
+          cmake -B build -DCMAKE_Fortran_COMPILER=gfortran-$GCC_VERSION
+          cmake --build build
+          cmake --build build --target test
+          set +x
+
+      - name: Cache installation directories
+        id: kokkos-cache
+        uses: actions/cache/save@v3
+        with:
+          key: ${{ matrix.os }}-kokkos${{ matrix.kokkos_version }}-${{ matrix.exec_model }}-${{ github.ref }}-${{ github.sha }}
+          path: ${{ github.workspace }}/Kokkos_install/${{ matrix.exec_model }}
+
+  cpptestswithKokkos:
+    if: ${{ github.sha == 'xxxxxxxx' }}
+    needs: [builddeps]
+    strategy:
+      matrix:
+        os: [ubuntu-22.04]
+        pl_backend: ["lightning_kokkos"]
+        exec_model: ["CUDA"]
+        kokkos_version: ["4.1.00"]
+
+    name: C++ tests (Kokkos)
+    runs-on: 
+      - ${{ matrix.os }}
+      - self-hosted
+      - gpu
+
+    steps:
+      - name: Validate GPU version and installed compiler
+        run: |
+          source /etc/profile.d/modules.sh
+          module use /opt/modules
+          module load cuda/12.2
+          echo "${PATH}" >> $GITHUB_PATH
+          echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+          nvcc --version
+          nvidia-smi
+
+      - name: Restoring cached dependencies
+        id: kokkos-cache
+        uses: actions/cache/restore@v3
+        with:
+          path: ${{ github.workspace }}/Kokkos_install/${{ matrix.exec_model }}
+          key: ${{ matrix.os }}-kokkos${{ matrix.kokkos_version }}-${{ matrix.exec_model }}-${{ github.ref }}-${{ github.sha }}
+          restore-keys: ${{ matrix.os }}-kokkos${{ matrix.kokkos_version }}-${{ matrix.exec_model }}-${{ github.ref }}-
+
+      - name: Copy cached libraries
+        run: |
+          mkdir -p ${{ github.workspace }}/Kokkos
+          ls -ltrh
+          cp -r ${{ github.workspace }}/Kokkos_install/${{ matrix.exec_model }}/* ${{ github.workspace }}/Kokkos
+
+      - name: Checkout PennyLane-Lightning
+        uses: actions/checkout@v3
+        with:
+          path: main
+          fetch-depth: 2
+
+      - uses: actions/setup-python@v4
+        name: Install Python
+        with:
+          python-version: '3.9'
+
+      - name: Install dependencies
+        run: sudo apt-get update && sudo apt-get -y -q install cmake gcc-$GCC_VERSION  g++-$GCC_VERSION  ninja-build gcovr lcov
+
+      - name: Build and run unit tests
+        run: |
+          cd main
+          cmake . -BBuild -G Ninja \
+            -DCMAKE_BUILD_TYPE=Debug \
+            -DBUILD_TESTS=ON \
+            -DENABLE_PYTHON=OFF \
+            -DCMAKE_PREFIX_PATH=${{ github.workspace }}/Kokkos \
+            -DPL_BACKEND=${{ matrix.pl_backend }} \
+            -DCMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION) \
+            -DENABLE_COVERAGE=ON
+          cmake --build ./Build
+          cd ./Build
+          mkdir -p ./tests/results_${{ github.job }}_${{ matrix.pl_backend }}
+          for file in *runner ; do ./$file --order lex --reporter junit --out ./tests/results_${{ github.job }}_${{ matrix.pl_backend }}/report_$file.xml; done;
+          lcov --directory . -b ../pennylane_lightning/core/src --capture --output-file coverage.info
+          lcov --remove coverage.info '/usr/*' --output-file coverage.info
+          mv coverage.info coverage-${{ github.job }}-${{ matrix.pl_backend }}.info
+
+      - name: Upload test results
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: ubuntu-tests-reports-${{ github.job }}_${{ matrix.pl_backend }}
+          path: ./main/Build/tests/results_${{ github.job }}_${{ matrix.pl_backend }}
+          if-no-files-found: error
+
+      - name: Upload code coverage results
+        uses: actions/upload-artifact@v3
+        with:
+          name: ubuntu-codecov-results-cpp
+          path: ./main/Build/coverage-${{ github.job }}-${{ matrix.pl_backend }}.info
+          if-no-files-found: error
+
+  pythontestswithKokkos:
+    needs: [builddeps]
+    strategy:
+      matrix:
+        os: [ubuntu-22.04]
+        pl_backend: ["lightning_kokkos", "all"]
+        exec_model: ["CUDA"]
+        kokkos_version: ["4.1.00"]
+
+    name: Python tests with Kokkos
+    runs-on: 
+      - ${{ matrix.os }}
+      - self-hosted
+      - gpu
+
+    steps:
+      - name: Validate GPU version and installed compiler
+        run: |
+          source /etc/profile.d/modules.sh
+          module use /opt/modules
+          module load cuda/12.2
+          echo "${PATH}" >> $GITHUB_PATH
+          echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+          nvcc --version
+          nvidia-smi
+
+      - name: Restoring cached dependencies
+        id: kokkos-cache
+        uses: actions/cache/restore@v3
+        with:
+          path: ${{ github.workspace }}/Kokkos_install/${{ matrix.exec_model }}
+          key: ${{ matrix.os }}-kokkos${{ matrix.kokkos_version }}-${{ matrix.exec_model }}-${{ github.ref }}-${{ github.sha }}
+          restore-keys: ${{ matrix.os }}-kokkos${{ matrix.kokkos_version }}-${{ matrix.exec_model }}-${{ github.ref }}-
+
+      - name: Copy cached libraries
+        run: |
+          mkdir -p ${{ github.workspace }}/Kokkos
+          ls -ltrh
+          cp -r ${{ github.workspace }}/Kokkos_install/${{ matrix.exec_model }}/* ${{ github.workspace }}/Kokkos
+
+      - name: Checkout PennyLane-Lightning
+        uses: actions/checkout@v3
+        with:
+          path: main
+          fetch-depth: 2
+
+      - uses: actions/setup-python@v4
+        name: Install Python
+        with:
+          python-version: '3.9'
+
+      - name: Install dependencies
+        run: sudo apt-get update && sudo apt-get -y -q install cmake gcc-$GCC_VERSION  g++-$GCC_VERSION
+
+      - name: Get required Python packages
+        run: |
+          cd main
+          python -m pip install -r requirements-dev.txt
+
+      - name: Install ML libraries for interfaces
+        run: |
+          python -m pip install --upgrade torch==$TORCH_VERSION -f https://download.pytorch.org/whl/cpu/torch_stable.html
+          python -m pip install --upgrade "jax[cpu]"  # This also installs jaxlib
+          python -m pip install --upgrade tensorflow~=$TF_VERSION keras~=$TF_VERSION
+
+      - name: Install backend device
+        if: ${{ matrix.pl_backend != 'all'}}
+        run: |
+          ls -l ${{ github.workspace }}/Kokkos
+          cd main
+          SKIP_COMPILATION=True PL_BACKEND="lightning_qubit" pip install -e . -vv
+          CMAKE_ARGS="-DPL_BACKEND=${{ matrix.pl_backend }} -DCMAKE_PREFIX_PATH=${{ github.workspace }}/Kokkos -DENABLE_PYTHON=ON -DCMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION)" \
+          python -m pip install -e . -vv
+
+      - name: Run PennyLane-Lightning unit tests
+        if: ${{ matrix.pl_backend != 'all'}}
+        env: 
+          OMP_NUM_THREADS: 1
+          OMP_PROC_BIND: false
+        run: |
+          cd main/
+          DEVICENAME=`echo ${{ matrix.pl_backend }} | sed "s/_/./g"`
+          pl-device-test --device ${DEVICENAME} --skip-ops --shots=20000 $COVERAGE_FLAGS --cov-append 2> /dev/null || echo Something went wrong with pl-device-test shot=20000
+          pl-device-test --device ${DEVICENAME} --shots=None --skip-ops $COVERAGE_FLAGS --cov-append 2> /dev/null || echo Something went wrong with pl-device-test shot=None
+          PL_DEVICE=${DEVICENAME} python -m pytest tests/ $COVERAGE_FLAGS 2> /dev/null || echo Something went wrong with Pytest
+          mv coverage.xml coverage-${{ github.job }}.xml
+
+      - name: Install all backend devices
+        if: ${{ matrix.pl_backend == 'all' }}
+        run: |
+          cd main
+          CMAKE_ARGS="-DPL_BACKEND="lightning_qubit" -DCMAKE_PREFIX_PATH=${{ github.workspace }}/Kokkos -DENABLE_KOKKOS=ON -DENABLE_PYTHON=ON -DCMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION)" \
+          python -m pip install -e . -vv
+          rm -rf build
+          CMAKE_ARGS="-DPL_BACKEND="lightning_kokkos" -DCMAKE_PREFIX_PATH=${{ github.workspace }}/Kokkos -DENABLE_KOKKOS=ON -DENABLE_PYTHON=ON -DCMAKE_CXX_COMPILER=$(which g++-$GCC_VERSION)" \
+          python -m pip install -e . -vv
+
+      - name: Run PennyLane-Lightning unit tests for lightning.qubit with all devices installed
+        if: ${{ matrix.pl_backend == 'all' }}
+        env: 
+          OMP_NUM_THREADS: 1
+          OMP_PROC_BIND: false
+        run: |
+          cd main/
+          PL_DEVICE=lightning.qubit python -m pytest tests/ $COVERAGE_FLAGS
+          pl-device-test --device lightning.qubit --skip-ops --shots=20000 $COVERAGE_FLAGS --cov-append
+          pl-device-test --device lightning.qubit --shots=None --skip-ops $COVERAGE_FLAGS --cov-append
+          PL_DEVICE=lightning.kokkos python -m pytest tests/ $COVERAGE_FLAGS
+          pl-device-test --device lightning.kokkos --skip-ops --shots=20000 $COVERAGE_FLAGS --cov-append
+          pl-device-test --device lightning.kokkos --shots=None --skip-ops $COVERAGE_FLAGS --cov-append
+          mv coverage.xml coverage-${{ github.job }}-${{ matrix.pl_backend }}.xml
+
+      - name: Upload code coverage results
+        uses: actions/upload-artifact@v3
+        with:
+          name: ubuntu-codecov-results-python
+          path: ./main/coverage-${{ github.job }}-${{ matrix.pl_backend }}.xml