From 7b4222430542fe87b103ba58bf94dde5c875e739 Mon Sep 17 00:00:00 2001 From: Bettina Heim Date: Thu, 7 Nov 2024 20:30:23 +0100 Subject: [PATCH] Python metapackages (#2356) --- .github/workflows/ci.yml | 10 + .github/workflows/publishing.yml | 278 +++++++++++++--------- .github/workflows/python_metapackages.yml | 244 +++++++++++++++++++ .github/workflows/python_wheels.yml | 30 ++- docker/release/cudaq.ext.Dockerfile | 9 +- docker/release/cudaq.wheel.Dockerfile | 16 +- pyproject.toml | 6 +- python/CMakeLists.txt | 12 + python/README-cu12.md | 135 ----------- python/{README-cu11.md => README.md.in} | 57 ++--- python/cudaq/__init__.py | 8 +- python/extension/CMakeLists.txt | 5 - python/extension/CUDAQuantumExtension.cpp | 3 - python/metadata.cmake | 23 ++ python/metapackages/pyproject.toml | 58 +++++ python/metapackages/setup.py | 170 +++++++++++++ scripts/validate_wheel.sh | 13 +- 17 files changed, 763 insertions(+), 314 deletions(-) create mode 100644 .github/workflows/python_metapackages.yml delete mode 100644 python/README-cu12.md rename python/{README-cu11.md => README.md.in} (70%) create mode 100644 python/metadata.cmake create mode 100644 python/metapackages/pyproject.toml create mode 100644 python/metapackages/setup.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a623691b62..5d3c94c5aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -249,6 +249,16 @@ jobs: devdeps_cache: ${{ fromJson(needs.config.outputs.json).cache_key[format('{0}-cu{1}-python', matrix.platform, matrix.cuda_version)] }} devdeps_archive: ${{ fromJson(needs.config.outputs.json).tar_archive[format('{0}-cu{1}-python', matrix.platform, matrix.cuda_version)] }} + python_metapackages: + name: Create Python metapackages + needs: python_wheels + uses: ./.github/workflows/python_metapackages.yml + with: + cudaq_version: ${{ needs.python_wheels.outputs.cudaq_version }} + python_versions: "['3.10', '3.12']" + cuda_versions: "['', '11.8', '12.0']" + wheel_artifacts: 'pycudaq-*' + binaries: name: Create CUDA Quantum installer needs: [metadata, config] diff --git a/.github/workflows/publishing.yml b/.github/workflows/publishing.yml index 8ac5e4327b..944db3b887 100644 --- a/.github/workflows/publishing.yml +++ b/.github/workflows/publishing.yml @@ -286,8 +286,8 @@ jobs: retention-days: 1 if-no-files-found: warn - cudaq_hpc: - name: CUDA Quantum Docker image + cudaq_images: + name: CUDA-Q Docker image if: ${{ toJson(fromJson(needs.assets.outputs.docker_images).info_files) != '[]' }} needs: assets runs-on: ubuntu-latest @@ -471,8 +471,8 @@ jobs: outputs: | image_hash: ${{ steps.release_info.outputs.image_name }}@${{ steps.cudaq_build.outputs.digest }} - cudaq_installer: - name: CUDA Quantum installer + cudaq_installers: + name: CUDA-Q installer if: ${{ toJson(fromJson(needs.assets.outputs.installers).info_files) != '[]' }} needs: assets permissions: @@ -580,7 +580,7 @@ jobs: if-no-files-found: error cudaq_wheels: - name: CUDA Quantum Python wheels + name: CUDA-Q Python wheels if: ${{ toJson(fromJson(needs.assets.outputs.python_wheels).info_files) != '[]' }} needs: assets permissions: @@ -683,7 +683,7 @@ jobs: if-no-files-found: error cudaq_wheels_release_info: - name: Update release info of CUDA Quantum Python wheels + name: Update release info of CUDA-Q Python wheels needs: [assets, cudaq_wheels] runs-on: ubuntu-latest @@ -708,9 +708,20 @@ jobs: env: GH_TOKEN: ${{ secrets.REPO_BOT_ACCESS_TOKEN }} - config: - name: Configure validation - needs: [cudaq_hpc, cudaq_installer, cudaq_wheels] + cudaq_metapackages: + name: CUDA-Q metapackages + needs: [assets, cudaq_wheels] + uses: ./.github/workflows/python_metapackages.yml + with: + cudaq_version: ${{ needs.assets.outputs.cudaq_version }} + python_versions: "['3.10', '3.11', '3.12']" + cuda_versions: "['', '11.8', '12.0']" + wheel_artifacts: '*-wheels' + github_commit: ${{ inputs.github_commit || needs.assets.outputs.github_commit }} + + image_validation_config: + name: Configure image validation + needs: cudaq_images runs-on: ubuntu-latest outputs: @@ -729,9 +740,95 @@ jobs: images=`echo '${{ steps.read_json.outputs.result }}' | jq '[.image_hash[] | select(. != "")]'` echo "docker_images=$(echo $images)" >> $GITHUB_OUTPUT + image_validation: + name: Docker image validation + needs: [assets, image_validation_config] + runs-on: linux-amd64-gpu-v100-latest-1 + permissions: + contents: read + packages: read + + strategy: + matrix: + image_hash: ${{ fromJson(needs.image_validation_config.outputs.docker_images) }} + fail-fast: false + + container: + image: ${{ matrix.image_hash }} + options: --user root # otherwise step summary doesn't work + env: + NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} + TERM: xterm + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.github_commit || needs.assets.outputs.github_commit }} + path: github-repo + sparse-checkout: | + scripts + docs + + - name: Basic validation (GPU backends) + shell: bash + run: | + backends_to_test=`\ + for file in $(ls $CUDA_QUANTUM_PATH/targets/*.yml); \ + do + if [ -n "$(cat $file | grep "gpu-requirements")" ]; then \ + basename $file | cut -d "." -f 1; \ + elif [ -n "$(basename $file | grep mqpu)" ]; then \ + echo remote-mqpu; \ + fi; \ + done` + + rm -rf examples applications targets && mv github-repo/docs/sphinx/examples examples && mv github-repo/docs/sphinx/applications applications && mv github-repo/docs/sphinx/targets targets + mv github-repo/docs/notebook_validation.py . + GITHUB_STEP_SUMMARY=$GITHUB_STEP_SUMMARY \ + bash github-repo/scripts/validate_container.sh $backends_to_test | tee /tmp/validation.out + + # Check that the tests included the nvidia-mgpu backend: + relevant_line=`grep -n "Testing backends:" /tmp/validation.out | cut -d : -f1` + tested_backends=`cat /tmp/validation.out | tail -n +$relevant_line | sed -e '/^$/,$d'` + if [ -z "$(echo $tested_backends | grep nvidia-mgpu)" ]; then + echo "::error::Missing tests for nvidia-mgpu backend." + exit 1 + fi + + - name: MPI validation + shell: bash + run: | + status_sum=0 && set +e # Allow script to keep going through errors + for ex in `find /home/cudaq/examples/other/distributed/ -name '*.cpp'`; do + # Set CUDAQ_ENABLE_MPI_EXAMPLE to activate these examples. + nvq++ -DCUDAQ_ENABLE_MPI_EXAMPLE=1 $ex + status=$? + if [ $status -eq 0 ]; then + # Run with mpiexec + mpiexec --allow-run-as-root -np 4 ./a.out + status=$? + filename=$(basename -- "$ex") + if [ $status -eq 0 ]; then + echo ":white_check_mark: Successfully ran $filename." >> $GITHUB_STEP_SUMMARY + else + echo ":x: Failed to execute $filename." >> $GITHUB_STEP_SUMMARY + status_sum=$((status_sum+1)) + fi + else + echo ":x: Compilation failed for $filename." >> $GITHUB_STEP_SUMMARY + status_sum=$((status_sum+1)) + fi + done + set -e # Re-enable exit code error checking + if [ ! $status_sum -eq 0 ]; then + echo "::error::$status_sum examples failed; see step summary for a list of failures." + exit $status_sum + fi + installer_validation: name: Installer validation - needs: [assets, cudaq_installer] + needs: [assets, cudaq_installers] runs-on: linux-amd64-gpu-v100-latest-1 permissions: contents: read @@ -834,7 +931,7 @@ jobs: fi wheel_validation_conda: - name: Python instructions validation (conda) + name: Wheel validation, conda needs: [assets, cudaq_wheels] runs-on: linux-amd64-gpu-v100-latest-1 permissions: @@ -867,8 +964,16 @@ jobs: - name: Run validation shell: bash run: | - apt-get update && apt-get install -y --no-install-recommends ca-certificates vim wget openssh-client - cd /tmp + apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates vim wget unzip openssh-client + + # Extract README from wheel + cudaq_wheel=/tmp/install/cuda_quantum_cu${{ matrix.cuda_major }}*.whl + wheelname=${cudaq_wheel##*/} && archive_name=${wheelname%.*}.zip + cp $cudaq_wheel /tmp/$wheelname && mv /tmp/$wheelname /tmp/$archive_name + unzip /tmp/$archive_name -d /tmp/cudaq_wheel && rm -rf /tmp/$archive_name + metadata=/tmp/cudaq_wheel/*.dist-info/METADATA + # Setup links for validate_wheel.sh script ln -s $GITHUB_WORKSPACE/scripts/validate_wheel.sh . ln -s $GITHUB_WORKSPACE/docs/sphinx/examples/python /tmp/examples @@ -876,19 +981,20 @@ jobs: ln -s $GITHUB_WORKSPACE/docs/sphinx/targets/python /tmp/targets ln -s $GITHUB_WORKSPACE/docs/sphinx/snippets/python /tmp/snippets ln -s $GITHUB_WORKSPACE/python/tests /tmp/tests - ln -s $GITHUB_WORKSPACE/python/README-cu${{ matrix.cuda_major }}.md . + ln -s $metadata /tmp/README.md + # Run the script w/ -q to run a shortened test set +e # Allow script to keep going through errors (needed for skipped tests) - source validate_wheel.sh -w /tmp/install/cuda_quantum_cu${{ matrix.cuda_major }}*.whl -f /tmp -p 3.10 -c ${{ matrix.cuda_major }} + source validate_wheel.sh -w $cudaq_wheel -f /tmp -p 3.10 set -e # Re-enable exit code error checking if [ "$status_sum" -ne "0" ]; then echo "::error::Error running validation script" exit $status_sum fi - wheel_validation_x86: - name: Simple Python wheel installation validation - needs: [assets, cudaq_wheels] + metapackage_validation_piponly: + name: Python metapackage validation, pip only + needs: [assets, cudaq_wheels, cudaq_metapackages] runs-on: linux-amd64-gpu-v100-latest-1 permissions: contents: read @@ -911,12 +1017,19 @@ jobs: with: ref: ${{ inputs.github_commit || needs.assets.outputs.github_commit }} - - name: Load wheel + - name: Load wheels uses: actions/download-artifact@v4 with: - name: x86_64-cu${{ matrix.cuda_major }}-py3.10-wheels - path: /tmp/install - + pattern: '*py3.10-wheels' + path: /tmp/wheels + merge-multiple: true + + - name: Load metapackage + uses: actions/download-artifact@v4 + with: + name: ${{ needs.cudaq_metapackages.outputs.artifact_name }} + path: /tmp/packages + - name: Run x86 validation shell: bash run: | @@ -924,7 +1037,18 @@ jobs: # targets and test cases that don't require MPI. # Create clean python3 environment. apt-get update && apt-get install -y --no-install-recommends python3 python3-pip - pip install /tmp/install/cuda_quantum_cu${{ matrix.cuda_major }}*.whl + mv /tmp/wheels/* /tmp/packages && rmdir /tmp/wheels + + python3 -m pip install pypiserver + server=`find / -name pypi-server -executable -type f` + $server run -p 8080 /tmp/packages & + + pip install cudaq --extra-index-url http://localhost:8080 + if [ -z "$(pip list | grep cuda-quantum-cu${{ matrix.cuda_major }})" ]; then + echo "::error::Missing installation of cuda-quantum-cu${{ matrix.cuda_major }} package." + exit 1 + fi + status_sum=0 set +e # Allow script to keep going through errors # Verify that the necessary GPU targets are installed and usable @@ -942,96 +1066,10 @@ jobs: echo "::error::Error running validation script" exit $status_sum fi - - image_validation: - name: Docker image validation - needs: [assets, config] - runs-on: linux-amd64-gpu-v100-latest-1 - permissions: - contents: read - packages: read - - strategy: - matrix: - image_hash: ${{ fromJson(needs.config.outputs.docker_images) }} - fail-fast: false - - container: - image: ${{ matrix.image_hash }} - options: --user root # otherwise step summary doesn't work - env: - NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} - TERM: xterm - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ inputs.github_commit || needs.assets.outputs.github_commit }} - path: github-repo - sparse-checkout: | - scripts - docs - - - name: Basic validation (GPU backends) - shell: bash - run: | - backends_to_test=`\ - for file in $(ls $CUDA_QUANTUM_PATH/targets/*.yml); \ - do - if [ -n "$(cat $file | grep "gpu-requirements")" ]; then \ - basename $file | cut -d "." -f 1; \ - elif [ -n "$(basename $file | grep mqpu)" ]; then \ - echo remote-mqpu; \ - fi; \ - done` - - rm -rf examples applications targets && mv github-repo/docs/sphinx/examples examples && mv github-repo/docs/sphinx/applications applications && mv github-repo/docs/sphinx/targets targets - mv github-repo/docs/notebook_validation.py . - GITHUB_STEP_SUMMARY=$GITHUB_STEP_SUMMARY \ - bash github-repo/scripts/validate_container.sh $backends_to_test | tee /tmp/validation.out - - # Check that the tests included the nvidia-mgpu backend: - relevant_line=`grep -n "Testing backends:" /tmp/validation.out | cut -d : -f1` - tested_backends=`cat /tmp/validation.out | tail -n +$relevant_line | sed -e '/^$/,$d'` - if [ -z "$(echo $tested_backends | grep nvidia-mgpu)" ]; then - echo "::error::Missing tests for nvidia-mgpu backend." - exit 1 - fi - - - name: MPI validation - shell: bash - run: | - status_sum=0 && set +e # Allow script to keep going through errors - for ex in `find /home/cudaq/examples/other/distributed/ -name '*.cpp'`; do - # Set CUDAQ_ENABLE_MPI_EXAMPLE to activate these examples. - nvq++ -DCUDAQ_ENABLE_MPI_EXAMPLE=1 $ex - status=$? - if [ $status -eq 0 ]; then - # Run with mpiexec - mpiexec --allow-run-as-root -np 4 ./a.out - status=$? - filename=$(basename -- "$ex") - if [ $status -eq 0 ]; then - echo ":white_check_mark: Successfully ran $filename." >> $GITHUB_STEP_SUMMARY - else - echo ":x: Failed to execute $filename." >> $GITHUB_STEP_SUMMARY - status_sum=$((status_sum+1)) - fi - else - echo ":x: Compilation failed for $filename." >> $GITHUB_STEP_SUMMARY - status_sum=$((status_sum+1)) - fi - done - set -e # Re-enable exit code error checking - if [ ! $status_sum -eq 0 ]; then - echo "::error::$status_sum examples failed; see step summary for a list of failures." - exit $status_sum - fi create_release: - name: CUDA Quantum Release - needs: [assets, cudaq_hpc, cudaq_installer, cudaq_wheels] + name: CUDA-Q Release + needs: [assets, cudaq_images, cudaq_installers, cudaq_wheels, cudaq_metapackages] if: needs.assets.outputs.release_title && inputs.github_commit == '' && inputs.assets_from_run == '' && inputs.nvidia_mgpu_commit == '' runs-on: ubuntu-latest @@ -1040,17 +1078,23 @@ jobs: url: ${{ vars.deployment_url }} steps: + - name: Download CUDA-Q installer + uses: actions/download-artifact@v4 + with: + pattern: '*-installer' + path: installers + - name: Download CUDA-Q Python wheels uses: actions/download-artifact@v4 with: pattern: '*-wheels' path: wheelhouse - - name: Download CUDA-Q installer + - name: Download CUDA-Q metapackages uses: actions/download-artifact@v4 with: - pattern: '*-installer' - path: installers + name: ${{ needs.cudaq_metapackages.outputs.artifact_name }} + path: metapackages # The python wheels are uploaded as a release asset, but not pushed to anywhere else. # Note that PyPI packages cannot be updated once pushed; @@ -1068,6 +1112,7 @@ jobs: mv -v "wheelhouse/$dir"/* wheelhouse/ && rmdir "wheelhouse/$dir" done zip -r wheelhouse.zip wheelhouse + zip -r metapackages.zip metapackages release_id=${{ inputs.assets_from_run || github.run_id }} release_title="${{ needs.assets.outputs.release_title }}" @@ -1082,8 +1127,9 @@ jobs: gh release create $release_id --title $release_id -R ${{ github.repository }} \ --target $github_commit --draft $prerelease \ --generate-notes --notes-start-tag $latest_tag --notes "$rel_notes" - gh release upload $release_id -R ${{ github.repository }} wheelhouse.zip --clobber gh release upload $release_id -R ${{ github.repository }} install_cuda_quantum.* --clobber + gh release upload $release_id -R ${{ github.repository }} wheelhouse.zip --clobber + gh release upload $release_id -R ${{ github.repository }} metapackages.zip --clobber gh release edit $release_id -R ${{ github.repository }} \ --title "$release_title" --tag $version $prerelease # --draft=false env: @@ -1091,7 +1137,7 @@ jobs: clean_up: name: Clean up - needs: [assets, cudaq_hpc, cudaq_installer, cudaq_wheels, wheel_validation_conda, wheel_validation_x86, create_release, installer_validation, image_validation] + needs: [assets, cudaq_images, cudaq_installers, cudaq_wheels, cudaq_metapackages, image_validation, installer_validation, wheel_validation_conda, metapackage_validation_piponly, create_release] # Force this job to run even when some of the dependencies above are skipped. if: always() && !cancelled() && needs.assets.result != 'skipped' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') runs-on: ubuntu-latest diff --git a/.github/workflows/python_metapackages.yml b/.github/workflows/python_metapackages.yml new file mode 100644 index 0000000000..0521e80a31 --- /dev/null +++ b/.github/workflows/python_metapackages.yml @@ -0,0 +1,244 @@ +on: + workflow_call: + inputs: + cudaq_version: + required: true + type: string + description: The version of the built CUDA-Q packages. + python_versions: + required: true + type: string + description: Json array of the Python versions to test the packages with (e.g. ['3.10', '3.12']). + cuda_versions: + required: true + type: string + description: Json array of the CUDA versions to test the packages with (e.g. ['11.8', '12.0']). + wheel_artifacts: + required: true + type: string + description: A pattern that matches all artifacts that contain wheels that the metapackages depend on. + github_commit: + required: false + type: string + description: Optional argument to set the GitHub commit to use for the build. + outputs: + artifact_name: + description: "The name of the artifact that contains the built metapackages." + value: ${{ jobs.build_metapackages.outputs.artifact_name }} + +name: Python metapackages + +jobs: + build_metapackages: + name: Build Python metapackages + runs-on: ubuntu-latest + permissions: + contents: read + + outputs: + artifact_name: ${{ steps.metapackage_build.outputs.artifact_name }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: "${{ inputs.github_commit || '' }}" + + - name: Build metapackages + id: metapackage_build + run: | + sed -i "s/README.md.in/README.md/g" python/metapackages/pyproject.toml + for file in LICENSE NOTICE CITATION.cff; do + cp $file python/metapackages/$file + done + + apt-get update && apt-get install -y --no-install-recommends \ + python3 python3-pip python3-venv + python3 -m pip install build + + echo "Creating README.md for cudaq package" + package_name=cudaq + cuda_version_requirement="11.x (where x >= 8) or 12.x" + cuda_version_conda=11.8.0 # only used as example in the install script + cat python/README.md.in > python/metapackages/README.md + for variable in package_name cuda_version_requirement cuda_version_conda; do + sed -i "s/.{{[ ]*$variable[ ]*}}/${!variable}/g" python/metapackages/README.md + done + if [ -n "$(cat python/metapackages/README.md | grep -e '.{{.*}}')" ]; then + echo "::error file=python_metapackages.yml::Incomplete template substitutions in README." + exit 1 + fi + + echo "Building cudaq metapackage ..." + cd python/metapackages && echo ${{ inputs.cudaq_version }} > _version.txt + CUDAQ_META_WHEEL_BUILD=1 python3 -m build . --sdist + mkdir /tmp/packages && mv dist/cudaq-* /tmp/packages/ + rm -rf cudaq.egg-info dist + + echo "Creating README.md for cuda-quantum package" + echo "# Welcome to the CUDA-Q Python API" > README.md + echo "This package is deprecated and new versions are published under the name `cudaq` instead." >> README.md + echo "For more information, please see [CUDA-Q on PyPI](https://pypi.org/project/cudaq)." >> README.md + + echo "Building cuda-quantum metapackage ..." + sed -i 's/name = "cudaq"/name = "cuda-quantum"/' pyproject.toml + echo 'Please remove the cuda-quantum package and `pip install cudaq` instead.' > _deprecated.txt + CUDAQ_META_WHEEL_BUILD=1 python3 -m build . --sdist + mv dist/cuda_quantum-* /tmp/packages/ + + echo "artifact_name=cudaq-metapackage-${{ inputs.cudaq_version }}" >> $GITHUB_OUTPUT + + - name: Upload metapackages + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.metapackage_build.outputs.artifact_name }} + path: /tmp/packages/ + retention-days: 1 + if-no-files-found: error + + test_metapackages: + name: Test Python metapackages + needs: build_metapackages + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + + container: + image: ghcr.io/nvidia/pypa/manylinux_2_28_x86_64:latest + credentials: + username: ${{ github.actor }} + password: ${{ github.token }} + + strategy: + matrix: + cuda_version: ${{ fromJSON(inputs.cuda_versions) }} + python_version: ${{ fromJSON(inputs.python_versions) }} + fail-fast: false + + steps: + - name: Load metapackages + uses: actions/download-artifact@v4 + with: + pattern: ${{ needs.build_metapackages.outputs.artifact_name }} + path: /tmp/metapackages/ + merge-multiple: true + + - name: Load wheels + uses: actions/download-artifact@v4 + with: + pattern: ${{ inputs.wheel_artifacts }} + path: /tmp/wheels/ + merge-multiple: true + + - name: Test installation + run: | + mkdir /tmp/packages + mv /tmp/wheels/* /tmp/packages && mv /tmp/metapackages/* /tmp/packages + rmdir /tmp/wheels /tmp/metapackages + + if [ -n "${{ matrix.cuda_version }}" ]; then + CUDA_DOWNLOAD_URL=https://developer.download.nvidia.com/compute/cuda/repos + CUDA_ARCH_FOLDER=$([ "$(uname -m)" == "aarch64" ] && echo sbsa || echo x86_64) + CUDA_DISTRIBUTION=rhel8 + dnf config-manager --add-repo \ + "${CUDA_DOWNLOAD_URL}/${CUDA_DISTRIBUTION}/${CUDA_ARCH_FOLDER}/cuda-${CUDA_DISTRIBUTION}.repo" + + cuda_version_suffix=`echo ${{ matrix.cuda_version }} | tr . -` + dnf install -y --nobest --setopt=install_weak_deps=False \ + cuda-cudart-${cuda_version_suffix} + fi + + python=python${{ matrix.python_version }} + $python -m pip install pip3-autoremove + # autoremove is in an odd location that isn't found unless we search for it... + autoremove="$python $($python -m pip show pip3-autoremove | grep -e 'Location: .*$' | cut -d ' ' -f2)/pip_autoremove.py" + $python -m pip install pypiserver + server="$python $($python -m pip show pypiserver | grep -e 'Location: .*$' | cut -d ' ' -f2)/pypiserver_main__.py" + $server run -p 8080 /tmp/packages & + + package_dep=`echo cuda-quantum${cuda_version_suffix:+-cu$cuda_version_suffix} | cut -d '-' -f1-2` + if [ -n "$($python -m pip list | grep $package_dep)" ]; then + echo "::error file=python_metapackages.yml::Unexpected installation of $package_dep package." + exit 1 + fi + + $python -m pip install cudaq==${{ inputs.cudaq_version }} \ + --extra-index-url http://localhost:8080 + + if [ -z "$($python -m pip list | grep $package_dep)" ]; then + echo "::error file=python_metapackages.yml::Missing installation of $package_dep package." + exit 1 + elif [ "$($python -c 'import cudaq; print(cudaq.get_target().name)')" != "qpp-cpu" ]; then + echo "::error file=python_metapackages.yml::Unexpected output for cudaq.get_target()." + exit 1 + fi + + $autoremove -y cudaq + if [ -n "$($python -m pip list | grep $package_dep)" ]; then + echo "::error file=python_metapackages.yml::Unexpected installation of $package_dep package." + exit 1 + fi + + if [ "$package_dep" != "cuda-quantum" ]; then + $python -m pip install ${package_dep}==${{ inputs.cudaq_version }} \ + --extra-index-url http://localhost:8080 + $python -m pip install cudaq==${{ inputs.cudaq_version }} \ + --extra-index-url http://localhost:8080 + $python -c 'import cudaq; print(cudaq.get_target().name)' + $autoremove -y cudaq + if [ -z "$($python -m pip list | grep $package_dep)" ]; then + echo "::error file=python_metapackages.yml::Missing installation of $package_dep package." + exit 1 + fi + $autoremove -y $package_dep + fi + + - name: Test installation error + run: | + #dnf install -y --nobest --setopt=install_weak_deps=False curl jq + #PACKAGE_JSON_URL="https://pypi.org/pypi/cuda-quantum/json" + #curl -L -s "$PACKAGE_JSON_URL" | jq -r '.releases | keys | .[]' + + python=python${{ matrix.python_version }} + $python -m pip install pypiserver + server="$python $($python -m pip show pypiserver | grep -e 'Location: .*$' | cut -d ' ' -f2)/pypiserver_main__.py" + $server run -p 8080 /tmp/packages & + + case "${{ matrix.cuda_version }}" in + "") + set +e # continue on error + $python -m pip install cuda-quantum==${{ inputs.cudaq_version }} \ + --extra-index-url http://localhost:8080 + set -e && check_package=cuda-quantum + ;; + 12.*) + $python -m pip install cuda-quantum-cu11==${{ inputs.cudaq_version }} \ + --extra-index-url http://localhost:8080 + set +e # continue on error + $python -m pip install cudaq==${{ inputs.cudaq_version }} \ + --extra-index-url http://localhost:8080 + set -e && check_package=cudaq + ;; + *) + set +e # continue on error + # test that we get an error for an existing cuda-quantum installation + $python -m pip install cuda-quantum==0.8.0 + set -e + if [ -z "$($python -m pip list | grep cuda-quantum)" ]; then + # if we don't have a 0.8.0 version for this python version, test other conflict + $python -m pip install cuda-quantum-cu12==${{ inputs.cudaq_version }} \ + --extra-index-url http://localhost:8080 + fi + set +e # continue on error + $python -m pip install cudaq==${{ inputs.cudaq_version }} \ + --extra-index-url http://localhost:8080 + set -e && check_package=cudaq + ;; + esac + + if [ -n "$($python -m pip list | grep ${check_package})" ]; then + echo "::error file=python_metapackages.yml::Unexpected installation of ${check_package} package." + exit 1 + fi + diff --git a/.github/workflows/python_wheels.yml b/.github/workflows/python_wheels.yml index ce9fa80f57..8e542a0b2e 100644 --- a/.github/workflows/python_wheels.yml +++ b/.github/workflows/python_wheels.yml @@ -25,6 +25,10 @@ on: required: false type: boolean default: false + outputs: + cudaq_version: + description: "The version of the built wheels." + value: ${{ jobs.build_wheel.outputs.cudaq_version }} secrets: DOCKERHUB_USERNAME: required: true @@ -43,6 +47,7 @@ jobs: outputs: artifact_name: ${{ steps.prereqs.outputs.artifact_name }} + cudaq_version: ${{ steps.prereqs.outputs.cudaq_version }} # Needed for making local images available to the docker/build-push-action. # See also https://stackoverflow.com/a/63927832. @@ -266,34 +271,41 @@ jobs: # hence we don't want to mix conda and pip packages. cuda_major=$(echo ${{ inputs.cuda_version }} | cut -d . -f1) python${{ inputs.python_version }} -m pip uninstall -y cuda-quantum-cu${cuda_major} + # Install cuda-quantum wheel and all dependencies (including MPI) using conda (README instructions) - # Install conda - dnf install -y --nobest --setopt=install_weak_deps=False wget openssh-clients + dnf install -y --nobest --setopt=install_weak_deps=False wget unzip openssh-clients mkdir -p ~/miniconda3 wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-$(uname -m).sh -O ~/miniconda3/miniconda.sh bash ~/miniconda3/miniconda.sh -b -u -p ~/miniconda3 ~/miniconda3/bin/conda init bash source ~/.bashrc + # Extract the setup script from Python wheel's readme - readme_file="/tmp/README-cu${cuda_major}.md" - python_version="${{ inputs.python_version }}" - # Parse README file to install openmpi - conda_script="$(awk '/(Begin conda install)/{flag=1;next}/(End conda install)/{flag=0}flag' $readme_file | grep . | sed '/^```/d')" + cudaq_wheel=/tmp/cuda_quantum_cu*-manylinux_*_$(uname -m).whl + wheelname=${cudaq_wheel##*/} && archive_name=${wheelname%.*}.zip + tmploc=/tmp/scratch && mkdir $tmploc + cp $cudaq_wheel $tmploc/$wheelname && mv $tmploc/$wheelname $tmploc/$archive_name + unzip $tmploc/$archive_name -d $tmploc/cudaq_wheel && rm -rf $tmploc/$archive_name + metadata=$tmploc/cudaq_wheel/*.dist-info/METADATA + + # Parse README content to install openmpi + conda_script="$(awk '/(Begin conda install)/{flag=1;next}/(End conda install)/{flag=0}flag' $metadata | grep . | sed '/^```/d')" # Skip the installation of CUDA since we don't need CUDA for this test. regex='conda install -y -n cuda-quantum.*cuda' while IFS= read -r line; do if [[ ! "$line" =~ $regex ]]; then # Replace Python version - line=$(echo $line | sed -E "s/python(=)?3.[0-9]{1,}/python\1$python_version/g") + line=$(echo $line | sed -E "s/python(=)?3.[0-9]{1,}/python\1${{ inputs.python_version }}/g") # Install the wheel file - line=${line//pip install cuda-quantum-cu${cuda_major}/pip install /tmp/cuda_quantum*-manylinux_*_$(uname -m).whl} + line=${line//pip install cuda-quantum-cu${cuda_major}/pip install ${cudaq_wheel}} eval "$line" fi done <<< "$conda_script" - ompi_script="$(awk '/(Begin ompi setup)/{flag=1;next}/(End ompi setup)/{flag=0}flag' $readme_file | grep . | sed '/^```/d')" + ompi_script="$(awk '/(Begin ompi setup)/{flag=1;next}/(End ompi setup)/{flag=0}flag' $metadata | grep . | sed '/^```/d')" while IFS= read -r line; do eval "$line" done <<< "$ompi_script" + # Run the MPI test python${{ inputs.python_version }} -m pip install pytest numpy mpirun --allow-run-as-root -np 4 python${{ inputs.python_version }} -m pytest -v /tmp/tests/parallel/test_mpi_api.py diff --git a/docker/release/cudaq.ext.Dockerfile b/docker/release/cudaq.ext.Dockerfile index aed0c53389..185ec4d35f 100644 --- a/docker/release/cudaq.ext.Dockerfile +++ b/docker/release/cudaq.ext.Dockerfile @@ -29,12 +29,17 @@ RUN if [ -d "$CUDA_QUANTUM_PATH/assets/documentation" ]; then \ && rm -rf "$CUDA_QUANTUM_PATH/assets" "$CUDA_QUANTUM_PATH/bin/migrate_assets.sh" # Install additional runtime dependencies. -RUN cuda_version_suffix=$(echo $CUDA_VERSION | tr . -) && \ +RUN cuda_version_suffix=$(echo ${CUDA_VERSION} | tr . -) && \ for cudart_dependency in libcusolver libcublas cuda-cudart; do \ if [ -z "$(apt list --installed | grep -o ${cudart_dependency}-${cuda_version_suffix})" ]; then \ - apt-get install -y --no-install-recommends ${cudart_dependency}-${cuda_version_suffix}; \ + apt-get install -y --no-install-recommends \ + ${cudart_dependency}-${cuda_version_suffix}; \ fi \ done && \ + if [ $(echo ${CUDA_VERSION} | cut -d . -f1) -gt 11 ]; then \ + apt-get install -y --no-install-recommends \ + libnvjitlink-${cuda_version_suffix}; \ + fi && \ # just here for convenience: apt-get install -y --no-install-recommends curl jq RUN if [ -x "$(command -v pip)" ]; then \ diff --git a/docker/release/cudaq.wheel.Dockerfile b/docker/release/cudaq.wheel.Dockerfile index 8c92abc2b2..8e7717de00 100644 --- a/docker/release/cudaq.wheel.Dockerfile +++ b/docker/release/cudaq.wheel.Dockerfile @@ -38,12 +38,26 @@ RUN echo "Building MLIR bindings for python${python_version}" && \ bash /scripts/build_llvm.sh -c Release -v # Patch the pyproject.toml file to change the CUDA version if needed -RUN if [ "${CUDA_VERSION#12.}" != "${CUDA_VERSION}" ]; then \ +RUN sed -i "s/README.md.in/README.md/g" cuda-quantum/pyproject.toml && \ + if [ "${CUDA_VERSION#12.}" != "${CUDA_VERSION}" ]; then \ sed -i "s/-cu11/-cu12/g" cuda-quantum/pyproject.toml && \ sed -i -E "s/(nvidia-cublas-cu[0-9]* ~= )[0-9\.]*/\1${CUDA_VERSION}/g" cuda-quantum/pyproject.toml; \ sed -i -E "s/(nvidia-cuda-runtime-cu[0-9]* ~= )[0-9\.]*/\1${CUDA_VERSION}/g" cuda-quantum/pyproject.toml; \ fi +# Create the README +RUN cd cuda-quantum && cat python/README.md.in > python/README.md && \ + package_name=cuda-quantum-cu$(echo ${CUDA_VERSION} | cut -d . -f1) && \ + cuda_version_requirement=">= ${CUDA_VERSION}" && \ + cuda_version_conda=${CUDA_VERSION}.0 && \ + for variable in package_name cuda_version_requirement cuda_version_conda; do \ + sed -i "s/.{{[ ]*$variable[ ]*}}/${!variable}/g" python/README.md; \ + done && \ + if [ -n "$(cat python/README.md | grep -e '.{{.*}}')" ]; then \ + echo "Incomplete template substitutions in README." && \ + exit 1; \ + fi + # Build the wheel RUN echo "Building wheel for python${python_version}." \ && rm ~/.cache/pip -rf \ diff --git a/pyproject.toml b/pyproject.toml index 194750ad58..bd01bd68f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,17 +9,16 @@ [project] name = "cuda-quantum-cu11" dynamic = ["version"] -keywords = [ "cuda-quantum", "cuda", "quantum", "quantum computing", "nvidia", "high-performance computing" ] +keywords = [ "cudaq", "cuda-quantum", "cuda", "quantum", "quantum computing", "nvidia", "high-performance computing" ] description="Python bindings for the CUDA-Q toolkit for heterogeneous quantum-classical workflows." authors = [{name = "NVIDIA Corporation & Affiliates"}] maintainers = [{name = "NVIDIA Corporation & Affiliates"}] -readme = "python/README-cu11.md" +readme = { file="python/README.md.in", content-type = "text/markdown"} requires-python = ">=3.10" license = { file="LICENSE" } dependencies = [ 'astpretty ~= 3.0', 'cuquantum-cu11 ~= 24.08', - 'graphlib-backport >= 1.0', 'numpy >= 1.24', 'requests >= 2.31', 'nvidia-cublas-cu11 ~= 11.11; platform_machine == "x86_64"', @@ -62,7 +61,6 @@ build-backend = "scikit_build_core.build" wheel.packages = ["python/cudaq"] wheel.license-files = [ "LICENSE", "NOTICE", "CITATION.cff" ] build-dir = "_skbuild" -sdist.include = ["_version.py"] metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" cmake.minimum-version = "3.26" cmake.build-type = "Release" diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 6efc9e8257..b8acea7a79 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -30,12 +30,23 @@ add_subdirectory(extension) file(GLOB_RECURSE PYTHON_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/*.py") +if (CUDA_FOUND) + enable_language(CUDA) + find_package(CUDAToolkit REQUIRED) +endif() + +set(METADATA_FILE "${CMAKE_BINARY_DIR}/python/cudaq/_metadata.py" ) add_custom_target( CopyPythonFiles ALL COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_BINARY_DIR}/python + COMMAND ${CMAKE_COMMAND} + -DMETADATA_FILE="${METADATA_FILE}" + -DCUDA_VERSION_MAJOR=${CUDAToolkit_VERSION_MAJOR} + -P ${CMAKE_CURRENT_SOURCE_DIR}/metadata.cmake DEPENDS ${PYTHON_SOURCES} + BYPRODUCTS "${METADATA_FILE}" ) add_dependencies(CUDAQuantumPythonModules CopyPythonFiles) @@ -45,6 +56,7 @@ add_subdirectory(runtime/cudaq/domains/plugins) if (NOT SKBUILD) install(DIRECTORY cudaq DESTINATION .) endif() +install(FILES "${METADATA_FILE}" DESTINATION cudaq) # Do we have pytest and numpy? execute_process(COMMAND ${Python_EXECUTABLE} -m pytest --version diff --git a/python/README-cu12.md b/python/README-cu12.md deleted file mode 100644 index 83b03f41d7..0000000000 --- a/python/README-cu12.md +++ /dev/null @@ -1,135 +0,0 @@ -# Welcome to the CUDA-Q Python API - -CUDA-Q is a comprehensive framework for quantum programming. It features: - -- A programming model which extends C++ and Python with quantum kernels, - enabling high-level programming in familiar languages -- A high-performance quantum compiler, `nvq++`, based on the industry standard - LLVM toolchain -- Interoperability with all of the leading models and tools for accelerated - computing, including CUDA, ISO standard parallelism, OpenMP, and OpenACC -- The ability to utilize and seamlessly switch between different quantum - technologies, including state-of-the-art simulator backends with NVIDIA - cuQuantum and a number of different physical quantum processors (QPUs) - -The CUDA-Q Python wheels contain the Python API and core components of -CUDA-Q. More information about available packages as well as a link to the -documentation and examples for each version can be found in the [release -notes][cudaq_docs_releases]. System and compatibility requirements -are listed in the Installation Guide of the linked documentation. - -**Note**: CUDA-Q is currently only supported on Linux operating systems using -`x86_64` or `aarch64`/`arm64` processors. - -[cudaq_docs_releases]: - https://nvidia.github.io/cuda-quantum/latest/releases.html - -## Installation Including GPU-Acceleration - -[//]: # (Begin complete install) - -CUDA-Q does not require a GPU to use, but some components are GPU-accelerated. - -Getting started with CUDA-Q on `x86_64` platforms simply requires -`pip install cuda-quantum-cu12`. If you have an NVIDIA GPU on your host system, you -will be able to use it without any further installation steps. - -However, if you want to perform multi-GPU simulations, or if you are using -`aarch64`/`arm64` processors, additional components must be installed. -We recommend using [Conda](https://docs.conda.io/en/latest/) to do so. -If you are not already using Conda, -you can install a minimal version following the instructions -[here](https://docs.anaconda.com/miniconda/). The -following commands will create and activate a complete environment for CUDA-Q -with all its dependencies: - -[//]: # (Begin conda install) - -```console -conda create -y -n cuda-quantum python=3.12 pip -conda install -y -n cuda-quantum -c "nvidia/label/cuda-12.0.0" cuda -conda install -y -n cuda-quantum -c conda-forge mpi4py openmpi cxx-compiler -conda env config vars set -n cuda-quantum LD_LIBRARY_PATH="$CONDA_PREFIX/envs/cuda-quantum/lib:$LD_LIBRARY_PATH" -conda env config vars set -n cuda-quantum MPI_PATH=$CONDA_PREFIX/envs/cuda-quantum -conda run -n cuda-quantum pip install cuda-quantum-cu12 -conda activate cuda-quantum -source $CONDA_PREFIX/lib/python3.12/site-packages/distributed_interfaces/activate_custom_mpi.sh -``` - -[//]: # (End conda install) - -You must configure MPI by setting the following environment variables: - -[//]: # (Begin ompi setup) - -```console -export OMPI_MCA_opal_cuda_support=true OMPI_MCA_btl='^openib' -``` - -[//]: # (End ompi setup) - -*If you do not set these variables you may encounter a segmentation fault.* - -**Important**: It is *not* sufficient to set these variable within the Conda -environment, like the commands above do for `LD_LIBRARY_PATH`. To avoid having -to set them every time you launch a new shell, we recommend adding them to -`~/.profile` (create the file if it does not exist), and to `~/.bash_profile` or -`~/.bash_login` if such a file exists. - -[//]: # (End complete install) - -MPI uses [SSH](https://en.wikipedia.org/wiki/Secure_Shell) or -[RSH](https://en.wikipedia.org/wiki/Remote_Shell) to communicate with each node -unless another resource manager, such as -[SLURM](https://slurm.schedmd.com/overview.html), is used. If you are -encountering an error "The value of the MCA parameter `plm_rsh_agent` was set to -a path that could not be found", please make sure you have an SSH Client -installed. - -## Running CUDA-Q - -You should now be able to import CUDA-Q and start building quantum -programs in Python! - -```console -import cudaq - -kernel = cudaq.make_kernel() -qubit = kernel.qalloc() -kernel.x(qubit) -kernel.mz(qubit) - -result = cudaq.sample(kernel) -``` - -Additional examples and documentation are linked in the [release -notes][cudaq_docs_releases]. - -## Contributing - -There are many ways in which you can get involved with CUDA-Q. If you are -interested in developing quantum applications with CUDA-Q, our [GitHub -repository][github_link] is a great place to get started! For more information -about contributing to the CUDA-Q platform, please take a look at -[Contributing.md](https://github.com/NVIDIA/cuda-quantum/blob/main/Contributing.md). - -## License - -CUDA-Q is an open source project. The source code is available on -[GitHub][github_link] and licensed under [Apache License -2.0](https://github.com/NVIDIA/cuda-quantum/blob/main/LICENSE). CUDA-Q -makes use of the NVIDIA cuQuantum SDK to enable high-performance simulation, -which is held to its own respective license. - -[github_link]: https://github.com/NVIDIA/cuda-quantum/ - -## Feedback - -Please let us know your feedback and ideas for the CUDA-Q platform in the -[Discussions][discussions] tab of our [GitHub repository][github_repo], or [file -an issue][cuda_quantum_issues]. To report security concerns please reach out to -[cuda-quantum@nvidia.com](mailto:cuda-quantum@nvidia.com). - -[discussions]: https://github.com/NVIDIA/cuda-quantum/discussions -[cuda_quantum_issues]: https://github.com/NVIDIA/cuda-quantum/issues -[github_repo]: https://github.com/NVIDIA/cuda-quantum diff --git a/python/README-cu11.md b/python/README.md.in similarity index 70% rename from python/README-cu11.md rename to python/README.md.in index 517c318c38..ddbbfd3b75 100644 --- a/python/README-cu11.md +++ b/python/README.md.in @@ -12,11 +12,12 @@ CUDA-Q is a comprehensive framework for quantum programming. It features: technologies, including state-of-the-art simulator backends with NVIDIA cuQuantum and a number of different physical quantum processors (QPUs) -The CUDA-Q Python wheels contain the Python API and core components of -CUDA-Q. More information about available packages as well as a link to the -documentation and examples for each version can be found in the [release -notes][cudaq_docs_releases]. System and compatibility requirements -are listed in the Installation Guide of the linked documentation. +The CUDA-Q Python wheels contain the Python API and core components of CUDA-Q. +This package installs CUDA-Q binaries that are compatible with a CUDA version +${{ cuda_version_requirement }}. More information about available packages as +well as a link to the documentation and examples for each version can be found +in the [release notes][cudaq_docs_releases]. System and compatibility +requirements are listed in the Installation Guide of the linked documentation. **Note**: CUDA-Q is currently only supported on Linux operating systems using `x86_64` or `aarch64`/`arm64` processors. @@ -30,30 +31,30 @@ are listed in the Installation Guide of the linked documentation. CUDA-Q does not require a GPU to use, but some components are GPU-accelerated. -Getting started with CUDA-Q on `x86_64` platforms simply requires -`pip install cuda-quantum-cu11`. If you have an NVIDIA GPU on your host system, you -will be able to use it without any further installation steps. +Getting started with CUDA-Q on `x86_64` platforms simply requires `pip install +${{ package_name }}`. If you have an NVIDIA GPU on your host system, you will be +able to use it without any further installation steps. However, if you want to perform multi-GPU simulations, or if you are using -`aarch64`/`arm64` processors, additional components must be installed. -We recommend using [Conda](https://docs.conda.io/en/latest/) to do so. -If you are not already using Conda, -you can install a minimal version following the instructions -[here](https://docs.anaconda.com/miniconda/). The -following commands will create and activate a complete environment for CUDA-Q -with all its dependencies: +`aarch64`/`arm64` processors, additional components must be installed. We +recommend using [Conda](https://docs.conda.io/en/latest/) to do so. If you are +not already using Conda, you can install a minimal version following the +instructions [here](https://docs.anaconda.com/miniconda/). The following +commands will create and activate a complete environment for CUDA-Q with all its +dependencies: [//]: # (Begin conda install) ```console -conda create -y -n cuda-quantum python=3.10 pip -conda install -y -n cuda-quantum -c "nvidia/label/cuda-11.8.0" cuda -conda install -y -n cuda-quantum -c conda-forge mpi4py openmpi cxx-compiler -conda env config vars set -n cuda-quantum LD_LIBRARY_PATH="$CONDA_PREFIX/envs/cuda-quantum/lib:$LD_LIBRARY_PATH" -conda env config vars set -n cuda-quantum MPI_PATH=$CONDA_PREFIX/envs/cuda-quantum -conda run -n cuda-quantum pip install cuda-quantum-cu11 -conda activate cuda-quantum -source $CONDA_PREFIX/lib/python3.10/site-packages/distributed_interfaces/activate_custom_mpi.sh +cuda_version=${{ cuda_version_conda }} # set this variable to version ${{ cuda_version_requirement }} +conda create -y -n cudaq-env python=3.11 pip +conda install -y -n cudaq-env -c "nvidia/label/cuda-${cuda_version}" cuda +conda install -y -n cudaq-env -c conda-forge mpi4py openmpi cxx-compiler +conda env config vars set -n cudaq-env LD_LIBRARY_PATH="$CONDA_PREFIX/envs/cudaq-env/lib:$LD_LIBRARY_PATH" +conda env config vars set -n cudaq-env MPI_PATH=$CONDA_PREFIX/envs/cudaq-env +conda run -n cudaq-env pip install ${{ package_name }} +conda activate cudaq-env +source $CONDA_PREFIX/lib/python3.11/site-packages/distributed_interfaces/activate_custom_mpi.sh ``` [//]: # (End conda install) @@ -88,8 +89,8 @@ installed. ## Running CUDA-Q -You should now be able to import CUDA-Q and start building quantum -programs in Python! +You should now be able to import CUDA-Q and start building quantum programs in +Python! ```console import cudaq @@ -117,9 +118,9 @@ about contributing to the CUDA-Q platform, please take a look at CUDA-Q is an open source project. The source code is available on [GitHub][github_link] and licensed under [Apache License -2.0](https://github.com/NVIDIA/cuda-quantum/blob/main/LICENSE). CUDA-Q -makes use of the NVIDIA cuQuantum SDK to enable high-performance simulation, -which is held to its own respective license. +2.0](https://github.com/NVIDIA/cuda-quantum/blob/main/LICENSE). CUDA-Q makes use +of the NVIDIA cuQuantum SDK to enable high-performance simulation, which is held +to its own respective license. [github_link]: https://github.com/NVIDIA/cuda-quantum/ diff --git a/python/cudaq/__init__.py b/python/cudaq/__init__.py index a63f03f6b5..bfefe6bea0 100644 --- a/python/cudaq/__init__.py +++ b/python/cudaq/__init__.py @@ -7,8 +7,8 @@ # ============================================================================ # import sys, os, numpy, platform, multiprocessing -from ._packages import * -from .mlir._mlir_libs._quakeDialects import cudaq_runtime +from ._packages import get_library_path +from ._metadata import cuda_major # Set the multiprocessing start method to 'spawn' if not already set if multiprocessing.get_start_method(allow_none=True) is None: @@ -16,9 +16,8 @@ # CUDAQ_DYNLIBS must be set before any other imports that would initialize # LinkedLibraryHolder. -if not "CUDAQ_DYNLIBS" in os.environ: +if not "CUDAQ_DYNLIBS" in os.environ and not cuda_major is None: try: - cuda_major = cudaq_runtime.__cuda_major__ custatevec_libs = get_library_path(f"custatevec-cu{cuda_major}") custatevec_path = os.path.join(custatevec_libs, "libcustatevec.so.1") @@ -54,6 +53,7 @@ from .runtime.observe import observe from .runtime.state import to_cupy from .kernel.register_op import register_operation +from .mlir._mlir_libs._quakeDialects import cudaq_runtime try: from qutip import Qobj, Bloch diff --git a/python/extension/CMakeLists.txt b/python/extension/CMakeLists.txt index bdd991ce21..2cde8f98df 100644 --- a/python/extension/CMakeLists.txt +++ b/python/extension/CMakeLists.txt @@ -104,11 +104,6 @@ target_link_libraries(CUDAQuantumPythonSources.Extension INTERFACE fmt::fmt-header-only unzip_util ) -if (CUDA_FOUND) - enable_language(CUDA) - find_package(CUDAToolkit REQUIRED) - target_compile_definitions(CUDAQuantumPythonSources.Extension INTERFACE -DCUDAQ_CUDA_MAJOR=${CUDAToolkit_VERSION_MAJOR}) -endif() ################################################################################ # Common CAPI diff --git a/python/extension/CUDAQuantumExtension.cpp b/python/extension/CUDAQuantumExtension.cpp index fd5150e6fc..daf29bfe60 100644 --- a/python/extension/CUDAQuantumExtension.cpp +++ b/python/extension/CUDAQuantumExtension.cpp @@ -118,9 +118,6 @@ PYBIND11_MODULE(_quakeDialects, m) { ss << "CUDA-Q Version " << cudaq::getVersion() << " (" << cudaq::getFullRepositoryVersion() << ")"; cudaqRuntime.attr("__version__") = ss.str(); -#ifdef CUDAQ_CUDA_MAJOR - cudaqRuntime.attr("__cuda_major__") = CUDAQ_CUDA_MAJOR; -#endif auto mpiSubmodule = cudaqRuntime.def_submodule("mpi"); mpiSubmodule.def( diff --git a/python/metadata.cmake b/python/metadata.cmake new file mode 100644 index 0000000000..396dfc58a3 --- /dev/null +++ b/python/metadata.cmake @@ -0,0 +1,23 @@ +# ============================================================================ # +# Copyright (c) 2022 - 2024 NVIDIA Corporation & Affiliates. # +# All rights reserved. # +# # +# This source code and the accompanying materials are made available under # +# the terms of the Apache License 2.0 which accompanies this distribution. # +# ============================================================================ # + +# We need to be able to access the metadata without loading the cudaq_runtime. +# To do so, we generate a separate (Python) file that is then included in the package. +# This script is used/needed to run it as a command in a custom CMake target, +# which copies all Python files to the build directory. + +if(NOT METADATA_FILE) + message(FATAL_ERROR "METADATA_FILE is not defined") +endif() + +message(STATUS "Creating metadata file in ${METADATA_FILE}.") +if(CUDA_VERSION_MAJOR) + file(WRITE ${METADATA_FILE} "cuda_major=${CUDA_VERSION_MAJOR}") +else() + file(WRITE ${METADATA_FILE} "cuda_major=None") +endif() diff --git a/python/metapackages/pyproject.toml b/python/metapackages/pyproject.toml new file mode 100644 index 0000000000..32543b320b --- /dev/null +++ b/python/metapackages/pyproject.toml @@ -0,0 +1,58 @@ +# ============================================================================ # +# Copyright (c) 2022 - 2024 NVIDIA Corporation & Affiliates. # +# All rights reserved. # +# # +# This source code and the accompanying materials are made available under # +# the terms of the Apache License 2.0 which accompanies this distribution. # +# ============================================================================ # + +[project] +name = "cudaq" +dynamic = ["version", "dependencies"] +keywords = [ "cudaq", "cuda-quantum", "cuda", "quantum", "quantum computing", "nvidia", "high-performance computing" ] +description="Python bindings for the CUDA-Q toolkit for heterogeneous quantum-classical workflows." +authors = [{name = "NVIDIA Corporation & Affiliates"}] +maintainers = [{name = "NVIDIA Corporation & Affiliates"}] +readme = { file="README.md.in", content-type = "text/markdown"} +requires-python = ">=3.10" +license = { file="LICENSE" } +classifiers = [ + 'Intended Audience :: Science/Research', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License 2.0 (Apache-2.0)', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + "Environment :: GPU :: NVIDIA CUDA", + "Environment :: GPU :: NVIDIA CUDA :: 11", + "Environment :: GPU :: NVIDIA CUDA :: 12", + 'Topic :: Software Development', + 'Topic :: Scientific/Engineering', +] + +[project.urls] +Homepage = "https://developer.nvidia.com/cuda-quantum" +Documentation = "https://nvidia.github.io/cuda-quantum" +Repository = "https://github.com/NVIDIA/cuda-quantum" +Releases = "https://nvidia.github.io/cuda-quantum/latest/releases.html" + +# We must use h5py<3.11 because 3.11 doesn't include aarch64 Linux wheels. +# https://github.com/h5py/h5py/issues/2408 +[project.optional-dependencies] +chemistry = [ "scipy==1.10.1", "openfermionpyscf==0.5", "h5py<3.11" ] +visualization = [ "qutip<5" , "matplotlib>=3.5" ] + +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +packages = [] +include-package-data = true +license-files = [ "LICENSE", "NOTICE", "CITATION.cff" ] + +[tool.setuptools.dynamic] +version = {file = "_version.txt"} + diff --git a/python/metapackages/setup.py b/python/metapackages/setup.py new file mode 100644 index 0000000000..4670620753 --- /dev/null +++ b/python/metapackages/setup.py @@ -0,0 +1,170 @@ +# ============================================================================ # +# Copyright (c) 2022 - 2024 NVIDIA Corporation & Affiliates. # +# All rights reserved. # +# # +# This source code and the accompanying materials are made available under # +# the terms of the Apache License 2.0 which accompanies this distribution. # +# ============================================================================ # + +import ctypes, pkg_resources, os, sys +from setuptools import setup +from typing import Optional + + +def _log(msg: str) -> None: + sys.stdout.write(f'[cudaq] {msg}\n') + sys.stdout.flush() + + +def _get_version_from_library( + libnames: list[str], + funcname: str, + nvrtc: bool = False, +) -> Optional[int]: + """Returns the library version from list of candidate libraries.""" + + for libname in libnames: + try: + _log(f'Looking for library: {libname}') + runtime_so = ctypes.CDLL(libname) + break + except Exception as e: + _log(f'Failed to open {libname}: {e}') + else: + _log('No more candidate library to find') + return None + + func = getattr(runtime_so, funcname, None) + if func is None: + raise Exception(f'{libname}: {func} could not be found') + func.restype = ctypes.c_int + + if nvrtc: + func.argtypes = [ + ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(ctypes.c_int), + ] + major = ctypes.c_int() + minor = ctypes.c_int() + retval = func(major, minor) + version = major.value * 1000 + minor.value * 10 + else: + func.argtypes = [ + ctypes.POINTER(ctypes.c_int), + ] + version_ref = ctypes.c_int() + retval = func(version_ref) + version = version_ref.value + + if retval != 0: + raise Exception(f'{libname}: {func} returned error: {retval}') + _log(f'Detected version: {version}') + return version + + +def _get_cuda_version() -> Optional[int]: + """Returns the detected CUDA version or None.""" + + version = None + + # First try NVRTC + libnames = [ + 'libnvrtc.so.12', + 'libnvrtc.so.11.2', + 'libnvrtc.so.11.1', + 'libnvrtc.so.11.0', + ] + _log(f'Trying to detect CUDA version from libraries: {libnames}') + try: + version = _get_version_from_library(libnames, 'nvrtcVersion', True) + except Exception as e: + _log(f"Error: {e}") # log and move on + if version is not None: + return version + + # Next try CUDART (side-effect: a CUDA context will be initialized) + libnames = [ + 'libcudart.so.12', + 'libcudart.so.11.0', + ] + _log(f'Trying to detect CUDA version from libraries: {libnames}') + try: + version = _get_version_from_library(libnames, 'cudaRuntimeGetVersion', + False) + except Exception as e: + _log(f"Error: {e}") # log and move on + if version is not None: + return version + + _log("Autodetection failed") + return None + + +def _infer_best_package() -> str: + + # Find the existing wheel installation + installed = [] + for pkg_suffix in ['', '-cu11', '-cu12']: + try: + pkg_resources.get_distribution(f"cuda-quantum{pkg_suffix}") + installed.append(f"cuda-quantum{pkg_suffix}") + except pkg_resources.DistributionNotFound: + pass + + cuda_version = _get_cuda_version() + if cuda_version is None: + cudaq_bdist = 'cuda-quantum-cu12' + elif cuda_version < 11000: + raise Exception(f'Your CUDA version ({cuda_version}) is too old.') + elif cuda_version < 12000: + cudaq_bdist = 'cuda-quantum-cu11' + elif cuda_version < 13000: + cudaq_bdist = 'cuda-quantum-cu12' + else: + raise Exception(f'Your CUDA version ({cuda_version}) is too new.') + + # Disallow -cu11 & -cu12 wheels from coexisting + conflicting = ", ".join((pkg for pkg in installed if pkg != cudaq_bdist)) + if conflicting != '': + raise Exception( + f'You have a conflicting CUDA-Q version installed.' + f'Please remove the following package(s): {conflicting}') + return cudaq_bdist + + +# This setup handles 3 cases: +# 1. At the release time, we use it to generate source distribution (which contains +# this script). +# 2. If the source distribution is generated for the deprecated cuda-quantum package +# name, this script raises an exception at install time asking to install cudaq. +# 3. If the source distribution is generated for a valid cudaq version, +# this script identifies the installed CUDA version at cudaq install time +# and downloads the corresponding CUDA-Q binary distribution. +# For case 1, `CUDAQ_META_WHEEL_BUILD` is set to 1. +setup_dir = os.path.dirname(os.path.abspath(__file__)) +data_files = [] +install_requires = [] +if os.environ.get('CUDAQ_META_WHEEL_BUILD', '0') == '1': + # Case 1: create source distribution + if os.path.exists(os.path.join(setup_dir, "_deprecated.txt")): + data_files = [('', [ + '_deprecated.txt', + ])] # extra files to be copied into the distribution +else: + # Case 2: install cuda-quantum source distribution + if os.path.exists(os.path.join(setup_dir, "_deprecated.txt")): + with open(os.path.join(setup_dir, "_deprecated.txt"), "r") as f: + deprecation = f.read() + raise Exception(f'This package is deprecated. \n' + deprecation) + # Case 3: install cudaq source distribution + with open(os.path.join(setup_dir, "_version.txt"), "r") as f: + __version__ = f.read() + install_requires = [ + f"{_infer_best_package()}=={__version__}", + ] + +setup( + zip_safe=False, + data_files=data_files, + install_requires=install_requires, +) diff --git a/scripts/validate_wheel.sh b/scripts/validate_wheel.sh index 3b513bf2ea..5aea564b92 100644 --- a/scripts/validate_wheel.sh +++ b/scripts/validate_wheel.sh @@ -28,17 +28,15 @@ # COPY docs/sphinx/targets/python /tmp/targets/ # COPY docs/sphinx/snippets/python /tmp/snippets/ # COPY python/tests /tmp/tests/ -# COPY python/README-cu11.md /tmp/ +# COPY python/README.md.in /tmp/README.md # RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates vim wget openssh-client __optind__=$OPTIND OPTIND=1 python_version=3.11 quick_test=false -while getopts ":c:f:p:qw:" opt; do +while getopts ":f:p:qw:" opt; do case $opt in - c) cuda_major="$OPTARG" - ;; f) root_folder="$OPTARG" ;; p) python_version="$OPTARG" @@ -55,7 +53,7 @@ done OPTIND=$__optind__ # FIXME: check validation with src dist (subsequent PR) -readme_file="$root_folder/README-cu$cuda_major.md" +readme_file="$root_folder/README.md" if [ ! -d "$root_folder" ] || [ ! -f "$readme_file" ] ; then echo -e "\e[01;31mDid not find Python root folder. Please pass the folder containing the README and test with -f.\e[0m" >&2 (return 0 2>/dev/null) && return 100 || exit 100 @@ -77,7 +75,7 @@ fi conda_script="$(awk '/(Begin conda install)/{flag=1;next}/(End conda install)/{flag=0}flag' "$readme_file" | grep . | sed '/^```/d')" while IFS= read -r line; do line=$(echo $line | sed -E "s/python(=)?3.[0-9]{1,}/python\13.10/g") - line=$(echo $line | sed -E "s/pip install cuda-quantum-cu[0-9]{2,}/pip install \"$cudaq_wheel\"/g") + line=$(echo $line | sed -E "s/pip install cuda-quantum-cu[0-9]{2}/pip install \"${cudaq_wheel//\//\\/}\"/g") if [ -n "$(echo $line | grep "conda activate")" ]; then conda_env=$(echo "$line" | sed "s#conda activate##" | tr -d '[:space:]') source $(conda info --base)/bin/activate $conda_env @@ -171,7 +169,8 @@ done # Note that a derivative of this code is in # docs/sphinx/using/backends/platform.rst, so if you update it here, you need to # check if any docs updates are needed. -cudaq_location=`python3 -m pip show cuda-quantum | grep -e 'Location: .*$'` +cudaq_package=`python3 -m pip list | grep -oE 'cuda-quantum-cu[0-9]{2}'` +cudaq_location=`python3 -m pip show ${cudaq_package} | grep -e 'Location: .*$'` qpud_py="${cudaq_location#Location: }/bin/cudaq-qpud.py" if [ -x "$(command -v nvidia-smi)" ]; then nr_gpus=`nvidia-smi --list-gpus | wc -l`