diff --git a/.github/actions/run-in-docker/action.yml b/.github/actions/run-in-docker/action.yml index 923335452f..ac7eb11072 100644 --- a/.github/actions/run-in-docker/action.yml +++ b/.github/actions/run-in-docker/action.yml @@ -11,6 +11,10 @@ inputs: shell: default: "bash" required: false + # Adding a new optional parameter for directory mapping + volume: + default: "" + required: false runs: using: "composite" @@ -21,6 +25,12 @@ runs: then additional_run_args="--gpus all" fi + # Check if a volume mapping is specified and set the option + volume_option="" + if [ -n "${{ inputs.volume }}" ]; then + volume_option="-v ${{ inputs.volume }}" + fi + # Note: "bash -s << EOF" does not play nice with mpirun/mpiexec. It # silently skips any shell commands that come after the mpirun/mpiexec, # so don't use it. Use this instead, which seems to work better. @@ -29,7 +39,7 @@ runs: ${{ inputs.run }} EOF - container=$(docker run --user ${{ inputs.user }} $additional_run_args -id ${{ inputs.image }}) + container=$(docker run --user ${{ inputs.user }} $additional_run_args $volume_option -id ${{ inputs.image }}) docker cp $tmpFile $container:$tmpFile docker exec --user root $container chown -R ${{ inputs.user }} $tmpFile docker exec $container ${{ inputs.shell }} $tmpFile diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 0000000000..3be32b64e3 --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,10 @@ +coverage: + status: + project: + default: + target: 70% + threshold: 1% +ignore: + - "tpls" +fixes: + - "/workspaces/cuda-quantum::" \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f6df56136..a5be434c09 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -193,6 +193,24 @@ jobs: devdeps_archive: ${{ fromJson(needs.config.outputs.json).tar_archive[format('{0}-{1}', matrix.platform, matrix.toolchain)] }} export_environment: ${{ github.event_name == 'workflow_dispatch' && inputs.export_environment }} + gen_code_coverage: + name: Gen code coverage + needs: config + strategy: + matrix: + platform: [amd64] + toolchain: [clang16] + fail-fast: false + uses: ./.github/workflows/generate_cc.yml + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + platform: linux/${{ matrix.platform }} + devdeps_image: ${{ fromJson(needs.config.outputs.json).image_hash[format('{0}-{1}', matrix.platform, matrix.toolchain)] }} + devdeps_cache: ${{ fromJson(needs.config.outputs.json).cache_key[format('{0}-{1}', matrix.platform, matrix.toolchain)] }} + devdeps_archive: ${{ fromJson(needs.config.outputs.json).tar_archive[format('{0}-{1}', matrix.platform, matrix.toolchain)] }} + export_environment: ${{ github.event_name == 'workflow_dispatch' && inputs.export_environment }} + docker_image: name: Create Docker images needs: config diff --git a/.github/workflows/generate_cc.yml b/.github/workflows/generate_cc.yml new file mode 100644 index 0000000000..463d0e0c6a --- /dev/null +++ b/.github/workflows/generate_cc.yml @@ -0,0 +1,135 @@ +on: + workflow_call: + inputs: + platform: + type: string + required: false + default: linux/amd64 + devdeps_image: + required: false + type: string + devdeps_cache: + required: true + type: string + devdeps_archive: + required: true + type: string + export_environment: + required: false + type: boolean + secrets: + CODECOV_TOKEN: + required: false + +name: Show Code Coverage Diff + +jobs: + gen_code_coverage: + runs-on: ${{ (contains(inputs.platform, 'arm') && 'linux-arm64-cpu8') || 'linux-amd64-cpu8' }} + permissions: + contents: read + packages: read + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Restore environment + id: restore_devdeps + if: inputs.devdeps_image == '' + uses: actions/cache/restore@v4 + with: + path: ${{ inputs.devdeps_archive }} + key: ${{ inputs.devdeps_cache }} + fail-on-cache-miss: true + + - name: Log in to GitHub CR + if: inputs.devdeps_image != '' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Set up context for buildx + run: | + docker context create builder_context + + - name: Set up buildx runner + uses: docker/setup-buildx-action@v3 + with: + endpoint: builder_context + + - name: Build CUDA Quantum + id: cudaq_build + run: | + if ${{ steps.restore_devdeps.outcome != 'skipped' }}; then + load_output=`docker load --input "${{ inputs.devdeps_archive }}"` + base_image=`echo "$load_output" | grep -o 'Loaded image: \S*:\S*' | head -1 | cut -d ' ' -f 3` + elif ${{ inputs.devdeps_image != '' }}; then + base_image=${{ inputs.devdeps_image }} + else + echo "::error file=generate_cc.yml::Missing configuration for development dependencies. Either specify the image (i.e. provide devdeps_image) or cache (i.e. provide devdeps_cache and devdeps_archive) that should be used for the build." + exit 1 + fi + + DOCKER_BUILDKIT=1 docker build --platform ${{ inputs.platform }} \ + -t cuda-quantum-cc:local -f docker/build/cudaq.dev.Dockerfile . \ + --build-arg base_image=$base_image + + devdeps_tag=`echo $base_image | rev | cut -d ":" -f 1 | rev` + echo "devdeps_tag=$devdeps_tag" >> $GITHUB_OUTPUT + + - name: Create Shared Dir + run: | + mkdir -p ${{ github.workspace }}/shared + - name: Test CUDA Quantum And Generate CC + uses: ./.github/actions/run-in-docker + with: + image: cuda-quantum-cc:local + shell: bash + volume: ${{ github.workspace }}/shared:/shared + run: | + cd $CUDAQ_REPO_ROOT + bash scripts/generate_cc.sh -v -c -p + if [ ! $? -eq 0 ]; then + echo "generate_cc status = " $? + else + chmod -R 777 ./build/ccoverage + chmod -R 777 ./build/pycoverage + cp ./build/ccoverage/coverage.txt /shared + cp ./build/pycoverage/coverage.xml /shared + fi + + - name: Upload C/C++ & Python Coverage To Codecov + uses: codecov/codecov-action@v4 + with: + fail_ci_if_error: false + files: ${{ github.workspace }}/shared/coverage.txt,${{ github.workspace }}/shared/coverage.xml + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + + - name: Delete Shared Dir + run: | + rm -r ${{ github.workspace }}/shared + + - name: Save environment + id: env_save + if: inputs.export_environment + run: | + output_directory=/tmp + filename=${{ steps.cudaq_build.outputs.devdeps_tag }}_cc_build + docker run --name cuda-quantum-cc cuda-quantum-cc:local + docker export cuda-quantum-cc > $output_directory/$filename.tar + docker rm -f cuda-quantum-cc + + echo "filename=$filename" >> $GITHUB_OUTPUT + echo "output_directory=$output_directory" >> $GITHUB_OUTPUT + + - name: Upload environment + uses: actions/upload-artifact@v4 + if: inputs.export_environment + with: + name: ${{ steps.env_save.outputs.filename }} + path: ${{ steps.env_save.outputs.output_directory }}/${{ steps.env_save.outputs.filename }}.tar + retention-days: 1 diff --git a/scripts/generate_cc.sh b/scripts/generate_cc.sh index 28eb4e7418..a75fcac42a 100644 --- a/scripts/generate_cc.sh +++ b/scripts/generate_cc.sh @@ -16,37 +16,45 @@ # bash scripts/generate_cc.sh -c -p # -c flag generates coverage information for C and C++ codes. # -p flag generates coverage information for Python codes. +# -v flag generates data format for uploading to codecov # C and C++ coverage reports are generated in the directory 'build/ccoverage' # Python coverage reports are generated in the directory 'build/pycoverage' # # Note: # The script should be run in the cuda-quantum-devdeps container environment. -# Currently, Python code cannot display the coverage of kernel functions. +# current tested image: ghcr.io/nvidia/cuda-quantum-devdeps:clang16-main +# Don't enable GPU +# C/C++ coverage is located in the ./build/ccoverage directory +# Python coverage is located in the ./build/pycoverage directory if [ $# -lt 1 ]; then - echo "Please provide at least one parameter" - exit 1 + echo "Please provide at least one parameter" + exit 1 fi gen_cpp_coverage=false gen_py_coverage=false +is_codecov_format=false # Process command line arguments __optind__=$OPTIND OPTIND=1 -while getopts ":cp" opt; do - case $opt in - c) - gen_cpp_coverage=true - ;; - p) - gen_py_coverage=true - ;; - \?) - echo "Invalid command line option -$OPTARG" >&2 - exit 1 - ;; - esac +while getopts ":cpv" opt; do + case $opt in + c) + gen_cpp_coverage=true + ;; + p) + gen_py_coverage=true + ;; + v) + is_codecov_format=true + ;; + \?) + echo "Invalid command line option -$OPTARG" >&2 + exit 1 + ;; + esac done OPTIND=$__optind__ @@ -56,8 +64,10 @@ repo_root=$(cd "$this_file_dir" && git rev-parse --show-toplevel) # Set envs if $gen_cpp_coverage; then - export CUDAQ_ENABLE_CC=ON - export LLVM_PROFILE_FILE=${repo_root}/build/tmp/cudaq-cc/profile-%9m.profraw + export CUDAQ_ENABLE_CC=ON + mkdir -p /usr/lib/llvm-16/lib/clang/16/lib/linux + ln -s /usr/local/llvm/lib/clang/16/lib/x86_64-unknown-linux-gnu/libclang_rt.profile.a /usr/lib/llvm-16/lib/clang/16/lib/linux/libclang_rt.profile-x86_64.a + export LLVM_PROFILE_FILE=${repo_root}/build/tmp/cudaq-cc/profile-%9m.profraw fi # Build project @@ -67,79 +77,126 @@ if [ $? -ne 0 ]; then exit 1 fi +# Function to run the llvm-cov command +gen_cplusplus_report() { + if $is_codecov_format; then + mkdir -p ${repo_root}/build/ccoverage + llvm-cov show ${objects} -instr-profile=${repo_root}/build/coverage.profdata --ignore-filename-regex="${repo_root}/tpls/*" \ + --ignore-filename-regex="${repo_root}/build/*" --ignore-filename-regex="${repo_root}/unittests/*" 2>&1 > ${repo_root}/build/ccoverage/coverage.txt + else + llvm-cov show -format=html ${objects} -instr-profile=${repo_root}/build/coverage.profdata --ignore-filename-regex="${repo_root}/tpls/*" \ + --ignore-filename-regex="${repo_root}/build/*" --ignore-filename-regex="${repo_root}/unittests/*" -o ${repo_root}/build/ccoverage 2>&1 + fi +} + if $gen_cpp_coverage; then - # Detect toolchain - use_llvm_cov=false - toolchain_contents=$(cat "$LLVM_INSTALL_PREFIX/bootstrap/cc") - if [[ $toolchain_contents == *"$LLVM_INSTALL_PREFIX/bin/clang"* ]]; then use_llvm_cov=true - else - echo "Currently not supported, running tests using llvm-lit fails" - exit 1 - fi - - # Run tests (C++ Unittests) - python3 -m pip install iqm-client==16.1 - ctest --output-on-failure --test-dir ${repo_root}/build -E ctest-nvqpp -E ctest-targettests - ctest_status=$? - $LLVM_INSTALL_PREFIX/bin/llvm-lit -v --param nvqpp_site_config=${repo_root}/build/test/lit.site.cfg.py ${repo_root}/build/test - lit_status=$? - $LLVM_INSTALL_PREFIX/bin/llvm-lit -v --param nvqpp_site_config=${repo_root}/build/targettests/lit.site.cfg.py ${repo_root}/build/targettests - targ_status=$? - $LLVM_INSTALL_PREFIX/bin/llvm-lit -v --param nvqpp_site_config=${repo_root}/build/python/tests/mlir/lit.site.cfg.py ${repo_root}/build/python/tests/mlir - pymlir_status=$? - if [ ! $ctest_status -eq 0 ] || [ ! $lit_status -eq 0 ] || [ $targ_status -ne 0 ] || [ $pymlir_status -ne 0 ]; then - echo "::error C++ tests failed (ctest status $ctest_status, llvm-lit status $lit_status, \ + + # Run tests (C++ Unittests) + python3 -m pip install iqm-client==16.1 + ctest --output-on-failure --test-dir ${repo_root}/build -E ctest-nvqpp -E ctest-targettests + ctest_status=$? + /usr/local/llvm/bin/llvm-lit -v --param nvqpp_site_config=${repo_root}/build/test/lit.site.cfg.py ${repo_root}/build/test + lit_status=$? + /usr/local/llvm/bin/llvm-lit -v --param nvqpp_site_config=${repo_root}/build/targettests/lit.site.cfg.py ${repo_root}/build/targettests + targ_status=$? + /usr/local/llvm/bin/llvm-lit -v --param nvqpp_site_config=${repo_root}/build/python/tests/mlir/lit.site.cfg.py ${repo_root}/build/python/tests/mlir + pymlir_status=$? + if [ ! $ctest_status -eq 0 ] || [ ! $lit_status -eq 0 ] || [ $targ_status -ne 0 ] || [ $pymlir_status -ne 0 ]; then + echo "::error C++ tests failed (ctest status $ctest_status, llvm-lit status $lit_status, \ target tests status $targ_status, Python MLIR status $pymlir_status)." - exit 1 - fi - - # Run tests (Python tests) - rm -rf ${repo_root}/_skbuild - pip install ${repo_root} --user -vvv - python3 -m pytest -v ${repo_root}/python/tests/ --ignore ${repo_root}/python/tests/backends - for backendTest in ${repo_root}/python/tests/backends/*.py; do - python3 -m pytest -v $backendTest - pytest_status=$? - if [ ! $pytest_status -eq 0 ] && [ ! $pytest_status -eq 5 ]; then - echo "::error $backendTest tests failed with status $pytest_status." - exit 1 + exit 1 fi - done - - # Generate report - if $use_llvm_cov; then - llvm-profdata merge -sparse ${repo_root}/build/tmp/cudaq-cc/profile-*.profraw -o ${repo_root}/build/coverage.profdata - binarys=($(sed -n -e '/Linking CXX shared library/s/^.*Linking CXX shared library //p' \ - -e '/Linking CXX static library/s/^.*Linking CXX static library //p' \ - -e '/Linking CXX shared module/s/^.*Linking CXX shared module //p' \ - -e '/Linking CXX executable/s/^.*Linking CXX executable //p' ${repo_root}/build/logs/ninja_output.txt)) - objects="" - for item in "${binarys[@]}"; do - if [ "$item" != "lib/libCUDAQuantumMLIRCAPI.a" ] && [ "$item" != "lib/libOptTransforms.a" ]; then - objects+="-object ${repo_root}/build/$item " - fi + + # Run tests (Python tests) + rm -rf ${repo_root}/_skbuild + pip install ${repo_root} --user -vvv + python3 -m pytest -v ${repo_root}/python/tests/ --ignore ${repo_root}/python/tests/backends + for backendTest in ${repo_root}/python/tests/backends/*.py; do + python3 -m pytest -v $backendTest + pytest_status=$? + if [ ! $pytest_status -eq 0 ] && [ ! $pytest_status -eq 5 ]; then + echo "::error $backendTest tests failed with status $pytest_status." + exit 1 + fi done - llvm-cov show -format=html ${objects} -instr-profile=${repo_root}/build/coverage.profdata --ignore-filename-regex="${repo_root}/tpls/*" \ - --ignore-filename-regex="${repo_root}/build/*" --ignore-filename-regex="${repo_root}/unittests/*" -o ${repo_root}/build/ccoverage - else - # Use gcov - echo "Currently not supported, running tests using llvm-lit fails" - exit 1 - fi + + # Generate report + if $use_llvm_cov; then + llvm-profdata merge -sparse ${repo_root}/build/tmp/cudaq-cc/profile-*.profraw -o ${repo_root}/build/coverage.profdata + binarys=($(sed -n -e '/Linking CXX shared library/s/^.*Linking CXX shared library //p' \ + -e '/Linking CXX static library/s/^.*Linking CXX static library //p' \ + -e '/Linking CXX shared module/s/^.*Linking CXX shared module //p' \ + -e '/Linking CXX executable/s/^.*Linking CXX executable //p' ${repo_root}/build/logs/ninja_output.txt)) + objects="" + for item in "${binarys[@]}"; do + objects+="-object ${repo_root}/build/$item " + done + + # The purpose of adding this code is to avoid the llvm-cov show command + # from being unable to generate a report due to a malformed format error of an object. + # This is mainly an error caused by a static library, but it has little impact on the coverage rate. + # Loop until the command succeeds + while true; do + output=$(gen_cplusplus_report ${objects}) + status=$? + + # Check if the command failed due to malformed coverage data + if [ $status -ne 0 ]; then + echo "Error detected. Attempting to remove problematic object and retry." + echo "$output" + + # Extract the problematic object from the error message + problematic_object=$(echo "$output" | grep -oP "error: Failed to load coverage: '\K[^']+") + echo $problematic_object + + if [ -n "$problematic_object" ]; then + # Remove the problematic object from the objects variable + objects=$(echo $objects | sed "s|-object $problematic_object||") + + # Check if the problematic object was successfully removed + if [[ $objects != *"-object $problematic_object"* ]]; then + echo "Problematic object '$problematic_object' removed. Retrying..." + else + echo "Failed to remove problematic object '$problematic_object'. Exiting..." + exit 1 + fi + else + echo "No problematic object found in the error message. Exiting..." + exit 1 + fi + else + echo "Command succeeded." + break + fi + done + else + # Use gcov + echo "Currently not supported, running tests using llvm-lit fails" + exit 1 + fi fi if $gen_py_coverage; then - pip install pytest-cov - pip install iqm_client==16.1 --user -vvv - pip install . --user -vvv - python3 -m pytest -v python/tests/ --ignore python/tests/backends --cov=./python --cov-report=html:${repo_root}/build/pycoverage --cov-append - for backendTest in python/tests/backends/*.py; do - python3 -m pytest -v $backendTest --cov=./python --cov-report=html:${repo_root}/build/pycoverage --cov-append - pytest_status=$? - if [ ! $pytest_status -eq 0 ] && [ ! $pytest_status -eq 5 ]; then - echo "::error $backendTest tests failed with status $pytest_status." - exit 1 + pip install pytest-cov + pip install iqm_client==16.1 --user -vvv + rm -rf ${repo_root}/_skbuild + pip install . --user -vvv + if $is_codecov_format; then + python3 -m pytest -v python/tests/ --ignore python/tests/backends --cov=cudaq --cov-report=xml:${repo_root}/build/pycoverage/coverage.xml --cov-append + else + python3 -m pytest -v python/tests/ --ignore python/tests/backends --cov=cudaq --cov-report=html:${repo_root}/build/pycoverage --cov-append fi - done + for backendTest in python/tests/backends/*.py; do + if $is_codecov_format; then + python3 -m pytest -v $backendTest --cov=cudaq --cov-report=xml:${repo_root}/build/pycoverage/coverage.xml --cov-append + else + python3 -m pytest -v $backendTest --cov=cudaq --cov-report=html:${repo_root}/build/pycoverage --cov-append + fi + pytest_status=$? + if [ ! $pytest_status -eq 0 ] && [ ! $pytest_status -eq 5 ]; then + echo "::error $backendTest tests failed with status $pytest_status." + exit 1 + fi + done fi