Enable build of linux-aarch64 wheels #691
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Wheels | |
| on: | |
| push: | |
| tags: | |
| - '*' | |
| branches: | |
| - develop | |
| - ci/* | |
| pull_request: | |
| types: | |
| - opened | |
| - synchronize | |
| branches: | |
| - develop | |
| env: | |
| PYTHON_VERSION: "3.12" | |
| jobs: | |
| build-wheels: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ ubuntu-latest, windows-latest, macOS-latest ] | |
| env: | |
| CIBW_SKIP: "*-musllinux*" | |
| CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest | |
| CIBW_ARCHS_MACOS: "x86_64 arm64" | |
| CIBW_ARCHS_LINUX: "x86_64" | |
| CIBW_TEST_SKIP: "*-macosx_arm64" # Apple Silicon wheels cannot be tested | |
| CIBW_REPAIR_WHEEL_COMMAND_MACOS: >- | |
| DYLD_LIBRARY_PATH=$LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} | |
| CIBW_BEFORE_BUILD_WINDOWS: "python -m pip install delvewheel" | |
| CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: >- | |
| delvewheel repair --add-path C:\vcpkg\installed\x64-windows\bin -w {dest_dir} {wheel} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Cache Vcpkg | |
| uses: actions/cache@v3 | |
| id: cache-vcpkg | |
| if: runner.os == 'macOS' || runner.os == 'Windows' | |
| with: | |
| path: | | |
| /usr/local/share/vcpkg/installed | |
| C:\vcpkg\installed | |
| key: ${{ runner.os }}-vcpkg-14 # INCREMENT ME!! | |
| - name: Install Vcpkg Dependencies | |
| if: (runner.os == 'macOS' || runner.os == 'Windows') && steps.cache-vcpkg.outputs.cache-hit != 'true' | |
| shell: bash | |
| run: | | |
| set -e | |
| VCPKG_CMD="vcpkg --overlay-ports .vcpkg/ports --overlay-triplets .vcpkg/triplets install" | |
| PKG_LIST="lexbor lz4 re2 uchardet zlib" | |
| if [ -d /usr/local/share/vcpkg/ports ]; then | |
| # MacOS | |
| $VCPKG_CMD --triplet=x64-osx $PKG_LIST | |
| $VCPKG_CMD --triplet=arm64-osx $PKG_LIST | |
| elif [ -d /c/vcpkg ]; then | |
| # Windows | |
| $VCPKG_CMD --triplet=x64-windows $PKG_LIST | |
| else | |
| echo "Unsupported platform." >&2 | |
| exit 1 | |
| fi | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Build FastWARC | |
| uses: pypa/[email protected] | |
| with: | |
| package-dir: fastwarc | |
| output-dir: wheelhouse | |
| env: | |
| CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/fastwarc | |
| - name: Build Resiliparse | |
| uses: pypa/[email protected] | |
| with: | |
| package-dir: resiliparse | |
| output-dir: wheelhouse | |
| env: | |
| CIBW_BEFORE_TEST: >- | |
| python -c "import glob, platform; open('fastwarc.txt', 'w').write(glob.glob('wheelhouse/FastWARC-*cp' + ''.join(map(str, platform.python_version_tuple()[:2])) + '-*_' + platform.machine().lower() + '.whl')[0])" && | |
| python -m pip install -r fastwarc.txt | |
| CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/resiliparse | |
| - name: Upload Wheels | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: wheels | |
| path: ./wheelhouse/*.whl | |
| build-wheels-qemu: | |
| runs-on: ubuntu-latest | |
| env: | |
| CIBW_SKIP: "*-musllinux*" | |
| CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_aarch64:latest | |
| CIBW_ARCHS_LINUX: "aarch64" | |
| CIBW_TEST_SKIP: "*-manylinux_aarch64" | |
| # https://github.com/pypa/cibuildwheel/issues/1771#issuecomment-1973003145 | |
| CIBW_CONTAINER_ENGINE: "docker; create_args: --platform linux/arm64/v8" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: all | |
| - name: Build FastWARC | |
| uses: pypa/[email protected] | |
| with: | |
| package-dir: fastwarc | |
| output-dir: wheelhouse | |
| env: | |
| CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/fastwarc | |
| - name: Build Resiliparse | |
| uses: pypa/[email protected] | |
| with: | |
| package-dir: resiliparse | |
| output-dir: wheelhouse | |
| env: | |
| CIBW_BEFORE_TEST: >- | |
| python -c "import glob, platform; open('fastwarc.txt', 'w').write(glob.glob('wheelhouse/FastWARC-*cp' + ''.join(map(str, platform.python_version_tuple()[:2])) + '-*_' + platform.machine().lower() + '.whl')[0])" && | |
| python -m pip install -r fastwarc.txt | |
| CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/resiliparse | |
| - name: Upload Wheels | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: wheels | |
| path: ./wheelhouse/*.whl | |
| build-sdist: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install Build Module | |
| run: python -m pip install build | |
| - name: Build FastWARC Source Dist | |
| run: python -m build --sdist --outdir dist fastwarc | |
| - name: Build Resiliparse Source Dist | |
| run: python -m build --sdist --outdir dist resiliparse | |
| - name: Upload Source Dists | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: sdist | |
| path: ./dist/*.tar.gz | |
| build-asan: | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest | |
| env: | |
| DEBUG: "1" | |
| ASAN: "1" | |
| ASAN_OPTIONS: leak_check_at_exit=0 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 # TODO: Upgrade to @v4 causes GLIBC failures | |
| with: | |
| fetch-depth: 0 | |
| - name: Build Extensions | |
| run: | | |
| set -e | |
| python${PYTHON_VERSION} -m pip install -e "fastwarc[all,test]" | |
| python${PYTHON_VERSION} -m pip install -e "resiliparse[all,test]" | |
| - name: Run Tests | |
| run: | | |
| export LD_PRELOAD="$(ldconfig -p | grep libasan | head -n1 | awk '{print $4}')" | |
| python${PYTHON_VERSION} -m pytest --capture=sys --verbose tests/ | |
| build-coverage: | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest | |
| needs: [ build-wheels, build-asan, build-sdist ] | |
| env: | |
| TRACE: "1" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 # TODO: Upgrade to @v4 causes GLIBC failures | |
| with: | |
| fetch-depth: 0 | |
| - name: Build Extensions | |
| run: | | |
| set -e | |
| python${PYTHON_VERSION} -m pip install cython | |
| python${PYTHON_VERSION} -m pip install -e "fastwarc[all,test]" | |
| python${PYTHON_VERSION} -m pip install -e "resiliparse[all,test]" | |
| - name: Run Tests | |
| run: python${PYTHON_VERSION} -m pytest --cov=resiliparse/ --cov=fastwarc/ --cov=resiliparse_common/ --cov-report xml tests/ | |
| - name: Upload Artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: coverage | |
| path: coverage.xml | |
| - name: Upload to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| fail_ci_if_error: true | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| build-documentation: | |
| runs-on: ubuntu-latest | |
| needs: build-wheels | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Download Artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: wheels | |
| path: wheelhouse | |
| - name: Build Documentation | |
| run: | | |
| set -e | |
| python -m pip install twine | |
| grep -vE "fastwarc|resiliparse" docs/requirements.txt | xargs python -m pip install | |
| PYTHON_ABI="cp${PYTHON_VERSION/./}" | |
| find wheelhouse -name "FastWARC-*-${PYTHON_ABI}-*-manylinux_x86_64.whl" | xargs -I% python -m pip install "%[all]" | |
| find wheelhouse -name "Resiliparse-*-${PYTHON_ABI}-*-manylinux_x86_64.whl" | xargs -I% python -m pip install "%[all]" | |
| cd docs | |
| make html | |
| - name: Trigger Readthedocs Build | |
| if: github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/tags/v') | |
| uses: dfm/rtds-action@v1 | |
| with: | |
| webhook_url: ${{ secrets.RTDS_WEBHOOK_URL }} | |
| webhook_token: ${{ secrets.RTDS_WEBHOOK_TOKEN }} | |
| commit_ref: ${{ github.ref }} | |
| publish-wheels: | |
| runs-on: ubuntu-latest | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| needs: [ build-wheels, build-wheels-qemu, build-sdist, build-documentation ] | |
| steps: | |
| - name: Download Wheels | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: wheels | |
| - name: Download Source Dist | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: sdist | |
| - name: Publish to PyPi | |
| env: | |
| TWINE_USERNAME: __token__ | |
| TWINE_PASSWORD1: ${{ secrets.PYPI_FASTWARC_API_TOKEN }} | |
| TWINE_PASSWORD2: ${{ secrets.PYPI_RESILIPARSE_API_TOKEN }} | |
| run: | | |
| set -e | |
| python3 -m pip install twine | |
| TWINE_PASSWORD="$TWINE_PASSWORD1" python3 -m twine upload FastWARC-*.whl FastWARC-*.tar.gz | |
| TWINE_PASSWORD="$TWINE_PASSWORD2" python3 -m twine upload Resiliparse-*.whl Resiliparse-*.tar.gz | |
| - name: Wait | |
| run: sleep 30 | |
| - name: Trigger Readthedocs Build | |
| uses: dfm/rtds-action@v1 | |
| with: | |
| webhook_url: ${{ secrets.RTDS_WEBHOOK_URL }} | |
| webhook_token: ${{ secrets.RTDS_WEBHOOK_TOKEN }} | |
| commit_ref: ${{ github.ref }} |