Include py.typed file & fix and/or add missing/wrong type hints in stub files #778

Workflow file for this run

.github/workflows/build-wheels.yml at 04e3438

	name: Build Wheels

	on:
	push:
	tags:
	- '*'
	branches:
	- develop
	- ci/*

	pull_request:
	types:
	- opened
	- synchronize
	branches:
	- develop

	env:
	PYTHON_VERSION: "3.12"
	FORCE_COLOR: "1"

	jobs:
	build-wheels:
	runs-on: ${{ matrix.os }}
	strategy:
	matrix:
	os: [ ubuntu-latest, windows-latest, macos-12 ]
	env:
	CIBW_SKIP: "-musllinux"
	CIBW_ARCHS_MACOS: "x86_64 arm64"
	CIBW_ARCHS_LINUX: "x86_64"
	# CIBW_ARCHS_LINUX: "x86_64 aarch64" # https://github.com/pypa/cibuildwheel/issues/1771#issuecomment-1973003145
	CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest
	CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_aarch64:latest
	CIBW_TEST_SKIP: "-macosx_arm64 -manylinux_aarch64" # ARM64 wheels cannot be tested
	CIBW_REPAIR_WHEEL_COMMAND_MACOS: >-
	DYLD_LIBRARY_PATH=$LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}
	CIBW_BEFORE_BUILD_WINDOWS: "python -m pip install delvewheel"
	CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: >-
	delvewheel repair --add-path C:\vcpkg\installed\x64-windows\bin -w {dest_dir} {wheel}

	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Cache Vcpkg
	uses: actions/cache@v4
	id: cache-vcpkg
	if: runner.os == 'macOS' \|\| runner.os == 'Windows'
	with:
	path: \|
	/usr/local/share/vcpkg/installed
	C:\vcpkg\installed
	key: ${{ runner.os }}-vcpkg-15 # INCREMENT ME!!

	- name: Install Vcpkg Dependencies
	if: (runner.os == 'macOS' \|\| runner.os == 'Windows') && steps.cache-vcpkg.outputs.cache-hit != 'true'
	shell: bash
	run: \|
	set -e

	VCPKG_CMD="vcpkg --overlay-ports .vcpkg/ports --overlay-triplets .vcpkg/triplets install"
	PKG_LIST="lexbor lz4 re2 uchardet zlib"

	if [ -d /usr/local/share/vcpkg/ports ]; then
	# MacOS
	$VCPKG_CMD --triplet=x64-osx $PKG_LIST
	$VCPKG_CMD --triplet=arm64-osx $PKG_LIST
	elif [ -d /c/vcpkg ]; then
	# Windows
	$VCPKG_CMD --triplet=x64-windows $PKG_LIST
	else
	echo "Unsupported platform." >&2
	exit 1
	fi

	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: ${{ env.PYTHON_VERSION }}

	# - name: Set up QEMU
	# if: runner.os == 'Linux'
	# uses: docker/setup-qemu-action@v3
	# with:
	# platforms: aarch64

	- name: Build FastWARC
	uses: pypa/[email protected]
	with:
	package-dir: fastwarc
	output-dir: wheelhouse
	env:
	CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/fastwarc

	- name: Build Resiliparse
	uses: pypa/[email protected]
	with:
	package-dir: resiliparse
	output-dir: wheelhouse
	env:
	CIBW_BEFORE_TEST: >-
	python -c "import glob, platform; open('fastwarc.txt', 'w').write(glob.glob('wheelhouse/FastWARC-cp' + ''.join(map(str, platform.python_version_tuple()[:2])) + '-_' + platform.machine().lower() + '.whl')[0])" &&
	python -m pip install -r fastwarc.txt
	CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/resiliparse

	- name: Upload Wheels
	uses: actions/upload-artifact@v4
	with:
	name: wheels-${{ matrix.os }}
	path: ./wheelhouse/*.whl

	# To be merged with the above when https://github.com/pypa/cibuildwheel/issues/1771#issuecomment-1973003145 is fixed
	build-wheels-aarch64:
	runs-on: ubuntu-latest
	if: startsWith(github.ref, 'refs/tags/v')
	env:
	CIBW_SKIP: "cp38 cp39 -musllinux"
	CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_aarch64:latest
	CIBW_ARCHS_LINUX: "aarch64"
	CIBW_TEST_SKIP: "*"
	CIBW_CONTAINER_ENGINE: "docker; create_args: --platform linux/arm64/v8"

	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: ${{ env.PYTHON_VERSION }}

	- name: Set up QEMU
	uses: docker/setup-qemu-action@v3
	with:
	platforms: all

	- name: Build FastWARC
	uses: pypa/[email protected]
	with:
	package-dir: fastwarc
	output-dir: wheelhouse
	env:
	CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/fastwarc

	- name: Build Resiliparse
	uses: pypa/[email protected]
	with:
	package-dir: resiliparse
	output-dir: wheelhouse
	env:
	CIBW_BEFORE_TEST: >-
	python -c "import glob, platform; open('fastwarc.txt', 'w').write(glob.glob('wheelhouse/FastWARC-cp' + ''.join(map(str, platform.python_version_tuple()[:2])) + '-_' + platform.machine().lower() + '.whl')[0])" &&
	python -m pip install -r fastwarc.txt
	CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/resiliparse

	- name: Upload Wheels
	uses: actions/upload-artifact@v4
	with:
	name: wheels-ubuntu-latest-aarch64
	path: ./wheelhouse/*.whl

	build-sdist:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: ${{ env.PYTHON_VERSION }}

	- name: Install Build Module
	run: python -m pip install build

	- name: Build FastWARC Source Dist
	run: python -m build --sdist --outdir dist fastwarc

	- name: Build Resiliparse Source Dist
	run: python -m build --sdist --outdir dist resiliparse

	- name: Upload Source Dists
	uses: actions/upload-artifact@v4
	with:
	name: sdist
	path: ./dist/*.tar.gz

	build-asan:
	runs-on: ubuntu-latest
	container:
	image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest
	env:
	DEBUG: "1"
	ASAN: "1"
	ASAN_OPTIONS: leak_check_at_exit=0

	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Build Extensions
	run: \|
	set -e
	python${PYTHON_VERSION} -m pip install -e "fastwarc[all,test]"
	python${PYTHON_VERSION} -m pip install -e "resiliparse[all,test]"

	- name: Run Tests
	run: \|
	export LD_PRELOAD="$(ldconfig -p \| grep libasan \| head -n1 \| awk '{print $4}')"
	python${PYTHON_VERSION} -m pytest --capture=sys --verbose tests/

	build-coverage:
	runs-on: ubuntu-latest
	container:
	image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest
	env:
	TRACE: "1"

	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Build Extensions
	run: \|
	set -e
	python${PYTHON_VERSION} -m pip install cython codecov-cli
	python${PYTHON_VERSION} -m pip install -e "fastwarc[all,test]"
	python${PYTHON_VERSION} -m pip install -e "resiliparse[all,test]"

	- name: Run Tests
	run: python${PYTHON_VERSION} -m pytest --cov --cov-report xml --junitxml=report.junit.xml tests/

	# We cannot use the Codecov GitHub action due to incompatible Glibc versions
	- name: Upload to Codecov
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
	TOKENLESS: ${{ github.event.pull_request.head.label }}
	run: \|
	set -e
	python${PYTHON_VERSION} -m codecov_cli.main do-upload \
	--file report.junit.xml \
	--report-type test_results \
	--sha ${{ github.sha }} \
	--branch ${{ github.event.pull_request.head.label \|\| github.ref_name }} \
	--slug ${{ github.repository }} \
	--git-service github \
	--fail-on-error

	python${PYTHON_VERSION} -m codecov_cli.main upload-process \
	--file coverage.xml \
	--report-type coverage \
	--sha ${{ github.sha }} \
	--branch ${{ github.event.pull_request.head.label \|\| github.ref_name }} \
	--slug ${{ github.repository }} \
	--git-service github \
	--fail-on-error

	- name: Upload Artifacts
	uses: actions/upload-artifact@v4
	with:
	name: coverage
	path: \|
	report.junit.xml
	coverage.xml

	build-documentation:
	runs-on: ubuntu-latest
	needs: build-wheels
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: ${{ env.PYTHON_VERSION }}

	- name: Download Artifacts
	uses: actions/download-artifact@v4
	with:
	name: wheels-ubuntu-latest
	path: wheelhouse

	- name: Build Documentation
	run: \|
	set -e

	python -m pip install twine
	grep -vE "fastwarc\|resiliparse" docs/requirements.txt \| xargs python -m pip install

	PYTHON_ABI="cp${PYTHON_VERSION/./}"
	find wheelhouse -name "FastWARC--${PYTHON_ABI}--manylinux_x86_64.whl" \| xargs -I% python -m pip install "%[all]"
	find wheelhouse -name "Resiliparse--${PYTHON_ABI}--manylinux_x86_64.whl" \| xargs -I% python -m pip install "%[all]"

	cd docs
	make html

	- name: Trigger Readthedocs Build
	if: github.ref == 'refs/heads/develop' \|\| startsWith(github.ref, 'refs/tags/v')
	uses: dfm/rtds-action@v1
	with:
	webhook_url: ${{ secrets.RTDS_WEBHOOK_URL }}
	webhook_token: ${{ secrets.RTDS_WEBHOOK_TOKEN }}
	commit_ref: ${{ github.ref }}

	publish-wheels:
	runs-on: ubuntu-latest
	if: startsWith(github.ref, 'refs/tags/v')
	needs: [ build-wheels, build-wheels-aarch64, build-asan, build-sdist, build-documentation ]
	steps:
	- name: Download Wheels
	uses: actions/download-artifact@v4
	with:
	pattern: wheels-*
	merge-multiple: true

	- name: Download Source Dist
	uses: actions/download-artifact@v4
	with:
	name: sdist

	- name: Publish to PyPi
	env:
	TWINE_USERNAME: __token__
	TWINE_PASSWORD1: ${{ secrets.PYPI_FASTWARC_API_TOKEN }}
	TWINE_PASSWORD2: ${{ secrets.PYPI_RESILIPARSE_API_TOKEN }}
	run: \|
	set -e

	python3 -m pip install twine

	TWINE_PASSWORD="$TWINE_PASSWORD1" python3 -m twine upload FastWARC-.whl fastwarc-.tar.gz
	TWINE_PASSWORD="$TWINE_PASSWORD2" python3 -m twine upload Resiliparse-.whl resiliparse-.tar.gz

	- name: Wait
	run: sleep 30

	- name: Trigger Readthedocs Build
	uses: dfm/rtds-action@v1
	with:
	webhook_url: ${{ secrets.RTDS_WEBHOOK_URL }}
	webhook_token: ${{ secrets.RTDS_WEBHOOK_TOKEN }}
	commit_ref: ${{ github.ref }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Include py.typed file & fix and/or add missing/wrong type hints in stub files #778

Workflow file

Include py.typed file & fix and/or add missing/wrong type hints in stub files #778

Jobs

Run details

Workflow file for this run