From 220dae2938ff157a8095cb4b1b512b9f39f2c81f Mon Sep 17 00:00:00 2001 From: Dan Albert Date: Tue, 25 Jun 2024 14:29:24 -0700 Subject: [PATCH] Add configuration for jitpack. Jitpack is a pseudo-repository for Android (and other) artifacts that builds AARs from GitHub repos for projects that do not themselves publish their artifacts to somewhere like Maven Central. Adding this allows Android app developers to add a dependency on com.github.khronos:vulkan-validationlayers:$TAG to their build.gradle file rather than needing to manage the VVL libraries themselves. See the comment at the top of jitpack.yml for more information. Fixes https://github.com/KhronosGroup/Vulkan-ValidationLayers/issues/8167 --- .github/workflows/sdk_android_build.yml | 123 +++++++++++++++++---- jitpack.yml | 17 +++ scripts/android.py | 59 ++++++---- scripts/jitpack.py | 136 ++++++++++++++++++++++++ 4 files changed, 294 insertions(+), 41 deletions(-) create mode 100644 jitpack.yml create mode 100644 scripts/jitpack.py diff --git a/.github/workflows/sdk_android_build.yml b/.github/workflows/sdk_android_build.yml index abdcd916595..a2d884742d1 100644 --- a/.github/workflows/sdk_android_build.yml +++ b/.github/workflows/sdk_android_build.yml @@ -19,7 +19,10 @@ name: SDK Android Build # artifacts to the release when a Vulkan SDK tag is pushed. The # Vulkan SDK does not include binaries for Android, so we publish # them here to provide Android binaries built from the same source -# used to build the Vulkan SDK. +# used to build the Vulkan SDK. The artifacts will also be bundled into an AAR +# (the library counterpart the APK application format) and uploaded to GitHub +# Packages so app developers can include the validation layers in their +# application the same way they would any Java dependencies. # # The tag needs to be pushed by name, as `git push --tags` to push all # tags does not appear to trigger the action. @@ -48,7 +51,23 @@ on: tags: - vulkan-sdk-* +env: + MIN_SDK_VERSION: 26 + ARTIFACT_ID: vulkan-validation-layers + jobs: + sdk-version: + name: Get SDK version + runs-on: ubuntu-22.04 + outputs: + sdk_version: ${{ steps.get_sdk_version.outputs.sdk_version}} + steps: + - name: Get sdk version string + id: get_sdk_version + run: | + sdk_version=`echo "${{ github.ref }}" | cut -d "-" -f 3` + echo "sdk_version=$sdk_version" >> $GITHUB_OUTPUT + android: name: Android SDK Release runs-on: ubuntu-22.04 @@ -63,24 +82,85 @@ jobs: with: python-version: '3.10' - name: CMake Build - run: python scripts/android.py --config Release --app-abi ${{ matrix.abi }} --app-stl c++_static + run: python scripts/android.py --config Release --app-abi ${{ matrix.abi }} --app-stl c++_static --min-sdk-version $MIN_SDK_VERSION - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: vvl-android-${{ matrix.abi }} path: ./build-android/libs/lib/ + aar: + name: Create AAR + runs-on: ubuntu-22.04 + needs: [android, sdk-version] + steps: + - name: Clone repository + uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: ./libs + merge-multiple: true + pattern: vvl-android-* + - name: Assemble AAR + # GROUP_ID must be configured as a repoistory variable in Settings -> + # Actions -> Variables. + run: | + python scripts/aar.py \ + --group-id ${{ vars.GROUP_ID }} \ + --artifact-id ${{ env.ARTIFACT_ID }} \ + --min-sdk-version $MIN_SDK_VERSION \ + -o vulkan-validation-layers-${{ needs.sdk-version.outputs.sdk_version }}.aar \ + libs + - name: Upload AAR + uses: actions/upload-artifact@v4 + with: + name: vulkan-validation-layers-aar + path: vulkan-validation-layers-${{ needs.sdk-version.outputs.sdk_version }}.aar + if-no-files-found: error + + maven: + name: Push AAR to GitHub Packages + runs-on: ubuntu-22.04 + needs: [aar, sdk-version] + steps: + - name: Set up Java + uses: actions/setup-java@v4 + with: + # Neither are really important. We need the mvn tool, but we aren't + # going to use it to build anything. + distribution: "temurin" + java-version: "21" + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: aar + name: vulkan-validation-layers-aar + - name: Publish + # Useful docs for this section: + # https://maven.apache.org/guides/mini/guide-3rd-party-jars-remote.html + # https://docs.github.com/en/actions/publishing-packages/publishing-java-packages-with-maven + run: | + mvn --batch-mode deploy:deploy-file \ + -DgroupId=${{ vars.GROUP_ID }} \ + -DartifactId=$ARTIFACT_ID \ + -Dversion=${{ needs.sdk-version.outputs.sdk_version }} \ + -Dpackaging=aar \ + -DrepositoryId=github \ + -Durl=https://maven.pkg.github.com/${{ github.repository }} \ + -Dfile=aar/vulkan-validation-layers-${{ needs.sdk-version.outputs.sdk_version }}.aar + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + release: name: Create Release for Tag permissions: write-all runs-on: ubuntu-22.04 - needs: android + needs: [android, sdk-version] steps: - - name: Get sdk version string - id: get_sdk_version - run: | - sdk_version=`echo "${{ github.ref }}" | cut -d "-" -f 3` - echo "sdk_version=$sdk_version" >> $GITHUB_OUTPUT - name: Create release id: create_release uses: actions/create-release@v1 @@ -88,11 +168,19 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: tag_name: ${{ github.ref }} - release_name: Android binaries for ${{ steps.get_sdk_version.outputs.sdk_version }} SDK release + release_name: Android binaries for ${{ needs.sdk-version.outputs.sdk_version }} SDK release body: | These Android Validation Layer binaries were built with ndk version 25.2.9519653 - The validation binaries can only be used with a device that supports Android API version 26 or higher. + The validation binaries can only be used with a device that supports Android API version ${{ env.MIN_SDK_VERSION }} or higher. + + If you're using Android Gradle to build your app, it will be easier + to use the validation layers direcetly from the GitHub Package + Repository: ${{ github.repositoryUrl }}/packages/. See + https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-gradle-registry#using-a-published-package + for instructions on using packages from this repository. To include + the validation layers only in your debug APK (recommended), use + `debugImplementation` rather than `implementation` as the docs say. draft: false prerelease: false - name: Get release URL @@ -107,7 +195,7 @@ jobs: publish: runs-on: ubuntu-22.04 permissions: write-all - needs: release + needs: [release, sdk-version] strategy: fail-fast: false matrix: @@ -123,20 +211,15 @@ jobs: suffix: "zip" type: "application/zip" steps: - - name: Get sdk version string - id: get_sdk_version - run: | - sdk_version=`echo "${{ github.ref }}" | cut -d "-" -f 3` - echo "sdk_version=$sdk_version" >> $GITHUB_OUTPUT - name: Download artifacts uses: actions/download-artifact@v4 with: - path: ./android-binaries-${{ steps.get_sdk_version.outputs.sdk_version }} + path: ./android-binaries-${{ needs.sdk-version.outputs.sdk_version }} merge-multiple: true pattern: ${{ matrix.config.artifact }}-* - name: Make release artifacts run: | - ${{ matrix.config.command }} android-binaries-${{ steps.get_sdk_version.outputs.sdk_version }}.${{ matrix.config.suffix }} android-binaries-${{ steps.get_sdk_version.outputs.sdk_version }} + ${{ matrix.config.command }} android-binaries-${{ needs.sdk-version.outputs.sdk_version }}.${{ matrix.config.suffix }} android-binaries-${{ needs.sdk-version.outputs.sdk_version }} - name: Download release URL uses: actions/download-artifact@v4 with: @@ -153,6 +236,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.set_upload_url.outputs.upload_url }} - asset_name: android-binaries-${{ steps.get_sdk_version.outputs.sdk_version }}.${{ matrix.config.suffix }} - asset_path: ./android-binaries-${{ steps.get_sdk_version.outputs.sdk_version }}.${{ matrix.config.suffix }} + asset_name: android-binaries-${{ needs.sdk-version.outputs.sdk_version }}.${{ matrix.config.suffix }} + asset_path: ./android-binaries-${{ needs.sdk-version.outputs.sdk_version }}.${{ matrix.config.suffix }} asset_content_type: ${{ matrix.config.type }} diff --git a/jitpack.yml b/jitpack.yml new file mode 100644 index 00000000000..66d6a310c51 --- /dev/null +++ b/jitpack.yml @@ -0,0 +1,17 @@ +# This file enables jitpack.io to build this repository into an AAR (Android +# Archive) for easier consumption by app developers. Instead of needing to +# download the VVL release artifacts from GitHub and check those libraries into +# their repository, they can instead depend on +# com.github.khronos:vulkan-validationlayers:$TAG. Jitpack will build the +# repository into an AAR and serve that artifact to developers. +# +# One caveat: if the VVL build is not completely deterministic (unlikely), the +# artifacts served from jitpack will not exactly match those hosted on the +# GitHub Release page, since jitpack will build the artifacts rather than serve +# the ones from the release. +# +# https://jitpack.io/docs/BUILDING/ +# https://github.com/KhronosGroup/Vulkan-ValidationLayers/issues/8167 +# https://github.com/KhronosGroup/Vulkan-ValidationLayers/pull/8303 +install: + - python scripts/jitpack.py diff --git a/scripts/android.py b/scripts/android.py index 7e59f4d95e0..1486f4993a2 100755 --- a/scripts/android.py +++ b/scripts/android.py @@ -22,6 +22,7 @@ import argparse import os +from pathlib import Path import sys import shutil import common_ci @@ -77,23 +78,14 @@ def generate_apk(SDK_ROOT : str, CMAKE_INSTALL_DIR : str) -> str: # https://en.wikipedia.org/wiki/Apk_(file_format)#Package_contents # # As a result CMake will need to be run multiple times to create a complete test APK that can be run on any Android device. -def main(): - configs = ['Release', 'Debug', 'MinSizeRel'] - - parser = argparse.ArgumentParser() - parser.add_argument('--config', type=str, choices=configs, default=configs[0]) - parser.add_argument('--app-abi', dest='android_abi', type=str, default="arm64-v8a") - parser.add_argument('--app-stl', dest='android_stl', type=str, choices=["c++_static", "c++_shared"], default="c++_static") - parser.add_argument('--apk', action='store_true', help='Generate an APK as a post build step.') - parser.add_argument('--clean', action='store_true', help='Cleans CMake build artifacts') - args = parser.parse_args() - - cmake_config = args.config - android_abis = args.android_abi.split(" ") - android_stl = args.android_stl - create_apk = args.apk - clean = args.clean - +def build( + cmake_config: str, + abis: list[str], + min_sdk_version: int, + stl: str, + create_apk: bool = False, + clean: bool = False, +) -> Path: if "ANDROID_NDK_HOME" not in os.environ: print("Cannot find ANDROID_NDK_HOME!") sys.exit(1) @@ -115,7 +107,7 @@ def main(): required_cli_tools += ['aapt', 'zipalign', 'keytool', 'apksigner'] print(f"ANDROID_NDK_HOME = {android_ndk_home}") - print(f"Build configured for {cmake_config} | {android_stl} | {android_abis} | APK {create_apk}") + print(f"Build configured for {cmake_config} | {stl} | {abis} | APK {create_apk}") if not os.path.isfile(android_toolchain): print(f'Unable to find android.toolchain.cmake at {android_toolchain}') @@ -136,7 +128,7 @@ def main(): print("Cleaning CMake install") shutil.rmtree(cmake_install_dir) - for abi in android_abis: + for abi in abis: build_dir = common_ci.RepoRelative(f'build-android/cmake/{abi}') lib_dir = f'lib/{abi}' @@ -157,9 +149,9 @@ def main(): cmake_cmd += f' -D CMAKE_ANDROID_ARCH_ABI={abi}' cmake_cmd += f' -D CMAKE_INSTALL_LIBDIR={lib_dir}' cmake_cmd += f' -D BUILD_TESTS={create_apk}' - cmake_cmd += f' -D CMAKE_ANDROID_STL_TYPE={android_stl}' + cmake_cmd += f' -D CMAKE_ANDROID_STL_TYPE={stl}' - cmake_cmd += ' -D ANDROID_PLATFORM=26' + cmake_cmd += f' -D ANDROID_PLATFORM={min_sdk_version}' cmake_cmd += ' -D ANDROID_USE_LEGACY_TOOLCHAIN_FILE=NO' common_ci.RunShellCmd(cmake_cmd) @@ -175,5 +167,30 @@ def main(): if create_apk: generate_apk(SDK_ROOT = android_sdk_root, CMAKE_INSTALL_DIR = cmake_install_dir) + return Path(cmake_install_dir) + + +def main() -> None: + + configs = ['Release', 'Debug', 'MinSizeRel'] + + parser = argparse.ArgumentParser() + parser.add_argument('--config', type=str, choices=configs, default=configs[0]) + parser.add_argument('--app-abi', dest='android_abi', type=str, default="arm64-v8a") + parser.add_argument('--min-sdk-version', type=int, default=26, help='The minSdkVersion of the built artifacts') + parser.add_argument('--app-stl', dest='android_stl', type=str, choices=["c++_static", "c++_shared"], default="c++_static") + parser.add_argument('--apk', action='store_true', help='Generate an APK as a post build step.') + parser.add_argument('--clean', action='store_true', help='Cleans CMake build artifacts') + args = parser.parse_args() + + build( + args.config, + args.android_abi.split(" "), + args.min_sdk_version, + args.android_sdk, + args.apk, + args.clean, + ) + if __name__ == '__main__': main() diff --git a/scripts/jitpack.py b/scripts/jitpack.py new file mode 100644 index 00000000000..fa86fe7967f --- /dev/null +++ b/scripts/jitpack.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2024 LunarG, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Entry point for jitpack.yml. + +This implements the custom build command used by the jitpack.yml in the top level of +this repo. See the documentation in that file for more information. +""" +import argparse +import json +import os +import sys +import textwrap +from collections.abc import Iterator +from pathlib import Path +from zipfile import ZipFile + +import android + + +def ndk_default_abis() -> Iterator[str]: + """Yields each default NDK ABI. + + The NDK includes a meta/abis.json file that enumerates each ABI the NDK is capable + of building for, and includes some descriptive data for each ABI. One of those + fields is "default", which will be true if the ABI is one that the NDK maintainers + recommend including support for by default. Most of the time all ABIs are + recommended to be built by default, but in the rare case where an ABI is under + development, or an old one is deprecated, we can key off that field to avoid them. + + At the time of writing (July 2024), the only non-default ABI is riscv64. The NDK r27 + changelog says that it's only there for bringup and shouldn't be used for release + artifacts yet. + """ + try: + ndk = Path(os.environ["ANDROID_NDK_HOME"]) + except KeyError: + sys.exit("ANDROID_NDK_HOME must be set in the environment") + + abis_meta_path = ndk / "meta/abis.json" + if not abis_meta_path.exists(): + sys.exit( + f"{abis_meta_path} does not exist. Does ANDROID_NDK_HOME ({ndk}) point to " + "a valid NDK?" + ) + + with abis_meta_path.open("r", encoding="utf-8") as abis_meta_file: + abis_meta = json.load(abis_meta_file) + + for name, data in abis_meta.items(): + if data["default"]: + yield name + + +def generate_aar( + output_path: Path, + library_dir: Path, + group_id: str, + artifact_id: str, + min_sdk_version: int, +) -> None: + """Creates an AAR from the CMake binaries.""" + with ZipFile(output_path, mode="w") as zip_file: + zip_file.writestr( + "AndroidManifest.xml", + textwrap.dedent( + f"""\ + + + + + """ + ), + ) + + for abi_dir in library_dir.iterdir(): + libs = list(abi_dir.glob("*.so")) + if not libs: + raise RuntimeError(f"No libraries found matching {abi_dir}/*.so") + for lib in libs: + zip_file.write(lib, arcname=f"jni/{abi_dir.name}/{lib.name}") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--group-id", + help="The group ID of the AAR that will be published", + ) + parser.add_argument( + "--artifact-id", + help="The artifact ID of the AAR that will be published", + ) + parser.add_argument( + "--min-sdk-version", + type=int, + default=26, + help="The minSdkVersion of the built artifacts", + ) + parser.add_argument( + "-o", + "--output", + required=True, + type=Path, + help="Output file name", + ) + args = parser.parse_args() + + build_dir = android.build( + "Release", list(ndk_default_abis()), args.min_sdk_version, "c++_static" + ) + + generate_aar( + args.output, + build_dir / "lib", + args.group_id, + args.artifact_id, + args.min_sdk_version, + ) + + +if __name__ == "__main__": + main()