Other OS Build #58
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # OpenMLDB build on other os, only trigger by dispatch event and only upload artifacts, not release. | |
| # You should download artifacts manually. And we won't do test in this workflow. | |
| name: Other OS Build | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| OS: | |
| description: 'os name: centos6, macos11, macos10' | |
| required: true | |
| default: 'centos6' | |
| JAVA_SDK: | |
| description: 'java sdk enable: ON, OFF' | |
| required: true | |
| default: 'ON' | |
| PYTHON_SDK: | |
| description: 'python sdk enable: ON, OFF' | |
| required: true | |
| default: 'ON' | |
| REF: | |
| description: 'The branch, tag or SHA to checkout, otherwise use the branch' | |
| required: false | |
| default: '' | |
| env: | |
| NPROC: 2 # default Parallel build number for GitHub's Linux runner | |
| EXAMPLES_ENABLE: OFF # turn off hybridse's example code | |
| HYBRIDSE_TESTING_ENABLE: OFF # turn off hybridse's test code | |
| jobs: | |
| centos6: # glibc version is 2.12, glibcxx version is 3.4.13 | |
| if: ${{ github.event.inputs.OS == 'centos6' }} | |
| runs-on: ubuntu-latest | |
| env: | |
| OS: linux | |
| SQL_JAVASDK_ENABLE: ${{ github.event.inputs.JAVA_SDK }} | |
| SQL_PYSDK_ENABLE: ${{ github.event.inputs.PYTHON_SDK }} # python whl will be built when make, no prerequirement | |
| TESTING_ENABLE: OFF | |
| NPROC: 8 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.inputs.REF }} | |
| submodules: true | |
| - name: prepare release | |
| run: | | |
| VERSION="snapshot" | |
| TAG=$(git tag -l | grep $(git describe HEAD)) | |
| if [[ "${TAG}" == "v"* ]]; then | |
| VERSION=${TAG#v} | |
| bash steps/prepare_release.sh "$VERSION" | |
| bash java/prepare_release.sh "$VERSION" | |
| fi | |
| echo "OPENMLDB_PREFIX=openmldb-$VERSION-${{ env.OS }}" >> $GITHUB_ENV | |
| - name: Cache thirdparty | |
| id: deps-cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| .deps/ | |
| key: centos6-thirdparty-${{ hashFiles('third-party/**/CMakeLists.txt', 'third-party/**/*.cmake', 'third-party/**/*.sh') }} | |
| # use docker in a step, not in a job, cuz we want use higher version actions(cache/upload...), centos6's nodejs is too old | |
| # use raw cmake to build main, avoid thirdparty build again and parallel limit | |
| # thirdparty -j8 is ok | |
| # build -j2 is ok, >=4 will oom | |
| # No space left on device when java sdk on, python sdk is small, rm /depends 400M, not enough, | |
| # so we disable java when build binary(+python) and then build java without binary build(target sql_javasdk_package, build/ cost ~6.2G, java/ cost ~4.6G) | |
| # P.S. we can package only openmldb-native, but it's better to build the whole, to avoid code error | |
| # if you want to download other jars, upload them | |
| # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources | |
| - uses: addnab/docker-run-action@v3 | |
| with: | |
| image: ghcr.io/4paradigm/centos6_gcc7_hybridsql | |
| options: -v ${{ github.workspace }}:/root/OpenMLDB -e USE_DEPS_CACHE=${{ steps.deps-cache.outputs.cache-hit }} -e OPENMLDB_PREFIX=${{ env.OPENMLDB_PREFIX }} -e SQL_PYSDK_ENABLE=${{ env.SQL_PYSDK_ENABLE }} -e SQL_JAVASDK_ENABLE=${{ env.SQL_JAVASDK_ENABLE }} | |
| shell: bash | |
| run: | | |
| cd /root/OpenMLDB | |
| # centos6_build.sh will try build zetasql even cache hit, just ignore the failure | |
| IN_WORKFLOW=true bash steps/centos6_build.sh | |
| # bazel bin | |
| export PATH=$PATH:`pwd` | |
| source /opt/rh/devtoolset-8/enable | |
| if [[ "${USE_DEPS_CACHE}" != "true" ]]; then | |
| echo "build thirdparty, make opt is better than nproc?" | |
| make thirdparty CMAKE_INSTALL_PREFIX=${OPENMLDB_PREFIX} BUILD_BUNDLE=ON THIRD_PARTY_CMAKE_FLAGS=-DMAKEOPTS=-j8 | |
| # 5.8G ./.deps, avail 8G | |
| rm -rf .deps/build # GitHub runner disk space is limited | |
| fi | |
| echo "build" | |
| # 1.4G ./.deps, avail 13G | |
| # will failed if openmldb_sdk is on | |
| cmake -S . -B `pwd`/build -DCMAKE_PREFIX_PATH=`pwd`/.deps/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo \ | |
| -DSQL_PYSDK_ENABLE=${SQL_PYSDK_ENABLE} -DSQL_JAVASDK_ENABLE=OFF \ | |
| -DTESTING_ENABLE=OFF -DCMAKE_INSTALL_PREFIX=${OPENMLDB_PREFIX} \ | |
| -DHYBRIDSE_TESTING_ENABLE=OFF -DEXAMPLES_ENABLE=OFF -DEXAMPLES_TESTING_ENABLE=OFF | |
| # target openmldb 6.7G ./build(no py/java), avail 5.2G | |
| # openmldb+cp_python_sdk_so 7.7G ./build(has py), python just ~180M | |
| # target 'install' cost more, preinstall/fast won't build all, so use install/fast if needed | |
| # or https://cmake.org/cmake/help/latest/variable/CMAKE_SKIP_INSTALL_ALL_DEPENDENCY.html | |
| cmake --build build --target openmldb cp_python_sdk_so -- -j2 | |
| du -h --max-depth=1 | |
| df -h | |
| # if target above cost too much disk, make java build failed, try to rm build cache | |
| # don't rm cache now cuz build java from emtpy will cost 20min | |
| # rm build/hybridse build/src -rf | |
| if [[ "${SQL_JAVASDK_ENABLE}" == "ON" ]]; then | |
| echo "build java sdk" | |
| cmake -S . -B `pwd`/build -DCMAKE_PREFIX_PATH=`pwd`/.deps/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo \ | |
| -DSQL_PYSDK_ENABLE=OFF -DSQL_JAVASDK_ENABLE=ON \ | |
| -DTESTING_ENABLE=OFF -DCMAKE_INSTALL_PREFIX=${OPENMLDB_PREFIX} \ | |
| -DHYBRIDSE_TESTING_ENABLE=OFF -DEXAMPLES_ENABLE=OFF -DEXAMPLES_TESTING_ENABLE=OFF | |
| # if build the whole java, 7.6G ./build, 5.7G ./java, avail 331M | |
| # so split it and build native only | |
| # 7.6G ./build, 1.8G ./java, avail 5.2G | |
| cmake --build build --target cp_native_so -- -j2 | |
| du -h --max-depth=1 | |
| df -h | |
| rm build/hybridse build/src -rf | |
| cd java | |
| ./mvnw -pl openmldb-native clean package -DskipTests=true -Dscalatest.skip=true -Dwagon.skip=true -Dmaven.test.skip=true --batch-mode | |
| fi | |
| rm build/hybridse build/src -rf | |
| du -h --max-depth=1 | |
| df -h | |
| - name: upload binary | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| path: build/bin/openmldb | |
| name: binary | |
| - name: upload java native | |
| if: ${{ env.SQL_JAVASDK_ENABLE == 'ON' }} | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: native-jar | |
| path: java/openmldb-native/target/openmldb-native-*.jar | |
| - name: upload python whl | |
| if: ${{ env.SQL_PYSDK_ENABLE == 'ON' }} | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: python-whl | |
| path: | | |
| python/openmldb_sdk/dist/openmldb*.whl | |
| # TODO(hw): upload cxx sdk | |
| # macos no need to build thirdparty, but binary/os needs to be built on each os | |
| macos-11: | |
| if: ${{ github.event.inputs.OS == 'macos11' }} | |
| runs-on: macos-11 | |
| env: | |
| OS: darwin | |
| # if macos, set VARIANT_TYPE=macos | |
| VARIANT_TYPE: macos | |
| SQL_JAVASDK_ENABLE: ${{ github.event.inputs.JAVA_SDK }} | |
| SQL_PYSDK_ENABLE: ${{ github.event.inputs.PYTHON_SDK }} # python whl will be built when make, no prerequirement | |
| TESTING_ENABLE: OFF | |
| ARCH: x86_64 | |
| NPROC: 3 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.inputs.REF }} | |
| submodules: true | |
| # it's ok to prepare version even java sdk disabled | |
| - name: prepare release | |
| run: | | |
| VERSION="snapshot" | |
| # use repo ref not event ref | |
| TAG=$(git tag -l | grep $(git describe HEAD)) | |
| echo $TAG | |
| if [[ "${TAG}" == "v"* ]]; then | |
| echo "set version with tag ${TAG}" | |
| VERSION=${TAG#v} | |
| bash steps/prepare_release.sh "$VERSION" | |
| bash java/prepare_release.sh "$VERSION" | |
| fi | |
| echo "OPENMLDB_PREFIX=openmldb-$VERSION-${{ env.OS }}" >> $GITHUB_ENV | |
| - name: build | |
| run: | | |
| make build CMAKE_INSTALL_PREFIX=${{ env.OPENMLDB_PREFIX }} | |
| # GitHub runner disk space is limited | |
| # delete thirdparty build directory($ROOT/.deps/) to save disk space | |
| # make thirdpartybuild-clean | |
| - name: install | |
| run: | | |
| make install | |
| - name: package | |
| run: | | |
| tar czf ${{ env.OPENMLDB_PREFIX }}.tar.gz ${{ env.OPENMLDB_PREFIX }}/ | |
| - name: upload binary | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| path: openmldb-*.tar.gz | |
| name: binary-package | |
| - name: upload java native | |
| if: ${{ env.SQL_JAVASDK_ENABLE == 'ON' }} | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: native-jar | |
| path: java/openmldb-native/target/openmldb-native-*.jar | |
| - name: upload python whl | |
| if: ${{ env.SQL_PYSDK_ENABLE == 'ON' }} | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: python-whl | |
| path: | | |
| python/openmldb_sdk/dist/openmldb*.whl | |
| macos-10: | |
| if: ${{ github.event.inputs.OS == 'macos10' }} | |
| runs-on: macos-11 | |
| env: | |
| OS: darwin | |
| # if macos, set VARIANT_TYPE=macos | |
| VARIANT_TYPE: macos | |
| SQL_JAVASDK_ENABLE: ${{ github.event.inputs.JAVA_SDK }} | |
| SQL_PYSDK_ENABLE: ${{ github.event.inputs.PYTHON_SDK }} # python whl will be built when make, no prerequirement | |
| TESTING_ENABLE: OFF | |
| ARCH: x86_64 | |
| NPROC: 3 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.inputs.REF }} | |
| submodules: true | |
| # ref https://github.com/actions/runner-images/blob/main/images/macos/macos-11-Readme.md macOS 10.15 macosx10.15 11.7 | macOS 11.1 macosx11.1 12.4 | |
| - name: xcode select | |
| uses: maxim-lobanov/setup-xcode@v1 | |
| with: | |
| xcode-version: '11.7' | |
| # it's ok to prepare version even java sdk disabled | |
| - name: prepare release | |
| run: | | |
| VERSION="snapshot" | |
| TAG=$(git tag -l | grep $(git describe HEAD)) | |
| if [[ "${TAG}" == "v"* ]]; then | |
| VERSION=${TAG#v} | |
| bash steps/prepare_release.sh "$VERSION" | |
| bash java/prepare_release.sh "$VERSION" | |
| fi | |
| echo "OPENMLDB_PREFIX=openmldb-$VERSION-${{ env.OS }}" >> $GITHUB_ENV | |
| - name: build | |
| run: | | |
| make build CMAKE_INSTALL_PREFIX=${{ env.OPENMLDB_PREFIX }} | |
| # GitHub runner disk space is limited | |
| # delete thirdparty build directory($ROOT/.deps/) to save disk space | |
| # make thirdpartybuild-clean | |
| - name: install | |
| run: | | |
| make install | |
| - name: package | |
| run: | | |
| tar czf ${{ env.OPENMLDB_PREFIX }}.tar.gz ${{ env.OPENMLDB_PREFIX }}/ | |
| - name: upload binary | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| path: openmldb-*.tar.gz | |
| name: binary-package | |
| - name: upload java native | |
| if: ${{ env.SQL_JAVASDK_ENABLE == 'ON' }} | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: native-jar | |
| path: java/openmldb-native/target/openmldb-native-*.jar | |
| - name: upload python whl | |
| if: ${{ env.SQL_PYSDK_ENABLE == 'ON' }} | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: python-whl | |
| path: | | |
| python/openmldb_sdk/dist/openmldb*.whl |