Other OS Build #73
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # OpenMLDB build on other os, only trigger by dispatch event and only upload artifacts, not release. | |
| # You should download artifacts manually. And we won't do test in this workflow. | |
| name: Other OS Build | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| OS: | |
| description: 'os name: centos6, macos11, macos10' | |
| required: true | |
| default: 'centos6' | |
| JAVA_SDK: | |
| description: 'java sdk enable: ON, OFF' | |
| required: true | |
| default: 'ON' | |
| PYTHON_SDK: | |
| description: 'python sdk enable: ON, OFF' | |
| required: true | |
| default: 'ON' | |
| REF: | |
| description: 'The branch, tag or SHA to checkout, otherwise use the branch' | |
| required: false | |
| default: '' | |
| ZETASQL_LIB_URL: | |
| description: 'zetasql lib url, default is built by src, if you want to use prebuilt, set it. Only for centos6' | |
| required: false | |
| default: '' | |
| env: | |
| NPROC: 2 # default Parallel build number for GitHub's Linux runner | |
| EXAMPLES_ENABLE: OFF # turn off hybridse's example code | |
| HYBRIDSE_TESTING_ENABLE: OFF # turn off hybridse's test code | |
| jobs: | |
| centos6: # glibc version is 2.12, glibcxx version is 3.4.13 | |
| if: ${{ github.event.inputs.OS == 'centos6' }} | |
| runs-on: ubuntu-latest | |
| env: | |
| OS: linux | |
| SQL_JAVASDK_ENABLE: ${{ github.event.inputs.JAVA_SDK }} | |
| SQL_PYSDK_ENABLE: ${{ github.event.inputs.PYTHON_SDK }} # python whl will be built when make, no prerequirement | |
| ZETASQL_LIB_URL: ${{ github.event.inputs.ZETASQL_LIB_URL }} | |
| TESTING_ENABLE: OFF | |
| NPROC: 8 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.inputs.REF }} | |
| submodules: true | |
| - name: prepare release | |
| run: | | |
| VERSION="snapshot" | |
| # if not tag, return error, just ignore it to use snapshot as the version | |
| TAG=$(git tag -l | grep $(git describe HEAD)) || true | |
| if [[ "${TAG}" == "v"* ]]; then | |
| VERSION=${TAG#v} | |
| bash steps/prepare_release.sh "$VERSION" | |
| bash java/prepare_release.sh "$VERSION" | |
| fi | |
| echo "OPENMLDB_PREFIX=openmldb-$VERSION-${{ env.OS }}" >> $GITHUB_ENV | |
| - name: Cache thirdparty | |
| id: deps-cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| .deps/ | |
| key: centos6-thirdparty-${{ hashFiles('third-party/**/CMakeLists.txt', 'third-party/**/*.cmake', 'third-party/**/*.sh') }} | |
| # use docker in a step, not in a job, cuz we want use higher version actions(cache/upload...), centos6's nodejs is too old | |
| # use raw cmake to build main, avoid thirdparty build again and parallel limit | |
| # thirdparty -j8 is ok | |
| # build -j2 is ok, >=4 will oom | |
| # No space left on device when java sdk on, python sdk is small, rm /depends 400M, not enough, | |
| # so we disable java when build binary(+python) and then build java without binary build(target sql_javasdk_package, build/ cost ~6.2G, java/ cost ~4.6G) | |
| # P.S. we can package only openmldb-native, but it's better to build the whole, to avoid code error | |
| # if you want to download other jars, upload them | |
| # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources | |
| - uses: addnab/docker-run-action@v3 | |
| with: | |
| image: ghcr.io/4paradigm/centos6_gcc7_hybridsql | |
| options: -v ${{ github.workspace }}:/root/OpenMLDB -e USE_DEPS_CACHE=${{ steps.deps-cache.outputs.cache-hit }} -e OPENMLDB_PREFIX=${{ env.OPENMLDB_PREFIX }} -e SQL_PYSDK_ENABLE=${{ env.SQL_PYSDK_ENABLE }} -e SQL_JAVASDK_ENABLE=${{ env.SQL_JAVASDK_ENABLE }} -e ZETASQL_LIB_URL=${{ env.ZETASQL_LIB_URL }} | |
| shell: bash | |
| run: | | |
| cd /root/OpenMLDB | |
| # centos6_build.sh won't try build zetasql when cache hit(USE_DEPS_CACHE==true) | |
| IN_WORKFLOW=true bash steps/centos6_build.sh | |
| # bazel bin | |
| export PATH=$PATH:`pwd` | |
| source /opt/rh/devtoolset-8/enable | |
| if [[ "${USE_DEPS_CACHE}" != "true" ]]; then | |
| echo "build thirdparty, configure in centos6_build, make opt is better than nproc" | |
| cmake --build $(pwd)/.deps | |
| # 5.8G ./.deps, avail 8G | |
| rm -rf .deps/build # GitHub runner disk space is limited | |
| fi | |
| if [[ "${ZETASQL_LIB_URL}" != "" ]]; then | |
| echo "replace zetasql lib by ${ZETASQL_LIB_URL}" | |
| curl -SLo .deps/usr/lib/libzetasql.a ${ZETASQL_LIB_URL} | |
| fi | |
| echo "build openmldb" | |
| # 1.4G ./.deps, avail 13G | |
| # will failed if openmldb_sdk is on | |
| cmake -S . -B `pwd`/build -DCMAKE_PREFIX_PATH=`pwd`/.deps/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo \ | |
| -DSQL_PYSDK_ENABLE=${SQL_PYSDK_ENABLE} -DSQL_JAVASDK_ENABLE=OFF \ | |
| -DTESTING_ENABLE=OFF -DCMAKE_INSTALL_PREFIX=${OPENMLDB_PREFIX} \ | |
| -DHYBRIDSE_TESTING_ENABLE=OFF -DEXAMPLES_ENABLE=OFF -DEXAMPLES_TESTING_ENABLE=OFF | |
| # target openmldb 6.7G ./build(no py/java), avail 5.2G | |
| # openmldb+cp_python_sdk_so 7.7G ./build(has py), python just ~180M | |
| # target 'install' cost more, preinstall/fast won't build all, so use install/fast if needed | |
| # or https://cmake.org/cmake/help/latest/variable/CMAKE_SKIP_INSTALL_ALL_DEPENDENCY.html | |
| cmake --build build --target openmldb cp_python_sdk_so -- -j2 | |
| du -h --max-depth=1 | |
| df -h | |
| # if target above cost too much disk, make java build failed, try to rm build cache | |
| # don't rm cache now cuz build java from emtpy will cost 20min | |
| # rm build/hybridse build/src -rf | |
| if [[ "${SQL_JAVASDK_ENABLE}" == "ON" ]]; then | |
| echo "build java sdk" | |
| cmake -S . -B `pwd`/build -DCMAKE_PREFIX_PATH=`pwd`/.deps/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo \ | |
| -DSQL_PYSDK_ENABLE=OFF -DSQL_JAVASDK_ENABLE=ON \ | |
| -DTESTING_ENABLE=OFF -DCMAKE_INSTALL_PREFIX=${OPENMLDB_PREFIX} \ | |
| -DHYBRIDSE_TESTING_ENABLE=OFF -DEXAMPLES_ENABLE=OFF -DEXAMPLES_TESTING_ENABLE=OFF | |
| # if build the whole java, 7.6G ./build, 5.7G ./java, avail 331M | |
| # so split it and build native only | |
| # 7.6G ./build, 1.8G ./java, avail 5.2G | |
| cmake --build build --target cp_native_so -- -j2 | |
| du -h --max-depth=1 | |
| df -h | |
| rm build/hybridse build/src -rf | |
| cd java | |
| ./mvnw -pl openmldb-native clean package -DskipTests=true -Dscalatest.skip=true -Dwagon.skip=true -Dmaven.test.skip=true --batch-mode | |
| fi | |
| rm build/hybridse build/src -rf | |
| du -h --max-depth=1 | |
| df -h | |
| # - name: tar thirdparty | |
| # run: | | |
| # cd .deps | |
| # tar czf usr.tar.gz usr/ | |
| - name: upload built thirdparty | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| path: .deps/usr/lib/libzetasql.a | |
| name: built-thirdparty | |
| - name: upload binary | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| path: build/bin/openmldb | |
| name: binary | |
| - name: upload java native | |
| if: ${{ env.SQL_JAVASDK_ENABLE == 'ON' }} | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: native-jar | |
| path: java/openmldb-native/target/openmldb-native-*.jar | |
| - name: upload python whl | |
| if: ${{ env.SQL_PYSDK_ENABLE == 'ON' }} | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: python-whl | |
| path: | | |
| python/openmldb_sdk/dist/openmldb*.whl | |
| # TODO(hw): upload cxx sdk | |
| # macos no need to build thirdparty, but binary/os needs to be built on each os | |
| macos-11: | |
| if: ${{ github.event.inputs.OS == 'macos11' }} | |
| runs-on: macos-11 | |
| env: | |
| OS: darwin | |
| # if macos, set VARIANT_TYPE=macos | |
| VARIANT_TYPE: macos | |
| SQL_JAVASDK_ENABLE: ${{ github.event.inputs.JAVA_SDK }} | |
| SQL_PYSDK_ENABLE: ${{ github.event.inputs.PYTHON_SDK }} # python whl will be built when make, no prerequirement | |
| TESTING_ENABLE: OFF | |
| ARCH: x86_64 | |
| NPROC: 3 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.inputs.REF }} | |
| submodules: true | |
| # it's ok to prepare version even java sdk disabled | |
| - name: prepare release | |
| run: | | |
| VERSION="snapshot" | |
| # use repo ref not event ref | |
| TAG=$(git tag -l | grep $(git describe HEAD)) || true | |
| if [[ "${TAG}" == "v"* ]]; then | |
| VERSION=${TAG#v} | |
| bash steps/prepare_release.sh "$VERSION" | |
| bash java/prepare_release.sh "$VERSION" | |
| fi | |
| echo "OPENMLDB_PREFIX=openmldb-$VERSION-${{ env.OS }}" >> $GITHUB_ENV | |
| - name: build | |
| run: | | |
| make build CMAKE_INSTALL_PREFIX=${{ env.OPENMLDB_PREFIX }} | |
| # GitHub runner disk space is limited | |
| # delete thirdparty build directory($ROOT/.deps/) to save disk space | |
| # make thirdpartybuild-clean | |
| - name: install | |
| run: | | |
| make install | |
| - name: package | |
| run: | | |
| tar czf ${{ env.OPENMLDB_PREFIX }}.tar.gz ${{ env.OPENMLDB_PREFIX }}/ | |
| - name: upload binary | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| path: openmldb-*.tar.gz | |
| name: binary-package | |
| - name: upload java native | |
| if: ${{ env.SQL_JAVASDK_ENABLE == 'ON' }} | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: native-jar | |
| path: java/openmldb-native/target/openmldb-native-*.jar | |
| - name: upload python whl | |
| if: ${{ env.SQL_PYSDK_ENABLE == 'ON' }} | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: python-whl | |
| path: | | |
| python/openmldb_sdk/dist/openmldb*.whl | |
| macos-10: | |
| if: ${{ github.event.inputs.OS == 'macos10' }} | |
| runs-on: macos-11 | |
| env: | |
| OS: darwin | |
| # if macos, set VARIANT_TYPE=macos | |
| VARIANT_TYPE: macos | |
| SQL_JAVASDK_ENABLE: ${{ github.event.inputs.JAVA_SDK }} | |
| SQL_PYSDK_ENABLE: ${{ github.event.inputs.PYTHON_SDK }} # python whl will be built when make, no prerequirement | |
| TESTING_ENABLE: OFF | |
| ARCH: x86_64 | |
| NPROC: 3 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.inputs.REF }} | |
| submodules: true | |
| # ref https://github.com/actions/runner-images/blob/main/images/macos/macos-11-Readme.md macOS 10.15 macosx10.15 11.7 | macOS 11.1 macosx11.1 12.4 | |
| - name: xcode select | |
| uses: maxim-lobanov/setup-xcode@v1 | |
| with: | |
| xcode-version: '11.7' | |
| # it's ok to prepare version even java sdk disabled | |
| - name: prepare release | |
| run: | | |
| VERSION="snapshot" | |
| TAG=$(git tag -l | grep $(git describe HEAD)) || true | |
| if [[ "${TAG}" == "v"* ]]; then | |
| VERSION=${TAG#v} | |
| bash steps/prepare_release.sh "$VERSION" | |
| bash java/prepare_release.sh "$VERSION" | |
| fi | |
| echo "OPENMLDB_PREFIX=openmldb-$VERSION-${{ env.OS }}" >> $GITHUB_ENV | |
| - name: build | |
| run: | | |
| make build CMAKE_INSTALL_PREFIX=${{ env.OPENMLDB_PREFIX }} | |
| # GitHub runner disk space is limited | |
| # delete thirdparty build directory($ROOT/.deps/) to save disk space | |
| # make thirdpartybuild-clean | |
| - name: install | |
| run: | | |
| make install | |
| - name: package | |
| run: | | |
| tar czf ${{ env.OPENMLDB_PREFIX }}.tar.gz ${{ env.OPENMLDB_PREFIX }}/ | |
| - name: upload binary | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| path: openmldb-*.tar.gz | |
| name: binary-package | |
| - name: upload java native | |
| if: ${{ env.SQL_JAVASDK_ENABLE == 'ON' }} | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: native-jar | |
| path: java/openmldb-native/target/openmldb-native-*.jar | |
| - name: upload python whl | |
| if: ${{ env.SQL_PYSDK_ENABLE == 'ON' }} | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: python-whl | |
| path: | | |
| python/openmldb_sdk/dist/openmldb*.whl |