From 98c805a71a13dbbb3513a4f55204fcf87a545f64 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 8 Feb 2024 12:41:17 -0800 Subject: [PATCH] Move macos jobs to gha (#8707) Summary: I removed the caching of homebrew because `brew install` only takes ~1 min so we don't need to spend the cache space for that. I tried using brew folly but that fails due to what looks like [int128 not being turned on](https://github.com/assignUser/velox/actions/runs/7824878934/job/21348161704#step:6:3453)? Bundled folly solves this and will be cached by ccache. Building and caching it separately will have the same cache misses as ccache so there is no advantage to that. Currently these jobs run on the public macos 13 (intel) and 14 (arm) runners which makes this very economical. A build without cache takes 90 and 40 mins respectively. On main we should always have a cache hit and in PRs we should also benefit from the caches on main via partial key matching (`restore-keys`). If we want to improve build times we can of course upgrade to the larger runners at any time. With the current rate of Apple Silicon adoption we could also think about dropping intel mac CI entirely (outside of pyvelox wheels if we publish those at some point). Closes https://github.com/facebookincubator/velox/issues/8547 Pull Request resolved: https://github.com/facebookincubator/velox/pull/8707 Reviewed By: Yuhta Differential Revision: D53577489 Pulled By: kgpai fbshipit-source-id: b9f965d1398c6e8a197dffef7f8d7f46e986dbae --- .circleci/dist_compile.yml | 101 ------------------------------------ .github/workflows/macos.yml | 81 +++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 101 deletions(-) create mode 100644 .github/workflows/macos.yml diff --git a/.circleci/dist_compile.yml b/.circleci/dist_compile.yml index 3278a67f6565..94a71eab4f7e 100644 --- a/.circleci/dist_compile.yml +++ b/.circleci/dist_compile.yml @@ -138,85 +138,8 @@ executors: check: docker: - image : ghcr.io/facebookincubator/velox-dev:check-avx - macos-intel: - macos: - xcode: "14.3.0" - resource_class: macos.x86.medium.gen2 - macos-m1: - macos: - xcode: "14.2.0" - resource_class: macos.m1.large.gen1 jobs: - macos-build: - parameters: - os: - type: executor - executor: << parameters.os >> - environment: - ICU_SOURCE: BUNDLED - simdjson_SOURCE: BUNDLED - steps: - - checkout - - update-submodules - - restore_cache: - name: "Restore Dependency Cache" - # The version number in the key can be incremented - # to manually avoid the case where bad dependencies - # are cached, and has no other meaning. - # If you update it, be sure to update save_cache too. - key: velox-circleci-macos-{{ arch }}-deps-v1-{{ checksum ".circleci/config.yml" }}-{{ checksum "scripts/setup-macos.sh" }} - - run: - name: "Install dependencies" - command: | - set -xu - mkdir -p ~/deps ~/deps-src - curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C ~/deps - PATH=~/deps/bin:${PATH} DEPENDENCY_DIR=~/deps-src INSTALL_PREFIX=~/deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-macos.sh - rm -rf ~/deps/.git ~/deps/Library/Taps/ # Reduce cache size by 70%. - no_output_timeout: 20m - - save_cache: - name: "Save Dependency Cache" - # The version number in the key can be incremented - # to manually avoid the case where bad dependencies - # are cached, and has no other meaning. - # If you update it, be sure to update restore_cache too. - key: velox-circleci-macos-{{ arch }}-deps-v1-{{ checksum ".circleci/config.yml" }}-{{ checksum "scripts/setup-macos.sh" }} - paths: - - ~/deps - - run: - name: "Calculate merge-base date for CCache" - command: git show -s --format=%cd --date="format:%Y%m%d" $(git merge-base origin/main HEAD) | tee merge-base-date - - restore_cache: - name: "Restore CCache cache" - keys: - - velox-ccache-debug-{{ arch }}-{{ checksum "merge-base-date" }} - - run: - name: "Build on MacOS" - command: | - export PATH=~/deps/bin:~/deps/opt/bison/bin:~/deps/opt/flex/bin:${PATH} - mkdir -p .ccache - export CCACHE_DIR=$(pwd)/.ccache - ccache -sz -M 5Gi - cmake \ - -B _build/debug \ - -GNinja \ - -DTREAT_WARNINGS_AS_ERRORS=1 \ - -DENABLE_ALL_WARNINGS=1 \ - -DVELOX_ENABLE_PARQUET=ON \ - -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_PREFIX_PATH=~/deps \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DFLEX_INCLUDE_DIR=~/deps/opt/flex/include - ninja -C _build/debug - ccache -s - no_output_timeout: 1h - - save_cache: - name: "Save CCache cache" - key: velox-ccache-debug-{{ arch }}-{{ checksum "merge-base-date" }} - paths: - - .ccache/ - linux-build: executor: build environment: @@ -681,10 +604,6 @@ workflows: - linux-build-options - linux-adapters - linux-presto-fuzzer-run - - macos-build: - matrix: - parameters: - os: [macos-intel] - format-check - header-check - doc-gen-job: @@ -692,14 +611,6 @@ workflows: branches: only: - main - - macos-build: - matrix: - parameters: - os: [ macos-m1 ] - filters: - branches: - only: - - main shorter-fuzzer: unless: << pipeline.parameters.run-longer-expression-fuzzer >> @@ -708,10 +619,6 @@ workflows: - linux-pr-fuzzer-run - linux-build-options - linux-adapters - - macos-build: - matrix: - parameters: - os: [ macos-intel ] - format-check - header-check - doc-gen-job: @@ -719,11 +626,3 @@ workflows: branches: only: - main - - macos-build: - matrix: - parameters: - os: [ macos-m1 ] - filters: - branches: - only: - - main diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml new file mode 100644 index 000000000000..7c6c87661cd7 --- /dev/null +++ b/.github/workflows/macos.yml @@ -0,0 +1,81 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: macOS Build + +on: + push: + pull_request: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.repository }}-${{ github.head_ref || github.sha }} + cancel-in-progress: true + +jobs: + macos-build: + name: "${{ matrix.os }}" + strategy: + fail-fast: false + matrix: + # macos-13 = x86_64 Mac + # macos-14 = arm64 Mac + os: [macos-13, macos-14] + runs-on: ${{ matrix.os }} + env: + CCACHE_DIR: '${{ github.workspace }}/.ccache' + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install Dependencies + run: | + brew install \ + bison boost ccache double-conversion flex fmt gflags glog \ + icu4c libevent libsodium lz4 lzo ninja openssl range-v3 simdjson \ + snappy thrift xz xsimd zstd + + echo "NJOBS=`sysctl -n hw.ncpu`" >> $GITHUB_ENV + + - name: Cache ccache + uses: actions/cache@v4 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-macos-${{ matrix.os }}-${{ hashFiles('velox/*') }} + restore-keys: ccache-macos-${{ matrix.os }} + + - name: Configure Build + env: + folly_SOURCE: BUNDLED + run: | + ccache -sz -M 5Gi + cmake \ + -B _build/debug \ + -GNinja \ + -DTREAT_WARNINGS_AS_ERRORS=1 \ + -DENABLE_ALL_WARNINGS=1 \ + -DVELOX_ENABLE_PARQUET=ON \ + -DCMAKE_BUILD_TYPE=Debug + + - name: Build + run: | + cmake --build _build/debug -j $NJOBS + ccache -s + - name: Run Tests + if: false + run: ctest -j $NJOBS --test-dir _build/debug --output-on-failure +