Skip to content
Build and Test

Flexibility for docker builds #1214

Sign in to view logs

Flexibility for docker builds

Flexibility for docker builds #1214

Workflow file for this run

.github/workflows/github-ci.yml at 9379a85
# Copyright (C) 2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
name: Build and Test
on:
# By default this will run when the activity type is "opened", "synchronize",
# or "reopened".
pull_request:
branches:
- main
# Run when protected branches are pushed to, e.g. via merge
push:
branches:
- main
# Manually run this workflow on any specified branch.
workflow_dispatch:
################
# Ubuntu 22.04 #
################
jobs:
format:
name: Format
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
# Required for pre-commit
- run: pip3 install bandit black cpplint docker mypy pydantic==1.10.4 pylint pylint-pydantic types-toml
- run: sudo snap install shfmt
- run: sudo apt install shellcheck
# NOTE: This is deprecated in favor of pre-commit.ci
- uses: pre-commit/[email protected]
with:
extra_args: --all-files
test-hekit-bash:
name: Test hekit commands (bash)
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
- name: Setup
run: |
pip3 install docker pytest pytest-mock pytest-xdist toml
echo "$HOME/.local/bin" >> $GITHUB_PATH
- name: Run pytests
run: pytest tests/ -n 8 --dist loadgroup
- name: hekit init
run: |
export SHELL=/bin/bash
./hekit init <<< y
source /home/runner/.bash_profile
cd .. && hekit -h
test-hekit-zshell:
name: Test hekit commands (zshell)
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
- name: Setup
run: |
pip3 install docker pytest pytest-mock pytest-xdist toml
sudo apt install zsh -y
touch ~/.zshrc
echo "$HOME/.local/bin" >> $GITHUB_PATH
- name: Run pytests
shell: zsh {0}
run: pytest tests/ -n 8 --dist loadgroup
- name: hekit init
shell: zsh {0}
run: |
export SHELL=/bin/zsh
./hekit --debug init <<< y
source /home/runner/.zshrc
cd .. && hekit -h
build-and-test-toolkit-palisade:
name: HEKit PALISADE | ${{ matrix.os }} ${{ matrix.c_compiler }} py${{ matrix.python-version }}
# Disable this job for now as it fails due to HEXL-PALISADE issue
# (fixed in HEXL 1.2.5 re-enable when we can update the HEXL version)
if: ${{ false }}
runs-on: ${{ matrix.os }}
defaults:
run:
shell: bash
strategy:
matrix:
# python3.10 is default on Ubuntu-22.04
python-version: ["3.10"]
c_compiler: [clang-14, gcc-11]
os: [ubuntu-22.04]
include:
# pairing cc compiler with relevant cxx compiler
# clang-14 and gcc-11 default on Ubuntu-22.04
- c_compiler: clang-14
cxx_compiler: clang++-14
- c_compiler: gcc-11
cxx_compiler: g++-11
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Setup
run: |
pip3 install toml
echo "$HOME/.local/bin" >> $GITHUB_PATH
echo "CC=${{matrix.c_compiler}}" >> $GITHUB_ENV
echo "CXX=${{matrix.cxx_compiler}}" >> $GITHUB_ENV
- name: Build and install PALISADE and sample kernels recipe
run: |
./hekit --config default.config install .github/hekit-recipes/palisade.toml --recipe_arg "toolkit-path=${GITHUB_WORKSPACE}"
./hekit --config default.config list
echo "PALISADE_KERNELS_BUILD_DIR=$HOME/.hekit/components/sample-kernels/palisade/build" >> $GITHUB_ENV
- name: Run unit tests
run: ${PALISADE_KERNELS_BUILD_DIR}/test/unit-test
# PALISADE sample kernels
- name: Run PALISADE sample kernels
run: KMP_WARNINGS=0 OMP_NUM_THREADS=5 ${PALISADE_KERNELS_BUILD_DIR}/sample-kernels-palisade --benchmark_out_format=json --benchmark_out="${GITHUB_JOB}_sample-kernels-palisade_${GITHUB_SHA}.json"
- name: Archive PALISADE sample kernel results
uses: actions/upload-artifact@v3
with:
name: ${{github.job}}_sample-kernels-palisade_${{github.sha}}.json
path: ${{github.job}}_sample-kernels-palisade_${{github.sha}}.json
retention-days: 90 # Maximum for free version
build-and-test-toolkit-helib:
name: HEKit HElib | ${{ matrix.os }} ${{ matrix.c_compiler }} py${{ matrix.python-version }}
runs-on: ${{ matrix.os }}
defaults:
run:
shell: bash
strategy:
matrix:
# python3.10 is default on Ubuntu-22.04
python-version: ["3.10"]
c_compiler: [clang-14, gcc-11]
os: [ubuntu-22.04]
include:
# pairing cc compiler with relevant cxx compiler
# clang-14 and gcc-11 default on Ubuntu-22.04
- c_compiler: clang-14
cxx_compiler: clang++-14
- c_compiler: gcc-11
cxx_compiler: g++-11
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Setup
run: |
pip3 install toml pydantic==1.10.4 pytest
echo "$HOME/.local/bin" >> $GITHUB_PATH
echo "CC=${{matrix.c_compiler}}" >> $GITHUB_ENV
echo "CXX=${{matrix.cxx_compiler}}" >> $GITHUB_ENV
echo "python-version=${{matrix.python-version}}" >> $GITHUB_ENV
python --version
- name: Build and install HElib and example recipe
run: |
./hekit --config default.config install .github/hekit-recipes/helib.toml --recipe_arg "toolkit-path=${GITHUB_WORKSPACE}"
./hekit --config default.config list
echo "PSI_BUILD_DIR=$HOME/.hekit/components/examples/psi/build" >> $GITHUB_ENV
- name: Run PSI example
run: |
echo -e "apple\nBanana\ncat\nBa-Ra\nCalifornia" > client.set
${PSI_BUILD_DIR}/psi client.set --server ${PSI_BUILD_DIR}/datasets/fruits.set | grep -B 1 "apple"
${PSI_BUILD_DIR}/psi client.set --server ${PSI_BUILD_DIR}/datasets/ancient_egyptian_gods.set | grep -B 1 "Ba-Ra"
${PSI_BUILD_DIR}/psi client.set --server ${PSI_BUILD_DIR}/datasets/us_states.set | grep -B 1 "California"
- name: Build PSI config tests
run: |
./hekit --config default.config install recipes/config-psi.toml --recipe_arg "toolkit-path=${GITHUB_WORKSPACE}"
./hekit --config default.config list
echo "CONFIG_PSI_TEST_DIR=$HOME/.hekit/components/psi/configurable-psi/build" >> $GITHUB_ENV
- name: Run PSI config unit tests
run: ${CONFIG_PSI_TEST_DIR}/bin/unit-test
- name: Run PSI config pytests
run: |
source deployments/config_psi/setenv.sh
pytest deployments/config_psi/tests/
pytest deployments/config_psi/scripts/tests/
build-and-test-toolkit-seal:
name: HEKit SEAL | ${{ matrix.os }} ${{ matrix.c_compiler }} py${{ matrix.python-version }}
runs-on: ${{ matrix.os }}
defaults:
run:
shell: bash
strategy:
matrix:
# python3.10 is default on Ubuntu-22.04
python-version: ["3.10"]
c_compiler: [clang-14, gcc-11]
os: [ubuntu-22.04]
include:
# pairing cc compiler with relevant cxx compiler
# clang-14 and gcc-11 default on Ubuntu-22.04
- c_compiler: clang-14
cxx_compiler: clang++-14
- c_compiler: gcc-11
cxx_compiler: g++-11
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Setup
run: |
pip3 install toml
echo "$HOME/.local/bin" >> $GITHUB_PATH
echo "CC=${{matrix.c_compiler}}" >> $GITHUB_ENV
echo "CXX=${{matrix.cxx_compiler}}" >> $GITHUB_ENV
echo "python-version=${{ matrix.python-version }}">> $GITHUB_ENV
python --version
- name: he-toolkit-seal-ubuntu-22
if: matrix.os == 'ubuntu-22.04' && matrix.c_compiler == 'clang-14'
run: |
sudo apt install libomp-dev
- name: Build and install SEAL, examples, and sample kernels recipe
run: |
./hekit --config default.config install .github/hekit-recipes/seal.toml --recipe_arg "toolkit-path=${GITHUB_WORKSPACE}"
./hekit --config default.config list
echo "QUERY_BUILD_DIR=$HOME/.hekit/components/examples/secure-query/build" >> $GITHUB_ENV
echo "LR_BUILD_DIR=$HOME/.hekit/components/examples/logistic-regression/build" >> $GITHUB_ENV
echo "SEAL_KERNELS_BUILD_DIR=$HOME/.hekit/components/sample-kernels/seal/build" >> $GITHUB_ENV
- name: Run Secure Query example
run: |
cd ${QUERY_BUILD_DIR}
echo -n "Input: Louisiana, " && ./secure-query <<< $'\n\nLouisiana\n' | grep "Baton Rouge"
echo -n "Input: North Dakota, " && ./secure-query <<< $'\n\nNorth Dakota\n' | grep "Bismarck"
echo -n "Input: Wyoming, " && ./secure-query <<< $'\n\nWyoming\n' | grep "Cheyenne"
- name: Run LR example
run: |
cd ${LR_BUILD_DIR}
./lr_test --compare --data lrtest_large | grep "All match!"
- name: Run sample kernels unit test
run: ${SEAL_KERNELS_BUILD_DIR}/test/unit-test
# SEAL sample kernels
- name: Run SEAL sample kernels
run: KMP_WARNINGS=0 OMP_NUM_THREADS=5 ${SEAL_KERNELS_BUILD_DIR}/sample-kernels-seal --benchmark_out_format=json --benchmark_out="${GITHUB_JOB}_sample-kernels-seal_${GITHUB_SHA}.json"
- name: Archive SEAL sample kernel results
uses: actions/upload-artifact@v3
with:
name: ${{github.job}}_sample-kernels-seal_${{github.sha}}.json
path: ${{github.job}}_sample-kernels-seal_${{github.sha}}.json
retention-days: 90 # Maximum for free version
##############
# IceLake CI #
##############
format-icelake:
name: Format (IceLake)
# Disable this job for now and until a self-hosted icelake becomes available
if: ${{ false }}
runs-on: [self-hosted, Linux, X64, ice-lake]
steps:
# Add local bin for cpplint
- name: Setup
run: echo "$HOME/.local/bin" >> $GITHUB_PATH
- uses: actions/checkout@v3
- name: pre-commit
run: pre-commit run --all-files
test-hekit-icelake:
name: Test hekit commands (IceLake)
# Disable this job for now and until a self-hosted icelake becomes available
if: ${{ false }}
runs-on: [self-hosted, Linux, X64, ice-lake]
defaults:
run:
shell: bash
steps:
# Add local bin for pytest
- name: Setup
run: echo "$HOME/.local/bin" >> $GITHUB_PATH
- uses: actions/checkout@v3
- name: Run pytests
run: pytest tests/ -n 8 --dist loadgroup
build-and-test-toolkit-icelake:
name: Build, test and run HE Toolkit (IceLake)
# Disable this job for now and until a self-hosted icelake becomes available
if: ${{ false }}
runs-on: [self-hosted, Linux, X64, ice-lake]
defaults:
run:
shell: bash
steps:
# Add local bin for pip modules and create temp workspace
- name: Setup
run: |
temp_workspace="$(pwd)/../$(date +%s)"
mkdir $temp_workspace
echo "$HOME/.local/bin" >> $GITHUB_PATH
echo "TEMP_DIRECTORY=$temp_workspace" >> $GITHUB_ENV
echo "HE_WORKSPACE=$temp_workspace/.hekit/components" >> $GITHUB_ENV
echo "CONFIG_FILE=$temp_workspace/default.config" >> $GITHUB_ENV
- uses: actions/checkout@v3
- name: Init hekit
run: echo "repo_location = \"$HE_WORKSPACE\"" > $CONFIG_FILE
- name: Build and install default recipe
run: |
./hekit --config $CONFIG_FILE install recipes/default.toml
./hekit --config $CONFIG_FILE list
- name: Build examples
run: |
./hekit --config $CONFIG_FILE build recipes/examples.toml --recipe_arg "toolkit-path=${GITHUB_WORKSPACE}"
./hekit --config $CONFIG_FILE list
echo "PSI_BUILD_DIR=$HE_WORKSPACE/examples/psi/build" >> $GITHUB_ENV
echo "QUERY_BUILD_DIR=$HE_WORKSPACE/examples/secure-query/build" >> $GITHUB_ENV
echo "LR_BUILD_DIR=$HE_WORKSPACE/examples/logistic-regression/build" >> $GITHUB_ENV
- name: Run PSI example
run: |
echo -e "apple\nBanana\ncat\nBa-Ra\nCalifornia" > client.set
${PSI_BUILD_DIR}/psi client.set --server ${PSI_BUILD_DIR}/datasets/fruits.set | grep -B 1 "apple"
${PSI_BUILD_DIR}/psi client.set --server ${PSI_BUILD_DIR}/datasets/ancient_egyptian_gods.set | grep -B 1 "Ba-Ra"
${PSI_BUILD_DIR}/psi client.set --server ${PSI_BUILD_DIR}/datasets/us_states.set | grep -B 1 "California"
- name: Run Secure Query example
run: |
cd ${QUERY_BUILD_DIR}
echo -n "Input: Louisiana, " && ./secure-query <<< $'\n\nLouisiana\n' | grep "Baton Rouge"
echo -n "Input: North Dakota, " && ./secure-query <<< $'\n\nNorth Dakota\n' | grep "Bismarck"
echo -n "Input: Wyoming, " && ./secure-query <<< $'\n\nWyoming\n' | grep "Cheyenne"
- name: Run LR example
run: |
cd ${LR_BUILD_DIR}
./lr_test --compare --data lrtest_large | grep "All match!"
- name: Build PSI config tests
run: |
./hekit --config $CONFIG_FILE install recipes/config-psi.toml --recipe_arg "toolkit-path=${GITHUB_WORKSPACE}"
./hekit --config $CONFIG_FILE list
echo "CONFIG_PSI_TEST_DIR=$HOME/.hekit/components/psi/configurable-psi/build" >> $GITHUB_ENV
- name: Run PSI config unit tests
run: ${CONFIG_PSI_TEST_DIR}/bin/unit-test
- name: Run PSI config pytests
run: |
source deployments/config_psi/setenv.sh
pytest deployments/config_psi/tests/
pytest deployments/config_psi/scripts/tests/
- name: Build sample kernels
run: |
./hekit --config $CONFIG_FILE build recipes/sample-kernels.toml --recipe_arg "toolkit-path=${GITHUB_WORKSPACE}"
./hekit --config $CONFIG_FILE list
echo "SEAL_KERNELS_BUILD_DIR=$HE_WORKSPACE/sample-kernels/seal/build" >> $GITHUB_ENV
echo "PALISADE_KERNELS_BUILD_DIR=$HE_WORKSPACE/sample-kernels/palisade/build" >> $GITHUB_ENV
- name: Run sample kernels unit test
run: |
${SEAL_KERNELS_BUILD_DIR}/test/unit-test
${PALISADE_KERNELS_BUILD_DIR}/test/unit-test
# PALISADE sample kernels
- name: Run PALISADE sample kernels
run: KMP_WARNINGS=0 OMP_NUM_THREADS=5 ${PALISADE_KERNELS_BUILD_DIR}/sample-kernels-palisade --benchmark_out_format=json --benchmark_out="${GITHUB_JOB}_sample-kernels-palisade_${GITHUB_SHA}.json"
- name: Archive PALISADE sample kernel results
uses: actions/upload-artifact@v3
with:
name: ${{github.job}}_sample-kernels-palisade_${{github.sha}}.json
path: ${{github.job}}_sample-kernels-palisade_${{github.sha}}.json
retention-days: 90 # Maximum for free version
# SEAL sample kernels
- name: Run SEAL sample kernels
run: KMP_WARNINGS=0 OMP_NUM_THREADS=5 ${SEAL_KERNELS_BUILD_DIR}/sample-kernels-seal --benchmark_out_format=json --benchmark_out="${GITHUB_JOB}_sample-kernels-seal_${GITHUB_SHA}.json"
- name: Archive SEAL sample kernel results
uses: actions/upload-artifact@v3
with:
name: ${{github.job}}_sample-kernels-seal_${{github.sha}}.json
path: ${{github.job}}_sample-kernels-seal_${{github.sha}}.json
retention-days: 90 # Maximum for free version
- name: Clean up hekit directory
if: always()
run: |
./hekit --config $CONFIG_FILE remove --all -y
rm -rf $TEMP_DIRECTORY