Skip to content
This repository has been archived by the owner on Aug 16, 2023. It is now read-only.

Commit

Permalink
optimize lapack library (#472)
Browse files Browse the repository at this point in the history 
Signed-off-by: yusheng.ma <[email protected]>

Signed-off-by: yusheng.ma <[email protected]>
  • Loading branch information
@Presburger
Presburger authored Sep 15, 2022
1 parent 5d9dea6 commit 3501024
Show file tree
Hide file tree
Showing 3 changed files with 96 additions and 25 deletions.
36 changes: 28 additions & 8 deletions ci/docker/ubuntu/scripts/deps/build.sh
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,39 @@
CMAKE_VERSION="v3.23"
CMAKE_TAR="cmake-3.23.0-linux-x86_64.tar.gz"
apt-get update && apt-get install -y --no-install-recommends wget curl g++ gcc ca-certificates \
make ccache python3-dev gfortran python3-setuptools swig pip \
libaio-dev libgoogle-perftools-dev clang-format libboost-all-dev liblapack-dev \
gpg make ccache python3-dev gfortran python3-setuptools swig pip \
libaio-dev libgoogle-perftools-dev clang-format libboost-program-options-dev \
&& wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \
2>/dev/null | gpg --dearmor - | tee \
/usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null \
&& echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ focal main' | tee \
/etc/apt/sources.list.d/kitware.list >/dev/null \
&& apt update && apt install -y cmake \
&& apt-get remove --purge -y \
&& rm -rf /var/lib/apt/lists/* \
&& wget https://github.com/Reference-LAPACK/lapack/archive/refs/tags/v3.10.1.tar.gz \
&& tar zxvf v3.10.1.tar.gz && cd lapack-3.10.1/ \
&& cmake -B build -S . -DCMAKE_SKIP_RPATH=ON -DBUILD_SHARED_LIBS=ON \
-DBUILD_TESTING=OFF -DCMAKE_INSTALL_PREFIX=/usr/local \
-DCMAKE_Fortran_COMPILER=gfortran -DLAPACKE_WITH_TMG=ON -DCBLAS=OFF \
-DBUILD_DEPRECATED=OFF \
&& cmake --build build \
&& cmake --install build \
&& rm -r /usr/local/lib/libblas.* \
&& rm -r /usr/local/lib/liblapacke.* \
&& rm -r /usr/local/lib/pkgconfig/blas.* \
&& rm -r /usr/local/lib/pkgconfig/lapacke.* \
&& rm -r /usr/local/lib/cmake/lapacke* \
&& rm -r /usr/local/include/* \
&& cd .. && rm -rf lapack-3.10.1 && rm v3.10.1.tar.gz \
&& wget https://github.com/xianyi/OpenBLAS/archive/v0.3.21.tar.gz && \
tar zxvf v0.3.21.tar.gz && cd OpenBLAS-0.3.21 && \
make NO_STATIC=1 NO_LAPACK=1 NO_LAPACKE=1 NO_CBLAS=1 NO_AFFINITY=1 USE_OPENMP=1 \
make NO_STATIC=1 NO_LAPACK=1 NO_LAPACKE=1 NO_AFFINITY=1 USE_OPENMP=1 \
TARGET=HASWELL DYNAMIC_ARCH=1 \
NUM_THREADS=64 MAJOR_VERSION=3 libs shared && \
make PREFIX=/usr/local NUM_THREADS=64 MAJOR_VERSION=3 install && \
rm -f /usr/local/include/cblas.h /usr/local/include/lapack* &&\
rm -f /usr/local/include/lapack* && \
ln -s /usr/local/lib/libopenblasp-r0.3.21.so /lib/libblas.so && \
ln -s /usr/local/lib/libopenblasp-r0.3.21.so /lib/libblas.so.3 && \
ln -s /usr/local/lib/pkgconfig/openblas.pc /usr/local/lib/pkgconfig/blas.pc && \
cd .. && rm -rf OpenBLAS-0.3.21 && rm v0.3.21.tar.gz \
&& cd /tmp && wget --tries=3 --retry-connrefused "https://cmake.org/files/${CMAKE_VERSION}/${CMAKE_TAR}" \
&& tar --strip-components=1 -xz -C /usr/local -f ${CMAKE_TAR} \
&& rm -f ${CMAKE_TAR} \
&& pip install twine
39 changes: 32 additions & 7 deletions ci/docker/ubuntu/scripts/deps/e2e.sh
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,38 @@
apt-get update && apt-get install -y --no-install-recommends wget curl g++ gcc make ca-certificates \
python3-dev gfortran python3-setuptools swig liblapack-dev pip git vim \
libaio-dev libboost-all-dev \
python3-dev gfortran python3-setuptools swig pip git vim \
libaio-dev libboost-program-options-dev gpg \
&& wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \
2>/dev/null | gpg --dearmor - | tee \
/usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null \
&& echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ focal main' | tee \
/etc/apt/sources.list.d/kitware.list >/dev/null \
&& apt update && apt install -y cmake \
&& apt-get remove --purge -y \
&& rm -rf /var/lib/apt/lists/* \
&& wget https://github.com/Reference-LAPACK/lapack/archive/refs/tags/v3.10.1.tar.gz \
&& tar zxvf v3.10.1.tar.gz && cd lapack-3.10.1/ \
&& cmake -B build -S . -DCMAKE_SKIP_RPATH=ON -DBUILD_SHARED_LIBS=ON \
-DBUILD_TESTING=OFF -DCMAKE_INSTALL_PREFIX=/usr/local \
-DCMAKE_Fortran_COMPILER=gfortran -DLAPACKE_WITH_TMG=ON -DCBLAS=OFF \
-DBUILD_DEPRECATED=OFF \
&& cmake --build build \
&& cmake --install build \
&& rm -r /usr/local/lib/libblas.* \
&& rm -r /usr/local/lib/liblapacke.* \
&& rm -r /usr/local/lib/pkgconfig/blas.* \
&& rm -r /usr/local/lib/pkgconfig/lapacke.* \
&& rm -r /usr/local/lib/cmake/lapacke* \
&& rm -r /usr/local/include/* \
&& cd .. && rm -rf lapack-3.10.1 && rm v3.10.1.tar.gz \
&& wget https://github.com/xianyi/OpenBLAS/archive/v0.3.21.tar.gz && \
tar zxvf v0.3.21.tar.gz && cd OpenBLAS-0.3.21 && \
make NO_STATIC=1 NO_LAPACK=1 NO_LAPACKE=1 NO_CBLAS=1 NO_AFFINITY=1 USE_OPENMP=1 \
TARGET=HASWELL DYNAMIC_ARCH=1 \
NUM_THREADS=64 MAJOR_VERSION=3 libs shared && \
tar zxvf v0.3.21.tar.gz && cd OpenBLAS-0.3.21 && \
make NO_STATIC=1 NO_LAPACK=1 NO_LAPACKE=1 NO_AFFINITY=1 USE_OPENMP=1 \
TARGET=HASWELL DYNAMIC_ARCH=1 \
NUM_THREADS=64 MAJOR_VERSION=3 libs shared && \
make PREFIX=/usr/local NUM_THREADS=64 MAJOR_VERSION=3 install && \
rm -f /usr/local/include/cblas.h /usr/local/include/lapack* && \
rm -f /usr/local/include/cblas.h /usr/local/include/lapack* &&\
rm -f /usr/local/include/lapack* && \
ln -s /usr/local/lib/libopenblasp-r0.3.21.so /lib/libblas.so && \
ln -s /usr/local/lib/libopenblasp-r0.3.21.so /lib/libblas.so.3 && \
ln -s /usr/local/lib/pkgconfig/openblas.pc /usr/local/lib/pkgconfig/blas.pc && \
cd .. && rm -rf OpenBLAS-0.3.21 && rm v0.3.21.tar.gz
46 changes: 36 additions & 10 deletions scripts/install_deps.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,22 +34,48 @@ if [[ "${MACHINE}" == "Linux" ]]; then
if [[ -x "$(command -v apt)" ]]; then
# for Ubuntu 18.04
release_num=$(lsb_release -r --short)
sudo apt install -y g++ gcc make ccache python3-dev gfortran liblapack-dev
ubuntu_alias="bionic"
sudo apt install -y g++ gcc make ccache python3-dev gfortran gpg wget
if [ "$release_num" == "20.04" ];then
sudo apt install -y python3-setuptools swig
fi
ubuntu_alias="focal"
fi
wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \
2>/dev/null | gpg --dearmor - | sudo tee \
/usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null \
&& echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ '$ubuntu_alias' main' | sudo tee \
/etc/apt/sources.list.d/kitware.list >/dev/null \
&& apt update && apt install -y cmake \
&& wget https://github.com/Reference-LAPACK/lapack/archive/refs/tags/v3.10.1.tar.gz \
&& tar zxvf v3.10.1.tar.gz && cd lapack-3.10.1/ \
&& cmake -B build -S . -DCMAKE_SKIP_RPATH=ON -DBUILD_SHARED_LIBS=ON \
-DBUILD_TESTING=OFF -DCMAKE_INSTALL_PREFIX=/usr/local \
-DCMAKE_Fortran_COMPILER=gfortran -DLAPACKE_WITH_TMG=ON -DCBLAS=OFF \
-DBUILD_DEPRECATED=OFF \
&& cmake --build build \
&& sudo cmake --install build \
&& sudo rm -r /usr/local/lib/libblas.* \
&& sudo rm -r /usr/local/lib/liblapacke.* \
&& sudo rm -r /usr/local/lib/pkgconfig/blas.* \
&& sudo rm -r /usr/local/lib/pkgconfig/lapacke.* \
&& sudo rm -r /usr/local/lib/cmake/lapacke* \
&& sudo rm -r /usr/local/include/* \
&& cd .. && rm -rf lapack-3.10.1 && rm v3.10.1.tar.gz \
# Pre-installation of openblas can save about 15 minutes of openblas building time.
# But the apt-installed openblas version is 0.2.20, while the latest openblas version is 0.3.19.
# So we only pre-install openblas in Unittest, and compile openblas-0.3.19 when release.
if [[ "${INSTALL_OPENBLAS}" == "true" ]]; then
sudo wget https://github.com/xianyi/OpenBLAS/archive/v0.3.21.tar.gz && \
tar zxvf v0.3.21.tar.gz && cd OpenBLAS-0.3.21 && \
make NO_STATIC=1 NO_LAPACK=1 NO_LAPACKE=1 NO_CBLAS=1 NO_AFFINITY=1 USE_OPENMP=1 \
TARGET=HASWELL DYNAMIC_ARCH=1 \
NUM_THREADS=64 MAJOR_VERSION=3 libs shared && \
sudo make PREFIX=/usr/local NUM_THREADS=64 MAJOR_VERSION=3 install && \
sudo rm -f /usr/local/include/cblas.h /usr/local/include/lapack* &&\
cd .. && rm -rf OpenBLAS-0.3.21 && rm v0.3.21.tar.gz
wget https://github.com/xianyi/OpenBLAS/archive/v0.3.21.tar.gz && \
tar zxvf v0.3.21.tar.gz && cd OpenBLAS-0.3.21 && \
make NO_STATIC=1 NO_LAPACK=1 NO_LAPACKE=1 NO_AFFINITY=1 USE_OPENMP=1 \
TARGET=HASWELL DYNAMIC_ARCH=1 \
NUM_THREADS=64 MAJOR_VERSION=3 libs shared && \
sudo make PREFIX=/usr/local NUM_THREADS=64 MAJOR_VERSION=3 install && \
sudo rm -f /usr/local/include/lapack* && \
sudo ln -s /usr/local/lib/libopenblasp-r0.3.21.so /lib/libblas.so && \
sudo ln -s /usr/local/lib/libopenblasp-r0.3.21.so /lib/libblas.so.3 && \
sudo ln -s /usr/local/lib/pkgconfig/openblas.pc /usr/local/lib/pkgconfig/blas.pc && \
cd .. && rm -rf OpenBLAS-0.3.21 && rm v0.3.21.tar.gz
fi
#DiskANN dependencies
sudo apt-get install -y libboost-program-options-dev
Expand Down

0 comments on commit 3501024

Please sign in to comment.