optimize openblas compile in pod (#469)

Signed-off-by: Yusheng.Ma <[email protected]>

Signed-off-by: Yusheng.Ma <[email protected]>

ci/docker/ubuntu/scripts/deps/build.sh

@@ -8,11 +8,12 @@ libaio-dev libgoogle-perftools-dev clang-format libboost-all-dev liblapack-dev \
 && wget https://github.com/xianyi/OpenBLAS/archive/v0.3.21.tar.gz && \
 tar zxvf v0.3.21.tar.gz && cd OpenBLAS-0.3.21 && \
 make NO_STATIC=1 NO_LAPACK=1 NO_LAPACKE=1 NO_CBLAS=1 NO_AFFINITY=1 USE_OPENMP=1 \
-    CFLAGS="-O3 -fPIC" TARGET=CORE2 DYNAMIC_ARCH=1 \
+    TARGET=HASWELL DYNAMIC_ARCH=1 \
     NUM_THREADS=64 MAJOR_VERSION=3 libs shared && \
-make -j4 PREFIX=/usr NO_STATIC=1 install && \
+make PREFIX=/usr/local NUM_THREADS=64 MAJOR_VERSION=3 install && \
+rm -f /usr/local/include/cblas.h /usr/local/include/lapack* &&\
 cd .. && rm -rf OpenBLAS-0.3.21 && rm v0.3.21.tar.gz \
 && cd /tmp && wget  --tries=3 --retry-connrefused "https://cmake.org/files/${CMAKE_VERSION}/${CMAKE_TAR}" \
 && tar --strip-components=1 -xz -C /usr/local -f ${CMAKE_TAR} \
 && rm -f ${CMAKE_TAR} \
-&& pip install twine
+&& pip install twine

ci/docker/ubuntu/scripts/deps/e2e.sh

@@ -1,12 +1,13 @@
 apt-get update && apt-get install -y --no-install-recommends wget curl g++ gcc make ca-certificates \
- python3-dev gfortran python3-setuptools swig libopenblas-dev liblapack-dev pip git vim \
+ python3-dev gfortran python3-setuptools swig liblapack-dev pip git vim \
  libaio-dev libboost-all-dev \
 && apt-get remove --purge -y  \
 && rm -rf /var/lib/apt/lists/* \
 && wget https://github.com/xianyi/OpenBLAS/archive/v0.3.21.tar.gz && \
 tar zxvf v0.3.21.tar.gz && cd OpenBLAS-0.3.21 && \
 make NO_STATIC=1 NO_LAPACK=1 NO_LAPACKE=1 NO_CBLAS=1 NO_AFFINITY=1 USE_OPENMP=1 \
-    CFLAGS="-O3 -fPIC" TARGET=CORE2 DYNAMIC_ARCH=1 \
+    TARGET=HASWELL DYNAMIC_ARCH=1 \
     NUM_THREADS=64 MAJOR_VERSION=3 libs shared && \
-make -j4 PREFIX=/usr NO_STATIC=1 install && \
-cd .. && rm -rf OpenBLAS-0.3.21 && rm v0.3.21.tar.gz 
+make PREFIX=/usr/local NUM_THREADS=64 MAJOR_VERSION=3 install && \
+rm -f /usr/local/include/cblas.h /usr/local/include/lapack* && \
+cd .. && rm -rf OpenBLAS-0.3.21 && rm v0.3.21.tar.gz 

knowhere/CMakeLists.txt

@@ -72,10 +72,14 @@ set(vector_offset_index_srcs
         index/vector_offset_index/IndexIVF_NM.cpp
         )
 
+set(BLA_VENDOR "")
+find_package(LAPACK REQUIRED)
+
 set(depend_libs
         faiss
         pthread
         knowhere_utils
+        ${LAPACK_LIBRARIES}
         )
 
 if (KNOWHERE_WITH_DISKANN)

scripts/install_deps.sh

@@ -34,7 +34,7 @@ if [[ "${MACHINE}" == "Linux" ]]; then
     if [[ -x "$(command -v apt)" ]]; then
         # for Ubuntu 18.04
         release_num=$(lsb_release -r --short)
-        sudo apt install -y g++ gcc make ccache python3-dev gfortran
+        sudo apt install -y g++ gcc make ccache python3-dev gfortran liblapack-dev 
         if [ "$release_num" == "20.04" ];then
             sudo apt install -y python3-setuptools swig
         fi
@@ -45,9 +45,10 @@ if [[ "${MACHINE}" == "Linux" ]]; then
           sudo wget https://github.com/xianyi/OpenBLAS/archive/v0.3.21.tar.gz && \
                 tar zxvf v0.3.21.tar.gz && cd OpenBLAS-0.3.21 && \
                 make NO_STATIC=1 NO_LAPACK=1 NO_LAPACKE=1 NO_CBLAS=1 NO_AFFINITY=1 USE_OPENMP=1 \
-                    CFLAGS="-O3 -fPIC" TARGET=CORE2 DYNAMIC_ARCH=1 \
+                    TARGET=HASWELL  DYNAMIC_ARCH=1 \
                     NUM_THREADS=64 MAJOR_VERSION=3 libs shared && \
-                make -j4 PREFIX=/usr NO_STATIC=1 install && \
+                sudo make PREFIX=/usr/local NUM_THREADS=64 MAJOR_VERSION=3 install  && \
+                sudo rm -f /usr/local/include/cblas.h /usr/local/include/lapack* &&\
                 cd .. && rm -rf OpenBLAS-0.3.21 && rm v0.3.21.tar.gz 
         fi
         #DiskANN dependencies