Skip to content

Commit

Permalink
Use internal zlib for blosc filter (#90)
Browse files Browse the repository at this point in the history
* Customize blosc zlib internal build and fix lz4 conflicts
  • Loading branch information
byrnHDF authored Jan 5, 2024
1 parent 5c492e9 commit 5740469
Show file tree
Hide file tree
Showing 38 changed files with 297 additions and 217 deletions.
37 changes: 20 additions & 17 deletions BLOSC/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -126,9 +126,9 @@ endif ()
#-----------------------------------------------------------------------------
# Option for ZLib support
#-----------------------------------------------------------------------------
option (HDF_ENABLE_BLOSC_ZLIB_SUPPORT "Enable BLOSC Zlib Filters" OFF)
option (HDF_ENABLE_BLOSC_ZLIB_SUPPORT "Enable BLOSC Zlib Filters" ON)
if (HDF_ENABLE_BLOSC_ZLIB_SUPPORT)
if (NOT H5_ZLIB_HEADER)
if (NOT BLOSC_ZLIB_HEADER)
if (NOT BLOSC_ZLIB_USE_EXTERNAL)
find_package (ZLIB NAMES ${BLOSC_ZLIB_PACKAGE_NAME}${HDF_PACKAGE_EXT} COMPONENTS static)
if (NOT ZLIB_FOUND)
Expand All @@ -137,37 +137,40 @@ if (HDF_ENABLE_BLOSC_ZLIB_SUPPORT)
set (H5PL_LINK_LIBS ${H5PL_LINK_LIBS} ${ZLIB_LIBRARIES})
endif ()
endif ()
set (BLOSC_ZLIB_FOUND ${ZLIB_FOUND})
endif ()
if (ZLIB_FOUND)
set (H5_HAVE_FILTER_DEFLATE 1)
set (H5_HAVE_ZLIB_H 1)
set (H5_HAVE_LIBZ 1)
set (H5_ZLIB_HEADER "zlib.h")
if (BLOSC_ZLIB_FOUND)
set (BLOSC_HAVE_FILTER_DEFLATE 1)
set (BLOSC_HAVE_ZLIB_H 1)
set (BLOSC_HAVE_LIBZ 1)
set (BLOSC_ZLIB_HEADER "zlib.h")
set (BLOSC_ZLIB_INCLUDE_DIR_GEN ${ZLIB_INCLUDE_DIR})
set (BLOSC_ZLIB_INCLUDE_DIRS ${ZLIB_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIR})
else ()
if (H5PL_ALLOW_EXTERNAL_SUPPORT MATCHES "GIT" OR H5PL_ALLOW_EXTERNAL_SUPPORT MATCHES "TGZ")
EXTERNAL_BLOSC_ZLIB_LIBRARY (${H5PL_ALLOW_EXTERNAL_SUPPORT} STATIC)
set (H5_HAVE_FILTER_DEFLATE 1)
set (H5_HAVE_ZLIB_H 1)
set (H5_HAVE_LIBZ 1)
set (BLOSC_HAVE_FILTER_DEFLATE 1)
set (BLOSC_HAVE_ZLIB_H 1)
set (BLOSC_HAVE_LIBZ 1)
message (STATUS "Filter BLOSC_ZLIB is built")
message (VERBOSE "BLOSC_ZLIB Includes: ${BLOSC_ZLIB_INCLUDE_DIRS}")
else ()
message (FATAL_ERROR " BLOSC_ZLib is Required for BLOSC_ZLib support in blosc")
endif ()
endif ()
else (NOT H5_ZLIB_HEADER)
else (NOT BLOSC_ZLIB_HEADER)
# This project is being called from within another and ZLib is already configured
set (H5_HAVE_FILTER_DEFLATE 1)
set (H5_HAVE_ZLIB_H 1)
set (H5_HAVE_LIBZ 1)
set (BLOSC_HAVE_FILTER_DEFLATE 1)
set (BLOSC_HAVE_ZLIB_H 1)
set (BLOSC_HAVE_LIBZ 1)
endif ()
if (H5_HAVE_FILTER_DEFLATE)
if (BLOSC_HAVE_FILTER_DEFLATE)
set (EXTERNAL_FILTERS "${EXTERNAL_FILTERS} DEFLATE")
endif ()
set (H5PL_LINK_LIBS ${H5PL_LINK_LIBS} ${ZLIB_LIBRARY})
set (H5BLOSC_INCLUDE_DIRS ${H5BLOSC_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS})
set (H5PL_LINK_LIBS ${H5PL_LINK_LIBS} ${BLOSC_ZLIB_LIBRARY})
set (H5BLOSC_INCLUDE_DIRS ${H5BLOSC_INCLUDE_DIRS} ${BLOSC_ZLIB_INCLUDE_DIRS})
message (STATUS "Filter BLOSC_ZLIB is ON")
message (VERBOSE "BLOSC_ZLIB Includes: ${H5BLOSC_INCLUDE_DIRS}")
endif ()

#-----------------------------------------------------------------------------
Expand Down
17 changes: 8 additions & 9 deletions BLOSC/config/cmake/H5BLOSCMacros.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -80,21 +80,20 @@ macro (EXTERNAL_BLOSC_ZLIB_LIBRARY compress_type)
if(NOT blosc_zlib_POPULATED)
FetchContent_Populate(BLOSC_ZLIB)

# Copy an additional/replacement files into the populated source
file(COPY ${H5BLOSC_SOURCE_DIR}/config/zlib-CMakeLists.txt DESTINATION ${blosc_zlib_SOURCE_DIR})
file(RENAME ${blosc_zlib_SOURCE_DIR}/zlib-CMakeLists.txt ${blosc_zlib_SOURCE_DIR}/CMakeLists.txt)

add_subdirectory(${blosc_zlib_SOURCE_DIR} ${blosc_zlib_BINARY_DIR})
endif()

if (WIN32)
set (BLOSC_ZLIB_LIB_NAME "zlib")
else ()
set (BLOSC_ZLIB_LIB_NAME "z")
endif ()

set ({BLOSC_ZLIB_STATIC_LIBRARY "blosc_zlib-static")
set ({BLOSC_ZLIB_LIBRARIES ${{BLOSC_ZLIB_STATIC_LIBRARY})
set (BLOSC_ZLIB_STATIC_LIBRARY "zlibstatic")
set (BLOSC_ZLIB_LIBRARY ${BLOSC_ZLIB_STATIC_LIBRARY})
set (BLOSC_ZLIB_LIBRARIES ${BLOSC_ZLIB_LIBRARY})

set (BLOSC_ZLIB_INCLUDE_DIR_GEN "${blosc_zlib_BINARY_DIR}")
set (BLOSC_ZLIB_INCLUDE_DIR "${blosc_zlib_SOURCE_DIR}")
set (BLOSC_ZLIB_FOUND 1)
set (BLOSC_ZLIB_LIBRARIES ${BLOSC_ZLIB_LIBRARY})
set (BLOSC_ZLIB_INCLUDE_DIRS ${BLOSC_ZLIB_INCLUDE_DIR_GEN} ${BLOSC_ZLIB_INCLUDE_DIR})
message (VERBOSE "BLOSC_ZLIB Includes: ${BLOSC_ZLIB_INCLUDE_DIRS}")
endmacro ()
9 changes: 2 additions & 7 deletions BLOSC/config/cmake/HDFPluginMacros.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -325,14 +325,9 @@ macro (HDF5_SUPPORT link_hdf)
#plugin source needs to be linked with HDF5
set (H5PL_LINK_LIBS ${H5PL_LINK_LIBS} ${H5PL_HDF5_LINK_LIBS})
endif ()
set (USE_SHARED_LIBS ON CACHE BOOL "Use Shared Libraries" FORCE)
endif ()
message (STATUS "HDF5 link libs: ${H5PL_HDF5_LINK_LIBS} Includes: ${H5PL_HDF5_INCLUDE_DIRS}")

if (USE_SHARED_LIBS)
set (H5_LIB_TYPE SHARED)
else ()
set (H5_LIB_TYPE STATIC)
endif ()
message (STATUS "HDF5 link libs: ${H5PL_LINK_LIBS} Includes: ${H5PL_HDF5_INCLUDE_DIRS}")
endmacro ()

macro (INSTALL_SUPPORT varname)
Expand Down
15 changes: 7 additions & 8 deletions BLOSC/config/cmake/binex/config/cmake/HDFPluginMacros.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -260,20 +260,19 @@ macro (HDF5_SUPPORT link_hdf)
set (H5PL_HDF5_HAVE_H5PUBCONF_H 1)
set (H5PL_HDF5_HAVE_HDF5 1)
set (H5PL_HDF5_HEADER "h5pubconf.h")
message (STATUS "HDF5 found: INC=${HDF5_INCLUDE_DIRS} TOOLS=${HDF5_TOOLS_DIR}")
message (STATUS "HDF5-${HDF5_VERSION_STRING} found: INC=${HDF5_INCLUDE_DIRS} TOOLS=${HDF5_TOOLS_DIR}")
else ()
message (FATAL_ERROR " HDF5 is Required for plugin library")
endif ()
else ()
# This project is being called from within another and HDF5 is already configured
set (H5PL_HDF5_HAVE_H5PUBCONF_H 1)
set (H5PL_HDF5_HAVE_HDF5 1)
if (${link_hdf})
#plugin source needs to be linked with HDF5
set (H5PL_LINK_LIBS ${H5PL_LINK_LIBS} ${H5PL_HDF5_LINK_LIBS})
endif ()
set (USE_SHARED_LIBS ON CACHE BOOL "Use Shared Libraries" FORCE)
endif ()
message (STATUS "HDF5 link libs: ${H5PL_HDF5_LINK_LIBS} Includes: ${H5PL_HDF5_INCLUDE_DIRS}")

if (USE_SHARED_LIBS)
set (H5_LIB_TYPE SHARED)
else ()
set (H5_LIB_TYPE STATIC)
endif ()
message (STATUS "HDF5 link libs: ${H5PL_LINK_LIBS} Includes: ${H5PL_HDF5_INCLUDE_DIRS}")
endmacro ()
144 changes: 144 additions & 0 deletions BLOSC/config/zlib-CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
cmake_minimum_required(VERSION 2.4.4...3.15.0)
set(CMAKE_ALLOW_LOOSE_LOOP_CONSTRUCTS ON)

project(zlib C)

set(VERSION "1.3")

include(CheckTypeSize)
include(CheckFunctionExists)
include(CheckIncludeFile)
include(CheckCSourceCompiles)
enable_testing()

check_include_file(sys/types.h HAVE_SYS_TYPES_H)
check_include_file(stdint.h HAVE_STDINT_H)
check_include_file(stddef.h HAVE_STDDEF_H)

#
# Check to see if we have large file support
#
set(CMAKE_REQUIRED_DEFINITIONS -D_LARGEFILE64_SOURCE=1)
# We add these other definitions here because CheckTypeSize.cmake
# in CMake 2.4.x does not automatically do so and we want
# compatibility with CMake 2.4.x.
if(HAVE_SYS_TYPES_H)
list(APPEND CMAKE_REQUIRED_DEFINITIONS -DHAVE_SYS_TYPES_H)
endif()
if(HAVE_STDINT_H)
list(APPEND CMAKE_REQUIRED_DEFINITIONS -DHAVE_STDINT_H)
endif()
if(HAVE_STDDEF_H)
list(APPEND CMAKE_REQUIRED_DEFINITIONS -DHAVE_STDDEF_H)
endif()
check_type_size(off64_t OFF64_T)
if(HAVE_OFF64_T)
add_definitions(-D_LARGEFILE64_SOURCE=1)
endif()
set(CMAKE_REQUIRED_DEFINITIONS) # clear variable

#
# Check for fseeko
#
check_function_exists(fseeko HAVE_FSEEKO)
if(NOT HAVE_FSEEKO)
add_definitions(-DNO_FSEEKO)
endif()

#
# Check for unistd.h
#
check_include_file(unistd.h Z_HAVE_UNISTD_H)

if(MSVC)
set(CMAKE_DEBUG_POSTFIX "d")
add_definitions(-D_CRT_SECURE_NO_DEPRECATE)
add_definitions(-D_CRT_NONSTDC_NO_DEPRECATE)
include_directories(${CMAKE_CURRENT_SOURCE_DIR})
endif()

if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR)
# If we're doing an out of source build and the user has a zconf.h
# in their source tree...
if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/zconf.h)
message(STATUS "Renaming")
message(STATUS " ${CMAKE_CURRENT_SOURCE_DIR}/zconf.h")
message(STATUS "to 'zconf.h.included' because this file is included with zlib")
message(STATUS "but CMake generates it automatically in the build directory.")
file(RENAME ${CMAKE_CURRENT_SOURCE_DIR}/zconf.h ${CMAKE_CURRENT_SOURCE_DIR}/zconf.h.included)
endif()
endif()

set(ZLIB_PC ${CMAKE_CURRENT_BINARY_DIR}/zlib.pc)
configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/zlib.pc.cmakein
${ZLIB_PC} @ONLY)
configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/zconf.h.cmakein
${CMAKE_CURRENT_BINARY_DIR}/zconf.h @ONLY)
include_directories(${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_SOURCE_DIR})


#============================================================================
# zlib
#============================================================================

set(ZLIB_PUBLIC_HDRS
${CMAKE_CURRENT_BINARY_DIR}/zconf.h
zlib.h
)
set(ZLIB_PRIVATE_HDRS
crc32.h
deflate.h
gzguts.h
inffast.h
inffixed.h
inflate.h
inftrees.h
trees.h
zutil.h
)
set(ZLIB_SRCS
adler32.c
compress.c
crc32.c
deflate.c
gzclose.c
gzlib.c
gzread.c
gzwrite.c
inflate.c
infback.c
inftrees.c
inffast.c
trees.c
uncompr.c
zutil.c
)

if(NOT MINGW)
set(ZLIB_DLL_SRCS
win32/zlib1.rc # If present will override custom build rule below.
)
endif()

# parse the full version number from zlib.h and include in ZLIB_FULL_VERSION
file(READ ${CMAKE_CURRENT_SOURCE_DIR}/zlib.h _zlib_h_contents)
string(REGEX REPLACE ".*#define[ \t]+ZLIB_VERSION[ \t]+\"([-0-9A-Za-z.]+)\".*"
"\\1" ZLIB_FULL_VERSION ${_zlib_h_contents})

if(MINGW)
# This gets us DLL resource information when compiling on MinGW.
if(NOT CMAKE_RC_COMPILER)
set(CMAKE_RC_COMPILER windres.exe)
endif()

add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/zlib1rc.obj
COMMAND ${CMAKE_RC_COMPILER}
-D GCC_WINDRES
-I ${CMAKE_CURRENT_SOURCE_DIR}
-I ${CMAKE_CURRENT_BINARY_DIR}
-o ${CMAKE_CURRENT_BINARY_DIR}/zlib1rc.obj
-i ${CMAKE_CURRENT_SOURCE_DIR}/win32/zlib1.rc)
set(ZLIB_DLL_SRCS ${CMAKE_CURRENT_BINARY_DIR}/zlib1rc.obj)
endif(MINGW)

add_library(zlibstatic STATIC ${ZLIB_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS})
12 changes: 6 additions & 6 deletions BSHUF/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -88,10 +88,10 @@ option (H5PL_ALLOW_EXTERNAL_SUPPORT "Allow External Library Building (NO GIT TGZ
set (H5PL_ALLOW_EXTERNAL_SUPPORT "NO" CACHE STRING "Allow External Library Building (NO GIT TGZ)")
set_property (CACHE H5PL_ALLOW_EXTERNAL_SUPPORT PROPERTY STRINGS NO GIT TGZ)

option (BUILD_LZ4_LIBRARY_SOURCE "build the lz4 library within the plugin" ON)
if (NOT BUILD_LZ4_LIBRARY_SOURCE)
option (BUILD_BSHUF_LZ4_LIBRARY_SOURCE "build the lz4 library within the BSHUF plugin" ON)
if (NOT BUILD_BSHUF_LZ4_LIBRARY_SOURCE)
if (H5PL_ALLOW_EXTERNAL_SUPPORT MATCHES "GIT" OR H5PL_ALLOW_EXTERNAL_SUPPORT MATCHES "TGZ")
option (LZ4_USE_EXTERNAL "Use External Library Building for LZ4" 1)
option (BSHUF_LZ4_USE_EXTERNAL "Use External Library Building for BSHUF LZ4" 1)
if (H5PL_ALLOW_EXTERNAL_SUPPORT MATCHES "GIT")
set (LZ4_URL ${LZ4_GIT_URL} CACHE STRING "Path to lz4 git repository")
set (LZ4_BRANCH ${LZ4_GIT_BRANCH})
Expand All @@ -101,13 +101,13 @@ if (NOT BUILD_LZ4_LIBRARY_SOURCE)
endif ()
set (LZ4_URL ${H5PL_COMP_TGZPATH}/${LZ4_TGZ_NAME})
else ()
set (LZ4_USE_EXTERNAL 0)
set (BSHUF_LZ4_USE_EXTERNAL 0)
endif ()
endif ()
#-----------------------------------------------------------------------------
# lz4 Library Settings
#-----------------------------------------------------------------------------
if (NOT LZ4_USE_EXTERNAL)
if (NOT BSHUF_LZ4_USE_EXTERNAL)
find_package (LZ4 NAMES ${LZ4_PACKAGE_NAME})
if (NOT LZ4_FOUND)
find_package (LZ4) # Legacy find
Expand All @@ -127,7 +127,7 @@ if (NOT BUILD_LZ4_LIBRARY_SOURCE)
EXTERNAL_LZ4_LIBRARY (${H5PL_ALLOW_EXTERNAL_SUPPORT} STATIC)
set (H5LZ4_HAVE_LZ4_H 1)
set (H5LZ4_HAVE_LZ4 1)
message (STATUS "Filter LZ4 is built")
message (STATUS "Filter BSHUF_LZ4 is built")
else ()
message (FATAL_ERROR " LZ4 is Required for ${H5BSHUF_PACKAGE_NAME} library")
endif ()
Expand Down
9 changes: 2 additions & 7 deletions BSHUF/config/cmake/HDFPluginMacros.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -325,14 +325,9 @@ macro (HDF5_SUPPORT link_hdf)
#plugin source needs to be linked with HDF5
set (H5PL_LINK_LIBS ${H5PL_LINK_LIBS} ${H5PL_HDF5_LINK_LIBS})
endif ()
set (USE_SHARED_LIBS ON CACHE BOOL "Use Shared Libraries" FORCE)
endif ()
message (STATUS "HDF5 link libs: ${H5PL_HDF5_LINK_LIBS} Includes: ${H5PL_HDF5_INCLUDE_DIRS}")

if (USE_SHARED_LIBS)
set (H5_LIB_TYPE SHARED)
else ()
set (H5_LIB_TYPE STATIC)
endif ()
message (STATUS "HDF5 link libs: ${H5PL_LINK_LIBS} Includes: ${H5PL_HDF5_INCLUDE_DIRS}")
endmacro ()

macro (INSTALL_SUPPORT varname)
Expand Down
15 changes: 7 additions & 8 deletions BSHUF/config/cmake/binex/config/cmake/HDFPluginMacros.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -260,20 +260,19 @@ macro (HDF5_SUPPORT link_hdf)
set (H5PL_HDF5_HAVE_H5PUBCONF_H 1)
set (H5PL_HDF5_HAVE_HDF5 1)
set (H5PL_HDF5_HEADER "h5pubconf.h")
message (STATUS "HDF5 found: INC=${HDF5_INCLUDE_DIRS} TOOLS=${HDF5_TOOLS_DIR}")
message (STATUS "HDF5-${HDF5_VERSION_STRING} found: INC=${HDF5_INCLUDE_DIRS} TOOLS=${HDF5_TOOLS_DIR}")
else ()
message (FATAL_ERROR " HDF5 is Required for plugin library")
endif ()
else ()
# This project is being called from within another and HDF5 is already configured
set (H5PL_HDF5_HAVE_H5PUBCONF_H 1)
set (H5PL_HDF5_HAVE_HDF5 1)
if (${link_hdf})
#plugin source needs to be linked with HDF5
set (H5PL_LINK_LIBS ${H5PL_LINK_LIBS} ${H5PL_HDF5_LINK_LIBS})
endif ()
set (USE_SHARED_LIBS ON CACHE BOOL "Use Shared Libraries" FORCE)
endif ()
message (STATUS "HDF5 link libs: ${H5PL_HDF5_LINK_LIBS} Includes: ${H5PL_HDF5_INCLUDE_DIRS}")

if (USE_SHARED_LIBS)
set (H5_LIB_TYPE SHARED)
else ()
set (H5_LIB_TYPE STATIC)
endif ()
message (STATUS "HDF5 link libs: ${H5PL_LINK_LIBS} Includes: ${H5PL_HDF5_INCLUDE_DIRS}")
endmacro ()
2 changes: 1 addition & 1 deletion BSHUF/config/cmake/cacheinit.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ set (HDF5_PACKAGE_NAME "hdf5" CACHE STRING "Name of HDF5 package" FORCE)

set (HDF5_NAMESPACE "hdf5::" CACHE STRING "Name space of HDF5 library" FORCE)

set (BUILD_LZ4_LIBRARY_SOURCE ON CACHE BOOL "build the lz4 library within the plugin" FORCE)
set (BUILD_BSHUF_LZ4_LIBRARY_SOURCE ON CACHE BOOL "build the lz4 library within the BSHUF plugin" FORCE)

set (LZ4_GIT_URL "https://github.com/lz4/lz4.git" CACHE STRING "Use LZ4 from GitHub repository" FORCE)
set (LZ4_GIT_BRANCH "dev" CACHE STRING "" FORCE)
Expand Down
2 changes: 1 addition & 1 deletion BSHUF/src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ if (BUILD_BSHUF_LIBRARY_SOURCE)
${H5BSHUF_SRC_SOURCE_DIR}
)
endif ()
if (BUILD_LZ4_LIBRARY_SOURCE)
if (BUILD_BSHUF_LZ4_LIBRARY_SOURCE)
set (H5BSHUF_SRCS ${H5BSHUF_SRCS}
${H5BSHUF_SRC_SOURCE_DIR}/lib/lz4.c
${H5BSHUF_SRC_SOURCE_DIR}/lib/lz4.h
Expand Down
Loading

0 comments on commit 5740469

Please sign in to comment.