|
|
08740b8 |
diff --git a/c-blosc/hdf5/CMakeLists.txt b/c-blosc/hdf5/CMakeLists.txt
|
|
|
08740b8 |
index d9cd847e89..d91194fcf7 100644
|
|
|
08740b8 |
--- a/c-blosc/hdf5/CMakeLists.txt
|
|
|
08740b8 |
+++ b/c-blosc/hdf5/CMakeLists.txt
|
|
|
08740b8 |
@@ -4,8 +4,19 @@ set(SOURCES blosc_filter.c)
|
|
|
08740b8 |
include_directories("${PROJECT_SOURCE_DIR}/blosc")
|
|
|
08740b8 |
|
|
|
08740b8 |
# dependencies
|
|
|
08740b8 |
-find_package(HDF5 REQUIRED)
|
|
|
08740b8 |
-include_directories(HDF5_INCLIDE_DIRS)
|
|
|
08740b8 |
+if(MSVC)
|
|
|
08740b8 |
+ # FindHDF5.cmake does not find Windows installations. Try to
|
|
|
08740b8 |
+ # use an environment variable instead until the official "find"
|
|
|
08740b8 |
+ # file can be updated for Windows.
|
|
|
08740b8 |
+ #
|
|
|
08740b8 |
+ # Note that you have to set this environment variable by hand.
|
|
|
08740b8 |
+ file(TO_CMAKE_PATH "$ENV{HDF5_DIR}" HDF5_HINT)
|
|
|
08740b8 |
+ set(HDF5_DIR ${HDF5_HINT} CACHE STRING "Path to HDF5 CMake config directory.")
|
|
|
08740b8 |
+ find_package(HDF5 REQUIRED HINTS ${HDF5_DIR})
|
|
|
08740b8 |
+else(MSVC)
|
|
|
08740b8 |
+ find_package(HDF5 REQUIRED)
|
|
|
08740b8 |
+endif(MSVC)
|
|
|
08740b8 |
+include_directories(${HDF5_INCLUDE_DIRS})
|
|
|
08740b8 |
|
|
|
08740b8 |
|
|
|
08740b8 |
# targets
|
|
|
08740b8 |
@@ -17,16 +28,26 @@ if(BUILD_STATIC)
|
|
|
08740b8 |
add_library(blosc_filter_static ${SOURCES})
|
|
|
08740b8 |
set_target_properties(
|
|
|
08740b8 |
blosc_filter_static PROPERTIES OUTPUT_NAME blosc_filter)
|
|
|
08740b8 |
- target_link_libraries(blosc_filter_static blosc_static)
|
|
|
08740b8 |
+ target_link_libraries(blosc_filter_static blosc_static ${HDF5_LIBRARIES})
|
|
|
08740b8 |
endif(BUILD_STATIC)
|
|
|
08740b8 |
|
|
|
08740b8 |
+# have to copy blosc dlls for Visual Studio
|
|
|
08740b8 |
+if(MSVC)
|
|
|
08740b8 |
+ add_custom_command(
|
|
|
08740b8 |
+ TARGET blosc_filter_shared
|
|
|
08740b8 |
+ POST_BUILD
|
|
|
08740b8 |
+ COMMAND ${CMAKE_COMMAND}
|
|
|
08740b8 |
+ ARGS -E copy_if_different
|
|
|
08740b8 |
+ "${PROJECT_BINARY_DIR}/blosc/\$\(Configuration\)/blosc.dll"
|
|
|
08740b8 |
+ "${CMAKE_CURRENT_BINARY_DIR}/\$\(Configuration\)/blosc.dll")
|
|
|
08740b8 |
+endif(MSVC)
|
|
|
08740b8 |
|
|
|
08740b8 |
# install
|
|
|
08740b8 |
install(FILES blosc_filter.h DESTINATION include COMPONENT HDF5_FILTER_DEV)
|
|
|
08740b8 |
-install(TARGETS blosc_filter_static DESTINATION lib COMPONENT HDF5_FILTER)
|
|
|
08740b8 |
+install(TARGETS blosc_filter_shared DESTINATION lib COMPONENT HDF5_FILTER)
|
|
|
08740b8 |
if(BUILD_STATIC)
|
|
|
08740b8 |
install(
|
|
|
08740b8 |
- TARGETS blosc_filter_shared DESTINATION lib COMPONENT HDF5_FILTER_DEV)
|
|
|
08740b8 |
+ TARGETS blosc_filter_static DESTINATION lib COMPONENT HDF5_FILTER_DEV)
|
|
|
08740b8 |
endif(BUILD_STATIC)
|
|
|
08740b8 |
|
|
|
08740b8 |
|
|
|
08740b8 |
diff --git a/c-blosc/hdf5/README.rst b/c-blosc/hdf5/README.rst
|
|
|
08740b8 |
index 15c6b35fc0..7aae585b2a 100644
|
|
|
08740b8 |
--- a/c-blosc/hdf5/README.rst
|
|
|
08740b8 |
+++ b/c-blosc/hdf5/README.rst
|
|
|
08740b8 |
@@ -17,10 +17,16 @@ value is returned.
|
|
|
08740b8 |
An example C program ("example.c") is included which demonstrates the
|
|
|
08740b8 |
proper use of the filter.
|
|
|
08740b8 |
|
|
|
08740b8 |
+Alternatively, instead of registering the Blosc filter, you can use the
|
|
|
08740b8 |
+automatically detectable `HDF5 filter plugin`_ which is supported in HDF5
|
|
|
08740b8 |
+1.8.11 and later.
|
|
|
08740b8 |
+
|
|
|
08740b8 |
This filter has been tested against HDF5 versions 1.6.5 through
|
|
|
08740b8 |
1.8.10. It is released under the MIT license (see LICENSE.txt for
|
|
|
08740b8 |
details).
|
|
|
08740b8 |
|
|
|
08740b8 |
+.. _`HDF5 filter plugin`: http://www.hdfgroup.org/HDF5/doc/Advanced/DynamicallyLoadedFilters/HDF5DynamicallyLoadedFilters.pdf
|
|
|
08740b8 |
+
|
|
|
08740b8 |
|
|
|
08740b8 |
Compiling
|
|
|
08740b8 |
=========
|
|
|
08740b8 |
@@ -50,13 +56,101 @@ Using Windows and MSVC (2008 or higher recommended):
|
|
|
08740b8 |
|
|
|
08740b8 |
Intel ICC compilers should work too.
|
|
|
08740b8 |
|
|
|
08740b8 |
+On Windows, you'll need to copy hdf5.dll and possibly the msvc*.dll files
|
|
|
08740b8 |
+to your filter's directory if you do not have HDF5 installed in your PATH.
|
|
|
08740b8 |
+
|
|
|
08740b8 |
For activating the support for other compressors than the integrated
|
|
|
08740b8 |
BloscLZ (like LZ4, LZ4HC, Snappy or Zlib) see the README file in the
|
|
|
08740b8 |
main Blosc directory.
|
|
|
08740b8 |
|
|
|
08740b8 |
|
|
|
08740b8 |
+Compiling dynamically loadable filter plugin
|
|
|
08740b8 |
+============================================
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+Compile blosc_plugin.c and blosc_filter.c to a shared library and then
|
|
|
08740b8 |
+let HDF5 know where to find it.
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+To complie using GCC on UNIX:
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+ gcc -O3 -msse2 -lhdf5 -lpthread ../blosc/*.c \
|
|
|
08740b8 |
+ blosc_filter.c blosc_plugin.c -fPIC -shared \
|
|
|
08740b8 |
+ -o libblosch5plugin.so
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+Then ether move the shared library to HDF5's default search location for
|
|
|
08740b8 |
+plugins (on UNIX ``/usr/local/hdf5/lib/plugin``) or to a directory pointed to
|
|
|
08740b8 |
+by the ``HDF5_PLUGIN_PATH`` environment variable.
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+IMPORTANT WINDOWS NOTE
|
|
|
08740b8 |
+======================
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+On Windows, the configuration (Release vs. Debug) and Visual Studio version
|
|
|
08740b8 |
+of HDF5 and the c-blosc filter must match EXACTLY or you will experience
|
|
|
08740b8 |
+crashes. You should also ensure that the C run-time is dynamically linked
|
|
|
08740b8 |
+to both HDF5 and c-blosc.
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+This is due to the way Microsoft implements its C library. On Windows, the
|
|
|
08740b8 |
+standard C library is not a fundamental part of the operating system, as it
|
|
|
08740b8 |
+is on Unix-like systems. Instead, the C library is implemented in separate
|
|
|
08740b8 |
+shared libraries (dlls - called the C run-time (CRT) by Microsoft), which
|
|
|
08740b8 |
+differ by Visual Studio and configuration. For example, msvcr110d.dll is the
|
|
|
08740b8 |
+Visual Studio 2012 debug C run-time and msvcr90.dll is the Visual Studio
|
|
|
08740b8 |
+2008 release C run-time. Since there is no shared state between these
|
|
|
08740b8 |
+independent libraries, allocating memory in one library and freeing it in
|
|
|
08740b8 |
+another (as the c-blosc HDF5 filter does) will corrupt the heap and cause
|
|
|
08740b8 |
+crashes.
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+There is currently no way around this issue since a fix involves exposing
|
|
|
08740b8 |
+the HDF5 library's memory management functions for filter author use, which
|
|
|
08740b8 |
+would ensure that both the filter and HDF5 use the same allocation and
|
|
|
08740b8 |
+free functions. The HDF Group is aware of the problem and hopes to have a
|
|
|
08740b8 |
+fix in HDF5 1.8.15 (May 2015).
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+To duplicate the problem
|
|
|
08740b8 |
+------------------------
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+* Install the HDF5 binary distribution. The HDF5 binaries are built in release mode (even though they include debugging symbols) and link to the release C run-time.
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+* Configure and build c-blosc using the debug configuration. Ensure that CMake uses the installed release-configuration HDF5.
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+* You may need to copy hdf5.dll and the msvc*.dll libraries to the filter's binary directory if the HDF5 bin directory is not in your PATH.
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+* At this point, HDF5 will be using the release C run-time and c-blosc will be using the debug C run-time. You can confirm this using the Visual Studio tool 'dumpbin /imports'.
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+* Run example.exe. It should crash.
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+If you build the HDF5 library from source in the debug configuration,
|
|
|
08740b8 |
+you can confirm that debug HDF5 and release c-blosc will also cause
|
|
|
08740b8 |
+example.exe to fail.
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+Note that the crashes may not be deterministic. Your mileage may vary.
|
|
|
08740b8 |
+Regardless of the behavior on your particular system, this is a serious
|
|
|
08740b8 |
+problem and will crash many, if not most, systems.
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+To demonstrate proper behavior
|
|
|
08740b8 |
+------------------------------
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+* Build c-blosc in the configuration that matches HDF5.
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+* example.exe should now run normally.
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+To confirm that it is a C run-time mismatch issue, you can modify the
|
|
|
08740b8 |
+src/H5.c and src/H5public.h files in the HDF5 source distribution to
|
|
|
08740b8 |
+expose the HDF5 library's allocator (H5free_memory() already exists).
|
|
|
08740b8 |
+Simply copy and modify the H5free_memory() function to something like
|
|
|
08740b8 |
+H5malloc() that wraps malloc(). You'll need to run 'bin/trace src/H5.c'
|
|
|
08740b8 |
+in the source root to generate a TRACE macro for the new API call
|
|
|
08740b8 |
+(requires Perl). Modify the filter to use H5malloc() and H5free_memory()
|
|
|
08740b8 |
+in place of malloc() and free() and rebuild c-blosc. You will now be
|
|
|
08740b8 |
+able to combine release and debug configurations without example.exe
|
|
|
08740b8 |
+crashing.
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+
|
|
|
08740b8 |
Acknowledgments
|
|
|
08740b8 |
===============
|
|
|
08740b8 |
|
|
|
08740b8 |
This HDF5 filter interface and its example is based in the LZF interface
|
|
|
08740b8 |
(http://h5py.alfven.org) by Andrew Collette.
|
|
|
08740b8 |
+
|
|
|
08740b8 |
+Dana Robinson made nice improvements on existing CMake files for
|
|
|
08740b8 |
+Windows/MSVC.
|
|
|
08740b8 |
diff --git a/c-blosc/hdf5/blosc_filter.c b/c-blosc/hdf5/blosc_filter.c
|
|
|
08740b8 |
index aa3521545d..3ab5237365 100644
|
|
|
08740b8 |
--- a/c-blosc/hdf5/blosc_filter.c
|
|
|
08740b8 |
+++ b/c-blosc/hdf5/blosc_filter.c
|
|
|
08740b8 |
@@ -105,7 +105,7 @@ herr_t blosc_set_local(hid_t dcpl, hid_t type, hid_t space){
|
|
|
08740b8 |
size_t nelements = 8;
|
|
|
08740b8 |
unsigned int values[] = {0,0,0,0,0,0,0,0};
|
|
|
08740b8 |
hid_t super_type;
|
|
|
08740b8 |
- H5T_class_t class;
|
|
|
08740b8 |
+ H5T_class_t classt;
|
|
|
08740b8 |
|
|
|
08740b8 |
r = GET_FILTER(dcpl, FILTER_BLOSC, &flags, &nelements, values, 0, NULL);
|
|
|
08740b8 |
if(r<0) return -1;
|
|
|
08740b8 |
@@ -126,8 +126,8 @@ herr_t blosc_set_local(hid_t dcpl, hid_t type, hid_t space){
|
|
|
08740b8 |
typesize = H5Tget_size(type);
|
|
|
08740b8 |
if (typesize==0) return -1;
|
|
|
08740b8 |
/* Get the size of the base type, even for ARRAY types */
|
|
|
08740b8 |
- class = H5Tget_class(type);
|
|
|
08740b8 |
- if (class == H5T_ARRAY) {
|
|
|
08740b8 |
+ classt = H5Tget_class(type);
|
|
|
08740b8 |
+ if (classt == H5T_ARRAY) {
|
|
|
08740b8 |
/* Get the array base component */
|
|
|
08740b8 |
super_type = H5Tget_super(type);
|
|
|
08740b8 |
basetypesize = H5Tget_size(super_type);
|
|
|
08740b8 |
@@ -175,7 +175,7 @@ size_t blosc_filter(unsigned flags, size_t cd_nelmts,
|
|
|
08740b8 |
int doshuffle = 1; /* Shuffle default */
|
|
|
08740b8 |
int compcode; /* Blosc compressor */
|
|
|
08740b8 |
int code;
|
|
|
08740b8 |
- char *compname = NULL;
|
|
|
08740b8 |
+ char *compname = "blosclz"; /* The compressor by default */
|
|
|
08740b8 |
char *complist;
|
|
|
08740b8 |
char errmsg[256];
|
|
|
08740b8 |
|
|
|
08740b8 |
@@ -234,10 +234,6 @@ size_t blosc_filter(unsigned flags, size_t cd_nelmts,
|
|
|
08740b8 |
}
|
|
|
08740b8 |
|
|
|
08740b8 |
#if ( (BLOSC_VERSION_MAJOR <= 1) && (BLOSC_VERSION_MINOR < 5) )
|
|
|
08740b8 |
- /* Select the correct compressor to use */
|
|
|
08740b8 |
- if (compname != NULL)
|
|
|
08740b8 |
- blosc_set_compressor(compname);
|
|
|
08740b8 |
-
|
|
|
08740b8 |
status = blosc_compress(clevel, doshuffle, typesize, nbytes,
|
|
|
08740b8 |
*buf, outbuf, nbytes);
|
|
|
08740b8 |
#else
|
|
|
08740b8 |
@@ -281,7 +277,7 @@ size_t blosc_filter(unsigned flags, size_t cd_nelmts,
|
|
|
08740b8 |
}
|
|
|
08740b8 |
|
|
|
08740b8 |
#if ( (BLOSC_VERSION_MAJOR <= 1) && (BLOSC_VERSION_MINOR < 5) )
|
|
|
08740b8 |
- status = blosc_decompress(*buf, outbuf, outbuf_size);
|
|
|
08740b8 |
+ status = blosc_decompress(*buf, outbuf, outbuf_size);
|
|
|
08740b8 |
#else
|
|
|
08740b8 |
/* Starting from Blosc 1.5 on, there is not an internal global
|
|
|
08740b8 |
lock anymore, so do not try to run in multithreading mode
|
|
|
08740b8 |
diff --git a/c-blosc/hdf5/example.c b/c-blosc/hdf5/example.c
|
|
|
08740b8 |
index 3b386e330d..d72b3fc7b8 100644
|
|
|
08740b8 |
--- a/c-blosc/hdf5/example.c
|
|
|
08740b8 |
+++ b/c-blosc/hdf5/example.c
|
|
|
08740b8 |
@@ -1,6 +1,6 @@
|
|
|
08740b8 |
/*
|
|
|
08740b8 |
Copyright (C) 2010 Francesc Alted
|
|
|
08740b8 |
- http://blosc.pytables.org
|
|
|
08740b8 |
+ http://blosc.org
|
|
|
08740b8 |
License: MIT (see LICENSE.txt)
|
|
|
08740b8 |
|
|
|
08740b8 |
Example program demonstrating use of the Blosc filter from C code.
|
|
|
08740b8 |
@@ -9,8 +9,7 @@
|
|
|
08740b8 |
|
|
|
08740b8 |
To compile this program:
|
|
|
08740b8 |
|
|
|
08740b8 |
- h5cc [-DH5_USE_16_API] -lblosc blosc_filter.c example.c \
|
|
|
08740b8 |
- -o example -lpthread
|
|
|
08740b8 |
+ h5cc blosc_filter.c example.c -o example -lblosc -lpthread
|
|
|
08740b8 |
|
|
|
08740b8 |
To run:
|
|
|
08740b8 |
|
|
|
08740b8 |
@@ -73,7 +72,7 @@ int main(){
|
|
|
08740b8 |
r = H5Pset_chunk(plist, 3, chunkshape);
|
|
|
08740b8 |
if(r<0) goto failed;
|
|
|
08740b8 |
|
|
|
08740b8 |
- /* Using the blosc filter in combianation with other ones also works */
|
|
|
08740b8 |
+ /* Using the blosc filter in combination with other ones also works */
|
|
|
08740b8 |
/*
|
|
|
08740b8 |
r = H5Pset_fletcher32(plist);
|
|
|
08740b8 |
if(r<0) goto failed;
|
|
|
08740b8 |
@@ -81,7 +80,7 @@ int main(){
|
|
|
08740b8 |
|
|
|
08740b8 |
/* This is the easiest way to call Blosc with default values: 5
|
|
|
08740b8 |
for BloscLZ and shuffle active. */
|
|
|
08740b8 |
- /* r = H5Pset_filter(plist, FILTER_BLOSC, H5Z_FLAG_OPTIONAL, 0, NULL); */
|
|
|
08740b8 |
+ /* r = H5Pset_filter(plist, FILTER_BLOSC, H5Z_FLAG_OPTIONAL, 0, NULL); */
|
|
|
08740b8 |
|
|
|
08740b8 |
/* But you can also taylor Blosc parameters to your needs */
|
|
|
08740b8 |
/* 0 to 3 (inclusive) param slots are reserved. */
|