diff --git a/c-blosc/hdf5/CMakeLists.txt b/c-blosc/hdf5/CMakeLists.txt index d9cd847e89..d91194fcf7 100644 --- a/c-blosc/hdf5/CMakeLists.txt +++ b/c-blosc/hdf5/CMakeLists.txt @@ -4,8 +4,19 @@ set(SOURCES blosc_filter.c) include_directories("${PROJECT_SOURCE_DIR}/blosc") # dependencies -find_package(HDF5 REQUIRED) -include_directories(HDF5_INCLIDE_DIRS) +if(MSVC) + # FindHDF5.cmake does not find Windows installations. Try to + # use an environment variable instead until the official "find" + # file can be updated for Windows. + # + # Note that you have to set this environment variable by hand. + file(TO_CMAKE_PATH "$ENV{HDF5_DIR}" HDF5_HINT) + set(HDF5_DIR ${HDF5_HINT} CACHE STRING "Path to HDF5 CMake config directory.") + find_package(HDF5 REQUIRED HINTS ${HDF5_DIR}) +else(MSVC) + find_package(HDF5 REQUIRED) +endif(MSVC) +include_directories(${HDF5_INCLUDE_DIRS}) # targets @@ -17,16 +28,26 @@ if(BUILD_STATIC) add_library(blosc_filter_static ${SOURCES}) set_target_properties( blosc_filter_static PROPERTIES OUTPUT_NAME blosc_filter) - target_link_libraries(blosc_filter_static blosc_static) + target_link_libraries(blosc_filter_static blosc_static ${HDF5_LIBRARIES}) endif(BUILD_STATIC) +# have to copy blosc dlls for Visual Studio +if(MSVC) + add_custom_command( + TARGET blosc_filter_shared + POST_BUILD + COMMAND ${CMAKE_COMMAND} + ARGS -E copy_if_different + "${PROJECT_BINARY_DIR}/blosc/\$\(Configuration\)/blosc.dll" + "${CMAKE_CURRENT_BINARY_DIR}/\$\(Configuration\)/blosc.dll") +endif(MSVC) # install install(FILES blosc_filter.h DESTINATION include COMPONENT HDF5_FILTER_DEV) -install(TARGETS blosc_filter_static DESTINATION lib COMPONENT HDF5_FILTER) +install(TARGETS blosc_filter_shared DESTINATION lib COMPONENT HDF5_FILTER) if(BUILD_STATIC) install( - TARGETS blosc_filter_shared DESTINATION lib COMPONENT HDF5_FILTER_DEV) + TARGETS blosc_filter_static DESTINATION lib COMPONENT HDF5_FILTER_DEV) endif(BUILD_STATIC) diff --git a/c-blosc/hdf5/README.rst b/c-blosc/hdf5/README.rst index 15c6b35fc0..7aae585b2a 100644 --- a/c-blosc/hdf5/README.rst +++ b/c-blosc/hdf5/README.rst @@ -17,10 +17,16 @@ value is returned. An example C program ("example.c") is included which demonstrates the proper use of the filter. +Alternatively, instead of registering the Blosc filter, you can use the +automatically detectable `HDF5 filter plugin`_ which is supported in HDF5 +1.8.11 and later. + This filter has been tested against HDF5 versions 1.6.5 through 1.8.10. It is released under the MIT license (see LICENSE.txt for details). +.. _`HDF5 filter plugin`: http://www.hdfgroup.org/HDF5/doc/Advanced/DynamicallyLoadedFilters/HDF5DynamicallyLoadedFilters.pdf + Compiling ========= @@ -50,13 +56,101 @@ Using Windows and MSVC (2008 or higher recommended): Intel ICC compilers should work too. +On Windows, you'll need to copy hdf5.dll and possibly the msvc*.dll files +to your filter's directory if you do not have HDF5 installed in your PATH. + For activating the support for other compressors than the integrated BloscLZ (like LZ4, LZ4HC, Snappy or Zlib) see the README file in the main Blosc directory. +Compiling dynamically loadable filter plugin +============================================ + +Compile blosc_plugin.c and blosc_filter.c to a shared library and then +let HDF5 know where to find it. + +To complie using GCC on UNIX: + + gcc -O3 -msse2 -lhdf5 -lpthread ../blosc/*.c \ + blosc_filter.c blosc_plugin.c -fPIC -shared \ + -o libblosch5plugin.so + +Then ether move the shared library to HDF5's default search location for +plugins (on UNIX ``/usr/local/hdf5/lib/plugin``) or to a directory pointed to +by the ``HDF5_PLUGIN_PATH`` environment variable. + + +IMPORTANT WINDOWS NOTE +====================== + +On Windows, the configuration (Release vs. Debug) and Visual Studio version +of HDF5 and the c-blosc filter must match EXACTLY or you will experience +crashes. You should also ensure that the C run-time is dynamically linked +to both HDF5 and c-blosc. + +This is due to the way Microsoft implements its C library. On Windows, the +standard C library is not a fundamental part of the operating system, as it +is on Unix-like systems. Instead, the C library is implemented in separate +shared libraries (dlls - called the C run-time (CRT) by Microsoft), which +differ by Visual Studio and configuration. For example, msvcr110d.dll is the +Visual Studio 2012 debug C run-time and msvcr90.dll is the Visual Studio +2008 release C run-time. Since there is no shared state between these +independent libraries, allocating memory in one library and freeing it in +another (as the c-blosc HDF5 filter does) will corrupt the heap and cause +crashes. + +There is currently no way around this issue since a fix involves exposing +the HDF5 library's memory management functions for filter author use, which +would ensure that both the filter and HDF5 use the same allocation and +free functions. The HDF Group is aware of the problem and hopes to have a +fix in HDF5 1.8.15 (May 2015). + +To duplicate the problem +------------------------ + +* Install the HDF5 binary distribution. The HDF5 binaries are built in release mode (even though they include debugging symbols) and link to the release C run-time. + +* Configure and build c-blosc using the debug configuration. Ensure that CMake uses the installed release-configuration HDF5. + +* You may need to copy hdf5.dll and the msvc*.dll libraries to the filter's binary directory if the HDF5 bin directory is not in your PATH. + +* At this point, HDF5 will be using the release C run-time and c-blosc will be using the debug C run-time. You can confirm this using the Visual Studio tool 'dumpbin /imports'. + +* Run example.exe. It should crash. + +If you build the HDF5 library from source in the debug configuration, +you can confirm that debug HDF5 and release c-blosc will also cause +example.exe to fail. + +Note that the crashes may not be deterministic. Your mileage may vary. +Regardless of the behavior on your particular system, this is a serious +problem and will crash many, if not most, systems. + +To demonstrate proper behavior +------------------------------ + +* Build c-blosc in the configuration that matches HDF5. + +* example.exe should now run normally. + +To confirm that it is a C run-time mismatch issue, you can modify the +src/H5.c and src/H5public.h files in the HDF5 source distribution to +expose the HDF5 library's allocator (H5free_memory() already exists). +Simply copy and modify the H5free_memory() function to something like +H5malloc() that wraps malloc(). You'll need to run 'bin/trace src/H5.c' +in the source root to generate a TRACE macro for the new API call +(requires Perl). Modify the filter to use H5malloc() and H5free_memory() +in place of malloc() and free() and rebuild c-blosc. You will now be +able to combine release and debug configurations without example.exe +crashing. + + Acknowledgments =============== This HDF5 filter interface and its example is based in the LZF interface (http://h5py.alfven.org) by Andrew Collette. + +Dana Robinson made nice improvements on existing CMake files for +Windows/MSVC. diff --git a/c-blosc/hdf5/blosc_filter.c b/c-blosc/hdf5/blosc_filter.c index aa3521545d..3ab5237365 100644 --- a/c-blosc/hdf5/blosc_filter.c +++ b/c-blosc/hdf5/blosc_filter.c @@ -105,7 +105,7 @@ herr_t blosc_set_local(hid_t dcpl, hid_t type, hid_t space){ size_t nelements = 8; unsigned int values[] = {0,0,0,0,0,0,0,0}; hid_t super_type; - H5T_class_t class; + H5T_class_t classt; r = GET_FILTER(dcpl, FILTER_BLOSC, &flags, &nelements, values, 0, NULL); if(r<0) return -1; @@ -126,8 +126,8 @@ herr_t blosc_set_local(hid_t dcpl, hid_t type, hid_t space){ typesize = H5Tget_size(type); if (typesize==0) return -1; /* Get the size of the base type, even for ARRAY types */ - class = H5Tget_class(type); - if (class == H5T_ARRAY) { + classt = H5Tget_class(type); + if (classt == H5T_ARRAY) { /* Get the array base component */ super_type = H5Tget_super(type); basetypesize = H5Tget_size(super_type); @@ -175,7 +175,7 @@ size_t blosc_filter(unsigned flags, size_t cd_nelmts, int doshuffle = 1; /* Shuffle default */ int compcode; /* Blosc compressor */ int code; - char *compname = NULL; + char *compname = "blosclz"; /* The compressor by default */ char *complist; char errmsg[256]; @@ -234,10 +234,6 @@ size_t blosc_filter(unsigned flags, size_t cd_nelmts, } #if ( (BLOSC_VERSION_MAJOR <= 1) && (BLOSC_VERSION_MINOR < 5) ) - /* Select the correct compressor to use */ - if (compname != NULL) - blosc_set_compressor(compname); - status = blosc_compress(clevel, doshuffle, typesize, nbytes, *buf, outbuf, nbytes); #else @@ -281,7 +277,7 @@ size_t blosc_filter(unsigned flags, size_t cd_nelmts, } #if ( (BLOSC_VERSION_MAJOR <= 1) && (BLOSC_VERSION_MINOR < 5) ) - status = blosc_decompress(*buf, outbuf, outbuf_size); + status = blosc_decompress(*buf, outbuf, outbuf_size); #else /* Starting from Blosc 1.5 on, there is not an internal global lock anymore, so do not try to run in multithreading mode diff --git a/c-blosc/hdf5/example.c b/c-blosc/hdf5/example.c index 3b386e330d..d72b3fc7b8 100644 --- a/c-blosc/hdf5/example.c +++ b/c-blosc/hdf5/example.c @@ -1,6 +1,6 @@ /* Copyright (C) 2010 Francesc Alted - http://blosc.pytables.org + http://blosc.org License: MIT (see LICENSE.txt) Example program demonstrating use of the Blosc filter from C code. @@ -9,8 +9,7 @@ To compile this program: - h5cc [-DH5_USE_16_API] -lblosc blosc_filter.c example.c \ - -o example -lpthread + h5cc blosc_filter.c example.c -o example -lblosc -lpthread To run: @@ -73,7 +72,7 @@ int main(){ r = H5Pset_chunk(plist, 3, chunkshape); if(r<0) goto failed; - /* Using the blosc filter in combianation with other ones also works */ + /* Using the blosc filter in combination with other ones also works */ /* r = H5Pset_fletcher32(plist); if(r<0) goto failed; @@ -81,7 +80,7 @@ int main(){ /* This is the easiest way to call Blosc with default values: 5 for BloscLZ and shuffle active. */ - /* r = H5Pset_filter(plist, FILTER_BLOSC, H5Z_FLAG_OPTIONAL, 0, NULL); */ + /* r = H5Pset_filter(plist, FILTER_BLOSC, H5Z_FLAG_OPTIONAL, 0, NULL); */ /* But you can also taylor Blosc parameters to your needs */ /* 0 to 3 (inclusive) param slots are reserved. */