diff --git a/MANIFEST.in b/MANIFEST.in index 0df736a7..e605d418 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,5 +3,5 @@ include *.txt include setup.py VERSION recursive-include blosc *.py *.c -recursive-include c-blosc *.c *.h +recursive-include c-blosc/blosc *.c *.h recursive-include LICENSES * diff --git a/README.rst b/README.rst index aa5ba026..73701159 100644 --- a/README.rst +++ b/README.rst @@ -59,6 +59,47 @@ Also, some examples are available on python-blosc wiki page: http://github.com/FrancescAlted/python-blosc/wiki +Merging Blosc sources from upstream +=================================== + +We use the `subtree merge technique +`_ to maintain the +upstream Blosc sources. In case you need to synchronise, the following recipe +may help to get setup the first time. + +1) Add the upstream Blosc sources as an additional remote called +``c-blosc-origin``:: + + $ git remote add -f c-blosc-origin git://github.com/FrancescAlted/blosc.git + +2) Checkout the ``master`` branch as ``c-blosc``:: + + $ git co -b c-blosc c-blosc-origin/master + +3) Reset the ``c-blosc`` branch to the desired tag:: + + $ git reset --hard vX.Y.Z + +4) Checkout the branch you want to subtree merge to:: + + $ git checkout master + +5) Actually perform the subtree merge:: + + $ git merge --squash -s subtree --no-commit c-blosc + +6) Finalize the subtree merge with a commit:: + + $ git commit -m "subtree merge blosc vX.Y.Z" + +If you alread have the ``c-blosc-origin`` remote set up and the ``c-blosc`` +branch created, you can just update it:: + + $ git checkout c-blosc + $ git pull + +And then proceed with step 3 above. + Mailing list ============ diff --git a/c-blosc/.gitignore b/c-blosc/.gitignore new file mode 100644 index 00000000..faf2156b --- /dev/null +++ b/c-blosc/.gitignore @@ -0,0 +1 @@ +bench/bench diff --git a/c-blosc/ANNOUNCE.rst b/c-blosc/ANNOUNCE.rst new file mode 100644 index 00000000..87d5d7cf --- /dev/null +++ b/c-blosc/ANNOUNCE.rst @@ -0,0 +1,59 @@ +=============================================================== + Announcing Blosc 1.1.5 + A blocking, shuffling and lossless compression library +=============================================================== + +What is new? +============ + +This is maintenance release fixing an issue that avoided compilation +with MSVC. + +For more info, please see the release notes in: + +https://github.com/FrancescAlted/blosc/wiki/Release-notes + +What is it? +=========== + +Blosc (http://blosc.pytables.org) is a high performance compressor +optimized for binary data. It has been designed to transmit data to +the processor cache faster than the traditional, non-compressed, +direct memory fetch approach via a memcpy() OS call. + +Blosc is the first compressor (that I'm aware of) that is meant not +only to reduce the size of large datasets on-disk or in-memory, but +also to accelerate object manipulations that are memory-bound. + +It also comes with a filter for HDF5 (http://www.hdfgroup.org/HDF5) so +that you can easily implement support for Blosc in your favourite HDF5 +tool. + +Download sources +================ + +Please go to main web site: + +http://blosc.pytables.org/sources/ + +or the github repository: + +https://github.com/FrancescAlted/blosc + +and download the most recent release from there. + +Blosc is distributed using the MIT license, see LICENSES/BLOSC.txt for +details. + +Mailing list +============ + +There is an official Blosc blosc mailing list at: + +blosc@googlegroups.com +http://groups.google.es/group/blosc + + +---- + + **Enjoy data!** diff --git a/c-blosc/LICENSES/BLOSC.txt b/c-blosc/LICENSES/BLOSC.txt new file mode 100644 index 00000000..ddf571f0 --- /dev/null +++ b/c-blosc/LICENSES/BLOSC.txt @@ -0,0 +1,22 @@ +Blosc - A blocking, shuffling and lossless compression library + +Copyright (C) 2009-2010 Francesc Alted (faltet@pytables.org) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/c-blosc/LICENSES/FASTLZ.txt b/c-blosc/LICENSES/FASTLZ.txt new file mode 100644 index 00000000..4a6abd6a --- /dev/null +++ b/c-blosc/LICENSES/FASTLZ.txt @@ -0,0 +1,24 @@ +FastLZ - lightning-fast lossless compression library + +Copyright (C) 2007 Ariya Hidayat (ariya@kde.org) +Copyright (C) 2006 Ariya Hidayat (ariya@kde.org) +Copyright (C) 2005 Ariya Hidayat (ariya@kde.org) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/c-blosc/LICENSES/H5PY.txt b/c-blosc/LICENSES/H5PY.txt new file mode 100644 index 00000000..15b30f29 --- /dev/null +++ b/c-blosc/LICENSES/H5PY.txt @@ -0,0 +1,34 @@ +Copyright Notice and Statement for the h5py Project + +Copyright (c) 2008 Andrew Collette +http://h5py.alfven.org +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +a. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +b. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the + distribution. + +c. Neither the name of the author nor the names of contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/c-blosc/LICENSES/STDINT.txt b/c-blosc/LICENSES/STDINT.txt new file mode 100644 index 00000000..7e9941ad --- /dev/null +++ b/c-blosc/LICENSES/STDINT.txt @@ -0,0 +1,25 @@ +Copyright (c) 2006-2008 Alexander Chemeris + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. The name of the author may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/c-blosc/README.rst b/c-blosc/README.rst new file mode 100644 index 00000000..799e3fe3 --- /dev/null +++ b/c-blosc/README.rst @@ -0,0 +1,163 @@ +=============================================================== + Blosc: A blocking, shuffling and lossless compression library +=============================================================== + +:Author: Francesc Alted i Abad +:Contact: faltet@pytables.org +:URL: http://blosc.pytables.org + +What is it? +=========== + +Blosc [1]_ is a high performance compressor optimized for binary data. +It has been designed to transmit data to the processor cache faster +than the traditional, non-compressed, direct memory fetch approach via +a memcpy() OS call. Blosc is the first compressor (that I'm aware of) +that is meant not only to reduce the size of large datasets on-disk or +in-memory, but also to accelerate memory-bound computations. + +It uses the blocking technique (as described in [2]_) to reduce +activity on the memory bus as much as possible. In short, this +technique works by dividing datasets in blocks that are small enough +to fit in caches of modern processors and perform compression / +decompression there. It also leverages, if available, SIMD +instructions (SSE2) and multi-threading capabilities of CPUs, in order +to accelerate the compression / decompression process to a maximum. + +You can see some recent benchmarks about Blosc performance in [3]_ + +Blosc is distributed using the MIT license, see LICENSES/BLOSC.txt for +details. + +.. [1] http://blosc.pytables.org +.. [2] http://www.pytables.org/docs/CISE-12-2-ScientificPro.pdf +.. [3] http://blosc.pytables.org/trac/wiki/SyntheticBenchmarks + +Meta-compression and other advantages over existing compressors +=============================================================== + +Blosc is not like other compressors: it should rather be called a +meta-compressor. This is so because it can use different compressors +and pre-conditioners (programs that generally improve compression +ratio). At any rate, it can also be called a compressor because it +happens that it already integrates one compressor and one +pre-conditioner, so it can actually work like so. + +Currently it uses BloscLZ, a compressor heavily based on FastLZ +(http://fastlz.org/), and a highly optimized (it can use SSE2 +instructions, if available) Shuffle pre-conditioner. However, +different compressors or pre-conditioners may be added in the future. + +Blosc is in charge of coordinating the compressor and pre-conditioners +so that they can leverage the blocking technique (described above) as +well as multi-threaded execution (if several cores are available) +automatically. That makes that every compressor and pre-conditioner +will work at very high speeds, even if it was not initially designed +for doing blocking or multi-threading. + +Other advantages of Blosc are: + +* Meant for binary data: can take advantage of the type size + meta-information for improved compression ratio (using the + integrated shuffle pre-conditioner). + +* Small overhead on non-compressible data: only a maximum of 16 + additional bytes over the source buffer length are needed to + compress *every* input. + +* Maximum destination length: contrarily to many other + compressors, both compression and decompression routines have + support for maximum size lengths for the destination buffer. + +* Replacement for memcpy(): it supports a 0 compression level that + does not compress at all and only adds 16 bytes of overhead. In + this mode Blosc can copy memory usually faster than a plain + memcpy(). + +When taken together, all these features set Blosc apart from other +similar solutions. + +Compiling your application with Blosc +===================================== + +Blosc consists of the next files (in blosc/ directory):: + + blosc.h and blosc.c -- the main routines + blosclz.h and blosclz.c -- the actual compressor + shuffle.h and shuffle.c -- the shuffle code + +Just add these files to your project in order to use Blosc. For +information on compression and decompression routines, see blosc.h. + +To compile using GCC (4.4 or higher recommended) on Unix:: + + gcc -O3 -msse2 -o myprog myprog.c blosc/*.c -lpthread + +Using Windows and MINGW:: + + gcc -O3 -msse2 -o myprog myprog.c blosc\*.c + +Using Windows and MSVC (2008 or higher recommended):: + + cl /Ox /Femyprog.exe myprog.c blosc\*.c + +A simple usage example is the benchmark in the bench/bench.c file. +Also, another example for using Blosc as a generic HDF5 filter is in +the hdf5/ directory. + +I have not tried to compile this with compilers other than GCC, MINGW, +Intel ICC or MSVC yet. Please report your experiences with your own +platforms. + +Testing Blosc +============= + +Go to the test/ directory and issue:: + + $ make test + +These tests are very basic, and only valid for platforms where GNU +make/gcc tools are available. If you really want to test Blosc the +hard way, look at: + +http://blosc.pytables.org/trac/wiki/SyntheticBenchmarks + +where instructions on how to intensively test (and benchmark) Blosc +are given. If while running these tests you get some error, please +report it back! + +Wrapper for Python +================== + +Blosc has an official wrapper for Python. See: + +https://github.com/FrancescAlted/python-blosc + +Filter for HDF5 +=============== + +For those that want to use Blosc as a filter in the HDF5 library, +there is a sample implementation in the hdf5/ directory. + +Mailing list +============ + +There is an official mailing list for Blosc at: + +blosc@googlegroups.com +http://groups.google.es/group/blosc + +Acknowledgments +=============== + +I'd like to thank the PyTables community that have collaborated in the +exhaustive testing of Blosc. With an aggregate amount of more than +300 TB of different datasets compressed *and* decompressed +successfully, I can say that Blosc is pretty safe now and ready for +production purposes. Also, Valentin Haenel did a terrific work fixing +typos and improving docs and the plotting script. + + +---- + + **Enjoy data!** diff --git a/c-blosc/README_HEADER.rst b/c-blosc/README_HEADER.rst new file mode 100644 index 00000000..315467f0 --- /dev/null +++ b/c-blosc/README_HEADER.rst @@ -0,0 +1,37 @@ +Blosc Header Format +=================== + +Blosc (as of Version 1.0.0) has the following 16 byte header that stores +information about the compressed buffer:: + + |-0-|-1-|-2-|-3-|-4-|-5-|-6-|-7-|-8-|-9-|-A-|-B-|-C-|-D-|-E-|-F-| + ^ ^ ^ ^ | nbytes | blocksize | ctbytes | + | | | | + | | | +--typesize + | | +------flags + | +----------versionlz + +--------------version + +Datatypes of the Header Entries +------------------------------- + +All entries are little endian. + +:version: + ``uint8`` +:versionlz: + ``uint8`` +:flags (bitfield): + :bit 0 (``0x01``): + whether the shuffle filter has been applied or not + :bit 1 (``0x02``): + whether the internal buffer is a pure memcpy or not +:typesize: + ``uint8`` +:nbytes: + ``uint32`` +:blocksize: + ``uint32`` +:ctbytes: + ``uint32`` + diff --git a/c-blosc/README_THREADED.rst b/c-blosc/README_THREADED.rst new file mode 100644 index 00000000..4d427f94 --- /dev/null +++ b/c-blosc/README_THREADED.rst @@ -0,0 +1,33 @@ +Blosc supports threading +======================== + +Threads are the most efficient way to program parallel code for +multi-core processors, but also the more difficult to program well. +Also, they has a non-negligible start-up time that does not fit well +with a high-performance compressor as Blosc tries to be. + +In order to reduce the overhead of threads as much as possible, I've +decided to implement a pool of threads (the workers) that are waiting +for the main process (the master) to send them jobs (basically, +compressing and decompressing small blocks of the initial buffer). + +Despite this and many other internal optimizations in the threaded +code, it does not work faster than the serial version for buffer sizes +around 64/128 KB or less. This is for Intel Quad Core2 (Q8400 @ 2.66 +GHz) / Linux (openSUSE 11.2, 64 bit), but your mileage may vary (and +will vary!) for other processors / operating systems. + +In contrast, for buffers larger than 64/128 KB, the threaded version +starts to perform significantly better, being the sweet point at 1 MB +(again, this is with my setup). For larger buffer sizes than 1 MB, +the threaded code slows down again, but it is probably due to a cache +size issue and besides, it is still considerably faster than serial +code. + +This is why Blosc falls back to use the serial version for such a +'small' buffers. So, you don't have to worry too much about deciding +whether you should set the number of threads to 1 (serial) or more +(parallel). Just set it to the number of cores in your processor and +your are done! + +Francesc Alted diff --git a/c-blosc/RELEASE_NOTES.rst b/c-blosc/RELEASE_NOTES.rst new file mode 100644 index 00000000..e9544017 --- /dev/null +++ b/c-blosc/RELEASE_NOTES.rst @@ -0,0 +1,186 @@ +=============================== + Release notes for Blosc 1.1.5 +=============================== + +:Author: Francesc Alted i Abad +:Contact: faltet@pytables.org +:URL: http://blosc.pytables.org + + +Changes from 1.1.4 to 1.1.5 +=========================== + +- Fix compile error with msvc compilers (Christoph Gohlke) + + +Changes from 1.1.3 to 1.1.4 +=========================== + +- Redefinition of the BLOSC_MAX_BUFFERSIZE constant as (INT_MAX - + BLOSC_MAX_OVERHEAD) instead of just INT_MAX. This prevents to produce + outputs larger than INT_MAX, which is not supported. + +- `exit()` call has been replaced by a ``return -1`` in blosc_compress() + when checking for buffer sizes. Now programs will not just exit when + the buffer is too large, but return a negative code. + +- Improvements in explicit casts. Blosc compiles without warnings + (with GCC) now. + +- Lots of improvements in docs, in particular a nice ascii-art diagram + of the Blosc format (Valentin Haenel). + +- Improvements to the plot-speeds.py (Valentin Haenel). + +- [HDF5 filter] Adapted HDF5 filter to use HDF5 1.8 by default + (Antonio Valentino). + +- [HDF5 filter] New version of H5Z_class_t definition (Antonio Valentino). + + +Changes from 1.1.2 to 1.1.3 +=========================== + +- Much improved compression ratio when using large blocks (> 64 KB) and + high compression levels (> 6) under some circumstances (special data + distribution). Closes #7. + + +Changes from 1.1.1 to 1.1.2 +=========================== + +- Fixes for small typesizes (#6 and #1 of python-blosc). + + +Changes from 1.1 to 1.1.1 +========================= + +- Added code to avoid calling blosc_set_nthreads more than necessary. + That will improve performance up to 3x or more, specially for small + chunksizes (< 1 MB). + + +Changes from 1.0 to 1.1 +======================= + +- Added code for emulating pthreads API on Windows. No need to link + explicitly with pthreads lib on Windows anymore. However, performance + is a somewhat worse because the new emulation layer does not support + the `pthread_barrier_wait()` call natively. But the big improvement + in installation easiness is worth this penalty (most specially on + 64-bit Windows, where pthreads-win32 support is flaky). + +- New BLOSC_MAX_BUFFERSIZE, BLOSC_MAX_TYPESIZE and BLOSC_MAX_THREADS + symbols are available in blosc.h. These can be useful for validating + parameters in clients. Thanks to Robert Smallshire for suggesting + that. + +- A new BLOSC_MIN_HEADER_LENGTH symbol in blosc.h tells how many bytes + long is the minimum length of a Blosc header. `blosc_cbuffer_sizes()` + only needs these bytes to be passed to work correctly. + +- Removed many warnings (related with potentially dangerous type-casting + code) issued by MSVC 2008 in 64-bit mode. + +- Fixed a problem with the computation of the blocksize in the Blosc + filter for HDF5. + +- Fixed a problem with large datatypes. See + http://www.pytables.org/trac/ticket/288 for more info. + +- Now Blosc is able to work well even if you fork an existing process + with a pool of threads. Bug discovered when PyTables runs in + multiprocess environments. See http://pytables.org/trac/ticket/295 + for details. + +- Added a new `blosc_getitem()` call to allow the retrieval of items in + sizes smaller than the complete buffer. That is useful for the carray + project, but certainly for others too. + + +Changes from 0.9.5 to 1.0 +========================= + +- Added a filter for HDF5 so that people can use Blosc outside PyTables, + if they want to. + +- Many small improvements, specially in README files. + +- Do not assume that size_t is uint_32 for every platform. + +- Added more protection for large buffers or in allocation memory + routines. + +- The src/ directory has been renamed to blosc/. + +- The `maxbytes` parameter in `blosc_compress()` has been renamed to + `destsize`. This is for consistency with the `blosc_decompress()` + parameters. + + +Changes from 0.9.4 to 0.9.5 +=========================== + +- Now, compression level 0 is allowed, meaning not compression at all. + The overhead of this mode will be always BLOSC_MAX_OVERHEAD (16) + bytes. This mode actually represents using Blosc as a basic memory + container. + +- Supported a new parameter `maxbytes` for ``blosc_compress()``. It + represents a maximum of bytes for output. Tests unit added too. + +- Added 3 new functions for querying different metadata on compressed + buffers. A test suite for testing the new API has been added too. + + +Changes from 0.9.3 to 0.9.4 +=========================== + +- Support for cross-platform big/little endian compatibility in Blosc + headers has been added. + +- Fixed several failures exposed by the extremesuite. The problem was a + bad check for limits in the buffer size while compressing. + +- Added a new suite in bench.c called ``debugsuite`` that is + appropriate for debugging purposes. Now, the ``extremesuite`` can be + used for running the complete (and extremely long) suite. + + +Changes from 0.9.0 to 0.9.3 +=========================== + +- Fixed several nasty bugs uncovered by the new suites in bench.c. + Thanks to Tony Theodore and Gabriel Beckers for their (very) + responsive beta testing and feedback. + +- Added several modes (suites), namely ``suite``, ``hardsuite`` and + ``extremehardsuite`` in bench.c so as to allow different levels of + testing. + + +Changes from 0.8.0 to 0.9 +========================= + +- Internal format version bumped to 2 in order to allow an easy way to + indicate that a buffer is being saved uncompressed. This is not + supported yet, but it might be in the future. + +- Blosc can use threads now for leveraging the increasing number of + multi-core processors out there. See README-threaded.txt for more + info. + +- Added a protection for MacOSX so that it has to not link against + posix_memalign() funtion, which seems not available in old versions of + MacOSX (for example, Tiger). At nay rate, posix_memalign() is not + necessary on Mac because 16 bytes alignment is ensured by default. + Thanks to Ivan Vilata. Fixes #3. + + + + +.. Local Variables: +.. mode: rst +.. coding: utf-8 +.. fill-column: 72 +.. End: diff --git a/c-blosc/RELEASING.rst b/c-blosc/RELEASING.rst new file mode 100644 index 00000000..b7c65f38 --- /dev/null +++ b/c-blosc/RELEASING.rst @@ -0,0 +1,104 @@ +================ +Releasing Blosc +================ + +:Author: Francesc Alted +:Contact: faltet@pytables.org +:Date: 2012-09-16 + + +Preliminaries +------------- + +- Make sure that ``RELEASE_NOTES.rst`` and ``ANNOUNCE.rst`` are up to + date with the latest news in the release. + +- Check that *VERSION* symbols in blosc/blosc.h contains the correct info. + +Testing +------- + +Go to the test/ directory and issue:: + + $ make test + +These tests are very basic, and only valid for platforms where GNU +make/gcc tools are available. To actually test Blosc the hard way, +look at: + +http://blosc.pytables.org/trac/wiki/SyntheticBenchmarks + +where instructions on how to intensively test (and benchmark) Blosc +are given. + +Packaging +--------- + +- Unpack the archive of the repository in a temporary directory:: + + $ export VERSION="the version number" + $ mkdir /tmp/blosc-$VERSION + $ git archive master | tar -x -C /tmp/blosc-$VERSION + +- And package the repo:: + + $ cd /tmp + $ tar cvfz blosc-$VERSION.tar.gz blosc-$VERSION + +Do a quick check that the tarball is sane. + + +Uploading +--------- + +- Go to the downloads section of the blosc project in github and + upload the source tarball. + +- Also, for backward compatibility, upload a copy of the tarball in: + +http://blosc.pytables.org/sources/ + +Tagging +------- + +- Create a tag ``X.Y.Z`` from ``master``. Use the next message:: + + $ git tag -a vX.Y.Z -m "Tagging version X.Y.Z" + +- Push the tag to the github repo:: + + $ git push --tags + + +Announcing +---------- + +- Update the release notes in the github wiki: + +https://github.com/FrancescAlted/blosc/wiki/Release-notes + +- Send an announcement to the blosc, pytables, carray and + comp.compression lists. Use the ``ANNOUNCE.rst`` file as skeleton + (possibly as the definitive version). + +Post-release actions +-------------------- + +- Edit *VERSION* symbols in blosc/blosc.h in master to increment the + version to the next minor one (i.e. X.Y.Z --> X.Y.(Z+1).dev). + +- Create new headers for adding new features in ``RELEASE_NOTES.rst`` + and empty the release-specific information in ``ANNOUNCE.rst`` and + add this place-holder instead: + + #XXX version-specific blurb XXX# + + +That's all folks! + + +.. Local Variables: +.. mode: rst +.. coding: utf-8 +.. fill-column: 70 +.. End: diff --git a/c-blosc/bench/Makefile b/c-blosc/bench/Makefile new file mode 100644 index 00000000..4dae287b --- /dev/null +++ b/c-blosc/bench/Makefile @@ -0,0 +1,14 @@ +CC=gcc +CFLAGS=-O3 -g -msse2 -Wall +LDFLAGS=-lpthread +BLOSC_LIB= $(wildcard ../blosc/*.c) +SOURCES=bench.c $(BLOSC_LIB) +EXECUTABLE=bench + +all: $(SOURCES) $(EXECUTABLE) + +$(EXECUTABLE): $(SOURCES) + $(CC) $(CFLAGS) $(LDFLAGS) $(SOURCES) -o $@ + +clean: + rm -rf $(EXECUTABLE) diff --git a/c-blosc/bench/bench.c b/c-blosc/bench/bench.c new file mode 100644 index 00000000..f90544b9 --- /dev/null +++ b/c-blosc/bench/bench.c @@ -0,0 +1,459 @@ +/********************************************************************* + Small benchmark for testing basic capabilities of Blosc. + + You can select different degrees of 'randomness' in input buffer, as + well as external datafiles (uncomment the lines after "For data + coming from a file" comment). + + For usage instructions of this benchmark, please see: + + http://blosc.pytables.org/trac/wiki/SyntheticBenchmarks + + I'm collecting speeds for different machines, so the output of your + benchmarks and your processor specifications are welcome! + + Author: Francesc Alted (faltet@pytables.org) + + See LICENSES/BLOSC.txt for details about copyright and rights to use. +**********************************************************************/ + +#include +#include +#include +#include +#include +#include +#if defined(_WIN32) && !defined(__MINGW32__) + #include +#else + #include + #include +#endif +#include +#include "../blosc/blosc.h" + +#define KB 1024 +#define MB (1024*KB) +#define GB (1024*MB) + +#define NCHUNKS (32*1024) /* maximum number of chunks */ + + +int nchunks = NCHUNKS; +int niter = 3; /* default number of iterations */ +double totalsize = 0.; /* total compressed/decompressed size */ + +#if defined(_WIN32) && !defined(__MINGW32__) +#include +#if defined(_MSC_VER) || defined(_MSC_EXTENSIONS) + #define DELTA_EPOCH_IN_MICROSECS 11644473600000000Ui64 +#else + #define DELTA_EPOCH_IN_MICROSECS 11644473600000000ULL +#endif + +struct timezone +{ + int tz_minuteswest; /* minutes W of Greenwich */ + int tz_dsttime; /* type of dst correction */ +}; + +int gettimeofday(struct timeval *tv, struct timezone *tz) +{ + FILETIME ft; + unsigned __int64 tmpres = 0; + static int tzflag; + + if (NULL != tv) + { + GetSystemTimeAsFileTime(&ft); + + tmpres |= ft.dwHighDateTime; + tmpres <<= 32; + tmpres |= ft.dwLowDateTime; + + /*converting file time to unix epoch*/ + tmpres -= DELTA_EPOCH_IN_MICROSECS; + tmpres /= 10; /*convert into microseconds*/ + tv->tv_sec = (long)(tmpres / 1000000UL); + tv->tv_usec = (long)(tmpres % 1000000UL); + } + + if (NULL != tz) + { + if (!tzflag) + { + _tzset(); + tzflag++; + } + tz->tz_minuteswest = _timezone / 60; + tz->tz_dsttime = _daylight; + } + + return 0; +} +#endif /* _WIN32 */ + + +/* Given two timeval stamps, return the difference in seconds */ +float getseconds(struct timeval last, struct timeval current) { + int sec, usec; + + sec = current.tv_sec - last.tv_sec; + usec = current.tv_usec - last.tv_usec; + return (float)(((double)sec + usec*1e-6)); +} + +/* Given two timeval stamps, return the time per chunk in usec */ +float get_usec_chunk(struct timeval last, struct timeval current) { + return (float)(getseconds(last, current)/(niter*nchunks)*1e6); +} + + +int get_value(int i, int rshift) { + int v; + + v = (i<<26)^(i<<18)^(i<<11)^(i<<3)^i; + if (rshift < 32) { + v &= (1 << rshift) - 1; + } + return v; +} + + +void init_buffer(void *src, int size, int rshift) { + unsigned int i; + int *_src = (int *)src; + + /* To have reproducible results */ + srand(1); + + /* Initialize the original buffer */ + for (i = 0; i < size/sizeof(int); ++i) { + /* Choose one below */ + //_src[i] = 0; + //_src[i] = 0x01010101; + //_src[i] = 0x01020304; + //_src[i] = i * 1/.3; + //_src[i] = i; + //_src[i] = rand() >> (32-rshift); + _src[i] = get_value(i, rshift); + } +} + + +void do_bench(int nthreads, int size, int elsize, int rshift) { + void *src, *srccpy; + void **dest[NCHUNKS], *dest2; + int nbytes = 0, cbytes = 0; + int i, j; + struct timeval last, current; + float tmemcpy, tshuf, tunshuf; + int clevel, doshuffle=1; + unsigned char *orig, *round; + + blosc_set_nthreads(nthreads); + + /* Initialize buffers */ + src = malloc(size); + srccpy = malloc(size); + dest2 = malloc(size); + /* zero src to initialize byte on it, and not only multiples of 4 */ + memset(src, 0, size); + init_buffer(src, size, rshift); + memcpy(srccpy, src, size); + for (j = 0; j < nchunks; j++) { + dest[j] = malloc(size+BLOSC_MAX_OVERHEAD); + } + + /* Warm destination memory (memcpy() will go a bit faster later on) */ + for (j = 0; j < nchunks; j++) { + memcpy(dest[j], src, size); + } + + printf("--> %d, %d, %d, %d\n", nthreads, size, elsize, rshift); + printf("********************** Run info ******************************\n"); + printf("Blosc version: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE); + printf("Using synthetic data with %d significant bits (out of 32)\n", rshift); + printf("Dataset size: %d bytes\tType size: %d bytes\n", size, elsize); + printf("Working set: %.1f MB\t\t", (size*nchunks) / (float)MB); + printf("Number of threads: %d\n", nthreads); + printf("********************** Running benchmarks *********************\n"); + + gettimeofday(&last, NULL); + for (i = 0; i < niter; i++) { + for (j = 0; j < nchunks; j++) { + memcpy(dest[j], src, size); + } + } + gettimeofday(¤t, NULL); + tmemcpy = get_usec_chunk(last, current); + printf("memcpy(write):\t\t %6.1f us, %.1f MB/s\n", + tmemcpy, size/(tmemcpy*MB/1e6)); + + gettimeofday(&last, NULL); + for (i = 0; i < niter; i++) { + for (j = 0; j < nchunks; j++) { + memcpy(dest2, dest[j], size); + } + } + gettimeofday(¤t, NULL); + tmemcpy = get_usec_chunk(last, current); + printf("memcpy(read):\t\t %6.1f us, %.1f MB/s\n", + tmemcpy, size/(tmemcpy*MB/1e6)); + + for (clevel=0; clevel<10; clevel++) { + + printf("Compression level: %d\n", clevel); + + gettimeofday(&last, NULL); + for (i = 0; i < niter; i++) { + for (j = 0; j < nchunks; j++) { + cbytes = blosc_compress(clevel, doshuffle, elsize, size, src, + dest[j], size+BLOSC_MAX_OVERHEAD); + } + } + gettimeofday(¤t, NULL); + tshuf = get_usec_chunk(last, current); + printf("comp(write):\t %6.1f us, %.1f MB/s\t ", + tshuf, size/(tshuf*MB/1e6)); + printf("Final bytes: %d ", cbytes); + if (cbytes > 0) { + printf("Ratio: %3.2f", size/(float)cbytes); + } + printf("\n"); + + /* Compressor was unable to compress. Copy the buffer manually. */ + if (cbytes == 0) { + for (j = 0; j < nchunks; j++) { + memcpy(dest[j], src, size); + } + } + + gettimeofday(&last, NULL); + for (i = 0; i < niter; i++) { + for (j = 0; j < nchunks; j++) { + if (cbytes == 0) { + memcpy(dest2, dest[j], size); + nbytes = size; + } + else { + nbytes = blosc_decompress(dest[j], dest2, size); + } + } + } + gettimeofday(¤t, NULL); + tunshuf = get_usec_chunk(last, current); + printf("decomp(read):\t %6.1f us, %.1f MB/s\t ", + tunshuf, nbytes/(tunshuf*MB/1e6)); + if (nbytes < 0) { + printf("FAILED. Error code: %d\n", nbytes); + } + /* printf("Orig bytes: %d\tFinal bytes: %d\n", cbytes, nbytes); */ + + /* Check if data has had a good roundtrip */ + orig = (unsigned char *)srccpy; + round = (unsigned char *)dest2; + for(i = 0; i %x, round-trip--> %x\n", orig[i], round[i]); + break; + } + } + + if (i == size) printf("OK\n"); + + } /* End clevel loop */ + + + /* To compute the totalsize, we should take into account the 10 + compression levels */ + totalsize += (size * nchunks * niter * 10.); + + free(src); free(srccpy); free(dest2); + for (i = 0; i < nchunks; i++) { + free(dest[i]); + } + +} + + +/* Compute a sensible value for nchunks */ +int get_nchunks(int size_, int ws) { + int nchunks; + + nchunks = ws / size_; + if (nchunks > NCHUNKS) nchunks = NCHUNKS; + if (nchunks < 1) nchunks = 1; + return nchunks; +} + + +int main(int argc, char *argv[]) { + int single = 1; + int suite = 0; + int hard_suite = 0; + int extreme_suite = 0; + int debug_suite = 0; + int nthreads = 1; /* The number of threads */ + int size = 2*MB; /* Buffer size */ + int elsize = 8; /* Datatype size */ + int rshift = 19; /* Significant bits */ + int workingset = 256*MB; /* The maximum allocated memory */ + int nthreads_, size_, elsize_, rshift_, i; + struct timeval last, current; + float totaltime; + char *usage = "Usage: bench ['single' | 'suite' | 'hardsuite' | 'extremesuite' | 'debugsuite'] [nthreads [bufsize(bytes) [typesize [sbits ]]]]"; + + + if (argc == 1) { + printf("%s\n", usage); + exit(1); + } + + if (strcmp(argv[1], "single") == 0) { + single = 1; + } + else if (strcmp(argv[1], "suite") == 0) { + suite = 1; + } + else if (strcmp(argv[1], "hardsuite") == 0) { + hard_suite = 1; + workingset = 64*MB; + /* Values here are ending points for loops */ + nthreads = 2; + size = 8*MB; + elsize = 32; + rshift = 32; + } + else if (strcmp(argv[1], "extremesuite") == 0) { + extreme_suite = 1; + workingset = 32*MB; + niter = 1; + /* Values here are ending points for loops */ + nthreads = 4; + size = 16*MB; + elsize = 32; + rshift = 32; + } + else if (strcmp(argv[1], "debugsuite") == 0) { + debug_suite = 1; + workingset = 32*MB; + niter = 1; + /* Warning: values here are starting points for loops. This is + useful for debugging. */ + nthreads = 1; + size = 16*KB; + elsize = 1; + rshift = 0; + } + else { + printf("%s\n", usage); + exit(1); + } + + if (argc >= 3) { + nthreads = atoi(argv[2]); + } + if (argc >= 4) { + size = atoi(argv[3]); + } + if (argc >= 5) { + elsize = atoi(argv[4]); + } + if (argc >= 6) { + rshift = atoi(argv[5]); + } + + if ((argc >= 7) || !(single || suite || hard_suite || extreme_suite)) { + printf("%s\n", usage); + exit(1); + } + + nchunks = get_nchunks(size, workingset); + gettimeofday(&last, NULL); + + if (suite) { + for (nthreads_=1; nthreads_ <= nthreads; nthreads_++) { + do_bench(nthreads_, size, elsize, rshift); + } + } + else if (hard_suite) { + /* Let's start the rshift loop by 4 so that 19 is visited. This + is to allow a direct comparison with the plain suite, that runs + precisely at 19 significant bits. */ + for (rshift_ = 4; rshift_ <= rshift; rshift_ += 5) { + for (elsize_ = 1; elsize_ <= elsize; elsize_ *= 2) { + /* The next loop is for getting sizes that are not power of 2 */ + for (i = -elsize_; i <= elsize_; i += elsize_) { + for (size_ = 32*KB; size_ <= size; size_ *= 2) { + nchunks = get_nchunks(size_+i, workingset); + niter = 1; + for (nthreads_ = 1; nthreads_ <= nthreads; nthreads_++) { + do_bench(nthreads_, size_+i, elsize_, rshift_); + gettimeofday(¤t, NULL); + totaltime = getseconds(last, current); + printf("Elapsed time:\t %6.1f s. Processed data: %.1f GB\n", + totaltime, totalsize / GB); + } + } + } + } + } + } + else if (extreme_suite) { + for (rshift_ = 0; rshift_ <= rshift; rshift_++) { + for (elsize_ = 1; elsize_ <= elsize; elsize_++) { + /* The next loop is for getting sizes that are not power of 2 */ + for (i = -elsize_*2; i <= elsize_*2; i += elsize_) { + for (size_ = 32*KB; size_ <= size; size_ *= 2) { + nchunks = get_nchunks(size_+i, workingset); + for (nthreads_ = 1; nthreads_ <= nthreads; nthreads_++) { + do_bench(nthreads_, size_+i, elsize_, rshift_); + gettimeofday(¤t, NULL); + totaltime = getseconds(last, current); + printf("Elapsed time:\t %6.1f s. Processed data: %.1f GB\n", + totaltime, totalsize / GB); + } + } + } + } + } + } + else if (debug_suite) { + for (rshift_ = rshift; rshift_ <= 32; rshift_++) { + for (elsize_ = elsize; elsize_ <= 32; elsize_++) { + /* The next loop is for getting sizes that are not power of 2 */ + for (i = -elsize_*2; i <= elsize_*2; i += elsize_) { + for (size_ = size; size_ <= 16*MB; size_ *= 2) { + nchunks = get_nchunks(size_+i, workingset); + for (nthreads_ = nthreads; nthreads_ <= 6; nthreads_++) { + do_bench(nthreads_, size_+i, elsize_, rshift_); + gettimeofday(¤t, NULL); + totaltime = getseconds(last, current); + printf("Elapsed time:\t %6.1f s. Processed data: %.1f GB\n", + totaltime, totalsize / GB); + } + } + } + } + } + } + /* Single mode */ + else { + do_bench(nthreads, size, elsize, rshift); + } + + /* Print out some statistics */ + gettimeofday(¤t, NULL); + totaltime = getseconds(last, current); + printf("\nRound-trip compr/decompr on %.1f GB\n", totalsize / GB); + printf("Elapsed time:\t %6.1f s, %.1f MB/s\n", + totaltime, totalsize*2*1.1/(MB*totaltime)); + + /* Free blosc resources */ + blosc_free_resources(); + + return 0; +} diff --git a/c-blosc/bench/plot-speeds.py b/c-blosc/bench/plot-speeds.py new file mode 100644 index 00000000..5be9e1a1 --- /dev/null +++ b/c-blosc/bench/plot-speeds.py @@ -0,0 +1,196 @@ +"""Script for plotting the results of the 'suite' benchmark. +Invoke without parameters for usage hints. + +:Author: Francesc Alted +:Date: 2010-06-01 +""" + +import matplotlib as mpl +from pylab import * + +KB_ = 1024 +MB_ = 1024*KB_ +GB_ = 1024*MB_ +NCHUNKS = 128 # keep in sync with bench.c + +linewidth=2 +#markers= ['+', ',', 'o', '.', 's', 'v', 'x', '>', '<', '^'] +#markers= [ 'x', '+', 'o', 's', 'v', '^', '>', '<', ] +markers= [ 's', 'o', 'v', '^', '+', 'x', '>', '<', '.', ',' ] +markersize = 8 + +def get_values(filename): + f = open(filename) + values = {"memcpyw": [], "memcpyr": []} + + for line in f: + if line.startswith('-->'): + tmp = line.split('-->')[1] + nthreads, size, elsize, sbits = [int(i) for i in tmp.split(', ')] + values["size"] = size * NCHUNKS / MB_; + values["elsize"] = elsize; + values["sbits"] = sbits; + # New run for nthreads + (ratios, speedsw, speedsr) = ([], [], []) + # Add a new entry for (ratios, speedw, speedr) + values[nthreads] = (ratios, speedsw, speedsr) + #print "-->", nthreads, size, elsize, sbits + elif line.startswith('memcpy(write):'): + tmp = line.split(',')[1] + memcpyw = float(tmp.split(' ')[1]) + values["memcpyw"].append(memcpyw) + elif line.startswith('memcpy(read):'): + tmp = line.split(',')[1] + memcpyr = float(tmp.split(' ')[1]) + values["memcpyr"].append(memcpyr) + elif line.startswith('comp(write):'): + tmp = line.split(',')[1] + speedw = float(tmp.split(' ')[1]) + ratio = float(line.split(':')[-1]) + speedsw.append(speedw) + ratios.append(ratio) + elif line.startswith('decomp(read):'): + tmp = line.split(',')[1] + speedr = float(tmp.split(' ')[1]) + speedsr.append(speedr) + if "OK" not in line: + print "WARNING! OK not found in decomp line!" + + f.close() + return nthreads, values + + +def show_plot(plots, yaxis, legends, gtitle, xmax=None): + xlabel('Compresssion ratio') + ylabel('Speed (MB/s)') + title(gtitle) + xlim(0, xmax) + #ylim(0, 10000) + ylim(0, None) + grid(True) + +# legends = [f[f.find('-'):f.index('.out')] for f in filenames] +# legends = [l.replace('-', ' ') for l in legends] + #legend([p[0] for p in plots], legends, loc = "upper left") + legend([p[0] for p in plots + if not isinstance(p, mpl.lines.Line2D)], + legends, loc = "best") + + + #subplots_adjust(bottom=0.2, top=None, wspace=0.2, hspace=0.2) + if outfile: + print "Saving plot to:", outfile + savefig(outfile) + else: + show() + +if __name__ == '__main__': + + from optparse import OptionParser + + usage = "usage: %prog [-o outfile] [-t title ] [-d|-c] filename" + compress_title = 'Compression speed' + decompress_title = 'Decompression speed' + yaxis = 'No axis name' + + parser = OptionParser(usage=usage) + parser.add_option('-o', + '--outfile', + dest='outfile', + help='filename for output') + + parser.add_option('-t', + '--title', + dest='title', + help='title of the plot',) + + parser.add_option('-l', + '--limit', + dest='limit', + help='expression to limit number of threads showen',) + + parser.add_option('-x', + '--xmax', + dest='xmax', + help='limit the x-axis', + default=None) + + parser.add_option('-d', '--decompress', action='store_true', + dest='dspeed', + help='plot decompression data', + default=False) + parser.add_option('-c', '--compress', action='store_true', + dest='cspeed', + help='plot compression data', + default=False) + + (options, args) = parser.parse_args() + if len(args) == 0: + parser.error("No input arguments") + elif len(args) > 1: + parser.error("Too many input arguments") + else: + pass + + if options.dspeed and options.cspeed: + parser.error("Can only select one of [-d, -c]") + elif options.cspeed: + options.dspeed = False + plot_title = compress_title + else: # either neither or dspeed + options.dspeed = True + plot_title = decompress_title + + filename = args[0] + outfile = options.outfile + cspeed = options.cspeed + dspeed = options.dspeed + + plots = [] + legends = [] + nthreads, values = get_values(filename) + #print "Values:", values + + if options.limit: + thread_range = eval(options.limit) + else: + thread_range = range(1, nthreads+1) + + if options.title: + plot_title = options.title + else: + plot_title += " (%(size).1f MB, %(elsize)d bytes, %(sbits)d bits)" % values + + gtitle = plot_title + + for nt in thread_range: + #print "Values for %s threads --> %s" % (nt, values[nt]) + (ratios, speedw, speedr) = values[nt] + if cspeed: + speed = speedw + else: + speed = speedr + #plot_ = semilogx(ratios, speed, linewidth=2) + plot_ = plot(ratios, speed, linewidth=2) + plots.append(plot_) + nmarker = nt + if nt >= len(markers): + nmarker = nt%len(markers) + setp(plot_, marker=markers[nmarker], markersize=markersize, + linewidth=linewidth) + legends.append("%d threads" % nt) + + # Add memcpy lines + if cspeed: + mean = sum(values["memcpyw"]) / nthreads + message = "memcpy (write to memory)" + else: + mean = sum(values["memcpyr"]) / nthreads + message = "memcpy (read from memory)" + plot_ = axhline(mean, linewidth=3, linestyle='-.', color='black') + text(4.0, mean+50, message) + plots.append(plot_) + show_plot(plots, yaxis, legends, gtitle, xmax=int(options.xmax) if + options.xmax else None) + + diff --git a/c-blosc/blosc.c b/c-blosc/blosc/blosc.c similarity index 100% rename from c-blosc/blosc.c rename to c-blosc/blosc/blosc.c diff --git a/c-blosc/blosc.h b/c-blosc/blosc/blosc.h similarity index 97% rename from c-blosc/blosc.h rename to c-blosc/blosc/blosc.h index 41f80ffd..8d64a46c 100644 --- a/c-blosc/blosc.h +++ b/c-blosc/blosc/blosc.h @@ -14,11 +14,11 @@ /* Version numbers */ #define BLOSC_VERSION_MAJOR 1 /* for major interface/format changes */ #define BLOSC_VERSION_MINOR 1 /* for minor interface/format changes */ -#define BLOSC_VERSION_RELEASE 4 /* for tweaks, bug-fixes, or development */ +#define BLOSC_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */ -#define BLOSC_VERSION_STRING "1.1.4" /* string version. Sync with above! */ +#define BLOSC_VERSION_STRING "1.1.5" /* string version. Sync with above! */ #define BLOSC_VERSION_REVISION "$Rev$" /* revision version */ -#define BLOSC_VERSION_DATE "$Date:: 2012-09-16 #$" /* date version */ +#define BLOSC_VERSION_DATE "$Date:: 2012-09-21 #$" /* date version */ /* The *_VERS_FORMAT should be just 1-byte long */ #define BLOSC_VERSION_FORMAT 2 /* Blosc format version, starting at 1 */ diff --git a/c-blosc/blosclz.c b/c-blosc/blosc/blosclz.c similarity index 100% rename from c-blosc/blosclz.c rename to c-blosc/blosc/blosclz.c diff --git a/c-blosc/blosclz.h b/c-blosc/blosc/blosclz.h similarity index 100% rename from c-blosc/blosclz.h rename to c-blosc/blosc/blosclz.h diff --git a/c-blosc/shuffle.c b/c-blosc/blosc/shuffle.c similarity index 100% rename from c-blosc/shuffle.c rename to c-blosc/blosc/shuffle.c diff --git a/c-blosc/shuffle.h b/c-blosc/blosc/shuffle.h similarity index 100% rename from c-blosc/shuffle.h rename to c-blosc/blosc/shuffle.h diff --git a/c-blosc/win32/pthread.c b/c-blosc/blosc/win32/pthread.c similarity index 100% rename from c-blosc/win32/pthread.c rename to c-blosc/blosc/win32/pthread.c diff --git a/c-blosc/win32/pthread.h b/c-blosc/blosc/win32/pthread.h similarity index 100% rename from c-blosc/win32/pthread.h rename to c-blosc/blosc/win32/pthread.h diff --git a/c-blosc/win32/stdint-windows.h b/c-blosc/blosc/win32/stdint-windows.h similarity index 100% rename from c-blosc/win32/stdint-windows.h rename to c-blosc/blosc/win32/stdint-windows.h diff --git a/c-blosc/hdf5/README.rst b/c-blosc/hdf5/README.rst new file mode 100644 index 00000000..7bf24e77 --- /dev/null +++ b/c-blosc/hdf5/README.rst @@ -0,0 +1,52 @@ +Using the Blosc filter from HDF5 +================================ + +In order to register Blosc into your HDF5 application, you only need +to call a function in blosc_filter.h, with the following signature: + + int register_blosc(char **version, char **date) + +Calling this will register the filter with the HDF5 library and will +return info about the Blosc release in `**version` and `**date` +char pointers. + +A non-negative return value indicates success. If the registration +fails, an error is pushed onto the current error stack and a negative +value is returned. + +An example C program ("example.c") is included which demonstrates the +proper use of the filter. + +This filter has been tested against HDF5 versions 1.6.5 through 1.8.5. It +is released under the MIT license (see LICENSE.txt for details). + + +Compiling +========= + +The filter consists of a single '.c' source file and '.h' header, +along with an embedded version of the BLOSC compression library. +Also, as Blosc uses SSE2 and multithreading, you must remember to use +special flags and libraries to make sure that these features are used. + +To compile using GCC on UNIX: + + gcc -O3 -msse2 -lhdf5 ../blosc/*.c blosc_filter.c \ + myprog.c -o myprog -lpthread + +To compile using MINGW on Windows: + + gcc -O3 -msse2 -lhdf5 ..\blosc\*.c blosc_filter.c myprog.c -o myprog + +Using Windows and MSVC (2008 or higher recommended): + + cl /Ox /Femyprog.exe myprog.c ..\blosc\*.c blosc_filter.c + +Intel ICC compilers should work too. + + +Acknowledgments +=============== + +This HDF5 filter interface and its example is base the LZF interface +(http://h5py.alfven.org) by Andrew Collette. diff --git a/c-blosc/hdf5/blosc_filter.c b/c-blosc/hdf5/blosc_filter.c new file mode 100644 index 00000000..363b03cb --- /dev/null +++ b/c-blosc/hdf5/blosc_filter.c @@ -0,0 +1,255 @@ +/* + Copyright (C) 2010 Francesc Alted + http://blosc.pytables.org + License: MIT (see LICENSE.txt) + + Filter program that allows the use of the Blosc filter in HDF5. + + This is based on the LZF filter interface (http://h5py.alfven.org) + by Andrew Collette. + +*/ + + +#include +#include +#include +#include +#include "hdf5.h" +#include "../blosc/blosc.h" +#include "blosc_filter.h" + +#if H5Epush_vers == 2 +/* 1.8.x */ +#define PUSH_ERR(func, minor, str) H5Epush(H5E_DEFAULT, __FILE__, func, __LINE__, H5E_ERR_CLS, H5E_PLINE, minor, str) +#else +/* 1.6.x */ +#define PUSH_ERR(func, minor, str) H5Epush(__FILE__, func, __LINE__, H5E_PLINE, minor, str) +#endif + +#if H5Pget_filter_by_id_vers == 2 +/* 1.8.x */ +#define GET_FILTER(a,b,c,d,e,f,g) H5Pget_filter_by_id(a,b,c,d,e,f,g,NULL) +#else +/* 1.6.x */ +#define GET_FILTER H5Pget_filter_by_id +#endif + +#if H5Z_class_t_vers == 2 +/* 1.8.x where x >= 3 */ +#define H5Z_16API 0 +#else +/* 1.6.x and 1.8.x with x < 3*/ +#define H5Z_16API 1 +#endif + +size_t blosc_filter(unsigned flags, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, + size_t *buf_size, void **buf); + +herr_t blosc_set_local(hid_t dcpl, hid_t type, hid_t space); + + +/* Register the filter, passing on the HDF5 return value */ +int register_blosc(char **version, char **date){ + + int retval; + +#if H5Z_16API + H5Z_class_t filter_class = { + (H5Z_filter_t)(FILTER_BLOSC), + "blosc", + NULL, + (H5Z_set_local_func_t)(blosc_set_local), + (H5Z_func_t)(blosc_filter) + }; +#else + H5Z_class_t filter_class = { + H5Z_CLASS_T_VERS, + (H5Z_filter_t)(FILTER_BLOSC), + 1, 1, + "blosc", + NULL, + (H5Z_set_local_func_t)(blosc_set_local), + (H5Z_func_t)(blosc_filter) + }; +#endif + + retval = H5Zregister(&filter_class); + if(retval<0){ + PUSH_ERR("register_blosc", H5E_CANTREGISTER, "Can't register Blosc filter"); + } + *version = strdup(BLOSC_VERSION_STRING); + *date = strdup(BLOSC_VERSION_DATE); + return 1; /* lib is available */ +} + +/* Filter setup. Records the following inside the DCPL: + + 1. If version information is not present, set slots 0 and 1 to the filter + revision and Blosc version, respectively. + + 2. Compute the type size in bytes and store it in slot 2. + + 3. Compute the chunk size in bytes and store it in slot 3. +*/ +herr_t blosc_set_local(hid_t dcpl, hid_t type, hid_t space){ + + int ndims; + int i; + herr_t r; + + unsigned int typesize, basetypesize; + unsigned int bufsize; + hsize_t chunkdims[32]; + unsigned int flags; + size_t nelements = 8; + unsigned int values[] = {0,0,0,0,0,0,0,0}; + hid_t super_type; + H5T_class_t class; + + r = GET_FILTER(dcpl, FILTER_BLOSC, &flags, &nelements, values, 0, NULL); + if(r<0) return -1; + + if(nelements < 4) nelements = 4; /* First 4 slots reserved. */ + + /* Set Blosc info in first two slots */ + values[0] = FILTER_BLOSC_VERSION; + values[1] = BLOSC_VERSION_FORMAT; + + ndims = H5Pget_chunk(dcpl, 32, chunkdims); + if(ndims<0) return -1; + if(ndims>32){ + PUSH_ERR("blosc_set_local", H5E_CALLBACK, "Chunk rank exceeds limit"); + return -1; + } + + typesize = H5Tget_size(type); + if (typesize==0) return -1; + /* Get the size of the base type, even for ARRAY types */ + class = H5Tget_class(type); + if (class == H5T_ARRAY) { + /* Get the array base component */ + super_type = H5Tget_super(type); + basetypesize = H5Tget_size(super_type); + /* Release resources */ + H5Tclose(super_type); + } + else { + basetypesize = typesize; + } + + /* Limit large typesizes (they are pretty inneficient to shuffle + and, in addition, Blosc does not handle typesizes larger than + blocksizes). */ + if (basetypesize > BLOSC_MAX_TYPESIZE) basetypesize = 1; + values[2] = basetypesize; + + /* Get the size of the chunk */ + bufsize = typesize; + for (i=0; i= 5) { + clevel = cd_values[4]; /* The compression level */ + } + if (cd_nelmts >= 6) { + doshuffle = cd_values[5]; /* Shuffle? */ + } + + /* We're compressing */ + if(!(flags & H5Z_FLAG_REVERSE)){ + +#ifdef BLOSC_DEBUG + fprintf(stderr, "Blosc: Compress %zd chunk w/buffer %zd\n", nbytes, outbuf_size); +#endif + + /* Allocate an output buffer exactly as long as the input data; if + the result is larger, we simply return 0. The filter is flagged + as optional, so HDF5 marks the chunk as uncompressed and + proceeds. + */ + + outbuf_size = (*buf_size); + outbuf = malloc(outbuf_size); + + if(outbuf == NULL){ + PUSH_ERR("blosc_filter", H5E_CALLBACK, + "Can't allocate compression buffer"); + goto failed; + } + + status = blosc_compress(clevel, doshuffle, typesize, nbytes, + *buf, outbuf, nbytes); + if (status < 0) { + PUSH_ERR("blosc_filter", H5E_CALLBACK, "Blosc compression error"); + goto failed; + } + + /* We're decompressing */ + } else { + + +#ifdef BLOSC_DEBUG + fprintf(stderr, "Blosc: Decompress %zd chunk w/buffer %zd\n", nbytes, outbuf_size); +#endif + + free(outbuf); + outbuf = malloc(outbuf_size); + + if(outbuf == NULL){ + PUSH_ERR("blosc_filter", H5E_CALLBACK, "Can't allocate decompression buffer"); + goto failed; + } + + status = blosc_decompress(*buf, outbuf, outbuf_size); + + if(status <= 0){ /* decompression failed */ + PUSH_ERR("blosc_filter", H5E_CALLBACK, "Blosc decompression error"); + goto failed; + } /* if !status */ + + } /* compressing vs decompressing */ + + if(status != 0){ + free(*buf); + *buf = outbuf; + *buf_size = outbuf_size; + return status; /* Size of compressed/decompressed data */ + } + + failed: + free(outbuf); + return 0; + +} /* End filter function */ diff --git a/c-blosc/hdf5/blosc_filter.h b/c-blosc/hdf5/blosc_filter.h new file mode 100644 index 00000000..8bf560ba --- /dev/null +++ b/c-blosc/hdf5/blosc_filter.h @@ -0,0 +1,22 @@ +#ifndef FILTER_BLOSC_H +#define FILTER_BLOSC_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* Filter revision number, starting at 1 */ +#define FILTER_BLOSC_VERSION 1 + +/* Filter ID registered with the HDF Group */ +#define FILTER_BLOSC 32001 + +/* Register the filter with the library */ +int register_blosc(char **version, char **date); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/c-blosc/hdf5/example.c b/c-blosc/hdf5/example.c new file mode 100644 index 00000000..e17b6039 --- /dev/null +++ b/c-blosc/hdf5/example.c @@ -0,0 +1,119 @@ +/* + Copyright (C) 2010 Francesc Alted + http://blosc.pytables.org + License: MIT (see LICENSE.txt) + + Example program demonstrating use of the Blosc filter from C code. + This is based on the LZF example (http://h5py.alfven.org) by + Andrew Collette. + + To compile this program: + + h5cc [-DH5_USE_16_API] -msse2 ../blosc/*.c blosc_filter.c example.c \ + -o example -lpthread + + To run: + + $ ./example + Blosc version info: 1.1.1 ($Date:: 2010-10-01 #$) + Success! + $ h5ls -v example.h5 + Opened "example.h5" with sec2 driver. + dset Dataset {100/100, 100/100, 100/100} + Location: 1:800 + Links: 1 + Chunks: {1, 100, 100} 40000 bytes + Storage: 4000000 logical bytes, 142097 allocated bytes, 2814.98% utilization + Filter-0: blosc-32001 OPT {1, 2, 4, 40000, 4, 1} + Type: native float + +*/ + +#include +#include "hdf5.h" +#include "blosc_filter.h" + +#define SIZE 100*100*100 +#define SHAPE {100,100,100} +#define CHUNKSHAPE {1,100,100} + +int main(){ + + static float data[SIZE]; + static float data_out[SIZE]; + const hsize_t shape[] = SHAPE; + const hsize_t chunkshape[] = CHUNKSHAPE; + char *version, *date; + int r, i; + unsigned int cd_values[6]; + int return_code = 1; + + hid_t fid, sid, dset, plist = 0; + + for(i=0; i0) H5Dclose(dset); + if(sid>0) H5Sclose(sid); + if(plist>0) H5Pclose(plist); + if(fid>0) H5Fclose(fid); + + return return_code; +} diff --git a/c-blosc/tests/Makefile b/c-blosc/tests/Makefile new file mode 100644 index 00000000..cb0fa348 --- /dev/null +++ b/c-blosc/tests/Makefile @@ -0,0 +1,21 @@ +CC=gcc +CFLAGS=-O3 -msse2 -Wall +LDFLAGS=-lpthread +BLOSC_LIB= $(wildcard ../blosc/*.c) + +# The list of executables +# Generated PNG (intermediate) files +SOURCES := $(wildcard *.c) +EXECUTABLES := $(patsubst %.c, %.exe, $(SOURCES)) + +.PHONY: all +all: $(EXECUTABLES) + +test: $(EXECUTABLES) + sh test_all.sh + +%.exe: %.c $(BLOSC_LIB) + $(CC) $(CFLAGS) $(LDFLAGS) "$<" $(BLOSC_LIB) -o "$@" + +clean: + rm -rf $(EXECUTABLES) diff --git a/c-blosc/tests/test_all.sh b/c-blosc/tests/test_all.sh new file mode 100644 index 00000000..ae2c8b1a --- /dev/null +++ b/c-blosc/tests/test_all.sh @@ -0,0 +1,14 @@ +#********************************************************************* +# Blosc - Blocked Suffling and Compression Library +# +# Unit tests for basic features in Blosc. +# +# Creation date: 2010-06-07 +# Author: Francesc Alted (faltet@pytables.org) +# +# See LICENSES/BLOSC.txt for details about copyright and rights to use. +#********************************************************************** + +for exe in $(ls *.exe); do + ./$exe +done diff --git a/c-blosc/tests/test_api.c b/c-blosc/tests/test_api.c new file mode 100644 index 00000000..def27603 --- /dev/null +++ b/c-blosc/tests/test_api.c @@ -0,0 +1,99 @@ +/********************************************************************* + Blosc - Blocked Suffling and Compression Library + + Unit tests for Blosc API. + + Creation date: 2010-06-07 + Author: Francesc Alted (faltet@pytables.org) + + See LICENSES/BLOSC.txt for details about copyright and rights to use. +**********************************************************************/ + +#include "test_common.h" + +int tests_run = 0; + +/* Global vars */ +void *src, *srccpy, *dest, *dest2; +size_t nbytes, cbytes; +int clevel = 3; +int doshuffle = 1; +size_t typesize = 4; +size_t size = 1*MB; + + + +static char *test_cbuffer_sizes() { + size_t nbytes_, cbytes_, blocksize; + + blosc_cbuffer_sizes(dest, &nbytes_, &cbytes_, &blocksize); + mu_assert("ERROR: nbytes incorrect(1)", nbytes == size); + mu_assert("ERROR: nbytes incorrect(2)", nbytes_ == nbytes); + mu_assert("ERROR: cbytes incorrect", cbytes == cbytes_); + mu_assert("ERROR: blocksize incorrect", blocksize >= 128); + return 0; +} + +static char *test_cbuffer_metainfo() { + size_t typesize_; + int flags; + + blosc_cbuffer_metainfo(dest, &typesize_, &flags); + mu_assert("ERROR: typesize incorrect", typesize_ == typesize); + mu_assert("ERROR: shuffle incorrect", (flags & BLOSC_DOSHUFFLE) == doshuffle); + return 0; +} + + +static char *test_cbuffer_versions() { + int version_; + int versionlz_; + + blosc_cbuffer_versions(dest, &version_, &versionlz_); + mu_assert("ERROR: version incorrect", version_ == BLOSC_VERSION_FORMAT); + mu_assert("ERROR: versionlz incorrect", versionlz_ == BLOSCLZ_VERSION_FORMAT); + return 0; +} + + +static char *all_tests() { + mu_run_test(test_cbuffer_sizes); + mu_run_test(test_cbuffer_metainfo); + mu_run_test(test_cbuffer_versions); + return 0; +} + +int main(int argc, char **argv) { + + printf("STARTING TESTS for %s", argv[0]); + + blosc_set_nthreads(1); + + /* Initialize buffers */ + src = malloc(size); + srccpy = malloc(size); + dest = malloc(size); + dest2 = malloc(size); + memset(src, 0, size); + memcpy(srccpy, src, size); + + /* Get a compressed buffer */ + cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size); + + /* Get a decompressed buffer */ + nbytes = blosc_decompress(dest, dest2, size); + + /* Run all the suite */ + char *result = all_tests(); + if (result != 0) { + printf(" (%s)\n", result); + } + else { + printf(" ALL TESTS PASSED"); + } + printf("\tTests run: %d\n", tests_run); + + free(src); free(srccpy); free(dest); free(dest2); + return result != 0; +} + diff --git a/c-blosc/tests/test_basics.c b/c-blosc/tests/test_basics.c new file mode 100644 index 00000000..f5dc668c --- /dev/null +++ b/c-blosc/tests/test_basics.c @@ -0,0 +1,105 @@ +/********************************************************************* + Blosc - Blocked Suffling and Compression Library + + Unit tests for basic features in Blosc. + + Creation date: 2010-06-07 + Author: Francesc Alted (faltet@pytables.org) + + See LICENSES/BLOSC.txt for details about copyright and rights to use. +**********************************************************************/ + +#include "test_common.h" + +int tests_run = 0; + +/* Global vars */ +void *src, *srccpy, *dest, *dest2; +size_t nbytes, cbytes; +int clevel = 1; +int doshuffle = 0; +size_t typesize = 4; +size_t size = 1000; /* must be divisible by 4 */ + + +/* Check maxout with maxout < size */ +static char *test_maxout_less() { + + /* Get a compressed buffer */ + cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, + dest, size+15); + mu_assert("ERROR: cbytes is not 0", cbytes == 0); + + return 0; +} + +/* Check maxout with maxout == size */ +static char *test_maxout_equal() { + + /* Get a compressed buffer */ + cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, + dest, size+16); + mu_assert("ERROR: cbytes is not correct", cbytes == size+16); + + /* Decompress the buffer */ + nbytes = blosc_decompress(dest, dest2, size); + mu_assert("ERROR: nbytes incorrect(1)", nbytes == size); + + return 0; +} + + +/* Check maxout with maxout > size */ +static char *test_maxout_great() { + /* Get a compressed buffer */ + cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, + dest, size+17); + mu_assert("ERROR: cbytes is not 0", cbytes == size+16); + + /* Decompress the buffer */ + nbytes = blosc_decompress(dest, dest2, size); + mu_assert("ERROR: nbytes incorrect(1)", nbytes == size); + + return 0; +} + + +static char *all_tests() { + mu_run_test(test_maxout_less); + mu_run_test(test_maxout_equal); + mu_run_test(test_maxout_great); + return 0; +} + +int main(int argc, char **argv) { + int i; + int32_t *_src; + + printf("STARTING TESTS for %s", argv[0]); + + blosc_set_nthreads(1); + + /* Initialize buffers */ + src = malloc(size); + srccpy = malloc(size); + dest = malloc(size+16); + dest2 = malloc(size); + _src = (int32_t *)src; + for (i=0; i < (size/4); i++) { + _src[i] = i; + } + memcpy(srccpy, src, size); + + /* Run all the suite */ + char *result = all_tests(); + if (result != 0) { + printf(" (%s)\n", result); + } + else { + printf(" ALL TESTS PASSED"); + } + printf("\tTests run: %d\n", tests_run); + + free(src); free(srccpy); free(dest); free(dest2); + return result != 0; +} diff --git a/c-blosc/tests/test_common.h b/c-blosc/tests/test_common.h new file mode 100644 index 00000000..1465f66b --- /dev/null +++ b/c-blosc/tests/test_common.h @@ -0,0 +1,40 @@ +/********************************************************************* + Blosc - Blocked Suffling and Compression Library + + Unit tests for basic features in Blosc. + + Creation date: 2010-06-07 + Author: Francesc Alted (faltet@pytables.org) + + See LICENSES/BLOSC.txt for details about copyright and rights to use. +**********************************************************************/ + +#include +#include +#include +#include +#include +#include +#if defined(_WIN32) && !defined(__MINGW32__) + #include + #include "stdint-windows.h" +#else + #include + #include +#endif +#include +#include "../blosc/blosc.h" + + +/* This is MinUnit in action (http://www.jera.com/techinfo/jtns/jtn002.html) */ +#define mu_assert(message, test) do { if (!(test)) return message; } while (0) +#define mu_run_test(test) do \ + { char *message = test(); tests_run++; \ + if (message) { printf("%c", 'F'); return message;} \ + else printf("%c", '.'); } while (0) + +extern int tests_run; + +#define KB 1024 +#define MB (1024*KB) +#define GB (1024*MB) diff --git a/setup.py b/setup.py index f3f74657..0704d8fa 100644 --- a/setup.py +++ b/setup.py @@ -31,13 +31,19 @@ # Create the version.py file open('blosc/version.py', 'w').write('__version__ = "%s"\n' % VERSION) +# c-blosc sources and header +c_blosc_source_dir = os.path.join('c-blosc', 'blosc') +c_blosc_sources = [os.path.join(c_blosc_source_dir, source) + for source in ('blosc.c', 'blosclz.c', 'shuffle.c')] +c_blosc_headers = [os.path.join(c_blosc_source_dir, header) + for header in ('blosc.h', 'blosclz.h', 'shuffle.h')] # Global variables CFLAGS = os.environ.get('CFLAGS', '').split() LFLAGS = os.environ.get('LFLAGS', '').split() lib_dirs = [] libs = [] -inc_dirs = ['c-blosc'] +inc_dirs = [c_blosc_source_dir] optional_libs = [] # Handle --lflags=[FLAGS] --cflags=[FLAGS] @@ -91,11 +97,8 @@ Extension( "blosc.blosc_extension", include_dirs=inc_dirs, define_macros=def_macros, - sources = [ "blosc/blosc_extension.c", - "c-blosc/blosc.c", "c-blosc/blosclz.c", - "c-blosc/shuffle.c" ], - depends = [ "c-blosc/blosc.h", "c-blosc/blosclz.h", - "c-blosc/shuffle.h" ], + sources = ["blosc/blosc_extension.c"] + c_blosc_sources, + depends = c_blosc_headers, library_dirs=lib_dirs, libraries=libs, extra_link_args=LFLAGS,