Permalink
Browse files

Merge branch 'snappy'

  • Loading branch information...
Mortal committed Apr 24, 2014
2 parents df0220a + d900c88 commit 1023c90e1288b4e2003eb82768aa4e9d70e3c51b
Showing with 6,906 additions and 919 deletions.
  1. +7 −0 CMakeLists.txt
  2. +16 −0 cmake_modules/FindSnappy.cmake
  3. +103 −0 cmake_modules/LibFindMacros.cmake
  4. +1 −0 doc/Doxyfile.in
  5. +20 −0 doc/compressed.dox
  6. +1 −0 doc/index.dox
  7. +4 −0 test/CMakeLists.txt
  8. +8 −0 test/speed_regression/CMakeLists.txt
  9. +101 −0 test/speed_regression/compressed.cpp
  10. +2 −2 test/speed_regression/stream.cpp
  11. +184 −0 test/speed_regression/stream2.cpp
  12. +93 −0 test/test_atomic_stats.cpp
  13. +7 −1 test/unit/CMakeLists.txt
  14. +893 −0 test/unit/test_compressed_stream.cpp
  15. +3 −3 test/unit/test_file_count.cpp
  16. +8 −8 test/unit/test_pipelining.cpp
  17. +46 −0 test/unit/test_snappy.cpp
  18. +79 −23 test/unit/test_stream.cpp
  19. +22 −1 tpie/CMakeLists.txt
  20. +49 −0 tpie/atomic.cpp
  21. +14 −0 tpie/atomic.h
  22. +131 −0 tpie/compressed/buffer.cpp
  23. +411 −0 tpie/compressed/buffer.h
  24. +41 −0 tpie/compressed/direction.h
  25. +101 −0 tpie/compressed/position.h
  26. +53 −0 tpie/compressed/predeclare.h
  27. +29 −0 tpie/compressed/request.cpp
  28. +458 −0 tpie/compressed/request.h
  29. +101 −0 tpie/compressed/scheme.h
  30. +58 −0 tpie/compressed/scheme_none.cpp
  31. +89 −0 tpie/compressed/scheme_snappy.cpp
  32. +1,274 −0 tpie/compressed/stream.h
  33. +227 −0 tpie/compressed/stream_base.cpp
  34. +379 −0 tpie/compressed/thread.cpp
  35. +83 −0 tpie/compressed/thread.h
  36. +2 −0 tpie/config.h.cmake
  37. +109 −0 tpie/file_accessor/byte_stream_accessor.h
  38. +3 −3 tpie/file_accessor/file_accessor.h
  39. +2 −1 tpie/file_accessor/posix.h
  40. +12 −2 tpie/file_accessor/posix.inl
  41. +31 −164 tpie/file_accessor/stream_accessor.h
  42. +217 −0 tpie/file_accessor/stream_accessor_base.h
  43. +22 −37 tpie/file_accessor/{stream_accessor.inl → stream_accessor_base.inl}
  44. +2 −1 tpie/file_accessor/win32.h
  45. +13 −1 tpie/file_accessor/win32.inl
  46. +8 −1 tpie/file_base_crtp.h
  47. +6 −168 tpie/file_stream.h
  48. +185 −0 tpie/pipelining/merge_sorter.cpp
  49. +127 −9 tpie/pipelining/merge_sorter.h
  50. +1 −0 tpie/pipelining/merger.h
  51. +3 −0 tpie/pipelining/sort_parameters.h
  52. +114 −11 tpie/sort.h
  53. +1 −0 tpie/sort_deprecated.h
  54. +31 −27 tpie/stack.h
  55. +20 −12 tpie/stats.cpp
  56. +156 −441 tpie/stream.h
  57. +12 −2 tpie/stream_header.h
  58. +523 −0 tpie/stream_old.h
  59. +7 −0 tpie/sysinfo.h
  60. +12 −0 tpie/tpie.cpp
  61. +3 −1 tpie/tpie.h
  62. +188 −0 tpie/uncompressed_stream.h
View
@@ -48,6 +48,13 @@ check_include_files("sys/unistd.h" TPIE_HAVE_SYS_UNISTD_H)
include(GetGitRevisionDescription)
get_git_head_revision(GIT_REFSPEC GIT_COMMIT)
## Snappy
option(TPIE_USE_SNAPPY "Use Snappy, a fast compressor/decompressor" ON)
if(TPIE_USE_SNAPPY)
find_package(Snappy REQUIRED)
include_directories(${Snappy_INCLUDE_DIR})
endif(TPIE_USE_SNAPPY)
#### Installation paths
#Default paths
set(BIN_INSTALL_DIR bin)
@@ -0,0 +1,16 @@
# Snappy, a fast compressor/decompressor
include(LibFindMacros)
find_path(Snappy_INCLUDE_DIR
NAMES snappy.h
)
find_library(Snappy_LIBRARY
NAMES snappy
)
set(Snappy_PROCESS_INCLUDES Snappy_INCLUDE_DIR)
set(Snappy_PROCESS_LIBS Snappy_LIBRARY)
libfind_process(Snappy)
@@ -0,0 +1,103 @@
# Version 1.0 (2013-04-12)
# Public Domain, originally written by Lasse Kärkkäinen <tronic@zi.fi>
# Published at http://www.cmake.org/Wiki/CMake:How_To_Find_Libraries
# Works the same as find_package, but forwards the "REQUIRED" and "QUIET" arguments
# used for the current package. For this to work, the first parameter must be the
# prefix of the current package, then the prefix of the new package etc, which are
# passed to find_package.
macro (libfind_package PREFIX)
set (LIBFIND_PACKAGE_ARGS ${ARGN})
if (${PREFIX}_FIND_QUIETLY)
set (LIBFIND_PACKAGE_ARGS ${LIBFIND_PACKAGE_ARGS} QUIET)
endif (${PREFIX}_FIND_QUIETLY)
if (${PREFIX}_FIND_REQUIRED)
set (LIBFIND_PACKAGE_ARGS ${LIBFIND_PACKAGE_ARGS} REQUIRED)
endif (${PREFIX}_FIND_REQUIRED)
find_package(${LIBFIND_PACKAGE_ARGS})
endmacro (libfind_package)
# CMake developers made the UsePkgConfig system deprecated in the same release (2.6)
# where they added pkg_check_modules. Consequently I need to support both in my scripts
# to avoid those deprecated warnings. Here's a helper that does just that.
# Works identically to pkg_check_modules, except that no checks are needed prior to use.
macro (libfind_pkg_check_modules PREFIX PKGNAME)
if (${CMAKE_MAJOR_VERSION} EQUAL 2 AND ${CMAKE_MINOR_VERSION} EQUAL 4)
include(UsePkgConfig)
pkgconfig(${PKGNAME} ${PREFIX}_INCLUDE_DIRS ${PREFIX}_LIBRARY_DIRS ${PREFIX}_LDFLAGS ${PREFIX}_CFLAGS)
else (${CMAKE_MAJOR_VERSION} EQUAL 2 AND ${CMAKE_MINOR_VERSION} EQUAL 4)
find_package(PkgConfig)
if (PKG_CONFIG_FOUND)
pkg_check_modules(${PREFIX} ${PKGNAME})
endif (PKG_CONFIG_FOUND)
endif (${CMAKE_MAJOR_VERSION} EQUAL 2 AND ${CMAKE_MINOR_VERSION} EQUAL 4)
endmacro (libfind_pkg_check_modules)
# Do the final processing once the paths have been detected.
# If include dirs are needed, ${PREFIX}_PROCESS_INCLUDES should be set to contain
# all the variables, each of which contain one include directory.
# Ditto for ${PREFIX}_PROCESS_LIBS and library files.
# Will set ${PREFIX}_FOUND, ${PREFIX}_INCLUDE_DIRS and ${PREFIX}_LIBRARIES.
# Also handles errors in case library detection was required, etc.
macro (libfind_process PREFIX)
# Skip processing if already processed during this run
if (NOT ${PREFIX}_FOUND)
# Start with the assumption that the library was found
set (${PREFIX}_FOUND TRUE)
# Process all includes and set _FOUND to false if any are missing
foreach (i ${${PREFIX}_PROCESS_INCLUDES})
if (${i})
set (${PREFIX}_INCLUDE_DIRS ${${PREFIX}_INCLUDE_DIRS} ${${i}})
mark_as_advanced(${i})
else (${i})
set (${PREFIX}_FOUND FALSE)
endif (${i})
endforeach (i)
# Process all libraries and set _FOUND to false if any are missing
foreach (i ${${PREFIX}_PROCESS_LIBS})
if (${i})
set (${PREFIX}_LIBRARIES ${${PREFIX}_LIBRARIES} ${${i}})
mark_as_advanced(${i})
else (${i})
set (${PREFIX}_FOUND FALSE)
endif (${i})
endforeach (i)
# Print message and/or exit on fatal error
if (${PREFIX}_FOUND)
if (NOT ${PREFIX}_FIND_QUIETLY)
message (STATUS "Found ${PREFIX} ${${PREFIX}_VERSION}")
endif (NOT ${PREFIX}_FIND_QUIETLY)
else (${PREFIX}_FOUND)
if (${PREFIX}_FIND_REQUIRED)
foreach (i ${${PREFIX}_PROCESS_INCLUDES} ${${PREFIX}_PROCESS_LIBS})
message("${i}=${${i}}")
endforeach (i)
message (FATAL_ERROR "Required library ${PREFIX} NOT FOUND.\nInstall the library (dev version) and try again. If the library is already installed, use ccmake to set the missing variables manually.")
endif (${PREFIX}_FIND_REQUIRED)
endif (${PREFIX}_FOUND)
endif (NOT ${PREFIX}_FOUND)
endmacro (libfind_process)
macro(libfind_library PREFIX basename)
set(TMP "")
if(MSVC80)
set(TMP -vc80)
endif(MSVC80)
if(MSVC90)
set(TMP -vc90)
endif(MSVC90)
set(${PREFIX}_LIBNAMES ${basename}${TMP})
if(${ARGC} GREATER 2)
set(${PREFIX}_LIBNAMES ${basename}${TMP}-${ARGV2})
string(REGEX REPLACE "\\." "_" TMP ${${PREFIX}_LIBNAMES})
set(${PREFIX}_LIBNAMES ${${PREFIX}_LIBNAMES} ${TMP})
endif(${ARGC} GREATER 2)
find_library(${PREFIX}_LIBRARY
NAMES ${${PREFIX}_LIBNAMES}
PATHS ${${PREFIX}_PKGCONF_LIBRARY_DIRS}
)
endmacro(libfind_library)
View
@@ -683,6 +683,7 @@ INPUT += @tpie_BINARY_DIR@/doc/fractiondb.dox
INPUT += @tpie_BINARY_DIR@/doc/queue.dox
INPUT += @tpie_SOURCE_DIR@/doc/pipelining.dox
INPUT += @tpie_BINARY_DIR@/doc/serialization.dox
INPUT += @tpie_SOURCE_DIR@/doc/compressed.dox
# This tag can be used to specify the character encoding of the source files
# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is
View
@@ -0,0 +1,20 @@
/**
\page compressed Compressed streams
The TPIE stream header has a bit that indicates whether compression is used for
the stream.
An uncompressed stream has blocks that begin on regular B-sized boundaries.
A compressed stream has blocks of varying length that have a block header in
the beginning and in the end of the block.
The block header indicates what compression algorithm has been used for the block.
When opening a new stream for writing, the \c compressionFlags parameter sets
whether compression should be enabled for the stream or not.
If compression is enabled, blocks are compressed with the currently preferred
compression scheme, which can be changed as follows:
\code
tpie::compressor_thread_lock l(tpie::the_compressor_thread());
tpie::the_compressor_thread().set_preferred_compression(l, tpie::compression_scheme::snappy);
\endcode
where \c snappy may be replaced with any scheme defined in \c tpie/compressed/scheme.h.
*/
View
@@ -41,6 +41,7 @@ file \c example/CMakeLists.txt in the repository.
- \ref memory (example snippets)
- \ref progress (example snippets)
- \ref fractiondb (example snippets)
- \ref compressed
\subsection sec_history History
View
@@ -52,3 +52,7 @@ target_link_libraries(test_parallel_sort_threshold tpie)
add_executable(test_job_overhead test_job_overhead.cpp)
set_target_properties(test_job_overhead PROPERTIES FOLDER tpie/test)
target_link_libraries(test_job_overhead tpie)
add_executable(atomic_stats test_atomic_stats.cpp)
set_target_properties(atomic_stats PROPERTIES FOLDER tpie/test)
target_link_libraries(atomic_stats tpie ${Boost_LIBRARIES})
@@ -2,10 +2,18 @@ set (SPEED_DEPS testtime.h stat.h)
list(APPEND SPEED_DEPS "${PROJECT_SOURCE_DIR}/tpie/sysinfo.h")
add_executable(compressed_speed_test compressed.cpp ${SPEED_DEPS})
target_link_libraries(compressed_speed_test tpie)
set_target_properties(compressed_speed_test PROPERTIES FOLDER tpie/test)
add_executable(stream_speed_test stream.cpp ${SPEED_DEPS})
target_link_libraries(stream_speed_test tpie)
set_target_properties(stream_speed_test PROPERTIES FOLDER tpie/test)
add_executable(stream_speed_test_2 stream2.cpp ${SPEED_DEPS})
target_link_libraries(stream_speed_test_2 tpie)
set_target_properties(stream_speed_test_2 PROPERTIES FOLDER tpie/test)
add_executable(progress_speed_test progress.cpp ${SPEED_DEPS})
target_link_libraries(progress_speed_test tpie)
set_target_properties(progress_speed_test PROPERTIES FOLDER tpie/test)
@@ -0,0 +1,101 @@
// -*- mode: c++; tab-width: 4; indent-tabs-mode: t; c-file-style: "stroustrup"; -*-
// vi:set ts=4 sts=4 sw=4 noet :
// Copyright 2013, The TPIE development team
//
// This file is part of TPIE.
//
// TPIE is free software: you can redistribute it and/or modify it under
// the terms of the GNU Lesser General Public License as published by the
// Free Software Foundation, either version 3 of the License, or (at your
// option) any later version.
//
// TPIE is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
// License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with TPIE. If not, see <http://www.gnu.org/licenses/>
#include <iostream>
#include <tpie/file_stream.h>
#include <tpie/compressed/stream.h>
struct file_stream {
typedef tpie::file_stream<size_t> Stream;
static void open(Stream & s, const std::string & fileName) {
s.open(fileName);
}
};
struct uncompressed_stream {
typedef tpie::file_stream<size_t> Stream;
static void open(Stream & s, const std::string & fileName) {
s.open(fileName, tpie::access_read_write, 0, tpie::access_sequential, tpie::compression_none);
}
};
struct compressed_stream {
typedef tpie::file_stream<size_t> Stream;
static void open(Stream & s, const std::string & fileName) {
s.open(fileName);
}
};
size_t f(size_t prev, size_t n) {
return ((42 + prev) * n) >> 3;
}
void usage() {
std::cout << "Parameters: <file_stream|uncompressed|compressed> <read|write> <filename> <items>" << std::endl;
}
template <typename Stream>
void write(Stream & s, size_t items) {
size_t prev = 0;
for (size_t i = 0; i < items; ++i) {
s.write(prev);
prev = f(prev, i);
}
}
template <typename Stream>
void read(Stream & s, size_t items) {
size_t prev = 0;
for (size_t i = 0; i < items; ++i) {
size_t x = s.read();
if (x != prev) {
std::cerr << "bad value at " << i << std::endl;
}
prev = f(prev, i);
}
}
template <typename Traits>
void go_class(const std::string & command, const std::string fileName, size_t items) {
typename Traits::Stream s;
Traits::open(s, fileName);
if (command == "read") read(s, items);
else if (command == "write") write(s, items);
else usage();
}
void go(std::string cls, std::string command, std::string fileName, std::string itemsString) {
size_t items;
std::stringstream(itemsString) >> items;
if (cls == "file_stream") go_class<file_stream>(command, fileName, items);
else if (cls == "uncompressed") go_class<uncompressed_stream>(command, fileName, items);
else if (cls == "compressed") go_class<compressed_stream>(command, fileName, items);
else usage();
}
int main(int argc, char ** argv) {
if (argc != 5) {
usage();
return 1;
}
tpie::tpie_init();
go(argv[1], argv[2], argv[3], argv[4]);
tpie::tpie_finish();
return 0;
}
@@ -154,7 +154,7 @@ struct test_file_accessor {
inline void write() {
tpie::default_file_accessor fa;
fa.open("tmp", false, true, sizeof(test_t), sysinfo::blocksize_bytes(), 0, access_sequential);
fa.open("tmp", false, true, sizeof(test_t), sysinfo::blocksize_bytes(), 0, access_sequential, false);
for (count_t j = 0; j < block_count; ++j) {
for (count_t k = 0; k < itemsPerBlock; ++k) {
block[k] = 42;
@@ -165,7 +165,7 @@ struct test_file_accessor {
inline void read() {
tpie::default_file_accessor fa;
fa.open("tmp", true, false, sizeof(test_t), sysinfo::blocksize_bytes(), 0, access_sequential);
fa.open("tmp", true, false, sizeof(test_t), sysinfo::blocksize_bytes(), 0, access_sequential, false);
for (count_t j = 0; j < block_count; ++j) {
fa.read_block(block, j, itemsPerBlock);
for (count_t k = 0; k < itemsPerBlock; ++k) {
Oops, something went wrong.

0 comments on commit 1023c90

Please sign in to comment.