Merge commit 'dfdf0e6595f19c2f5e1229cd022df140ddd98e5d'

# Please enter a commit message to explain why this merge is necessary,
# especially if it merges an updated upstream into a topic branch.
#
# Lines starting with '#' will be ignored, and an empty message aborts
# the commit.

keyvi/3rdparty/tpie/.travis.yml

@@ -1,8 +1,17 @@
+matrix:
+  include:
+    - os: linux
+      compiler: gcc
+    - os: linux
+      compiler: clang
+    - os: osx
+      compiler: clang
 language: cpp
-compiler:
-- gcc
+before install:
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update          ; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install boost snappy; fi
 install:
-- if [ "$CXX" = "g++" ]; then export CXX="g++-4.8" CC="gcc-4.8"; fi
+  - if [ "$CXX" = "g++" ]; then export CXX="g++-4.8" CC="gcc-4.8"; fi
 addons:
   apt:
       sources:
@@ -17,4 +26,4 @@ addons:
       - libboost-system1.55-dev
       - libsnappy-dev
 script: mkdir build && cd build && cmake .. && make -j2 && make test
-sudo: false
+sudo: false

keyvi/3rdparty/tpie/README.md

@@ -1,6 +1,9 @@
 TPIE - The Templated Portable I/O Environment
 =============================================
 
+NIX: [![Build Status](https://travis-ci.org/thomasmoelhave/tpie.svg?branch=master)](https://travis-ci.org/thomasmoelhave/tpie)
+Windows: [![Build status Windows](https://ci.appveyor.com/api/projects/status/570fltgf1ko5twna/branch/master?svg=true)](https://ci.appveyor.com/project/antialize/tpie/branch/master)
+
 The TPIE (Templated Portable I/O Environment) library is a tool box providing
 efficient and convenient tools to ease the implementation of algorithm and data
 structures on very large sets of data.

keyvi/3rdparty/tpie/appveyor.yml

@@ -0,0 +1,54 @@
+# Specify version format
+version: "{build}"
+
+# Operating system (build VM template)
+os: Visual Studio 2015
+
+# build platform, i.e. Win32 (instead of x86), x64, Any CPU. This setting is optional.
+platform:
+  - x64
+
+# specify custom environment variables
+environment:
+  BOOST_ROOT: C:\Libraries\boost_1_60_0
+  BOOST_LIBRARYDIR: C:\Libraries\boost_1_60_0\lib64-msvc-14.0
+
+# build configuration, i.e. Debug, Release, etc.
+configuration:
+  - Debug
+  - Release
+
+# scripts that are called at very beginning, before repo cloning
+init:
+  - cmd: cmake --version
+  - cmd: msbuild /version
+
+# clone directory
+clone_folder: C:\projects\tpie
+
+# branches to build
+branches:
+  # whitelist
+  # only:
+  #   - master
+  # blacklist
+  except:
+    - gh-pages
+
+# scripts to run before build
+before_build:
+  - cmd: cd C:\projects\tpie
+  - cmd: md build
+  - cmd: cd build
+  - cmd: if "%platform%"=="Win32" set CMAKE_GENERATOR_NAME=Visual Studio 14 2015
+  - cmd: if "%platform%"=="x64"   set CMAKE_GENERATOR_NAME=Visual Studio 14 2015 Win64
+  - cmd: cmake -G "%CMAKE_GENERATOR_NAME%" -DCMAKE_BUILD_TYPE=%configuration% -DBOOST_ROOT="%BOOST_ROOT%" -DBOOST_LIBRARYDIR="%BOOST_LIBRARYDIR%" ..
+
+build:
+  parallel: true                           # enable MSBuild parallel builds
+  project: C:\projects\tpie\build\tpie.sln # path to Visual Studio solution or project
+
+test_script:
+  - cmd: cd C:\projects\tpie
+  - cmd: cd build
+  - cmd: ctest --timeout 30 -j 2 --output-on-failure -C "%CONFIG%" ctest 

keyvi/3rdparty/tpie/doc/CMakeLists.txt

@@ -18,8 +18,8 @@ else (INTERNAL_DOCS)
 endif(INTERNAL_DOCS)
 
 ADD_CUSTOM_TARGET(apidocs
-	sh -c "sed -e \"s/@VERSION@/`git describe --always`/g\" ${PROJECT_SOURCE_DIR}/doc/header.html > ${PROJECT_BINARY_DIR}/doc/header.html"
-	COMMAND sh -c "sed -e \"s/@VERSION@/`git describe --always`/g\" ${PROJECT_SOURCE_DIR}/doc/footer.html > ${PROJECT_BINARY_DIR}/doc/footer.html"
+	sh -c "sed -e \"s/\@VERSION\@/`git describe --always`/g\" ${PROJECT_SOURCE_DIR}/doc/header.html > ${PROJECT_BINARY_DIR}/doc/header.html"
+	COMMAND sh -c "sed -e \"s/\@VERSION\@/`git describe --always`/g\" ${PROJECT_SOURCE_DIR}/doc/footer.html > ${PROJECT_BINARY_DIR}/doc/footer.html"
 	COMMAND ${DOXYGEN} ${DOXYFILE}
 	VERBATIM)
 

keyvi/3rdparty/tpie/doc/authors.dox

@@ -6,6 +6,7 @@ The TPIE development team consists of the following people:
 \section sec_current Current contributors
 
   - <a href="http://cs.swarthmore.edu/~adanner/">Andrew Danner</a>
+  - <a href="http://cs.au.dk/~tyilo/">Asger Hautop Drewsen</a>
   - <a href="http://madalgo.au.dk/~jakobt/">Jakob Truelsen</a>
   - <a href="http://ls11-www.cs.tu-dortmund.de/staff/jv">Jan Vahrenhold</a>
   - <a href="http://madalgo.au.dk/~large/">Lars Arge</a>

keyvi/3rdparty/tpie/doc/index.dox

@@ -57,6 +57,6 @@ as a group of contributors in Europe and the US.
 A full list of current and past contributors can be found in the \ref authors section.
 
 The current maintainer is
-<a href="http://cs.au.dk/~rav/">Mathias Rav</a>.
+<a href="http://cs.au.dk/~tyilo/">Asger Hautop Drewsen</a>.
 
 */

keyvi/3rdparty/tpie/doc/pipelining.dox

@@ -74,7 +74,7 @@ private:
 	dest_t dest;
 };
 
-typedef tp::pipe_middle<tp::factory_0<hello_world_type> > hello_world;
+typedef tp::pipe_middle<tp::factory<hello_world_type> > hello_world;
 \endcode
 
 A node implementation may supply the following extra information to the framework
@@ -185,7 +185,7 @@ private:
 	dest_t dest;
 };
 
-typedef tp::pipe_middle<tp::factory_0<hello_world_type> > hello_world;
+typedef tp::pipe_middle<tp::factory<hello_world_type> > hello_world;
 \endcode
 
 \section sec_pull Pull nodes
@@ -207,28 +207,23 @@ The implementation details of pull nodes are similar to regular nodes. The follo
 template <typename source_t>
 class pull_hello_world_type : public node {
 public:
-	typedef int item_type;
-
-	pull_hello_world_type(source_t source) : source(std::move(source)) {
-		add_pull_source(source);
-	}
-
-	item_type pull() {
-		int item = source.can_pull();
-		if (item % 2 == 0)
-			return item/2;
-		return (3 * item + 1);
-	}
-
-	bool can_pull() {
-		return source.can_pull();
-	}
-
+    typedef int item_type;
+    pull_hello_world_type(source_t source) : source(std::move(source)) {
+        add_pull_source(source);
+    }
+    item_type pull() {
+        int item = source.pull();
+        if (item % 2 == 0)
+            return item/2;
+        return (3 * item + 1);
+    }
+    bool can_pull() {
+        return source.can_pull();
+    }
 private:
-	source_t source;
+    source_t source;
 };
-
-typedef pullpipe_middle<factory_0<pull_hello_world_type> > pull_hello_world;
+typedef pullpipe_middle<factory<pull_hello_world_type> > pull_hello_world;
 \endcode
 
 \section sec_forming_pipelines Forming pipelines
@@ -359,8 +354,8 @@ public:
 };
 
 template <typename source_t>
-inline pipe_begin<tempfactory_1<square_t<source_t>, source_t> > square(const source_t & source) {
-	return tempfactory_1<square_t<source_t>, source_t >(source);
+inline pipe_begin<tempfactory<square_t<source_t>, source_t> > square(const source_t & source) {
+	return tempfactory<square_t<source_t>, source_t >(source);
 }
 \endcode
 \code
@@ -671,7 +666,7 @@ instantiate the node implementations.
 Usually, the built-in factories contained in \c factory_helpers.h will suffice:
 
 \code
-typedef tpie::pipelining::factory_0<hello_world_type> hello_world_factory;
+typedef tpie::pipelining::factory<hello_world_type> hello_world_factory;
 \endcode
 
 but in some cases it is helpful to implement one's own factory.
@@ -731,7 +726,7 @@ To use the above defined factories, we might write the following:
 \code
 using namespace tpie;
 using namespace tpie::pipelining;
-factory_1<input_t, file_stream<memory_size_type> &> fact0(inputstream);
+factory<input_t, file_stream<memory_size_type> &> fact0(inputstream);
 hello_world_factory fact1;
 goodbye_world_factory fact2;
 pipeline p = fact0.construct(fact1.construct(fact2.construct()));
@@ -743,14 +738,14 @@ helper classes to ease the construction of pipelines, namely the descendants of
 \c pipe_base which are called \c pipe_begin, \c pipe_middle and \c pipe_end.
 
 \code
-inline pipe_middle<factory_0<hello_world_type> >
+inline pipe_middle<factory<hello_world_type> >
 hello_world() {
-	return factory_0<hello_world_type>();
+	return factory<hello_world_type>();
 }
 
-inline pipe_end<termfactory_0<goodbye_world_type> >
+inline pipe_end<termfactory<goodbye_world_type> >
 goodbye_world() {
-	return termfactory_0<goodbye_world_type>();
+	return termfactory<goodbye_world_type>();
 }
 \endcode
 

keyvi/3rdparty/tpie/doc/pipelining_nodes.dox

@@ -44,6 +44,7 @@
 \ref tpie::pipelining::store_sort
 \ref tpie::pipelining::tee
 \ref tpie::pipelining::unzip
+\ref tpie::pipelining::visit
 \ref tpie::pipelining::zip
 \endsecreflist
 

keyvi/3rdparty/tpie/doc/setup.dox

@@ -68,8 +68,7 @@ sudo tar xvf path/to/tpie/build/tpie-1.0.0-Linux.tar.gz
 \endverbatim
 
 To link your application with TPIE, may manually specify the linker flags
-<tt>-ltpie -lboost_system -lboost_filesystem -lboost_date_time
--lboost_thread</tt>, or you may use the example \c CMakeLists.txt from the
+<tt>-ltpie -lboost_system -lboost_filesystem -lboost_date_time</tt>, or you may use the example \c CMakeLists.txt from the
 \c tpie/example/ directory to find the TPIE installation using CMake.
 
 <h2>Windows</h2>

keyvi/3rdparty/tpie/test/speed_regression/btree.cpp

@@ -59,10 +59,10 @@ void test(size_t times, size_t size) {
 		btree<int, btree_external> tree(tmp.path());
 
 		// pre-protocol
-		int x[count];
+		std::vector<int> x(count);
 		for(size_t i = 0; i < count; ++i)
 			x[i] = i;
-		std::random_shuffle(x, x + count);
+		std::random_shuffle(x.begin(), x.end());
 
 		// insertion
 		getTestRealtime(start);
@@ -81,7 +81,7 @@ void test(size_t times, size_t size) {
 		s(testRealtimeDiff(start,end));
 
 		// deletion
-		std::random_shuffle(x, x + count);
+		std::random_shuffle(x.begin(), x.end());
 
 		getTestRealtime(start);
 		for(size_t i = 0; i < count; ++i) {

keyvi/3rdparty/tpie/test/speed_regression/stream2.cpp

@@ -63,7 +63,7 @@ class series_crtp : public series_base {
 		self().end(n);
 	}
 
-	virtual const char * name() const { return self().get_name(); }
+	const char * name() const override { return self().get_name(); }
 };
 
 class series_random : public series_crtp<series_random> {

keyvi/3rdparty/tpie/test/speed_regression/testtime.h

@@ -90,7 +90,7 @@ namespace tpie {
 		/// Calculate real time difference in micro seconds
 		///////////////////////////////////////////////////////////////////
 		inline uint_fast64_t testRealtimeDiff(const test_realtime_t a, const test_realtime_t b) {
-			return tpie::test_millisecs(a, b);
+			return (uint_fast64_t)tpie::test_millisecs(a, b);
 		}
 	}
 }

keyvi/3rdparty/tpie/test/unit/CMakeLists.txt

@@ -53,6 +53,8 @@ add_unittest(compressed_stream
 
 	odd_block_size write_only
 	write_peek
+
+	lockstep_reverse
 )
 add_unittest(btree
 	internal_augment
@@ -95,6 +97,7 @@ add_unittest(merge_sort
 	evacuate_before_merge
 	evacuate_before_report
 	sort_upper_bound
+	sort_faulty_upper_bound
 	temp_file_usage
 	tall_tree
 	)
@@ -110,6 +113,7 @@ add_unittest(serialization_sort
 	small_final_fanout
 	evacuate_before_merge
 	evacuate_before_report
+	file_limit
 	)
 add_unittest(stats simple)
 add_unittest(stream
@@ -151,6 +155,9 @@ add_unittest(pipelining
 	merger_memory
 	bound_fetch_forward
 	fetch_forward
+	forward_multiple_pipelines
+	forward_unique_ptr
+	pipe_base_forward
 	virtual
 	virtual_cref_item_type
 	virtual_fork
@@ -165,16 +172,27 @@ add_unittest(pipelining
 	parallel_push_in_end
 	node_map
 	join
+	split
 	copy_ctor
 	datastructures
 	phase_priority_test
 	set_flush_priority_test
 	node_map
+	subpipeline
+	file_limit_sort
+	passive_virtual_management
+	join_split_dealloc
 	)
-add_unittest(pipelining_runtime evacuate get_phase_graph)
+add_unittest(pipelining_runtime evacuate get_phase_graph evacuate_phase_graph)
 add_unittest(pipelining_serialization basic reverse sort)
 add_unittest(maybe basic unique_ptr)
-add_unittest(close_file internal serialization_writer_close serialization_writer_dtor serialization_reader_dtor)
+add_unittest(close_file
+	internal
+	serialization_writer_close
+	serialization_writer_dtor
+	serialization_reader_dtor
+	file_limit_enforcement
+)
 add_unittest(node_name gcc msvc)
 add_unittest(snappy basic)
 

keyvi/3rdparty/tpie/test/unit/merge_sort.h

@@ -75,7 +75,8 @@ static bool sort_test(memory_size_type m1,
 					  double mb_data,
 					  memory_size_type extraMemory = 0,
 					  bool evacuateBeforeMerge = false,
-					  bool evacuateBeforeReport = false)
+					  bool evacuateBeforeReport = false,
+					  memory_size_type file_limit = 0)
 {
 	m1 *= 1024*1024;
 	m2 *= 1024*1024;
@@ -88,6 +89,7 @@ static bool sort_test(memory_size_type m1,
 	relative_memory_usage m(extraMemory);
 	sorter s;
 	s.set_available_memory(m1, m2, m3);
+	s.set_available_files(file_limit);
 
 	log_debug() << "Begin phase 1" << std::endl;
 	m.set_threshold(m1);
@@ -177,6 +179,13 @@ static bool evacuate_before_report_test() {
 	return sort_test(20,20,20,50, 0, false, true);
 }
 
+static bool file_limit_test(int limit) {
+	get_file_manager().set_limit(limit);
+	get_file_manager().set_enforcement(file_manager::ENFORCE_THROW);
+
+	return sort_test(15,15,15,40, 0, false, false, limit);
+}
+
 public:
 
 static tests & add_all(tests & t) {
@@ -192,6 +201,10 @@ static tests & add_all(tests & t) {
 		;
 }
 
+static tests & add_file_limit_test(tests & t, int limit) {
+	return t.test(file_limit_test, "file_limit", "limit", limit);
+}
+
 };
 
 #endif // TPIE_TEST_MERGE_SORT_H