Merge 1764767 into d9b357f

CMakeLists.txt

@@ -19,6 +19,7 @@ set(COMPILE_TEST OFF CACHE BOOL "")
 set(TPIE_PARALLEL_SORT 1 CACHE BOOL "")
 add_subdirectory(keyvi/3rdparty/tpie EXCLUDE_FROM_ALL)
 
+add_subdirectory(keyvi/3rdparty/tiny-process-library)
 
 set(Boost_USE_STATIC_LIBS OFF)
 set(Boost_USE_MULTITHREADED ON)
@@ -51,7 +52,9 @@ include_directories(keyvi/3rdparty/misc)
 include_directories(keyvi/3rdparty/rapidjson/include)
 include_directories(keyvi/3rdparty/msgpack-c/include)
 include_directories(keyvi/3rdparty/xchange/src)
+include_directories(keyvi/3rdparty/tiny-process-library/)
 include_directories(${CMAKE_BINARY_DIR}/keyvi/3rdparty/tpie)
+include_directories(${CMAKE_BINARY_DIR}/keyvi/3rdparty/tiny-process-library/)
 
 
 FILE(GLOB_RECURSE UNIT_TEST_SOURCES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} keyvi/tests/keyvi/*.cpp)
@@ -64,7 +67,7 @@ add_executable(unit_test_all ${UNIT_TEST_SOURCES})
 target_link_libraries(keyvicompiler tpie ${Boost_LIBRARIES} ${ZLIB_LIBRARIES})
 target_link_libraries(keyviinspector tpie ${Boost_LIBRARIES} ${ZLIB_LIBRARIES})
 target_link_libraries(keyvimerger tpie ${Boost_LIBRARIES} ${ZLIB_LIBRARIES})
-target_link_libraries(unit_test_all tpie ${Boost_LIBRARIES} ${ZLIB_LIBRARIES})
+target_link_libraries(unit_test_all tiny-process-library tpie ${Boost_LIBRARIES} ${ZLIB_LIBRARIES})
 
 install (TARGETS keyvicompiler DESTINATION bin)
 install (TARGETS keyviinspector DESTINATION bin)

keyvi/include/keyvi/index/index_reader.h

@@ -0,0 +1,57 @@
+//
+// keyvi - A key value store.
+//
+// Copyright 2015 Hendrik Muhs<hendrik.muhs@gmail.com>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+/*
+ * index_reader.h
+ *
+ *  Created on: Jan 11, 2017
+ *      Author: hendrik
+ */
+
+#ifndef KEYVI_INDEX_INDEX_READER_H_
+#define KEYVI_INDEX_INDEX_READER_H_
+
+#include <string>
+
+#include "index/internal/base_index_reader.h"
+#include "index/internal/index_reader_worker.h"
+
+// #define ENABLE_TRACING
+#include "dictionary/util/trace.h"
+
+namespace keyvi {
+namespace index {
+
+class IndexReader final : public internal::BaseIndexReader<internal::IndexReaderWorker> {
+ public:
+  explicit IndexReader(const std::string index_directory, size_t refresh_interval = 1 /*, optional external logger*/)
+      : worker_(index_directory, refresh_interval), BaseIndexReader(worker_) {
+    worker_.StartWorkerThread();
+  }
+
+  ~IndexReader() { worker_.StopWorkerThread(); }
+
+  void Reload() { worker_.Reload(); }
+
+ private:
+  internal::IndexReaderWorker worker_;
+};
+} /* namespace index */
+} /* namespace keyvi */
+
+#endif  // KEYVI_INDEX_INDEX_READER_H_

keyvi/include/keyvi/index/index_writer.h

@@ -0,0 +1,112 @@
+//
+// keyvi - A key value store.
+//
+// Copyright 2015 Hendrik Muhs<hendrik.muhs@gmail.com>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+/*
+ * index_writer.h
+ *
+ *  Created on: Jan 11, 2017
+ *      Author: hendrik
+ */
+
+#ifndef KEYVI_INDEX_INDEX_WRITER_H_
+#define KEYVI_INDEX_INDEX_WRITER_H_
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>              //NOLINT
+#include <condition_variable>  //NOLINT
+#include <ctime>
+#include <string>
+#include <thread>  //NOLINT
+#include <vector>
+
+#include <boost/filesystem.hpp>
+#include <boost/filesystem/operations.hpp>
+#include <boost/interprocess/sync/file_lock.hpp>
+#include <boost/property_tree/json_parser.hpp>
+#include <boost/property_tree/ptree.hpp>
+
+#include "dictionary/dictionary.h"
+#include "dictionary/dictionary_compiler.h"
+#include "dictionary/dictionary_types.h"
+#include "dictionary/fsa/internal/serialization_utils.h"
+#include "dictionary/match.h"
+#include "index/internal/base_index_reader.h"
+#include "index/internal/index_writer_worker.h"
+
+#define ENABLE_TRACING
+#include "dictionary/util/trace.h"
+
+namespace keyvi {
+namespace index {
+
+class IndexWriter final : public internal::BaseIndexReader<internal::IndexWriterWorker> {
+ public:
+  explicit IndexWriter(const std::string& index_directory)
+      : index_finalizer_(index_directory), BaseIndexReader(index_finalizer_) {
+    index_directory_ = index_directory;
+
+    index_toc_file_ = index_directory_;
+    index_toc_file_ /= "index.toc";
+
+    // lock the index (filesystem lock)
+    boost::filesystem::path index_lock_file = index_directory_;
+
+    // create dir if it does not exist yet
+    boost::filesystem::create_directories(index_directory_);
+
+    index_lock_file /= "index.lock";
+
+    TRACE("locking index %s", index_lock_file.string().c_str());
+
+    std::ofstream o(index_lock_file.string(), std::ios_base::app);
+
+    index_lock_ = boost::interprocess::file_lock(index_lock_file.string().c_str());
+    index_lock_.lock();
+  }
+
+  ~IndexWriter() {
+    // todo: happens to early, move into own class, destruct after worker is destructed
+    TRACE("Unlock Index");
+    try {
+      index_lock_.unlock();
+    } catch (std::exception& e) {
+      TRACE("exception %s", e.what());
+    }
+  }
+
+  void Set(const std::string& key, const std::string& value) { index_finalizer_.Add(key, value); }
+
+  void Delete(const std::string& key) { index_finalizer_.Delete(key); }
+
+  void Flush(bool async = true) {
+    TRACE("Flush (manually)");
+    index_finalizer_.Flush(async);
+  }
+
+ private:
+  boost::filesystem::path index_directory_;
+  boost::filesystem::path index_toc_file_;
+  boost::interprocess::file_lock index_lock_;
+  internal::IndexWriterWorker index_finalizer_;
+};
+
+} /* namespace index */
+} /* namespace keyvi */
+
+#endif  // KEYVI_INDEX_INDEX_WRITER_H_

keyvi/include/keyvi/index/internal/base_index_reader.h

@@ -0,0 +1,76 @@
+//
+// keyvi - A key value store.
+//
+// Copyright 2015 Hendrik Muhs<hendrik.muhs@gmail.com>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+/*
+ * base_index_reader.h
+ *
+ *  Created on: Nov 1, 2017
+ *      Author: hendrik
+ */
+
+#ifndef KEYVI_INDEX_INTERNAL_BASE_INDEX_READER_H_
+#define KEYVI_INDEX_INTERNAL_BASE_INDEX_READER_H_
+
+#include <string>
+
+#include "dictionary/match.h"
+#include "index/internal/segment.h"
+
+namespace keyvi {
+namespace index {
+namespace internal {
+
+template <class SegmentHolderT>
+class BaseIndexReader {
+ public:
+  explicit BaseIndexReader(const SegmentHolderT& segementHolder) : holder_(segementHolder) {}
+
+  dictionary::Match operator[](const std::string& key) const {
+    dictionary::Match m;
+    segments_t segments = holder_.Segments();
+
+    for (auto it = segments->crbegin(); it != segments->crend(); ++it) {
+      m = (*it)->GetDictionary()->operator[](key);
+      if (!m.IsEmpty()) {
+        return m;
+      }
+    }
+
+    return m;
+  }
+
+  bool Contains(const std::string& key) const {
+    segments_t segments = holder_.Segments();
+    for (auto it = segments->crbegin(); it != segments->crend(); it++) {
+      if ((*it)->GetDictionary()->Contains(key)) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+ private:
+  const SegmentHolderT& holder_;
+};
+
+} /* namespace internal */
+} /* namespace index */
+} /* namespace keyvi */
+
+#endif  // KEYVI_INDEX_INTERNAL_BASE_INDEX_READER_H_