Permalink
Browse files

Merge branch 'release/0.3.3'

  • Loading branch information...
2 parents f3a80c8 + 6388a1e commit 22ce13ec7a523c65897266b09a51ef9807ac9a9c @rimms rimms committed Oct 29, 2012
Showing with 4,518 additions and 2,621 deletions.
  1. +4 −1 Makefile
  2. +19 −0 README.rst
  3. +2 −2 configure
  4. +125 −112 src/common/cached_zk.cpp
  5. +23 −22 src/common/cached_zk.hpp
  6. +35 −30 src/common/cached_zk_test.cpp
  7. +108 −97 src/common/cht.cpp
  8. +44 −43 src/common/cht.hpp
  9. +2 −2 src/common/cht_test.cpp
  10. +16 −5 src/common/global_id_generator.cpp
  11. +6 −5 src/common/global_id_generator.hpp
  12. +6 −5 src/common/hash.hpp
  13. +25 −18 src/common/lock_service.cpp
  14. +60 −60 src/common/lock_service.hpp
  15. +108 −98 src/common/membership.cpp
  16. +28 −27 src/common/membership.hpp
  17. +77 −0 src/common/mprpc/rpc_client.cpp
  18. +21 −1 src/common/mprpc/rpc_client.hpp
  19. +3 −2 src/common/mprpc/rpc_client_test.cpp
  20. +2 −2 src/common/mprpc/wscript
  21. +1 −1 src/common/rpc_util.hpp
  22. +67 −50 src/common/util.cpp
  23. +10 −3 src/common/util.hpp
  24. +14 −0 src/common/util_test.cpp
  25. +16 −2 src/common/wscript
  26. +248 −219 src/common/zk.cpp
  27. +72 −70 src/common/zk.hpp
  28. +11 −10 src/common/zk_test.cpp
  29. +2 −1 src/framework.hpp
  30. +0 −409 src/framework/jubatus_serv.cpp
  31. +0 −111 src/framework/jubatus_serv.hpp
  32. +6 −2 src/framework/keeper.cpp
  33. +18 −18 src/framework/keeper.hpp
  34. +34 −65 src/framework/mixable.hpp
  35. +89 −0 src/framework/mixable_test.cpp
  36. +0 −99 src/framework/mixer.cpp
  37. +0 −87 src/framework/mixer.hpp
  38. +51 −0 src/framework/mixer/dummy_mixer.hpp
  39. +257 −0 src/framework/mixer/linear_mixer.cpp
  40. +97 −0 src/framework/mixer/linear_mixer.hpp
  41. +86 −0 src/framework/mixer/linear_mixer_test.cpp
  42. +48 −0 src/framework/mixer/mixer.hpp
  43. +43 −0 src/framework/mixer/mixer_factory.cpp
  44. +34 −0 src/framework/mixer/mixer_factory.hpp
  45. +31 −0 src/framework/mixer/wscript
  46. +96 −0 src/framework/server_base.cpp
  47. +70 −0 src/framework/server_base.hpp
  48. +78 −0 src/framework/server_helper.cpp
  49. +145 −0 src/framework/server_helper.hpp
  50. +19 −6 src/framework/server_util.cpp
  51. +5 −22 src/framework/server_util.hpp
  52. +14 −0 src/framework/server_util_test.cpp
  53. +27 −7 src/framework/wscript
  54. +1 −1 src/fv_converter/character_ngram.cpp
  55. +1 −1 src/fv_converter/character_ngram.hpp
  56. +10 −0 src/fv_converter/converter_config.cpp
  57. +5 −1 src/fv_converter/converter_config.hpp
  58. +27 −0 src/fv_converter/converter_config_test.cpp
  59. +52 −6 src/fv_converter/datum_to_fv_converter.cpp
  60. +9 −1 src/fv_converter/datum_to_fv_converter.hpp
  61. +37 −38 src/fv_converter/datum_to_fv_converter_test.cpp
  62. +12 −1 src/fv_converter/dynamic_loader.cpp
  63. +1 −1 src/fv_converter/dynamic_splitter.cpp
  64. +1 −1 src/fv_converter/dynamic_splitter.hpp
  65. +47 −0 src/fv_converter/feature_hasher.cpp
  66. +37 −0 src/fv_converter/feature_hasher.hpp
  67. +48 −0 src/fv_converter/feature_hasher_test.cpp
  68. +0 −3 src/fv_converter/re2_filter.hpp
  69. +1 −1 src/fv_converter/space_splitter.cpp
  70. +1 −1 src/fv_converter/space_splitter.hpp
  71. +1 −1 src/fv_converter/test_splitter.cpp
  72. +1 −1 src/fv_converter/weight_manager.cpp
  73. +2 −2 src/fv_converter/weight_manager_test.cpp
  74. +1 −1 src/fv_converter/without_split.cpp
  75. +1 −1 src/fv_converter/without_split.hpp
  76. +1 −1 src/fv_converter/word_splitter.hpp
  77. +3 −1 src/fv_converter/wscript
  78. +21 −8 src/plugin/fv_converter/mecab_splitter.cpp
  79. +2 −2 src/plugin/fv_converter/mecab_splitter.hpp
  80. +31 −0 src/plugin/fv_converter/mecab_splitter_test.cpp
  81. +1 −1 src/plugin/fv_converter/re2_splitter.cpp
  82. +1 −1 src/plugin/fv_converter/re2_splitter.hpp
  83. +1 −1 src/plugin/fv_converter/ux_splitter.cpp
  84. +1 −1 src/plugin/fv_converter/ux_splitter.hpp
  85. +13 −1 src/plugin/fv_converter/wscript
  86. +1 −0 src/recommender/inverted_index.cpp
  87. +1 −0 src/recommender/lsh.cpp
  88. +1 −1 src/recommender/minhash.cpp
  89. +10 −0 src/server/classifier_client.hpp
  90. +9 −9 src/server/classifier_impl.cpp
  91. +15 −9 src/server/classifier_keeper.cpp
  92. +48 −104 src/server/classifier_serv.cpp
  93. +25 −55 src/server/classifier_serv.hpp
  94. +0 −7 src/server/diffv.hpp
  95. +10 −0 src/server/graph_client.hpp
  96. +24 −24 src/server/graph_impl.cpp
  97. +27 −21 src/server/graph_keeper.cpp
  98. +158 −136 src/server/graph_serv.cpp
  99. +36 −32 src/server/graph_serv.hpp
  100. +59 −0 src/server/linear_function_mixer.cpp
  101. +24 −0 src/server/linear_function_mixer.hpp
  102. +96 −0 src/server/linear_function_mixer_test.cpp
  103. +32 −0 src/server/mixable_weight_manager.cpp
  104. +16 −32 src/server/mixable_weight_manager.hpp
  105. +10 −0 src/server/recommender_client.hpp
  106. +18 −18 src/server/recommender_impl.cpp
  107. +24 −18 src/server/recommender_keeper.cpp
  108. +51 −63 src/server/recommender_serv.cpp
  109. +32 −34 src/server/recommender_serv.hpp
  110. +50 −0 src/server/recommender_serv_test.cpp
  111. +22 −1 src/server/recommender_test.cpp
  112. +10 −0 src/server/regression_client.hpp
  113. +9 −9 src/server/regression_impl.cpp
  114. +15 −9 src/server/regression_keeper.cpp
  115. +47 −88 src/server/regression_serv.cpp
  116. +28 −58 src/server/regression_serv.hpp
  117. +10 −0 src/server/stat_client.hpp
  118. +14 −14 src/server/stat_impl.cpp
  119. +20 −14 src/server/stat_keeper.cpp
  120. +32 −13 src/server/stat_serv.cpp
  121. +30 −33 src/server/stat_serv.hpp
  122. +21 −1 src/server/test_util.hpp
  123. +32 −10 src/server/wscript
  124. +1 −1 src/stat/mixable_stat.cpp
  125. +1 −0 src/storage/storage_type.hpp
  126. +28 −0 src/storage/storage_type_test.cpp
  127. +1 −0 src/storage/wscript
  128. +14 −5 tools/generator/impl_generator.ml
  129. +15 −7 tools/generator/keeper_generator.ml
  130. +1 −1 tools/generator/server_generator.ml
  131. +4 −0 tools/packaging/allinone/.gitignore
  132. +445 −0 tools/packaging/allinone/jubapkg
  133. +20 −0 tools/packaging/allinone/jubapkg_version
  134. +1 −0 tools/packaging/allinone/metadata/debian/compat
  135. +15 −0 tools/packaging/allinone/metadata/debian/control
  136. +4 −0 tools/packaging/allinone/metadata/debian/jubatus.lintian-overrides
  137. +22 −0 tools/packaging/allinone/metadata/debian/rules
  138. +75 −0 tools/packaging/homebrew/jubatus.rb
  139. +25 −0 tools/packaging/homebrew/pficommon.rb
  140. BIN unittest_gtest.py
  141. +5 −1 wscript
View
@@ -1,4 +1,4 @@
-.PHONY: all
+.PHONY: all clean build check install
all:
./waf
@@ -11,3 +11,6 @@ build:
check:
./waf --checkall
+
+install:
+ ./waf install
View
@@ -31,6 +31,25 @@ LGPL 2.1
Update history
--------------
+Release 0.3.3 2012/10/29
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Improvements
+ - deb package (#14)
+ - Jubatus loads plugin from default directory (#57)
+ - Add hash_max_size option to learn in fixed-size memory (#67)
+ - OS X Homebrew packaging (#116)
+ - GCC compilation version <= 4.2 when zookeeper enabled (#60)
+ - Experimental support for Clang compilation (#100)
+ - Make the timeout smaller in unittest
+ - libmecab_splitter works well in multi-thread environment, and now only support mecab ver. 0.99 or later
+ - word_splitter::split method is now constant
+ - global_id_generator(standalone-mode) for graph, that supports 32 bit environment (#135)
+ - Use (document_frequency + 1) to calculate IDF weight to avoid inifinity
+
+Bugfix
+ - #94, #104, #106, #108, #110, #111, #113, #114, #117, #118, #124, #126, #129, #133, #138, #139, #146, #148
+
Release 0.3.2 2012/9/21
~~~~~~~~~~~~~~~~~~~~~~~
View
@@ -1,3 +1,3 @@
-#! /bin/sh
+#!/bin/sh
-./waf configure $*
+./waf configure "$@"
View
@@ -22,132 +22,145 @@ using pfi::concurrent::scoped_lock;
using std::vector;
using std::string;
-namespace jubatus{
-namespace common{
-
- cached_zk::cached_zk(const std::string& hosts, int timeout, const std::string& logfile):
- zk(hosts, timeout, logfile)
- {
- };
-
- cached_zk::~cached_zk(){
- };
-
- void cached_zk::list(const std::string& path, std::vector<std::string>& out){
- out.clear();
- scoped_lock lk(m_);
- std::map<std::string, std::set<std::string> >::const_iterator it = list_cache_.find(path);
- if(it == list_cache_.end()){
- //first time. get list to create cache and set the watcher at the same time.
- {DLOG(INFO) << "creating cache: " << path; }
- std::set<std::string> tmp_list;
- list_(path, tmp_list);
- list_cache_[path] = tmp_list;
-
- for(std::set<std::string>::const_iterator i=tmp_list.begin();
- i!=tmp_list.end(); ++i){
- out.push_back(*i);
- }
-
- }else{
- for(std::set<std::string>::const_iterator i=it->second.begin();
- i!=it->second.end(); ++i){
- out.push_back(*i);
- }
+namespace jubatus {
+namespace common {
+
+cached_zk::cached_zk(const string& hosts, int timeout, const string& logfile):
+ zk(hosts, timeout, logfile)
+{
+}
+
+cached_zk::~cached_zk()
+{
+}
+
+void cached_zk::list(const string& path, vector<string>& out)
+{
+ out.clear();
+ scoped_lock lk(m_);
+ std::map<string, std::set<string> >::const_iterator it = list_cache_.find(path);
+ if (it == list_cache_.end()) {
+ //first time. get list to create cache and set the watcher at the same time.
+ { DLOG(INFO) << "creating cache: " << path; }
+ std::set<string> tmp_list;
+ list_(path, tmp_list);
+ list_cache_[path] = tmp_list;
+
+ for (std::set<string>::const_iterator i=tmp_list.begin();
+ i!=tmp_list.end(); ++i) {
+ out.push_back(*i);
}
- }
- void cached_zk::list_(const std::string& path, std::set<std::string>& out){
- out.clear();
- struct String_vector s;
- int rc = zoo_wget_children(zh_, path.c_str(), cached_zk::update_cache, this, &s);
-
- if(rc == ZOK){
- std::set<std::string> cache;
- for(int i=0; i<s.count; ++i){
- out.insert(s.data[i]);
- }
- }else{
- LOG(ERROR) << zerror(rc) << " (" << path << ")";
+ } else {
+ for (std::set<string>::const_iterator i=it->second.begin();
+ i!=it->second.end(); ++i) {
+ out.push_back(*i);
}
}
-
- void cached_zk::hd_list(const std::string& path, std::string& out){
- out.clear();
- scoped_lock lk(m_);
- const std::set<std::string>& list(list_cache_[path]);
- if(!list.empty())
- out = *(list.begin());
- };
- const std::string cached_zk::type()const{
- return "cached_zk";
- }
+}
- void cached_zk::reload_cache(const std::string& path){
- scoped_lock lk(m_);
- list_(path, list_cache_[path]);
- }
- void cached_zk::clear_cache(const char* path){
- scoped_lock lk(m_);
- if(path == NULL){
- list_cache_.clear();
- znode_cache_.clear();
- }else{
- list_cache_.erase(std::string(path));
- znode_cache_.erase(std::string(path));
+void cached_zk::list_(const string& path, std::set<std::string>& out)
+{
+ out.clear();
+ struct String_vector s;
+ int rc = zoo_wget_children(zh_, path.c_str(), cached_zk::update_cache, this, &s);
+
+ if (rc == ZOK) {
+ std::set<string> cache;
+ for (int i=0; i<s.count; ++i) {
+ out.insert(s.data[i]);
}
+ } else {
+ LOG(ERROR) << zerror(rc) << " (" << path << ")";
}
+}
- void cached_zk::update_cache(zhandle_t* zh, int type, int state,
- const char* path, void* ctx){
- cached_zk* zk = reinterpret_cast<cached_zk*>(ctx);
-
- if( type == ZOO_CHILD_EVENT ){
- // update cache
-
- { LOG(INFO) << "ZOO_CHILD_EVENT: " << path; }
- zk->reload_cache(path);
-
- }else if( type == ZOO_DELETED_EVENT || // clear one cache
- type == ZOO_NOTWATCHING_EVENT || // clear one cache?
- type == ZOO_SESSION_EVENT ){ // path == NULL, clear all cache
- // clear cache
- { LOG(INFO) << "ZOO_(DELETED|NOTWATCHING|SESSION)_EVENT: " << path; }
- zk->clear_cache(path);
- }
- // ZOO_CHANGED_EVENT => ignore (FIXME, when read() cache going to be modified this needs fix)
- // ZOO_CREATED_EVENT => ignore
+void cached_zk::hd_list(const string& path, string& out)
+{
+ out.clear();
+ scoped_lock lk(m_);
+ const std::set<string>& list(list_cache_[path]);
+ if (!list.empty())
+ out = *(list.begin());
+}
+
+const string cached_zk::type() const
+{
+ return "cached_zk";
+}
+
+void cached_zk::reload_cache(const string& path)
+{
+ scoped_lock lk(m_);
+ list_(path, list_cache_[path]);
+}
+
+void cached_zk::clear_cache(const char* path)
+{
+ scoped_lock lk(m_);
+ if (path == NULL) {
+ list_cache_.clear();
+ znode_cache_.clear();
+ } else {
+ list_cache_.erase(string(path));
+ znode_cache_.erase(string(path));
}
+}
+void cached_zk::update_cache(zhandle_t* zh, int type, int state,
+ const char* path, void* ctx)
+{
+ cached_zk* zk = static_cast<cached_zk*>(ctx);
- bool cached_zk::read(const std::string& path, std::string& out){
- scoped_lock lk(m_);
- std::map<std::string, std::string>::const_iterator it = znode_cache_.find(path);
+ if (type == ZOO_CHILD_EVENT) {
+ // update cache
- if(it == znode_cache_.end()){
- {DLOG(INFO) << "creating cache: " << path; }
+ { LOG(INFO) << "ZOO_CHILD_EVENT: " << path; }
+ zk->reload_cache(path);
- if(read_(path, out))
- znode_cache_[path] = out;
+ } else if (type == ZOO_DELETED_EVENT || // clear one cache
+ type == ZOO_NOTWATCHING_EVENT || // clear one cache?
+ type == ZOO_SESSION_EVENT) { // path == NULL, clear all cache
+ // clear cache
+ { LOG(INFO) << "ZOO_(DELETED|NOTWATCHING|SESSION)_EVENT: " << path; }
+ zk->clear_cache(path);
+ }
+ // ZOO_CHANGED_EVENT => ignore (FIXME, when read() cache going to be modified this needs fix)
+ // ZOO_CREATED_EVENT => ignore
+}
- }else{
- out = it->second;
- }
- return true;
+
+bool cached_zk::read(const string& path, string& out)
+{
+ scoped_lock lk(m_);
+ std::map<string, string>::const_iterator it = znode_cache_.find(path);
+
+ if (it == znode_cache_.end()) {
+ { DLOG(INFO) << "creating cache: " << path; }
+
+ if (read_(path, out))
+ znode_cache_[path] = out;
+
+ } else {
+ out = it->second;
}
- bool zk::read_(const std::string& path, std::string& out){
- char buf[1024];
- int buflen = 1024;
- int rc = zoo_wget(zh_, path.c_str(), cached_zk::update_cache, this, buf, &buflen, NULL);
-
- if(rc == ZOK){
- out = string(buf, buflen);
- return buflen <= 1024;
- }else{
- LOG(ERROR) << zerror(rc);
- return false;
- }
- };
-
+ return true;
}
+
+bool zk::read_(const string& path, string& out)
+{
+ char buf[1024];
+ int buflen = 1024;
+ int rc = zoo_wget(zh_, path.c_str(), cached_zk::update_cache, this, buf, &buflen, NULL);
+
+ if (rc == ZOK) {
+ out = string(buf, buflen);
+ return buflen <= 1024;
+ } else {
+ LOG(ERROR) << zerror(rc);
+ return false;
+ }
}
+
+} // common
+} // jubatus
View
@@ -21,34 +21,35 @@
#include <set>
#include "zk.hpp"
-namespace jubatus{
+namespace jubatus {
namespace common {
// TODO: write zk mock and test them all?
- class cached_zk : zk {
- public:
- // timeout [ms]
- cached_zk(const std::string& hosts, int timeout = 10, const std::string& logfile = "");
- virtual ~cached_zk();
+class cached_zk : public zk {
+public:
+ // timeout [ms]
+ cached_zk(const std::string& hosts, int timeout = 10, const std::string& logfile = "");
+ virtual ~cached_zk();
- void list(const std::string& path, std::vector<std::string>& out);
- void hd_list(const std::string& path, std::string& out);
+ void list(const std::string& path, std::vector<std::string>& out);
+ void hd_list(const std::string& path, std::string& out);
- // reads data (should be smaller than 1024B)
- bool read(const std::string& path, std::string& out);
+ // reads data (should be smaller than 1024B)
+ bool read(const std::string& path, std::string& out);
- const std::string type() const;
+ const std::string type() const;
- void check_and_update(const std::string& path);
- void clear_cache(const char* path);
- static void update_cache(zhandle_t*, int, int, const char*, void*);
- void reload_cache(const std::string& path);
+ void check_and_update(const std::string& path);
+ void clear_cache(const char* path);
+ static void update_cache(zhandle_t*, int, int, const char*, void*);
+ void reload_cache(const std::string& path);
- private:
- void list_(const std::string& path, std::set<std::string>& out);
- std::map<std::string, std::set<std::string> > list_cache_;
- std::map<std::string, std::string> znode_cache_;
+private:
+ void list_(const std::string& path, std::set<std::string>& out);
+ std::map<std::string, std::set<std::string> > list_cache_;
+ std::map<std::string, std::string> znode_cache_;
- };
-}
-}
+};
+
+} // common
+} // jubatus
Oops, something went wrong.

0 comments on commit 22ce13e

Please sign in to comment.