Permalink
Browse files

Merge branch 'release/0.3.0'

  • Loading branch information...
2 parents a55e2f4 + ddf3f58 commit e2a2397fb206304a9b9f82e265bddc781277752d @beam2d beam2d committed Jun 29, 2012
Showing with 5,618 additions and 635 deletions.
  1. +0 −1 .gitignore
  2. +13 −0 Makefile
  3. +37 −0 README.rst
  4. +3 −0 configure
  5. +11 −0 jubatus.pc.in
  6. +1 −1 src/classifier/classifier_factory.cpp
  7. +4 −1 src/cmd/jubaconv.cpp
  8. +6 −2 src/cmd/jubactl.cpp
  9. +7 −8 src/cmd/wscript
  10. +48 −11 src/common/cht.cpp
  11. +16 −2 src/common/cht.hpp
  12. +53 −0 src/common/exception.cpp
  13. +311 −24 src/common/exception.hpp
  14. +142 −0 src/common/exception_info.hpp
  15. +291 −0 src/common/exception_test.cpp
  16. +65 −0 src/common/global_id_generator.cpp
  17. +54 −0 src/common/global_id_generator.hpp
  18. +3 −2 src/common/lock_service.cpp
  19. +3 −0 src/common/lock_service.hpp
  20. +6 −4 src/common/mprpc/rpc_client_test.cpp
  21. +1 −1 src/common/mprpc/wscript
  22. +15 −5 src/common/util.cpp
  23. +2 −0 src/common/util.hpp
  24. +2 −1 src/common/wscript
  25. +27 −4 src/common/zk.cpp
  26. +2 −0 src/common/zk.hpp
  27. +5 −1 src/framework/aggregators.hpp
  28. +105 −27 src/framework/jubatus_serv.cpp
  29. +10 −9 src/framework/jubatus_serv.hpp
  30. +13 −6 src/framework/keeper.cpp
  31. +69 −38 src/framework/keeper.hpp
  32. +16 −4 src/framework/mixable.hpp
  33. +48 −48 src/framework/mixer.cpp
  34. +7 −5 src/framework/mixer.hpp
  35. +4 −4 src/framework/server_util.cpp
  36. +29 −22 src/framework/server_util.hpp
  37. +7 −7 src/fv_converter/converter_config.cpp
  38. +2 −2 src/fv_converter/converter_config_test.cpp
  39. +17 −7 src/fv_converter/datum_to_fv_converter.cpp
  40. +29 −0 src/fv_converter/datum_to_fv_converter_test.cpp
  41. +9 −3 src/fv_converter/dynamic_loader.cpp
  42. +3 −6 src/fv_converter/exception.hpp
  43. +1 −1 src/fv_converter/key_matcher_factory.cpp
  44. +1 −1 src/fv_converter/libsvm_converter.cpp
  45. +1 −1 src/fv_converter/num_feature_factory.cpp
  46. +1 −1 src/fv_converter/num_filter_factory.cpp
  47. +1 −1 src/fv_converter/re2_filter.cpp
  48. +1 −1 src/fv_converter/re2_match.cpp
  49. +2 −2 src/fv_converter/splitter_factory.cpp
  50. +1 −1 src/fv_converter/string_filter_factory.cpp
  51. +2 −2 src/fv_converter/util.cpp
  52. +20 −0 src/graph/graph.hpp
  53. +39 −0 src/graph/graph_base.cpp
  54. +72 −0 src/graph/graph_base.hpp
  55. +58 −0 src/graph/graph_base_test.cpp
  56. +11 −22 src/{fv_converter/dynamic_load.cpp → graph/graph_factory.cpp}
  57. +30 −0 src/graph/graph_factory.hpp
  58. +232 −0 src/graph/graph_type.hpp
  59. +718 −0 src/graph/graph_wo_index.cpp
  60. +106 −0 src/graph/graph_wo_index.hpp
  61. +824 −0 src/graph/graph_wo_index_test.cpp
  62. +27 −0 src/graph/wscript
  63. +77 −13 src/jubavisor/jubavisor.cpp
  64. +7 −1 src/jubavisor/jubavisor.hpp
  65. +5 −2 src/jubavisor/main.cpp
  66. +5 −3 src/jubavisor/process.cpp
  67. +5 −4 src/jubavisor/process.hpp
  68. +1 −1 src/plugin/fv_converter/mecab_splitter.cpp
  69. +5 −5 src/plugin/fv_converter/re2_splitter.cpp
  70. +2 −1 src/plugin/fv_converter/ux_splitter.cpp
  71. +1 −1 src/recommender/lsh.cpp
  72. +142 −0 src/recommender/minhash.cpp
  73. +55 −0 src/recommender/minhash.hpp
  74. +1 −0 src/recommender/recommender.hpp
  75. +3 −1 src/recommender/recommender_factory.cpp
  76. +1 −1 src/recommender/recommender_random_test.cpp
  77. +1 −0 src/recommender/wscript
  78. +1 −1 src/regression/regression_factory.cpp
  79. +4 −1 src/regression/regression_performance_test.cpp
  80. +4 −4 src/server/classifier.idl
  81. +4 −4 src/server/classifier_client.hpp
  82. +1 −1 src/server/classifier_serv.cpp
  83. +6 −5 src/server/classifier_serv.hpp
  84. +8 −8 src/server/classifier_test.cpp
  85. +3 −3 src/server/classifier_types.hpp
  86. +118 −0 src/server/graph.idl
  87. +113 −0 src/server/graph_client.hpp
  88. +95 −0 src/server/graph_impl.cpp
  89. +33 −0 src/server/graph_keeper.cpp
  90. +379 −0 src/server/graph_serv.cpp
  91. +120 −0 src/server/graph_serv.hpp
  92. +55 −0 src/server/graph_server.hpp
  93. +115 −0 src/server/graph_test.cpp
  94. +75 −0 src/server/graph_types.hpp
  95. +11 −11 src/server/recommender.idl
  96. +10 −10 src/server/recommender_client.hpp
  97. +5 −5 src/server/recommender_impl.cpp
  98. +7 −8 src/server/recommender_keeper.cpp
  99. +2 −1 src/server/recommender_serv.cpp
  100. +6 −5 src/server/recommender_serv.hpp
  101. +1 −1 src/server/recommender_test.cpp
  102. +3 −3 src/server/recommender_types.hpp
  103. +4 −4 src/server/regression.idl
  104. +4 −4 src/server/regression_client.hpp
  105. +0 −1 src/server/regression_keeper.cpp
  106. +1 −1 src/server/regression_serv.cpp
  107. +6 −5 src/server/regression_serv.hpp
  108. +2 −2 src/server/regression_test.cpp
  109. +3 −3 src/server/regression_types.hpp
  110. +9 −9 src/server/stat.idl
  111. +18 −18 src/server/stat_client.hpp
  112. +18 −18 src/server/stat_impl.cpp
  113. +7 −8 src/server/stat_keeper.cpp
  114. +18 −9 src/server/stat_serv.cpp
  115. +32 −2 src/server/stat_serv.hpp
  116. +5 −1 src/server/wscript
  117. +78 −0 src/stat/mixable_stat.cpp
  118. +52 −0 src/stat/mixable_stat.hpp
  119. +29 −0 src/stat/mixable_stat_test.cpp
  120. +9 −10 src/stat/stat.cpp
  121. +4 −2 src/stat/stat.hpp
  122. +0 −14 src/stat/stat.idl
  123. +7 −0 src/stat/stat_test.cpp
  124. +2 −2 src/stat/wscript
  125. +1 −1 src/storage/norm_factory.cpp
  126. +2 −1 src/storage/storage_base.hpp
  127. +3 −1 src/storage/storage_factory.cpp
  128. +1 −0 src/storage/storage_type.hpp
  129. +1 −0 src/wscript
  130. +11 −4 tools/generator/OMakefile
  131. +49 −46 tools/generator/generator.ml
  132. +3 −11 tools/generator/jdl_lexer.mll
  133. +15 −9 tools/generator/jdl_parser.mly
  134. +5 −5 tools/generator/jubatus_idl.ml
  135. +3 −4 tools/generator/small.idl
  136. +61 −6 tools/generator/stree.ml
  137. +11 −36 tools/generator/validator.ml
  138. +10 −5 wscript
View
@@ -8,7 +8,6 @@ build/
.waf3-*
.lock-*
.DS_Store
-Makefile
cscope.*
callgrind.*
.unittest-gtest
View
@@ -0,0 +1,13 @@
+.PHONY: all
+
+all:
+ ./waf
+
+clean:
+ ./waf clean
+
+build:
+ ./waf build
+
+check:
+ ./waf check
View
@@ -5,6 +5,24 @@ The Jubatus library is a online machine learning framework which runs in distrib
See http://jubat.us for details.
+QuickStart
+----------
+
+Jubatus publicly supports Ubuntu >= 12.04 , Redhat Linux >= 6.2 . In other platforms, some may work if they have gcc >= 4.4 .
+
+For minimal standalone mode, Jubatus requires libevent >= 1.4 , pficommon >= 1.3.1 , google-glog , libmsgpack, python >= 2.6 . Please install these softwares with headers before installation.
+
+::
+
+ $ git clone git://github.com/jubatus/jubatus.git
+ $ cd jubatus
+ $ ./waf configure --disable-re2
+ $ ./waf build
+ $ sudo ./waf install
+ $ jubaclassifier
+
+Here Jubatus classifier server have started. Enjoy!
+
LICENSE
-------
@@ -13,6 +31,25 @@ LGPL 2.1
Update history
--------------
+Release 0.3.0 2012/6/29
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Improvements
+ - Distributed Graph Processing
+
+ - Centrality computation
+ - Shortest path search
+
+ - New neighbor-search algorithm for recommender: minhash
+ - installs pkgconfig file (#42, jubatus.pc)
+ - jubavisor handling signal (#53)
+ - Exception handling and error outputs (#55)
+
+Bugfix
+ - internal API get_storage implemented again (#21)
+ - Forgot ignoring signal in keeper #47
+ - #54, #45, #15
+
Release 0.2.3 2012/6/8
~~~~~~~~~~~~~~~~~~~~~~~
View
@@ -0,0 +1,3 @@
+#! /bin/sh
+
+./waf configure $*
View
@@ -0,0 +1,11 @@
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: @PACKAGE@
+Description: Framework and Library for Distributed Online Machine Learning
+Version: @VERSION@
+Requires: pficommon >= 1.3.1, libglog >= 0.3.0
+Cflags: -I${includedir}
+Libs: -L${libdir} -ljubatus_framework
@@ -39,7 +39,7 @@ classifier_base* classifier_factory::create_classifier(const std::string& name,
} else if (name == "NHERD"){
return static_cast<classifier_base*>(new NHERD(storage));
} else {
- throw unsupported_method(name);
+ throw JUBATUS_EXCEPTION(unsupported_method(name));
}
}
View
@@ -16,6 +16,7 @@
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include <pficommon/text/json.h>
+#include "../common/exception.hpp"
#include "../common/cmdline.h"
#include "../fv_converter/datum_to_fv_converter.hpp"
#include "../fv_converter/datum.hpp"
@@ -89,7 +90,7 @@ void read_datum(datum& datum) {
}
}
-int main(int argc, char* argv[]) {
+int main(int argc, char* argv[]) try {
cmdline::parser p;
p.add<string>("input-format", 'i', "input format (json/datum)", false, "json",
cmdline::oneof<string>("json", "datum"));
@@ -151,4 +152,6 @@ int main(int argc, char* argv[]) {
}
return -1;
+} catch (const jubatus::exception::jubatus_exception& e) {
+ std::cout << e.diagnostic_information(true) << std::endl;
}
View
@@ -24,6 +24,7 @@
#include <pficommon/network/mprpc.h>
#include <pficommon/lang/function.h>
+#include "../common/exception.hpp"
#include "../common/zk.hpp"
#include "../common/membership.hpp"
#include "../common/cmdline.h"
@@ -40,7 +41,7 @@ void send2supervisor(const string& , const string&, const string&, unsigned int
void send2server(const string& , const string&, const string&);
void status(const string&, const string&);
-int main(int args, char** argv){
+int main(int args, char** argv) try {
cmdline::parser p;
p.add<std::string>("cmd", 'c', "command to send servers(start|stop|save|load)", true);
p.add<std::string>("server", 's', "server exec file of learning machine (jubaclassifier, ...)", true);
@@ -84,7 +85,10 @@ int main(int args, char** argv){
}else if(cmd == "save" or cmd == "load"){ //or set_config?
send2server(cmd, name, zk);
}
- //cout << cmd << name << zk << endl;
+
+ return 0;
+} catch (const jubatus::exception::jubatus_exception& e) {
+ std::cout << e.diagnostic_information(true) << std::endl;
}
int do_request(const string& cmd, const string& name, const string& ip_port, unsigned int n){
View
@@ -3,14 +3,13 @@ def options(opt): pass
def configure(conf): pass
def build(bld):
- if not bld.env.HAVE_ZOOKEEPER_H: return
-
- bld.program(
- source = 'jubactl.cpp',
- target = 'jubactl',
- includes = '.',
- use = 'MSGPACK PFICOMMON LIBGLOG ZOOKEEPER_MT jubacommon'
- )
+ if bld.env.HAVE_ZOOKEEPER_H:
+ bld.program(
+ source = 'jubactl.cpp',
+ target = 'jubactl',
+ includes = '.',
+ use = 'MSGPACK PFICOMMON LIBGLOG ZOOKEEPER_MT jubacommon'
+ )
bld.program(
source = 'jubaconv.cpp',
View
@@ -17,6 +17,7 @@
#include "cht.hpp"
#include "membership.hpp"
+#include "exception.hpp"
#include <stdlib.h>
#include <glog/logging.h>
@@ -51,46 +52,82 @@ namespace common{
}
}
- bool cht::find(const std::string& host, int port, std::vector<std::pair<std::string,int> >& out){
- return find(build_loc_str(host, port), out);
+ bool cht::find(const std::string& host, int port, std::vector<std::pair<std::string,int> >& out, size_t s){
+ return find(build_loc_str(host, port), out, s);
}
+ // return at most n nodes, if theres nodes less than n, return size is also less than n.
// find(hash) :: lock_service -> key -> [node] where hash(node0) <= hash(key) < hash(node1)
- bool cht::find(const std::string& key, std::vector<std::pair<std::string,int> >& out){
+ bool cht::find(const std::string& key, std::vector<std::pair<std::string,int> >& out, size_t n){
out.clear();
- std::string path = ACTOR_BASE_PATH + "/" + name_ + "/cht";
- std::string hash = make_hash(key);
- std::vector<std::pair<std::string, int> > ret;
std::vector<std::string> hlist;
- lock_service_->list(path, hlist);
-
- if(hlist.empty()) return false;
- std::sort(hlist.begin(), hlist.end());
+ if(! get_hashlist_(key, hlist)){
+ throw JUBATUS_EXCEPTION(not_found(key));
+ }
+ std::string hash = make_hash(key);
+ std::string path = ACTOR_BASE_PATH + "/" + name_ + "/cht";
std::vector<std::string>::iterator node0 = std::lower_bound(hlist.begin(), hlist.end(), hash);
size_t idx = int(node0 - hlist.begin()) % hlist.size();
std::string loc;
- for(int i=0; i<2; ++i){
+ for(size_t i=0; i<n; ++i){
std::string ip;
int port;
if(lock_service_->read(path + "/" + hlist[idx], loc)){
revert(loc, ip, port);
out.push_back(make_pair(ip,port));
}else{
// TODO: output log
+ throw JUBATUS_EXCEPTION(not_found(path));
}
idx++;
idx %= hlist.size();
}
return !hlist.size();
}
+ std::pair<std::string,int> cht::find_predecessor(const std::string& host, int port){
+ return find_predecessor(build_loc_str(host, port));
+ }
+ std::pair<std::string,int> cht::find_predecessor(const std::string& key){
+ std::vector<std::string> hlist;
+ get_hashlist_(key, hlist);
+
+ std::string hash = make_hash(key);
+ std::string path = ACTOR_BASE_PATH + "/" + name_ + "/cht";
+
+ std::vector<std::string>::iterator node0 = std::lower_bound(hlist.begin(), hlist.end(), hash);
+ size_t idx = (int(node0 - hlist.begin())+ hlist.size() -1) % hlist.size();
+
+ std::string ip;
+ int port;
+ std::string loc;
+ if(lock_service_->read(path + "/" + hlist[idx], loc)){
+ revert(loc, ip, port);
+ return make_pair(ip, port);
+ }else{
+ throw JUBATUS_EXCEPTION(not_found(path));
+ // TODO: output log and throw exception
+ }
+ }
+
void cht::setup_cht_dir(lock_service& ls, const std::string& name){
std::string path = ACTOR_BASE_PATH + "/" + name;
ls.create(path, "");
path += "/cht";
ls.create(path, "");
}
+ bool cht::get_hashlist_(const std::string& key, std::vector<std::string>& hlist){
+ hlist.clear();
+ std::string path = ACTOR_BASE_PATH + "/" + name_ + "/cht";
+ std::vector<std::pair<std::string, int> > ret;
+ lock_service_->list(path, hlist);
+
+ if(hlist.empty()) return false;
+ std::sort(hlist.begin(), hlist.end());
+ return true;
+ }
+
}
}
View
@@ -25,6 +25,8 @@
#include "shared_ptr.hpp"
#include "lock_service.hpp"
+#include <pficommon/lang/cast.h>
+
namespace jubatus{
namespace common{
@@ -41,15 +43,27 @@ namespace common{
// node :: ip_port
// register_node :: node -> bool;
void register_node(const std::string&, int);
+
+ template <typename T>
+ bool find(const T& t, std::vector<std::pair<std::string,int> > & ret, size_t s){
+ std::string k = pfi::lang::lexical_cast<std::string>(t);
+ return find(k, ret, s);
+ };
// find(hash) :: key -> [node] where hash(node0) <= hash(key) < hash(node1) < hash(node2) < ...
- bool find(const std::string& host, int port, std::vector<std::pair<std::string,int> >&);
- bool find(const std::string&, std::vector<std::pair<std::string,int> >&);
+ bool find(const std::string& host, int port, std::vector<std::pair<std::string,int> >&, size_t);
+ bool find(const std::string&, std::vector<std::pair<std::string,int> >&, size_t);
+
+ std::pair<std::string,int> find_predecessor(const std::string& host, int port);
+ std::pair<std::string,int> find_predecessor(const std::string&);
// run just once in starting up the process: creates <name>/cht directory.
static void setup_cht_dir(lock_service&, const std::string&);
private:
+
+ bool get_hashlist_(const std::string& key, std::vector<std::string>&);
+
std::string name_;
cshared_ptr<lock_service> lock_service_;
}; //cht
View
@@ -0,0 +1,53 @@
+// Jubatus: Online machine learning framework for distributed environment
+// Copyright (C) 2011,2012 Preferred Infrastructure and Nippon Telegraph and Telephone Corporation.
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "exception.hpp"
+#include <cstring>
+
+namespace jubatus {
+namespace exception {
+
+error_info_list_t jubatus_exception::error_info() const
+{
+ return info_list_;
+}
+
+std::string jubatus_exception::diagnostic_information(bool display_what) const
+{
+ std::ostringstream tmp;
+
+ tmp << "Dynamic exception type: ";
+ tmp << detail::demangle_symbol(typeid(*this).name());
+
+ if (display_what && strcmp(what(), ""))
+ tmp << "::what: " << what();
+ tmp << '\n';
+
+ size_t frame = 0;
+ for (error_info_list_t::const_iterator it = info_list_.begin(), end = info_list_.end();
+ it != end; ++it) {
+ if ((*it)->splitter()) {
+ frame++;
+ continue;
+ }
+ tmp << '#' << frame << " [" << (*it)->tag_typeid_name() << "] = " << (*it)->as_string() << '\n';
+ }
+ return tmp.str();
+}
+
+} // exception
+} // jubatus
Oops, something went wrong.

0 comments on commit e2a2397

Please sign in to comment.