Merge branch 'release/1.0.3'

CONTRIBUTING.md

@@ -1,12 +1,10 @@
-How to contribute
-======================
+How to Contribute
+=================
 
-Please check our Contibution guideline first.
-
-In English : http://jubat.us/en/howtocontribute.html
-
-In Japanese: http://jubat.us/ja/howtocontribute.html
+Please check our Contibution Guideline first.
 
+* English : http://jubat.us/en/developers/howtocontribute.html
+* Japanese: http://jubat.us/ja/developers/howtocontribute.html
 
 Copyright
 ---------

ChangeLog.rst

@@ -1,3 +1,17 @@
+Release 1.0.3 - 2017/04/24
+--------------------------
+
+* Changes to Supported Systems
+    * Deprecate Ubuntu 12.04 LTS support (#1189)
+
+* Improvements
+    * Validate save/load RPC arguments (#1188, #1192)
+    * Model files generated in earlier versions of Jubatus can now be loaded (#1179, #1187)
+    * Improve packaging process (#1185, #1186)
+
+* Bug fixes
+    * classifier/regression: fix performance regression in standalone mode (#1190, #1193)
+
 Release 1.0.2 - 2017/02/27
 --------------------------
 

README.rst

@@ -1,14 +1,17 @@
 Jubatus
 =======
 
+.. image:: https://api.travis-ci.org/jubatus/jubatus.svg?branch=master
+    :target: https://api.travis-ci.org/jubatus/jubatus
+
 The Jubatus library is an online machine learning framework which runs in distributed environment.
 
 See http://jubat.us/ for details.
 
 Quick Start
 -----------
 
-We officially support Red Hat Enterprise Linux (RHEL) 6.2 or later (64-bit) and Ubuntu Server 12.04 LTS / 14.04 LTS / 16.04 LTS (64-bit).
+We officially support Red Hat Enterprise Linux (RHEL) 6.2 or later (64-bit) and Ubuntu Server 14.04 LTS / 16.04 LTS (64-bit).
 On supported systems, you can install all components of Jubatus using binary packages.
 
 See `QuickStart <http://jubat.us/en/quickstart.html>`_ for detailed description.
@@ -45,7 +48,7 @@ Write the following line to ``/etc/apt/sources.list.d/jubatus.list`` to register
 
 ::
 
-  // For Ubuntu 12.04 (Precise)
+  // For Ubuntu 12.04 (Precise) - Deprecated (unsupported)
   deb http://download.jubat.us/apt/ubuntu/precise binary/
 
   // For Ubuntu 14.04 (Trusty)

jubatus/server/framework/save_load.cpp

@@ -183,16 +183,13 @@ void load_server(
   uint32_t jubatus_minor_read = read_big_endian<uint32_t>(&header_buf[20]);
   uint32_t jubatus_maintenance_read =
       read_big_endian<uint32_t>(&header_buf[24]);
-  if (jubatus_major_read != jubatus_version_major ||
-      jubatus_minor_read != jubatus_version_minor ||
-      jubatus_maintenance_read != jubatus_version_maintenance) {
+  if (jubatus_major_read != 1) {
     throw JUBATUS_EXCEPTION(
         core::common::exception::runtime_error(
-          "jubatus version mismatched: " +
+          "cannot load model file created before v1.0.0: " +
           lexical_cast<std::string>(jubatus_major_read) + "." +
           lexical_cast<std::string>(jubatus_minor_read) + "." +
-          lexical_cast<std::string>(jubatus_maintenance_read) +
-          ", expected (current) version: " JUBATUS_VERSION));
+          lexical_cast<std::string>(jubatus_maintenance_read)));
   }
   uint32_t crc32_expected = read_big_endian<uint32_t>(&header_buf[28]);
   uint64_t system_data_size = read_big_endian<uint64_t>(&header_buf[32]);

jubatus/server/framework/server_base.cpp

@@ -136,6 +136,9 @@ std::map<std::string, std::string> server_base::save(const std::string& id) {
   if (id == "") {
     throw JUBATUS_EXCEPTION(
         core::common::exception::runtime_error("empty id is not allowed"));
+  } else if (!validate_model_id(id)) {
+    throw JUBATUS_EXCEPTION(core::common::exception::runtime_error(
+        "model ID contains invalid character"));
   }
 
   const std::string path = build_local_path(argv_, argv_.type, id);
@@ -197,6 +200,9 @@ bool server_base::load(const std::string& id) {
   if (id == "") {
     throw JUBATUS_EXCEPTION(
         core::common::exception::runtime_error("empty id is not allowed"));
+  } else if (!validate_model_id(id)) {
+    throw JUBATUS_EXCEPTION(core::common::exception::runtime_error(
+        "model ID contains invalid character"));
   }
 
   load_file_impl(*this, build_local_path(argv_, argv_.type, id), id, false);
@@ -230,6 +236,15 @@ void server_base::update_loaded_status(const std::string& path) {
   last_loaded_path_ = path;
 }
 
+bool validate_model_id(const std::string& id) {
+    for (size_t i = 0; i < id.length(); ++i) {
+        if (id[i] == '/' || id[i] < ' ' || '~' < id[i]) {
+            return false;
+        }
+    }
+    return true;
+}
+
 }  // namespace framework
 }  // namespace server
 }  // namespace jubatus

jubatus/server/framework/server_base.hpp

@@ -108,6 +108,8 @@ class server_base {
   mutable jubatus::util::concurrent::rw_mutex status_mutex_;
 };
 
+bool validate_model_id(const std::string& id);
+
 }  // namespace framework
 }  // namespace server
 }  // namespace jubatus

jubatus/server/framework/server_base_test.cpp

@@ -0,0 +1,41 @@
+// Jubatus: Online machine learning framework for distributed environment
+// Copyright (C) 2017 Preferred Networks and Nippon Telegraph and Telephone Corporation.
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License version 2.1 as published by the Free Software Foundation.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+#include <string>
+#include <gtest/gtest.h>
+
+#include "server_base.hpp"
+
+namespace jubatus {
+namespace server {
+namespace framework {
+
+TEST(validate_model_id, simple) {
+    EXPECT_TRUE(validate_model_id("valid_model_id"));
+    EXPECT_TRUE(validate_model_id(
+        "valid_chars !\"#$%&'()*+,-."
+        "0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]"
+        "^_`abcdefghijklmnopqrstuvwxyz{|}~"));
+    EXPECT_FALSE(validate_model_id("invalid/../model/id/"));
+
+    std::string nullstr("null");
+    nullstr.push_back(0);
+    EXPECT_FALSE(validate_model_id(nullstr));
+}
+
+}  // namespace framework
+}  // namespace server
+}  // namespace jubatus

jubatus/server/framework/wscript

@@ -22,15 +22,22 @@ def build(bld):
     vnum = bld.env['ABI_VERSION'],
     )
 
+  test_source = [
+    'server_base_test.cpp',
+  ]
+
   def make_test(t):
     bld.program(
       features='gtest',
-      source=t + '.cpp',
-      target=t,
+      source=t,
+      target = s[0:s.rfind('.')],
       includes='.',
       use='jubaserv_framework'
       )
 
+  for s in test_source:
+    make_test(s)
+
   header_files = [
     'save_load.hpp',
     'server_base.hpp',

jubatus/server/server/classifier_serv.cpp

@@ -66,7 +66,8 @@ struct classifier_serv_config {
 
 shared_ptr<core::storage::storage_base> make_model(
     const framework::server_argv& arg) {
-  return core::storage::storage_factory::create_storage("local_mixture");
+  return core::storage::storage_factory::create_storage(
+      (arg.is_standalone()) ? "local" : "local_mixture");
 }
 
 }  // namespace

jubatus/server/server/regression_serv.cpp

@@ -61,7 +61,8 @@ struct regression_serv_config {
 
 shared_ptr<core::storage::storage_base> make_model(
     const framework::server_argv& arg) {
-  return core::storage::storage_factory::create_storage("local_mixture");
+  return core::storage::storage_factory::create_storage(
+      (arg.is_standalone()) ? "local" : "local_mixture");
 }
 
 }  // namespace

tools/packaging/allinone/.dockerignore

@@ -0,0 +1 @@
+*

tools/packaging/rpm/.dockerignore

@@ -0,0 +1 @@
+*

tools/packaging/rpm/README.rst

@@ -1,6 +1,33 @@
 RPM Packaging
 =============
 
+Building RPM Packages with Docker
+---------------------------------
+
+You can build RPM packages in Docker.
+
+First create Docker images for building package:
+
+::
+
+  docker build -t jubatus-rpmbuild-centos6 -f docker/Dockerfile.centos6 .
+  docker build -t jubatus-rpmbuild-centos7 -f docker/Dockerfile.centos7 .
+
+Modify ``package-config`` as appropriate (see below).
+
+Then build packages:
+
+::
+
+  docker run --rm -v $PWD:/pkg jubatus-rpmbuild-centos6 sh -c "cd pkg && chown root: -R rpmbuild && ./package.sh -cai"
+  docker run --rm -v $PWD:/pkg jubatus-rpmbuild-centos7 sh -c "cd pkg && chown root: -R rpmbuild && ./package.sh -cai"
+
+Do not run these commands simultaneously!
+These two ``docker run`` commands share the current directory.
+
+Building RPM Packages without Docker
+------------------------------------
+
 1. On RHEL 6 or 7, install the following packages.
 
 ::

tools/packaging/rpm/docker/Dockerfile.centos6

@@ -0,0 +1,5 @@
+FROM centos:6
+
+RUN yum -y update && \
+    yum -y groupinstall "Development Tools" && \
+    yum -y install sudo git-core ruby rpm-build rpmdevtools spectool createrepo apr-devel apr-util-devel opencv-devel

tools/packaging/rpm/docker/Dockerfile.centos7

@@ -0,0 +1,5 @@
+FROM centos:7
+
+RUN yum -y update && \
+    yum -y groupinstall "Development Tools" && \
+    yum -y install sudo wget git-core ruby rpm-build rpmdevtools spectool createrepo apr-devel apr-util-devel opencv-devel

wscript

@@ -4,7 +4,7 @@ from waflib.Errors import TaskNotReady
 import os
 import sys
 
-VERSION = '1.0.2'
+VERSION = '1.0.3'
 ABI_VERSION = VERSION
 APPNAME = 'jubatus'