Permalink
Browse files

Optimizations. It now takes 1.6 seconds to insert 200000 simple docum…

…ents

W00t!
  • Loading branch information...
Cyberax committed Dec 1, 2011
1 parent 383add8 commit 5c127ecaad0cef3f2c5d5dbe2f90e0bc98aac376
View
@@ -9,7 +9,7 @@ ENABLE_TESTING()
SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
#SET(CMAKE_BUILD_TYPE MinSizeRel)
-#SET(CMAKE_BUILD_TYPE RelWithDebInfo)
+SET(CMAKE_BUILD_TYPE RelWithDebInfo)
IF(NOT CMAKE_BUILD_TYPE)
SET(CMAKE_BUILD_TYPE Debug CACHE STRING
"Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel."
View
@@ -15,6 +15,7 @@ FILE(GLOB libsofadb_INCLUDES
database.h
engine.h
errors.h
+ json_stream.h
native_json.h
native_json_helpers.h
scope_guard.h
View
@@ -1,5 +1,6 @@
#include "database.h"
#include "storage_interface.h"
+#include "json_stream.h"
#include <openssl/md5.h>
#include <time.h>
#include <boost/lexical_cast.hpp>
@@ -187,21 +188,20 @@ revision_num_t Database::store_data(storage_t *ifc,
bool deleted,
const json_value &content)
{
- //No need to call constructors each time!
- const static std::string deleted_text = "deleted";
- const static std::string prev_revid_text = "prev_revid";
- const static std::string atts_text = "atts";
- const static std::string data_text = "data";
-
- json_value serialized(submap_d);
- serialized.insert(deleted_text, json_value(deleted));
- serialized.insert(prev_revid_text, prev_rev_.full_string());
- serialized.insert(atts_text, json_value()); //TODO: attachments
- serialized.insert(data_text, graft_t(&content));
-
- jstring_t body=json_to_string(serialized);
- revision_num_t rev=compute_revision(prev_rev_, body);
+ jstring_t body;
+ body.reserve(128);
+
+ //Format is [deleted, prev_rev, attachments, content]
+ std::auto_ptr<json_stream> str=make_stream(body, false);
+ str->start_list();
+ str->write_bool(deleted);
+ str->write_string(prev_rev_.full_string());
//TODO: attachments
+ str->write_null();
+ str->write_json(content);
+ str->end_list();
+
+ revision_num_t rev=compute_revision(prev_rev_, body);
//Write the document
std::string doc_data_path=doc_data_path_base+rev.full_string();
@@ -275,14 +275,16 @@ bool Database::get(storage_t *ifc,
}
json_value serialized=string_to_json(val);
+ sublist_t &lst = serialized.get_sublist();
+ //Format is [deleted, prev_rev, attachments, content]
if (content)
- *content = std::move(serialized["data"]);
+ *content = std::move(lst.at(3));
if (rev)
{
rev->id_ = id;
- rev->deleted_ = serialized["deleted"].get_bool();
- rev->previous_rev_ = revision_num_t(serialized["prev_revid"].get_str());
+ rev->deleted_ = lst.at(0).get_bool();
+ rev->previous_rev_ = revision_num_t(lst.at(1).get_str());
//serialized["atts"] = json_value(); //TODO: attachments
rev->rev_ = num;
}
View
@@ -0,0 +1,56 @@
+#ifndef JSON_STREAM_H
+#define JSON_STREAM_H
+
+#include "common.h"
+
+namespace sofadb {
+ class json_value;
+
+ class json_stream
+ {
+ public:
+ virtual ~json_stream() {}
+
+ virtual void write_null() = 0;
+ virtual void write_bool(bool val) = 0;
+ virtual void write_int(int64_t i) = 0;
+ virtual void write_double(double d) = 0;
+ virtual void write_string(const char *str, size_t ln) = 0;
+ virtual void write_digits(const char *str, size_t ln) = 0;
+ virtual void start_map() = 0;
+ virtual void end_map() = 0;
+ virtual void start_list() = 0;
+ virtual void end_list() = 0;
+
+ void write_string(const jstring_t &str)
+ {
+ write_string(str.data(), str.length());
+ }
+
+ virtual void write_json(const json_value &val)=0;
+ };
+
+ class json_read_stream
+ {
+ public:
+ virtual ~json_read_stream() {}
+
+ virtual void read_null() = 0;
+ virtual bool read_bool() = 0;
+ virtual int64_t read_int() = 0;
+ virtual double read_double() = 0;
+ virtual jstring_t read_string() = 0;
+ virtual jstring_t read_digits() = 0;
+
+ virtual void expect_map() = 0;
+ virtual void unexpect_map() = 0;
+ virtual void expect_list() = 0;
+ virtual void unexpect_list() = 0;
+ };
+
+ SOFADB_PUBLIC std::auto_ptr<json_stream> make_stream(
+ jstring_t &append_to, bool pretty=false);
+
+}; //namespace sofadb
+
+#endif //JSON_STREAM_H
View
@@ -1,4 +1,5 @@
#include "native_json.h"
+#include "json_stream.h"
#include "errors.h"
#include "native_json_helpers.h"
@@ -249,7 +250,8 @@ struct rapid_read_handler
}
};
-template<class Stream> json_value parse_from_stream(Stream &istr)
+template<class Stream, unsigned Flags>
+ json_value parse_from_stream(Stream &istr)
{
char alloc_buf[8192];
MemoryPoolAllocator<> alloc(alloc_buf, 8192);
@@ -259,7 +261,7 @@ template<class Stream> json_value parse_from_stream(Stream &istr)
rapid_read_handler hndl;
hndl.values_.push_back(&res);
- reader.Parse<0>(istr, hndl);
+ reader.Parse<Flags>(istr, hndl);
if (reader.HasParseError())
{
size_t offset=reader.GetErrorOffset();
@@ -293,19 +295,19 @@ template<class Stream> json_value parse_from_stream(Stream &istr)
json_value sofadb::string_to_json(const jstring_t &str)
{
BufReadStream istr((char*)str.data(), str.size());
- return parse_from_stream(istr);
+ return parse_from_stream<BufReadStream, 0>(istr);
}
json_value sofadb::json_from_stream(std::istream &val)
{
StdStreamReadStream istr(val);
- return parse_from_stream(istr);
+ return parse_from_stream<StdStreamReadStream, kParseIgnoreTrailing>(istr);
}
-struct rapid_json_printer
+template<class Writer> struct rapid_json_printer
{
- Writer<StringWriteStream> &writer_;
- rapid_json_printer(Writer<StringWriteStream> &writer) : writer_(writer) {}
+ Writer &writer_;
+ rapid_json_printer(Writer &writer) : writer_(writer) {}
void operator()()
{
@@ -364,13 +366,105 @@ void sofadb::json_to_string(jstring_t &append_to,
{
PrettyWriter<StringWriteStream> writer(stream, &alloc);
writer.SetIndent(' ', 2);
- rapid_json_printer vis(writer);
+ rapid_json_printer< PrettyWriter<StringWriteStream> > vis(writer);
val.apply_visitor(vis);
} else
{
Writer<StringWriteStream> writer(stream, &alloc);
- rapid_json_printer vis(writer);
+ rapid_json_printer< Writer<StringWriteStream> > vis(writer);
val.apply_visitor(vis);
}
alloc.Clear();
}
+
+template <class Writer> class str_json_stream : public json_stream
+{
+ jstring_t &str_;
+ Writer &writer_;
+public:
+ str_json_stream(jstring_t &str, Writer &writer) :
+ str_(str), writer_(writer)
+ {
+ }
+
+ virtual ~str_json_stream() {}
+
+ virtual void write_null()
+ {
+ writer_.Null();
+ }
+
+ virtual void write_bool(bool val)
+ {
+ writer_.Bool(val);
+ }
+
+ virtual void write_int(int64_t i)
+ {
+ writer_.Int64(i);
+ }
+
+ virtual void write_double(double d)
+ {
+ writer_.Double(d);
+ }
+
+ virtual void write_string(const char *str, size_t ln)
+ {
+ writer_.String(str, ln);
+ }
+
+ virtual void write_digits(const char *str, size_t ln)
+ {
+ writer_.BigInt(str, ln);
+ }
+
+ virtual void start_map()
+ {
+ writer_.StartObject();
+ }
+
+ virtual void end_map()
+ {
+ writer_.EndObject();
+ }
+
+ virtual void start_list()
+ {
+ writer_.StartArray();
+ }
+
+ virtual void end_list()
+ {
+ writer_.EndArray();
+ }
+
+ virtual void write_json(const json_value &val)
+ {
+ rapid_json_printer< Writer > p(writer_);
+ val.apply_visitor(p);
+ }
+};
+
+std::auto_ptr<json_stream> sofadb::make_stream(jstring_t &append_to,
+ bool pretty)
+{
+ char buf[8192];
+ MemoryPoolAllocator<> alloc(buf, 8192);
+ StringWriteStream stream(append_to);
+
+ if (pretty)
+ {
+ PrettyWriter<StringWriteStream> writer(stream, &alloc);
+ writer.SetIndent(' ', 2);
+ json_stream *s=new str_json_stream< PrettyWriter<StringWriteStream> >(
+ append_to, writer);
+ return std::auto_ptr<json_stream>(s);
+ } else
+ {
+ Writer<StringWriteStream> writer(stream, &alloc);
+ json_stream *s=new str_json_stream< Writer<StringWriteStream> >(
+ append_to, writer);
+ return std::auto_ptr<json_stream>(s);
+ }
+}
View
@@ -15,7 +15,7 @@ namespace sofadb {
class json_value;
typedef std::map<jstring_t, json_value> submap_t;
//typedef boost::unordered_map<jstring_t, json_value> submap_t;
- //typedef vector_map<jstring_t, json_value> submap_t;
+ //typedef utils::vector_map<jstring_t, json_value> submap_t;
typedef std::vector<json_value> sublist_t;
SOFADB_PUBLIC std::string int_to_string(int64_t in);
@@ -29,6 +29,7 @@ enum ParseFlag {
kParseDefaultFlags = 0, //!< Default parse flags. Non-destructive parsing. Text strings are decoded into allocated buffer.
kParseInsituFlag = 1, //!< In-situ(destructive) parsing.
kParseValidateEncodingFlag = 2, //!< Validate encoding of JSON strings.
+ kParseIgnoreTrailing = 4, //!< Ignore trailing junk after the root object
};
///////////////////////////////////////////////////////////////////////////////
@@ -230,7 +231,7 @@ class GenericReader {
}
SkipWhitespace(stream);
- if (stream.Peek() != '\0')
+ if (!(parseFlags & kParseIgnoreTrailing) && stream.Peek() != '\0')
RAPIDJSON_PARSE_ERROR("Nothing should follow the root object or array.", stream.Tell());
}
View
@@ -34,15 +34,16 @@ namespace utils {
values_.reserve(sz);
}
- void insert(pair_t && pair)
+ pair_t& insert(pair_t && pair)
{
for(auto iter=begin(), iend=end(); iter!=iend; ++iter)
if (iter->first == pair.first)
{
iter->second = std::move(pair.second);
- return;
+ return (*iter);
}
values_.push_back(std::move(pair));
+ return values_.back();
}
V& operator [] (const K& key)
{
View
@@ -140,7 +140,7 @@ BOOST_AUTO_TEST_CASE(test_bench)
revision_num_t old;
storage_ptr_t stg=engine.create_storage(false);
- for(int f=0;f<2000; ++f)
+ for(int f=0;f<200000; ++f)
{
revision_t rev=ptr->put(stg.get(),
id+int_to_string(f),

0 comments on commit 5c127ec

Please sign in to comment.