From da6606406637bead267eb2dbfd2e8c712a045735 Mon Sep 17 00:00:00 2001 From: John Langford Date: Thu, 29 Dec 2016 14:20:13 -0500 Subject: [PATCH] consistent indentation with astyle (before releasing 8.3.1) --- cs/cli/clr_io.cpp | 130 +- cs/cli/clr_io.h | 46 +- cs/cli/clr_io_memory.cpp | 96 +- cs/cli/clr_io_memory.h | 42 +- cs/cli/resource.h | 2 +- cs/cli/spanning_tree_clr.cpp | 63 +- cs/cli/spanning_tree_clr.h | 62 +- cs/cli/vld_clr.cpp | 77 +- cs/cli/vld_clr.h | 35 +- cs/cli/vowpalwabbit.cpp | 1188 +++-- cs/cli/vowpalwabbit.h | 448 +- cs/cli/vw_arguments.h | 212 +- cs/cli/vw_base.cpp | 395 +- cs/cli/vw_base.h | 291 +- cs/cli/vw_builder.cpp | 190 +- cs/cli/vw_builder.h | 234 +- cs/cli/vw_cbutil.cpp | 9 +- cs/cli/vw_cbutil.h | 10 +- cs/cli/vw_example.cpp | 777 ++- cs/cli/vw_example.h | 489 +- cs/cli/vw_exception.cpp | 22 +- cs/cli/vw_interface.h | 40 +- cs/cli/vw_label.h | 604 ++- cs/cli/vw_labelcomparator.h | 104 +- cs/cli/vw_model.cpp | 49 +- cs/cli/vw_model.h | 28 +- cs/cli/vw_prediction.cpp | 306 +- cs/cli/vw_prediction.h | 776 ++- cs/cli/vw_settings.h | 256 +- java/src/main/c++/jni_base_learner.h | 4 +- java/src/main/c++/vowpalWabbit_VW.h | 5 +- ...owpalWabbit_learner_VWActionProbsLearner.h | 7 +- ...wpalWabbit_learner_VWActionScoresLearner.h | 7 +- .../c++/vowpalWabbit_learner_VWLearners.h | 9 +- ...owpalWabbit_learner_VWMulticlassLearner.cc | 2 +- ...vowpalWabbit_learner_VWMulticlassLearner.h | 7 +- ...owpalWabbit_learner_VWMultilabelsLearner.h | 7 +- .../c++/vowpalWabbit_learner_VWProbLearner.h | 7 +- .../vowpalWabbit_learner_VWScalarLearner.h | 7 +- .../vowpalWabbit_learner_VWScalarsLearner.h | 7 +- library/gd_mf_weights.cc | 20 +- library/search_generate.cc | 22 +- python/pylibvw.cc | 60 +- rapidjson/example/capitalize/capitalize.cpp | 84 +- rapidjson/example/condense/condense.cpp | 32 +- rapidjson/example/filterkey/filterkey.cpp | 211 +- .../example/filterkeydom/filterkeydom.cpp | 262 +- rapidjson/example/jsonx/jsonx.cpp | 366 +- .../example/messagereader/messagereader.cpp | 104 +- .../example/parsebyparts/parsebyparts.cpp | 241 +- rapidjson/example/pretty/pretty.cpp | 30 +- rapidjson/example/prettyauto/prettyauto.cpp | 57 +- .../schemavalidator/schemavalidator.cpp | 105 +- rapidjson/example/serialize/serialize.cpp | 218 +- rapidjson/example/simpledom/simpledom.cpp | 30 +- .../example/simplereader/simplereader.cpp | 62 +- .../example/simplewriter/simplewriter.cpp | 52 +- rapidjson/example/tutorial/tutorial.cpp | 261 +- rapidjson/include/rapidjson/allocators.h | 361 +- rapidjson/include/rapidjson/document.h | 4197 +++++++++-------- rapidjson/include/rapidjson/encodedstream.h | 373 +- rapidjson/include/rapidjson/encodings.h | 1033 ++-- rapidjson/include/rapidjson/error/en.h | 42 +- rapidjson/include/rapidjson/error/error.h | 99 +- rapidjson/include/rapidjson/filereadstream.h | 103 +- rapidjson/include/rapidjson/filewritestream.h | 113 +- rapidjson/include/rapidjson/fwd.h | 18 +- .../include/rapidjson/internal/biginteger.h | 460 +- rapidjson/include/rapidjson/internal/diyfp.h | 360 +- rapidjson/include/rapidjson/internal/dtoa.h | 379 +- .../include/rapidjson/internal/ieee754.h | 92 +- rapidjson/include/rapidjson/internal/itoa.h | 531 +-- rapidjson/include/rapidjson/internal/meta.h | 51 +- rapidjson/include/rapidjson/internal/pow10.h | 55 +- rapidjson/include/rapidjson/internal/regex.h | 1063 +++-- rapidjson/include/rapidjson/internal/stack.h | 360 +- .../include/rapidjson/internal/strfunc.h | 45 +- rapidjson/include/rapidjson/internal/strtod.h | 451 +- rapidjson/include/rapidjson/internal/swap.h | 11 +- rapidjson/include/rapidjson/istreamwrapper.h | 107 +- rapidjson/include/rapidjson/memorybuffer.h | 44 +- rapidjson/include/rapidjson/memorystream.h | 44 +- .../include/rapidjson/msinttypes/inttypes.h | 47 +- .../include/rapidjson/msinttypes/stdint.h | 57 +- rapidjson/include/rapidjson/ostreamwrapper.h | 47 +- rapidjson/include/rapidjson/pointer.h | 2000 ++++---- rapidjson/include/rapidjson/prettywriter.h | 369 +- rapidjson/include/rapidjson/rapidjson.h | 32 +- rapidjson/include/rapidjson/reader.h | 2884 ++++++----- rapidjson/include/rapidjson/schema.h | 3157 ++++++------- rapidjson/include/rapidjson/stream.h | 106 +- rapidjson/include/rapidjson/stringbuffer.h | 91 +- rapidjson/include/rapidjson/writer.h | 920 ++-- rapidjson/test/perftest/misctest.cpp | 1452 +++--- rapidjson/test/perftest/perftest.cpp | 19 +- rapidjson/test/perftest/perftest.h | 213 +- rapidjson/test/perftest/platformtest.cpp | 201 +- rapidjson/test/perftest/rapidjsontest.cpp | 599 +-- rapidjson/test/perftest/schematest.cpp | 367 +- rapidjson/test/unittest/allocatorstest.cpp | 131 +- rapidjson/test/unittest/bigintegertest.cpp | 188 +- rapidjson/test/unittest/documenttest.cpp | 1000 ++-- rapidjson/test/unittest/dtoatest.cpp | 112 +- rapidjson/test/unittest/encodedstreamtest.cpp | 496 +- rapidjson/test/unittest/encodingstest.cpp | 752 ++- rapidjson/test/unittest/filestreamtest.cpp | 149 +- rapidjson/test/unittest/fwdtest.cpp | 276 +- .../test/unittest/istreamwrappertest.cpp | 237 +- rapidjson/test/unittest/itoatest.cpp | 200 +- rapidjson/test/unittest/jsoncheckertest.cpp | 148 +- rapidjson/test/unittest/namespacetest.cpp | 68 +- .../test/unittest/ostreamwrappertest.cpp | 95 +- rapidjson/test/unittest/pointertest.cpp | 2542 +++++----- rapidjson/test/unittest/prettywritertest.cpp | 303 +- rapidjson/test/unittest/readertest.cpp | 2538 +++++----- rapidjson/test/unittest/regextest.cpp | 1105 ++--- rapidjson/test/unittest/schematest.cpp | 2151 +++++---- rapidjson/test/unittest/simdtest.cpp | 296 +- rapidjson/test/unittest/strfunctest.cpp | 26 +- rapidjson/test/unittest/stringbuffertest.cpp | 196 +- rapidjson/test/unittest/strtodtest.cpp | 208 +- rapidjson/test/unittest/unittest.cpp | 32 +- rapidjson/test/unittest/unittest.h | 81 +- rapidjson/test/unittest/valuetest.cpp | 3061 ++++++------ rapidjson/test/unittest/writertest.cpp | 696 ++- vowpalwabbit/OjaNewton.cc | 941 ++-- vowpalwabbit/accumulate.cc | 52 +- vowpalwabbit/action_score.cc | 5 +- vowpalwabbit/action_score.h | 54 +- vowpalwabbit/active.cc | 7 +- vowpalwabbit/allreduce.h | 2 +- vowpalwabbit/array_parameters.h | 336 +- vowpalwabbit/audit_regressor.cc | 353 +- vowpalwabbit/autolink.cc | 5 +- vowpalwabbit/bfgs.cc | 170 +- vowpalwabbit/bs.cc | 2 +- vowpalwabbit/cache.cc | 40 +- vowpalwabbit/cb.cc | 14 +- vowpalwabbit/cb.h | 2 +- vowpalwabbit/cb_adf.cc | 83 +- vowpalwabbit/cb_adf.h | 7 +- vowpalwabbit/cb_algs.cc | 13 +- vowpalwabbit/cb_algs.h | 10 +- vowpalwabbit/cb_explore.cc | 548 ++- vowpalwabbit/cb_explore.h | 7 +- vowpalwabbit/cb_explore_adf.cc | 857 ++-- vowpalwabbit/cb_explore_adf.h | 5 +- vowpalwabbit/cbify.cc | 17 +- vowpalwabbit/confidence.cc | 24 +- vowpalwabbit/cost_sensitive.cc | 26 +- vowpalwabbit/csoaa.cc | 122 +- vowpalwabbit/ect.cc | 2 +- vowpalwabbit/example.cc | 21 +- vowpalwabbit/example.h | 21 +- vowpalwabbit/explore_eval.cc | 376 +- vowpalwabbit/expreplay.h | 2 +- vowpalwabbit/ezexample.h | 12 +- vowpalwabbit/feature_group.h | 92 +- vowpalwabbit/ftrl.cc | 27 +- vowpalwabbit/gd.cc | 181 +- vowpalwabbit/gd.h | 22 +- vowpalwabbit/gd_mf.cc | 45 +- vowpalwabbit/gen_cs_example.cc | 169 +- vowpalwabbit/gen_cs_example.h | 278 +- vowpalwabbit/global_data.h | 26 +- vowpalwabbit/interact.cc | 24 +- vowpalwabbit/interactions.cc | 2 +- vowpalwabbit/interactions.h | 425 +- vowpalwabbit/io_buf.cc | 2 +- vowpalwabbit/io_buf.h | 6 +- vowpalwabbit/kernel_svm.cc | 77 +- vowpalwabbit/label_dictionary.cc | 54 +- vowpalwabbit/lda_core.cc | 679 ++- vowpalwabbit/learner.cc | 37 +- vowpalwabbit/learner.h | 44 +- vowpalwabbit/log_multi.cc | 26 +- vowpalwabbit/loss_functions.cc | 36 +- vowpalwabbit/lrq.cc | 97 +- vowpalwabbit/lrqfa.cc | 75 +- vowpalwabbit/marginal.cc | 197 +- vowpalwabbit/memory.h | 7 +- vowpalwabbit/mf.cc | 12 +- vowpalwabbit/multiclass.cc | 89 +- vowpalwabbit/multilabel.cc | 22 +- vowpalwabbit/mwt.cc | 366 +- vowpalwabbit/mwt.h | 3 +- vowpalwabbit/nn.cc | 14 +- vowpalwabbit/oaa.cc | 71 +- vowpalwabbit/parse_args.cc | 25 +- vowpalwabbit/parse_example.cc | 156 +- vowpalwabbit/parse_example.h | 5 +- vowpalwabbit/parse_example_json.cc | 827 ++-- vowpalwabbit/parse_example_json.h | 260 +- vowpalwabbit/parse_primitives.h | 3 +- vowpalwabbit/parse_regressor.cc | 220 +- vowpalwabbit/parser.cc | 146 +- vowpalwabbit/rand48.cc | 6 +- vowpalwabbit/recall_tree.cc | 408 +- vowpalwabbit/scorer.cc | 3 +- vowpalwabbit/search.cc | 59 +- vowpalwabbit/search.h | 6 +- vowpalwabbit/search_dep_parser.cc | 95 +- vowpalwabbit/search_dep_parser.h | 4 +- vowpalwabbit/search_entityrelationtask.h | 2 +- vowpalwabbit/search_graph.cc | 6 +- vowpalwabbit/search_graph.h | 6 +- vowpalwabbit/search_hooktask.h | 6 +- vowpalwabbit/search_multiclasstask.h | 2 +- vowpalwabbit/search_sequencetask.cc | 2 +- vowpalwabbit/search_sequencetask.h | 14 +- vowpalwabbit/simple_label.cc | 6 +- vowpalwabbit/stagewise_poly.cc | 51 +- vowpalwabbit/unique_sort.cc | 6 +- vowpalwabbit/v_array.h | 18 +- vowpalwabbit/vw.h | 29 +- vowpalwabbit/vw_exception.cc | 20 +- vowpalwabbit/vw_exception.h | 10 +- vowpalwabbit/vw_validate.cc | 13 +- vowpalwabbit/vwdll.cpp | 395 +- vowpalwabbit/vwdll.h | 104 +- 220 files changed, 29331 insertions(+), 30230 deletions(-) mode change 100755 => 100644 vowpalwabbit/cb_explore.h mode change 100755 => 100644 vowpalwabbit/gen_cs_example.cc mode change 100755 => 100644 vowpalwabbit/gen_cs_example.h diff --git a/cs/cli/clr_io.cpp b/cs/cli/clr_io.cpp index 43a1403c88e..f7d541eab64 100644 --- a/cs/cli/clr_io.cpp +++ b/cs/cli/clr_io.cpp @@ -12,74 +12,64 @@ using namespace System::Runtime::InteropServices; namespace VW { - clr_io_buf::clr_io_buf(Stream^ stream) : m_stream(stream), m_buffer(nullptr) - { - if (stream == nullptr) - throw gcnew ArgumentNullException("stream"); - - files.push_back(0); - } - - void clr_io_buf::ensure_buffer_size(size_t nbytes) - { - if (m_buffer != nullptr && m_buffer->Length >= nbytes) - return; - - m_buffer = gcnew array((int)nbytes); - } - - int clr_io_buf::open_file(const char* name, bool stdin_off, int flag) - { - return 0; - } - - void clr_io_buf::reset_file(int f) - { - m_stream->Seek(0, SeekOrigin::Begin); - head = space.begin(); - - space.end() = space.begin(); - } - - ssize_t clr_io_buf::read_file(int f, void* buf, size_t nbytes) - { - ensure_buffer_size(nbytes); - - int readBytes = m_stream->Read(m_buffer, 0, (int)nbytes); - Marshal::Copy(m_buffer, 0, IntPtr(buf), (int)nbytes); - - return readBytes; - } - - size_t clr_io_buf::num_files() - { - return 1; - } - - ssize_t clr_io_buf::write_file(int file, const void* buf, size_t nbytes) - { - ensure_buffer_size(nbytes); - - Marshal::Copy(IntPtr((void*)buf), m_buffer, 0, (int)nbytes); - m_stream->Write(m_buffer, 0, (int)nbytes); - - return nbytes; - } - - bool clr_io_buf::compressed() - { - return false; - } - - void clr_io_buf::flush() - { - io_buf::flush(); - m_stream->Flush(); - } - - bool clr_io_buf::close_file() - { - // don't close stream on purpose. Caller of SaveModel should have control when to close. - return true; - } +clr_io_buf::clr_io_buf(Stream^ stream) : m_stream(stream), m_buffer(nullptr) +{ if (stream == nullptr) + throw gcnew ArgumentNullException("stream"); + + files.push_back(0); +} + +void clr_io_buf::ensure_buffer_size(size_t nbytes) +{ if (m_buffer != nullptr && m_buffer->Length >= nbytes) + return; + + m_buffer = gcnew array((int)nbytes); +} + +int clr_io_buf::open_file(const char* name, bool stdin_off, int flag) +{ return 0; +} + +void clr_io_buf::reset_file(int f) +{ m_stream->Seek(0, SeekOrigin::Begin); + head = space.begin(); + + space.end() = space.begin(); +} + +ssize_t clr_io_buf::read_file(int f, void* buf, size_t nbytes) +{ ensure_buffer_size(nbytes); + + int readBytes = m_stream->Read(m_buffer, 0, (int)nbytes); + Marshal::Copy(m_buffer, 0, IntPtr(buf), (int)nbytes); + + return readBytes; +} + +size_t clr_io_buf::num_files() +{ return 1; +} + +ssize_t clr_io_buf::write_file(int file, const void* buf, size_t nbytes) +{ ensure_buffer_size(nbytes); + + Marshal::Copy(IntPtr((void*)buf), m_buffer, 0, (int)nbytes); + m_stream->Write(m_buffer, 0, (int)nbytes); + + return nbytes; +} + +bool clr_io_buf::compressed() +{ return false; +} + +void clr_io_buf::flush() +{ io_buf::flush(); + m_stream->Flush(); +} + +bool clr_io_buf::close_file() +{ // don't close stream on purpose. Caller of SaveModel should have control when to close. + return true; +} } diff --git a/cs/cli/clr_io.h b/cs/cli/clr_io.h index 05fe9292983..586a3404107 100644 --- a/cs/cli/clr_io.h +++ b/cs/cli/clr_io.h @@ -11,37 +11,37 @@ using namespace System::IO; namespace VW { - /// - /// C++ wrapper for managed . - /// - class clr_io_buf : public io_buf - { - private: - gcroot m_stream; - gcroot^> m_buffer; +/// +/// C++ wrapper for managed . +/// +class clr_io_buf : public io_buf +{ +private: + gcroot m_stream; + gcroot^> m_buffer; - void ensure_buffer_size(size_t nbytes); + void ensure_buffer_size(size_t nbytes); - public: - /// - /// Initializes a new instance. - /// - clr_io_buf(Stream^ stream); +public: + /// + /// Initializes a new instance. + /// + clr_io_buf(Stream^ stream); - virtual int open_file(const char* name, bool stdin_off, int flag = READ); + virtual int open_file(const char* name, bool stdin_off, int flag = READ); - virtual void reset_file(int f); + virtual void reset_file(int f); - virtual ssize_t read_file(int f, void* buf, size_t nbytes); + virtual ssize_t read_file(int f, void* buf, size_t nbytes); - virtual size_t num_files(); + virtual size_t num_files(); - virtual ssize_t write_file(int file, const void* buf, size_t nbytes); + virtual ssize_t write_file(int file, const void* buf, size_t nbytes); - virtual bool compressed(); + virtual bool compressed(); - virtual void flush(); + virtual void flush(); - virtual bool close_file(); - }; + virtual bool close_file(); +}; } diff --git a/cs/cli/clr_io_memory.cpp b/cs/cli/clr_io_memory.cpp index b21f568494c..d617dee1f93 100644 --- a/cs/cli/clr_io_memory.cpp +++ b/cs/cli/clr_io_memory.cpp @@ -8,56 +8,48 @@ license as described in the file LICENSE. namespace VW { - clr_io_memory_buf::clr_io_memory_buf() - { - files.push_back(0); - m_iterator = m_data.begin(); - } - - int clr_io_memory_buf::open_file(const char* name, bool stdin_off, int flag) - { - m_iterator = m_data.begin(); - return 0; - } - - void clr_io_memory_buf::reset_file(int f) - { - size_t count = m_data.size(); - m_iterator = m_data.begin(); - } - - ssize_t clr_io_memory_buf::read_file(int f, void* buf, size_t nbytes) - { - size_t left_over = min(nbytes, m_data.end() - m_iterator); - - if (left_over == 0) - return 0; - - memcpy_s(buf, nbytes, &*m_iterator, left_over); - - m_iterator += left_over; - - return left_over; - } - - size_t clr_io_memory_buf::num_files() - { - return 1; - } - - ssize_t clr_io_memory_buf::write_file(int file, const void* buf, size_t nbytes) - { - m_data.insert(m_data.end(), (char*)buf, (char*)buf + nbytes); - return nbytes; - } - - bool clr_io_memory_buf::compressed() - { - return false; - } - - bool clr_io_memory_buf::close_file() - { - return true; - } +clr_io_memory_buf::clr_io_memory_buf() +{ files.push_back(0); + m_iterator = m_data.begin(); +} + +int clr_io_memory_buf::open_file(const char* name, bool stdin_off, int flag) +{ m_iterator = m_data.begin(); + return 0; +} + +void clr_io_memory_buf::reset_file(int f) +{ size_t count = m_data.size(); + m_iterator = m_data.begin(); +} + +ssize_t clr_io_memory_buf::read_file(int f, void* buf, size_t nbytes) +{ size_t left_over = min(nbytes, m_data.end() - m_iterator); + + if (left_over == 0) + return 0; + + memcpy_s(buf, nbytes, &*m_iterator, left_over); + + m_iterator += left_over; + + return left_over; +} + +size_t clr_io_memory_buf::num_files() +{ return 1; +} + +ssize_t clr_io_memory_buf::write_file(int file, const void* buf, size_t nbytes) +{ m_data.insert(m_data.end(), (char*)buf, (char*)buf + nbytes); + return nbytes; +} + +bool clr_io_memory_buf::compressed() +{ return false; +} + +bool clr_io_memory_buf::close_file() +{ return true; +} } diff --git a/cs/cli/clr_io_memory.h b/cs/cli/clr_io_memory.h index 1aedd430aa8..da80710401d 100644 --- a/cs/cli/clr_io_memory.h +++ b/cs/cli/clr_io_memory.h @@ -10,34 +10,34 @@ license as described in the file LICENSE. namespace VW { - /// - /// IO Buffer keeping data in memory. Used by VowpalWabbit::Reload. - /// - class clr_io_memory_buf : public io_buf - { - private: - std::vector m_data; +/// +/// IO Buffer keeping data in memory. Used by VowpalWabbit::Reload. +/// +class clr_io_memory_buf : public io_buf +{ +private: + std::vector m_data; - std::vector::const_iterator m_iterator; + std::vector::const_iterator m_iterator; - public: - /// - /// Initializes a new instance. - /// - clr_io_memory_buf(); +public: + /// + /// Initializes a new instance. + /// + clr_io_memory_buf(); - virtual int open_file(const char* name, bool stdin_off, int flag = READ); + virtual int open_file(const char* name, bool stdin_off, int flag = READ); - virtual void reset_file(int f); + virtual void reset_file(int f); - virtual ssize_t read_file(int f, void* buf, size_t nbytes); + virtual ssize_t read_file(int f, void* buf, size_t nbytes); - virtual size_t num_files(); + virtual size_t num_files(); - virtual ssize_t write_file(int file, const void* buf, size_t nbytes); + virtual ssize_t write_file(int file, const void* buf, size_t nbytes); - virtual bool compressed(); + virtual bool compressed(); - virtual bool close_file(); - }; + virtual bool close_file(); +}; } diff --git a/cs/cli/resource.h b/cs/cli/resource.h index 7ca31da7405..d42b43ecc2b 100644 --- a/cs/cli/resource.h +++ b/cs/cli/resource.h @@ -3,7 +3,7 @@ // Used by Resource.rc // Next default values for new objects -// +// #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS #define _APS_NEXT_RESOURCE_VALUE 101 diff --git a/cs/cli/spanning_tree_clr.cpp b/cs/cli/spanning_tree_clr.cpp index 801b7a85e1c..fe9d9ac1c2c 100644 --- a/cs/cli/spanning_tree_clr.cpp +++ b/cs/cli/spanning_tree_clr.cpp @@ -11,44 +11,35 @@ using namespace std; namespace VW { - SpanningTreeClr::SpanningTreeClr() - { - m_spanningTree = new SpanningTree; - } +SpanningTreeClr::SpanningTreeClr() +{ m_spanningTree = new SpanningTree; +} - SpanningTreeClr::~SpanningTreeClr() - { - try - { - delete m_spanningTree; - } - CATCHRETHROW - } +SpanningTreeClr::~SpanningTreeClr() +{ try + { delete m_spanningTree; + } + CATCHRETHROW +} - void SpanningTreeClr::Start() - { - try - { - m_spanningTree->Start(); - } - CATCHRETHROW - } +void SpanningTreeClr::Start() +{ try + { m_spanningTree->Start(); + } + CATCHRETHROW +} - void SpanningTreeClr::Stop() - { - try - { - m_spanningTree->Stop(); - } - CATCHRETHROW - } +void SpanningTreeClr::Stop() +{ try + { m_spanningTree->Stop(); + } + CATCHRETHROW +} - void SpanningTreeClr::Run() - { - try - { - m_spanningTree->Run(); - } - CATCHRETHROW - } +void SpanningTreeClr::Run() +{ try + { m_spanningTree->Run(); + } + CATCHRETHROW +} } diff --git a/cs/cli/spanning_tree_clr.h b/cs/cli/spanning_tree_clr.h index 1a39be3fa32..3844c2dcdd7 100644 --- a/cs/cli/spanning_tree_clr.h +++ b/cs/cli/spanning_tree_clr.h @@ -10,35 +10,35 @@ license as described in the file LICENSE. namespace VW { - /// - /// Managed wrapper for AllReduce spanning tree implementation. - /// - public ref class SpanningTreeClr - { - private: - SpanningTree* m_spanningTree; - - public: - /// - /// Initializes a new instance. - /// - SpanningTreeClr(); - - ~SpanningTreeClr(); - - /// - /// Starts the server on a background thread. - /// - void Start(); - - /// - /// Runs the server on the calling thread. - /// - void Run(); - - /// - /// Stops the background thread. - /// - void Stop(); - }; +/// +/// Managed wrapper for AllReduce spanning tree implementation. +/// +public ref class SpanningTreeClr +{ +private: + SpanningTree* m_spanningTree; + +public: + /// + /// Initializes a new instance. + /// + SpanningTreeClr(); + + ~SpanningTreeClr(); + + /// + /// Starts the server on a background thread. + /// + void Start(); + + /// + /// Runs the server on the calling thread. + /// + void Run(); + + /// + /// Stops the background thread. + /// + void Stop(); +}; } diff --git a/cs/cli/vld_clr.cpp b/cs/cli/vld_clr.cpp index b5c1e2cf136..3c07d374992 100644 --- a/cs/cli/vld_clr.cpp +++ b/cs/cli/vld_clr.cpp @@ -2,58 +2,49 @@ namespace VLD { - int VldReportHook(int reportType, wchar_t *message, int *returnValue) - { - auto msg = gcnew String(message); - System::Diagnostics::Debug::Write(msg); +int VldReportHook(int reportType, wchar_t *message, int *returnValue) +{ auto msg = gcnew String(message); + System::Diagnostics::Debug::Write(msg); - if (VisualLeakDetector::Instance) - VisualLeakDetector::Instance->ReportInternal(reportType, msg); + if (VisualLeakDetector::Instance) + VisualLeakDetector::Instance->ReportInternal(reportType, msg); - *returnValue = 0; /* don't debug break */ - return 1; /* handled */ - } + *returnValue = 0; /* don't debug break */ + return 1; /* handled */ +} - VisualLeakDetector::VisualLeakDetector() : m_messages(gcnew List^>) - { - if (Instance != nullptr) - { - throw gcnew NotSupportedException("Only a single instance is supported."); - } +VisualLeakDetector::VisualLeakDetector() : m_messages(gcnew List^>) +{ if (Instance != nullptr) + { throw gcnew NotSupportedException("Only a single instance is supported."); + } - Instance = this; + Instance = this; - VLDSetReportHook(VLD_RPTHOOK_INSTALL, VldReportHook); - } + VLDSetReportHook(VLD_RPTHOOK_INSTALL, VldReportHook); +} - VisualLeakDetector::~VisualLeakDetector() - { - this->!VisualLeakDetector(); - } +VisualLeakDetector::~VisualLeakDetector() +{ this->!VisualLeakDetector(); +} - VisualLeakDetector::!VisualLeakDetector() - { - VLDSetReportHook(VLD_RPTHOOK_REMOVE, VldReportHook); - Instance = nullptr; - } +VisualLeakDetector::!VisualLeakDetector() +{ VLDSetReportHook(VLD_RPTHOOK_REMOVE, VldReportHook); + Instance = nullptr; +} - void VisualLeakDetector::ReportInternal(int reportType, String^ msg) - { - m_messages->Add(Tuple::Create(reportType, msg)); - } +void VisualLeakDetector::ReportInternal(int reportType, String^ msg) +{ m_messages->Add(Tuple::Create(reportType, msg)); +} - void VisualLeakDetector::ReportLeaks() - { - VLDReportLeaks(); - } +void VisualLeakDetector::ReportLeaks() +{ VLDReportLeaks(); +} - List^>^ VisualLeakDetector::Messages::get() - { - return m_messages; - } +List^>^ VisualLeakDetector::Messages::get() +{ return m_messages; +} - void VisualLeakDetector::MarkAllLeaksAsReported() - { - VLDMarkAllLeaksAsReported(); - } +void VisualLeakDetector::MarkAllLeaksAsReported() +{ VLDMarkAllLeaksAsReported(); +} } diff --git a/cs/cli/vld_clr.h b/cs/cli/vld_clr.h index bc026493b98..eceef50c096 100644 --- a/cs/cli/vld_clr.h +++ b/cs/cli/vld_clr.h @@ -8,31 +8,30 @@ using namespace System::Runtime::InteropServices; namespace VLD { - int VldReportHook(int reportType, wchar_t *message, int *returnValue); +int VldReportHook(int reportType, wchar_t *message, int *returnValue); - public ref class VisualLeakDetector - { - private: - initonly List^>^ m_messages; +public ref class VisualLeakDetector +{ +private: + initonly List^>^ m_messages; - !VisualLeakDetector(); + !VisualLeakDetector(); - public: - VisualLeakDetector(); +public: + VisualLeakDetector(); - ~VisualLeakDetector(); + ~VisualLeakDetector(); - static VisualLeakDetector^ Instance; + static VisualLeakDetector^ Instance; - void ReportInternal(int reportType, String^ msg); + void ReportInternal(int reportType, String^ msg); - property List^>^ Messages - { - List^>^ get(); - } + property List^>^ Messages + { List^>^ get(); + } - void ReportLeaks(); + void ReportLeaks(); - void MarkAllLeaksAsReported(); - }; + void MarkAllLeaksAsReported(); +}; } diff --git a/cs/cli/vowpalwabbit.cpp b/cs/cli/vowpalwabbit.cpp index 00ec7cbdb1f..764564900f2 100644 --- a/cs/cli/vowpalwabbit.cpp +++ b/cs/cli/vowpalwabbit.cpp @@ -22,637 +22,574 @@ using namespace System::Text; namespace VW { - VowpalWabbit::VowpalWabbit(VowpalWabbitSettings^ settings) - : VowpalWabbitBase(settings) - { - if (settings == nullptr) - { - throw gcnew ArgumentNullException("settings"); - } +VowpalWabbit::VowpalWabbit(VowpalWabbitSettings^ settings) + : VowpalWabbitBase(settings) +{ if (settings == nullptr) + { throw gcnew ArgumentNullException("settings"); + } - if (settings->ParallelOptions != nullptr) - { - m_vw->all_reduce_type = AllReduceType::Thread; - auto total = settings->ParallelOptions->MaxDegreeOfParallelism; - - if (settings->Root == nullptr) - { - m_vw->all_reduce = new AllReduceThreads(total, settings->Node); - } - else - { - auto parent_all_reduce = (AllReduceThreads*)settings->Root->m_vw->all_reduce; - - m_vw->all_reduce = new AllReduceThreads(parent_all_reduce, total, settings->Node); - } + if (settings->ParallelOptions != nullptr) + { m_vw->all_reduce_type = AllReduceType::Thread; + auto total = settings->ParallelOptions->MaxDegreeOfParallelism; + + if (settings->Root == nullptr) + { m_vw->all_reduce = new AllReduceThreads(total, settings->Node); } + else + { auto parent_all_reduce = (AllReduceThreads*)settings->Root->m_vw->all_reduce; - try - { - m_hasher = GetHasher(); + m_vw->all_reduce = new AllReduceThreads(parent_all_reduce, total, settings->Node); } - CATCHRETHROW } - VowpalWabbit::VowpalWabbit(String^ args) - : VowpalWabbit(gcnew VowpalWabbitSettings(args)) - { + try + { m_hasher = GetHasher(); } + CATCHRETHROW +} - void VowpalWabbit::Driver() - { - try - { - LEARNER::generic_driver(*m_vw); - } - CATCHRETHROW +VowpalWabbit::VowpalWabbit(String^ args) + : VowpalWabbit(gcnew VowpalWabbitSettings(args)) +{ +} + +void VowpalWabbit::Driver() +{ try + { LEARNER::generic_driver(*m_vw); } + CATCHRETHROW +} - void VowpalWabbit::RunMultiPass() - { - if (m_vw->numpasses > 1) - { - try - { - adjust_used_index(*m_vw); - m_vw->do_reset_source = true; - VW::start_parser(*m_vw); - LEARNER::generic_driver(*m_vw); - VW::end_parser(*m_vw); - } - CATCHRETHROW +void VowpalWabbit::RunMultiPass() +{ if (m_vw->numpasses > 1) + { try + { adjust_used_index(*m_vw); + m_vw->do_reset_source = true; + VW::start_parser(*m_vw); + LEARNER::generic_driver(*m_vw); + VW::end_parser(*m_vw); } + CATCHRETHROW } +} - VowpalWabbitPerformanceStatistics^ VowpalWabbit::PerformanceStatistics::get() - { - // see parse_args.cc:finish(...) - auto stats = gcnew VowpalWabbitPerformanceStatistics(); +VowpalWabbitPerformanceStatistics^ VowpalWabbit::PerformanceStatistics::get() +{ // see parse_args.cc:finish(...) + auto stats = gcnew VowpalWabbitPerformanceStatistics(); - if (m_vw->current_pass == 0) - { - stats->NumberOfExamplesPerPass = m_vw->sd->example_number; - } - else - { - stats->NumberOfExamplesPerPass = m_vw->sd->example_number / m_vw->current_pass; - } + if (m_vw->current_pass == 0) + { stats->NumberOfExamplesPerPass = m_vw->sd->example_number; + } + else + { stats->NumberOfExamplesPerPass = m_vw->sd->example_number / m_vw->current_pass; + } - stats->WeightedExampleSum = m_vw->sd->weighted_examples; - stats->WeightedLabelSum = m_vw->sd->weighted_labels; + stats->WeightedExampleSum = m_vw->sd->weighted_examples; + stats->WeightedLabelSum = m_vw->sd->weighted_labels; - if (m_vw->holdout_set_off || (m_vw->sd->holdout_best_loss == FLT_MAX)) - { - stats->AverageLoss = m_vw->sd->sum_loss / m_vw->sd->weighted_examples; - } - else - { - stats->AverageLoss = m_vw->sd->holdout_best_loss; - } + if (m_vw->holdout_set_off || (m_vw->sd->holdout_best_loss == FLT_MAX)) + { stats->AverageLoss = m_vw->sd->sum_loss / m_vw->sd->weighted_examples; + } + else + { stats->AverageLoss = m_vw->sd->holdout_best_loss; + } - float best_constant; float best_constant_loss; - if (get_best_constant(*m_vw, best_constant, best_constant_loss)) - { - stats->BestConstant = best_constant; - if (best_constant_loss != FLT_MIN) - { - stats->BestConstantLoss = best_constant_loss; - } + float best_constant; float best_constant_loss; + if (get_best_constant(*m_vw, best_constant, best_constant_loss)) + { stats->BestConstant = best_constant; + if (best_constant_loss != FLT_MIN) + { stats->BestConstantLoss = best_constant_loss; } + } - stats->TotalNumberOfFeatures = m_vw->sd->total_features; + stats->TotalNumberOfFeatures = m_vw->sd->total_features; - return stats; - } + return stats; +} - uint64_t VowpalWabbit::HashSpace(String^ s) - { - auto newHash = m_hasher(s, hash_base); +uint64_t VowpalWabbit::HashSpace(String^ s) +{ auto newHash = m_hasher(s, hash_base); #ifdef _DEBUG - auto oldHash = HashSpaceNative(s); - assert(newHash == oldHash); + auto oldHash = HashSpaceNative(s); + assert(newHash == oldHash); #endif - return (uint32_t)newHash; - } + return (uint32_t)newHash; +} - uint64_t VowpalWabbit::HashFeature(String^ s, size_t u) - { - auto newHash = m_hasher(s, u) & m_vw->parse_mask; +uint64_t VowpalWabbit::HashFeature(String^ s, size_t u) +{ auto newHash = m_hasher(s, u) & m_vw->parse_mask; #ifdef _DEBUG - auto oldHash = HashFeatureNative(s, u); - assert(newHash == oldHash); + auto oldHash = HashFeatureNative(s, u); + assert(newHash == oldHash); #endif - return (uint64_t)newHash; - } + return (uint64_t)newHash; +} - uint64_t VowpalWabbit::HashSpaceNative(String^ s) - { - auto bytes = System::Text::Encoding::UTF8->GetBytes(s); - auto handle = GCHandle::Alloc(bytes, GCHandleType::Pinned); +uint64_t VowpalWabbit::HashSpaceNative(String^ s) +{ auto bytes = System::Text::Encoding::UTF8->GetBytes(s); + auto handle = GCHandle::Alloc(bytes, GCHandleType::Pinned); - try - { - return VW::hash_space(*m_vw, reinterpret_cast(handle.AddrOfPinnedObject().ToPointer())); - } - CATCHRETHROW - finally - { - handle.Free(); - } + try + { return VW::hash_space(*m_vw, reinterpret_cast(handle.AddrOfPinnedObject().ToPointer())); + } + CATCHRETHROW + finally + { handle.Free(); } +} - uint64_t VowpalWabbit::HashFeatureNative(String^ s, uint64_t u) - { - auto bytes = System::Text::Encoding::UTF8->GetBytes(s); - auto handle = GCHandle::Alloc(bytes, GCHandleType::Pinned); +uint64_t VowpalWabbit::HashFeatureNative(String^ s, uint64_t u) +{ auto bytes = System::Text::Encoding::UTF8->GetBytes(s); + auto handle = GCHandle::Alloc(bytes, GCHandleType::Pinned); - try - { - return VW::hash_feature(*m_vw, reinterpret_cast(handle.AddrOfPinnedObject().ToPointer()), u); - } - CATCHRETHROW - finally - { handle.Free(); - } + try + { return VW::hash_feature(*m_vw, reinterpret_cast(handle.AddrOfPinnedObject().ToPointer()), u); + } + CATCHRETHROW + finally + { handle.Free(); } +} - void VowpalWabbit::Learn(VowpalWabbitExample^ ex) - { +void VowpalWabbit::Learn(VowpalWabbitExample^ ex) +{ #if _DEBUG - if (ex == nullptr) - { - throw gcnew ArgumentNullException("ex"); - } + if (ex == nullptr) + { throw gcnew ArgumentNullException("ex"); + } #endif - try - { - m_vw->learn(ex->m_example); + try + { m_vw->learn(ex->m_example); - // as this is not a ring-based example it is not free'd - m_vw->l->finish_example(*m_vw, *ex->m_example); - } - CATCHRETHROW + // as this is not a ring-based example it is not free'd + m_vw->l->finish_example(*m_vw, *ex->m_example); } + CATCHRETHROW +} - generic T VowpalWabbit::Learn(VowpalWabbitExample^ ex, IVowpalWabbitPredictionFactory^ predictionFactory) - { +generic T VowpalWabbit::Learn(VowpalWabbitExample^ ex, IVowpalWabbitPredictionFactory^ predictionFactory) +{ #if _DEBUG - if (ex == nullptr) - throw gcnew ArgumentNullException("ex"); + if (ex == nullptr) + throw gcnew ArgumentNullException("ex"); - if (nullptr == predictionFactory) - throw gcnew ArgumentNullException("predictionFactory"); + if (nullptr == predictionFactory) + throw gcnew ArgumentNullException("predictionFactory"); #endif - try - { - m_vw->learn(ex->m_example); + try + { m_vw->learn(ex->m_example); - auto prediction = predictionFactory->Create(m_vw, ex->m_example); + auto prediction = predictionFactory->Create(m_vw, ex->m_example); - // as this is not a ring-based example it is not free'd - m_vw->l->finish_example(*m_vw, *ex->m_example); + // as this is not a ring-based example it is not free'd + m_vw->l->finish_example(*m_vw, *ex->m_example); - return prediction; - } - CATCHRETHROW + return prediction; } + CATCHRETHROW +} - void VowpalWabbit::Predict(VowpalWabbitExample^ ex) - { +void VowpalWabbit::Predict(VowpalWabbitExample^ ex) +{ #if _DEBUG - if (ex == nullptr) - throw gcnew ArgumentNullException("ex"); + if (ex == nullptr) + throw gcnew ArgumentNullException("ex"); #endif - try - { - m_vw->l->predict(*ex->m_example); + try + { m_vw->l->predict(*ex->m_example); - // as this is not a ring-based example it is not free'd - m_vw->l->finish_example(*m_vw, *ex->m_example); - } - CATCHRETHROW + // as this is not a ring-based example it is not free'd + m_vw->l->finish_example(*m_vw, *ex->m_example); } + CATCHRETHROW +} - generic T VowpalWabbit::Predict(VowpalWabbitExample^ ex, IVowpalWabbitPredictionFactory^ predictionFactory) - { +generic T VowpalWabbit::Predict(VowpalWabbitExample^ ex, IVowpalWabbitPredictionFactory^ predictionFactory) +{ #if _DEBUG - if (ex == nullptr) - throw gcnew ArgumentNullException("ex"); + if (ex == nullptr) + throw gcnew ArgumentNullException("ex"); #endif - try - { - m_vw->l->predict(*ex->m_example); + try + { m_vw->l->predict(*ex->m_example); - auto prediction = predictionFactory->Create(m_vw, ex->m_example); + auto prediction = predictionFactory->Create(m_vw, ex->m_example); - // as this is not a ring-based example it is not free'd - m_vw->l->finish_example(*m_vw, *ex->m_example); + // as this is not a ring-based example it is not free'd + m_vw->l->finish_example(*m_vw, *ex->m_example); - return prediction; - } - CATCHRETHROW + return prediction; } + CATCHRETHROW +} - public ref struct ParseJsonState - { - VowpalWabbit^ vw; - List^ examples; - }; +public ref struct ParseJsonState +{ VowpalWabbit^ vw; + List^ examples; +}; - example& get_example_from_pool(void* v) - { - interior_ptr state = (interior_ptr)v; +example& get_example_from_pool(void* v) +{ interior_ptr state = (interior_ptr)v; - auto ex = (*state)->vw->GetOrCreateNativeExample(); - (*state)->examples->Add(ex); + auto ex = (*state)->vw->GetOrCreateNativeExample(); + (*state)->examples->Add(ex); - return *ex->m_example; - } + return *ex->m_example; +} - List^ VowpalWabbit::ParseJson(String^ line) - { -#if _DEBUG - if (line == nullptr) - throw gcnew ArgumentNullException("line"); -#endif - auto bytes = System::Text::Encoding::UTF8->GetBytes(line); - auto valueHandle = GCHandle::Alloc(bytes, GCHandleType::Pinned); - - try - { - ParseJsonState^ state = gcnew ParseJsonState(); - state->vw = this; - state->examples = gcnew List(); - - try - { - auto ex = GetOrCreateNativeExample(); - state->examples->Add(ex); - - v_array examples = v_init(); - example* native_example = ex->m_example; - examples.push_back(native_example); - - interior_ptr state_ptr = &state; - - VW::read_line_json( - *m_vw, - examples, - reinterpret_cast(valueHandle.AddrOfPinnedObject().ToPointer()), - get_example_from_pool, - &state); - - // finalize example - VW::setup_examples(*m_vw, examples); - - // remember the input string for debugging purposes - ex->VowpalWabbitString = line; - - return state->examples; - } - catch (...) - { - // cleanup - for each (auto ex in state->examples) - delete ex; - throw; - } - } - CATCHRETHROW - finally - { - valueHandle.Free(); - } - } - - VowpalWabbitExample^ VowpalWabbit::ParseLine(String^ line) - { +List^ VowpalWabbit::ParseJson(String^ line) +{ #if _DEBUG - if (line == nullptr) - throw gcnew ArgumentNullException("line"); + if (line == nullptr) + throw gcnew ArgumentNullException("line"); #endif + auto bytes = System::Text::Encoding::UTF8->GetBytes(line); + auto valueHandle = GCHandle::Alloc(bytes, GCHandleType::Pinned); - auto ex = GetOrCreateNativeExample(); - auto bytes = System::Text::Encoding::UTF8->GetBytes(line); - auto valueHandle = GCHandle::Alloc(bytes, GCHandleType::Pinned); + try + { ParseJsonState^ state = gcnew ParseJsonState(); + state->vw = this; + state->examples = gcnew List(); try - { - try - { - VW::read_line(*m_vw, ex->m_example, reinterpret_cast(valueHandle.AddrOfPinnedObject().ToPointer())); + { auto ex = GetOrCreateNativeExample(); + state->examples->Add(ex); - // finalize example - VW::setup_example(*m_vw, ex->m_example); + v_array examples = v_init(); + example* native_example = ex->m_example; + examples.push_back(native_example); - // remember the input string for debugging purposes - ex->VowpalWabbitString = line; + interior_ptr state_ptr = &state; - return ex; - } - catch (...) - { - delete ex; - throw; - } + VW::read_line_json( + *m_vw, + examples, + reinterpret_cast(valueHandle.AddrOfPinnedObject().ToPointer()), + get_example_from_pool, + &state); + + // finalize example + VW::setup_examples(*m_vw, examples); + + // remember the input string for debugging purposes + ex->VowpalWabbitString = line; + + return state->examples; } - CATCHRETHROW - finally - { - valueHandle.Free(); + catch (...) + { // cleanup + for each (auto ex in state->examples) + delete ex; + throw; } } + CATCHRETHROW + finally + { valueHandle.Free(); + } +} - void VowpalWabbit::Learn(String^ line) - { +VowpalWabbitExample^ VowpalWabbit::ParseLine(String^ line) +{ #if _DEBUG - if (String::IsNullOrEmpty(line)) - throw gcnew ArgumentException("lines must not be empty. For multi-line examples use Learn(IEnumerable) overload."); + if (line == nullptr) + throw gcnew ArgumentNullException("line"); #endif - VowpalWabbitExample^ example = nullptr; + auto ex = GetOrCreateNativeExample(); + auto bytes = System::Text::Encoding::UTF8->GetBytes(line); + auto valueHandle = GCHandle::Alloc(bytes, GCHandleType::Pinned); - try - { - example = ParseLine(line); - Learn(example); + try + { try + { VW::read_line(*m_vw, ex->m_example, reinterpret_cast(valueHandle.AddrOfPinnedObject().ToPointer())); + + // finalize example + VW::setup_example(*m_vw, ex->m_example); + + // remember the input string for debugging purposes + ex->VowpalWabbitString = line; + + return ex; } - finally - { - delete example; + catch (...) + { delete ex; + throw; } } + CATCHRETHROW + finally + { valueHandle.Free(); + } +} - void VowpalWabbit::Predict(String^ line) - { +void VowpalWabbit::Learn(String^ line) +{ #if _DEBUG - if (String::IsNullOrEmpty(line)) - throw gcnew ArgumentException("lines must not be empty. For multi-line examples use Predict(IEnumerable) overload."); + if (String::IsNullOrEmpty(line)) + throw gcnew ArgumentException("lines must not be empty. For multi-line examples use Learn(IEnumerable) overload."); #endif - VowpalWabbitExample^ example = nullptr; + VowpalWabbitExample^ example = nullptr; - try - { - example = ParseLine(line); - Predict(example); - } - finally - { - delete example; - } + try + { example = ParseLine(line); + Learn(example); + } + finally + { delete example; } +} - generic TPrediction VowpalWabbit::Learn(String^ line, IVowpalWabbitPredictionFactory^ predictionFactory) - { +void VowpalWabbit::Predict(String^ line) +{ #if _DEBUG - if (String::IsNullOrEmpty(line)) - throw gcnew ArgumentException("lines must not be empty. For multi-line examples use Learn(IEnumerable) overload."); + if (String::IsNullOrEmpty(line)) + throw gcnew ArgumentException("lines must not be empty. For multi-line examples use Predict(IEnumerable) overload."); #endif - VowpalWabbitExample^ example = nullptr; + VowpalWabbitExample^ example = nullptr; - try - { - example = ParseLine(line); - return Learn(example, predictionFactory); - } - finally - { - delete example; - } + try + { example = ParseLine(line); + Predict(example); + } + finally + { delete example; } +} - generic T VowpalWabbit::Predict(String^ line, IVowpalWabbitPredictionFactory^ predictionFactory) - { +generic TPrediction VowpalWabbit::Learn(String^ line, IVowpalWabbitPredictionFactory^ predictionFactory) +{ #if _DEBUG - if (String::IsNullOrEmpty(line)) - throw gcnew ArgumentException("lines must not be empty. For multi-line examples use Learn(IEnumerable) overload."); + if (String::IsNullOrEmpty(line)) + throw gcnew ArgumentException("lines must not be empty. For multi-line examples use Learn(IEnumerable) overload."); #endif - VowpalWabbitExample^ example = nullptr; + VowpalWabbitExample^ example = nullptr; - try - { - example = ParseLine(line); - return Predict(example, predictionFactory); - } - finally - { delete example; - } + try + { example = ParseLine(line); + return Learn(example, predictionFactory); } + finally + { delete example; + } +} - void VowpalWabbit::Learn(IEnumerable^ lines) - { +generic T VowpalWabbit::Predict(String^ line, IVowpalWabbitPredictionFactory^ predictionFactory) +{ #if _DEBUG - if (lines == nullptr) - throw gcnew ArgumentNullException("lines"); + if (String::IsNullOrEmpty(line)) + throw gcnew ArgumentException("lines must not be empty. For multi-line examples use Learn(IEnumerable) overload."); #endif - auto examples = gcnew List; + VowpalWabbitExample^ example = nullptr; - try - { - for each (auto line in lines) - { - auto ex = ParseLine(line); - examples->Add(ex); - - Learn(ex); - } - - auto empty = GetOrCreateNativeExample(); - examples->Add(empty); - empty->MakeEmpty(this); - Learn(empty); - } - finally - { - for each (auto ex in examples) - { - delete ex; - } - } + try + { example = ParseLine(line); + return Predict(example, predictionFactory); + } + finally + { delete example; } +} - void VowpalWabbit::Predict(IEnumerable^ lines) - { +void VowpalWabbit::Learn(IEnumerable^ lines) +{ #if _DEBUG - if (lines == nullptr) - throw gcnew ArgumentNullException("lines"); + if (lines == nullptr) + throw gcnew ArgumentNullException("lines"); #endif - auto examples = gcnew List; + auto examples = gcnew List; - try - { - for each (auto line in lines) - { - auto ex = ParseLine(line); - examples->Add(ex); - - Predict(ex); - } - - auto empty = GetOrCreateNativeExample(); - examples->Add(empty); - empty->MakeEmpty(this); - Predict(empty); - } - finally - { for each (auto ex in examples) - { - delete ex; + try + { for each (auto line in lines) + { auto ex = ParseLine(line); + examples->Add(ex); + + Learn(ex); } + + auto empty = GetOrCreateNativeExample(); + examples->Add(empty); + empty->MakeEmpty(this); + Learn(empty); + } + finally + { for each (auto ex in examples) + { delete ex; } } +} - generic T VowpalWabbit::Learn(IEnumerable^ lines, IVowpalWabbitPredictionFactory^ predictionFactory) - { +void VowpalWabbit::Predict(IEnumerable^ lines) +{ #if _DEBUG - if (lines == nullptr) - throw gcnew ArgumentNullException("lines"); + if (lines == nullptr) + throw gcnew ArgumentNullException("lines"); #endif - auto examples = gcnew List; + auto examples = gcnew List; - try - { - for each (auto line in lines) - { - auto ex = ParseLine(line); - examples->Add(ex); + try + { for each (auto line in lines) + { auto ex = ParseLine(line); + examples->Add(ex); - Learn(ex); - } - - auto empty = GetOrCreateNativeExample(); - examples->Add(empty); - empty->MakeEmpty(this); - Learn(empty); - - return examples[0]->GetPrediction(this, predictionFactory); - } - finally - { for each (auto ex in examples) - { - delete ex; + Predict(ex); } + + auto empty = GetOrCreateNativeExample(); + examples->Add(empty); + empty->MakeEmpty(this); + Predict(empty); + } + finally + { for each (auto ex in examples) + { delete ex; } } +} - generic T VowpalWabbit::Predict(IEnumerable^ lines, IVowpalWabbitPredictionFactory^ predictionFactory) - { +generic T VowpalWabbit::Learn(IEnumerable^ lines, IVowpalWabbitPredictionFactory^ predictionFactory) +{ #if _DEBUG - if (lines == nullptr) - throw gcnew ArgumentNullException("lines"); + if (lines == nullptr) + throw gcnew ArgumentNullException("lines"); #endif - auto examples = gcnew List; + auto examples = gcnew List; - try - { - for each (auto line in lines) - { - auto ex = ParseLine(line); - examples->Add(ex); + try + { for each (auto line in lines) + { auto ex = ParseLine(line); + examples->Add(ex); - Predict(ex); - } + Learn(ex); + } - auto empty = GetOrCreateNativeExample(); - examples->Add(empty); - empty->MakeEmpty(this); - Predict(empty); + auto empty = GetOrCreateNativeExample(); + examples->Add(empty); + empty->MakeEmpty(this); + Learn(empty); - return examples[0]->GetPrediction(this, predictionFactory); - } - finally - { for each (auto ex in examples) - { - delete ex; - } - } + return examples[0]->GetPrediction(this, predictionFactory); } - - void VowpalWabbit::EndOfPass() - { - try - { - m_vw->l->end_pass(); - sync_stats(*m_vw); + finally + { for each (auto ex in examples) + { delete ex; } - CATCHRETHROW } +} + +generic T VowpalWabbit::Predict(IEnumerable^ lines, IVowpalWabbitPredictionFactory^ predictionFactory) +{ +#if _DEBUG + if (lines == nullptr) + throw gcnew ArgumentNullException("lines"); +#endif - /// - /// Hashes the given value . - /// - /// String to be hashed. - /// Hash offset. - /// The resulting hash code. - //template - uint64_t hashall(String^ s, int offset, int count, uint64_t u) - { // get raw bytes from string - auto keys = gcnew cli::array(Encoding::UTF8->GetMaxByteCount(count)); - int length = Encoding::UTF8->GetBytes(s, offset, count, keys, 0); - - // TOOD: benchmark and verify correctness - //if (replaceSpace) - //{ - // for (int j = 0; j < length;) - // { - // var k = keys[j]; - // if (k == ' ') - // { - // keys[j] = '_'; - // } - - // j++; - - // // take care of UTF-8 multi-byte characters - // while (k & 0xC == 0xC) - // { - // j++; - // k <<= 1; - // } - // } - //} - - uint32_t h1 = (uint32_t)u; - uint32_t k1 = 0; - - const uint32_t c1 = 0xcc9e2d51; - const uint32_t c2 = 0x1b873593; - - int i = 0; - while (i <= length - 4) - { // convert byte array to integer - k1 = (uint32_t)(keys[i] | keys[i + 1] << 8 | keys[i + 2] << 16 | keys[i + 3] << 24); + auto examples = gcnew List; - k1 *= c1; - k1 = ROTL32(k1, 15); - k1 *= c2; + try + { for each (auto line in lines) + { auto ex = ParseLine(line); + examples->Add(ex); - h1 ^= k1; - h1 = ROTL32(h1, 13); - h1 = h1 * 5 + 0xe6546b64; + Predict(ex); + } + + auto empty = GetOrCreateNativeExample(); + examples->Add(empty); + empty->MakeEmpty(this); + Predict(empty); - i += 4; + return examples[0]->GetPrediction(this, predictionFactory); + } + finally + { for each (auto ex in examples) + { delete ex; } + } +} - k1 = 0; - int tail = length - length % 4; - switch (length & 3) - { - case 3: +void VowpalWabbit::EndOfPass() +{ try + { m_vw->l->end_pass(); + sync_stats(*m_vw); + } + CATCHRETHROW +} + +/// +/// Hashes the given value . +/// +/// String to be hashed. +/// Hash offset. +/// The resulting hash code. +//template +uint64_t hashall(String^ s, int offset, int count, uint64_t u) +{ // get raw bytes from string + auto keys = gcnew cli::array(Encoding::UTF8->GetMaxByteCount(count)); + int length = Encoding::UTF8->GetBytes(s, offset, count, keys, 0); + + // TOOD: benchmark and verify correctness + //if (replaceSpace) + //{ + // for (int j = 0; j < length;) + // { + // var k = keys[j]; + // if (k == ' ') + // { + // keys[j] = '_'; + // } + + // j++; + + // // take care of UTF-8 multi-byte characters + // while (k & 0xC == 0xC) + // { + // j++; + // k <<= 1; + // } + // } + //} + + uint32_t h1 = (uint32_t)u; + uint32_t k1 = 0; + + const uint32_t c1 = 0xcc9e2d51; + const uint32_t c2 = 0x1b873593; + + int i = 0; + while (i <= length - 4) + { // convert byte array to integer + k1 = (uint32_t)(keys[i] | keys[i + 1] << 8 | keys[i + 2] << 16 | keys[i + 3] << 24); + + k1 *= c1; + k1 = ROTL32(k1, 15); + k1 *= c2; + + h1 ^= k1; + h1 = ROTL32(h1, 13); + h1 = h1 * 5 + 0xe6546b64; + + i += 4; + } + + k1 = 0; + int tail = length - length % 4; + switch (length & 3) + { case 3: k1 ^= (uint32_t)(keys[tail + 2] << 16); case 2: k1 ^= (uint32_t)(keys[tail + 1] << 8); @@ -663,177 +600,160 @@ namespace VW k1 *= c2; h1 ^= k1; break; - } + } - // finalization - h1 ^= (uint32_t)length; - - return MURMUR_HASH_3::fmix(h1); - } - - uint64_t hashall(String^ s, uint64_t u) - { - return hashall(s, 0, s->Length, u); - } - - /// - /// Hashes the given value . - /// - /// String to be hashed. - /// Hash offset. - /// The resulting hash code. - size_t hashstring(String^ s, size_t u) - { - int offset = 0; - int end = s->Length; - if (end == 0) - return u; - - //trim leading whitespace but not UTF-8 - for (;offset < s->Length && s[offset] <= 0x20;offset++); - for (;end >= offset && s[end - 1] <= 0x20;end--); - - int sInt = 0; - for (int i = offset;i < end;i++) - { - auto c = s[i]; - if (c >= '0' && c <= '9') - sInt = 10 * sInt + (c - '0'); - else - return hashall(s, offset, end - offset, u); - } - - return sInt + u; - } - - Func^ VowpalWabbit::GetHasher() - { - //feature manipulation - string hash_function("strings"); - if (m_vw->vm.count("hash")) - { - hash_function = m_vw->vm["hash"].as(); - } + // finalization + h1 ^= (uint32_t)length; - if (hash_function == "strings") - { - return gcnew Func(&hashstring); - } - else if (hash_function == "all") - { - return gcnew Func(&hashall); - } + return MURMUR_HASH_3::fmix(h1); +} + +uint64_t hashall(String^ s, uint64_t u) +{ return hashall(s, 0, s->Length, u); +} + +/// +/// Hashes the given value . +/// +/// String to be hashed. +/// Hash offset. +/// The resulting hash code. +size_t hashstring(String^ s, size_t u) +{ int offset = 0; + int end = s->Length; + if (end == 0) + return u; + + //trim leading whitespace but not UTF-8 + for (; offset < s->Length && s[offset] <= 0x20; offset++); + for (; end >= offset && s[end - 1] <= 0x20; end--); + + int sInt = 0; + for (int i = offset; i < end; i++) + { auto c = s[i]; + if (c >= '0' && c <= '9') + sInt = 10 * sInt + (c - '0'); else - { - THROW("Unsupported hash function: " << hash_function); - } + return hashall(s, offset, end - offset, u); } - VowpalWabbit^ VowpalWabbit::Native::get() - { - return this; + return sInt + u; +} + +Func^ VowpalWabbit::GetHasher() +{ //feature manipulation + string hash_function("strings"); + if (m_vw->vm.count("hash")) + { hash_function = m_vw->vm["hash"].as(); } + if (hash_function == "strings") + { return gcnew Func(&hashstring); + } + else if (hash_function == "all") + { return gcnew Func(&hashall); + } + else + { THROW("Unsupported hash function: " << hash_function); + } +} - VowpalWabbitExample^ VowpalWabbit::GetOrCreateNativeExample() - { - auto ex = m_examples->Remove(); +VowpalWabbit^ VowpalWabbit::Native::get() +{ return this; +} - if (ex == nullptr) - { - try - { - auto ex = VW::alloc_examples(0, 1); - m_vw->p->lp.default_label(&ex->l); - return gcnew VowpalWabbitExample(this, ex); - } - CATCHRETHROW - } - try - { - VW::empty_example(*m_vw, *ex->m_example); - m_vw->p->lp.default_label(&ex->m_example->l); +VowpalWabbitExample^ VowpalWabbit::GetOrCreateNativeExample() +{ auto ex = m_examples->Remove(); - return ex; + if (ex == nullptr) + { try + { auto ex = VW::alloc_examples(0, 1); + m_vw->p->lp.default_label(&ex->l); + return gcnew VowpalWabbitExample(this, ex); } CATCHRETHROW } - void VowpalWabbit::ReturnExampleToPool(VowpalWabbitExample^ ex) - { + try + { VW::empty_example(*m_vw, *ex->m_example); + m_vw->p->lp.default_label(&ex->m_example->l); + + return ex; + } + CATCHRETHROW +} + +void VowpalWabbit::ReturnExampleToPool(VowpalWabbitExample^ ex) +{ #if _DEBUG - if (m_vw == nullptr) - throw gcnew ObjectDisposedException("VowpalWabbitExample was not properly disposed as the owner is already disposed"); + if (m_vw == nullptr) + throw gcnew ObjectDisposedException("VowpalWabbitExample was not properly disposed as the owner is already disposed"); #endif - if (ex == nullptr) - throw gcnew ArgumentNullException("ex"); + if (ex == nullptr) + throw gcnew ArgumentNullException("ex"); - // make sure we're not a ring based example - assert(!VW::is_ring_example(*m_vw, ex->m_example)); + // make sure we're not a ring based example + assert(!VW::is_ring_example(*m_vw, ex->m_example)); - // the bag might have reached it's limit - if (m_examples != nullptr) - { - if (!m_examples->TryAdd(ex)) - DisposeExample(ex); - } + // the bag might have reached it's limit + if (m_examples != nullptr) + { if (!m_examples->TryAdd(ex)) + DisposeExample(ex); + } #if _DEBUG - else // this should not happen as m_vw is already set to null - throw gcnew ObjectDisposedException("VowpalWabbitExample was disposed after the owner is disposed"); + else // this should not happen as m_vw is already set to null + throw gcnew ObjectDisposedException("VowpalWabbitExample was disposed after the owner is disposed"); #endif +} + +cli::array^>^ VowpalWabbit::GetTopicAllocation(int top) +{ uint64_t length = (uint64_t)1 << m_vw->num_bits; + // using jagged array to enable LINQ + auto K = (int)m_vw->lda; + auto allocation = gcnew cli::array^>(K); + + // TODO: better way of peaking into lda? + auto lda_rho = m_vw->vm["lda_rho"].as(); + + std::vector top_weights; + // over topics + for (int topic = 0; topic < K; topic++) + { get_top_weights(m_vw, top, topic, top_weights); + + auto clr_weights = gcnew List(top); + allocation[topic] = clr_weights; + for (auto& pair : top_weights) + clr_weights->Add(gcnew VowpalWabbitFeature(this, pair.x, pair.weight_index)); } - - cli::array^>^ VowpalWabbit::GetTopicAllocation(int top) - { - uint64_t length = (uint64_t)1 << m_vw->num_bits; - // using jagged array to enable LINQ - auto K = (int)m_vw->lda; - auto allocation = gcnew cli::array^>(K); - - // TODO: better way of peaking into lda? - auto lda_rho = m_vw->vm["lda_rho"].as(); - - std::vector top_weights; - // over topics - for (int topic = 0; topic < K; topic++) - { - get_top_weights(m_vw, top, topic, top_weights); - - auto clr_weights = gcnew List(top); - allocation[topic] = clr_weights; - for (auto& pair : top_weights) - clr_weights->Add(gcnew VowpalWabbitFeature(this, pair.x, pair.weight_index)); - } - - return allocation; - } - - - cli::array^>^ VowpalWabbit::GetTopicAllocation() - { - uint64_t length = (uint64_t)1 << m_vw->num_bits; - - // using jagged array to enable LINQ - auto K = (int)m_vw->lda; - auto allocation = gcnew cli::array^>(K); - for (int k = 0; k < K; k++) - allocation[k] = gcnew cli::array((int)length); - - // TODO: better way of peaking into lda? - auto lda_rho = m_vw->vm["lda_rho"].as(); - - // over weights - weight_parameters& weights = m_vw->weights; - weight_parameters::iterator iter = weights.begin(); - for (uint64_t i = 0; i < length; i++, ++iter) - { // over topics - weight_parameters::iterator::w_iter v = iter.begin(); - for (uint64_t k = 0; k < K; k++, ++v) - allocation[(int)k][(int)i] = *v + lda_rho; - } - - return allocation; + + return allocation; +} + + +cli::array^>^ VowpalWabbit::GetTopicAllocation() +{ uint64_t length = (uint64_t)1 << m_vw->num_bits; + + // using jagged array to enable LINQ + auto K = (int)m_vw->lda; + auto allocation = gcnew cli::array^>(K); + for (int k = 0; k < K; k++) + allocation[k] = gcnew cli::array((int)length); + + // TODO: better way of peaking into lda? + auto lda_rho = m_vw->vm["lda_rho"].as(); + + // over weights + weight_parameters& weights = m_vw->weights; + weight_parameters::iterator iter = weights.begin(); + for (uint64_t i = 0; i < length; i++, ++iter) + { // over topics + weight_parameters::iterator::w_iter v = iter.begin(); + for (uint64_t k = 0; k < K; k++, ++v) + allocation[(int)k][(int)i] = *v + lda_rho; } + + return allocation; +} } diff --git a/cs/cli/vowpalwabbit.h b/cs/cli/vowpalwabbit.h index d973edcf391..5ae2af534ab 100644 --- a/cs/cli/vowpalwabbit.h +++ b/cs/cli/vowpalwabbit.h @@ -14,229 +14,227 @@ license as described in the file LICENSE. namespace VW { - ref class VowpalWabbitExampleBuilder; - ref struct VowpalWabbitFeature; - - /// - /// Simple string example based wrapper for vowpal wabbit. - /// - /// If possible use VowpalWabbit{T} types as this wrapper suffers from marshalling performance wise. - public ref class VowpalWabbit : VowpalWabbitBase, IVowpalWabbitExamplePool - { - private: - /// - /// Select the right hash method based on args. - /// - Func^ GetHasher(); - - /// - /// The selected hasher method. - /// - /// - /// Avoiding if-else for hash function selection. Delegates outperform function pointers according to http://stackoverflow.com/questions/13443250/performance-of-c-cli-function-pointers-versus-net-delegates - /// - initonly Func^ m_hasher; - - public: - /// - /// Initializes a new instance. - /// - /// The settings. - VowpalWabbit(VowpalWabbitSettings^ settings); - - /// - /// Initializes a new instance. - /// - /// Command line arguments. - VowpalWabbit(String^ args); - - /// - /// Run multi-passe training. - /// - void RunMultiPass(); - - /// - /// Gets Collected performance statistics. - /// - property VowpalWabbitPerformanceStatistics^ PerformanceStatistics - { - VowpalWabbitPerformanceStatistics^ get(); - } - - /// - /// Parses using the C++ parser. - /// - /// - /// Returns a ready to be used for or . - /// - VowpalWabbitExample^ ParseLine(String^ line); - - /// - /// Parses using the C++ parser. - /// TODO: this should return VowpalWabbitExampleCollection, but that would require moving VowpalWaabitExampleCollection to C++/CLI - /// - /// - /// Returns a ready to be used for or . - /// - List^ ParseJson(String^ line); - - /// - /// Hashes the given namespace . - /// - /// String to be hashed. - /// The resulting hash code. - /// The hash code depends on the vowpal wabbit instance as different has functions can be configured. - uint64_t HashSpaceNative(String^ s); - - /// - /// Hashes the given namespace . - /// - /// String to be hashed. - /// The resulting hash code. - /// The hash code depends on the vowpal wabbit instance as different has functions can be configured. - uint64_t HashSpace(String^ s); - - /// - /// Hash the given feature . - /// - /// String to be hashed. - /// Hash offset. - /// The resulting hash code. - /// The hash code depends on the vowpal wabbit instance as different has functions can be configured. - uint64_t HashFeatureNative(String^ s, size_t u); - - /// - /// Hash the given feature . - /// - /// String to be hashed. - /// Hash offset. - /// The resulting hash code. - /// The hash code depends on the vowpal wabbit instance as different has functions can be configured. - uint64_t HashFeature(String^ s, size_t u); - - /// - /// Return full topic allocation [topic, feature]. - /// - cli::array^>^ GetTopicAllocation(); - - /// - /// Return the topic weights. - /// - cli::array^>^ GetTopicAllocation(int top); - - /// - /// The associated instance learns from this example and returns the prediction result for this example. - /// - /// The prediction result. - /// The prediction result type. - generic T Learn(VowpalWabbitExample^ example, IVowpalWabbitPredictionFactory^ predictionFactory); - - /// - /// Predicts for the given example. - /// - /// The prediction type. - /// Example to predict for. - /// The prediction factory to be used. See . - /// The prediction for the given . - generic T Predict(VowpalWabbitExample^ example, IVowpalWabbitPredictionFactory^ predictionFactory); - - /// - /// Learns from the given example. - /// - /// Example to learn from. - void Learn(VowpalWabbitExample^ example); - - /// - /// Predicts for the given example. - /// - /// Example to predict for. - void Predict(VowpalWabbitExample^ example); - - /// - /// Learns from string data. - /// - /// Data in vw string format. - void Learn(String^ line); - - /// - /// Predicts for string data. - /// - /// Data in vw string format. - void Predict(String^ line); - - /// - /// Learns from string data. - /// - /// The prediction type. - /// Data in vw string format. - /// The prediction factory to be used. See . - /// The prediction for the given . - generic T Learn(String^ line, IVowpalWabbitPredictionFactory^ predictionFactory); - - /// - /// Predicts for string data. - /// - /// The prediction type. - /// Data in vw string format. - /// The prediction factory to be used. See . - /// The prediction for the given . - generic T Predict(String^ line, IVowpalWabbitPredictionFactory^ predictionFactory); - - /// - /// Learns from multi-line examples. - /// - /// Data in vw string format. - void Learn(IEnumerable^ lines); - - /// - /// Predicts for multi-line examples. - /// - /// Data in vw string format. - void Predict(IEnumerable^ lines); - - /// - /// Learns from multi-line examples. - /// - /// The prediction type. - /// Data in vw string format. - /// The prediction factory to be used. See . - /// The prediction for the given . - generic T Learn(IEnumerable^ lines, IVowpalWabbitPredictionFactory^ predictionFactory); - - /// - /// Predicts for the given lines. - /// - /// The prediction type. - /// Data in vw string format. - /// The prediction factory to be used. See . - /// The prediction for the given . - generic T Predict(IEnumerable^ lines, IVowpalWabbitPredictionFactory^ predictionFactory); - - /// - /// Signals the end of a pass. - /// - void EndOfPass(); - - /// - /// Invokes the driver. - /// - void Driver(); - - virtual property VowpalWabbit^ Native - { - virtual VowpalWabbit^ get() sealed; - } - - /// - /// Gets or creates a native example from a CLR maintained, but natively allocated pool. - /// - /// A ready to use cleared native example data structure. - virtual VowpalWabbitExample^ GetOrCreateNativeExample() sealed; - - /// - /// Puts a native example data structure back into the pool. - /// - /// The example to be returned. - virtual void ReturnExampleToPool(VowpalWabbitExample^ example) sealed; - }; +ref class VowpalWabbitExampleBuilder; +ref struct VowpalWabbitFeature; + +/// +/// Simple string example based wrapper for vowpal wabbit. +/// +/// If possible use VowpalWabbit{T} types as this wrapper suffers from marshalling performance wise. +public ref class VowpalWabbit : VowpalWabbitBase, IVowpalWabbitExamplePool +{ +private: + /// + /// Select the right hash method based on args. + /// + Func^ GetHasher(); + + /// + /// The selected hasher method. + /// + /// + /// Avoiding if-else for hash function selection. Delegates outperform function pointers according to http://stackoverflow.com/questions/13443250/performance-of-c-cli-function-pointers-versus-net-delegates + /// + initonly Func^ m_hasher; + +public: + /// + /// Initializes a new instance. + /// + /// The settings. + VowpalWabbit(VowpalWabbitSettings^ settings); + + /// + /// Initializes a new instance. + /// + /// Command line arguments. + VowpalWabbit(String^ args); + + /// + /// Run multi-passe training. + /// + void RunMultiPass(); + + /// + /// Gets Collected performance statistics. + /// + property VowpalWabbitPerformanceStatistics^ PerformanceStatistics + { VowpalWabbitPerformanceStatistics^ get(); + } + + /// + /// Parses using the C++ parser. + /// + /// + /// Returns a ready to be used for or . + /// + VowpalWabbitExample^ ParseLine(String^ line); + + /// + /// Parses using the C++ parser. + /// TODO: this should return VowpalWabbitExampleCollection, but that would require moving VowpalWaabitExampleCollection to C++/CLI + /// + /// + /// Returns a ready to be used for or . + /// + List^ ParseJson(String^ line); + + /// + /// Hashes the given namespace . + /// + /// String to be hashed. + /// The resulting hash code. + /// The hash code depends on the vowpal wabbit instance as different has functions can be configured. + uint64_t HashSpaceNative(String^ s); + + /// + /// Hashes the given namespace . + /// + /// String to be hashed. + /// The resulting hash code. + /// The hash code depends on the vowpal wabbit instance as different has functions can be configured. + uint64_t HashSpace(String^ s); + + /// + /// Hash the given feature . + /// + /// String to be hashed. + /// Hash offset. + /// The resulting hash code. + /// The hash code depends on the vowpal wabbit instance as different has functions can be configured. + uint64_t HashFeatureNative(String^ s, size_t u); + + /// + /// Hash the given feature . + /// + /// String to be hashed. + /// Hash offset. + /// The resulting hash code. + /// The hash code depends on the vowpal wabbit instance as different has functions can be configured. + uint64_t HashFeature(String^ s, size_t u); + + /// + /// Return full topic allocation [topic, feature]. + /// + cli::array^>^ GetTopicAllocation(); + + /// + /// Return the topic weights. + /// + cli::array^>^ GetTopicAllocation(int top); + + /// + /// The associated instance learns from this example and returns the prediction result for this example. + /// + /// The prediction result. + /// The prediction result type. + generic T Learn(VowpalWabbitExample^ example, IVowpalWabbitPredictionFactory^ predictionFactory); + + /// + /// Predicts for the given example. + /// + /// The prediction type. + /// Example to predict for. + /// The prediction factory to be used. See . + /// The prediction for the given . + generic T Predict(VowpalWabbitExample^ example, IVowpalWabbitPredictionFactory^ predictionFactory); + + /// + /// Learns from the given example. + /// + /// Example to learn from. + void Learn(VowpalWabbitExample^ example); + + /// + /// Predicts for the given example. + /// + /// Example to predict for. + void Predict(VowpalWabbitExample^ example); + + /// + /// Learns from string data. + /// + /// Data in vw string format. + void Learn(String^ line); + + /// + /// Predicts for string data. + /// + /// Data in vw string format. + void Predict(String^ line); + + /// + /// Learns from string data. + /// + /// The prediction type. + /// Data in vw string format. + /// The prediction factory to be used. See . + /// The prediction for the given . + generic T Learn(String^ line, IVowpalWabbitPredictionFactory^ predictionFactory); + + /// + /// Predicts for string data. + /// + /// The prediction type. + /// Data in vw string format. + /// The prediction factory to be used. See . + /// The prediction for the given . + generic T Predict(String^ line, IVowpalWabbitPredictionFactory^ predictionFactory); + + /// + /// Learns from multi-line examples. + /// + /// Data in vw string format. + void Learn(IEnumerable^ lines); + + /// + /// Predicts for multi-line examples. + /// + /// Data in vw string format. + void Predict(IEnumerable^ lines); + + /// + /// Learns from multi-line examples. + /// + /// The prediction type. + /// Data in vw string format. + /// The prediction factory to be used. See . + /// The prediction for the given . + generic T Learn(IEnumerable^ lines, IVowpalWabbitPredictionFactory^ predictionFactory); + + /// + /// Predicts for the given lines. + /// + /// The prediction type. + /// Data in vw string format. + /// The prediction factory to be used. See . + /// The prediction for the given . + generic T Predict(IEnumerable^ lines, IVowpalWabbitPredictionFactory^ predictionFactory); + + /// + /// Signals the end of a pass. + /// + void EndOfPass(); + + /// + /// Invokes the driver. + /// + void Driver(); + + virtual property VowpalWabbit^ Native + { virtual VowpalWabbit^ get() sealed; + } + + /// + /// Gets or creates a native example from a CLR maintained, but natively allocated pool. + /// + /// A ready to use cleared native example data structure. + virtual VowpalWabbitExample^ GetOrCreateNativeExample() sealed; + + /// + /// Puts a native example data structure back into the pool. + /// + /// The example to be returned. + virtual void ReturnExampleToPool(VowpalWabbitExample^ example) sealed; +}; } diff --git a/cs/cli/vw_arguments.h b/cs/cli/vw_arguments.h index 6a55da5a989..a62283b8d7d 100644 --- a/cs/cli/vw_arguments.h +++ b/cs/cli/vw_arguments.h @@ -17,118 +17,102 @@ using namespace System::Collections::Generic; namespace VW { - /// - /// command line arguments extracted from native C++. - /// - public ref class VowpalWabbitArguments - { - private: - initonly String^ m_data; - initonly String^ m_finalRegressor; - const bool m_testonly; - const int m_passes; - List^ m_regressors; - String^ m_commandLine; - - int m_numberOfActions; - - internal: - VowpalWabbitArguments(vw* vw) : - m_data(gcnew String(vw->data_filename.c_str())), - m_finalRegressor(gcnew String(vw->final_regressor_name.c_str())), - m_testonly(!vw->training), - m_passes((int)vw->numpasses) - { - po::variables_map& vm = vw->vm; - if (vm.count("initial_regressor") || vm.count("i")) - { - m_regressors = gcnew List; - - vector regs = vm["initial_regressor"].as< vector >(); - for (auto& r : regs) - m_regressors->Add(gcnew String(r.c_str())); - } - - StringBuilder^ sb = gcnew StringBuilder(); - for (auto& s : vw->args) - sb->AppendFormat("{0} ", gcnew String(s.c_str())); - - m_commandLine = sb->ToString()->TrimEnd(); - - if (vw->vm.count("cb")) - m_numberOfActions = (int)vw->vm["cb"].as(); - } - - public: - /// - /// The input data file. - /// - property String^ Data - { - String^ get() - { - return m_data; - } - } - - /// - /// True if "-t" for test only mode supplied as part of arguments. - /// - property bool TestOnly - { - bool get() - { - return m_testonly; - } - } - - /// - /// Number of passes. - /// - property int NumPasses - { - int get() - { - return m_passes; - } - } - - /// - /// The output model filename. - /// - property String^ FinalRegressor - { - String^ get() - { - return m_finalRegressor; - } - } - - /// - ///The list of input model filenames. - /// - property List^ InitialRegressors - { - List^ get() - { - return m_regressors; - } - } - - property String^ CommandLine - { - String^ get() - { - return m_commandLine; - } - } - - property int ContextualBanditNumberOfActions - { - int get() - { - return m_numberOfActions; - } - } - }; +/// +/// command line arguments extracted from native C++. +/// +public ref class VowpalWabbitArguments +{ +private: + initonly String^ m_data; + initonly String^ m_finalRegressor; + const bool m_testonly; + const int m_passes; + List^ m_regressors; + String^ m_commandLine; + + int m_numberOfActions; + +internal: + VowpalWabbitArguments(vw* vw) : + m_data(gcnew String(vw->data_filename.c_str())), + m_finalRegressor(gcnew String(vw->final_regressor_name.c_str())), + m_testonly(!vw->training), + m_passes((int)vw->numpasses) + { po::variables_map& vm = vw->vm; + if (vm.count("initial_regressor") || vm.count("i")) + { m_regressors = gcnew List; + + vector regs = vm["initial_regressor"].as< vector >(); + for (auto& r : regs) + m_regressors->Add(gcnew String(r.c_str())); + } + + StringBuilder^ sb = gcnew StringBuilder(); + for (auto& s : vw->args) + sb->AppendFormat("{0} ", gcnew String(s.c_str())); + + m_commandLine = sb->ToString()->TrimEnd(); + + if (vw->vm.count("cb")) + m_numberOfActions = (int)vw->vm["cb"].as(); + } + +public: + /// + /// The input data file. + /// + property String^ Data + { String^ get() + { return m_data; + } + } + + /// + /// True if "-t" for test only mode supplied as part of arguments. + /// + property bool TestOnly + { bool get() + { return m_testonly; + } + } + + /// + /// Number of passes. + /// + property int NumPasses + { int get() + { return m_passes; + } + } + + /// + /// The output model filename. + /// + property String^ FinalRegressor + { String^ get() + { return m_finalRegressor; + } + } + + /// + ///The list of input model filenames. + /// + property List^ InitialRegressors + { List^ get() + { return m_regressors; + } + } + + property String^ CommandLine + { String^ get() + { return m_commandLine; + } + } + + property int ContextualBanditNumberOfActions + { int get() + { return m_numberOfActions; + } + } +}; } diff --git a/cs/cli/vw_base.cpp b/cs/cli/vw_base.cpp index e6fe5fe3e6e..f0aaacbe564 100644 --- a/cs/cli/vw_base.cpp +++ b/cs/cli/vw_base.cpp @@ -22,265 +22,226 @@ using namespace System::Text; namespace VW { - static VowpalWabbitBase::VowpalWabbitBase() - { - // make sure zlib.dll is loaded before anybody changes the current directory and we can't load anymore... - auto str = System::IO::Path::Combine(System::IO::Path::GetDirectoryName(VowpalWabbitBase::typeid->Assembly->Location), "zlib.dll"); - wstring path = msclr::interop::marshal_as(str); - LoadLibrary(path.c_str()); - } +static VowpalWabbitBase::VowpalWabbitBase() +{ // make sure zlib.dll is loaded before anybody changes the current directory and we can't load anymore... + auto str = System::IO::Path::Combine(System::IO::Path::GetDirectoryName(VowpalWabbitBase::typeid->Assembly->Location), "zlib.dll"); + wstring path = msclr::interop::marshal_as(str); + LoadLibrary(path.c_str()); +} + +VowpalWabbitBase::VowpalWabbitBase(VowpalWabbitSettings^ settings) + : m_examples(nullptr), m_vw(nullptr), m_model(nullptr), m_settings(settings != nullptr ? settings : gcnew VowpalWabbitSettings), m_instanceCount(0) +{ if (m_settings->EnableThreadSafeExamplePooling) + m_examples = Bag::CreateLockFree(); + else + m_examples = Bag::Create(m_settings->MaxExamples); + + try + { try + { std::string string; + if (settings->Arguments != nullptr) + string = msclr::interop::marshal_as(settings->Arguments); + + if (settings->Model != nullptr) + { m_model = settings->Model; + if (!settings->Verbose && !settings->Arguments->Contains("--quiet") && !m_model->Arguments->CommandLine->Contains("--quiet")) + string.append(" --quiet"); + m_vw = VW::seed_vw_model(m_model->m_vw, string); + m_model->IncrementReference(); + } + else + { if (settings->ModelStream == nullptr) + { if (!settings->Verbose && !settings->Arguments->Contains("--quiet")) + string.append(" --quiet"); - VowpalWabbitBase::VowpalWabbitBase(VowpalWabbitSettings^ settings) - : m_examples(nullptr), m_vw(nullptr), m_model(nullptr), m_settings(settings != nullptr ? settings : gcnew VowpalWabbitSettings), m_instanceCount(0) - { - if (m_settings->EnableThreadSafeExamplePooling) - m_examples = Bag::CreateLockFree(); + m_vw = VW::initialize(string); + } else - m_examples = Bag::Create(m_settings->MaxExamples); - - try - { - try - { - std::string string; - if (settings->Arguments != nullptr) - string = msclr::interop::marshal_as(settings->Arguments); - - if (settings->Model != nullptr) - { - m_model = settings->Model; - if (!settings->Verbose && !settings->Arguments->Contains("--quiet") && !m_model->Arguments->CommandLine->Contains("--quiet")) - string.append(" --quiet"); - m_vw = VW::seed_vw_model(m_model->m_vw, string); - m_model->IncrementReference(); - } - else - { - if (settings->ModelStream == nullptr) - { - if (!settings->Verbose && !settings->Arguments->Contains("--quiet")) - string.append(" --quiet"); - - m_vw = VW::initialize(string); - } - else - { - clr_io_buf model(settings->ModelStream); - if (!settings->Arguments->Contains("--no_stdin")) - string += " --no_stdin"; - m_vw = VW::initialize(string, &model); - settings->ModelStream->Close(); - } - } - } - catch (...) - { - // memory leak, but better than crashing - m_vw = nullptr; - throw; - } + { clr_io_buf model(settings->ModelStream); + if (!settings->Arguments->Contains("--no_stdin")) + string += " --no_stdin"; + m_vw = VW::initialize(string, &model); + settings->ModelStream->Close(); } - CATCHRETHROW + } } - - VowpalWabbitBase::~VowpalWabbitBase() - { - this->!VowpalWabbitBase(); + catch (...) + { // memory leak, but better than crashing + m_vw = nullptr; + throw; } + } + CATCHRETHROW +} - VowpalWabbitBase::!VowpalWabbitBase() - { - if (m_instanceCount <= 0) - { - this->InternalDispose(); - } - } +VowpalWabbitBase::~VowpalWabbitBase() +{ this->!VowpalWabbitBase(); +} - void VowpalWabbitBase::IncrementReference() - { - // thread-safe increase of model reference counter - System::Threading::Interlocked::Increment(m_instanceCount); - } +VowpalWabbitBase::!VowpalWabbitBase() +{ if (m_instanceCount <= 0) + { this->InternalDispose(); + } +} - void VowpalWabbitBase::DecrementReference() - { - // thread-safe decrease of model reference counter - if (System::Threading::Interlocked::Decrement(m_instanceCount) <= 0) - { - this->InternalDispose(); - } - } +void VowpalWabbitBase::IncrementReference() +{ // thread-safe increase of model reference counter + System::Threading::Interlocked::Increment(m_instanceCount); +} - void VowpalWabbitBase::DisposeExample(VowpalWabbitExample^ ex) - { - VW::dealloc_example(m_vw->p->lp.delete_label, *ex->m_example); - ::free_it(ex->m_example); - - // cleanup pointers in example chain - auto inner = ex; - while ((inner = inner->InnerExample) != nullptr) - { - inner->m_owner = nullptr; - inner->m_example = nullptr; - } +void VowpalWabbitBase::DecrementReference() +{ // thread-safe decrease of model reference counter + if (System::Threading::Interlocked::Decrement(m_instanceCount) <= 0) + { this->InternalDispose(); + } +} - ex->m_example = nullptr; +void VowpalWabbitBase::DisposeExample(VowpalWabbitExample^ ex) +{ VW::dealloc_example(m_vw->p->lp.delete_label, *ex->m_example); + ::free_it(ex->m_example); - // avoid that this example is returned again - ex->m_owner = nullptr; - } + // cleanup pointers in example chain + auto inner = ex; + while ((inner = inner->InnerExample) != nullptr) + { inner->m_owner = nullptr; + inner->m_example = nullptr; + } - void VowpalWabbitBase::InternalDispose() - { - if (m_vw != nullptr) - { - // de-allocate example pools that are managed for each even shared instances - if (m_examples != nullptr) - { - for each (auto ex in m_examples->RemoveAll()) - DisposeExample(ex); - - m_examples = nullptr; - } - - if (m_model != nullptr) - { - // this object doesn't own the VW instance - m_model->DecrementReference(); - m_model = nullptr; - } - } + ex->m_example = nullptr; - try - { - if (m_vw != nullptr) - { - reset_source(*m_vw, m_vw->num_bits); - release_parser_datastructures(*m_vw); + // avoid that this example is returned again + ex->m_owner = nullptr; +} - // make sure don't try to free m_vw twice in case VW::finish throws. - vw* vw_tmp = m_vw; - m_vw = nullptr; - VW::finish(*vw_tmp); - } +void VowpalWabbitBase::InternalDispose() +{ if (m_vw != nullptr) + { // de-allocate example pools that are managed for each even shared instances + if (m_examples != nullptr) + { for each (auto ex in m_examples->RemoveAll()) + DisposeExample(ex); - // don't add code here as in the case of VW::finish throws an exception it won't be called - } - CATCHRETHROW + m_examples = nullptr; } - VowpalWabbitSettings^ VowpalWabbitBase::Settings::get() - { - return m_settings; + if (m_model != nullptr) + { // this object doesn't own the VW instance + m_model->DecrementReference(); + m_model = nullptr; } + } - VowpalWabbitArguments^ VowpalWabbitBase::Arguments::get() - { - if (m_arguments == nullptr) - { - m_arguments = gcnew VowpalWabbitArguments(m_vw); - } + try + { if (m_vw != nullptr) + { reset_source(*m_vw, m_vw->num_bits); + release_parser_datastructures(*m_vw); - return m_arguments; + // make sure don't try to free m_vw twice in case VW::finish throws. + vw* vw_tmp = m_vw; + m_vw = nullptr; + VW::finish(*vw_tmp); } - void VowpalWabbitBase::Reload([System::Runtime::InteropServices::Optional] String^ args) - { - if (m_settings->ParallelOptions != nullptr) - { - throw gcnew NotSupportedException("Cannot reload model if AllRecude is enabled."); - } + // don't add code here as in the case of VW::finish throws an exception it won't be called + } + CATCHRETHROW +} - clr_io_memory_buf mem_buf; +VowpalWabbitSettings^ VowpalWabbitBase::Settings::get() +{ return m_settings; +} - if (args == nullptr) - args = String::Empty; +VowpalWabbitArguments^ VowpalWabbitBase::Arguments::get() +{ if (m_arguments == nullptr) + { m_arguments = gcnew VowpalWabbitArguments(m_vw); + } - auto stringArgs = msclr::interop::marshal_as(args); + return m_arguments; +} - try - { - reset_source(*m_vw, m_vw->num_bits); +void VowpalWabbitBase::Reload([System::Runtime::InteropServices::Optional] String^ args) +{ if (m_settings->ParallelOptions != nullptr) + { throw gcnew NotSupportedException("Cannot reload model if AllRecude is enabled."); + } - VW::save_predictor(*m_vw, mem_buf); - mem_buf.flush(); + clr_io_memory_buf mem_buf; - release_parser_datastructures(*m_vw); + if (args == nullptr) + args = String::Empty; - // make sure don't try to free m_vw twice in case VW::finish throws. - vw* vw_tmp = m_vw; - m_vw = nullptr; - VW::finish(*vw_tmp); + auto stringArgs = msclr::interop::marshal_as(args); - // reload from model - // seek to beginning - mem_buf.reset_file(0); - m_vw = VW::initialize(stringArgs.c_str(), &mem_buf); - } - CATCHRETHROW - } + try + { reset_source(*m_vw, m_vw->num_bits); - String^ VowpalWabbitBase::AreFeaturesCompatible(VowpalWabbitBase^ other) - { - auto diff = VW::are_features_compatible(*m_vw, *other->m_vw); + VW::save_predictor(*m_vw, mem_buf); + mem_buf.flush(); - return diff == nullptr ? nullptr : gcnew String(diff); - } + release_parser_datastructures(*m_vw); - String^ VowpalWabbitBase::ID::get() - { - return gcnew String(m_vw->id.c_str()); - } + // make sure don't try to free m_vw twice in case VW::finish throws. + vw* vw_tmp = m_vw; + m_vw = nullptr; + VW::finish(*vw_tmp); - void VowpalWabbitBase::ID::set(String^ value) - { - m_vw->id = msclr::interop::marshal_as(value); - } + // reload from model + // seek to beginning + mem_buf.reset_file(0); + m_vw = VW::initialize(stringArgs.c_str(), &mem_buf); + } + CATCHRETHROW +} - void VowpalWabbitBase::SaveModel() - { - string name = m_vw->final_regressor_name; - if (name.empty()) - { - return; - } +String^ VowpalWabbitBase::AreFeaturesCompatible(VowpalWabbitBase^ other) +{ auto diff = VW::are_features_compatible(*m_vw, *other->m_vw); - // this results in extra marshaling but should be fine here - this->SaveModel(gcnew String(name.c_str())); - } + return diff == nullptr ? nullptr : gcnew String(diff); +} - void VowpalWabbitBase::SaveModel(String^ filename) - { - if (String::IsNullOrEmpty(filename)) - throw gcnew ArgumentException("Filename must not be null or empty"); +String^ VowpalWabbitBase::ID::get() +{ return gcnew String(m_vw->id.c_str()); +} - String^ directoryName = System::IO::Path::GetDirectoryName(filename); +void VowpalWabbitBase::ID::set(String^ value) +{ m_vw->id = msclr::interop::marshal_as(value); +} - if (!String::IsNullOrEmpty(directoryName)) - { - System::IO::Directory::CreateDirectory(directoryName); - } +void VowpalWabbitBase::SaveModel() +{ string name = m_vw->final_regressor_name; + if (name.empty()) + { return; + } - auto name = msclr::interop::marshal_as(filename); + // this results in extra marshaling but should be fine here + this->SaveModel(gcnew String(name.c_str())); +} - try - { - VW::save_predictor(*m_vw, name); - } - CATCHRETHROW - } +void VowpalWabbitBase::SaveModel(String^ filename) +{ if (String::IsNullOrEmpty(filename)) + throw gcnew ArgumentException("Filename must not be null or empty"); - void VowpalWabbitBase::SaveModel(Stream^ stream) - { - if (stream == nullptr) - throw gcnew ArgumentException("stream"); + String^ directoryName = System::IO::Path::GetDirectoryName(filename); - try - { - VW::clr_io_buf buf(stream); + if (!String::IsNullOrEmpty(directoryName)) + { System::IO::Directory::CreateDirectory(directoryName); + } - VW::save_predictor(*m_vw, buf); - } - CATCHRETHROW - } + auto name = msclr::interop::marshal_as(filename); + + try + { VW::save_predictor(*m_vw, name); + } + CATCHRETHROW +} + +void VowpalWabbitBase::SaveModel(Stream^ stream) +{ if (stream == nullptr) + throw gcnew ArgumentException("stream"); + + try + { VW::clr_io_buf buf(stream); + + VW::save_predictor(*m_vw, buf); + } + CATCHRETHROW +} } diff --git a/cs/cli/vw_base.h b/cs/cli/vw_base.h index 42f4db63a68..306bd4884bb 100644 --- a/cs/cli/vw_base.h +++ b/cs/cli/vw_base.h @@ -16,151 +16,148 @@ using namespace System::Collections::Generic; namespace VW { - ref class VowpalWabbitPrediction; - ref class VowpalWabbitModel; - - /// - /// A base wrapper around vowpal wabbit machine learning instance. - /// - /// - /// Since the model class must delay diposal of until all referencing - /// VowpalWabbit instances are disposed, the base class does not dispose . - /// - public ref class VowpalWabbitBase abstract - { - private: - /// - /// The settings used for this instance. - /// - initonly VowpalWabbitSettings^ m_settings; - - /// - /// An optional shared model. - /// - VowpalWabbitModel^ m_model; - - /// - /// Extracted command line arguments. - /// - VowpalWabbitArguments^ m_arguments; - - /// - /// Reference count to native data structure. - /// - System::Int32 m_instanceCount; - - internal: - /// - /// The native vowpal wabbit data structure. - /// - vw* m_vw; - - /// - /// Thread-safe increment of reference count. - /// - void IncrementReference(); - - /// - /// Thread-safe decrement of reference count. - /// - void DecrementReference(); - - protected: - /// - /// True if all nativedata structures are disposed. - /// - bool m_isDisposed; - - /// - /// Example pool. Kept in base to simplify deallocation. - /// - IBag^ m_examples; - - /// - /// Initializes a new instance. - /// - /// Command line arguments. - VowpalWabbitBase(VowpalWabbitSettings^ settings); - - /// - /// Cleanup. - /// - !VowpalWabbitBase(); - - /// - /// Internal dipose using reference counting to delay disposal of shared native data structures. - /// - void InternalDispose(); - - void DisposeExample(VowpalWabbitExample^ ex); - - public: - static VowpalWabbitBase(); - - /// - /// Cleanup. - /// - virtual ~VowpalWabbitBase(); - - /// - /// The settings used for this instance. - /// - property VowpalWabbitSettings^ Settings - { - VowpalWabbitSettings^ get(); - } - - /// - /// Extracted command line arguments. - /// - property VowpalWabbitArguments^ Arguments - { - VowpalWabbitArguments^ get(); - } - - /// - /// The read/writable model id. - /// - property String^ ID - { - String^ get(); - void set(String^ id); - } - - /// - /// Performs the following steps to reset the learning state: - /// - /// - Save model to in-memory buffer - /// - Dispose existing instance - /// - Initialize new instance from in-memory buffer - /// - void Reload([System::Runtime::InteropServices::Optional] String^ args); - - /// - /// Compares features created by current instance are compatible to features created by . - /// - /// - /// Null if compatible, otherwise the difference - /// - String^ AreFeaturesCompatible(VowpalWabbitBase^ other); - - - /// - /// Persist model to file specified by -i. - /// - void SaveModel(); - - /// - /// Persist model to . - /// - /// The destination filename for the model. - void SaveModel(String^ filename); - - /// - /// Persist model to . - /// - /// The destination stream for the model. - /// The stream is not closed to support embedded schemes. - void SaveModel(Stream^ stream); - }; +ref class VowpalWabbitPrediction; +ref class VowpalWabbitModel; + +/// +/// A base wrapper around vowpal wabbit machine learning instance. +/// +/// +/// Since the model class must delay diposal of until all referencing +/// VowpalWabbit instances are disposed, the base class does not dispose . +/// +public ref class VowpalWabbitBase abstract +{ +private: + /// + /// The settings used for this instance. + /// + initonly VowpalWabbitSettings^ m_settings; + + /// + /// An optional shared model. + /// + VowpalWabbitModel^ m_model; + + /// + /// Extracted command line arguments. + /// + VowpalWabbitArguments^ m_arguments; + + /// + /// Reference count to native data structure. + /// + System::Int32 m_instanceCount; + +internal: + /// + /// The native vowpal wabbit data structure. + /// + vw* m_vw; + + /// + /// Thread-safe increment of reference count. + /// + void IncrementReference(); + + /// + /// Thread-safe decrement of reference count. + /// + void DecrementReference(); + +protected: + /// + /// True if all nativedata structures are disposed. + /// + bool m_isDisposed; + + /// + /// Example pool. Kept in base to simplify deallocation. + /// + IBag^ m_examples; + + /// + /// Initializes a new instance. + /// + /// Command line arguments. + VowpalWabbitBase(VowpalWabbitSettings^ settings); + + /// + /// Cleanup. + /// + !VowpalWabbitBase(); + + /// + /// Internal dipose using reference counting to delay disposal of shared native data structures. + /// + void InternalDispose(); + + void DisposeExample(VowpalWabbitExample^ ex); + +public: + static VowpalWabbitBase(); + + /// + /// Cleanup. + /// + virtual ~VowpalWabbitBase(); + + /// + /// The settings used for this instance. + /// + property VowpalWabbitSettings^ Settings + { VowpalWabbitSettings^ get(); + } + + /// + /// Extracted command line arguments. + /// + property VowpalWabbitArguments^ Arguments + { VowpalWabbitArguments^ get(); + } + + /// + /// The read/writable model id. + /// + property String^ ID + { String^ get(); + void set(String^ id); + } + + /// + /// Performs the following steps to reset the learning state: + /// + /// - Save model to in-memory buffer + /// - Dispose existing instance + /// - Initialize new instance from in-memory buffer + /// + void Reload([System::Runtime::InteropServices::Optional] String^ args); + + /// + /// Compares features created by current instance are compatible to features created by . + /// + /// + /// Null if compatible, otherwise the difference + /// + String^ AreFeaturesCompatible(VowpalWabbitBase^ other); + + + /// + /// Persist model to file specified by -i. + /// + void SaveModel(); + + /// + /// Persist model to . + /// + /// The destination filename for the model. + void SaveModel(String^ filename); + + /// + /// Persist model to . + /// + /// The destination stream for the model. + /// The stream is not closed to support embedded schemes. + void SaveModel(Stream^ stream); +}; } diff --git a/cs/cli/vw_builder.cpp b/cs/cli/vw_builder.cpp index a3149c6725a..0aff8bee8fb 100644 --- a/cs/cli/vw_builder.cpp +++ b/cs/cli/vw_builder.cpp @@ -9,130 +9,112 @@ license as described in the file LICENSE. namespace VW { - VowpalWabbitExampleBuilder::VowpalWabbitExampleBuilder(IVowpalWabbitExamplePool^ vw) : - m_vw(vw), m_example(nullptr) - { - if (vw == nullptr) - throw gcnew ArgumentNullException("vw"); +VowpalWabbitExampleBuilder::VowpalWabbitExampleBuilder(IVowpalWabbitExamplePool^ vw) : + m_vw(vw), m_example(nullptr) +{ if (vw == nullptr) + throw gcnew ArgumentNullException("vw"); - m_example = vw->GetOrCreateNativeExample(); - } + m_example = vw->GetOrCreateNativeExample(); +} - VowpalWabbitExampleBuilder::~VowpalWabbitExampleBuilder() - { - this->!VowpalWabbitExampleBuilder(); - } +VowpalWabbitExampleBuilder::~VowpalWabbitExampleBuilder() +{ this->!VowpalWabbitExampleBuilder(); +} - VowpalWabbitExampleBuilder::!VowpalWabbitExampleBuilder() - { - if (m_example != nullptr) - { - // in case CreateExample is not getting called - delete m_example; +VowpalWabbitExampleBuilder::!VowpalWabbitExampleBuilder() +{ if (m_example != nullptr) + { // in case CreateExample is not getting called + delete m_example; - m_example = nullptr; - } - } + m_example = nullptr; + } +} - VowpalWabbitExample^ VowpalWabbitExampleBuilder::CreateExample() - { - if (m_example == nullptr) - return nullptr; +VowpalWabbitExample^ VowpalWabbitExampleBuilder::CreateExample() +{ if (m_example == nullptr) + return nullptr; - try - { - // finalize example - VW::setup_example(*m_vw->Native->m_vw, m_example->m_example); - } - CATCHRETHROW + try + { // finalize example + VW::setup_example(*m_vw->Native->m_vw, m_example->m_example); + } + CATCHRETHROW - // hand memory management off to VowpalWabbitExample - auto ret = m_example; - m_example = nullptr; + // hand memory management off to VowpalWabbitExample + auto ret = m_example; + m_example = nullptr; - return ret; - } + return ret; +} - void VowpalWabbitExampleBuilder::ApplyLabel(ILabel^ label) - { - if (label == nullptr) - return; +void VowpalWabbitExampleBuilder::ApplyLabel(ILabel^ label) +{ if (label == nullptr) + return; - label->UpdateExample(m_vw->Native->m_vw, m_example->m_example); - } + label->UpdateExample(m_vw->Native->m_vw, m_example->m_example); +} - VowpalWabbitNamespaceBuilder^ VowpalWabbitExampleBuilder::AddNamespace(Char featureGroup) - { - return AddNamespace((Byte)featureGroup); - } +VowpalWabbitNamespaceBuilder^ VowpalWabbitExampleBuilder::AddNamespace(Char featureGroup) +{ return AddNamespace((Byte)featureGroup); +} - VowpalWabbitNamespaceBuilder^ VowpalWabbitExampleBuilder::AddNamespace(Byte featureGroup) - { - uint32_t index = featureGroup; - example* ex = m_example->m_example; +VowpalWabbitNamespaceBuilder^ VowpalWabbitExampleBuilder::AddNamespace(Byte featureGroup) +{ uint32_t index = featureGroup; + example* ex = m_example->m_example; - return gcnew VowpalWabbitNamespaceBuilder(ex->feature_space + index, featureGroup, m_example->m_example); - } + return gcnew VowpalWabbitNamespaceBuilder(ex->feature_space + index, featureGroup, m_example->m_example); +} - VowpalWabbitNamespaceBuilder::VowpalWabbitNamespaceBuilder(features* features, - unsigned char index, example* example) - : m_features(features), m_index(index), m_example(example) - { - } +VowpalWabbitNamespaceBuilder::VowpalWabbitNamespaceBuilder(features* features, + unsigned char index, example* example) + : m_features(features), m_index(index), m_example(example) +{ +} - VowpalWabbitNamespaceBuilder::~VowpalWabbitNamespaceBuilder() - { - this->!VowpalWabbitNamespaceBuilder(); - } +VowpalWabbitNamespaceBuilder::~VowpalWabbitNamespaceBuilder() +{ this->!VowpalWabbitNamespaceBuilder(); +} - VowpalWabbitNamespaceBuilder::!VowpalWabbitNamespaceBuilder() - { - if (m_features->size() > 0) - { - unsigned char temp = m_index; - - // avoid duplicate insertion - // can't check at the beginning, because multiple builders can be open - // at the same time - for (unsigned char ns : m_example->indices) - if (ns == temp) - return; - - m_example->indices.push_back(temp); - } - } +VowpalWabbitNamespaceBuilder::!VowpalWabbitNamespaceBuilder() +{ if (m_features->size() > 0) + { unsigned char temp = m_index; - void VowpalWabbitNamespaceBuilder::AddFeaturesUnchecked(uint64_t weight_index_base, float* begin, float* end) - { - for (; begin != end; begin++) - { - float x = *begin; - if (x != 0) - { - m_features->values.push_back_unchecked(x); - m_features->indicies.push_back_unchecked(weight_index_base); - } - weight_index_base++; - } - } + // avoid duplicate insertion + // can't check at the beginning, because multiple builders can be open + // at the same time + for (unsigned char ns : m_example->indices) + if (ns == temp) + return; - void VowpalWabbitNamespaceBuilder::AddFeature(uint64_t weight_index, float x) - { - // filter out 0-values - if (x == 0) - return; + m_example->indices.push_back(temp); + } +} - m_features->push_back(x, weight_index); +void VowpalWabbitNamespaceBuilder::AddFeaturesUnchecked(uint64_t weight_index_base, float* begin, float* end) +{ for (; begin != end; begin++) + { float x = *begin; + if (x != 0) + { m_features->values.push_back_unchecked(x); + m_features->indicies.push_back_unchecked(weight_index_base); } + weight_index_base++; + } +} - void VowpalWabbitNamespaceBuilder::PreAllocate(int size) - { - m_features->values.resize(m_features->values.end() - m_features->values.begin() + size); - m_features->indicies.resize(m_features->indicies.end() - m_features->indicies.begin() + size); - } +void VowpalWabbitNamespaceBuilder::AddFeature(uint64_t weight_index, float x) +{ // filter out 0-values + if (x == 0) + return; - size_t VowpalWabbitNamespaceBuilder::FeatureCount::get() - { - return m_features->size(); - } + m_features->push_back(x, weight_index); +} + +void VowpalWabbitNamespaceBuilder::PreAllocate(int size) +{ m_features->values.resize(m_features->values.end() - m_features->values.begin() + size); + m_features->indicies.resize(m_features->indicies.end() - m_features->indicies.begin() + size); +} + +size_t VowpalWabbitNamespaceBuilder::FeatureCount::get() +{ return m_features->size(); +} } diff --git a/cs/cli/vw_builder.h b/cs/cli/vw_builder.h index e88a2f46847..79e76687165 100644 --- a/cs/cli/vw_builder.h +++ b/cs/cli/vw_builder.h @@ -14,121 +14,121 @@ license as described in the file LICENSE. namespace VW { - using namespace VW::Labels; - - /// - /// Helper class to ease construction of native vowpal wabbit namespace data structure. - /// - public ref class VowpalWabbitNamespaceBuilder sealed - { - private: - /// - /// Features. - /// - features* m_features; - - /// - /// The namespace index. - /// - unsigned char m_index; - - /// - /// The native example. - /// - example* m_example; - - // float(*m_sum_of_squares)(float*, float*); - - !VowpalWabbitNamespaceBuilder(); - - internal: - /// - /// Initializes a new instance. - /// - /// Pointer into features owned by . - /// The namespace index. - /// The native example to build up. - VowpalWabbitNamespaceBuilder(features* features, unsigned char index, example* example); - - public: - ~VowpalWabbitNamespaceBuilder(); - - /// - /// Add feature entry. - /// - /// The weight index. - /// The value. - void AddFeature(uint64_t weight_index, float x); - - /// - /// Adds a dense array to the example. - /// - /// The base weight index. Each element is then placed relative to this index. - /// The start pointer of the float array. - /// The end pointer of the float array. - void AddFeaturesUnchecked(uint64_t weight_index_base, float* begin, float* end); - - /// - /// Pre-allocate features of . - /// - /// The number of features to pre-allocate. - void PreAllocate(int size); - - property size_t FeatureCount { size_t get(); } - }; - - /// - /// Helper class to ease construction of native vowpal wabbit example data structure. - /// - public ref class VowpalWabbitExampleBuilder sealed - { - private: - IVowpalWabbitExamplePool^ m_vw; - - /// - /// The produced CLR example data structure. - /// - VowpalWabbitExample^ m_example; - - protected: - /// - /// Cleanup. - /// - !VowpalWabbitExampleBuilder(); - - public: - /// - /// Initializes a new instance. - /// - /// The parent vowpal wabbit instance. - VowpalWabbitExampleBuilder(IVowpalWabbitExamplePool^ vw); - - /// - /// Cleanup. - /// - ~VowpalWabbitExampleBuilder(); - - /// - /// Creates the managed example representation. - /// - /// Creates the managed example. - VowpalWabbitExample^ CreateExample(); - - /// - /// Sets the label for the resulting example. - /// - void ApplyLabel(ILabel^ label); - - /// - /// Creates and adds a new namespace to this example. - /// - VowpalWabbitNamespaceBuilder^ AddNamespace(Byte featureGroup); - - /// - /// Creates and adds a new namespace to this example. - /// - /// The feature group of the new namespace. - /// Casts to System::Byte. - VowpalWabbitNamespaceBuilder^ AddNamespace(Char featureGroup); - }; +using namespace VW::Labels; + +/// +/// Helper class to ease construction of native vowpal wabbit namespace data structure. +/// +public ref class VowpalWabbitNamespaceBuilder sealed +{ +private: + /// + /// Features. + /// + features* m_features; + + /// + /// The namespace index. + /// + unsigned char m_index; + + /// + /// The native example. + /// + example* m_example; + + // float(*m_sum_of_squares)(float*, float*); + + !VowpalWabbitNamespaceBuilder(); + +internal: + /// + /// Initializes a new instance. + /// + /// Pointer into features owned by . + /// The namespace index. + /// The native example to build up. + VowpalWabbitNamespaceBuilder(features* features, unsigned char index, example* example); + +public: + ~VowpalWabbitNamespaceBuilder(); + + /// + /// Add feature entry. + /// + /// The weight index. + /// The value. + void AddFeature(uint64_t weight_index, float x); + + /// + /// Adds a dense array to the example. + /// + /// The base weight index. Each element is then placed relative to this index. + /// The start pointer of the float array. + /// The end pointer of the float array. + void AddFeaturesUnchecked(uint64_t weight_index_base, float* begin, float* end); + + /// + /// Pre-allocate features of . + /// + /// The number of features to pre-allocate. + void PreAllocate(int size); + + property size_t FeatureCount { size_t get(); } +}; + +/// +/// Helper class to ease construction of native vowpal wabbit example data structure. +/// +public ref class VowpalWabbitExampleBuilder sealed +{ +private: + IVowpalWabbitExamplePool^ m_vw; + + /// + /// The produced CLR example data structure. + /// + VowpalWabbitExample^ m_example; + +protected: + /// + /// Cleanup. + /// + !VowpalWabbitExampleBuilder(); + +public: + /// + /// Initializes a new instance. + /// + /// The parent vowpal wabbit instance. + VowpalWabbitExampleBuilder(IVowpalWabbitExamplePool^ vw); + + /// + /// Cleanup. + /// + ~VowpalWabbitExampleBuilder(); + + /// + /// Creates the managed example representation. + /// + /// Creates the managed example. + VowpalWabbitExample^ CreateExample(); + + /// + /// Sets the label for the resulting example. + /// + void ApplyLabel(ILabel^ label); + + /// + /// Creates and adds a new namespace to this example. + /// + VowpalWabbitNamespaceBuilder^ AddNamespace(Byte featureGroup); + + /// + /// Creates and adds a new namespace to this example. + /// + /// The feature group of the new namespace. + /// Casts to System::Byte. + VowpalWabbitNamespaceBuilder^ AddNamespace(Char featureGroup); +}; } diff --git a/cs/cli/vw_cbutil.cpp b/cs/cli/vw_cbutil.cpp index d3f6649418f..18230303e25 100644 --- a/cs/cli/vw_cbutil.cpp +++ b/cs/cli/vw_cbutil.cpp @@ -9,10 +9,9 @@ license as described in the file LICENSE. namespace VW { - float VowpalWabbitContextualBanditUtil::GetUnbiasedCost(uint32_t actionObservered, uint32_t actionTaken, float cost, float probability) - { - CB::cb_class observation = { cost, actionObservered, probability }; +float VowpalWabbitContextualBanditUtil::GetUnbiasedCost(uint32_t actionObservered, uint32_t actionTaken, float cost, float probability) +{ CB::cb_class observation = { cost, actionObservered, probability }; - return CB_ALGS::get_unbiased_cost(&observation, actionTaken); - } + return CB_ALGS::get_unbiased_cost(&observation, actionTaken); +} } diff --git a/cs/cli/vw_cbutil.h b/cs/cli/vw_cbutil.h index 1469a6feea8..e3d1d201d98 100644 --- a/cs/cli/vw_cbutil.h +++ b/cs/cli/vw_cbutil.h @@ -10,9 +10,9 @@ license as described in the file LICENSE. namespace VW { - public ref class VowpalWabbitContextualBanditUtil abstract sealed - { - public: - static float GetUnbiasedCost(uint32_t actionObservered, uint32_t actionTaken, float cost, float probability); - }; +public ref class VowpalWabbitContextualBanditUtil abstract sealed +{ +public: + static float GetUnbiasedCost(uint32_t actionObservered, uint32_t actionTaken, float cost, float probability); +}; } diff --git a/cs/cli/vw_example.cpp b/cs/cli/vw_example.cpp index 3322a9ba189..4319b6d2aa0 100644 --- a/cs/cli/vw_example.cpp +++ b/cs/cli/vw_example.cpp @@ -11,517 +11,450 @@ license as described in the file LICENSE. namespace VW { - using namespace Labels; +using namespace Labels; - VowpalWabbitExample::VowpalWabbitExample(IVowpalWabbitExamplePool^ owner, example* example) : - m_owner(owner), m_example(example), m_innerExample(nullptr) - { - } +VowpalWabbitExample::VowpalWabbitExample(IVowpalWabbitExamplePool^ owner, example* example) : + m_owner(owner), m_example(example), m_innerExample(nullptr) +{ +} - VowpalWabbitExample::VowpalWabbitExample(IVowpalWabbitExamplePool^ owner, VowpalWabbitExample^ example) : - m_owner(owner), m_example(example->m_example), m_innerExample(example), m_string(example->m_string) - { - } +VowpalWabbitExample::VowpalWabbitExample(IVowpalWabbitExamplePool^ owner, VowpalWabbitExample^ example) : + m_owner(owner), m_example(example->m_example), m_innerExample(example), m_string(example->m_string) +{ +} - VowpalWabbitExample::!VowpalWabbitExample() - { - if (m_owner != nullptr) - m_owner->ReturnExampleToPool(this); - } +VowpalWabbitExample::!VowpalWabbitExample() +{ if (m_owner != nullptr) + m_owner->ReturnExampleToPool(this); +} - VowpalWabbitExample::~VowpalWabbitExample() - { - this->!VowpalWabbitExample(); - } +VowpalWabbitExample::~VowpalWabbitExample() +{ this->!VowpalWabbitExample(); +} - VowpalWabbitExample^ VowpalWabbitExample::InnerExample::get() - { - return m_innerExample; - } +VowpalWabbitExample^ VowpalWabbitExample::InnerExample::get() +{ return m_innerExample; +} - IVowpalWabbitExamplePool^ VowpalWabbitExample::Owner::get() - { - return m_owner; - } +IVowpalWabbitExamplePool^ VowpalWabbitExample::Owner::get() +{ return m_owner; +} - size_t VowpalWabbitExample::NumberOfFeatures::get() - { - return m_example->num_features; - } +size_t VowpalWabbitExample::NumberOfFeatures::get() +{ return m_example->num_features; +} - generic T VowpalWabbitExample::GetPrediction(VowpalWabbit^ vw, IVowpalWabbitPredictionFactory^ factory) - { +generic T VowpalWabbitExample::GetPrediction(VowpalWabbit^ vw, IVowpalWabbitPredictionFactory^ factory) +{ #ifdef _DEBUG - if (vw == nullptr) - throw gcnew ArgumentNullException("vw"); + if (vw == nullptr) + throw gcnew ArgumentNullException("vw"); #endif - return factory->Create(vw->m_vw, m_example); - } + return factory->Create(vw->m_vw, m_example); +} - String^ VowpalWabbitExample::VowpalWabbitString::get() - { - return m_string; - } +String^ VowpalWabbitExample::VowpalWabbitString::get() +{ return m_string; +} - void VowpalWabbitExample::VowpalWabbitString::set(String^ value) - { - m_string = value; - } +void VowpalWabbitExample::VowpalWabbitString::set(String^ value) +{ m_string = value; +} - bool VowpalWabbitExample::IsNewLine::get() - { - return example_is_newline(*m_example) != 0; - } +bool VowpalWabbitExample::IsNewLine::get() +{ return example_is_newline(*m_example) != 0; +} - ILabel^ VowpalWabbitExample::Label::get() - { - ILabel^ label; - auto lp = m_owner->Native->m_vw->p->lp; - if (!memcmp(&lp, &simple_label, sizeof(lp))) - label = gcnew SimpleLabel(); - else if (!memcmp(&lp, &CB::cb_label, sizeof(lp))) - label = gcnew ContextualBanditLabel(); - else if (!memcmp(&lp, &CB_EVAL::cb_eval, sizeof(lp))) - label = gcnew SimpleLabel(); - else if (!memcmp(&lp, &COST_SENSITIVE::cs_label, sizeof(lp))) - label = gcnew SimpleLabel(); - else - return nullptr; - - // TODO: - //else if (!memcmp(&lp, &MULTICLASS::multilabel, sizeof(lp))) - // label = gcnew MulticlassLabel; - //else if (!memcmp(&lp, &MC::multilabel, sizeof(lp))) - - label->ReadFromExample(this->m_example); - - return label; - } +ILabel^ VowpalWabbitExample::Label::get() +{ ILabel^ label; + auto lp = m_owner->Native->m_vw->p->lp; + if (!memcmp(&lp, &simple_label, sizeof(lp))) + label = gcnew SimpleLabel(); + else if (!memcmp(&lp, &CB::cb_label, sizeof(lp))) + label = gcnew ContextualBanditLabel(); + else if (!memcmp(&lp, &CB_EVAL::cb_eval, sizeof(lp))) + label = gcnew SimpleLabel(); + else if (!memcmp(&lp, &COST_SENSITIVE::cs_label, sizeof(lp))) + label = gcnew SimpleLabel(); + else + return nullptr; + + // TODO: + //else if (!memcmp(&lp, &MULTICLASS::multilabel, sizeof(lp))) + // label = gcnew MulticlassLabel; + //else if (!memcmp(&lp, &MC::multilabel, sizeof(lp))) + + label->ReadFromExample(this->m_example); + + return label; +} - void VowpalWabbitExample::MakeEmpty(VowpalWabbit^ vw) - { - char empty = '\0'; - VW::read_line(*vw->m_vw, m_example, &empty); +void VowpalWabbitExample::MakeEmpty(VowpalWabbit^ vw) +{ char empty = '\0'; + VW::read_line(*vw->m_vw, m_example, &empty); - VW::setup_example(*vw->m_vw, m_example); - } + VW::setup_example(*vw->m_vw, m_example); +} - void FormatIndices(example* a, System::Text::StringBuilder^ sb) - { - for (auto ns : a->indices) - { - if (ns == 0) - sb->Append("NULL:0,"); - else - sb->AppendFormat("'{0}':{1},", gcnew System::Char(ns), (int)ns); - } - } +void FormatIndices(example* a, System::Text::StringBuilder^ sb) +{ for (auto ns : a->indices) + { if (ns == 0) + sb->Append("NULL:0,"); + else + sb->AppendFormat("'{0}':{1},", gcnew System::Char(ns), (int)ns); + } +} - System::String^ FormatIndices(example* a, example *b) - { - auto sb = gcnew System::Text::StringBuilder(); +System::String^ FormatIndices(example* a, example *b) +{ auto sb = gcnew System::Text::StringBuilder(); - sb->AppendFormat("Namespace indicies differ: {0} vs {1}. this.indices: [", - a->indices.size(), - b->indices.size()); + sb->AppendFormat("Namespace indicies differ: {0} vs {1}. this.indices: [", + a->indices.size(), + b->indices.size()); - FormatIndices(a, sb); + FormatIndices(a, sb); - sb->Append("] other.indices: ["); + sb->Append("] other.indices: ["); - FormatIndices(b, sb); + FormatIndices(b, sb); - sb->Append("]"); + sb->Append("]"); - return sb->ToString(); - } + return sb->ToString(); +} - System::String^ FormatFeature(vw* vw, feature_value& f1, feature_index& i1) - { - uint64_t masked_weight_index1 = i1 & vw->weights.mask(); +System::String^ FormatFeature(vw* vw, feature_value& f1, feature_index& i1) +{ uint64_t masked_weight_index1 = i1 & vw->weights.mask(); - return System::String::Format( - "weight_index = {0}/{1}, x = {2}", - masked_weight_index1, - i1, - gcnew System::Single(f1)); - } + return System::String::Format( + "weight_index = {0}/{1}, x = {2}", + masked_weight_index1, + i1, + gcnew System::Single(f1)); +} - System::String^ FormatFeature(vw* vw, feature_value& f1, feature_index& i1, feature_value& f2, feature_index& i2) - { - return System::String::Format( - "Feature differ: this({0}) vs other({1})", - FormatFeature(vw, f1, i1), - FormatFeature(vw, f2, i2)); - } +System::String^ FormatFeature(vw* vw, feature_value& f1, feature_index& i1, feature_value& f2, feature_index& i2) +{ return System::String::Format( + "Feature differ: this({0}) vs other({1})", + FormatFeature(vw, f1, i1), + FormatFeature(vw, f2, i2)); +} - bool FloatEqual(float a, float b) - { - if ((abs(a) < 1e-20 && abs(b) < 1e-20) || - (isinf(a) && isinf(b))) - { - return true; - } +bool FloatEqual(float a, float b) +{ if ((abs(a) < 1e-20 && abs(b) < 1e-20) || + (isinf(a) && isinf(b))) + { return true; + } - return abs(a - b) / max(a, b) < 1e-6; - } + return abs(a - b) / max(a, b) < 1e-6; +} - System::String^ FormatFeatures(vw* vw, features& arr) - { - auto sb = gcnew System::Text::StringBuilder(); - for (size_t i = 0; i < arr.values.size(); i++) - { - sb->Append(FormatFeature(vw, arr.values[i], arr.indicies[i]))->Append(" "); - } +System::String^ FormatFeatures(vw* vw, features& arr) +{ auto sb = gcnew System::Text::StringBuilder(); + for (size_t i = 0; i < arr.values.size(); i++) + { sb->Append(FormatFeature(vw, arr.values[i], arr.indicies[i]))->Append(" "); + } - return sb->ToString(); - } + return sb->ToString(); +} - System::String^ CompareFeatures(vw* vw, features& fa, features& fb, unsigned char ns) - { - vector fa_missing; - for (size_t ia = 0, ib = 0; ia < fa.values.size(); ia++) - { - auto masked_weight_index = fa.indicies[ia] & vw->weights.mask(); - auto other_masked_weight_index = fb.indicies[ib] & vw->weights.mask(); - - /*System::Diagnostics::Debug::WriteLine(System::String::Format("{0} -> {1} vs {2} -> {3}", - fa.indicies[ia], masked_weight_index, - fb.indicies[ib], other_masked_weight_index - ));*/ - - if (masked_weight_index == other_masked_weight_index && FloatEqual(fa.values[ia], fb.values[ib])) - ib++; - else - { - // fallback to search - size_t ib_old = ib; - bool found = false; - for (ib = 0; ib < fb.values.size(); ib++) - { - auto other_masked_weight_index = fb.indicies[ib] & vw->weights.mask(); - if (masked_weight_index == other_masked_weight_index) - { - if (!FloatEqual(fa.values[ia], fb.values[ib])) - { - return FormatFeature(vw, fa.values[ia], fa.indicies[ia], fb.values[ib], fb.indicies[ib]); - } - else - { - found = true; - break; - } - } - } - - if (!found) - { - fa_missing.push_back(ia); - } - - ib = ib_old + 1; - } +System::String^ CompareFeatures(vw* vw, features& fa, features& fb, unsigned char ns) +{ vector fa_missing; + for (size_t ia = 0, ib = 0; ia < fa.values.size(); ia++) + { auto masked_weight_index = fa.indicies[ia] & vw->weights.mask(); + auto other_masked_weight_index = fb.indicies[ib] & vw->weights.mask(); + + /*System::Diagnostics::Debug::WriteLine(System::String::Format("{0} -> {1} vs {2} -> {3}", + fa.indicies[ia], masked_weight_index, + fb.indicies[ib], other_masked_weight_index + ));*/ + + if (masked_weight_index == other_masked_weight_index && FloatEqual(fa.values[ia], fb.values[ib])) + ib++; + else + { // fallback to search + size_t ib_old = ib; + bool found = false; + for (ib = 0; ib < fb.values.size(); ib++) + { auto other_masked_weight_index = fb.indicies[ib] & vw->weights.mask(); + if (masked_weight_index == other_masked_weight_index) + { if (!FloatEqual(fa.values[ia], fb.values[ib])) + { return FormatFeature(vw, fa.values[ia], fa.indicies[ia], fb.values[ib], fb.indicies[ib]); + } + else + { found = true; + break; + } } + } - if (!fa_missing.empty()) - { - auto diff = gcnew System::Text::StringBuilder(); - diff->AppendFormat("missing features in ns '{0}'/'{1}': ", ns, gcnew Char(ns)); - for (size_t& ia : fa_missing) - { - diff->AppendFormat("this.weight_index = {0}, x = {1}, ", - fa.indicies[ia] & vw->weights.mask(), - fa.values[ia]); - } - - return diff->ToString(); - } + if (!found) + { fa_missing.push_back(ia); + } - return nullptr; + ib = ib_old + 1; } + } - System::String^ VowpalWabbitExample::Diff(VowpalWabbit^ vw, VowpalWabbitExample^ other, IVowpalWabbitLabelComparator^ labelComparator) - { - auto a = this->m_example; - auto b = other->m_example; + if (!fa_missing.empty()) + { auto diff = gcnew System::Text::StringBuilder(); + diff->AppendFormat("missing features in ns '{0}'/'{1}': ", ns, gcnew Char(ns)); + for (size_t& ia : fa_missing) + { diff->AppendFormat("this.weight_index = {0}, x = {1}, ", + fa.indicies[ia] & vw->weights.mask(), + fa.values[ia]); + } - if (a->indices.size() != b->indices.size()) - { - return FormatIndices(a, b); - } + return diff->ToString(); + } - for (auto i = a->indices.begin(), j = b->indices.begin(); i != a->indices.end(); i++) - { - if (*i == *j) - j++; - else - { - // fall back on search - auto j_old = j; - - j = b->indices.begin(); - bool found = false; - for (; j != b->indices.end(); j++) - { - if (*i == *j) - { - found = true; - break; - } - } - - if (!found) - return FormatIndices(a, b); - - j = j_old + 1; - } - - // compare features - features& fa = a->feature_space[*i]; - features& fb = b->feature_space[*i]; - - if (fa.size() != fb.size()) - return System::String::Format("Feature length differ {0} vs {1}. this({2}) vs other({3})", - fa.size(), fb.size(), FormatFeatures(vw->m_vw, fa), FormatFeatures(vw->m_vw, fb)); - - auto diff = CompareFeatures(vw->m_vw, fa, fb, *i); - if (diff != nullptr) - return diff; - - diff = CompareFeatures(vw->m_vw, fb, fa, *i); - if (diff != nullptr) - return diff; - } + return nullptr; +} - if (labelComparator != nullptr) - { - // Compare the label - auto diff = labelComparator->Diff(this, other); - if (diff != nullptr) - return diff; +System::String^ VowpalWabbitExample::Diff(VowpalWabbit^ vw, VowpalWabbitExample^ other, IVowpalWabbitLabelComparator^ labelComparator) +{ auto a = this->m_example; + auto b = other->m_example; + + if (a->indices.size() != b->indices.size()) + { return FormatIndices(a, b); + } + + for (auto i = a->indices.begin(), j = b->indices.begin(); i != a->indices.end(); i++) + { if (*i == *j) + j++; + else + { // fall back on search + auto j_old = j; + + j = b->indices.begin(); + bool found = false; + for (; j != b->indices.end(); j++) + { if (*i == *j) + { found = true; + break; } + } + + if (!found) + return FormatIndices(a, b); - return nullptr; + j = j_old + 1; } - String^ VowpalWabbitSimpleLabelComparator::Diff(VowpalWabbitExample^ ex1, VowpalWabbitExample^ ex2) - { - auto s1 = ex1->m_example->l.simple; - auto s2 = ex2->m_example->l.simple; - - if (!(FloatEqual(s1.initial, s2.initial) && - FloatEqual(s1.label, s2.label) && - FloatEqual(s1.weight, s2.weight))) - { - return System::String::Format("Label differ. label {0} vs {1}. initial {2} vs {3}. weight {4} vs {5}", - s1.label, s2.label, - s1.initial, s2.initial, - s1.weight, s2.weight); - } + // compare features + features& fa = a->feature_space[*i]; + features& fb = b->feature_space[*i]; - return nullptr; - } + if (fa.size() != fb.size()) + return System::String::Format("Feature length differ {0} vs {1}. this({2}) vs other({3})", + fa.size(), fb.size(), FormatFeatures(vw->m_vw, fa), FormatFeatures(vw->m_vw, fb)); - String^ VowpalWabbitContextualBanditLabelComparator::Diff(VowpalWabbitExample^ ex1, VowpalWabbitExample^ ex2) - { - auto s1 = ex1->m_example->l.cb; - auto s2 = ex2->m_example->l.cb; + auto diff = CompareFeatures(vw->m_vw, fa, fb, *i); + if (diff != nullptr) + return diff; - if (s1.costs.size() != s2.costs.size()) - { - return System::String::Format("Cost size differ: {0} vs {1}", s1.costs.size(), s2.costs.size()); - } + diff = CompareFeatures(vw->m_vw, fb, fa, *i); + if (diff != nullptr) + return diff; + } - for (size_t i = 0; i < s1.costs.size(); i++) - { - auto c1 = s1.costs[i]; - auto c2 = s2.costs[i]; - if (c1.action != c2.action) - { - return System::String::Format("Action differ: {0} vs {1}", c1.action, c2.action); - } - - if (c1.cost != c2.cost) - { - return System::String::Format("Cost differ: {0} vs {1}", c1.cost, c2.cost); - } - - if (abs(c1.probability - c2.probability) / max(c1.probability, c2.probability) > 0.01) - { - return System::String::Format("Probability differ: {0} vs {1}", c1.probability, c2.probability); - } - } + if (labelComparator != nullptr) + { // Compare the label + auto diff = labelComparator->Diff(this, other); + if (diff != nullptr) + return diff; + } - return nullptr; - } + return nullptr; +} - System::Collections::IEnumerator^ VowpalWabbitExample::EnumerableGetEnumerator::get() - { - return GetEnumerator(); - } +String^ VowpalWabbitSimpleLabelComparator::Diff(VowpalWabbitExample^ ex1, VowpalWabbitExample^ ex2) +{ auto s1 = ex1->m_example->l.simple; + auto s2 = ex2->m_example->l.simple; - IEnumerator^ VowpalWabbitExample::GetEnumerator() - { - return gcnew NamespaceEnumerator(this); - } + if (!(FloatEqual(s1.initial, s2.initial) && + FloatEqual(s1.label, s2.label) && + FloatEqual(s1.weight, s2.weight))) + { return System::String::Format("Label differ. label {0} vs {1}. initial {2} vs {3}. weight {4} vs {5}", + s1.label, s2.label, + s1.initial, s2.initial, + s1.weight, s2.weight); + } - VowpalWabbitExample::NamespaceEnumerator::NamespaceEnumerator(VowpalWabbitExample^ example) - : m_example(example) - { - Reset(); - } + return nullptr; +} - VowpalWabbitExample::NamespaceEnumerator::~NamespaceEnumerator() - { } +String^ VowpalWabbitContextualBanditLabelComparator::Diff(VowpalWabbitExample^ ex1, VowpalWabbitExample^ ex2) +{ auto s1 = ex1->m_example->l.cb; + auto s2 = ex2->m_example->l.cb; - bool VowpalWabbitExample::NamespaceEnumerator::MoveNext() - { - m_current++; + if (s1.costs.size() != s2.costs.size()) + { return System::String::Format("Cost size differ: {0} vs {1}", s1.costs.size(), s2.costs.size()); + } - return m_current < m_example->m_example->indices.end(); + for (size_t i = 0; i < s1.costs.size(); i++) + { auto c1 = s1.costs[i]; + auto c2 = s2.costs[i]; + if (c1.action != c2.action) + { return System::String::Format("Action differ: {0} vs {1}", c1.action, c2.action); } - void VowpalWabbitExample::NamespaceEnumerator::Reset() - { - // position before the beginning. - m_current = m_example->m_example->indices.begin() - 1; + if (c1.cost != c2.cost) + { return System::String::Format("Cost differ: {0} vs {1}", c1.cost, c2.cost); } - VowpalWabbitNamespace^ VowpalWabbitExample::NamespaceEnumerator::Current::get() - { - if (m_current < m_example->m_example->indices.begin() || m_current >= m_example->m_example->indices.end()) - throw gcnew InvalidOperationException(); - - return gcnew VowpalWabbitNamespace(m_example, *m_current, &m_example->m_example->feature_space[*m_current]); + if (abs(c1.probability - c2.probability) / max(c1.probability, c2.probability) > 0.01) + { return System::String::Format("Probability differ: {0} vs {1}", c1.probability, c2.probability); } + } - System::Object^ VowpalWabbitExample::NamespaceEnumerator::IEnumeratorCurrent::get() - { - return Current; - } + return nullptr; +} - VowpalWabbitFeature::VowpalWabbitFeature(VowpalWabbitExample^ example, feature_value x, uint64_t weight_index) - : m_example(example), m_vw(m_example->Owner->Native), m_x(x), m_weight_index(weight_index) - { } +System::Collections::IEnumerator^ VowpalWabbitExample::EnumerableGetEnumerator::get() +{ return GetEnumerator(); +} - VowpalWabbitFeature::VowpalWabbitFeature(VowpalWabbit^ vw, feature_value x, uint64_t weight_index) - : m_vw(vw), m_x(x), m_weight_index(weight_index) - { } +IEnumerator^ VowpalWabbitExample::GetEnumerator() +{ return gcnew NamespaceEnumerator(this); +} - float VowpalWabbitFeature::X::get() - { - return m_x; - } +VowpalWabbitExample::NamespaceEnumerator::NamespaceEnumerator(VowpalWabbitExample^ example) + : m_example(example) +{ Reset(); +} - uint64_t VowpalWabbitFeature::FeatureIndex::get() - { - return m_weight_index; - } +VowpalWabbitExample::NamespaceEnumerator::~NamespaceEnumerator() +{ } - uint64_t VowpalWabbitFeature::WeightIndex::get() - { - if (m_example == nullptr) - throw gcnew InvalidOperationException("VowpalWabbitFeature must be initialized with example"); +bool VowpalWabbitExample::NamespaceEnumerator::MoveNext() +{ m_current++; - vw* vw = m_example->Owner->Native->m_vw; - return ((m_weight_index + m_example->m_example->ft_offset) >> vw->weights.stride_shift()) & vw->parse_mask; - } + return m_current < m_example->m_example->indices.end(); +} - float VowpalWabbitFeature::Weight::get() - { - if (m_example == nullptr) - throw gcnew InvalidOperationException("VowpalWabbitFeature must be initialized with example"); +void VowpalWabbitExample::NamespaceEnumerator::Reset() +{ // position before the beginning. + m_current = m_example->m_example->indices.begin() - 1; +} - vw* vw = m_example->Owner->Native->m_vw; +VowpalWabbitNamespace^ VowpalWabbitExample::NamespaceEnumerator::Current::get() +{ if (m_current < m_example->m_example->indices.begin() || m_current >= m_example->m_example->indices.end()) + throw gcnew InvalidOperationException(); - uint64_t weightIndex = m_weight_index + m_example->m_example->ft_offset; - return vw->weights[weightIndex]; - } + return gcnew VowpalWabbitNamespace(m_example, *m_current, &m_example->m_example->feature_space[*m_current]); +} +System::Object^ VowpalWabbitExample::NamespaceEnumerator::IEnumeratorCurrent::get() +{ return Current; +} - float VowpalWabbitFeature::AuditWeight::get() - { - vw* vw = m_vw->m_vw; +VowpalWabbitFeature::VowpalWabbitFeature(VowpalWabbitExample^ example, feature_value x, uint64_t weight_index) + : m_example(example), m_vw(m_example->Owner->Native), m_x(x), m_weight_index(weight_index) +{ } - return GD::trunc_weight(Weight, (float)vw->sd->gravity) * (float)vw->sd->contraction; - } +VowpalWabbitFeature::VowpalWabbitFeature(VowpalWabbit^ vw, feature_value x, uint64_t weight_index) + : m_vw(vw), m_x(x), m_weight_index(weight_index) +{ } - bool VowpalWabbitFeature::Equals(Object^ o) - { - VowpalWabbitFeature^ other = dynamic_cast(o); +float VowpalWabbitFeature::X::get() +{ return m_x; +} - return other != nullptr && - other->m_x == m_x && - other->m_weight_index == m_weight_index; - } +uint64_t VowpalWabbitFeature::FeatureIndex::get() +{ return m_weight_index; +} - int VowpalWabbitFeature::GetHashCode() - { - return (int)(m_x + m_weight_index); - } +uint64_t VowpalWabbitFeature::WeightIndex::get() +{ if (m_example == nullptr) + throw gcnew InvalidOperationException("VowpalWabbitFeature must be initialized with example"); + vw* vw = m_example->Owner->Native->m_vw; + return ((m_weight_index + m_example->m_example->ft_offset) >> vw->weights.stride_shift()) & vw->parse_mask; +} - VowpalWabbitNamespace::VowpalWabbitNamespace(VowpalWabbitExample^ example, namespace_index ns, features* features) - : m_example(example), m_ns(ns), m_features(features) - { } +float VowpalWabbitFeature::Weight::get() +{ if (m_example == nullptr) + throw gcnew InvalidOperationException("VowpalWabbitFeature must be initialized with example"); - VowpalWabbitNamespace::~VowpalWabbitNamespace() - { } + vw* vw = m_example->Owner->Native->m_vw; - namespace_index VowpalWabbitNamespace::Index::get() - { - return m_ns; - } + uint64_t weightIndex = m_weight_index + m_example->m_example->ft_offset; + return vw->weights[weightIndex]; +} - System::Collections::IEnumerator^ VowpalWabbitNamespace::EnumerableGetEnumerator::get() - { - return GetEnumerator(); - } - IEnumerator^ VowpalWabbitNamespace::GetEnumerator() - { - return gcnew FeatureEnumerator(m_example, m_features); - } +float VowpalWabbitFeature::AuditWeight::get() +{ vw* vw = m_vw->m_vw; - VowpalWabbitNamespace::FeatureEnumerator::FeatureEnumerator(VowpalWabbitExample^ example, features* features) - : m_example(example), m_features(features), m_iterator(nullptr) - { - m_end = new Holder{ features->end() }; - } + return GD::trunc_weight(Weight, (float)vw->sd->gravity) * (float)vw->sd->contraction; +} - VowpalWabbitNamespace::FeatureEnumerator::~FeatureEnumerator() - { - delete m_end; - delete m_iterator; - } +bool VowpalWabbitFeature::Equals(Object^ o) +{ VowpalWabbitFeature^ other = dynamic_cast(o); - void VowpalWabbitNamespace::FeatureEnumerator::Reset() - { - delete m_iterator; - m_iterator = nullptr; - } + return other != nullptr && + other->m_x == m_x && + other->m_weight_index == m_weight_index; +} - bool VowpalWabbitNamespace::FeatureEnumerator::MoveNext() - { - if (m_iterator) - ++m_iterator->value; - else - m_iterator = new Holder{ m_features->begin() }; +int VowpalWabbitFeature::GetHashCode() +{ return (int)(m_x + m_weight_index); +} - return m_iterator->value != m_end->value; - } - System::Object^ VowpalWabbitNamespace::FeatureEnumerator::IEnumeratorCurrent::get() - { - return Current; - } +VowpalWabbitNamespace::VowpalWabbitNamespace(VowpalWabbitExample^ example, namespace_index ns, features* features) + : m_example(example), m_ns(ns), m_features(features) +{ } - VowpalWabbitFeature^ VowpalWabbitNamespace::FeatureEnumerator::Current::get() - { - if (!m_iterator || m_iterator->value == m_end->value) - throw gcnew InvalidOperationException(); +VowpalWabbitNamespace::~VowpalWabbitNamespace() +{ } - return gcnew VowpalWabbitFeature(m_example, m_iterator->value.value(), m_iterator->value.index()); - } +namespace_index VowpalWabbitNamespace::Index::get() +{ return m_ns; +} + +System::Collections::IEnumerator^ VowpalWabbitNamespace::EnumerableGetEnumerator::get() +{ return GetEnumerator(); +} + +IEnumerator^ VowpalWabbitNamespace::GetEnumerator() +{ return gcnew FeatureEnumerator(m_example, m_features); +} + +VowpalWabbitNamespace::FeatureEnumerator::FeatureEnumerator(VowpalWabbitExample^ example, features* features) + : m_example(example), m_features(features), m_iterator(nullptr) +{ m_end = new Holder { features->end() }; +} + +VowpalWabbitNamespace::FeatureEnumerator::~FeatureEnumerator() +{ delete m_end; + delete m_iterator; +} + +void VowpalWabbitNamespace::FeatureEnumerator::Reset() +{ delete m_iterator; + m_iterator = nullptr; +} + +bool VowpalWabbitNamespace::FeatureEnumerator::MoveNext() +{ if (m_iterator) + ++m_iterator->value; + else + m_iterator = new Holder { m_features->begin() }; + + return m_iterator->value != m_end->value; +} + +System::Object^ VowpalWabbitNamespace::FeatureEnumerator::IEnumeratorCurrent::get() +{ return Current; +} + +VowpalWabbitFeature^ VowpalWabbitNamespace::FeatureEnumerator::Current::get() +{ if (!m_iterator || m_iterator->value == m_end->value) + throw gcnew InvalidOperationException(); + + return gcnew VowpalWabbitFeature(m_example, m_iterator->value.value(), m_iterator->value.index()); +} } diff --git a/cs/cli/vw_example.h b/cs/cli/vw_example.h index 2014e9b82ea..fec4722c4c7 100644 --- a/cs/cli/vw_example.h +++ b/cs/cli/vw_example.h @@ -13,258 +13,239 @@ license as described in the file LICENSE. namespace VW { - using namespace System::Collections::Generic; - using namespace VW::Labels; - - ref class VowpalWabbitExample; - ref class VowpalWabbit; - - [System::Diagnostics::DebuggerDisplay("{m_weight_index}:{m_x}")] - public ref struct VowpalWabbitFeature - { - private: - feature_value m_x; - uint64_t m_weight_index; - VowpalWabbitExample^ m_example; - VowpalWabbit^ m_vw; - - public: - VowpalWabbitFeature(VowpalWabbitExample^ example, feature_value x, uint64_t weight_index); - VowpalWabbitFeature(VowpalWabbit^ vw, feature_value x, uint64_t weight_index); - - property feature_value X - { - float get(); - } - - property uint64_t FeatureIndex - { - uint64_t get(); - } - - property uint64_t WeightIndex - { - uint64_t get(); - } - - property float Weight - { - float get(); - } - - property float AuditWeight - { - float get(); - } - - virtual bool Equals(Object^ o) override; - - virtual int GetHashCode() override; - }; - - template - struct Holder - { - T value; - }; - - [System::Diagnostics::DebuggerDisplay("{Index} = '{(char)Index}'")] - public ref struct VowpalWabbitNamespace : public IEnumerable - { - private: - ref class FeatureEnumerator : public IEnumerator - { - private: - VowpalWabbitExample^ m_example; - features* m_features; - Holder* m_iterator; - Holder* m_end; - - internal: - FeatureEnumerator(VowpalWabbitExample^ example, features* features); - ~FeatureEnumerator(); - - property System::Object^ IEnumeratorCurrent - { - virtual System::Object^ get() sealed = System::Collections::IEnumerator::Current::get; - } - - public: - virtual bool MoveNext(); - - virtual void Reset(); - - property VowpalWabbitFeature^ Current - { - virtual VowpalWabbitFeature^ get(); - } - }; - - namespace_index m_ns; - features* m_features; - VowpalWabbitExample^ m_example; - - property System::Collections::IEnumerator^ EnumerableGetEnumerator - { - virtual System::Collections::IEnumerator^ get() sealed = System::Collections::IEnumerable::GetEnumerator; - } - - public: - VowpalWabbitNamespace(VowpalWabbitExample^ m_example, namespace_index ns, features* features); - ~VowpalWabbitNamespace(); - - property namespace_index Index - { - namespace_index get(); - } - - - virtual IEnumerator^ GetEnumerator(); - }; - - /// - /// A CLR representation of a vowpal wabbit example. - /// - /// - /// Underlying memory is allocated by native code, but examples are not part of the ring. - /// - [System::Diagnostics::DebuggerDisplay("{m_example}: '{m_string}'")] - public ref class VowpalWabbitExample : public IEnumerable - { - private: - /// - /// Reference to an optional underlying example. - /// - /// If this instance owns this is null. - initonly VowpalWabbitExample^ m_innerExample; - - ref class NamespaceEnumerator : public IEnumerator - { - private: - VowpalWabbitExample^ m_example; - namespace_index* m_current; - - internal: - NamespaceEnumerator(VowpalWabbitExample^ example); - ~NamespaceEnumerator(); - - property System::Object^ IEnumeratorCurrent - { - virtual System::Object^ get() sealed = System::Collections::IEnumerator::Current::get; - } - - public: - virtual bool MoveNext(); - - virtual void Reset(); - - property VowpalWabbitNamespace^ Current - { - virtual VowpalWabbitNamespace^ get(); - } - }; - - protected: - /// - /// Returns native example data structure to owning instance. - /// - !VowpalWabbitExample(); - - internal: - /// - /// Initializes a new instance of . - /// - /// The parent instance. Examples cannot be shared between vw instances. - /// The already allocated example structure - VowpalWabbitExample(IVowpalWabbitExamplePool^ owner, example* example); - - /// - /// The native example data structure. - /// - example* m_example; - - /// - /// The owner of this example. - /// - IVowpalWabbitExamplePool^ m_owner; - - /// - /// The optional string version of the example. - /// - String^ m_string; - - public: - /// - /// Initializes a new instance of . - /// - /// The parent instance. Examples cannot be shared between instances. - /// The inner example this instance wraps. - VowpalWabbitExample(IVowpalWabbitExamplePool^ owner, VowpalWabbitExample^ example); - - /// - /// Returns native example data structure to owning pool. - /// - ~VowpalWabbitExample(); - - /// - /// Extracts the prediction from this example using the given prediction factory. - /// - /// The prediction stored in this example. - generic T GetPrediction(VowpalWabbit^ vw, IVowpalWabbitPredictionFactory^ factory); - - /// - /// An optional inner example this example wraps. - /// - property VowpalWabbitExample^ InnerExample - { - VowpalWabbitExample^ get(); - } - - /// - /// The owner of this example. - /// - property IVowpalWabbitExamplePool^ Owner - { - IVowpalWabbitExamplePool^ get(); - } - - /// - /// The corresponding VowpalWabbitString for this example. - /// - property String^ VowpalWabbitString - { - String^ get(); - void set(String^ value); - } - - /// - /// True if this is a new line example, otherwise false. - /// - /// A example without features is considered a new line example. - property bool IsNewLine - { - bool get(); - } - - String^ Diff(VowpalWabbit^ vw, VowpalWabbitExample^ other, IVowpalWabbitLabelComparator^ labelComparator); - - void MakeEmpty(VowpalWabbit^ vw); - - property System::Collections::IEnumerator^ EnumerableGetEnumerator - { - virtual System::Collections::IEnumerator^ get() sealed = System::Collections::IEnumerable::GetEnumerator; - } - - virtual IEnumerator^ GetEnumerator(); - - property size_t NumberOfFeatures - { - size_t get(); - } - - property ILabel^ Label - { - ILabel^ get(); - } - }; +using namespace System::Collections::Generic; +using namespace VW::Labels; + +ref class VowpalWabbitExample; +ref class VowpalWabbit; + +[System::Diagnostics::DebuggerDisplay("{m_weight_index}:{m_x}")] +public ref struct VowpalWabbitFeature +{ +private: + feature_value m_x; + uint64_t m_weight_index; + VowpalWabbitExample^ m_example; + VowpalWabbit^ m_vw; + +public: + VowpalWabbitFeature(VowpalWabbitExample^ example, feature_value x, uint64_t weight_index); + VowpalWabbitFeature(VowpalWabbit^ vw, feature_value x, uint64_t weight_index); + + property feature_value X + { float get(); + } + + property uint64_t FeatureIndex + { uint64_t get(); + } + + property uint64_t WeightIndex + { uint64_t get(); + } + + property float Weight + { float get(); + } + + property float AuditWeight + { float get(); + } + + virtual bool Equals(Object^ o) override; + + virtual int GetHashCode() override; +}; + +template +struct Holder +{ T value; +}; + +[System::Diagnostics::DebuggerDisplay("{Index} = '{(char)Index}'")] +public ref struct VowpalWabbitNamespace : public IEnumerable +{ +private: + ref class FeatureEnumerator : public IEnumerator + { + private: + VowpalWabbitExample^ m_example; + features* m_features; + Holder* m_iterator; + Holder* m_end; + + internal: + FeatureEnumerator(VowpalWabbitExample^ example, features* features); + ~FeatureEnumerator(); + + property System::Object^ IEnumeratorCurrent + { virtual System::Object^ get() sealed = System::Collections::IEnumerator::Current::get; + } + + public: + virtual bool MoveNext(); + + virtual void Reset(); + + property VowpalWabbitFeature^ Current + { virtual VowpalWabbitFeature^ get(); + } + }; + + namespace_index m_ns; + features* m_features; + VowpalWabbitExample^ m_example; + + property System::Collections::IEnumerator^ EnumerableGetEnumerator + { virtual System::Collections::IEnumerator^ get() sealed = System::Collections::IEnumerable::GetEnumerator; + } + +public: + VowpalWabbitNamespace(VowpalWabbitExample^ m_example, namespace_index ns, features* features); + ~VowpalWabbitNamespace(); + + property namespace_index Index + { namespace_index get(); + } + + + virtual IEnumerator^ GetEnumerator(); +}; + +/// +/// A CLR representation of a vowpal wabbit example. +/// +/// +/// Underlying memory is allocated by native code, but examples are not part of the ring. +/// +[System::Diagnostics::DebuggerDisplay("{m_example}: '{m_string}'")] +public ref class VowpalWabbitExample : public IEnumerable +{ +private: + /// + /// Reference to an optional underlying example. + /// + /// If this instance owns this is null. + initonly VowpalWabbitExample^ m_innerExample; + + ref class NamespaceEnumerator : public IEnumerator + { + private: + VowpalWabbitExample^ m_example; + namespace_index* m_current; + + internal: + NamespaceEnumerator(VowpalWabbitExample^ example); + ~NamespaceEnumerator(); + + property System::Object^ IEnumeratorCurrent + { virtual System::Object^ get() sealed = System::Collections::IEnumerator::Current::get; + } + + public: + virtual bool MoveNext(); + + virtual void Reset(); + + property VowpalWabbitNamespace^ Current + { virtual VowpalWabbitNamespace^ get(); + } + }; + +protected: + /// + /// Returns native example data structure to owning instance. + /// + !VowpalWabbitExample(); + +internal: + /// + /// Initializes a new instance of . + /// + /// The parent instance. Examples cannot be shared between vw instances. + /// The already allocated example structure + VowpalWabbitExample(IVowpalWabbitExamplePool^ owner, example* example); + + /// + /// The native example data structure. + /// + example* m_example; + + /// + /// The owner of this example. + /// + IVowpalWabbitExamplePool^ m_owner; + + /// + /// The optional string version of the example. + /// + String^ m_string; + +public: + /// + /// Initializes a new instance of . + /// + /// The parent instance. Examples cannot be shared between instances. + /// The inner example this instance wraps. + VowpalWabbitExample(IVowpalWabbitExamplePool^ owner, VowpalWabbitExample^ example); + + /// + /// Returns native example data structure to owning pool. + /// + ~VowpalWabbitExample(); + + /// + /// Extracts the prediction from this example using the given prediction factory. + /// + /// The prediction stored in this example. + generic T GetPrediction(VowpalWabbit^ vw, IVowpalWabbitPredictionFactory^ factory); + + /// + /// An optional inner example this example wraps. + /// + property VowpalWabbitExample^ InnerExample + { VowpalWabbitExample^ get(); + } + + /// + /// The owner of this example. + /// + property IVowpalWabbitExamplePool^ Owner + { IVowpalWabbitExamplePool^ get(); + } + + /// + /// The corresponding VowpalWabbitString for this example. + /// + property String^ VowpalWabbitString + { String^ get(); + void set(String^ value); + } + + /// + /// True if this is a new line example, otherwise false. + /// + /// A example without features is considered a new line example. + property bool IsNewLine + { bool get(); + } + + String^ Diff(VowpalWabbit^ vw, VowpalWabbitExample^ other, IVowpalWabbitLabelComparator^ labelComparator); + + void MakeEmpty(VowpalWabbit^ vw); + + property System::Collections::IEnumerator^ EnumerableGetEnumerator + { virtual System::Collections::IEnumerator^ get() sealed = System::Collections::IEnumerable::GetEnumerator; + } + + virtual IEnumerator^ GetEnumerator(); + + property size_t NumberOfFeatures + { size_t get(); + } + + property ILabel^ Label + { ILabel^ get(); + } +}; } diff --git a/cs/cli/vw_exception.cpp b/cs/cli/vw_exception.cpp index f065c9b742a..1a6015ba1e6 100644 --- a/cs/cli/vw_exception.cpp +++ b/cs/cli/vw_exception.cpp @@ -8,18 +8,16 @@ license as described in the file LICENSE. namespace VW { - VowpalWabbitException::VowpalWabbitException(const vw_exception& ex) - : Exception(gcnew System::String(ex.what())), m_filename(gcnew System::String(ex.Filename())), m_lineNumber(ex.LineNumber()) - { - } +VowpalWabbitException::VowpalWabbitException(const vw_exception& ex) + : Exception(gcnew System::String(ex.what())), m_filename(gcnew System::String(ex.Filename())), m_lineNumber(ex.LineNumber()) +{ +} - String^ VowpalWabbitException::Filename::get() - { - return m_filename; - } +String^ VowpalWabbitException::Filename::get() +{ return m_filename; +} - Int32 VowpalWabbitException::LineNumber::get() - { - return m_lineNumber; - } +Int32 VowpalWabbitException::LineNumber::get() +{ return m_lineNumber; +} } diff --git a/cs/cli/vw_interface.h b/cs/cli/vw_interface.h index 1708d28084e..c95da3d3d17 100644 --- a/cs/cli/vw_interface.h +++ b/cs/cli/vw_interface.h @@ -12,28 +12,26 @@ license as described in the file LICENSE. namespace VW { - ref class VowpalWabbitExample; - ref class VowpalWabbitBase; +ref class VowpalWabbitExample; +ref class VowpalWabbitBase; - /// - /// Owners of example must implement this interface. - /// - public interface class IVowpalWabbitExamplePool : public System::IDisposable - { - /// - /// Gets or creates a new native examples. - /// - VowpalWabbitExample^ GetOrCreateNativeExample(); +/// +/// Owners of example must implement this interface. +/// +public interface class IVowpalWabbitExamplePool : public System::IDisposable +{ /// + /// Gets or creates a new native examples. + /// + VowpalWabbitExample^ GetOrCreateNativeExample(); - property VowpalWabbit^ Native - { - VowpalWabbit^ get(); - } + property VowpalWabbit^ Native + { VowpalWabbit^ get(); + } - /// - /// Puts a native example data structure back into the pool. - /// - /// The example to be returned. - void ReturnExampleToPool(VowpalWabbitExample^ example); - }; + /// + /// Puts a native example data structure back into the pool. + /// + /// The example to be returned. + void ReturnExampleToPool(VowpalWabbitExample^ example); +}; } diff --git a/cs/cli/vw_label.h b/cs/cli/vw_label.h index d93fb935ebd..e8560db03dc 100644 --- a/cs/cli/vw_label.h +++ b/cs/cli/vw_label.h @@ -16,416 +16,362 @@ namespace VW { namespace Labels { - // The label classes are a replication of the parse_label function pointers provided by individual - // modules. Main reason for creation is thread-saftey. The C++ version use a shared v_array in parser - // and thus need to be synchronized. - // These label classes are thread-safe and even more efficient as they avoid marshalling. - - using namespace System; - using namespace System::Collections::Generic; - using namespace System::Globalization; - using namespace System::Text; - using namespace CB; - using namespace MULTICLASS; - using namespace Newtonsoft::Json; - - public interface class ILabel - { - void UpdateExample(vw* vw, example* ex); - void ReadFromExample(example* ex); - }; - - public ref class ContextualBanditLabel sealed : ILabel - { - private: - uint32_t m_action; - float m_cost; - float m_probability; - - public: - ContextualBanditLabel() - : m_action(0), m_cost(0), m_probability(0) - { } - - ContextualBanditLabel(uint32_t action, float cost, float probability) - : m_action(action), m_cost(cost), m_probability(0) - { - Probability = probability; +// The label classes are a replication of the parse_label function pointers provided by individual +// modules. Main reason for creation is thread-saftey. The C++ version use a shared v_array in parser +// and thus need to be synchronized. +// These label classes are thread-safe and even more efficient as they avoid marshalling. + +using namespace System; +using namespace System::Collections::Generic; +using namespace System::Globalization; +using namespace System::Text; +using namespace CB; +using namespace MULTICLASS; +using namespace Newtonsoft::Json; + +public interface class ILabel +{ void UpdateExample(vw* vw, example* ex); + void ReadFromExample(example* ex); +}; + +public ref class ContextualBanditLabel sealed : ILabel +{ +private: + uint32_t m_action; + float m_cost; + float m_probability; + +public: + ContextualBanditLabel() + : m_action(0), m_cost(0), m_probability(0) + { } + + ContextualBanditLabel(uint32_t action, float cost, float probability) + : m_action(action), m_cost(cost), m_probability(0) + { Probability = probability; + } + + [JsonProperty] + property uint32_t Action + { uint32_t get() + { return m_action; } - [JsonProperty] - property uint32_t Action - { - uint32_t get() - { - return m_action; - } - - void set(uint32_t value) - { - m_action = value; - } + void set(uint32_t value) + { m_action = value; } + } - [JsonProperty] - property float Probability - { - float get() - { - return m_probability; - } + [JsonProperty] + property float Probability + { float get() + { return m_probability; + } - void set(float value) - { - if (value < 0 || value >1) - throw gcnew ArgumentOutOfRangeException("invalid probability: " + value); + void set(float value) + { if (value < 0 || value >1) + throw gcnew ArgumentOutOfRangeException("invalid probability: " + value); - m_probability = value; - } + m_probability = value; } + } - [JsonProperty] - property float Cost - { - float get() - { - return m_cost; - } + [JsonProperty] + property float Cost + { float get() + { return m_cost; + } - void set(float value) - { - m_cost = value; - } + void set(float value) + { m_cost = value; } + } - [JsonIgnore] - property bool IsShared - { - bool get() - { - return m_cost == FLT_MAX && m_probability == -1.f; - } + [JsonIgnore] + property bool IsShared + { bool get() + { return m_cost == FLT_MAX && m_probability == -1.f; } + } - virtual void ReadFromExample(example* ex) - { - CB::label* ld = (CB::label*)&ex->l; - if (ld->costs.size() > 0) - { - cb_class& f = ld->costs[0]; + virtual void ReadFromExample(example* ex) + { CB::label* ld = (CB::label*)&ex->l; + if (ld->costs.size() > 0) + { cb_class& f = ld->costs[0]; - m_action = f.action; - m_cost = f.cost; - m_probability = f.probability; - } + m_action = f.action; + m_cost = f.cost; + m_probability = f.probability; } + } + + virtual void UpdateExample(vw* vw, example* ex) + { CB::label* ld = (CB::label*)&ex->l; + cb_class f; + + f.partial_prediction = 0.; + f.action = m_action; + f.cost = m_cost; + f.probability = m_probability; + + ld->costs.push_back(f); + } + + virtual String^ ToString() override + { auto sb = gcnew StringBuilder; + + sb->Append(m_action.ToString(CultureInfo::InvariantCulture)); + sb->Append(L':'); + sb->Append(m_cost.ToString(CultureInfo::InvariantCulture)); + sb->Append(L':'); + sb->Append(m_probability.ToString(CultureInfo::InvariantCulture)); + + return sb->ToString(); + } +}; + +/// +/// In multi-line scenarios the first example can contain a set of shared features. This first example must be +/// marked using a 'shared' label. +/// +public ref class SharedLabel sealed : ILabel +{ +private: + uint32_t m_action; - virtual void UpdateExample(vw* vw, example* ex) - { - CB::label* ld = (CB::label*)&ex->l; - cb_class f; + SharedLabel() : m_action((uint32_t)uniform_hash("shared", 6, 0)) + { } - f.partial_prediction = 0.; - f.action = m_action; - f.cost = m_cost; - f.probability = m_probability; +public: + static SharedLabel^ Instance = gcnew SharedLabel; - ld->costs.push_back(f); - } + virtual void UpdateExample(vw* vw, example* ex) + { CB::label* ld = (CB::label*)&ex->l; + cb_class f; - virtual String^ ToString() override - { - auto sb = gcnew StringBuilder; + f.partial_prediction = 0.; + f.action = m_action; + f.cost = FLT_MAX; + f.probability = -1.f; - sb->Append(m_action.ToString(CultureInfo::InvariantCulture)); - sb->Append(L':'); - sb->Append(m_cost.ToString(CultureInfo::InvariantCulture)); - sb->Append(L':'); - sb->Append(m_probability.ToString(CultureInfo::InvariantCulture)); + ld->costs.push_back(f); + } - return sb->ToString(); - } - }; + virtual String^ ToString() override + { return "shared"; + } - /// - /// In multi-line scenarios the first example can contain a set of shared features. This first example must be - /// marked using a 'shared' label. - /// - public ref class SharedLabel sealed : ILabel + virtual void ReadFromExample(example* ex) { - private: - uint32_t m_action; + } +}; - SharedLabel() : m_action((uint32_t)uniform_hash("shared", 6, 0)) - { } +public ref class SimpleLabel sealed : ILabel +{ +private: + float m_label; - public: - static SharedLabel^ Instance = gcnew SharedLabel; + Nullable m_weight; - virtual void UpdateExample(vw* vw, example* ex) - { - CB::label* ld = (CB::label*)&ex->l; - cb_class f; + Nullable m_initial; - f.partial_prediction = 0.; - f.action = m_action; - f.cost = FLT_MAX; - f.probability = -1.f; +public: + SimpleLabel() : m_label(0) + { } - ld->costs.push_back(f); + [JsonProperty] + property float Label + { float get() + { return m_label; } - virtual String^ ToString() override - { - return "shared"; + void set(float value) + { m_label = value; } + } - virtual void ReadFromExample(example* ex) - { + [JsonProperty(NullValueHandling = NullValueHandling::Ignore)] + property Nullable Weight + { Nullable get() + { return m_weight; } - }; - - public ref class SimpleLabel sealed : ILabel - { - private: - float m_label; - - Nullable m_weight; - Nullable m_initial; - - public: - SimpleLabel() : m_label(0) - { } - - [JsonProperty] - property float Label - { - float get() - { - return m_label; - } - - void set(float value) - { - m_label = value; - } + void set(Nullable value) + { m_weight = value; } + } - [JsonProperty(NullValueHandling = NullValueHandling::Ignore)] - property Nullable Weight - { - Nullable get() - { - return m_weight; - } - - void set(Nullable value) - { - m_weight = value; - } + [JsonProperty(NullValueHandling = NullValueHandling::Ignore)] + property Nullable Initial + { Nullable get() + { return m_initial; } - [JsonProperty(NullValueHandling = NullValueHandling::Ignore)] - property Nullable Initial - { - Nullable get() - { - return m_initial; - } - - void set(Nullable value) - { - m_initial = value; - } + void set(Nullable value) + { m_initial = value; } + } - virtual void ReadFromExample(example* ex) - { - label_data* ld = (label_data*)&ex->l; + virtual void ReadFromExample(example* ex) + { label_data* ld = (label_data*)&ex->l; - m_label = ld->label; - m_weight = ld->weight; - m_initial = ld->initial; - } + m_label = ld->label; + m_weight = ld->weight; + m_initial = ld->initial; + } - virtual void UpdateExample(vw* vw, example* ex) - { - label_data* ld = (label_data*)&ex->l; - ld->label = m_label; + virtual void UpdateExample(vw* vw, example* ex) + { label_data* ld = (label_data*)&ex->l; + ld->label = m_label; - if (m_weight.HasValue) - ld->weight = m_weight.Value; + if (m_weight.HasValue) + ld->weight = m_weight.Value; - if (m_initial.HasValue) - ld->initial = m_initial.Value; + if (m_initial.HasValue) + ld->initial = m_initial.Value; - // not sure if I should complain... external... - count_label(ld->label); - } + // not sure if I should complain... external... + count_label(ld->label); + } - virtual String^ ToString() override - { - auto sb = gcnew StringBuilder; + virtual String^ ToString() override + { auto sb = gcnew StringBuilder; - sb->Append(m_label.ToString(CultureInfo::InvariantCulture)); + sb->Append(m_label.ToString(CultureInfo::InvariantCulture)); - if (m_weight.HasValue) - { - sb->Append(L' '); - sb->Append(m_weight.Value.ToString(CultureInfo::InvariantCulture)); + if (m_weight.HasValue) + { sb->Append(L' '); + sb->Append(m_weight.Value.ToString(CultureInfo::InvariantCulture)); - if (m_initial.HasValue) - { - sb->Append(L' '); - sb->Append(m_initial.Value.ToString(CultureInfo::InvariantCulture)); - } + if (m_initial.HasValue) + { sb->Append(L' '); + sb->Append(m_initial.Value.ToString(CultureInfo::InvariantCulture)); } - - return sb->ToString(); } - }; - public ref class MulticlassLabel sealed : ILabel + return sb->ToString(); + } +}; + +public ref class MulticlassLabel sealed : ILabel +{ +public: + ref class Label sealed { + private: + uint32_t m_class; + Nullable m_weight; + public: - ref class Label sealed - { - private: - uint32_t m_class; - Nullable m_weight; - - public: - property uint32_t Class - { - uint32_t get() - { - return m_class; - } - - void set(uint32_t value) - { - m_class = value; - } + property uint32_t Class + { uint32_t get() + { return m_class; } - [JsonProperty(NullValueHandling = NullValueHandling::Ignore)] - property Nullable Weight - { - Nullable get() - { - return m_weight; - } - - void set(Nullable value) - { - m_weight = value; - } + void set(uint32_t value) + { m_class = value; } - }; - - private: - List^ m_classes; + } - public: - [JsonProperty] - property List^ Classes - { - List^ get() - { - return m_classes; + [JsonProperty(NullValueHandling = NullValueHandling::Ignore)] + property Nullable Weight + { Nullable get() + { return m_weight; } - void set(List^ value) - { - m_classes = value; + void set(Nullable value) + { m_weight = value; } } + }; - virtual void ReadFromExample(example* ex) - { - throw gcnew NotImplementedException("to be done..."); +private: + List^ m_classes; + +public: + [JsonProperty] + property List^ Classes + { List^ get() + { return m_classes; } - virtual void UpdateExample(vw* vw, example* ex) - { - throw gcnew NotImplementedException("to be done..."); + void set(List^ value) + { m_classes = value; } + } - virtual String^ ToString() override - { - auto sb = gcnew StringBuilder; + virtual void ReadFromExample(example* ex) + { throw gcnew NotImplementedException("to be done..."); + } - for each (Label^ label in m_classes) - { - sb->Append(L' '); - sb->Append(label->Class.ToString(CultureInfo::InvariantCulture)); + virtual void UpdateExample(vw* vw, example* ex) + { throw gcnew NotImplementedException("to be done..."); + } - if (label->Weight.HasValue) - { - sb->Append(L' '); - sb->Append(label->Weight.Value.ToString(CultureInfo::InvariantCulture)); - } - } + virtual String^ ToString() override + { auto sb = gcnew StringBuilder; - // strip first space - if (sb->Length > 0) - sb->Remove(0, 1); + for each (Label^ label in m_classes) + { sb->Append(L' '); + sb->Append(label->Class.ToString(CultureInfo::InvariantCulture)); - return sb->ToString(); + if (label->Weight.HasValue) + { sb->Append(L' '); + sb->Append(label->Weight.Value.ToString(CultureInfo::InvariantCulture)); + } } - }; - public ref class StringLabel sealed : ILabel - { - private: - String^ m_label; + // strip first space + if (sb->Length > 0) + sb->Remove(0, 1); - public: - StringLabel() - { } - - StringLabel(String^ label) : m_label(label) - { } - - [JsonProperty] - property String^ Label - { - String^ get() - { - return m_label; - } + return sb->ToString(); + } +}; - void set(String^ value) - { - m_label = value; - } +public ref class StringLabel sealed : ILabel +{ +private: + String^ m_label; + +public: + StringLabel() + { } + + StringLabel(String^ label) : m_label(label) + { } + + [JsonProperty] + property String^ Label + { String^ get() + { return m_label; } - virtual void ReadFromExample(example* ex) - { - throw gcnew NotImplementedException("to be done..."); + void set(String^ value) + { m_label = value; } + } - virtual void UpdateExample(vw* vw, example* ex) - { - auto bytes = System::Text::Encoding::UTF8->GetBytes(m_label); - auto valueHandle = GCHandle::Alloc(bytes, GCHandleType::Pinned); + virtual void ReadFromExample(example* ex) + { throw gcnew NotImplementedException("to be done..."); + } - try - { - VW::parse_example_label(*vw, *ex, reinterpret_cast(valueHandle.AddrOfPinnedObject().ToPointer())); - } - CATCHRETHROW - finally - { valueHandle.Free(); - } - } + virtual void UpdateExample(vw* vw, example* ex) + { auto bytes = System::Text::Encoding::UTF8->GetBytes(m_label); + auto valueHandle = GCHandle::Alloc(bytes, GCHandleType::Pinned); - virtual String^ ToString() override - { - return m_label; + try + { VW::parse_example_label(*vw, *ex, reinterpret_cast(valueHandle.AddrOfPinnedObject().ToPointer())); } - }; + CATCHRETHROW + finally + { valueHandle.Free(); + } + } + + virtual String^ ToString() override + { return m_label; + } +}; } } diff --git a/cs/cli/vw_labelcomparator.h b/cs/cli/vw_labelcomparator.h index 6ea319c466e..21774b7932a 100644 --- a/cs/cli/vw_labelcomparator.h +++ b/cs/cli/vw_labelcomparator.h @@ -10,61 +10,61 @@ license as described in the file LICENSE. namespace VW { - ref class VowpalWabbitExample; +ref class VowpalWabbitExample; - /// - /// Interface for label comparators. - /// - public interface class IVowpalWabbitLabelComparator - { - public: - /// - /// Compares labels of and . - /// - /// Returns null if labels are equivalent, otherwise returns the difference description. - String^ Diff(VowpalWabbitExample^ ex1, VowpalWabbitExample^ ex2); - }; +/// +/// Interface for label comparators. +/// +public interface class IVowpalWabbitLabelComparator +{ +public: + /// + /// Compares labels of and . + /// + /// Returns null if labels are equivalent, otherwise returns the difference description. + String^ Diff(VowpalWabbitExample^ ex1, VowpalWabbitExample^ ex2); +}; - /// - /// A label comparer for simple labels. - /// - public ref class VowpalWabbitSimpleLabelComparator sealed : IVowpalWabbitLabelComparator - { - public: - /// - /// Compares labels of and . - /// - /// Returns null if labels are equivalent, otherwise returns the difference description. - virtual String^ Diff(VowpalWabbitExample^ ex1, VowpalWabbitExample^ ex2) sealed; - }; +/// +/// A label comparer for simple labels. +/// +public ref class VowpalWabbitSimpleLabelComparator sealed : IVowpalWabbitLabelComparator +{ +public: + /// + /// Compares labels of and . + /// + /// Returns null if labels are equivalent, otherwise returns the difference description. + virtual String^ Diff(VowpalWabbitExample^ ex1, VowpalWabbitExample^ ex2) sealed; +}; - /// - /// A label comparer for contextual bandit label. - /// - public ref class VowpalWabbitContextualBanditLabelComparator sealed : IVowpalWabbitLabelComparator - { - public: - /// - /// Compares labels of and . - /// - /// Returns null if labels are equivalent, otherwise returns the difference description. - virtual String^ Diff(VowpalWabbitExample^ ex1, VowpalWabbitExample^ ex2) sealed; - }; +/// +/// A label comparer for contextual bandit label. +/// +public ref class VowpalWabbitContextualBanditLabelComparator sealed : IVowpalWabbitLabelComparator +{ +public: + /// + /// Compares labels of and . + /// + /// Returns null if labels are equivalent, otherwise returns the difference description. + virtual String^ Diff(VowpalWabbitExample^ ex1, VowpalWabbitExample^ ex2) sealed; +}; - /// - /// Label comparator factory. - /// - public ref class VowpalWabbitLabelComparator sealed abstract - { - public: - /// - /// Simple label comparator. - /// - static initonly IVowpalWabbitLabelComparator^ Simple = gcnew VowpalWabbitSimpleLabelComparator; +/// +/// Label comparator factory. +/// +public ref class VowpalWabbitLabelComparator sealed abstract +{ +public: + /// + /// Simple label comparator. + /// + static initonly IVowpalWabbitLabelComparator^ Simple = gcnew VowpalWabbitSimpleLabelComparator; - /// - /// Contextual bandit label comparator. - /// - static initonly IVowpalWabbitLabelComparator^ ContextualBandit = gcnew VowpalWabbitContextualBanditLabelComparator; - }; + /// + /// Contextual bandit label comparator. + /// + static initonly IVowpalWabbitLabelComparator^ ContextualBandit = gcnew VowpalWabbitContextualBanditLabelComparator; +}; } diff --git a/cs/cli/vw_model.cpp b/cs/cli/vw_model.cpp index fb0be489346..9a5ac6da4e3 100644 --- a/cs/cli/vw_model.cpp +++ b/cs/cli/vw_model.cpp @@ -12,34 +12,31 @@ license as described in the file LICENSE. namespace VW { - VowpalWabbitSettings^ AddTestOnly(VowpalWabbitSettings^ settings) - { - // VowpalWabbitModel and VowpalWabbit instances seeded from VowpalWabbitModel - // need to have the same "test" setting, otherwise the stride shift is different - // and all hell breaks loose. - if (!settings->Arguments->Contains("-t ") && - !settings->Arguments->Contains("--testonly ") && - !settings->Arguments->EndsWith("-t") && - !settings->Arguments->EndsWith("--testonly")) - { - settings->Arguments += " -t"; - } +VowpalWabbitSettings^ AddTestOnly(VowpalWabbitSettings^ settings) +{ // VowpalWabbitModel and VowpalWabbit instances seeded from VowpalWabbitModel + // need to have the same "test" setting, otherwise the stride shift is different + // and all hell breaks loose. + if (!settings->Arguments->Contains("-t ") && + !settings->Arguments->Contains("--testonly ") && + !settings->Arguments->EndsWith("-t") && + !settings->Arguments->EndsWith("--testonly")) + { settings->Arguments += " -t"; + } - return settings; - } + return settings; +} - VowpalWabbitModel::VowpalWabbitModel(VowpalWabbitSettings^ settings) - : VowpalWabbitBase(AddTestOnly(settings)) - { - if (settings == nullptr) - throw gcnew ArgumentNullException("settings"); +VowpalWabbitModel::VowpalWabbitModel(VowpalWabbitSettings^ settings) + : VowpalWabbitBase(AddTestOnly(settings)) +{ if (settings == nullptr) + throw gcnew ArgumentNullException("settings"); - if (settings->Model != nullptr) - throw gcnew ArgumentNullException("VowpalWabbitModel cannot be initialized from another model"); - } + if (settings->Model != nullptr) + throw gcnew ArgumentNullException("VowpalWabbitModel cannot be initialized from another model"); +} - VowpalWabbitModel::VowpalWabbitModel(String^ args) - : VowpalWabbitModel(gcnew VowpalWabbitSettings(args)) - { - } +VowpalWabbitModel::VowpalWabbitModel(String^ args) + : VowpalWabbitModel(gcnew VowpalWabbitSettings(args)) +{ +} } diff --git a/cs/cli/vw_model.h b/cs/cli/vw_model.h index 03054ac0e21..cf0d1866598 100644 --- a/cs/cli/vw_model.h +++ b/cs/cli/vw_model.h @@ -11,19 +11,19 @@ license as described in the file LICENSE. namespace VW { - /// - /// VowpalWabbit model wrapper used in multi-threaded scenarios. - /// - public ref class VowpalWabbitModel : public VowpalWabbitBase - { - public: - /// - /// Initializes a new instance. - /// - /// Arguments passed to native instance. - VowpalWabbitModel(VowpalWabbitSettings^ settings); +/// +/// VowpalWabbit model wrapper used in multi-threaded scenarios. +/// +public ref class VowpalWabbitModel : public VowpalWabbitBase +{ +public: + /// + /// Initializes a new instance. + /// + /// Arguments passed to native instance. + VowpalWabbitModel(VowpalWabbitSettings^ settings); - /// Command line arguments passed to native instance. - VowpalWabbitModel(String^ args); - }; + /// Command line arguments passed to native instance. + VowpalWabbitModel(String^ args); +}; } diff --git a/cs/cli/vw_prediction.cpp b/cs/cli/vw_prediction.cpp index 672644a3afd..f95a0a36c7b 100644 --- a/cs/cli/vw_prediction.cpp +++ b/cs/cli/vw_prediction.cpp @@ -11,202 +11,182 @@ license as described in the file LICENSE. namespace VW { - void CheckExample(vw* vw, example* ex, prediction_type::prediction_type_t type) - { - if (vw == nullptr) - throw gcnew ArgumentNullException("vw"); - - if (ex == nullptr) - throw gcnew ArgumentNullException("ex"); - - auto ex_pred_type = vw->l->pred_type; - if (ex_pred_type != type) - { - auto sb = gcnew StringBuilder(); - sb->Append("Prediction type must be "); - sb->Append(gcnew String(prediction_type::to_string(type))); - sb->Append(" but is "); - sb->Append(gcnew String(prediction_type::to_string(ex_pred_type))); - - throw gcnew ArgumentException(sb->ToString()); - } - } - - float VowpalWabbitScalarPredictionFactory::Create(vw* vw, example* ex) - { - CheckExample(vw, ex, PredictionType); - - try - { - return VW::get_prediction(ex); - } - CATCHRETHROW - } +void CheckExample(vw* vw, example* ex, prediction_type::prediction_type_t type) +{ if (vw == nullptr) + throw gcnew ArgumentNullException("vw"); + + if (ex == nullptr) + throw gcnew ArgumentNullException("ex"); + + auto ex_pred_type = vw->l->pred_type; + if (ex_pred_type != type) + { auto sb = gcnew StringBuilder(); + sb->Append("Prediction type must be "); + sb->Append(gcnew String(prediction_type::to_string(type))); + sb->Append(" but is "); + sb->Append(gcnew String(prediction_type::to_string(ex_pred_type))); + + throw gcnew ArgumentException(sb->ToString()); + } +} +float VowpalWabbitScalarPredictionFactory::Create(vw* vw, example* ex) +{ CheckExample(vw, ex, PredictionType); - VowpalWabbitScalar VowpalWabbitScalarConfidencePredictionFactory::Create(vw* vw, example* ex) - { - CheckExample(vw, ex, PredictionType); + try + { return VW::get_prediction(ex); + } + CATCHRETHROW +} - try - { - VowpalWabbitScalar ret; - ret.Value = VW::get_prediction(ex); - ret.Confidence = ex->confidence; +VowpalWabbitScalar VowpalWabbitScalarConfidencePredictionFactory::Create(vw* vw, example* ex) +{ CheckExample(vw, ex, PredictionType); - return ret; - } - CATCHRETHROW - } + try + { VowpalWabbitScalar ret; - cli::array^ VowpalWabbitScalarsPredictionFactory::Create(vw* vw, example* ex) - { - CheckExample(vw, ex, PredictionType); + ret.Value = VW::get_prediction(ex); + ret.Confidence = ex->confidence; - try - { - auto& scalars = ex->pred.scalars; - auto values = gcnew cli::array((int)scalars.size()); - int index = 0; - for (float s : scalars) - values[index++] = s; + return ret; + } + CATCHRETHROW +} - return values; - } - CATCHRETHROW - } +cli::array^ VowpalWabbitScalarsPredictionFactory::Create(vw* vw, example* ex) +{ CheckExample(vw, ex, PredictionType); - float VowpalWabbitProbabilityPredictionFactory::Create(vw* vw, example* ex) - { - CheckExample(vw, ex, PredictionType); + try + { auto& scalars = ex->pred.scalars; + auto values = gcnew cli::array((int)scalars.size()); + int index = 0; + for (float s : scalars) + values[index++] = s; - return ex->pred.prob; - } + return values; + } + CATCHRETHROW +} - float VowpalWabbitCostSensitivePredictionFactory::Create(vw* vw, example* ex) - { - CheckExample(vw, ex, PredictionType); +float VowpalWabbitProbabilityPredictionFactory::Create(vw* vw, example* ex) +{ CheckExample(vw, ex, PredictionType); - try - { - return VW::get_cost_sensitive_prediction(ex); - } - CATCHRETHROW - } + return ex->pred.prob; +} + +float VowpalWabbitCostSensitivePredictionFactory::Create(vw* vw, example* ex) +{ CheckExample(vw, ex, PredictionType); + + try + { return VW::get_cost_sensitive_prediction(ex); + } + CATCHRETHROW +} - Dictionary^ VowpalWabbitMulticlassProbabilitiesPredictionFactory::Create(vw* vw, example* ex) - { +Dictionary^ VowpalWabbitMulticlassProbabilitiesPredictionFactory::Create(vw* vw, example* ex) +{ #if _DEBUG - if (ex == nullptr) - throw gcnew ArgumentNullException("ex"); + if (ex == nullptr) + throw gcnew ArgumentNullException("ex"); #endif - v_array confidence_scores; + v_array confidence_scores; - try { - confidence_scores = VW::get_cost_sensitive_prediction_confidence_scores(ex); - } - CATCHRETHROW + try + { confidence_scores = VW::get_cost_sensitive_prediction_confidence_scores(ex); + } + CATCHRETHROW - auto values = gcnew Dictionary(); - int i = 0; - for (auto& val : confidence_scores) { - values->Add(++i, val); - } + auto values = gcnew Dictionary(); + int i = 0; + for (auto& val : confidence_scores) + { values->Add(++i, val); + } - return values; - } + return values; +} - uint32_t VowpalWabbitMulticlassPredictionFactory::Create(vw* vw, example* ex) - { - CheckExample(vw, ex, PredictionType); +uint32_t VowpalWabbitMulticlassPredictionFactory::Create(vw* vw, example* ex) +{ CheckExample(vw, ex, PredictionType); - return ex->pred.multiclass; - } + return ex->pred.multiclass; +} - cli::array^ VowpalWabbitMultilabelPredictionFactory::Create(vw* vw, example* ex) - { - CheckExample(vw, ex, prediction_type::multilabels); +cli::array^ VowpalWabbitMultilabelPredictionFactory::Create(vw* vw, example* ex) +{ CheckExample(vw, ex, prediction_type::multilabels); - size_t length; - uint32_t* labels; + size_t length; + uint32_t* labels; - try - { - labels = VW::get_multilabel_predictions(ex, length); - } - CATCHRETHROW + try + { labels = VW::get_multilabel_predictions(ex, length); + } + CATCHRETHROW - if (length > Int32::MaxValue) - throw gcnew ArgumentOutOfRangeException("Multi-label predictions too large"); + if (length > Int32::MaxValue) + throw gcnew ArgumentOutOfRangeException("Multi-label predictions too large"); - auto values = gcnew cli::array((int)length); + auto values = gcnew cli::array((int)length); - if (length > 0) - Marshal::Copy(IntPtr(labels), values, 0, (int)length); + if (length > 0) + Marshal::Copy(IntPtr(labels), values, 0, (int)length); - return values; - } + return values; +} - cli::array^ VowpalWabbitActionScoreBasePredictionFactory::Create(vw* vw, example* ex) - { - CheckExample(vw, ex, PredictionType); +cli::array^ VowpalWabbitActionScoreBasePredictionFactory::Create(vw* vw, example* ex) +{ CheckExample(vw, ex, PredictionType); - auto& a_s = ex->pred.a_s; - auto values = gcnew cli::array((int)a_s.size()); + auto& a_s = ex->pred.a_s; + auto values = gcnew cli::array((int)a_s.size()); - auto index = 0; - for (auto& as : a_s) - { - values[index].Action = as.action; - values[index].Score = as.score; - index++; - } + auto index = 0; + for (auto& as : a_s) + { values[index].Action = as.action; + values[index].Score = as.score; + index++; + } - return values; - } + return values; +} - cli::array^ VowpalWabbitTopicPredictionFactory::Create(vw* vw, example* ex) - { - if (ex == nullptr) - throw gcnew ArgumentNullException("ex"); +cli::array^ VowpalWabbitTopicPredictionFactory::Create(vw* vw, example* ex) +{ if (ex == nullptr) + throw gcnew ArgumentNullException("ex"); - auto values = gcnew cli::array(vw->lda); - Marshal::Copy(IntPtr(ex->pred.scalars.begin()), values, 0, vw->lda); + auto values = gcnew cli::array(vw->lda); + Marshal::Copy(IntPtr(ex->pred.scalars.begin()), values, 0, vw->lda); - return values; - } + return values; +} - System::Object^ VowpalWabbitDynamicPredictionFactory::Create(vw* vw, example* ex) - { - if (ex == nullptr) - throw gcnew ArgumentNullException("ex"); - - switch (vw->l->pred_type) - { - case prediction_type::scalar: - return VowpalWabbitPredictionType::Scalar->Create(vw, ex); - case prediction_type::scalars: - return VowpalWabbitPredictionType::Scalars->Create(vw, ex); - case prediction_type::multiclass: - return VowpalWabbitPredictionType::Multiclass->Create(vw, ex); - case prediction_type::multilabels: - return VowpalWabbitPredictionType::Multilabel->Create(vw, ex); - case prediction_type::action_scores: - return VowpalWabbitPredictionType::ActionScore->Create(vw, ex); - case prediction_type::action_probs: - return VowpalWabbitPredictionType::ActionProbabilities->Create(vw, ex); - case prediction_type::prob: - return VowpalWabbitPredictionType::Probability->Create(vw, ex); - case prediction_type::multiclassprobs: - return VowpalWabbitPredictionType::MultiClassProbabilities->Create(vw, ex); - default: - { - auto sb = gcnew StringBuilder(); - sb->Append("Unsupported prediction type: "); - sb->Append(gcnew String(prediction_type::to_string(vw->l->pred_type))); - throw gcnew ArgumentException(sb->ToString()); - } - } - } +System::Object^ VowpalWabbitDynamicPredictionFactory::Create(vw* vw, example* ex) +{ if (ex == nullptr) + throw gcnew ArgumentNullException("ex"); + + switch (vw->l->pred_type) + { case prediction_type::scalar: + return VowpalWabbitPredictionType::Scalar->Create(vw, ex); + case prediction_type::scalars: + return VowpalWabbitPredictionType::Scalars->Create(vw, ex); + case prediction_type::multiclass: + return VowpalWabbitPredictionType::Multiclass->Create(vw, ex); + case prediction_type::multilabels: + return VowpalWabbitPredictionType::Multilabel->Create(vw, ex); + case prediction_type::action_scores: + return VowpalWabbitPredictionType::ActionScore->Create(vw, ex); + case prediction_type::action_probs: + return VowpalWabbitPredictionType::ActionProbabilities->Create(vw, ex); + case prediction_type::prob: + return VowpalWabbitPredictionType::Probability->Create(vw, ex); + case prediction_type::multiclassprobs: + return VowpalWabbitPredictionType::MultiClassProbabilities->Create(vw, ex); + default: + { auto sb = gcnew StringBuilder(); + sb->Append("Unsupported prediction type: "); + sb->Append(gcnew String(prediction_type::to_string(vw->l->pred_type))); + throw gcnew ArgumentException(sb->ToString()); + } + } +} } diff --git a/cs/cli/vw_prediction.h b/cs/cli/vw_prediction.h index 77df686d622..e442f95aaf3 100644 --- a/cs/cli/vw_prediction.h +++ b/cs/cli/vw_prediction.h @@ -10,405 +10,379 @@ license as described in the file LICENSE. namespace VW { - ref class VowpalWabbitExample; - ref class VowpalWabbit; - - using namespace System::Collections::Generic; - - /// - /// Interface for prediction factories enabling read-out of various prediction results in an extendable manner. - /// - generic - public interface class IVowpalWabbitPredictionFactory - { - public: - /// - /// Creates a new prediction result from an example and the associated VW instance. - /// - /// A prediction result. - /// Implementation must be thread-safe. - T Create(vw* vw, example* ex); - - /// - /// Returns the supported prediction type. - /// - property prediction_type::prediction_type_t PredictionType - { - prediction_type::prediction_type_t get(); - } - }; - - /// - /// A scalar prediction result. - /// - public ref class VowpalWabbitDynamicPredictionFactory sealed : IVowpalWabbitPredictionFactory - { - public: - /// - /// Extracts prediction results from example. - /// - virtual System::Object^ Create(vw* vw, example* ex) sealed; - - /// - /// Returns the supported prediction type. - /// - property prediction_type::prediction_type_t PredictionType - { - virtual prediction_type::prediction_type_t get() sealed - { - throw gcnew NotSupportedException("Prediction type is not available."); - } - } - }; - - /// - /// A scalar prediction result. - /// - public ref class VowpalWabbitScalarPredictionFactory sealed : IVowpalWabbitPredictionFactory - { - public: - /// - /// Extracts prediction results from example. - /// - virtual float Create(vw* vw, example* ex) sealed; - - /// - /// Returns the supported prediction type. - /// - property prediction_type::prediction_type_t PredictionType - { - virtual prediction_type::prediction_type_t get() sealed - { - return prediction_type::scalar; - } - } - }; - - public value struct VowpalWabbitScalar - { - public: - float Value; - - float Confidence; - }; - - /// - /// A scalar prediction result. - /// - public ref class VowpalWabbitScalarConfidencePredictionFactory sealed : IVowpalWabbitPredictionFactory - { - public: - /// - /// Extracts prediction results from example. - /// - virtual VowpalWabbitScalar Create(vw* vw, example* ex) sealed; - - /// - /// Returns the supported prediction type. - /// - property prediction_type::prediction_type_t PredictionType - { - virtual prediction_type::prediction_type_t get() sealed - { - return prediction_type::scalar; - } - } - }; - - /// - /// A scalar prediction result. - /// - public ref class VowpalWabbitScalarsPredictionFactory sealed : IVowpalWabbitPredictionFactory^> - { - public: - /// - /// Extracts prediction results from example. - /// - virtual cli::array^ Create(vw* vw, example* ex) sealed; - - /// - /// Returns the supported prediction type. - /// - property prediction_type::prediction_type_t PredictionType - { - virtual prediction_type::prediction_type_t get() sealed - { - return prediction_type::scalars; - } - } - }; - - /// - /// A scalar prediction result. - /// - public ref class VowpalWabbitProbabilityPredictionFactory sealed : IVowpalWabbitPredictionFactory - { - public: - /// - /// Extracts prediction results from example. - /// - virtual float Create(vw* vw, example* ex) sealed; - - /// - /// Returns the supported prediction type. - /// - property prediction_type::prediction_type_t PredictionType - { - virtual prediction_type::prediction_type_t get() sealed - { - return prediction_type::prob; - } - } - }; - - /// - /// A cost sensitive prediction result. - /// - public ref class VowpalWabbitCostSensitivePredictionFactory sealed : IVowpalWabbitPredictionFactory - { - public: - /// - /// Extracts cost sensitive prediction results from example. - /// - virtual float Create(vw* vw, example* ex) sealed; - - /// - /// Returns the supported prediction type. - /// - property prediction_type::prediction_type_t PredictionType - { - virtual prediction_type::prediction_type_t get() sealed - { - return prediction_type::multiclass; - } - } - }; - - /// - /// A cost sensitive prediction result. - /// - public ref class VowpalWabbitMulticlassPredictionFactory sealed : IVowpalWabbitPredictionFactory - { - public: - /// - /// Extracts cost sensitive prediction results from example. - /// - virtual uint32_t Create(vw* vw, example* ex) sealed; - - /// - /// Returns the supported prediction type. - /// - property prediction_type::prediction_type_t PredictionType - { - virtual prediction_type::prediction_type_t get() sealed - { - return prediction_type::multiclass; - } - } - }; - - /// - /// A cost sensitive prediction result with associated confidence score - /// For -oaa --probabilities - /// - public ref class VowpalWabbitMulticlassProbabilitiesPredictionFactory sealed : IVowpalWabbitPredictionFactory^> - { - public: - /// - /// Extracts cost sensitive prediction results from example, including confidence score. - /// - virtual Dictionary^ Create(vw* vw, example* ex) sealed; - - /// - /// Returns the supported prediction type. - /// - property prediction_type::prediction_type_t PredictionType - { - virtual prediction_type::prediction_type_t get() sealed - { - return prediction_type::multiclassprobs; - } - } - }; - - /// - /// A multi label prediction result. - /// - public ref class VowpalWabbitMultilabelPredictionFactory sealed : IVowpalWabbitPredictionFactory^> - { - public: - /// - /// Extracts multilabel prediction results from example. - /// - virtual cli::array^ Create(vw* vw, example* ex) sealed; - - /// - /// Returns the supported prediction type. - /// - property prediction_type::prediction_type_t PredictionType - { - virtual prediction_type::prediction_type_t get() sealed - { - return prediction_type::multilabels; - } - } - }; - - [System::Diagnostics::DebuggerDisplay("{Action}:{Score}")] - public value struct ActionScore sealed - { - public: - property uint32_t Action; - - property float Score; - }; - - /// - /// A action score/probability result. - /// - public ref class VowpalWabbitActionScoreBasePredictionFactory abstract - : IVowpalWabbitPredictionFactory^> - { - public: - /// - /// Extracts multilabel prediction results from example. - /// - virtual cli::array^ Create(vw* vw, example* ex) sealed; - - /// - /// Returns the supported prediction type. - /// - property prediction_type::prediction_type_t PredictionType - { - virtual prediction_type::prediction_type_t get() abstract; - } - }; - - /// - /// A action score prediction result. - /// - public ref class VowpalWabbitActionScorePredictionFactory sealed - : public VowpalWabbitActionScoreBasePredictionFactory - { - public: - /// - /// Returns the supported prediction type. - /// - property prediction_type::prediction_type_t PredictionType - { - virtual prediction_type::prediction_type_t get() override sealed - { - return prediction_type::action_scores; - } - } - }; - - /// - /// A multi label prediction result. - /// - public ref class VowpalWabbitActionProbabilitiesPredictionFactory sealed - : public VowpalWabbitActionScoreBasePredictionFactory - { - public: - /// - /// Returns the supported prediction type. - /// - property prediction_type::prediction_type_t PredictionType - { - virtual prediction_type::prediction_type_t get() override sealed - { - return prediction_type::action_probs; - } - } - }; - - /// - /// A topic prediction result. - /// - public ref class VowpalWabbitTopicPredictionFactory sealed : IVowpalWabbitPredictionFactory^> - { - public: - /// - /// Extracts prediction results from example. The predicted topics. - /// - virtual cli::array^ Create(vw* vw, example* ex) sealed; - - /// - /// Returns the supported prediction type. - /// - property prediction_type::prediction_type_t PredictionType - { - virtual prediction_type::prediction_type_t get() sealed - { - throw gcnew NotSupportedException("Prediction type is not available."); - } - } - }; - - /// - /// Provides convenient collection of all prediction types. - /// - public ref class VowpalWabbitPredictionType sealed abstract - { - public: - /// - /// Use for scalar predictions. - /// - static initonly VowpalWabbitScalarPredictionFactory^ Scalar = gcnew VowpalWabbitScalarPredictionFactory; - - /// - /// Use for scalar predictions. - /// - static initonly VowpalWabbitScalarConfidencePredictionFactory^ ScalarConfidence = gcnew VowpalWabbitScalarConfidencePredictionFactory; - - /// - /// Use for scalar predictions. - /// - static initonly VowpalWabbitScalarsPredictionFactory^ Scalars = gcnew VowpalWabbitScalarsPredictionFactory; - - /// - /// Use for cost sensitive predictions. - /// - static initonly VowpalWabbitCostSensitivePredictionFactory^ CostSensitive = gcnew VowpalWabbitCostSensitivePredictionFactory; - - /// - /// Use for multi label predictions. - /// - static initonly VowpalWabbitMultilabelPredictionFactory^ Multilabel = gcnew VowpalWabbitMultilabelPredictionFactory; - - /// - /// Use for multi class predictions. - /// - static initonly VowpalWabbitMulticlassPredictionFactory^ Multiclass = gcnew VowpalWabbitMulticlassPredictionFactory; - - /// - /// Use for action score predictions. - /// - static initonly VowpalWabbitActionScorePredictionFactory^ ActionScore = gcnew VowpalWabbitActionScorePredictionFactory; - - /// - /// Use for action score predictions. - /// - static initonly VowpalWabbitActionProbabilitiesPredictionFactory^ ActionProbabilities = gcnew VowpalWabbitActionProbabilitiesPredictionFactory; - - /// - /// Use for LDA topic predictions. - /// - static initonly VowpalWabbitTopicPredictionFactory^ Topic = gcnew VowpalWabbitTopicPredictionFactory; - - /// - /// Use for dynamicially determined predictions. - /// - static initonly VowpalWabbitDynamicPredictionFactory^ Dynamic = gcnew VowpalWabbitDynamicPredictionFactory; - - /// - /// Use for dynamicially determined predictions. - /// - static initonly VowpalWabbitProbabilityPredictionFactory^ Probability = gcnew VowpalWabbitProbabilityPredictionFactory; - - /// - /// Use for multiclass predictions with probabilities - /// - static initonly VowpalWabbitMulticlassProbabilitiesPredictionFactory^ MultiClassProbabilities = gcnew VowpalWabbitMulticlassProbabilitiesPredictionFactory; - }; +ref class VowpalWabbitExample; +ref class VowpalWabbit; + +using namespace System::Collections::Generic; + +/// +/// Interface for prediction factories enabling read-out of various prediction results in an extendable manner. +/// +generic +public interface class IVowpalWabbitPredictionFactory +{ +public: + /// + /// Creates a new prediction result from an example and the associated VW instance. + /// + /// A prediction result. + /// Implementation must be thread-safe. + T Create(vw* vw, example* ex); + + /// + /// Returns the supported prediction type. + /// + property prediction_type::prediction_type_t PredictionType + { prediction_type::prediction_type_t get(); + } +}; + +/// +/// A scalar prediction result. +/// +public ref class VowpalWabbitDynamicPredictionFactory sealed : IVowpalWabbitPredictionFactory +{ +public: + /// + /// Extracts prediction results from example. + /// + virtual System::Object^ Create(vw* vw, example* ex) sealed; + + /// + /// Returns the supported prediction type. + /// + property prediction_type::prediction_type_t PredictionType + { virtual prediction_type::prediction_type_t get() sealed + { throw gcnew NotSupportedException("Prediction type is not available."); + } + } +}; + +/// +/// A scalar prediction result. +/// +public ref class VowpalWabbitScalarPredictionFactory sealed : IVowpalWabbitPredictionFactory +{ +public: + /// + /// Extracts prediction results from example. + /// + virtual float Create(vw* vw, example* ex) sealed; + + /// + /// Returns the supported prediction type. + /// + property prediction_type::prediction_type_t PredictionType + { virtual prediction_type::prediction_type_t get() sealed + { return prediction_type::scalar; + } + } +}; + +public value struct VowpalWabbitScalar +{ +public: + float Value; + + float Confidence; +}; + +/// +/// A scalar prediction result. +/// +public ref class VowpalWabbitScalarConfidencePredictionFactory sealed : IVowpalWabbitPredictionFactory +{ +public: + /// + /// Extracts prediction results from example. + /// + virtual VowpalWabbitScalar Create(vw* vw, example* ex) sealed; + + /// + /// Returns the supported prediction type. + /// + property prediction_type::prediction_type_t PredictionType + { virtual prediction_type::prediction_type_t get() sealed + { return prediction_type::scalar; + } + } +}; + +/// +/// A scalar prediction result. +/// +public ref class VowpalWabbitScalarsPredictionFactory sealed : IVowpalWabbitPredictionFactory^> +{ +public: + /// + /// Extracts prediction results from example. + /// + virtual cli::array^ Create(vw* vw, example* ex) sealed; + + /// + /// Returns the supported prediction type. + /// + property prediction_type::prediction_type_t PredictionType + { virtual prediction_type::prediction_type_t get() sealed + { return prediction_type::scalars; + } + } +}; + +/// +/// A scalar prediction result. +/// +public ref class VowpalWabbitProbabilityPredictionFactory sealed : IVowpalWabbitPredictionFactory +{ +public: + /// + /// Extracts prediction results from example. + /// + virtual float Create(vw* vw, example* ex) sealed; + + /// + /// Returns the supported prediction type. + /// + property prediction_type::prediction_type_t PredictionType + { virtual prediction_type::prediction_type_t get() sealed + { return prediction_type::prob; + } + } +}; + +/// +/// A cost sensitive prediction result. +/// +public ref class VowpalWabbitCostSensitivePredictionFactory sealed : IVowpalWabbitPredictionFactory +{ +public: + /// + /// Extracts cost sensitive prediction results from example. + /// + virtual float Create(vw* vw, example* ex) sealed; + + /// + /// Returns the supported prediction type. + /// + property prediction_type::prediction_type_t PredictionType + { virtual prediction_type::prediction_type_t get() sealed + { return prediction_type::multiclass; + } + } +}; + +/// +/// A cost sensitive prediction result. +/// +public ref class VowpalWabbitMulticlassPredictionFactory sealed : IVowpalWabbitPredictionFactory +{ +public: + /// + /// Extracts cost sensitive prediction results from example. + /// + virtual uint32_t Create(vw* vw, example* ex) sealed; + + /// + /// Returns the supported prediction type. + /// + property prediction_type::prediction_type_t PredictionType + { virtual prediction_type::prediction_type_t get() sealed + { return prediction_type::multiclass; + } + } +}; + +/// +/// A cost sensitive prediction result with associated confidence score +/// For -oaa --probabilities +/// +public ref class VowpalWabbitMulticlassProbabilitiesPredictionFactory sealed : IVowpalWabbitPredictionFactory^> +{ +public: + /// + /// Extracts cost sensitive prediction results from example, including confidence score. + /// + virtual Dictionary^ Create(vw* vw, example* ex) sealed; + + /// + /// Returns the supported prediction type. + /// + property prediction_type::prediction_type_t PredictionType + { virtual prediction_type::prediction_type_t get() sealed + { return prediction_type::multiclassprobs; + } + } +}; + +/// +/// A multi label prediction result. +/// +public ref class VowpalWabbitMultilabelPredictionFactory sealed : IVowpalWabbitPredictionFactory^> +{ +public: + /// + /// Extracts multilabel prediction results from example. + /// + virtual cli::array^ Create(vw* vw, example* ex) sealed; + + /// + /// Returns the supported prediction type. + /// + property prediction_type::prediction_type_t PredictionType + { virtual prediction_type::prediction_type_t get() sealed + { return prediction_type::multilabels; + } + } +}; + +[System::Diagnostics::DebuggerDisplay("{Action}:{Score}")] +public value struct ActionScore sealed +{ +public: + property uint32_t Action; + + property float Score; +}; + +/// +/// A action score/probability result. +/// +public ref class VowpalWabbitActionScoreBasePredictionFactory abstract + : IVowpalWabbitPredictionFactory^> +{ +public: + /// + /// Extracts multilabel prediction results from example. + /// + virtual cli::array^ Create(vw* vw, example* ex) sealed; + + /// + /// Returns the supported prediction type. + /// + property prediction_type::prediction_type_t PredictionType + { virtual prediction_type::prediction_type_t get() abstract; + } +}; + +/// +/// A action score prediction result. +/// +public ref class VowpalWabbitActionScorePredictionFactory sealed + : public VowpalWabbitActionScoreBasePredictionFactory +{ +public: + /// + /// Returns the supported prediction type. + /// + property prediction_type::prediction_type_t PredictionType + { virtual prediction_type::prediction_type_t get() override sealed + { return prediction_type::action_scores; + } + } +}; + +/// +/// A multi label prediction result. +/// +public ref class VowpalWabbitActionProbabilitiesPredictionFactory sealed + : public VowpalWabbitActionScoreBasePredictionFactory +{ +public: + /// + /// Returns the supported prediction type. + /// + property prediction_type::prediction_type_t PredictionType + { virtual prediction_type::prediction_type_t get() override sealed + { return prediction_type::action_probs; + } + } +}; + +/// +/// A topic prediction result. +/// +public ref class VowpalWabbitTopicPredictionFactory sealed : IVowpalWabbitPredictionFactory^> +{ +public: + /// + /// Extracts prediction results from example. The predicted topics. + /// + virtual cli::array^ Create(vw* vw, example* ex) sealed; + + /// + /// Returns the supported prediction type. + /// + property prediction_type::prediction_type_t PredictionType + { virtual prediction_type::prediction_type_t get() sealed + { throw gcnew NotSupportedException("Prediction type is not available."); + } + } +}; + +/// +/// Provides convenient collection of all prediction types. +/// +public ref class VowpalWabbitPredictionType sealed abstract +{ +public: + /// + /// Use for scalar predictions. + /// + static initonly VowpalWabbitScalarPredictionFactory^ Scalar = gcnew VowpalWabbitScalarPredictionFactory; + + /// + /// Use for scalar predictions. + /// + static initonly VowpalWabbitScalarConfidencePredictionFactory^ ScalarConfidence = gcnew VowpalWabbitScalarConfidencePredictionFactory; + + /// + /// Use for scalar predictions. + /// + static initonly VowpalWabbitScalarsPredictionFactory^ Scalars = gcnew VowpalWabbitScalarsPredictionFactory; + + /// + /// Use for cost sensitive predictions. + /// + static initonly VowpalWabbitCostSensitivePredictionFactory^ CostSensitive = gcnew VowpalWabbitCostSensitivePredictionFactory; + + /// + /// Use for multi label predictions. + /// + static initonly VowpalWabbitMultilabelPredictionFactory^ Multilabel = gcnew VowpalWabbitMultilabelPredictionFactory; + + /// + /// Use for multi class predictions. + /// + static initonly VowpalWabbitMulticlassPredictionFactory^ Multiclass = gcnew VowpalWabbitMulticlassPredictionFactory; + + /// + /// Use for action score predictions. + /// + static initonly VowpalWabbitActionScorePredictionFactory^ ActionScore = gcnew VowpalWabbitActionScorePredictionFactory; + + /// + /// Use for action score predictions. + /// + static initonly VowpalWabbitActionProbabilitiesPredictionFactory^ ActionProbabilities = gcnew VowpalWabbitActionProbabilitiesPredictionFactory; + + /// + /// Use for LDA topic predictions. + /// + static initonly VowpalWabbitTopicPredictionFactory^ Topic = gcnew VowpalWabbitTopicPredictionFactory; + + /// + /// Use for dynamicially determined predictions. + /// + static initonly VowpalWabbitDynamicPredictionFactory^ Dynamic = gcnew VowpalWabbitDynamicPredictionFactory; + + /// + /// Use for dynamicially determined predictions. + /// + static initonly VowpalWabbitProbabilityPredictionFactory^ Probability = gcnew VowpalWabbitProbabilityPredictionFactory; + + /// + /// Use for multiclass predictions with probabilities + /// + static initonly VowpalWabbitMulticlassProbabilitiesPredictionFactory^ MultiClassProbabilities = gcnew VowpalWabbitMulticlassProbabilitiesPredictionFactory; +}; } \ No newline at end of file diff --git a/cs/cli/vw_settings.h b/cs/cli/vw_settings.h index 6f316cc751c..cb072d88124 100644 --- a/cs/cli/vw_settings.h +++ b/cs/cli/vw_settings.h @@ -16,134 +16,130 @@ using namespace VW::Serializer; namespace VW { - ref class VowpalWabbit; - ref class VowpalWabbitModel; - ref class VowpalWabbitSettings; - - public enum class VowpalWabbitExampleDistribution - { - /// - /// Statistically safer option. - /// - UniformRandom = 0, - - /// - /// Better runtime performance. - /// - RoundRobin = 1 - }; - - public interface class ITypeInspector - { - public: - Schema^ CreateSchema(VowpalWabbitSettings^ settings, Type^ type); - }; - - /// - /// Settings for wrapper. - /// - /// Constructor with optional arguments was dropped as it broke version remapping (signature changed with the introduction of new options). - public ref class VowpalWabbitSettings : public ICloneable - { - public: - VowpalWabbitSettings() - { - Arguments = String::Empty; - ExampleCountPerRun = 1000; - MaxExampleCacheSize = UINT32_MAX; - MaxExampleQueueLengthPerInstance = UINT32_MAX; - EnableExampleCaching = false; - // default to the statistically more safe option - ExampleDistribution = VowpalWabbitExampleDistribution::UniformRandom; - EnableStringExampleGeneration = false; - EnableStringFloatCompact = false; - PropertyConfiguration = ::PropertyConfiguration::Default; - EnableThreadSafeExamplePooling = false; - MaxExamples = INT32_MAX; - Verbose = false; - } - - VowpalWabbitSettings(String^ arguments) - : VowpalWabbitSettings() - { - if (arguments != nullptr) - Arguments = arguments; - } - - /// - /// Command line arguments. - /// - property String^ Arguments; - - /// - /// Model used for initialization. - /// - property Stream^ ModelStream; - - /// - /// Shared native vowpwal wabbit data structure. - /// - property VowpalWabbitModel^ Model; - - property ParallelOptions^ ParallelOptions; - - /// - /// Set to true to disable example caching when used with a serializer. Defaults to true. - /// - property bool EnableExampleCaching; - - /// - /// Maximum number of serialized examples cached. Defaults to UINT32_MAX. - /// - property uint32_t MaxExampleCacheSize; - - /// - /// Maximum number of examples accepted by VowpalWabbitManager until Learn/Predict/... start to block. Defaults to UINT32_MAX. - /// - property uint32_t MaxExampleQueueLengthPerInstance; - - property uint32_t Node; - - property VowpalWabbit^ Root; - - property VowpalWabbitExampleDistribution ExampleDistribution; - - /// - /// In multi-threaded mode, this is the number of examples processed per run. - /// After ecah run the models are synchronized. - /// Defaults to 1000. - /// - property uint32_t ExampleCountPerRun; - - /// - /// Enable Vowpal Wabbit native string generation. - /// - property bool EnableStringExampleGeneration; - - /// - /// Enable compact float serialization for Vowpal Wabbit native string generation. - /// - property bool EnableStringFloatCompact; - - property VW::Serializer::Schema^ Schema; - - property VW::Serializer::Schema^ ActionDependentSchema; - - property List^ CustomFeaturizer; - - property ITypeInspector^ TypeInspector; - - property PropertyConfiguration^ PropertyConfiguration; - - property bool EnableThreadSafeExamplePooling; - - property int MaxExamples; - - property bool Verbose; - - virtual Object^ Clone() - { - return MemberwiseClone(); - } - }; +ref class VowpalWabbit; +ref class VowpalWabbitModel; +ref class VowpalWabbitSettings; + +public enum class VowpalWabbitExampleDistribution +{ /// + /// Statistically safer option. + /// + UniformRandom = 0, + + /// + /// Better runtime performance. + /// + RoundRobin = 1 +}; + +public interface class ITypeInspector +{ +public: + Schema^ CreateSchema(VowpalWabbitSettings^ settings, Type^ type); +}; + +/// +/// Settings for wrapper. +/// +/// Constructor with optional arguments was dropped as it broke version remapping (signature changed with the introduction of new options). +public ref class VowpalWabbitSettings : public ICloneable +{ +public: + VowpalWabbitSettings() + { Arguments = String::Empty; + ExampleCountPerRun = 1000; + MaxExampleCacheSize = UINT32_MAX; + MaxExampleQueueLengthPerInstance = UINT32_MAX; + EnableExampleCaching = false; + // default to the statistically more safe option + ExampleDistribution = VowpalWabbitExampleDistribution::UniformRandom; + EnableStringExampleGeneration = false; + EnableStringFloatCompact = false; + PropertyConfiguration = ::PropertyConfiguration::Default; + EnableThreadSafeExamplePooling = false; + MaxExamples = INT32_MAX; + Verbose = false; + } + + VowpalWabbitSettings(String^ arguments) + : VowpalWabbitSettings() + { if (arguments != nullptr) + Arguments = arguments; + } + + /// + /// Command line arguments. + /// + property String^ Arguments; + + /// + /// Model used for initialization. + /// + property Stream^ ModelStream; + + /// + /// Shared native vowpwal wabbit data structure. + /// + property VowpalWabbitModel^ Model; + + property ParallelOptions^ ParallelOptions; + + /// + /// Set to true to disable example caching when used with a serializer. Defaults to true. + /// + property bool EnableExampleCaching; + + /// + /// Maximum number of serialized examples cached. Defaults to UINT32_MAX. + /// + property uint32_t MaxExampleCacheSize; + + /// + /// Maximum number of examples accepted by VowpalWabbitManager until Learn/Predict/... start to block. Defaults to UINT32_MAX. + /// + property uint32_t MaxExampleQueueLengthPerInstance; + + property uint32_t Node; + + property VowpalWabbit^ Root; + + property VowpalWabbitExampleDistribution ExampleDistribution; + + /// + /// In multi-threaded mode, this is the number of examples processed per run. + /// After ecah run the models are synchronized. + /// Defaults to 1000. + /// + property uint32_t ExampleCountPerRun; + + /// + /// Enable Vowpal Wabbit native string generation. + /// + property bool EnableStringExampleGeneration; + + /// + /// Enable compact float serialization for Vowpal Wabbit native string generation. + /// + property bool EnableStringFloatCompact; + + property VW::Serializer::Schema^ Schema; + + property VW::Serializer::Schema^ ActionDependentSchema; + + property List^ CustomFeaturizer; + + property ITypeInspector^ TypeInspector; + + property PropertyConfiguration^ PropertyConfiguration; + + property bool EnableThreadSafeExamplePooling; + + property int MaxExamples; + + property bool Verbose; + + virtual Object^ Clone() + { return MemberwiseClone(); + } +}; } diff --git a/java/src/main/c++/jni_base_learner.h b/java/src/main/c++/jni_base_learner.h index 06dc58ca6a2..2e003466788 100644 --- a/java/src/main/c++/jni_base_learner.h +++ b/java/src/main/c++/jni_base_learner.h @@ -65,8 +65,8 @@ T base_predict( // When doing multiline prediction the final result is stored in the FIRST example parsed. example* first_example = NULL; - for (int i=0; iGetObjectArrayElement(example_strings, i)); + for (int i=0; iGetObjectArrayElement(example_strings, i)); example* ex = read_example(env, example_string, vwInstance); base_predict(env, ex, learn, vwInstance, predictor, false); if (i == 0) diff --git a/java/src/main/c++/vowpalWabbit_VW.h b/java/src/main/c++/vowpalWabbit_VW.h index 46f98092dc2..1035e024e79 100644 --- a/java/src/main/c++/vowpalWabbit_VW.h +++ b/java/src/main/c++/vowpalWabbit_VW.h @@ -5,7 +5,8 @@ #ifndef _Included_vowpalWabbit_VW #define _Included_vowpalWabbit_VW #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif /* * Class: vowpalWabbit_VW @@ -13,7 +14,7 @@ extern "C" { * Signature: ()Ljava/lang/String; */ JNIEXPORT jstring JNICALL Java_vowpalWabbit_VW_version - (JNIEnv *, jclass); +(JNIEnv *, jclass); #ifdef __cplusplus } diff --git a/java/src/main/c++/vowpalWabbit_learner_VWActionProbsLearner.h b/java/src/main/c++/vowpalWabbit_learner_VWActionProbsLearner.h index f1d09ccc53e..8db1c1fb522 100644 --- a/java/src/main/c++/vowpalWabbit_learner_VWActionProbsLearner.h +++ b/java/src/main/c++/vowpalWabbit_learner_VWActionProbsLearner.h @@ -5,7 +5,8 @@ #ifndef _Included_vowpalWabbit_learner_VWActionProbsLearner #define _Included_vowpalWabbit_learner_VWActionProbsLearner #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif /* * Class: vowpalWabbit_learner_VWActionProbsLearner @@ -13,7 +14,7 @@ extern "C" { * Signature: (Ljava/lang/String;ZJ)LvowpalWabbit/responses/ActionProbs; */ JNIEXPORT jobject JNICALL Java_vowpalWabbit_learner_VWActionProbsLearner_predict - (JNIEnv *, jobject, jstring, jboolean, jlong); +(JNIEnv *, jobject, jstring, jboolean, jlong); /* * Class: vowpalWabbit_learner_VWActionProbsLearner @@ -21,7 +22,7 @@ JNIEXPORT jobject JNICALL Java_vowpalWabbit_learner_VWActionProbsLearner_predict * Signature: ([Ljava/lang/String;ZJ)LvowpalWabbit/responses/ActionProbs; */ JNIEXPORT jobject JNICALL Java_vowpalWabbit_learner_VWActionProbsLearner_predictMultiline - (JNIEnv *, jobject, jobjectArray, jboolean, jlong); +(JNIEnv *, jobject, jobjectArray, jboolean, jlong); #ifdef __cplusplus } diff --git a/java/src/main/c++/vowpalWabbit_learner_VWActionScoresLearner.h b/java/src/main/c++/vowpalWabbit_learner_VWActionScoresLearner.h index 19a66256f4b..24263a0e92d 100644 --- a/java/src/main/c++/vowpalWabbit_learner_VWActionScoresLearner.h +++ b/java/src/main/c++/vowpalWabbit_learner_VWActionScoresLearner.h @@ -5,7 +5,8 @@ #ifndef _Included_vowpalWabbit_learner_VWActionScoresLearner #define _Included_vowpalWabbit_learner_VWActionScoresLearner #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif /* * Class: vowpalWabbit_learner_VWActionScoresLearner @@ -13,7 +14,7 @@ extern "C" { * Signature: (Ljava/lang/String;ZJ)LvowpalWabbit/responses/ActionScores; */ JNIEXPORT jobject JNICALL Java_vowpalWabbit_learner_VWActionScoresLearner_predict - (JNIEnv *, jobject, jstring, jboolean, jlong); +(JNIEnv *, jobject, jstring, jboolean, jlong); /* * Class: vowpalWabbit_learner_VWActionScoresLearner @@ -21,7 +22,7 @@ JNIEXPORT jobject JNICALL Java_vowpalWabbit_learner_VWActionScoresLearner_predic * Signature: ([Ljava/lang/String;ZJ)LvowpalWabbit/responses/ActionScores; */ JNIEXPORT jobject JNICALL Java_vowpalWabbit_learner_VWActionScoresLearner_predictMultiline - (JNIEnv *, jobject, jobjectArray, jboolean, jlong); +(JNIEnv *, jobject, jobjectArray, jboolean, jlong); #ifdef __cplusplus } diff --git a/java/src/main/c++/vowpalWabbit_learner_VWLearners.h b/java/src/main/c++/vowpalWabbit_learner_VWLearners.h index 553fe7700f7..25a841a4204 100644 --- a/java/src/main/c++/vowpalWabbit_learner_VWLearners.h +++ b/java/src/main/c++/vowpalWabbit_learner_VWLearners.h @@ -5,7 +5,8 @@ #ifndef _Included_vowpalWabbit_learner_VWLearners #define _Included_vowpalWabbit_learner_VWLearners #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif /* * Class: vowpalWabbit_learner_VWLearners @@ -13,7 +14,7 @@ extern "C" { * Signature: (Ljava/lang/String;)J */ JNIEXPORT jlong JNICALL Java_vowpalWabbit_learner_VWLearners_initialize - (JNIEnv *, jclass, jstring); +(JNIEnv *, jclass, jstring); /* * Class: vowpalWabbit_learner_VWLearners @@ -21,7 +22,7 @@ JNIEXPORT jlong JNICALL Java_vowpalWabbit_learner_VWLearners_initialize * Signature: (J)LvowpalWabbit/learner/VWLearners/VWReturnType; */ JNIEXPORT jobject JNICALL Java_vowpalWabbit_learner_VWLearners_getReturnType - (JNIEnv *, jclass, jlong); +(JNIEnv *, jclass, jlong); /* * Class: vowpalWabbit_learner_VWLearners @@ -29,7 +30,7 @@ JNIEXPORT jobject JNICALL Java_vowpalWabbit_learner_VWLearners_getReturnType * Signature: (J)V */ JNIEXPORT void JNICALL Java_vowpalWabbit_learner_VWLearners_closeInstance - (JNIEnv *, jclass, jlong); +(JNIEnv *, jclass, jlong); #ifdef __cplusplus } diff --git a/java/src/main/c++/vowpalWabbit_learner_VWMulticlassLearner.cc b/java/src/main/c++/vowpalWabbit_learner_VWMulticlassLearner.cc index 6bb38c835de..6b51c4d30fb 100644 --- a/java/src/main/c++/vowpalWabbit_learner_VWMulticlassLearner.cc +++ b/java/src/main/c++/vowpalWabbit_learner_VWMulticlassLearner.cc @@ -2,7 +2,7 @@ #include "../../../../vowpalwabbit/vw.h" #include "jni_base_learner.h" -jint multiclass_predictor(example* vec, JNIEnv *env){ return vec->pred.multiclass; } +jint multiclass_predictor(example* vec, JNIEnv *env) { return vec->pred.multiclass; } JNIEXPORT jint JNICALL Java_vowpalWabbit_learner_VWMulticlassLearner_predict(JNIEnv *env, jobject obj, jstring example_string, jboolean learn, jlong vwPtr) { return base_predict(env, example_string, learn, vwPtr, multiclass_predictor); diff --git a/java/src/main/c++/vowpalWabbit_learner_VWMulticlassLearner.h b/java/src/main/c++/vowpalWabbit_learner_VWMulticlassLearner.h index 74dba3eead0..05204d53e91 100644 --- a/java/src/main/c++/vowpalWabbit_learner_VWMulticlassLearner.h +++ b/java/src/main/c++/vowpalWabbit_learner_VWMulticlassLearner.h @@ -5,7 +5,8 @@ #ifndef _Included_vowpalWabbit_learner_VWMulticlassLearner #define _Included_vowpalWabbit_learner_VWMulticlassLearner #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif /* * Class: vowpalWabbit_learner_VWMulticlassLearner @@ -13,7 +14,7 @@ extern "C" { * Signature: (Ljava/lang/String;ZJ)I */ JNIEXPORT jint JNICALL Java_vowpalWabbit_learner_VWMulticlassLearner_predict - (JNIEnv *, jobject, jstring, jboolean, jlong); +(JNIEnv *, jobject, jstring, jboolean, jlong); /* * Class: vowpalWabbit_learner_VWMulticlassLearner @@ -21,7 +22,7 @@ JNIEXPORT jint JNICALL Java_vowpalWabbit_learner_VWMulticlassLearner_predict * Signature: ([Ljava/lang/String;ZJ)I */ JNIEXPORT jint JNICALL Java_vowpalWabbit_learner_VWMulticlassLearner_predictMultiline - (JNIEnv *, jobject, jobjectArray, jboolean, jlong); +(JNIEnv *, jobject, jobjectArray, jboolean, jlong); #ifdef __cplusplus } diff --git a/java/src/main/c++/vowpalWabbit_learner_VWMultilabelsLearner.h b/java/src/main/c++/vowpalWabbit_learner_VWMultilabelsLearner.h index cb5661d0425..365394e0e15 100644 --- a/java/src/main/c++/vowpalWabbit_learner_VWMultilabelsLearner.h +++ b/java/src/main/c++/vowpalWabbit_learner_VWMultilabelsLearner.h @@ -5,7 +5,8 @@ #ifndef _Included_vowpalWabbit_learner_VWMultilabelsLearner #define _Included_vowpalWabbit_learner_VWMultilabelsLearner #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif /* * Class: vowpalWabbit_learner_VWMultilabelsLearner @@ -13,7 +14,7 @@ extern "C" { * Signature: (Ljava/lang/String;ZJ)LvowpalWabbit/responses/Multilabels; */ JNIEXPORT jobject JNICALL Java_vowpalWabbit_learner_VWMultilabelsLearner_predict - (JNIEnv *, jobject, jstring, jboolean, jlong); +(JNIEnv *, jobject, jstring, jboolean, jlong); /* * Class: vowpalWabbit_learner_VWMultilabelsLearner @@ -21,7 +22,7 @@ JNIEXPORT jobject JNICALL Java_vowpalWabbit_learner_VWMultilabelsLearner_predict * Signature: ([Ljava/lang/String;ZJ)LvowpalWabbit/responses/Multilabels; */ JNIEXPORT jobject JNICALL Java_vowpalWabbit_learner_VWMultilabelsLearner_predictMultiline - (JNIEnv *, jobject, jobjectArray, jboolean, jlong); +(JNIEnv *, jobject, jobjectArray, jboolean, jlong); #ifdef __cplusplus } diff --git a/java/src/main/c++/vowpalWabbit_learner_VWProbLearner.h b/java/src/main/c++/vowpalWabbit_learner_VWProbLearner.h index 3c7a7c14c28..e74d507fd3a 100644 --- a/java/src/main/c++/vowpalWabbit_learner_VWProbLearner.h +++ b/java/src/main/c++/vowpalWabbit_learner_VWProbLearner.h @@ -5,7 +5,8 @@ #ifndef _Included_vowpalWabbit_learner_VWProbLearner #define _Included_vowpalWabbit_learner_VWProbLearner #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif /* * Class: vowpalWabbit_learner_VWProbLearner @@ -13,7 +14,7 @@ extern "C" { * Signature: (Ljava/lang/String;ZJ)F */ JNIEXPORT jfloat JNICALL Java_vowpalWabbit_learner_VWProbLearner_predict - (JNIEnv *, jobject, jstring, jboolean, jlong); +(JNIEnv *, jobject, jstring, jboolean, jlong); /* * Class: vowpalWabbit_learner_VWProbLearner @@ -21,7 +22,7 @@ JNIEXPORT jfloat JNICALL Java_vowpalWabbit_learner_VWProbLearner_predict * Signature: ([Ljava/lang/String;ZJ)F */ JNIEXPORT jfloat JNICALL Java_vowpalWabbit_learner_VWProbLearner_predictMultiline - (JNIEnv *, jobject, jobjectArray, jboolean, jlong); +(JNIEnv *, jobject, jobjectArray, jboolean, jlong); #ifdef __cplusplus } diff --git a/java/src/main/c++/vowpalWabbit_learner_VWScalarLearner.h b/java/src/main/c++/vowpalWabbit_learner_VWScalarLearner.h index 06d14c1742a..cac82662760 100644 --- a/java/src/main/c++/vowpalWabbit_learner_VWScalarLearner.h +++ b/java/src/main/c++/vowpalWabbit_learner_VWScalarLearner.h @@ -5,7 +5,8 @@ #ifndef _Included_vowpalWabbit_learner_VWScalarLearner #define _Included_vowpalWabbit_learner_VWScalarLearner #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif /* * Class: vowpalWabbit_learner_VWScalarLearner @@ -13,7 +14,7 @@ extern "C" { * Signature: (Ljava/lang/String;ZJ)F */ JNIEXPORT jfloat JNICALL Java_vowpalWabbit_learner_VWScalarLearner_predict - (JNIEnv *, jobject, jstring, jboolean, jlong); +(JNIEnv *, jobject, jstring, jboolean, jlong); /* * Class: vowpalWabbit_learner_VWScalarLearner @@ -21,7 +22,7 @@ JNIEXPORT jfloat JNICALL Java_vowpalWabbit_learner_VWScalarLearner_predict * Signature: ([Ljava/lang/String;ZJ)F */ JNIEXPORT jfloat JNICALL Java_vowpalWabbit_learner_VWScalarLearner_predictMultiline - (JNIEnv *, jobject, jobjectArray, jboolean, jlong); +(JNIEnv *, jobject, jobjectArray, jboolean, jlong); #ifdef __cplusplus } diff --git a/java/src/main/c++/vowpalWabbit_learner_VWScalarsLearner.h b/java/src/main/c++/vowpalWabbit_learner_VWScalarsLearner.h index 93c091ed391..461bd946740 100644 --- a/java/src/main/c++/vowpalWabbit_learner_VWScalarsLearner.h +++ b/java/src/main/c++/vowpalWabbit_learner_VWScalarsLearner.h @@ -5,7 +5,8 @@ #ifndef _Included_vowpalWabbit_learner_VWScalarsLearner #define _Included_vowpalWabbit_learner_VWScalarsLearner #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif /* * Class: vowpalWabbit_learner_VWScalarsLearner @@ -13,7 +14,7 @@ extern "C" { * Signature: (Ljava/lang/String;ZJ)[F */ JNIEXPORT jfloatArray JNICALL Java_vowpalWabbit_learner_VWScalarsLearner_predict - (JNIEnv *, jobject, jstring, jboolean, jlong); +(JNIEnv *, jobject, jstring, jboolean, jlong); /* * Class: vowpalWabbit_learner_VWScalarsLearner @@ -21,7 +22,7 @@ JNIEXPORT jfloatArray JNICALL Java_vowpalWabbit_learner_VWScalarsLearner_predict * Signature: ([Ljava/lang/String;ZJ)[F */ JNIEXPORT jfloatArray JNICALL Java_vowpalWabbit_learner_VWScalarsLearner_predictMultiline - (JNIEnv *, jobject, jobjectArray, jboolean, jlong); +(JNIEnv *, jobject, jobjectArray, jboolean, jlong); #ifdef __cplusplus } diff --git a/library/gd_mf_weights.cc b/library/gd_mf_weights.cc index d6447113319..65b8c5920f2 100644 --- a/library/gd_mf_weights.cc +++ b/library/gd_mf_weights.cc @@ -82,24 +82,24 @@ int main(int argc, char *argv[]) // write out features for left namespace features& left = ec->feature_space[left_ns]; for (size_t i = 0; i < left.size(); ++i) - { left_linear << left.space_names[i].get()->second << '\t' << weights[left.indicies[i]]; + { left_linear << left.space_names[i].get()->second << '\t' << weights[left.indicies[i]]; - left_quadratic << left.space_names[i].get()->second; - for (size_t k = 1; k <= rank; k++) - left_quadratic << '\t' << weights[(left.indicies[i] + k)]; - } + left_quadratic << left.space_names[i].get()->second; + for (size_t k = 1; k <= rank; k++) + left_quadratic << '\t' << weights[(left.indicies[i] + k)]; + } left_linear << endl; left_quadratic << endl; // write out features for right namespace features& right = ec->feature_space[right_ns]; for (size_t i = 0; i < right.size(); ++i) - { right_linear << right.space_names[i].get()->second << '\t' << weights[left.indicies[i]]; + { right_linear << right.space_names[i].get()->second << '\t' << weights[left.indicies[i]]; - right_quadratic << right.space_names[i].get()->second; - for (size_t k = 1; k <= rank; k++) - right_quadratic << '\t' << weights[(left.indicies[i] + k + rank)]; - } + right_quadratic << right.space_names[i].get()->second; + for (size_t k = 1; k <= rank; k++) + right_quadratic << '\t' << weights[(left.indicies[i] + k + rank)]; + } right_linear << endl; right_quadratic << endl; diff --git a/library/search_generate.cc b/library/search_generate.cc index b2025a950e3..61fd83f487f 100644 --- a/library/search_generate.cc +++ b/library/search_generate.cc @@ -38,7 +38,7 @@ class Trie Trie() : terminus(false), count(0), max_count(0), max_string("") {} ~Trie() -{ for (Trie* t : children) + { for (Trie* t : children) delete t; } @@ -150,7 +150,7 @@ class IncrementalEditDistance prev_row = tmp; } -void append(string s) { for (char c : s) append(c); } + void append(string s) { for (char c : s) append(c); } vector& next() { A.clear(); @@ -193,7 +193,7 @@ void append(string s) { for (char c : s) append(c); } string output_string; vector A; -inline size_t min3(size_t a, size_t b, size_t c) { return (a < b) ? (a < c) ? a : c : (b < c) ? b : c; } + inline size_t min3(size_t a, size_t b, size_t c) { return (a < b) ? (a < c) ? a : c : (b < c) ? b : c; } }; struct input @@ -213,7 +213,7 @@ float max_cost = 100.; float get_or_one(vector< pair >& v, char c) { // TODO: could binary search -for (auto& p : v) + for (auto& p : v) if (p.first == c) return minf(max_cost, (float)p.second); return 1.; @@ -258,13 +258,13 @@ class Generator : public SearchTask // characters thus far ex(vw_namespace('c')); -for (char c : out) ex("c=" + string(1,c)); + for (char c : out) ex("c=" + string(1,c)); ex("c=$"); // words thus far ex(vw_namespace('w')); tmp = ""; -for (char c : out) + for (char c : out) { if (c == '^') continue; if (c == ' ') { ex("w=" + tmp + "$"); @@ -280,7 +280,7 @@ for (char c : out) cdict->get_next(nullptr, next); ex(vw_namespace('d')); char best_char = '~'; float best_count = 0.; -for (auto xx : next) + for (auto xx : next) { if (xx.cw > 0.) ex("c=" + string(1,xx.c), xx.cw); if (xx.sw > 0.) ex("mc=" + xx.s, xx.sw); if (xx.sw > best_count) { best_count = xx.sw; best_char = xx.c; } @@ -299,7 +299,7 @@ for (auto xx : next) */ ex(vw_namespace('i')); tmp = ""; -for (char c : in.in) + for (char c : in.in) { if (c == ' ') { ex("w=" + tmp); tmp = ""; @@ -366,16 +366,16 @@ void run_easy() for (size_t i=0; i<100; i++) { //if (i == 9999) max_cost = 1.; if (i % 10 == 0) cerr << '.'; -for (auto x : training_data) + for (auto x : training_data) task.learn(x, out); } cerr << endl; -for (auto x : training_data) + for (auto x : training_data) { task.predict(x, out); cerr << "output = " << out << endl; } -for (auto x : test_data) + for (auto x : test_data) { task.predict(x, out); cerr << "output = " << out << endl; } diff --git a/python/pylibvw.cc b/python/pylibvw.cc index 8bfeacf22c3..35cd09522c9 100644 --- a/python/pylibvw.cc +++ b/python/pylibvw.cc @@ -71,16 +71,20 @@ label_parser* get_label_parser(vw*all, size_t labelType) size_t my_get_label_type(vw*all) { label_parser* lp = &all->p->lp; - if (lp->parse_label == simple_label.parse_label) { - return lBINARY; - } else if (lp->parse_label == MULTICLASS::mc_label.parse_label) { - return lMULTICLASS; - } else if (lp->parse_label == COST_SENSITIVE::cs_label.parse_label) { - return lCOST_SENSITIVE; - } else if (lp->parse_label == CB::cb_label.parse_label) { - return lCONTEXTUAL_BANDIT; - } else { - cerr << "unsupported label parser used" << endl; throw exception(); + if (lp->parse_label == simple_label.parse_label) + { return lBINARY; + } + else if (lp->parse_label == MULTICLASS::mc_label.parse_label) + { return lMULTICLASS; + } + else if (lp->parse_label == COST_SENSITIVE::cs_label.parse_label) + { return lCOST_SENSITIVE; + } + else if (lp->parse_label == CB::cb_label.parse_label) + { return lCONTEXTUAL_BANDIT; + } + else + { cerr << "unsupported label parser used" << endl; throw exception(); } } @@ -239,9 +243,9 @@ void ex_push_feature_list(example_ptr ec, vw_ptr vw, unsigned char ns, py::list& else { cerr << "warning: malformed feature in list" << endl; continue; } } if (got) - { ec->feature_space[ns].push_back(f.x, f.weight_index); + { ec->feature_space[ns].push_back(f.x, f.weight_index); count++; - sum_sq += f.x*f.x; + sum_sq += f.x*f.x; } } } @@ -260,13 +264,11 @@ void ex_ensure_namespace_exists(example_ptr ec, unsigned char ns) } void ex_push_dictionary(example_ptr ec, vw_ptr vw, py::dict& dict) -{ - const py::object objectKeys = py::object(py::handle<>(PyObject_GetIter(dict.keys().ptr()))); +{ const py::object objectKeys = py::object(py::handle<>(PyObject_GetIter(dict.keys().ptr()))); const py::object objectVals = py::object(py::handle<>(PyObject_GetIter(dict.values().ptr()))); unsigned long ulCount = boost::python::extract(dict.attr("__len__")()); for (size_t u=0; u(PyIter_Next(objectKeys.ptr()))); + { py::object objectKey = py::object(py::handle<>(PyIter_Next(objectKeys.ptr()))); py::object objectVal = py::object(py::handle<>(PyIter_Next(objectVals.ptr()))); char chCheckKey = objectKey.ptr()->ob_type->tp_name[0]; @@ -333,7 +335,7 @@ void unsetup_example(vw_ptr vwP, example_ptr ae) } if (all.add_constant) - { ae->feature_space[constant_namespace].erase(); + { ae->feature_space[constant_namespace].erase(); int hit_constant = -1; size_t N = ae->indices.size(); for (size_t i=0; il.multi.label; } float ex_get_multiclass_weight(example_ptr ec) { return ec->l.multi.weight; } uint32_t ex_get_multiclass_prediction(example_ptr ec) { return ec->pred.multiclass; } -py::list ex_get_scalars(example_ptr ec) { - py::list values; +py::list ex_get_scalars(example_ptr ec) +{ py::list values; v_array scalars = ec->pred.scalars; - for (float s : scalars) { - values.append(s); + for (float s : scalars) + { values.append(s); } return values; } -py::list ex_get_action_scores(example_ptr ec) { - py::list values; +py::list ex_get_action_scores(example_ptr ec) +{ py::list values; v_array scores = ec->pred.a_s; - for (ACTION_SCORE::action_score s : scores) { - values.append(s.score); + for (ACTION_SCORE::action_score s : scores) + { values.append(s.score); } return values; } -py::list ex_get_multilabel_predictions(example_ptr ec) { - py::list values; +py::list ex_get_multilabel_predictions(example_ptr ec) +{ py::list values; MULTILABEL::labels labels = ec->pred.multilabels; - for (uint32_t l : labels.label_v) { - values.append(l); + for (uint32_t l : labels.label_v) + { values.append(l); } return values; } diff --git a/rapidjson/example/capitalize/capitalize.cpp b/rapidjson/example/capitalize/capitalize.cpp index 7da37e9c504..9c7df69a51f 100644 --- a/rapidjson/example/capitalize/capitalize.cpp +++ b/rapidjson/example/capitalize/capitalize.cpp @@ -1,6 +1,6 @@ // JSON condenser example -// This example parses JSON from stdin with validation, +// This example parses JSON from stdin with validation, // and re-output the JSON content to stdout with all string capitalized, and without whitespace. #include "rapidjson/reader.h" @@ -14,54 +14,54 @@ using namespace rapidjson; template -struct CapitalizeFilter { - CapitalizeFilter(OutputHandler& out) : out_(out), buffer_() {} +struct CapitalizeFilter +{ CapitalizeFilter(OutputHandler& out) : out_(out), buffer_() {} - bool Null() { return out_.Null(); } - bool Bool(bool b) { return out_.Bool(b); } - bool Int(int i) { return out_.Int(i); } - bool Uint(unsigned u) { return out_.Uint(u); } - bool Int64(int64_t i) { return out_.Int64(i); } - bool Uint64(uint64_t u) { return out_.Uint64(u); } - bool Double(double d) { return out_.Double(d); } - bool RawNumber(const char* str, SizeType length, bool copy) { return out_.RawNumber(str, length, copy); } - bool String(const char* str, SizeType length, bool) { - buffer_.clear(); - for (SizeType i = 0; i < length; i++) - buffer_.push_back(static_cast(std::toupper(str[i]))); - return out_.String(&buffer_.front(), length, true); // true = output handler need to copy the string - } - bool StartObject() { return out_.StartObject(); } - bool Key(const char* str, SizeType length, bool copy) { return String(str, length, copy); } - bool EndObject(SizeType memberCount) { return out_.EndObject(memberCount); } - bool StartArray() { return out_.StartArray(); } - bool EndArray(SizeType elementCount) { return out_.EndArray(elementCount); } + bool Null() { return out_.Null(); } + bool Bool(bool b) { return out_.Bool(b); } + bool Int(int i) { return out_.Int(i); } + bool Uint(unsigned u) { return out_.Uint(u); } + bool Int64(int64_t i) { return out_.Int64(i); } + bool Uint64(uint64_t u) { return out_.Uint64(u); } + bool Double(double d) { return out_.Double(d); } + bool RawNumber(const char* str, SizeType length, bool copy) { return out_.RawNumber(str, length, copy); } + bool String(const char* str, SizeType length, bool) + { buffer_.clear(); + for (SizeType i = 0; i < length; i++) + buffer_.push_back(static_cast(std::toupper(str[i]))); + return out_.String(&buffer_.front(), length, true); // true = output handler need to copy the string + } + bool StartObject() { return out_.StartObject(); } + bool Key(const char* str, SizeType length, bool copy) { return String(str, length, copy); } + bool EndObject(SizeType memberCount) { return out_.EndObject(memberCount); } + bool StartArray() { return out_.StartArray(); } + bool EndArray(SizeType elementCount) { return out_.EndArray(elementCount); } - OutputHandler& out_; - std::vector buffer_; + OutputHandler& out_; + std::vector buffer_; private: - CapitalizeFilter(const CapitalizeFilter&); - CapitalizeFilter& operator=(const CapitalizeFilter&); + CapitalizeFilter(const CapitalizeFilter&); + CapitalizeFilter& operator=(const CapitalizeFilter&); }; -int main(int, char*[]) { - // Prepare JSON reader and input stream. - Reader reader; - char readBuffer[65536]; - FileReadStream is(stdin, readBuffer, sizeof(readBuffer)); +int main(int, char*[]) +{ // Prepare JSON reader and input stream. + Reader reader; + char readBuffer[65536]; + FileReadStream is(stdin, readBuffer, sizeof(readBuffer)); - // Prepare JSON writer and output stream. - char writeBuffer[65536]; - FileWriteStream os(stdout, writeBuffer, sizeof(writeBuffer)); - Writer writer(os); + // Prepare JSON writer and output stream. + char writeBuffer[65536]; + FileWriteStream os(stdout, writeBuffer, sizeof(writeBuffer)); + Writer writer(os); - // JSON reader parse from the input stream and let writer generate the output. - CapitalizeFilter > filter(writer); - if (!reader.Parse(is, filter)) { - fprintf(stderr, "\nError(%u): %s\n", static_cast(reader.GetErrorOffset()), GetParseError_En(reader.GetParseErrorCode())); - return 1; - } + // JSON reader parse from the input stream and let writer generate the output. + CapitalizeFilter > filter(writer); + if (!reader.Parse(is, filter)) + { fprintf(stderr, "\nError(%u): %s\n", static_cast(reader.GetErrorOffset()), GetParseError_En(reader.GetParseErrorCode())); + return 1; + } - return 0; + return 0; } diff --git a/rapidjson/example/condense/condense.cpp b/rapidjson/example/condense/condense.cpp index 46dc3504397..83002deb96f 100644 --- a/rapidjson/example/condense/condense.cpp +++ b/rapidjson/example/condense/condense.cpp @@ -1,6 +1,6 @@ // JSON condenser example -// This example parses JSON text from stdin with validation, +// This example parses JSON text from stdin with validation, // and re-output the JSON content to stdout without whitespace. #include "rapidjson/reader.h" @@ -11,22 +11,22 @@ using namespace rapidjson; -int main(int, char*[]) { - // Prepare JSON reader and input stream. - Reader reader; - char readBuffer[65536]; - FileReadStream is(stdin, readBuffer, sizeof(readBuffer)); +int main(int, char*[]) +{ // Prepare JSON reader and input stream. + Reader reader; + char readBuffer[65536]; + FileReadStream is(stdin, readBuffer, sizeof(readBuffer)); - // Prepare JSON writer and output stream. - char writeBuffer[65536]; - FileWriteStream os(stdout, writeBuffer, sizeof(writeBuffer)); - Writer writer(os); + // Prepare JSON writer and output stream. + char writeBuffer[65536]; + FileWriteStream os(stdout, writeBuffer, sizeof(writeBuffer)); + Writer writer(os); - // JSON reader parse from the input stream and let writer generate the output. - if (!reader.Parse(is, writer)) { - fprintf(stderr, "\nError(%u): %s\n", static_cast(reader.GetErrorOffset()), GetParseError_En(reader.GetParseErrorCode())); - return 1; - } + // JSON reader parse from the input stream and let writer generate the output. + if (!reader.Parse(is, writer)) + { fprintf(stderr, "\nError(%u): %s\n", static_cast(reader.GetErrorOffset()), GetParseError_En(reader.GetParseErrorCode())); + return 1; + } - return 0; + return 0; } diff --git a/rapidjson/example/filterkey/filterkey.cpp b/rapidjson/example/filterkey/filterkey.cpp index c34a050dc8d..4a8fccaf3f3 100644 --- a/rapidjson/example/filterkey/filterkey.cpp +++ b/rapidjson/example/filterkey/filterkey.cpp @@ -15,121 +15,122 @@ using namespace rapidjson; // This handler forwards event into an output handler, with filtering the descendent events of specified key. template -class FilterKeyHandler { +class FilterKeyHandler +{ public: - typedef char Ch; - - FilterKeyHandler(OutputHandler& outputHandler, const Ch* keyString, SizeType keyLength) : - outputHandler_(outputHandler), keyString_(keyString), keyLength_(keyLength), filterValueDepth_(), filteredKeyCount_() - {} - - bool Null() { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Null() && EndValue(); } - bool Bool(bool b) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Bool(b) && EndValue(); } - bool Int(int i) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Int(i) && EndValue(); } - bool Uint(unsigned u) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Uint(u) && EndValue(); } - bool Int64(int64_t i) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Int64(i) && EndValue(); } - bool Uint64(uint64_t u) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Uint64(u) && EndValue(); } - bool Double(double d) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Double(d) && EndValue(); } - bool RawNumber(const Ch* str, SizeType len, bool copy) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.RawNumber(str, len, copy) && EndValue(); } - bool String (const Ch* str, SizeType len, bool copy) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.String (str, len, copy) && EndValue(); } - - bool StartObject() { - if (filterValueDepth_ > 0) { - filterValueDepth_++; - return true; - } - else { - filteredKeyCount_.push(0); - return outputHandler_.StartObject(); - } + typedef char Ch; + + FilterKeyHandler(OutputHandler& outputHandler, const Ch* keyString, SizeType keyLength) : + outputHandler_(outputHandler), keyString_(keyString), keyLength_(keyLength), filterValueDepth_(), filteredKeyCount_() + {} + + bool Null() { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Null() && EndValue(); } + bool Bool(bool b) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Bool(b) && EndValue(); } + bool Int(int i) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Int(i) && EndValue(); } + bool Uint(unsigned u) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Uint(u) && EndValue(); } + bool Int64(int64_t i) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Int64(i) && EndValue(); } + bool Uint64(uint64_t u) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Uint64(u) && EndValue(); } + bool Double(double d) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Double(d) && EndValue(); } + bool RawNumber(const Ch* str, SizeType len, bool copy) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.RawNumber(str, len, copy) && EndValue(); } + bool String (const Ch* str, SizeType len, bool copy) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.String (str, len, copy) && EndValue(); } + + bool StartObject() + { if (filterValueDepth_ > 0) + { filterValueDepth_++; + return true; } - - bool Key(const Ch* str, SizeType len, bool copy) { - if (filterValueDepth_ > 0) - return true; - else if (len == keyLength_ && std::memcmp(str, keyString_, len) == 0) { - filterValueDepth_ = 1; - return true; - } - else { - ++filteredKeyCount_.top(); - return outputHandler_.Key(str, len, copy); - } + else + { filteredKeyCount_.push(0); + return outputHandler_.StartObject(); } - - bool EndObject(SizeType) { - if (filterValueDepth_ > 0) { - filterValueDepth_--; - return EndValue(); - } - else { - // Use our own filtered memberCount - SizeType memberCount = filteredKeyCount_.top(); - filteredKeyCount_.pop(); - return outputHandler_.EndObject(memberCount) && EndValue(); - } + } + + bool Key(const Ch* str, SizeType len, bool copy) + { if (filterValueDepth_ > 0) + return true; + else if (len == keyLength_ && std::memcmp(str, keyString_, len) == 0) + { filterValueDepth_ = 1; + return true; } - - bool StartArray() { - if (filterValueDepth_ > 0) { - filterValueDepth_++; - return true; - } - else - return outputHandler_.StartArray(); + else + { ++filteredKeyCount_.top(); + return outputHandler_.Key(str, len, copy); } + } - bool EndArray(SizeType elementCount) { - if (filterValueDepth_ > 0) { - filterValueDepth_--; - return EndValue(); - } - else - return outputHandler_.EndArray(elementCount) && EndValue(); + bool EndObject(SizeType) + { if (filterValueDepth_ > 0) + { filterValueDepth_--; + return EndValue(); } - -private: - FilterKeyHandler(const FilterKeyHandler&); - FilterKeyHandler& operator=(const FilterKeyHandler&); - - bool EndValue() { - if (filterValueDepth_ == 1) // Just at the end of value after filtered key - filterValueDepth_ = 0; - return true; + else + { // Use our own filtered memberCount + SizeType memberCount = filteredKeyCount_.top(); + filteredKeyCount_.pop(); + return outputHandler_.EndObject(memberCount) && EndValue(); } - - OutputHandler& outputHandler_; - const char* keyString_; - const SizeType keyLength_; - unsigned filterValueDepth_; - std::stack filteredKeyCount_; -}; + } -int main(int argc, char* argv[]) { - if (argc != 2) { - fprintf(stderr, "filterkey key < input.json > output.json\n"); - return 1; + bool StartArray() + { if (filterValueDepth_ > 0) + { filterValueDepth_++; + return true; } - - // Prepare JSON reader and input stream. - Reader reader; - char readBuffer[65536]; - FileReadStream is(stdin, readBuffer, sizeof(readBuffer)); - - // Prepare JSON writer and output stream. - char writeBuffer[65536]; - FileWriteStream os(stdout, writeBuffer, sizeof(writeBuffer)); - Writer writer(os); - - // Prepare Filter - FilterKeyHandler > filter(writer, argv[1], static_cast(strlen(argv[1]))); - - // JSON reader parse from the input stream, filter handler filters the events, and forward to writer. - // i.e. the events flow is: reader -> filter -> writer - if (!reader.Parse(is, filter)) { - fprintf(stderr, "\nError(%u): %s\n", static_cast(reader.GetErrorOffset()), GetParseError_En(reader.GetParseErrorCode())); - return 1; + else + return outputHandler_.StartArray(); + } + + bool EndArray(SizeType elementCount) + { if (filterValueDepth_ > 0) + { filterValueDepth_--; + return EndValue(); } + else + return outputHandler_.EndArray(elementCount) && EndValue(); + } + +private: + FilterKeyHandler(const FilterKeyHandler&); + FilterKeyHandler& operator=(const FilterKeyHandler&); + + bool EndValue() + { if (filterValueDepth_ == 1) // Just at the end of value after filtered key + filterValueDepth_ = 0; + return true; + } + + OutputHandler& outputHandler_; + const char* keyString_; + const SizeType keyLength_; + unsigned filterValueDepth_; + std::stack filteredKeyCount_; +}; - return 0; +int main(int argc, char* argv[]) +{ if (argc != 2) + { fprintf(stderr, "filterkey key < input.json > output.json\n"); + return 1; + } + + // Prepare JSON reader and input stream. + Reader reader; + char readBuffer[65536]; + FileReadStream is(stdin, readBuffer, sizeof(readBuffer)); + + // Prepare JSON writer and output stream. + char writeBuffer[65536]; + FileWriteStream os(stdout, writeBuffer, sizeof(writeBuffer)); + Writer writer(os); + + // Prepare Filter + FilterKeyHandler > filter(writer, argv[1], static_cast(strlen(argv[1]))); + + // JSON reader parse from the input stream, filter handler filters the events, and forward to writer. + // i.e. the events flow is: reader -> filter -> writer + if (!reader.Parse(is, filter)) + { fprintf(stderr, "\nError(%u): %s\n", static_cast(reader.GetErrorOffset()), GetParseError_En(reader.GetParseErrorCode())); + return 1; + } + + return 0; } diff --git a/rapidjson/example/filterkeydom/filterkeydom.cpp b/rapidjson/example/filterkeydom/filterkeydom.cpp index 732cc81f136..e06380b1128 100644 --- a/rapidjson/example/filterkeydom/filterkeydom.cpp +++ b/rapidjson/example/filterkeydom/filterkeydom.cpp @@ -16,155 +16,157 @@ using namespace rapidjson; // This handler forwards event into an output handler, with filtering the descendent events of specified key. template -class FilterKeyHandler { +class FilterKeyHandler +{ public: - typedef char Ch; - - FilterKeyHandler(OutputHandler& outputHandler, const Ch* keyString, SizeType keyLength) : - outputHandler_(outputHandler), keyString_(keyString), keyLength_(keyLength), filterValueDepth_(), filteredKeyCount_() - {} - - bool Null() { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Null() && EndValue(); } - bool Bool(bool b) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Bool(b) && EndValue(); } - bool Int(int i) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Int(i) && EndValue(); } - bool Uint(unsigned u) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Uint(u) && EndValue(); } - bool Int64(int64_t i) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Int64(i) && EndValue(); } - bool Uint64(uint64_t u) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Uint64(u) && EndValue(); } - bool Double(double d) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Double(d) && EndValue(); } - bool RawNumber(const Ch* str, SizeType len, bool copy) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.RawNumber(str, len, copy) && EndValue(); } - bool String (const Ch* str, SizeType len, bool copy) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.String (str, len, copy) && EndValue(); } - - bool StartObject() { - if (filterValueDepth_ > 0) { - filterValueDepth_++; - return true; - } - else { - filteredKeyCount_.push(0); - return outputHandler_.StartObject(); - } + typedef char Ch; + + FilterKeyHandler(OutputHandler& outputHandler, const Ch* keyString, SizeType keyLength) : + outputHandler_(outputHandler), keyString_(keyString), keyLength_(keyLength), filterValueDepth_(), filteredKeyCount_() + {} + + bool Null() { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Null() && EndValue(); } + bool Bool(bool b) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Bool(b) && EndValue(); } + bool Int(int i) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Int(i) && EndValue(); } + bool Uint(unsigned u) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Uint(u) && EndValue(); } + bool Int64(int64_t i) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Int64(i) && EndValue(); } + bool Uint64(uint64_t u) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Uint64(u) && EndValue(); } + bool Double(double d) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.Double(d) && EndValue(); } + bool RawNumber(const Ch* str, SizeType len, bool copy) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.RawNumber(str, len, copy) && EndValue(); } + bool String (const Ch* str, SizeType len, bool copy) { return filterValueDepth_ > 0 ? EndValue() : outputHandler_.String (str, len, copy) && EndValue(); } + + bool StartObject() + { if (filterValueDepth_ > 0) + { filterValueDepth_++; + return true; } - - bool Key(const Ch* str, SizeType len, bool copy) { - if (filterValueDepth_ > 0) - return true; - else if (len == keyLength_ && std::memcmp(str, keyString_, len) == 0) { - filterValueDepth_ = 1; - return true; - } - else { - ++filteredKeyCount_.top(); - return outputHandler_.Key(str, len, copy); - } + else + { filteredKeyCount_.push(0); + return outputHandler_.StartObject(); } - - bool EndObject(SizeType) { - if (filterValueDepth_ > 0) { - filterValueDepth_--; - return EndValue(); - } - else { - // Use our own filtered memberCount - SizeType memberCount = filteredKeyCount_.top(); - filteredKeyCount_.pop(); - return outputHandler_.EndObject(memberCount) && EndValue(); - } + } + + bool Key(const Ch* str, SizeType len, bool copy) + { if (filterValueDepth_ > 0) + return true; + else if (len == keyLength_ && std::memcmp(str, keyString_, len) == 0) + { filterValueDepth_ = 1; + return true; } - - bool StartArray() { - if (filterValueDepth_ > 0) { - filterValueDepth_++; - return true; - } - else - return outputHandler_.StartArray(); + else + { ++filteredKeyCount_.top(); + return outputHandler_.Key(str, len, copy); } + } - bool EndArray(SizeType elementCount) { - if (filterValueDepth_ > 0) { - filterValueDepth_--; - return EndValue(); - } - else - return outputHandler_.EndArray(elementCount) && EndValue(); + bool EndObject(SizeType) + { if (filterValueDepth_ > 0) + { filterValueDepth_--; + return EndValue(); } + else + { // Use our own filtered memberCount + SizeType memberCount = filteredKeyCount_.top(); + filteredKeyCount_.pop(); + return outputHandler_.EndObject(memberCount) && EndValue(); + } + } -private: - FilterKeyHandler(const FilterKeyHandler&); - FilterKeyHandler& operator=(const FilterKeyHandler&); - - bool EndValue() { - if (filterValueDepth_ == 1) // Just at the end of value after filtered key - filterValueDepth_ = 0; - return true; + bool StartArray() + { if (filterValueDepth_ > 0) + { filterValueDepth_++; + return true; } + else + return outputHandler_.StartArray(); + } + + bool EndArray(SizeType elementCount) + { if (filterValueDepth_ > 0) + { filterValueDepth_--; + return EndValue(); + } + else + return outputHandler_.EndArray(elementCount) && EndValue(); + } - OutputHandler& outputHandler_; - const char* keyString_; - const SizeType keyLength_; - unsigned filterValueDepth_; - std::stack filteredKeyCount_; +private: + FilterKeyHandler(const FilterKeyHandler&); + FilterKeyHandler& operator=(const FilterKeyHandler&); + + bool EndValue() + { if (filterValueDepth_ == 1) // Just at the end of value after filtered key + filterValueDepth_ = 0; + return true; + } + + OutputHandler& outputHandler_; + const char* keyString_; + const SizeType keyLength_; + unsigned filterValueDepth_; + std::stack filteredKeyCount_; }; // Implements a generator for Document::Populate() template -class FilterKeyReader { +class FilterKeyReader +{ public: - typedef char Ch; - - FilterKeyReader(InputStream& is, const Ch* keyString, SizeType keyLength) : - is_(is), keyString_(keyString), keyLength_(keyLength), parseResult_() - {} - - // SAX event flow: reader -> filter -> handler - template - bool operator()(Handler& handler) { - FilterKeyHandler filter(handler, keyString_, keyLength_); - Reader reader; - parseResult_ = reader.Parse(is_, filter); - return parseResult_; - } + typedef char Ch; - const ParseResult& GetParseResult() const { return parseResult_; } + FilterKeyReader(InputStream& is, const Ch* keyString, SizeType keyLength) : + is_(is), keyString_(keyString), keyLength_(keyLength), parseResult_() + {} -private: - FilterKeyReader(const FilterKeyReader&); - FilterKeyReader& operator=(const FilterKeyReader&); + // SAX event flow: reader -> filter -> handler + template + bool operator()(Handler& handler) + { FilterKeyHandler filter(handler, keyString_, keyLength_); + Reader reader; + parseResult_ = reader.Parse(is_, filter); + return parseResult_; + } - InputStream& is_; - const char* keyString_; - const SizeType keyLength_; - ParseResult parseResult_; -}; - -int main(int argc, char* argv[]) { - if (argc != 2) { - fprintf(stderr, "filterkeydom key < input.json > output.json\n"); - return 1; - } - - // Prepare input stream. - char readBuffer[65536]; - FileReadStream is(stdin, readBuffer, sizeof(readBuffer)); + const ParseResult& GetParseResult() const { return parseResult_; } - // Prepare Filter - FilterKeyReader reader(is, argv[1], static_cast(strlen(argv[1]))); - - // Populates the filtered events from reader - Document document; - document.Populate(reader); - ParseResult pr = reader.GetParseResult(); - if (!pr) { - fprintf(stderr, "\nError(%u): %s\n", static_cast(pr.Offset()), GetParseError_En(pr.Code())); - return 1; - } +private: + FilterKeyReader(const FilterKeyReader&); + FilterKeyReader& operator=(const FilterKeyReader&); - // Prepare JSON writer and output stream. - char writeBuffer[65536]; - FileWriteStream os(stdout, writeBuffer, sizeof(writeBuffer)); - Writer writer(os); + InputStream& is_; + const char* keyString_; + const SizeType keyLength_; + ParseResult parseResult_; +}; - // Write the document to standard output - document.Accept(writer); - return 0; +int main(int argc, char* argv[]) +{ if (argc != 2) + { fprintf(stderr, "filterkeydom key < input.json > output.json\n"); + return 1; + } + + // Prepare input stream. + char readBuffer[65536]; + FileReadStream is(stdin, readBuffer, sizeof(readBuffer)); + + // Prepare Filter + FilterKeyReader reader(is, argv[1], static_cast(strlen(argv[1]))); + + // Populates the filtered events from reader + Document document; + document.Populate(reader); + ParseResult pr = reader.GetParseResult(); + if (!pr) + { fprintf(stderr, "\nError(%u): %s\n", static_cast(pr.Offset()), GetParseError_En(pr.Code())); + return 1; + } + + // Prepare JSON writer and output stream. + char writeBuffer[65536]; + FileWriteStream os(stdout, writeBuffer, sizeof(writeBuffer)); + Writer writer(os); + + // Write the document to standard output + document.Accept(writer); + return 0; } diff --git a/rapidjson/example/jsonx/jsonx.cpp b/rapidjson/example/jsonx/jsonx.cpp index 1346b578c39..50a64609d49 100644 --- a/rapidjson/example/jsonx/jsonx.cpp +++ b/rapidjson/example/jsonx/jsonx.cpp @@ -1,7 +1,7 @@ // JSON to JSONx conversion exmaple, using SAX API. // JSONx is an IBM standard format to represent JSON as XML. // https://www-01.ibm.com/support/knowledgecenter/SS9H2Y_7.1.0/com.ibm.dp.doc/json_jsonx.html -// This example parses JSON text from stdin with validation, +// This example parses JSON text from stdin with validation, // and convert to JSONx format to stdout. // Need compile with -D__STDC_FORMAT_MACROS for defining PRId64 and PRIu64 macros. @@ -16,192 +16,194 @@ using namespace rapidjson; // For simplicity, this example only read/write in UTF-8 encoding template -class JsonxWriter { +class JsonxWriter +{ public: - JsonxWriter(OutputStream& os) : os_(os), name_(), level_(0), hasName_(false) { - } - - bool Null() { - return WriteStartElement("null", true); - } - - bool Bool(bool b) { - return - WriteStartElement("boolean") && - WriteString(b ? "true" : "false") && - WriteEndElement("boolean"); - } - - bool Int(int i) { - char buffer[12]; - return WriteNumberElement(buffer, sprintf(buffer, "%d", i)); - } - - bool Uint(unsigned i) { - char buffer[11]; - return WriteNumberElement(buffer, sprintf(buffer, "%u", i)); - } - - bool Int64(int64_t i) { - char buffer[21]; - return WriteNumberElement(buffer, sprintf(buffer, "%" PRId64, i)); - } - - bool Uint64(uint64_t i) { - char buffer[21]; - return WriteNumberElement(buffer, sprintf(buffer, "%" PRIu64, i)); - } - - bool Double(double d) { - char buffer[30]; - return WriteNumberElement(buffer, sprintf(buffer, "%.17g", d)); - } - - bool RawNumber(const char* str, SizeType length, bool) { - return - WriteStartElement("number") && - WriteEscapedText(str, length) && - WriteEndElement("number"); - } - - bool String(const char* str, SizeType length, bool) { - return - WriteStartElement("string") && - WriteEscapedText(str, length) && - WriteEndElement("string"); - } - - bool StartObject() { - return WriteStartElement("object"); - } - - bool Key(const char* str, SizeType length, bool) { - // backup key to name_ - name_.Clear(); - for (SizeType i = 0; i < length; i++) - name_.Put(str[i]); - hasName_ = true; - return true; - } - - bool EndObject(SizeType) { - return WriteEndElement("object"); - } - - bool StartArray() { - return WriteStartElement("array"); - } - - bool EndArray(SizeType) { - return WriteEndElement("array"); - } + JsonxWriter(OutputStream& os) : os_(os), name_(), level_(0), hasName_(false) + { + } + + bool Null() + { return WriteStartElement("null", true); + } + + bool Bool(bool b) + { return + WriteStartElement("boolean") && + WriteString(b ? "true" : "false") && + WriteEndElement("boolean"); + } + + bool Int(int i) + { char buffer[12]; + return WriteNumberElement(buffer, sprintf(buffer, "%d", i)); + } + + bool Uint(unsigned i) + { char buffer[11]; + return WriteNumberElement(buffer, sprintf(buffer, "%u", i)); + } + + bool Int64(int64_t i) + { char buffer[21]; + return WriteNumberElement(buffer, sprintf(buffer, "%" PRId64, i)); + } + + bool Uint64(uint64_t i) + { char buffer[21]; + return WriteNumberElement(buffer, sprintf(buffer, "%" PRIu64, i)); + } + + bool Double(double d) + { char buffer[30]; + return WriteNumberElement(buffer, sprintf(buffer, "%.17g", d)); + } + + bool RawNumber(const char* str, SizeType length, bool) + { return + WriteStartElement("number") && + WriteEscapedText(str, length) && + WriteEndElement("number"); + } + + bool String(const char* str, SizeType length, bool) + { return + WriteStartElement("string") && + WriteEscapedText(str, length) && + WriteEndElement("string"); + } + + bool StartObject() + { return WriteStartElement("object"); + } + + bool Key(const char* str, SizeType length, bool) + { // backup key to name_ + name_.Clear(); + for (SizeType i = 0; i < length; i++) + name_.Put(str[i]); + hasName_ = true; + return true; + } + + bool EndObject(SizeType) + { return WriteEndElement("object"); + } + + bool StartArray() + { return WriteStartElement("array"); + } + + bool EndArray(SizeType) + { return WriteEndElement("array"); + } private: - bool WriteString(const char* s) { - while (*s) - os_.Put(*s++); - return true; - } - - bool WriteEscapedAttributeValue(const char* s, size_t length) { - for (size_t i = 0; i < length; i++) { - switch (s[i]) { - case '&': WriteString("&"); break; - case '<': WriteString("<"); break; - case '"': WriteString("""); break; - default: os_.Put(s[i]); break; - } - } - return true; - } - - bool WriteEscapedText(const char* s, size_t length) { - for (size_t i = 0; i < length; i++) { - switch (s[i]) { - case '&': WriteString("&"); break; - case '<': WriteString("<"); break; - default: os_.Put(s[i]); break; - } - } - return true; - } - - bool WriteStartElement(const char* type, bool emptyElement = false) { - if (level_ == 0) - if (!WriteString("")) - return false; - - if (!WriteString(""); - else { - level_++; - return WriteString(">"); - } - } - - bool WriteEndElement(const char* type) { - if (!WriteString("")) - return false; - - // For the last end tag, flush the output stream. - if (--level_ == 0) - os_.Flush(); - - return true; - } + bool WriteString(const char* s) + { while (*s) + os_.Put(*s++); + return true; + } + + bool WriteEscapedAttributeValue(const char* s, size_t length) + { for (size_t i = 0; i < length; i++) + { switch (s[i]) + { case '&': WriteString("&"); break; + case '<': WriteString("<"); break; + case '"': WriteString("""); break; + default: os_.Put(s[i]); break; + } + } + return true; + } + + bool WriteEscapedText(const char* s, size_t length) + { for (size_t i = 0; i < length; i++) + { switch (s[i]) + { case '&': WriteString("&"); break; + case '<': WriteString("<"); break; + default: os_.Put(s[i]); break; + } + } + return true; + } + + bool WriteStartElement(const char* type, bool emptyElement = false) + { if (level_ == 0) + if (!WriteString("")) + return false; + + if (!WriteString(""); + else + { level_++; + return WriteString(">"); + } + } + + bool WriteEndElement(const char* type) + { if (!WriteString("")) + return false; + + // For the last end tag, flush the output stream. + if (--level_ == 0) + os_.Flush(); + + return true; + } + + bool WriteNumberElement(const char* buffer, int length) + { if (!WriteStartElement("number")) + return false; + for (int j = 0; j < length; j++) + os_.Put(buffer[j]); + return WriteEndElement("number"); + } + + OutputStream& os_; + StringBuffer name_; + unsigned level_; + bool hasName_; +}; - bool WriteNumberElement(const char* buffer, int length) { - if (!WriteStartElement("number")) - return false; - for (int j = 0; j < length; j++) - os_.Put(buffer[j]); - return WriteEndElement("number"); - } +int main(int, char*[]) +{ // Prepare JSON reader and input stream. + Reader reader; + char readBuffer[65536]; + FileReadStream is(stdin, readBuffer, sizeof(readBuffer)); - OutputStream& os_; - StringBuffer name_; - unsigned level_; - bool hasName_; -}; + // Prepare JSON writer and output stream. + char writeBuffer[65536]; + FileWriteStream os(stdout, writeBuffer, sizeof(writeBuffer)); + JsonxWriter writer(os); -int main(int, char*[]) { - // Prepare JSON reader and input stream. - Reader reader; - char readBuffer[65536]; - FileReadStream is(stdin, readBuffer, sizeof(readBuffer)); - - // Prepare JSON writer and output stream. - char writeBuffer[65536]; - FileWriteStream os(stdout, writeBuffer, sizeof(writeBuffer)); - JsonxWriter writer(os); - - // JSON reader parse from the input stream and let writer generate the output. - if (!reader.Parse(is, writer)) { - fprintf(stderr, "\nError(%u): %s\n", static_cast(reader.GetErrorOffset()), GetParseError_En(reader.GetParseErrorCode())); - return 1; - } + // JSON reader parse from the input stream and let writer generate the output. + if (!reader.Parse(is, writer)) + { fprintf(stderr, "\nError(%u): %s\n", static_cast(reader.GetErrorOffset()), GetParseError_En(reader.GetParseErrorCode())); + return 1; + } - return 0; + return 0; } diff --git a/rapidjson/example/messagereader/messagereader.cpp b/rapidjson/example/messagereader/messagereader.cpp index 3399bc9400e..ff163ae3e65 100644 --- a/rapidjson/example/messagereader/messagereader.cpp +++ b/rapidjson/example/messagereader/messagereader.cpp @@ -22,33 +22,33 @@ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(switch-enum) #endif -struct MessageHandler - : public BaseReaderHandler, MessageHandler> { - MessageHandler() : messages_(), state_(kExpectObjectStart), name_() {} - - bool StartObject() { - switch (state_) { - case kExpectObjectStart: - state_ = kExpectNameOrObjectEnd; - return true; + struct MessageHandler + : public BaseReaderHandler, MessageHandler> +{ MessageHandler() : messages_(), state_(kExpectObjectStart), name_() {} + + bool StartObject() + { switch (state_) + { case kExpectObjectStart: + state_ = kExpectNameOrObjectEnd; + return true; default: - return false; - } + return false; + } } - bool String(const char* str, SizeType length, bool) { - switch (state_) { - case kExpectNameOrObjectEnd: - name_ = string(str, length); - state_ = kExpectValue; - return true; + bool String(const char* str, SizeType length, bool) + { switch (state_) + { case kExpectNameOrObjectEnd: + name_ = string(str, length); + state_ = kExpectValue; + return true; case kExpectValue: - messages_.insert(MessageMap::value_type(name_, string(str, length))); - state_ = kExpectNameOrObjectEnd; - return true; + messages_.insert(MessageMap::value_type(name_, string(str, length))); + state_ = kExpectNameOrObjectEnd; + return true; default: - return false; - } + return false; + } } bool EndObject(SizeType) { return state_ == kExpectNameOrObjectEnd; } @@ -56,13 +56,13 @@ struct MessageHandler bool Default() { return false; } // All other events are invalid. MessageMap messages_; - enum State { - kExpectObjectStart, - kExpectNameOrObjectEnd, - kExpectValue - }state_; + enum State + { kExpectObjectStart, + kExpectNameOrObjectEnd, + kExpectValue + } state_; std::string name_; -}; + }; #if defined(__GNUC__) RAPIDJSON_DIAG_POP @@ -72,34 +72,34 @@ RAPIDJSON_DIAG_POP RAPIDJSON_DIAG_POP #endif -static void ParseMessages(const char* json, MessageMap& messages) { - Reader reader; - MessageHandler handler; - StringStream ss(json); - if (reader.Parse(ss, handler)) - messages.swap(handler.messages_); // Only change it if success. - else { - ParseErrorCode e = reader.GetParseErrorCode(); - size_t o = reader.GetErrorOffset(); - cout << "Error: " << GetParseError_En(e) << endl;; - cout << " at offset " << o << " near '" << string(json).substr(o, 10) << "...'" << endl; - } +static void ParseMessages(const char* json, MessageMap& messages) +{ Reader reader; + MessageHandler handler; + StringStream ss(json); + if (reader.Parse(ss, handler)) + messages.swap(handler.messages_); // Only change it if success. + else + { ParseErrorCode e = reader.GetParseErrorCode(); + size_t o = reader.GetErrorOffset(); + cout << "Error: " << GetParseError_En(e) << endl;; + cout << " at offset " << o << " near '" << string(json).substr(o, 10) << "...'" << endl; + } } -int main() { - MessageMap messages; +int main() +{ MessageMap messages; - const char* json1 = "{ \"greeting\" : \"Hello!\", \"farewell\" : \"bye-bye!\" }"; - cout << json1 << endl; - ParseMessages(json1, messages); + const char* json1 = "{ \"greeting\" : \"Hello!\", \"farewell\" : \"bye-bye!\" }"; + cout << json1 << endl; + ParseMessages(json1, messages); - for (MessageMap::const_iterator itr = messages.begin(); itr != messages.end(); ++itr) - cout << itr->first << ": " << itr->second << endl; + for (MessageMap::const_iterator itr = messages.begin(); itr != messages.end(); ++itr) + cout << itr->first << ": " << itr->second << endl; - cout << endl << "Parse a JSON with invalid schema." << endl; - const char* json2 = "{ \"greeting\" : \"Hello!\", \"farewell\" : \"bye-bye!\", \"foo\" : {} }"; - cout << json2 << endl; - ParseMessages(json2, messages); + cout << endl << "Parse a JSON with invalid schema." << endl; + const char* json2 = "{ \"greeting\" : \"Hello!\", \"farewell\" : \"bye-bye!\", \"foo\" : {} }"; + cout << json2 << endl; + ParseMessages(json2, messages); - return 0; + return 0; } diff --git a/rapidjson/example/parsebyparts/parsebyparts.cpp b/rapidjson/example/parsebyparts/parsebyparts.cpp index 57eed005dea..b06d30005dd 100644 --- a/rapidjson/example/parsebyparts/parsebyparts.cpp +++ b/rapidjson/example/parsebyparts/parsebyparts.cpp @@ -16,158 +16,157 @@ using namespace rapidjson; template -class AsyncDocumentParser { +class AsyncDocumentParser +{ public: - AsyncDocumentParser(Document& d) - : stream_(*this) - , d_(d) - , parseThread_(&AsyncDocumentParser::Parse, this) - , mutex_() - , notEmpty_() - , finish_() - , completed_() - {} - - ~AsyncDocumentParser() { - if (!parseThread_.joinable()) - return; - - { - std::unique_lock lock(mutex_); - - // Wait until the buffer is read up (or parsing is completed) - while (!stream_.Empty() && !completed_) - finish_.wait(lock); - - // Automatically append '\0' as the terminator in the stream. - static const char terminator[] = ""; - stream_.src_ = terminator; - stream_.end_ = terminator + 1; - notEmpty_.notify_one(); // unblock the AsyncStringStream - } - - parseThread_.join(); + AsyncDocumentParser(Document& d) + : stream_(*this) + , d_(d) + , parseThread_(&AsyncDocumentParser::Parse, this) + , mutex_() + , notEmpty_() + , finish_() + , completed_() + {} + + ~AsyncDocumentParser() + { if (!parseThread_.joinable()) + return; + + { std::unique_lock lock(mutex_); + + // Wait until the buffer is read up (or parsing is completed) + while (!stream_.Empty() && !completed_) + finish_.wait(lock); + + // Automatically append '\0' as the terminator in the stream. + static const char terminator[] = ""; + stream_.src_ = terminator; + stream_.end_ = terminator + 1; + notEmpty_.notify_one(); // unblock the AsyncStringStream } - void ParsePart(const char* buffer, size_t length) { - std::unique_lock lock(mutex_); - - // Wait until the buffer is read up (or parsing is completed) - while (!stream_.Empty() && !completed_) - finish_.wait(lock); - - // Stop further parsing if the parsing process is completed. - if (completed_) - return; - - // Set the buffer to stream and unblock the AsyncStringStream - stream_.src_ = buffer; - stream_.end_ = buffer + length; - notEmpty_.notify_one(); - } + parseThread_.join(); + } -private: - void Parse() { - d_.ParseStream(stream_); + void ParsePart(const char* buffer, size_t length) + { std::unique_lock lock(mutex_); - // The stream may not be fully read, notify finish anyway to unblock ParsePart() - std::unique_lock lock(mutex_); - completed_ = true; // Parsing process is completed - finish_.notify_one(); // Unblock ParsePart() or destructor if they are waiting. - } + // Wait until the buffer is read up (or parsing is completed) + while (!stream_.Empty() && !completed_) + finish_.wait(lock); - struct AsyncStringStream { - typedef char Ch; + // Stop further parsing if the parsing process is completed. + if (completed_) + return; - AsyncStringStream(AsyncDocumentParser& parser) : parser_(parser), src_(), end_(), count_() {} + // Set the buffer to stream and unblock the AsyncStringStream + stream_.src_ = buffer; + stream_.end_ = buffer + length; + notEmpty_.notify_one(); + } - char Peek() const { - std::unique_lock lock(parser_.mutex_); +private: + void Parse() + { d_.ParseStream(stream_); - // If nothing in stream, block to wait. - while (Empty()) - parser_.notEmpty_.wait(lock); + // The stream may not be fully read, notify finish anyway to unblock ParsePart() + std::unique_lock lock(mutex_); + completed_ = true; // Parsing process is completed + finish_.notify_one(); // Unblock ParsePart() or destructor if they are waiting. + } - return *src_; - } + struct AsyncStringStream + { typedef char Ch; - char Take() { - std::unique_lock lock(parser_.mutex_); + AsyncStringStream(AsyncDocumentParser& parser) : parser_(parser), src_(), end_(), count_() {} - // If nothing in stream, block to wait. - while (Empty()) - parser_.notEmpty_.wait(lock); + char Peek() const + { std::unique_lock lock(parser_.mutex_); - count_++; - char c = *src_++; + // If nothing in stream, block to wait. + while (Empty()) + parser_.notEmpty_.wait(lock); - // If all stream is read up, notify that the stream is finish. - if (Empty()) - parser_.finish_.notify_one(); + return *src_; + } - return c; - } + char Take() + { std::unique_lock lock(parser_.mutex_); - size_t Tell() const { return count_; } + // If nothing in stream, block to wait. + while (Empty()) + parser_.notEmpty_.wait(lock); - // Not implemented - char* PutBegin() { return 0; } - void Put(char) {} - void Flush() {} - size_t PutEnd(char*) { return 0; } + count_++; + char c = *src_++; - bool Empty() const { return src_ == end_; } + // If all stream is read up, notify that the stream is finish. + if (Empty()) + parser_.finish_.notify_one(); - AsyncDocumentParser& parser_; - const char* src_; //!< Current read position. - const char* end_; //!< End of buffer - size_t count_; //!< Number of characters taken so far. - }; + return c; + } - AsyncStringStream stream_; - Document& d_; - std::thread parseThread_; - std::mutex mutex_; - std::condition_variable notEmpty_; - std::condition_variable finish_; - bool completed_; + size_t Tell() const { return count_; } + + // Not implemented + char* PutBegin() { return 0; } + void Put(char) {} + void Flush() {} + size_t PutEnd(char*) { return 0; } + + bool Empty() const { return src_ == end_; } + + AsyncDocumentParser& parser_; + const char* src_; //!< Current read position. + const char* end_; //!< End of buffer + size_t count_; //!< Number of characters taken so far. + }; + + AsyncStringStream stream_; + Document& d_; + std::thread parseThread_; + std::mutex mutex_; + std::condition_variable notEmpty_; + std::condition_variable finish_; + bool completed_; }; -int main() { - Document d; +int main() +{ Document d; - { - AsyncDocumentParser<> parser(d); + { AsyncDocumentParser<> parser(d); - const char json1[] = " { \"hello\" : \"world\", \"t\" : tr"; - //const char json1[] = " { \"hello\" : \"world\", \"t\" : trX"; // Fot test parsing error - const char json2[] = "ue, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.14"; - const char json3[] = "16, \"a\":[1, 2, 3, 4] } "; + const char json1[] = " { \"hello\" : \"world\", \"t\" : tr"; + //const char json1[] = " { \"hello\" : \"world\", \"t\" : trX"; // Fot test parsing error + const char json2[] = "ue, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.14"; + const char json3[] = "16, \"a\":[1, 2, 3, 4] } "; - parser.ParsePart(json1, sizeof(json1) - 1); - parser.ParsePart(json2, sizeof(json2) - 1); - parser.ParsePart(json3, sizeof(json3) - 1); - } + parser.ParsePart(json1, sizeof(json1) - 1); + parser.ParsePart(json2, sizeof(json2) - 1); + parser.ParsePart(json3, sizeof(json3) - 1); + } - if (d.HasParseError()) { - std::cout << "Error at offset " << d.GetErrorOffset() << ": " << GetParseError_En(d.GetParseError()) << std::endl; - return EXIT_FAILURE; - } - - // Stringify the JSON to cout - OStreamWrapper os(std::cout); - Writer writer(os); - d.Accept(writer); - std::cout << std::endl; - - return EXIT_SUCCESS; + if (d.HasParseError()) + { std::cout << "Error at offset " << d.GetErrorOffset() << ": " << GetParseError_En(d.GetParseError()) << std::endl; + return EXIT_FAILURE; + } + + // Stringify the JSON to cout + OStreamWrapper os(std::cout); + Writer writer(os); + d.Accept(writer); + std::cout << std::endl; + + return EXIT_SUCCESS; } #else // Not supporting C++11 #include -int main() { - std::cout << "This example requires C++11 compiler" << std::endl; +int main() +{ std::cout << "This example requires C++11 compiler" << std::endl; } #endif diff --git a/rapidjson/example/pretty/pretty.cpp b/rapidjson/example/pretty/pretty.cpp index 2feff5d02ea..73de8f849f4 100644 --- a/rapidjson/example/pretty/pretty.cpp +++ b/rapidjson/example/pretty/pretty.cpp @@ -9,22 +9,22 @@ using namespace rapidjson; -int main(int, char*[]) { - // Prepare reader and input stream. - Reader reader; - char readBuffer[65536]; - FileReadStream is(stdin, readBuffer, sizeof(readBuffer)); +int main(int, char*[]) +{ // Prepare reader and input stream. + Reader reader; + char readBuffer[65536]; + FileReadStream is(stdin, readBuffer, sizeof(readBuffer)); - // Prepare writer and output stream. - char writeBuffer[65536]; - FileWriteStream os(stdout, writeBuffer, sizeof(writeBuffer)); - PrettyWriter writer(os); + // Prepare writer and output stream. + char writeBuffer[65536]; + FileWriteStream os(stdout, writeBuffer, sizeof(writeBuffer)); + PrettyWriter writer(os); - // JSON reader parse from the input stream and let writer generate the output. - if (!reader.Parse(is, writer)) { - fprintf(stderr, "\nError(%u): %s\n", static_cast(reader.GetErrorOffset()), GetParseError_En(reader.GetParseErrorCode())); - return 1; - } + // JSON reader parse from the input stream and let writer generate the output. + if (!reader.Parse(is, writer)) + { fprintf(stderr, "\nError(%u): %s\n", static_cast(reader.GetErrorOffset()), GetParseError_En(reader.GetParseErrorCode())); + return 1; + } - return 0; + return 0; } diff --git a/rapidjson/example/prettyauto/prettyauto.cpp b/rapidjson/example/prettyauto/prettyauto.cpp index 1687bae5559..17f8301687b 100644 --- a/rapidjson/example/prettyauto/prettyauto.cpp +++ b/rapidjson/example/prettyauto/prettyauto.cpp @@ -15,42 +15,43 @@ using namespace rapidjson; -int main(int, char*[]) { +int main(int, char*[]) +{ #ifdef _WIN32 - // Prevent Windows converting between CR+LF and LF - _setmode(_fileno(stdin), _O_BINARY); // NEW - _setmode(_fileno(stdout), _O_BINARY); // NEW + // Prevent Windows converting between CR+LF and LF + _setmode(_fileno(stdin), _O_BINARY); // NEW + _setmode(_fileno(stdout), _O_BINARY); // NEW #endif - // Prepare reader and input stream. - //Reader reader; - GenericReader, UTF8<> > reader; // CHANGED - char readBuffer[65536]; - FileReadStream is(stdin, readBuffer, sizeof(readBuffer)); - AutoUTFInputStream eis(is); // NEW + // Prepare reader and input stream. + //Reader reader; + GenericReader, UTF8<> > reader; // CHANGED + char readBuffer[65536]; + FileReadStream is(stdin, readBuffer, sizeof(readBuffer)); + AutoUTFInputStream eis(is); // NEW - // Prepare writer and output stream. - char writeBuffer[65536]; - FileWriteStream os(stdout, writeBuffer, sizeof(writeBuffer)); + // Prepare writer and output stream. + char writeBuffer[65536]; + FileWriteStream os(stdout, writeBuffer, sizeof(writeBuffer)); #if 1 - // Use the same Encoding of the input. Also use BOM according to input. - typedef AutoUTFOutputStream OutputStream; // NEW - OutputStream eos(os, eis.GetType(), eis.HasBOM()); // NEW - PrettyWriter, AutoUTF > writer(eos); // CHANGED + // Use the same Encoding of the input. Also use BOM according to input. + typedef AutoUTFOutputStream OutputStream; // NEW + OutputStream eos(os, eis.GetType(), eis.HasBOM()); // NEW + PrettyWriter, AutoUTF > writer(eos); // CHANGED #else - // You may also use static bound encoding type, such as output to UTF-16LE with BOM - typedef EncodedOutputStream,FileWriteStream> OutputStream; // NEW - OutputStream eos(os, true); // NEW - PrettyWriter, UTF16LE<> > writer(eos); // CHANGED + // You may also use static bound encoding type, such as output to UTF-16LE with BOM + typedef EncodedOutputStream,FileWriteStream> OutputStream; // NEW + OutputStream eos(os, true); // NEW + PrettyWriter, UTF16LE<> > writer(eos); // CHANGED #endif - // JSON reader parse from the input stream and let writer generate the output. - //if (!reader.Parse(is, writer)) { - if (!reader.Parse(eis, writer)) { // CHANGED - fprintf(stderr, "\nError(%u): %s\n", static_cast(reader.GetErrorOffset()), GetParseError_En(reader.GetParseErrorCode())); - return 1; - } + // JSON reader parse from the input stream and let writer generate the output. + //if (!reader.Parse(is, writer)) { + if (!reader.Parse(eis, writer)) // CHANGED + { fprintf(stderr, "\nError(%u): %s\n", static_cast(reader.GetErrorOffset()), GetParseError_En(reader.GetParseErrorCode())); + return 1; + } - return 0; + return 0; } diff --git a/rapidjson/example/schemavalidator/schemavalidator.cpp b/rapidjson/example/schemavalidator/schemavalidator.cpp index ce36ea95f00..e9f42cfb287 100644 --- a/rapidjson/example/schemavalidator/schemavalidator.cpp +++ b/rapidjson/example/schemavalidator/schemavalidator.cpp @@ -9,64 +9,63 @@ using namespace rapidjson; -int main(int argc, char *argv[]) { - if (argc != 2) { - fprintf(stderr, "Usage: schemavalidator schema.json < input.json\n"); - return EXIT_FAILURE; - } +int main(int argc, char *argv[]) +{ if (argc != 2) + { fprintf(stderr, "Usage: schemavalidator schema.json < input.json\n"); + return EXIT_FAILURE; + } - // Read a JSON schema from file into Document - Document d; - char buffer[4096]; + // Read a JSON schema from file into Document + Document d; + char buffer[4096]; - { - FILE *fp = fopen(argv[1], "r"); - if (!fp) { - printf("Schema file '%s' not found\n", argv[1]); - return -1; - } - FileReadStream fs(fp, buffer, sizeof(buffer)); - d.ParseStream(fs); - if (d.HasParseError()) { - fprintf(stderr, "Schema file '%s' is not a valid JSON\n", argv[1]); - fprintf(stderr, "Error(offset %u): %s\n", - static_cast(d.GetErrorOffset()), - GetParseError_En(d.GetParseError())); - fclose(fp); - return EXIT_FAILURE; - } - fclose(fp); + { FILE *fp = fopen(argv[1], "r"); + if (!fp) + { printf("Schema file '%s' not found\n", argv[1]); + return -1; + } + FileReadStream fs(fp, buffer, sizeof(buffer)); + d.ParseStream(fs); + if (d.HasParseError()) + { fprintf(stderr, "Schema file '%s' is not a valid JSON\n", argv[1]); + fprintf(stderr, "Error(offset %u): %s\n", + static_cast(d.GetErrorOffset()), + GetParseError_En(d.GetParseError())); + fclose(fp); + return EXIT_FAILURE; } - - // Then convert the Document into SchemaDocument - SchemaDocument sd(d); + fclose(fp); + } + + // Then convert the Document into SchemaDocument + SchemaDocument sd(d); - // Use reader to parse the JSON in stdin, and forward SAX events to validator - SchemaValidator validator(sd); - Reader reader; - FileReadStream is(stdin, buffer, sizeof(buffer)); - if (!reader.Parse(is, validator) && reader.GetParseErrorCode() != kParseErrorTermination) { - // Schema validator error would cause kParseErrorTermination, which will handle it in next step. - fprintf(stderr, "Input is not a valid JSON\n"); - fprintf(stderr, "Error(offset %u): %s\n", + // Use reader to parse the JSON in stdin, and forward SAX events to validator + SchemaValidator validator(sd); + Reader reader; + FileReadStream is(stdin, buffer, sizeof(buffer)); + if (!reader.Parse(is, validator) && reader.GetParseErrorCode() != kParseErrorTermination) + { // Schema validator error would cause kParseErrorTermination, which will handle it in next step. + fprintf(stderr, "Input is not a valid JSON\n"); + fprintf(stderr, "Error(offset %u): %s\n", static_cast(reader.GetErrorOffset()), GetParseError_En(reader.GetParseErrorCode())); - } + } - // Check the validation result - if (validator.IsValid()) { - printf("Input JSON is valid.\n"); - return EXIT_SUCCESS; - } - else { - printf("Input JSON is invalid.\n"); - StringBuffer sb; - validator.GetInvalidSchemaPointer().StringifyUriFragment(sb); - fprintf(stderr, "Invalid schema: %s\n", sb.GetString()); - fprintf(stderr, "Invalid keyword: %s\n", validator.GetInvalidSchemaKeyword()); - sb.Clear(); - validator.GetInvalidDocumentPointer().StringifyUriFragment(sb); - fprintf(stderr, "Invalid document: %s\n", sb.GetString()); - return EXIT_FAILURE; - } + // Check the validation result + if (validator.IsValid()) + { printf("Input JSON is valid.\n"); + return EXIT_SUCCESS; + } + else + { printf("Input JSON is invalid.\n"); + StringBuffer sb; + validator.GetInvalidSchemaPointer().StringifyUriFragment(sb); + fprintf(stderr, "Invalid schema: %s\n", sb.GetString()); + fprintf(stderr, "Invalid keyword: %s\n", validator.GetInvalidSchemaKeyword()); + sb.Clear(); + validator.GetInvalidDocumentPointer().StringifyUriFragment(sb); + fprintf(stderr, "Invalid document: %s\n", sb.GetString()); + return EXIT_FAILURE; + } } diff --git a/rapidjson/example/serialize/serialize.cpp b/rapidjson/example/serialize/serialize.cpp index 12d87151e64..0ff77a51b62 100644 --- a/rapidjson/example/serialize/serialize.cpp +++ b/rapidjson/example/serialize/serialize.cpp @@ -8,166 +8,172 @@ using namespace rapidjson; -class Person { +class Person +{ public: - Person(const std::string& name, unsigned age) : name_(name), age_(age) {} - Person(const Person& rhs) : name_(rhs.name_), age_(rhs.age_) {} - virtual ~Person(); + Person(const std::string& name, unsigned age) : name_(name), age_(age) {} + Person(const Person& rhs) : name_(rhs.name_), age_(rhs.age_) {} + virtual ~Person(); - Person& operator=(const Person& rhs) { - name_ = rhs.name_; - age_ = rhs.age_; - return *this; - } + Person& operator=(const Person& rhs) + { name_ = rhs.name_; + age_ = rhs.age_; + return *this; + } protected: - template - void Serialize(Writer& writer) const { - // This base class just write out name-value pairs, without wrapping within an object. - writer.String("name"); + template + void Serialize(Writer& writer) const + { // This base class just write out name-value pairs, without wrapping within an object. + writer.String("name"); #if RAPIDJSON_HAS_STDSTRING - writer.String(name_); + writer.String(name_); #else - writer.String(name_.c_str(), static_cast(name_.length())); // Supplying length of string is faster. + writer.String(name_.c_str(), static_cast(name_.length())); // Supplying length of string is faster. #endif - writer.String("age"); - writer.Uint(age_); - } + writer.String("age"); + writer.Uint(age_); + } private: - std::string name_; - unsigned age_; + std::string name_; + unsigned age_; }; -Person::~Person() { +Person::~Person() +{ } -class Education { +class Education +{ public: - Education(const std::string& school, double GPA) : school_(school), GPA_(GPA) {} - Education(const Education& rhs) : school_(rhs.school_), GPA_(rhs.GPA_) {} - - template - void Serialize(Writer& writer) const { - writer.StartObject(); - - writer.String("school"); + Education(const std::string& school, double GPA) : school_(school), GPA_(GPA) {} + Education(const Education& rhs) : school_(rhs.school_), GPA_(rhs.GPA_) {} + + template + void Serialize(Writer& writer) const + { writer.StartObject(); + + writer.String("school"); #if RAPIDJSON_HAS_STDSTRING - writer.String(school_); + writer.String(school_); #else - writer.String(school_.c_str(), static_cast(school_.length())); + writer.String(school_.c_str(), static_cast(school_.length())); #endif - writer.String("GPA"); - writer.Double(GPA_); + writer.String("GPA"); + writer.Double(GPA_); - writer.EndObject(); - } + writer.EndObject(); + } private: - std::string school_; - double GPA_; + std::string school_; + double GPA_; }; -class Dependent : public Person { +class Dependent : public Person +{ public: - Dependent(const std::string& name, unsigned age, Education* education = 0) : Person(name, age), education_(education) {} - Dependent(const Dependent& rhs) : Person(rhs), education_(0) { education_ = (rhs.education_ == 0) ? 0 : new Education(*rhs.education_); } - virtual ~Dependent(); + Dependent(const std::string& name, unsigned age, Education* education = 0) : Person(name, age), education_(education) {} + Dependent(const Dependent& rhs) : Person(rhs), education_(0) { education_ = (rhs.education_ == 0) ? 0 : new Education(*rhs.education_); } + virtual ~Dependent(); - Dependent& operator=(const Dependent& rhs) { - if (this == &rhs) - return *this; - delete education_; - education_ = (rhs.education_ == 0) ? 0 : new Education(*rhs.education_); - return *this; - } + Dependent& operator=(const Dependent& rhs) + { if (this == &rhs) + return *this; + delete education_; + education_ = (rhs.education_ == 0) ? 0 : new Education(*rhs.education_); + return *this; + } - template - void Serialize(Writer& writer) const { - writer.StartObject(); + template + void Serialize(Writer& writer) const + { writer.StartObject(); - Person::Serialize(writer); + Person::Serialize(writer); - writer.String("education"); - if (education_) - education_->Serialize(writer); - else - writer.Null(); + writer.String("education"); + if (education_) + education_->Serialize(writer); + else + writer.Null(); - writer.EndObject(); - } + writer.EndObject(); + } private: - Education *education_; + Education *education_; }; -Dependent::~Dependent() { - delete education_; +Dependent::~Dependent() +{ delete education_; } -class Employee : public Person { +class Employee : public Person +{ public: - Employee(const std::string& name, unsigned age, bool married) : Person(name, age), dependents_(), married_(married) {} - Employee(const Employee& rhs) : Person(rhs), dependents_(rhs.dependents_), married_(rhs.married_) {} - virtual ~Employee(); + Employee(const std::string& name, unsigned age, bool married) : Person(name, age), dependents_(), married_(married) {} + Employee(const Employee& rhs) : Person(rhs), dependents_(rhs.dependents_), married_(rhs.married_) {} + virtual ~Employee(); - Employee& operator=(const Employee& rhs) { - static_cast(*this) = rhs; - dependents_ = rhs.dependents_; - married_ = rhs.married_; - return *this; - } + Employee& operator=(const Employee& rhs) + { static_cast(*this) = rhs; + dependents_ = rhs.dependents_; + married_ = rhs.married_; + return *this; + } - void AddDependent(const Dependent& dependent) { - dependents_.push_back(dependent); - } + void AddDependent(const Dependent& dependent) + { dependents_.push_back(dependent); + } - template - void Serialize(Writer& writer) const { - writer.StartObject(); + template + void Serialize(Writer& writer) const + { writer.StartObject(); - Person::Serialize(writer); + Person::Serialize(writer); - writer.String("married"); - writer.Bool(married_); + writer.String("married"); + writer.Bool(married_); - writer.String(("dependents")); - writer.StartArray(); - for (std::vector::const_iterator dependentItr = dependents_.begin(); dependentItr != dependents_.end(); ++dependentItr) - dependentItr->Serialize(writer); - writer.EndArray(); + writer.String(("dependents")); + writer.StartArray(); + for (std::vector::const_iterator dependentItr = dependents_.begin(); dependentItr != dependents_.end(); ++dependentItr) + dependentItr->Serialize(writer); + writer.EndArray(); - writer.EndObject(); - } + writer.EndObject(); + } private: - std::vector dependents_; - bool married_; + std::vector dependents_; + bool married_; }; -Employee::~Employee() { +Employee::~Employee() +{ } -int main(int, char*[]) { - std::vector employees; +int main(int, char*[]) +{ std::vector employees; - employees.push_back(Employee("Milo YIP", 34, true)); - employees.back().AddDependent(Dependent("Lua YIP", 3, new Education("Happy Kindergarten", 3.5))); - employees.back().AddDependent(Dependent("Mio YIP", 1)); + employees.push_back(Employee("Milo YIP", 34, true)); + employees.back().AddDependent(Dependent("Lua YIP", 3, new Education("Happy Kindergarten", 3.5))); + employees.back().AddDependent(Dependent("Mio YIP", 1)); - employees.push_back(Employee("Percy TSE", 30, false)); + employees.push_back(Employee("Percy TSE", 30, false)); - StringBuffer sb; - PrettyWriter writer(sb); + StringBuffer sb; + PrettyWriter writer(sb); - writer.StartArray(); - for (std::vector::const_iterator employeeItr = employees.begin(); employeeItr != employees.end(); ++employeeItr) - employeeItr->Serialize(writer); - writer.EndArray(); + writer.StartArray(); + for (std::vector::const_iterator employeeItr = employees.begin(); employeeItr != employees.end(); ++employeeItr) + employeeItr->Serialize(writer); + writer.EndArray(); - puts(sb.GetString()); + puts(sb.GetString()); - return 0; + return 0; } diff --git a/rapidjson/example/simpledom/simpledom.cpp b/rapidjson/example/simpledom/simpledom.cpp index 80384199a92..c587ae56f9f 100644 --- a/rapidjson/example/simpledom/simpledom.cpp +++ b/rapidjson/example/simpledom/simpledom.cpp @@ -8,22 +8,22 @@ using namespace rapidjson; -int main() { - // 1. Parse a JSON string into DOM. - const char* json = "{\"project\":\"rapidjson\",\"stars\":10}"; - Document d; - d.Parse(json); +int main() +{ // 1. Parse a JSON string into DOM. + const char* json = "{\"project\":\"rapidjson\",\"stars\":10}"; + Document d; + d.Parse(json); - // 2. Modify it by DOM. - Value& s = d["stars"]; - s.SetInt(s.GetInt() + 1); + // 2. Modify it by DOM. + Value& s = d["stars"]; + s.SetInt(s.GetInt() + 1); - // 3. Stringify the DOM - StringBuffer buffer; - Writer writer(buffer); - d.Accept(writer); + // 3. Stringify the DOM + StringBuffer buffer; + Writer writer(buffer); + d.Accept(writer); - // Output {"project":"rapidjson","stars":11} - std::cout << buffer.GetString() << std::endl; - return 0; + // Output {"project":"rapidjson","stars":11} + std::cout << buffer.GetString() << std::endl; + return 0; } diff --git a/rapidjson/example/simplereader/simplereader.cpp b/rapidjson/example/simplereader/simplereader.cpp index 5aae8a1c0ac..2a79d973ba6 100644 --- a/rapidjson/example/simplereader/simplereader.cpp +++ b/rapidjson/example/simplereader/simplereader.cpp @@ -4,39 +4,39 @@ using namespace rapidjson; using namespace std; -struct MyHandler { - bool Null() { cout << "Null()" << endl; return true; } - bool Bool(bool b) { cout << "Bool(" << boolalpha << b << ")" << endl; return true; } - bool Int(int i) { cout << "Int(" << i << ")" << endl; return true; } - bool Uint(unsigned u) { cout << "Uint(" << u << ")" << endl; return true; } - bool Int64(int64_t i) { cout << "Int64(" << i << ")" << endl; return true; } - bool Uint64(uint64_t u) { cout << "Uint64(" << u << ")" << endl; return true; } - bool Double(double d) { cout << "Double(" << d << ")" << endl; return true; } - bool RawNumber(const char* str, SizeType length, bool copy) { - cout << "Number(" << str << ", " << length << ", " << boolalpha << copy << ")" << endl; - return true; - } - bool String(const char* str, SizeType length, bool copy) { - cout << "String(" << str << ", " << length << ", " << boolalpha << copy << ")" << endl; - return true; - } - bool StartObject() { cout << "StartObject()" << endl; return true; } - bool Key(const char* str, SizeType length, bool copy) { - cout << "Key(" << str << ", " << length << ", " << boolalpha << copy << ")" << endl; - return true; - } - bool EndObject(SizeType memberCount) { cout << "EndObject(" << memberCount << ")" << endl; return true; } - bool StartArray() { cout << "StartArray()" << endl; return true; } - bool EndArray(SizeType elementCount) { cout << "EndArray(" << elementCount << ")" << endl; return true; } +struct MyHandler +{ bool Null() { cout << "Null()" << endl; return true; } + bool Bool(bool b) { cout << "Bool(" << boolalpha << b << ")" << endl; return true; } + bool Int(int i) { cout << "Int(" << i << ")" << endl; return true; } + bool Uint(unsigned u) { cout << "Uint(" << u << ")" << endl; return true; } + bool Int64(int64_t i) { cout << "Int64(" << i << ")" << endl; return true; } + bool Uint64(uint64_t u) { cout << "Uint64(" << u << ")" << endl; return true; } + bool Double(double d) { cout << "Double(" << d << ")" << endl; return true; } + bool RawNumber(const char* str, SizeType length, bool copy) + { cout << "Number(" << str << ", " << length << ", " << boolalpha << copy << ")" << endl; + return true; + } + bool String(const char* str, SizeType length, bool copy) + { cout << "String(" << str << ", " << length << ", " << boolalpha << copy << ")" << endl; + return true; + } + bool StartObject() { cout << "StartObject()" << endl; return true; } + bool Key(const char* str, SizeType length, bool copy) + { cout << "Key(" << str << ", " << length << ", " << boolalpha << copy << ")" << endl; + return true; + } + bool EndObject(SizeType memberCount) { cout << "EndObject(" << memberCount << ")" << endl; return true; } + bool StartArray() { cout << "StartArray()" << endl; return true; } + bool EndArray(SizeType elementCount) { cout << "EndArray(" << elementCount << ")" << endl; return true; } }; -int main() { - const char json[] = " { \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3, 4] } "; +int main() +{ const char json[] = " { \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3, 4] } "; - MyHandler handler; - Reader reader; - StringStream ss(json); - reader.Parse(ss, handler); + MyHandler handler; + Reader reader; + StringStream ss(json); + reader.Parse(ss, handler); - return 0; + return 0; } diff --git a/rapidjson/example/simplewriter/simplewriter.cpp b/rapidjson/example/simplewriter/simplewriter.cpp index 8d1275c2923..8f9f51e4af8 100644 --- a/rapidjson/example/simplewriter/simplewriter.cpp +++ b/rapidjson/example/simplewriter/simplewriter.cpp @@ -5,32 +5,32 @@ using namespace rapidjson; using namespace std; -int main() { - StringBuffer s; - Writer writer(s); - - writer.StartObject(); // Between StartObject()/EndObject(), - writer.Key("hello"); // output a key, - writer.String("world"); // follow by a value. - writer.Key("t"); - writer.Bool(true); - writer.Key("f"); - writer.Bool(false); - writer.Key("n"); - writer.Null(); - writer.Key("i"); - writer.Uint(123); - writer.Key("pi"); - writer.Double(3.1416); - writer.Key("a"); - writer.StartArray(); // Between StartArray()/EndArray(), - for (unsigned i = 0; i < 4; i++) - writer.Uint(i); // all values are elements of the array. - writer.EndArray(); - writer.EndObject(); +int main() +{ StringBuffer s; + Writer writer(s); - // {"hello":"world","t":true,"f":false,"n":null,"i":123,"pi":3.1416,"a":[0,1,2,3]} - cout << s.GetString() << endl; + writer.StartObject(); // Between StartObject()/EndObject(), + writer.Key("hello"); // output a key, + writer.String("world"); // follow by a value. + writer.Key("t"); + writer.Bool(true); + writer.Key("f"); + writer.Bool(false); + writer.Key("n"); + writer.Null(); + writer.Key("i"); + writer.Uint(123); + writer.Key("pi"); + writer.Double(3.1416); + writer.Key("a"); + writer.StartArray(); // Between StartArray()/EndArray(), + for (unsigned i = 0; i < 4; i++) + writer.Uint(i); // all values are elements of the array. + writer.EndArray(); + writer.EndObject(); - return 0; + // {"hello":"world","t":true,"f":false,"n":null,"i":123,"pi":3.1416,"a":[0,1,2,3]} + cout << s.GetString() << endl; + + return 0; } diff --git a/rapidjson/example/tutorial/tutorial.cpp b/rapidjson/example/tutorial/tutorial.cpp index c8bfcc14c18..bbb50e81ac7 100644 --- a/rapidjson/example/tutorial/tutorial.cpp +++ b/rapidjson/example/tutorial/tutorial.cpp @@ -8,144 +8,139 @@ using namespace rapidjson; using namespace std; -int main(int, char*[]) { - //////////////////////////////////////////////////////////////////////////// - // 1. Parse a JSON text string to a document. +int main(int, char*[]) +{ //////////////////////////////////////////////////////////////////////////// + // 1. Parse a JSON text string to a document. - const char json[] = " { \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3, 4] } "; - printf("Original JSON:\n %s\n", json); + const char json[] = " { \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3, 4] } "; + printf("Original JSON:\n %s\n", json); - Document document; // Default template parameter uses UTF8 and MemoryPoolAllocator. + Document document; // Default template parameter uses UTF8 and MemoryPoolAllocator. #if 0 - // "normal" parsing, decode strings to new buffers. Can use other input stream via ParseStream(). - if (document.Parse(json).HasParseError()) - return 1; + // "normal" parsing, decode strings to new buffers. Can use other input stream via ParseStream(). + if (document.Parse(json).HasParseError()) + return 1; #else - // In-situ parsing, decode strings directly in the source string. Source must be string. - char buffer[sizeof(json)]; - memcpy(buffer, json, sizeof(json)); - if (document.ParseInsitu(buffer).HasParseError()) - return 1; + // In-situ parsing, decode strings directly in the source string. Source must be string. + char buffer[sizeof(json)]; + memcpy(buffer, json, sizeof(json)); + if (document.ParseInsitu(buffer).HasParseError()) + return 1; #endif - printf("\nParsing to document succeeded.\n"); - - //////////////////////////////////////////////////////////////////////////// - // 2. Access values in document. - - printf("\nAccess values in document:\n"); - assert(document.IsObject()); // Document is a JSON value represents the root of DOM. Root can be either an object or array. - - assert(document.HasMember("hello")); - assert(document["hello"].IsString()); - printf("hello = %s\n", document["hello"].GetString()); - - // Since version 0.2, you can use single lookup to check the existing of member and its value: - Value::MemberIterator hello = document.FindMember("hello"); - assert(hello != document.MemberEnd()); - assert(hello->value.IsString()); - assert(strcmp("world", hello->value.GetString()) == 0); - (void)hello; - - assert(document["t"].IsBool()); // JSON true/false are bool. Can also uses more specific function IsTrue(). - printf("t = %s\n", document["t"].GetBool() ? "true" : "false"); - - assert(document["f"].IsBool()); - printf("f = %s\n", document["f"].GetBool() ? "true" : "false"); - - printf("n = %s\n", document["n"].IsNull() ? "null" : "?"); - - assert(document["i"].IsNumber()); // Number is a JSON type, but C++ needs more specific type. - assert(document["i"].IsInt()); // In this case, IsUint()/IsInt64()/IsUInt64() also return true. - printf("i = %d\n", document["i"].GetInt()); // Alternative (int)document["i"] - - assert(document["pi"].IsNumber()); - assert(document["pi"].IsDouble()); - printf("pi = %g\n", document["pi"].GetDouble()); - - { - const Value& a = document["a"]; // Using a reference for consecutive access is handy and faster. - assert(a.IsArray()); - for (SizeType i = 0; i < a.Size(); i++) // rapidjson uses SizeType instead of size_t. - printf("a[%d] = %d\n", i, a[i].GetInt()); - - int y = a[0].GetInt(); - (void)y; - - // Iterating array with iterators - printf("a = "); - for (Value::ConstValueIterator itr = a.Begin(); itr != a.End(); ++itr) - printf("%d ", itr->GetInt()); - printf("\n"); - } - - // Iterating object members - static const char* kTypeNames[] = { "Null", "False", "True", "Object", "Array", "String", "Number" }; - for (Value::ConstMemberIterator itr = document.MemberBegin(); itr != document.MemberEnd(); ++itr) - printf("Type of member %s is %s\n", itr->name.GetString(), kTypeNames[itr->value.GetType()]); - - //////////////////////////////////////////////////////////////////////////// - // 3. Modify values in document. - - // Change i to a bigger number - { - uint64_t f20 = 1; // compute factorial of 20 - for (uint64_t j = 1; j <= 20; j++) - f20 *= j; - document["i"] = f20; // Alternate form: document["i"].SetUint64(f20) - assert(!document["i"].IsInt()); // No longer can be cast as int or uint. - } - - // Adding values to array. - { - Value& a = document["a"]; // This time we uses non-const reference. - Document::AllocatorType& allocator = document.GetAllocator(); - for (int i = 5; i <= 10; i++) - a.PushBack(i, allocator); // May look a bit strange, allocator is needed for potentially realloc. We normally uses the document's. - - // Fluent API - a.PushBack("Lua", allocator).PushBack("Mio", allocator); - } - - // Making string values. - - // This version of SetString() just store the pointer to the string. - // So it is for literal and string that exists within value's life-cycle. - { - document["hello"] = "rapidjson"; // This will invoke strlen() - // Faster version: - // document["hello"].SetString("rapidjson", 9); - } - - // This version of SetString() needs an allocator, which means it will allocate a new buffer and copy the the string into the buffer. - Value author; - { - char buffer2[10]; - int len = sprintf(buffer2, "%s %s", "Milo", "Yip"); // synthetic example of dynamically created string. - - author.SetString(buffer2, static_cast(len), document.GetAllocator()); - // Shorter but slower version: - // document["hello"].SetString(buffer, document.GetAllocator()); - - // Constructor version: - // Value author(buffer, len, document.GetAllocator()); - // Value author(buffer, document.GetAllocator()); - memset(buffer2, 0, sizeof(buffer2)); // For demonstration purpose. - } - // Variable 'buffer' is unusable now but 'author' has already made a copy. - document.AddMember("author", author, document.GetAllocator()); - - assert(author.IsNull()); // Move semantic for assignment. After this variable is assigned as a member, the variable becomes null. - - //////////////////////////////////////////////////////////////////////////// - // 4. Stringify JSON - - printf("\nModified JSON with reformatting:\n"); - StringBuffer sb; - PrettyWriter writer(sb); - document.Accept(writer); // Accept() traverses the DOM and generates Handler events. - puts(sb.GetString()); - - return 0; + printf("\nParsing to document succeeded.\n"); + + //////////////////////////////////////////////////////////////////////////// + // 2. Access values in document. + + printf("\nAccess values in document:\n"); + assert(document.IsObject()); // Document is a JSON value represents the root of DOM. Root can be either an object or array. + + assert(document.HasMember("hello")); + assert(document["hello"].IsString()); + printf("hello = %s\n", document["hello"].GetString()); + + // Since version 0.2, you can use single lookup to check the existing of member and its value: + Value::MemberIterator hello = document.FindMember("hello"); + assert(hello != document.MemberEnd()); + assert(hello->value.IsString()); + assert(strcmp("world", hello->value.GetString()) == 0); + (void)hello; + + assert(document["t"].IsBool()); // JSON true/false are bool. Can also uses more specific function IsTrue(). + printf("t = %s\n", document["t"].GetBool() ? "true" : "false"); + + assert(document["f"].IsBool()); + printf("f = %s\n", document["f"].GetBool() ? "true" : "false"); + + printf("n = %s\n", document["n"].IsNull() ? "null" : "?"); + + assert(document["i"].IsNumber()); // Number is a JSON type, but C++ needs more specific type. + assert(document["i"].IsInt()); // In this case, IsUint()/IsInt64()/IsUInt64() also return true. + printf("i = %d\n", document["i"].GetInt()); // Alternative (int)document["i"] + + assert(document["pi"].IsNumber()); + assert(document["pi"].IsDouble()); + printf("pi = %g\n", document["pi"].GetDouble()); + + { const Value& a = document["a"]; // Using a reference for consecutive access is handy and faster. + assert(a.IsArray()); + for (SizeType i = 0; i < a.Size(); i++) // rapidjson uses SizeType instead of size_t. + printf("a[%d] = %d\n", i, a[i].GetInt()); + + int y = a[0].GetInt(); + (void)y; + + // Iterating array with iterators + printf("a = "); + for (Value::ConstValueIterator itr = a.Begin(); itr != a.End(); ++itr) + printf("%d ", itr->GetInt()); + printf("\n"); + } + + // Iterating object members + static const char* kTypeNames[] = { "Null", "False", "True", "Object", "Array", "String", "Number" }; + for (Value::ConstMemberIterator itr = document.MemberBegin(); itr != document.MemberEnd(); ++itr) + printf("Type of member %s is %s\n", itr->name.GetString(), kTypeNames[itr->value.GetType()]); + + //////////////////////////////////////////////////////////////////////////// + // 3. Modify values in document. + + // Change i to a bigger number + { uint64_t f20 = 1; // compute factorial of 20 + for (uint64_t j = 1; j <= 20; j++) + f20 *= j; + document["i"] = f20; // Alternate form: document["i"].SetUint64(f20) + assert(!document["i"].IsInt()); // No longer can be cast as int or uint. + } + + // Adding values to array. + { Value& a = document["a"]; // This time we uses non-const reference. + Document::AllocatorType& allocator = document.GetAllocator(); + for (int i = 5; i <= 10; i++) + a.PushBack(i, allocator); // May look a bit strange, allocator is needed for potentially realloc. We normally uses the document's. + + // Fluent API + a.PushBack("Lua", allocator).PushBack("Mio", allocator); + } + + // Making string values. + + // This version of SetString() just store the pointer to the string. + // So it is for literal and string that exists within value's life-cycle. + { document["hello"] = "rapidjson"; // This will invoke strlen() + // Faster version: + // document["hello"].SetString("rapidjson", 9); + } + + // This version of SetString() needs an allocator, which means it will allocate a new buffer and copy the the string into the buffer. + Value author; + { char buffer2[10]; + int len = sprintf(buffer2, "%s %s", "Milo", "Yip"); // synthetic example of dynamically created string. + + author.SetString(buffer2, static_cast(len), document.GetAllocator()); + // Shorter but slower version: + // document["hello"].SetString(buffer, document.GetAllocator()); + + // Constructor version: + // Value author(buffer, len, document.GetAllocator()); + // Value author(buffer, document.GetAllocator()); + memset(buffer2, 0, sizeof(buffer2)); // For demonstration purpose. + } + // Variable 'buffer' is unusable now but 'author' has already made a copy. + document.AddMember("author", author, document.GetAllocator()); + + assert(author.IsNull()); // Move semantic for assignment. After this variable is assigned as a member, the variable becomes null. + + //////////////////////////////////////////////////////////////////////////// + // 4. Stringify JSON + + printf("\nModified JSON with reformatting:\n"); + StringBuffer sb; + PrettyWriter writer(sb); + document.Accept(writer); // Accept() traverses the DOM and generates Handler events. + puts(sb.GetString()); + + return 0; } diff --git a/rapidjson/include/rapidjson/allocators.h b/rapidjson/include/rapidjson/allocators.h index 98affe03fbf..9507379a609 100644 --- a/rapidjson/include/rapidjson/allocators.h +++ b/rapidjson/include/rapidjson/allocators.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_ALLOCATORS_H_ @@ -24,10 +24,10 @@ RAPIDJSON_NAMESPACE_BEGIN /*! \class rapidjson::Allocator \brief Concept for allocating, resizing and freeing memory block. - + Note that Malloc() and Realloc() are non-static but Free() is static. - - So if an allocator need to support Free(), it needs to put its pointer in + + So if an allocator need to support Free(), it needs to put its pointer in the header of memory block. \code @@ -59,31 +59,32 @@ concept Allocator { /*! This class is just wrapper for standard C library memory routines. \note implements Allocator concept */ -class CrtAllocator { +class CrtAllocator +{ public: - static const bool kNeedFree = true; - void* Malloc(size_t size) { - if (size) // behavior of malloc(0) is implementation defined. - return std::malloc(size); - else - return NULL; // standardize to returning NULL. + static const bool kNeedFree = true; + void* Malloc(size_t size) + { if (size) // behavior of malloc(0) is implementation defined. + return std::malloc(size); + else + return NULL; // standardize to returning NULL. + } + void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) + { (void)originalSize; + if (newSize == 0) + { std::free(originalPtr); + return NULL; } - void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { - (void)originalSize; - if (newSize == 0) { - std::free(originalPtr); - return NULL; - } - return std::realloc(originalPtr, newSize); - } - static void Free(void *ptr) { std::free(ptr); } + return std::realloc(originalPtr, newSize); + } + static void Free(void *ptr) { std::free(ptr); } }; /////////////////////////////////////////////////////////////////////////////// // MemoryPoolAllocator //! Default memory allocator used by the parser and DOM. -/*! This allocator allocate memory blocks from pre-allocated memory chunks. +/*! This allocator allocate memory blocks from pre-allocated memory chunks. It does not free memory blocks. And Realloc() only allocate new memory. @@ -99,171 +100,171 @@ class CrtAllocator { \note implements Allocator concept */ template -class MemoryPoolAllocator { +class MemoryPoolAllocator +{ public: - static const bool kNeedFree = false; //!< Tell users that no need to call Free() with this allocator. (concept Allocator) - - //! Constructor with chunkSize. - /*! \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. - \param baseAllocator The allocator for allocating memory chunks. - */ - MemoryPoolAllocator(size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : - chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(0), baseAllocator_(baseAllocator), ownBaseAllocator_(0) - { - } - - //! Constructor with user-supplied buffer. - /*! The user buffer will be used firstly. When it is full, memory pool allocates new chunk with chunk size. - - The user buffer will not be deallocated when this allocator is destructed. - - \param buffer User supplied buffer. - \param size Size of the buffer in bytes. It must at least larger than sizeof(ChunkHeader). - \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. - \param baseAllocator The allocator for allocating memory chunks. - */ - MemoryPoolAllocator(void *buffer, size_t size, size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : - chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(buffer), baseAllocator_(baseAllocator), ownBaseAllocator_(0) - { - RAPIDJSON_ASSERT(buffer != 0); - RAPIDJSON_ASSERT(size > sizeof(ChunkHeader)); - chunkHead_ = reinterpret_cast(buffer); - chunkHead_->capacity = size - sizeof(ChunkHeader); - chunkHead_->size = 0; - chunkHead_->next = 0; - } - - //! Destructor. - /*! This deallocates all memory chunks, excluding the user-supplied buffer. - */ - ~MemoryPoolAllocator() { - Clear(); - RAPIDJSON_DELETE(ownBaseAllocator_); - } - - //! Deallocates all memory chunks, excluding the user-supplied buffer. - void Clear() { - while (chunkHead_ && chunkHead_ != userBuffer_) { - ChunkHeader* next = chunkHead_->next; - baseAllocator_->Free(chunkHead_); - chunkHead_ = next; - } - if (chunkHead_ && chunkHead_ == userBuffer_) - chunkHead_->size = 0; // Clear user buffer + static const bool kNeedFree = false; //!< Tell users that no need to call Free() with this allocator. (concept Allocator) + + //! Constructor with chunkSize. + /*! \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. + \param baseAllocator The allocator for allocating memory chunks. + */ + MemoryPoolAllocator(size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : + chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(0), baseAllocator_(baseAllocator), ownBaseAllocator_(0) + { + } + + //! Constructor with user-supplied buffer. + /*! The user buffer will be used firstly. When it is full, memory pool allocates new chunk with chunk size. + + The user buffer will not be deallocated when this allocator is destructed. + + \param buffer User supplied buffer. + \param size Size of the buffer in bytes. It must at least larger than sizeof(ChunkHeader). + \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. + \param baseAllocator The allocator for allocating memory chunks. + */ + MemoryPoolAllocator(void *buffer, size_t size, size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : + chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(buffer), baseAllocator_(baseAllocator), ownBaseAllocator_(0) + { RAPIDJSON_ASSERT(buffer != 0); + RAPIDJSON_ASSERT(size > sizeof(ChunkHeader)); + chunkHead_ = reinterpret_cast(buffer); + chunkHead_->capacity = size - sizeof(ChunkHeader); + chunkHead_->size = 0; + chunkHead_->next = 0; + } + + //! Destructor. + /*! This deallocates all memory chunks, excluding the user-supplied buffer. + */ + ~MemoryPoolAllocator() + { Clear(); + RAPIDJSON_DELETE(ownBaseAllocator_); + } + + //! Deallocates all memory chunks, excluding the user-supplied buffer. + void Clear() + { while (chunkHead_ && chunkHead_ != userBuffer_) + { ChunkHeader* next = chunkHead_->next; + baseAllocator_->Free(chunkHead_); + chunkHead_ = next; } - - //! Computes the total capacity of allocated memory chunks. - /*! \return total capacity in bytes. - */ - size_t Capacity() const { - size_t capacity = 0; - for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) - capacity += c->capacity; - return capacity; + if (chunkHead_ && chunkHead_ == userBuffer_) + chunkHead_->size = 0; // Clear user buffer + } + + //! Computes the total capacity of allocated memory chunks. + /*! \return total capacity in bytes. + */ + size_t Capacity() const + { size_t capacity = 0; + for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) + capacity += c->capacity; + return capacity; + } + + //! Computes the memory blocks allocated. + /*! \return total used bytes. + */ + size_t Size() const + { size_t size = 0; + for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) + size += c->size; + return size; + } + + //! Allocates a memory block. (concept Allocator) + void* Malloc(size_t size) + { if (!size) + return NULL; + + size = RAPIDJSON_ALIGN(size); + if (chunkHead_ == 0 || chunkHead_->size + size > chunkHead_->capacity) + if (!AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size)) + return NULL; + + void *buffer = reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size; + chunkHead_->size += size; + return buffer; + } + + //! Resizes a memory block (concept Allocator) + void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) + { if (originalPtr == 0) + return Malloc(newSize); + + if (newSize == 0) + return NULL; + + originalSize = RAPIDJSON_ALIGN(originalSize); + newSize = RAPIDJSON_ALIGN(newSize); + + // Do not shrink if new size is smaller than original + if (originalSize >= newSize) + return originalPtr; + + // Simply expand it if it is the last allocation and there is sufficient space + if (originalPtr == reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size - originalSize) + { size_t increment = static_cast(newSize - originalSize); + if (chunkHead_->size + increment <= chunkHead_->capacity) + { chunkHead_->size += increment; + return originalPtr; + } } - //! Computes the memory blocks allocated. - /*! \return total used bytes. - */ - size_t Size() const { - size_t size = 0; - for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) - size += c->size; - return size; + // Realloc process: allocate and copy memory, do not free original buffer. + if (void* newBuffer = Malloc(newSize)) + { if (originalSize) + std::memcpy(newBuffer, originalPtr, originalSize); + return newBuffer; } + else + return NULL; + } - //! Allocates a memory block. (concept Allocator) - void* Malloc(size_t size) { - if (!size) - return NULL; - - size = RAPIDJSON_ALIGN(size); - if (chunkHead_ == 0 || chunkHead_->size + size > chunkHead_->capacity) - if (!AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size)) - return NULL; - - void *buffer = reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size; - chunkHead_->size += size; - return buffer; - } - - //! Resizes a memory block (concept Allocator) - void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { - if (originalPtr == 0) - return Malloc(newSize); - - if (newSize == 0) - return NULL; - - originalSize = RAPIDJSON_ALIGN(originalSize); - newSize = RAPIDJSON_ALIGN(newSize); - - // Do not shrink if new size is smaller than original - if (originalSize >= newSize) - return originalPtr; - - // Simply expand it if it is the last allocation and there is sufficient space - if (originalPtr == reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size - originalSize) { - size_t increment = static_cast(newSize - originalSize); - if (chunkHead_->size + increment <= chunkHead_->capacity) { - chunkHead_->size += increment; - return originalPtr; - } - } - - // Realloc process: allocate and copy memory, do not free original buffer. - if (void* newBuffer = Malloc(newSize)) { - if (originalSize) - std::memcpy(newBuffer, originalPtr, originalSize); - return newBuffer; - } - else - return NULL; - } - - //! Frees a memory block (concept Allocator) - static void Free(void *ptr) { (void)ptr; } // Do nothing + //! Frees a memory block (concept Allocator) + static void Free(void *ptr) { (void)ptr; } // Do nothing private: - //! Copy constructor is not permitted. - MemoryPoolAllocator(const MemoryPoolAllocator& rhs) /* = delete */; - //! Copy assignment operator is not permitted. - MemoryPoolAllocator& operator=(const MemoryPoolAllocator& rhs) /* = delete */; - - //! Creates a new chunk. - /*! \param capacity Capacity of the chunk in bytes. - \return true if success. - */ - bool AddChunk(size_t capacity) { - if (!baseAllocator_) - ownBaseAllocator_ = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator()); - if (ChunkHeader* chunk = reinterpret_cast(baseAllocator_->Malloc(RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + capacity))) { - chunk->capacity = capacity; - chunk->size = 0; - chunk->next = chunkHead_; - chunkHead_ = chunk; - return true; - } - else - return false; + //! Copy constructor is not permitted. + MemoryPoolAllocator(const MemoryPoolAllocator& rhs) /* = delete */; + //! Copy assignment operator is not permitted. + MemoryPoolAllocator& operator=(const MemoryPoolAllocator& rhs) /* = delete */; + + //! Creates a new chunk. + /*! \param capacity Capacity of the chunk in bytes. + \return true if success. + */ + bool AddChunk(size_t capacity) + { if (!baseAllocator_) + ownBaseAllocator_ = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator()); + if (ChunkHeader* chunk = reinterpret_cast(baseAllocator_->Malloc(RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + capacity))) + { chunk->capacity = capacity; + chunk->size = 0; + chunk->next = chunkHead_; + chunkHead_ = chunk; + return true; } - - static const int kDefaultChunkCapacity = 64 * 1024; //!< Default chunk capacity. - - //! Chunk header for perpending to each chunk. - /*! Chunks are stored as a singly linked list. - */ - struct ChunkHeader { - size_t capacity; //!< Capacity of the chunk in bytes (excluding the header itself). - size_t size; //!< Current size of allocated memory in bytes. - ChunkHeader *next; //!< Next chunk in the linked list. - }; - - ChunkHeader *chunkHead_; //!< Head of the chunk linked-list. Only the head chunk serves allocation. - size_t chunk_capacity_; //!< The minimum capacity of chunk when they are allocated. - void *userBuffer_; //!< User supplied buffer. - BaseAllocator* baseAllocator_; //!< base allocator for allocating memory chunks. - BaseAllocator* ownBaseAllocator_; //!< base allocator created by this object. + else + return false; + } + + static const int kDefaultChunkCapacity = 64 * 1024; //!< Default chunk capacity. + + //! Chunk header for perpending to each chunk. + /*! Chunks are stored as a singly linked list. + */ + struct ChunkHeader + { size_t capacity; //!< Capacity of the chunk in bytes (excluding the header itself). + size_t size; //!< Current size of allocated memory in bytes. + ChunkHeader *next; //!< Next chunk in the linked list. + }; + + ChunkHeader *chunkHead_; //!< Head of the chunk linked-list. Only the head chunk serves allocation. + size_t chunk_capacity_; //!< The minimum capacity of chunk when they are allocated. + void *userBuffer_; //!< User supplied buffer. + BaseAllocator* baseAllocator_; //!< base allocator for allocating memory chunks. + BaseAllocator* ownBaseAllocator_; //!< base allocator created by this object. }; RAPIDJSON_NAMESPACE_END diff --git a/rapidjson/include/rapidjson/document.h b/rapidjson/include/rapidjson/document.h index e3e20dfbdc9..9a127e7db5e 100644 --- a/rapidjson/include/rapidjson/document.h +++ b/rapidjson/include/rapidjson/document.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_DOCUMENT_H_ @@ -34,13 +34,13 @@ RAPIDJSON_DIAG_OFF(4244) // conversion from kXxxFlags to 'uint16_t', possible lo #ifdef __clang__ RAPIDJSON_DIAG_OFF(padded) RAPIDJSON_DIAG_OFF(switch-enum) -RAPIDJSON_DIAG_OFF(c++98-compat) + RAPIDJSON_DIAG_OFF(c++98-compat) #endif #ifdef __GNUC__ -RAPIDJSON_DIAG_OFF(effc++) + RAPIDJSON_DIAG_OFF(effc++) #if __GNUC__ >= 6 -RAPIDJSON_DIAG_OFF(terminate) // ignore throwing RAPIDJSON_ASSERT in RAPIDJSON_NOEXCEPT functions + RAPIDJSON_DIAG_OFF(terminate) // ignore throwing RAPIDJSON_ASSERT in RAPIDJSON_NOEXCEPT functions #endif #endif // __GNUC__ @@ -52,10 +52,10 @@ RAPIDJSON_DIAG_OFF(terminate) // ignore throwing RAPIDJSON_ASSERT in RAPIDJSON_N #include // std::move #endif -RAPIDJSON_NAMESPACE_BEGIN + RAPIDJSON_NAMESPACE_BEGIN // Forward declaration. -template + template class GenericValue; template @@ -67,10 +67,10 @@ class GenericDocument; But a compiler (IBM XL C/C++ for AIX) have reported to have problem with that so it moved as a namespace scope struct. https://code.google.com/p/rapidjson/issues/detail?id=64 */ -template -struct GenericMember { - GenericValue name; //!< name of member (must be a string) - GenericValue value; //!< value of member. +template +struct GenericMember +{ GenericValue name; //!< name of member (must be a string) + GenericValue value; //!< value of member. }; /////////////////////////////////////////////////////////////////////////////// @@ -99,98 +99,99 @@ struct GenericMember { */ template class GenericMemberIterator - : public std::iterator >::Type> { + : public std::iterator >::Type> +{ - friend class GenericValue; - template friend class GenericMemberIterator; + friend class GenericValue; + template friend class GenericMemberIterator; - typedef GenericMember PlainType; - typedef typename internal::MaybeAddConst::Type ValueType; - typedef std::iterator BaseType; + typedef GenericMember PlainType; + typedef typename internal::MaybeAddConst::Type ValueType; + typedef std::iterator BaseType; public: - //! Iterator type itself - typedef GenericMemberIterator Iterator; - //! Constant iterator type - typedef GenericMemberIterator ConstIterator; - //! Non-constant iterator type - typedef GenericMemberIterator NonConstIterator; - - //! Pointer to (const) GenericMember - typedef typename BaseType::pointer Pointer; - //! Reference to (const) GenericMember - typedef typename BaseType::reference Reference; - //! Signed integer type (e.g. \c ptrdiff_t) - typedef typename BaseType::difference_type DifferenceType; - - //! Default constructor (singular value) - /*! Creates an iterator pointing to no element. - \note All operations, except for comparisons, are undefined on such values. - */ - GenericMemberIterator() : ptr_() {} - - //! Iterator conversions to more const - /*! - \param it (Non-const) iterator to copy from - - Allows the creation of an iterator from another GenericMemberIterator - that is "less const". Especially, creating a non-constant iterator - from a constant iterator are disabled: - \li const -> non-const (not ok) - \li const -> const (ok) - \li non-const -> const (ok) - \li non-const -> non-const (ok) - - \note If the \c Const template parameter is already \c false, this - constructor effectively defines a regular copy-constructor. - Otherwise, the copy constructor is implicitly defined. - */ - GenericMemberIterator(const NonConstIterator & it) : ptr_(it.ptr_) {} - Iterator& operator=(const NonConstIterator & it) { ptr_ = it.ptr_; return *this; } - - //! @name stepping - //@{ - Iterator& operator++(){ ++ptr_; return *this; } - Iterator& operator--(){ --ptr_; return *this; } - Iterator operator++(int){ Iterator old(*this); ++ptr_; return old; } - Iterator operator--(int){ Iterator old(*this); --ptr_; return old; } - //@} - - //! @name increment/decrement - //@{ - Iterator operator+(DifferenceType n) const { return Iterator(ptr_+n); } - Iterator operator-(DifferenceType n) const { return Iterator(ptr_-n); } - - Iterator& operator+=(DifferenceType n) { ptr_+=n; return *this; } - Iterator& operator-=(DifferenceType n) { ptr_-=n; return *this; } - //@} - - //! @name relations - //@{ - bool operator==(ConstIterator that) const { return ptr_ == that.ptr_; } - bool operator!=(ConstIterator that) const { return ptr_ != that.ptr_; } - bool operator<=(ConstIterator that) const { return ptr_ <= that.ptr_; } - bool operator>=(ConstIterator that) const { return ptr_ >= that.ptr_; } - bool operator< (ConstIterator that) const { return ptr_ < that.ptr_; } - bool operator> (ConstIterator that) const { return ptr_ > that.ptr_; } - //@} - - //! @name dereference - //@{ - Reference operator*() const { return *ptr_; } - Pointer operator->() const { return ptr_; } - Reference operator[](DifferenceType n) const { return ptr_[n]; } - //@} - - //! Distance - DifferenceType operator-(ConstIterator that) const { return ptr_-that.ptr_; } + //! Iterator type itself + typedef GenericMemberIterator Iterator; + //! Constant iterator type + typedef GenericMemberIterator ConstIterator; + //! Non-constant iterator type + typedef GenericMemberIterator NonConstIterator; + + //! Pointer to (const) GenericMember + typedef typename BaseType::pointer Pointer; + //! Reference to (const) GenericMember + typedef typename BaseType::reference Reference; + //! Signed integer type (e.g. \c ptrdiff_t) + typedef typename BaseType::difference_type DifferenceType; + + //! Default constructor (singular value) + /*! Creates an iterator pointing to no element. + \note All operations, except for comparisons, are undefined on such values. + */ + GenericMemberIterator() : ptr_() {} + + //! Iterator conversions to more const + /*! + \param it (Non-const) iterator to copy from + + Allows the creation of an iterator from another GenericMemberIterator + that is "less const". Especially, creating a non-constant iterator + from a constant iterator are disabled: + \li const -> non-const (not ok) + \li const -> const (ok) + \li non-const -> const (ok) + \li non-const -> non-const (ok) + + \note If the \c Const template parameter is already \c false, this + constructor effectively defines a regular copy-constructor. + Otherwise, the copy constructor is implicitly defined. + */ + GenericMemberIterator(const NonConstIterator & it) : ptr_(it.ptr_) {} + Iterator& operator=(const NonConstIterator & it) { ptr_ = it.ptr_; return *this; } + + //! @name stepping + //@{ + Iterator& operator++() { ++ptr_; return *this; } + Iterator& operator--() { --ptr_; return *this; } + Iterator operator++(int) { Iterator old(*this); ++ptr_; return old; } + Iterator operator--(int) { Iterator old(*this); --ptr_; return old; } + //@} + + //! @name increment/decrement + //@{ + Iterator operator+(DifferenceType n) const { return Iterator(ptr_+n); } + Iterator operator-(DifferenceType n) const { return Iterator(ptr_-n); } + + Iterator& operator+=(DifferenceType n) { ptr_+=n; return *this; } + Iterator& operator-=(DifferenceType n) { ptr_-=n; return *this; } + //@} + + //! @name relations + //@{ + bool operator==(ConstIterator that) const { return ptr_ == that.ptr_; } + bool operator!=(ConstIterator that) const { return ptr_ != that.ptr_; } + bool operator<=(ConstIterator that) const { return ptr_ <= that.ptr_; } + bool operator>=(ConstIterator that) const { return ptr_ >= that.ptr_; } + bool operator< (ConstIterator that) const { return ptr_ < that.ptr_; } + bool operator> (ConstIterator that) const { return ptr_ > that.ptr_; } + //@} + + //! @name dereference + //@{ + Reference operator*() const { return *ptr_; } + Pointer operator->() const { return ptr_; } + Reference operator[](DifferenceType n) const { return ptr_[n]; } + //@} + + //! Distance + DifferenceType operator-(ConstIterator that) const { return ptr_-that.ptr_; } private: - //! Internal constructor from plain pointer - explicit GenericMemberIterator(Pointer p) : ptr_(p) {} + //! Internal constructor from plain pointer + explicit GenericMemberIterator(Pointer p) : ptr_(p) {} - Pointer ptr_; //!< raw pointer + Pointer ptr_; //!< raw pointer }; #else // RAPIDJSON_NOMEMBERITERATORCLASS @@ -202,15 +203,15 @@ struct GenericMemberIterator; //! non-const GenericMemberIterator template -struct GenericMemberIterator { - //! use plain pointer as iterator type - typedef GenericMember* Iterator; +struct GenericMemberIterator +{ //! use plain pointer as iterator type + typedef GenericMember* Iterator; }; //! const GenericMemberIterator template -struct GenericMemberIterator { - //! use plain const pointer as iterator type - typedef const GenericMember* Iterator; +struct GenericMemberIterator +{ //! use plain const pointer as iterator type + typedef const GenericMember* Iterator; }; #endif // RAPIDJSON_NOMEMBERITERATORCLASS @@ -246,88 +247,88 @@ struct GenericMemberIterator { \see StringRef, GenericValue::SetString */ template -struct GenericStringRef { - typedef CharType Ch; //!< character type of the string +struct GenericStringRef +{ typedef CharType Ch; //!< character type of the string - //! Create string reference from \c const character array + //! Create string reference from \c const character array #ifndef __clang__ // -Wdocumentation - /*! - This constructor implicitly creates a constant string reference from - a \c const character array. It has better performance than - \ref StringRef(const CharType*) by inferring the string \ref length - from the array length, and also supports strings containing null - characters. - - \tparam N length of the string, automatically inferred - - \param str Constant character array, lifetime assumed to be longer - than the use of the string in e.g. a GenericValue - - \post \ref s == str - - \note Constant complexity. - \note There is a hidden, private overload to disallow references to - non-const character arrays to be created via this constructor. - By this, e.g. function-scope arrays used to be filled via - \c snprintf are excluded from consideration. - In such cases, the referenced string should be \b copied to the - GenericValue instead. - */ + /*! + This constructor implicitly creates a constant string reference from + a \c const character array. It has better performance than + \ref StringRef(const CharType*) by inferring the string \ref length + from the array length, and also supports strings containing null + characters. + + \tparam N length of the string, automatically inferred + + \param str Constant character array, lifetime assumed to be longer + than the use of the string in e.g. a GenericValue + + \post \ref s == str + + \note Constant complexity. + \note There is a hidden, private overload to disallow references to + non-const character arrays to be created via this constructor. + By this, e.g. function-scope arrays used to be filled via + \c snprintf are excluded from consideration. + In such cases, the referenced string should be \b copied to the + GenericValue instead. + */ #endif - template - GenericStringRef(const CharType (&str)[N]) RAPIDJSON_NOEXCEPT - : s(str), length(N-1) {} + template + GenericStringRef(const CharType (&str)[N]) RAPIDJSON_NOEXCEPT +: s(str), length(N-1) {} - //! Explicitly create string reference from \c const character pointer + //! Explicitly create string reference from \c const character pointer #ifndef __clang__ // -Wdocumentation - /*! - This constructor can be used to \b explicitly create a reference to - a constant string pointer. + /*! + This constructor can be used to \b explicitly create a reference to + a constant string pointer. - \see StringRef(const CharType*) + \see StringRef(const CharType*) - \param str Constant character pointer, lifetime assumed to be longer - than the use of the string in e.g. a GenericValue + \param str Constant character pointer, lifetime assumed to be longer + than the use of the string in e.g. a GenericValue - \post \ref s == str + \post \ref s == str - \note There is a hidden, private overload to disallow references to - non-const character arrays to be created via this constructor. - By this, e.g. function-scope arrays used to be filled via - \c snprintf are excluded from consideration. - In such cases, the referenced string should be \b copied to the - GenericValue instead. - */ + \note There is a hidden, private overload to disallow references to + non-const character arrays to be created via this constructor. + By this, e.g. function-scope arrays used to be filled via + \c snprintf are excluded from consideration. + In such cases, the referenced string should be \b copied to the + GenericValue instead. + */ #endif - explicit GenericStringRef(const CharType* str) - : s(str), length(internal::StrLen(str)){ RAPIDJSON_ASSERT(s != 0); } + explicit GenericStringRef(const CharType* str) + : s(str), length(internal::StrLen(str)) { RAPIDJSON_ASSERT(s != 0); } - //! Create constant string reference from pointer and length + //! Create constant string reference from pointer and length #ifndef __clang__ // -Wdocumentation - /*! \param str constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue - \param len length of the string, excluding the trailing NULL terminator + /*! \param str constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue + \param len length of the string, excluding the trailing NULL terminator - \post \ref s == str && \ref length == len - \note Constant complexity. - */ + \post \ref s == str && \ref length == len + \note Constant complexity. + */ #endif - GenericStringRef(const CharType* str, SizeType len) - : s(str), length(len) { RAPIDJSON_ASSERT(s != 0); } + GenericStringRef(const CharType* str, SizeType len) + : s(str), length(len) { RAPIDJSON_ASSERT(s != 0); } - GenericStringRef(const GenericStringRef& rhs) : s(rhs.s), length(rhs.length) {} + GenericStringRef(const GenericStringRef& rhs) : s(rhs.s), length(rhs.length) {} - GenericStringRef& operator=(const GenericStringRef& rhs) { s = rhs.s; length = rhs.length; } + GenericStringRef& operator=(const GenericStringRef& rhs) { s = rhs.s; length = rhs.length; } - //! implicit conversion to plain CharType pointer - operator const Ch *() const { return s; } + //! implicit conversion to plain CharType pointer + operator const Ch *() const { return s; } - const Ch* const s; //!< plain CharType pointer - const SizeType length; //!< length of the string (excluding the trailing NULL terminator) + const Ch* const s; //!< plain CharType pointer + const SizeType length; //!< length of the string (excluding the trailing NULL terminator) private: - //! Disallow construction from non-const array - template - GenericStringRef(CharType (&str)[N]) /* = delete */; + //! Disallow construction from non-const array + template + GenericStringRef(CharType (&str)[N]) /* = delete */; }; //! Mark a character pointer as constant string @@ -343,8 +344,8 @@ struct GenericStringRef { \see GenericValue::GenericValue(StringRefType), GenericValue::operator=(StringRefType), GenericValue::SetString(StringRefType), GenericValue::PushBack(StringRefType, Allocator&), GenericValue::AddMember */ template -inline GenericStringRef StringRef(const CharType* str) { - return GenericStringRef(str, internal::StrLen(str)); +inline GenericStringRef StringRef(const CharType* str) +{ return GenericStringRef(str, internal::StrLen(str)); } //! Mark a character pointer as constant string @@ -363,8 +364,8 @@ inline GenericStringRef StringRef(const CharType* str) { \relatesalso GenericStringRef */ template -inline GenericStringRef StringRef(const CharType* str, size_t length) { - return GenericStringRef(str, SizeType(length)); +inline GenericStringRef StringRef(const CharType* str, size_t length) +{ return GenericStringRef(str, SizeType(length)); } #if RAPIDJSON_HAS_STDSTRING @@ -381,21 +382,22 @@ inline GenericStringRef StringRef(const CharType* str, size_t length) \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. */ template -inline GenericStringRef StringRef(const std::basic_string& str) { - return GenericStringRef(str.data(), SizeType(str.size())); +inline GenericStringRef StringRef(const std::basic_string& str) +{ return GenericStringRef(str.data(), SizeType(str.size())); } #endif /////////////////////////////////////////////////////////////////////////////// // GenericValue type traits -namespace internal { +namespace internal +{ template struct IsGenericValueImpl : FalseType {}; // select candidates according to nested encoding and allocator types template struct IsGenericValueImpl::Type, typename Void::Type> - : IsBaseOf, T>::Type {}; + : IsBaseOf, T>::Type {}; // helper to match arbitrary GenericValue instantiations, including derived classes template struct IsGenericValue : IsGenericValueImpl::Type {}; @@ -405,116 +407,117 @@ template struct IsGenericValue : IsGenericValueImpl::Type {}; /////////////////////////////////////////////////////////////////////////////// // TypeHelper -namespace internal { +namespace internal +{ template struct TypeHelper {}; -template -struct TypeHelper { - static bool Is(const ValueType& v) { return v.IsBool(); } - static bool Get(const ValueType& v) { return v.GetBool(); } - static ValueType& Set(ValueType& v, bool data) { return v.SetBool(data); } - static ValueType& Set(ValueType& v, bool data, typename ValueType::AllocatorType&) { return v.SetBool(data); } +template +struct TypeHelper +{ static bool Is(const ValueType& v) { return v.IsBool(); } + static bool Get(const ValueType& v) { return v.GetBool(); } + static ValueType& Set(ValueType& v, bool data) { return v.SetBool(data); } + static ValueType& Set(ValueType& v, bool data, typename ValueType::AllocatorType&) { return v.SetBool(data); } }; -template -struct TypeHelper { - static bool Is(const ValueType& v) { return v.IsInt(); } - static int Get(const ValueType& v) { return v.GetInt(); } - static ValueType& Set(ValueType& v, int data) { return v.SetInt(data); } - static ValueType& Set(ValueType& v, int data, typename ValueType::AllocatorType&) { return v.SetInt(data); } +template +struct TypeHelper +{ static bool Is(const ValueType& v) { return v.IsInt(); } + static int Get(const ValueType& v) { return v.GetInt(); } + static ValueType& Set(ValueType& v, int data) { return v.SetInt(data); } + static ValueType& Set(ValueType& v, int data, typename ValueType::AllocatorType&) { return v.SetInt(data); } }; -template -struct TypeHelper { - static bool Is(const ValueType& v) { return v.IsUint(); } - static unsigned Get(const ValueType& v) { return v.GetUint(); } - static ValueType& Set(ValueType& v, unsigned data) { return v.SetUint(data); } - static ValueType& Set(ValueType& v, unsigned data, typename ValueType::AllocatorType&) { return v.SetUint(data); } +template +struct TypeHelper +{ static bool Is(const ValueType& v) { return v.IsUint(); } + static unsigned Get(const ValueType& v) { return v.GetUint(); } + static ValueType& Set(ValueType& v, unsigned data) { return v.SetUint(data); } + static ValueType& Set(ValueType& v, unsigned data, typename ValueType::AllocatorType&) { return v.SetUint(data); } }; -template -struct TypeHelper { - static bool Is(const ValueType& v) { return v.IsInt64(); } - static int64_t Get(const ValueType& v) { return v.GetInt64(); } - static ValueType& Set(ValueType& v, int64_t data) { return v.SetInt64(data); } - static ValueType& Set(ValueType& v, int64_t data, typename ValueType::AllocatorType&) { return v.SetInt64(data); } +template +struct TypeHelper +{ static bool Is(const ValueType& v) { return v.IsInt64(); } + static int64_t Get(const ValueType& v) { return v.GetInt64(); } + static ValueType& Set(ValueType& v, int64_t data) { return v.SetInt64(data); } + static ValueType& Set(ValueType& v, int64_t data, typename ValueType::AllocatorType&) { return v.SetInt64(data); } }; -template -struct TypeHelper { - static bool Is(const ValueType& v) { return v.IsUint64(); } - static uint64_t Get(const ValueType& v) { return v.GetUint64(); } - static ValueType& Set(ValueType& v, uint64_t data) { return v.SetUint64(data); } - static ValueType& Set(ValueType& v, uint64_t data, typename ValueType::AllocatorType&) { return v.SetUint64(data); } +template +struct TypeHelper +{ static bool Is(const ValueType& v) { return v.IsUint64(); } + static uint64_t Get(const ValueType& v) { return v.GetUint64(); } + static ValueType& Set(ValueType& v, uint64_t data) { return v.SetUint64(data); } + static ValueType& Set(ValueType& v, uint64_t data, typename ValueType::AllocatorType&) { return v.SetUint64(data); } }; -template -struct TypeHelper { - static bool Is(const ValueType& v) { return v.IsDouble(); } - static double Get(const ValueType& v) { return v.GetDouble(); } - static ValueType& Set(ValueType& v, double data) { return v.SetDouble(data); } - static ValueType& Set(ValueType& v, double data, typename ValueType::AllocatorType&) { return v.SetDouble(data); } +template +struct TypeHelper +{ static bool Is(const ValueType& v) { return v.IsDouble(); } + static double Get(const ValueType& v) { return v.GetDouble(); } + static ValueType& Set(ValueType& v, double data) { return v.SetDouble(data); } + static ValueType& Set(ValueType& v, double data, typename ValueType::AllocatorType&) { return v.SetDouble(data); } }; -template -struct TypeHelper { - static bool Is(const ValueType& v) { return v.IsFloat(); } - static float Get(const ValueType& v) { return v.GetFloat(); } - static ValueType& Set(ValueType& v, float data) { return v.SetFloat(data); } - static ValueType& Set(ValueType& v, float data, typename ValueType::AllocatorType&) { return v.SetFloat(data); } +template +struct TypeHelper +{ static bool Is(const ValueType& v) { return v.IsFloat(); } + static float Get(const ValueType& v) { return v.GetFloat(); } + static ValueType& Set(ValueType& v, float data) { return v.SetFloat(data); } + static ValueType& Set(ValueType& v, float data, typename ValueType::AllocatorType&) { return v.SetFloat(data); } }; -template -struct TypeHelper { - typedef const typename ValueType::Ch* StringType; - static bool Is(const ValueType& v) { return v.IsString(); } - static StringType Get(const ValueType& v) { return v.GetString(); } - static ValueType& Set(ValueType& v, const StringType data) { return v.SetString(typename ValueType::StringRefType(data)); } - static ValueType& Set(ValueType& v, const StringType data, typename ValueType::AllocatorType& a) { return v.SetString(data, a); } +template +struct TypeHelper +{ typedef const typename ValueType::Ch* StringType; + static bool Is(const ValueType& v) { return v.IsString(); } + static StringType Get(const ValueType& v) { return v.GetString(); } + static ValueType& Set(ValueType& v, const StringType data) { return v.SetString(typename ValueType::StringRefType(data)); } + static ValueType& Set(ValueType& v, const StringType data, typename ValueType::AllocatorType& a) { return v.SetString(data, a); } }; #if RAPIDJSON_HAS_STDSTRING -template -struct TypeHelper > { - typedef std::basic_string StringType; - static bool Is(const ValueType& v) { return v.IsString(); } - static StringType Get(const ValueType& v) { return StringType(v.GetString(), v.GetStringLength()); } - static ValueType& Set(ValueType& v, const StringType& data, typename ValueType::AllocatorType& a) { return v.SetString(data, a); } +template +struct TypeHelper > +{ typedef std::basic_string StringType; + static bool Is(const ValueType& v) { return v.IsString(); } + static StringType Get(const ValueType& v) { return StringType(v.GetString(), v.GetStringLength()); } + static ValueType& Set(ValueType& v, const StringType& data, typename ValueType::AllocatorType& a) { return v.SetString(data, a); } }; #endif -template -struct TypeHelper { - typedef typename ValueType::Array ArrayType; - static bool Is(const ValueType& v) { return v.IsArray(); } - static ArrayType Get(ValueType& v) { return v.GetArray(); } - static ValueType& Set(ValueType& v, ArrayType data) { return v = data; } - static ValueType& Set(ValueType& v, ArrayType data, typename ValueType::AllocatorType&) { return v = data; } +template +struct TypeHelper +{ typedef typename ValueType::Array ArrayType; + static bool Is(const ValueType& v) { return v.IsArray(); } + static ArrayType Get(ValueType& v) { return v.GetArray(); } + static ValueType& Set(ValueType& v, ArrayType data) { return v = data; } + static ValueType& Set(ValueType& v, ArrayType data, typename ValueType::AllocatorType&) { return v = data; } }; -template -struct TypeHelper { - typedef typename ValueType::ConstArray ArrayType; - static bool Is(const ValueType& v) { return v.IsArray(); } - static ArrayType Get(const ValueType& v) { return v.GetArray(); } +template +struct TypeHelper +{ typedef typename ValueType::ConstArray ArrayType; + static bool Is(const ValueType& v) { return v.IsArray(); } + static ArrayType Get(const ValueType& v) { return v.GetArray(); } }; -template -struct TypeHelper { - typedef typename ValueType::Object ObjectType; - static bool Is(const ValueType& v) { return v.IsObject(); } - static ObjectType Get(ValueType& v) { return v.GetObject(); } - static ValueType& Set(ValueType& v, ObjectType data) { return v = data; } - static ValueType& Set(ValueType& v, ObjectType data, typename ValueType::AllocatorType&) { v = data; } +template +struct TypeHelper +{ typedef typename ValueType::Object ObjectType; + static bool Is(const ValueType& v) { return v.IsObject(); } + static ObjectType Get(ValueType& v) { return v.GetObject(); } + static ValueType& Set(ValueType& v, ObjectType data) { return v = data; } + static ValueType& Set(ValueType& v, ObjectType data, typename ValueType::AllocatorType&) { v = data; } }; -template -struct TypeHelper { - typedef typename ValueType::ConstObject ObjectType; - static bool Is(const ValueType& v) { return v.IsObject(); } - static ObjectType Get(const ValueType& v) { return v.GetObject(); } +template +struct TypeHelper +{ typedef typename ValueType::ConstObject ObjectType; + static bool Is(const ValueType& v) { return v.IsObject(); } + static ObjectType Get(const ValueType& v) { return v.GetObject(); } }; } // namespace internal @@ -536,1481 +539,1484 @@ template class GenericObject; \tparam Encoding Encoding of the value. (Even non-string values need to have the same encoding in a document) \tparam Allocator Allocator type for allocating memory of object, array and string. */ -template > -class GenericValue { +template > +class GenericValue +{ public: - //! Name-value pair in an object. - typedef GenericMember Member; - typedef Encoding EncodingType; //!< Encoding type from template parameter. - typedef Allocator AllocatorType; //!< Allocator type from template parameter. - typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. - typedef GenericStringRef StringRefType; //!< Reference to a constant string - typedef typename GenericMemberIterator::Iterator MemberIterator; //!< Member iterator for iterating in object. - typedef typename GenericMemberIterator::Iterator ConstMemberIterator; //!< Constant member iterator for iterating in object. - typedef GenericValue* ValueIterator; //!< Value iterator for iterating in array. - typedef const GenericValue* ConstValueIterator; //!< Constant value iterator for iterating in array. - typedef GenericValue ValueType; //!< Value type of itself. - typedef GenericArray Array; - typedef GenericArray ConstArray; - typedef GenericObject Object; - typedef GenericObject ConstObject; - - //!@name Constructors and destructor. - //@{ - - //! Default constructor creates a null value. - GenericValue() RAPIDJSON_NOEXCEPT : data_() { data_.f.flags = kNullFlag; } + //! Name-value pair in an object. + typedef GenericMember Member; + typedef Encoding EncodingType; //!< Encoding type from template parameter. + typedef Allocator AllocatorType; //!< Allocator type from template parameter. + typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. + typedef GenericStringRef StringRefType; //!< Reference to a constant string + typedef typename GenericMemberIterator::Iterator MemberIterator; //!< Member iterator for iterating in object. + typedef typename GenericMemberIterator::Iterator ConstMemberIterator; //!< Constant member iterator for iterating in object. + typedef GenericValue* ValueIterator; //!< Value iterator for iterating in array. + typedef const GenericValue* ConstValueIterator; //!< Constant value iterator for iterating in array. + typedef GenericValue ValueType; //!< Value type of itself. + typedef GenericArray Array; + typedef GenericArray ConstArray; + typedef GenericObject Object; + typedef GenericObject ConstObject; + + //!@name Constructors and destructor. + //@{ + + //! Default constructor creates a null value. +GenericValue() RAPIDJSON_NOEXCEPT : data_() { data_.f.flags = kNullFlag; } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Move constructor in C++11 - GenericValue(GenericValue&& rhs) RAPIDJSON_NOEXCEPT : data_(rhs.data_) { - rhs.data_.f.flags = kNullFlag; // give up contents - } + //! Move constructor in C++11 +GenericValue(GenericValue&& rhs) RAPIDJSON_NOEXCEPT : data_(rhs.data_) + { rhs.data_.f.flags = kNullFlag; // give up contents + } #endif private: - //! Copy constructor is not permitted. - GenericValue(const GenericValue& rhs); + //! Copy constructor is not permitted. + GenericValue(const GenericValue& rhs); #if RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Moving from a GenericDocument is not permitted. - template - GenericValue(GenericDocument&& rhs); + //! Moving from a GenericDocument is not permitted. + template + GenericValue(GenericDocument&& rhs); - //! Move assignment from a GenericDocument is not permitted. - template - GenericValue& operator=(GenericDocument&& rhs); + //! Move assignment from a GenericDocument is not permitted. + template + GenericValue& operator=(GenericDocument&& rhs); #endif public: - //! Constructor with JSON value type. - /*! This creates a Value of specified type with default content. - \param type Type of the value. - \note Default content for number is zero. - */ - explicit GenericValue(Type type) RAPIDJSON_NOEXCEPT : data_() { - static const uint16_t defaultFlags[7] = { - kNullFlag, kFalseFlag, kTrueFlag, kObjectFlag, kArrayFlag, kShortStringFlag, - kNumberAnyFlag - }; - RAPIDJSON_ASSERT(type <= kNumberType); - data_.f.flags = defaultFlags[type]; - - // Use ShortString to store empty string. - if (type == kStringType) - data_.ss.SetLength(0); - } - - //! Explicit copy constructor (with allocator) - /*! Creates a copy of a Value by using the given Allocator - \tparam SourceAllocator allocator of \c rhs - \param rhs Value to copy from (read-only) - \param allocator Allocator for allocating copied elements and buffers. Commonly use GenericDocument::GetAllocator(). - \see CopyFrom() - */ - template< typename SourceAllocator > - GenericValue(const GenericValue& rhs, Allocator & allocator); - - //! Constructor for boolean value. - /*! \param b Boolean value - \note This constructor is limited to \em real boolean values and rejects - implicitly converted types like arbitrary pointers. Use an explicit cast - to \c bool, if you want to construct a boolean JSON value in such cases. - */ + //! Constructor with JSON value type. + /*! This creates a Value of specified type with default content. + \param type Type of the value. + \note Default content for number is zero. + */ +explicit GenericValue(Type type) RAPIDJSON_NOEXCEPT : data_() + { static const uint16_t defaultFlags[7] = + { kNullFlag, kFalseFlag, kTrueFlag, kObjectFlag, kArrayFlag, kShortStringFlag, + kNumberAnyFlag + }; + RAPIDJSON_ASSERT(type <= kNumberType); + data_.f.flags = defaultFlags[type]; + + // Use ShortString to store empty string. + if (type == kStringType) + data_.ss.SetLength(0); + } + + //! Explicit copy constructor (with allocator) + /*! Creates a copy of a Value by using the given Allocator + \tparam SourceAllocator allocator of \c rhs + \param rhs Value to copy from (read-only) + \param allocator Allocator for allocating copied elements and buffers. Commonly use GenericDocument::GetAllocator(). + \see CopyFrom() + */ + template< typename SourceAllocator > + GenericValue(const GenericValue& rhs, Allocator & allocator); + + //! Constructor for boolean value. + /*! \param b Boolean value + \note This constructor is limited to \em real boolean values and rejects + implicitly converted types like arbitrary pointers. Use an explicit cast + to \c bool, if you want to construct a boolean JSON value in such cases. + */ #ifndef RAPIDJSON_DOXYGEN_RUNNING // hide SFINAE from Doxygen - template - explicit GenericValue(T b, RAPIDJSON_ENABLEIF((internal::IsSame))) RAPIDJSON_NOEXCEPT // See #472 + template + explicit GenericValue(T b, RAPIDJSON_ENABLEIF((internal::IsSame))) RAPIDJSON_NOEXCEPT // See #472 #else - explicit GenericValue(bool b) RAPIDJSON_NOEXCEPT + explicit GenericValue(bool b) RAPIDJSON_NOEXCEPT #endif - : data_() { - // safe-guard against failing SFINAE - RAPIDJSON_STATIC_ASSERT((internal::IsSame::Value)); - data_.f.flags = b ? kTrueFlag : kFalseFlag; - } - - //! Constructor for int value. - explicit GenericValue(int i) RAPIDJSON_NOEXCEPT : data_() { - data_.n.i64 = i; - data_.f.flags = (i >= 0) ? (kNumberIntFlag | kUintFlag | kUint64Flag) : kNumberIntFlag; - } - - //! Constructor for unsigned value. - explicit GenericValue(unsigned u) RAPIDJSON_NOEXCEPT : data_() { - data_.n.u64 = u; - data_.f.flags = (u & 0x80000000) ? kNumberUintFlag : (kNumberUintFlag | kIntFlag | kInt64Flag); - } - - //! Constructor for int64_t value. - explicit GenericValue(int64_t i64) RAPIDJSON_NOEXCEPT : data_() { - data_.n.i64 = i64; - data_.f.flags = kNumberInt64Flag; - if (i64 >= 0) { - data_.f.flags |= kNumberUint64Flag; - if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) - data_.f.flags |= kUintFlag; - if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) - data_.f.flags |= kIntFlag; - } - else if (i64 >= static_cast(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) - data_.f.flags |= kIntFlag; - } - - //! Constructor for uint64_t value. - explicit GenericValue(uint64_t u64) RAPIDJSON_NOEXCEPT : data_() { - data_.n.u64 = u64; - data_.f.flags = kNumberUint64Flag; - if (!(u64 & RAPIDJSON_UINT64_C2(0x80000000, 0x00000000))) - data_.f.flags |= kInt64Flag; - if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) - data_.f.flags |= kUintFlag; - if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) - data_.f.flags |= kIntFlag; - } - - //! Constructor for double value. - explicit GenericValue(double d) RAPIDJSON_NOEXCEPT : data_() { data_.n.d = d; data_.f.flags = kNumberDoubleFlag; } - - //! Constructor for constant string (i.e. do not make a copy of string) - GenericValue(const Ch* s, SizeType length) RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(StringRef(s, length)); } - - //! Constructor for constant string (i.e. do not make a copy of string) - explicit GenericValue(StringRefType s) RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(s); } - - //! Constructor for copy-string (i.e. do make a copy of string) - GenericValue(const Ch* s, SizeType length, Allocator& allocator) : data_() { SetStringRaw(StringRef(s, length), allocator); } - - //! Constructor for copy-string (i.e. do make a copy of string) - GenericValue(const Ch*s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); } +: data_() + { // safe-guard against failing SFINAE + RAPIDJSON_STATIC_ASSERT((internal::IsSame::Value)); + data_.f.flags = b ? kTrueFlag : kFalseFlag; + } + + //! Constructor for int value. +explicit GenericValue(int i) RAPIDJSON_NOEXCEPT : data_() + { data_.n.i64 = i; + data_.f.flags = (i >= 0) ? (kNumberIntFlag | kUintFlag | kUint64Flag) : kNumberIntFlag; + } + + //! Constructor for unsigned value. +explicit GenericValue(unsigned u) RAPIDJSON_NOEXCEPT : data_() + { data_.n.u64 = u; + data_.f.flags = (u & 0x80000000) ? kNumberUintFlag : (kNumberUintFlag | kIntFlag | kInt64Flag); + } + + //! Constructor for int64_t value. +explicit GenericValue(int64_t i64) RAPIDJSON_NOEXCEPT : data_() + { data_.n.i64 = i64; + data_.f.flags = kNumberInt64Flag; + if (i64 >= 0) + { data_.f.flags |= kNumberUint64Flag; + if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) + data_.f.flags |= kUintFlag; + if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) + data_.f.flags |= kIntFlag; + } + else if (i64 >= static_cast(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) + data_.f.flags |= kIntFlag; + } + + //! Constructor for uint64_t value. +explicit GenericValue(uint64_t u64) RAPIDJSON_NOEXCEPT : data_() + { data_.n.u64 = u64; + data_.f.flags = kNumberUint64Flag; + if (!(u64 & RAPIDJSON_UINT64_C2(0x80000000, 0x00000000))) + data_.f.flags |= kInt64Flag; + if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) + data_.f.flags |= kUintFlag; + if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) + data_.f.flags |= kIntFlag; + } + + //! Constructor for double value. +explicit GenericValue(double d) RAPIDJSON_NOEXCEPT : data_() { data_.n.d = d; data_.f.flags = kNumberDoubleFlag; } + + //! Constructor for constant string (i.e. do not make a copy of string) +GenericValue(const Ch* s, SizeType length) RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(StringRef(s, length)); } + + //! Constructor for constant string (i.e. do not make a copy of string) +explicit GenericValue(StringRefType s) RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(s); } + + //! Constructor for copy-string (i.e. do make a copy of string) + GenericValue(const Ch* s, SizeType length, Allocator& allocator) : data_() { SetStringRaw(StringRef(s, length), allocator); } + + //! Constructor for copy-string (i.e. do make a copy of string) + GenericValue(const Ch*s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); } #if RAPIDJSON_HAS_STDSTRING - //! Constructor for copy-string from a string object (i.e. do make a copy of string) - /*! \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. - */ - GenericValue(const std::basic_string& s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); } + //! Constructor for copy-string from a string object (i.e. do make a copy of string) + /*! \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. + */ + GenericValue(const std::basic_string& s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); } #endif - //! Constructor for Array. - /*! - \param a An array obtained by \c GetArray(). - \note \c Array is always pass-by-value. - \note the source array is moved into this value and the sourec array becomes empty. - */ - GenericValue(Array a) RAPIDJSON_NOEXCEPT : data_(a.value_.data_) { - a.value_.data_ = Data(); - a.value_.data_.f.flags = kArrayFlag; - } - - //! Constructor for Object. - /*! - \param o An object obtained by \c GetObject(). - \note \c Object is always pass-by-value. - \note the source object is moved into this value and the sourec object becomes empty. - */ - GenericValue(Object o) RAPIDJSON_NOEXCEPT : data_(o.value_.data_) { - o.value_.data_ = Data(); - o.value_.data_.f.flags = kObjectFlag; - } - - //! Destructor. - /*! Need to destruct elements of array, members of object, or copy-string. - */ - ~GenericValue() { - if (Allocator::kNeedFree) { // Shortcut by Allocator's trait - switch(data_.f.flags) { - case kArrayFlag: - { - GenericValue* e = GetElementsPointer(); - for (GenericValue* v = e; v != e + data_.a.size; ++v) - v->~GenericValue(); - Allocator::Free(e); - } - break; - - case kObjectFlag: - for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) - m->~Member(); - Allocator::Free(GetMembersPointer()); - break; - - case kCopyStringFlag: - Allocator::Free(const_cast(GetStringPointer())); - break; - - default: - break; // Do nothing for other types. - } + //! Constructor for Array. + /*! + \param a An array obtained by \c GetArray(). + \note \c Array is always pass-by-value. + \note the source array is moved into this value and the sourec array becomes empty. + */ +GenericValue(Array a) RAPIDJSON_NOEXCEPT : data_(a.value_.data_) + { a.value_.data_ = Data(); + a.value_.data_.f.flags = kArrayFlag; + } + + //! Constructor for Object. + /*! + \param o An object obtained by \c GetObject(). + \note \c Object is always pass-by-value. + \note the source object is moved into this value and the sourec object becomes empty. + */ +GenericValue(Object o) RAPIDJSON_NOEXCEPT : data_(o.value_.data_) + { o.value_.data_ = Data(); + o.value_.data_.f.flags = kObjectFlag; + } + + //! Destructor. + /*! Need to destruct elements of array, members of object, or copy-string. + */ + ~GenericValue() + { if (Allocator::kNeedFree) // Shortcut by Allocator's trait + { switch(data_.f.flags) + { case kArrayFlag: + { GenericValue* e = GetElementsPointer(); + for (GenericValue* v = e; v != e + data_.a.size; ++v) + v->~GenericValue(); + Allocator::Free(e); } - } - - //@} + break; - //!@name Assignment operators - //@{ + case kObjectFlag: + for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) + m->~Member(); + Allocator::Free(GetMembersPointer()); + break; - //! Assignment with move semantics. - /*! \param rhs Source of the assignment. It will become a null value after assignment. - */ - GenericValue& operator=(GenericValue& rhs) RAPIDJSON_NOEXCEPT { - RAPIDJSON_ASSERT(this != &rhs); - this->~GenericValue(); - RawAssign(rhs); - return *this; - } + case kCopyStringFlag: + Allocator::Free(const_cast(GetStringPointer())); + break; -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Move assignment in C++11 - GenericValue& operator=(GenericValue&& rhs) RAPIDJSON_NOEXCEPT { - return *this = rhs.Move(); + default: + break; // Do nothing for other types. + } } -#endif + } - //! Assignment of constant string reference (no copy) - /*! \param str Constant string reference to be assigned - \note This overload is needed to avoid clashes with the generic primitive type assignment overload below. - \see GenericStringRef, operator=(T) - */ - GenericValue& operator=(StringRefType str) RAPIDJSON_NOEXCEPT { - GenericValue s(str); - return *this = s; - } + //@} - //! Assignment with primitive types. - /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t - \param value The value to be assigned. - - \note The source type \c T explicitly disallows all pointer types, - especially (\c const) \ref Ch*. This helps avoiding implicitly - referencing character strings with insufficient lifetime, use - \ref SetString(const Ch*, Allocator&) (for copying) or - \ref StringRef() (to explicitly mark the pointer as constant) instead. - All other pointer types would implicitly convert to \c bool, - use \ref SetBool() instead. - */ - template - RAPIDJSON_DISABLEIF_RETURN((internal::IsPointer), (GenericValue&)) - operator=(T value) { - GenericValue v(value); - return *this = v; - } + //!@name Assignment operators + //@{ - //! Deep-copy assignment from Value - /*! Assigns a \b copy of the Value to the current Value object - \tparam SourceAllocator Allocator type of \c rhs - \param rhs Value to copy from (read-only) - \param allocator Allocator to use for copying - */ - template - GenericValue& CopyFrom(const GenericValue& rhs, Allocator& allocator) { - RAPIDJSON_ASSERT(static_cast(this) != static_cast(&rhs)); - this->~GenericValue(); - new (this) GenericValue(rhs, allocator); - return *this; - } + //! Assignment with move semantics. + /*! \param rhs Source of the assignment. It will become a null value after assignment. + */ + GenericValue& operator=(GenericValue& rhs) RAPIDJSON_NOEXCEPT + { RAPIDJSON_ASSERT(this != &rhs); + this->~GenericValue(); + RawAssign(rhs); + return *this; + } - //! Exchange the contents of this value with those of other. - /*! - \param other Another value. - \note Constant complexity. - */ - GenericValue& Swap(GenericValue& other) RAPIDJSON_NOEXCEPT { - GenericValue temp; - temp.RawAssign(*this); - RawAssign(other); - other.RawAssign(temp); - return *this; - } +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move assignment in C++11 + GenericValue& operator=(GenericValue&& rhs) RAPIDJSON_NOEXCEPT + { return *this = rhs.Move(); + } +#endif - //! free-standing swap function helper - /*! - Helper function to enable support for common swap implementation pattern based on \c std::swap: - \code - void swap(MyClass& a, MyClass& b) { - using std::swap; - swap(a.value, b.value); - // ... - } - \endcode - \see Swap() - */ - friend inline void swap(GenericValue& a, GenericValue& b) RAPIDJSON_NOEXCEPT { a.Swap(b); } - - //! Prepare Value for move semantics - /*! \return *this */ - GenericValue& Move() RAPIDJSON_NOEXCEPT { return *this; } - //@} - - //!@name Equal-to and not-equal-to operators - //@{ - //! Equal-to operator - /*! - \note If an object contains duplicated named member, comparing equality with any object is always \c false. - \note Linear time complexity (number of all values in the subtree and total lengths of all strings). - */ - template - bool operator==(const GenericValue& rhs) const { - typedef GenericValue RhsType; - if (GetType() != rhs.GetType()) + //! Assignment of constant string reference (no copy) + /*! \param str Constant string reference to be assigned + \note This overload is needed to avoid clashes with the generic primitive type assignment overload below. + \see GenericStringRef, operator=(T) + */ + GenericValue& operator=(StringRefType str) RAPIDJSON_NOEXCEPT + { GenericValue s(str); + return *this = s; + } + + //! Assignment with primitive types. + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t + \param value The value to be assigned. + + \note The source type \c T explicitly disallows all pointer types, + especially (\c const) \ref Ch*. This helps avoiding implicitly + referencing character strings with insufficient lifetime, use + \ref SetString(const Ch*, Allocator&) (for copying) or + \ref StringRef() (to explicitly mark the pointer as constant) instead. + All other pointer types would implicitly convert to \c bool, + use \ref SetBool() instead. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::IsPointer), (GenericValue&)) + operator=(T value) + { GenericValue v(value); + return *this = v; + } + + //! Deep-copy assignment from Value + /*! Assigns a \b copy of the Value to the current Value object + \tparam SourceAllocator Allocator type of \c rhs + \param rhs Value to copy from (read-only) + \param allocator Allocator to use for copying + */ + template + GenericValue& CopyFrom(const GenericValue& rhs, Allocator& allocator) + { RAPIDJSON_ASSERT(static_cast(this) != static_cast(&rhs)); + this->~GenericValue(); + new (this) GenericValue(rhs, allocator); + return *this; + } + + //! Exchange the contents of this value with those of other. + /*! + \param other Another value. + \note Constant complexity. + */ + GenericValue& Swap(GenericValue& other) RAPIDJSON_NOEXCEPT + { GenericValue temp; + temp.RawAssign(*this); + RawAssign(other); + other.RawAssign(temp); + return *this; + } + + //! free-standing swap function helper + /*! + Helper function to enable support for common swap implementation pattern based on \c std::swap: + \code + void swap(MyClass& a, MyClass& b) { + using std::swap; + swap(a.value, b.value); + // ... + } + \endcode + \see Swap() + */ + friend inline void swap(GenericValue& a, GenericValue& b) RAPIDJSON_NOEXCEPT { a.Swap(b); } + + //! Prepare Value for move semantics + /*! \return *this */ + GenericValue& Move() RAPIDJSON_NOEXCEPT { return *this; } + //@} + + //!@name Equal-to and not-equal-to operators + //@{ + //! Equal-to operator + /*! + \note If an object contains duplicated named member, comparing equality with any object is always \c false. + \note Linear time complexity (number of all values in the subtree and total lengths of all strings). + */ + template + bool operator==(const GenericValue& rhs) const + { typedef GenericValue RhsType; + if (GetType() != rhs.GetType()) + return false; + + switch (GetType()) + { case kObjectType: // Warning: O(n^2) inner-loop + if (data_.o.size != rhs.data_.o.size) + return false; + for (ConstMemberIterator lhsMemberItr = MemberBegin(); lhsMemberItr != MemberEnd(); ++lhsMemberItr) + { typename RhsType::ConstMemberIterator rhsMemberItr = rhs.FindMember(lhsMemberItr->name); + if (rhsMemberItr == rhs.MemberEnd() || lhsMemberItr->value != rhsMemberItr->value) return false; - - switch (GetType()) { - case kObjectType: // Warning: O(n^2) inner-loop - if (data_.o.size != rhs.data_.o.size) - return false; - for (ConstMemberIterator lhsMemberItr = MemberBegin(); lhsMemberItr != MemberEnd(); ++lhsMemberItr) { - typename RhsType::ConstMemberIterator rhsMemberItr = rhs.FindMember(lhsMemberItr->name); - if (rhsMemberItr == rhs.MemberEnd() || lhsMemberItr->value != rhsMemberItr->value) - return false; - } - return true; - - case kArrayType: - if (data_.a.size != rhs.data_.a.size) - return false; - for (SizeType i = 0; i < data_.a.size; i++) - if ((*this)[i] != rhs[i]) - return false; - return true; - - case kStringType: - return StringEqual(rhs); - - case kNumberType: - if (IsDouble() || rhs.IsDouble()) { - double a = GetDouble(); // May convert from integer to double. - double b = rhs.GetDouble(); // Ditto - return a >= b && a <= b; // Prevent -Wfloat-equal - } - else - return data_.n.u64 == rhs.data_.n.u64; - - default: - return true; } - } + return true; - //! Equal-to operator with const C-string pointer - bool operator==(const Ch* rhs) const { return *this == GenericValue(StringRef(rhs)); } + case kArrayType: + if (data_.a.size != rhs.data_.a.size) + return false; + for (SizeType i = 0; i < data_.a.size; i++) + if ((*this)[i] != rhs[i]) + return false; + return true; -#if RAPIDJSON_HAS_STDSTRING - //! Equal-to operator with string object - /*! \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. - */ - bool operator==(const std::basic_string& rhs) const { return *this == GenericValue(StringRef(rhs)); } -#endif + case kStringType: + return StringEqual(rhs); - //! Equal-to operator with primitive types - /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c double, \c true, \c false - */ - template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr,internal::IsGenericValue >), (bool)) operator==(const T& rhs) const { return *this == GenericValue(rhs); } - - //! Not-equal-to operator - /*! \return !(*this == rhs) - */ - template - bool operator!=(const GenericValue& rhs) const { return !(*this == rhs); } - - //! Not-equal-to operator with const C-string pointer - bool operator!=(const Ch* rhs) const { return !(*this == rhs); } - - //! Not-equal-to operator with arbitrary types - /*! \return !(*this == rhs) - */ - template RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& rhs) const { return !(*this == rhs); } - - //! Equal-to operator with arbitrary types (symmetric version) - /*! \return (rhs == lhs) - */ - template friend RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator==(const T& lhs, const GenericValue& rhs) { return rhs == lhs; } - - //! Not-Equal-to operator with arbitrary types (symmetric version) - /*! \return !(rhs == lhs) - */ - template friend RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& lhs, const GenericValue& rhs) { return !(rhs == lhs); } - //@} - - //!@name Type - //@{ - - Type GetType() const { return static_cast(data_.f.flags & kTypeMask); } - bool IsNull() const { return data_.f.flags == kNullFlag; } - bool IsFalse() const { return data_.f.flags == kFalseFlag; } - bool IsTrue() const { return data_.f.flags == kTrueFlag; } - bool IsBool() const { return (data_.f.flags & kBoolFlag) != 0; } - bool IsObject() const { return data_.f.flags == kObjectFlag; } - bool IsArray() const { return data_.f.flags == kArrayFlag; } - bool IsNumber() const { return (data_.f.flags & kNumberFlag) != 0; } - bool IsInt() const { return (data_.f.flags & kIntFlag) != 0; } - bool IsUint() const { return (data_.f.flags & kUintFlag) != 0; } - bool IsInt64() const { return (data_.f.flags & kInt64Flag) != 0; } - bool IsUint64() const { return (data_.f.flags & kUint64Flag) != 0; } - bool IsDouble() const { return (data_.f.flags & kDoubleFlag) != 0; } - bool IsString() const { return (data_.f.flags & kStringFlag) != 0; } - - // Checks whether a number can be losslessly converted to a double. - bool IsLosslessDouble() const { - if (!IsNumber()) return false; - if (IsUint64()) { - uint64_t u = GetUint64(); - volatile double d = static_cast(u); - return (d >= 0.0) - && (d < static_cast(std::numeric_limits::max())) - && (u == static_cast(d)); - } - if (IsInt64()) { - int64_t i = GetInt64(); - volatile double d = static_cast(i); - return (d >= static_cast(std::numeric_limits::min())) - && (d < static_cast(std::numeric_limits::max())) - && (i == static_cast(d)); + case kNumberType: + if (IsDouble() || rhs.IsDouble()) + { double a = GetDouble(); // May convert from integer to double. + double b = rhs.GetDouble(); // Ditto + return a >= b && a <= b; // Prevent -Wfloat-equal } - return true; // double, int, uint are always lossless - } + else + return data_.n.u64 == rhs.data_.n.u64; - // Checks whether a number is a float (possible lossy). - bool IsFloat() const { - if ((data_.f.flags & kDoubleFlag) == 0) - return false; - double d = GetDouble(); - return d >= -3.4028234e38 && d <= 3.4028234e38; - } - // Checks whether a number can be losslessly converted to a float. - bool IsLosslessFloat() const { - if (!IsNumber()) return false; - double a = GetDouble(); - if (a < static_cast(-std::numeric_limits::max()) - || a > static_cast(std::numeric_limits::max())) - return false; - double b = static_cast(static_cast(a)); - return a >= b && a <= b; // Prevent -Wfloat-equal + default: + return true; } + } - //@} - - //!@name Null - //@{ + //! Equal-to operator with const C-string pointer + bool operator==(const Ch* rhs) const { return *this == GenericValue(StringRef(rhs)); } - GenericValue& SetNull() { this->~GenericValue(); new (this) GenericValue(); return *this; } - - //@} - - //!@name Bool - //@{ - - bool GetBool() const { RAPIDJSON_ASSERT(IsBool()); return data_.f.flags == kTrueFlag; } - //!< Set boolean value - /*! \post IsBool() == true */ - GenericValue& SetBool(bool b) { this->~GenericValue(); new (this) GenericValue(b); return *this; } - - //@} - - //!@name Object - //@{ - - //! Set this value as an empty object. - /*! \post IsObject() == true */ - GenericValue& SetObject() { this->~GenericValue(); new (this) GenericValue(kObjectType); return *this; } - - //! Get the number of members in the object. - SizeType MemberCount() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.size; } - - //! Check whether the object is empty. - bool ObjectEmpty() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.size == 0; } +#if RAPIDJSON_HAS_STDSTRING + //! Equal-to operator with string object + /*! \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. + */ + bool operator==(const std::basic_string& rhs) const { return *this == GenericValue(StringRef(rhs)); } +#endif - //! Get a value from an object associated with the name. - /*! \pre IsObject() == true - \tparam T Either \c Ch or \c const \c Ch (template used for disambiguation with \ref operator[](SizeType)) - \note In version 0.1x, if the member is not found, this function returns a null value. This makes issue 7. - Since 0.2, if the name is not correct, it will assert. - If user is unsure whether a member exists, user should use HasMember() first. - A better approach is to use FindMember(). - \note Linear time complexity. - */ - template - RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >),(GenericValue&)) operator[](T* name) { - GenericValue n(StringRef(name)); - return (*this)[n]; - } - template - RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >),(const GenericValue&)) operator[](T* name) const { return const_cast(*this)[name]; } - - //! Get a value from an object associated with the name. - /*! \pre IsObject() == true - \tparam SourceAllocator Allocator of the \c name value - - \note Compared to \ref operator[](T*), this version is faster because it does not need a StrLen(). - And it can also handle strings with embedded null characters. - - \note Linear time complexity. - */ - template - GenericValue& operator[](const GenericValue& name) { - MemberIterator member = FindMember(name); - if (member != MemberEnd()) - return member->value; - else { - RAPIDJSON_ASSERT(false); // see above note - - // This will generate -Wexit-time-destructors in clang - // static GenericValue NullValue; - // return NullValue; - - // Use static buffer and placement-new to prevent destruction - static char buffer[sizeof(GenericValue)]; - return *new (buffer) GenericValue(); - } - } - template - const GenericValue& operator[](const GenericValue& name) const { return const_cast(*this)[name]; } + //! Equal-to operator with primitive types + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c double, \c true, \c false + */ + template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr,internal::IsGenericValue >), (bool)) operator==(const T& rhs) const { return *this == GenericValue(rhs); } + + //! Not-equal-to operator + /*! \return !(*this == rhs) + */ + template + bool operator!=(const GenericValue& rhs) const { return !(*this == rhs); } + + //! Not-equal-to operator with const C-string pointer + bool operator!=(const Ch* rhs) const { return !(*this == rhs); } + + //! Not-equal-to operator with arbitrary types + /*! \return !(*this == rhs) + */ + template RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& rhs) const { return !(*this == rhs); } + + //! Equal-to operator with arbitrary types (symmetric version) + /*! \return (rhs == lhs) + */ + template friend RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator==(const T& lhs, const GenericValue& rhs) { return rhs == lhs; } + + //! Not-Equal-to operator with arbitrary types (symmetric version) + /*! \return !(rhs == lhs) + */ + template friend RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& lhs, const GenericValue& rhs) { return !(rhs == lhs); } + //@} + + //!@name Type + //@{ + + Type GetType() const { return static_cast(data_.f.flags & kTypeMask); } + bool IsNull() const { return data_.f.flags == kNullFlag; } + bool IsFalse() const { return data_.f.flags == kFalseFlag; } + bool IsTrue() const { return data_.f.flags == kTrueFlag; } + bool IsBool() const { return (data_.f.flags & kBoolFlag) != 0; } + bool IsObject() const { return data_.f.flags == kObjectFlag; } + bool IsArray() const { return data_.f.flags == kArrayFlag; } + bool IsNumber() const { return (data_.f.flags & kNumberFlag) != 0; } + bool IsInt() const { return (data_.f.flags & kIntFlag) != 0; } + bool IsUint() const { return (data_.f.flags & kUintFlag) != 0; } + bool IsInt64() const { return (data_.f.flags & kInt64Flag) != 0; } + bool IsUint64() const { return (data_.f.flags & kUint64Flag) != 0; } + bool IsDouble() const { return (data_.f.flags & kDoubleFlag) != 0; } + bool IsString() const { return (data_.f.flags & kStringFlag) != 0; } + + // Checks whether a number can be losslessly converted to a double. + bool IsLosslessDouble() const + { if (!IsNumber()) return false; + if (IsUint64()) + { uint64_t u = GetUint64(); + volatile double d = static_cast(u); + return (d >= 0.0) + && (d < static_cast(std::numeric_limits::max())) + && (u == static_cast(d)); + } + if (IsInt64()) + { int64_t i = GetInt64(); + volatile double d = static_cast(i); + return (d >= static_cast(std::numeric_limits::min())) + && (d < static_cast(std::numeric_limits::max())) + && (i == static_cast(d)); + } + return true; // double, int, uint are always lossless + } + + // Checks whether a number is a float (possible lossy). + bool IsFloat() const + { if ((data_.f.flags & kDoubleFlag) == 0) + return false; + double d = GetDouble(); + return d >= -3.4028234e38 && d <= 3.4028234e38; + } + // Checks whether a number can be losslessly converted to a float. + bool IsLosslessFloat() const + { if (!IsNumber()) return false; + double a = GetDouble(); + if (a < static_cast(-std::numeric_limits::max()) + || a > static_cast(std::numeric_limits::max())) + return false; + double b = static_cast(static_cast(a)); + return a >= b && a <= b; // Prevent -Wfloat-equal + } + + //@} + + //!@name Null + //@{ + + GenericValue& SetNull() { this->~GenericValue(); new (this) GenericValue(); return *this; } + + //@} + + //!@name Bool + //@{ + + bool GetBool() const { RAPIDJSON_ASSERT(IsBool()); return data_.f.flags == kTrueFlag; } + //!< Set boolean value + /*! \post IsBool() == true */ + GenericValue& SetBool(bool b) { this->~GenericValue(); new (this) GenericValue(b); return *this; } + + //@} + + //!@name Object + //@{ + + //! Set this value as an empty object. + /*! \post IsObject() == true */ + GenericValue& SetObject() { this->~GenericValue(); new (this) GenericValue(kObjectType); return *this; } + + //! Get the number of members in the object. + SizeType MemberCount() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.size; } + + //! Check whether the object is empty. + bool ObjectEmpty() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.size == 0; } + + //! Get a value from an object associated with the name. + /*! \pre IsObject() == true + \tparam T Either \c Ch or \c const \c Ch (template used for disambiguation with \ref operator[](SizeType)) + \note In version 0.1x, if the member is not found, this function returns a null value. This makes issue 7. + Since 0.2, if the name is not correct, it will assert. + If user is unsure whether a member exists, user should use HasMember() first. + A better approach is to use FindMember(). + \note Linear time complexity. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >),(GenericValue&)) operator[](T* name) + { GenericValue n(StringRef(name)); + return (*this)[n]; + } + template + RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >),(const GenericValue&)) operator[](T* name) const { return const_cast(*this)[name]; } + + //! Get a value from an object associated with the name. + /*! \pre IsObject() == true + \tparam SourceAllocator Allocator of the \c name value + + \note Compared to \ref operator[](T*), this version is faster because it does not need a StrLen(). + And it can also handle strings with embedded null characters. + + \note Linear time complexity. + */ + template + GenericValue& operator[](const GenericValue& name) + { MemberIterator member = FindMember(name); + if (member != MemberEnd()) + return member->value; + else + { RAPIDJSON_ASSERT(false); // see above note + + // This will generate -Wexit-time-destructors in clang + // static GenericValue NullValue; + // return NullValue; + + // Use static buffer and placement-new to prevent destruction + static char buffer[sizeof(GenericValue)]; + return *new (buffer) GenericValue(); + } + } + template + const GenericValue& operator[](const GenericValue& name) const { return const_cast(*this)[name]; } #if RAPIDJSON_HAS_STDSTRING - //! Get a value from an object associated with name (string object). - GenericValue& operator[](const std::basic_string& name) { return (*this)[GenericValue(StringRef(name))]; } - const GenericValue& operator[](const std::basic_string& name) const { return (*this)[GenericValue(StringRef(name))]; } + //! Get a value from an object associated with name (string object). + GenericValue& operator[](const std::basic_string& name) { return (*this)[GenericValue(StringRef(name))]; } + const GenericValue& operator[](const std::basic_string& name) const { return (*this)[GenericValue(StringRef(name))]; } #endif - //! Const member iterator - /*! \pre IsObject() == true */ - ConstMemberIterator MemberBegin() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(GetMembersPointer()); } - //! Const \em past-the-end member iterator - /*! \pre IsObject() == true */ - ConstMemberIterator MemberEnd() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(GetMembersPointer() + data_.o.size); } - //! Member iterator - /*! \pre IsObject() == true */ - MemberIterator MemberBegin() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(GetMembersPointer()); } - //! \em Past-the-end member iterator - /*! \pre IsObject() == true */ - MemberIterator MemberEnd() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(GetMembersPointer() + data_.o.size); } - - //! Check whether a member exists in the object. - /*! - \param name Member name to be searched. - \pre IsObject() == true - \return Whether a member with that name exists. - \note It is better to use FindMember() directly if you need the obtain the value as well. - \note Linear time complexity. - */ - bool HasMember(const Ch* name) const { return FindMember(name) != MemberEnd(); } + //! Const member iterator + /*! \pre IsObject() == true */ + ConstMemberIterator MemberBegin() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(GetMembersPointer()); } + //! Const \em past-the-end member iterator + /*! \pre IsObject() == true */ + ConstMemberIterator MemberEnd() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(GetMembersPointer() + data_.o.size); } + //! Member iterator + /*! \pre IsObject() == true */ + MemberIterator MemberBegin() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(GetMembersPointer()); } + //! \em Past-the-end member iterator + /*! \pre IsObject() == true */ + MemberIterator MemberEnd() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(GetMembersPointer() + data_.o.size); } + + //! Check whether a member exists in the object. + /*! + \param name Member name to be searched. + \pre IsObject() == true + \return Whether a member with that name exists. + \note It is better to use FindMember() directly if you need the obtain the value as well. + \note Linear time complexity. + */ + bool HasMember(const Ch* name) const { return FindMember(name) != MemberEnd(); } #if RAPIDJSON_HAS_STDSTRING - //! Check whether a member exists in the object with string object. - /*! - \param name Member name to be searched. - \pre IsObject() == true - \return Whether a member with that name exists. - \note It is better to use FindMember() directly if you need the obtain the value as well. - \note Linear time complexity. - */ - bool HasMember(const std::basic_string& name) const { return FindMember(name) != MemberEnd(); } + //! Check whether a member exists in the object with string object. + /*! + \param name Member name to be searched. + \pre IsObject() == true + \return Whether a member with that name exists. + \note It is better to use FindMember() directly if you need the obtain the value as well. + \note Linear time complexity. + */ + bool HasMember(const std::basic_string& name) const { return FindMember(name) != MemberEnd(); } #endif - //! Check whether a member exists in the object with GenericValue name. - /*! - This version is faster because it does not need a StrLen(). It can also handle string with null character. - \param name Member name to be searched. - \pre IsObject() == true - \return Whether a member with that name exists. - \note It is better to use FindMember() directly if you need the obtain the value as well. - \note Linear time complexity. - */ - template - bool HasMember(const GenericValue& name) const { return FindMember(name) != MemberEnd(); } - - //! Find member by name. - /*! - \param name Member name to be searched. - \pre IsObject() == true - \return Iterator to member, if it exists. - Otherwise returns \ref MemberEnd(). - - \note Earlier versions of Rapidjson returned a \c NULL pointer, in case - the requested member doesn't exist. For consistency with e.g. - \c std::map, this has been changed to MemberEnd() now. - \note Linear time complexity. - */ - MemberIterator FindMember(const Ch* name) { - GenericValue n(StringRef(name)); - return FindMember(n); - } - - ConstMemberIterator FindMember(const Ch* name) const { return const_cast(*this).FindMember(name); } - - //! Find member by name. - /*! - This version is faster because it does not need a StrLen(). It can also handle string with null character. - \param name Member name to be searched. - \pre IsObject() == true - \return Iterator to member, if it exists. - Otherwise returns \ref MemberEnd(). - - \note Earlier versions of Rapidjson returned a \c NULL pointer, in case - the requested member doesn't exist. For consistency with e.g. - \c std::map, this has been changed to MemberEnd() now. - \note Linear time complexity. - */ - template - MemberIterator FindMember(const GenericValue& name) { - RAPIDJSON_ASSERT(IsObject()); - RAPIDJSON_ASSERT(name.IsString()); - MemberIterator member = MemberBegin(); - for ( ; member != MemberEnd(); ++member) - if (name.StringEqual(member->name)) - break; - return member; - } - template ConstMemberIterator FindMember(const GenericValue& name) const { return const_cast(*this).FindMember(name); } + //! Check whether a member exists in the object with GenericValue name. + /*! + This version is faster because it does not need a StrLen(). It can also handle string with null character. + \param name Member name to be searched. + \pre IsObject() == true + \return Whether a member with that name exists. + \note It is better to use FindMember() directly if you need the obtain the value as well. + \note Linear time complexity. + */ + template + bool HasMember(const GenericValue& name) const { return FindMember(name) != MemberEnd(); } + + //! Find member by name. + /*! + \param name Member name to be searched. + \pre IsObject() == true + \return Iterator to member, if it exists. + Otherwise returns \ref MemberEnd(). + + \note Earlier versions of Rapidjson returned a \c NULL pointer, in case + the requested member doesn't exist. For consistency with e.g. + \c std::map, this has been changed to MemberEnd() now. + \note Linear time complexity. + */ + MemberIterator FindMember(const Ch* name) + { GenericValue n(StringRef(name)); + return FindMember(n); + } + + ConstMemberIterator FindMember(const Ch* name) const { return const_cast(*this).FindMember(name); } + + //! Find member by name. + /*! + This version is faster because it does not need a StrLen(). It can also handle string with null character. + \param name Member name to be searched. + \pre IsObject() == true + \return Iterator to member, if it exists. + Otherwise returns \ref MemberEnd(). + + \note Earlier versions of Rapidjson returned a \c NULL pointer, in case + the requested member doesn't exist. For consistency with e.g. + \c std::map, this has been changed to MemberEnd() now. + \note Linear time complexity. + */ + template + MemberIterator FindMember(const GenericValue& name) + { RAPIDJSON_ASSERT(IsObject()); + RAPIDJSON_ASSERT(name.IsString()); + MemberIterator member = MemberBegin(); + for ( ; member != MemberEnd(); ++member) + if (name.StringEqual(member->name)) + break; + return member; + } + template ConstMemberIterator FindMember(const GenericValue& name) const { return const_cast(*this).FindMember(name); } #if RAPIDJSON_HAS_STDSTRING - //! Find member by string object name. - /*! - \param name Member name to be searched. - \pre IsObject() == true - \return Iterator to member, if it exists. - Otherwise returns \ref MemberEnd(). - */ - MemberIterator FindMember(const std::basic_string& name) { return FindMember(GenericValue(StringRef(name))); } - ConstMemberIterator FindMember(const std::basic_string& name) const { return FindMember(GenericValue(StringRef(name))); } + //! Find member by string object name. + /*! + \param name Member name to be searched. + \pre IsObject() == true + \return Iterator to member, if it exists. + Otherwise returns \ref MemberEnd(). + */ + MemberIterator FindMember(const std::basic_string& name) { return FindMember(GenericValue(StringRef(name))); } + ConstMemberIterator FindMember(const std::basic_string& name) const { return FindMember(GenericValue(StringRef(name))); } #endif - //! Add a member (name-value pair) to the object. - /*! \param name A string value as name of member. - \param value Value of any type. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \note The ownership of \c name and \c value will be transferred to this object on success. - \pre IsObject() && name.IsString() - \post name.IsNull() && value.IsNull() - \note Amortized Constant time complexity. - */ - GenericValue& AddMember(GenericValue& name, GenericValue& value, Allocator& allocator) { - RAPIDJSON_ASSERT(IsObject()); - RAPIDJSON_ASSERT(name.IsString()); - - ObjectData& o = data_.o; - if (o.size >= o.capacity) { - if (o.capacity == 0) { - o.capacity = kDefaultObjectCapacity; - SetMembersPointer(reinterpret_cast(allocator.Malloc(o.capacity * sizeof(Member)))); - } - else { - SizeType oldCapacity = o.capacity; - o.capacity += (oldCapacity + 1) / 2; // grow by factor 1.5 - SetMembersPointer(reinterpret_cast(allocator.Realloc(GetMembersPointer(), oldCapacity * sizeof(Member), o.capacity * sizeof(Member)))); - } - } - Member* members = GetMembersPointer(); - members[o.size].name.RawAssign(name); - members[o.size].value.RawAssign(value); - o.size++; - return *this; - } - - //! Add a constant string value as member (name-value pair) to the object. - /*! \param name A string value as name of member. - \param value constant string reference as value of member. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \pre IsObject() - \note This overload is needed to avoid clashes with the generic primitive type AddMember(GenericValue&,T,Allocator&) overload below. - \note Amortized Constant time complexity. - */ - GenericValue& AddMember(GenericValue& name, StringRefType value, Allocator& allocator) { - GenericValue v(value); - return AddMember(name, v, allocator); - } + //! Add a member (name-value pair) to the object. + /*! \param name A string value as name of member. + \param value Value of any type. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \note The ownership of \c name and \c value will be transferred to this object on success. + \pre IsObject() && name.IsString() + \post name.IsNull() && value.IsNull() + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(GenericValue& name, GenericValue& value, Allocator& allocator) + { RAPIDJSON_ASSERT(IsObject()); + RAPIDJSON_ASSERT(name.IsString()); + + ObjectData& o = data_.o; + if (o.size >= o.capacity) + { if (o.capacity == 0) + { o.capacity = kDefaultObjectCapacity; + SetMembersPointer(reinterpret_cast(allocator.Malloc(o.capacity * sizeof(Member)))); + } + else + { SizeType oldCapacity = o.capacity; + o.capacity += (oldCapacity + 1) / 2; // grow by factor 1.5 + SetMembersPointer(reinterpret_cast(allocator.Realloc(GetMembersPointer(), oldCapacity * sizeof(Member), o.capacity * sizeof(Member)))); + } + } + Member* members = GetMembersPointer(); + members[o.size].name.RawAssign(name); + members[o.size].value.RawAssign(value); + o.size++; + return *this; + } + + //! Add a constant string value as member (name-value pair) to the object. + /*! \param name A string value as name of member. + \param value constant string reference as value of member. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + \note This overload is needed to avoid clashes with the generic primitive type AddMember(GenericValue&,T,Allocator&) overload below. + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(GenericValue& name, StringRefType value, Allocator& allocator) + { GenericValue v(value); + return AddMember(name, v, allocator); + } #if RAPIDJSON_HAS_STDSTRING - //! Add a string object as member (name-value pair) to the object. - /*! \param name A string value as name of member. - \param value constant string reference as value of member. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \pre IsObject() - \note This overload is needed to avoid clashes with the generic primitive type AddMember(GenericValue&,T,Allocator&) overload below. - \note Amortized Constant time complexity. - */ - GenericValue& AddMember(GenericValue& name, std::basic_string& value, Allocator& allocator) { - GenericValue v(value, allocator); - return AddMember(name, v, allocator); - } + //! Add a string object as member (name-value pair) to the object. + /*! \param name A string value as name of member. + \param value constant string reference as value of member. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + \note This overload is needed to avoid clashes with the generic primitive type AddMember(GenericValue&,T,Allocator&) overload below. + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(GenericValue& name, std::basic_string& value, Allocator& allocator) + { GenericValue v(value, allocator); + return AddMember(name, v, allocator); + } #endif - //! Add any primitive value as member (name-value pair) to the object. - /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t - \param name A string value as name of member. - \param value Value of primitive type \c T as value of member - \param allocator Allocator for reallocating memory. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \pre IsObject() - - \note The source type \c T explicitly disallows all pointer types, - especially (\c const) \ref Ch*. This helps avoiding implicitly - referencing character strings with insufficient lifetime, use - \ref AddMember(StringRefType, GenericValue&, Allocator&) or \ref - AddMember(StringRefType, StringRefType, Allocator&). - All other pointer types would implicitly convert to \c bool, - use an explicit cast instead, if needed. - \note Amortized Constant time complexity. - */ - template - RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) - AddMember(GenericValue& name, T value, Allocator& allocator) { - GenericValue v(value); - return AddMember(name, v, allocator); - } + //! Add any primitive value as member (name-value pair) to the object. + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t + \param name A string value as name of member. + \param value Value of primitive type \c T as value of member + \param allocator Allocator for reallocating memory. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + + \note The source type \c T explicitly disallows all pointer types, + especially (\c const) \ref Ch*. This helps avoiding implicitly + referencing character strings with insufficient lifetime, use + \ref AddMember(StringRefType, GenericValue&, Allocator&) or \ref + AddMember(StringRefType, StringRefType, Allocator&). + All other pointer types would implicitly convert to \c bool, + use an explicit cast instead, if needed. + \note Amortized Constant time complexity. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) + AddMember(GenericValue& name, T value, Allocator& allocator) + { GenericValue v(value); + return AddMember(name, v, allocator); + } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS - GenericValue& AddMember(GenericValue&& name, GenericValue&& value, Allocator& allocator) { - return AddMember(name, value, allocator); - } - GenericValue& AddMember(GenericValue&& name, GenericValue& value, Allocator& allocator) { - return AddMember(name, value, allocator); - } - GenericValue& AddMember(GenericValue& name, GenericValue&& value, Allocator& allocator) { - return AddMember(name, value, allocator); - } - GenericValue& AddMember(StringRefType name, GenericValue&& value, Allocator& allocator) { - GenericValue n(name); - return AddMember(n, value, allocator); - } + GenericValue& AddMember(GenericValue&& name, GenericValue&& value, Allocator& allocator) + { return AddMember(name, value, allocator); + } + GenericValue& AddMember(GenericValue&& name, GenericValue& value, Allocator& allocator) + { return AddMember(name, value, allocator); + } + GenericValue& AddMember(GenericValue& name, GenericValue&& value, Allocator& allocator) + { return AddMember(name, value, allocator); + } + GenericValue& AddMember(StringRefType name, GenericValue&& value, Allocator& allocator) + { GenericValue n(name); + return AddMember(n, value, allocator); + } #endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Add a member (name-value pair) to the object. - /*! \param name A constant string reference as name of member. - \param value Value of any type. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \note The ownership of \c value will be transferred to this object on success. - \pre IsObject() - \post value.IsNull() - \note Amortized Constant time complexity. - */ - GenericValue& AddMember(StringRefType name, GenericValue& value, Allocator& allocator) { - GenericValue n(name); - return AddMember(n, value, allocator); - } - - //! Add a constant string value as member (name-value pair) to the object. - /*! \param name A constant string reference as name of member. - \param value constant string reference as value of member. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \pre IsObject() - \note This overload is needed to avoid clashes with the generic primitive type AddMember(StringRefType,T,Allocator&) overload below. - \note Amortized Constant time complexity. - */ - GenericValue& AddMember(StringRefType name, StringRefType value, Allocator& allocator) { - GenericValue v(value); - return AddMember(name, v, allocator); - } - - //! Add any primitive value as member (name-value pair) to the object. - /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t - \param name A constant string reference as name of member. - \param value Value of primitive type \c T as value of member - \param allocator Allocator for reallocating memory. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \pre IsObject() - - \note The source type \c T explicitly disallows all pointer types, - especially (\c const) \ref Ch*. This helps avoiding implicitly - referencing character strings with insufficient lifetime, use - \ref AddMember(StringRefType, GenericValue&, Allocator&) or \ref - AddMember(StringRefType, StringRefType, Allocator&). - All other pointer types would implicitly convert to \c bool, - use an explicit cast instead, if needed. - \note Amortized Constant time complexity. - */ - template - RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) - AddMember(StringRefType name, T value, Allocator& allocator) { - GenericValue n(name); - return AddMember(n, value, allocator); - } - - //! Remove all members in the object. - /*! This function do not deallocate memory in the object, i.e. the capacity is unchanged. - \note Linear time complexity. - */ - void RemoveAllMembers() { - RAPIDJSON_ASSERT(IsObject()); - for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) - m->~Member(); - data_.o.size = 0; - } - - //! Remove a member in object by its name. - /*! \param name Name of member to be removed. - \return Whether the member existed. - \note This function may reorder the object members. Use \ref - EraseMember(ConstMemberIterator) if you need to preserve the - relative order of the remaining members. - \note Linear time complexity. - */ - bool RemoveMember(const Ch* name) { - GenericValue n(StringRef(name)); - return RemoveMember(n); - } + //! Add a member (name-value pair) to the object. + /*! \param name A constant string reference as name of member. + \param value Value of any type. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \note The ownership of \c value will be transferred to this object on success. + \pre IsObject() + \post value.IsNull() + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(StringRefType name, GenericValue& value, Allocator& allocator) + { GenericValue n(name); + return AddMember(n, value, allocator); + } + + //! Add a constant string value as member (name-value pair) to the object. + /*! \param name A constant string reference as name of member. + \param value constant string reference as value of member. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + \note This overload is needed to avoid clashes with the generic primitive type AddMember(StringRefType,T,Allocator&) overload below. + \note Amortized Constant time complexity. + */ + GenericValue& AddMember(StringRefType name, StringRefType value, Allocator& allocator) + { GenericValue v(value); + return AddMember(name, v, allocator); + } + + //! Add any primitive value as member (name-value pair) to the object. + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t + \param name A constant string reference as name of member. + \param value Value of primitive type \c T as value of member + \param allocator Allocator for reallocating memory. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \pre IsObject() + + \note The source type \c T explicitly disallows all pointer types, + especially (\c const) \ref Ch*. This helps avoiding implicitly + referencing character strings with insufficient lifetime, use + \ref AddMember(StringRefType, GenericValue&, Allocator&) or \ref + AddMember(StringRefType, StringRefType, Allocator&). + All other pointer types would implicitly convert to \c bool, + use an explicit cast instead, if needed. + \note Amortized Constant time complexity. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) + AddMember(StringRefType name, T value, Allocator& allocator) + { GenericValue n(name); + return AddMember(n, value, allocator); + } + + //! Remove all members in the object. + /*! This function do not deallocate memory in the object, i.e. the capacity is unchanged. + \note Linear time complexity. + */ + void RemoveAllMembers() + { RAPIDJSON_ASSERT(IsObject()); + for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) + m->~Member(); + data_.o.size = 0; + } + + //! Remove a member in object by its name. + /*! \param name Name of member to be removed. + \return Whether the member existed. + \note This function may reorder the object members. Use \ref + EraseMember(ConstMemberIterator) if you need to preserve the + relative order of the remaining members. + \note Linear time complexity. + */ + bool RemoveMember(const Ch* name) + { GenericValue n(StringRef(name)); + return RemoveMember(n); + } #if RAPIDJSON_HAS_STDSTRING - bool RemoveMember(const std::basic_string& name) { return RemoveMember(GenericValue(StringRef(name))); } + bool RemoveMember(const std::basic_string& name) { return RemoveMember(GenericValue(StringRef(name))); } #endif - template - bool RemoveMember(const GenericValue& name) { - MemberIterator m = FindMember(name); - if (m != MemberEnd()) { - RemoveMember(m); - return true; - } - else - return false; - } - - //! Remove a member in object by iterator. - /*! \param m member iterator (obtained by FindMember() or MemberBegin()). - \return the new iterator after removal. - \note This function may reorder the object members. Use \ref - EraseMember(ConstMemberIterator) if you need to preserve the - relative order of the remaining members. - \note Constant time complexity. - */ - MemberIterator RemoveMember(MemberIterator m) { - RAPIDJSON_ASSERT(IsObject()); - RAPIDJSON_ASSERT(data_.o.size > 0); - RAPIDJSON_ASSERT(GetMembersPointer() != 0); - RAPIDJSON_ASSERT(m >= MemberBegin() && m < MemberEnd()); - - MemberIterator last(GetMembersPointer() + (data_.o.size - 1)); - if (data_.o.size > 1 && m != last) - *m = *last; // Move the last one to this place - else - m->~Member(); // Only one left, just destroy - --data_.o.size; - return m; - } - - //! Remove a member from an object by iterator. - /*! \param pos iterator to the member to remove - \pre IsObject() == true && \ref MemberBegin() <= \c pos < \ref MemberEnd() - \return Iterator following the removed element. - If the iterator \c pos refers to the last element, the \ref MemberEnd() iterator is returned. - \note This function preserves the relative order of the remaining object - members. If you do not need this, use the more efficient \ref RemoveMember(MemberIterator). - \note Linear time complexity. - */ - MemberIterator EraseMember(ConstMemberIterator pos) { - return EraseMember(pos, pos +1); - } - - //! Remove members in the range [first, last) from an object. - /*! \param first iterator to the first member to remove - \param last iterator following the last member to remove - \pre IsObject() == true && \ref MemberBegin() <= \c first <= \c last <= \ref MemberEnd() - \return Iterator following the last removed element. - \note This function preserves the relative order of the remaining object - members. - \note Linear time complexity. - */ - MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) { - RAPIDJSON_ASSERT(IsObject()); - RAPIDJSON_ASSERT(data_.o.size > 0); - RAPIDJSON_ASSERT(GetMembersPointer() != 0); - RAPIDJSON_ASSERT(first >= MemberBegin()); - RAPIDJSON_ASSERT(first <= last); - RAPIDJSON_ASSERT(last <= MemberEnd()); - - MemberIterator pos = MemberBegin() + (first - MemberBegin()); - for (MemberIterator itr = pos; itr != last; ++itr) - itr->~Member(); - std::memmove(&*pos, &*last, static_cast(MemberEnd() - last) * sizeof(Member)); - data_.o.size -= static_cast(last - first); - return pos; - } - - //! Erase a member in object by its name. - /*! \param name Name of member to be removed. - \return Whether the member existed. - \note Linear time complexity. - */ - bool EraseMember(const Ch* name) { - GenericValue n(StringRef(name)); - return EraseMember(n); - } + template + bool RemoveMember(const GenericValue& name) + { MemberIterator m = FindMember(name); + if (m != MemberEnd()) + { RemoveMember(m); + return true; + } + else + return false; + } + + //! Remove a member in object by iterator. + /*! \param m member iterator (obtained by FindMember() or MemberBegin()). + \return the new iterator after removal. + \note This function may reorder the object members. Use \ref + EraseMember(ConstMemberIterator) if you need to preserve the + relative order of the remaining members. + \note Constant time complexity. + */ + MemberIterator RemoveMember(MemberIterator m) + { RAPIDJSON_ASSERT(IsObject()); + RAPIDJSON_ASSERT(data_.o.size > 0); + RAPIDJSON_ASSERT(GetMembersPointer() != 0); + RAPIDJSON_ASSERT(m >= MemberBegin() && m < MemberEnd()); + + MemberIterator last(GetMembersPointer() + (data_.o.size - 1)); + if (data_.o.size > 1 && m != last) + *m = *last; // Move the last one to this place + else + m->~Member(); // Only one left, just destroy + --data_.o.size; + return m; + } + + //! Remove a member from an object by iterator. + /*! \param pos iterator to the member to remove + \pre IsObject() == true && \ref MemberBegin() <= \c pos < \ref MemberEnd() + \return Iterator following the removed element. + If the iterator \c pos refers to the last element, the \ref MemberEnd() iterator is returned. + \note This function preserves the relative order of the remaining object + members. If you do not need this, use the more efficient \ref RemoveMember(MemberIterator). + \note Linear time complexity. + */ + MemberIterator EraseMember(ConstMemberIterator pos) + { return EraseMember(pos, pos +1); + } + + //! Remove members in the range [first, last) from an object. + /*! \param first iterator to the first member to remove + \param last iterator following the last member to remove + \pre IsObject() == true && \ref MemberBegin() <= \c first <= \c last <= \ref MemberEnd() + \return Iterator following the last removed element. + \note This function preserves the relative order of the remaining object + members. + \note Linear time complexity. + */ + MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) + { RAPIDJSON_ASSERT(IsObject()); + RAPIDJSON_ASSERT(data_.o.size > 0); + RAPIDJSON_ASSERT(GetMembersPointer() != 0); + RAPIDJSON_ASSERT(first >= MemberBegin()); + RAPIDJSON_ASSERT(first <= last); + RAPIDJSON_ASSERT(last <= MemberEnd()); + + MemberIterator pos = MemberBegin() + (first - MemberBegin()); + for (MemberIterator itr = pos; itr != last; ++itr) + itr->~Member(); + std::memmove(&*pos, &*last, static_cast(MemberEnd() - last) * sizeof(Member)); + data_.o.size -= static_cast(last - first); + return pos; + } + + //! Erase a member in object by its name. + /*! \param name Name of member to be removed. + \return Whether the member existed. + \note Linear time complexity. + */ + bool EraseMember(const Ch* name) + { GenericValue n(StringRef(name)); + return EraseMember(n); + } #if RAPIDJSON_HAS_STDSTRING - bool EraseMember(const std::basic_string& name) { return EraseMember(GenericValue(StringRef(name))); } + bool EraseMember(const std::basic_string& name) { return EraseMember(GenericValue(StringRef(name))); } #endif - template - bool EraseMember(const GenericValue& name) { - MemberIterator m = FindMember(name); - if (m != MemberEnd()) { - EraseMember(m); - return true; - } - else - return false; - } - - Object GetObject() { RAPIDJSON_ASSERT(IsObject()); return Object(*this); } - ConstObject GetObject() const { RAPIDJSON_ASSERT(IsObject()); return ConstObject(*this); } - - //@} - - //!@name Array - //@{ - - //! Set this value as an empty array. - /*! \post IsArray == true */ - GenericValue& SetArray() { this->~GenericValue(); new (this) GenericValue(kArrayType); return *this; } - - //! Get the number of elements in array. - SizeType Size() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.size; } - - //! Get the capacity of array. - SizeType Capacity() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.capacity; } - - //! Check whether the array is empty. - bool Empty() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.size == 0; } - - //! Remove all elements in the array. - /*! This function do not deallocate memory in the array, i.e. the capacity is unchanged. - \note Linear time complexity. - */ - void Clear() { - RAPIDJSON_ASSERT(IsArray()); - GenericValue* e = GetElementsPointer(); - for (GenericValue* v = e; v != e + data_.a.size; ++v) - v->~GenericValue(); - data_.a.size = 0; - } - - //! Get an element from array by index. - /*! \pre IsArray() == true - \param index Zero-based index of element. - \see operator[](T*) - */ - GenericValue& operator[](SizeType index) { - RAPIDJSON_ASSERT(IsArray()); - RAPIDJSON_ASSERT(index < data_.a.size); - return GetElementsPointer()[index]; - } - const GenericValue& operator[](SizeType index) const { return const_cast(*this)[index]; } - - //! Element iterator - /*! \pre IsArray() == true */ - ValueIterator Begin() { RAPIDJSON_ASSERT(IsArray()); return GetElementsPointer(); } - //! \em Past-the-end element iterator - /*! \pre IsArray() == true */ - ValueIterator End() { RAPIDJSON_ASSERT(IsArray()); return GetElementsPointer() + data_.a.size; } - //! Constant element iterator - /*! \pre IsArray() == true */ - ConstValueIterator Begin() const { return const_cast(*this).Begin(); } - //! Constant \em past-the-end element iterator - /*! \pre IsArray() == true */ - ConstValueIterator End() const { return const_cast(*this).End(); } - - //! Request the array to have enough capacity to store elements. - /*! \param newCapacity The capacity that the array at least need to have. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \note Linear time complexity. - */ - GenericValue& Reserve(SizeType newCapacity, Allocator &allocator) { - RAPIDJSON_ASSERT(IsArray()); - if (newCapacity > data_.a.capacity) { - SetElementsPointer(reinterpret_cast(allocator.Realloc(GetElementsPointer(), data_.a.capacity * sizeof(GenericValue), newCapacity * sizeof(GenericValue)))); - data_.a.capacity = newCapacity; - } - return *this; - } - - //! Append a GenericValue at the end of the array. - /*! \param value Value to be appended. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \pre IsArray() == true - \post value.IsNull() == true - \return The value itself for fluent API. - \note The ownership of \c value will be transferred to this array on success. - \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. - \note Amortized constant time complexity. - */ - GenericValue& PushBack(GenericValue& value, Allocator& allocator) { - RAPIDJSON_ASSERT(IsArray()); - if (data_.a.size >= data_.a.capacity) - Reserve(data_.a.capacity == 0 ? kDefaultArrayCapacity : (data_.a.capacity + (data_.a.capacity + 1) / 2), allocator); - GetElementsPointer()[data_.a.size++].RawAssign(value); - return *this; - } + template + bool EraseMember(const GenericValue& name) + { MemberIterator m = FindMember(name); + if (m != MemberEnd()) + { EraseMember(m); + return true; + } + else + return false; + } + + Object GetObject() { RAPIDJSON_ASSERT(IsObject()); return Object(*this); } + ConstObject GetObject() const { RAPIDJSON_ASSERT(IsObject()); return ConstObject(*this); } + + //@} + + //!@name Array + //@{ + + //! Set this value as an empty array. + /*! \post IsArray == true */ + GenericValue& SetArray() { this->~GenericValue(); new (this) GenericValue(kArrayType); return *this; } + + //! Get the number of elements in array. + SizeType Size() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.size; } + + //! Get the capacity of array. + SizeType Capacity() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.capacity; } + + //! Check whether the array is empty. + bool Empty() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.size == 0; } + + //! Remove all elements in the array. + /*! This function do not deallocate memory in the array, i.e. the capacity is unchanged. + \note Linear time complexity. + */ + void Clear() + { RAPIDJSON_ASSERT(IsArray()); + GenericValue* e = GetElementsPointer(); + for (GenericValue* v = e; v != e + data_.a.size; ++v) + v->~GenericValue(); + data_.a.size = 0; + } + + //! Get an element from array by index. + /*! \pre IsArray() == true + \param index Zero-based index of element. + \see operator[](T*) + */ + GenericValue& operator[](SizeType index) + { RAPIDJSON_ASSERT(IsArray()); + RAPIDJSON_ASSERT(index < data_.a.size); + return GetElementsPointer()[index]; + } + const GenericValue& operator[](SizeType index) const { return const_cast(*this)[index]; } + + //! Element iterator + /*! \pre IsArray() == true */ + ValueIterator Begin() { RAPIDJSON_ASSERT(IsArray()); return GetElementsPointer(); } + //! \em Past-the-end element iterator + /*! \pre IsArray() == true */ + ValueIterator End() { RAPIDJSON_ASSERT(IsArray()); return GetElementsPointer() + data_.a.size; } + //! Constant element iterator + /*! \pre IsArray() == true */ + ConstValueIterator Begin() const { return const_cast(*this).Begin(); } + //! Constant \em past-the-end element iterator + /*! \pre IsArray() == true */ + ConstValueIterator End() const { return const_cast(*this).End(); } + + //! Request the array to have enough capacity to store elements. + /*! \param newCapacity The capacity that the array at least need to have. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \note Linear time complexity. + */ + GenericValue& Reserve(SizeType newCapacity, Allocator &allocator) + { RAPIDJSON_ASSERT(IsArray()); + if (newCapacity > data_.a.capacity) + { SetElementsPointer(reinterpret_cast(allocator.Realloc(GetElementsPointer(), data_.a.capacity * sizeof(GenericValue), newCapacity * sizeof(GenericValue)))); + data_.a.capacity = newCapacity; + } + return *this; + } + + //! Append a GenericValue at the end of the array. + /*! \param value Value to be appended. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \pre IsArray() == true + \post value.IsNull() == true + \return The value itself for fluent API. + \note The ownership of \c value will be transferred to this array on success. + \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. + \note Amortized constant time complexity. + */ + GenericValue& PushBack(GenericValue& value, Allocator& allocator) + { RAPIDJSON_ASSERT(IsArray()); + if (data_.a.size >= data_.a.capacity) + Reserve(data_.a.capacity == 0 ? kDefaultArrayCapacity : (data_.a.capacity + (data_.a.capacity + 1) / 2), allocator); + GetElementsPointer()[data_.a.size++].RawAssign(value); + return *this; + } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS - GenericValue& PushBack(GenericValue&& value, Allocator& allocator) { - return PushBack(value, allocator); - } + GenericValue& PushBack(GenericValue&& value, Allocator& allocator) + { return PushBack(value, allocator); + } #endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Append a constant string reference at the end of the array. - /*! \param value Constant string reference to be appended. - \param allocator Allocator for reallocating memory. It must be the same one used previously. Commonly use GenericDocument::GetAllocator(). - \pre IsArray() == true - \return The value itself for fluent API. - \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. - \note Amortized constant time complexity. - \see GenericStringRef - */ - GenericValue& PushBack(StringRefType value, Allocator& allocator) { - return (*this).template PushBack(value, allocator); - } - - //! Append a primitive value at the end of the array. - /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t - \param value Value of primitive type T to be appended. - \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). - \pre IsArray() == true - \return The value itself for fluent API. - \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. - - \note The source type \c T explicitly disallows all pointer types, - especially (\c const) \ref Ch*. This helps avoiding implicitly - referencing character strings with insufficient lifetime, use - \ref PushBack(GenericValue&, Allocator&) or \ref - PushBack(StringRefType, Allocator&). - All other pointer types would implicitly convert to \c bool, - use an explicit cast instead, if needed. - \note Amortized constant time complexity. - */ - template - RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) - PushBack(T value, Allocator& allocator) { - GenericValue v(value); - return PushBack(v, allocator); - } - - //! Remove the last element in the array. - /*! - \note Constant time complexity. - */ - GenericValue& PopBack() { - RAPIDJSON_ASSERT(IsArray()); - RAPIDJSON_ASSERT(!Empty()); - GetElementsPointer()[--data_.a.size].~GenericValue(); - return *this; - } - - //! Remove an element of array by iterator. - /*! - \param pos iterator to the element to remove - \pre IsArray() == true && \ref Begin() <= \c pos < \ref End() - \return Iterator following the removed element. If the iterator pos refers to the last element, the End() iterator is returned. - \note Linear time complexity. - */ - ValueIterator Erase(ConstValueIterator pos) { - return Erase(pos, pos + 1); - } - - //! Remove elements in the range [first, last) of the array. - /*! - \param first iterator to the first element to remove - \param last iterator following the last element to remove - \pre IsArray() == true && \ref Begin() <= \c first <= \c last <= \ref End() - \return Iterator following the last removed element. - \note Linear time complexity. - */ - ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) { - RAPIDJSON_ASSERT(IsArray()); - RAPIDJSON_ASSERT(data_.a.size > 0); - RAPIDJSON_ASSERT(GetElementsPointer() != 0); - RAPIDJSON_ASSERT(first >= Begin()); - RAPIDJSON_ASSERT(first <= last); - RAPIDJSON_ASSERT(last <= End()); - ValueIterator pos = Begin() + (first - Begin()); - for (ValueIterator itr = pos; itr != last; ++itr) - itr->~GenericValue(); - std::memmove(pos, last, static_cast(End() - last) * sizeof(GenericValue)); - data_.a.size -= static_cast(last - first); - return pos; - } - - Array GetArray() { RAPIDJSON_ASSERT(IsArray()); return Array(*this); } - ConstArray GetArray() const { RAPIDJSON_ASSERT(IsArray()); return ConstArray(*this); } - - //@} - - //!@name Number - //@{ - - int GetInt() const { RAPIDJSON_ASSERT(data_.f.flags & kIntFlag); return data_.n.i.i; } - unsigned GetUint() const { RAPIDJSON_ASSERT(data_.f.flags & kUintFlag); return data_.n.u.u; } - int64_t GetInt64() const { RAPIDJSON_ASSERT(data_.f.flags & kInt64Flag); return data_.n.i64; } - uint64_t GetUint64() const { RAPIDJSON_ASSERT(data_.f.flags & kUint64Flag); return data_.n.u64; } - - //! Get the value as double type. - /*! \note If the value is 64-bit integer type, it may lose precision. Use \c IsLosslessDouble() to check whether the converison is lossless. - */ - double GetDouble() const { - RAPIDJSON_ASSERT(IsNumber()); - if ((data_.f.flags & kDoubleFlag) != 0) return data_.n.d; // exact type, no conversion. - if ((data_.f.flags & kIntFlag) != 0) return data_.n.i.i; // int -> double - if ((data_.f.flags & kUintFlag) != 0) return data_.n.u.u; // unsigned -> double - if ((data_.f.flags & kInt64Flag) != 0) return static_cast(data_.n.i64); // int64_t -> double (may lose precision) - RAPIDJSON_ASSERT((data_.f.flags & kUint64Flag) != 0); return static_cast(data_.n.u64); // uint64_t -> double (may lose precision) - } - - //! Get the value as float type. - /*! \note If the value is 64-bit integer type, it may lose precision. Use \c IsLosslessFloat() to check whether the converison is lossless. - */ - float GetFloat() const { - return static_cast(GetDouble()); - } - - GenericValue& SetInt(int i) { this->~GenericValue(); new (this) GenericValue(i); return *this; } - GenericValue& SetUint(unsigned u) { this->~GenericValue(); new (this) GenericValue(u); return *this; } - GenericValue& SetInt64(int64_t i64) { this->~GenericValue(); new (this) GenericValue(i64); return *this; } - GenericValue& SetUint64(uint64_t u64) { this->~GenericValue(); new (this) GenericValue(u64); return *this; } - GenericValue& SetDouble(double d) { this->~GenericValue(); new (this) GenericValue(d); return *this; } - GenericValue& SetFloat(float f) { this->~GenericValue(); new (this) GenericValue(f); return *this; } - - //@} - - //!@name String - //@{ - - const Ch* GetString() const { RAPIDJSON_ASSERT(IsString()); return (data_.f.flags & kInlineStrFlag) ? data_.ss.str : GetStringPointer(); } - - //! Get the length of string. - /*! Since rapidjson permits "\\u0000" in the json string, strlen(v.GetString()) may not equal to v.GetStringLength(). - */ - SizeType GetStringLength() const { RAPIDJSON_ASSERT(IsString()); return ((data_.f.flags & kInlineStrFlag) ? (data_.ss.GetLength()) : data_.s.length); } - - //! Set this value as a string without copying source string. - /*! This version has better performance with supplied length, and also support string containing null character. - \param s source string pointer. - \param length The length of source string, excluding the trailing null terminator. - \return The value itself for fluent API. - \post IsString() == true && GetString() == s && GetStringLength() == length - \see SetString(StringRefType) - */ - GenericValue& SetString(const Ch* s, SizeType length) { return SetString(StringRef(s, length)); } - - //! Set this value as a string without copying source string. - /*! \param s source string reference - \return The value itself for fluent API. - \post IsString() == true && GetString() == s && GetStringLength() == s.length - */ - GenericValue& SetString(StringRefType s) { this->~GenericValue(); SetStringRaw(s); return *this; } - - //! Set this value as a string by copying from source string. - /*! This version has better performance with supplied length, and also support string containing null character. - \param s source string. - \param length The length of source string, excluding the trailing null terminator. - \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length - */ - GenericValue& SetString(const Ch* s, SizeType length, Allocator& allocator) { this->~GenericValue(); SetStringRaw(StringRef(s, length), allocator); return *this; } - - //! Set this value as a string by copying from source string. - /*! \param s source string. - \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length - */ - GenericValue& SetString(const Ch* s, Allocator& allocator) { return SetString(s, internal::StrLen(s), allocator); } + //! Append a constant string reference at the end of the array. + /*! \param value Constant string reference to be appended. + \param allocator Allocator for reallocating memory. It must be the same one used previously. Commonly use GenericDocument::GetAllocator(). + \pre IsArray() == true + \return The value itself for fluent API. + \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. + \note Amortized constant time complexity. + \see GenericStringRef + */ + GenericValue& PushBack(StringRefType value, Allocator& allocator) + { return (*this).template PushBack(value, allocator); + } + + //! Append a primitive value at the end of the array. + /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t + \param value Value of primitive type T to be appended. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \pre IsArray() == true + \return The value itself for fluent API. + \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. + + \note The source type \c T explicitly disallows all pointer types, + especially (\c const) \ref Ch*. This helps avoiding implicitly + referencing character strings with insufficient lifetime, use + \ref PushBack(GenericValue&, Allocator&) or \ref + PushBack(StringRefType, Allocator&). + All other pointer types would implicitly convert to \c bool, + use an explicit cast instead, if needed. + \note Amortized constant time complexity. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) + PushBack(T value, Allocator& allocator) + { GenericValue v(value); + return PushBack(v, allocator); + } + + //! Remove the last element in the array. + /*! + \note Constant time complexity. + */ + GenericValue& PopBack() + { RAPIDJSON_ASSERT(IsArray()); + RAPIDJSON_ASSERT(!Empty()); + GetElementsPointer()[--data_.a.size].~GenericValue(); + return *this; + } + + //! Remove an element of array by iterator. + /*! + \param pos iterator to the element to remove + \pre IsArray() == true && \ref Begin() <= \c pos < \ref End() + \return Iterator following the removed element. If the iterator pos refers to the last element, the End() iterator is returned. + \note Linear time complexity. + */ + ValueIterator Erase(ConstValueIterator pos) + { return Erase(pos, pos + 1); + } + + //! Remove elements in the range [first, last) of the array. + /*! + \param first iterator to the first element to remove + \param last iterator following the last element to remove + \pre IsArray() == true && \ref Begin() <= \c first <= \c last <= \ref End() + \return Iterator following the last removed element. + \note Linear time complexity. + */ + ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) + { RAPIDJSON_ASSERT(IsArray()); + RAPIDJSON_ASSERT(data_.a.size > 0); + RAPIDJSON_ASSERT(GetElementsPointer() != 0); + RAPIDJSON_ASSERT(first >= Begin()); + RAPIDJSON_ASSERT(first <= last); + RAPIDJSON_ASSERT(last <= End()); + ValueIterator pos = Begin() + (first - Begin()); + for (ValueIterator itr = pos; itr != last; ++itr) + itr->~GenericValue(); + std::memmove(pos, last, static_cast(End() - last) * sizeof(GenericValue)); + data_.a.size -= static_cast(last - first); + return pos; + } + + Array GetArray() { RAPIDJSON_ASSERT(IsArray()); return Array(*this); } + ConstArray GetArray() const { RAPIDJSON_ASSERT(IsArray()); return ConstArray(*this); } + + //@} + + //!@name Number + //@{ + + int GetInt() const { RAPIDJSON_ASSERT(data_.f.flags & kIntFlag); return data_.n.i.i; } + unsigned GetUint() const { RAPIDJSON_ASSERT(data_.f.flags & kUintFlag); return data_.n.u.u; } + int64_t GetInt64() const { RAPIDJSON_ASSERT(data_.f.flags & kInt64Flag); return data_.n.i64; } + uint64_t GetUint64() const { RAPIDJSON_ASSERT(data_.f.flags & kUint64Flag); return data_.n.u64; } + + //! Get the value as double type. + /*! \note If the value is 64-bit integer type, it may lose precision. Use \c IsLosslessDouble() to check whether the converison is lossless. + */ + double GetDouble() const + { RAPIDJSON_ASSERT(IsNumber()); + if ((data_.f.flags & kDoubleFlag) != 0) return data_.n.d; // exact type, no conversion. + if ((data_.f.flags & kIntFlag) != 0) return data_.n.i.i; // int -> double + if ((data_.f.flags & kUintFlag) != 0) return data_.n.u.u; // unsigned -> double + if ((data_.f.flags & kInt64Flag) != 0) return static_cast(data_.n.i64); // int64_t -> double (may lose precision) + RAPIDJSON_ASSERT((data_.f.flags & kUint64Flag) != 0); return static_cast(data_.n.u64); // uint64_t -> double (may lose precision) + } + + //! Get the value as float type. + /*! \note If the value is 64-bit integer type, it may lose precision. Use \c IsLosslessFloat() to check whether the converison is lossless. + */ + float GetFloat() const + { return static_cast(GetDouble()); + } + + GenericValue& SetInt(int i) { this->~GenericValue(); new (this) GenericValue(i); return *this; } + GenericValue& SetUint(unsigned u) { this->~GenericValue(); new (this) GenericValue(u); return *this; } + GenericValue& SetInt64(int64_t i64) { this->~GenericValue(); new (this) GenericValue(i64); return *this; } + GenericValue& SetUint64(uint64_t u64) { this->~GenericValue(); new (this) GenericValue(u64); return *this; } + GenericValue& SetDouble(double d) { this->~GenericValue(); new (this) GenericValue(d); return *this; } + GenericValue& SetFloat(float f) { this->~GenericValue(); new (this) GenericValue(f); return *this; } + + //@} + + //!@name String + //@{ + + const Ch* GetString() const { RAPIDJSON_ASSERT(IsString()); return (data_.f.flags & kInlineStrFlag) ? data_.ss.str : GetStringPointer(); } + + //! Get the length of string. + /*! Since rapidjson permits "\\u0000" in the json string, strlen(v.GetString()) may not equal to v.GetStringLength(). + */ + SizeType GetStringLength() const { RAPIDJSON_ASSERT(IsString()); return ((data_.f.flags & kInlineStrFlag) ? (data_.ss.GetLength()) : data_.s.length); } + + //! Set this value as a string without copying source string. + /*! This version has better performance with supplied length, and also support string containing null character. + \param s source string pointer. + \param length The length of source string, excluding the trailing null terminator. + \return The value itself for fluent API. + \post IsString() == true && GetString() == s && GetStringLength() == length + \see SetString(StringRefType) + */ + GenericValue& SetString(const Ch* s, SizeType length) { return SetString(StringRef(s, length)); } + + //! Set this value as a string without copying source string. + /*! \param s source string reference + \return The value itself for fluent API. + \post IsString() == true && GetString() == s && GetStringLength() == s.length + */ + GenericValue& SetString(StringRefType s) { this->~GenericValue(); SetStringRaw(s); return *this; } + + //! Set this value as a string by copying from source string. + /*! This version has better performance with supplied length, and also support string containing null character. + \param s source string. + \param length The length of source string, excluding the trailing null terminator. + \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length + */ + GenericValue& SetString(const Ch* s, SizeType length, Allocator& allocator) { this->~GenericValue(); SetStringRaw(StringRef(s, length), allocator); return *this; } + + //! Set this value as a string by copying from source string. + /*! \param s source string. + \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length + */ + GenericValue& SetString(const Ch* s, Allocator& allocator) { return SetString(s, internal::StrLen(s), allocator); } #if RAPIDJSON_HAS_STDSTRING - //! Set this value as a string by copying from source string. - /*! \param s source string. - \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). - \return The value itself for fluent API. - \post IsString() == true && GetString() != s.data() && strcmp(GetString(),s.data() == 0 && GetStringLength() == s.size() - \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. - */ - GenericValue& SetString(const std::basic_string& s, Allocator& allocator) { return SetString(s.data(), SizeType(s.size()), allocator); } + //! Set this value as a string by copying from source string. + /*! \param s source string. + \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \post IsString() == true && GetString() != s.data() && strcmp(GetString(),s.data() == 0 && GetStringLength() == s.size() + \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. + */ + GenericValue& SetString(const std::basic_string& s, Allocator& allocator) { return SetString(s.data(), SizeType(s.size()), allocator); } #endif - //@} - - //!@name Array - //@{ - - //! Templated version for checking whether this value is type T. - /*! - \tparam T Either \c bool, \c int, \c unsigned, \c int64_t, \c uint64_t, \c double, \c float, \c const \c char*, \c std::basic_string - */ - template - bool Is() const { return internal::TypeHelper::Is(*this); } - - template - T Get() const { return internal::TypeHelper::Get(*this); } - - template - T Get() { return internal::TypeHelper::Get(*this); } - - template - ValueType& Set(const T& data) { return internal::TypeHelper::Set(*this, data); } - - template - ValueType& Set(const T& data, AllocatorType& allocator) { return internal::TypeHelper::Set(*this, data, allocator); } - - //@} - - //! Generate events of this value to a Handler. - /*! This function adopts the GoF visitor pattern. - Typical usage is to output this JSON value as JSON text via Writer, which is a Handler. - It can also be used to deep clone this value via GenericDocument, which is also a Handler. - \tparam Handler type of handler. - \param handler An object implementing concept Handler. - */ - template - bool Accept(Handler& handler) const { - switch(GetType()) { - case kNullType: return handler.Null(); - case kFalseType: return handler.Bool(false); - case kTrueType: return handler.Bool(true); - - case kObjectType: - if (RAPIDJSON_UNLIKELY(!handler.StartObject())) - return false; - for (ConstMemberIterator m = MemberBegin(); m != MemberEnd(); ++m) { - RAPIDJSON_ASSERT(m->name.IsString()); // User may change the type of name by MemberIterator. - if (RAPIDJSON_UNLIKELY(!handler.Key(m->name.GetString(), m->name.GetStringLength(), (m->name.data_.f.flags & kCopyFlag) != 0))) - return false; - if (RAPIDJSON_UNLIKELY(!m->value.Accept(handler))) - return false; - } - return handler.EndObject(data_.o.size); - - case kArrayType: - if (RAPIDJSON_UNLIKELY(!handler.StartArray())) - return false; - for (const GenericValue* v = Begin(); v != End(); ++v) - if (RAPIDJSON_UNLIKELY(!v->Accept(handler))) - return false; - return handler.EndArray(data_.a.size); - - case kStringType: - return handler.String(GetString(), GetStringLength(), (data_.f.flags & kCopyFlag) != 0); - - default: - RAPIDJSON_ASSERT(GetType() == kNumberType); - if (IsDouble()) return handler.Double(data_.n.d); - else if (IsInt()) return handler.Int(data_.n.i.i); - else if (IsUint()) return handler.Uint(data_.n.u.u); - else if (IsInt64()) return handler.Int64(data_.n.i64); - else return handler.Uint64(data_.n.u64); + //@} + + //!@name Array + //@{ + + //! Templated version for checking whether this value is type T. + /*! + \tparam T Either \c bool, \c int, \c unsigned, \c int64_t, \c uint64_t, \c double, \c float, \c const \c char*, \c std::basic_string + */ + template + bool Is() const { return internal::TypeHelper::Is(*this); } + + template + T Get() const { return internal::TypeHelper::Get(*this); } + + template + T Get() { return internal::TypeHelper::Get(*this); } + + template + ValueType& Set(const T& data) { return internal::TypeHelper::Set(*this, data); } + + template + ValueType& Set(const T& data, AllocatorType& allocator) { return internal::TypeHelper::Set(*this, data, allocator); } + + //@} + + //! Generate events of this value to a Handler. + /*! This function adopts the GoF visitor pattern. + Typical usage is to output this JSON value as JSON text via Writer, which is a Handler. + It can also be used to deep clone this value via GenericDocument, which is also a Handler. + \tparam Handler type of handler. + \param handler An object implementing concept Handler. + */ + template + bool Accept(Handler& handler) const + { switch(GetType()) + { case kNullType: return handler.Null(); + case kFalseType: return handler.Bool(false); + case kTrueType: return handler.Bool(true); + + case kObjectType: + if (RAPIDJSON_UNLIKELY(!handler.StartObject())) + return false; + for (ConstMemberIterator m = MemberBegin(); m != MemberEnd(); ++m) + { RAPIDJSON_ASSERT(m->name.IsString()); // User may change the type of name by MemberIterator. + if (RAPIDJSON_UNLIKELY(!handler.Key(m->name.GetString(), m->name.GetStringLength(), (m->name.data_.f.flags & kCopyFlag) != 0))) + return false; + if (RAPIDJSON_UNLIKELY(!m->value.Accept(handler))) + return false; } - } + return handler.EndObject(data_.o.size); -private: - template friend class GenericValue; - template friend class GenericDocument; - - enum { - kBoolFlag = 0x0008, - kNumberFlag = 0x0010, - kIntFlag = 0x0020, - kUintFlag = 0x0040, - kInt64Flag = 0x0080, - kUint64Flag = 0x0100, - kDoubleFlag = 0x0200, - kStringFlag = 0x0400, - kCopyFlag = 0x0800, - kInlineStrFlag = 0x1000, - - // Initial flags of different types. - kNullFlag = kNullType, - kTrueFlag = kTrueType | kBoolFlag, - kFalseFlag = kFalseType | kBoolFlag, - kNumberIntFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag, - kNumberUintFlag = kNumberType | kNumberFlag | kUintFlag | kUint64Flag | kInt64Flag, - kNumberInt64Flag = kNumberType | kNumberFlag | kInt64Flag, - kNumberUint64Flag = kNumberType | kNumberFlag | kUint64Flag, - kNumberDoubleFlag = kNumberType | kNumberFlag | kDoubleFlag, - kNumberAnyFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag | kUintFlag | kUint64Flag | kDoubleFlag, - kConstStringFlag = kStringType | kStringFlag, - kCopyStringFlag = kStringType | kStringFlag | kCopyFlag, - kShortStringFlag = kStringType | kStringFlag | kCopyFlag | kInlineStrFlag, - kObjectFlag = kObjectType, - kArrayFlag = kArrayType, - - kTypeMask = 0x07 - }; + case kArrayType: + if (RAPIDJSON_UNLIKELY(!handler.StartArray())) + return false; + for (const GenericValue* v = Begin(); v != End(); ++v) + if (RAPIDJSON_UNLIKELY(!v->Accept(handler))) + return false; + return handler.EndArray(data_.a.size); - static const SizeType kDefaultArrayCapacity = 16; - static const SizeType kDefaultObjectCapacity = 16; + case kStringType: + return handler.String(GetString(), GetStringLength(), (data_.f.flags & kCopyFlag) != 0); + + default: + RAPIDJSON_ASSERT(GetType() == kNumberType); + if (IsDouble()) return handler.Double(data_.n.d); + else if (IsInt()) return handler.Int(data_.n.i.i); + else if (IsUint()) return handler.Uint(data_.n.u.u); + else if (IsInt64()) return handler.Int64(data_.n.i64); + else return handler.Uint64(data_.n.u64); + } + } - struct Flag { +private: + template friend class GenericValue; + template friend class GenericDocument; + + enum + { kBoolFlag = 0x0008, + kNumberFlag = 0x0010, + kIntFlag = 0x0020, + kUintFlag = 0x0040, + kInt64Flag = 0x0080, + kUint64Flag = 0x0100, + kDoubleFlag = 0x0200, + kStringFlag = 0x0400, + kCopyFlag = 0x0800, + kInlineStrFlag = 0x1000, + + // Initial flags of different types. + kNullFlag = kNullType, + kTrueFlag = kTrueType | kBoolFlag, + kFalseFlag = kFalseType | kBoolFlag, + kNumberIntFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag, + kNumberUintFlag = kNumberType | kNumberFlag | kUintFlag | kUint64Flag | kInt64Flag, + kNumberInt64Flag = kNumberType | kNumberFlag | kInt64Flag, + kNumberUint64Flag = kNumberType | kNumberFlag | kUint64Flag, + kNumberDoubleFlag = kNumberType | kNumberFlag | kDoubleFlag, + kNumberAnyFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag | kUintFlag | kUint64Flag | kDoubleFlag, + kConstStringFlag = kStringType | kStringFlag, + kCopyStringFlag = kStringType | kStringFlag | kCopyFlag, + kShortStringFlag = kStringType | kStringFlag | kCopyFlag | kInlineStrFlag, + kObjectFlag = kObjectType, + kArrayFlag = kArrayType, + + kTypeMask = 0x07 + }; + + static const SizeType kDefaultArrayCapacity = 16; + static const SizeType kDefaultObjectCapacity = 16; + + struct Flag + { #if RAPIDJSON_48BITPOINTER_OPTIMIZATION - char payload[sizeof(SizeType) * 2 + 6]; // 2 x SizeType + lower 48-bit pointer + char payload[sizeof(SizeType) * 2 + 6]; // 2 x SizeType + lower 48-bit pointer #elif RAPIDJSON_64BIT - char payload[sizeof(SizeType) * 2 + sizeof(void*) + 6]; // 6 padding bytes + char payload[sizeof(SizeType) * 2 + sizeof(void*) + 6]; // 6 padding bytes #else - char payload[sizeof(SizeType) * 2 + sizeof(void*) + 2]; // 2 padding bytes + char payload[sizeof(SizeType) * 2 + sizeof(void*) + 2]; // 2 padding bytes #endif - uint16_t flags; - }; - - struct String { - SizeType length; - SizeType hashcode; //!< reserved - const Ch* str; - }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode - - // implementation detail: ShortString can represent zero-terminated strings up to MaxSize chars - // (excluding the terminating zero) and store a value to determine the length of the contained - // string in the last character str[LenPos] by storing "MaxSize - length" there. If the string - // to store has the maximal length of MaxSize then str[LenPos] will be 0 and therefore act as - // the string terminator as well. For getting the string length back from that value just use - // "MaxSize - str[LenPos]". - // This allows to store 13-chars strings in 32-bit mode, 21-chars strings in 64-bit mode, - // 13-chars strings for RAPIDJSON_48BITPOINTER_OPTIMIZATION=1 inline (for `UTF8`-encoded strings). - struct ShortString { - enum { MaxChars = sizeof(static_cast(0)->payload) / sizeof(Ch), MaxSize = MaxChars - 1, LenPos = MaxSize }; - Ch str[MaxChars]; - - inline static bool Usable(SizeType len) { return (MaxSize >= len); } - inline void SetLength(SizeType len) { str[LenPos] = static_cast(MaxSize - len); } - inline SizeType GetLength() const { return static_cast(MaxSize - str[LenPos]); } - }; // at most as many bytes as "String" above => 12 bytes in 32-bit mode, 16 bytes in 64-bit mode - - // By using proper binary layout, retrieval of different integer types do not need conversions. - union Number { + uint16_t flags; + }; + + struct String + { SizeType length; + SizeType hashcode; //!< reserved + const Ch* str; + }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + + // implementation detail: ShortString can represent zero-terminated strings up to MaxSize chars + // (excluding the terminating zero) and store a value to determine the length of the contained + // string in the last character str[LenPos] by storing "MaxSize - length" there. If the string + // to store has the maximal length of MaxSize then str[LenPos] will be 0 and therefore act as + // the string terminator as well. For getting the string length back from that value just use + // "MaxSize - str[LenPos]". + // This allows to store 13-chars strings in 32-bit mode, 21-chars strings in 64-bit mode, + // 13-chars strings for RAPIDJSON_48BITPOINTER_OPTIMIZATION=1 inline (for `UTF8`-encoded strings). + struct ShortString + { enum { MaxChars = sizeof(static_cast(0)->payload) / sizeof(Ch), MaxSize = MaxChars - 1, LenPos = MaxSize }; + Ch str[MaxChars]; + + inline static bool Usable(SizeType len) { return (MaxSize >= len); } + inline void SetLength(SizeType len) { str[LenPos] = static_cast(MaxSize - len); } + inline SizeType GetLength() const { return static_cast(MaxSize - str[LenPos]); } + }; // at most as many bytes as "String" above => 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + + // By using proper binary layout, retrieval of different integer types do not need conversions. + union Number + { #if RAPIDJSON_ENDIAN == RAPIDJSON_LITTLEENDIAN - struct I { - int i; - char padding[4]; - }i; - struct U { - unsigned u; - char padding2[4]; - }u; + struct I + { int i; + char padding[4]; + } i; + struct U + { unsigned u; + char padding2[4]; + } u; #else - struct I { - char padding[4]; - int i; - }i; - struct U { - char padding2[4]; - unsigned u; - }u; + struct I + { char padding[4]; + int i; + } i; + struct U + { char padding2[4]; + unsigned u; + } u; #endif - int64_t i64; - uint64_t u64; - double d; - }; // 8 bytes - - struct ObjectData { - SizeType size; - SizeType capacity; - Member* members; - }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode - - struct ArrayData { - SizeType size; - SizeType capacity; - GenericValue* elements; - }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode - - union Data { - String s; - ShortString ss; - Number n; - ObjectData o; - ArrayData a; - Flag f; - }; // 16 bytes in 32-bit mode, 24 bytes in 64-bit mode, 16 bytes in 64-bit with RAPIDJSON_48BITPOINTER_OPTIMIZATION - - RAPIDJSON_FORCEINLINE const Ch* GetStringPointer() const { return RAPIDJSON_GETPOINTER(Ch, data_.s.str); } - RAPIDJSON_FORCEINLINE const Ch* SetStringPointer(const Ch* str) { return RAPIDJSON_SETPOINTER(Ch, data_.s.str, str); } - RAPIDJSON_FORCEINLINE GenericValue* GetElementsPointer() const { return RAPIDJSON_GETPOINTER(GenericValue, data_.a.elements); } - RAPIDJSON_FORCEINLINE GenericValue* SetElementsPointer(GenericValue* elements) { return RAPIDJSON_SETPOINTER(GenericValue, data_.a.elements, elements); } - RAPIDJSON_FORCEINLINE Member* GetMembersPointer() const { return RAPIDJSON_GETPOINTER(Member, data_.o.members); } - RAPIDJSON_FORCEINLINE Member* SetMembersPointer(Member* members) { return RAPIDJSON_SETPOINTER(Member, data_.o.members, members); } - - // Initialize this value as array with initial data, without calling destructor. - void SetArrayRaw(GenericValue* values, SizeType count, Allocator& allocator) { - data_.f.flags = kArrayFlag; - if (count) { - GenericValue* e = static_cast(allocator.Malloc(count * sizeof(GenericValue))); - SetElementsPointer(e); - std::memcpy(e, values, count * sizeof(GenericValue)); - } - else - SetElementsPointer(0); - data_.a.size = data_.a.capacity = count; - } - - //! Initialize this value as object with initial data, without calling destructor. - void SetObjectRaw(Member* members, SizeType count, Allocator& allocator) { - data_.f.flags = kObjectFlag; - if (count) { - Member* m = static_cast(allocator.Malloc(count * sizeof(Member))); - SetMembersPointer(m); - std::memcpy(m, members, count * sizeof(Member)); - } - else - SetMembersPointer(0); - data_.o.size = data_.o.capacity = count; - } - - //! Initialize this value as constant string, without calling destructor. - void SetStringRaw(StringRefType s) RAPIDJSON_NOEXCEPT { - data_.f.flags = kConstStringFlag; - SetStringPointer(s); - data_.s.length = s.length; - } - - //! Initialize this value as copy string with initial data, without calling destructor. - void SetStringRaw(StringRefType s, Allocator& allocator) { - Ch* str = 0; - if (ShortString::Usable(s.length)) { - data_.f.flags = kShortStringFlag; - data_.ss.SetLength(s.length); - str = data_.ss.str; - } else { - data_.f.flags = kCopyStringFlag; - data_.s.length = s.length; - str = static_cast(allocator.Malloc((s.length + 1) * sizeof(Ch))); - SetStringPointer(str); - } - std::memcpy(str, s, s.length * sizeof(Ch)); - str[s.length] = '\0'; - } - - //! Assignment without calling destructor - void RawAssign(GenericValue& rhs) RAPIDJSON_NOEXCEPT { - data_ = rhs.data_; - // data_.f.flags = rhs.data_.f.flags; - rhs.data_.f.flags = kNullFlag; - } - - template - bool StringEqual(const GenericValue& rhs) const { - RAPIDJSON_ASSERT(IsString()); - RAPIDJSON_ASSERT(rhs.IsString()); - - const SizeType len1 = GetStringLength(); - const SizeType len2 = rhs.GetStringLength(); - if(len1 != len2) { return false; } - - const Ch* const str1 = GetString(); - const Ch* const str2 = rhs.GetString(); - if(str1 == str2) { return true; } // fast path for constant string - - return (std::memcmp(str1, str2, sizeof(Ch) * len1) == 0); - } - - Data data_; + int64_t i64; + uint64_t u64; + double d; + }; // 8 bytes + + struct ObjectData + { SizeType size; + SizeType capacity; + Member* members; + }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + + struct ArrayData + { SizeType size; + SizeType capacity; + GenericValue* elements; + }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode + + union Data + { String s; + ShortString ss; + Number n; + ObjectData o; + ArrayData a; + Flag f; + }; // 16 bytes in 32-bit mode, 24 bytes in 64-bit mode, 16 bytes in 64-bit with RAPIDJSON_48BITPOINTER_OPTIMIZATION + + RAPIDJSON_FORCEINLINE const Ch* GetStringPointer() const { return RAPIDJSON_GETPOINTER(Ch, data_.s.str); } + RAPIDJSON_FORCEINLINE const Ch* SetStringPointer(const Ch* str) { return RAPIDJSON_SETPOINTER(Ch, data_.s.str, str); } + RAPIDJSON_FORCEINLINE GenericValue* GetElementsPointer() const { return RAPIDJSON_GETPOINTER(GenericValue, data_.a.elements); } + RAPIDJSON_FORCEINLINE GenericValue* SetElementsPointer(GenericValue* elements) { return RAPIDJSON_SETPOINTER(GenericValue, data_.a.elements, elements); } + RAPIDJSON_FORCEINLINE Member* GetMembersPointer() const { return RAPIDJSON_GETPOINTER(Member, data_.o.members); } + RAPIDJSON_FORCEINLINE Member* SetMembersPointer(Member* members) { return RAPIDJSON_SETPOINTER(Member, data_.o.members, members); } + + // Initialize this value as array with initial data, without calling destructor. + void SetArrayRaw(GenericValue* values, SizeType count, Allocator& allocator) + { data_.f.flags = kArrayFlag; + if (count) + { GenericValue* e = static_cast(allocator.Malloc(count * sizeof(GenericValue))); + SetElementsPointer(e); + std::memcpy(e, values, count * sizeof(GenericValue)); + } + else + SetElementsPointer(0); + data_.a.size = data_.a.capacity = count; + } + + //! Initialize this value as object with initial data, without calling destructor. + void SetObjectRaw(Member* members, SizeType count, Allocator& allocator) + { data_.f.flags = kObjectFlag; + if (count) + { Member* m = static_cast(allocator.Malloc(count * sizeof(Member))); + SetMembersPointer(m); + std::memcpy(m, members, count * sizeof(Member)); + } + else + SetMembersPointer(0); + data_.o.size = data_.o.capacity = count; + } + + //! Initialize this value as constant string, without calling destructor. + void SetStringRaw(StringRefType s) RAPIDJSON_NOEXCEPT + { data_.f.flags = kConstStringFlag; + SetStringPointer(s); + data_.s.length = s.length; + } + + //! Initialize this value as copy string with initial data, without calling destructor. + void SetStringRaw(StringRefType s, Allocator& allocator) + { Ch* str = 0; + if (ShortString::Usable(s.length)) + { data_.f.flags = kShortStringFlag; + data_.ss.SetLength(s.length); + str = data_.ss.str; + } + else + { data_.f.flags = kCopyStringFlag; + data_.s.length = s.length; + str = static_cast(allocator.Malloc((s.length + 1) * sizeof(Ch))); + SetStringPointer(str); + } + std::memcpy(str, s, s.length * sizeof(Ch)); + str[s.length] = '\0'; + } + + //! Assignment without calling destructor + void RawAssign(GenericValue& rhs) RAPIDJSON_NOEXCEPT + { data_ = rhs.data_; + // data_.f.flags = rhs.data_.f.flags; + rhs.data_.f.flags = kNullFlag; + } + + template + bool StringEqual(const GenericValue& rhs) const + { RAPIDJSON_ASSERT(IsString()); + RAPIDJSON_ASSERT(rhs.IsString()); + + const SizeType len1 = GetStringLength(); + const SizeType len2 = rhs.GetStringLength(); + if(len1 != len2) { return false; } + + const Ch* const str1 = GetString(); + const Ch* const str2 = rhs.GetString(); + if(str1 == str2) { return true; } // fast path for constant string + + return (std::memcmp(str1, str2, sizeof(Ch) * len1) == 0); + } + + Data data_; }; //! GenericValue with UTF8 encoding typedef GenericValue > Value; /////////////////////////////////////////////////////////////////////////////// -// GenericDocument +// GenericDocument //! A document for parsing JSON text as DOM. /*! @@ -2021,381 +2027,378 @@ typedef GenericValue > Value; \warning Although GenericDocument inherits from GenericValue, the API does \b not provide any virtual functions, especially no virtual destructor. To avoid memory leaks, do not \c delete a GenericDocument object via a pointer to a GenericValue. */ template , typename StackAllocator = CrtAllocator> -class GenericDocument : public GenericValue { +class GenericDocument : public GenericValue +{ public: - typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. - typedef GenericValue ValueType; //!< Value type of the document. - typedef Allocator AllocatorType; //!< Allocator type from template parameter. - - //! Constructor - /*! Creates an empty document of specified type. - \param type Mandatory type of object to create. - \param allocator Optional allocator for allocating memory. - \param stackCapacity Optional initial capacity of stack in bytes. - \param stackAllocator Optional allocator for allocating memory for stack. - */ - explicit GenericDocument(Type type, Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity, StackAllocator* stackAllocator = 0) : - GenericValue(type), allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() - { - if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); - } - - //! Constructor - /*! Creates an empty document which type is Null. - \param allocator Optional allocator for allocating memory. - \param stackCapacity Optional initial capacity of stack in bytes. - \param stackAllocator Optional allocator for allocating memory for stack. - */ - GenericDocument(Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity, StackAllocator* stackAllocator = 0) : - allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() - { - if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); - } + typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. + typedef GenericValue ValueType; //!< Value type of the document. + typedef Allocator AllocatorType; //!< Allocator type from template parameter. + + //! Constructor + /*! Creates an empty document of specified type. + \param type Mandatory type of object to create. + \param allocator Optional allocator for allocating memory. + \param stackCapacity Optional initial capacity of stack in bytes. + \param stackAllocator Optional allocator for allocating memory for stack. + */ + explicit GenericDocument(Type type, Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity, StackAllocator* stackAllocator = 0) : + GenericValue(type), allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() + { if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + } + + //! Constructor + /*! Creates an empty document which type is Null. + \param allocator Optional allocator for allocating memory. + \param stackCapacity Optional initial capacity of stack in bytes. + \param stackAllocator Optional allocator for allocating memory for stack. + */ + GenericDocument(Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity, StackAllocator* stackAllocator = 0) : + allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() + { if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Move constructor in C++11 - GenericDocument(GenericDocument&& rhs) RAPIDJSON_NOEXCEPT - : ValueType(std::forward(rhs)), // explicit cast to avoid prohibited move from Document - allocator_(rhs.allocator_), - ownAllocator_(rhs.ownAllocator_), - stack_(std::move(rhs.stack_)), - parseResult_(rhs.parseResult_) - { - rhs.allocator_ = 0; - rhs.ownAllocator_ = 0; - rhs.parseResult_ = ParseResult(); - } + //! Move constructor in C++11 + GenericDocument(GenericDocument&& rhs) RAPIDJSON_NOEXCEPT +: ValueType(std::forward(rhs)), // explicit cast to avoid prohibited move from Document + allocator_(rhs.allocator_), + ownAllocator_(rhs.ownAllocator_), + stack_(std::move(rhs.stack_)), + parseResult_(rhs.parseResult_) + { rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.parseResult_ = ParseResult(); + } #endif - ~GenericDocument() { - Destroy(); - } + ~GenericDocument() + { Destroy(); + } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Move assignment in C++11 - GenericDocument& operator=(GenericDocument&& rhs) RAPIDJSON_NOEXCEPT - { - // The cast to ValueType is necessary here, because otherwise it would - // attempt to call GenericValue's templated assignment operator. - ValueType::operator=(std::forward(rhs)); - - // Calling the destructor here would prematurely call stack_'s destructor - Destroy(); - - allocator_ = rhs.allocator_; - ownAllocator_ = rhs.ownAllocator_; - stack_ = std::move(rhs.stack_); - parseResult_ = rhs.parseResult_; - - rhs.allocator_ = 0; - rhs.ownAllocator_ = 0; - rhs.parseResult_ = ParseResult(); - - return *this; - } + //! Move assignment in C++11 + GenericDocument& operator=(GenericDocument&& rhs) RAPIDJSON_NOEXCEPT + { // The cast to ValueType is necessary here, because otherwise it would + // attempt to call GenericValue's templated assignment operator. + ValueType::operator=(std::forward(rhs)); + + // Calling the destructor here would prematurely call stack_'s destructor + Destroy(); + + allocator_ = rhs.allocator_; + ownAllocator_ = rhs.ownAllocator_; + stack_ = std::move(rhs.stack_); + parseResult_ = rhs.parseResult_; + + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.parseResult_ = ParseResult(); + + return *this; + } #endif - //! Exchange the contents of this document with those of another. - /*! - \param rhs Another document. - \note Constant complexity. - \see GenericValue::Swap - */ - GenericDocument& Swap(GenericDocument& rhs) RAPIDJSON_NOEXCEPT { - ValueType::Swap(rhs); - stack_.Swap(rhs.stack_); - internal::Swap(allocator_, rhs.allocator_); - internal::Swap(ownAllocator_, rhs.ownAllocator_); - internal::Swap(parseResult_, rhs.parseResult_); - return *this; - } - - //! free-standing swap function helper - /*! - Helper function to enable support for common swap implementation pattern based on \c std::swap: - \code - void swap(MyClass& a, MyClass& b) { - using std::swap; - swap(a.doc, b.doc); - // ... - } - \endcode - \see Swap() - */ - friend inline void swap(GenericDocument& a, GenericDocument& b) RAPIDJSON_NOEXCEPT { a.Swap(b); } - - //! Populate this document by a generator which produces SAX events. - /*! \tparam Generator A functor with bool f(Handler) prototype. - \param g Generator functor which sends SAX events to the parameter. - \return The document itself for fluent API. - */ - template - GenericDocument& Populate(Generator& g) { - ClearStackOnExit scope(*this); - if (g(*this)) { - RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object - ValueType::operator=(*stack_.template Pop(1));// Move value from stack to document - } - return *this; - } - - //!@name Parse from stream - //!@{ - - //! Parse JSON text from an input stream (with Encoding conversion) - /*! \tparam parseFlags Combination of \ref ParseFlag. - \tparam SourceEncoding Encoding of input stream - \tparam InputStream Type of input stream, implementing Stream concept - \param is Input stream to be parsed. - \return The document itself for fluent API. - */ - template - GenericDocument& ParseStream(InputStream& is) { - GenericReader reader( - stack_.HasAllocator() ? &stack_.GetAllocator() : 0); - ClearStackOnExit scope(*this); - parseResult_ = reader.template Parse(is, *this); - if (parseResult_) { - RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object - ValueType::operator=(*stack_.template Pop(1));// Move value from stack to document - } - return *this; - } - - //! Parse JSON text from an input stream - /*! \tparam parseFlags Combination of \ref ParseFlag. - \tparam InputStream Type of input stream, implementing Stream concept - \param is Input stream to be parsed. - \return The document itself for fluent API. - */ - template - GenericDocument& ParseStream(InputStream& is) { - return ParseStream(is); - } - - //! Parse JSON text from an input stream (with \ref kParseDefaultFlags) - /*! \tparam InputStream Type of input stream, implementing Stream concept - \param is Input stream to be parsed. - \return The document itself for fluent API. - */ - template - GenericDocument& ParseStream(InputStream& is) { - return ParseStream(is); - } - //!@} - - //!@name Parse in-place from mutable string - //!@{ - - //! Parse JSON text from a mutable string - /*! \tparam parseFlags Combination of \ref ParseFlag. - \param str Mutable zero-terminated string to be parsed. - \return The document itself for fluent API. - */ - template - GenericDocument& ParseInsitu(Ch* str) { - GenericInsituStringStream s(str); - return ParseStream(s); - } - - //! Parse JSON text from a mutable string (with \ref kParseDefaultFlags) - /*! \param str Mutable zero-terminated string to be parsed. - \return The document itself for fluent API. - */ - GenericDocument& ParseInsitu(Ch* str) { - return ParseInsitu(str); - } - //!@} - - //!@name Parse from read-only string - //!@{ - - //! Parse JSON text from a read-only string (with Encoding conversion) - /*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag). - \tparam SourceEncoding Transcoding from input Encoding - \param str Read-only zero-terminated string to be parsed. - */ - template - GenericDocument& Parse(const typename SourceEncoding::Ch* str) { - RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); - GenericStringStream s(str); - return ParseStream(s); - } - - //! Parse JSON text from a read-only string - /*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag). - \param str Read-only zero-terminated string to be parsed. - */ - template - GenericDocument& Parse(const Ch* str) { - return Parse(str); - } - - //! Parse JSON text from a read-only string (with \ref kParseDefaultFlags) - /*! \param str Read-only zero-terminated string to be parsed. - */ - GenericDocument& Parse(const Ch* str) { - return Parse(str); - } - - template - GenericDocument& Parse(const typename SourceEncoding::Ch* str, size_t length) { - RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); - MemoryStream ms(static_cast(str), length * sizeof(typename SourceEncoding::Ch)); - EncodedInputStream is(ms); - ParseStream(is); - return *this; - } - - template - GenericDocument& Parse(const Ch* str, size_t length) { - return Parse(str, length); - } - - GenericDocument& Parse(const Ch* str, size_t length) { - return Parse(str, length); - } + //! Exchange the contents of this document with those of another. + /*! + \param rhs Another document. + \note Constant complexity. + \see GenericValue::Swap + */ + GenericDocument& Swap(GenericDocument& rhs) RAPIDJSON_NOEXCEPT + { ValueType::Swap(rhs); + stack_.Swap(rhs.stack_); + internal::Swap(allocator_, rhs.allocator_); + internal::Swap(ownAllocator_, rhs.ownAllocator_); + internal::Swap(parseResult_, rhs.parseResult_); + return *this; + } + + //! free-standing swap function helper + /*! + Helper function to enable support for common swap implementation pattern based on \c std::swap: + \code + void swap(MyClass& a, MyClass& b) { + using std::swap; + swap(a.doc, b.doc); + // ... + } + \endcode + \see Swap() + */ + friend inline void swap(GenericDocument& a, GenericDocument& b) RAPIDJSON_NOEXCEPT { a.Swap(b); } + + //! Populate this document by a generator which produces SAX events. + /*! \tparam Generator A functor with bool f(Handler) prototype. + \param g Generator functor which sends SAX events to the parameter. + \return The document itself for fluent API. + */ + template + GenericDocument& Populate(Generator& g) + { ClearStackOnExit scope(*this); + if (g(*this)) + { RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object + ValueType::operator=(*stack_.template Pop(1));// Move value from stack to document + } + return *this; + } + + //!@name Parse from stream + //!@{ + + //! Parse JSON text from an input stream (with Encoding conversion) + /*! \tparam parseFlags Combination of \ref ParseFlag. + \tparam SourceEncoding Encoding of input stream + \tparam InputStream Type of input stream, implementing Stream concept + \param is Input stream to be parsed. + \return The document itself for fluent API. + */ + template + GenericDocument& ParseStream(InputStream& is) + { GenericReader reader( + stack_.HasAllocator() ? &stack_.GetAllocator() : 0); + ClearStackOnExit scope(*this); + parseResult_ = reader.template Parse(is, *this); + if (parseResult_) + { RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object + ValueType::operator=(*stack_.template Pop(1));// Move value from stack to document + } + return *this; + } + + //! Parse JSON text from an input stream + /*! \tparam parseFlags Combination of \ref ParseFlag. + \tparam InputStream Type of input stream, implementing Stream concept + \param is Input stream to be parsed. + \return The document itself for fluent API. + */ + template + GenericDocument& ParseStream(InputStream& is) + { return ParseStream(is); + } + + //! Parse JSON text from an input stream (with \ref kParseDefaultFlags) + /*! \tparam InputStream Type of input stream, implementing Stream concept + \param is Input stream to be parsed. + \return The document itself for fluent API. + */ + template + GenericDocument& ParseStream(InputStream& is) + { return ParseStream(is); + } + //!@} + + //!@name Parse in-place from mutable string + //!@{ + + //! Parse JSON text from a mutable string + /*! \tparam parseFlags Combination of \ref ParseFlag. + \param str Mutable zero-terminated string to be parsed. + \return The document itself for fluent API. + */ + template + GenericDocument& ParseInsitu(Ch* str) + { GenericInsituStringStream s(str); + return ParseStream(s); + } + + //! Parse JSON text from a mutable string (with \ref kParseDefaultFlags) + /*! \param str Mutable zero-terminated string to be parsed. + \return The document itself for fluent API. + */ + GenericDocument& ParseInsitu(Ch* str) + { return ParseInsitu(str); + } + //!@} + + //!@name Parse from read-only string + //!@{ + + //! Parse JSON text from a read-only string (with Encoding conversion) + /*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag). + \tparam SourceEncoding Transcoding from input Encoding + \param str Read-only zero-terminated string to be parsed. + */ + template + GenericDocument& Parse(const typename SourceEncoding::Ch* str) + { RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); + GenericStringStream s(str); + return ParseStream(s); + } + + //! Parse JSON text from a read-only string + /*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag). + \param str Read-only zero-terminated string to be parsed. + */ + template + GenericDocument& Parse(const Ch* str) + { return Parse(str); + } + + //! Parse JSON text from a read-only string (with \ref kParseDefaultFlags) + /*! \param str Read-only zero-terminated string to be parsed. + */ + GenericDocument& Parse(const Ch* str) + { return Parse(str); + } + + template + GenericDocument& Parse(const typename SourceEncoding::Ch* str, size_t length) + { RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); + MemoryStream ms(static_cast(str), length * sizeof(typename SourceEncoding::Ch)); + EncodedInputStream is(ms); + ParseStream(is); + return *this; + } + + template + GenericDocument& Parse(const Ch* str, size_t length) + { return Parse(str, length); + } + + GenericDocument& Parse(const Ch* str, size_t length) + { return Parse(str, length); + } #if RAPIDJSON_HAS_STDSTRING - template - GenericDocument& Parse(const std::basic_string& str) { - // c_str() is constant complexity according to standard. Should be faster than Parse(const char*, size_t) - return Parse(str.c_str()); - } - - template - GenericDocument& Parse(const std::basic_string& str) { - return Parse(str.c_str()); - } - - GenericDocument& Parse(const std::basic_string& str) { - return Parse(str); - } + template + GenericDocument& Parse(const std::basic_string& str) + { // c_str() is constant complexity according to standard. Should be faster than Parse(const char*, size_t) + return Parse(str.c_str()); + } + + template + GenericDocument& Parse(const std::basic_string& str) + { return Parse(str.c_str()); + } + + GenericDocument& Parse(const std::basic_string& str) + { return Parse(str); + } #endif // RAPIDJSON_HAS_STDSTRING - //!@} + //!@} - //!@name Handling parse errors - //!@{ + //!@name Handling parse errors + //!@{ - //! Whether a parse error has occured in the last parsing. - bool HasParseError() const { return parseResult_.IsError(); } + //! Whether a parse error has occured in the last parsing. + bool HasParseError() const { return parseResult_.IsError(); } - //! Get the \ref ParseErrorCode of last parsing. - ParseErrorCode GetParseError() const { return parseResult_.Code(); } + //! Get the \ref ParseErrorCode of last parsing. + ParseErrorCode GetParseError() const { return parseResult_.Code(); } - //! Get the position of last parsing error in input, 0 otherwise. - size_t GetErrorOffset() const { return parseResult_.Offset(); } + //! Get the position of last parsing error in input, 0 otherwise. + size_t GetErrorOffset() const { return parseResult_.Offset(); } - //! Implicit conversion to get the last parse result + //! Implicit conversion to get the last parse result #ifndef __clang // -Wdocumentation - /*! \return \ref ParseResult of the last parse operation - - \code - Document doc; - ParseResult ok = doc.Parse(json); - if (!ok) - printf( "JSON parse error: %s (%u)\n", GetParseError_En(ok.Code()), ok.Offset()); - \endcode - */ + /*! \return \ref ParseResult of the last parse operation + + \code + Document doc; + ParseResult ok = doc.Parse(json); + if (!ok) + printf( "JSON parse error: %s (%u)\n", GetParseError_En(ok.Code()), ok.Offset()); + \endcode + */ #endif - operator ParseResult() const { return parseResult_; } - //!@} + operator ParseResult() const { return parseResult_; } + //!@} - //! Get the allocator of this document. - Allocator& GetAllocator() { - RAPIDJSON_ASSERT(allocator_); - return *allocator_; - } + //! Get the allocator of this document. + Allocator& GetAllocator() + { RAPIDJSON_ASSERT(allocator_); + return *allocator_; + } - //! Get the capacity of stack in bytes. - size_t GetStackCapacity() const { return stack_.GetCapacity(); } + //! Get the capacity of stack in bytes. + size_t GetStackCapacity() const { return stack_.GetCapacity(); } private: - // clear stack on any exit from ParseStream, e.g. due to exception - struct ClearStackOnExit { - explicit ClearStackOnExit(GenericDocument& d) : d_(d) {} - ~ClearStackOnExit() { d_.ClearStack(); } - private: - ClearStackOnExit(const ClearStackOnExit&); - ClearStackOnExit& operator=(const ClearStackOnExit&); - GenericDocument& d_; - }; - - // callers of the following private Handler functions - // template friend class GenericReader; // for parsing - template friend class GenericValue; // for deep copying + // clear stack on any exit from ParseStream, e.g. due to exception + struct ClearStackOnExit +{ explicit ClearStackOnExit(GenericDocument& d) : d_(d) {} + ~ClearStackOnExit() { d_.ClearStack(); } + private: + ClearStackOnExit(const ClearStackOnExit&); + ClearStackOnExit& operator=(const ClearStackOnExit&); + GenericDocument& d_; + }; + + // callers of the following private Handler functions + // template friend class GenericReader; // for parsing + template friend class GenericValue; // for deep copying public: - // Implementation of Handler - bool Null() { new (stack_.template Push()) ValueType(); return true; } - bool Bool(bool b) { new (stack_.template Push()) ValueType(b); return true; } - bool Int(int i) { new (stack_.template Push()) ValueType(i); return true; } - bool Uint(unsigned i) { new (stack_.template Push()) ValueType(i); return true; } - bool Int64(int64_t i) { new (stack_.template Push()) ValueType(i); return true; } - bool Uint64(uint64_t i) { new (stack_.template Push()) ValueType(i); return true; } - bool Double(double d) { new (stack_.template Push()) ValueType(d); return true; } - - bool RawNumber(const Ch* str, SizeType length, bool copy) { - if (copy) - new (stack_.template Push()) ValueType(str, length, GetAllocator()); - else - new (stack_.template Push()) ValueType(str, length); - return true; - } - - bool String(const Ch* str, SizeType length, bool copy) { - if (copy) - new (stack_.template Push()) ValueType(str, length, GetAllocator()); - else - new (stack_.template Push()) ValueType(str, length); - return true; - } - - bool StartObject() { new (stack_.template Push()) ValueType(kObjectType); return true; } - - bool Key(const Ch* str, SizeType length, bool copy) { return String(str, length, copy); } - - bool EndObject(SizeType memberCount) { - typename ValueType::Member* members = stack_.template Pop(memberCount); - stack_.template Top()->SetObjectRaw(members, memberCount, GetAllocator()); - return true; - } - - bool StartArray() { new (stack_.template Push()) ValueType(kArrayType); return true; } - - bool EndArray(SizeType elementCount) { - ValueType* elements = stack_.template Pop(elementCount); - stack_.template Top()->SetArrayRaw(elements, elementCount, GetAllocator()); - return true; - } + // Implementation of Handler + bool Null() { new (stack_.template Push()) ValueType(); return true; } + bool Bool(bool b) { new (stack_.template Push()) ValueType(b); return true; } + bool Int(int i) { new (stack_.template Push()) ValueType(i); return true; } + bool Uint(unsigned i) { new (stack_.template Push()) ValueType(i); return true; } + bool Int64(int64_t i) { new (stack_.template Push()) ValueType(i); return true; } + bool Uint64(uint64_t i) { new (stack_.template Push()) ValueType(i); return true; } + bool Double(double d) { new (stack_.template Push()) ValueType(d); return true; } + + bool RawNumber(const Ch* str, SizeType length, bool copy) + { if (copy) + new (stack_.template Push()) ValueType(str, length, GetAllocator()); + else + new (stack_.template Push()) ValueType(str, length); + return true; + } + + bool String(const Ch* str, SizeType length, bool copy) + { if (copy) + new (stack_.template Push()) ValueType(str, length, GetAllocator()); + else + new (stack_.template Push()) ValueType(str, length); + return true; + } + + bool StartObject() { new (stack_.template Push()) ValueType(kObjectType); return true; } + + bool Key(const Ch* str, SizeType length, bool copy) { return String(str, length, copy); } + + bool EndObject(SizeType memberCount) + { typename ValueType::Member* members = stack_.template Pop(memberCount); + stack_.template Top()->SetObjectRaw(members, memberCount, GetAllocator()); + return true; + } + + bool StartArray() { new (stack_.template Push()) ValueType(kArrayType); return true; } + + bool EndArray(SizeType elementCount) + { ValueType* elements = stack_.template Pop(elementCount); + stack_.template Top()->SetArrayRaw(elements, elementCount, GetAllocator()); + return true; + } private: - //! Prohibit copying - GenericDocument(const GenericDocument&); - //! Prohibit assignment - GenericDocument& operator=(const GenericDocument&); - - void ClearStack() { - if (Allocator::kNeedFree) - while (stack_.GetSize() > 0) // Here assumes all elements in stack array are GenericValue (Member is actually 2 GenericValue objects) - (stack_.template Pop(1))->~ValueType(); - else - stack_.Clear(); - stack_.ShrinkToFit(); - } - - void Destroy() { - RAPIDJSON_DELETE(ownAllocator_); - } - - static const size_t kDefaultStackCapacity = 1024; - Allocator* allocator_; - Allocator* ownAllocator_; - internal::Stack stack_; - ParseResult parseResult_; + //! Prohibit copying + GenericDocument(const GenericDocument&); + //! Prohibit assignment + GenericDocument& operator=(const GenericDocument&); + + void ClearStack() + { if (Allocator::kNeedFree) + while (stack_.GetSize() > 0) // Here assumes all elements in stack array are GenericValue (Member is actually 2 GenericValue objects) + (stack_.template Pop(1))->~ValueType(); + else + stack_.Clear(); + stack_.ShrinkToFit(); + } + + void Destroy() + { RAPIDJSON_DELETE(ownAllocator_); + } + + static const size_t kDefaultStackCapacity = 1024; + Allocator* allocator_; + Allocator* ownAllocator_; + internal::Stack stack_; + ParseResult parseResult_; }; //! GenericDocument with UTF8 encoding @@ -2406,28 +2409,28 @@ template template inline GenericValue::GenericValue(const GenericValue& rhs, Allocator& allocator) -{ - switch (rhs.GetType()) { - case kObjectType: - case kArrayType: { // perform deep copy via SAX Handler - GenericDocument d(&allocator); - rhs.Accept(d); - RawAssign(*d.stack_.template Pop(1)); - } - break; +{ switch (rhs.GetType()) + { case kObjectType: + case kArrayType: // perform deep copy via SAX Handler + { GenericDocument d(&allocator); + rhs.Accept(d); + RawAssign(*d.stack_.template Pop(1)); + } + break; case kStringType: - if (rhs.data_.f.flags == kConstStringFlag) { - data_.f.flags = rhs.data_.f.flags; - data_ = *reinterpret_cast(&rhs.data_); - } else { - SetStringRaw(StringRef(rhs.GetString(), rhs.GetStringLength()), allocator); - } - break; - default: - data_.f.flags = rhs.data_.f.flags; + if (rhs.data_.f.flags == kConstStringFlag) + { data_.f.flags = rhs.data_.f.flags; data_ = *reinterpret_cast(&rhs.data_); - break; - } + } + else + { SetStringRaw(StringRef(rhs.GetString(), rhs.GetStringLength()), allocator); + } + break; + default: + data_.f.flags = rhs.data_.f.flags; + data_ = *reinterpret_cast(&rhs.data_); + break; + } } //! Helper class for accessing Value of array type. @@ -2436,51 +2439,52 @@ GenericValue::GenericValue(const GenericValue -class GenericArray { +class GenericArray +{ public: - typedef GenericArray ConstArray; - typedef GenericArray Array; - typedef ValueT PlainType; - typedef typename internal::MaybeAddConst::Type ValueType; - typedef ValueType* ValueIterator; // This may be const or non-const iterator - typedef const ValueT* ConstValueIterator; - typedef typename ValueType::AllocatorType AllocatorType; - typedef typename ValueType::StringRefType StringRefType; - - template - friend class GenericValue; - - GenericArray(const GenericArray& rhs) : value_(rhs.value_) {} - GenericArray& operator=(const GenericArray& rhs) { value_ = rhs.value_; return *this; } - ~GenericArray() {} - - SizeType Size() const { return value_.Size(); } - SizeType Capacity() const { return value_.Capacity(); } - bool Empty() const { return value_.Empty(); } - void Clear() const { value_.Clear(); } - ValueType& operator[](SizeType index) const { return value_[index]; } - ValueIterator Begin() const { return value_.Begin(); } - ValueIterator End() const { return value_.End(); } - GenericArray Reserve(SizeType newCapacity, AllocatorType &allocator) const { value_.Reserve(newCapacity, allocator); return *this; } - GenericArray PushBack(ValueType& value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } + typedef GenericArray ConstArray; + typedef GenericArray Array; + typedef ValueT PlainType; + typedef typename internal::MaybeAddConst::Type ValueType; + typedef ValueType* ValueIterator; // This may be const or non-const iterator + typedef const ValueT* ConstValueIterator; + typedef typename ValueType::AllocatorType AllocatorType; + typedef typename ValueType::StringRefType StringRefType; + + template + friend class GenericValue; + + GenericArray(const GenericArray& rhs) : value_(rhs.value_) {} + GenericArray& operator=(const GenericArray& rhs) { value_ = rhs.value_; return *this; } + ~GenericArray() {} + + SizeType Size() const { return value_.Size(); } + SizeType Capacity() const { return value_.Capacity(); } + bool Empty() const { return value_.Empty(); } + void Clear() const { value_.Clear(); } + ValueType& operator[](SizeType index) const { return value_[index]; } + ValueIterator Begin() const { return value_.Begin(); } + ValueIterator End() const { return value_.End(); } + GenericArray Reserve(SizeType newCapacity, AllocatorType &allocator) const { value_.Reserve(newCapacity, allocator); return *this; } + GenericArray PushBack(ValueType& value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS - GenericArray PushBack(ValueType&& value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } + GenericArray PushBack(ValueType&& value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } #endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS - GenericArray PushBack(StringRefType value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } - template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (const GenericArray&)) PushBack(T value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } - GenericArray PopBack() const { value_.PopBack(); return *this; } - ValueIterator Erase(ConstValueIterator pos) const { return value_.Erase(pos); } - ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) const { return value_.Erase(first, last); } + GenericArray PushBack(StringRefType value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } + template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (const GenericArray&)) PushBack(T value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } + GenericArray PopBack() const { value_.PopBack(); return *this; } + ValueIterator Erase(ConstValueIterator pos) const { return value_.Erase(pos); } + ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) const { return value_.Erase(first, last); } #if RAPIDJSON_HAS_CXX11_RANGE_FOR - ValueIterator begin() const { return value_.Begin(); } - ValueIterator end() const { return value_.End(); } + ValueIterator begin() const { return value_.Begin(); } + ValueIterator end() const { return value_.End(); } #endif private: - GenericArray(); - GenericArray(ValueType& value) : value_(value) {} - ValueType& value_; + GenericArray(); + GenericArray(ValueType& value) : value_(value) {} + ValueType& value_; }; //! Helper class for accessing Value of object type. @@ -2489,84 +2493,85 @@ class GenericArray { In addition to all APIs for array type, it provides range-based for loop if \c RAPIDJSON_HAS_CXX11_RANGE_FOR=1. */ template -class GenericObject { +class GenericObject +{ public: - typedef GenericObject ConstObject; - typedef GenericObject Object; - typedef ValueT PlainType; - typedef typename internal::MaybeAddConst::Type ValueType; - typedef GenericMemberIterator MemberIterator; // This may be const or non-const iterator - typedef GenericMemberIterator ConstMemberIterator; - typedef typename ValueType::AllocatorType AllocatorType; - typedef typename ValueType::StringRefType StringRefType; - typedef typename ValueType::EncodingType EncodingType; - typedef typename ValueType::Ch Ch; - - template - friend class GenericValue; - - GenericObject(const GenericObject& rhs) : value_(rhs.value_) {} - GenericObject& operator=(const GenericObject& rhs) { value_ = rhs.value_; return *this; } - ~GenericObject() {} - - SizeType MemberCount() const { return value_.MemberCount(); } - bool ObjectEmpty() const { return value_.ObjectEmpty(); } - template ValueType& operator[](T* name) const { return value_[name]; } - template ValueType& operator[](const GenericValue& name) const { return value_[name]; } + typedef GenericObject ConstObject; + typedef GenericObject Object; + typedef ValueT PlainType; + typedef typename internal::MaybeAddConst::Type ValueType; + typedef GenericMemberIterator MemberIterator; // This may be const or non-const iterator + typedef GenericMemberIterator ConstMemberIterator; + typedef typename ValueType::AllocatorType AllocatorType; + typedef typename ValueType::StringRefType StringRefType; + typedef typename ValueType::EncodingType EncodingType; + typedef typename ValueType::Ch Ch; + + template + friend class GenericValue; + + GenericObject(const GenericObject& rhs) : value_(rhs.value_) {} + GenericObject& operator=(const GenericObject& rhs) { value_ = rhs.value_; return *this; } + ~GenericObject() {} + + SizeType MemberCount() const { return value_.MemberCount(); } + bool ObjectEmpty() const { return value_.ObjectEmpty(); } + template ValueType& operator[](T* name) const { return value_[name]; } + template ValueType& operator[](const GenericValue& name) const { return value_[name]; } #if RAPIDJSON_HAS_STDSTRING - ValueType& operator[](const std::basic_string& name) const { return value_[name]; } + ValueType& operator[](const std::basic_string& name) const { return value_[name]; } #endif - MemberIterator MemberBegin() const { return value_.MemberBegin(); } - MemberIterator MemberEnd() const { return value_.MemberEnd(); } - bool HasMember(const Ch* name) const { return value_.HasMember(name); } + MemberIterator MemberBegin() const { return value_.MemberBegin(); } + MemberIterator MemberEnd() const { return value_.MemberEnd(); } + bool HasMember(const Ch* name) const { return value_.HasMember(name); } #if RAPIDJSON_HAS_STDSTRING - bool HasMember(const std::basic_string& name) const { return value_.HasMember(name); } + bool HasMember(const std::basic_string& name) const { return value_.HasMember(name); } #endif - template bool HasMember(const GenericValue& name) const { return value_.HasMember(name); } - MemberIterator FindMember(const Ch* name) const { return value_.FindMember(name); } - template MemberIterator FindMember(const GenericValue& name) const { return value_.FindMember(name); } + template bool HasMember(const GenericValue& name) const { return value_.HasMember(name); } + MemberIterator FindMember(const Ch* name) const { return value_.FindMember(name); } + template MemberIterator FindMember(const GenericValue& name) const { return value_.FindMember(name); } #if RAPIDJSON_HAS_STDSTRING - MemberIterator FindMember(const std::basic_string& name) const { return value_.FindMember(name); } + MemberIterator FindMember(const std::basic_string& name) const { return value_.FindMember(name); } #endif - GenericObject AddMember(ValueType& name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - GenericObject AddMember(ValueType& name, StringRefType value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(ValueType& name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(ValueType& name, StringRefType value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } #if RAPIDJSON_HAS_STDSTRING - GenericObject AddMember(ValueType& name, std::basic_string& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(ValueType& name, std::basic_string& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } #endif - template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) AddMember(ValueType& name, T value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) AddMember(ValueType& name, T value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS - GenericObject AddMember(ValueType&& name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - GenericObject AddMember(ValueType&& name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - GenericObject AddMember(ValueType& name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - GenericObject AddMember(StringRefType name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(ValueType&& name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(ValueType&& name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(ValueType& name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(StringRefType name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } #endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS - GenericObject AddMember(StringRefType name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - GenericObject AddMember(StringRefType name, StringRefType value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericObject)) AddMember(StringRefType name, T value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - void RemoveAllMembers() { return value_.RemoveAllMembers(); } - bool RemoveMember(const Ch* name) const { return value_.RemoveMember(name); } + GenericObject AddMember(StringRefType name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + GenericObject AddMember(StringRefType name, StringRefType value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericObject)) AddMember(StringRefType name, T value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } + void RemoveAllMembers() { return value_.RemoveAllMembers(); } + bool RemoveMember(const Ch* name) const { return value_.RemoveMember(name); } #if RAPIDJSON_HAS_STDSTRING - bool RemoveMember(const std::basic_string& name) const { return value_.RemoveMember(name); } + bool RemoveMember(const std::basic_string& name) const { return value_.RemoveMember(name); } #endif - template bool RemoveMember(const GenericValue& name) const { return value_.RemoveMember(name); } - MemberIterator RemoveMember(MemberIterator m) const { return value_.RemoveMember(m); } - MemberIterator EraseMember(ConstMemberIterator pos) const { return value_.EraseMember(pos); } - MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) const { return value_.EraseMember(first, last); } - bool EraseMember(const Ch* name) const { return value_.EraseMember(name); } + template bool RemoveMember(const GenericValue& name) const { return value_.RemoveMember(name); } + MemberIterator RemoveMember(MemberIterator m) const { return value_.RemoveMember(m); } + MemberIterator EraseMember(ConstMemberIterator pos) const { return value_.EraseMember(pos); } + MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) const { return value_.EraseMember(first, last); } + bool EraseMember(const Ch* name) const { return value_.EraseMember(name); } #if RAPIDJSON_HAS_STDSTRING - bool EraseMember(const std::basic_string& name) const { return EraseMember(ValueType(StringRef(name))); } + bool EraseMember(const std::basic_string& name) const { return EraseMember(ValueType(StringRef(name))); } #endif - template bool EraseMember(const GenericValue& name) const { return value_.EraseMember(name); } + template bool EraseMember(const GenericValue& name) const { return value_.EraseMember(name); } #if RAPIDJSON_HAS_CXX11_RANGE_FOR - MemberIterator begin() const { return value_.MemberBegin(); } - MemberIterator end() const { return value_.MemberEnd(); } + MemberIterator begin() const { return value_.MemberBegin(); } + MemberIterator end() const { return value_.MemberEnd(); } #endif private: - GenericObject(); - GenericObject(ValueType& value) : value_(value) {} - ValueType& value_; + GenericObject(); + GenericObject(ValueType& value) : value_(value) {} + ValueType& value_; }; RAPIDJSON_NAMESPACE_END diff --git a/rapidjson/include/rapidjson/encodedstream.h b/rapidjson/include/rapidjson/encodedstream.h index 145068386a0..06d30a469c6 100644 --- a/rapidjson/include/rapidjson/encodedstream.h +++ b/rapidjson/include/rapidjson/encodedstream.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_ENCODEDSTREAM_H_ @@ -36,59 +36,60 @@ RAPIDJSON_NAMESPACE_BEGIN \tparam InputByteStream Type of input byte stream. For example, FileReadStream. */ template -class EncodedInputStream { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); +class EncodedInputStream +{ RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); public: - typedef typename Encoding::Ch Ch; + typedef typename Encoding::Ch Ch; - EncodedInputStream(InputByteStream& is) : is_(is) { - current_ = Encoding::TakeBOM(is_); - } + EncodedInputStream(InputByteStream& is) : is_(is) + { current_ = Encoding::TakeBOM(is_); + } - Ch Peek() const { return current_; } - Ch Take() { Ch c = current_; current_ = Encoding::Take(is_); return c; } - size_t Tell() const { return is_.Tell(); } + Ch Peek() const { return current_; } + Ch Take() { Ch c = current_; current_ = Encoding::Take(is_); return c; } + size_t Tell() const { return is_.Tell(); } - // Not implemented - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + // Not implemented + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } private: - EncodedInputStream(const EncodedInputStream&); - EncodedInputStream& operator=(const EncodedInputStream&); + EncodedInputStream(const EncodedInputStream&); + EncodedInputStream& operator=(const EncodedInputStream&); - InputByteStream& is_; - Ch current_; + InputByteStream& is_; + Ch current_; }; //! Specialized for UTF8 MemoryStream. template <> -class EncodedInputStream, MemoryStream> { +class EncodedInputStream, MemoryStream> +{ public: - typedef UTF8<>::Ch Ch; + typedef UTF8<>::Ch Ch; - EncodedInputStream(MemoryStream& is) : is_(is) { - if (static_cast(is_.Peek()) == 0xEFu) is_.Take(); - if (static_cast(is_.Peek()) == 0xBBu) is_.Take(); - if (static_cast(is_.Peek()) == 0xBFu) is_.Take(); - } - Ch Peek() const { return is_.Peek(); } - Ch Take() { return is_.Take(); } - size_t Tell() const { return is_.Tell(); } + EncodedInputStream(MemoryStream& is) : is_(is) + { if (static_cast(is_.Peek()) == 0xEFu) is_.Take(); + if (static_cast(is_.Peek()) == 0xBBu) is_.Take(); + if (static_cast(is_.Peek()) == 0xBFu) is_.Take(); + } + Ch Peek() const { return is_.Peek(); } + Ch Take() { return is_.Take(); } + size_t Tell() const { return is_.Tell(); } - // Not implemented - void Put(Ch) {} - void Flush() {} - Ch* PutBegin() { return 0; } - size_t PutEnd(Ch*) { return 0; } + // Not implemented + void Put(Ch) {} + void Flush() {} + Ch* PutBegin() { return 0; } + size_t PutEnd(Ch*) { return 0; } - MemoryStream& is_; + MemoryStream& is_; private: - EncodedInputStream(const EncodedInputStream&); - EncodedInputStream& operator=(const EncodedInputStream&); + EncodedInputStream(const EncodedInputStream&); + EncodedInputStream& operator=(const EncodedInputStream&); }; //! Output byte stream wrapper with statically bound encoding. @@ -97,31 +98,31 @@ class EncodedInputStream, MemoryStream> { \tparam OutputByteStream Type of input byte stream. For example, FileWriteStream. */ template -class EncodedOutputStream { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); +class EncodedOutputStream +{ RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); public: - typedef typename Encoding::Ch Ch; + typedef typename Encoding::Ch Ch; - EncodedOutputStream(OutputByteStream& os, bool putBOM = true) : os_(os) { - if (putBOM) - Encoding::PutBOM(os_); - } + EncodedOutputStream(OutputByteStream& os, bool putBOM = true) : os_(os) + { if (putBOM) + Encoding::PutBOM(os_); + } - void Put(Ch c) { Encoding::Put(os_, c); } - void Flush() { os_.Flush(); } + void Put(Ch c) { Encoding::Put(os_, c); } + void Flush() { os_.Flush(); } - // Not implemented - Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;} - Ch Take() { RAPIDJSON_ASSERT(false); return 0;} - size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + // Not implemented + Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;} + Ch Take() { RAPIDJSON_ASSERT(false); return 0;} + size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } private: - EncodedOutputStream(const EncodedOutputStream&); - EncodedOutputStream& operator=(const EncodedOutputStream&); + EncodedOutputStream(const EncodedOutputStream&); + EncodedOutputStream& operator=(const EncodedOutputStream&); - OutputByteStream& os_; + OutputByteStream& os_; }; #define RAPIDJSON_ENCODINGS_FUNC(x) UTF8::x, UTF16LE::x, UTF16BE::x, UTF32LE::x, UTF32BE::x @@ -132,96 +133,96 @@ class EncodedOutputStream { \tparam InputByteStream type of input byte stream to be wrapped. */ template -class AutoUTFInputStream { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); +class AutoUTFInputStream +{ RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); public: - typedef CharType Ch; - - //! Constructor. - /*! - \param is input stream to be wrapped. - \param type UTF encoding type if it is not detected from the stream. - */ - AutoUTFInputStream(InputByteStream& is, UTFType type = kUTF8) : is_(&is), type_(type), hasBOM_(false) { - RAPIDJSON_ASSERT(type >= kUTF8 && type <= kUTF32BE); - DetectType(); - static const TakeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Take) }; - takeFunc_ = f[type_]; - current_ = takeFunc_(*is_); - } - - UTFType GetType() const { return type_; } - bool HasBOM() const { return hasBOM_; } - - Ch Peek() const { return current_; } - Ch Take() { Ch c = current_; current_ = takeFunc_(*is_); return c; } - size_t Tell() const { return is_->Tell(); } - - // Not implemented - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + typedef CharType Ch; + + //! Constructor. + /*! + \param is input stream to be wrapped. + \param type UTF encoding type if it is not detected from the stream. + */ + AutoUTFInputStream(InputByteStream& is, UTFType type = kUTF8) : is_(&is), type_(type), hasBOM_(false) + { RAPIDJSON_ASSERT(type >= kUTF8 && type <= kUTF32BE); + DetectType(); + static const TakeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Take) }; + takeFunc_ = f[type_]; + current_ = takeFunc_(*is_); + } + + UTFType GetType() const { return type_; } + bool HasBOM() const { return hasBOM_; } + + Ch Peek() const { return current_; } + Ch Take() { Ch c = current_; current_ = takeFunc_(*is_); return c; } + size_t Tell() const { return is_->Tell(); } + + // Not implemented + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } private: - AutoUTFInputStream(const AutoUTFInputStream&); - AutoUTFInputStream& operator=(const AutoUTFInputStream&); - - // Detect encoding type with BOM or RFC 4627 - void DetectType() { - // BOM (Byte Order Mark): - // 00 00 FE FF UTF-32BE - // FF FE 00 00 UTF-32LE - // FE FF UTF-16BE - // FF FE UTF-16LE - // EF BB BF UTF-8 - - const unsigned char* c = reinterpret_cast(is_->Peek4()); - if (!c) - return; - - unsigned bom = static_cast(c[0] | (c[1] << 8) | (c[2] << 16) | (c[3] << 24)); - hasBOM_ = false; - if (bom == 0xFFFE0000) { type_ = kUTF32BE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } - else if (bom == 0x0000FEFF) { type_ = kUTF32LE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } - else if ((bom & 0xFFFF) == 0xFFFE) { type_ = kUTF16BE; hasBOM_ = true; is_->Take(); is_->Take(); } - else if ((bom & 0xFFFF) == 0xFEFF) { type_ = kUTF16LE; hasBOM_ = true; is_->Take(); is_->Take(); } - else if ((bom & 0xFFFFFF) == 0xBFBBEF) { type_ = kUTF8; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); } - - // RFC 4627: Section 3 - // "Since the first two characters of a JSON text will always be ASCII - // characters [RFC0020], it is possible to determine whether an octet - // stream is UTF-8, UTF-16 (BE or LE), or UTF-32 (BE or LE) by looking - // at the pattern of nulls in the first four octets." - // 00 00 00 xx UTF-32BE - // 00 xx 00 xx UTF-16BE - // xx 00 00 00 UTF-32LE - // xx 00 xx 00 UTF-16LE - // xx xx xx xx UTF-8 - - if (!hasBOM_) { - unsigned pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0); - switch (pattern) { - case 0x08: type_ = kUTF32BE; break; - case 0x0A: type_ = kUTF16BE; break; - case 0x01: type_ = kUTF32LE; break; - case 0x05: type_ = kUTF16LE; break; - case 0x0F: type_ = kUTF8; break; - default: break; // Use type defined by user. - } - } - - // Runtime check whether the size of character type is sufficient. It only perform checks with assertion. - if (type_ == kUTF16LE || type_ == kUTF16BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 2); - if (type_ == kUTF32LE || type_ == kUTF32BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 4); + AutoUTFInputStream(const AutoUTFInputStream&); + AutoUTFInputStream& operator=(const AutoUTFInputStream&); + + // Detect encoding type with BOM or RFC 4627 + void DetectType() + { // BOM (Byte Order Mark): + // 00 00 FE FF UTF-32BE + // FF FE 00 00 UTF-32LE + // FE FF UTF-16BE + // FF FE UTF-16LE + // EF BB BF UTF-8 + + const unsigned char* c = reinterpret_cast(is_->Peek4()); + if (!c) + return; + + unsigned bom = static_cast(c[0] | (c[1] << 8) | (c[2] << 16) | (c[3] << 24)); + hasBOM_ = false; + if (bom == 0xFFFE0000) { type_ = kUTF32BE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } + else if (bom == 0x0000FEFF) { type_ = kUTF32LE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } + else if ((bom & 0xFFFF) == 0xFFFE) { type_ = kUTF16BE; hasBOM_ = true; is_->Take(); is_->Take(); } + else if ((bom & 0xFFFF) == 0xFEFF) { type_ = kUTF16LE; hasBOM_ = true; is_->Take(); is_->Take(); } + else if ((bom & 0xFFFFFF) == 0xBFBBEF) { type_ = kUTF8; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); } + + // RFC 4627: Section 3 + // "Since the first two characters of a JSON text will always be ASCII + // characters [RFC0020], it is possible to determine whether an octet + // stream is UTF-8, UTF-16 (BE or LE), or UTF-32 (BE or LE) by looking + // at the pattern of nulls in the first four octets." + // 00 00 00 xx UTF-32BE + // 00 xx 00 xx UTF-16BE + // xx 00 00 00 UTF-32LE + // xx 00 xx 00 UTF-16LE + // xx xx xx xx UTF-8 + + if (!hasBOM_) + { unsigned pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0); + switch (pattern) + { case 0x08: type_ = kUTF32BE; break; + case 0x0A: type_ = kUTF16BE; break; + case 0x01: type_ = kUTF32LE; break; + case 0x05: type_ = kUTF16LE; break; + case 0x0F: type_ = kUTF8; break; + default: break; // Use type defined by user. + } } - typedef Ch (*TakeFunc)(InputByteStream& is); - InputByteStream* is_; - UTFType type_; - Ch current_; - TakeFunc takeFunc_; - bool hasBOM_; + // Runtime check whether the size of character type is sufficient. It only perform checks with assertion. + if (type_ == kUTF16LE || type_ == kUTF16BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 2); + if (type_ == kUTF32LE || type_ == kUTF32BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 4); + } + + typedef Ch (*TakeFunc)(InputByteStream& is); + InputByteStream* is_; + UTFType type_; + Ch current_; + TakeFunc takeFunc_; + bool hasBOM_; }; //! Output stream wrapper with dynamically bound encoding and automatic encoding detection. @@ -230,58 +231,58 @@ class AutoUTFInputStream { \tparam OutputByteStream type of output byte stream to be wrapped. */ template -class AutoUTFOutputStream { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); +class AutoUTFOutputStream +{ RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); public: - typedef CharType Ch; - - //! Constructor. - /*! - \param os output stream to be wrapped. - \param type UTF encoding type. - \param putBOM Whether to write BOM at the beginning of the stream. - */ - AutoUTFOutputStream(OutputByteStream& os, UTFType type, bool putBOM) : os_(&os), type_(type) { - RAPIDJSON_ASSERT(type >= kUTF8 && type <= kUTF32BE); - - // Runtime check whether the size of character type is sufficient. It only perform checks with assertion. - if (type_ == kUTF16LE || type_ == kUTF16BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 2); - if (type_ == kUTF32LE || type_ == kUTF32BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 4); - - static const PutFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Put) }; - putFunc_ = f[type_]; - - if (putBOM) - PutBOM(); - } + typedef CharType Ch; + + //! Constructor. + /*! + \param os output stream to be wrapped. + \param type UTF encoding type. + \param putBOM Whether to write BOM at the beginning of the stream. + */ + AutoUTFOutputStream(OutputByteStream& os, UTFType type, bool putBOM) : os_(&os), type_(type) + { RAPIDJSON_ASSERT(type >= kUTF8 && type <= kUTF32BE); + + // Runtime check whether the size of character type is sufficient. It only perform checks with assertion. + if (type_ == kUTF16LE || type_ == kUTF16BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 2); + if (type_ == kUTF32LE || type_ == kUTF32BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 4); - UTFType GetType() const { return type_; } + static const PutFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Put) }; + putFunc_ = f[type_]; - void Put(Ch c) { putFunc_(*os_, c); } - void Flush() { os_->Flush(); } + if (putBOM) + PutBOM(); + } - // Not implemented - Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;} - Ch Take() { RAPIDJSON_ASSERT(false); return 0;} - size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + UTFType GetType() const { return type_; } + + void Put(Ch c) { putFunc_(*os_, c); } + void Flush() { os_->Flush(); } + + // Not implemented + Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;} + Ch Take() { RAPIDJSON_ASSERT(false); return 0;} + size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } private: - AutoUTFOutputStream(const AutoUTFOutputStream&); - AutoUTFOutputStream& operator=(const AutoUTFOutputStream&); + AutoUTFOutputStream(const AutoUTFOutputStream&); + AutoUTFOutputStream& operator=(const AutoUTFOutputStream&); - void PutBOM() { - typedef void (*PutBOMFunc)(OutputByteStream&); - static const PutBOMFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(PutBOM) }; - f[type_](*os_); - } + void PutBOM() + { typedef void (*PutBOMFunc)(OutputByteStream&); + static const PutBOMFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(PutBOM) }; + f[type_](*os_); + } - typedef void (*PutFunc)(OutputByteStream&, Ch); + typedef void (*PutFunc)(OutputByteStream&, Ch); - OutputByteStream* os_; - UTFType type_; - PutFunc putFunc_; + OutputByteStream* os_; + UTFType type_; + PutFunc putFunc_; }; #undef RAPIDJSON_ENCODINGS_FUNC diff --git a/rapidjson/include/rapidjson/encodings.h b/rapidjson/include/rapidjson/encodings.h index baa7c2b17f8..c44e891084c 100644 --- a/rapidjson/include/rapidjson/encodings.h +++ b/rapidjson/include/rapidjson/encodings.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_ENCODINGS_H_ @@ -93,164 +93,167 @@ concept Encoding { \note implements Encoding concept */ template -struct UTF8 { - typedef CharType Ch; - - enum { supportUnicode = 1 }; - - template - static void Encode(OutputStream& os, unsigned codepoint) { - if (codepoint <= 0x7F) - os.Put(static_cast(codepoint & 0xFF)); - else if (codepoint <= 0x7FF) { - os.Put(static_cast(0xC0 | ((codepoint >> 6) & 0xFF))); - os.Put(static_cast(0x80 | ((codepoint & 0x3F)))); - } - else if (codepoint <= 0xFFFF) { - os.Put(static_cast(0xE0 | ((codepoint >> 12) & 0xFF))); - os.Put(static_cast(0x80 | ((codepoint >> 6) & 0x3F))); - os.Put(static_cast(0x80 | (codepoint & 0x3F))); - } - else { - RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); - os.Put(static_cast(0xF0 | ((codepoint >> 18) & 0xFF))); - os.Put(static_cast(0x80 | ((codepoint >> 12) & 0x3F))); - os.Put(static_cast(0x80 | ((codepoint >> 6) & 0x3F))); - os.Put(static_cast(0x80 | (codepoint & 0x3F))); - } - } - - template - static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { - if (codepoint <= 0x7F) - PutUnsafe(os, static_cast(codepoint & 0xFF)); - else if (codepoint <= 0x7FF) { - PutUnsafe(os, static_cast(0xC0 | ((codepoint >> 6) & 0xFF))); - PutUnsafe(os, static_cast(0x80 | ((codepoint & 0x3F)))); - } - else if (codepoint <= 0xFFFF) { - PutUnsafe(os, static_cast(0xE0 | ((codepoint >> 12) & 0xFF))); - PutUnsafe(os, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); - PutUnsafe(os, static_cast(0x80 | (codepoint & 0x3F))); - } - else { - RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); - PutUnsafe(os, static_cast(0xF0 | ((codepoint >> 18) & 0xFF))); - PutUnsafe(os, static_cast(0x80 | ((codepoint >> 12) & 0x3F))); - PutUnsafe(os, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); - PutUnsafe(os, static_cast(0x80 | (codepoint & 0x3F))); - } - } - - template - static bool Decode(InputStream& is, unsigned* codepoint) { +struct UTF8 +{ typedef CharType Ch; + + enum { supportUnicode = 1 }; + + template + static void Encode(OutputStream& os, unsigned codepoint) + { if (codepoint <= 0x7F) + os.Put(static_cast(codepoint & 0xFF)); + else if (codepoint <= 0x7FF) + { os.Put(static_cast(0xC0 | ((codepoint >> 6) & 0xFF))); + os.Put(static_cast(0x80 | ((codepoint & 0x3F)))); + } + else if (codepoint <= 0xFFFF) + { os.Put(static_cast(0xE0 | ((codepoint >> 12) & 0xFF))); + os.Put(static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + os.Put(static_cast(0x80 | (codepoint & 0x3F))); + } + else + { RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + os.Put(static_cast(0xF0 | ((codepoint >> 18) & 0xFF))); + os.Put(static_cast(0x80 | ((codepoint >> 12) & 0x3F))); + os.Put(static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + os.Put(static_cast(0x80 | (codepoint & 0x3F))); + } + } + + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) + { if (codepoint <= 0x7F) + PutUnsafe(os, static_cast(codepoint & 0xFF)); + else if (codepoint <= 0x7FF) + { PutUnsafe(os, static_cast(0xC0 | ((codepoint >> 6) & 0xFF))); + PutUnsafe(os, static_cast(0x80 | ((codepoint & 0x3F)))); + } + else if (codepoint <= 0xFFFF) + { PutUnsafe(os, static_cast(0xE0 | ((codepoint >> 12) & 0xFF))); + PutUnsafe(os, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + PutUnsafe(os, static_cast(0x80 | (codepoint & 0x3F))); + } + else + { RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + PutUnsafe(os, static_cast(0xF0 | ((codepoint >> 18) & 0xFF))); + PutUnsafe(os, static_cast(0x80 | ((codepoint >> 12) & 0x3F))); + PutUnsafe(os, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + PutUnsafe(os, static_cast(0x80 | (codepoint & 0x3F))); + } + } + + template + static bool Decode(InputStream& is, unsigned* codepoint) + { #define COPY() c = is.Take(); *codepoint = (*codepoint << 6) | (static_cast(c) & 0x3Fu) #define TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) #define TAIL() COPY(); TRANS(0x70) - typename InputStream::Ch c = is.Take(); - if (!(c & 0x80)) { - *codepoint = static_cast(c); - return true; - } - - unsigned char type = GetRange(static_cast(c)); - if (type >= 32) { - *codepoint = 0; - } else { - *codepoint = (0xFF >> type) & static_cast(c); - } - bool result = true; - switch (type) { - case 2: TAIL(); return result; - case 3: TAIL(); TAIL(); return result; - case 4: COPY(); TRANS(0x50); TAIL(); return result; - case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); return result; - case 6: TAIL(); TAIL(); TAIL(); return result; - case 10: COPY(); TRANS(0x20); TAIL(); return result; - case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); return result; - default: return false; - } + typename InputStream::Ch c = is.Take(); + if (!(c & 0x80)) + { *codepoint = static_cast(c); + return true; + } + + unsigned char type = GetRange(static_cast(c)); + if (type >= 32) + { *codepoint = 0; + } + else + { *codepoint = (0xFF >> type) & static_cast(c); + } + bool result = true; + switch (type) + { case 2: TAIL(); return result; + case 3: TAIL(); TAIL(); return result; + case 4: COPY(); TRANS(0x50); TAIL(); return result; + case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); return result; + case 6: TAIL(); TAIL(); TAIL(); return result; + case 10: COPY(); TRANS(0x20); TAIL(); return result; + case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); return result; + default: return false; + } #undef COPY #undef TRANS #undef TAIL - } + } - template - static bool Validate(InputStream& is, OutputStream& os) { + template + static bool Validate(InputStream& is, OutputStream& os) + { #define COPY() os.Put(c = is.Take()) #define TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) #define TAIL() COPY(); TRANS(0x70) - Ch c; - COPY(); - if (!(c & 0x80)) - return true; - - bool result = true; - switch (GetRange(static_cast(c))) { - case 2: TAIL(); return result; - case 3: TAIL(); TAIL(); return result; - case 4: COPY(); TRANS(0x50); TAIL(); return result; - case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); return result; - case 6: TAIL(); TAIL(); TAIL(); return result; - case 10: COPY(); TRANS(0x20); TAIL(); return result; - case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); return result; - default: return false; - } + Ch c; + COPY(); + if (!(c & 0x80)) + return true; + + bool result = true; + switch (GetRange(static_cast(c))) + { case 2: TAIL(); return result; + case 3: TAIL(); TAIL(); return result; + case 4: COPY(); TRANS(0x50); TAIL(); return result; + case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); return result; + case 6: TAIL(); TAIL(); TAIL(); return result; + case 10: COPY(); TRANS(0x20); TAIL(); return result; + case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); return result; + default: return false; + } #undef COPY #undef TRANS #undef TAIL - } - - static unsigned char GetRange(unsigned char c) { - // Referring to DFA of http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ - // With new mapping 1 -> 0x10, 7 -> 0x20, 9 -> 0x40, such that AND operation can test multiple types. - static const unsigned char type[] = { - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10, - 0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40, - 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, - 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, - 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, - 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, - }; - return type[c]; - } - - template - static CharType TakeBOM(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - typename InputByteStream::Ch c = Take(is); - if (static_cast(c) != 0xEFu) return c; - c = is.Take(); - if (static_cast(c) != 0xBBu) return c; - c = is.Take(); - if (static_cast(c) != 0xBFu) return c; - c = is.Take(); - return c; - } - - template - static Ch Take(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - return static_cast(is.Take()); - } - - template - static void PutBOM(OutputByteStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast(0xEFu)); - os.Put(static_cast(0xBBu)); - os.Put(static_cast(0xBFu)); - } - - template - static void Put(OutputByteStream& os, Ch c) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast(c)); - } + } + + static unsigned char GetRange(unsigned char c) + { // Referring to DFA of http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ + // With new mapping 1 -> 0x10, 7 -> 0x20, 9 -> 0x40, such that AND operation can test multiple types. + static const unsigned char type[] = + { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10, + 0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40, + 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, + 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, + 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, + 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, + }; + return type[c]; + } + + template + static CharType TakeBOM(InputByteStream& is) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + typename InputByteStream::Ch c = Take(is); + if (static_cast(c) != 0xEFu) return c; + c = is.Take(); + if (static_cast(c) != 0xBBu) return c; + c = is.Take(); + if (static_cast(c) != 0xBFu) return c; + c = is.Take(); + return c; + } + + template + static Ch Take(InputByteStream& is) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + return static_cast(is.Take()); + } + + template + static void PutBOM(OutputByteStream& os) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(0xEFu)); + os.Put(static_cast(0xBBu)); + os.Put(static_cast(0xBFu)); + } + + template + static void Put(OutputByteStream& os, Ch c) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(c)); + } }; /////////////////////////////////////////////////////////////////////////////// @@ -266,147 +269,147 @@ struct UTF8 { For streaming, use UTF16LE and UTF16BE, which handle endianness. */ template -struct UTF16 { - typedef CharType Ch; - RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 2); - - enum { supportUnicode = 1 }; - - template - static void Encode(OutputStream& os, unsigned codepoint) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); - if (codepoint <= 0xFFFF) { - RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair - os.Put(static_cast(codepoint)); - } - else { - RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); - unsigned v = codepoint - 0x10000; - os.Put(static_cast((v >> 10) | 0xD800)); - os.Put((v & 0x3FF) | 0xDC00); - } - } - - - template - static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); - if (codepoint <= 0xFFFF) { - RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair - PutUnsafe(os, static_cast(codepoint)); - } - else { - RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); - unsigned v = codepoint - 0x10000; - PutUnsafe(os, static_cast((v >> 10) | 0xD800)); - PutUnsafe(os, (v & 0x3FF) | 0xDC00); - } - } - - template - static bool Decode(InputStream& is, unsigned* codepoint) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); - typename InputStream::Ch c = is.Take(); - if (c < 0xD800 || c > 0xDFFF) { - *codepoint = static_cast(c); - return true; - } - else if (c <= 0xDBFF) { - *codepoint = (static_cast(c) & 0x3FF) << 10; - c = is.Take(); - *codepoint |= (static_cast(c) & 0x3FF); - *codepoint += 0x10000; - return c >= 0xDC00 && c <= 0xDFFF; - } - return false; - } - - template - static bool Validate(InputStream& is, OutputStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); - typename InputStream::Ch c; - os.Put(static_cast(c = is.Take())); - if (c < 0xD800 || c > 0xDFFF) - return true; - else if (c <= 0xDBFF) { - os.Put(c = is.Take()); - return c >= 0xDC00 && c <= 0xDFFF; - } - return false; - } +struct UTF16 +{ typedef CharType Ch; + RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 2); + + enum { supportUnicode = 1 }; + + template + static void Encode(OutputStream& os, unsigned codepoint) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); + if (codepoint <= 0xFFFF) + { RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair + os.Put(static_cast(codepoint)); + } + else + { RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + unsigned v = codepoint - 0x10000; + os.Put(static_cast((v >> 10) | 0xD800)); + os.Put((v & 0x3FF) | 0xDC00); + } + } + + + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); + if (codepoint <= 0xFFFF) + { RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair + PutUnsafe(os, static_cast(codepoint)); + } + else + { RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + unsigned v = codepoint - 0x10000; + PutUnsafe(os, static_cast((v >> 10) | 0xD800)); + PutUnsafe(os, (v & 0x3FF) | 0xDC00); + } + } + + template + static bool Decode(InputStream& is, unsigned* codepoint) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); + typename InputStream::Ch c = is.Take(); + if (c < 0xD800 || c > 0xDFFF) + { *codepoint = static_cast(c); + return true; + } + else if (c <= 0xDBFF) + { *codepoint = (static_cast(c) & 0x3FF) << 10; + c = is.Take(); + *codepoint |= (static_cast(c) & 0x3FF); + *codepoint += 0x10000; + return c >= 0xDC00 && c <= 0xDFFF; + } + return false; + } + + template + static bool Validate(InputStream& is, OutputStream& os) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); + RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); + typename InputStream::Ch c; + os.Put(static_cast(c = is.Take())); + if (c < 0xD800 || c > 0xDFFF) + return true; + else if (c <= 0xDBFF) + { os.Put(c = is.Take()); + return c >= 0xDC00 && c <= 0xDFFF; + } + return false; + } }; //! UTF-16 little endian encoding. template -struct UTF16LE : UTF16 { - template - static CharType TakeBOM(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = Take(is); - return static_cast(c) == 0xFEFFu ? Take(is) : c; - } - - template - static CharType Take(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - unsigned c = static_cast(is.Take()); - c |= static_cast(static_cast(is.Take())) << 8; - return static_cast(c); - } - - template - static void PutBOM(OutputByteStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast(0xFFu)); - os.Put(static_cast(0xFEu)); - } - - template - static void Put(OutputByteStream& os, CharType c) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast(static_cast(c) & 0xFFu)); - os.Put(static_cast((static_cast(c) >> 8) & 0xFFu)); - } +struct UTF16LE : UTF16 +{ template + static CharType TakeBOM(InputByteStream& is) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = Take(is); + return static_cast(c) == 0xFEFFu ? Take(is) : c; + } + + template + static CharType Take(InputByteStream& is) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + unsigned c = static_cast(is.Take()); + c |= static_cast(static_cast(is.Take())) << 8; + return static_cast(c); + } + + template + static void PutBOM(OutputByteStream& os) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(0xFFu)); + os.Put(static_cast(0xFEu)); + } + + template + static void Put(OutputByteStream& os, CharType c) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(static_cast(c) & 0xFFu)); + os.Put(static_cast((static_cast(c) >> 8) & 0xFFu)); + } }; //! UTF-16 big endian encoding. template -struct UTF16BE : UTF16 { - template - static CharType TakeBOM(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = Take(is); - return static_cast(c) == 0xFEFFu ? Take(is) : c; - } - - template - static CharType Take(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - unsigned c = static_cast(static_cast(is.Take())) << 8; - c |= static_cast(is.Take()); - return static_cast(c); - } - - template - static void PutBOM(OutputByteStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast(0xFEu)); - os.Put(static_cast(0xFFu)); - } - - template - static void Put(OutputByteStream& os, CharType c) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast((static_cast(c) >> 8) & 0xFFu)); - os.Put(static_cast(static_cast(c) & 0xFFu)); - } +struct UTF16BE : UTF16 +{ template + static CharType TakeBOM(InputByteStream& is) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = Take(is); + return static_cast(c) == 0xFEFFu ? Take(is) : c; + } + + template + static CharType Take(InputByteStream& is) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + unsigned c = static_cast(static_cast(is.Take())) << 8; + c |= static_cast(is.Take()); + return static_cast(c); + } + + template + static void PutBOM(OutputByteStream& os) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(0xFEu)); + os.Put(static_cast(0xFFu)); + } + + template + static void Put(OutputByteStream& os, CharType c) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast((static_cast(c) >> 8) & 0xFFu)); + os.Put(static_cast(static_cast(c) & 0xFFu)); + } }; /////////////////////////////////////////////////////////////////////////////// // UTF32 -//! UTF-32 encoding. +//! UTF-32 encoding. /*! http://en.wikipedia.org/wiki/UTF-32 \tparam CharType Type for storing 32-bit UTF-32 data. Default is unsigned. C++11 may use char32_t instead. \note implements Encoding concept @@ -415,119 +418,119 @@ struct UTF16BE : UTF16 { For streaming, use UTF32LE and UTF32BE, which handle endianness. */ template -struct UTF32 { - typedef CharType Ch; - RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 4); - - enum { supportUnicode = 1 }; - - template - static void Encode(OutputStream& os, unsigned codepoint) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4); - RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); - os.Put(codepoint); - } - - template - static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4); - RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); - PutUnsafe(os, codepoint); - } - - template - static bool Decode(InputStream& is, unsigned* codepoint) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); - Ch c = is.Take(); - *codepoint = c; - return c <= 0x10FFFF; - } - - template - static bool Validate(InputStream& is, OutputStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); - Ch c; - os.Put(c = is.Take()); - return c <= 0x10FFFF; - } +struct UTF32 +{ typedef CharType Ch; + RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 4); + + enum { supportUnicode = 1 }; + + template + static void Encode(OutputStream& os, unsigned codepoint) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4); + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + os.Put(codepoint); + } + + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4); + RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); + PutUnsafe(os, codepoint); + } + + template + static bool Decode(InputStream& is, unsigned* codepoint) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); + Ch c = is.Take(); + *codepoint = c; + return c <= 0x10FFFF; + } + + template + static bool Validate(InputStream& is, OutputStream& os) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); + Ch c; + os.Put(c = is.Take()); + return c <= 0x10FFFF; + } }; //! UTF-32 little endian enocoding. template -struct UTF32LE : UTF32 { - template - static CharType TakeBOM(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = Take(is); - return static_cast(c) == 0x0000FEFFu ? Take(is) : c; - } - - template - static CharType Take(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - unsigned c = static_cast(is.Take()); - c |= static_cast(static_cast(is.Take())) << 8; - c |= static_cast(static_cast(is.Take())) << 16; - c |= static_cast(static_cast(is.Take())) << 24; - return static_cast(c); - } - - template - static void PutBOM(OutputByteStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast(0xFFu)); - os.Put(static_cast(0xFEu)); - os.Put(static_cast(0x00u)); - os.Put(static_cast(0x00u)); - } - - template - static void Put(OutputByteStream& os, CharType c) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast(c & 0xFFu)); - os.Put(static_cast((c >> 8) & 0xFFu)); - os.Put(static_cast((c >> 16) & 0xFFu)); - os.Put(static_cast((c >> 24) & 0xFFu)); - } +struct UTF32LE : UTF32 +{ template + static CharType TakeBOM(InputByteStream& is) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = Take(is); + return static_cast(c) == 0x0000FEFFu ? Take(is) : c; + } + + template + static CharType Take(InputByteStream& is) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + unsigned c = static_cast(is.Take()); + c |= static_cast(static_cast(is.Take())) << 8; + c |= static_cast(static_cast(is.Take())) << 16; + c |= static_cast(static_cast(is.Take())) << 24; + return static_cast(c); + } + + template + static void PutBOM(OutputByteStream& os) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(0xFFu)); + os.Put(static_cast(0xFEu)); + os.Put(static_cast(0x00u)); + os.Put(static_cast(0x00u)); + } + + template + static void Put(OutputByteStream& os, CharType c) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(c & 0xFFu)); + os.Put(static_cast((c >> 8) & 0xFFu)); + os.Put(static_cast((c >> 16) & 0xFFu)); + os.Put(static_cast((c >> 24) & 0xFFu)); + } }; //! UTF-32 big endian encoding. template -struct UTF32BE : UTF32 { - template - static CharType TakeBOM(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - CharType c = Take(is); - return static_cast(c) == 0x0000FEFFu ? Take(is) : c; - } - - template - static CharType Take(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - unsigned c = static_cast(static_cast(is.Take())) << 24; - c |= static_cast(static_cast(is.Take())) << 16; - c |= static_cast(static_cast(is.Take())) << 8; - c |= static_cast(static_cast(is.Take())); - return static_cast(c); - } - - template - static void PutBOM(OutputByteStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast(0x00u)); - os.Put(static_cast(0x00u)); - os.Put(static_cast(0xFEu)); - os.Put(static_cast(0xFFu)); - } - - template - static void Put(OutputByteStream& os, CharType c) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast((c >> 24) & 0xFFu)); - os.Put(static_cast((c >> 16) & 0xFFu)); - os.Put(static_cast((c >> 8) & 0xFFu)); - os.Put(static_cast(c & 0xFFu)); - } +struct UTF32BE : UTF32 +{ template + static CharType TakeBOM(InputByteStream& is) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + CharType c = Take(is); + return static_cast(c) == 0x0000FEFFu ? Take(is) : c; + } + + template + static CharType Take(InputByteStream& is) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + unsigned c = static_cast(static_cast(is.Take())) << 24; + c |= static_cast(static_cast(is.Take())) << 16; + c |= static_cast(static_cast(is.Take())) << 8; + c |= static_cast(static_cast(is.Take())); + return static_cast(c); + } + + template + static void PutBOM(OutputByteStream& os) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(0x00u)); + os.Put(static_cast(0x00u)); + os.Put(static_cast(0xFEu)); + os.Put(static_cast(0xFFu)); + } + + template + static void Put(OutputByteStream& os, CharType c) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast((c >> 24) & 0xFFu)); + os.Put(static_cast((c >> 16) & 0xFFu)); + os.Put(static_cast((c >> 8) & 0xFFu)); + os.Put(static_cast(c & 0xFFu)); + } }; /////////////////////////////////////////////////////////////////////////////// @@ -539,113 +542,113 @@ struct UTF32BE : UTF32 { \note implements Encoding concept */ template -struct ASCII { - typedef CharType Ch; - - enum { supportUnicode = 0 }; - - template - static void Encode(OutputStream& os, unsigned codepoint) { - RAPIDJSON_ASSERT(codepoint <= 0x7F); - os.Put(static_cast(codepoint & 0xFF)); - } - - template - static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { - RAPIDJSON_ASSERT(codepoint <= 0x7F); - PutUnsafe(os, static_cast(codepoint & 0xFF)); - } - - template - static bool Decode(InputStream& is, unsigned* codepoint) { - uint8_t c = static_cast(is.Take()); - *codepoint = c; - return c <= 0X7F; - } - - template - static bool Validate(InputStream& is, OutputStream& os) { - uint8_t c = static_cast(is.Take()); - os.Put(static_cast(c)); - return c <= 0x7F; - } - - template - static CharType TakeBOM(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - uint8_t c = static_cast(Take(is)); - return static_cast(c); - } - - template - static Ch Take(InputByteStream& is) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); - return static_cast(is.Take()); - } - - template - static void PutBOM(OutputByteStream& os) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - (void)os; - } - - template - static void Put(OutputByteStream& os, Ch c) { - RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); - os.Put(static_cast(c)); - } +struct ASCII +{ typedef CharType Ch; + + enum { supportUnicode = 0 }; + + template + static void Encode(OutputStream& os, unsigned codepoint) + { RAPIDJSON_ASSERT(codepoint <= 0x7F); + os.Put(static_cast(codepoint & 0xFF)); + } + + template + static void EncodeUnsafe(OutputStream& os, unsigned codepoint) + { RAPIDJSON_ASSERT(codepoint <= 0x7F); + PutUnsafe(os, static_cast(codepoint & 0xFF)); + } + + template + static bool Decode(InputStream& is, unsigned* codepoint) + { uint8_t c = static_cast(is.Take()); + *codepoint = c; + return c <= 0X7F; + } + + template + static bool Validate(InputStream& is, OutputStream& os) + { uint8_t c = static_cast(is.Take()); + os.Put(static_cast(c)); + return c <= 0x7F; + } + + template + static CharType TakeBOM(InputByteStream& is) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + uint8_t c = static_cast(Take(is)); + return static_cast(c); + } + + template + static Ch Take(InputByteStream& is) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); + return static_cast(is.Take()); + } + + template + static void PutBOM(OutputByteStream& os) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + (void)os; + } + + template + static void Put(OutputByteStream& os, Ch c) + { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); + os.Put(static_cast(c)); + } }; /////////////////////////////////////////////////////////////////////////////// // AutoUTF //! Runtime-specified UTF encoding type of a stream. -enum UTFType { - kUTF8 = 0, //!< UTF-8. - kUTF16LE = 1, //!< UTF-16 little endian. - kUTF16BE = 2, //!< UTF-16 big endian. - kUTF32LE = 3, //!< UTF-32 little endian. - kUTF32BE = 4 //!< UTF-32 big endian. +enum UTFType +{ kUTF8 = 0, //!< UTF-8. + kUTF16LE = 1, //!< UTF-16 little endian. + kUTF16BE = 2, //!< UTF-16 big endian. + kUTF32LE = 3, //!< UTF-32 little endian. + kUTF32BE = 4 //!< UTF-32 big endian. }; //! Dynamically select encoding according to stream's runtime-specified UTF encoding type. /*! \note This class can be used with AutoUTFInputtStream and AutoUTFOutputStream, which provides GetType(). */ template -struct AutoUTF { - typedef CharType Ch; +struct AutoUTF +{ typedef CharType Ch; - enum { supportUnicode = 1 }; + enum { supportUnicode = 1 }; #define RAPIDJSON_ENCODINGS_FUNC(x) UTF8::x, UTF16LE::x, UTF16BE::x, UTF32LE::x, UTF32BE::x - template - RAPIDJSON_FORCEINLINE static void Encode(OutputStream& os, unsigned codepoint) { - typedef void (*EncodeFunc)(OutputStream&, unsigned); - static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Encode) }; - (*f[os.GetType()])(os, codepoint); - } - - template - RAPIDJSON_FORCEINLINE static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { - typedef void (*EncodeFunc)(OutputStream&, unsigned); - static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(EncodeUnsafe) }; - (*f[os.GetType()])(os, codepoint); - } - - template - RAPIDJSON_FORCEINLINE static bool Decode(InputStream& is, unsigned* codepoint) { - typedef bool (*DecodeFunc)(InputStream&, unsigned*); - static const DecodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Decode) }; - return (*f[is.GetType()])(is, codepoint); - } - - template - RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { - typedef bool (*ValidateFunc)(InputStream&, OutputStream&); - static const ValidateFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Validate) }; - return (*f[is.GetType()])(is, os); - } + template + RAPIDJSON_FORCEINLINE static void Encode(OutputStream& os, unsigned codepoint) + { typedef void (*EncodeFunc)(OutputStream&, unsigned); + static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Encode) }; + (*f[os.GetType()])(os, codepoint); + } + + template + RAPIDJSON_FORCEINLINE static void EncodeUnsafe(OutputStream& os, unsigned codepoint) + { typedef void (*EncodeFunc)(OutputStream&, unsigned); + static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(EncodeUnsafe) }; + (*f[os.GetType()])(os, codepoint); + } + + template + RAPIDJSON_FORCEINLINE static bool Decode(InputStream& is, unsigned* codepoint) + { typedef bool (*DecodeFunc)(InputStream&, unsigned*); + static const DecodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Decode) }; + return (*f[is.GetType()])(is, codepoint); + } + + template + RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) + { typedef bool (*ValidateFunc)(InputStream&, OutputStream&); + static const ValidateFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Validate) }; + return (*f[is.GetType()])(is, os); + } #undef RAPIDJSON_ENCODINGS_FUNC }; @@ -655,31 +658,31 @@ struct AutoUTF { //! Encoding conversion. template -struct Transcoder { - //! Take one Unicode codepoint from source encoding, convert it to target encoding and put it to the output stream. - template - RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { - unsigned codepoint; - if (!SourceEncoding::Decode(is, &codepoint)) - return false; - TargetEncoding::Encode(os, codepoint); - return true; - } - - template - RAPIDJSON_FORCEINLINE static bool TranscodeUnsafe(InputStream& is, OutputStream& os) { - unsigned codepoint; - if (!SourceEncoding::Decode(is, &codepoint)) - return false; - TargetEncoding::EncodeUnsafe(os, codepoint); - return true; - } - - //! Validate one Unicode codepoint from an encoded stream. - template - RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { - return Transcode(is, os); // Since source/target encoding is different, must transcode. - } +struct Transcoder +{ //! Take one Unicode codepoint from source encoding, convert it to target encoding and put it to the output stream. + template + RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) + { unsigned codepoint; + if (!SourceEncoding::Decode(is, &codepoint)) + return false; + TargetEncoding::Encode(os, codepoint); + return true; + } + + template + RAPIDJSON_FORCEINLINE static bool TranscodeUnsafe(InputStream& is, OutputStream& os) + { unsigned codepoint; + if (!SourceEncoding::Decode(is, &codepoint)) + return false; + TargetEncoding::EncodeUnsafe(os, codepoint); + return true; + } + + //! Validate one Unicode codepoint from an encoded stream. + template + RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) + { return Transcode(is, os); // Since source/target encoding is different, must transcode. + } }; // Forward declaration. @@ -688,23 +691,23 @@ inline void PutUnsafe(Stream& stream, typename Stream::Ch c); //! Specialization of Transcoder with same source and target encoding. template -struct Transcoder { - template - RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { - os.Put(is.Take()); // Just copy one code unit. This semantic is different from primary template class. - return true; - } - - template - RAPIDJSON_FORCEINLINE static bool TranscodeUnsafe(InputStream& is, OutputStream& os) { - PutUnsafe(os, is.Take()); // Just copy one code unit. This semantic is different from primary template class. - return true; - } - - template - RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { - return Encoding::Validate(is, os); // source/target encoding are the same - } +struct Transcoder +{ template + RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) + { os.Put(is.Take()); // Just copy one code unit. This semantic is different from primary template class. + return true; + } + + template + RAPIDJSON_FORCEINLINE static bool TranscodeUnsafe(InputStream& is, OutputStream& os) + { PutUnsafe(os, is.Take()); // Just copy one code unit. This semantic is different from primary template class. + return true; + } + + template + RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) + { return Encoding::Validate(is, os); // source/target encoding are the same + } }; RAPIDJSON_NAMESPACE_END diff --git a/rapidjson/include/rapidjson/error/en.h b/rapidjson/include/rapidjson/error/en.h index 2db838bff23..8d531bb02c3 100644 --- a/rapidjson/include/rapidjson/error/en.h +++ b/rapidjson/include/rapidjson/error/en.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_ERROR_EN_H_ @@ -20,32 +20,32 @@ #ifdef __clang__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(switch-enum) -RAPIDJSON_DIAG_OFF(covered-switch-default) + RAPIDJSON_DIAG_OFF(covered-switch-default) #endif -RAPIDJSON_NAMESPACE_BEGIN + RAPIDJSON_NAMESPACE_BEGIN //! Maps error code of parsing into error message. -/*! - \ingroup RAPIDJSON_ERRORS - \param parseErrorCode Error code obtained in parsing. - \return the error message. - \note User can make a copy of this function for localization. - Using switch-case is safer for future modification of error codes. -*/ -inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErrorCode) { - switch (parseErrorCode) { - case kParseErrorNone: return RAPIDJSON_ERROR_STRING("No error."); + /*! + \ingroup RAPIDJSON_ERRORS + \param parseErrorCode Error code obtained in parsing. + \return the error message. + \note User can make a copy of this function for localization. + Using switch-case is safer for future modification of error codes. + */ + inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErrorCode) + { switch (parseErrorCode) + { case kParseErrorNone: return RAPIDJSON_ERROR_STRING("No error."); case kParseErrorDocumentEmpty: return RAPIDJSON_ERROR_STRING("The document is empty."); case kParseErrorDocumentRootNotSingular: return RAPIDJSON_ERROR_STRING("The document root must not be followed by other values."); - + case kParseErrorValueInvalid: return RAPIDJSON_ERROR_STRING("Invalid value."); - + case kParseErrorObjectMissName: return RAPIDJSON_ERROR_STRING("Missing a name for object member."); case kParseErrorObjectMissColon: return RAPIDJSON_ERROR_STRING("Missing a colon after a name of object member."); case kParseErrorObjectMissCommaOrCurlyBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or '}' after an object member."); - + case kParseErrorArrayMissCommaOrSquareBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or ']' after an array element."); case kParseErrorStringUnicodeEscapeInvalidHex: return RAPIDJSON_ERROR_STRING("Incorrect hex digit after \\u escape in string."); @@ -62,8 +62,8 @@ inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErro case kParseErrorUnspecificSyntaxError: return RAPIDJSON_ERROR_STRING("Unspecific syntax error."); default: return RAPIDJSON_ERROR_STRING("Unknown error."); - } -} + } + } RAPIDJSON_NAMESPACE_END diff --git a/rapidjson/include/rapidjson/error/error.h b/rapidjson/include/rapidjson/error/error.h index 95cb31a72fe..fa89dbeb630 100644 --- a/rapidjson/include/rapidjson/error/error.h +++ b/rapidjson/include/rapidjson/error/error.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_ERROR_ERROR_H_ @@ -61,32 +61,32 @@ RAPIDJSON_NAMESPACE_BEGIN /*! \ingroup RAPIDJSON_ERRORS \see GenericReader::Parse, GenericReader::GetParseErrorCode */ -enum ParseErrorCode { - kParseErrorNone = 0, //!< No error. +enum ParseErrorCode +{ kParseErrorNone = 0, //!< No error. - kParseErrorDocumentEmpty, //!< The document is empty. - kParseErrorDocumentRootNotSingular, //!< The document root must not follow by other values. + kParseErrorDocumentEmpty, //!< The document is empty. + kParseErrorDocumentRootNotSingular, //!< The document root must not follow by other values. - kParseErrorValueInvalid, //!< Invalid value. + kParseErrorValueInvalid, //!< Invalid value. - kParseErrorObjectMissName, //!< Missing a name for object member. - kParseErrorObjectMissColon, //!< Missing a colon after a name of object member. - kParseErrorObjectMissCommaOrCurlyBracket, //!< Missing a comma or '}' after an object member. + kParseErrorObjectMissName, //!< Missing a name for object member. + kParseErrorObjectMissColon, //!< Missing a colon after a name of object member. + kParseErrorObjectMissCommaOrCurlyBracket, //!< Missing a comma or '}' after an object member. - kParseErrorArrayMissCommaOrSquareBracket, //!< Missing a comma or ']' after an array element. + kParseErrorArrayMissCommaOrSquareBracket, //!< Missing a comma or ']' after an array element. - kParseErrorStringUnicodeEscapeInvalidHex, //!< Incorrect hex digit after \\u escape in string. - kParseErrorStringUnicodeSurrogateInvalid, //!< The surrogate pair in string is invalid. - kParseErrorStringEscapeInvalid, //!< Invalid escape character in string. - kParseErrorStringMissQuotationMark, //!< Missing a closing quotation mark in string. - kParseErrorStringInvalidEncoding, //!< Invalid encoding in string. + kParseErrorStringUnicodeEscapeInvalidHex, //!< Incorrect hex digit after \\u escape in string. + kParseErrorStringUnicodeSurrogateInvalid, //!< The surrogate pair in string is invalid. + kParseErrorStringEscapeInvalid, //!< Invalid escape character in string. + kParseErrorStringMissQuotationMark, //!< Missing a closing quotation mark in string. + kParseErrorStringInvalidEncoding, //!< Invalid encoding in string. - kParseErrorNumberTooBig, //!< Number too big to be stored in double. - kParseErrorNumberMissFraction, //!< Miss fraction part in number. - kParseErrorNumberMissExponent, //!< Miss exponent in number. + kParseErrorNumberTooBig, //!< Number too big to be stored in double. + kParseErrorNumberMissFraction, //!< Miss fraction part in number. + kParseErrorNumberMissExponent, //!< Miss exponent in number. - kParseErrorTermination, //!< Parsing was terminated. - kParseErrorUnspecificSyntaxError //!< Unspecific syntax error. + kParseErrorTermination, //!< Parsing was terminated. + kParseErrorUnspecificSyntaxError //!< Unspecific syntax error. }; //! Result of parsing (wraps ParseErrorCode) @@ -103,35 +103,36 @@ enum ParseErrorCode { \endcode \see GenericReader::Parse, GenericDocument::Parse */ -struct ParseResult { +struct ParseResult +{ public: - //! Default constructor, no error. - ParseResult() : code_(kParseErrorNone), offset_(0) {} - //! Constructor to set an error. - ParseResult(ParseErrorCode code, size_t offset) : code_(code), offset_(offset) {} - - //! Get the error code. - ParseErrorCode Code() const { return code_; } - //! Get the error offset, if \ref IsError(), 0 otherwise. - size_t Offset() const { return offset_; } - - //! Conversion to \c bool, returns \c true, iff !\ref IsError(). - operator bool() const { return !IsError(); } - //! Whether the result is an error. - bool IsError() const { return code_ != kParseErrorNone; } - - bool operator==(const ParseResult& that) const { return code_ == that.code_; } - bool operator==(ParseErrorCode code) const { return code_ == code; } - friend bool operator==(ParseErrorCode code, const ParseResult & err) { return code == err.code_; } - - //! Reset error code. - void Clear() { Set(kParseErrorNone); } - //! Update error code and offset. - void Set(ParseErrorCode code, size_t offset = 0) { code_ = code; offset_ = offset; } + //! Default constructor, no error. + ParseResult() : code_(kParseErrorNone), offset_(0) {} + //! Constructor to set an error. + ParseResult(ParseErrorCode code, size_t offset) : code_(code), offset_(offset) {} + + //! Get the error code. + ParseErrorCode Code() const { return code_; } + //! Get the error offset, if \ref IsError(), 0 otherwise. + size_t Offset() const { return offset_; } + + //! Conversion to \c bool, returns \c true, iff !\ref IsError(). + operator bool() const { return !IsError(); } + //! Whether the result is an error. + bool IsError() const { return code_ != kParseErrorNone; } + + bool operator==(const ParseResult& that) const { return code_ == that.code_; } + bool operator==(ParseErrorCode code) const { return code_ == code; } + friend bool operator==(ParseErrorCode code, const ParseResult & err) { return code == err.code_; } + + //! Reset error code. + void Clear() { Set(kParseErrorNone); } + //! Update error code and offset. + void Set(ParseErrorCode code, size_t offset = 0) { code_ = code; offset_ = offset; } private: - ParseErrorCode code_; - size_t offset_; + ParseErrorCode code_; + size_t offset_; }; //! Function pointer type of GetParseError(). diff --git a/rapidjson/include/rapidjson/filereadstream.h b/rapidjson/include/rapidjson/filereadstream.h index b56ea13b342..df67b6b7157 100644 --- a/rapidjson/include/rapidjson/filereadstream.h +++ b/rapidjson/include/rapidjson/filereadstream.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_FILEREADSTREAM_H_ @@ -31,63 +31,64 @@ RAPIDJSON_NAMESPACE_BEGIN /*! \note implements Stream concept */ -class FileReadStream { +class FileReadStream +{ public: - typedef char Ch; //!< Character type (byte). + typedef char Ch; //!< Character type (byte). - //! Constructor. - /*! - \param fp File pointer opened for read. - \param buffer user-supplied buffer. - \param bufferSize size of buffer in bytes. Must >=4 bytes. - */ - FileReadStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { - RAPIDJSON_ASSERT(fp_ != 0); - RAPIDJSON_ASSERT(bufferSize >= 4); - Read(); - } + //! Constructor. + /*! + \param fp File pointer opened for read. + \param buffer user-supplied buffer. + \param bufferSize size of buffer in bytes. Must >=4 bytes. + */ + FileReadStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) + { RAPIDJSON_ASSERT(fp_ != 0); + RAPIDJSON_ASSERT(bufferSize >= 4); + Read(); + } - Ch Peek() const { return *current_; } - Ch Take() { Ch c = *current_; Read(); return c; } - size_t Tell() const { return count_ + static_cast(current_ - buffer_); } + Ch Peek() const { return *current_; } + Ch Take() { Ch c = *current_; Read(); return c; } + size_t Tell() const { return count_ + static_cast(current_ - buffer_); } - // Not implemented - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + // Not implemented + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - // For encoding detection only. - const Ch* Peek4() const { - return (current_ + 4 <= bufferLast_) ? current_ : 0; - } + // For encoding detection only. + const Ch* Peek4() const + { return (current_ + 4 <= bufferLast_) ? current_ : 0; + } private: - void Read() { - if (current_ < bufferLast_) - ++current_; - else if (!eof_) { - count_ += readCount_; - readCount_ = fread(buffer_, 1, bufferSize_, fp_); - bufferLast_ = buffer_ + readCount_ - 1; - current_ = buffer_; + void Read() + { if (current_ < bufferLast_) + ++current_; + else if (!eof_) + { count_ += readCount_; + readCount_ = fread(buffer_, 1, bufferSize_, fp_); + bufferLast_ = buffer_ + readCount_ - 1; + current_ = buffer_; - if (readCount_ < bufferSize_) { - buffer_[readCount_] = '\0'; - ++bufferLast_; - eof_ = true; - } - } + if (readCount_ < bufferSize_) + { buffer_[readCount_] = '\0'; + ++bufferLast_; + eof_ = true; + } } + } - std::FILE* fp_; - Ch *buffer_; - size_t bufferSize_; - Ch *bufferLast_; - Ch *current_; - size_t readCount_; - size_t count_; //!< Number of characters read - bool eof_; + std::FILE* fp_; + Ch *buffer_; + size_t bufferSize_; + Ch *bufferLast_; + Ch *current_; + size_t readCount_; + size_t count_; //!< Number of characters read + bool eof_; }; RAPIDJSON_NAMESPACE_END diff --git a/rapidjson/include/rapidjson/filewritestream.h b/rapidjson/include/rapidjson/filewritestream.h index 6378dd60ed4..3caa4934e9d 100644 --- a/rapidjson/include/rapidjson/filewritestream.h +++ b/rapidjson/include/rapidjson/filewritestream.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_FILEWRITESTREAM_H_ @@ -29,70 +29,71 @@ RAPIDJSON_NAMESPACE_BEGIN /*! \note implements Stream concept */ -class FileWriteStream { +class FileWriteStream +{ public: - typedef char Ch; //!< Character type. Only support char. - - FileWriteStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferEnd_(buffer + bufferSize), current_(buffer_) { - RAPIDJSON_ASSERT(fp_ != 0); - } - - void Put(char c) { - if (current_ >= bufferEnd_) - Flush(); - - *current_++ = c; + typedef char Ch; //!< Character type. Only support char. + + FileWriteStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferEnd_(buffer + bufferSize), current_(buffer_) + { RAPIDJSON_ASSERT(fp_ != 0); + } + + void Put(char c) + { if (current_ >= bufferEnd_) + Flush(); + + *current_++ = c; + } + + void PutN(char c, size_t n) + { size_t avail = static_cast(bufferEnd_ - current_); + while (n > avail) + { std::memset(current_, c, avail); + current_ += avail; + Flush(); + n -= avail; + avail = static_cast(bufferEnd_ - current_); } - void PutN(char c, size_t n) { - size_t avail = static_cast(bufferEnd_ - current_); - while (n > avail) { - std::memset(current_, c, avail); - current_ += avail; - Flush(); - n -= avail; - avail = static_cast(bufferEnd_ - current_); - } - - if (n > 0) { - std::memset(current_, c, n); - current_ += n; - } + if (n > 0) + { std::memset(current_, c, n); + current_ += n; } - - void Flush() { - if (current_ != buffer_) { - size_t result = fwrite(buffer_, 1, static_cast(current_ - buffer_), fp_); - if (result < static_cast(current_ - buffer_)) { - // failure deliberately ignored at this time - // added to avoid warn_unused_result build errors - } - current_ = buffer_; - } + } + + void Flush() + { if (current_ != buffer_) + { size_t result = fwrite(buffer_, 1, static_cast(current_ - buffer_), fp_); + if (result < static_cast(current_ - buffer_)) + { // failure deliberately ignored at this time + // added to avoid warn_unused_result build errors + } + current_ = buffer_; } + } - // Not implemented - char Peek() const { RAPIDJSON_ASSERT(false); return 0; } - char Take() { RAPIDJSON_ASSERT(false); return 0; } - size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } - char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } + // Not implemented + char Peek() const { RAPIDJSON_ASSERT(false); return 0; } + char Take() { RAPIDJSON_ASSERT(false); return 0; } + size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } + char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } private: - // Prohibit copy constructor & assignment operator. - FileWriteStream(const FileWriteStream&); - FileWriteStream& operator=(const FileWriteStream&); - - std::FILE* fp_; - char *buffer_; - char *bufferEnd_; - char *current_; + // Prohibit copy constructor & assignment operator. + FileWriteStream(const FileWriteStream&); + FileWriteStream& operator=(const FileWriteStream&); + + std::FILE* fp_; + char *buffer_; + char *bufferEnd_; + char *current_; }; //! Implement specialized version of PutN() with memset() for better performance. template<> -inline void PutN(FileWriteStream& stream, char c, size_t n) { - stream.PutN(c, n); +inline void PutN(FileWriteStream& stream, char c, size_t n) +{ stream.PutN(c, n); } RAPIDJSON_NAMESPACE_END diff --git a/rapidjson/include/rapidjson/fwd.h b/rapidjson/include/rapidjson/fwd.h index e8104e841bc..f280a4ea30b 100644 --- a/rapidjson/include/rapidjson/fwd.h +++ b/rapidjson/include/rapidjson/fwd.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_FWD_H_ @@ -101,7 +101,7 @@ class PrettyWriter; // document.h -template +template struct GenericMember; template @@ -110,7 +110,7 @@ class GenericMemberIterator; template struct GenericStringRef; -template +template class GenericValue; typedef GenericValue, MemoryPoolAllocator > Value; @@ -139,9 +139,9 @@ typedef GenericSchemaDocument SchemaDocument; typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProvider; template < - typename SchemaDocumentType, - typename OutputHandler, - typename StateAllocator> + typename SchemaDocumentType, + typename OutputHandler, + typename StateAllocator> class GenericSchemaValidator; typedef GenericSchemaValidator, void>, CrtAllocator> SchemaValidator; diff --git a/rapidjson/include/rapidjson/internal/biginteger.h b/rapidjson/include/rapidjson/internal/biginteger.h index 9d3e88c9981..cbf45c641f6 100644 --- a/rapidjson/include/rapidjson/internal/biginteger.h +++ b/rapidjson/include/rapidjson/internal/biginteger.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_BIGINTEGER_H_ @@ -23,265 +23,267 @@ #endif RAPIDJSON_NAMESPACE_BEGIN -namespace internal { +namespace internal +{ -class BigInteger { +class BigInteger +{ public: - typedef uint64_t Type; - - BigInteger(const BigInteger& rhs) : count_(rhs.count_) { - std::memcpy(digits_, rhs.digits_, count_ * sizeof(Type)); + typedef uint64_t Type; + + BigInteger(const BigInteger& rhs) : count_(rhs.count_) + { std::memcpy(digits_, rhs.digits_, count_ * sizeof(Type)); + } + + explicit BigInteger(uint64_t u) : count_(1) + { digits_[0] = u; + } + + BigInteger(const char* decimals, size_t length) : count_(1) + { RAPIDJSON_ASSERT(length > 0); + digits_[0] = 0; + size_t i = 0; + const size_t kMaxDigitPerIteration = 19; // 2^64 = 18446744073709551616 > 10^19 + while (length >= kMaxDigitPerIteration) + { AppendDecimal64(decimals + i, decimals + i + kMaxDigitPerIteration); + length -= kMaxDigitPerIteration; + i += kMaxDigitPerIteration; } - explicit BigInteger(uint64_t u) : count_(1) { - digits_[0] = u; - } + if (length > 0) + AppendDecimal64(decimals + i, decimals + i + length); + } - BigInteger(const char* decimals, size_t length) : count_(1) { - RAPIDJSON_ASSERT(length > 0); - digits_[0] = 0; - size_t i = 0; - const size_t kMaxDigitPerIteration = 19; // 2^64 = 18446744073709551616 > 10^19 - while (length >= kMaxDigitPerIteration) { - AppendDecimal64(decimals + i, decimals + i + kMaxDigitPerIteration); - length -= kMaxDigitPerIteration; - i += kMaxDigitPerIteration; - } - - if (length > 0) - AppendDecimal64(decimals + i, decimals + i + length); + BigInteger& operator=(const BigInteger &rhs) + { if (this != &rhs) + { count_ = rhs.count_; + std::memcpy(digits_, rhs.digits_, count_ * sizeof(Type)); } - - BigInteger& operator=(const BigInteger &rhs) - { - if (this != &rhs) { - count_ = rhs.count_; - std::memcpy(digits_, rhs.digits_, count_ * sizeof(Type)); - } - return *this; - } - - BigInteger& operator=(uint64_t u) { - digits_[0] = u; - count_ = 1; - return *this; + return *this; + } + + BigInteger& operator=(uint64_t u) + { digits_[0] = u; + count_ = 1; + return *this; + } + + BigInteger& operator+=(uint64_t u) + { Type backup = digits_[0]; + digits_[0] += u; + for (size_t i = 0; i < count_ - 1; i++) + { if (digits_[i] >= backup) + return *this; // no carry + backup = digits_[i + 1]; + digits_[i + 1] += 1; } - BigInteger& operator+=(uint64_t u) { - Type backup = digits_[0]; - digits_[0] += u; - for (size_t i = 0; i < count_ - 1; i++) { - if (digits_[i] >= backup) - return *this; // no carry - backup = digits_[i + 1]; - digits_[i + 1] += 1; - } - - // Last carry - if (digits_[count_ - 1] < backup) - PushBack(1); - - return *this; - } + // Last carry + if (digits_[count_ - 1] < backup) + PushBack(1); - BigInteger& operator*=(uint64_t u) { - if (u == 0) return *this = 0; - if (u == 1) return *this; - if (*this == 1) return *this = u; - - uint64_t k = 0; - for (size_t i = 0; i < count_; i++) { - uint64_t hi; - digits_[i] = MulAdd64(digits_[i], u, k, &hi); - k = hi; - } - - if (k > 0) - PushBack(k); - - return *this; - } + return *this; + } - BigInteger& operator*=(uint32_t u) { - if (u == 0) return *this = 0; - if (u == 1) return *this; - if (*this == 1) return *this = u; - - uint64_t k = 0; - for (size_t i = 0; i < count_; i++) { - const uint64_t c = digits_[i] >> 32; - const uint64_t d = digits_[i] & 0xFFFFFFFF; - const uint64_t uc = u * c; - const uint64_t ud = u * d; - const uint64_t p0 = ud + k; - const uint64_t p1 = uc + (p0 >> 32); - digits_[i] = (p0 & 0xFFFFFFFF) | (p1 << 32); - k = p1 >> 32; - } - - if (k > 0) - PushBack(k); - - return *this; - } + BigInteger& operator*=(uint64_t u) + { if (u == 0) return *this = 0; + if (u == 1) return *this; + if (*this == 1) return *this = u; - BigInteger& operator<<=(size_t shift) { - if (IsZero() || shift == 0) return *this; - - size_t offset = shift / kTypeBit; - size_t interShift = shift % kTypeBit; - RAPIDJSON_ASSERT(count_ + offset <= kCapacity); - - if (interShift == 0) { - std::memmove(&digits_[count_ - 1 + offset], &digits_[count_ - 1], count_ * sizeof(Type)); - count_ += offset; - } - else { - digits_[count_] = 0; - for (size_t i = count_; i > 0; i--) - digits_[i + offset] = (digits_[i] << interShift) | (digits_[i - 1] >> (kTypeBit - interShift)); - digits_[offset] = digits_[0] << interShift; - count_ += offset; - if (digits_[count_]) - count_++; - } - - std::memset(digits_, 0, offset * sizeof(Type)); - - return *this; + uint64_t k = 0; + for (size_t i = 0; i < count_; i++) + { uint64_t hi; + digits_[i] = MulAdd64(digits_[i], u, k, &hi); + k = hi; } - bool operator==(const BigInteger& rhs) const { - return count_ == rhs.count_ && std::memcmp(digits_, rhs.digits_, count_ * sizeof(Type)) == 0; + if (k > 0) + PushBack(k); + + return *this; + } + + BigInteger& operator*=(uint32_t u) + { if (u == 0) return *this = 0; + if (u == 1) return *this; + if (*this == 1) return *this = u; + + uint64_t k = 0; + for (size_t i = 0; i < count_; i++) + { const uint64_t c = digits_[i] >> 32; + const uint64_t d = digits_[i] & 0xFFFFFFFF; + const uint64_t uc = u * c; + const uint64_t ud = u * d; + const uint64_t p0 = ud + k; + const uint64_t p1 = uc + (p0 >> 32); + digits_[i] = (p0 & 0xFFFFFFFF) | (p1 << 32); + k = p1 >> 32; } - bool operator==(const Type rhs) const { - return count_ == 1 && digits_[0] == rhs; - } + if (k > 0) + PushBack(k); + + return *this; + } - BigInteger& MultiplyPow5(unsigned exp) { - static const uint32_t kPow5[12] = { - 5, - 5 * 5, - 5 * 5 * 5, - 5 * 5 * 5 * 5, - 5 * 5 * 5 * 5 * 5, - 5 * 5 * 5 * 5 * 5 * 5, - 5 * 5 * 5 * 5 * 5 * 5 * 5, - 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, - 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, - 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, - 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, - 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 - }; - if (exp == 0) return *this; - for (; exp >= 27; exp -= 27) *this *= RAPIDJSON_UINT64_C2(0X6765C793, 0XFA10079D); // 5^27 - for (; exp >= 13; exp -= 13) *this *= static_cast(1220703125u); // 5^13 - if (exp > 0) *this *= kPow5[exp - 1]; - return *this; + BigInteger& operator<<=(size_t shift) + { if (IsZero() || shift == 0) return *this; + + size_t offset = shift / kTypeBit; + size_t interShift = shift % kTypeBit; + RAPIDJSON_ASSERT(count_ + offset <= kCapacity); + + if (interShift == 0) + { std::memmove(&digits_[count_ - 1 + offset], &digits_[count_ - 1], count_ * sizeof(Type)); + count_ += offset; + } + else + { digits_[count_] = 0; + for (size_t i = count_; i > 0; i--) + digits_[i + offset] = (digits_[i] << interShift) | (digits_[i - 1] >> (kTypeBit - interShift)); + digits_[offset] = digits_[0] << interShift; + count_ += offset; + if (digits_[count_]) + count_++; } - // Compute absolute difference of this and rhs. - // Assume this != rhs - bool Difference(const BigInteger& rhs, BigInteger* out) const { - int cmp = Compare(rhs); - RAPIDJSON_ASSERT(cmp != 0); - const BigInteger *a, *b; // Makes a > b - bool ret; - if (cmp < 0) { a = &rhs; b = this; ret = true; } - else { a = this; b = &rhs; ret = false; } - - Type borrow = 0; - for (size_t i = 0; i < a->count_; i++) { - Type d = a->digits_[i] - borrow; - if (i < b->count_) - d -= b->digits_[i]; - borrow = (d > a->digits_[i]) ? 1 : 0; - out->digits_[i] = d; - if (d != 0) - out->count_ = i + 1; - } - - return ret; + std::memset(digits_, 0, offset * sizeof(Type)); + + return *this; + } + + bool operator==(const BigInteger& rhs) const + { return count_ == rhs.count_ && std::memcmp(digits_, rhs.digits_, count_ * sizeof(Type)) == 0; + } + + bool operator==(const Type rhs) const + { return count_ == 1 && digits_[0] == rhs; + } + + BigInteger& MultiplyPow5(unsigned exp) + { static const uint32_t kPow5[12] = + { 5, + 5 * 5, + 5 * 5 * 5, + 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 + }; + if (exp == 0) return *this; + for (; exp >= 27; exp -= 27) *this *= RAPIDJSON_UINT64_C2(0X6765C793, 0XFA10079D); // 5^27 + for (; exp >= 13; exp -= 13) *this *= static_cast(1220703125u); // 5^13 + if (exp > 0) *this *= kPow5[exp - 1]; + return *this; + } + + // Compute absolute difference of this and rhs. + // Assume this != rhs + bool Difference(const BigInteger& rhs, BigInteger* out) const + { int cmp = Compare(rhs); + RAPIDJSON_ASSERT(cmp != 0); + const BigInteger *a, *b; // Makes a > b + bool ret; + if (cmp < 0) { a = &rhs; b = this; ret = true; } + else { a = this; b = &rhs; ret = false; } + + Type borrow = 0; + for (size_t i = 0; i < a->count_; i++) + { Type d = a->digits_[i] - borrow; + if (i < b->count_) + d -= b->digits_[i]; + borrow = (d > a->digits_[i]) ? 1 : 0; + out->digits_[i] = d; + if (d != 0) + out->count_ = i + 1; } - int Compare(const BigInteger& rhs) const { - if (count_ != rhs.count_) - return count_ < rhs.count_ ? -1 : 1; + return ret; + } - for (size_t i = count_; i-- > 0;) - if (digits_[i] != rhs.digits_[i]) - return digits_[i] < rhs.digits_[i] ? -1 : 1; + int Compare(const BigInteger& rhs) const + { if (count_ != rhs.count_) + return count_ < rhs.count_ ? -1 : 1; - return 0; - } + for (size_t i = count_; i-- > 0;) + if (digits_[i] != rhs.digits_[i]) + return digits_[i] < rhs.digits_[i] ? -1 : 1; - size_t GetCount() const { return count_; } - Type GetDigit(size_t index) const { RAPIDJSON_ASSERT(index < count_); return digits_[index]; } - bool IsZero() const { return count_ == 1 && digits_[0] == 0; } + return 0; + } -private: - void AppendDecimal64(const char* begin, const char* end) { - uint64_t u = ParseUint64(begin, end); - if (IsZero()) - *this = u; - else { - unsigned exp = static_cast(end - begin); - (MultiplyPow5(exp) <<= exp) += u; // *this = *this * 10^exp + u - } - } + size_t GetCount() const { return count_; } + Type GetDigit(size_t index) const { RAPIDJSON_ASSERT(index < count_); return digits_[index]; } + bool IsZero() const { return count_ == 1 && digits_[0] == 0; } - void PushBack(Type digit) { - RAPIDJSON_ASSERT(count_ < kCapacity); - digits_[count_++] = digit; +private: + void AppendDecimal64(const char* begin, const char* end) + { uint64_t u = ParseUint64(begin, end); + if (IsZero()) + *this = u; + else + { unsigned exp = static_cast(end - begin); + (MultiplyPow5(exp) <<= exp) += u; // *this = *this * 10^exp + u } - - static uint64_t ParseUint64(const char* begin, const char* end) { - uint64_t r = 0; - for (const char* p = begin; p != end; ++p) { - RAPIDJSON_ASSERT(*p >= '0' && *p <= '9'); - r = r * 10u + static_cast(*p - '0'); - } - return r; + } + + void PushBack(Type digit) + { RAPIDJSON_ASSERT(count_ < kCapacity); + digits_[count_++] = digit; + } + + static uint64_t ParseUint64(const char* begin, const char* end) + { uint64_t r = 0; + for (const char* p = begin; p != end; ++p) + { RAPIDJSON_ASSERT(*p >= '0' && *p <= '9'); + r = r * 10u + static_cast(*p - '0'); } + return r; + } - // Assume a * b + k < 2^128 - static uint64_t MulAdd64(uint64_t a, uint64_t b, uint64_t k, uint64_t* outHigh) { + // Assume a * b + k < 2^128 + static uint64_t MulAdd64(uint64_t a, uint64_t b, uint64_t k, uint64_t* outHigh) + { #if defined(_MSC_VER) && defined(_M_AMD64) - uint64_t low = _umul128(a, b, outHigh) + k; - if (low < k) - (*outHigh)++; - return low; + uint64_t low = _umul128(a, b, outHigh) + k; + if (low < k) + (*outHigh)++; + return low; #elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) - __extension__ typedef unsigned __int128 uint128; - uint128 p = static_cast(a) * static_cast(b); - p += k; - *outHigh = static_cast(p >> 64); - return static_cast(p); + __extension__ typedef unsigned __int128 uint128; + uint128 p = static_cast(a) * static_cast(b); + p += k; + *outHigh = static_cast(p >> 64); + return static_cast(p); #else - const uint64_t a0 = a & 0xFFFFFFFF, a1 = a >> 32, b0 = b & 0xFFFFFFFF, b1 = b >> 32; - uint64_t x0 = a0 * b0, x1 = a0 * b1, x2 = a1 * b0, x3 = a1 * b1; - x1 += (x0 >> 32); // can't give carry - x1 += x2; - if (x1 < x2) - x3 += (static_cast(1) << 32); - uint64_t lo = (x1 << 32) + (x0 & 0xFFFFFFFF); - uint64_t hi = x3 + (x1 >> 32); - - lo += k; - if (lo < k) - hi++; - *outHigh = hi; - return lo; + const uint64_t a0 = a & 0xFFFFFFFF, a1 = a >> 32, b0 = b & 0xFFFFFFFF, b1 = b >> 32; + uint64_t x0 = a0 * b0, x1 = a0 * b1, x2 = a1 * b0, x3 = a1 * b1; + x1 += (x0 >> 32); // can't give carry + x1 += x2; + if (x1 < x2) + x3 += (static_cast(1) << 32); + uint64_t lo = (x1 << 32) + (x0 & 0xFFFFFFFF); + uint64_t hi = x3 + (x1 >> 32); + + lo += k; + if (lo < k) + hi++; + *outHigh = hi; + return lo; #endif - } + } - static const size_t kBitCount = 3328; // 64bit * 54 > 10^1000 - static const size_t kCapacity = kBitCount / sizeof(Type); - static const size_t kTypeBit = sizeof(Type) * 8; + static const size_t kBitCount = 3328; // 64bit * 54 > 10^1000 + static const size_t kCapacity = kBitCount / sizeof(Type); + static const size_t kTypeBit = sizeof(Type) * 8; - Type digits_[kCapacity]; - size_t count_; + Type digits_[kCapacity]; + size_t count_; }; } // namespace internal diff --git a/rapidjson/include/rapidjson/internal/diyfp.h b/rapidjson/include/rapidjson/internal/diyfp.h index c9fefdc6139..c6ad49ff75f 100644 --- a/rapidjson/include/rapidjson/internal/diyfp.h +++ b/rapidjson/include/rapidjson/internal/diyfp.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. // This is a C++ header-only implementation of Grisu2 algorithm from the publication: @@ -28,7 +28,8 @@ #endif RAPIDJSON_NAMESPACE_BEGIN -namespace internal { +namespace internal +{ #ifdef __GNUC__ RAPIDJSON_DIAG_PUSH @@ -40,208 +41,211 @@ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(padded) #endif -struct DiyFp { - DiyFp() : f(), e() {} +struct DiyFp +{ DiyFp() : f(), e() {} - DiyFp(uint64_t fp, int exp) : f(fp), e(exp) {} + DiyFp(uint64_t fp, int exp) : f(fp), e(exp) {} - explicit DiyFp(double d) { - union { - double d; - uint64_t u64; - } u = { d }; + explicit DiyFp(double d) + { union + { double d; + uint64_t u64; + } u = { d }; - int biased_e = static_cast((u.u64 & kDpExponentMask) >> kDpSignificandSize); - uint64_t significand = (u.u64 & kDpSignificandMask); - if (biased_e != 0) { - f = significand + kDpHiddenBit; - e = biased_e - kDpExponentBias; - } - else { - f = significand; - e = kDpMinExponent + 1; - } + int biased_e = static_cast((u.u64 & kDpExponentMask) >> kDpSignificandSize); + uint64_t significand = (u.u64 & kDpSignificandMask); + if (biased_e != 0) + { f = significand + kDpHiddenBit; + e = biased_e - kDpExponentBias; } - - DiyFp operator-(const DiyFp& rhs) const { - return DiyFp(f - rhs.f, e); + else + { f = significand; + e = kDpMinExponent + 1; } + } + + DiyFp operator-(const DiyFp& rhs) const + { return DiyFp(f - rhs.f, e); + } - DiyFp operator*(const DiyFp& rhs) const { + DiyFp operator*(const DiyFp& rhs) const + { #if defined(_MSC_VER) && defined(_M_AMD64) - uint64_t h; - uint64_t l = _umul128(f, rhs.f, &h); - if (l & (uint64_t(1) << 63)) // rounding - h++; - return DiyFp(h, e + rhs.e + 64); + uint64_t h; + uint64_t l = _umul128(f, rhs.f, &h); + if (l & (uint64_t(1) << 63)) // rounding + h++; + return DiyFp(h, e + rhs.e + 64); #elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) - __extension__ typedef unsigned __int128 uint128; - uint128 p = static_cast(f) * static_cast(rhs.f); - uint64_t h = static_cast(p >> 64); - uint64_t l = static_cast(p); - if (l & (uint64_t(1) << 63)) // rounding - h++; - return DiyFp(h, e + rhs.e + 64); + __extension__ typedef unsigned __int128 uint128; + uint128 p = static_cast(f) * static_cast(rhs.f); + uint64_t h = static_cast(p >> 64); + uint64_t l = static_cast(p); + if (l & (uint64_t(1) << 63)) // rounding + h++; + return DiyFp(h, e + rhs.e + 64); #else - const uint64_t M32 = 0xFFFFFFFF; - const uint64_t a = f >> 32; - const uint64_t b = f & M32; - const uint64_t c = rhs.f >> 32; - const uint64_t d = rhs.f & M32; - const uint64_t ac = a * c; - const uint64_t bc = b * c; - const uint64_t ad = a * d; - const uint64_t bd = b * d; - uint64_t tmp = (bd >> 32) + (ad & M32) + (bc & M32); - tmp += 1U << 31; /// mult_round - return DiyFp(ac + (ad >> 32) + (bc >> 32) + (tmp >> 32), e + rhs.e + 64); + const uint64_t M32 = 0xFFFFFFFF; + const uint64_t a = f >> 32; + const uint64_t b = f & M32; + const uint64_t c = rhs.f >> 32; + const uint64_t d = rhs.f & M32; + const uint64_t ac = a * c; + const uint64_t bc = b * c; + const uint64_t ad = a * d; + const uint64_t bd = b * d; + uint64_t tmp = (bd >> 32) + (ad & M32) + (bc & M32); + tmp += 1U << 31; /// mult_round + return DiyFp(ac + (ad >> 32) + (bc >> 32) + (tmp >> 32), e + rhs.e + 64); #endif - } + } - DiyFp Normalize() const { + DiyFp Normalize() const + { #if defined(_MSC_VER) && defined(_M_AMD64) - unsigned long index; - _BitScanReverse64(&index, f); - return DiyFp(f << (63 - index), e - (63 - index)); + unsigned long index; + _BitScanReverse64(&index, f); + return DiyFp(f << (63 - index), e - (63 - index)); #elif defined(__GNUC__) && __GNUC__ >= 4 - int s = __builtin_clzll(f); - return DiyFp(f << s, e - s); + int s = __builtin_clzll(f); + return DiyFp(f << s, e - s); #else - DiyFp res = *this; - while (!(res.f & (static_cast(1) << 63))) { - res.f <<= 1; - res.e--; - } - return res; -#endif + DiyFp res = *this; + while (!(res.f & (static_cast(1) << 63))) + { res.f <<= 1; + res.e--; } + return res; +#endif + } - DiyFp NormalizeBoundary() const { - DiyFp res = *this; - while (!(res.f & (kDpHiddenBit << 1))) { - res.f <<= 1; - res.e--; - } - res.f <<= (kDiySignificandSize - kDpSignificandSize - 2); - res.e = res.e - (kDiySignificandSize - kDpSignificandSize - 2); - return res; + DiyFp NormalizeBoundary() const + { DiyFp res = *this; + while (!(res.f & (kDpHiddenBit << 1))) + { res.f <<= 1; + res.e--; } + res.f <<= (kDiySignificandSize - kDpSignificandSize - 2); + res.e = res.e - (kDiySignificandSize - kDpSignificandSize - 2); + return res; + } - void NormalizedBoundaries(DiyFp* minus, DiyFp* plus) const { - DiyFp pl = DiyFp((f << 1) + 1, e - 1).NormalizeBoundary(); - DiyFp mi = (f == kDpHiddenBit) ? DiyFp((f << 2) - 1, e - 2) : DiyFp((f << 1) - 1, e - 1); - mi.f <<= mi.e - pl.e; - mi.e = pl.e; - *plus = pl; - *minus = mi; - } + void NormalizedBoundaries(DiyFp* minus, DiyFp* plus) const + { DiyFp pl = DiyFp((f << 1) + 1, e - 1).NormalizeBoundary(); + DiyFp mi = (f == kDpHiddenBit) ? DiyFp((f << 2) - 1, e - 2) : DiyFp((f << 1) - 1, e - 1); + mi.f <<= mi.e - pl.e; + mi.e = pl.e; + *plus = pl; + *minus = mi; + } - double ToDouble() const { - union { - double d; - uint64_t u64; - }u; - const uint64_t be = (e == kDpDenormalExponent && (f & kDpHiddenBit) == 0) ? 0 : - static_cast(e + kDpExponentBias); - u.u64 = (f & kDpSignificandMask) | (be << kDpSignificandSize); - return u.d; - } + double ToDouble() const + { union + { double d; + uint64_t u64; + } u; + const uint64_t be = (e == kDpDenormalExponent && (f & kDpHiddenBit) == 0) ? 0 : + static_cast(e + kDpExponentBias); + u.u64 = (f & kDpSignificandMask) | (be << kDpSignificandSize); + return u.d; + } - static const int kDiySignificandSize = 64; - static const int kDpSignificandSize = 52; - static const int kDpExponentBias = 0x3FF + kDpSignificandSize; - static const int kDpMaxExponent = 0x7FF - kDpExponentBias; - static const int kDpMinExponent = -kDpExponentBias; - static const int kDpDenormalExponent = -kDpExponentBias + 1; - static const uint64_t kDpExponentMask = RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000); - static const uint64_t kDpSignificandMask = RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF); - static const uint64_t kDpHiddenBit = RAPIDJSON_UINT64_C2(0x00100000, 0x00000000); + static const int kDiySignificandSize = 64; + static const int kDpSignificandSize = 52; + static const int kDpExponentBias = 0x3FF + kDpSignificandSize; + static const int kDpMaxExponent = 0x7FF - kDpExponentBias; + static const int kDpMinExponent = -kDpExponentBias; + static const int kDpDenormalExponent = -kDpExponentBias + 1; + static const uint64_t kDpExponentMask = RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000); + static const uint64_t kDpSignificandMask = RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF); + static const uint64_t kDpHiddenBit = RAPIDJSON_UINT64_C2(0x00100000, 0x00000000); - uint64_t f; - int e; + uint64_t f; + int e; }; -inline DiyFp GetCachedPowerByIndex(size_t index) { - // 10^-348, 10^-340, ..., 10^340 - static const uint64_t kCachedPowers_F[] = { - RAPIDJSON_UINT64_C2(0xfa8fd5a0, 0x081c0288), RAPIDJSON_UINT64_C2(0xbaaee17f, 0xa23ebf76), - RAPIDJSON_UINT64_C2(0x8b16fb20, 0x3055ac76), RAPIDJSON_UINT64_C2(0xcf42894a, 0x5dce35ea), - RAPIDJSON_UINT64_C2(0x9a6bb0aa, 0x55653b2d), RAPIDJSON_UINT64_C2(0xe61acf03, 0x3d1a45df), - RAPIDJSON_UINT64_C2(0xab70fe17, 0xc79ac6ca), RAPIDJSON_UINT64_C2(0xff77b1fc, 0xbebcdc4f), - RAPIDJSON_UINT64_C2(0xbe5691ef, 0x416bd60c), RAPIDJSON_UINT64_C2(0x8dd01fad, 0x907ffc3c), - RAPIDJSON_UINT64_C2(0xd3515c28, 0x31559a83), RAPIDJSON_UINT64_C2(0x9d71ac8f, 0xada6c9b5), - RAPIDJSON_UINT64_C2(0xea9c2277, 0x23ee8bcb), RAPIDJSON_UINT64_C2(0xaecc4991, 0x4078536d), - RAPIDJSON_UINT64_C2(0x823c1279, 0x5db6ce57), RAPIDJSON_UINT64_C2(0xc2109436, 0x4dfb5637), - RAPIDJSON_UINT64_C2(0x9096ea6f, 0x3848984f), RAPIDJSON_UINT64_C2(0xd77485cb, 0x25823ac7), - RAPIDJSON_UINT64_C2(0xa086cfcd, 0x97bf97f4), RAPIDJSON_UINT64_C2(0xef340a98, 0x172aace5), - RAPIDJSON_UINT64_C2(0xb23867fb, 0x2a35b28e), RAPIDJSON_UINT64_C2(0x84c8d4df, 0xd2c63f3b), - RAPIDJSON_UINT64_C2(0xc5dd4427, 0x1ad3cdba), RAPIDJSON_UINT64_C2(0x936b9fce, 0xbb25c996), - RAPIDJSON_UINT64_C2(0xdbac6c24, 0x7d62a584), RAPIDJSON_UINT64_C2(0xa3ab6658, 0x0d5fdaf6), - RAPIDJSON_UINT64_C2(0xf3e2f893, 0xdec3f126), RAPIDJSON_UINT64_C2(0xb5b5ada8, 0xaaff80b8), - RAPIDJSON_UINT64_C2(0x87625f05, 0x6c7c4a8b), RAPIDJSON_UINT64_C2(0xc9bcff60, 0x34c13053), - RAPIDJSON_UINT64_C2(0x964e858c, 0x91ba2655), RAPIDJSON_UINT64_C2(0xdff97724, 0x70297ebd), - RAPIDJSON_UINT64_C2(0xa6dfbd9f, 0xb8e5b88f), RAPIDJSON_UINT64_C2(0xf8a95fcf, 0x88747d94), - RAPIDJSON_UINT64_C2(0xb9447093, 0x8fa89bcf), RAPIDJSON_UINT64_C2(0x8a08f0f8, 0xbf0f156b), - RAPIDJSON_UINT64_C2(0xcdb02555, 0x653131b6), RAPIDJSON_UINT64_C2(0x993fe2c6, 0xd07b7fac), - RAPIDJSON_UINT64_C2(0xe45c10c4, 0x2a2b3b06), RAPIDJSON_UINT64_C2(0xaa242499, 0x697392d3), - RAPIDJSON_UINT64_C2(0xfd87b5f2, 0x8300ca0e), RAPIDJSON_UINT64_C2(0xbce50864, 0x92111aeb), - RAPIDJSON_UINT64_C2(0x8cbccc09, 0x6f5088cc), RAPIDJSON_UINT64_C2(0xd1b71758, 0xe219652c), - RAPIDJSON_UINT64_C2(0x9c400000, 0x00000000), RAPIDJSON_UINT64_C2(0xe8d4a510, 0x00000000), - RAPIDJSON_UINT64_C2(0xad78ebc5, 0xac620000), RAPIDJSON_UINT64_C2(0x813f3978, 0xf8940984), - RAPIDJSON_UINT64_C2(0xc097ce7b, 0xc90715b3), RAPIDJSON_UINT64_C2(0x8f7e32ce, 0x7bea5c70), - RAPIDJSON_UINT64_C2(0xd5d238a4, 0xabe98068), RAPIDJSON_UINT64_C2(0x9f4f2726, 0x179a2245), - RAPIDJSON_UINT64_C2(0xed63a231, 0xd4c4fb27), RAPIDJSON_UINT64_C2(0xb0de6538, 0x8cc8ada8), - RAPIDJSON_UINT64_C2(0x83c7088e, 0x1aab65db), RAPIDJSON_UINT64_C2(0xc45d1df9, 0x42711d9a), - RAPIDJSON_UINT64_C2(0x924d692c, 0xa61be758), RAPIDJSON_UINT64_C2(0xda01ee64, 0x1a708dea), - RAPIDJSON_UINT64_C2(0xa26da399, 0x9aef774a), RAPIDJSON_UINT64_C2(0xf209787b, 0xb47d6b85), - RAPIDJSON_UINT64_C2(0xb454e4a1, 0x79dd1877), RAPIDJSON_UINT64_C2(0x865b8692, 0x5b9bc5c2), - RAPIDJSON_UINT64_C2(0xc83553c5, 0xc8965d3d), RAPIDJSON_UINT64_C2(0x952ab45c, 0xfa97a0b3), - RAPIDJSON_UINT64_C2(0xde469fbd, 0x99a05fe3), RAPIDJSON_UINT64_C2(0xa59bc234, 0xdb398c25), - RAPIDJSON_UINT64_C2(0xf6c69a72, 0xa3989f5c), RAPIDJSON_UINT64_C2(0xb7dcbf53, 0x54e9bece), - RAPIDJSON_UINT64_C2(0x88fcf317, 0xf22241e2), RAPIDJSON_UINT64_C2(0xcc20ce9b, 0xd35c78a5), - RAPIDJSON_UINT64_C2(0x98165af3, 0x7b2153df), RAPIDJSON_UINT64_C2(0xe2a0b5dc, 0x971f303a), - RAPIDJSON_UINT64_C2(0xa8d9d153, 0x5ce3b396), RAPIDJSON_UINT64_C2(0xfb9b7cd9, 0xa4a7443c), - RAPIDJSON_UINT64_C2(0xbb764c4c, 0xa7a44410), RAPIDJSON_UINT64_C2(0x8bab8eef, 0xb6409c1a), - RAPIDJSON_UINT64_C2(0xd01fef10, 0xa657842c), RAPIDJSON_UINT64_C2(0x9b10a4e5, 0xe9913129), - RAPIDJSON_UINT64_C2(0xe7109bfb, 0xa19c0c9d), RAPIDJSON_UINT64_C2(0xac2820d9, 0x623bf429), - RAPIDJSON_UINT64_C2(0x80444b5e, 0x7aa7cf85), RAPIDJSON_UINT64_C2(0xbf21e440, 0x03acdd2d), - RAPIDJSON_UINT64_C2(0x8e679c2f, 0x5e44ff8f), RAPIDJSON_UINT64_C2(0xd433179d, 0x9c8cb841), - RAPIDJSON_UINT64_C2(0x9e19db92, 0xb4e31ba9), RAPIDJSON_UINT64_C2(0xeb96bf6e, 0xbadf77d9), - RAPIDJSON_UINT64_C2(0xaf87023b, 0x9bf0ee6b) - }; - static const int16_t kCachedPowers_E[] = { - -1220, -1193, -1166, -1140, -1113, -1087, -1060, -1034, -1007, -980, - -954, -927, -901, -874, -847, -821, -794, -768, -741, -715, - -688, -661, -635, -608, -582, -555, -529, -502, -475, -449, - -422, -396, -369, -343, -316, -289, -263, -236, -210, -183, - -157, -130, -103, -77, -50, -24, 3, 30, 56, 83, - 109, 136, 162, 189, 216, 242, 269, 295, 322, 348, - 375, 402, 428, 455, 481, 508, 534, 561, 588, 614, - 641, 667, 694, 720, 747, 774, 800, 827, 853, 880, - 907, 933, 960, 986, 1013, 1039, 1066 - }; - return DiyFp(kCachedPowers_F[index], kCachedPowers_E[index]); +inline DiyFp GetCachedPowerByIndex(size_t index) +{ // 10^-348, 10^-340, ..., 10^340 + static const uint64_t kCachedPowers_F[] = + { RAPIDJSON_UINT64_C2(0xfa8fd5a0, 0x081c0288), RAPIDJSON_UINT64_C2(0xbaaee17f, 0xa23ebf76), + RAPIDJSON_UINT64_C2(0x8b16fb20, 0x3055ac76), RAPIDJSON_UINT64_C2(0xcf42894a, 0x5dce35ea), + RAPIDJSON_UINT64_C2(0x9a6bb0aa, 0x55653b2d), RAPIDJSON_UINT64_C2(0xe61acf03, 0x3d1a45df), + RAPIDJSON_UINT64_C2(0xab70fe17, 0xc79ac6ca), RAPIDJSON_UINT64_C2(0xff77b1fc, 0xbebcdc4f), + RAPIDJSON_UINT64_C2(0xbe5691ef, 0x416bd60c), RAPIDJSON_UINT64_C2(0x8dd01fad, 0x907ffc3c), + RAPIDJSON_UINT64_C2(0xd3515c28, 0x31559a83), RAPIDJSON_UINT64_C2(0x9d71ac8f, 0xada6c9b5), + RAPIDJSON_UINT64_C2(0xea9c2277, 0x23ee8bcb), RAPIDJSON_UINT64_C2(0xaecc4991, 0x4078536d), + RAPIDJSON_UINT64_C2(0x823c1279, 0x5db6ce57), RAPIDJSON_UINT64_C2(0xc2109436, 0x4dfb5637), + RAPIDJSON_UINT64_C2(0x9096ea6f, 0x3848984f), RAPIDJSON_UINT64_C2(0xd77485cb, 0x25823ac7), + RAPIDJSON_UINT64_C2(0xa086cfcd, 0x97bf97f4), RAPIDJSON_UINT64_C2(0xef340a98, 0x172aace5), + RAPIDJSON_UINT64_C2(0xb23867fb, 0x2a35b28e), RAPIDJSON_UINT64_C2(0x84c8d4df, 0xd2c63f3b), + RAPIDJSON_UINT64_C2(0xc5dd4427, 0x1ad3cdba), RAPIDJSON_UINT64_C2(0x936b9fce, 0xbb25c996), + RAPIDJSON_UINT64_C2(0xdbac6c24, 0x7d62a584), RAPIDJSON_UINT64_C2(0xa3ab6658, 0x0d5fdaf6), + RAPIDJSON_UINT64_C2(0xf3e2f893, 0xdec3f126), RAPIDJSON_UINT64_C2(0xb5b5ada8, 0xaaff80b8), + RAPIDJSON_UINT64_C2(0x87625f05, 0x6c7c4a8b), RAPIDJSON_UINT64_C2(0xc9bcff60, 0x34c13053), + RAPIDJSON_UINT64_C2(0x964e858c, 0x91ba2655), RAPIDJSON_UINT64_C2(0xdff97724, 0x70297ebd), + RAPIDJSON_UINT64_C2(0xa6dfbd9f, 0xb8e5b88f), RAPIDJSON_UINT64_C2(0xf8a95fcf, 0x88747d94), + RAPIDJSON_UINT64_C2(0xb9447093, 0x8fa89bcf), RAPIDJSON_UINT64_C2(0x8a08f0f8, 0xbf0f156b), + RAPIDJSON_UINT64_C2(0xcdb02555, 0x653131b6), RAPIDJSON_UINT64_C2(0x993fe2c6, 0xd07b7fac), + RAPIDJSON_UINT64_C2(0xe45c10c4, 0x2a2b3b06), RAPIDJSON_UINT64_C2(0xaa242499, 0x697392d3), + RAPIDJSON_UINT64_C2(0xfd87b5f2, 0x8300ca0e), RAPIDJSON_UINT64_C2(0xbce50864, 0x92111aeb), + RAPIDJSON_UINT64_C2(0x8cbccc09, 0x6f5088cc), RAPIDJSON_UINT64_C2(0xd1b71758, 0xe219652c), + RAPIDJSON_UINT64_C2(0x9c400000, 0x00000000), RAPIDJSON_UINT64_C2(0xe8d4a510, 0x00000000), + RAPIDJSON_UINT64_C2(0xad78ebc5, 0xac620000), RAPIDJSON_UINT64_C2(0x813f3978, 0xf8940984), + RAPIDJSON_UINT64_C2(0xc097ce7b, 0xc90715b3), RAPIDJSON_UINT64_C2(0x8f7e32ce, 0x7bea5c70), + RAPIDJSON_UINT64_C2(0xd5d238a4, 0xabe98068), RAPIDJSON_UINT64_C2(0x9f4f2726, 0x179a2245), + RAPIDJSON_UINT64_C2(0xed63a231, 0xd4c4fb27), RAPIDJSON_UINT64_C2(0xb0de6538, 0x8cc8ada8), + RAPIDJSON_UINT64_C2(0x83c7088e, 0x1aab65db), RAPIDJSON_UINT64_C2(0xc45d1df9, 0x42711d9a), + RAPIDJSON_UINT64_C2(0x924d692c, 0xa61be758), RAPIDJSON_UINT64_C2(0xda01ee64, 0x1a708dea), + RAPIDJSON_UINT64_C2(0xa26da399, 0x9aef774a), RAPIDJSON_UINT64_C2(0xf209787b, 0xb47d6b85), + RAPIDJSON_UINT64_C2(0xb454e4a1, 0x79dd1877), RAPIDJSON_UINT64_C2(0x865b8692, 0x5b9bc5c2), + RAPIDJSON_UINT64_C2(0xc83553c5, 0xc8965d3d), RAPIDJSON_UINT64_C2(0x952ab45c, 0xfa97a0b3), + RAPIDJSON_UINT64_C2(0xde469fbd, 0x99a05fe3), RAPIDJSON_UINT64_C2(0xa59bc234, 0xdb398c25), + RAPIDJSON_UINT64_C2(0xf6c69a72, 0xa3989f5c), RAPIDJSON_UINT64_C2(0xb7dcbf53, 0x54e9bece), + RAPIDJSON_UINT64_C2(0x88fcf317, 0xf22241e2), RAPIDJSON_UINT64_C2(0xcc20ce9b, 0xd35c78a5), + RAPIDJSON_UINT64_C2(0x98165af3, 0x7b2153df), RAPIDJSON_UINT64_C2(0xe2a0b5dc, 0x971f303a), + RAPIDJSON_UINT64_C2(0xa8d9d153, 0x5ce3b396), RAPIDJSON_UINT64_C2(0xfb9b7cd9, 0xa4a7443c), + RAPIDJSON_UINT64_C2(0xbb764c4c, 0xa7a44410), RAPIDJSON_UINT64_C2(0x8bab8eef, 0xb6409c1a), + RAPIDJSON_UINT64_C2(0xd01fef10, 0xa657842c), RAPIDJSON_UINT64_C2(0x9b10a4e5, 0xe9913129), + RAPIDJSON_UINT64_C2(0xe7109bfb, 0xa19c0c9d), RAPIDJSON_UINT64_C2(0xac2820d9, 0x623bf429), + RAPIDJSON_UINT64_C2(0x80444b5e, 0x7aa7cf85), RAPIDJSON_UINT64_C2(0xbf21e440, 0x03acdd2d), + RAPIDJSON_UINT64_C2(0x8e679c2f, 0x5e44ff8f), RAPIDJSON_UINT64_C2(0xd433179d, 0x9c8cb841), + RAPIDJSON_UINT64_C2(0x9e19db92, 0xb4e31ba9), RAPIDJSON_UINT64_C2(0xeb96bf6e, 0xbadf77d9), + RAPIDJSON_UINT64_C2(0xaf87023b, 0x9bf0ee6b) + }; + static const int16_t kCachedPowers_E[] = + { -1220, -1193, -1166, -1140, -1113, -1087, -1060, -1034, -1007, -980, + -954, -927, -901, -874, -847, -821, -794, -768, -741, -715, + -688, -661, -635, -608, -582, -555, -529, -502, -475, -449, + -422, -396, -369, -343, -316, -289, -263, -236, -210, -183, + -157, -130, -103, -77, -50, -24, 3, 30, 56, 83, + 109, 136, 162, 189, 216, 242, 269, 295, 322, 348, + 375, 402, 428, 455, 481, 508, 534, 561, 588, 614, + 641, 667, 694, 720, 747, 774, 800, 827, 853, 880, + 907, 933, 960, 986, 1013, 1039, 1066 + }; + return DiyFp(kCachedPowers_F[index], kCachedPowers_E[index]); } - -inline DiyFp GetCachedPower(int e, int* K) { - //int k = static_cast(ceil((-61 - e) * 0.30102999566398114)) + 374; - double dk = (-61 - e) * 0.30102999566398114 + 347; // dk must be positive, so can do ceiling in positive - int k = static_cast(dk); - if (dk - k > 0.0) - k++; +inline DiyFp GetCachedPower(int e, int* K) +{ - unsigned index = static_cast((k >> 3) + 1); - *K = -(-348 + static_cast(index << 3)); // decimal exponent no need lookup table + //int k = static_cast(ceil((-61 - e) * 0.30102999566398114)) + 374; + double dk = (-61 - e) * 0.30102999566398114 + 347; // dk must be positive, so can do ceiling in positive + int k = static_cast(dk); + if (dk - k > 0.0) + k++; - return GetCachedPowerByIndex(index); + unsigned index = static_cast((k >> 3) + 1); + *K = -(-348 + static_cast(index << 3)); // decimal exponent no need lookup table + + return GetCachedPowerByIndex(index); } -inline DiyFp GetCachedPower10(int exp, int *outExp) { - unsigned index = (static_cast(exp) + 348u) / 8u; - *outExp = -348 + static_cast(index) * 8; - return GetCachedPowerByIndex(index); - } +inline DiyFp GetCachedPower10(int exp, int *outExp) +{ unsigned index = (static_cast(exp) + 348u) / 8u; + *outExp = -348 + static_cast(index) * 8; + return GetCachedPowerByIndex(index); +} #ifdef __GNUC__ RAPIDJSON_DIAG_POP diff --git a/rapidjson/include/rapidjson/internal/dtoa.h b/rapidjson/include/rapidjson/internal/dtoa.h index 8d6350e626d..0a3c87faa3d 100644 --- a/rapidjson/include/rapidjson/internal/dtoa.h +++ b/rapidjson/include/rapidjson/internal/dtoa.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. // This is a C++ header-only implementation of Grisu2 algorithm from the publication: @@ -24,7 +24,8 @@ #include "ieee754.h" RAPIDJSON_NAMESPACE_BEGIN -namespace internal { +namespace internal +{ #ifdef __GNUC__ RAPIDJSON_DIAG_PUSH @@ -32,207 +33,207 @@ RAPIDJSON_DIAG_OFF(effc++) RAPIDJSON_DIAG_OFF(array-bounds) // some gcc versions generate wrong warnings https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59124 #endif -inline void GrisuRound(char* buffer, int len, uint64_t delta, uint64_t rest, uint64_t ten_kappa, uint64_t wp_w) { - while (rest < wp_w && delta - rest >= ten_kappa && - (rest + ten_kappa < wp_w || /// closer - wp_w - rest > rest + ten_kappa - wp_w)) { - buffer[len - 1]--; - rest += ten_kappa; - } +inline void GrisuRound(char* buffer, int len, uint64_t delta, uint64_t rest, uint64_t ten_kappa, uint64_t wp_w) +{ while (rest < wp_w && delta - rest >= ten_kappa && + (rest + ten_kappa < wp_w || /// closer + wp_w - rest > rest + ten_kappa - wp_w)) + { buffer[len - 1]--; + rest += ten_kappa; + } } -inline unsigned CountDecimalDigit32(uint32_t n) { - // Simple pure C++ implementation was faster than __builtin_clz version in this situation. - if (n < 10) return 1; - if (n < 100) return 2; - if (n < 1000) return 3; - if (n < 10000) return 4; - if (n < 100000) return 5; - if (n < 1000000) return 6; - if (n < 10000000) return 7; - if (n < 100000000) return 8; - // Will not reach 10 digits in DigitGen() - //if (n < 1000000000) return 9; - //return 10; - return 9; +inline unsigned CountDecimalDigit32(uint32_t n) +{ // Simple pure C++ implementation was faster than __builtin_clz version in this situation. + if (n < 10) return 1; + if (n < 100) return 2; + if (n < 1000) return 3; + if (n < 10000) return 4; + if (n < 100000) return 5; + if (n < 1000000) return 6; + if (n < 10000000) return 7; + if (n < 100000000) return 8; + // Will not reach 10 digits in DigitGen() + //if (n < 1000000000) return 9; + //return 10; + return 9; } -inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buffer, int* len, int* K) { - static const uint32_t kPow10[] = { 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000 }; - const DiyFp one(uint64_t(1) << -Mp.e, Mp.e); - const DiyFp wp_w = Mp - W; - uint32_t p1 = static_cast(Mp.f >> -one.e); - uint64_t p2 = Mp.f & (one.f - 1); - unsigned kappa = CountDecimalDigit32(p1); // kappa in [0, 9] - *len = 0; - - while (kappa > 0) { - uint32_t d = 0; - switch (kappa) { - case 9: d = p1 / 100000000; p1 %= 100000000; break; - case 8: d = p1 / 10000000; p1 %= 10000000; break; - case 7: d = p1 / 1000000; p1 %= 1000000; break; - case 6: d = p1 / 100000; p1 %= 100000; break; - case 5: d = p1 / 10000; p1 %= 10000; break; - case 4: d = p1 / 1000; p1 %= 1000; break; - case 3: d = p1 / 100; p1 %= 100; break; - case 2: d = p1 / 10; p1 %= 10; break; - case 1: d = p1; p1 = 0; break; - default:; - } - if (d || *len) - buffer[(*len)++] = static_cast('0' + static_cast(d)); - kappa--; - uint64_t tmp = (static_cast(p1) << -one.e) + p2; - if (tmp <= delta) { - *K += kappa; - GrisuRound(buffer, *len, delta, tmp, static_cast(kPow10[kappa]) << -one.e, wp_w.f); - return; - } +inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buffer, int* len, int* K) +{ static const uint32_t kPow10[] = { 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000 }; + const DiyFp one(uint64_t(1) << -Mp.e, Mp.e); + const DiyFp wp_w = Mp - W; + uint32_t p1 = static_cast(Mp.f >> -one.e); + uint64_t p2 = Mp.f & (one.f - 1); + unsigned kappa = CountDecimalDigit32(p1); // kappa in [0, 9] + *len = 0; + + while (kappa > 0) + { uint32_t d = 0; + switch (kappa) + { case 9: d = p1 / 100000000; p1 %= 100000000; break; + case 8: d = p1 / 10000000; p1 %= 10000000; break; + case 7: d = p1 / 1000000; p1 %= 1000000; break; + case 6: d = p1 / 100000; p1 %= 100000; break; + case 5: d = p1 / 10000; p1 %= 10000; break; + case 4: d = p1 / 1000; p1 %= 1000; break; + case 3: d = p1 / 100; p1 %= 100; break; + case 2: d = p1 / 10; p1 %= 10; break; + case 1: d = p1; p1 = 0; break; + default:; } - - // kappa = 0 - for (;;) { - p2 *= 10; - delta *= 10; - char d = static_cast(p2 >> -one.e); - if (d || *len) - buffer[(*len)++] = static_cast('0' + d); - p2 &= one.f - 1; - kappa--; - if (p2 < delta) { - *K += kappa; - int index = -static_cast(kappa); - GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * (index < 9 ? kPow10[-static_cast(kappa)] : 0)); - return; - } + if (d || *len) + buffer[(*len)++] = static_cast('0' + static_cast(d)); + kappa--; + uint64_t tmp = (static_cast(p1) << -one.e) + p2; + if (tmp <= delta) + { *K += kappa; + GrisuRound(buffer, *len, delta, tmp, static_cast(kPow10[kappa]) << -one.e, wp_w.f); + return; } + } + + // kappa = 0 + for (;;) + { p2 *= 10; + delta *= 10; + char d = static_cast(p2 >> -one.e); + if (d || *len) + buffer[(*len)++] = static_cast('0' + d); + p2 &= one.f - 1; + kappa--; + if (p2 < delta) + { *K += kappa; + int index = -static_cast(kappa); + GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * (index < 9 ? kPow10[-static_cast(kappa)] : 0)); + return; + } + } } -inline void Grisu2(double value, char* buffer, int* length, int* K) { - const DiyFp v(value); - DiyFp w_m, w_p; - v.NormalizedBoundaries(&w_m, &w_p); - - const DiyFp c_mk = GetCachedPower(w_p.e, K); - const DiyFp W = v.Normalize() * c_mk; - DiyFp Wp = w_p * c_mk; - DiyFp Wm = w_m * c_mk; - Wm.f++; - Wp.f--; - DigitGen(W, Wp, Wp.f - Wm.f, buffer, length, K); +inline void Grisu2(double value, char* buffer, int* length, int* K) +{ const DiyFp v(value); + DiyFp w_m, w_p; + v.NormalizedBoundaries(&w_m, &w_p); + + const DiyFp c_mk = GetCachedPower(w_p.e, K); + const DiyFp W = v.Normalize() * c_mk; + DiyFp Wp = w_p * c_mk; + DiyFp Wm = w_m * c_mk; + Wm.f++; + Wp.f--; + DigitGen(W, Wp, Wp.f - Wm.f, buffer, length, K); } -inline char* WriteExponent(int K, char* buffer) { - if (K < 0) { - *buffer++ = '-'; - K = -K; - } - - if (K >= 100) { - *buffer++ = static_cast('0' + static_cast(K / 100)); - K %= 100; - const char* d = GetDigitsLut() + K * 2; - *buffer++ = d[0]; - *buffer++ = d[1]; - } - else if (K >= 10) { - const char* d = GetDigitsLut() + K * 2; - *buffer++ = d[0]; - *buffer++ = d[1]; - } - else - *buffer++ = static_cast('0' + static_cast(K)); - - return buffer; +inline char* WriteExponent(int K, char* buffer) +{ if (K < 0) + { *buffer++ = '-'; + K = -K; + } + + if (K >= 100) + { *buffer++ = static_cast('0' + static_cast(K / 100)); + K %= 100; + const char* d = GetDigitsLut() + K * 2; + *buffer++ = d[0]; + *buffer++ = d[1]; + } + else if (K >= 10) + { const char* d = GetDigitsLut() + K * 2; + *buffer++ = d[0]; + *buffer++ = d[1]; + } + else + *buffer++ = static_cast('0' + static_cast(K)); + + return buffer; } -inline char* Prettify(char* buffer, int length, int k, int maxDecimalPlaces) { - const int kk = length + k; // 10^(kk-1) <= v < 10^kk - - if (0 <= k && kk <= 21) { - // 1234e7 -> 12340000000 - for (int i = length; i < kk; i++) - buffer[i] = '0'; - buffer[kk] = '.'; - buffer[kk + 1] = '0'; - return &buffer[kk + 2]; - } - else if (0 < kk && kk <= 21) { - // 1234e-2 -> 12.34 - std::memmove(&buffer[kk + 1], &buffer[kk], static_cast(length - kk)); - buffer[kk] = '.'; - if (0 > k + maxDecimalPlaces) { - // When maxDecimalPlaces = 2, 1.2345 -> 1.23, 1.102 -> 1.1 - // Remove extra trailing zeros (at least one) after truncation. - for (int i = kk + maxDecimalPlaces; i > kk + 1; i--) - if (buffer[i] != '0') - return &buffer[i + 1]; - return &buffer[kk + 2]; // Reserve one zero - } - else - return &buffer[length + 1]; - } - else if (-6 < kk && kk <= 0) { - // 1234e-6 -> 0.001234 - const int offset = 2 - kk; - std::memmove(&buffer[offset], &buffer[0], static_cast(length)); - buffer[0] = '0'; - buffer[1] = '.'; - for (int i = 2; i < offset; i++) - buffer[i] = '0'; - if (length - kk > maxDecimalPlaces) { - // When maxDecimalPlaces = 2, 0.123 -> 0.12, 0.102 -> 0.1 - // Remove extra trailing zeros (at least one) after truncation. - for (int i = maxDecimalPlaces + 1; i > 2; i--) - if (buffer[i] != '0') - return &buffer[i + 1]; - return &buffer[3]; // Reserve one zero - } - else - return &buffer[length + offset]; - } - else if (kk < -maxDecimalPlaces) { - // Truncate to zero - buffer[0] = '0'; - buffer[1] = '.'; - buffer[2] = '0'; - return &buffer[3]; +inline char* Prettify(char* buffer, int length, int k, int maxDecimalPlaces) +{ const int kk = length + k; // 10^(kk-1) <= v < 10^kk + + if (0 <= k && kk <= 21) + { // 1234e7 -> 12340000000 + for (int i = length; i < kk; i++) + buffer[i] = '0'; + buffer[kk] = '.'; + buffer[kk + 1] = '0'; + return &buffer[kk + 2]; + } + else if (0 < kk && kk <= 21) + { // 1234e-2 -> 12.34 + std::memmove(&buffer[kk + 1], &buffer[kk], static_cast(length - kk)); + buffer[kk] = '.'; + if (0 > k + maxDecimalPlaces) + { // When maxDecimalPlaces = 2, 1.2345 -> 1.23, 1.102 -> 1.1 + // Remove extra trailing zeros (at least one) after truncation. + for (int i = kk + maxDecimalPlaces; i > kk + 1; i--) + if (buffer[i] != '0') + return &buffer[i + 1]; + return &buffer[kk + 2]; // Reserve one zero } - else if (length == 1) { - // 1e30 - buffer[1] = 'e'; - return WriteExponent(kk - 1, &buffer[2]); - } - else { - // 1234e30 -> 1.234e33 - std::memmove(&buffer[2], &buffer[1], static_cast(length - 1)); - buffer[1] = '.'; - buffer[length + 1] = 'e'; - return WriteExponent(kk - 1, &buffer[0 + length + 2]); + else + return &buffer[length + 1]; + } + else if (-6 < kk && kk <= 0) + { // 1234e-6 -> 0.001234 + const int offset = 2 - kk; + std::memmove(&buffer[offset], &buffer[0], static_cast(length)); + buffer[0] = '0'; + buffer[1] = '.'; + for (int i = 2; i < offset; i++) + buffer[i] = '0'; + if (length - kk > maxDecimalPlaces) + { // When maxDecimalPlaces = 2, 0.123 -> 0.12, 0.102 -> 0.1 + // Remove extra trailing zeros (at least one) after truncation. + for (int i = maxDecimalPlaces + 1; i > 2; i--) + if (buffer[i] != '0') + return &buffer[i + 1]; + return &buffer[3]; // Reserve one zero } + else + return &buffer[length + offset]; + } + else if (kk < -maxDecimalPlaces) + { // Truncate to zero + buffer[0] = '0'; + buffer[1] = '.'; + buffer[2] = '0'; + return &buffer[3]; + } + else if (length == 1) + { // 1e30 + buffer[1] = 'e'; + return WriteExponent(kk - 1, &buffer[2]); + } + else + { // 1234e30 -> 1.234e33 + std::memmove(&buffer[2], &buffer[1], static_cast(length - 1)); + buffer[1] = '.'; + buffer[length + 1] = 'e'; + return WriteExponent(kk - 1, &buffer[0 + length + 2]); + } } -inline char* dtoa(double value, char* buffer, int maxDecimalPlaces = 324) { - RAPIDJSON_ASSERT(maxDecimalPlaces >= 1); - Double d(value); - if (d.IsZero()) { - if (d.Sign()) - *buffer++ = '-'; // -0.0, Issue #289 - buffer[0] = '0'; - buffer[1] = '.'; - buffer[2] = '0'; - return &buffer[3]; - } - else { - if (value < 0) { - *buffer++ = '-'; - value = -value; - } - int length, K; - Grisu2(value, buffer, &length, &K); - return Prettify(buffer, length, K, maxDecimalPlaces); +inline char* dtoa(double value, char* buffer, int maxDecimalPlaces = 324) +{ RAPIDJSON_ASSERT(maxDecimalPlaces >= 1); + Double d(value); + if (d.IsZero()) + { if (d.Sign()) + *buffer++ = '-'; // -0.0, Issue #289 + buffer[0] = '0'; + buffer[1] = '.'; + buffer[2] = '0'; + return &buffer[3]; + } + else + { if (value < 0) + { *buffer++ = '-'; + value = -value; } + int length, K; + Grisu2(value, buffer, &length, &K); + return Prettify(buffer, length, K, maxDecimalPlaces); + } } #ifdef __GNUC__ diff --git a/rapidjson/include/rapidjson/internal/ieee754.h b/rapidjson/include/rapidjson/internal/ieee754.h index 82bb0b99e5c..b437e3d9129 100644 --- a/rapidjson/include/rapidjson/internal/ieee754.h +++ b/rapidjson/include/rapidjson/internal/ieee754.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_IEEE754_ @@ -18,58 +18,60 @@ #include "../rapidjson.h" RAPIDJSON_NAMESPACE_BEGIN -namespace internal { +namespace internal +{ -class Double { +class Double +{ public: - Double() {} - Double(double d) : d_(d) {} - Double(uint64_t u) : u_(u) {} + Double() {} + Double(double d) : d_(d) {} + Double(uint64_t u) : u_(u) {} - double Value() const { return d_; } - uint64_t Uint64Value() const { return u_; } + double Value() const { return d_; } + uint64_t Uint64Value() const { return u_; } - double NextPositiveDouble() const { - RAPIDJSON_ASSERT(!Sign()); - return Double(u_ + 1).Value(); - } + double NextPositiveDouble() const + { RAPIDJSON_ASSERT(!Sign()); + return Double(u_ + 1).Value(); + } - bool Sign() const { return (u_ & kSignMask) != 0; } - uint64_t Significand() const { return u_ & kSignificandMask; } - int Exponent() const { return static_cast(((u_ & kExponentMask) >> kSignificandSize) - kExponentBias); } + bool Sign() const { return (u_ & kSignMask) != 0; } + uint64_t Significand() const { return u_ & kSignificandMask; } + int Exponent() const { return static_cast(((u_ & kExponentMask) >> kSignificandSize) - kExponentBias); } - bool IsNan() const { return (u_ & kExponentMask) == kExponentMask && Significand() != 0; } - bool IsInf() const { return (u_ & kExponentMask) == kExponentMask && Significand() == 0; } - bool IsNanOrInf() const { return (u_ & kExponentMask) == kExponentMask; } - bool IsNormal() const { return (u_ & kExponentMask) != 0 || Significand() == 0; } - bool IsZero() const { return (u_ & (kExponentMask | kSignificandMask)) == 0; } + bool IsNan() const { return (u_ & kExponentMask) == kExponentMask && Significand() != 0; } + bool IsInf() const { return (u_ & kExponentMask) == kExponentMask && Significand() == 0; } + bool IsNanOrInf() const { return (u_ & kExponentMask) == kExponentMask; } + bool IsNormal() const { return (u_ & kExponentMask) != 0 || Significand() == 0; } + bool IsZero() const { return (u_ & (kExponentMask | kSignificandMask)) == 0; } - uint64_t IntegerSignificand() const { return IsNormal() ? Significand() | kHiddenBit : Significand(); } - int IntegerExponent() const { return (IsNormal() ? Exponent() : kDenormalExponent) - kSignificandSize; } - uint64_t ToBias() const { return (u_ & kSignMask) ? ~u_ + 1 : u_ | kSignMask; } + uint64_t IntegerSignificand() const { return IsNormal() ? Significand() | kHiddenBit : Significand(); } + int IntegerExponent() const { return (IsNormal() ? Exponent() : kDenormalExponent) - kSignificandSize; } + uint64_t ToBias() const { return (u_ & kSignMask) ? ~u_ + 1 : u_ | kSignMask; } - static unsigned EffectiveSignificandSize(int order) { - if (order >= -1021) - return 53; - else if (order <= -1074) - return 0; - else - return static_cast(order) + 1074; - } + static unsigned EffectiveSignificandSize(int order) + { if (order >= -1021) + return 53; + else if (order <= -1074) + return 0; + else + return static_cast(order) + 1074; + } private: - static const int kSignificandSize = 52; - static const int kExponentBias = 0x3FF; - static const int kDenormalExponent = 1 - kExponentBias; - static const uint64_t kSignMask = RAPIDJSON_UINT64_C2(0x80000000, 0x00000000); - static const uint64_t kExponentMask = RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000); - static const uint64_t kSignificandMask = RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF); - static const uint64_t kHiddenBit = RAPIDJSON_UINT64_C2(0x00100000, 0x00000000); + static const int kSignificandSize = 52; + static const int kExponentBias = 0x3FF; + static const int kDenormalExponent = 1 - kExponentBias; + static const uint64_t kSignMask = RAPIDJSON_UINT64_C2(0x80000000, 0x00000000); + static const uint64_t kExponentMask = RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000); + static const uint64_t kSignificandMask = RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF); + static const uint64_t kHiddenBit = RAPIDJSON_UINT64_C2(0x00100000, 0x00000000); - union { - double d_; - uint64_t u_; - }; + union + { double d_; + uint64_t u_; + }; }; } // namespace internal diff --git a/rapidjson/include/rapidjson/internal/itoa.h b/rapidjson/include/rapidjson/internal/itoa.h index 01a4e7e72d7..51e579a2e3c 100644 --- a/rapidjson/include/rapidjson/internal/itoa.h +++ b/rapidjson/include/rapidjson/internal/itoa.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_ITOA_ @@ -18,284 +18,285 @@ #include "../rapidjson.h" RAPIDJSON_NAMESPACE_BEGIN -namespace internal { - -inline const char* GetDigitsLut() { - static const char cDigitsLut[200] = { - '0','0','0','1','0','2','0','3','0','4','0','5','0','6','0','7','0','8','0','9', - '1','0','1','1','1','2','1','3','1','4','1','5','1','6','1','7','1','8','1','9', - '2','0','2','1','2','2','2','3','2','4','2','5','2','6','2','7','2','8','2','9', - '3','0','3','1','3','2','3','3','3','4','3','5','3','6','3','7','3','8','3','9', - '4','0','4','1','4','2','4','3','4','4','4','5','4','6','4','7','4','8','4','9', - '5','0','5','1','5','2','5','3','5','4','5','5','5','6','5','7','5','8','5','9', - '6','0','6','1','6','2','6','3','6','4','6','5','6','6','6','7','6','8','6','9', - '7','0','7','1','7','2','7','3','7','4','7','5','7','6','7','7','7','8','7','9', - '8','0','8','1','8','2','8','3','8','4','8','5','8','6','8','7','8','8','8','9', - '9','0','9','1','9','2','9','3','9','4','9','5','9','6','9','7','9','8','9','9' - }; - return cDigitsLut; +namespace internal +{ + +inline const char* GetDigitsLut() +{ static const char cDigitsLut[200] = + { '0','0','0','1','0','2','0','3','0','4','0','5','0','6','0','7','0','8','0','9', + '1','0','1','1','1','2','1','3','1','4','1','5','1','6','1','7','1','8','1','9', + '2','0','2','1','2','2','2','3','2','4','2','5','2','6','2','7','2','8','2','9', + '3','0','3','1','3','2','3','3','3','4','3','5','3','6','3','7','3','8','3','9', + '4','0','4','1','4','2','4','3','4','4','4','5','4','6','4','7','4','8','4','9', + '5','0','5','1','5','2','5','3','5','4','5','5','5','6','5','7','5','8','5','9', + '6','0','6','1','6','2','6','3','6','4','6','5','6','6','6','7','6','8','6','9', + '7','0','7','1','7','2','7','3','7','4','7','5','7','6','7','7','7','8','7','9', + '8','0','8','1','8','2','8','3','8','4','8','5','8','6','8','7','8','8','8','9', + '9','0','9','1','9','2','9','3','9','4','9','5','9','6','9','7','9','8','9','9' + }; + return cDigitsLut; } -inline char* u32toa(uint32_t value, char* buffer) { - const char* cDigitsLut = GetDigitsLut(); - - if (value < 10000) { - const uint32_t d1 = (value / 100) << 1; - const uint32_t d2 = (value % 100) << 1; - - if (value >= 1000) - *buffer++ = cDigitsLut[d1]; - if (value >= 100) - *buffer++ = cDigitsLut[d1 + 1]; - if (value >= 10) - *buffer++ = cDigitsLut[d2]; - *buffer++ = cDigitsLut[d2 + 1]; - } - else if (value < 100000000) { - // value = bbbbcccc - const uint32_t b = value / 10000; - const uint32_t c = value % 10000; - - const uint32_t d1 = (b / 100) << 1; - const uint32_t d2 = (b % 100) << 1; - - const uint32_t d3 = (c / 100) << 1; - const uint32_t d4 = (c % 100) << 1; - - if (value >= 10000000) - *buffer++ = cDigitsLut[d1]; - if (value >= 1000000) - *buffer++ = cDigitsLut[d1 + 1]; - if (value >= 100000) - *buffer++ = cDigitsLut[d2]; - *buffer++ = cDigitsLut[d2 + 1]; - - *buffer++ = cDigitsLut[d3]; - *buffer++ = cDigitsLut[d3 + 1]; - *buffer++ = cDigitsLut[d4]; - *buffer++ = cDigitsLut[d4 + 1]; +inline char* u32toa(uint32_t value, char* buffer) +{ const char* cDigitsLut = GetDigitsLut(); + + if (value < 10000) + { const uint32_t d1 = (value / 100) << 1; + const uint32_t d2 = (value % 100) << 1; + + if (value >= 1000) + *buffer++ = cDigitsLut[d1]; + if (value >= 100) + *buffer++ = cDigitsLut[d1 + 1]; + if (value >= 10) + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + } + else if (value < 100000000) + { // value = bbbbcccc + const uint32_t b = value / 10000; + const uint32_t c = value % 10000; + + const uint32_t d1 = (b / 100) << 1; + const uint32_t d2 = (b % 100) << 1; + + const uint32_t d3 = (c / 100) << 1; + const uint32_t d4 = (c % 100) << 1; + + if (value >= 10000000) + *buffer++ = cDigitsLut[d1]; + if (value >= 1000000) + *buffer++ = cDigitsLut[d1 + 1]; + if (value >= 100000) + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + + *buffer++ = cDigitsLut[d3]; + *buffer++ = cDigitsLut[d3 + 1]; + *buffer++ = cDigitsLut[d4]; + *buffer++ = cDigitsLut[d4 + 1]; + } + else + { // value = aabbbbcccc in decimal + + const uint32_t a = value / 100000000; // 1 to 42 + value %= 100000000; + + if (a >= 10) + { const unsigned i = a << 1; + *buffer++ = cDigitsLut[i]; + *buffer++ = cDigitsLut[i + 1]; } - else { - // value = aabbbbcccc in decimal - - const uint32_t a = value / 100000000; // 1 to 42 - value %= 100000000; - - if (a >= 10) { - const unsigned i = a << 1; - *buffer++ = cDigitsLut[i]; - *buffer++ = cDigitsLut[i + 1]; - } - else - *buffer++ = static_cast('0' + static_cast(a)); - - const uint32_t b = value / 10000; // 0 to 9999 - const uint32_t c = value % 10000; // 0 to 9999 - - const uint32_t d1 = (b / 100) << 1; - const uint32_t d2 = (b % 100) << 1; - - const uint32_t d3 = (c / 100) << 1; - const uint32_t d4 = (c % 100) << 1; - + else + *buffer++ = static_cast('0' + static_cast(a)); + + const uint32_t b = value / 10000; // 0 to 9999 + const uint32_t c = value % 10000; // 0 to 9999 + + const uint32_t d1 = (b / 100) << 1; + const uint32_t d2 = (b % 100) << 1; + + const uint32_t d3 = (c / 100) << 1; + const uint32_t d4 = (c % 100) << 1; + + *buffer++ = cDigitsLut[d1]; + *buffer++ = cDigitsLut[d1 + 1]; + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + *buffer++ = cDigitsLut[d3]; + *buffer++ = cDigitsLut[d3 + 1]; + *buffer++ = cDigitsLut[d4]; + *buffer++ = cDigitsLut[d4 + 1]; + } + return buffer; +} + +inline char* i32toa(int32_t value, char* buffer) +{ uint32_t u = static_cast(value); + if (value < 0) + { *buffer++ = '-'; + u = ~u + 1; + } + + return u32toa(u, buffer); +} + +inline char* u64toa(uint64_t value, char* buffer) +{ const char* cDigitsLut = GetDigitsLut(); + const uint64_t kTen8 = 100000000; + const uint64_t kTen9 = kTen8 * 10; + const uint64_t kTen10 = kTen8 * 100; + const uint64_t kTen11 = kTen8 * 1000; + const uint64_t kTen12 = kTen8 * 10000; + const uint64_t kTen13 = kTen8 * 100000; + const uint64_t kTen14 = kTen8 * 1000000; + const uint64_t kTen15 = kTen8 * 10000000; + const uint64_t kTen16 = kTen8 * kTen8; + + if (value < kTen8) + { uint32_t v = static_cast(value); + if (v < 10000) + { const uint32_t d1 = (v / 100) << 1; + const uint32_t d2 = (v % 100) << 1; + + if (v >= 1000) *buffer++ = cDigitsLut[d1]; + if (v >= 100) *buffer++ = cDigitsLut[d1 + 1]; + if (v >= 10) *buffer++ = cDigitsLut[d2]; - *buffer++ = cDigitsLut[d2 + 1]; - *buffer++ = cDigitsLut[d3]; - *buffer++ = cDigitsLut[d3 + 1]; - *buffer++ = cDigitsLut[d4]; - *buffer++ = cDigitsLut[d4 + 1]; + *buffer++ = cDigitsLut[d2 + 1]; } - return buffer; -} + else + { // value = bbbbcccc + const uint32_t b = v / 10000; + const uint32_t c = v % 10000; -inline char* i32toa(int32_t value, char* buffer) { - uint32_t u = static_cast(value); - if (value < 0) { - *buffer++ = '-'; - u = ~u + 1; - } + const uint32_t d1 = (b / 100) << 1; + const uint32_t d2 = (b % 100) << 1; - return u32toa(u, buffer); -} + const uint32_t d3 = (c / 100) << 1; + const uint32_t d4 = (c % 100) << 1; -inline char* u64toa(uint64_t value, char* buffer) { - const char* cDigitsLut = GetDigitsLut(); - const uint64_t kTen8 = 100000000; - const uint64_t kTen9 = kTen8 * 10; - const uint64_t kTen10 = kTen8 * 100; - const uint64_t kTen11 = kTen8 * 1000; - const uint64_t kTen12 = kTen8 * 10000; - const uint64_t kTen13 = kTen8 * 100000; - const uint64_t kTen14 = kTen8 * 1000000; - const uint64_t kTen15 = kTen8 * 10000000; - const uint64_t kTen16 = kTen8 * kTen8; - - if (value < kTen8) { - uint32_t v = static_cast(value); - if (v < 10000) { - const uint32_t d1 = (v / 100) << 1; - const uint32_t d2 = (v % 100) << 1; - - if (v >= 1000) - *buffer++ = cDigitsLut[d1]; - if (v >= 100) - *buffer++ = cDigitsLut[d1 + 1]; - if (v >= 10) - *buffer++ = cDigitsLut[d2]; - *buffer++ = cDigitsLut[d2 + 1]; - } - else { - // value = bbbbcccc - const uint32_t b = v / 10000; - const uint32_t c = v % 10000; - - const uint32_t d1 = (b / 100) << 1; - const uint32_t d2 = (b % 100) << 1; - - const uint32_t d3 = (c / 100) << 1; - const uint32_t d4 = (c % 100) << 1; - - if (value >= 10000000) - *buffer++ = cDigitsLut[d1]; - if (value >= 1000000) - *buffer++ = cDigitsLut[d1 + 1]; - if (value >= 100000) - *buffer++ = cDigitsLut[d2]; - *buffer++ = cDigitsLut[d2 + 1]; - - *buffer++ = cDigitsLut[d3]; - *buffer++ = cDigitsLut[d3 + 1]; - *buffer++ = cDigitsLut[d4]; - *buffer++ = cDigitsLut[d4 + 1]; - } - } - else if (value < kTen16) { - const uint32_t v0 = static_cast(value / kTen8); - const uint32_t v1 = static_cast(value % kTen8); - - const uint32_t b0 = v0 / 10000; - const uint32_t c0 = v0 % 10000; - - const uint32_t d1 = (b0 / 100) << 1; - const uint32_t d2 = (b0 % 100) << 1; - - const uint32_t d3 = (c0 / 100) << 1; - const uint32_t d4 = (c0 % 100) << 1; - - const uint32_t b1 = v1 / 10000; - const uint32_t c1 = v1 % 10000; - - const uint32_t d5 = (b1 / 100) << 1; - const uint32_t d6 = (b1 % 100) << 1; - - const uint32_t d7 = (c1 / 100) << 1; - const uint32_t d8 = (c1 % 100) << 1; - - if (value >= kTen15) - *buffer++ = cDigitsLut[d1]; - if (value >= kTen14) - *buffer++ = cDigitsLut[d1 + 1]; - if (value >= kTen13) - *buffer++ = cDigitsLut[d2]; - if (value >= kTen12) - *buffer++ = cDigitsLut[d2 + 1]; - if (value >= kTen11) - *buffer++ = cDigitsLut[d3]; - if (value >= kTen10) - *buffer++ = cDigitsLut[d3 + 1]; - if (value >= kTen9) - *buffer++ = cDigitsLut[d4]; - if (value >= kTen8) - *buffer++ = cDigitsLut[d4 + 1]; - - *buffer++ = cDigitsLut[d5]; - *buffer++ = cDigitsLut[d5 + 1]; - *buffer++ = cDigitsLut[d6]; - *buffer++ = cDigitsLut[d6 + 1]; - *buffer++ = cDigitsLut[d7]; - *buffer++ = cDigitsLut[d7 + 1]; - *buffer++ = cDigitsLut[d8]; - *buffer++ = cDigitsLut[d8 + 1]; - } - else { - const uint32_t a = static_cast(value / kTen16); // 1 to 1844 - value %= kTen16; - - if (a < 10) - *buffer++ = static_cast('0' + static_cast(a)); - else if (a < 100) { - const uint32_t i = a << 1; - *buffer++ = cDigitsLut[i]; - *buffer++ = cDigitsLut[i + 1]; - } - else if (a < 1000) { - *buffer++ = static_cast('0' + static_cast(a / 100)); - - const uint32_t i = (a % 100) << 1; - *buffer++ = cDigitsLut[i]; - *buffer++ = cDigitsLut[i + 1]; - } - else { - const uint32_t i = (a / 100) << 1; - const uint32_t j = (a % 100) << 1; - *buffer++ = cDigitsLut[i]; - *buffer++ = cDigitsLut[i + 1]; - *buffer++ = cDigitsLut[j]; - *buffer++ = cDigitsLut[j + 1]; - } - - const uint32_t v0 = static_cast(value / kTen8); - const uint32_t v1 = static_cast(value % kTen8); - - const uint32_t b0 = v0 / 10000; - const uint32_t c0 = v0 % 10000; - - const uint32_t d1 = (b0 / 100) << 1; - const uint32_t d2 = (b0 % 100) << 1; - - const uint32_t d3 = (c0 / 100) << 1; - const uint32_t d4 = (c0 % 100) << 1; - - const uint32_t b1 = v1 / 10000; - const uint32_t c1 = v1 % 10000; - - const uint32_t d5 = (b1 / 100) << 1; - const uint32_t d6 = (b1 % 100) << 1; - - const uint32_t d7 = (c1 / 100) << 1; - const uint32_t d8 = (c1 % 100) << 1; - + if (value >= 10000000) *buffer++ = cDigitsLut[d1]; + if (value >= 1000000) *buffer++ = cDigitsLut[d1 + 1]; + if (value >= 100000) *buffer++ = cDigitsLut[d2]; - *buffer++ = cDigitsLut[d2 + 1]; - *buffer++ = cDigitsLut[d3]; - *buffer++ = cDigitsLut[d3 + 1]; - *buffer++ = cDigitsLut[d4]; - *buffer++ = cDigitsLut[d4 + 1]; - *buffer++ = cDigitsLut[d5]; - *buffer++ = cDigitsLut[d5 + 1]; - *buffer++ = cDigitsLut[d6]; - *buffer++ = cDigitsLut[d6 + 1]; - *buffer++ = cDigitsLut[d7]; - *buffer++ = cDigitsLut[d7 + 1]; - *buffer++ = cDigitsLut[d8]; - *buffer++ = cDigitsLut[d8 + 1]; + *buffer++ = cDigitsLut[d2 + 1]; + + *buffer++ = cDigitsLut[d3]; + *buffer++ = cDigitsLut[d3 + 1]; + *buffer++ = cDigitsLut[d4]; + *buffer++ = cDigitsLut[d4 + 1]; } - - return buffer; -} + } + else if (value < kTen16) + { const uint32_t v0 = static_cast(value / kTen8); + const uint32_t v1 = static_cast(value % kTen8); + + const uint32_t b0 = v0 / 10000; + const uint32_t c0 = v0 % 10000; + + const uint32_t d1 = (b0 / 100) << 1; + const uint32_t d2 = (b0 % 100) << 1; + + const uint32_t d3 = (c0 / 100) << 1; + const uint32_t d4 = (c0 % 100) << 1; + + const uint32_t b1 = v1 / 10000; + const uint32_t c1 = v1 % 10000; + + const uint32_t d5 = (b1 / 100) << 1; + const uint32_t d6 = (b1 % 100) << 1; + + const uint32_t d7 = (c1 / 100) << 1; + const uint32_t d8 = (c1 % 100) << 1; + + if (value >= kTen15) + *buffer++ = cDigitsLut[d1]; + if (value >= kTen14) + *buffer++ = cDigitsLut[d1 + 1]; + if (value >= kTen13) + *buffer++ = cDigitsLut[d2]; + if (value >= kTen12) + *buffer++ = cDigitsLut[d2 + 1]; + if (value >= kTen11) + *buffer++ = cDigitsLut[d3]; + if (value >= kTen10) + *buffer++ = cDigitsLut[d3 + 1]; + if (value >= kTen9) + *buffer++ = cDigitsLut[d4]; + if (value >= kTen8) + *buffer++ = cDigitsLut[d4 + 1]; + + *buffer++ = cDigitsLut[d5]; + *buffer++ = cDigitsLut[d5 + 1]; + *buffer++ = cDigitsLut[d6]; + *buffer++ = cDigitsLut[d6 + 1]; + *buffer++ = cDigitsLut[d7]; + *buffer++ = cDigitsLut[d7 + 1]; + *buffer++ = cDigitsLut[d8]; + *buffer++ = cDigitsLut[d8 + 1]; + } + else + { const uint32_t a = static_cast(value / kTen16); // 1 to 1844 + value %= kTen16; + + if (a < 10) + *buffer++ = static_cast('0' + static_cast(a)); + else if (a < 100) + { const uint32_t i = a << 1; + *buffer++ = cDigitsLut[i]; + *buffer++ = cDigitsLut[i + 1]; + } + else if (a < 1000) + { *buffer++ = static_cast('0' + static_cast(a / 100)); -inline char* i64toa(int64_t value, char* buffer) { - uint64_t u = static_cast(value); - if (value < 0) { - *buffer++ = '-'; - u = ~u + 1; + const uint32_t i = (a % 100) << 1; + *buffer++ = cDigitsLut[i]; + *buffer++ = cDigitsLut[i + 1]; } + else + { const uint32_t i = (a / 100) << 1; + const uint32_t j = (a % 100) << 1; + *buffer++ = cDigitsLut[i]; + *buffer++ = cDigitsLut[i + 1]; + *buffer++ = cDigitsLut[j]; + *buffer++ = cDigitsLut[j + 1]; + } + + const uint32_t v0 = static_cast(value / kTen8); + const uint32_t v1 = static_cast(value % kTen8); + + const uint32_t b0 = v0 / 10000; + const uint32_t c0 = v0 % 10000; + + const uint32_t d1 = (b0 / 100) << 1; + const uint32_t d2 = (b0 % 100) << 1; + + const uint32_t d3 = (c0 / 100) << 1; + const uint32_t d4 = (c0 % 100) << 1; + + const uint32_t b1 = v1 / 10000; + const uint32_t c1 = v1 % 10000; + + const uint32_t d5 = (b1 / 100) << 1; + const uint32_t d6 = (b1 % 100) << 1; + + const uint32_t d7 = (c1 / 100) << 1; + const uint32_t d8 = (c1 % 100) << 1; + + *buffer++ = cDigitsLut[d1]; + *buffer++ = cDigitsLut[d1 + 1]; + *buffer++ = cDigitsLut[d2]; + *buffer++ = cDigitsLut[d2 + 1]; + *buffer++ = cDigitsLut[d3]; + *buffer++ = cDigitsLut[d3 + 1]; + *buffer++ = cDigitsLut[d4]; + *buffer++ = cDigitsLut[d4 + 1]; + *buffer++ = cDigitsLut[d5]; + *buffer++ = cDigitsLut[d5 + 1]; + *buffer++ = cDigitsLut[d6]; + *buffer++ = cDigitsLut[d6 + 1]; + *buffer++ = cDigitsLut[d7]; + *buffer++ = cDigitsLut[d7 + 1]; + *buffer++ = cDigitsLut[d8]; + *buffer++ = cDigitsLut[d8 + 1]; + } + + return buffer; +} + +inline char* i64toa(int64_t value, char* buffer) +{ uint64_t u = static_cast(value); + if (value < 0) + { *buffer++ = '-'; + u = ~u + 1; + } - return u64toa(u, buffer); + return u64toa(u, buffer); } } // namespace internal diff --git a/rapidjson/include/rapidjson/internal/meta.h b/rapidjson/include/rapidjson/internal/meta.h index 5a9aaa42866..f2e7af3391b 100644 --- a/rapidjson/include/rapidjson/internal/meta.h +++ b/rapidjson/include/rapidjson/internal/meta.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_INTERNAL_META_H_ @@ -32,7 +32,8 @@ RAPIDJSON_DIAG_OFF(6334) //@cond RAPIDJSON_INTERNAL RAPIDJSON_NAMESPACE_BEGIN -namespace internal { +namespace internal +{ // Helper to wrap/convert arbitrary types to void, useful for arbitrary type matching template struct Void { typedef void Type; }; @@ -40,9 +41,9 @@ template struct Void { typedef void Type; }; /////////////////////////////////////////////////////////////////////////////// // BoolType, TrueType, FalseType // -template struct BoolType { - static const bool Value = Cond; - typedef BoolType Type; +template struct BoolType +{ static const bool Value = Cond; + typedef BoolType Type; }; typedef BoolType TrueType; typedef BoolType FalseType; @@ -87,8 +88,8 @@ template struct IsConst : TrueType {}; template struct IsMoreConst - : AndExpr::Type, typename RemoveConst::Type>, - BoolType::Value >= IsConst::Value> >::Type {}; + : AndExpr::Type, typename RemoveConst::Type>, + BoolType::Value >= IsConst::Value> >::Type {}; template struct IsPointer : FalseType {}; template struct IsPointer : TrueType {}; @@ -99,31 +100,31 @@ template struct IsPointer : TrueType {}; #if RAPIDJSON_HAS_CXX11_TYPETRAITS template struct IsBaseOf - : BoolType< ::std::is_base_of::value> {}; + : BoolType< ::std::is_base_of::value> {}; #else // simplified version adopted from Boost -template struct IsBaseOfImpl { - RAPIDJSON_STATIC_ASSERT(sizeof(B) != 0); - RAPIDJSON_STATIC_ASSERT(sizeof(D) != 0); +template struct IsBaseOfImpl +{ RAPIDJSON_STATIC_ASSERT(sizeof(B) != 0); + RAPIDJSON_STATIC_ASSERT(sizeof(D) != 0); - typedef char (&Yes)[1]; - typedef char (&No) [2]; + typedef char (&Yes)[1]; + typedef char (&No) [2]; - template - static Yes Check(const D*, T); - static No Check(const B*, int); + template + static Yes Check(const D*, T); + static No Check(const B*, int); - struct Host { - operator const B*() const; - operator const D*(); - }; + struct Host + { operator const B*() const; + operator const D*(); + }; - enum { Value = (sizeof(Check(Host(), 0)) == sizeof(Yes)) }; + enum { Value = (sizeof(Check(Host(), 0)) == sizeof(Yes)) }; }; template struct IsBaseOf - : OrExpr, BoolExpr > >::Type {}; + : OrExpr, BoolExpr > >::Type {}; #endif // RAPIDJSON_HAS_CXX11_TYPETRAITS diff --git a/rapidjson/include/rapidjson/internal/pow10.h b/rapidjson/include/rapidjson/internal/pow10.h index 02f475d705f..278df44b376 100644 --- a/rapidjson/include/rapidjson/internal/pow10.h +++ b/rapidjson/include/rapidjson/internal/pow10.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_POW10_ @@ -18,35 +18,36 @@ #include "../rapidjson.h" RAPIDJSON_NAMESPACE_BEGIN -namespace internal { +namespace internal +{ //! Computes integer powers of 10 in double (10.0^n). /*! This function uses lookup table for fast and accurate results. \param n non-negative exponent. Must <= 308. \return 10.0^n */ -inline double Pow10(int n) { - static const double e[] = { // 1e-0...1e308: 309 * 8 bytes = 2472 bytes - 1e+0, - 1e+1, 1e+2, 1e+3, 1e+4, 1e+5, 1e+6, 1e+7, 1e+8, 1e+9, 1e+10, 1e+11, 1e+12, 1e+13, 1e+14, 1e+15, 1e+16, 1e+17, 1e+18, 1e+19, 1e+20, - 1e+21, 1e+22, 1e+23, 1e+24, 1e+25, 1e+26, 1e+27, 1e+28, 1e+29, 1e+30, 1e+31, 1e+32, 1e+33, 1e+34, 1e+35, 1e+36, 1e+37, 1e+38, 1e+39, 1e+40, - 1e+41, 1e+42, 1e+43, 1e+44, 1e+45, 1e+46, 1e+47, 1e+48, 1e+49, 1e+50, 1e+51, 1e+52, 1e+53, 1e+54, 1e+55, 1e+56, 1e+57, 1e+58, 1e+59, 1e+60, - 1e+61, 1e+62, 1e+63, 1e+64, 1e+65, 1e+66, 1e+67, 1e+68, 1e+69, 1e+70, 1e+71, 1e+72, 1e+73, 1e+74, 1e+75, 1e+76, 1e+77, 1e+78, 1e+79, 1e+80, - 1e+81, 1e+82, 1e+83, 1e+84, 1e+85, 1e+86, 1e+87, 1e+88, 1e+89, 1e+90, 1e+91, 1e+92, 1e+93, 1e+94, 1e+95, 1e+96, 1e+97, 1e+98, 1e+99, 1e+100, - 1e+101,1e+102,1e+103,1e+104,1e+105,1e+106,1e+107,1e+108,1e+109,1e+110,1e+111,1e+112,1e+113,1e+114,1e+115,1e+116,1e+117,1e+118,1e+119,1e+120, - 1e+121,1e+122,1e+123,1e+124,1e+125,1e+126,1e+127,1e+128,1e+129,1e+130,1e+131,1e+132,1e+133,1e+134,1e+135,1e+136,1e+137,1e+138,1e+139,1e+140, - 1e+141,1e+142,1e+143,1e+144,1e+145,1e+146,1e+147,1e+148,1e+149,1e+150,1e+151,1e+152,1e+153,1e+154,1e+155,1e+156,1e+157,1e+158,1e+159,1e+160, - 1e+161,1e+162,1e+163,1e+164,1e+165,1e+166,1e+167,1e+168,1e+169,1e+170,1e+171,1e+172,1e+173,1e+174,1e+175,1e+176,1e+177,1e+178,1e+179,1e+180, - 1e+181,1e+182,1e+183,1e+184,1e+185,1e+186,1e+187,1e+188,1e+189,1e+190,1e+191,1e+192,1e+193,1e+194,1e+195,1e+196,1e+197,1e+198,1e+199,1e+200, - 1e+201,1e+202,1e+203,1e+204,1e+205,1e+206,1e+207,1e+208,1e+209,1e+210,1e+211,1e+212,1e+213,1e+214,1e+215,1e+216,1e+217,1e+218,1e+219,1e+220, - 1e+221,1e+222,1e+223,1e+224,1e+225,1e+226,1e+227,1e+228,1e+229,1e+230,1e+231,1e+232,1e+233,1e+234,1e+235,1e+236,1e+237,1e+238,1e+239,1e+240, - 1e+241,1e+242,1e+243,1e+244,1e+245,1e+246,1e+247,1e+248,1e+249,1e+250,1e+251,1e+252,1e+253,1e+254,1e+255,1e+256,1e+257,1e+258,1e+259,1e+260, - 1e+261,1e+262,1e+263,1e+264,1e+265,1e+266,1e+267,1e+268,1e+269,1e+270,1e+271,1e+272,1e+273,1e+274,1e+275,1e+276,1e+277,1e+278,1e+279,1e+280, - 1e+281,1e+282,1e+283,1e+284,1e+285,1e+286,1e+287,1e+288,1e+289,1e+290,1e+291,1e+292,1e+293,1e+294,1e+295,1e+296,1e+297,1e+298,1e+299,1e+300, - 1e+301,1e+302,1e+303,1e+304,1e+305,1e+306,1e+307,1e+308 - }; - RAPIDJSON_ASSERT(n >= 0 && n <= 308); - return e[n]; +inline double Pow10(int n) +{ static const double e[] = // 1e-0...1e308: 309 * 8 bytes = 2472 bytes + { 1e+0, + 1e+1, 1e+2, 1e+3, 1e+4, 1e+5, 1e+6, 1e+7, 1e+8, 1e+9, 1e+10, 1e+11, 1e+12, 1e+13, 1e+14, 1e+15, 1e+16, 1e+17, 1e+18, 1e+19, 1e+20, + 1e+21, 1e+22, 1e+23, 1e+24, 1e+25, 1e+26, 1e+27, 1e+28, 1e+29, 1e+30, 1e+31, 1e+32, 1e+33, 1e+34, 1e+35, 1e+36, 1e+37, 1e+38, 1e+39, 1e+40, + 1e+41, 1e+42, 1e+43, 1e+44, 1e+45, 1e+46, 1e+47, 1e+48, 1e+49, 1e+50, 1e+51, 1e+52, 1e+53, 1e+54, 1e+55, 1e+56, 1e+57, 1e+58, 1e+59, 1e+60, + 1e+61, 1e+62, 1e+63, 1e+64, 1e+65, 1e+66, 1e+67, 1e+68, 1e+69, 1e+70, 1e+71, 1e+72, 1e+73, 1e+74, 1e+75, 1e+76, 1e+77, 1e+78, 1e+79, 1e+80, + 1e+81, 1e+82, 1e+83, 1e+84, 1e+85, 1e+86, 1e+87, 1e+88, 1e+89, 1e+90, 1e+91, 1e+92, 1e+93, 1e+94, 1e+95, 1e+96, 1e+97, 1e+98, 1e+99, 1e+100, + 1e+101,1e+102,1e+103,1e+104,1e+105,1e+106,1e+107,1e+108,1e+109,1e+110,1e+111,1e+112,1e+113,1e+114,1e+115,1e+116,1e+117,1e+118,1e+119,1e+120, + 1e+121,1e+122,1e+123,1e+124,1e+125,1e+126,1e+127,1e+128,1e+129,1e+130,1e+131,1e+132,1e+133,1e+134,1e+135,1e+136,1e+137,1e+138,1e+139,1e+140, + 1e+141,1e+142,1e+143,1e+144,1e+145,1e+146,1e+147,1e+148,1e+149,1e+150,1e+151,1e+152,1e+153,1e+154,1e+155,1e+156,1e+157,1e+158,1e+159,1e+160, + 1e+161,1e+162,1e+163,1e+164,1e+165,1e+166,1e+167,1e+168,1e+169,1e+170,1e+171,1e+172,1e+173,1e+174,1e+175,1e+176,1e+177,1e+178,1e+179,1e+180, + 1e+181,1e+182,1e+183,1e+184,1e+185,1e+186,1e+187,1e+188,1e+189,1e+190,1e+191,1e+192,1e+193,1e+194,1e+195,1e+196,1e+197,1e+198,1e+199,1e+200, + 1e+201,1e+202,1e+203,1e+204,1e+205,1e+206,1e+207,1e+208,1e+209,1e+210,1e+211,1e+212,1e+213,1e+214,1e+215,1e+216,1e+217,1e+218,1e+219,1e+220, + 1e+221,1e+222,1e+223,1e+224,1e+225,1e+226,1e+227,1e+228,1e+229,1e+230,1e+231,1e+232,1e+233,1e+234,1e+235,1e+236,1e+237,1e+238,1e+239,1e+240, + 1e+241,1e+242,1e+243,1e+244,1e+245,1e+246,1e+247,1e+248,1e+249,1e+250,1e+251,1e+252,1e+253,1e+254,1e+255,1e+256,1e+257,1e+258,1e+259,1e+260, + 1e+261,1e+262,1e+263,1e+264,1e+265,1e+266,1e+267,1e+268,1e+269,1e+270,1e+271,1e+272,1e+273,1e+274,1e+275,1e+276,1e+277,1e+278,1e+279,1e+280, + 1e+281,1e+282,1e+283,1e+284,1e+285,1e+286,1e+287,1e+288,1e+289,1e+290,1e+291,1e+292,1e+293,1e+294,1e+295,1e+296,1e+297,1e+298,1e+299,1e+300, + 1e+301,1e+302,1e+303,1e+304,1e+305,1e+306,1e+307,1e+308 + }; + RAPIDJSON_ASSERT(n >= 0 && n <= 308); + return e[n]; } } // namespace internal diff --git a/rapidjson/include/rapidjson/internal/regex.h b/rapidjson/include/rapidjson/internal/regex.h index 422a5240bf5..95633be91d2 100644 --- a/rapidjson/include/rapidjson/internal/regex.h +++ b/rapidjson/include/rapidjson/internal/regex.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_INTERNAL_REGEX_H_ @@ -23,650 +23,647 @@ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(padded) RAPIDJSON_DIAG_OFF(switch-enum) -RAPIDJSON_DIAG_OFF(implicit-fallthrough) + RAPIDJSON_DIAG_OFF(implicit-fallthrough) #endif #ifdef __GNUC__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(effc++) + RAPIDJSON_DIAG_PUSH + RAPIDJSON_DIAG_OFF(effc++) #endif #ifdef _MSC_VER -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated + RAPIDJSON_DIAG_PUSH + RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated #endif #ifndef RAPIDJSON_REGEX_VERBOSE #define RAPIDJSON_REGEX_VERBOSE 0 #endif -RAPIDJSON_NAMESPACE_BEGIN -namespace internal { + RAPIDJSON_NAMESPACE_BEGIN + namespace internal + { /////////////////////////////////////////////////////////////////////////////// // GenericRegex -static const SizeType kRegexInvalidState = ~SizeType(0); //!< Represents an invalid index in GenericRegex::State::out, out1 -static const SizeType kRegexInvalidRange = ~SizeType(0); + static const SizeType kRegexInvalidState = ~SizeType(0); //!< Represents an invalid index in GenericRegex::State::out, out1 + static const SizeType kRegexInvalidRange = ~SizeType(0); //! Regular expression engine with subset of ECMAscript grammar. -/*! - Supported regular expression syntax: - - \c ab Concatenation - - \c a|b Alternation - - \c a? Zero or one - - \c a* Zero or more - - \c a+ One or more - - \c a{3} Exactly 3 times - - \c a{3,} At least 3 times - - \c a{3,5} 3 to 5 times - - \c (ab) Grouping - - \c ^a At the beginning - - \c a$ At the end - - \c . Any character - - \c [abc] Character classes - - \c [a-c] Character class range - - \c [a-z0-9_] Character class combination - - \c [^abc] Negated character classes - - \c [^a-c] Negated character class range - - \c [\b] Backspace (U+0008) - - \c \\| \\\\ ... Escape characters - - \c \\f Form feed (U+000C) - - \c \\n Line feed (U+000A) - - \c \\r Carriage return (U+000D) - - \c \\t Tab (U+0009) - - \c \\v Vertical tab (U+000B) - - \note This is a Thompson NFA engine, implemented with reference to - Cox, Russ. "Regular Expression Matching Can Be Simple And Fast (but is slow in Java, Perl, PHP, Python, Ruby,...).", - https://swtch.com/~rsc/regexp/regexp1.html -*/ -template -class GenericRegex { -public: + /*! + Supported regular expression syntax: + - \c ab Concatenation + - \c a|b Alternation + - \c a? Zero or one + - \c a* Zero or more + - \c a+ One or more + - \c a{3} Exactly 3 times + - \c a{3,} At least 3 times + - \c a{3,5} 3 to 5 times + - \c (ab) Grouping + - \c ^a At the beginning + - \c a$ At the end + - \c . Any character + - \c [abc] Character classes + - \c [a-c] Character class range + - \c [a-z0-9_] Character class combination + - \c [^abc] Negated character classes + - \c [^a-c] Negated character class range + - \c [\b] Backspace (U+0008) + - \c \\| \\\\ ... Escape characters + - \c \\f Form feed (U+000C) + - \c \\n Line feed (U+000A) + - \c \\r Carriage return (U+000D) + - \c \\t Tab (U+0009) + - \c \\v Vertical tab (U+000B) + + \note This is a Thompson NFA engine, implemented with reference to + Cox, Russ. "Regular Expression Matching Can Be Simple And Fast (but is slow in Java, Perl, PHP, Python, Ruby,...).", + https://swtch.com/~rsc/regexp/regexp1.html + */ + template + class GenericRegex + { + public: typedef typename Encoding::Ch Ch; - GenericRegex(const Ch* source, Allocator* allocator = 0) : - states_(allocator, 256), ranges_(allocator, 256), root_(kRegexInvalidState), stateCount_(), rangeCount_(), - stateSet_(), state0_(allocator, 0), state1_(allocator, 0), anchorBegin_(), anchorEnd_() - { - GenericStringStream ss(source); - DecodedStream > ds(ss); - Parse(ds); + GenericRegex(const Ch* source, Allocator* allocator = 0) : + states_(allocator, 256), ranges_(allocator, 256), root_(kRegexInvalidState), stateCount_(), rangeCount_(), + stateSet_(), state0_(allocator, 0), state1_(allocator, 0), anchorBegin_(), anchorEnd_() + { GenericStringStream ss(source); + DecodedStream > ds(ss); + Parse(ds); } - ~GenericRegex() { - Allocator::Free(stateSet_); + ~GenericRegex() + { Allocator::Free(stateSet_); } - bool IsValid() const { - return root_ != kRegexInvalidState; + bool IsValid() const + { return root_ != kRegexInvalidState; } template - bool Match(InputStream& is) const { - return SearchWithAnchoring(is, true, true); + bool Match(InputStream& is) const + { return SearchWithAnchoring(is, true, true); } - bool Match(const Ch* s) const { - GenericStringStream is(s); - return Match(is); + bool Match(const Ch* s) const + { GenericStringStream is(s); + return Match(is); } template - bool Search(InputStream& is) const { - return SearchWithAnchoring(is, anchorBegin_, anchorEnd_); + bool Search(InputStream& is) const + { return SearchWithAnchoring(is, anchorBegin_, anchorEnd_); } - bool Search(const Ch* s) const { - GenericStringStream is(s); - return Search(is); + bool Search(const Ch* s) const + { GenericStringStream is(s); + return Search(is); } -private: - enum Operator { - kZeroOrOne, - kZeroOrMore, - kOneOrMore, - kConcatenation, - kAlternation, - kLeftParenthesis + private: + enum Operator + { kZeroOrOne, + kZeroOrMore, + kOneOrMore, + kConcatenation, + kAlternation, + kLeftParenthesis }; static const unsigned kAnyCharacterClass = 0xFFFFFFFF; //!< For '.' static const unsigned kRangeCharacterClass = 0xFFFFFFFE; static const unsigned kRangeNegationFlag = 0x80000000; - struct Range { - unsigned start; // - unsigned end; - SizeType next; + struct Range + { unsigned start; // + unsigned end; + SizeType next; }; - struct State { - SizeType out; //!< Equals to kInvalid for matching state - SizeType out1; //!< Equals to non-kInvalid for split - SizeType rangeStart; - unsigned codepoint; + struct State + { SizeType out; //!< Equals to kInvalid for matching state + SizeType out1; //!< Equals to non-kInvalid for split + SizeType rangeStart; + unsigned codepoint; }; - struct Frag { - Frag(SizeType s, SizeType o, SizeType m) : start(s), out(o), minIndex(m) {} - SizeType start; - SizeType out; //!< link-list of all output states - SizeType minIndex; + struct Frag + { Frag(SizeType s, SizeType o, SizeType m) : start(s), out(o), minIndex(m) {} + SizeType start; + SizeType out; //!< link-list of all output states + SizeType minIndex; }; template - class DecodedStream { + class DecodedStream + { public: - DecodedStream(SourceStream& ss) : ss_(ss), codepoint_() { Decode(); } - unsigned Peek() { return codepoint_; } - unsigned Take() { - unsigned c = codepoint_; - if (c) // No further decoding when '\0' - Decode(); - return c; - } + DecodedStream(SourceStream& ss) : ss_(ss), codepoint_() { Decode(); } + unsigned Peek() { return codepoint_; } + unsigned Take() + { unsigned c = codepoint_; + if (c) // No further decoding when '\0' + Decode(); + return c; + } private: - void Decode() { - if (!Encoding::Decode(ss_, &codepoint_)) - codepoint_ = 0; - } + void Decode() + { if (!Encoding::Decode(ss_, &codepoint_)) + codepoint_ = 0; + } - SourceStream& ss_; - unsigned codepoint_; + SourceStream& ss_; + unsigned codepoint_; }; - State& GetState(SizeType index) { - RAPIDJSON_ASSERT(index < stateCount_); - return states_.template Bottom()[index]; + State& GetState(SizeType index) + { RAPIDJSON_ASSERT(index < stateCount_); + return states_.template Bottom()[index]; } - const State& GetState(SizeType index) const { - RAPIDJSON_ASSERT(index < stateCount_); - return states_.template Bottom()[index]; + const State& GetState(SizeType index) const + { RAPIDJSON_ASSERT(index < stateCount_); + return states_.template Bottom()[index]; } - Range& GetRange(SizeType index) { - RAPIDJSON_ASSERT(index < rangeCount_); - return ranges_.template Bottom()[index]; + Range& GetRange(SizeType index) + { RAPIDJSON_ASSERT(index < rangeCount_); + return ranges_.template Bottom()[index]; } - const Range& GetRange(SizeType index) const { - RAPIDJSON_ASSERT(index < rangeCount_); - return ranges_.template Bottom()[index]; + const Range& GetRange(SizeType index) const + { RAPIDJSON_ASSERT(index < rangeCount_); + return ranges_.template Bottom()[index]; } template - void Parse(DecodedStream& ds) { - Allocator allocator; - Stack operandStack(&allocator, 256); // Frag - Stack operatorStack(&allocator, 256); // Operator - Stack atomCountStack(&allocator, 256); // unsigned (Atom per parenthesis) - - *atomCountStack.template Push() = 0; - - unsigned codepoint; - while (ds.Peek() != 0) { - switch (codepoint = ds.Take()) { - case '^': - anchorBegin_ = true; - break; - - case '$': - anchorEnd_ = true; - break; - - case '|': - while (!operatorStack.Empty() && *operatorStack.template Top() < kAlternation) - if (!Eval(operandStack, *operatorStack.template Pop(1))) - return; - *operatorStack.template Push() = kAlternation; - *atomCountStack.template Top() = 0; - break; - - case '(': - *operatorStack.template Push() = kLeftParenthesis; - *atomCountStack.template Push() = 0; - break; - - case ')': - while (!operatorStack.Empty() && *operatorStack.template Top() != kLeftParenthesis) - if (!Eval(operandStack, *operatorStack.template Pop(1))) - return; - if (operatorStack.Empty()) - return; - operatorStack.template Pop(1); - atomCountStack.template Pop(1); - ImplicitConcatenation(atomCountStack, operatorStack); - break; - - case '?': - if (!Eval(operandStack, kZeroOrOne)) - return; - break; - - case '*': - if (!Eval(operandStack, kZeroOrMore)) - return; - break; - - case '+': - if (!Eval(operandStack, kOneOrMore)) - return; - break; - - case '{': - { - unsigned n, m; - if (!ParseUnsigned(ds, &n)) - return; - - if (ds.Peek() == ',') { - ds.Take(); - if (ds.Peek() == '}') - m = kInfinityQuantifier; - else if (!ParseUnsigned(ds, &m) || m < n) - return; - } - else - m = n; - - if (!EvalQuantifier(operandStack, n, m) || ds.Peek() != '}') - return; - ds.Take(); - } - break; - - case '.': - PushOperand(operandStack, kAnyCharacterClass); - ImplicitConcatenation(atomCountStack, operatorStack); - break; - - case '[': - { - SizeType range; - if (!ParseRange(ds, &range)) - return; - SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, kRangeCharacterClass); - GetState(s).rangeStart = range; - *operandStack.template Push() = Frag(s, s, s); - } - ImplicitConcatenation(atomCountStack, operatorStack); - break; - - case '\\': // Escape character - if (!CharacterEscape(ds, &codepoint)) - return; // Unsupported escape character - // fall through to default - - default: // Pattern character - PushOperand(operandStack, codepoint); - ImplicitConcatenation(atomCountStack, operatorStack); + void Parse(DecodedStream& ds) + { Allocator allocator; + Stack operandStack(&allocator, 256); // Frag + Stack operatorStack(&allocator, 256); // Operator + Stack atomCountStack(&allocator, 256); // unsigned (Atom per parenthesis) + + *atomCountStack.template Push() = 0; + + unsigned codepoint; + while (ds.Peek() != 0) + { switch (codepoint = ds.Take()) + { case '^': + anchorBegin_ = true; + break; + + case '$': + anchorEnd_ = true; + break; + + case '|': + while (!operatorStack.Empty() && *operatorStack.template Top() < kAlternation) + if (!Eval(operandStack, *operatorStack.template Pop(1))) + return; + *operatorStack.template Push() = kAlternation; + *atomCountStack.template Top() = 0; + break; + + case '(': + *operatorStack.template Push() = kLeftParenthesis; + *atomCountStack.template Push() = 0; + break; + + case ')': + while (!operatorStack.Empty() && *operatorStack.template Top() != kLeftParenthesis) + if (!Eval(operandStack, *operatorStack.template Pop(1))) + return; + if (operatorStack.Empty()) + return; + operatorStack.template Pop(1); + atomCountStack.template Pop(1); + ImplicitConcatenation(atomCountStack, operatorStack); + break; + + case '?': + if (!Eval(operandStack, kZeroOrOne)) + return; + break; + + case '*': + if (!Eval(operandStack, kZeroOrMore)) + return; + break; + + case '+': + if (!Eval(operandStack, kOneOrMore)) + return; + break; + + case '{': + { unsigned n, m; + if (!ParseUnsigned(ds, &n)) + return; + + if (ds.Peek() == ',') + { ds.Take(); + if (ds.Peek() == '}') + m = kInfinityQuantifier; + else if (!ParseUnsigned(ds, &m) || m < n) + return; } + else + m = n; + + if (!EvalQuantifier(operandStack, n, m) || ds.Peek() != '}') + return; + ds.Take(); + } + break; + + case '.': + PushOperand(operandStack, kAnyCharacterClass); + ImplicitConcatenation(atomCountStack, operatorStack); + break; + + case '[': + { SizeType range; + if (!ParseRange(ds, &range)) + return; + SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, kRangeCharacterClass); + GetState(s).rangeStart = range; + *operandStack.template Push() = Frag(s, s, s); + } + ImplicitConcatenation(atomCountStack, operatorStack); + break; + + case '\\': // Escape character + if (!CharacterEscape(ds, &codepoint)) + return; // Unsupported escape character + // fall through to default + + default: // Pattern character + PushOperand(operandStack, codepoint); + ImplicitConcatenation(atomCountStack, operatorStack); } + } - while (!operatorStack.Empty()) - if (!Eval(operandStack, *operatorStack.template Pop(1))) - return; + while (!operatorStack.Empty()) + if (!Eval(operandStack, *operatorStack.template Pop(1))) + return; - // Link the operand to matching state. - if (operandStack.GetSize() == sizeof(Frag)) { - Frag* e = operandStack.template Pop(1); - Patch(e->out, NewState(kRegexInvalidState, kRegexInvalidState, 0)); - root_ = e->start; + // Link the operand to matching state. + if (operandStack.GetSize() == sizeof(Frag)) + { Frag* e = operandStack.template Pop(1); + Patch(e->out, NewState(kRegexInvalidState, kRegexInvalidState, 0)); + root_ = e->start; #if RAPIDJSON_REGEX_VERBOSE - printf("root: %d\n", root_); - for (SizeType i = 0; i < stateCount_ ; i++) { - State& s = GetState(i); - printf("[%2d] out: %2d out1: %2d c: '%c'\n", i, s.out, s.out1, (char)s.codepoint); - } - printf("\n"); -#endif - } - - // Preallocate buffer for SearchWithAnchoring() - RAPIDJSON_ASSERT(stateSet_ == 0); - if (stateCount_ > 0) { - stateSet_ = static_cast(states_.GetAllocator().Malloc(GetStateSetSize())); - state0_.template Reserve(stateCount_); - state1_.template Reserve(stateCount_); + printf("root: %d\n", root_); + for (SizeType i = 0; i < stateCount_ ; i++) + { State& s = GetState(i); + printf("[%2d] out: %2d out1: %2d c: '%c'\n", i, s.out, s.out1, (char)s.codepoint); } + printf("\n"); +#endif + } + + // Preallocate buffer for SearchWithAnchoring() + RAPIDJSON_ASSERT(stateSet_ == 0); + if (stateCount_ > 0) + { stateSet_ = static_cast(states_.GetAllocator().Malloc(GetStateSetSize())); + state0_.template Reserve(stateCount_); + state1_.template Reserve(stateCount_); + } } - SizeType NewState(SizeType out, SizeType out1, unsigned codepoint) { - State* s = states_.template Push(); - s->out = out; - s->out1 = out1; - s->codepoint = codepoint; - s->rangeStart = kRegexInvalidRange; - return stateCount_++; + SizeType NewState(SizeType out, SizeType out1, unsigned codepoint) + { State* s = states_.template Push(); + s->out = out; + s->out1 = out1; + s->codepoint = codepoint; + s->rangeStart = kRegexInvalidRange; + return stateCount_++; } - void PushOperand(Stack& operandStack, unsigned codepoint) { - SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, codepoint); - *operandStack.template Push() = Frag(s, s, s); + void PushOperand(Stack& operandStack, unsigned codepoint) + { SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, codepoint); + *operandStack.template Push() = Frag(s, s, s); } - void ImplicitConcatenation(Stack& atomCountStack, Stack& operatorStack) { - if (*atomCountStack.template Top()) - *operatorStack.template Push() = kConcatenation; - (*atomCountStack.template Top())++; + void ImplicitConcatenation(Stack& atomCountStack, Stack& operatorStack) + { if (*atomCountStack.template Top()) + *operatorStack.template Push() = kConcatenation; + (*atomCountStack.template Top())++; } - SizeType Append(SizeType l1, SizeType l2) { - SizeType old = l1; - while (GetState(l1).out != kRegexInvalidState) - l1 = GetState(l1).out; - GetState(l1).out = l2; - return old; + SizeType Append(SizeType l1, SizeType l2) + { SizeType old = l1; + while (GetState(l1).out != kRegexInvalidState) + l1 = GetState(l1).out; + GetState(l1).out = l2; + return old; } - void Patch(SizeType l, SizeType s) { - for (SizeType next; l != kRegexInvalidState; l = next) { - next = GetState(l).out; - GetState(l).out = s; - } - } - - bool Eval(Stack& operandStack, Operator op) { - switch (op) { - case kConcatenation: - RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag) * 2); - { - Frag e2 = *operandStack.template Pop(1); - Frag e1 = *operandStack.template Pop(1); - Patch(e1.out, e2.start); - *operandStack.template Push() = Frag(e1.start, e2.out, Min(e1.minIndex, e2.minIndex)); - } - return true; - - case kAlternation: - if (operandStack.GetSize() >= sizeof(Frag) * 2) { - Frag e2 = *operandStack.template Pop(1); - Frag e1 = *operandStack.template Pop(1); - SizeType s = NewState(e1.start, e2.start, 0); - *operandStack.template Push() = Frag(s, Append(e1.out, e2.out), Min(e1.minIndex, e2.minIndex)); - return true; - } - return false; - - case kZeroOrOne: - if (operandStack.GetSize() >= sizeof(Frag)) { - Frag e = *operandStack.template Pop(1); - SizeType s = NewState(kRegexInvalidState, e.start, 0); - *operandStack.template Push() = Frag(s, Append(e.out, s), e.minIndex); - return true; - } - return false; - - case kZeroOrMore: - if (operandStack.GetSize() >= sizeof(Frag)) { - Frag e = *operandStack.template Pop(1); - SizeType s = NewState(kRegexInvalidState, e.start, 0); - Patch(e.out, s); - *operandStack.template Push() = Frag(s, s, e.minIndex); - return true; - } - return false; - - default: - RAPIDJSON_ASSERT(op == kOneOrMore); - if (operandStack.GetSize() >= sizeof(Frag)) { - Frag e = *operandStack.template Pop(1); - SizeType s = NewState(kRegexInvalidState, e.start, 0); - Patch(e.out, s); - *operandStack.template Push() = Frag(e.start, s, e.minIndex); - return true; - } - return false; - } + void Patch(SizeType l, SizeType s) + { for (SizeType next; l != kRegexInvalidState; l = next) + { next = GetState(l).out; + GetState(l).out = s; + } } - bool EvalQuantifier(Stack& operandStack, unsigned n, unsigned m) { - RAPIDJSON_ASSERT(n <= m); - RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag)); - - if (n == 0) { - if (m == 0) // a{0} not support - return false; - else if (m == kInfinityQuantifier) - Eval(operandStack, kZeroOrMore); // a{0,} -> a* - else { - Eval(operandStack, kZeroOrOne); // a{0,5} -> a? - for (unsigned i = 0; i < m - 1; i++) - CloneTopOperand(operandStack); // a{0,5} -> a? a? a? a? a? - for (unsigned i = 0; i < m - 1; i++) - Eval(operandStack, kConcatenation); // a{0,5} -> a?a?a?a?a? - } + bool Eval(Stack& operandStack, Operator op) + { switch (op) + { case kConcatenation: + RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag) * 2); + { Frag e2 = *operandStack.template Pop(1); + Frag e1 = *operandStack.template Pop(1); + Patch(e1.out, e2.start); + *operandStack.template Push() = Frag(e1.start, e2.out, Min(e1.minIndex, e2.minIndex)); + } + return true; + + case kAlternation: + if (operandStack.GetSize() >= sizeof(Frag) * 2) + { Frag e2 = *operandStack.template Pop(1); + Frag e1 = *operandStack.template Pop(1); + SizeType s = NewState(e1.start, e2.start, 0); + *operandStack.template Push() = Frag(s, Append(e1.out, e2.out), Min(e1.minIndex, e2.minIndex)); return true; - } + } + return false; + + case kZeroOrOne: + if (operandStack.GetSize() >= sizeof(Frag)) + { Frag e = *operandStack.template Pop(1); + SizeType s = NewState(kRegexInvalidState, e.start, 0); + *operandStack.template Push() = Frag(s, Append(e.out, s), e.minIndex); + return true; + } + return false; + + case kZeroOrMore: + if (operandStack.GetSize() >= sizeof(Frag)) + { Frag e = *operandStack.template Pop(1); + SizeType s = NewState(kRegexInvalidState, e.start, 0); + Patch(e.out, s); + *operandStack.template Push() = Frag(s, s, e.minIndex); + return true; + } + return false; + + default: + RAPIDJSON_ASSERT(op == kOneOrMore); + if (operandStack.GetSize() >= sizeof(Frag)) + { Frag e = *operandStack.template Pop(1); + SizeType s = NewState(kRegexInvalidState, e.start, 0); + Patch(e.out, s); + *operandStack.template Push() = Frag(e.start, s, e.minIndex); + return true; + } + return false; + } + } - for (unsigned i = 0; i < n - 1; i++) // a{3} -> a a a - CloneTopOperand(operandStack); - - if (m == kInfinityQuantifier) - Eval(operandStack, kOneOrMore); // a{3,} -> a a a+ - else if (m > n) { - CloneTopOperand(operandStack); // a{3,5} -> a a a a - Eval(operandStack, kZeroOrOne); // a{3,5} -> a a a a? - for (unsigned i = n; i < m - 1; i++) - CloneTopOperand(operandStack); // a{3,5} -> a a a a? a? - for (unsigned i = n; i < m; i++) - Eval(operandStack, kConcatenation); // a{3,5} -> a a aa?a? + bool EvalQuantifier(Stack& operandStack, unsigned n, unsigned m) + { RAPIDJSON_ASSERT(n <= m); + RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag)); + + if (n == 0) + { if (m == 0) // a{0} not support + return false; + else if (m == kInfinityQuantifier) + Eval(operandStack, kZeroOrMore); // a{0,} -> a* + else + { Eval(operandStack, kZeroOrOne); // a{0,5} -> a? + for (unsigned i = 0; i < m - 1; i++) + CloneTopOperand(operandStack); // a{0,5} -> a? a? a? a? a? + for (unsigned i = 0; i < m - 1; i++) + Eval(operandStack, kConcatenation); // a{0,5} -> a?a?a?a?a? } - - for (unsigned i = 0; i < n - 1; i++) - Eval(operandStack, kConcatenation); // a{3} -> aaa, a{3,} -> aaa+, a{3.5} -> aaaa?a? - return true; + } + + for (unsigned i = 0; i < n - 1; i++) // a{3} -> a a a + CloneTopOperand(operandStack); + + if (m == kInfinityQuantifier) + Eval(operandStack, kOneOrMore); // a{3,} -> a a a+ + else if (m > n) + { CloneTopOperand(operandStack); // a{3,5} -> a a a a + Eval(operandStack, kZeroOrOne); // a{3,5} -> a a a a? + for (unsigned i = n; i < m - 1; i++) + CloneTopOperand(operandStack); // a{3,5} -> a a a a? a? + for (unsigned i = n; i < m; i++) + Eval(operandStack, kConcatenation); // a{3,5} -> a a aa?a? + } + + for (unsigned i = 0; i < n - 1; i++) + Eval(operandStack, kConcatenation); // a{3} -> aaa, a{3,} -> aaa+, a{3.5} -> aaaa?a? + + return true; } static SizeType Min(SizeType a, SizeType b) { return a < b ? a : b; } - void CloneTopOperand(Stack& operandStack) { - const Frag src = *operandStack.template Top(); // Copy constructor to prevent invalidation - SizeType count = stateCount_ - src.minIndex; // Assumes top operand contains states in [src->minIndex, stateCount_) - State* s = states_.template Push(count); - memcpy(s, &GetState(src.minIndex), count * sizeof(State)); - for (SizeType j = 0; j < count; j++) { - if (s[j].out != kRegexInvalidState) - s[j].out += count; - if (s[j].out1 != kRegexInvalidState) - s[j].out1 += count; - } - *operandStack.template Push() = Frag(src.start + count, src.out + count, src.minIndex + count); - stateCount_ += count; + void CloneTopOperand(Stack& operandStack) + { const Frag src = *operandStack.template Top(); // Copy constructor to prevent invalidation + SizeType count = stateCount_ - src.minIndex; // Assumes top operand contains states in [src->minIndex, stateCount_) + State* s = states_.template Push(count); + memcpy(s, &GetState(src.minIndex), count * sizeof(State)); + for (SizeType j = 0; j < count; j++) + { if (s[j].out != kRegexInvalidState) + s[j].out += count; + if (s[j].out1 != kRegexInvalidState) + s[j].out1 += count; + } + *operandStack.template Push() = Frag(src.start + count, src.out + count, src.minIndex + count); + stateCount_ += count; } template - bool ParseUnsigned(DecodedStream& ds, unsigned* u) { - unsigned r = 0; - if (ds.Peek() < '0' || ds.Peek() > '9') - return false; - while (ds.Peek() >= '0' && ds.Peek() <= '9') { - if (r >= 429496729 && ds.Peek() > '5') // 2^32 - 1 = 4294967295 - return false; // overflow - r = r * 10 + (ds.Take() - '0'); - } - *u = r; - return true; + bool ParseUnsigned(DecodedStream& ds, unsigned* u) + { unsigned r = 0; + if (ds.Peek() < '0' || ds.Peek() > '9') + return false; + while (ds.Peek() >= '0' && ds.Peek() <= '9') + { if (r >= 429496729 && ds.Peek() > '5') // 2^32 - 1 = 4294967295 + return false; // overflow + r = r * 10 + (ds.Take() - '0'); + } + *u = r; + return true; } template - bool ParseRange(DecodedStream& ds, SizeType* range) { - bool isBegin = true; - bool negate = false; - int step = 0; - SizeType start = kRegexInvalidRange; - SizeType current = kRegexInvalidRange; - unsigned codepoint; - while ((codepoint = ds.Take()) != 0) { - if (isBegin) { - isBegin = false; - if (codepoint == '^') { - negate = true; - continue; - } + bool ParseRange(DecodedStream& ds, SizeType* range) + { bool isBegin = true; + bool negate = false; + int step = 0; + SizeType start = kRegexInvalidRange; + SizeType current = kRegexInvalidRange; + unsigned codepoint; + while ((codepoint = ds.Take()) != 0) + { if (isBegin) + { isBegin = false; + if (codepoint == '^') + { negate = true; + continue; + } + } + + switch (codepoint) + { case ']': + if (start == kRegexInvalidRange) + return false; // Error: nothing inside [] + if (step == 2) // Add trailing '-' + { SizeType r = NewRange('-'); + RAPIDJSON_ASSERT(current != kRegexInvalidRange); + GetRange(current).next = r; } + if (negate) + GetRange(start).start |= kRangeNegationFlag; + *range = start; + return true; - switch (codepoint) { - case ']': - if (start == kRegexInvalidRange) - return false; // Error: nothing inside [] - if (step == 2) { // Add trailing '-' - SizeType r = NewRange('-'); - RAPIDJSON_ASSERT(current != kRegexInvalidRange); - GetRange(current).next = r; - } - if (negate) - GetRange(start).start |= kRangeNegationFlag; - *range = start; - return true; - - case '\\': - if (ds.Peek() == 'b') { - ds.Take(); - codepoint = 0x0008; // Escape backspace character - } - else if (!CharacterEscape(ds, &codepoint)) - return false; - // fall through to default - - default: - switch (step) { - case 1: - if (codepoint == '-') { - step++; - break; - } - // fall through to step 0 for other characters - - case 0: - { - SizeType r = NewRange(codepoint); - if (current != kRegexInvalidRange) - GetRange(current).next = r; - if (start == kRegexInvalidRange) - start = r; - current = r; - } - step = 1; - break; - - default: - RAPIDJSON_ASSERT(step == 2); - GetRange(current).end = codepoint; - step = 0; + case '\\': + if (ds.Peek() == 'b') + { ds.Take(); + codepoint = 0x0008; // Escape backspace character + } + else if (!CharacterEscape(ds, &codepoint)) + return false; + // fall through to default + + default: + switch (step) + { case 1: + if (codepoint == '-') + { step++; + break; } + // fall through to step 0 for other characters + + case 0: + { SizeType r = NewRange(codepoint); + if (current != kRegexInvalidRange) + GetRange(current).next = r; + if (start == kRegexInvalidRange) + start = r; + current = r; + } + step = 1; + break; + + default: + RAPIDJSON_ASSERT(step == 2); + GetRange(current).end = codepoint; + step = 0; } } - return false; + } + return false; } - - SizeType NewRange(unsigned codepoint) { - Range* r = ranges_.template Push(); - r->start = r->end = codepoint; - r->next = kRegexInvalidRange; - return rangeCount_++; + + SizeType NewRange(unsigned codepoint) + { Range* r = ranges_.template Push(); + r->start = r->end = codepoint; + r->next = kRegexInvalidRange; + return rangeCount_++; } template - bool CharacterEscape(DecodedStream& ds, unsigned* escapedCodepoint) { - unsigned codepoint; - switch (codepoint = ds.Take()) { - case '^': - case '$': - case '|': - case '(': - case ')': - case '?': - case '*': - case '+': - case '.': - case '[': - case ']': - case '{': - case '}': - case '\\': - *escapedCodepoint = codepoint; return true; - case 'f': *escapedCodepoint = 0x000C; return true; - case 'n': *escapedCodepoint = 0x000A; return true; - case 'r': *escapedCodepoint = 0x000D; return true; - case 't': *escapedCodepoint = 0x0009; return true; - case 'v': *escapedCodepoint = 0x000B; return true; - default: - return false; // Unsupported escape character - } + bool CharacterEscape(DecodedStream& ds, unsigned* escapedCodepoint) + { unsigned codepoint; + switch (codepoint = ds.Take()) + { case '^': + case '$': + case '|': + case '(': + case ')': + case '?': + case '*': + case '+': + case '.': + case '[': + case ']': + case '{': + case '}': + case '\\': + *escapedCodepoint = codepoint; return true; + case 'f': *escapedCodepoint = 0x000C; return true; + case 'n': *escapedCodepoint = 0x000A; return true; + case 'r': *escapedCodepoint = 0x000D; return true; + case 't': *escapedCodepoint = 0x0009; return true; + case 'v': *escapedCodepoint = 0x000B; return true; + default: + return false; // Unsupported escape character + } } template - bool SearchWithAnchoring(InputStream& is, bool anchorBegin, bool anchorEnd) const { - RAPIDJSON_ASSERT(IsValid()); - DecodedStream ds(is); - - state0_.Clear(); - Stack *current = &state0_, *next = &state1_; - const size_t stateSetSize = GetStateSetSize(); - std::memset(stateSet_, 0, stateSetSize); - - bool matched = AddState(*current, root_); - unsigned codepoint; - while (!current->Empty() && (codepoint = ds.Take()) != 0) { - std::memset(stateSet_, 0, stateSetSize); - next->Clear(); - matched = false; - for (const SizeType* s = current->template Bottom(); s != current->template End(); ++s) { - const State& sr = GetState(*s); - if (sr.codepoint == codepoint || - sr.codepoint == kAnyCharacterClass || - (sr.codepoint == kRangeCharacterClass && MatchRange(sr.rangeStart, codepoint))) - { - matched = AddState(*next, sr.out) || matched; - if (!anchorEnd && matched) - return true; - } - if (!anchorBegin) - AddState(*next, root_); - } - internal::Swap(current, next); + bool SearchWithAnchoring(InputStream& is, bool anchorBegin, bool anchorEnd) const + { RAPIDJSON_ASSERT(IsValid()); + DecodedStream ds(is); + + state0_.Clear(); + Stack *current = &state0_, *next = &state1_; + const size_t stateSetSize = GetStateSetSize(); + std::memset(stateSet_, 0, stateSetSize); + + bool matched = AddState(*current, root_); + unsigned codepoint; + while (!current->Empty() && (codepoint = ds.Take()) != 0) + { std::memset(stateSet_, 0, stateSetSize); + next->Clear(); + matched = false; + for (const SizeType* s = current->template Bottom(); s != current->template End(); ++s) + { const State& sr = GetState(*s); + if (sr.codepoint == codepoint || + sr.codepoint == kAnyCharacterClass || + (sr.codepoint == kRangeCharacterClass && MatchRange(sr.rangeStart, codepoint))) + { matched = AddState(*next, sr.out) || matched; + if (!anchorEnd && matched) + return true; + } + if (!anchorBegin) + AddState(*next, root_); } + internal::Swap(current, next); + } - return matched; + return matched; } - size_t GetStateSetSize() const { - return (stateCount_ + 31) / 32 * 4; + size_t GetStateSetSize() const + { return (stateCount_ + 31) / 32 * 4; } // Return whether the added states is a match state - bool AddState(Stack& l, SizeType index) const { - RAPIDJSON_ASSERT(index != kRegexInvalidState); - - const State& s = GetState(index); - if (s.out1 != kRegexInvalidState) { // Split - bool matched = AddState(l, s.out); - return AddState(l, s.out1) || matched; - } - else if (!(stateSet_[index >> 5] & (1 << (index & 31)))) { - stateSet_[index >> 5] |= (1 << (index & 31)); - *l.template PushUnsafe() = index; - } - return s.out == kRegexInvalidState; // by using PushUnsafe() above, we can ensure s is not validated due to reallocation. + bool AddState(Stack& l, SizeType index) const + { RAPIDJSON_ASSERT(index != kRegexInvalidState); + + const State& s = GetState(index); + if (s.out1 != kRegexInvalidState) // Split + { bool matched = AddState(l, s.out); + return AddState(l, s.out1) || matched; + } + else if (!(stateSet_[index >> 5] & (1 << (index & 31)))) + { stateSet_[index >> 5] |= (1 << (index & 31)); + *l.template PushUnsafe() = index; + } + return s.out == kRegexInvalidState; // by using PushUnsafe() above, we can ensure s is not validated due to reallocation. } - bool MatchRange(SizeType rangeIndex, unsigned codepoint) const { - bool yes = (GetRange(rangeIndex).start & kRangeNegationFlag) == 0; - while (rangeIndex != kRegexInvalidRange) { - const Range& r = GetRange(rangeIndex); - if (codepoint >= (r.start & ~kRangeNegationFlag) && codepoint <= r.end) - return yes; - rangeIndex = r.next; - } - return !yes; + bool MatchRange(SizeType rangeIndex, unsigned codepoint) const + { bool yes = (GetRange(rangeIndex).start & kRangeNegationFlag) == 0; + while (rangeIndex != kRegexInvalidRange) + { const Range& r = GetRange(rangeIndex); + if (codepoint >= (r.start & ~kRangeNegationFlag) && codepoint <= r.end) + return yes; + rangeIndex = r.next; + } + return !yes; } Stack states_; @@ -683,11 +680,11 @@ class GenericRegex { mutable Stack state1_; bool anchorBegin_; bool anchorEnd_; -}; + }; -typedef GenericRegex > Regex; + typedef GenericRegex > Regex; -} // namespace internal + } // namespace internal RAPIDJSON_NAMESPACE_END #ifdef __clang__ diff --git a/rapidjson/include/rapidjson/internal/stack.h b/rapidjson/include/rapidjson/internal/stack.h index 022c9aab411..e3bead3465e 100644 --- a/rapidjson/include/rapidjson/internal/stack.h +++ b/rapidjson/include/rapidjson/internal/stack.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_INTERNAL_STACK_H_ @@ -24,7 +24,8 @@ RAPIDJSON_DIAG_OFF(c++98-compat) #endif RAPIDJSON_NAMESPACE_BEGIN -namespace internal { +namespace internal +{ /////////////////////////////////////////////////////////////////////////////// // Stack @@ -33,191 +34,192 @@ namespace internal { /*! \tparam Allocator Allocator for allocating stack memory. */ template -class Stack { +class Stack +{ public: - // Optimization note: Do not allocate memory for stack_ in constructor. - // Do it lazily when first Push() -> Expand() -> Resize(). - Stack(Allocator* allocator, size_t stackCapacity) : allocator_(allocator), ownAllocator_(0), stack_(0), stackTop_(0), stackEnd_(0), initialCapacity_(stackCapacity) { - } + // Optimization note: Do not allocate memory for stack_ in constructor. + // Do it lazily when first Push() -> Expand() -> Resize(). + Stack(Allocator* allocator, size_t stackCapacity) : allocator_(allocator), ownAllocator_(0), stack_(0), stackTop_(0), stackEnd_(0), initialCapacity_(stackCapacity) + { + } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS - Stack(Stack&& rhs) - : allocator_(rhs.allocator_), - ownAllocator_(rhs.ownAllocator_), - stack_(rhs.stack_), - stackTop_(rhs.stackTop_), - stackEnd_(rhs.stackEnd_), - initialCapacity_(rhs.initialCapacity_) - { - rhs.allocator_ = 0; - rhs.ownAllocator_ = 0; - rhs.stack_ = 0; - rhs.stackTop_ = 0; - rhs.stackEnd_ = 0; - rhs.initialCapacity_ = 0; - } + Stack(Stack&& rhs) + : allocator_(rhs.allocator_), + ownAllocator_(rhs.ownAllocator_), + stack_(rhs.stack_), + stackTop_(rhs.stackTop_), + stackEnd_(rhs.stackEnd_), + initialCapacity_(rhs.initialCapacity_) + { rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.stack_ = 0; + rhs.stackTop_ = 0; + rhs.stackEnd_ = 0; + rhs.initialCapacity_ = 0; + } #endif - ~Stack() { - Destroy(); - } + ~Stack() + { Destroy(); + } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS - Stack& operator=(Stack&& rhs) { - if (&rhs != this) - { - Destroy(); - - allocator_ = rhs.allocator_; - ownAllocator_ = rhs.ownAllocator_; - stack_ = rhs.stack_; - stackTop_ = rhs.stackTop_; - stackEnd_ = rhs.stackEnd_; - initialCapacity_ = rhs.initialCapacity_; - - rhs.allocator_ = 0; - rhs.ownAllocator_ = 0; - rhs.stack_ = 0; - rhs.stackTop_ = 0; - rhs.stackEnd_ = 0; - rhs.initialCapacity_ = 0; - } - return *this; - } + Stack& operator=(Stack&& rhs) + { if (&rhs != this) + { Destroy(); + + allocator_ = rhs.allocator_; + ownAllocator_ = rhs.ownAllocator_; + stack_ = rhs.stack_; + stackTop_ = rhs.stackTop_; + stackEnd_ = rhs.stackEnd_; + initialCapacity_ = rhs.initialCapacity_; + + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.stack_ = 0; + rhs.stackTop_ = 0; + rhs.stackEnd_ = 0; + rhs.initialCapacity_ = 0; + } + return *this; + } #endif - void Swap(Stack& rhs) RAPIDJSON_NOEXCEPT { - internal::Swap(allocator_, rhs.allocator_); - internal::Swap(ownAllocator_, rhs.ownAllocator_); - internal::Swap(stack_, rhs.stack_); - internal::Swap(stackTop_, rhs.stackTop_); - internal::Swap(stackEnd_, rhs.stackEnd_); - internal::Swap(initialCapacity_, rhs.initialCapacity_); - } - - void Clear() { stackTop_ = stack_; } - - void ShrinkToFit() { - if (Empty()) { - // If the stack is empty, completely deallocate the memory. - Allocator::Free(stack_); - stack_ = 0; - stackTop_ = 0; - stackEnd_ = 0; - } - else - Resize(GetSize()); - } - - // Optimization note: try to minimize the size of this function for force inline. - // Expansion is run very infrequently, so it is moved to another (probably non-inline) function. - template - RAPIDJSON_FORCEINLINE void Reserve(size_t count = 1) { - // Expand the stack if needed - if (RAPIDJSON_UNLIKELY(stackTop_ + sizeof(T) * count > stackEnd_)) - Expand(count); - } - - template - RAPIDJSON_FORCEINLINE T* Push(size_t count = 1) { - Reserve(count); - return PushUnsafe(count); - } - - template - RAPIDJSON_FORCEINLINE T* PushUnsafe(size_t count = 1) { - RAPIDJSON_ASSERT(stackTop_ + sizeof(T) * count <= stackEnd_); - T* ret = reinterpret_cast(stackTop_); - stackTop_ += sizeof(T) * count; - return ret; - } - - template - T* Pop(size_t count) { - RAPIDJSON_ASSERT(GetSize() >= count * sizeof(T)); - stackTop_ -= count * sizeof(T); - return reinterpret_cast(stackTop_); - } - - template - T* Top() { - RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); - return reinterpret_cast(stackTop_ - sizeof(T)); - } - - template - const T* Top() const { - RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); - return reinterpret_cast(stackTop_ - sizeof(T)); - } - - template - T* End() { return reinterpret_cast(stackTop_); } - - template - const T* End() const { return reinterpret_cast(stackTop_); } - - template - T* Bottom() { return reinterpret_cast(stack_); } - - template - const T* Bottom() const { return reinterpret_cast(stack_); } - - bool HasAllocator() const { - return allocator_ != 0; - } - - Allocator& GetAllocator() { - RAPIDJSON_ASSERT(allocator_); - return *allocator_; - } - - bool Empty() const { return stackTop_ == stack_; } - size_t GetSize() const { return static_cast(stackTop_ - stack_); } - size_t GetCapacity() const { return static_cast(stackEnd_ - stack_); } + void Swap(Stack& rhs) RAPIDJSON_NOEXCEPT + { internal::Swap(allocator_, rhs.allocator_); + internal::Swap(ownAllocator_, rhs.ownAllocator_); + internal::Swap(stack_, rhs.stack_); + internal::Swap(stackTop_, rhs.stackTop_); + internal::Swap(stackEnd_, rhs.stackEnd_); + internal::Swap(initialCapacity_, rhs.initialCapacity_); + } + + void Clear() { stackTop_ = stack_; } + + void ShrinkToFit() + { if (Empty()) + { // If the stack is empty, completely deallocate the memory. + Allocator::Free(stack_); + stack_ = 0; + stackTop_ = 0; + stackEnd_ = 0; + } + else + Resize(GetSize()); + } + + // Optimization note: try to minimize the size of this function for force inline. + // Expansion is run very infrequently, so it is moved to another (probably non-inline) function. + template + RAPIDJSON_FORCEINLINE void Reserve(size_t count = 1) + { // Expand the stack if needed + if (RAPIDJSON_UNLIKELY(stackTop_ + sizeof(T) * count > stackEnd_)) + Expand(count); + } + + template + RAPIDJSON_FORCEINLINE T* Push(size_t count = 1) + { Reserve(count); + return PushUnsafe(count); + } + + template + RAPIDJSON_FORCEINLINE T* PushUnsafe(size_t count = 1) + { RAPIDJSON_ASSERT(stackTop_ + sizeof(T) * count <= stackEnd_); + T* ret = reinterpret_cast(stackTop_); + stackTop_ += sizeof(T) * count; + return ret; + } + + template + T* Pop(size_t count) + { RAPIDJSON_ASSERT(GetSize() >= count * sizeof(T)); + stackTop_ -= count * sizeof(T); + return reinterpret_cast(stackTop_); + } + + template + T* Top() + { RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); + return reinterpret_cast(stackTop_ - sizeof(T)); + } + + template + const T* Top() const + { RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); + return reinterpret_cast(stackTop_ - sizeof(T)); + } + + template + T* End() { return reinterpret_cast(stackTop_); } + + template + const T* End() const { return reinterpret_cast(stackTop_); } + + template + T* Bottom() { return reinterpret_cast(stack_); } + + template + const T* Bottom() const { return reinterpret_cast(stack_); } + + bool HasAllocator() const + { return allocator_ != 0; + } + + Allocator& GetAllocator() + { RAPIDJSON_ASSERT(allocator_); + return *allocator_; + } + + bool Empty() const { return stackTop_ == stack_; } + size_t GetSize() const { return static_cast(stackTop_ - stack_); } + size_t GetCapacity() const { return static_cast(stackEnd_ - stack_); } private: - template - void Expand(size_t count) { - // Only expand the capacity if the current stack exists. Otherwise just create a stack with initial capacity. - size_t newCapacity; - if (stack_ == 0) { - if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); - newCapacity = initialCapacity_; - } else { - newCapacity = GetCapacity(); - newCapacity += (newCapacity + 1) / 2; - } - size_t newSize = GetSize() + sizeof(T) * count; - if (newCapacity < newSize) - newCapacity = newSize; - - Resize(newCapacity); - } - - void Resize(size_t newCapacity) { - const size_t size = GetSize(); // Backup the current size - stack_ = static_cast(allocator_->Realloc(stack_, GetCapacity(), newCapacity)); - stackTop_ = stack_ + size; - stackEnd_ = stack_ + newCapacity; - } - - void Destroy() { - Allocator::Free(stack_); - RAPIDJSON_DELETE(ownAllocator_); // Only delete if it is owned by the stack - } - - // Prohibit copy constructor & assignment operator. - Stack(const Stack&); - Stack& operator=(const Stack&); - - Allocator* allocator_; - Allocator* ownAllocator_; - char *stack_; - char *stackTop_; - char *stackEnd_; - size_t initialCapacity_; + template + void Expand(size_t count) + { // Only expand the capacity if the current stack exists. Otherwise just create a stack with initial capacity. + size_t newCapacity; + if (stack_ == 0) + { if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + newCapacity = initialCapacity_; + } + else + { newCapacity = GetCapacity(); + newCapacity += (newCapacity + 1) / 2; + } + size_t newSize = GetSize() + sizeof(T) * count; + if (newCapacity < newSize) + newCapacity = newSize; + + Resize(newCapacity); + } + + void Resize(size_t newCapacity) + { const size_t size = GetSize(); // Backup the current size + stack_ = static_cast(allocator_->Realloc(stack_, GetCapacity(), newCapacity)); + stackTop_ = stack_ + size; + stackEnd_ = stack_ + newCapacity; + } + + void Destroy() + { Allocator::Free(stack_); + RAPIDJSON_DELETE(ownAllocator_); // Only delete if it is owned by the stack + } + + // Prohibit copy constructor & assignment operator. + Stack(const Stack&); + Stack& operator=(const Stack&); + + Allocator* allocator_; + Allocator* ownAllocator_; + char *stack_; + char *stackTop_; + char *stackEnd_; + size_t initialCapacity_; }; } // namespace internal diff --git a/rapidjson/include/rapidjson/internal/strfunc.h b/rapidjson/include/rapidjson/internal/strfunc.h index 2edfae52678..45488d550d4 100644 --- a/rapidjson/include/rapidjson/internal/strfunc.h +++ b/rapidjson/include/rapidjson/internal/strfunc.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_INTERNAL_STRFUNC_H_ @@ -18,35 +18,36 @@ #include "../stream.h" RAPIDJSON_NAMESPACE_BEGIN -namespace internal { +namespace internal +{ //! Custom strlen() which works on different character types. /*! \tparam Ch Character type (e.g. char, wchar_t, short) \param s Null-terminated input string. - \return Number of characters in the string. + \return Number of characters in the string. \note This has the same semantics as strlen(), the return value is not number of Unicode codepoints. */ template -inline SizeType StrLen(const Ch* s) { - const Ch* p = s; - while (*p) ++p; - return SizeType(p - s); +inline SizeType StrLen(const Ch* s) +{ const Ch* p = s; + while (*p) ++p; + return SizeType(p - s); } //! Returns number of code points in a encoded string. template -bool CountStringCodePoint(const typename Encoding::Ch* s, SizeType length, SizeType* outCount) { - GenericStringStream is(s); - const typename Encoding::Ch* end = s + length; - SizeType count = 0; - while (is.src_ < end) { - unsigned codepoint; - if (!Encoding::Decode(is, &codepoint)) - return false; - count++; - } - *outCount = count; - return true; +bool CountStringCodePoint(const typename Encoding::Ch* s, SizeType length, SizeType* outCount) +{ GenericStringStream is(s); + const typename Encoding::Ch* end = s + length; + SizeType count = 0; + while (is.src_ < end) + { unsigned codepoint; + if (!Encoding::Decode(is, &codepoint)) + return false; + count++; + } + *outCount = count; + return true; } } // namespace internal diff --git a/rapidjson/include/rapidjson/internal/strtod.h b/rapidjson/include/rapidjson/internal/strtod.h index 289c413b07b..73fc1b4a3a7 100644 --- a/rapidjson/include/rapidjson/internal/strtod.h +++ b/rapidjson/include/rapidjson/internal/strtod.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_STRTOD_ @@ -21,246 +21,247 @@ #include "pow10.h" RAPIDJSON_NAMESPACE_BEGIN -namespace internal { - -inline double FastPath(double significand, int exp) { - if (exp < -308) - return 0.0; - else if (exp >= 0) - return significand * internal::Pow10(exp); - else - return significand / internal::Pow10(-exp); +namespace internal +{ + +inline double FastPath(double significand, int exp) +{ if (exp < -308) + return 0.0; + else if (exp >= 0) + return significand * internal::Pow10(exp); + else + return significand / internal::Pow10(-exp); } -inline double StrtodNormalPrecision(double d, int p) { - if (p < -308) { - // Prevent expSum < -308, making Pow10(p) = 0 - d = FastPath(d, -308); - d = FastPath(d, p + 308); - } - else - d = FastPath(d, p); - return d; +inline double StrtodNormalPrecision(double d, int p) +{ if (p < -308) + { // Prevent expSum < -308, making Pow10(p) = 0 + d = FastPath(d, -308); + d = FastPath(d, p + 308); + } + else + d = FastPath(d, p); + return d; } template -inline T Min3(T a, T b, T c) { - T m = a; - if (m > b) m = b; - if (m > c) m = c; - return m; +inline T Min3(T a, T b, T c) +{ T m = a; + if (m > b) m = b; + if (m > c) m = c; + return m; } -inline int CheckWithinHalfULP(double b, const BigInteger& d, int dExp) { - const Double db(b); - const uint64_t bInt = db.IntegerSignificand(); - const int bExp = db.IntegerExponent(); - const int hExp = bExp - 1; - - int dS_Exp2 = 0, dS_Exp5 = 0, bS_Exp2 = 0, bS_Exp5 = 0, hS_Exp2 = 0, hS_Exp5 = 0; - - // Adjust for decimal exponent - if (dExp >= 0) { - dS_Exp2 += dExp; - dS_Exp5 += dExp; - } - else { - bS_Exp2 -= dExp; - bS_Exp5 -= dExp; - hS_Exp2 -= dExp; - hS_Exp5 -= dExp; - } - - // Adjust for binary exponent - if (bExp >= 0) - bS_Exp2 += bExp; - else { - dS_Exp2 -= bExp; - hS_Exp2 -= bExp; - } - - // Adjust for half ulp exponent - if (hExp >= 0) - hS_Exp2 += hExp; - else { - dS_Exp2 -= hExp; - bS_Exp2 -= hExp; - } - - // Remove common power of two factor from all three scaled values - int common_Exp2 = Min3(dS_Exp2, bS_Exp2, hS_Exp2); - dS_Exp2 -= common_Exp2; - bS_Exp2 -= common_Exp2; - hS_Exp2 -= common_Exp2; - - BigInteger dS = d; - dS.MultiplyPow5(static_cast(dS_Exp5)) <<= static_cast(dS_Exp2); - - BigInteger bS(bInt); - bS.MultiplyPow5(static_cast(bS_Exp5)) <<= static_cast(bS_Exp2); - - BigInteger hS(1); - hS.MultiplyPow5(static_cast(hS_Exp5)) <<= static_cast(hS_Exp2); - - BigInteger delta(0); - dS.Difference(bS, &delta); - - return delta.Compare(hS); +inline int CheckWithinHalfULP(double b, const BigInteger& d, int dExp) +{ const Double db(b); + const uint64_t bInt = db.IntegerSignificand(); + const int bExp = db.IntegerExponent(); + const int hExp = bExp - 1; + + int dS_Exp2 = 0, dS_Exp5 = 0, bS_Exp2 = 0, bS_Exp5 = 0, hS_Exp2 = 0, hS_Exp5 = 0; + + // Adjust for decimal exponent + if (dExp >= 0) + { dS_Exp2 += dExp; + dS_Exp5 += dExp; + } + else + { bS_Exp2 -= dExp; + bS_Exp5 -= dExp; + hS_Exp2 -= dExp; + hS_Exp5 -= dExp; + } + + // Adjust for binary exponent + if (bExp >= 0) + bS_Exp2 += bExp; + else + { dS_Exp2 -= bExp; + hS_Exp2 -= bExp; + } + + // Adjust for half ulp exponent + if (hExp >= 0) + hS_Exp2 += hExp; + else + { dS_Exp2 -= hExp; + bS_Exp2 -= hExp; + } + + // Remove common power of two factor from all three scaled values + int common_Exp2 = Min3(dS_Exp2, bS_Exp2, hS_Exp2); + dS_Exp2 -= common_Exp2; + bS_Exp2 -= common_Exp2; + hS_Exp2 -= common_Exp2; + + BigInteger dS = d; + dS.MultiplyPow5(static_cast(dS_Exp5)) <<= static_cast(dS_Exp2); + + BigInteger bS(bInt); + bS.MultiplyPow5(static_cast(bS_Exp5)) <<= static_cast(bS_Exp2); + + BigInteger hS(1); + hS.MultiplyPow5(static_cast(hS_Exp5)) <<= static_cast(hS_Exp2); + + BigInteger delta(0); + dS.Difference(bS, &delta); + + return delta.Compare(hS); } -inline bool StrtodFast(double d, int p, double* result) { - // Use fast path for string-to-double conversion if possible - // see http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - if (p > 22 && p < 22 + 16) { - // Fast Path Cases In Disguise - d *= internal::Pow10(p - 22); - p = 22; - } - - if (p >= -22 && p <= 22 && d <= 9007199254740991.0) { // 2^53 - 1 - *result = FastPath(d, p); - return true; - } - else - return false; +inline bool StrtodFast(double d, int p, double* result) +{ // Use fast path for string-to-double conversion if possible + // see http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ + if (p > 22 && p < 22 + 16) + { // Fast Path Cases In Disguise + d *= internal::Pow10(p - 22); + p = 22; + } + + if (p >= -22 && p <= 22 && d <= 9007199254740991.0) // 2^53 - 1 + { *result = FastPath(d, p); + return true; + } + else + return false; } // Compute an approximation and see if it is within 1/2 ULP -inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosition, int exp, double* result) { - uint64_t significand = 0; - size_t i = 0; // 2^64 - 1 = 18446744073709551615, 1844674407370955161 = 0x1999999999999999 - for (; i < length; i++) { - if (significand > RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || - (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > '5')) - break; - significand = significand * 10u + static_cast(decimals[i] - '0'); - } - - if (i < length && decimals[i] >= '5') // Rounding - significand++; - - size_t remaining = length - i; - const unsigned kUlpShift = 3; - const unsigned kUlp = 1 << kUlpShift; - int64_t error = (remaining == 0) ? 0 : kUlp / 2; - - DiyFp v(significand, 0); - v = v.Normalize(); - error <<= -v.e; - - const int dExp = static_cast(decimalPosition) - static_cast(i) + exp; - - int actualExp; - DiyFp cachedPower = GetCachedPower10(dExp, &actualExp); - if (actualExp != dExp) { - static const DiyFp kPow10[] = { - DiyFp(RAPIDJSON_UINT64_C2(0xa0000000, 00000000), -60), // 10^1 - DiyFp(RAPIDJSON_UINT64_C2(0xc8000000, 00000000), -57), // 10^2 - DiyFp(RAPIDJSON_UINT64_C2(0xfa000000, 00000000), -54), // 10^3 - DiyFp(RAPIDJSON_UINT64_C2(0x9c400000, 00000000), -50), // 10^4 - DiyFp(RAPIDJSON_UINT64_C2(0xc3500000, 00000000), -47), // 10^5 - DiyFp(RAPIDJSON_UINT64_C2(0xf4240000, 00000000), -44), // 10^6 - DiyFp(RAPIDJSON_UINT64_C2(0x98968000, 00000000), -40) // 10^7 - }; - int adjustment = dExp - actualExp - 1; - RAPIDJSON_ASSERT(adjustment >= 0 && adjustment < 7); - v = v * kPow10[adjustment]; - if (length + static_cast(adjustment)> 19u) // has more digits than decimal digits in 64-bit - error += kUlp / 2; +inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosition, int exp, double* result) +{ uint64_t significand = 0; + size_t i = 0; // 2^64 - 1 = 18446744073709551615, 1844674407370955161 = 0x1999999999999999 + for (; i < length; i++) + { if (significand > RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || + (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > '5')) + break; + significand = significand * 10u + static_cast(decimals[i] - '0'); + } + + if (i < length && decimals[i] >= '5') // Rounding + significand++; + + size_t remaining = length - i; + const unsigned kUlpShift = 3; + const unsigned kUlp = 1 << kUlpShift; + int64_t error = (remaining == 0) ? 0 : kUlp / 2; + + DiyFp v(significand, 0); + v = v.Normalize(); + error <<= -v.e; + + const int dExp = static_cast(decimalPosition) - static_cast(i) + exp; + + int actualExp; + DiyFp cachedPower = GetCachedPower10(dExp, &actualExp); + if (actualExp != dExp) + { static const DiyFp kPow10[] = + { DiyFp(RAPIDJSON_UINT64_C2(0xa0000000, 00000000), -60), // 10^1 + DiyFp(RAPIDJSON_UINT64_C2(0xc8000000, 00000000), -57), // 10^2 + DiyFp(RAPIDJSON_UINT64_C2(0xfa000000, 00000000), -54), // 10^3 + DiyFp(RAPIDJSON_UINT64_C2(0x9c400000, 00000000), -50), // 10^4 + DiyFp(RAPIDJSON_UINT64_C2(0xc3500000, 00000000), -47), // 10^5 + DiyFp(RAPIDJSON_UINT64_C2(0xf4240000, 00000000), -44), // 10^6 + DiyFp(RAPIDJSON_UINT64_C2(0x98968000, 00000000), -40) // 10^7 + }; + int adjustment = dExp - actualExp - 1; + RAPIDJSON_ASSERT(adjustment >= 0 && adjustment < 7); + v = v * kPow10[adjustment]; + if (length + static_cast(adjustment)> 19u) // has more digits than decimal digits in 64-bit + error += kUlp / 2; + } + + v = v * cachedPower; + + error += kUlp + (error == 0 ? 0 : 1); + + const int oldExp = v.e; + v = v.Normalize(); + error <<= oldExp - v.e; + + const unsigned effectiveSignificandSize = Double::EffectiveSignificandSize(64 + v.e); + unsigned precisionSize = 64 - effectiveSignificandSize; + if (precisionSize + kUlpShift >= 64) + { unsigned scaleExp = (precisionSize + kUlpShift) - 63; + v.f >>= scaleExp; + v.e += scaleExp; + error = (error >> scaleExp) + 1 + static_cast(kUlp); + precisionSize -= scaleExp; + } + + DiyFp rounded(v.f >> precisionSize, v.e + static_cast(precisionSize)); + const uint64_t precisionBits = (v.f & ((uint64_t(1) << precisionSize) - 1)) * kUlp; + const uint64_t halfWay = (uint64_t(1) << (precisionSize - 1)) * kUlp; + if (precisionBits >= halfWay + static_cast(error)) + { rounded.f++; + if (rounded.f & (DiyFp::kDpHiddenBit << 1)) // rounding overflows mantissa (issue #340) + { rounded.f >>= 1; + rounded.e++; } + } - v = v * cachedPower; - - error += kUlp + (error == 0 ? 0 : 1); - - const int oldExp = v.e; - v = v.Normalize(); - error <<= oldExp - v.e; - - const unsigned effectiveSignificandSize = Double::EffectiveSignificandSize(64 + v.e); - unsigned precisionSize = 64 - effectiveSignificandSize; - if (precisionSize + kUlpShift >= 64) { - unsigned scaleExp = (precisionSize + kUlpShift) - 63; - v.f >>= scaleExp; - v.e += scaleExp; - error = (error >> scaleExp) + 1 + static_cast(kUlp); - precisionSize -= scaleExp; - } + *result = rounded.ToDouble(); - DiyFp rounded(v.f >> precisionSize, v.e + static_cast(precisionSize)); - const uint64_t precisionBits = (v.f & ((uint64_t(1) << precisionSize) - 1)) * kUlp; - const uint64_t halfWay = (uint64_t(1) << (precisionSize - 1)) * kUlp; - if (precisionBits >= halfWay + static_cast(error)) { - rounded.f++; - if (rounded.f & (DiyFp::kDpHiddenBit << 1)) { // rounding overflows mantissa (issue #340) - rounded.f >>= 1; - rounded.e++; - } - } - - *result = rounded.ToDouble(); - - return halfWay - static_cast(error) >= precisionBits || precisionBits >= halfWay + static_cast(error); + return halfWay - static_cast(error) >= precisionBits || precisionBits >= halfWay + static_cast(error); } -inline double StrtodBigInteger(double approx, const char* decimals, size_t length, size_t decimalPosition, int exp) { - const BigInteger dInt(decimals, length); - const int dExp = static_cast(decimalPosition) - static_cast(length) + exp; - Double a(approx); - int cmp = CheckWithinHalfULP(a.Value(), dInt, dExp); - if (cmp < 0) - return a.Value(); // within half ULP - else if (cmp == 0) { - // Round towards even - if (a.Significand() & 1) - return a.NextPositiveDouble(); - else - return a.Value(); - } - else // adjustment - return a.NextPositiveDouble(); +inline double StrtodBigInteger(double approx, const char* decimals, size_t length, size_t decimalPosition, int exp) +{ const BigInteger dInt(decimals, length); + const int dExp = static_cast(decimalPosition) - static_cast(length) + exp; + Double a(approx); + int cmp = CheckWithinHalfULP(a.Value(), dInt, dExp); + if (cmp < 0) + return a.Value(); // within half ULP + else if (cmp == 0) + { // Round towards even + if (a.Significand() & 1) + return a.NextPositiveDouble(); + else + return a.Value(); + } + else // adjustment + return a.NextPositiveDouble(); } -inline double StrtodFullPrecision(double d, int p, const char* decimals, size_t length, size_t decimalPosition, int exp) { - RAPIDJSON_ASSERT(d >= 0.0); - RAPIDJSON_ASSERT(length >= 1); - - double result; - if (StrtodFast(d, p, &result)) - return result; - - // Trim leading zeros - while (*decimals == '0' && length > 1) { - length--; - decimals++; - decimalPosition--; - } - - // Trim trailing zeros - while (decimals[length - 1] == '0' && length > 1) { - length--; - decimalPosition--; - exp++; - } - - // Trim right-most digits - const int kMaxDecimalDigit = 780; - if (static_cast(length) > kMaxDecimalDigit) { - int delta = (static_cast(length) - kMaxDecimalDigit); - exp += delta; - decimalPosition -= static_cast(delta); - length = kMaxDecimalDigit; - } - - // If too small, underflow to zero - if (int(length) + exp < -324) - return 0.0; - - if (StrtodDiyFp(decimals, length, decimalPosition, exp, &result)) - return result; - - // Use approximation from StrtodDiyFp and make adjustment with BigInteger comparison - return StrtodBigInteger(result, decimals, length, decimalPosition, exp); +inline double StrtodFullPrecision(double d, int p, const char* decimals, size_t length, size_t decimalPosition, int exp) +{ RAPIDJSON_ASSERT(d >= 0.0); + RAPIDJSON_ASSERT(length >= 1); + + double result; + if (StrtodFast(d, p, &result)) + return result; + + // Trim leading zeros + while (*decimals == '0' && length > 1) + { length--; + decimals++; + decimalPosition--; + } + + // Trim trailing zeros + while (decimals[length - 1] == '0' && length > 1) + { length--; + decimalPosition--; + exp++; + } + + // Trim right-most digits + const int kMaxDecimalDigit = 780; + if (static_cast(length) > kMaxDecimalDigit) + { int delta = (static_cast(length) - kMaxDecimalDigit); + exp += delta; + decimalPosition -= static_cast(delta); + length = kMaxDecimalDigit; + } + + // If too small, underflow to zero + if (int(length) + exp < -324) + return 0.0; + + if (StrtodDiyFp(decimals, length, decimalPosition, exp, &result)) + return result; + + // Use approximation from StrtodDiyFp and make adjustment with BigInteger comparison + return StrtodBigInteger(result, decimals, length, decimalPosition, exp); } } // namespace internal diff --git a/rapidjson/include/rapidjson/internal/swap.h b/rapidjson/include/rapidjson/internal/swap.h index 666e49f97b6..2ed578b295e 100644 --- a/rapidjson/include/rapidjson/internal/swap.h +++ b/rapidjson/include/rapidjson/internal/swap.h @@ -23,17 +23,18 @@ RAPIDJSON_DIAG_OFF(c++98-compat) #endif RAPIDJSON_NAMESPACE_BEGIN -namespace internal { +namespace internal +{ //! Custom swap() to avoid dependency on C++ header /*! \tparam T Type of the arguments to swap, should be instantiated with primitive C++ types only. \note This has the same semantics as std::swap(). */ template -inline void Swap(T& a, T& b) RAPIDJSON_NOEXCEPT { - T tmp = a; - a = b; - b = tmp; +inline void Swap(T& a, T& b) RAPIDJSON_NOEXCEPT +{ T tmp = a; + a = b; + b = tmp; } } // namespace internal diff --git a/rapidjson/include/rapidjson/istreamwrapper.h b/rapidjson/include/rapidjson/istreamwrapper.h index f5fe28977eb..a9204078898 100644 --- a/rapidjson/include/rapidjson/istreamwrapper.h +++ b/rapidjson/include/rapidjson/istreamwrapper.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_ISTREAMWRAPPER_H_ @@ -45,62 +45,63 @@ RAPIDJSON_NAMESPACE_BEGIN \tparam StreamType Class derived from \c std::basic_istream. */ - + template -class BasicIStreamWrapper { +class BasicIStreamWrapper +{ public: - typedef typename StreamType::char_type Ch; - BasicIStreamWrapper(StreamType& stream) : stream_(stream), count_(), peekBuffer_() {} - - Ch Peek() const { - typename StreamType::int_type c = stream_.peek(); - return RAPIDJSON_LIKELY(c != StreamType::traits_type::eof()) ? static_cast(c) : '\0'; + typedef typename StreamType::char_type Ch; + BasicIStreamWrapper(StreamType& stream) : stream_(stream), count_(), peekBuffer_() {} + + Ch Peek() const + { typename StreamType::int_type c = stream_.peek(); + return RAPIDJSON_LIKELY(c != StreamType::traits_type::eof()) ? static_cast(c) : '\0'; + } + + Ch Take() + { typename StreamType::int_type c = stream_.get(); + if (RAPIDJSON_LIKELY(c != StreamType::traits_type::eof())) + { count_++; + return static_cast(c); } - - Ch Take() { - typename StreamType::int_type c = stream_.get(); - if (RAPIDJSON_LIKELY(c != StreamType::traits_type::eof())) { - count_++; - return static_cast(c); - } - else - return '\0'; - } - - // tellg() may return -1 when failed. So we count by ourself. - size_t Tell() const { return count_; } - - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - - // For encoding detection only. - const Ch* Peek4() const { - RAPIDJSON_ASSERT(sizeof(Ch) == 1); // Only usable for byte stream. - int i; - bool hasError = false; - for (i = 0; i < 4; ++i) { - typename StreamType::int_type c = stream_.get(); - if (c == StreamType::traits_type::eof()) { - hasError = true; - stream_.clear(); - break; - } - peekBuffer_[i] = static_cast(c); - } - for (--i; i >= 0; --i) - stream_.putback(peekBuffer_[i]); - return !hasError ? peekBuffer_ : 0; + else + return '\0'; + } + + // tellg() may return -1 when failed. So we count by ourself. + size_t Tell() const { return count_; } + + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + + // For encoding detection only. + const Ch* Peek4() const + { RAPIDJSON_ASSERT(sizeof(Ch) == 1); // Only usable for byte stream. + int i; + bool hasError = false; + for (i = 0; i < 4; ++i) + { typename StreamType::int_type c = stream_.get(); + if (c == StreamType::traits_type::eof()) + { hasError = true; + stream_.clear(); + break; + } + peekBuffer_[i] = static_cast(c); } + for (--i; i >= 0; --i) + stream_.putback(peekBuffer_[i]); + return !hasError ? peekBuffer_ : 0; + } private: - BasicIStreamWrapper(const BasicIStreamWrapper&); - BasicIStreamWrapper& operator=(const BasicIStreamWrapper&); + BasicIStreamWrapper(const BasicIStreamWrapper&); + BasicIStreamWrapper& operator=(const BasicIStreamWrapper&); - StreamType& stream_; - size_t count_; //!< Number of characters read. Note: - mutable Ch peekBuffer_[4]; + StreamType& stream_; + size_t count_; //!< Number of characters read. Note: + mutable Ch peekBuffer_[4]; }; typedef BasicIStreamWrapper IStreamWrapper; diff --git a/rapidjson/include/rapidjson/memorybuffer.h b/rapidjson/include/rapidjson/memorybuffer.h index 39bee1dec1c..2d281dcfe9a 100644 --- a/rapidjson/include/rapidjson/memorybuffer.h +++ b/rapidjson/include/rapidjson/memorybuffer.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_MEMORYBUFFER_H_ @@ -27,42 +27,42 @@ RAPIDJSON_NAMESPACE_BEGIN It is similar to FileWriteBuffer but the destination is an in-memory buffer instead of a file. Differences between MemoryBuffer and StringBuffer: - 1. StringBuffer has Encoding but MemoryBuffer is only a byte buffer. + 1. StringBuffer has Encoding but MemoryBuffer is only a byte buffer. 2. StringBuffer::GetString() returns a null-terminated string. MemoryBuffer::GetBuffer() returns a buffer without terminator. \tparam Allocator type for allocating memory buffer. \note implements Stream concept */ template -struct GenericMemoryBuffer { - typedef char Ch; // byte +struct GenericMemoryBuffer +{ typedef char Ch; // byte - GenericMemoryBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} + GenericMemoryBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} - void Put(Ch c) { *stack_.template Push() = c; } - void Flush() {} + void Put(Ch c) { *stack_.template Push() = c; } + void Flush() {} - void Clear() { stack_.Clear(); } - void ShrinkToFit() { stack_.ShrinkToFit(); } - Ch* Push(size_t count) { return stack_.template Push(count); } - void Pop(size_t count) { stack_.template Pop(count); } + void Clear() { stack_.Clear(); } + void ShrinkToFit() { stack_.ShrinkToFit(); } + Ch* Push(size_t count) { return stack_.template Push(count); } + void Pop(size_t count) { stack_.template Pop(count); } - const Ch* GetBuffer() const { - return stack_.template Bottom(); - } + const Ch* GetBuffer() const + { return stack_.template Bottom(); + } - size_t GetSize() const { return stack_.GetSize(); } + size_t GetSize() const { return stack_.GetSize(); } - static const size_t kDefaultCapacity = 256; - mutable internal::Stack stack_; + static const size_t kDefaultCapacity = 256; + mutable internal::Stack stack_; }; typedef GenericMemoryBuffer<> MemoryBuffer; //! Implement specialized version of PutN() with memset() for better performance. template<> -inline void PutN(MemoryBuffer& memoryBuffer, char c, size_t n) { - std::memset(memoryBuffer.stack_.Push(n), c, n * sizeof(c)); +inline void PutN(MemoryBuffer& memoryBuffer, char c, size_t n) +{ std::memset(memoryBuffer.stack_.Push(n), c, n * sizeof(c)); } RAPIDJSON_NAMESPACE_END diff --git a/rapidjson/include/rapidjson/memorystream.h b/rapidjson/include/rapidjson/memorystream.h index 1d71d8a4f0e..8e371d56cfd 100644 --- a/rapidjson/include/rapidjson/memorystream.h +++ b/rapidjson/include/rapidjson/memorystream.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_MEMORYSTREAM_H_ @@ -37,29 +37,29 @@ RAPIDJSON_NAMESPACE_BEGIN 3. MemoryStream supports Peek4() for encoding detection. StringStream is specified with an encoding so it should not have Peek4(). \note implements Stream concept */ -struct MemoryStream { - typedef char Ch; // byte +struct MemoryStream +{ typedef char Ch; // byte - MemoryStream(const Ch *src, size_t size) : src_(src), begin_(src), end_(src + size), size_(size) {} + MemoryStream(const Ch *src, size_t size) : src_(src), begin_(src), end_(src + size), size_(size) {} - Ch Peek() const { return RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_; } - Ch Take() { return RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_++; } - size_t Tell() const { return static_cast(src_ - begin_); } + Ch Peek() const { return RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_; } + Ch Take() { return RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_++; } + size_t Tell() const { return static_cast(src_ - begin_); } - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - // For encoding detection only. - const Ch* Peek4() const { - return Tell() + 4 <= size_ ? src_ : 0; - } + // For encoding detection only. + const Ch* Peek4() const + { return Tell() + 4 <= size_ ? src_ : 0; + } - const Ch* src_; //!< Current read position. - const Ch* begin_; //!< Original head of the string. - const Ch* end_; //!< End of stream. - size_t size_; //!< Size of the stream. + const Ch* src_; //!< Current read position. + const Ch* begin_; //!< Original head of the string. + const Ch* end_; //!< End of stream. + size_t size_; //!< Size of the stream. }; RAPIDJSON_NAMESPACE_END diff --git a/rapidjson/include/rapidjson/msinttypes/inttypes.h b/rapidjson/include/rapidjson/msinttypes/inttypes.h index 18111286bf5..cb1f3693967 100644 --- a/rapidjson/include/rapidjson/msinttypes/inttypes.h +++ b/rapidjson/include/rapidjson/msinttypes/inttypes.h @@ -1,37 +1,37 @@ // ISO C9x compliant inttypes.h for Microsoft Visual Studio -// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 -// +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// // Copyright (c) 2006-2013 Alexander Chemeris -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are met: -// +// // 1. Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. -// +// // 3. Neither the name of the product nor the names of its contributors may // be used to endorse or promote products derived from this software // without specific prior written permission. -// +// // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO // EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// +// /////////////////////////////////////////////////////////////////////////////// -// The above software in this distribution may have been modified by -// THL A29 Limited ("Tencent Modifications"). +// The above software in this distribution may have been modified by +// THL A29 Limited ("Tencent Modifications"). // All Tencent Modifications are Copyright (C) 2015 THL A29 Limited. #ifndef _MSC_VER // [ @@ -54,9 +54,9 @@ // 7.8 Format conversion of integer types -typedef struct { - intmax_t quot; - intmax_t rem; +typedef struct +{ intmax_t quot; + intmax_t rem; } imaxdiv_t; // 7.8.1 Macros for format specifiers @@ -288,19 +288,18 @@ static _inline #endif // STATIC_IMAXDIV ] imaxdiv_t __cdecl imaxdiv(intmax_t numer, intmax_t denom) -{ - imaxdiv_t result; +{ imaxdiv_t result; - result.quot = numer / denom; - result.rem = numer % denom; + result.quot = numer / denom; + result.rem = numer % denom; - if (numer < 0 && result.rem > 0) { - // did division wrong; must fix up - ++result.quot; - result.rem -= denom; - } + if (numer < 0 && result.rem > 0) + { // did division wrong; must fix up + ++result.quot; + result.rem -= denom; + } - return result; + return result; } // 7.8.2.3 The strtoimax and strtoumax functions diff --git a/rapidjson/include/rapidjson/msinttypes/stdint.h b/rapidjson/include/rapidjson/msinttypes/stdint.h index 3d4477b9a02..a4a4116d4de 100644 --- a/rapidjson/include/rapidjson/msinttypes/stdint.h +++ b/rapidjson/include/rapidjson/msinttypes/stdint.h @@ -1,37 +1,37 @@ // ISO C9x compliant stdint.h for Microsoft Visual Studio -// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 -// +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// // Copyright (c) 2006-2013 Alexander Chemeris -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are met: -// +// // 1. Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. -// +// // 3. Neither the name of the product nor the names of its contributors may // be used to endorse or promote products derived from this software // without specific prior written permission. -// +// // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO // EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// +// /////////////////////////////////////////////////////////////////////////////// -// The above software in this distribution may have been modified by -// THL A29 Limited ("Tencent Modifications"). +// The above software in this distribution may have been modified by +// THL A29 Limited ("Tencent Modifications"). // All Tencent Modifications are Copyright (C) 2015 THL A29 Limited. #ifndef _MSC_VER // [ @@ -93,7 +93,8 @@ // or compiler would give many errors like this: // error C2733: second C linkage of overloaded function 'wmemchr' not allowed #if defined(__cplusplus) && !defined(_M_ARM) -extern "C" { +extern "C" +{ #endif # include #if defined(__cplusplus) && !defined(_M_ARM) @@ -118,19 +119,19 @@ extern "C" { // realize that, e.g. char has the same size as __int8 // so we give up on __intX for them. #if (_MSC_VER < 1300) - typedef signed char int8_t; - typedef signed short int16_t; - typedef signed int int32_t; - typedef unsigned char uint8_t; - typedef unsigned short uint16_t; - typedef unsigned int uint32_t; +typedef signed char int8_t; +typedef signed short int16_t; +typedef signed int int32_t; +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; #else - typedef signed __int8 int8_t; - typedef signed __int16 int16_t; - typedef signed __int32 int32_t; - typedef unsigned __int8 uint8_t; - typedef unsigned __int16 uint16_t; - typedef unsigned __int32 uint32_t; +typedef signed __int8 int8_t; +typedef signed __int16 int16_t; +typedef signed __int32 int32_t; +typedef unsigned __int8 uint8_t; +typedef unsigned __int16 uint16_t; +typedef unsigned __int32 uint32_t; #endif typedef signed __int64 int64_t; typedef unsigned __int64 uint64_t; @@ -158,11 +159,11 @@ typedef uint64_t uint_fast64_t; // 7.18.1.4 Integer types capable of holding object pointers #ifdef _WIN64 // [ - typedef signed __int64 intptr_t; - typedef unsigned __int64 uintptr_t; +typedef signed __int64 intptr_t; +typedef unsigned __int64 uintptr_t; #else // _WIN64 ][ - typedef _W64 signed int intptr_t; - typedef _W64 unsigned int uintptr_t; +typedef _W64 signed int intptr_t; +typedef _W64 unsigned int uintptr_t; #endif // _WIN64 ] // 7.18.1.5 Greatest-width integer types diff --git a/rapidjson/include/rapidjson/ostreamwrapper.h b/rapidjson/include/rapidjson/ostreamwrapper.h index 6f4667c08ad..a7273227d0d 100644 --- a/rapidjson/include/rapidjson/ostreamwrapper.h +++ b/rapidjson/include/rapidjson/ostreamwrapper.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_OSTREAMWRAPPER_H_ @@ -40,33 +40,34 @@ RAPIDJSON_NAMESPACE_BEGIN \tparam StreamType Class derived from \c std::basic_ostream. */ - + template -class BasicOStreamWrapper { +class BasicOStreamWrapper +{ public: - typedef typename StreamType::char_type Ch; - BasicOStreamWrapper(StreamType& stream) : stream_(stream) {} + typedef typename StreamType::char_type Ch; + BasicOStreamWrapper(StreamType& stream) : stream_(stream) {} - void Put(Ch c) { - stream_.put(c); - } + void Put(Ch c) + { stream_.put(c); + } - void Flush() { - stream_.flush(); - } + void Flush() + { stream_.flush(); + } - // Not implemented - char Peek() const { RAPIDJSON_ASSERT(false); return 0; } - char Take() { RAPIDJSON_ASSERT(false); return 0; } - size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } - char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } + // Not implemented + char Peek() const { RAPIDJSON_ASSERT(false); return 0; } + char Take() { RAPIDJSON_ASSERT(false); return 0; } + size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } + char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } private: - BasicOStreamWrapper(const BasicOStreamWrapper&); - BasicOStreamWrapper& operator=(const BasicOStreamWrapper&); + BasicOStreamWrapper(const BasicOStreamWrapper&); + BasicOStreamWrapper& operator=(const BasicOStreamWrapper&); - StreamType& stream_; + StreamType& stream_; }; typedef BasicOStreamWrapper OStreamWrapper; diff --git a/rapidjson/include/rapidjson/pointer.h b/rapidjson/include/rapidjson/pointer.h index 0206ac1c8b6..115ea4a60fa 100644 --- a/rapidjson/include/rapidjson/pointer.h +++ b/rapidjson/include/rapidjson/pointer.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_POINTER_H_ @@ -24,25 +24,25 @@ RAPIDJSON_DIAG_OFF(switch-enum) #endif #ifdef _MSC_VER -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated + RAPIDJSON_DIAG_PUSH + RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated #endif -RAPIDJSON_NAMESPACE_BEGIN + RAPIDJSON_NAMESPACE_BEGIN -static const SizeType kPointerInvalidIndex = ~SizeType(0); //!< Represents an invalid index in GenericPointer::Token + static const SizeType kPointerInvalidIndex = ~SizeType(0); //!< Represents an invalid index in GenericPointer::Token //! Error code of parsing. /*! \ingroup RAPIDJSON_ERRORS \see GenericPointer::GenericPointer, GenericPointer::GetParseErrorCode */ -enum PointerParseErrorCode { - kPointerParseErrorNone = 0, //!< The parse is successful +enum PointerParseErrorCode +{ kPointerParseErrorNone = 0, //!< The parse is successful - kPointerParseErrorTokenMustBeginWithSolidus, //!< A token must begin with a '/' - kPointerParseErrorInvalidEscape, //!< Invalid escape - kPointerParseErrorInvalidPercentEncoding, //!< Invalid percent encoding in URI fragment - kPointerParseErrorCharacterMustPercentEncode //!< A character must percent encoded in URI fragment + kPointerParseErrorTokenMustBeginWithSolidus, //!< A token must begin with a '/' + kPointerParseErrorInvalidEscape, //!< Invalid escape + kPointerParseErrorInvalidPercentEncoding, //!< Invalid percent encoding in URI fragment + kPointerParseErrorCharacterMustPercentEncode //!< A character must percent encoded in URI fragment }; /////////////////////////////////////////////////////////////////////////////// @@ -50,16 +50,16 @@ enum PointerParseErrorCode { //! Represents a JSON Pointer. Use Pointer for UTF8 encoding and default allocator. /*! - This class implements RFC 6901 "JavaScript Object Notation (JSON) Pointer" + This class implements RFC 6901 "JavaScript Object Notation (JSON) Pointer" (https://tools.ietf.org/html/rfc6901). A JSON pointer is for identifying a specific value in a JSON document (GenericDocument). It can simplify coding of DOM tree manipulation, because it can access multiple-level depth of DOM tree with single API call. - After it parses a string representation (e.g. "/foo/0" or URI fragment + After it parses a string representation (e.g. "/foo/0" or URI fragment representation (e.g. "#/foo/0") into its internal representation (tokens), - it can be used to resolve a specific value in multiple documents, or sub-tree + it can be used to resolve a specific value in multiple documents, or sub-tree of documents. Contrary to GenericValue, Pointer can be copy constructed and copy assigned. @@ -70,979 +70,979 @@ enum PointerParseErrorCode { supplied tokens eliminates these. GenericPointer depends on GenericDocument and GenericValue. - + \tparam ValueType The value type of the DOM tree. E.g. GenericValue > \tparam Allocator The allocator type for allocating memory for internal representation. - + \note GenericPointer uses same encoding of ValueType. However, Allocator of GenericPointer is independent of Allocator of Value. */ template -class GenericPointer { +class GenericPointer +{ public: - typedef typename ValueType::EncodingType EncodingType; //!< Encoding type from Value - typedef typename ValueType::Ch Ch; //!< Character type from Value - - //! A token is the basic units of internal representation. - /*! - A JSON pointer string representation "/foo/123" is parsed to two tokens: - "foo" and 123. 123 will be represented in both numeric form and string form. - They are resolved according to the actual value type (object or array). - - For token that are not numbers, or the numeric value is out of bound - (greater than limits of SizeType), they are only treated as string form - (i.e. the token's index will be equal to kPointerInvalidIndex). - - This struct is public so that user can create a Pointer without parsing and - allocation, using a special constructor. - */ - struct Token { - const Ch* name; //!< Name of the token. It has null character at the end but it can contain null character. - SizeType length; //!< Length of the name. - SizeType index; //!< A valid array index, if it is not equal to kPointerInvalidIndex. - }; - - //!@name Constructors and destructor. - //@{ - - //! Default constructor. - GenericPointer(Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} - - //! Constructor that parses a string or URI fragment representation. - /*! - \param source A null-terminated, string or URI fragment representation of JSON pointer. - \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. - */ - explicit GenericPointer(const Ch* source, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { - Parse(source, internal::StrLen(source)); - } + typedef typename ValueType::EncodingType EncodingType; //!< Encoding type from Value + typedef typename ValueType::Ch Ch; //!< Character type from Value + + //! A token is the basic units of internal representation. + /*! + A JSON pointer string representation "/foo/123" is parsed to two tokens: + "foo" and 123. 123 will be represented in both numeric form and string form. + They are resolved according to the actual value type (object or array). + + For token that are not numbers, or the numeric value is out of bound + (greater than limits of SizeType), they are only treated as string form + (i.e. the token's index will be equal to kPointerInvalidIndex). + + This struct is public so that user can create a Pointer without parsing and + allocation, using a special constructor. + */ + struct Token + { const Ch* name; //!< Name of the token. It has null character at the end but it can contain null character. + SizeType length; //!< Length of the name. + SizeType index; //!< A valid array index, if it is not equal to kPointerInvalidIndex. + }; + + //!@name Constructors and destructor. + //@{ + + //! Default constructor. + GenericPointer(Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} + + //! Constructor that parses a string or URI fragment representation. + /*! + \param source A null-terminated, string or URI fragment representation of JSON pointer. + \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. + */ + explicit GenericPointer(const Ch* source, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) + { Parse(source, internal::StrLen(source)); + } #if RAPIDJSON_HAS_STDSTRING - //! Constructor that parses a string or URI fragment representation. - /*! - \param source A string or URI fragment representation of JSON pointer. - \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. - \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. - */ - explicit GenericPointer(const std::basic_string& source, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { - Parse(source.c_str(), source.size()); - } + //! Constructor that parses a string or URI fragment representation. + /*! + \param source A string or URI fragment representation of JSON pointer. + \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. + \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. + */ + explicit GenericPointer(const std::basic_string& source, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) + { Parse(source.c_str(), source.size()); + } #endif - //! Constructor that parses a string or URI fragment representation, with length of the source string. - /*! - \param source A string or URI fragment representation of JSON pointer. - \param length Length of source. - \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. - \note Slightly faster than the overload without length. - */ - GenericPointer(const Ch* source, size_t length, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { - Parse(source, length); - } - - //! Constructor with user-supplied tokens. - /*! - This constructor let user supplies const array of tokens. - This prevents the parsing process and eliminates allocation. - This is preferred for memory constrained environments. - - \param tokens An constant array of tokens representing the JSON pointer. - \param tokenCount Number of tokens. - - \b Example - \code - #define NAME(s) { s, sizeof(s) / sizeof(s[0]) - 1, kPointerInvalidIndex } - #define INDEX(i) { #i, sizeof(#i) - 1, i } - - static const Pointer::Token kTokens[] = { NAME("foo"), INDEX(123) }; - static const Pointer p(kTokens, sizeof(kTokens) / sizeof(kTokens[0])); - // Equivalent to static const Pointer p("/foo/123"); - - #undef NAME - #undef INDEX - \endcode - */ - GenericPointer(const Token* tokens, size_t tokenCount) : allocator_(), ownAllocator_(), nameBuffer_(), tokens_(const_cast(tokens)), tokenCount_(tokenCount), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} - - //! Copy constructor. - GenericPointer(const GenericPointer& rhs, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { - *this = rhs; - } - - //! Destructor. - ~GenericPointer() { - if (nameBuffer_) // If user-supplied tokens constructor is used, nameBuffer_ is nullptr and tokens_ are not deallocated. - Allocator::Free(tokens_); - RAPIDJSON_DELETE(ownAllocator_); - } - - //! Assignment operator. - GenericPointer& operator=(const GenericPointer& rhs) { - if (this != &rhs) { - // Do not delete ownAllcator - if (nameBuffer_) - Allocator::Free(tokens_); - - tokenCount_ = rhs.tokenCount_; - parseErrorOffset_ = rhs.parseErrorOffset_; - parseErrorCode_ = rhs.parseErrorCode_; - - if (rhs.nameBuffer_) - CopyFromRaw(rhs); // Normally parsed tokens. - else { - tokens_ = rhs.tokens_; // User supplied const tokens. - nameBuffer_ = 0; - } - } - return *this; - } - - //@} - - //!@name Append token - //@{ - - //! Append a token and return a new Pointer - /*! - \param token Token to be appended. - \param allocator Allocator for the newly return Pointer. - \return A new Pointer with appended token. - */ - GenericPointer Append(const Token& token, Allocator* allocator = 0) const { - GenericPointer r; - r.allocator_ = allocator; - Ch *p = r.CopyFromRaw(*this, 1, token.length + 1); - std::memcpy(p, token.name, (token.length + 1) * sizeof(Ch)); - r.tokens_[tokenCount_].name = p; - r.tokens_[tokenCount_].length = token.length; - r.tokens_[tokenCount_].index = token.index; - return r; - } + //! Constructor that parses a string or URI fragment representation, with length of the source string. + /*! + \param source A string or URI fragment representation of JSON pointer. + \param length Length of source. + \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. + \note Slightly faster than the overload without length. + */ + GenericPointer(const Ch* source, size_t length, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) + { Parse(source, length); + } + + //! Constructor with user-supplied tokens. + /*! + This constructor let user supplies const array of tokens. + This prevents the parsing process and eliminates allocation. + This is preferred for memory constrained environments. + + \param tokens An constant array of tokens representing the JSON pointer. + \param tokenCount Number of tokens. + + \b Example + \code + #define NAME(s) { s, sizeof(s) / sizeof(s[0]) - 1, kPointerInvalidIndex } + #define INDEX(i) { #i, sizeof(#i) - 1, i } + + static const Pointer::Token kTokens[] = { NAME("foo"), INDEX(123) }; + static const Pointer p(kTokens, sizeof(kTokens) / sizeof(kTokens[0])); + // Equivalent to static const Pointer p("/foo/123"); + + #undef NAME + #undef INDEX + \endcode + */ + GenericPointer(const Token* tokens, size_t tokenCount) : allocator_(), ownAllocator_(), nameBuffer_(), tokens_(const_cast(tokens)), tokenCount_(tokenCount), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} + + //! Copy constructor. + GenericPointer(const GenericPointer& rhs, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) + { *this = rhs; + } + + //! Destructor. + ~GenericPointer() + { if (nameBuffer_) // If user-supplied tokens constructor is used, nameBuffer_ is nullptr and tokens_ are not deallocated. + Allocator::Free(tokens_); + RAPIDJSON_DELETE(ownAllocator_); + } + + //! Assignment operator. + GenericPointer& operator=(const GenericPointer& rhs) + { if (this != &rhs) + { // Do not delete ownAllcator + if (nameBuffer_) + Allocator::Free(tokens_); - //! Append a name token with length, and return a new Pointer - /*! - \param name Name to be appended. - \param length Length of name. - \param allocator Allocator for the newly return Pointer. - \return A new Pointer with appended token. - */ - GenericPointer Append(const Ch* name, SizeType length, Allocator* allocator = 0) const { - Token token = { name, length, kPointerInvalidIndex }; - return Append(token, allocator); - } + tokenCount_ = rhs.tokenCount_; + parseErrorOffset_ = rhs.parseErrorOffset_; + parseErrorCode_ = rhs.parseErrorCode_; - //! Append a name token without length, and return a new Pointer - /*! - \param name Name (const Ch*) to be appended. - \param allocator Allocator for the newly return Pointer. - \return A new Pointer with appended token. - */ - template - RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >), (GenericPointer)) - Append(T* name, Allocator* allocator = 0) const { - return Append(name, StrLen(name), allocator); + if (rhs.nameBuffer_) + CopyFromRaw(rhs); // Normally parsed tokens. + else + { tokens_ = rhs.tokens_; // User supplied const tokens. + nameBuffer_ = 0; + } } + return *this; + } + + //@} + + //!@name Append token + //@{ + + //! Append a token and return a new Pointer + /*! + \param token Token to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(const Token& token, Allocator* allocator = 0) const + { GenericPointer r; + r.allocator_ = allocator; + Ch *p = r.CopyFromRaw(*this, 1, token.length + 1); + std::memcpy(p, token.name, (token.length + 1) * sizeof(Ch)); + r.tokens_[tokenCount_].name = p; + r.tokens_[tokenCount_].length = token.length; + r.tokens_[tokenCount_].index = token.index; + return r; + } + + //! Append a name token with length, and return a new Pointer + /*! + \param name Name to be appended. + \param length Length of name. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(const Ch* name, SizeType length, Allocator* allocator = 0) const + { Token token = { name, length, kPointerInvalidIndex }; + return Append(token, allocator); + } + + //! Append a name token without length, and return a new Pointer + /*! + \param name Name (const Ch*) to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >), (GenericPointer)) + Append(T* name, Allocator* allocator = 0) const + { return Append(name, StrLen(name), allocator); + } #if RAPIDJSON_HAS_STDSTRING - //! Append a name token, and return a new Pointer - /*! - \param name Name to be appended. - \param allocator Allocator for the newly return Pointer. - \return A new Pointer with appended token. - */ - GenericPointer Append(const std::basic_string& name, Allocator* allocator = 0) const { - return Append(name.c_str(), static_cast(name.size()), allocator); - } + //! Append a name token, and return a new Pointer + /*! + \param name Name to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(const std::basic_string& name, Allocator* allocator = 0) const + { return Append(name.c_str(), static_cast(name.size()), allocator); + } #endif - //! Append a index token, and return a new Pointer - /*! - \param index Index to be appended. - \param allocator Allocator for the newly return Pointer. - \return A new Pointer with appended token. - */ - GenericPointer Append(SizeType index, Allocator* allocator = 0) const { - char buffer[21]; - char* end = sizeof(SizeType) == 4 ? internal::u32toa(index, buffer) : internal::u64toa(index, buffer); - SizeType length = static_cast(end - buffer); - buffer[length] = '\0'; - - if (sizeof(Ch) == 1) { - Token token = { reinterpret_cast(buffer), length, index }; - return Append(token, allocator); - } - else { - Ch name[21]; - for (size_t i = 0; i <= length; i++) - name[i] = buffer[i]; - Token token = { name, length, index }; - return Append(token, allocator); - } + //! Append a index token, and return a new Pointer + /*! + \param index Index to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(SizeType index, Allocator* allocator = 0) const + { char buffer[21]; + char* end = sizeof(SizeType) == 4 ? internal::u32toa(index, buffer) : internal::u64toa(index, buffer); + SizeType length = static_cast(end - buffer); + buffer[length] = '\0'; + + if (sizeof(Ch) == 1) + { Token token = { reinterpret_cast(buffer), length, index }; + return Append(token, allocator); } - - //! Append a token by value, and return a new Pointer - /*! - \param token token to be appended. - \param allocator Allocator for the newly return Pointer. - \return A new Pointer with appended token. - */ - GenericPointer Append(const ValueType& token, Allocator* allocator = 0) const { - if (token.IsString()) - return Append(token.GetString(), token.GetStringLength(), allocator); - else { - RAPIDJSON_ASSERT(token.IsUint64()); - RAPIDJSON_ASSERT(token.GetUint64() <= SizeType(~0)); - return Append(static_cast(token.GetUint64()), allocator); - } + else + { Ch name[21]; + for (size_t i = 0; i <= length; i++) + name[i] = buffer[i]; + Token token = { name, length, index }; + return Append(token, allocator); } + } + + //! Append a token by value, and return a new Pointer + /*! + \param token token to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(const ValueType& token, Allocator* allocator = 0) const + { if (token.IsString()) + return Append(token.GetString(), token.GetStringLength(), allocator); + else + { RAPIDJSON_ASSERT(token.IsUint64()); + RAPIDJSON_ASSERT(token.GetUint64() <= SizeType(~0)); + return Append(static_cast(token.GetUint64()), allocator); + } + } - //!@name Handling Parse Error - //@{ - - //! Check whether this is a valid pointer. - bool IsValid() const { return parseErrorCode_ == kPointerParseErrorNone; } + //!@name Handling Parse Error + //@{ - //! Get the parsing error offset in code unit. - size_t GetParseErrorOffset() const { return parseErrorOffset_; } + //! Check whether this is a valid pointer. + bool IsValid() const { return parseErrorCode_ == kPointerParseErrorNone; } - //! Get the parsing error code. - PointerParseErrorCode GetParseErrorCode() const { return parseErrorCode_; } + //! Get the parsing error offset in code unit. + size_t GetParseErrorOffset() const { return parseErrorOffset_; } - //@} + //! Get the parsing error code. + PointerParseErrorCode GetParseErrorCode() const { return parseErrorCode_; } - //! Get the allocator of this pointer. - Allocator& GetAllocator() { return *allocator_; } + //@} - //!@name Tokens - //@{ + //! Get the allocator of this pointer. + Allocator& GetAllocator() { return *allocator_; } - //! Get the token array (const version only). - const Token* GetTokens() const { return tokens_; } + //!@name Tokens + //@{ - //! Get the number of tokens. - size_t GetTokenCount() const { return tokenCount_; } + //! Get the token array (const version only). + const Token* GetTokens() const { return tokens_; } - //@} + //! Get the number of tokens. + size_t GetTokenCount() const { return tokenCount_; } - //!@name Equality/inequality operators - //@{ + //@} - //! Equality operator. - /*! - \note When any pointers are invalid, always returns false. - */ - bool operator==(const GenericPointer& rhs) const { - if (!IsValid() || !rhs.IsValid() || tokenCount_ != rhs.tokenCount_) - return false; + //!@name Equality/inequality operators + //@{ - for (size_t i = 0; i < tokenCount_; i++) { - if (tokens_[i].index != rhs.tokens_[i].index || - tokens_[i].length != rhs.tokens_[i].length || - (tokens_[i].length != 0 && std::memcmp(tokens_[i].name, rhs.tokens_[i].name, sizeof(Ch)* tokens_[i].length) != 0)) - { - return false; - } - } + //! Equality operator. + /*! + \note When any pointers are invalid, always returns false. + */ + bool operator==(const GenericPointer& rhs) const + { if (!IsValid() || !rhs.IsValid() || tokenCount_ != rhs.tokenCount_) + return false; - return true; - } - - //! Inequality operator. - /*! - \note When any pointers are invalid, always returns true. - */ - bool operator!=(const GenericPointer& rhs) const { return !(*this == rhs); } - - //@} - - //!@name Stringify - //@{ - - //! Stringify the pointer into string representation. - /*! - \tparam OutputStream Type of output stream. - \param os The output stream. - */ - template - bool Stringify(OutputStream& os) const { - return Stringify(os); - } - - //! Stringify the pointer into URI fragment representation. - /*! - \tparam OutputStream Type of output stream. - \param os The output stream. - */ - template - bool StringifyUriFragment(OutputStream& os) const { - return Stringify(os); + for (size_t i = 0; i < tokenCount_; i++) + { if (tokens_[i].index != rhs.tokens_[i].index || + tokens_[i].length != rhs.tokens_[i].length || + (tokens_[i].length != 0 && std::memcmp(tokens_[i].name, rhs.tokens_[i].name, sizeof(Ch)* tokens_[i].length) != 0)) + { return false; + } } - //@} - - //!@name Create value - //@{ - - //! Create a value in a subtree. - /*! - If the value is not exist, it creates all parent values and a JSON Null value. - So it always succeed and return the newly created or existing value. - - Remind that it may change types of parents according to tokens, so it - potentially removes previously stored values. For example, if a document - was an array, and "/foo" is used to create a value, then the document - will be changed to an object, and all existing array elements are lost. - - \param root Root value of a DOM subtree to be resolved. It can be any value other than document root. - \param allocator Allocator for creating the values if the specified value or its parents are not exist. - \param alreadyExist If non-null, it stores whether the resolved value is already exist. - \return The resolved newly created (a JSON Null value), or already exists value. - */ - ValueType& Create(ValueType& root, typename ValueType::AllocatorType& allocator, bool* alreadyExist = 0) const { - RAPIDJSON_ASSERT(IsValid()); - ValueType* v = &root; - bool exist = true; - for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { - if (v->IsArray() && t->name[0] == '-' && t->length == 1) { - v->PushBack(ValueType().Move(), allocator); - v = &((*v)[v->Size() - 1]); - exist = false; - } - else { - if (t->index == kPointerInvalidIndex) { // must be object name - if (!v->IsObject()) - v->SetObject(); // Change to Object - } - else { // object name or array index - if (!v->IsArray() && !v->IsObject()) - v->SetArray(); // Change to Array - } - - if (v->IsArray()) { - if (t->index >= v->Size()) { - v->Reserve(t->index + 1, allocator); - while (t->index >= v->Size()) - v->PushBack(ValueType().Move(), allocator); - exist = false; - } - v = &((*v)[t->index]); - } - else { - typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); - if (m == v->MemberEnd()) { - v->AddMember(ValueType(t->name, t->length, allocator).Move(), ValueType().Move(), allocator); - v = &(--v->MemberEnd())->value; // Assumes AddMember() appends at the end - exist = false; - } - else - v = &m->value; - } - } + return true; + } + + //! Inequality operator. + /*! + \note When any pointers are invalid, always returns true. + */ + bool operator!=(const GenericPointer& rhs) const { return !(*this == rhs); } + + //@} + + //!@name Stringify + //@{ + + //! Stringify the pointer into string representation. + /*! + \tparam OutputStream Type of output stream. + \param os The output stream. + */ + template + bool Stringify(OutputStream& os) const + { return Stringify(os); + } + + //! Stringify the pointer into URI fragment representation. + /*! + \tparam OutputStream Type of output stream. + \param os The output stream. + */ + template + bool StringifyUriFragment(OutputStream& os) const + { return Stringify(os); + } + + //@} + + //!@name Create value + //@{ + + //! Create a value in a subtree. + /*! + If the value is not exist, it creates all parent values and a JSON Null value. + So it always succeed and return the newly created or existing value. + + Remind that it may change types of parents according to tokens, so it + potentially removes previously stored values. For example, if a document + was an array, and "/foo" is used to create a value, then the document + will be changed to an object, and all existing array elements are lost. + + \param root Root value of a DOM subtree to be resolved. It can be any value other than document root. + \param allocator Allocator for creating the values if the specified value or its parents are not exist. + \param alreadyExist If non-null, it stores whether the resolved value is already exist. + \return The resolved newly created (a JSON Null value), or already exists value. + */ + ValueType& Create(ValueType& root, typename ValueType::AllocatorType& allocator, bool* alreadyExist = 0) const + { RAPIDJSON_ASSERT(IsValid()); + ValueType* v = &root; + bool exist = true; + for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) + { if (v->IsArray() && t->name[0] == '-' && t->length == 1) + { v->PushBack(ValueType().Move(), allocator); + v = &((*v)[v->Size() - 1]); + exist = false; + } + else + { if (t->index == kPointerInvalidIndex) // must be object name + { if (!v->IsObject()) + v->SetObject(); // Change to Object } - - if (alreadyExist) - *alreadyExist = exist; - - return *v; - } - - //! Creates a value in a document. - /*! - \param document A document to be resolved. - \param alreadyExist If non-null, it stores whether the resolved value is already exist. - \return The resolved newly created, or already exists value. - */ - template - ValueType& Create(GenericDocument& document, bool* alreadyExist = 0) const { - return Create(document, document.GetAllocator(), alreadyExist); - } - - //@} - - //!@name Query value - //@{ - - //! Query a value in a subtree. - /*! - \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. - \param unresolvedTokenIndex If the pointer cannot resolve a token in the pointer, this parameter can obtain the index of unresolved token. - \return Pointer to the value if it can be resolved. Otherwise null. - - \note - There are only 3 situations when a value cannot be resolved: - 1. A value in the path is not an array nor object. - 2. An object value does not contain the token. - 3. A token is out of range of an array value. - - Use unresolvedTokenIndex to retrieve the token index. - */ - ValueType* Get(ValueType& root, size_t* unresolvedTokenIndex = 0) const { - RAPIDJSON_ASSERT(IsValid()); - ValueType* v = &root; - for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { - switch (v->GetType()) { - case kObjectType: - { - typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); - if (m == v->MemberEnd()) - break; - v = &m->value; - } - continue; - case kArrayType: - if (t->index == kPointerInvalidIndex || t->index >= v->Size()) - break; - v = &((*v)[t->index]); - continue; - default: - break; - } - - // Error: unresolved token - if (unresolvedTokenIndex) - *unresolvedTokenIndex = static_cast(t - tokens_); - return 0; + else // object name or array index + { if (!v->IsArray() && !v->IsObject()) + v->SetArray(); // Change to Array } - return v; - } - //! Query a const value in a const subtree. - /*! - \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. - \return Pointer to the value if it can be resolved. Otherwise null. - */ - const ValueType* Get(const ValueType& root, size_t* unresolvedTokenIndex = 0) const { - return Get(const_cast(root), unresolvedTokenIndex); + if (v->IsArray()) + { if (t->index >= v->Size()) + { v->Reserve(t->index + 1, allocator); + while (t->index >= v->Size()) + v->PushBack(ValueType().Move(), allocator); + exist = false; + } + v = &((*v)[t->index]); + } + else + { typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); + if (m == v->MemberEnd()) + { v->AddMember(ValueType(t->name, t->length, allocator).Move(), ValueType().Move(), allocator); + v = &(--v->MemberEnd())->value; // Assumes AddMember() appends at the end + exist = false; + } + else + v = &m->value; + } + } } - //@} - - //!@name Query a value with default - //@{ - - //! Query a value in a subtree with default value. - /*! - Similar to Get(), but if the specified value do not exists, it creates all parents and clone the default value. - So that this function always succeed. - - \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. - \param defaultValue Default value to be cloned if the value was not exists. - \param allocator Allocator for creating the values if the specified value or its parents are not exist. - \see Create() - */ - ValueType& GetWithDefault(ValueType& root, const ValueType& defaultValue, typename ValueType::AllocatorType& allocator) const { - bool alreadyExist; - Value& v = Create(root, allocator, &alreadyExist); - return alreadyExist ? v : v.CopyFrom(defaultValue, allocator); - } + if (alreadyExist) + *alreadyExist = exist; + + return *v; + } + + //! Creates a value in a document. + /*! + \param document A document to be resolved. + \param alreadyExist If non-null, it stores whether the resolved value is already exist. + \return The resolved newly created, or already exists value. + */ + template + ValueType& Create(GenericDocument& document, bool* alreadyExist = 0) const + { return Create(document, document.GetAllocator(), alreadyExist); + } + + //@} + + //!@name Query value + //@{ + + //! Query a value in a subtree. + /*! + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param unresolvedTokenIndex If the pointer cannot resolve a token in the pointer, this parameter can obtain the index of unresolved token. + \return Pointer to the value if it can be resolved. Otherwise null. + + \note + There are only 3 situations when a value cannot be resolved: + 1. A value in the path is not an array nor object. + 2. An object value does not contain the token. + 3. A token is out of range of an array value. + + Use unresolvedTokenIndex to retrieve the token index. + */ + ValueType* Get(ValueType& root, size_t* unresolvedTokenIndex = 0) const + { RAPIDJSON_ASSERT(IsValid()); + ValueType* v = &root; + for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) + { switch (v->GetType()) + { case kObjectType: + { typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); + if (m == v->MemberEnd()) + break; + v = &m->value; + } + continue; + case kArrayType: + if (t->index == kPointerInvalidIndex || t->index >= v->Size()) + break; + v = &((*v)[t->index]); + continue; + default: + break; + } - //! Query a value in a subtree with default null-terminated string. - ValueType& GetWithDefault(ValueType& root, const Ch* defaultValue, typename ValueType::AllocatorType& allocator) const { - bool alreadyExist; - Value& v = Create(root, allocator, &alreadyExist); - return alreadyExist ? v : v.SetString(defaultValue, allocator); + // Error: unresolved token + if (unresolvedTokenIndex) + *unresolvedTokenIndex = static_cast(t - tokens_); + return 0; } + return v; + } + + //! Query a const value in a const subtree. + /*! + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \return Pointer to the value if it can be resolved. Otherwise null. + */ + const ValueType* Get(const ValueType& root, size_t* unresolvedTokenIndex = 0) const + { return Get(const_cast(root), unresolvedTokenIndex); + } + + //@} + + //!@name Query a value with default + //@{ + + //! Query a value in a subtree with default value. + /*! + Similar to Get(), but if the specified value do not exists, it creates all parents and clone the default value. + So that this function always succeed. + + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param defaultValue Default value to be cloned if the value was not exists. + \param allocator Allocator for creating the values if the specified value or its parents are not exist. + \see Create() + */ + ValueType& GetWithDefault(ValueType& root, const ValueType& defaultValue, typename ValueType::AllocatorType& allocator) const + { bool alreadyExist; + Value& v = Create(root, allocator, &alreadyExist); + return alreadyExist ? v : v.CopyFrom(defaultValue, allocator); + } + + //! Query a value in a subtree with default null-terminated string. + ValueType& GetWithDefault(ValueType& root, const Ch* defaultValue, typename ValueType::AllocatorType& allocator) const + { bool alreadyExist; + Value& v = Create(root, allocator, &alreadyExist); + return alreadyExist ? v : v.SetString(defaultValue, allocator); + } #if RAPIDJSON_HAS_STDSTRING - //! Query a value in a subtree with default std::basic_string. - ValueType& GetWithDefault(ValueType& root, const std::basic_string& defaultValue, typename ValueType::AllocatorType& allocator) const { - bool alreadyExist; - Value& v = Create(root, allocator, &alreadyExist); - return alreadyExist ? v : v.SetString(defaultValue, allocator); - } + //! Query a value in a subtree with default std::basic_string. + ValueType& GetWithDefault(ValueType& root, const std::basic_string& defaultValue, typename ValueType::AllocatorType& allocator) const + { bool alreadyExist; + Value& v = Create(root, allocator, &alreadyExist); + return alreadyExist ? v : v.SetString(defaultValue, allocator); + } #endif - //! Query a value in a subtree with default primitive value. - /*! - \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool - */ - template - RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) - GetWithDefault(ValueType& root, T defaultValue, typename ValueType::AllocatorType& allocator) const { - return GetWithDefault(root, ValueType(defaultValue).Move(), allocator); - } + //! Query a value in a subtree with default primitive value. + /*! + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) + GetWithDefault(ValueType& root, T defaultValue, typename ValueType::AllocatorType& allocator) const + { return GetWithDefault(root, ValueType(defaultValue).Move(), allocator); + } + + //! Query a value in a document with default value. + template + ValueType& GetWithDefault(GenericDocument& document, const ValueType& defaultValue) const + { return GetWithDefault(document, defaultValue, document.GetAllocator()); + } + + //! Query a value in a document with default null-terminated string. + template + ValueType& GetWithDefault(GenericDocument& document, const Ch* defaultValue) const + { return GetWithDefault(document, defaultValue, document.GetAllocator()); + } - //! Query a value in a document with default value. - template - ValueType& GetWithDefault(GenericDocument& document, const ValueType& defaultValue) const { - return GetWithDefault(document, defaultValue, document.GetAllocator()); - } - - //! Query a value in a document with default null-terminated string. - template - ValueType& GetWithDefault(GenericDocument& document, const Ch* defaultValue) const { - return GetWithDefault(document, defaultValue, document.GetAllocator()); - } - #if RAPIDJSON_HAS_STDSTRING - //! Query a value in a document with default std::basic_string. - template - ValueType& GetWithDefault(GenericDocument& document, const std::basic_string& defaultValue) const { - return GetWithDefault(document, defaultValue, document.GetAllocator()); - } + //! Query a value in a document with default std::basic_string. + template + ValueType& GetWithDefault(GenericDocument& document, const std::basic_string& defaultValue) const + { return GetWithDefault(document, defaultValue, document.GetAllocator()); + } #endif - //! Query a value in a document with default primitive value. - /*! - \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool - */ - template - RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) - GetWithDefault(GenericDocument& document, T defaultValue) const { - return GetWithDefault(document, defaultValue, document.GetAllocator()); - } - - //@} - - //!@name Set a value - //@{ - - //! Set a value in a subtree, with move semantics. - /*! - It creates all parents if they are not exist or types are different to the tokens. - So this function always succeeds but potentially remove existing values. - - \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. - \param value Value to be set. - \param allocator Allocator for creating the values if the specified value or its parents are not exist. - \see Create() - */ - ValueType& Set(ValueType& root, ValueType& value, typename ValueType::AllocatorType& allocator) const { - return Create(root, allocator) = value; - } - - //! Set a value in a subtree, with copy semantics. - ValueType& Set(ValueType& root, const ValueType& value, typename ValueType::AllocatorType& allocator) const { - return Create(root, allocator).CopyFrom(value, allocator); - } - - //! Set a null-terminated string in a subtree. - ValueType& Set(ValueType& root, const Ch* value, typename ValueType::AllocatorType& allocator) const { - return Create(root, allocator) = ValueType(value, allocator).Move(); - } + //! Query a value in a document with default primitive value. + /*! + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) + GetWithDefault(GenericDocument& document, T defaultValue) const + { return GetWithDefault(document, defaultValue, document.GetAllocator()); + } + + //@} + + //!@name Set a value + //@{ + + //! Set a value in a subtree, with move semantics. + /*! + It creates all parents if they are not exist or types are different to the tokens. + So this function always succeeds but potentially remove existing values. + + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param value Value to be set. + \param allocator Allocator for creating the values if the specified value or its parents are not exist. + \see Create() + */ + ValueType& Set(ValueType& root, ValueType& value, typename ValueType::AllocatorType& allocator) const + { return Create(root, allocator) = value; + } + + //! Set a value in a subtree, with copy semantics. + ValueType& Set(ValueType& root, const ValueType& value, typename ValueType::AllocatorType& allocator) const + { return Create(root, allocator).CopyFrom(value, allocator); + } + + //! Set a null-terminated string in a subtree. + ValueType& Set(ValueType& root, const Ch* value, typename ValueType::AllocatorType& allocator) const + { return Create(root, allocator) = ValueType(value, allocator).Move(); + } #if RAPIDJSON_HAS_STDSTRING - //! Set a std::basic_string in a subtree. - ValueType& Set(ValueType& root, const std::basic_string& value, typename ValueType::AllocatorType& allocator) const { - return Create(root, allocator) = ValueType(value, allocator).Move(); - } + //! Set a std::basic_string in a subtree. + ValueType& Set(ValueType& root, const std::basic_string& value, typename ValueType::AllocatorType& allocator) const + { return Create(root, allocator) = ValueType(value, allocator).Move(); + } #endif - //! Set a primitive value in a subtree. - /*! - \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool - */ - template - RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) - Set(ValueType& root, T value, typename ValueType::AllocatorType& allocator) const { - return Create(root, allocator) = ValueType(value).Move(); - } + //! Set a primitive value in a subtree. + /*! + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) + Set(ValueType& root, T value, typename ValueType::AllocatorType& allocator) const + { return Create(root, allocator) = ValueType(value).Move(); + } + + //! Set a value in a document, with move semantics. + template + ValueType& Set(GenericDocument& document, ValueType& value) const + { return Create(document) = value; + } + + //! Set a value in a document, with copy semantics. + template + ValueType& Set(GenericDocument& document, const ValueType& value) const + { return Create(document).CopyFrom(value, document.GetAllocator()); + } + + //! Set a null-terminated string in a document. + template + ValueType& Set(GenericDocument& document, const Ch* value) const + { return Create(document) = ValueType(value, document.GetAllocator()).Move(); + } - //! Set a value in a document, with move semantics. - template - ValueType& Set(GenericDocument& document, ValueType& value) const { - return Create(document) = value; - } +#if RAPIDJSON_HAS_STDSTRING + //! Sets a std::basic_string in a document. + template + ValueType& Set(GenericDocument& document, const std::basic_string& value) const + { return Create(document) = ValueType(value, document.GetAllocator()).Move(); + } +#endif - //! Set a value in a document, with copy semantics. - template - ValueType& Set(GenericDocument& document, const ValueType& value) const { - return Create(document).CopyFrom(value, document.GetAllocator()); + //! Set a primitive value in a document. + /*! + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) + Set(GenericDocument& document, T value) const + { return Create(document) = value; + } + + //@} + + //!@name Swap a value + //@{ + + //! Swap a value with a value in a subtree. + /*! + It creates all parents if they are not exist or types are different to the tokens. + So this function always succeeds but potentially remove existing values. + + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param value Value to be swapped. + \param allocator Allocator for creating the values if the specified value or its parents are not exist. + \see Create() + */ + ValueType& Swap(ValueType& root, ValueType& value, typename ValueType::AllocatorType& allocator) const + { return Create(root, allocator).Swap(value); + } + + //! Swap a value with a value in a document. + template + ValueType& Swap(GenericDocument& document, ValueType& value) const + { return Create(document).Swap(value); + } + + //@} + + //! Erase a value in a subtree. + /*! + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \return Whether the resolved value is found and erased. + + \note Erasing with an empty pointer \c Pointer(""), i.e. the root, always fail and return false. + */ + bool Erase(ValueType& root) const + { RAPIDJSON_ASSERT(IsValid()); + if (tokenCount_ == 0) // Cannot erase the root + return false; + + ValueType* v = &root; + const Token* last = tokens_ + (tokenCount_ - 1); + for (const Token *t = tokens_; t != last; ++t) + { switch (v->GetType()) + { case kObjectType: + { typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); + if (m == v->MemberEnd()) + return false; + v = &m->value; + } + break; + case kArrayType: + if (t->index == kPointerInvalidIndex || t->index >= v->Size()) + return false; + v = &((*v)[t->index]); + break; + default: + return false; + } } - //! Set a null-terminated string in a document. - template - ValueType& Set(GenericDocument& document, const Ch* value) const { - return Create(document) = ValueType(value, document.GetAllocator()).Move(); + switch (v->GetType()) + { case kObjectType: + return v->EraseMember(GenericStringRef(last->name, last->length)); + case kArrayType: + if (last->index == kPointerInvalidIndex || last->index >= v->Size()) + return false; + v->Erase(v->Begin() + last->index); + return true; + default: + return false; } + } -#if RAPIDJSON_HAS_STDSTRING - //! Sets a std::basic_string in a document. - template - ValueType& Set(GenericDocument& document, const std::basic_string& value) const { - return Create(document) = ValueType(value, document.GetAllocator()).Move(); +private: + //! Clone the content from rhs to this. + /*! + \param rhs Source pointer. + \param extraToken Extra tokens to be allocated. + \param extraNameBufferSize Extra name buffer size (in number of Ch) to be allocated. + \return Start of non-occupied name buffer, for storing extra names. + */ + Ch* CopyFromRaw(const GenericPointer& rhs, size_t extraToken = 0, size_t extraNameBufferSize = 0) + { if (!allocator_) // allocator is independently owned. + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + + size_t nameBufferSize = rhs.tokenCount_; // null terminators for tokens + for (Token *t = rhs.tokens_; t != rhs.tokens_ + rhs.tokenCount_; ++t) + nameBufferSize += t->length; + + tokenCount_ = rhs.tokenCount_ + extraToken; + tokens_ = static_cast(allocator_->Malloc(tokenCount_ * sizeof(Token) + (nameBufferSize + extraNameBufferSize) * sizeof(Ch))); + nameBuffer_ = reinterpret_cast(tokens_ + tokenCount_); + if (rhs.tokenCount_ > 0) + { std::memcpy(tokens_, rhs.tokens_, rhs.tokenCount_ * sizeof(Token)); } -#endif - - //! Set a primitive value in a document. - /*! - \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool - */ - template - RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) - Set(GenericDocument& document, T value) const { - return Create(document) = value; + if (nameBufferSize > 0) + { std::memcpy(nameBuffer_, rhs.nameBuffer_, nameBufferSize * sizeof(Ch)); } - //@} + // Adjust pointers to name buffer + std::ptrdiff_t diff = nameBuffer_ - rhs.nameBuffer_; + for (Token *t = tokens_; t != tokens_ + rhs.tokenCount_; ++t) + t->name += diff; - //!@name Swap a value - //@{ + return nameBuffer_ + nameBufferSize; + } - //! Swap a value with a value in a subtree. - /*! - It creates all parents if they are not exist or types are different to the tokens. - So this function always succeeds but potentially remove existing values. + //! Check whether a character should be percent-encoded. + /*! + According to RFC 3986 2.3 Unreserved Characters. + \param c The character (code unit) to be tested. + */ + bool NeedPercentEncode(Ch c) const + { return !((c >= '0' && c <= '9') || (c >= 'A' && c <='Z') || (c >= 'a' && c <= 'z') || c == '-' || c == '.' || c == '_' || c =='~'); + } - \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. - \param value Value to be swapped. - \param allocator Allocator for creating the values if the specified value or its parents are not exist. - \see Create() - */ - ValueType& Swap(ValueType& root, ValueType& value, typename ValueType::AllocatorType& allocator) const { - return Create(root, allocator).Swap(value); + //! Parse a JSON String or its URI fragment representation into tokens. +#ifndef __clang__ // -Wdocumentation + /*! + \param source Either a JSON Pointer string, or its URI fragment representation. Not need to be null terminated. + \param length Length of the source string. + \note Source cannot be JSON String Representation of JSON Pointer, e.g. In "/\u0000", \u0000 will not be unescaped. + */ +#endif + void Parse(const Ch* source, size_t length) + { RAPIDJSON_ASSERT(source != NULL); + RAPIDJSON_ASSERT(nameBuffer_ == 0); + RAPIDJSON_ASSERT(tokens_ == 0); + + // Create own allocator if user did not supply. + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + + // Count number of '/' as tokenCount + tokenCount_ = 0; + for (const Ch* s = source; s != source + length; s++) + if (*s == '/') + tokenCount_++; + + Token* token = tokens_ = static_cast(allocator_->Malloc(tokenCount_ * sizeof(Token) + length * sizeof(Ch))); + Ch* name = nameBuffer_ = reinterpret_cast(tokens_ + tokenCount_); + size_t i = 0; + + // Detect if it is a URI fragment + bool uriFragment = false; + if (source[i] == '#') + { uriFragment = true; + i++; } - //! Swap a value with a value in a document. - template - ValueType& Swap(GenericDocument& document, ValueType& value) const { - return Create(document).Swap(value); + if (i != length && source[i] != '/') + { parseErrorCode_ = kPointerParseErrorTokenMustBeginWithSolidus; + goto error; } - //@} + while (i < length) + { RAPIDJSON_ASSERT(source[i] == '/'); + i++; // consumes '/' - //! Erase a value in a subtree. - /*! - \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. - \return Whether the resolved value is found and erased. + token->name = name; + bool isNumber = true; - \note Erasing with an empty pointer \c Pointer(""), i.e. the root, always fail and return false. - */ - bool Erase(ValueType& root) const { - RAPIDJSON_ASSERT(IsValid()); - if (tokenCount_ == 0) // Cannot erase the root - return false; - - ValueType* v = &root; - const Token* last = tokens_ + (tokenCount_ - 1); - for (const Token *t = tokens_; t != last; ++t) { - switch (v->GetType()) { - case kObjectType: - { - typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); - if (m == v->MemberEnd()) - return false; - v = &m->value; - } - break; - case kArrayType: - if (t->index == kPointerInvalidIndex || t->index >= v->Size()) - return false; - v = &((*v)[t->index]); - break; - default: - return false; + while (i < length && source[i] != '/') + { Ch c = source[i]; + if (uriFragment) + { // Decoding percent-encoding for URI fragment + if (c == '%') + { PercentDecodeStream is(&source[i], source + length); + GenericInsituStringStream os(name); + Ch* begin = os.PutBegin(); + if (!Transcoder, EncodingType>().Validate(is, os) || !is.IsValid()) + { parseErrorCode_ = kPointerParseErrorInvalidPercentEncoding; + goto error; + } + size_t len = os.PutEnd(begin); + i += is.Tell() - 1; + if (len == 1) + c = *name; + else + { name += len; + isNumber = false; + i++; + continue; } + } + else if (NeedPercentEncode(c)) + { parseErrorCode_ = kPointerParseErrorCharacterMustPercentEncode; + goto error; + } } - switch (v->GetType()) { - case kObjectType: - return v->EraseMember(GenericStringRef(last->name, last->length)); - case kArrayType: - if (last->index == kPointerInvalidIndex || last->index >= v->Size()) - return false; - v->Erase(v->Begin() + last->index); - return true; - default: - return false; + i++; + + // Escaping "~0" -> '~', "~1" -> '/' + if (c == '~') + { if (i < length) + { c = source[i]; + if (c == '0') c = '~'; + else if (c == '1') c = '/'; + else + { parseErrorCode_ = kPointerParseErrorInvalidEscape; + goto error; + } + i++; + } + else + { parseErrorCode_ = kPointerParseErrorInvalidEscape; + goto error; + } } - } -private: - //! Clone the content from rhs to this. - /*! - \param rhs Source pointer. - \param extraToken Extra tokens to be allocated. - \param extraNameBufferSize Extra name buffer size (in number of Ch) to be allocated. - \return Start of non-occupied name buffer, for storing extra names. - */ - Ch* CopyFromRaw(const GenericPointer& rhs, size_t extraToken = 0, size_t extraNameBufferSize = 0) { - if (!allocator_) // allocator is independently owned. - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); - - size_t nameBufferSize = rhs.tokenCount_; // null terminators for tokens - for (Token *t = rhs.tokens_; t != rhs.tokens_ + rhs.tokenCount_; ++t) - nameBufferSize += t->length; - - tokenCount_ = rhs.tokenCount_ + extraToken; - tokens_ = static_cast(allocator_->Malloc(tokenCount_ * sizeof(Token) + (nameBufferSize + extraNameBufferSize) * sizeof(Ch))); - nameBuffer_ = reinterpret_cast(tokens_ + tokenCount_); - if (rhs.tokenCount_ > 0) { - std::memcpy(tokens_, rhs.tokens_, rhs.tokenCount_ * sizeof(Token)); + // First check for index: all of characters are digit + if (c < '0' || c > '9') + isNumber = false; + + *name++ = c; + } + token->length = static_cast(name - token->name); + if (token->length == 0) + isNumber = false; + *name++ = '\0'; // Null terminator + + // Second check for index: more than one digit cannot have leading zero + if (isNumber && token->length > 1 && token->name[0] == '0') + isNumber = false; + + // String to SizeType conversion + SizeType n = 0; + if (isNumber) + { for (size_t j = 0; j < token->length; j++) + { SizeType m = n * 10 + static_cast(token->name[j] - '0'); + if (m < n) // overflow detection + { isNumber = false; + break; + } + n = m; } - if (nameBufferSize > 0) { - std::memcpy(nameBuffer_, rhs.nameBuffer_, nameBufferSize * sizeof(Ch)); - } - - // Adjust pointers to name buffer - std::ptrdiff_t diff = nameBuffer_ - rhs.nameBuffer_; - for (Token *t = tokens_; t != tokens_ + rhs.tokenCount_; ++t) - t->name += diff; + } - return nameBuffer_ + nameBufferSize; + token->index = isNumber ? n : kPointerInvalidIndex; + token++; } - //! Check whether a character should be percent-encoded. - /*! - According to RFC 3986 2.3 Unreserved Characters. - \param c The character (code unit) to be tested. - */ - bool NeedPercentEncode(Ch c) const { - return !((c >= '0' && c <= '9') || (c >= 'A' && c <='Z') || (c >= 'a' && c <= 'z') || c == '-' || c == '.' || c == '_' || c =='~'); - } - - //! Parse a JSON String or its URI fragment representation into tokens. -#ifndef __clang__ // -Wdocumentation - /*! - \param source Either a JSON Pointer string, or its URI fragment representation. Not need to be null terminated. - \param length Length of the source string. - \note Source cannot be JSON String Representation of JSON Pointer, e.g. In "/\u0000", \u0000 will not be unescaped. - */ -#endif - void Parse(const Ch* source, size_t length) { - RAPIDJSON_ASSERT(source != NULL); - RAPIDJSON_ASSERT(nameBuffer_ == 0); - RAPIDJSON_ASSERT(tokens_ == 0); - - // Create own allocator if user did not supply. - if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); - - // Count number of '/' as tokenCount - tokenCount_ = 0; - for (const Ch* s = source; s != source + length; s++) - if (*s == '/') - tokenCount_++; - - Token* token = tokens_ = static_cast(allocator_->Malloc(tokenCount_ * sizeof(Token) + length * sizeof(Ch))); - Ch* name = nameBuffer_ = reinterpret_cast(tokens_ + tokenCount_); - size_t i = 0; - - // Detect if it is a URI fragment - bool uriFragment = false; - if (source[i] == '#') { - uriFragment = true; - i++; + RAPIDJSON_ASSERT(name <= nameBuffer_ + length); // Should not overflow buffer + parseErrorCode_ = kPointerParseErrorNone; + return; + +error: + Allocator::Free(tokens_); + nameBuffer_ = 0; + tokens_ = 0; + tokenCount_ = 0; + parseErrorOffset_ = i; + return; + } + + //! Stringify to string or URI fragment representation. + /*! + \tparam uriFragment True for stringifying to URI fragment representation. False for string representation. + \tparam OutputStream type of output stream. + \param os The output stream. + */ + template + bool Stringify(OutputStream& os) const + { RAPIDJSON_ASSERT(IsValid()); + + if (uriFragment) + os.Put('#'); + + for (Token *t = tokens_; t != tokens_ + tokenCount_; ++t) + { os.Put('/'); + for (size_t j = 0; j < t->length; j++) + { Ch c = t->name[j]; + if (c == '~') + { os.Put('~'); + os.Put('0'); } - - if (i != length && source[i] != '/') { - parseErrorCode_ = kPointerParseErrorTokenMustBeginWithSolidus; - goto error; + else if (c == '/') + { os.Put('~'); + os.Put('1'); } - - while (i < length) { - RAPIDJSON_ASSERT(source[i] == '/'); - i++; // consumes '/' - - token->name = name; - bool isNumber = true; - - while (i < length && source[i] != '/') { - Ch c = source[i]; - if (uriFragment) { - // Decoding percent-encoding for URI fragment - if (c == '%') { - PercentDecodeStream is(&source[i], source + length); - GenericInsituStringStream os(name); - Ch* begin = os.PutBegin(); - if (!Transcoder, EncodingType>().Validate(is, os) || !is.IsValid()) { - parseErrorCode_ = kPointerParseErrorInvalidPercentEncoding; - goto error; - } - size_t len = os.PutEnd(begin); - i += is.Tell() - 1; - if (len == 1) - c = *name; - else { - name += len; - isNumber = false; - i++; - continue; - } - } - else if (NeedPercentEncode(c)) { - parseErrorCode_ = kPointerParseErrorCharacterMustPercentEncode; - goto error; - } - } - - i++; - - // Escaping "~0" -> '~', "~1" -> '/' - if (c == '~') { - if (i < length) { - c = source[i]; - if (c == '0') c = '~'; - else if (c == '1') c = '/'; - else { - parseErrorCode_ = kPointerParseErrorInvalidEscape; - goto error; - } - i++; - } - else { - parseErrorCode_ = kPointerParseErrorInvalidEscape; - goto error; - } - } - - // First check for index: all of characters are digit - if (c < '0' || c > '9') - isNumber = false; - - *name++ = c; - } - token->length = static_cast(name - token->name); - if (token->length == 0) - isNumber = false; - *name++ = '\0'; // Null terminator - - // Second check for index: more than one digit cannot have leading zero - if (isNumber && token->length > 1 && token->name[0] == '0') - isNumber = false; - - // String to SizeType conversion - SizeType n = 0; - if (isNumber) { - for (size_t j = 0; j < token->length; j++) { - SizeType m = n * 10 + static_cast(token->name[j] - '0'); - if (m < n) { // overflow detection - isNumber = false; - break; - } - n = m; - } - } - - token->index = isNumber ? n : kPointerInvalidIndex; - token++; + else if (uriFragment && NeedPercentEncode(c)) + { // Transcode to UTF8 sequence + GenericStringStream source(&t->name[j]); + PercentEncodeStream target(os); + if (!Transcoder >().Validate(source, target)) + return false; + j += source.Tell() - 1; } - - RAPIDJSON_ASSERT(name <= nameBuffer_ + length); // Should not overflow buffer - parseErrorCode_ = kPointerParseErrorNone; - return; - - error: - Allocator::Free(tokens_); - nameBuffer_ = 0; - tokens_ = 0; - tokenCount_ = 0; - parseErrorOffset_ = i; - return; + else + os.Put(c); + } } - - //! Stringify to string or URI fragment representation. + return true; + } + + //! A helper stream for decoding a percent-encoded sequence into code unit. + /*! + This stream decodes %XY triplet into code unit (0-255). + If it encounters invalid characters, it sets output code unit as 0 and + mark invalid, and to be checked by IsValid(). + */ + class PercentDecodeStream + { + public: + typedef typename ValueType::Ch Ch; + + //! Constructor /*! - \tparam uriFragment True for stringifying to URI fragment representation. False for string representation. - \tparam OutputStream type of output stream. - \param os The output stream. + \param source Start of the stream + \param end Past-the-end of the stream. */ - template - bool Stringify(OutputStream& os) const { - RAPIDJSON_ASSERT(IsValid()); - - if (uriFragment) - os.Put('#'); - - for (Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { - os.Put('/'); - for (size_t j = 0; j < t->length; j++) { - Ch c = t->name[j]; - if (c == '~') { - os.Put('~'); - os.Put('0'); - } - else if (c == '/') { - os.Put('~'); - os.Put('1'); - } - else if (uriFragment && NeedPercentEncode(c)) { - // Transcode to UTF8 sequence - GenericStringStream source(&t->name[j]); - PercentEncodeStream target(os); - if (!Transcoder >().Validate(source, target)) - return false; - j += source.Tell() - 1; - } - else - os.Put(c); - } + PercentDecodeStream(const Ch* source, const Ch* end) : src_(source), head_(source), end_(end), valid_(true) {} + + Ch Take() + { if (*src_ != '%' || src_ + 3 > end_) // %XY triplet + { valid_ = false; + return 0; + } + src_++; + Ch c = 0; + for (int j = 0; j < 2; j++) + { c = static_cast(c << 4); + Ch h = *src_; + if (h >= '0' && h <= '9') c = static_cast(c + h - '0'); + else if (h >= 'A' && h <= 'F') c = static_cast(c + h - 'A' + 10); + else if (h >= 'a' && h <= 'f') c = static_cast(c + h - 'a' + 10); + else + { valid_ = false; + return 0; } - return true; + src_++; + } + return c; } - //! A helper stream for decoding a percent-encoded sequence into code unit. - /*! - This stream decodes %XY triplet into code unit (0-255). - If it encounters invalid characters, it sets output code unit as 0 and - mark invalid, and to be checked by IsValid(). - */ - class PercentDecodeStream { - public: - typedef typename ValueType::Ch Ch; - - //! Constructor - /*! - \param source Start of the stream - \param end Past-the-end of the stream. - */ - PercentDecodeStream(const Ch* source, const Ch* end) : src_(source), head_(source), end_(end), valid_(true) {} - - Ch Take() { - if (*src_ != '%' || src_ + 3 > end_) { // %XY triplet - valid_ = false; - return 0; - } - src_++; - Ch c = 0; - for (int j = 0; j < 2; j++) { - c = static_cast(c << 4); - Ch h = *src_; - if (h >= '0' && h <= '9') c = static_cast(c + h - '0'); - else if (h >= 'A' && h <= 'F') c = static_cast(c + h - 'A' + 10); - else if (h >= 'a' && h <= 'f') c = static_cast(c + h - 'a' + 10); - else { - valid_ = false; - return 0; - } - src_++; - } - return c; - } - - size_t Tell() const { return static_cast(src_ - head_); } - bool IsValid() const { return valid_; } - - private: - const Ch* src_; //!< Current read position. - const Ch* head_; //!< Original head of the string. - const Ch* end_; //!< Past-the-end position. - bool valid_; //!< Whether the parsing is valid. - }; - - //! A helper stream to encode character (UTF-8 code unit) into percent-encoded sequence. - template - class PercentEncodeStream { - public: - PercentEncodeStream(OutputStream& os) : os_(os) {} - void Put(char c) { // UTF-8 must be byte - unsigned char u = static_cast(c); - static const char hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; - os_.Put('%'); - os_.Put(hexDigits[u >> 4]); - os_.Put(hexDigits[u & 15]); - } - private: - OutputStream& os_; - }; - - Allocator* allocator_; //!< The current allocator. It is either user-supplied or equal to ownAllocator_. - Allocator* ownAllocator_; //!< Allocator owned by this Pointer. - Ch* nameBuffer_; //!< A buffer containing all names in tokens. - Token* tokens_; //!< A list of tokens. - size_t tokenCount_; //!< Number of tokens in tokens_. - size_t parseErrorOffset_; //!< Offset in code unit when parsing fail. - PointerParseErrorCode parseErrorCode_; //!< Parsing error code. + size_t Tell() const { return static_cast(src_ - head_); } + bool IsValid() const { return valid_; } + + private: + const Ch* src_; //!< Current read position. + const Ch* head_; //!< Original head of the string. + const Ch* end_; //!< Past-the-end position. + bool valid_; //!< Whether the parsing is valid. + }; + + //! A helper stream to encode character (UTF-8 code unit) into percent-encoded sequence. + template + class PercentEncodeStream + { + public: + PercentEncodeStream(OutputStream& os) : os_(os) {} + void Put(char c) // UTF-8 must be byte + { unsigned char u = static_cast(c); + static const char hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; + os_.Put('%'); + os_.Put(hexDigits[u >> 4]); + os_.Put(hexDigits[u & 15]); + } + private: + OutputStream& os_; + }; + + Allocator* allocator_; //!< The current allocator. It is either user-supplied or equal to ownAllocator_. + Allocator* ownAllocator_; //!< Allocator owned by this Pointer. + Ch* nameBuffer_; //!< A buffer containing all names in tokens. + Token* tokens_; //!< A list of tokens. + size_t tokenCount_; //!< Number of tokens in tokens_. + size_t parseErrorOffset_; //!< Offset in code unit when parsing fail. + PointerParseErrorCode parseErrorCode_; //!< Parsing error code. }; //! GenericPointer for Value (UTF-8, default allocator). @@ -1054,293 +1054,293 @@ typedef GenericPointer Pointer; ////////////////////////////////////////////////////////////////////////////// template -typename T::ValueType& CreateValueByPointer(T& root, const GenericPointer& pointer, typename T::AllocatorType& a) { - return pointer.Create(root, a); +typename T::ValueType& CreateValueByPointer(T& root, const GenericPointer& pointer, typename T::AllocatorType& a) +{ return pointer.Create(root, a); } template -typename T::ValueType& CreateValueByPointer(T& root, const CharType(&source)[N], typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).Create(root, a); +typename T::ValueType& CreateValueByPointer(T& root, const CharType(&source)[N], typename T::AllocatorType& a) +{ return GenericPointer(source, N - 1).Create(root, a); } // No allocator parameter template -typename DocumentType::ValueType& CreateValueByPointer(DocumentType& document, const GenericPointer& pointer) { - return pointer.Create(document); +typename DocumentType::ValueType& CreateValueByPointer(DocumentType& document, const GenericPointer& pointer) +{ return pointer.Create(document); } template -typename DocumentType::ValueType& CreateValueByPointer(DocumentType& document, const CharType(&source)[N]) { - return GenericPointer(source, N - 1).Create(document); +typename DocumentType::ValueType& CreateValueByPointer(DocumentType& document, const CharType(&source)[N]) +{ return GenericPointer(source, N - 1).Create(document); } ////////////////////////////////////////////////////////////////////////////// template -typename T::ValueType* GetValueByPointer(T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) { - return pointer.Get(root, unresolvedTokenIndex); +typename T::ValueType* GetValueByPointer(T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) +{ return pointer.Get(root, unresolvedTokenIndex); } template -const typename T::ValueType* GetValueByPointer(const T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) { - return pointer.Get(root, unresolvedTokenIndex); +const typename T::ValueType* GetValueByPointer(const T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) +{ return pointer.Get(root, unresolvedTokenIndex); } template -typename T::ValueType* GetValueByPointer(T& root, const CharType (&source)[N], size_t* unresolvedTokenIndex = 0) { - return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); +typename T::ValueType* GetValueByPointer(T& root, const CharType (&source)[N], size_t* unresolvedTokenIndex = 0) +{ return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); } template -const typename T::ValueType* GetValueByPointer(const T& root, const CharType(&source)[N], size_t* unresolvedTokenIndex = 0) { - return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); +const typename T::ValueType* GetValueByPointer(const T& root, const CharType(&source)[N], size_t* unresolvedTokenIndex = 0) +{ return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); } ////////////////////////////////////////////////////////////////////////////// template -typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const typename T::ValueType& defaultValue, typename T::AllocatorType& a) { - return pointer.GetWithDefault(root, defaultValue, a); +typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const typename T::ValueType& defaultValue, typename T::AllocatorType& a) +{ return pointer.GetWithDefault(root, defaultValue, a); } template -typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const typename T::Ch* defaultValue, typename T::AllocatorType& a) { - return pointer.GetWithDefault(root, defaultValue, a); +typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const typename T::Ch* defaultValue, typename T::AllocatorType& a) +{ return pointer.GetWithDefault(root, defaultValue, a); } #if RAPIDJSON_HAS_STDSTRING template -typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const std::basic_string& defaultValue, typename T::AllocatorType& a) { - return pointer.GetWithDefault(root, defaultValue, a); +typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const std::basic_string& defaultValue, typename T::AllocatorType& a) +{ return pointer.GetWithDefault(root, defaultValue, a); } #endif template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) -GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, T2 defaultValue, typename T::AllocatorType& a) { - return pointer.GetWithDefault(root, defaultValue, a); +GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, T2 defaultValue, typename T::AllocatorType& a) +{ return pointer.GetWithDefault(root, defaultValue, a); } template -typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const typename T::ValueType& defaultValue, typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); +typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const typename T::ValueType& defaultValue, typename T::AllocatorType& a) +{ return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); } template -typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const typename T::Ch* defaultValue, typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); +typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const typename T::Ch* defaultValue, typename T::AllocatorType& a) +{ return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); } #if RAPIDJSON_HAS_STDSTRING template -typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const std::basic_string& defaultValue, typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); +typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const std::basic_string& defaultValue, typename T::AllocatorType& a) +{ return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); } #endif template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) -GetValueByPointerWithDefault(T& root, const CharType(&source)[N], T2 defaultValue, typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); +GetValueByPointerWithDefault(T& root, const CharType(&source)[N], T2 defaultValue, typename T::AllocatorType& a) +{ return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); } // No allocator parameter template -typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::ValueType& defaultValue) { - return pointer.GetWithDefault(document, defaultValue); +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::ValueType& defaultValue) +{ return pointer.GetWithDefault(document, defaultValue); } template -typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::Ch* defaultValue) { - return pointer.GetWithDefault(document, defaultValue); +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::Ch* defaultValue) +{ return pointer.GetWithDefault(document, defaultValue); } #if RAPIDJSON_HAS_STDSTRING template -typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const std::basic_string& defaultValue) { - return pointer.GetWithDefault(document, defaultValue); +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const std::basic_string& defaultValue) +{ return pointer.GetWithDefault(document, defaultValue); } #endif template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) -GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, T2 defaultValue) { - return pointer.GetWithDefault(document, defaultValue); +GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, T2 defaultValue) +{ return pointer.GetWithDefault(document, defaultValue); } template -typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const typename DocumentType::ValueType& defaultValue) { - return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const typename DocumentType::ValueType& defaultValue) +{ return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); } template -typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const typename DocumentType::Ch* defaultValue) { - return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const typename DocumentType::Ch* defaultValue) +{ return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); } #if RAPIDJSON_HAS_STDSTRING template -typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const std::basic_string& defaultValue) { - return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const std::basic_string& defaultValue) +{ return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); } #endif template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) -GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], T2 defaultValue) { - return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); +GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], T2 defaultValue) +{ return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); } ////////////////////////////////////////////////////////////////////////////// template -typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, typename T::ValueType& value, typename T::AllocatorType& a) { - return pointer.Set(root, value, a); +typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, typename T::ValueType& value, typename T::AllocatorType& a) +{ return pointer.Set(root, value, a); } template -typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const typename T::ValueType& value, typename T::AllocatorType& a) { - return pointer.Set(root, value, a); +typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const typename T::ValueType& value, typename T::AllocatorType& a) +{ return pointer.Set(root, value, a); } template -typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const typename T::Ch* value, typename T::AllocatorType& a) { - return pointer.Set(root, value, a); +typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const typename T::Ch* value, typename T::AllocatorType& a) +{ return pointer.Set(root, value, a); } #if RAPIDJSON_HAS_STDSTRING template -typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const std::basic_string& value, typename T::AllocatorType& a) { - return pointer.Set(root, value, a); +typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const std::basic_string& value, typename T::AllocatorType& a) +{ return pointer.Set(root, value, a); } #endif template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) -SetValueByPointer(T& root, const GenericPointer& pointer, T2 value, typename T::AllocatorType& a) { - return pointer.Set(root, value, a); +SetValueByPointer(T& root, const GenericPointer& pointer, T2 value, typename T::AllocatorType& a) +{ return pointer.Set(root, value, a); } template -typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], typename T::ValueType& value, typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).Set(root, value, a); +typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], typename T::ValueType& value, typename T::AllocatorType& a) +{ return GenericPointer(source, N - 1).Set(root, value, a); } template -typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const typename T::ValueType& value, typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).Set(root, value, a); +typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const typename T::ValueType& value, typename T::AllocatorType& a) +{ return GenericPointer(source, N - 1).Set(root, value, a); } template -typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const typename T::Ch* value, typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).Set(root, value, a); +typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const typename T::Ch* value, typename T::AllocatorType& a) +{ return GenericPointer(source, N - 1).Set(root, value, a); } #if RAPIDJSON_HAS_STDSTRING template -typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const std::basic_string& value, typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).Set(root, value, a); +typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const std::basic_string& value, typename T::AllocatorType& a) +{ return GenericPointer(source, N - 1).Set(root, value, a); } #endif template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) -SetValueByPointer(T& root, const CharType(&source)[N], T2 value, typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).Set(root, value, a); +SetValueByPointer(T& root, const CharType(&source)[N], T2 value, typename T::AllocatorType& a) +{ return GenericPointer(source, N - 1).Set(root, value, a); } // No allocator parameter template -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, typename DocumentType::ValueType& value) { - return pointer.Set(document, value); +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, typename DocumentType::ValueType& value) +{ return pointer.Set(document, value); } template -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::ValueType& value) { - return pointer.Set(document, value); +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::ValueType& value) +{ return pointer.Set(document, value); } template -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::Ch* value) { - return pointer.Set(document, value); +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::Ch* value) +{ return pointer.Set(document, value); } #if RAPIDJSON_HAS_STDSTRING template -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const std::basic_string& value) { - return pointer.Set(document, value); +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const std::basic_string& value) +{ return pointer.Set(document, value); } #endif template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) -SetValueByPointer(DocumentType& document, const GenericPointer& pointer, T2 value) { - return pointer.Set(document, value); +SetValueByPointer(DocumentType& document, const GenericPointer& pointer, T2 value) +{ return pointer.Set(document, value); } template -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], typename DocumentType::ValueType& value) { - return GenericPointer(source, N - 1).Set(document, value); +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], typename DocumentType::ValueType& value) +{ return GenericPointer(source, N - 1).Set(document, value); } template -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const typename DocumentType::ValueType& value) { - return GenericPointer(source, N - 1).Set(document, value); +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const typename DocumentType::ValueType& value) +{ return GenericPointer(source, N - 1).Set(document, value); } template -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const typename DocumentType::Ch* value) { - return GenericPointer(source, N - 1).Set(document, value); +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const typename DocumentType::Ch* value) +{ return GenericPointer(source, N - 1).Set(document, value); } #if RAPIDJSON_HAS_STDSTRING template -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const std::basic_string& value) { - return GenericPointer(source, N - 1).Set(document, value); +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const std::basic_string& value) +{ return GenericPointer(source, N - 1).Set(document, value); } #endif template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) -SetValueByPointer(DocumentType& document, const CharType(&source)[N], T2 value) { - return GenericPointer(source, N - 1).Set(document, value); +SetValueByPointer(DocumentType& document, const CharType(&source)[N], T2 value) +{ return GenericPointer(source, N - 1).Set(document, value); } ////////////////////////////////////////////////////////////////////////////// template -typename T::ValueType& SwapValueByPointer(T& root, const GenericPointer& pointer, typename T::ValueType& value, typename T::AllocatorType& a) { - return pointer.Swap(root, value, a); +typename T::ValueType& SwapValueByPointer(T& root, const GenericPointer& pointer, typename T::ValueType& value, typename T::AllocatorType& a) +{ return pointer.Swap(root, value, a); } template -typename T::ValueType& SwapValueByPointer(T& root, const CharType(&source)[N], typename T::ValueType& value, typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).Swap(root, value, a); +typename T::ValueType& SwapValueByPointer(T& root, const CharType(&source)[N], typename T::ValueType& value, typename T::AllocatorType& a) +{ return GenericPointer(source, N - 1).Swap(root, value, a); } template -typename DocumentType::ValueType& SwapValueByPointer(DocumentType& document, const GenericPointer& pointer, typename DocumentType::ValueType& value) { - return pointer.Swap(document, value); +typename DocumentType::ValueType& SwapValueByPointer(DocumentType& document, const GenericPointer& pointer, typename DocumentType::ValueType& value) +{ return pointer.Swap(document, value); } template -typename DocumentType::ValueType& SwapValueByPointer(DocumentType& document, const CharType(&source)[N], typename DocumentType::ValueType& value) { - return GenericPointer(source, N - 1).Swap(document, value); +typename DocumentType::ValueType& SwapValueByPointer(DocumentType& document, const CharType(&source)[N], typename DocumentType::ValueType& value) +{ return GenericPointer(source, N - 1).Swap(document, value); } ////////////////////////////////////////////////////////////////////////////// template -bool EraseValueByPointer(T& root, const GenericPointer& pointer) { - return pointer.Erase(root); +bool EraseValueByPointer(T& root, const GenericPointer& pointer) +{ return pointer.Erase(root); } template -bool EraseValueByPointer(T& root, const CharType(&source)[N]) { - return GenericPointer(source, N - 1).Erase(root); +bool EraseValueByPointer(T& root, const CharType(&source)[N]) +{ return GenericPointer(source, N - 1).Erase(root); } //@} diff --git a/rapidjson/include/rapidjson/prettywriter.h b/rapidjson/include/rapidjson/prettywriter.h index 0dcb0fee923..5d1377dff00 100644 --- a/rapidjson/include/rapidjson/prettywriter.h +++ b/rapidjson/include/rapidjson/prettywriter.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_PRETTYWRITER_H_ @@ -27,9 +27,9 @@ RAPIDJSON_NAMESPACE_BEGIN //! Combination of PrettyWriter format flags. /*! \see PrettyWriter::SetFormatOptions */ -enum PrettyFormatOptions { - kFormatDefault = 0, //!< Default pretty formatting. - kFormatSingleLineArray = 1 //!< Format arrays on a single line. +enum PrettyFormatOptions +{ kFormatDefault = 0, //!< Default pretty formatting. + kFormatSingleLineArray = 1 //!< Format arrays on a single line. }; //! Writer with indentation and spacing. @@ -40,210 +40,211 @@ enum PrettyFormatOptions { \tparam StackAllocator Type of allocator for allocating memory of stack. */ template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> -class PrettyWriter : public Writer { +class PrettyWriter : public Writer +{ public: - typedef Writer Base; - typedef typename Base::Ch Ch; - - //! Constructor - /*! \param os Output stream. - \param allocator User supplied allocator. If it is null, it will create a private one. - \param levelDepth Initial capacity of stack. - */ - explicit PrettyWriter(OutputStream& os, StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : - Base(os, allocator, levelDepth), indentChar_(' '), indentCharCount_(4), formatOptions_(kFormatDefault) {} - - - explicit PrettyWriter(StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : - Base(allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} - - //! Set custom indentation. - /*! \param indentChar Character for indentation. Must be whitespace character (' ', '\\t', '\\n', '\\r'). - \param indentCharCount Number of indent characters for each indentation level. - \note The default indentation is 4 spaces. - */ - PrettyWriter& SetIndent(Ch indentChar, unsigned indentCharCount) { - RAPIDJSON_ASSERT(indentChar == ' ' || indentChar == '\t' || indentChar == '\n' || indentChar == '\r'); - indentChar_ = indentChar; - indentCharCount_ = indentCharCount; - return *this; - } - - //! Set pretty writer formatting options. - /*! \param options Formatting options. - */ - PrettyWriter& SetFormatOptions(PrettyFormatOptions options) { - formatOptions_ = options; - return *this; - } - - /*! @name Implementation of Handler - \see Handler - */ - //@{ - - bool Null() { PrettyPrefix(kNullType); return Base::WriteNull(); } - bool Bool(bool b) { PrettyPrefix(b ? kTrueType : kFalseType); return Base::WriteBool(b); } - bool Int(int i) { PrettyPrefix(kNumberType); return Base::WriteInt(i); } - bool Uint(unsigned u) { PrettyPrefix(kNumberType); return Base::WriteUint(u); } - bool Int64(int64_t i64) { PrettyPrefix(kNumberType); return Base::WriteInt64(i64); } - bool Uint64(uint64_t u64) { PrettyPrefix(kNumberType); return Base::WriteUint64(u64); } - bool Double(double d) { PrettyPrefix(kNumberType); return Base::WriteDouble(d); } - - bool RawNumber(const Ch* str, SizeType length, bool copy = false) { - (void)copy; - PrettyPrefix(kNumberType); - return Base::WriteString(str, length); - } - - bool String(const Ch* str, SizeType length, bool copy = false) { - (void)copy; - PrettyPrefix(kStringType); - return Base::WriteString(str, length); - } + typedef Writer Base; + typedef typename Base::Ch Ch; + + //! Constructor + /*! \param os Output stream. + \param allocator User supplied allocator. If it is null, it will create a private one. + \param levelDepth Initial capacity of stack. + */ + explicit PrettyWriter(OutputStream& os, StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : + Base(os, allocator, levelDepth), indentChar_(' '), indentCharCount_(4), formatOptions_(kFormatDefault) {} + + + explicit PrettyWriter(StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : + Base(allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} + + //! Set custom indentation. + /*! \param indentChar Character for indentation. Must be whitespace character (' ', '\\t', '\\n', '\\r'). + \param indentCharCount Number of indent characters for each indentation level. + \note The default indentation is 4 spaces. + */ + PrettyWriter& SetIndent(Ch indentChar, unsigned indentCharCount) + { RAPIDJSON_ASSERT(indentChar == ' ' || indentChar == '\t' || indentChar == '\n' || indentChar == '\r'); + indentChar_ = indentChar; + indentCharCount_ = indentCharCount; + return *this; + } + + //! Set pretty writer formatting options. + /*! \param options Formatting options. + */ + PrettyWriter& SetFormatOptions(PrettyFormatOptions options) + { formatOptions_ = options; + return *this; + } + + /*! @name Implementation of Handler + \see Handler + */ + //@{ + + bool Null() { PrettyPrefix(kNullType); return Base::WriteNull(); } + bool Bool(bool b) { PrettyPrefix(b ? kTrueType : kFalseType); return Base::WriteBool(b); } + bool Int(int i) { PrettyPrefix(kNumberType); return Base::WriteInt(i); } + bool Uint(unsigned u) { PrettyPrefix(kNumberType); return Base::WriteUint(u); } + bool Int64(int64_t i64) { PrettyPrefix(kNumberType); return Base::WriteInt64(i64); } + bool Uint64(uint64_t u64) { PrettyPrefix(kNumberType); return Base::WriteUint64(u64); } + bool Double(double d) { PrettyPrefix(kNumberType); return Base::WriteDouble(d); } + + bool RawNumber(const Ch* str, SizeType length, bool copy = false) + { (void)copy; + PrettyPrefix(kNumberType); + return Base::WriteString(str, length); + } + + bool String(const Ch* str, SizeType length, bool copy = false) + { (void)copy; + PrettyPrefix(kStringType); + return Base::WriteString(str, length); + } #if RAPIDJSON_HAS_STDSTRING - bool String(const std::basic_string& str) { - return String(str.data(), SizeType(str.size())); - } + bool String(const std::basic_string& str) + { return String(str.data(), SizeType(str.size())); + } #endif - bool StartObject() { - PrettyPrefix(kObjectType); - new (Base::level_stack_.template Push()) typename Base::Level(false); - return Base::WriteStartObject(); - } + bool StartObject() + { PrettyPrefix(kObjectType); + new (Base::level_stack_.template Push()) typename Base::Level(false); + return Base::WriteStartObject(); + } - bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } + bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } #if RAPIDJSON_HAS_STDSTRING - bool Key(const std::basic_string& str) { - return Key(str.data(), SizeType(str.size())); - } + bool Key(const std::basic_string& str) + { return Key(str.data(), SizeType(str.size())); + } #endif - - bool EndObject(SizeType memberCount = 0) { - (void)memberCount; - RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); - RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); - bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; - - if (!empty) { - Base::os_->Put('\n'); - WriteIndent(); - } - bool ret = Base::WriteEndObject(); - (void)ret; - RAPIDJSON_ASSERT(ret == true); - if (Base::level_stack_.Empty()) // end of json text - Base::os_->Flush(); - return true; - } - - bool StartArray() { - PrettyPrefix(kArrayType); - new (Base::level_stack_.template Push()) typename Base::Level(true); - return Base::WriteStartArray(); - } - bool EndArray(SizeType memberCount = 0) { - (void)memberCount; - RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); - RAPIDJSON_ASSERT(Base::level_stack_.template Top()->inArray); - bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; + bool EndObject(SizeType memberCount = 0) + { (void)memberCount; + RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); + RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); + bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; - if (!empty && !(formatOptions_ & kFormatSingleLineArray)) { - Base::os_->Put('\n'); - WriteIndent(); - } - bool ret = Base::WriteEndArray(); - (void)ret; - RAPIDJSON_ASSERT(ret == true); - if (Base::level_stack_.Empty()) // end of json text - Base::os_->Flush(); - return true; + if (!empty) + { Base::os_->Put('\n'); + WriteIndent(); } + bool ret = Base::WriteEndObject(); + (void)ret; + RAPIDJSON_ASSERT(ret == true); + if (Base::level_stack_.Empty()) // end of json text + Base::os_->Flush(); + return true; + } + + bool StartArray() + { PrettyPrefix(kArrayType); + new (Base::level_stack_.template Push()) typename Base::Level(true); + return Base::WriteStartArray(); + } + + bool EndArray(SizeType memberCount = 0) + { (void)memberCount; + RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); + RAPIDJSON_ASSERT(Base::level_stack_.template Top()->inArray); + bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; + + if (!empty && !(formatOptions_ & kFormatSingleLineArray)) + { Base::os_->Put('\n'); + WriteIndent(); + } + bool ret = Base::WriteEndArray(); + (void)ret; + RAPIDJSON_ASSERT(ret == true); + if (Base::level_stack_.Empty()) // end of json text + Base::os_->Flush(); + return true; + } - //@} + //@} - /*! @name Convenience extensions */ - //@{ + /*! @name Convenience extensions */ + //@{ - //! Simpler but slower overload. - bool String(const Ch* str) { return String(str, internal::StrLen(str)); } - bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } + //! Simpler but slower overload. + bool String(const Ch* str) { return String(str, internal::StrLen(str)); } + bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } - //@} + //@} - //! Write a raw JSON value. - /*! - For user to write a stringified JSON as a value. + //! Write a raw JSON value. + /*! + For user to write a stringified JSON as a value. - \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. - \param length Length of the json. - \param type Type of the root of json. - \note When using PrettyWriter::RawValue(), the result json may not be indented correctly. - */ - bool RawValue(const Ch* json, size_t length, Type type) { PrettyPrefix(type); return Base::WriteRawValue(json, length); } + \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. + \param length Length of the json. + \param type Type of the root of json. + \note When using PrettyWriter::RawValue(), the result json may not be indented correctly. + */ + bool RawValue(const Ch* json, size_t length, Type type) { PrettyPrefix(type); return Base::WriteRawValue(json, length); } protected: - void PrettyPrefix(Type type) { - (void)type; - if (Base::level_stack_.GetSize() != 0) { // this value is not at root - typename Base::Level* level = Base::level_stack_.template Top(); - - if (level->inArray) { - if (level->valueCount > 0) { - Base::os_->Put(','); // add comma if it is not the first element in array - if (formatOptions_ & kFormatSingleLineArray) - Base::os_->Put(' '); - } - - if (!(formatOptions_ & kFormatSingleLineArray)) { - Base::os_->Put('\n'); - WriteIndent(); - } - } - else { // in object - if (level->valueCount > 0) { - if (level->valueCount % 2 == 0) { - Base::os_->Put(','); - Base::os_->Put('\n'); - } - else { - Base::os_->Put(':'); - Base::os_->Put(' '); - } - } - else - Base::os_->Put('\n'); - - if (level->valueCount % 2 == 0) - WriteIndent(); - } - if (!level->inArray && level->valueCount % 2 == 0) - RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name - level->valueCount++; + void PrettyPrefix(Type type) + { (void)type; + if (Base::level_stack_.GetSize() != 0) // this value is not at root + { typename Base::Level* level = Base::level_stack_.template Top(); + + if (level->inArray) + { if (level->valueCount > 0) + { Base::os_->Put(','); // add comma if it is not the first element in array + if (formatOptions_ & kFormatSingleLineArray) + Base::os_->Put(' '); + } + + if (!(formatOptions_ & kFormatSingleLineArray)) + { Base::os_->Put('\n'); + WriteIndent(); } - else { - RAPIDJSON_ASSERT(!Base::hasRoot_); // Should only has one and only one root. - Base::hasRoot_ = true; + } + else // in object + { if (level->valueCount > 0) + { if (level->valueCount % 2 == 0) + { Base::os_->Put(','); + Base::os_->Put('\n'); + } + else + { Base::os_->Put(':'); + Base::os_->Put(' '); + } } + else + Base::os_->Put('\n'); + + if (level->valueCount % 2 == 0) + WriteIndent(); + } + if (!level->inArray && level->valueCount % 2 == 0) + RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name + level->valueCount++; } - - void WriteIndent() { - size_t count = (Base::level_stack_.GetSize() / sizeof(typename Base::Level)) * indentCharCount_; - PutN(*Base::os_, static_cast(indentChar_), count); + else + { RAPIDJSON_ASSERT(!Base::hasRoot_); // Should only has one and only one root. + Base::hasRoot_ = true; } + } + + void WriteIndent() + { size_t count = (Base::level_stack_.GetSize() / sizeof(typename Base::Level)) * indentCharCount_; + PutN(*Base::os_, static_cast(indentChar_), count); + } - Ch indentChar_; - unsigned indentCharCount_; - PrettyFormatOptions formatOptions_; + Ch indentChar_; + unsigned indentCharCount_; + PrettyFormatOptions formatOptions_; private: - // Prohibit copy constructor & assignment operator. - PrettyWriter(const PrettyWriter&); - PrettyWriter& operator=(const PrettyWriter&); + // Prohibit copy constructor & assignment operator. + PrettyWriter(const PrettyWriter&); + PrettyWriter& operator=(const PrettyWriter&); }; RAPIDJSON_NAMESPACE_END diff --git a/rapidjson/include/rapidjson/rapidjson.h b/rapidjson/include/rapidjson/rapidjson.h index 053b2ce43f9..e3c1509bb9e 100644 --- a/rapidjson/include/rapidjson/rapidjson.h +++ b/rapidjson/include/rapidjson/rapidjson.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_RAPIDJSON_H_ @@ -17,7 +17,7 @@ /*!\file rapidjson.h \brief common definitions and configuration - + \see RAPIDJSON_CONFIG */ @@ -241,7 +241,7 @@ # elif defined(RAPIDJSON_DOXYGEN_RUNNING) # define RAPIDJSON_ENDIAN # else -# error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. +# error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. # endif #endif // RAPIDJSON_ENDIAN @@ -423,7 +423,7 @@ RAPIDJSON_NAMESPACE_END #if defined(__GNUC__) #define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE __attribute__((unused)) #else -#define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE +#define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE #endif #ifndef __clang__ //!@endcond @@ -474,7 +474,7 @@ RAPIDJSON_NAMESPACE_END //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#define RAPIDJSON_MULTILINEMACRO_BEGIN do { +#define RAPIDJSON_MULTILINEMACRO_BEGIN do { #define RAPIDJSON_MULTILINEMACRO_END \ } while((void)0, 0) @@ -600,14 +600,14 @@ RAPIDJSON_NAMESPACE_END RAPIDJSON_NAMESPACE_BEGIN //! Type of JSON value -enum Type { - kNullType = 0, //!< null - kFalseType = 1, //!< false - kTrueType = 2, //!< true - kObjectType = 3, //!< object - kArrayType = 4, //!< array - kStringType = 5, //!< string - kNumberType = 6 //!< number +enum Type +{ kNullType = 0, //!< null + kFalseType = 1, //!< false + kTrueType = 2, //!< true + kObjectType = 3, //!< object + kArrayType = 4, //!< array + kStringType = 5, //!< string + kNumberType = 6 //!< number }; RAPIDJSON_NAMESPACE_END diff --git a/rapidjson/include/rapidjson/reader.h b/rapidjson/include/rapidjson/reader.h index 19f8849b14c..48c41d52c70 100644 --- a/rapidjson/include/rapidjson/reader.h +++ b/rapidjson/include/rapidjson/reader.h @@ -49,8 +49,8 @@ RAPIDJSON_DIAG_OFF(switch-enum) #endif #ifdef __GNUC__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(effc++) + RAPIDJSON_DIAG_PUSH + RAPIDJSON_DIAG_OFF(effc++) #endif //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN @@ -65,36 +65,36 @@ RAPIDJSON_DIAG_OFF(effc++) RAPIDJSON_PARSE_ERROR_EARLY_RETURN(RAPIDJSON_NOTHING) //!@endcond -/*! \def RAPIDJSON_PARSE_ERROR_NORETURN - \ingroup RAPIDJSON_ERRORS - \brief Macro to indicate a parse error. - \param parseErrorCode \ref rapidjson::ParseErrorCode of the error - \param offset position of the error in JSON input (\c size_t) + /*! \def RAPIDJSON_PARSE_ERROR_NORETURN + \ingroup RAPIDJSON_ERRORS + \brief Macro to indicate a parse error. + \param parseErrorCode \ref rapidjson::ParseErrorCode of the error + \param offset position of the error in JSON input (\c size_t) - This macros can be used as a customization point for the internal - error handling mechanism of RapidJSON. + This macros can be used as a customization point for the internal + error handling mechanism of RapidJSON. - A common usage model is to throw an exception instead of requiring the - caller to explicitly check the \ref rapidjson::GenericReader::Parse's - return value: + A common usage model is to throw an exception instead of requiring the + caller to explicitly check the \ref rapidjson::GenericReader::Parse's + return value: - \code - #define RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode,offset) \ - throw ParseException(parseErrorCode, #parseErrorCode, offset) + \code + #define RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode,offset) \ + throw ParseException(parseErrorCode, #parseErrorCode, offset) - #include // std::runtime_error - #include "rapidjson/error/error.h" // rapidjson::ParseResult + #include // std::runtime_error + #include "rapidjson/error/error.h" // rapidjson::ParseResult - struct ParseException : std::runtime_error, rapidjson::ParseResult { - ParseException(rapidjson::ParseErrorCode code, const char* msg, size_t offset) - : std::runtime_error(msg), ParseResult(code, offset) {} - }; + struct ParseException : std::runtime_error, rapidjson::ParseResult { + ParseException(rapidjson::ParseErrorCode code, const char* msg, size_t offset) + : std::runtime_error(msg), ParseResult(code, offset) {} + }; - #include "rapidjson/reader.h" - \endcode + #include "rapidjson/reader.h" + \endcode - \see RAPIDJSON_PARSE_ERROR, rapidjson::GenericReader::Parse - */ + \see RAPIDJSON_PARSE_ERROR, rapidjson::GenericReader::Parse + */ #ifndef RAPIDJSON_PARSE_ERROR_NORETURN #define RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode, offset) \ RAPIDJSON_MULTILINEMACRO_BEGIN \ @@ -103,17 +103,17 @@ RAPIDJSON_DIAG_OFF(effc++) RAPIDJSON_MULTILINEMACRO_END #endif -/*! \def RAPIDJSON_PARSE_ERROR - \ingroup RAPIDJSON_ERRORS - \brief (Internal) macro to indicate and handle a parse error. - \param parseErrorCode \ref rapidjson::ParseErrorCode of the error - \param offset position of the error in JSON input (\c size_t) + /*! \def RAPIDJSON_PARSE_ERROR + \ingroup RAPIDJSON_ERRORS + \brief (Internal) macro to indicate and handle a parse error. + \param parseErrorCode \ref rapidjson::ParseErrorCode of the error + \param offset position of the error in JSON input (\c size_t) - Invokes RAPIDJSON_PARSE_ERROR_NORETURN and stops the parsing. + Invokes RAPIDJSON_PARSE_ERROR_NORETURN and stops the parsing. - \see RAPIDJSON_PARSE_ERROR_NORETURN - \hideinitializer - */ + \see RAPIDJSON_PARSE_ERROR_NORETURN + \hideinitializer + */ #ifndef RAPIDJSON_PARSE_ERROR #define RAPIDJSON_PARSE_ERROR(parseErrorCode, offset) \ RAPIDJSON_MULTILINEMACRO_BEGIN \ @@ -124,36 +124,36 @@ RAPIDJSON_DIAG_OFF(effc++) #include "error/error.h" // ParseErrorCode, ParseResult -RAPIDJSON_NAMESPACE_BEGIN + RAPIDJSON_NAMESPACE_BEGIN /////////////////////////////////////////////////////////////////////////////// // ParseFlag -/*! \def RAPIDJSON_PARSE_DEFAULT_FLAGS - \ingroup RAPIDJSON_CONFIG - \brief User-defined kParseDefaultFlags definition. + /*! \def RAPIDJSON_PARSE_DEFAULT_FLAGS + \ingroup RAPIDJSON_CONFIG + \brief User-defined kParseDefaultFlags definition. - User can define this as any \c ParseFlag combinations. -*/ + User can define this as any \c ParseFlag combinations. + */ #ifndef RAPIDJSON_PARSE_DEFAULT_FLAGS #define RAPIDJSON_PARSE_DEFAULT_FLAGS kParseNoFlags #endif //! Combination of parseFlags -/*! \see Reader::Parse, Document::Parse, Document::ParseInsitu, Document::ParseStream - */ -enum ParseFlag { - kParseNoFlags = 0, //!< No flags are set. - kParseInsituFlag = 1, //!< In-situ(destructive) parsing. - kParseValidateEncodingFlag = 2, //!< Validate encoding of JSON strings. - kParseIterativeFlag = 4, //!< Iterative(constant complexity in terms of function call stack size) parsing. - kParseStopWhenDoneFlag = 8, //!< After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate kParseErrorDocumentRootNotSingular error. - kParseFullPrecisionFlag = 16, //!< Parse number in full precision (but slower). - kParseCommentsFlag = 32, //!< Allow one-line (//) and multi-line (/**/) comments. - kParseNumbersAsStringsFlag = 64, //!< Parse all numbers (ints/doubles) as strings. - kParseTrailingCommasFlag = 128, //!< Allow trailing commas at the end of objects and arrays. - kParseNanAndInfFlag = 256, //!< Allow parsing NaN, Inf, Infinity, -Inf and -Infinity as doubles. - kParseDefaultFlags = RAPIDJSON_PARSE_DEFAULT_FLAGS //!< Default parse flags. Can be customized by defining RAPIDJSON_PARSE_DEFAULT_FLAGS + /*! \see Reader::Parse, Document::Parse, Document::ParseInsitu, Document::ParseStream + */ + enum ParseFlag +{ kParseNoFlags = 0, //!< No flags are set. + kParseInsituFlag = 1, //!< In-situ(destructive) parsing. + kParseValidateEncodingFlag = 2, //!< Validate encoding of JSON strings. + kParseIterativeFlag = 4, //!< Iterative(constant complexity in terms of function call stack size) parsing. + kParseStopWhenDoneFlag = 8, //!< After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate kParseErrorDocumentRootNotSingular error. + kParseFullPrecisionFlag = 16, //!< Parse number in full precision (but slower). + kParseCommentsFlag = 32, //!< Allow one-line (//) and multi-line (/**/) comments. + kParseNumbersAsStringsFlag = 64, //!< Parse all numbers (ints/doubles) as strings. + kParseTrailingCommasFlag = 128, //!< Allow trailing commas at the end of objects and arrays. + kParseNanAndInfFlag = 256, //!< Allow parsing NaN, Inf, Infinity, -Inf and -Infinity as doubles. + kParseDefaultFlags = RAPIDJSON_PARSE_DEFAULT_FLAGS //!< Default parse flags. Can be customized by defining RAPIDJSON_PARSE_DEFAULT_FLAGS }; /////////////////////////////////////////////////////////////////////////////// @@ -193,62 +193,65 @@ concept Handler { \note implements Handler concept */ template, typename Derived = void> -struct BaseReaderHandler { - typedef typename Encoding::Ch Ch; - - typedef typename internal::SelectIf, BaseReaderHandler, Derived>::Type Override; - - bool Default() { return true; } - bool Null() { return static_cast(*this).Default(); } - bool Bool(bool) { return static_cast(*this).Default(); } - bool Int(int) { return static_cast(*this).Default(); } - bool Uint(unsigned) { return static_cast(*this).Default(); } - bool Int64(int64_t) { return static_cast(*this).Default(); } - bool Uint64(uint64_t) { return static_cast(*this).Default(); } - bool Double(double) { return static_cast(*this).Default(); } - /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) - bool RawNumber(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } - bool String(const Ch*, SizeType, bool) { return static_cast(*this).Default(); } - bool StartObject() { return static_cast(*this).Default(); } - bool Key(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } - bool EndObject(SizeType) { return static_cast(*this).Default(); } - bool StartArray() { return static_cast(*this).Default(); } - bool EndArray(SizeType) { return static_cast(*this).Default(); } +struct BaseReaderHandler +{ typedef typename Encoding::Ch Ch; + + typedef typename internal::SelectIf, BaseReaderHandler, Derived>::Type Override; + + bool Default() { return true; } + bool Null() { return static_cast(*this).Default(); } + bool Bool(bool) { return static_cast(*this).Default(); } + bool Int(int) { return static_cast(*this).Default(); } + bool Uint(unsigned) { return static_cast(*this).Default(); } + bool Int64(int64_t) { return static_cast(*this).Default(); } + bool Uint64(uint64_t) { return static_cast(*this).Default(); } + bool Double(double) { return static_cast(*this).Default(); } + /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) + bool RawNumber(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } + bool String(const Ch*, SizeType, bool) { return static_cast(*this).Default(); } + bool StartObject() { return static_cast(*this).Default(); } + bool Key(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } + bool EndObject(SizeType) { return static_cast(*this).Default(); } + bool StartArray() { return static_cast(*this).Default(); } + bool EndArray(SizeType) { return static_cast(*this).Default(); } }; /////////////////////////////////////////////////////////////////////////////// // StreamLocalCopy -namespace internal { +namespace internal +{ template::copyOptimization> class StreamLocalCopy; //! Do copy optimization. template -class StreamLocalCopy { +class StreamLocalCopy +{ public: - StreamLocalCopy(Stream& original) : s(original), original_(original) {} - ~StreamLocalCopy() { original_ = s; } + StreamLocalCopy(Stream& original) : s(original), original_(original) {} + ~StreamLocalCopy() { original_ = s; } - Stream s; + Stream s; private: - StreamLocalCopy& operator=(const StreamLocalCopy&) /* = delete */; + StreamLocalCopy& operator=(const StreamLocalCopy&) /* = delete */; - Stream& original_; + Stream& original_; }; //! Keep reference. template -class StreamLocalCopy { +class StreamLocalCopy +{ public: - StreamLocalCopy(Stream& original) : s(original) {} + StreamLocalCopy(Stream& original) : s(original) {} - Stream& s; + Stream& s; private: - StreamLocalCopy& operator=(const StreamLocalCopy&) /* = delete */; + StreamLocalCopy& operator=(const StreamLocalCopy&) /* = delete */; }; } // namespace internal @@ -261,185 +264,189 @@ class StreamLocalCopy { \note This function has SSE2/SSE4.2 specialization. */ template -void SkipWhitespace(InputStream& is) { - internal::StreamLocalCopy copy(is); - InputStream& s(copy.s); +void SkipWhitespace(InputStream& is) +{ internal::StreamLocalCopy copy(is); + InputStream& s(copy.s); - typename InputStream::Ch c; - while ((c = s.Peek()) == ' ' || c == '\n' || c == '\r' || c == '\t') - s.Take(); + typename InputStream::Ch c; + while ((c = s.Peek()) == ' ' || c == '\n' || c == '\r' || c == '\t') + s.Take(); } -inline const char* SkipWhitespace(const char* p, const char* end) { - while (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) - ++p; - return p; +inline const char* SkipWhitespace(const char* p, const char* end) +{ while (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + return p; } #ifdef RAPIDJSON_SSE42 //! Skip whitespace with SSE 4.2 pcmpistrm instruction, testing 16 8-byte characters at once. -inline const char *SkipWhitespace_SIMD(const char* p) { - // Fast return for single non-whitespace +inline const char *SkipWhitespace_SIMD(const char* p) +{ // Fast return for single non-whitespace + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // 16-byte align to the next boundary + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') - ++p; + ++p; else - return p; + return p; - // 16-byte align to the next boundary - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); - while (p != nextAligned) - if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') - ++p; - else - return p; - - // The rest of string using SIMD - static const char whitespace[16] = " \n\r\t"; - const __m128i w = _mm_loadu_si128(reinterpret_cast(&whitespace[0])); + // The rest of string using SIMD + static const char whitespace[16] = " \n\r\t"; + const __m128i w = _mm_loadu_si128(reinterpret_cast(&whitespace[0])); - for (;; p += 16) { - const __m128i s = _mm_load_si128(reinterpret_cast(p)); - const int r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY)); - if (r != 0) { // some of characters is non-whitespace + for (;; p += 16) + { const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const int r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY)); + if (r != 0) // some of characters is non-whitespace + { #ifdef _MSC_VER // Find the index of first non-whitespace - unsigned long offset; - _BitScanForward(&offset, r); - return p + offset; + unsigned long offset; + _BitScanForward(&offset, r); + return p + offset; #else - return p + __builtin_ffs(r) - 1; + return p + __builtin_ffs(r) - 1; #endif - } } + } } -inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { - // Fast return for single non-whitespace - if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) - ++p; - else - return p; +inline const char *SkipWhitespace_SIMD(const char* p, const char* end) +{ // Fast return for single non-whitespace + if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + else + return p; - // The middle of string using SIMD - static const char whitespace[16] = " \n\r\t"; - const __m128i w = _mm_loadu_si128(reinterpret_cast(&whitespace[0])); + // The middle of string using SIMD + static const char whitespace[16] = " \n\r\t"; + const __m128i w = _mm_loadu_si128(reinterpret_cast(&whitespace[0])); - for (; p <= end - 16; p += 16) { - const __m128i s = _mm_loadu_si128(reinterpret_cast(p)); - const int r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY)); - if (r != 0) { // some of characters is non-whitespace + for (; p <= end - 16; p += 16) + { const __m128i s = _mm_loadu_si128(reinterpret_cast(p)); + const int r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY)); + if (r != 0) // some of characters is non-whitespace + { #ifdef _MSC_VER // Find the index of first non-whitespace - unsigned long offset; - _BitScanForward(&offset, r); - return p + offset; + unsigned long offset; + _BitScanForward(&offset, r); + return p + offset; #else - return p + __builtin_ffs(r) - 1; + return p + __builtin_ffs(r) - 1; #endif - } } + } - return SkipWhitespace(p, end); + return SkipWhitespace(p, end); } #elif defined(RAPIDJSON_SSE2) //! Skip whitespace with SSE2 instructions, testing 16 8-byte characters at once. -inline const char *SkipWhitespace_SIMD(const char* p) { - // Fast return for single non-whitespace +inline const char *SkipWhitespace_SIMD(const char* p) +{ // Fast return for single non-whitespace + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // 16-byte align to the next boundary + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') - ++p; + ++p; else - return p; - - // 16-byte align to the next boundary - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); - while (p != nextAligned) - if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') - ++p; - else - return p; - - // The rest of string - #define C16(c) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c } - static const char whitespaces[4][16] = { C16(' '), C16('\n'), C16('\r'), C16('\t') }; - #undef C16 - - const __m128i w0 = _mm_loadu_si128(reinterpret_cast(&whitespaces[0][0])); - const __m128i w1 = _mm_loadu_si128(reinterpret_cast(&whitespaces[1][0])); - const __m128i w2 = _mm_loadu_si128(reinterpret_cast(&whitespaces[2][0])); - const __m128i w3 = _mm_loadu_si128(reinterpret_cast(&whitespaces[3][0])); - - for (;; p += 16) { - const __m128i s = _mm_load_si128(reinterpret_cast(p)); - __m128i x = _mm_cmpeq_epi8(s, w0); - x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); - x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); - x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); - unsigned short r = static_cast(~_mm_movemask_epi8(x)); - if (r != 0) { // some of characters may be non-whitespace + return p; + + // The rest of string +#define C16(c) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c } + static const char whitespaces[4][16] = { C16(' '), C16('\n'), C16('\r'), C16('\t') }; +#undef C16 + + const __m128i w0 = _mm_loadu_si128(reinterpret_cast(&whitespaces[0][0])); + const __m128i w1 = _mm_loadu_si128(reinterpret_cast(&whitespaces[1][0])); + const __m128i w2 = _mm_loadu_si128(reinterpret_cast(&whitespaces[2][0])); + const __m128i w3 = _mm_loadu_si128(reinterpret_cast(&whitespaces[3][0])); + + for (;; p += 16) + { const __m128i s = _mm_load_si128(reinterpret_cast(p)); + __m128i x = _mm_cmpeq_epi8(s, w0); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); + unsigned short r = static_cast(~_mm_movemask_epi8(x)); + if (r != 0) // some of characters may be non-whitespace + { #ifdef _MSC_VER // Find the index of first non-whitespace - unsigned long offset; - _BitScanForward(&offset, r); - return p + offset; + unsigned long offset; + _BitScanForward(&offset, r); + return p + offset; #else - return p + __builtin_ffs(r) - 1; + return p + __builtin_ffs(r) - 1; #endif - } } + } } -inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { - // Fast return for single non-whitespace - if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) - ++p; - else - return p; - - // The rest of string - #define C16(c) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c } - static const char whitespaces[4][16] = { C16(' '), C16('\n'), C16('\r'), C16('\t') }; - #undef C16 - - const __m128i w0 = _mm_loadu_si128(reinterpret_cast(&whitespaces[0][0])); - const __m128i w1 = _mm_loadu_si128(reinterpret_cast(&whitespaces[1][0])); - const __m128i w2 = _mm_loadu_si128(reinterpret_cast(&whitespaces[2][0])); - const __m128i w3 = _mm_loadu_si128(reinterpret_cast(&whitespaces[3][0])); - - for (; p <= end - 16; p += 16) { - const __m128i s = _mm_loadu_si128(reinterpret_cast(p)); - __m128i x = _mm_cmpeq_epi8(s, w0); - x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); - x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); - x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); - unsigned short r = static_cast(~_mm_movemask_epi8(x)); - if (r != 0) { // some of characters may be non-whitespace +inline const char *SkipWhitespace_SIMD(const char* p, const char* end) +{ // Fast return for single non-whitespace + if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + else + return p; + + // The rest of string +#define C16(c) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c } + static const char whitespaces[4][16] = { C16(' '), C16('\n'), C16('\r'), C16('\t') }; +#undef C16 + + const __m128i w0 = _mm_loadu_si128(reinterpret_cast(&whitespaces[0][0])); + const __m128i w1 = _mm_loadu_si128(reinterpret_cast(&whitespaces[1][0])); + const __m128i w2 = _mm_loadu_si128(reinterpret_cast(&whitespaces[2][0])); + const __m128i w3 = _mm_loadu_si128(reinterpret_cast(&whitespaces[3][0])); + + for (; p <= end - 16; p += 16) + { const __m128i s = _mm_loadu_si128(reinterpret_cast(p)); + __m128i x = _mm_cmpeq_epi8(s, w0); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); + unsigned short r = static_cast(~_mm_movemask_epi8(x)); + if (r != 0) // some of characters may be non-whitespace + { #ifdef _MSC_VER // Find the index of first non-whitespace - unsigned long offset; - _BitScanForward(&offset, r); - return p + offset; + unsigned long offset; + _BitScanForward(&offset, r); + return p + offset; #else - return p + __builtin_ffs(r) - 1; + return p + __builtin_ffs(r) - 1; #endif - } } + } - return SkipWhitespace(p, end); + return SkipWhitespace(p, end); } #endif // RAPIDJSON_SSE2 #ifdef RAPIDJSON_SIMD //! Template function specialization for InsituStringStream -template<> inline void SkipWhitespace(InsituStringStream& is) { - is.src_ = const_cast(SkipWhitespace_SIMD(is.src_)); +template<> inline void SkipWhitespace(InsituStringStream& is) +{ is.src_ = const_cast(SkipWhitespace_SIMD(is.src_)); } //! Template function specialization for StringStream -template<> inline void SkipWhitespace(StringStream& is) { - is.src_ = SkipWhitespace_SIMD(is.src_); +template<> inline void SkipWhitespace(StringStream& is) +{ is.src_ = SkipWhitespace_SIMD(is.src_); } -template<> inline void SkipWhitespace(EncodedInputStream, MemoryStream>& is) { - is.is_.src_ = SkipWhitespace_SIMD(is.is_.src_, is.is_.end_); +template<> inline void SkipWhitespace(EncodedInputStream, MemoryStream>& is) +{ is.is_.src_ = SkipWhitespace_SIMD(is.is_.src_, is.is_.end_); } #endif // RAPIDJSON_SIMD @@ -463,1399 +470,1388 @@ template<> inline void SkipWhitespace(EncodedInputStream, MemoryStream>& \tparam StackAllocator Allocator type for stack. */ template -class GenericReader { +class GenericReader +{ public: - typedef typename SourceEncoding::Ch Ch; //!< SourceEncoding character type - - //! Constructor. - /*! \param stackAllocator Optional allocator for allocating stack memory. (Only use for non-destructive parsing) - \param stackCapacity stack capacity in bytes for storing a single decoded string. (Only use for non-destructive parsing) - */ - GenericReader(StackAllocator* stackAllocator = 0, size_t stackCapacity = kDefaultStackCapacity) : stack_(stackAllocator, stackCapacity), parseResult_() {} - - //! Parse JSON text. - /*! \tparam parseFlags Combination of \ref ParseFlag. - \tparam InputStream Type of input stream, implementing Stream concept. - \tparam Handler Type of handler, implementing Handler concept. - \param is Input stream to be parsed. - \param handler The handler to receive events. - \return Whether the parsing is successful. - */ - template - ParseResult Parse(InputStream& is, Handler& handler) { - if (parseFlags & kParseIterativeFlag) - return IterativeParse(is, handler); - - parseResult_.Clear(); - - ClearStackOnExit scope(*this); - - SkipWhitespaceAndComments(is); + typedef typename SourceEncoding::Ch Ch; //!< SourceEncoding character type + + //! Constructor. + /*! \param stackAllocator Optional allocator for allocating stack memory. (Only use for non-destructive parsing) + \param stackCapacity stack capacity in bytes for storing a single decoded string. (Only use for non-destructive parsing) + */ + GenericReader(StackAllocator* stackAllocator = 0, size_t stackCapacity = kDefaultStackCapacity) : stack_(stackAllocator, stackCapacity), parseResult_() {} + + //! Parse JSON text. + /*! \tparam parseFlags Combination of \ref ParseFlag. + \tparam InputStream Type of input stream, implementing Stream concept. + \tparam Handler Type of handler, implementing Handler concept. + \param is Input stream to be parsed. + \param handler The handler to receive events. + \return Whether the parsing is successful. + */ + template + ParseResult Parse(InputStream& is, Handler& handler) + { if (parseFlags & kParseIterativeFlag) + return IterativeParse(is, handler); + + parseResult_.Clear(); + + ClearStackOnExit scope(*this); + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + + if (RAPIDJSON_UNLIKELY(is.Peek() == '\0')) + { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentEmpty, is.Tell()); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + } + else + { ParseValue(is, handler); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + + if (!(parseFlags & kParseStopWhenDoneFlag)) + { SkipWhitespaceAndComments(is); RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); - if (RAPIDJSON_UNLIKELY(is.Peek() == '\0')) { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentEmpty, is.Tell()); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); - } - else { - ParseValue(is, handler); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); - - if (!(parseFlags & kParseStopWhenDoneFlag)) { - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); - - if (RAPIDJSON_UNLIKELY(is.Peek() != '\0')) { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentRootNotSingular, is.Tell()); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); - } - } + if (RAPIDJSON_UNLIKELY(is.Peek() != '\0')) + { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentRootNotSingular, is.Tell()); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); } - - return parseResult_; + } } - //! Parse JSON text (with \ref kParseDefaultFlags) - /*! \tparam InputStream Type of input stream, implementing Stream concept - \tparam Handler Type of handler, implementing Handler concept. - \param is Input stream to be parsed. - \param handler The handler to receive events. - \return Whether the parsing is successful. - */ - template - ParseResult Parse(InputStream& is, Handler& handler) { - return Parse(is, handler); - } + return parseResult_; + } + + //! Parse JSON text (with \ref kParseDefaultFlags) + /*! \tparam InputStream Type of input stream, implementing Stream concept + \tparam Handler Type of handler, implementing Handler concept. + \param is Input stream to be parsed. + \param handler The handler to receive events. + \return Whether the parsing is successful. + */ + template + ParseResult Parse(InputStream& is, Handler& handler) + { return Parse(is, handler); + } - //! Whether a parse error has occured in the last parsing. - bool HasParseError() const { return parseResult_.IsError(); } + //! Whether a parse error has occured in the last parsing. + bool HasParseError() const { return parseResult_.IsError(); } - //! Get the \ref ParseErrorCode of last parsing. - ParseErrorCode GetParseErrorCode() const { return parseResult_.Code(); } + //! Get the \ref ParseErrorCode of last parsing. + ParseErrorCode GetParseErrorCode() const { return parseResult_.Code(); } - //! Get the position of last parsing error in input, 0 otherwise. - size_t GetErrorOffset() const { return parseResult_.Offset(); } + //! Get the position of last parsing error in input, 0 otherwise. + size_t GetErrorOffset() const { return parseResult_.Offset(); } protected: - void SetParseError(ParseErrorCode code, size_t offset) { parseResult_.Set(code, offset); } + void SetParseError(ParseErrorCode code, size_t offset) { parseResult_.Set(code, offset); } private: - // Prohibit copy constructor & assignment operator. - GenericReader(const GenericReader&); - GenericReader& operator=(const GenericReader&); - - void ClearStack() { stack_.Clear(); } - - // clear stack on any exit from ParseStream, e.g. due to exception - struct ClearStackOnExit { - explicit ClearStackOnExit(GenericReader& r) : r_(r) {} - ~ClearStackOnExit() { r_.ClearStack(); } - private: - GenericReader& r_; - ClearStackOnExit(const ClearStackOnExit&); - ClearStackOnExit& operator=(const ClearStackOnExit&); - }; - - template - void SkipWhitespaceAndComments(InputStream& is) { - SkipWhitespace(is); - - if (parseFlags & kParseCommentsFlag) { - while (RAPIDJSON_UNLIKELY(Consume(is, '/'))) { - if (Consume(is, '*')) { - while (true) { - if (RAPIDJSON_UNLIKELY(is.Peek() == '\0')) - RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); - else if (Consume(is, '*')) { - if (Consume(is, '/')) - break; - } - else - is.Take(); - } - } - else if (RAPIDJSON_LIKELY(Consume(is, '/'))) - while (is.Peek() != '\0' && is.Take() != '\n'); - else - RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); - - SkipWhitespace(is); + // Prohibit copy constructor & assignment operator. + GenericReader(const GenericReader&); + GenericReader& operator=(const GenericReader&); + + void ClearStack() { stack_.Clear(); } + + // clear stack on any exit from ParseStream, e.g. due to exception + struct ClearStackOnExit +{ explicit ClearStackOnExit(GenericReader& r) : r_(r) {} + ~ClearStackOnExit() { r_.ClearStack(); } + private: + GenericReader& r_; + ClearStackOnExit(const ClearStackOnExit&); + ClearStackOnExit& operator=(const ClearStackOnExit&); + }; + + template + void SkipWhitespaceAndComments(InputStream& is) + { SkipWhitespace(is); + + if (parseFlags & kParseCommentsFlag) + { while (RAPIDJSON_UNLIKELY(Consume(is, '/'))) + { if (Consume(is, '*')) + { while (true) + { if (RAPIDJSON_UNLIKELY(is.Peek() == '\0')) + RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); + else if (Consume(is, '*')) + { if (Consume(is, '/')) + break; } + else + is.Take(); + } } + else if (RAPIDJSON_LIKELY(Consume(is, '/'))) + while (is.Peek() != '\0' && is.Take() != '\n'); + else + RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); + + SkipWhitespace(is); + } } + } - // Parse object: { string : value, ... } - template - void ParseObject(InputStream& is, Handler& handler) { - RAPIDJSON_ASSERT(is.Peek() == '{'); - is.Take(); // Skip '{' + // Parse object: { string : value, ... } + template + void ParseObject(InputStream& is, Handler& handler) + { RAPIDJSON_ASSERT(is.Peek() == '{'); + is.Take(); // Skip '{' - if (RAPIDJSON_UNLIKELY(!handler.StartObject())) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + if (RAPIDJSON_UNLIKELY(!handler.StartObject())) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (Consume(is, '}')) { - if (RAPIDJSON_UNLIKELY(!handler.EndObject(0))) // empty object - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - return; - } + if (Consume(is, '}')) + { if (RAPIDJSON_UNLIKELY(!handler.EndObject(0))) // empty object + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + return; + } - for (SizeType memberCount = 0;;) { - if (RAPIDJSON_UNLIKELY(is.Peek() != '"')) - RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); + for (SizeType memberCount = 0;;) + { if (RAPIDJSON_UNLIKELY(is.Peek() != '"')) + RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); - ParseString(is, handler, true); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + ParseString(is, handler, true); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (RAPIDJSON_UNLIKELY(!Consume(is, ':'))) - RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); + if (RAPIDJSON_UNLIKELY(!Consume(is, ':'))) + RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - ParseValue(is, handler); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + ParseValue(is, handler); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - ++memberCount; - - switch (is.Peek()) { - case ',': - is.Take(); - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - break; - case '}': - is.Take(); - if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount))) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - return; - default: - RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); break; // This useless break is only for making warning and coverage happy - } + ++memberCount; - if (parseFlags & kParseTrailingCommasFlag) { - if (is.Peek() == '}') { - if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount))) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - is.Take(); - return; - } - } + switch (is.Peek()) + { case ',': + is.Take(); + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + break; + case '}': + is.Take(); + if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + return; + default: + RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); break; // This useless break is only for making warning and coverage happy + } + + if (parseFlags & kParseTrailingCommasFlag) + { if (is.Peek() == '}') + { if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + is.Take(); + return; } + } } + } - // Parse array: [ value, ... ] - template - void ParseArray(InputStream& is, Handler& handler) { - RAPIDJSON_ASSERT(is.Peek() == '['); - is.Take(); // Skip '[' + // Parse array: [ value, ... ] + template + void ParseArray(InputStream& is, Handler& handler) + { RAPIDJSON_ASSERT(is.Peek() == '['); + is.Take(); // Skip '[' - if (RAPIDJSON_UNLIKELY(!handler.StartArray())) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + if (RAPIDJSON_UNLIKELY(!handler.StartArray())) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (Consume(is, ']')) { - if (RAPIDJSON_UNLIKELY(!handler.EndArray(0))) // empty array - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - return; - } + if (Consume(is, ']')) + { if (RAPIDJSON_UNLIKELY(!handler.EndArray(0))) // empty array + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + return; + } - for (SizeType elementCount = 0;;) { - ParseValue(is, handler); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + for (SizeType elementCount = 0;;) + { ParseValue(is, handler); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - ++elementCount; - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + ++elementCount; + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (Consume(is, ',')) { - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - } - else if (Consume(is, ']')) { - if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount))) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - return; - } - else - RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); - - if (parseFlags & kParseTrailingCommasFlag) { - if (is.Peek() == ']') { - if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount))) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - is.Take(); - return; - } - } + if (Consume(is, ',')) + { SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + } + else if (Consume(is, ']')) + { if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + return; + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); + + if (parseFlags & kParseTrailingCommasFlag) + { if (is.Peek() == ']') + { if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); + is.Take(); + return; } + } } + } - template - void ParseNull(InputStream& is, Handler& handler) { - RAPIDJSON_ASSERT(is.Peek() == 'n'); - is.Take(); + template + void ParseNull(InputStream& is, Handler& handler) + { RAPIDJSON_ASSERT(is.Peek() == 'n'); + is.Take(); - if (RAPIDJSON_LIKELY(Consume(is, 'u') && Consume(is, 'l') && Consume(is, 'l'))) { - if (RAPIDJSON_UNLIKELY(!handler.Null())) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - } - else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); + if (RAPIDJSON_LIKELY(Consume(is, 'u') && Consume(is, 'l') && Consume(is, 'l'))) + { if (RAPIDJSON_UNLIKELY(!handler.Null())) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); } + else + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); + } - template - void ParseTrue(InputStream& is, Handler& handler) { - RAPIDJSON_ASSERT(is.Peek() == 't'); - is.Take(); + template + void ParseTrue(InputStream& is, Handler& handler) + { RAPIDJSON_ASSERT(is.Peek() == 't'); + is.Take(); - if (RAPIDJSON_LIKELY(Consume(is, 'r') && Consume(is, 'u') && Consume(is, 'e'))) { - if (RAPIDJSON_UNLIKELY(!handler.Bool(true))) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - } - else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); + if (RAPIDJSON_LIKELY(Consume(is, 'r') && Consume(is, 'u') && Consume(is, 'e'))) + { if (RAPIDJSON_UNLIKELY(!handler.Bool(true))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); } + else + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); + } - template - void ParseFalse(InputStream& is, Handler& handler) { - RAPIDJSON_ASSERT(is.Peek() == 'f'); - is.Take(); + template + void ParseFalse(InputStream& is, Handler& handler) + { RAPIDJSON_ASSERT(is.Peek() == 'f'); + is.Take(); - if (RAPIDJSON_LIKELY(Consume(is, 'a') && Consume(is, 'l') && Consume(is, 's') && Consume(is, 'e'))) { - if (RAPIDJSON_UNLIKELY(!handler.Bool(false))) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - } - else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); + if (RAPIDJSON_LIKELY(Consume(is, 'a') && Consume(is, 'l') && Consume(is, 's') && Consume(is, 'e'))) + { if (RAPIDJSON_UNLIKELY(!handler.Bool(false))) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); } - - template - RAPIDJSON_FORCEINLINE static bool Consume(InputStream& is, typename InputStream::Ch expect) { - if (RAPIDJSON_LIKELY(is.Peek() == expect)) { - is.Take(); - return true; - } - else - return false; + else + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); + } + + template + RAPIDJSON_FORCEINLINE static bool Consume(InputStream& is, typename InputStream::Ch expect) + { if (RAPIDJSON_LIKELY(is.Peek() == expect)) + { is.Take(); + return true; } - - // Helper function to parse four hexidecimal digits in \uXXXX in ParseString(). - template - unsigned ParseHex4(InputStream& is, size_t escapeOffset) { - unsigned codepoint = 0; - for (int i = 0; i < 4; i++) { - Ch c = is.Peek(); - codepoint <<= 4; - codepoint += static_cast(c); - if (c >= '0' && c <= '9') - codepoint -= '0'; - else if (c >= 'A' && c <= 'F') - codepoint -= 'A' - 10; - else if (c >= 'a' && c <= 'f') - codepoint -= 'a' - 10; - else { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorStringUnicodeEscapeInvalidHex, escapeOffset); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(0); - } - is.Take(); - } - return codepoint; + else + return false; + } + + // Helper function to parse four hexidecimal digits in \uXXXX in ParseString(). + template + unsigned ParseHex4(InputStream& is, size_t escapeOffset) + { unsigned codepoint = 0; + for (int i = 0; i < 4; i++) + { Ch c = is.Peek(); + codepoint <<= 4; + codepoint += static_cast(c); + if (c >= '0' && c <= '9') + codepoint -= '0'; + else if (c >= 'A' && c <= 'F') + codepoint -= 'A' - 10; + else if (c >= 'a' && c <= 'f') + codepoint -= 'a' - 10; + else + { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorStringUnicodeEscapeInvalidHex, escapeOffset); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(0); + } + is.Take(); + } + return codepoint; + } + + template + class StackStream + { + public: + typedef CharType Ch; + + StackStream(internal::Stack& stack) : stack_(stack), length_(0) {} + RAPIDJSON_FORCEINLINE void Put(Ch c) + { *stack_.template Push() = c; + ++length_; } - template - class StackStream { - public: - typedef CharType Ch; - - StackStream(internal::Stack& stack) : stack_(stack), length_(0) {} - RAPIDJSON_FORCEINLINE void Put(Ch c) { - *stack_.template Push() = c; - ++length_; - } - - RAPIDJSON_FORCEINLINE void* Push(SizeType count) { - length_ += count; - return stack_.template Push(count); - } - - size_t Length() const { return length_; } + RAPIDJSON_FORCEINLINE void* Push(SizeType count) + { length_ += count; + return stack_.template Push(count); + } - Ch* Pop() { - return stack_.template Pop(length_); - } + size_t Length() const { return length_; } - private: - StackStream(const StackStream&); - StackStream& operator=(const StackStream&); + Ch* Pop() + { return stack_.template Pop(length_); + } - internal::Stack& stack_; - SizeType length_; - }; + private: + StackStream(const StackStream&); + StackStream& operator=(const StackStream&); - // Parse string and generate String event. Different code paths for kParseInsituFlag. - template - void ParseString(InputStream& is, Handler& handler, bool isKey = false) { - internal::StreamLocalCopy copy(is); - InputStream& s(copy.s); + internal::Stack& stack_; + SizeType length_; + }; - RAPIDJSON_ASSERT(s.Peek() == '\"'); - s.Take(); // Skip '\"' + // Parse string and generate String event. Different code paths for kParseInsituFlag. + template + void ParseString(InputStream& is, Handler& handler, bool isKey = false) + { internal::StreamLocalCopy copy(is); + InputStream& s(copy.s); - bool success = false; - if (parseFlags & kParseInsituFlag) { - typename InputStream::Ch *head = s.PutBegin(); - ParseStringToStream(s, s); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - size_t length = s.PutEnd(head) - 1; - RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); - const typename TargetEncoding::Ch* const str = reinterpret_cast(head); - success = (isKey ? handler.Key(str, SizeType(length), false) : handler.String(str, SizeType(length), false)); - } - else { - StackStream stackStream(stack_); - ParseStringToStream(s, stackStream); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - SizeType length = static_cast(stackStream.Length()) - 1; - const typename TargetEncoding::Ch* const str = stackStream.Pop(); - success = (isKey ? handler.Key(str, length, true) : handler.String(str, length, true)); - } - if (RAPIDJSON_UNLIKELY(!success)) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, s.Tell()); + RAPIDJSON_ASSERT(s.Peek() == '\"'); + s.Take(); // Skip '\"' + + bool success = false; + if (parseFlags & kParseInsituFlag) + { typename InputStream::Ch *head = s.PutBegin(); + ParseStringToStream(s, s); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + size_t length = s.PutEnd(head) - 1; + RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); + const typename TargetEncoding::Ch* const str = reinterpret_cast(head); + success = (isKey ? handler.Key(str, SizeType(length), false) : handler.String(str, SizeType(length), false)); } - - // Parse string to an output is - // This function handles the prefix/suffix double quotes, escaping, and optional encoding validation. - template - RAPIDJSON_FORCEINLINE void ParseStringToStream(InputStream& is, OutputStream& os) { + else + { StackStream stackStream(stack_); + ParseStringToStream(s, stackStream); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + SizeType length = static_cast(stackStream.Length()) - 1; + const typename TargetEncoding::Ch* const str = stackStream.Pop(); + success = (isKey ? handler.Key(str, length, true) : handler.String(str, length, true)); + } + if (RAPIDJSON_UNLIKELY(!success)) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, s.Tell()); + } + + // Parse string to an output is + // This function handles the prefix/suffix double quotes, escaping, and optional encoding validation. + template + RAPIDJSON_FORCEINLINE void ParseStringToStream(InputStream& is, OutputStream& os) + { //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN #define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 - static const char escape[256] = { - Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'/', - Z16, Z16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, - 0, 0,'\b', 0, 0, 0,'\f', 0, 0, 0, 0, 0, 0, 0,'\n', 0, - 0, 0,'\r', 0,'\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 - }; + static const char escape[256] = + { Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'/', + Z16, Z16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, + 0, 0,'\b', 0, 0, 0,'\f', 0, 0, 0, 0, 0, 0, 0,'\n', 0, + 0, 0,'\r', 0,'\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 + }; #undef Z16 //!@endcond - for (;;) { - // Scan and copy string before "\\\"" or < 0x20. This is an optional optimzation. - if (!(parseFlags & kParseValidateEncodingFlag)) - ScanCopyUnescapedString(is, os); - - Ch c = is.Peek(); - if (RAPIDJSON_UNLIKELY(c == '\\')) { // Escape - size_t escapeOffset = is.Tell(); // For invalid escaping, report the inital '\\' as error offset - is.Take(); - Ch e = is.Peek(); - if ((sizeof(Ch) == 1 || unsigned(e) < 256) && RAPIDJSON_LIKELY(escape[static_cast(e)])) { - is.Take(); - os.Put(static_cast(escape[static_cast(e)])); - } - else if (RAPIDJSON_LIKELY(e == 'u')) { // Unicode - is.Take(); - unsigned codepoint = ParseHex4(is, escapeOffset); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (RAPIDJSON_UNLIKELY(codepoint >= 0xD800 && codepoint <= 0xDBFF)) { - // Handle UTF-16 surrogate pair - if (RAPIDJSON_UNLIKELY(!Consume(is, '\\') || !Consume(is, 'u'))) - RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); - unsigned codepoint2 = ParseHex4(is, escapeOffset); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (RAPIDJSON_UNLIKELY(codepoint2 < 0xDC00 || codepoint2 > 0xDFFF)) - RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); - codepoint = (((codepoint - 0xD800) << 10) | (codepoint2 - 0xDC00)) + 0x10000; - } - TEncoding::Encode(os, codepoint); - } - else - RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, escapeOffset); - } - else if (RAPIDJSON_UNLIKELY(c == '"')) { // Closing double quote - is.Take(); - os.Put('\0'); // null-terminate the string - return; - } - else if (RAPIDJSON_UNLIKELY(static_cast(c) < 0x20)) { // RFC 4627: unescaped = %x20-21 / %x23-5B / %x5D-10FFFF - if (c == '\0') - RAPIDJSON_PARSE_ERROR(kParseErrorStringMissQuotationMark, is.Tell()); - else - RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, is.Tell()); - } - else { - size_t offset = is.Tell(); - if (RAPIDJSON_UNLIKELY((parseFlags & kParseValidateEncodingFlag ? - !Transcoder::Validate(is, os) : - !Transcoder::Transcode(is, os)))) - RAPIDJSON_PARSE_ERROR(kParseErrorStringInvalidEncoding, offset); - } + for (;;) + { // Scan and copy string before "\\\"" or < 0x20. This is an optional optimzation. + if (!(parseFlags & kParseValidateEncodingFlag)) + ScanCopyUnescapedString(is, os); + + Ch c = is.Peek(); + if (RAPIDJSON_UNLIKELY(c == '\\')) // Escape + { size_t escapeOffset = is.Tell(); // For invalid escaping, report the inital '\\' as error offset + is.Take(); + Ch e = is.Peek(); + if ((sizeof(Ch) == 1 || unsigned(e) < 256) && RAPIDJSON_LIKELY(escape[static_cast(e)])) + { is.Take(); + os.Put(static_cast(escape[static_cast(e)])); } + else if (RAPIDJSON_LIKELY(e == 'u')) // Unicode + { is.Take(); + unsigned codepoint = ParseHex4(is, escapeOffset); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + if (RAPIDJSON_UNLIKELY(codepoint >= 0xD800 && codepoint <= 0xDBFF)) + { // Handle UTF-16 surrogate pair + if (RAPIDJSON_UNLIKELY(!Consume(is, '\\') || !Consume(is, 'u'))) + RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); + unsigned codepoint2 = ParseHex4(is, escapeOffset); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + if (RAPIDJSON_UNLIKELY(codepoint2 < 0xDC00 || codepoint2 > 0xDFFF)) + RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); + codepoint = (((codepoint - 0xD800) << 10) | (codepoint2 - 0xDC00)) + 0x10000; + } + TEncoding::Encode(os, codepoint); + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, escapeOffset); + } + else if (RAPIDJSON_UNLIKELY(c == '"')) // Closing double quote + { is.Take(); + os.Put('\0'); // null-terminate the string + return; + } + else if (RAPIDJSON_UNLIKELY(static_cast(c) < 0x20)) // RFC 4627: unescaped = %x20-21 / %x23-5B / %x5D-10FFFF + { if (c == '\0') + RAPIDJSON_PARSE_ERROR(kParseErrorStringMissQuotationMark, is.Tell()); + else + RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, is.Tell()); + } + else + { size_t offset = is.Tell(); + if (RAPIDJSON_UNLIKELY((parseFlags & kParseValidateEncodingFlag ? + !Transcoder::Validate(is, os) : + !Transcoder::Transcode(is, os)))) + RAPIDJSON_PARSE_ERROR(kParseErrorStringInvalidEncoding, offset); + } } + } - template - static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InputStream&, OutputStream&) { - // Do nothing for generic version - } + template + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InputStream&, OutputStream&) + { // Do nothing for generic version + } #if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) - // StringStream -> StackStream - static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(StringStream& is, StackStream& os) { - const char* p = is.src_; - - // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); - while (p != nextAligned) - if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { - is.src_ = p; - return; - } - else - os.Put(*p++); - - // The rest of string using SIMD - static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; - static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; - static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; - const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); - const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); - const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); - - for (;; p += 16) { - const __m128i s = _mm_load_si128(reinterpret_cast(p)); - const __m128i t1 = _mm_cmpeq_epi8(s, dq); - const __m128i t2 = _mm_cmpeq_epi8(s, bs); - const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 - const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); - unsigned short r = static_cast(_mm_movemask_epi8(x)); - if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped - SizeType length; - #ifdef _MSC_VER // Find the index of first escaped - unsigned long offset; - _BitScanForward(&offset, r); - length = offset; - #else - length = static_cast(__builtin_ffs(r) - 1); - #endif - char* q = reinterpret_cast(os.Push(length)); - for (size_t i = 0; i < length; i++) - q[i] = p[i]; - - p += length; - break; - } - _mm_storeu_si128(reinterpret_cast<__m128i *>(os.Push(16)), s); - } + // StringStream -> StackStream + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(StringStream& is, StackStream& os) + { const char* p = is.src_; - is.src_ = p; + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) + { is.src_ = p; + return; + } + else + os.Put(*p++); + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (;; p += 16) + { const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (RAPIDJSON_UNLIKELY(r != 0)) // some of characters is escaped + { SizeType length; +#ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + length = offset; +#else + length = static_cast(__builtin_ffs(r) - 1); +#endif + char* q = reinterpret_cast(os.Push(length)); + for (size_t i = 0; i < length; i++) + q[i] = p[i]; + + p += length; + break; + } + _mm_storeu_si128(reinterpret_cast<__m128i *>(os.Push(16)), s); } - // InsituStringStream -> InsituStringStream - static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InsituStringStream& is, InsituStringStream& os) { - RAPIDJSON_ASSERT(&is == &os); - (void)os; + is.src_ = p; + } - if (is.src_ == is.dst_) { - SkipUnescapedString(is); - return; - } + // InsituStringStream -> InsituStringStream + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InsituStringStream& is, InsituStringStream& os) + { RAPIDJSON_ASSERT(&is == &os); + (void)os; - char* p = is.src_; - char *q = is.dst_; + if (is.src_ == is.dst_) + { SkipUnescapedString(is); + return; + } - // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); - while (p != nextAligned) - if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { - is.src_ = p; - is.dst_ = q; - return; - } - else - *q++ = *p++; - - // The rest of string using SIMD - static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; - static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; - static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; - const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); - const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); - const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); - - for (;; p += 16, q += 16) { - const __m128i s = _mm_load_si128(reinterpret_cast(p)); - const __m128i t1 = _mm_cmpeq_epi8(s, dq); - const __m128i t2 = _mm_cmpeq_epi8(s, bs); - const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 - const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); - unsigned short r = static_cast(_mm_movemask_epi8(x)); - if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped - size_t length; + char* p = is.src_; + char *q = is.dst_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) + { is.src_ = p; + is.dst_ = q; + return; + } + else + *q++ = *p++; + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (;; p += 16, q += 16) + { const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (RAPIDJSON_UNLIKELY(r != 0)) // some of characters is escaped + { size_t length; #ifdef _MSC_VER // Find the index of first escaped - unsigned long offset; - _BitScanForward(&offset, r); - length = offset; + unsigned long offset; + _BitScanForward(&offset, r); + length = offset; #else - length = static_cast(__builtin_ffs(r) - 1); + length = static_cast(__builtin_ffs(r) - 1); #endif - for (const char* pend = p + length; p != pend; ) - *q++ = *p++; - break; - } - _mm_storeu_si128(reinterpret_cast<__m128i *>(q), s); - } - - is.src_ = p; - is.dst_ = q; + for (const char* pend = p + length; p != pend; ) + *q++ = *p++; + break; + } + _mm_storeu_si128(reinterpret_cast<__m128i *>(q), s); } - // When read/write pointers are the same for insitu stream, just skip unescaped characters - static RAPIDJSON_FORCEINLINE void SkipUnescapedString(InsituStringStream& is) { - RAPIDJSON_ASSERT(is.src_ == is.dst_); - char* p = is.src_; - - // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); - for (; p != nextAligned; p++) - if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { - is.src_ = is.dst_ = p; - return; - } + is.src_ = p; + is.dst_ = q; + } + + // When read/write pointers are the same for insitu stream, just skip unescaped characters + static RAPIDJSON_FORCEINLINE void SkipUnescapedString(InsituStringStream& is) + { RAPIDJSON_ASSERT(is.src_ == is.dst_); + char* p = is.src_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + for (; p != nextAligned; p++) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) + { is.src_ = is.dst_ = p; + return; + } - // The rest of string using SIMD - static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; - static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; - static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; - const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); - const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); - const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); - - for (;; p += 16) { - const __m128i s = _mm_load_si128(reinterpret_cast(p)); - const __m128i t1 = _mm_cmpeq_epi8(s, dq); - const __m128i t2 = _mm_cmpeq_epi8(s, bs); - const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 - const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); - unsigned short r = static_cast(_mm_movemask_epi8(x)); - if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped - size_t length; + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (;; p += 16) + { const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (RAPIDJSON_UNLIKELY(r != 0)) // some of characters is escaped + { size_t length; #ifdef _MSC_VER // Find the index of first escaped - unsigned long offset; - _BitScanForward(&offset, r); - length = offset; + unsigned long offset; + _BitScanForward(&offset, r); + length = offset; #else - length = static_cast(__builtin_ffs(r) - 1); + length = static_cast(__builtin_ffs(r) - 1); #endif - p += length; - break; - } - } - - is.src_ = is.dst_ = p; + p += length; + break; + } } -#endif - - template - class NumberStream; - - template - class NumberStream { - public: - typedef typename InputStream::Ch Ch; - NumberStream(GenericReader& reader, InputStream& s) : is(s) { (void)reader; } - ~NumberStream() {} + is.src_ = is.dst_ = p; + } +#endif - RAPIDJSON_FORCEINLINE Ch Peek() const { return is.Peek(); } - RAPIDJSON_FORCEINLINE Ch TakePush() { return is.Take(); } - RAPIDJSON_FORCEINLINE Ch Take() { return is.Take(); } - RAPIDJSON_FORCEINLINE void Push(char) {} + template + class NumberStream; - size_t Tell() { return is.Tell(); } - size_t Length() { return 0; } - const char* Pop() { return 0; } + template + class NumberStream + { + public: + typedef typename InputStream::Ch Ch; - protected: - NumberStream& operator=(const NumberStream&); + NumberStream(GenericReader& reader, InputStream& s) : is(s) { (void)reader; } + ~NumberStream() {} - InputStream& is; - }; + RAPIDJSON_FORCEINLINE Ch Peek() const { return is.Peek(); } + RAPIDJSON_FORCEINLINE Ch TakePush() { return is.Take(); } + RAPIDJSON_FORCEINLINE Ch Take() { return is.Take(); } + RAPIDJSON_FORCEINLINE void Push(char) {} - template - class NumberStream : public NumberStream { - typedef NumberStream Base; - public: - NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is), stackStream(reader.stack_) {} - ~NumberStream() {} + size_t Tell() { return is.Tell(); } + size_t Length() { return 0; } + const char* Pop() { return 0; } - RAPIDJSON_FORCEINLINE Ch TakePush() { - stackStream.Put(static_cast(Base::is.Peek())); - return Base::is.Take(); - } + protected: + NumberStream& operator=(const NumberStream&); - RAPIDJSON_FORCEINLINE void Push(char c) { - stackStream.Put(c); - } + InputStream& is; + }; - size_t Length() { return stackStream.Length(); } + template + class NumberStream : public NumberStream + { typedef NumberStream Base; + public: + NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is), stackStream(reader.stack_) {} + ~NumberStream() {} - const char* Pop() { - stackStream.Put('\0'); - return stackStream.Pop(); - } + RAPIDJSON_FORCEINLINE Ch TakePush() + { stackStream.Put(static_cast(Base::is.Peek())); + return Base::is.Take(); + } - private: - StackStream stackStream; - }; + RAPIDJSON_FORCEINLINE void Push(char c) + { stackStream.Put(c); + } - template - class NumberStream : public NumberStream { - typedef NumberStream Base; - public: - NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is) {} - ~NumberStream() {} + size_t Length() { return stackStream.Length(); } - RAPIDJSON_FORCEINLINE Ch Take() { return Base::TakePush(); } - }; + const char* Pop() + { stackStream.Put('\0'); + return stackStream.Pop(); + } - template - void ParseNumber(InputStream& is, Handler& handler) { - internal::StreamLocalCopy copy(is); - NumberStream s(*this, copy.s); - - size_t startOffset = s.Tell(); - double d = 0.0; - bool useNanOrInf = false; - - // Parse minus - bool minus = Consume(s, '-'); - - // Parse int: zero / ( digit1-9 *DIGIT ) - unsigned i = 0; - uint64_t i64 = 0; - bool use64bit = false; - int significandDigit = 0; - if (RAPIDJSON_UNLIKELY(s.Peek() == '0')) { - i = 0; - s.TakePush(); - } - else if (RAPIDJSON_LIKELY(s.Peek() >= '1' && s.Peek() <= '9')) { - i = static_cast(s.TakePush() - '0'); - - if (minus) - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - if (RAPIDJSON_UNLIKELY(i >= 214748364)) { // 2^31 = 2147483648 - if (RAPIDJSON_LIKELY(i != 214748364 || s.Peek() > '8')) { - i64 = i; - use64bit = true; - break; - } - } - i = i * 10 + static_cast(s.TakePush() - '0'); - significandDigit++; - } - else - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - if (RAPIDJSON_UNLIKELY(i >= 429496729)) { // 2^32 - 1 = 4294967295 - if (RAPIDJSON_LIKELY(i != 429496729 || s.Peek() > '5')) { - i64 = i; - use64bit = true; - break; - } - } - i = i * 10 + static_cast(s.TakePush() - '0'); - significandDigit++; - } - } - // Parse NaN or Infinity here - else if ((parseFlags & kParseNanAndInfFlag) && RAPIDJSON_LIKELY((s.Peek() == 'I' || s.Peek() == 'N'))) { - useNanOrInf = true; - if (RAPIDJSON_LIKELY(Consume(s, 'N') && Consume(s, 'a') && Consume(s, 'N'))) { - d = std::numeric_limits::quiet_NaN(); + private: + StackStream stackStream; + }; + + template + class NumberStream : public NumberStream + { typedef NumberStream Base; + public: + NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is) {} + ~NumberStream() {} + + RAPIDJSON_FORCEINLINE Ch Take() { return Base::TakePush(); } + }; + + template + void ParseNumber(InputStream& is, Handler& handler) + { internal::StreamLocalCopy copy(is); + NumberStream s(*this, copy.s); + + size_t startOffset = s.Tell(); + double d = 0.0; + bool useNanOrInf = false; + + // Parse minus + bool minus = Consume(s, '-'); + + // Parse int: zero / ( digit1-9 *DIGIT ) + unsigned i = 0; + uint64_t i64 = 0; + bool use64bit = false; + int significandDigit = 0; + if (RAPIDJSON_UNLIKELY(s.Peek() == '0')) + { i = 0; + s.TakePush(); + } + else if (RAPIDJSON_LIKELY(s.Peek() >= '1' && s.Peek() <= '9')) + { i = static_cast(s.TakePush() - '0'); + + if (minus) + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) + { if (RAPIDJSON_UNLIKELY(i >= 214748364)) // 2^31 = 2147483648 + { if (RAPIDJSON_LIKELY(i != 214748364 || s.Peek() > '8')) + { i64 = i; + use64bit = true; + break; } - else if (RAPIDJSON_LIKELY(Consume(s, 'I') && Consume(s, 'n') && Consume(s, 'f'))) { - d = (minus ? -std::numeric_limits::infinity() : std::numeric_limits::infinity()); - if (RAPIDJSON_UNLIKELY(s.Peek() == 'i' && !(Consume(s, 'i') && Consume(s, 'n') - && Consume(s, 'i') && Consume(s, 't') && Consume(s, 'y')))) - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + } + i = i * 10 + static_cast(s.TakePush() - '0'); + significandDigit++; + } + else + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) + { if (RAPIDJSON_UNLIKELY(i >= 429496729)) // 2^32 - 1 = 4294967295 + { if (RAPIDJSON_LIKELY(i != 429496729 || s.Peek() > '5')) + { i64 = i; + use64bit = true; + break; } - else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + } + i = i * 10 + static_cast(s.TakePush() - '0'); + significandDigit++; } - else - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); - - // Parse 64bit int - bool useDouble = false; - if (use64bit) { - if (minus) - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - if (RAPIDJSON_UNLIKELY(i64 >= RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC))) // 2^63 = 9223372036854775808 - if (RAPIDJSON_LIKELY(i64 != RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC) || s.Peek() > '8')) { - d = static_cast(i64); - useDouble = true; - break; - } - i64 = i64 * 10 + static_cast(s.TakePush() - '0'); - significandDigit++; - } - else - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - if (RAPIDJSON_UNLIKELY(i64 >= RAPIDJSON_UINT64_C2(0x19999999, 0x99999999))) // 2^64 - 1 = 18446744073709551615 - if (RAPIDJSON_LIKELY(i64 != RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || s.Peek() > '5')) { - d = static_cast(i64); - useDouble = true; - break; - } - i64 = i64 * 10 + static_cast(s.TakePush() - '0'); - significandDigit++; - } + } + // Parse NaN or Infinity here + else if ((parseFlags & kParseNanAndInfFlag) && RAPIDJSON_LIKELY((s.Peek() == 'I' || s.Peek() == 'N'))) + { useNanOrInf = true; + if (RAPIDJSON_LIKELY(Consume(s, 'N') && Consume(s, 'a') && Consume(s, 'N'))) + { d = std::numeric_limits::quiet_NaN(); + } + else if (RAPIDJSON_LIKELY(Consume(s, 'I') && Consume(s, 'n') && Consume(s, 'f'))) + { d = (minus ? -std::numeric_limits::infinity() : std::numeric_limits::infinity()); + if (RAPIDJSON_UNLIKELY(s.Peek() == 'i' && !(Consume(s, 'i') && Consume(s, 'n') + && Consume(s, 'i') && Consume(s, 't') && Consume(s, 'y')))) + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + + // Parse 64bit int + bool useDouble = false; + if (use64bit) + { if (minus) + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) + { if (RAPIDJSON_UNLIKELY(i64 >= RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC))) // 2^63 = 9223372036854775808 + if (RAPIDJSON_LIKELY(i64 != RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC) || s.Peek() > '8')) + { d = static_cast(i64); + useDouble = true; + break; + } + i64 = i64 * 10 + static_cast(s.TakePush() - '0'); + significandDigit++; } - - // Force double for big integer - if (useDouble) { - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - if (RAPIDJSON_UNLIKELY(d >= 1.7976931348623157e307)) // DBL_MAX / 10.0 - RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); - d = d * 10 + (s.TakePush() - '0'); + else + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) + { if (RAPIDJSON_UNLIKELY(i64 >= RAPIDJSON_UINT64_C2(0x19999999, 0x99999999))) // 2^64 - 1 = 18446744073709551615 + if (RAPIDJSON_LIKELY(i64 != RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || s.Peek() > '5')) + { d = static_cast(i64); + useDouble = true; + break; } + i64 = i64 * 10 + static_cast(s.TakePush() - '0'); + significandDigit++; } + } - // Parse frac = decimal-point 1*DIGIT - int expFrac = 0; - size_t decimalPosition; - if (Consume(s, '.')) { - decimalPosition = s.Length(); + // Force double for big integer + if (useDouble) + { while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) + { if (RAPIDJSON_UNLIKELY(d >= 1.7976931348623157e307)) // DBL_MAX / 10.0 + RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); + d = d * 10 + (s.TakePush() - '0'); + } + } + + // Parse frac = decimal-point 1*DIGIT + int expFrac = 0; + size_t decimalPosition; + if (Consume(s, '.')) + { decimalPosition = s.Length(); - if (RAPIDJSON_UNLIKELY(!(s.Peek() >= '0' && s.Peek() <= '9'))) - RAPIDJSON_PARSE_ERROR(kParseErrorNumberMissFraction, s.Tell()); + if (RAPIDJSON_UNLIKELY(!(s.Peek() >= '0' && s.Peek() <= '9'))) + RAPIDJSON_PARSE_ERROR(kParseErrorNumberMissFraction, s.Tell()); - if (!useDouble) { + if (!useDouble) + { #if RAPIDJSON_64BIT - // Use i64 to store significand in 64-bit architecture - if (!use64bit) - i64 = i; - - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - if (i64 > RAPIDJSON_UINT64_C2(0x1FFFFF, 0xFFFFFFFF)) // 2^53 - 1 for fast path - break; - else { - i64 = i64 * 10 + static_cast(s.TakePush() - '0'); - --expFrac; - if (i64 != 0) - significandDigit++; - } - } - - d = static_cast(i64); + // Use i64 to store significand in 64-bit architecture + if (!use64bit) + i64 = i; + + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) + { if (i64 > RAPIDJSON_UINT64_C2(0x1FFFFF, 0xFFFFFFFF)) // 2^53 - 1 for fast path + break; + else + { i64 = i64 * 10 + static_cast(s.TakePush() - '0'); + --expFrac; + if (i64 != 0) + significandDigit++; + } + } + + d = static_cast(i64); #else - // Use double to store significand in 32-bit architecture - d = static_cast(use64bit ? i64 : i); + // Use double to store significand in 32-bit architecture + d = static_cast(use64bit ? i64 : i); #endif - useDouble = true; - } - - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - if (significandDigit < 17) { - d = d * 10.0 + (s.TakePush() - '0'); - --expFrac; - if (RAPIDJSON_LIKELY(d > 0.0)) - significandDigit++; - } - else - s.TakePush(); - } + useDouble = true; + } + + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) + { if (significandDigit < 17) + { d = d * 10.0 + (s.TakePush() - '0'); + --expFrac; + if (RAPIDJSON_LIKELY(d > 0.0)) + significandDigit++; } else - decimalPosition = s.Length(); // decimal position at the end of integer. - - // Parse exp = e [ minus / plus ] 1*DIGIT - int exp = 0; - if (Consume(s, 'e') || Consume(s, 'E')) { - if (!useDouble) { - d = static_cast(use64bit ? i64 : i); - useDouble = true; + s.TakePush(); + } + } + else + decimalPosition = s.Length(); // decimal position at the end of integer. + + // Parse exp = e [ minus / plus ] 1*DIGIT + int exp = 0; + if (Consume(s, 'e') || Consume(s, 'E')) + { if (!useDouble) + { d = static_cast(use64bit ? i64 : i); + useDouble = true; + } + + bool expMinus = false; + if (Consume(s, '+')) + ; + else if (Consume(s, '-')) + expMinus = true; + + if (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) + { exp = static_cast(s.Take() - '0'); + if (expMinus) + { while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) + { exp = exp * 10 + static_cast(s.Take() - '0'); + if (exp >= 214748364) // Issue #313: prevent overflow exponent + { while (RAPIDJSON_UNLIKELY(s.Peek() >= '0' && s.Peek() <= '9')) // Consume the rest of exponent + s.Take(); } + } + } + else // positive exp + { int maxExp = 308 - expFrac; + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) + { exp = exp * 10 + static_cast(s.Take() - '0'); + if (RAPIDJSON_UNLIKELY(exp > maxExp)) + RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); + } + } + } + else + RAPIDJSON_PARSE_ERROR(kParseErrorNumberMissExponent, s.Tell()); - bool expMinus = false; - if (Consume(s, '+')) - ; - else if (Consume(s, '-')) - expMinus = true; - - if (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - exp = static_cast(s.Take() - '0'); - if (expMinus) { - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - exp = exp * 10 + static_cast(s.Take() - '0'); - if (exp >= 214748364) { // Issue #313: prevent overflow exponent - while (RAPIDJSON_UNLIKELY(s.Peek() >= '0' && s.Peek() <= '9')) // Consume the rest of exponent - s.Take(); - } - } - } - else { // positive exp - int maxExp = 308 - expFrac; - while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - exp = exp * 10 + static_cast(s.Take() - '0'); - if (RAPIDJSON_UNLIKELY(exp > maxExp)) - RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); - } - } - } - else - RAPIDJSON_PARSE_ERROR(kParseErrorNumberMissExponent, s.Tell()); + if (expMinus) + exp = -exp; + } - if (expMinus) - exp = -exp; + // Finish parsing, call event according to the type of number. + bool cont = true; + + if (parseFlags & kParseNumbersAsStringsFlag) + { if (parseFlags & kParseInsituFlag) + { s.Pop(); // Pop stack no matter if it will be used or not. + typename InputStream::Ch* head = is.PutBegin(); + const size_t length = s.Tell() - startOffset; + RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); + // unable to insert the \0 character here, it will erase the comma after this number + const typename TargetEncoding::Ch* const str = reinterpret_cast(head); + cont = handler.RawNumber(str, SizeType(length), false); + } + else + { SizeType numCharsToCopy = static_cast(s.Length()); + StringStream srcStream(s.Pop()); + StackStream dstStream(stack_); + while (numCharsToCopy--) + { Transcoder, TargetEncoding>::Transcode(srcStream, dstStream); } + dstStream.Put('\0'); + const typename TargetEncoding::Ch* str = dstStream.Pop(); + const SizeType length = static_cast(dstStream.Length()) - 1; + cont = handler.RawNumber(str, SizeType(length), true); + } + } + else + { size_t length = s.Length(); + const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not. - // Finish parsing, call event according to the type of number. - bool cont = true; - - if (parseFlags & kParseNumbersAsStringsFlag) { - if (parseFlags & kParseInsituFlag) { - s.Pop(); // Pop stack no matter if it will be used or not. - typename InputStream::Ch* head = is.PutBegin(); - const size_t length = s.Tell() - startOffset; - RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); - // unable to insert the \0 character here, it will erase the comma after this number - const typename TargetEncoding::Ch* const str = reinterpret_cast(head); - cont = handler.RawNumber(str, SizeType(length), false); - } - else { - SizeType numCharsToCopy = static_cast(s.Length()); - StringStream srcStream(s.Pop()); - StackStream dstStream(stack_); - while (numCharsToCopy--) { - Transcoder, TargetEncoding>::Transcode(srcStream, dstStream); - } - dstStream.Put('\0'); - const typename TargetEncoding::Ch* str = dstStream.Pop(); - const SizeType length = static_cast(dstStream.Length()) - 1; - cont = handler.RawNumber(str, SizeType(length), true); - } + if (useDouble) + { int p = exp + expFrac; + if (parseFlags & kParseFullPrecisionFlag) + d = internal::StrtodFullPrecision(d, p, decimal, length, decimalPosition, exp); + else + d = internal::StrtodNormalPrecision(d, p); + + cont = handler.Double(minus ? -d : d); + } + else if (useNanOrInf) + { cont = handler.Double(d); + } + else + { if (use64bit) + { if (minus) + cont = handler.Int64(static_cast(~i64 + 1)); + else + cont = handler.Uint64(i64); } - else { - size_t length = s.Length(); - const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not. - - if (useDouble) { - int p = exp + expFrac; - if (parseFlags & kParseFullPrecisionFlag) - d = internal::StrtodFullPrecision(d, p, decimal, length, decimalPosition, exp); - else - d = internal::StrtodNormalPrecision(d, p); - - cont = handler.Double(minus ? -d : d); - } - else if (useNanOrInf) { - cont = handler.Double(d); - } - else { - if (use64bit) { - if (minus) - cont = handler.Int64(static_cast(~i64 + 1)); - else - cont = handler.Uint64(i64); - } - else { - if (minus) - cont = handler.Int(static_cast(~i + 1)); - else - cont = handler.Uint(i); - } - } + else + { if (minus) + cont = handler.Int(static_cast(~i + 1)); + else + cont = handler.Uint(i); } - if (RAPIDJSON_UNLIKELY(!cont)) - RAPIDJSON_PARSE_ERROR(kParseErrorTermination, startOffset); + } } + if (RAPIDJSON_UNLIKELY(!cont)) + RAPIDJSON_PARSE_ERROR(kParseErrorTermination, startOffset); + } + + // Parse any JSON value + template + void ParseValue(InputStream& is, Handler& handler) + { switch (is.Peek()) + { case 'n': ParseNull (is, handler); break; + case 't': ParseTrue (is, handler); break; + case 'f': ParseFalse (is, handler); break; + case '"': ParseString(is, handler); break; + case '{': ParseObject(is, handler); break; + case '[': ParseArray (is, handler); break; + default : + ParseNumber(is, handler); + break; - // Parse any JSON value - template - void ParseValue(InputStream& is, Handler& handler) { - switch (is.Peek()) { - case 'n': ParseNull (is, handler); break; - case 't': ParseTrue (is, handler); break; - case 'f': ParseFalse (is, handler); break; - case '"': ParseString(is, handler); break; - case '{': ParseObject(is, handler); break; - case '[': ParseArray (is, handler); break; - default : - ParseNumber(is, handler); - break; - - } } + } - // Iterative Parsing - - // States - enum IterativeParsingState { - IterativeParsingStartState = 0, - IterativeParsingFinishState, - IterativeParsingErrorState, - - // Object states - IterativeParsingObjectInitialState, - IterativeParsingMemberKeyState, - IterativeParsingKeyValueDelimiterState, - IterativeParsingMemberValueState, - IterativeParsingMemberDelimiterState, - IterativeParsingObjectFinishState, - - // Array states - IterativeParsingArrayInitialState, - IterativeParsingElementState, - IterativeParsingElementDelimiterState, - IterativeParsingArrayFinishState, - - // Single value state - IterativeParsingValueState - }; + // Iterative Parsing - enum { cIterativeParsingStateCount = IterativeParsingValueState + 1 }; + // States + enum IterativeParsingState + { IterativeParsingStartState = 0, + IterativeParsingFinishState, + IterativeParsingErrorState, - // Tokens - enum Token { - LeftBracketToken = 0, - RightBracketToken, + // Object states + IterativeParsingObjectInitialState, + IterativeParsingMemberKeyState, + IterativeParsingKeyValueDelimiterState, + IterativeParsingMemberValueState, + IterativeParsingMemberDelimiterState, + IterativeParsingObjectFinishState, - LeftCurlyBracketToken, - RightCurlyBracketToken, + // Array states + IterativeParsingArrayInitialState, + IterativeParsingElementState, + IterativeParsingElementDelimiterState, + IterativeParsingArrayFinishState, - CommaToken, - ColonToken, + // Single value state + IterativeParsingValueState + }; - StringToken, - FalseToken, - TrueToken, - NullToken, - NumberToken, + enum { cIterativeParsingStateCount = IterativeParsingValueState + 1 }; - kTokenCount - }; + // Tokens + enum Token + { LeftBracketToken = 0, + RightBracketToken, - RAPIDJSON_FORCEINLINE Token Tokenize(Ch c) { + LeftCurlyBracketToken, + RightCurlyBracketToken, + + CommaToken, + ColonToken, + + StringToken, + FalseToken, + TrueToken, + NullToken, + NumberToken, + + kTokenCount + }; + + RAPIDJSON_FORCEINLINE Token Tokenize(Ch c) + { //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN #define N NumberToken #define N16 N,N,N,N,N,N,N,N,N,N,N,N,N,N,N,N - // Maps from ASCII to Token - static const unsigned char tokenMap[256] = { - N16, // 00~0F - N16, // 10~1F - N, N, StringToken, N, N, N, N, N, N, N, N, N, CommaToken, N, N, N, // 20~2F - N, N, N, N, N, N, N, N, N, N, ColonToken, N, N, N, N, N, // 30~3F - N16, // 40~4F - N, N, N, N, N, N, N, N, N, N, N, LeftBracketToken, N, RightBracketToken, N, N, // 50~5F - N, N, N, N, N, N, FalseToken, N, N, N, N, N, N, N, NullToken, N, // 60~6F - N, N, N, N, TrueToken, N, N, N, N, N, N, LeftCurlyBracketToken, N, RightCurlyBracketToken, N, N, // 70~7F - N16, N16, N16, N16, N16, N16, N16, N16 // 80~FF - }; + // Maps from ASCII to Token + static const unsigned char tokenMap[256] = + { N16, // 00~0F + N16, // 10~1F + N, N, StringToken, N, N, N, N, N, N, N, N, N, CommaToken, N, N, N, // 20~2F + N, N, N, N, N, N, N, N, N, N, ColonToken, N, N, N, N, N, // 30~3F + N16, // 40~4F + N, N, N, N, N, N, N, N, N, N, N, LeftBracketToken, N, RightBracketToken, N, N, // 50~5F + N, N, N, N, N, N, FalseToken, N, N, N, N, N, N, N, NullToken, N, // 60~6F + N, N, N, N, TrueToken, N, N, N, N, N, N, LeftCurlyBracketToken, N, RightCurlyBracketToken, N, N, // 70~7F + N16, N16, N16, N16, N16, N16, N16, N16 // 80~FF + }; #undef N #undef N16 //!@endcond - if (sizeof(Ch) == 1 || static_cast(c) < 256) - return static_cast(tokenMap[static_cast(c)]); + if (sizeof(Ch) == 1 || static_cast(c) < 256) + return static_cast(tokenMap[static_cast(c)]); + else + return NumberToken; + } + + RAPIDJSON_FORCEINLINE IterativeParsingState Predict(IterativeParsingState state, Token token) + { // current state x one lookahead token -> new state + static const char G[cIterativeParsingStateCount][kTokenCount] = + { // Start + { IterativeParsingArrayInitialState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingValueState, // String + IterativeParsingValueState, // False + IterativeParsingValueState, // True + IterativeParsingValueState, // Null + IterativeParsingValueState // Number + }, + // Finish(sink state) + { IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // Error(sink state) + { IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // ObjectInitial + { IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingObjectFinishState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberKeyState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // MemberKey + { IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingKeyValueDelimiterState, // Colon + IterativeParsingErrorState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // KeyValueDelimiter + { IterativeParsingArrayInitialState, // Left bracket(push MemberValue state) + IterativeParsingErrorState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push MemberValue state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberValueState, // String + IterativeParsingMemberValueState, // False + IterativeParsingMemberValueState, // True + IterativeParsingMemberValueState, // Null + IterativeParsingMemberValueState // Number + }, + // MemberValue + { IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingObjectFinishState, // Right curly bracket + IterativeParsingMemberDelimiterState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingErrorState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // MemberDelimiter + { IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingObjectFinishState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberKeyState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // ObjectFinish(sink state) + { IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // ArrayInitial + { IterativeParsingArrayInitialState, // Left bracket(push Element state) + IterativeParsingArrayFinishState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push Element state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingElementState, // String + IterativeParsingElementState, // False + IterativeParsingElementState, // True + IterativeParsingElementState, // Null + IterativeParsingElementState // Number + }, + // Element + { IterativeParsingErrorState, // Left bracket + IterativeParsingArrayFinishState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingErrorState, // Right curly bracket + IterativeParsingElementDelimiterState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingErrorState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number + }, + // ElementDelimiter + { IterativeParsingArrayInitialState, // Left bracket(push Element state) + IterativeParsingArrayFinishState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push Element state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingElementState, // String + IterativeParsingElementState, // False + IterativeParsingElementState, // True + IterativeParsingElementState, // Null + IterativeParsingElementState // Number + }, + // ArrayFinish(sink state) + { IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // Single Value (sink state) + { IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + } + }; // End of G + + return static_cast(G[state][token]); + } + + // Make an advance in the token stream and state based on the candidate destination state which was returned by Transit(). + // May return a new state on state pop. + template + RAPIDJSON_FORCEINLINE IterativeParsingState Transit(IterativeParsingState src, Token token, IterativeParsingState dst, InputStream& is, Handler& handler) + { (void)token; + + switch (dst) + { case IterativeParsingErrorState: + return dst; + + case IterativeParsingObjectInitialState: + case IterativeParsingArrayInitialState: + { // Push the state(Element or MemeberValue) if we are nested in another array or value of member. + // In this way we can get the correct state on ObjectFinish or ArrayFinish by frame pop. + IterativeParsingState n = src; + if (src == IterativeParsingArrayInitialState || src == IterativeParsingElementDelimiterState) + n = IterativeParsingElementState; + else if (src == IterativeParsingKeyValueDelimiterState) + n = IterativeParsingMemberValueState; + // Push current state. + *stack_.template Push(1) = n; + // Initialize and push the member/element count. + *stack_.template Push(1) = 0; + // Call handler + bool hr = (dst == IterativeParsingObjectInitialState) ? handler.StartObject() : handler.StartArray(); + // On handler short circuits the parsing. + if (!hr) + { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); + return IterativeParsingErrorState; + } else - return NumberToken; - } + { is.Take(); + return dst; + } + } - RAPIDJSON_FORCEINLINE IterativeParsingState Predict(IterativeParsingState state, Token token) { - // current state x one lookahead token -> new state - static const char G[cIterativeParsingStateCount][kTokenCount] = { - // Start - { - IterativeParsingArrayInitialState, // Left bracket - IterativeParsingErrorState, // Right bracket - IterativeParsingObjectInitialState, // Left curly bracket - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingValueState, // String - IterativeParsingValueState, // False - IterativeParsingValueState, // True - IterativeParsingValueState, // Null - IterativeParsingValueState // Number - }, - // Finish(sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - }, - // Error(sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - }, - // ObjectInitial - { - IterativeParsingErrorState, // Left bracket - IterativeParsingErrorState, // Right bracket - IterativeParsingErrorState, // Left curly bracket - IterativeParsingObjectFinishState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingMemberKeyState, // String - IterativeParsingErrorState, // False - IterativeParsingErrorState, // True - IterativeParsingErrorState, // Null - IterativeParsingErrorState // Number - }, - // MemberKey - { - IterativeParsingErrorState, // Left bracket - IterativeParsingErrorState, // Right bracket - IterativeParsingErrorState, // Left curly bracket - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingKeyValueDelimiterState, // Colon - IterativeParsingErrorState, // String - IterativeParsingErrorState, // False - IterativeParsingErrorState, // True - IterativeParsingErrorState, // Null - IterativeParsingErrorState // Number - }, - // KeyValueDelimiter - { - IterativeParsingArrayInitialState, // Left bracket(push MemberValue state) - IterativeParsingErrorState, // Right bracket - IterativeParsingObjectInitialState, // Left curly bracket(push MemberValue state) - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingMemberValueState, // String - IterativeParsingMemberValueState, // False - IterativeParsingMemberValueState, // True - IterativeParsingMemberValueState, // Null - IterativeParsingMemberValueState // Number - }, - // MemberValue - { - IterativeParsingErrorState, // Left bracket - IterativeParsingErrorState, // Right bracket - IterativeParsingErrorState, // Left curly bracket - IterativeParsingObjectFinishState, // Right curly bracket - IterativeParsingMemberDelimiterState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingErrorState, // String - IterativeParsingErrorState, // False - IterativeParsingErrorState, // True - IterativeParsingErrorState, // Null - IterativeParsingErrorState // Number - }, - // MemberDelimiter - { - IterativeParsingErrorState, // Left bracket - IterativeParsingErrorState, // Right bracket - IterativeParsingErrorState, // Left curly bracket - IterativeParsingObjectFinishState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingMemberKeyState, // String - IterativeParsingErrorState, // False - IterativeParsingErrorState, // True - IterativeParsingErrorState, // Null - IterativeParsingErrorState // Number - }, - // ObjectFinish(sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - }, - // ArrayInitial - { - IterativeParsingArrayInitialState, // Left bracket(push Element state) - IterativeParsingArrayFinishState, // Right bracket - IterativeParsingObjectInitialState, // Left curly bracket(push Element state) - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingElementState, // String - IterativeParsingElementState, // False - IterativeParsingElementState, // True - IterativeParsingElementState, // Null - IterativeParsingElementState // Number - }, - // Element - { - IterativeParsingErrorState, // Left bracket - IterativeParsingArrayFinishState, // Right bracket - IterativeParsingErrorState, // Left curly bracket - IterativeParsingErrorState, // Right curly bracket - IterativeParsingElementDelimiterState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingErrorState, // String - IterativeParsingErrorState, // False - IterativeParsingErrorState, // True - IterativeParsingErrorState, // Null - IterativeParsingErrorState // Number - }, - // ElementDelimiter - { - IterativeParsingArrayInitialState, // Left bracket(push Element state) - IterativeParsingArrayFinishState, // Right bracket - IterativeParsingObjectInitialState, // Left curly bracket(push Element state) - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingElementState, // String - IterativeParsingElementState, // False - IterativeParsingElementState, // True - IterativeParsingElementState, // Null - IterativeParsingElementState // Number - }, - // ArrayFinish(sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - }, - // Single Value (sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - } - }; // End of G + case IterativeParsingMemberKeyState: + ParseString(is, handler, true); + if (HasParseError()) + return IterativeParsingErrorState; + else + return dst; - return static_cast(G[state][token]); - } + case IterativeParsingKeyValueDelimiterState: + RAPIDJSON_ASSERT(token == ColonToken); + is.Take(); + return dst; - // Make an advance in the token stream and state based on the candidate destination state which was returned by Transit(). - // May return a new state on state pop. - template - RAPIDJSON_FORCEINLINE IterativeParsingState Transit(IterativeParsingState src, Token token, IterativeParsingState dst, InputStream& is, Handler& handler) { - (void)token; - - switch (dst) { - case IterativeParsingErrorState: - return dst; - - case IterativeParsingObjectInitialState: - case IterativeParsingArrayInitialState: - { - // Push the state(Element or MemeberValue) if we are nested in another array or value of member. - // In this way we can get the correct state on ObjectFinish or ArrayFinish by frame pop. - IterativeParsingState n = src; - if (src == IterativeParsingArrayInitialState || src == IterativeParsingElementDelimiterState) - n = IterativeParsingElementState; - else if (src == IterativeParsingKeyValueDelimiterState) - n = IterativeParsingMemberValueState; - // Push current state. - *stack_.template Push(1) = n; - // Initialize and push the member/element count. - *stack_.template Push(1) = 0; - // Call handler - bool hr = (dst == IterativeParsingObjectInitialState) ? handler.StartObject() : handler.StartArray(); - // On handler short circuits the parsing. - if (!hr) { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); - return IterativeParsingErrorState; - } - else { - is.Take(); - return dst; - } + case IterativeParsingMemberValueState: + // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. + ParseValue(is, handler); + if (HasParseError()) + { return IterativeParsingErrorState; } + return dst; - case IterativeParsingMemberKeyState: - ParseString(is, handler, true); - if (HasParseError()) - return IterativeParsingErrorState; - else - return dst; - - case IterativeParsingKeyValueDelimiterState: - RAPIDJSON_ASSERT(token == ColonToken); - is.Take(); - return dst; - - case IterativeParsingMemberValueState: - // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. - ParseValue(is, handler); - if (HasParseError()) { - return IterativeParsingErrorState; - } - return dst; - - case IterativeParsingElementState: - // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. - ParseValue(is, handler); - if (HasParseError()) { - return IterativeParsingErrorState; - } - return dst; - - case IterativeParsingMemberDelimiterState: - case IterativeParsingElementDelimiterState: - is.Take(); - // Update member/element count. - *stack_.template Top() = *stack_.template Top() + 1; - return dst; - - case IterativeParsingObjectFinishState: - { - // Transit from delimiter is only allowed when trailing commas are enabled - if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingMemberDelimiterState) { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorObjectMissName, is.Tell()); - return IterativeParsingErrorState; - } - // Get member count. - SizeType c = *stack_.template Pop(1); - // If the object is not empty, count the last member. - if (src == IterativeParsingMemberValueState) - ++c; - // Restore the state. - IterativeParsingState n = static_cast(*stack_.template Pop(1)); - // Transit to Finish state if this is the topmost scope. - if (n == IterativeParsingStartState) - n = IterativeParsingFinishState; - // Call handler - bool hr = handler.EndObject(c); - // On handler short circuits the parsing. - if (!hr) { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); - return IterativeParsingErrorState; - } - else { - is.Take(); - return n; - } + case IterativeParsingElementState: + // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. + ParseValue(is, handler); + if (HasParseError()) + { return IterativeParsingErrorState; } + return dst; - case IterativeParsingArrayFinishState: - { - // Transit from delimiter is only allowed when trailing commas are enabled - if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingElementDelimiterState) { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorValueInvalid, is.Tell()); - return IterativeParsingErrorState; - } - // Get element count. - SizeType c = *stack_.template Pop(1); - // If the array is not empty, count the last element. - if (src == IterativeParsingElementState) - ++c; - // Restore the state. - IterativeParsingState n = static_cast(*stack_.template Pop(1)); - // Transit to Finish state if this is the topmost scope. - if (n == IterativeParsingStartState) - n = IterativeParsingFinishState; - // Call handler - bool hr = handler.EndArray(c); - // On handler short circuits the parsing. - if (!hr) { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); - return IterativeParsingErrorState; - } - else { - is.Take(); - return n; - } + case IterativeParsingMemberDelimiterState: + case IterativeParsingElementDelimiterState: + is.Take(); + // Update member/element count. + *stack_.template Top() = *stack_.template Top() + 1; + return dst; + + case IterativeParsingObjectFinishState: + { // Transit from delimiter is only allowed when trailing commas are enabled + if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingMemberDelimiterState) + { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorObjectMissName, is.Tell()); + return IterativeParsingErrorState; } - - default: - // This branch is for IterativeParsingValueState actually. - // Use `default:` rather than - // `case IterativeParsingValueState:` is for code coverage. - - // The IterativeParsingStartState is not enumerated in this switch-case. - // It is impossible for that case. And it can be caught by following assertion. - - // The IterativeParsingFinishState is not enumerated in this switch-case either. - // It is a "derivative" state which cannot triggered from Predict() directly. - // Therefore it cannot happen here. And it can be caught by following assertion. - RAPIDJSON_ASSERT(dst == IterativeParsingValueState); - - // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. - ParseValue(is, handler); - if (HasParseError()) { - return IterativeParsingErrorState; - } - return IterativeParsingFinishState; + // Get member count. + SizeType c = *stack_.template Pop(1); + // If the object is not empty, count the last member. + if (src == IterativeParsingMemberValueState) + ++c; + // Restore the state. + IterativeParsingState n = static_cast(*stack_.template Pop(1)); + // Transit to Finish state if this is the topmost scope. + if (n == IterativeParsingStartState) + n = IterativeParsingFinishState; + // Call handler + bool hr = handler.EndObject(c); + // On handler short circuits the parsing. + if (!hr) + { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); + return IterativeParsingErrorState; } - } - - template - void HandleError(IterativeParsingState src, InputStream& is) { - if (HasParseError()) { - // Error flag has been set. - return; + else + { is.Take(); + return n; } + } - switch (src) { - case IterativeParsingStartState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentEmpty, is.Tell()); return; - case IterativeParsingFinishState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotSingular, is.Tell()); return; - case IterativeParsingObjectInitialState: - case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); return; - case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); return; - case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); return; - case IterativeParsingKeyValueDelimiterState: - case IterativeParsingArrayInitialState: - case IterativeParsingElementDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); return; - default: RAPIDJSON_ASSERT(src == IterativeParsingElementState); RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); return; + case IterativeParsingArrayFinishState: + { // Transit from delimiter is only allowed when trailing commas are enabled + if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingElementDelimiterState) + { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorValueInvalid, is.Tell()); + return IterativeParsingErrorState; } - } - - template - ParseResult IterativeParse(InputStream& is, Handler& handler) { - parseResult_.Clear(); - ClearStackOnExit scope(*this); - IterativeParsingState state = IterativeParsingStartState; + // Get element count. + SizeType c = *stack_.template Pop(1); + // If the array is not empty, count the last element. + if (src == IterativeParsingElementState) + ++c; + // Restore the state. + IterativeParsingState n = static_cast(*stack_.template Pop(1)); + // Transit to Finish state if this is the topmost scope. + if (n == IterativeParsingStartState) + n = IterativeParsingFinishState; + // Call handler + bool hr = handler.EndArray(c); + // On handler short circuits the parsing. + if (!hr) + { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); + return IterativeParsingErrorState; + } + else + { is.Take(); + return n; + } + } - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); - while (is.Peek() != '\0') { - Token t = Tokenize(is.Peek()); - IterativeParsingState n = Predict(state, t); - IterativeParsingState d = Transit(state, t, n, is, handler); + default: + // This branch is for IterativeParsingValueState actually. + // Use `default:` rather than + // `case IterativeParsingValueState:` is for code coverage. - if (d == IterativeParsingErrorState) { - HandleError(state, is); - break; - } + // The IterativeParsingStartState is not enumerated in this switch-case. + // It is impossible for that case. And it can be caught by following assertion. - state = d; + // The IterativeParsingFinishState is not enumerated in this switch-case either. + // It is a "derivative" state which cannot triggered from Predict() directly. + // Therefore it cannot happen here. And it can be caught by following assertion. + RAPIDJSON_ASSERT(dst == IterativeParsingValueState); - // Do not further consume streams if a root JSON has been parsed. - if ((parseFlags & kParseStopWhenDoneFlag) && state == IterativeParsingFinishState) - break; - - SkipWhitespaceAndComments(is); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. + ParseValue(is, handler); + if (HasParseError()) + { return IterativeParsingErrorState; } + return IterativeParsingFinishState; + } + } - // Handle the end of file. - if (state != IterativeParsingFinishState) - HandleError(state, is); + template + void HandleError(IterativeParsingState src, InputStream& is) + { if (HasParseError()) + { // Error flag has been set. + return; + } - return parseResult_; + switch (src) + { case IterativeParsingStartState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentEmpty, is.Tell()); return; + case IterativeParsingFinishState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotSingular, is.Tell()); return; + case IterativeParsingObjectInitialState: + case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); return; + case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); return; + case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); return; + case IterativeParsingKeyValueDelimiterState: + case IterativeParsingArrayInitialState: + case IterativeParsingElementDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); return; + default: RAPIDJSON_ASSERT(src == IterativeParsingElementState); RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); return; } + } + + template + ParseResult IterativeParse(InputStream& is, Handler& handler) + { parseResult_.Clear(); + ClearStackOnExit scope(*this); + IterativeParsingState state = IterativeParsingStartState; + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + while (is.Peek() != '\0') + { Token t = Tokenize(is.Peek()); + IterativeParsingState n = Predict(state, t); + IterativeParsingState d = Transit(state, t, n, is, handler); + + if (d == IterativeParsingErrorState) + { HandleError(state, is); + break; + } + + state = d; + + // Do not further consume streams if a root JSON has been parsed. + if ((parseFlags & kParseStopWhenDoneFlag) && state == IterativeParsingFinishState) + break; + + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + } + + // Handle the end of file. + if (state != IterativeParsingFinishState) + HandleError(state, is); + + return parseResult_; + } - static const size_t kDefaultStackCapacity = 256; //!< Default stack capacity in bytes for storing a single decoded string. - internal::Stack stack_; //!< A stack for storing decoded string temporarily during non-destructive parsing. - ParseResult parseResult_; + static const size_t kDefaultStackCapacity = 256; //!< Default stack capacity in bytes for storing a single decoded string. + internal::Stack stack_; //!< A stack for storing decoded string temporarily during non-destructive parsing. + ParseResult parseResult_; }; // class GenericReader //! Reader with UTF8 encoding and default allocator. diff --git a/rapidjson/include/rapidjson/schema.h b/rapidjson/include/rapidjson/schema.h index b182aa27f0b..a52dd0f78a0 100644 --- a/rapidjson/include/rapidjson/schema.h +++ b/rapidjson/include/rapidjson/schema.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available-> -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip-> All rights reserved-> // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource->org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied-> See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied-> See the License for the // specific language governing permissions and limitations under the License-> #ifndef RAPIDJSON_SCHEMA_H_ @@ -75,30 +75,31 @@ RAPIDJSON_NAMESPACE_BEGIN #if RAPIDJSON_SCHEMA_VERBOSE -namespace internal { +namespace internal +{ -inline void PrintInvalidKeyword(const char* keyword) { - printf("Fail keyword: %s\n", keyword); +inline void PrintInvalidKeyword(const char* keyword) +{ printf("Fail keyword: %s\n", keyword); } -inline void PrintInvalidKeyword(const wchar_t* keyword) { - wprintf(L"Fail keyword: %ls\n", keyword); +inline void PrintInvalidKeyword(const wchar_t* keyword) +{ wprintf(L"Fail keyword: %ls\n", keyword); } -inline void PrintInvalidDocument(const char* document) { - printf("Fail document: %s\n\n", document); +inline void PrintInvalidDocument(const char* document) +{ printf("Fail document: %s\n\n", document); } -inline void PrintInvalidDocument(const wchar_t* document) { - wprintf(L"Fail document: %ls\n\n", document); +inline void PrintInvalidDocument(const wchar_t* document) +{ wprintf(L"Fail document: %ls\n\n", document); } -inline void PrintValidatorPointers(unsigned depth, const char* s, const char* d) { - printf("S: %*s%s\nD: %*s%s\n\n", depth * 4, " ", s, depth * 4, " ", d); +inline void PrintValidatorPointers(unsigned depth, const char* s, const char* d) +{ printf("S: %*s%s\nD: %*s%s\n\n", depth * 4, " ", s, depth * 4, " ", d); } -inline void PrintValidatorPointers(unsigned depth, const wchar_t* s, const wchar_t* d) { - wprintf(L"S: %*ls%ls\nD: %*ls%ls\n\n", depth * 4, L" ", s, depth * 4, L" ", d); +inline void PrintValidatorPointers(unsigned depth, const wchar_t* s, const wchar_t* d) +{ wprintf(L"S: %*ls%ls\nD: %*ls%ls\n\n", depth * 4, L" ", s, depth * 4, L" ", d); } } // namespace internal @@ -127,7 +128,8 @@ RAPIDJSON_MULTILINEMACRO_END template class GenericSchemaDocument; -namespace internal { +namespace internal +{ template class Schema; @@ -135,26 +137,28 @@ class Schema; /////////////////////////////////////////////////////////////////////////////// // ISchemaValidator -class ISchemaValidator { +class ISchemaValidator +{ public: - virtual ~ISchemaValidator() {} - virtual bool IsValid() const = 0; + virtual ~ISchemaValidator() {} + virtual bool IsValid() const = 0; }; /////////////////////////////////////////////////////////////////////////////// // ISchemaStateFactory template -class ISchemaStateFactory { +class ISchemaStateFactory +{ public: - virtual ~ISchemaStateFactory() {} - virtual ISchemaValidator* CreateSchemaValidator(const SchemaType&) = 0; - virtual void DestroySchemaValidator(ISchemaValidator* validator) = 0; - virtual void* CreateHasher() = 0; - virtual uint64_t GetHashCode(void* hasher) = 0; - virtual void DestroryHasher(void* hasher) = 0; - virtual void* MallocState(size_t size) = 0; - virtual void FreeState(void* p) = 0; + virtual ~ISchemaStateFactory() {} + virtual ISchemaValidator* CreateSchemaValidator(const SchemaType&) = 0; + virtual void DestroySchemaValidator(ISchemaValidator* validator) = 0; + virtual void* CreateHasher() = 0; + virtual uint64_t GetHashCode(void* hasher) = 0; + virtual void DestroryHasher(void* hasher) = 0; + virtual void* MallocState(size_t size) = 0; + virtual void FreeState(void* p) = 0; }; /////////////////////////////////////////////////////////////////////////////// @@ -162,722 +166,722 @@ class ISchemaStateFactory { // For comparison of compound value template -class Hasher { +class Hasher +{ public: - typedef typename Encoding::Ch Ch; - - Hasher(Allocator* allocator = 0, size_t stackCapacity = kDefaultSize) : stack_(allocator, stackCapacity) {} - - bool Null() { return WriteType(kNullType); } - bool Bool(bool b) { return WriteType(b ? kTrueType : kFalseType); } - bool Int(int i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } - bool Uint(unsigned u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } - bool Int64(int64_t i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } - bool Uint64(uint64_t u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } - bool Double(double d) { - Number n; - if (d < 0) n.u.i = static_cast(d); - else n.u.u = static_cast(d); - n.d = d; - return WriteNumber(n); - } - - bool RawNumber(const Ch* str, SizeType len, bool) { - WriteBuffer(kNumberType, str, len * sizeof(Ch)); - return true; - } - - bool String(const Ch* str, SizeType len, bool) { - WriteBuffer(kStringType, str, len * sizeof(Ch)); - return true; - } - - bool StartObject() { return true; } - bool Key(const Ch* str, SizeType len, bool copy) { return String(str, len, copy); } - bool EndObject(SizeType memberCount) { - uint64_t h = Hash(0, kObjectType); - uint64_t* kv = stack_.template Pop(memberCount * 2); - for (SizeType i = 0; i < memberCount; i++) - h ^= Hash(kv[i * 2], kv[i * 2 + 1]); // Use xor to achieve member order insensitive - *stack_.template Push() = h; - return true; - } - - bool StartArray() { return true; } - bool EndArray(SizeType elementCount) { - uint64_t h = Hash(0, kArrayType); - uint64_t* e = stack_.template Pop(elementCount); - for (SizeType i = 0; i < elementCount; i++) - h = Hash(h, e[i]); // Use hash to achieve element order sensitive - *stack_.template Push() = h; - return true; - } - - bool IsValid() const { return stack_.GetSize() == sizeof(uint64_t); } - - uint64_t GetHashCode() const { - RAPIDJSON_ASSERT(IsValid()); - return *stack_.template Top(); - } + typedef typename Encoding::Ch Ch; + + Hasher(Allocator* allocator = 0, size_t stackCapacity = kDefaultSize) : stack_(allocator, stackCapacity) {} + + bool Null() { return WriteType(kNullType); } + bool Bool(bool b) { return WriteType(b ? kTrueType : kFalseType); } + bool Int(int i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } + bool Uint(unsigned u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } + bool Int64(int64_t i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } + bool Uint64(uint64_t u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } + bool Double(double d) + { Number n; + if (d < 0) n.u.i = static_cast(d); + else n.u.u = static_cast(d); + n.d = d; + return WriteNumber(n); + } + + bool RawNumber(const Ch* str, SizeType len, bool) + { WriteBuffer(kNumberType, str, len * sizeof(Ch)); + return true; + } + + bool String(const Ch* str, SizeType len, bool) + { WriteBuffer(kStringType, str, len * sizeof(Ch)); + return true; + } + + bool StartObject() { return true; } + bool Key(const Ch* str, SizeType len, bool copy) { return String(str, len, copy); } + bool EndObject(SizeType memberCount) + { uint64_t h = Hash(0, kObjectType); + uint64_t* kv = stack_.template Pop(memberCount * 2); + for (SizeType i = 0; i < memberCount; i++) + h ^= Hash(kv[i * 2], kv[i * 2 + 1]); // Use xor to achieve member order insensitive + *stack_.template Push() = h; + return true; + } + + bool StartArray() { return true; } + bool EndArray(SizeType elementCount) + { uint64_t h = Hash(0, kArrayType); + uint64_t* e = stack_.template Pop(elementCount); + for (SizeType i = 0; i < elementCount; i++) + h = Hash(h, e[i]); // Use hash to achieve element order sensitive + *stack_.template Push() = h; + return true; + } + + bool IsValid() const { return stack_.GetSize() == sizeof(uint64_t); } + + uint64_t GetHashCode() const + { RAPIDJSON_ASSERT(IsValid()); + return *stack_.template Top(); + } private: - static const size_t kDefaultSize = 256; - struct Number { - union U { - uint64_t u; - int64_t i; - }u; - double d; - }; - - bool WriteType(Type type) { return WriteBuffer(type, 0, 0); } - - bool WriteNumber(const Number& n) { return WriteBuffer(kNumberType, &n, sizeof(n)); } - - bool WriteBuffer(Type type, const void* data, size_t len) { - // FNV-1a from http://isthe.com/chongo/tech/comp/fnv/ - uint64_t h = Hash(RAPIDJSON_UINT64_C2(0x84222325, 0xcbf29ce4), type); - const unsigned char* d = static_cast(data); - for (size_t i = 0; i < len; i++) - h = Hash(h, d[i]); - *stack_.template Push() = h; - return true; - } - - static uint64_t Hash(uint64_t h, uint64_t d) { - static const uint64_t kPrime = RAPIDJSON_UINT64_C2(0x00000100, 0x000001b3); - h ^= d; - h *= kPrime; - return h; - } - - Stack stack_; + static const size_t kDefaultSize = 256; + struct Number + { union U + { uint64_t u; + int64_t i; + } u; + double d; + }; + + bool WriteType(Type type) { return WriteBuffer(type, 0, 0); } + + bool WriteNumber(const Number& n) { return WriteBuffer(kNumberType, &n, sizeof(n)); } + + bool WriteBuffer(Type type, const void* data, size_t len) + { // FNV-1a from http://isthe.com/chongo/tech/comp/fnv/ + uint64_t h = Hash(RAPIDJSON_UINT64_C2(0x84222325, 0xcbf29ce4), type); + const unsigned char* d = static_cast(data); + for (size_t i = 0; i < len; i++) + h = Hash(h, d[i]); + *stack_.template Push() = h; + return true; + } + + static uint64_t Hash(uint64_t h, uint64_t d) + { static const uint64_t kPrime = RAPIDJSON_UINT64_C2(0x00000100, 0x000001b3); + h ^= d; + h *= kPrime; + return h; + } + + Stack stack_; }; /////////////////////////////////////////////////////////////////////////////// // SchemaValidationContext template -struct SchemaValidationContext { - typedef Schema SchemaType; - typedef ISchemaStateFactory SchemaValidatorFactoryType; - typedef typename SchemaType::ValueType ValueType; - typedef typename ValueType::Ch Ch; - - enum PatternValidatorType { - kPatternValidatorOnly, - kPatternValidatorWithProperty, - kPatternValidatorWithAdditionalProperty - }; - - SchemaValidationContext(SchemaValidatorFactoryType& f, const SchemaType* s) : - factory(f), - schema(s), - valueSchema(), - invalidKeyword(), - hasher(), - arrayElementHashCodes(), - validators(), - validatorCount(), - patternPropertiesValidators(), - patternPropertiesValidatorCount(), - patternPropertiesSchemas(), - patternPropertiesSchemaCount(), - valuePatternValidatorType(kPatternValidatorOnly), - propertyExist(), - inArray(false), - valueUniqueness(false), - arrayUniqueness(false) - { - } - - ~SchemaValidationContext() { - if (hasher) - factory.DestroryHasher(hasher); - if (validators) { - for (SizeType i = 0; i < validatorCount; i++) - factory.DestroySchemaValidator(validators[i]); - factory.FreeState(validators); - } - if (patternPropertiesValidators) { - for (SizeType i = 0; i < patternPropertiesValidatorCount; i++) - factory.DestroySchemaValidator(patternPropertiesValidators[i]); - factory.FreeState(patternPropertiesValidators); - } - if (patternPropertiesSchemas) - factory.FreeState(patternPropertiesSchemas); - if (propertyExist) - factory.FreeState(propertyExist); - } - - SchemaValidatorFactoryType& factory; - const SchemaType* schema; - const SchemaType* valueSchema; - const Ch* invalidKeyword; - void* hasher; // Only validator access - void* arrayElementHashCodes; // Only validator access this - ISchemaValidator** validators; - SizeType validatorCount; - ISchemaValidator** patternPropertiesValidators; - SizeType patternPropertiesValidatorCount; - const SchemaType** patternPropertiesSchemas; - SizeType patternPropertiesSchemaCount; - PatternValidatorType valuePatternValidatorType; - PatternValidatorType objectPatternValidatorType; - SizeType arrayElementIndex; - bool* propertyExist; - bool inArray; - bool valueUniqueness; - bool arrayUniqueness; +struct SchemaValidationContext +{ typedef Schema SchemaType; + typedef ISchemaStateFactory SchemaValidatorFactoryType; + typedef typename SchemaType::ValueType ValueType; + typedef typename ValueType::Ch Ch; + + enum PatternValidatorType + { kPatternValidatorOnly, + kPatternValidatorWithProperty, + kPatternValidatorWithAdditionalProperty + }; + + SchemaValidationContext(SchemaValidatorFactoryType& f, const SchemaType* s) : + factory(f), + schema(s), + valueSchema(), + invalidKeyword(), + hasher(), + arrayElementHashCodes(), + validators(), + validatorCount(), + patternPropertiesValidators(), + patternPropertiesValidatorCount(), + patternPropertiesSchemas(), + patternPropertiesSchemaCount(), + valuePatternValidatorType(kPatternValidatorOnly), + propertyExist(), + inArray(false), + valueUniqueness(false), + arrayUniqueness(false) + { + } + + ~SchemaValidationContext() + { if (hasher) + factory.DestroryHasher(hasher); + if (validators) + { for (SizeType i = 0; i < validatorCount; i++) + factory.DestroySchemaValidator(validators[i]); + factory.FreeState(validators); + } + if (patternPropertiesValidators) + { for (SizeType i = 0; i < patternPropertiesValidatorCount; i++) + factory.DestroySchemaValidator(patternPropertiesValidators[i]); + factory.FreeState(patternPropertiesValidators); + } + if (patternPropertiesSchemas) + factory.FreeState(patternPropertiesSchemas); + if (propertyExist) + factory.FreeState(propertyExist); + } + + SchemaValidatorFactoryType& factory; + const SchemaType* schema; + const SchemaType* valueSchema; + const Ch* invalidKeyword; + void* hasher; // Only validator access + void* arrayElementHashCodes; // Only validator access this + ISchemaValidator** validators; + SizeType validatorCount; + ISchemaValidator** patternPropertiesValidators; + SizeType patternPropertiesValidatorCount; + const SchemaType** patternPropertiesSchemas; + SizeType patternPropertiesSchemaCount; + PatternValidatorType valuePatternValidatorType; + PatternValidatorType objectPatternValidatorType; + SizeType arrayElementIndex; + bool* propertyExist; + bool inArray; + bool valueUniqueness; + bool arrayUniqueness; }; /////////////////////////////////////////////////////////////////////////////// // Schema template -class Schema { +class Schema +{ public: - typedef typename SchemaDocumentType::ValueType ValueType; - typedef typename SchemaDocumentType::AllocatorType AllocatorType; - typedef typename SchemaDocumentType::PointerType PointerType; - typedef typename ValueType::EncodingType EncodingType; - typedef typename EncodingType::Ch Ch; - typedef SchemaValidationContext Context; - typedef Schema SchemaType; - typedef GenericValue SValue; - friend class GenericSchemaDocument; - - Schema(SchemaDocumentType* schemaDocument, const PointerType& p, const ValueType& value, const ValueType& document, AllocatorType* allocator) : - allocator_(allocator), - enum_(), - enumCount_(), - not_(), - type_((1 << kTotalSchemaType) - 1), // typeless - validatorCount_(), - properties_(), - additionalPropertiesSchema_(), - patternProperties_(), - patternPropertyCount_(), - propertyCount_(), - minProperties_(), - maxProperties_(SizeType(~0)), - additionalProperties_(true), - hasDependencies_(), - hasRequired_(), - hasSchemaDependencies_(), - additionalItemsSchema_(), - itemsList_(), - itemsTuple_(), - itemsTupleCount_(), - minItems_(), - maxItems_(SizeType(~0)), - additionalItems_(true), - uniqueItems_(false), - pattern_(), - minLength_(0), - maxLength_(~SizeType(0)), - exclusiveMinimum_(false), - exclusiveMaximum_(false) - { - typedef typename SchemaDocumentType::ValueType ValueType; - typedef typename ValueType::ConstValueIterator ConstValueIterator; - typedef typename ValueType::ConstMemberIterator ConstMemberIterator; - - if (!value.IsObject()) - return; - - if (const ValueType* v = GetMember(value, GetTypeString())) { - type_ = 0; - if (v->IsString()) - AddType(*v); - else if (v->IsArray()) - for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) - AddType(*itr); + typedef typename SchemaDocumentType::ValueType ValueType; + typedef typename SchemaDocumentType::AllocatorType AllocatorType; + typedef typename SchemaDocumentType::PointerType PointerType; + typedef typename ValueType::EncodingType EncodingType; + typedef typename EncodingType::Ch Ch; + typedef SchemaValidationContext Context; + typedef Schema SchemaType; + typedef GenericValue SValue; + friend class GenericSchemaDocument; + + Schema(SchemaDocumentType* schemaDocument, const PointerType& p, const ValueType& value, const ValueType& document, AllocatorType* allocator) : + allocator_(allocator), + enum_(), + enumCount_(), + not_(), + type_((1 << kTotalSchemaType) - 1), // typeless + validatorCount_(), + properties_(), + additionalPropertiesSchema_(), + patternProperties_(), + patternPropertyCount_(), + propertyCount_(), + minProperties_(), + maxProperties_(SizeType(~0)), + additionalProperties_(true), + hasDependencies_(), + hasRequired_(), + hasSchemaDependencies_(), + additionalItemsSchema_(), + itemsList_(), + itemsTuple_(), + itemsTupleCount_(), + minItems_(), + maxItems_(SizeType(~0)), + additionalItems_(true), + uniqueItems_(false), + pattern_(), + minLength_(0), + maxLength_(~SizeType(0)), + exclusiveMinimum_(false), + exclusiveMaximum_(false) + { typedef typename SchemaDocumentType::ValueType ValueType; + typedef typename ValueType::ConstValueIterator ConstValueIterator; + typedef typename ValueType::ConstMemberIterator ConstMemberIterator; + + if (!value.IsObject()) + return; + + if (const ValueType* v = GetMember(value, GetTypeString())) + { type_ = 0; + if (v->IsString()) + AddType(*v); + else if (v->IsArray()) + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) + AddType(*itr); + } + + if (const ValueType* v = GetMember(value, GetEnumString())) + if (v->IsArray() && v->Size() > 0) + { enum_ = static_cast(allocator_->Malloc(sizeof(uint64_t) * v->Size())); + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) + { typedef Hasher > EnumHasherType; + char buffer[256 + 24]; + MemoryPoolAllocator<> hasherAllocator(buffer, sizeof(buffer)); + EnumHasherType h(&hasherAllocator, 256); + itr->Accept(h); + enum_[enumCount_++] = h.GetHashCode(); } + } - if (const ValueType* v = GetMember(value, GetEnumString())) - if (v->IsArray() && v->Size() > 0) { - enum_ = static_cast(allocator_->Malloc(sizeof(uint64_t) * v->Size())); - for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) { - typedef Hasher > EnumHasherType; - char buffer[256 + 24]; - MemoryPoolAllocator<> hasherAllocator(buffer, sizeof(buffer)); - EnumHasherType h(&hasherAllocator, 256); - itr->Accept(h); - enum_[enumCount_++] = h.GetHashCode(); - } - } + if (schemaDocument) + { AssignIfExist(allOf_, *schemaDocument, p, value, GetAllOfString(), document); + AssignIfExist(anyOf_, *schemaDocument, p, value, GetAnyOfString(), document); + AssignIfExist(oneOf_, *schemaDocument, p, value, GetOneOfString(), document); + } - if (schemaDocument) { - AssignIfExist(allOf_, *schemaDocument, p, value, GetAllOfString(), document); - AssignIfExist(anyOf_, *schemaDocument, p, value, GetAnyOfString(), document); - AssignIfExist(oneOf_, *schemaDocument, p, value, GetOneOfString(), document); - } + if (const ValueType* v = GetMember(value, GetNotString())) + { schemaDocument->CreateSchema(¬_, p.Append(GetNotString(), allocator_), *v, document); + notValidatorIndex_ = validatorCount_; + validatorCount_++; + } - if (const ValueType* v = GetMember(value, GetNotString())) { - schemaDocument->CreateSchema(¬_, p.Append(GetNotString(), allocator_), *v, document); - notValidatorIndex_ = validatorCount_; - validatorCount_++; - } + // Object - // Object - - const ValueType* properties = GetMember(value, GetPropertiesString()); - const ValueType* required = GetMember(value, GetRequiredString()); - const ValueType* dependencies = GetMember(value, GetDependenciesString()); - { - // Gather properties from properties/required/dependencies - SValue allProperties(kArrayType); - - if (properties && properties->IsObject()) - for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) - AddUniqueElement(allProperties, itr->name); - - if (required && required->IsArray()) - for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) - if (itr->IsString()) - AddUniqueElement(allProperties, *itr); - - if (dependencies && dependencies->IsObject()) - for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { - AddUniqueElement(allProperties, itr->name); - if (itr->value.IsArray()) - for (ConstValueIterator i = itr->value.Begin(); i != itr->value.End(); ++i) - if (i->IsString()) - AddUniqueElement(allProperties, *i); - } + const ValueType* properties = GetMember(value, GetPropertiesString()); + const ValueType* required = GetMember(value, GetRequiredString()); + const ValueType* dependencies = GetMember(value, GetDependenciesString()); + { // Gather properties from properties/required/dependencies + SValue allProperties(kArrayType); - if (allProperties.Size() > 0) { - propertyCount_ = allProperties.Size(); - properties_ = static_cast(allocator_->Malloc(sizeof(Property) * propertyCount_)); - for (SizeType i = 0; i < propertyCount_; i++) { - new (&properties_[i]) Property(); - properties_[i].name = allProperties[i]; - properties_[i].schema = GetTypeless(); - } - } - } - - if (properties && properties->IsObject()) { - PointerType q = p.Append(GetPropertiesString(), allocator_); - for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) { - SizeType index; - if (FindPropertyIndex(itr->name, &index)) - schemaDocument->CreateSchema(&properties_[index].schema, q.Append(itr->name, allocator_), itr->value, document); - } - } + if (properties && properties->IsObject()) + for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) + AddUniqueElement(allProperties, itr->name); - if (const ValueType* v = GetMember(value, GetPatternPropertiesString())) { - PointerType q = p.Append(GetPatternPropertiesString(), allocator_); - patternProperties_ = static_cast(allocator_->Malloc(sizeof(PatternProperty) * v->MemberCount())); - patternPropertyCount_ = 0; + if (required && required->IsArray()) + for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) + if (itr->IsString()) + AddUniqueElement(allProperties, *itr); - for (ConstMemberIterator itr = v->MemberBegin(); itr != v->MemberEnd(); ++itr) { - new (&patternProperties_[patternPropertyCount_]) PatternProperty(); - patternProperties_[patternPropertyCount_].pattern = CreatePattern(itr->name); - schemaDocument->CreateSchema(&patternProperties_[patternPropertyCount_].schema, q.Append(itr->name, allocator_), itr->value, document); - patternPropertyCount_++; - } + if (dependencies && dependencies->IsObject()) + for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) + { AddUniqueElement(allProperties, itr->name); + if (itr->value.IsArray()) + for (ConstValueIterator i = itr->value.Begin(); i != itr->value.End(); ++i) + if (i->IsString()) + AddUniqueElement(allProperties, *i); } - if (required && required->IsArray()) - for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) - if (itr->IsString()) { - SizeType index; - if (FindPropertyIndex(*itr, &index)) { - properties_[index].required = true; - hasRequired_ = true; - } - } - - if (dependencies && dependencies->IsObject()) { - PointerType q = p.Append(GetDependenciesString(), allocator_); - hasDependencies_ = true; - for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { - SizeType sourceIndex; - if (FindPropertyIndex(itr->name, &sourceIndex)) { - if (itr->value.IsArray()) { - properties_[sourceIndex].dependencies = static_cast(allocator_->Malloc(sizeof(bool) * propertyCount_)); - std::memset(properties_[sourceIndex].dependencies, 0, sizeof(bool)* propertyCount_); - for (ConstValueIterator targetItr = itr->value.Begin(); targetItr != itr->value.End(); ++targetItr) { - SizeType targetIndex; - if (FindPropertyIndex(*targetItr, &targetIndex)) - properties_[sourceIndex].dependencies[targetIndex] = true; - } - } - else if (itr->value.IsObject()) { - hasSchemaDependencies_ = true; - schemaDocument->CreateSchema(&properties_[sourceIndex].dependenciesSchema, q.Append(itr->name, allocator_), itr->value, document); - properties_[sourceIndex].dependenciesValidatorIndex = validatorCount_; - validatorCount_++; - } - } - } + if (allProperties.Size() > 0) + { propertyCount_ = allProperties.Size(); + properties_ = static_cast(allocator_->Malloc(sizeof(Property) * propertyCount_)); + for (SizeType i = 0; i < propertyCount_; i++) + { new (&properties_[i]) Property(); + properties_[i].name = allProperties[i]; + properties_[i].schema = GetTypeless(); } - - if (const ValueType* v = GetMember(value, GetAdditionalPropertiesString())) { - if (v->IsBool()) - additionalProperties_ = v->GetBool(); - else if (v->IsObject()) - schemaDocument->CreateSchema(&additionalPropertiesSchema_, p.Append(GetAdditionalPropertiesString(), allocator_), *v, document); + } + } + + if (properties && properties->IsObject()) + { PointerType q = p.Append(GetPropertiesString(), allocator_); + for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) + { SizeType index; + if (FindPropertyIndex(itr->name, &index)) + schemaDocument->CreateSchema(&properties_[index].schema, q.Append(itr->name, allocator_), itr->value, document); + } + } + + if (const ValueType* v = GetMember(value, GetPatternPropertiesString())) + { PointerType q = p.Append(GetPatternPropertiesString(), allocator_); + patternProperties_ = static_cast(allocator_->Malloc(sizeof(PatternProperty) * v->MemberCount())); + patternPropertyCount_ = 0; + + for (ConstMemberIterator itr = v->MemberBegin(); itr != v->MemberEnd(); ++itr) + { new (&patternProperties_[patternPropertyCount_]) PatternProperty(); + patternProperties_[patternPropertyCount_].pattern = CreatePattern(itr->name); + schemaDocument->CreateSchema(&patternProperties_[patternPropertyCount_].schema, q.Append(itr->name, allocator_), itr->value, document); + patternPropertyCount_++; + } + } + + if (required && required->IsArray()) + for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) + if (itr->IsString()) + { SizeType index; + if (FindPropertyIndex(*itr, &index)) + { properties_[index].required = true; + hasRequired_ = true; + } } - AssignIfExist(minProperties_, value, GetMinPropertiesString()); - AssignIfExist(maxProperties_, value, GetMaxPropertiesString()); - - // Array - if (const ValueType* v = GetMember(value, GetItemsString())) { - PointerType q = p.Append(GetItemsString(), allocator_); - if (v->IsObject()) // List validation - schemaDocument->CreateSchema(&itemsList_, q, *v, document); - else if (v->IsArray()) { // Tuple validation - itemsTuple_ = static_cast(allocator_->Malloc(sizeof(const Schema*) * v->Size())); - SizeType index = 0; - for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr, index++) - schemaDocument->CreateSchema(&itemsTuple_[itemsTupleCount_++], q.Append(index, allocator_), *itr, document); + if (dependencies && dependencies->IsObject()) + { PointerType q = p.Append(GetDependenciesString(), allocator_); + hasDependencies_ = true; + for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) + { SizeType sourceIndex; + if (FindPropertyIndex(itr->name, &sourceIndex)) + { if (itr->value.IsArray()) + { properties_[sourceIndex].dependencies = static_cast(allocator_->Malloc(sizeof(bool) * propertyCount_)); + std::memset(properties_[sourceIndex].dependencies, 0, sizeof(bool)* propertyCount_); + for (ConstValueIterator targetItr = itr->value.Begin(); targetItr != itr->value.End(); ++targetItr) + { SizeType targetIndex; + if (FindPropertyIndex(*targetItr, &targetIndex)) + properties_[sourceIndex].dependencies[targetIndex] = true; } + } + else if (itr->value.IsObject()) + { hasSchemaDependencies_ = true; + schemaDocument->CreateSchema(&properties_[sourceIndex].dependenciesSchema, q.Append(itr->name, allocator_), itr->value, document); + properties_[sourceIndex].dependenciesValidatorIndex = validatorCount_; + validatorCount_++; + } } + } + } - AssignIfExist(minItems_, value, GetMinItemsString()); - AssignIfExist(maxItems_, value, GetMaxItemsString()); - - if (const ValueType* v = GetMember(value, GetAdditionalItemsString())) { - if (v->IsBool()) - additionalItems_ = v->GetBool(); - else if (v->IsObject()) - schemaDocument->CreateSchema(&additionalItemsSchema_, p.Append(GetAdditionalItemsString(), allocator_), *v, document); - } - - AssignIfExist(uniqueItems_, value, GetUniqueItemsString()); - - // String - AssignIfExist(minLength_, value, GetMinLengthString()); - AssignIfExist(maxLength_, value, GetMaxLengthString()); - - if (const ValueType* v = GetMember(value, GetPatternString())) - pattern_ = CreatePattern(*v); - - // Number - if (const ValueType* v = GetMember(value, GetMinimumString())) - if (v->IsNumber()) - minimum_.CopyFrom(*v, *allocator_); - - if (const ValueType* v = GetMember(value, GetMaximumString())) - if (v->IsNumber()) - maximum_.CopyFrom(*v, *allocator_); - - AssignIfExist(exclusiveMinimum_, value, GetExclusiveMinimumString()); - AssignIfExist(exclusiveMaximum_, value, GetExclusiveMaximumString()); - - if (const ValueType* v = GetMember(value, GetMultipleOfString())) - if (v->IsNumber() && v->GetDouble() > 0.0) - multipleOf_.CopyFrom(*v, *allocator_); + if (const ValueType* v = GetMember(value, GetAdditionalPropertiesString())) + { if (v->IsBool()) + additionalProperties_ = v->GetBool(); + else if (v->IsObject()) + schemaDocument->CreateSchema(&additionalPropertiesSchema_, p.Append(GetAdditionalPropertiesString(), allocator_), *v, document); } - ~Schema() { - if (allocator_) { - allocator_->Free(enum_); - } - if (properties_) { - for (SizeType i = 0; i < propertyCount_; i++) - properties_[i].~Property(); - AllocatorType::Free(properties_); - } - if (patternProperties_) { - for (SizeType i = 0; i < patternPropertyCount_; i++) - patternProperties_[i].~PatternProperty(); - AllocatorType::Free(patternProperties_); - } - AllocatorType::Free(itemsTuple_); -#if RAPIDJSON_SCHEMA_HAS_REGEX - if (pattern_) { - pattern_->~RegexType(); - allocator_->Free(pattern_); - } -#endif + AssignIfExist(minProperties_, value, GetMinPropertiesString()); + AssignIfExist(maxProperties_, value, GetMaxPropertiesString()); + + // Array + if (const ValueType* v = GetMember(value, GetItemsString())) + { PointerType q = p.Append(GetItemsString(), allocator_); + if (v->IsObject()) // List validation + schemaDocument->CreateSchema(&itemsList_, q, *v, document); + else if (v->IsArray()) // Tuple validation + { itemsTuple_ = static_cast(allocator_->Malloc(sizeof(const Schema*) * v->Size())); + SizeType index = 0; + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr, index++) + schemaDocument->CreateSchema(&itemsTuple_[itemsTupleCount_++], q.Append(index, allocator_), *itr, document); + } } - bool BeginValue(Context& context) const { - if (context.inArray) { - if (uniqueItems_) - context.valueUniqueness = true; - - if (itemsList_) - context.valueSchema = itemsList_; - else if (itemsTuple_) { - if (context.arrayElementIndex < itemsTupleCount_) - context.valueSchema = itemsTuple_[context.arrayElementIndex]; - else if (additionalItemsSchema_) - context.valueSchema = additionalItemsSchema_; - else if (additionalItems_) - context.valueSchema = GetTypeless(); - else - RAPIDJSON_INVALID_KEYWORD_RETURN(GetItemsString()); - } - else - context.valueSchema = GetTypeless(); + AssignIfExist(minItems_, value, GetMinItemsString()); + AssignIfExist(maxItems_, value, GetMaxItemsString()); - context.arrayElementIndex++; - } - return true; + if (const ValueType* v = GetMember(value, GetAdditionalItemsString())) + { if (v->IsBool()) + additionalItems_ = v->GetBool(); + else if (v->IsObject()) + schemaDocument->CreateSchema(&additionalItemsSchema_, p.Append(GetAdditionalItemsString(), allocator_), *v, document); } - RAPIDJSON_FORCEINLINE bool EndValue(Context& context) const { - if (context.patternPropertiesValidatorCount > 0) { - bool otherValid = false; - SizeType count = context.patternPropertiesValidatorCount; - if (context.objectPatternValidatorType != Context::kPatternValidatorOnly) - otherValid = context.patternPropertiesValidators[--count]->IsValid(); - - bool patternValid = true; - for (SizeType i = 0; i < count; i++) - if (!context.patternPropertiesValidators[i]->IsValid()) { - patternValid = false; - break; - } + AssignIfExist(uniqueItems_, value, GetUniqueItemsString()); - if (context.objectPatternValidatorType == Context::kPatternValidatorOnly) { - if (!patternValid) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); - } - else if (context.objectPatternValidatorType == Context::kPatternValidatorWithProperty) { - if (!patternValid || !otherValid) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); - } - else if (!patternValid && !otherValid) // kPatternValidatorWithAdditionalProperty) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); - } + // String + AssignIfExist(minLength_, value, GetMinLengthString()); + AssignIfExist(maxLength_, value, GetMaxLengthString()); - if (enum_) { - const uint64_t h = context.factory.GetHashCode(context.hasher); - for (SizeType i = 0; i < enumCount_; i++) - if (enum_[i] == h) - goto foundEnum; - RAPIDJSON_INVALID_KEYWORD_RETURN(GetEnumString()); - foundEnum:; - } + if (const ValueType* v = GetMember(value, GetPatternString())) + pattern_ = CreatePattern(*v); - if (allOf_.schemas) - for (SizeType i = allOf_.begin; i < allOf_.begin + allOf_.count; i++) - if (!context.validators[i]->IsValid()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetAllOfString()); - - if (anyOf_.schemas) { - for (SizeType i = anyOf_.begin; i < anyOf_.begin + anyOf_.count; i++) - if (context.validators[i]->IsValid()) - goto foundAny; - RAPIDJSON_INVALID_KEYWORD_RETURN(GetAnyOfString()); - foundAny:; - } + // Number + if (const ValueType* v = GetMember(value, GetMinimumString())) + if (v->IsNumber()) + minimum_.CopyFrom(*v, *allocator_); - if (oneOf_.schemas) { - bool oneValid = false; - for (SizeType i = oneOf_.begin; i < oneOf_.begin + oneOf_.count; i++) - if (context.validators[i]->IsValid()) { - if (oneValid) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); - else - oneValid = true; - } - if (!oneValid) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); - } + if (const ValueType* v = GetMember(value, GetMaximumString())) + if (v->IsNumber()) + maximum_.CopyFrom(*v, *allocator_); - if (not_ && context.validators[notValidatorIndex_]->IsValid()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetNotString()); + AssignIfExist(exclusiveMinimum_, value, GetExclusiveMinimumString()); + AssignIfExist(exclusiveMaximum_, value, GetExclusiveMaximumString()); - return true; - } + if (const ValueType* v = GetMember(value, GetMultipleOfString())) + if (v->IsNumber() && v->GetDouble() > 0.0) + multipleOf_.CopyFrom(*v, *allocator_); + } - bool Null(Context& context) const { - if (!(type_ & (1 << kNullSchemaType))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - return CreateParallelValidator(context); + ~Schema() + { if (allocator_) + { allocator_->Free(enum_); } - - bool Bool(Context& context, bool) const { - if (!(type_ & (1 << kBooleanSchemaType))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - return CreateParallelValidator(context); + if (properties_) + { for (SizeType i = 0; i < propertyCount_; i++) + properties_[i].~Property(); + AllocatorType::Free(properties_); } - - bool Int(Context& context, int i) const { - if (!CheckInt(context, i)) - return false; - return CreateParallelValidator(context); + if (patternProperties_) + { for (SizeType i = 0; i < patternPropertyCount_; i++) + patternProperties_[i].~PatternProperty(); + AllocatorType::Free(patternProperties_); } - - bool Uint(Context& context, unsigned u) const { - if (!CheckUint(context, u)) - return false; - return CreateParallelValidator(context); + AllocatorType::Free(itemsTuple_); +#if RAPIDJSON_SCHEMA_HAS_REGEX + if (pattern_) + { pattern_->~RegexType(); + allocator_->Free(pattern_); } +#endif + } + + bool BeginValue(Context& context) const + { if (context.inArray) + { if (uniqueItems_) + context.valueUniqueness = true; + + if (itemsList_) + context.valueSchema = itemsList_; + else if (itemsTuple_) + { if (context.arrayElementIndex < itemsTupleCount_) + context.valueSchema = itemsTuple_[context.arrayElementIndex]; + else if (additionalItemsSchema_) + context.valueSchema = additionalItemsSchema_; + else if (additionalItems_) + context.valueSchema = GetTypeless(); + else + RAPIDJSON_INVALID_KEYWORD_RETURN(GetItemsString()); + } + else + context.valueSchema = GetTypeless(); + + context.arrayElementIndex++; + } + return true; + } + + RAPIDJSON_FORCEINLINE bool EndValue(Context& context) const + { if (context.patternPropertiesValidatorCount > 0) + { bool otherValid = false; + SizeType count = context.patternPropertiesValidatorCount; + if (context.objectPatternValidatorType != Context::kPatternValidatorOnly) + otherValid = context.patternPropertiesValidators[--count]->IsValid(); + + bool patternValid = true; + for (SizeType i = 0; i < count; i++) + if (!context.patternPropertiesValidators[i]->IsValid()) + { patternValid = false; + break; + } - bool Int64(Context& context, int64_t i) const { - if (!CheckInt(context, i)) - return false; - return CreateParallelValidator(context); + if (context.objectPatternValidatorType == Context::kPatternValidatorOnly) + { if (!patternValid) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } + else if (context.objectPatternValidatorType == Context::kPatternValidatorWithProperty) + { if (!patternValid || !otherValid) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } + else if (!patternValid && !otherValid) // kPatternValidatorWithAdditionalProperty) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } + + if (enum_) + { const uint64_t h = context.factory.GetHashCode(context.hasher); + for (SizeType i = 0; i < enumCount_; i++) + if (enum_[i] == h) + goto foundEnum; + RAPIDJSON_INVALID_KEYWORD_RETURN(GetEnumString()); +foundEnum:; + } + + if (allOf_.schemas) + for (SizeType i = allOf_.begin; i < allOf_.begin + allOf_.count; i++) + if (!context.validators[i]->IsValid()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetAllOfString()); + + if (anyOf_.schemas) + { for (SizeType i = anyOf_.begin; i < anyOf_.begin + anyOf_.count; i++) + if (context.validators[i]->IsValid()) + goto foundAny; + RAPIDJSON_INVALID_KEYWORD_RETURN(GetAnyOfString()); +foundAny:; + } + + if (oneOf_.schemas) + { bool oneValid = false; + for (SizeType i = oneOf_.begin; i < oneOf_.begin + oneOf_.count; i++) + if (context.validators[i]->IsValid()) + { if (oneValid) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); + else + oneValid = true; + } + if (!oneValid) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); } - bool Uint64(Context& context, uint64_t u) const { - if (!CheckUint(context, u)) - return false; - return CreateParallelValidator(context); - } + if (not_ && context.validators[notValidatorIndex_]->IsValid()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetNotString()); - bool Double(Context& context, double d) const { - if (!(type_ & (1 << kNumberSchemaType))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + return true; + } - if (!minimum_.IsNull() && !CheckDoubleMinimum(context, d)) - return false; + bool Null(Context& context) const + { if (!(type_ & (1 << kNullSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + return CreateParallelValidator(context); + } + + bool Bool(Context& context, bool) const + { if (!(type_ & (1 << kBooleanSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + return CreateParallelValidator(context); + } - if (!maximum_.IsNull() && !CheckDoubleMaximum(context, d)) - return false; - - if (!multipleOf_.IsNull() && !CheckDoubleMultipleOf(context, d)) - return false; - - return CreateParallelValidator(context); - } - - bool String(Context& context, const Ch* str, SizeType length, bool) const { - if (!(type_ & (1 << kStringSchemaType))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - - if (minLength_ != 0 || maxLength_ != SizeType(~0)) { - SizeType count; - if (internal::CountStringCodePoint(str, length, &count)) { - if (count < minLength_) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinLengthString()); - if (count > maxLength_) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxLengthString()); - } - } + bool Int(Context& context, int i) const + { if (!CheckInt(context, i)) + return false; + return CreateParallelValidator(context); + } - if (pattern_ && !IsPatternMatch(pattern_, str, length)) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternString()); + bool Uint(Context& context, unsigned u) const + { if (!CheckUint(context, u)) + return false; + return CreateParallelValidator(context); + } + + bool Int64(Context& context, int64_t i) const + { if (!CheckInt(context, i)) + return false; + return CreateParallelValidator(context); + } - return CreateParallelValidator(context); - } + bool Uint64(Context& context, uint64_t u) const + { if (!CheckUint(context, u)) + return false; + return CreateParallelValidator(context); + } - bool StartObject(Context& context) const { - if (!(type_ & (1 << kObjectSchemaType))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + bool Double(Context& context, double d) const + { if (!(type_ & (1 << kNumberSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - if (hasDependencies_ || hasRequired_) { - context.propertyExist = static_cast(context.factory.MallocState(sizeof(bool) * propertyCount_)); - std::memset(context.propertyExist, 0, sizeof(bool) * propertyCount_); - } + if (!minimum_.IsNull() && !CheckDoubleMinimum(context, d)) + return false; - if (patternProperties_) { // pre-allocate schema array - SizeType count = patternPropertyCount_ + 1; // extra for valuePatternValidatorType - context.patternPropertiesSchemas = static_cast(context.factory.MallocState(sizeof(const SchemaType*) * count)); - context.patternPropertiesSchemaCount = 0; - std::memset(context.patternPropertiesSchemas, 0, sizeof(SchemaType*) * count); - } + if (!maximum_.IsNull() && !CheckDoubleMaximum(context, d)) + return false; - return CreateParallelValidator(context); - } - - bool Key(Context& context, const Ch* str, SizeType len, bool) const { - if (patternProperties_) { - context.patternPropertiesSchemaCount = 0; - for (SizeType i = 0; i < patternPropertyCount_; i++) - if (patternProperties_[i].pattern && IsPatternMatch(patternProperties_[i].pattern, str, len)) - context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = patternProperties_[i].schema; - } + if (!multipleOf_.IsNull() && !CheckDoubleMultipleOf(context, d)) + return false; - SizeType index; - if (FindPropertyIndex(ValueType(str, len).Move(), &index)) { - if (context.patternPropertiesSchemaCount > 0) { - context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = properties_[index].schema; - context.valueSchema = GetTypeless(); - context.valuePatternValidatorType = Context::kPatternValidatorWithProperty; - } - else - context.valueSchema = properties_[index].schema; + return CreateParallelValidator(context); + } - if (context.propertyExist) - context.propertyExist[index] = true; + bool String(Context& context, const Ch* str, SizeType length, bool) const + { if (!(type_ & (1 << kStringSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (minLength_ != 0 || maxLength_ != SizeType(~0)) + { SizeType count; + if (internal::CountStringCodePoint(str, length, &count)) + { if (count < minLength_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinLengthString()); + if (count > maxLength_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxLengthString()); + } + } + + if (pattern_ && !IsPatternMatch(pattern_, str, length)) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternString()); + + return CreateParallelValidator(context); + } + + bool StartObject(Context& context) const + { if (!(type_ & (1 << kObjectSchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (hasDependencies_ || hasRequired_) + { context.propertyExist = static_cast(context.factory.MallocState(sizeof(bool) * propertyCount_)); + std::memset(context.propertyExist, 0, sizeof(bool) * propertyCount_); + } + + if (patternProperties_) // pre-allocate schema array + { SizeType count = patternPropertyCount_ + 1; // extra for valuePatternValidatorType + context.patternPropertiesSchemas = static_cast(context.factory.MallocState(sizeof(const SchemaType*) * count)); + context.patternPropertiesSchemaCount = 0; + std::memset(context.patternPropertiesSchemas, 0, sizeof(SchemaType*) * count); + } + + return CreateParallelValidator(context); + } + + bool Key(Context& context, const Ch* str, SizeType len, bool) const + { if (patternProperties_) + { context.patternPropertiesSchemaCount = 0; + for (SizeType i = 0; i < patternPropertyCount_; i++) + if (patternProperties_[i].pattern && IsPatternMatch(patternProperties_[i].pattern, str, len)) + context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = patternProperties_[i].schema; + } + + SizeType index; + if (FindPropertyIndex(ValueType(str, len).Move(), &index)) + { if (context.patternPropertiesSchemaCount > 0) + { context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = properties_[index].schema; + context.valueSchema = GetTypeless(); + context.valuePatternValidatorType = Context::kPatternValidatorWithProperty; + } + else + context.valueSchema = properties_[index].schema; + + if (context.propertyExist) + context.propertyExist[index] = true; + + return true; + } + + if (additionalPropertiesSchema_) + { if (additionalPropertiesSchema_ && context.patternPropertiesSchemaCount > 0) + { context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = additionalPropertiesSchema_; + context.valueSchema = GetTypeless(); + context.valuePatternValidatorType = Context::kPatternValidatorWithAdditionalProperty; + } + else + context.valueSchema = additionalPropertiesSchema_; + return true; + } + else if (additionalProperties_) + { context.valueSchema = GetTypeless(); + return true; + } - return true; - } + if (context.patternPropertiesSchemaCount == 0) // patternProperties are not additional properties + RAPIDJSON_INVALID_KEYWORD_RETURN(GetAdditionalPropertiesString()); + + return true; + } - if (additionalPropertiesSchema_) { - if (additionalPropertiesSchema_ && context.patternPropertiesSchemaCount > 0) { - context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = additionalPropertiesSchema_; - context.valueSchema = GetTypeless(); - context.valuePatternValidatorType = Context::kPatternValidatorWithAdditionalProperty; - } - else - context.valueSchema = additionalPropertiesSchema_; - return true; - } - else if (additionalProperties_) { - context.valueSchema = GetTypeless(); - return true; - } + bool EndObject(Context& context, SizeType memberCount) const + { if (hasRequired_) + for (SizeType index = 0; index < propertyCount_; index++) + if (properties_[index].required) + if (!context.propertyExist[index]) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetRequiredString()); - if (context.patternPropertiesSchemaCount == 0) // patternProperties are not additional properties - RAPIDJSON_INVALID_KEYWORD_RETURN(GetAdditionalPropertiesString()); + if (memberCount < minProperties_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinPropertiesString()); - return true; + if (memberCount > maxProperties_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxPropertiesString()); + + if (hasDependencies_) + { for (SizeType sourceIndex = 0; sourceIndex < propertyCount_; sourceIndex++) + if (context.propertyExist[sourceIndex]) + { if (properties_[sourceIndex].dependencies) + { for (SizeType targetIndex = 0; targetIndex < propertyCount_; targetIndex++) + if (properties_[sourceIndex].dependencies[targetIndex] && !context.propertyExist[targetIndex]) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); + } + else if (properties_[sourceIndex].dependenciesSchema) + if (!context.validators[properties_[sourceIndex].dependenciesValidatorIndex]->IsValid()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); + } } - bool EndObject(Context& context, SizeType memberCount) const { - if (hasRequired_) - for (SizeType index = 0; index < propertyCount_; index++) - if (properties_[index].required) - if (!context.propertyExist[index]) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetRequiredString()); - - if (memberCount < minProperties_) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinPropertiesString()); - - if (memberCount > maxProperties_) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxPropertiesString()); - - if (hasDependencies_) { - for (SizeType sourceIndex = 0; sourceIndex < propertyCount_; sourceIndex++) - if (context.propertyExist[sourceIndex]) { - if (properties_[sourceIndex].dependencies) { - for (SizeType targetIndex = 0; targetIndex < propertyCount_; targetIndex++) - if (properties_[sourceIndex].dependencies[targetIndex] && !context.propertyExist[targetIndex]) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); - } - else if (properties_[sourceIndex].dependenciesSchema) - if (!context.validators[properties_[sourceIndex].dependenciesValidatorIndex]->IsValid()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); - } - } + return true; + } - return true; - } + bool StartArray(Context& context) const + { if (!(type_ & (1 << kArraySchemaType))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - bool StartArray(Context& context) const { - if (!(type_ & (1 << kArraySchemaType))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + context.arrayElementIndex = 0; + context.inArray = true; - context.arrayElementIndex = 0; - context.inArray = true; + return CreateParallelValidator(context); + } - return CreateParallelValidator(context); - } + bool EndArray(Context& context, SizeType elementCount) const + { context.inArray = false; - bool EndArray(Context& context, SizeType elementCount) const { - context.inArray = false; - - if (elementCount < minItems_) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinItemsString()); - - if (elementCount > maxItems_) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxItemsString()); + if (elementCount < minItems_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinItemsString()); - return true; - } + if (elementCount > maxItems_) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxItemsString()); - // Generate functions for string literal according to Ch + return true; + } + + // Generate functions for string literal according to Ch #define RAPIDJSON_STRING_(name, ...) \ static const ValueType& Get##name##String() {\ static const Ch s[] = { __VA_ARGS__, '\0' };\ @@ -885,409 +889,409 @@ class Schema { return v;\ } - RAPIDJSON_STRING_(Null, 'n', 'u', 'l', 'l') - RAPIDJSON_STRING_(Boolean, 'b', 'o', 'o', 'l', 'e', 'a', 'n') - RAPIDJSON_STRING_(Object, 'o', 'b', 'j', 'e', 'c', 't') - RAPIDJSON_STRING_(Array, 'a', 'r', 'r', 'a', 'y') - RAPIDJSON_STRING_(String, 's', 't', 'r', 'i', 'n', 'g') - RAPIDJSON_STRING_(Number, 'n', 'u', 'm', 'b', 'e', 'r') - RAPIDJSON_STRING_(Integer, 'i', 'n', 't', 'e', 'g', 'e', 'r') - RAPIDJSON_STRING_(Type, 't', 'y', 'p', 'e') - RAPIDJSON_STRING_(Enum, 'e', 'n', 'u', 'm') - RAPIDJSON_STRING_(AllOf, 'a', 'l', 'l', 'O', 'f') - RAPIDJSON_STRING_(AnyOf, 'a', 'n', 'y', 'O', 'f') - RAPIDJSON_STRING_(OneOf, 'o', 'n', 'e', 'O', 'f') - RAPIDJSON_STRING_(Not, 'n', 'o', 't') - RAPIDJSON_STRING_(Properties, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') - RAPIDJSON_STRING_(Required, 'r', 'e', 'q', 'u', 'i', 'r', 'e', 'd') - RAPIDJSON_STRING_(Dependencies, 'd', 'e', 'p', 'e', 'n', 'd', 'e', 'n', 'c', 'i', 'e', 's') - RAPIDJSON_STRING_(PatternProperties, 'p', 'a', 't', 't', 'e', 'r', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') - RAPIDJSON_STRING_(AdditionalProperties, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') - RAPIDJSON_STRING_(MinProperties, 'm', 'i', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') - RAPIDJSON_STRING_(MaxProperties, 'm', 'a', 'x', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') - RAPIDJSON_STRING_(Items, 'i', 't', 'e', 'm', 's') - RAPIDJSON_STRING_(MinItems, 'm', 'i', 'n', 'I', 't', 'e', 'm', 's') - RAPIDJSON_STRING_(MaxItems, 'm', 'a', 'x', 'I', 't', 'e', 'm', 's') - RAPIDJSON_STRING_(AdditionalItems, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'I', 't', 'e', 'm', 's') - RAPIDJSON_STRING_(UniqueItems, 'u', 'n', 'i', 'q', 'u', 'e', 'I', 't', 'e', 'm', 's') - RAPIDJSON_STRING_(MinLength, 'm', 'i', 'n', 'L', 'e', 'n', 'g', 't', 'h') - RAPIDJSON_STRING_(MaxLength, 'm', 'a', 'x', 'L', 'e', 'n', 'g', 't', 'h') - RAPIDJSON_STRING_(Pattern, 'p', 'a', 't', 't', 'e', 'r', 'n') - RAPIDJSON_STRING_(Minimum, 'm', 'i', 'n', 'i', 'm', 'u', 'm') - RAPIDJSON_STRING_(Maximum, 'm', 'a', 'x', 'i', 'm', 'u', 'm') - RAPIDJSON_STRING_(ExclusiveMinimum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'i', 'n', 'i', 'm', 'u', 'm') - RAPIDJSON_STRING_(ExclusiveMaximum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'a', 'x', 'i', 'm', 'u', 'm') - RAPIDJSON_STRING_(MultipleOf, 'm', 'u', 'l', 't', 'i', 'p', 'l', 'e', 'O', 'f') + RAPIDJSON_STRING_(Null, 'n', 'u', 'l', 'l') + RAPIDJSON_STRING_(Boolean, 'b', 'o', 'o', 'l', 'e', 'a', 'n') + RAPIDJSON_STRING_(Object, 'o', 'b', 'j', 'e', 'c', 't') + RAPIDJSON_STRING_(Array, 'a', 'r', 'r', 'a', 'y') + RAPIDJSON_STRING_(String, 's', 't', 'r', 'i', 'n', 'g') + RAPIDJSON_STRING_(Number, 'n', 'u', 'm', 'b', 'e', 'r') + RAPIDJSON_STRING_(Integer, 'i', 'n', 't', 'e', 'g', 'e', 'r') + RAPIDJSON_STRING_(Type, 't', 'y', 'p', 'e') + RAPIDJSON_STRING_(Enum, 'e', 'n', 'u', 'm') + RAPIDJSON_STRING_(AllOf, 'a', 'l', 'l', 'O', 'f') + RAPIDJSON_STRING_(AnyOf, 'a', 'n', 'y', 'O', 'f') + RAPIDJSON_STRING_(OneOf, 'o', 'n', 'e', 'O', 'f') + RAPIDJSON_STRING_(Not, 'n', 'o', 't') + RAPIDJSON_STRING_(Properties, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(Required, 'r', 'e', 'q', 'u', 'i', 'r', 'e', 'd') + RAPIDJSON_STRING_(Dependencies, 'd', 'e', 'p', 'e', 'n', 'd', 'e', 'n', 'c', 'i', 'e', 's') + RAPIDJSON_STRING_(PatternProperties, 'p', 'a', 't', 't', 'e', 'r', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(AdditionalProperties, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(MinProperties, 'm', 'i', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(MaxProperties, 'm', 'a', 'x', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(Items, 'i', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(MinItems, 'm', 'i', 'n', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(MaxItems, 'm', 'a', 'x', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(AdditionalItems, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(UniqueItems, 'u', 'n', 'i', 'q', 'u', 'e', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(MinLength, 'm', 'i', 'n', 'L', 'e', 'n', 'g', 't', 'h') + RAPIDJSON_STRING_(MaxLength, 'm', 'a', 'x', 'L', 'e', 'n', 'g', 't', 'h') + RAPIDJSON_STRING_(Pattern, 'p', 'a', 't', 't', 'e', 'r', 'n') + RAPIDJSON_STRING_(Minimum, 'm', 'i', 'n', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(Maximum, 'm', 'a', 'x', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(ExclusiveMinimum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'i', 'n', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(ExclusiveMaximum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'a', 'x', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(MultipleOf, 'm', 'u', 'l', 't', 'i', 'p', 'l', 'e', 'O', 'f') #undef RAPIDJSON_STRING_ private: - enum SchemaValueType { - kNullSchemaType, - kBooleanSchemaType, - kObjectSchemaType, - kArraySchemaType, - kStringSchemaType, - kNumberSchemaType, - kIntegerSchemaType, - kTotalSchemaType - }; + enum SchemaValueType + { kNullSchemaType, + kBooleanSchemaType, + kObjectSchemaType, + kArraySchemaType, + kStringSchemaType, + kNumberSchemaType, + kIntegerSchemaType, + kTotalSchemaType + }; #if RAPIDJSON_SCHEMA_USE_INTERNALREGEX - typedef internal::GenericRegex RegexType; + typedef internal::GenericRegex RegexType; #elif RAPIDJSON_SCHEMA_USE_STDREGEX - typedef std::basic_regex RegexType; + typedef std::basic_regex RegexType; #else - typedef char RegexType; + typedef char RegexType; #endif - struct SchemaArray { - SchemaArray() : schemas(), count() {} - ~SchemaArray() { AllocatorType::Free(schemas); } - const SchemaType** schemas; - SizeType begin; // begin index of context.validators - SizeType count; - }; - - static const SchemaType* GetTypeless() { - static SchemaType typeless(0, PointerType(), ValueType(kObjectType).Move(), ValueType(kObjectType).Move(), 0); - return &typeless; - } - - template - void AddUniqueElement(V1& a, const V2& v) { - for (typename V1::ConstValueIterator itr = a.Begin(); itr != a.End(); ++itr) - if (*itr == v) - return; - V1 c(v, *allocator_); - a.PushBack(c, *allocator_); - } - - static const ValueType* GetMember(const ValueType& value, const ValueType& name) { - typename ValueType::ConstMemberIterator itr = value.FindMember(name); - return itr != value.MemberEnd() ? &(itr->value) : 0; - } - - static void AssignIfExist(bool& out, const ValueType& value, const ValueType& name) { - if (const ValueType* v = GetMember(value, name)) - if (v->IsBool()) - out = v->GetBool(); - } - - static void AssignIfExist(SizeType& out, const ValueType& value, const ValueType& name) { - if (const ValueType* v = GetMember(value, name)) - if (v->IsUint64() && v->GetUint64() <= SizeType(~0)) - out = static_cast(v->GetUint64()); - } - - void AssignIfExist(SchemaArray& out, SchemaDocumentType& schemaDocument, const PointerType& p, const ValueType& value, const ValueType& name, const ValueType& document) { - if (const ValueType* v = GetMember(value, name)) { - if (v->IsArray() && v->Size() > 0) { - PointerType q = p.Append(name, allocator_); - out.count = v->Size(); - out.schemas = static_cast(allocator_->Malloc(out.count * sizeof(const Schema*))); - memset(out.schemas, 0, sizeof(Schema*)* out.count); - for (SizeType i = 0; i < out.count; i++) - schemaDocument.CreateSchema(&out.schemas[i], q.Append(i, allocator_), (*v)[i], document); - out.begin = validatorCount_; - validatorCount_ += out.count; - } - } - } + struct SchemaArray +{ SchemaArray() : schemas(), count() {} + ~SchemaArray() { AllocatorType::Free(schemas); } + const SchemaType** schemas; + SizeType begin; // begin index of context.validators + SizeType count; + }; + + static const SchemaType* GetTypeless() + { static SchemaType typeless(0, PointerType(), ValueType(kObjectType).Move(), ValueType(kObjectType).Move(), 0); + return &typeless; + } + + template + void AddUniqueElement(V1& a, const V2& v) + { for (typename V1::ConstValueIterator itr = a.Begin(); itr != a.End(); ++itr) + if (*itr == v) + return; + V1 c(v, *allocator_); + a.PushBack(c, *allocator_); + } + + static const ValueType* GetMember(const ValueType& value, const ValueType& name) + { typename ValueType::ConstMemberIterator itr = value.FindMember(name); + return itr != value.MemberEnd() ? &(itr->value) : 0; + } + + static void AssignIfExist(bool& out, const ValueType& value, const ValueType& name) + { if (const ValueType* v = GetMember(value, name)) + if (v->IsBool()) + out = v->GetBool(); + } + + static void AssignIfExist(SizeType& out, const ValueType& value, const ValueType& name) + { if (const ValueType* v = GetMember(value, name)) + if (v->IsUint64() && v->GetUint64() <= SizeType(~0)) + out = static_cast(v->GetUint64()); + } + + void AssignIfExist(SchemaArray& out, SchemaDocumentType& schemaDocument, const PointerType& p, const ValueType& value, const ValueType& name, const ValueType& document) + { if (const ValueType* v = GetMember(value, name)) + { if (v->IsArray() && v->Size() > 0) + { PointerType q = p.Append(name, allocator_); + out.count = v->Size(); + out.schemas = static_cast(allocator_->Malloc(out.count * sizeof(const Schema*))); + memset(out.schemas, 0, sizeof(Schema*)* out.count); + for (SizeType i = 0; i < out.count; i++) + schemaDocument.CreateSchema(&out.schemas[i], q.Append(i, allocator_), (*v)[i], document); + out.begin = validatorCount_; + validatorCount_ += out.count; + } + } + } #if RAPIDJSON_SCHEMA_USE_INTERNALREGEX - template - RegexType* CreatePattern(const ValueType& value) { - if (value.IsString()) { - RegexType* r = new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString()); - if (!r->IsValid()) { - r->~RegexType(); - AllocatorType::Free(r); - r = 0; - } - return r; - } - return 0; - } - - static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType) { - return pattern->Search(str); - } + template + RegexType* CreatePattern(const ValueType& value) + { if (value.IsString()) + { RegexType* r = new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString()); + if (!r->IsValid()) + { r->~RegexType(); + AllocatorType::Free(r); + r = 0; + } + return r; + } + return 0; + } + + static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType) + { return pattern->Search(str); + } #elif RAPIDJSON_SCHEMA_USE_STDREGEX - template - RegexType* CreatePattern(const ValueType& value) { - if (value.IsString()) - try { - return new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString(), std::size_t(value.GetStringLength()), std::regex_constants::ECMAScript); - } - catch (const std::regex_error&) { - } - return 0; - } - - static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType length) { - std::match_results r; - return std::regex_search(str, str + length, r, *pattern); - } + template + RegexType* CreatePattern(const ValueType& value) + { if (value.IsString()) + try + { return new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString(), std::size_t(value.GetStringLength()), std::regex_constants::ECMAScript); + } + catch (const std::regex_error&) + { + } + return 0; + } + + static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType length) + { std::match_results r; + return std::regex_search(str, str + length, r, *pattern); + } #else - template - RegexType* CreatePattern(const ValueType&) { return 0; } + template + RegexType* CreatePattern(const ValueType&) { return 0; } - static bool IsPatternMatch(const RegexType*, const Ch *, SizeType) { return true; } + static bool IsPatternMatch(const RegexType*, const Ch *, SizeType) { return true; } #endif // RAPIDJSON_SCHEMA_USE_STDREGEX - void AddType(const ValueType& type) { - if (type == GetNullString() ) type_ |= 1 << kNullSchemaType; - else if (type == GetBooleanString()) type_ |= 1 << kBooleanSchemaType; - else if (type == GetObjectString() ) type_ |= 1 << kObjectSchemaType; - else if (type == GetArrayString() ) type_ |= 1 << kArraySchemaType; - else if (type == GetStringString() ) type_ |= 1 << kStringSchemaType; - else if (type == GetIntegerString()) type_ |= 1 << kIntegerSchemaType; - else if (type == GetNumberString() ) type_ |= (1 << kNumberSchemaType) | (1 << kIntegerSchemaType); - } - - bool CreateParallelValidator(Context& context) const { - if (enum_ || context.arrayUniqueness) - context.hasher = context.factory.CreateHasher(); - - if (validatorCount_) { - RAPIDJSON_ASSERT(context.validators == 0); - context.validators = static_cast(context.factory.MallocState(sizeof(ISchemaValidator*) * validatorCount_)); - context.validatorCount = validatorCount_; - - if (allOf_.schemas) - CreateSchemaValidators(context, allOf_); - - if (anyOf_.schemas) - CreateSchemaValidators(context, anyOf_); - - if (oneOf_.schemas) - CreateSchemaValidators(context, oneOf_); - - if (not_) - context.validators[notValidatorIndex_] = context.factory.CreateSchemaValidator(*not_); - - if (hasSchemaDependencies_) { - for (SizeType i = 0; i < propertyCount_; i++) - if (properties_[i].dependenciesSchema) - context.validators[properties_[i].dependenciesValidatorIndex] = context.factory.CreateSchemaValidator(*properties_[i].dependenciesSchema); - } - } - + void AddType(const ValueType& type) + { if (type == GetNullString() ) type_ |= 1 << kNullSchemaType; + else if (type == GetBooleanString()) type_ |= 1 << kBooleanSchemaType; + else if (type == GetObjectString() ) type_ |= 1 << kObjectSchemaType; + else if (type == GetArrayString() ) type_ |= 1 << kArraySchemaType; + else if (type == GetStringString() ) type_ |= 1 << kStringSchemaType; + else if (type == GetIntegerString()) type_ |= 1 << kIntegerSchemaType; + else if (type == GetNumberString() ) type_ |= (1 << kNumberSchemaType) | (1 << kIntegerSchemaType); + } + + bool CreateParallelValidator(Context& context) const + { if (enum_ || context.arrayUniqueness) + context.hasher = context.factory.CreateHasher(); + + if (validatorCount_) + { RAPIDJSON_ASSERT(context.validators == 0); + context.validators = static_cast(context.factory.MallocState(sizeof(ISchemaValidator*) * validatorCount_)); + context.validatorCount = validatorCount_; + + if (allOf_.schemas) + CreateSchemaValidators(context, allOf_); + + if (anyOf_.schemas) + CreateSchemaValidators(context, anyOf_); + + if (oneOf_.schemas) + CreateSchemaValidators(context, oneOf_); + + if (not_) + context.validators[notValidatorIndex_] = context.factory.CreateSchemaValidator(*not_); + + if (hasSchemaDependencies_) + { for (SizeType i = 0; i < propertyCount_; i++) + if (properties_[i].dependenciesSchema) + context.validators[properties_[i].dependenciesValidatorIndex] = context.factory.CreateSchemaValidator(*properties_[i].dependenciesSchema); + } + } + + return true; + } + + void CreateSchemaValidators(Context& context, const SchemaArray& schemas) const + { for (SizeType i = 0; i < schemas.count; i++) + context.validators[schemas.begin + i] = context.factory.CreateSchemaValidator(*schemas.schemas[i]); + } + + // O(n) + bool FindPropertyIndex(const ValueType& name, SizeType* outIndex) const + { SizeType len = name.GetStringLength(); + const Ch* str = name.GetString(); + for (SizeType index = 0; index < propertyCount_; index++) + if (properties_[index].name.GetStringLength() == len && + (std::memcmp(properties_[index].name.GetString(), str, sizeof(Ch) * len) == 0)) + { *outIndex = index; return true; + } + return false; + } + + bool CheckInt(Context& context, int64_t i) const + { if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + + if (!minimum_.IsNull()) + { if (minimum_.IsInt64()) + { if (exclusiveMinimum_ ? i <= minimum_.GetInt64() : i < minimum_.GetInt64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + } + else if (minimum_.IsUint64()) + { RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); // i <= max(int64_t) < minimum.GetUint64() + } + else if (!CheckDoubleMinimum(context, static_cast(i))) + return false; } - void CreateSchemaValidators(Context& context, const SchemaArray& schemas) const { - for (SizeType i = 0; i < schemas.count; i++) - context.validators[schemas.begin + i] = context.factory.CreateSchemaValidator(*schemas.schemas[i]); - } - - // O(n) - bool FindPropertyIndex(const ValueType& name, SizeType* outIndex) const { - SizeType len = name.GetStringLength(); - const Ch* str = name.GetString(); - for (SizeType index = 0; index < propertyCount_; index++) - if (properties_[index].name.GetStringLength() == len && - (std::memcmp(properties_[index].name.GetString(), str, sizeof(Ch) * len) == 0)) - { - *outIndex = index; - return true; - } + if (!maximum_.IsNull()) + { if (maximum_.IsInt64()) + { if (exclusiveMaximum_ ? i >= maximum_.GetInt64() : i > maximum_.GetInt64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + } + else if (maximum_.IsUint64()) + /* do nothing */; // i <= max(int64_t) < maximum_.GetUint64() + else if (!CheckDoubleMaximum(context, static_cast(i))) return false; } - bool CheckInt(Context& context, int64_t i) const { - if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - - if (!minimum_.IsNull()) { - if (minimum_.IsInt64()) { - if (exclusiveMinimum_ ? i <= minimum_.GetInt64() : i < minimum_.GetInt64()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); - } - else if (minimum_.IsUint64()) { - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); // i <= max(int64_t) < minimum.GetUint64() - } - else if (!CheckDoubleMinimum(context, static_cast(i))) - return false; - } - - if (!maximum_.IsNull()) { - if (maximum_.IsInt64()) { - if (exclusiveMaximum_ ? i >= maximum_.GetInt64() : i > maximum_.GetInt64()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); - } - else if (maximum_.IsUint64()) - /* do nothing */; // i <= max(int64_t) < maximum_.GetUint64() - else if (!CheckDoubleMaximum(context, static_cast(i))) - return false; - } - - if (!multipleOf_.IsNull()) { - if (multipleOf_.IsUint64()) { - if (static_cast(i >= 0 ? i : -i) % multipleOf_.GetUint64() != 0) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); - } - else if (!CheckDoubleMultipleOf(context, static_cast(i))) - return false; - } - - return true; + if (!multipleOf_.IsNull()) + { if (multipleOf_.IsUint64()) + { if (static_cast(i >= 0 ? i : -i) % multipleOf_.GetUint64() != 0) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + } + else if (!CheckDoubleMultipleOf(context, static_cast(i))) + return false; } - bool CheckUint(Context& context, uint64_t i) const { - if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + return true; + } - if (!minimum_.IsNull()) { - if (minimum_.IsUint64()) { - if (exclusiveMinimum_ ? i <= minimum_.GetUint64() : i < minimum_.GetUint64()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); - } - else if (minimum_.IsInt64()) - /* do nothing */; // i >= 0 > minimum.Getint64() - else if (!CheckDoubleMinimum(context, static_cast(i))) - return false; - } - - if (!maximum_.IsNull()) { - if (maximum_.IsUint64()) { - if (exclusiveMaximum_ ? i >= maximum_.GetUint64() : i > maximum_.GetUint64()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); - } - else if (maximum_.IsInt64()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); // i >= 0 > maximum_ - else if (!CheckDoubleMaximum(context, static_cast(i))) - return false; - } + bool CheckUint(Context& context, uint64_t i) const + { if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - if (!multipleOf_.IsNull()) { - if (multipleOf_.IsUint64()) { - if (i % multipleOf_.GetUint64() != 0) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); - } - else if (!CheckDoubleMultipleOf(context, static_cast(i))) - return false; - } - - return true; + if (!minimum_.IsNull()) + { if (minimum_.IsUint64()) + { if (exclusiveMinimum_ ? i <= minimum_.GetUint64() : i < minimum_.GetUint64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + } + else if (minimum_.IsInt64()) + /* do nothing */; // i >= 0 > minimum.Getint64() + else if (!CheckDoubleMinimum(context, static_cast(i))) + return false; } - bool CheckDoubleMinimum(Context& context, double d) const { - if (exclusiveMinimum_ ? d <= minimum_.GetDouble() : d < minimum_.GetDouble()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); - return true; + if (!maximum_.IsNull()) + { if (maximum_.IsUint64()) + { if (exclusiveMaximum_ ? i >= maximum_.GetUint64() : i > maximum_.GetUint64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + } + else if (maximum_.IsInt64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); // i >= 0 > maximum_ + else if (!CheckDoubleMaximum(context, static_cast(i))) + return false; } - bool CheckDoubleMaximum(Context& context, double d) const { - if (exclusiveMaximum_ ? d >= maximum_.GetDouble() : d > maximum_.GetDouble()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); - return true; + if (!multipleOf_.IsNull()) + { if (multipleOf_.IsUint64()) + { if (i % multipleOf_.GetUint64() != 0) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + } + else if (!CheckDoubleMultipleOf(context, static_cast(i))) + return false; } - bool CheckDoubleMultipleOf(Context& context, double d) const { - double a = std::abs(d), b = std::abs(multipleOf_.GetDouble()); - double q = std::floor(a / b); - double r = a - q * b; - if (r > 0.0) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); - return true; + return true; + } + + bool CheckDoubleMinimum(Context& context, double d) const + { if (exclusiveMinimum_ ? d <= minimum_.GetDouble() : d < minimum_.GetDouble()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + return true; + } + + bool CheckDoubleMaximum(Context& context, double d) const + { if (exclusiveMaximum_ ? d >= maximum_.GetDouble() : d > maximum_.GetDouble()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + return true; + } + + bool CheckDoubleMultipleOf(Context& context, double d) const + { double a = std::abs(d), b = std::abs(multipleOf_.GetDouble()); + double q = std::floor(a / b); + double r = a - q * b; + if (r > 0.0) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + return true; + } + + struct Property +{ Property() : schema(), dependenciesSchema(), dependenciesValidatorIndex(), dependencies(), required(false) {} + ~Property() { AllocatorType::Free(dependencies); } + SValue name; + const SchemaType* schema; + const SchemaType* dependenciesSchema; + SizeType dependenciesValidatorIndex; + bool* dependencies; + bool required; + }; + + struct PatternProperty +{ PatternProperty() : schema(), pattern() {} + ~PatternProperty() + { if (pattern) + { pattern->~RegexType(); + AllocatorType::Free(pattern); + } } - - struct Property { - Property() : schema(), dependenciesSchema(), dependenciesValidatorIndex(), dependencies(), required(false) {} - ~Property() { AllocatorType::Free(dependencies); } - SValue name; - const SchemaType* schema; - const SchemaType* dependenciesSchema; - SizeType dependenciesValidatorIndex; - bool* dependencies; - bool required; - }; - - struct PatternProperty { - PatternProperty() : schema(), pattern() {} - ~PatternProperty() { - if (pattern) { - pattern->~RegexType(); - AllocatorType::Free(pattern); - } - } - const SchemaType* schema; - RegexType* pattern; - }; - - AllocatorType* allocator_; - uint64_t* enum_; - SizeType enumCount_; - SchemaArray allOf_; - SchemaArray anyOf_; - SchemaArray oneOf_; - const SchemaType* not_; - unsigned type_; // bitmask of kSchemaType - SizeType validatorCount_; - SizeType notValidatorIndex_; - - Property* properties_; - const SchemaType* additionalPropertiesSchema_; - PatternProperty* patternProperties_; - SizeType patternPropertyCount_; - SizeType propertyCount_; - SizeType minProperties_; - SizeType maxProperties_; - bool additionalProperties_; - bool hasDependencies_; - bool hasRequired_; - bool hasSchemaDependencies_; - - const SchemaType* additionalItemsSchema_; - const SchemaType* itemsList_; - const SchemaType** itemsTuple_; - SizeType itemsTupleCount_; - SizeType minItems_; - SizeType maxItems_; - bool additionalItems_; - bool uniqueItems_; - - RegexType* pattern_; - SizeType minLength_; - SizeType maxLength_; - - SValue minimum_; - SValue maximum_; - SValue multipleOf_; - bool exclusiveMinimum_; - bool exclusiveMaximum_; + const SchemaType* schema; + RegexType* pattern; + }; + + AllocatorType* allocator_; + uint64_t* enum_; + SizeType enumCount_; + SchemaArray allOf_; + SchemaArray anyOf_; + SchemaArray oneOf_; + const SchemaType* not_; + unsigned type_; // bitmask of kSchemaType + SizeType validatorCount_; + SizeType notValidatorIndex_; + + Property* properties_; + const SchemaType* additionalPropertiesSchema_; + PatternProperty* patternProperties_; + SizeType patternPropertyCount_; + SizeType propertyCount_; + SizeType minProperties_; + SizeType maxProperties_; + bool additionalProperties_; + bool hasDependencies_; + bool hasRequired_; + bool hasSchemaDependencies_; + + const SchemaType* additionalItemsSchema_; + const SchemaType* itemsList_; + const SchemaType** itemsTuple_; + SizeType itemsTupleCount_; + SizeType minItems_; + SizeType maxItems_; + bool additionalItems_; + bool uniqueItems_; + + RegexType* pattern_; + SizeType minLength_; + SizeType maxLength_; + + SValue minimum_; + SValue maximum_; + SValue multipleOf_; + bool exclusiveMinimum_; + bool exclusiveMaximum_; }; template -struct TokenHelper { - RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { - *documentStack.template Push() = '/'; - char buffer[21]; - size_t length = static_cast((sizeof(SizeType) == 4 ? u32toa(index, buffer) : u64toa(index, buffer)) - buffer); - for (size_t i = 0; i < length; i++) - *documentStack.template Push() = buffer[i]; - } +struct TokenHelper +{ RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) + { *documentStack.template Push() = '/'; + char buffer[21]; + size_t length = static_cast((sizeof(SizeType) == 4 ? u32toa(index, buffer) : u64toa(index, buffer)) - buffer); + for (size_t i = 0; i < length; i++) + *documentStack.template Push() = buffer[i]; + } }; // Partial specialized version for char to prevent buffer copying. template -struct TokenHelper { - RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { - if (sizeof(SizeType) == 4) { - char *buffer = documentStack.template Push(1 + 10); // '/' + uint - *buffer++ = '/'; - const char* end = internal::u32toa(index, buffer); - documentStack.template Pop(static_cast(10 - (end - buffer))); - } - else { - char *buffer = documentStack.template Push(1 + 20); // '/' + uint64 - *buffer++ = '/'; - const char* end = internal::u64toa(index, buffer); - documentStack.template Pop(static_cast(20 - (end - buffer))); - } - } +struct TokenHelper +{ RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) + { if (sizeof(SizeType) == 4) + { char *buffer = documentStack.template Push(1 + 10); // '/' + uint + *buffer++ = '/'; + const char* end = internal::u32toa(index, buffer); + documentStack.template Pop(static_cast(10 - (end - buffer))); + } + else + { char *buffer = documentStack.template Push(1 + 20); // '/' + uint64 + *buffer++ = '/'; + const char* end = internal::u64toa(index, buffer); + documentStack.template Pop(static_cast(20 - (end - buffer))); + } + } }; } // namespace internal @@ -1296,12 +1300,13 @@ struct TokenHelper { // IGenericRemoteSchemaDocumentProvider template -class IGenericRemoteSchemaDocumentProvider { +class IGenericRemoteSchemaDocumentProvider +{ public: - typedef typename SchemaDocumentType::Ch Ch; + typedef typename SchemaDocumentType::Ch Ch; - virtual ~IGenericRemoteSchemaDocumentProvider() {} - virtual const SchemaDocumentType* GetRemoteDocument(const Ch* uri, SizeType length) = 0; + virtual ~IGenericRemoteSchemaDocumentProvider() {} + virtual const SchemaDocumentType* GetRemoteDocument(const Ch* uri, SizeType length) = 0; }; /////////////////////////////////////////////////////////////////////////////// @@ -1317,213 +1322,212 @@ class IGenericRemoteSchemaDocumentProvider { \tparam Allocator Allocator type for allocating memory of this document. */ template -class GenericSchemaDocument { +class GenericSchemaDocument +{ public: - typedef ValueT ValueType; - typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProviderType; - typedef Allocator AllocatorType; - typedef typename ValueType::EncodingType EncodingType; - typedef typename EncodingType::Ch Ch; - typedef internal::Schema SchemaType; - typedef GenericPointer PointerType; - friend class internal::Schema; - template - friend class GenericSchemaValidator; - - //! Constructor. - /*! - Compile a JSON document into schema document. - - \param document A JSON document as source. - \param remoteProvider An optional remote schema document provider for resolving remote reference. Can be null. - \param allocator An optional allocator instance for allocating memory. Can be null. - */ - explicit GenericSchemaDocument(const ValueType& document, IRemoteSchemaDocumentProviderType* remoteProvider = 0, Allocator* allocator = 0) : - remoteProvider_(remoteProvider), - allocator_(allocator), - ownAllocator_(), - root_(), - schemaMap_(allocator, kInitialSchemaMapSize), - schemaRef_(allocator, kInitialSchemaRefSize) - { - if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); - - // Generate root schema, it will call CreateSchema() to create sub-schemas, - // And call AddRefSchema() if there are $ref. - CreateSchemaRecursive(&root_, PointerType(), document, document); - - // Resolve $ref - while (!schemaRef_.Empty()) { - SchemaRefEntry* refEntry = schemaRef_.template Pop(1); - if (const SchemaType* s = GetSchema(refEntry->target)) { - if (refEntry->schema) - *refEntry->schema = s; - - // Create entry in map if not exist - if (!GetSchema(refEntry->source)) { - new (schemaMap_.template Push()) SchemaEntry(refEntry->source, const_cast(s), false, allocator_); - } - } - refEntry->~SchemaRefEntry(); + typedef ValueT ValueType; + typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProviderType; + typedef Allocator AllocatorType; + typedef typename ValueType::EncodingType EncodingType; + typedef typename EncodingType::Ch Ch; + typedef internal::Schema SchemaType; + typedef GenericPointer PointerType; + friend class internal::Schema; + template + friend class GenericSchemaValidator; + + //! Constructor. + /*! + Compile a JSON document into schema document. + + \param document A JSON document as source. + \param remoteProvider An optional remote schema document provider for resolving remote reference. Can be null. + \param allocator An optional allocator instance for allocating memory. Can be null. + */ + explicit GenericSchemaDocument(const ValueType& document, IRemoteSchemaDocumentProviderType* remoteProvider = 0, Allocator* allocator = 0) : + remoteProvider_(remoteProvider), + allocator_(allocator), + ownAllocator_(), + root_(), + schemaMap_(allocator, kInitialSchemaMapSize), + schemaRef_(allocator, kInitialSchemaRefSize) + { if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + + // Generate root schema, it will call CreateSchema() to create sub-schemas, + // And call AddRefSchema() if there are $ref. + CreateSchemaRecursive(&root_, PointerType(), document, document); + + // Resolve $ref + while (!schemaRef_.Empty()) + { SchemaRefEntry* refEntry = schemaRef_.template Pop(1); + if (const SchemaType* s = GetSchema(refEntry->target)) + { if (refEntry->schema) + *refEntry->schema = s; + + // Create entry in map if not exist + if (!GetSchema(refEntry->source)) + { new (schemaMap_.template Push()) SchemaEntry(refEntry->source, const_cast(s), false, allocator_); } + } + refEntry->~SchemaRefEntry(); + } - RAPIDJSON_ASSERT(root_ != 0); + RAPIDJSON_ASSERT(root_ != 0); - schemaRef_.ShrinkToFit(); // Deallocate all memory for ref - } + schemaRef_.ShrinkToFit(); // Deallocate all memory for ref + } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Move constructor in C++11 - GenericSchemaDocument(GenericSchemaDocument&& rhs) RAPIDJSON_NOEXCEPT : - remoteProvider_(rhs.remoteProvider_), - allocator_(rhs.allocator_), - ownAllocator_(rhs.ownAllocator_), - root_(rhs.root_), - schemaMap_(std::move(rhs.schemaMap_)), - schemaRef_(std::move(rhs.schemaRef_)) - { - rhs.remoteProvider_ = 0; - rhs.allocator_ = 0; - rhs.ownAllocator_ = 0; - } + //! Move constructor in C++11 +GenericSchemaDocument(GenericSchemaDocument&& rhs) RAPIDJSON_NOEXCEPT : + remoteProvider_(rhs.remoteProvider_), + allocator_(rhs.allocator_), + ownAllocator_(rhs.ownAllocator_), + root_(rhs.root_), + schemaMap_(std::move(rhs.schemaMap_)), + schemaRef_(std::move(rhs.schemaRef_)) + { rhs.remoteProvider_ = 0; + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + } #endif - //! Destructor - ~GenericSchemaDocument() { - while (!schemaMap_.Empty()) - schemaMap_.template Pop(1)->~SchemaEntry(); + //! Destructor + ~GenericSchemaDocument() + { while (!schemaMap_.Empty()) + schemaMap_.template Pop(1)->~SchemaEntry(); - RAPIDJSON_DELETE(ownAllocator_); - } + RAPIDJSON_DELETE(ownAllocator_); + } - //! Get the root schema. - const SchemaType& GetRoot() const { return *root_; } + //! Get the root schema. + const SchemaType& GetRoot() const { return *root_; } private: - //! Prohibit copying - GenericSchemaDocument(const GenericSchemaDocument&); - //! Prohibit assignment - GenericSchemaDocument& operator=(const GenericSchemaDocument&); - - struct SchemaRefEntry { - SchemaRefEntry(const PointerType& s, const PointerType& t, const SchemaType** outSchema, Allocator *allocator) : source(s, allocator), target(t, allocator), schema(outSchema) {} - PointerType source; - PointerType target; - const SchemaType** schema; - }; - - struct SchemaEntry { - SchemaEntry(const PointerType& p, SchemaType* s, bool o, Allocator* allocator) : pointer(p, allocator), schema(s), owned(o) {} - ~SchemaEntry() { - if (owned) { - schema->~SchemaType(); - Allocator::Free(schema); - } - } - PointerType pointer; - SchemaType* schema; - bool owned; - }; - - void CreateSchemaRecursive(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { + //! Prohibit copying + GenericSchemaDocument(const GenericSchemaDocument&); + //! Prohibit assignment + GenericSchemaDocument& operator=(const GenericSchemaDocument&); + + struct SchemaRefEntry +{ SchemaRefEntry(const PointerType& s, const PointerType& t, const SchemaType** outSchema, Allocator *allocator) : source(s, allocator), target(t, allocator), schema(outSchema) {} + PointerType source; + PointerType target; + const SchemaType** schema; + }; + + struct SchemaEntry +{ SchemaEntry(const PointerType& p, SchemaType* s, bool o, Allocator* allocator) : pointer(p, allocator), schema(s), owned(o) {} + ~SchemaEntry() + { if (owned) + { schema->~SchemaType(); + Allocator::Free(schema); + } + } + PointerType pointer; + SchemaType* schema; + bool owned; + }; + + void CreateSchemaRecursive(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) + { if (schema) + *schema = SchemaType::GetTypeless(); + + if (v.GetType() == kObjectType) + { const SchemaType* s = GetSchema(pointer); + if (!s) + CreateSchema(schema, pointer, v, document); + + for (typename ValueType::ConstMemberIterator itr = v.MemberBegin(); itr != v.MemberEnd(); ++itr) + CreateSchemaRecursive(0, pointer.Append(itr->name, allocator_), itr->value, document); + } + else if (v.GetType() == kArrayType) + for (SizeType i = 0; i < v.Size(); i++) + CreateSchemaRecursive(0, pointer.Append(i, allocator_), v[i], document); + } + + void CreateSchema(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) + { RAPIDJSON_ASSERT(pointer.IsValid()); + if (v.IsObject()) + { if (!HandleRefSchema(pointer, schema, v, document)) + { SchemaType* s = new (allocator_->Malloc(sizeof(SchemaType))) SchemaType(this, pointer, v, document, allocator_); + new (schemaMap_.template Push()) SchemaEntry(pointer, s, true, allocator_); if (schema) - *schema = SchemaType::GetTypeless(); - - if (v.GetType() == kObjectType) { - const SchemaType* s = GetSchema(pointer); - if (!s) - CreateSchema(schema, pointer, v, document); - - for (typename ValueType::ConstMemberIterator itr = v.MemberBegin(); itr != v.MemberEnd(); ++itr) - CreateSchemaRecursive(0, pointer.Append(itr->name, allocator_), itr->value, document); - } - else if (v.GetType() == kArrayType) - for (SizeType i = 0; i < v.Size(); i++) - CreateSchemaRecursive(0, pointer.Append(i, allocator_), v[i], document); - } - - void CreateSchema(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { - RAPIDJSON_ASSERT(pointer.IsValid()); - if (v.IsObject()) { - if (!HandleRefSchema(pointer, schema, v, document)) { - SchemaType* s = new (allocator_->Malloc(sizeof(SchemaType))) SchemaType(this, pointer, v, document, allocator_); - new (schemaMap_.template Push()) SchemaEntry(pointer, s, true, allocator_); - if (schema) - *schema = s; - } - } - } - - bool HandleRefSchema(const PointerType& source, const SchemaType** schema, const ValueType& v, const ValueType& document) { - static const Ch kRefString[] = { '$', 'r', 'e', 'f', '\0' }; - static const ValueType kRefValue(kRefString, 4); - - typename ValueType::ConstMemberIterator itr = v.FindMember(kRefValue); - if (itr == v.MemberEnd()) - return false; - - if (itr->value.IsString()) { - SizeType len = itr->value.GetStringLength(); - if (len > 0) { - const Ch* s = itr->value.GetString(); - SizeType i = 0; - while (i < len && s[i] != '#') // Find the first # - i++; - - if (i > 0) { // Remote reference, resolve immediately - if (remoteProvider_) { - if (const GenericSchemaDocument* remoteDocument = remoteProvider_->GetRemoteDocument(s, i - 1)) { - PointerType pointer(&s[i], len - i, allocator_); - if (pointer.IsValid()) { - if (const SchemaType* sc = remoteDocument->GetSchema(pointer)) { - if (schema) - *schema = sc; - return true; - } - } - } - } - } - else if (s[i] == '#') { // Local reference, defer resolution - PointerType pointer(&s[i], len - i, allocator_); - if (pointer.IsValid()) { - if (const ValueType* nv = pointer.Get(document)) - if (HandleRefSchema(source, schema, *nv, document)) - return true; - - new (schemaRef_.template Push()) SchemaRefEntry(source, pointer, schema, allocator_); - return true; - } + *schema = s; + } + } + } + + bool HandleRefSchema(const PointerType& source, const SchemaType** schema, const ValueType& v, const ValueType& document) + { static const Ch kRefString[] = { '$', 'r', 'e', 'f', '\0' }; + static const ValueType kRefValue(kRefString, 4); + + typename ValueType::ConstMemberIterator itr = v.FindMember(kRefValue); + if (itr == v.MemberEnd()) + return false; + + if (itr->value.IsString()) + { SizeType len = itr->value.GetStringLength(); + if (len > 0) + { const Ch* s = itr->value.GetString(); + SizeType i = 0; + while (i < len && s[i] != '#') // Find the first # + i++; + + if (i > 0) // Remote reference, resolve immediately + { if (remoteProvider_) + { if (const GenericSchemaDocument* remoteDocument = remoteProvider_->GetRemoteDocument(s, i - 1)) + { PointerType pointer(&s[i], len - i, allocator_); + if (pointer.IsValid()) + { if (const SchemaType* sc = remoteDocument->GetSchema(pointer)) + { if (schema) + *schema = sc; + return true; } + } } + } } - return false; - } - - const SchemaType* GetSchema(const PointerType& pointer) const { - for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) - if (pointer == target->pointer) - return target->schema; - return 0; - } - - PointerType GetPointer(const SchemaType* schema) const { - for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) - if (schema == target->schema) - return target->pointer; - return PointerType(); - } - - static const size_t kInitialSchemaMapSize = 64; - static const size_t kInitialSchemaRefSize = 64; + else if (s[i] == '#') // Local reference, defer resolution + { PointerType pointer(&s[i], len - i, allocator_); + if (pointer.IsValid()) + { if (const ValueType* nv = pointer.Get(document)) + if (HandleRefSchema(source, schema, *nv, document)) + return true; - IRemoteSchemaDocumentProviderType* remoteProvider_; - Allocator *allocator_; - Allocator *ownAllocator_; - const SchemaType* root_; //!< Root schema. - internal::Stack schemaMap_; // Stores created Pointer -> Schemas - internal::Stack schemaRef_; // Stores Pointer from $ref and schema which holds the $ref + new (schemaRef_.template Push()) SchemaRefEntry(source, pointer, schema, allocator_); + return true; + } + } + } + } + return false; + } + + const SchemaType* GetSchema(const PointerType& pointer) const + { for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) + if (pointer == target->pointer) + return target->schema; + return 0; + } + + PointerType GetPointer(const SchemaType* schema) const + { for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) + if (schema == target->schema) + return target->pointer; + return PointerType(); + } + + static const size_t kInitialSchemaMapSize = 64; + static const size_t kInitialSchemaRefSize = 64; + + IRemoteSchemaDocumentProviderType* remoteProvider_; + Allocator *allocator_; + Allocator *ownAllocator_; + const SchemaType* root_; //!< Root schema. + internal::Stack schemaMap_; // Stores created Pointer -> Schemas + internal::Stack schemaRef_; // Stores Pointer from $ref and schema which holds the $ref }; //! GenericSchemaDocument using Value type. @@ -1547,106 +1551,106 @@ typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocume \tparam StateAllocator Allocator for storing the internal validation states. */ template < - typename SchemaDocumentType, - typename OutputHandler = BaseReaderHandler, - typename StateAllocator = CrtAllocator> + typename SchemaDocumentType, + typename OutputHandler = BaseReaderHandler, + typename StateAllocator = CrtAllocator> class GenericSchemaValidator : - public internal::ISchemaStateFactory, - public internal::ISchemaValidator + public internal::ISchemaStateFactory, + public internal::ISchemaValidator { public: - typedef typename SchemaDocumentType::SchemaType SchemaType; - typedef typename SchemaDocumentType::PointerType PointerType; - typedef typename SchemaType::EncodingType EncodingType; - typedef typename EncodingType::Ch Ch; - - //! Constructor without output handler. - /*! - \param schemaDocument The schema document to conform to. - \param allocator Optional allocator for storing internal validation states. - \param schemaStackCapacity Optional initial capacity of schema path stack. - \param documentStackCapacity Optional initial capacity of document path stack. - */ - GenericSchemaValidator( - const SchemaDocumentType& schemaDocument, - StateAllocator* allocator = 0, - size_t schemaStackCapacity = kDefaultSchemaStackCapacity, - size_t documentStackCapacity = kDefaultDocumentStackCapacity) - : - schemaDocument_(&schemaDocument), - root_(schemaDocument.GetRoot()), - outputHandler_(GetNullHandler()), - stateAllocator_(allocator), - ownStateAllocator_(0), - schemaStack_(allocator, schemaStackCapacity), - documentStack_(allocator, documentStackCapacity), - valid_(true) + typedef typename SchemaDocumentType::SchemaType SchemaType; + typedef typename SchemaDocumentType::PointerType PointerType; + typedef typename SchemaType::EncodingType EncodingType; + typedef typename EncodingType::Ch Ch; + + //! Constructor without output handler. + /*! + \param schemaDocument The schema document to conform to. + \param allocator Optional allocator for storing internal validation states. + \param schemaStackCapacity Optional initial capacity of schema path stack. + \param documentStackCapacity Optional initial capacity of document path stack. + */ + GenericSchemaValidator( + const SchemaDocumentType& schemaDocument, + StateAllocator* allocator = 0, + size_t schemaStackCapacity = kDefaultSchemaStackCapacity, + size_t documentStackCapacity = kDefaultDocumentStackCapacity) + : + schemaDocument_(&schemaDocument), + root_(schemaDocument.GetRoot()), + outputHandler_(GetNullHandler()), + stateAllocator_(allocator), + ownStateAllocator_(0), + schemaStack_(allocator, schemaStackCapacity), + documentStack_(allocator, documentStackCapacity), + valid_(true) #if RAPIDJSON_SCHEMA_VERBOSE - , depth_(0) + , depth_(0) #endif - { - } - - //! Constructor with output handler. - /*! - \param schemaDocument The schema document to conform to. - \param allocator Optional allocator for storing internal validation states. - \param schemaStackCapacity Optional initial capacity of schema path stack. - \param documentStackCapacity Optional initial capacity of document path stack. - */ - GenericSchemaValidator( - const SchemaDocumentType& schemaDocument, - OutputHandler& outputHandler, - StateAllocator* allocator = 0, - size_t schemaStackCapacity = kDefaultSchemaStackCapacity, - size_t documentStackCapacity = kDefaultDocumentStackCapacity) - : - schemaDocument_(&schemaDocument), - root_(schemaDocument.GetRoot()), - outputHandler_(outputHandler), - stateAllocator_(allocator), - ownStateAllocator_(0), - schemaStack_(allocator, schemaStackCapacity), - documentStack_(allocator, documentStackCapacity), - valid_(true) + { + } + + //! Constructor with output handler. + /*! + \param schemaDocument The schema document to conform to. + \param allocator Optional allocator for storing internal validation states. + \param schemaStackCapacity Optional initial capacity of schema path stack. + \param documentStackCapacity Optional initial capacity of document path stack. + */ + GenericSchemaValidator( + const SchemaDocumentType& schemaDocument, + OutputHandler& outputHandler, + StateAllocator* allocator = 0, + size_t schemaStackCapacity = kDefaultSchemaStackCapacity, + size_t documentStackCapacity = kDefaultDocumentStackCapacity) + : + schemaDocument_(&schemaDocument), + root_(schemaDocument.GetRoot()), + outputHandler_(outputHandler), + stateAllocator_(allocator), + ownStateAllocator_(0), + schemaStack_(allocator, schemaStackCapacity), + documentStack_(allocator, documentStackCapacity), + valid_(true) #if RAPIDJSON_SCHEMA_VERBOSE - , depth_(0) + , depth_(0) #endif - { - } - - //! Destructor. - ~GenericSchemaValidator() { - Reset(); - RAPIDJSON_DELETE(ownStateAllocator_); - } - - //! Reset the internal states. - void Reset() { - while (!schemaStack_.Empty()) - PopSchema(); - documentStack_.Clear(); - valid_ = true; - } - - //! Checks whether the current state is valid. - // Implementation of ISchemaValidator - virtual bool IsValid() const { return valid_; } - - //! Gets the JSON pointer pointed to the invalid schema. - PointerType GetInvalidSchemaPointer() const { - return schemaStack_.Empty() ? PointerType() : schemaDocument_->GetPointer(&CurrentSchema()); - } - - //! Gets the keyword of invalid schema. - const Ch* GetInvalidSchemaKeyword() const { - return schemaStack_.Empty() ? 0 : CurrentContext().invalidKeyword; - } - - //! Gets the JSON pointer pointed to the invalid value. - PointerType GetInvalidDocumentPointer() const { - return documentStack_.Empty() ? PointerType() : PointerType(documentStack_.template Bottom(), documentStack_.GetSize() / sizeof(Ch)); - } + { + } + + //! Destructor. + ~GenericSchemaValidator() + { Reset(); + RAPIDJSON_DELETE(ownStateAllocator_); + } + + //! Reset the internal states. + void Reset() + { while (!schemaStack_.Empty()) + PopSchema(); + documentStack_.Clear(); + valid_ = true; + } + + //! Checks whether the current state is valid. + // Implementation of ISchemaValidator + virtual bool IsValid() const { return valid_; } + + //! Gets the JSON pointer pointed to the invalid schema. + PointerType GetInvalidSchemaPointer() const + { return schemaStack_.Empty() ? PointerType() : schemaDocument_->GetPointer(&CurrentSchema()); + } + + //! Gets the keyword of invalid schema. + const Ch* GetInvalidSchemaKeyword() const + { return schemaStack_.Empty() ? 0 : CurrentContext().invalidKeyword; + } + + //! Gets the JSON pointer pointed to the invalid value. + PointerType GetInvalidDocumentPointer() const + { return documentStack_.Empty() ? PointerType() : PointerType(documentStack_.template Bottom(), documentStack_.GetSize() / sizeof(Ch)); + } #if RAPIDJSON_SCHEMA_VERBOSE #define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() \ @@ -1686,246 +1690,246 @@ RAPIDJSON_MULTILINEMACRO_END RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2);\ RAPIDJSON_SCHEMA_HANDLE_END_ (method, arg2) - bool Null() { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Null, (CurrentContext() ), ( )); } - bool Bool(bool b) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Bool, (CurrentContext(), b), (b)); } - bool Int(int i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int, (CurrentContext(), i), (i)); } - bool Uint(unsigned u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint, (CurrentContext(), u), (u)); } - bool Int64(int64_t i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int64, (CurrentContext(), i), (i)); } - bool Uint64(uint64_t u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint64, (CurrentContext(), u), (u)); } - bool Double(double d) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Double, (CurrentContext(), d), (d)); } - bool RawNumber(const Ch* str, SizeType length, bool copy) - { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } - bool String(const Ch* str, SizeType length, bool copy) - { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } - - bool StartObject() { - RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartObject, (CurrentContext())); - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartObject, ()); - return valid_ = outputHandler_.StartObject(); - } - - bool Key(const Ch* str, SizeType len, bool copy) { - if (!valid_) return false; - AppendToken(str, len); - if (!CurrentSchema().Key(CurrentContext(), str, len, copy)) return valid_ = false; - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(Key, (str, len, copy)); - return valid_ = outputHandler_.Key(str, len, copy); - } - - bool EndObject(SizeType memberCount) { - if (!valid_) return false; - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndObject, (memberCount)); - if (!CurrentSchema().EndObject(CurrentContext(), memberCount)) return valid_ = false; - RAPIDJSON_SCHEMA_HANDLE_END_(EndObject, (memberCount)); - } - - bool StartArray() { - RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartArray, (CurrentContext())); - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartArray, ()); - return valid_ = outputHandler_.StartArray(); - } - - bool EndArray(SizeType elementCount) { - if (!valid_) return false; - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndArray, (elementCount)); - if (!CurrentSchema().EndArray(CurrentContext(), elementCount)) return valid_ = false; - RAPIDJSON_SCHEMA_HANDLE_END_(EndArray, (elementCount)); - } + bool Null() { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Null, (CurrentContext() ), ( )); } + bool Bool(bool b) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Bool, (CurrentContext(), b), (b)); } + bool Int(int i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int, (CurrentContext(), i), (i)); } + bool Uint(unsigned u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint, (CurrentContext(), u), (u)); } + bool Int64(int64_t i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int64, (CurrentContext(), i), (i)); } + bool Uint64(uint64_t u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint64, (CurrentContext(), u), (u)); } + bool Double(double d) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Double, (CurrentContext(), d), (d)); } + bool RawNumber(const Ch* str, SizeType length, bool copy) + { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } + bool String(const Ch* str, SizeType length, bool copy) + { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } + + bool StartObject() + { RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartObject, (CurrentContext())); + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartObject, ()); + return valid_ = outputHandler_.StartObject(); + } + + bool Key(const Ch* str, SizeType len, bool copy) + { if (!valid_) return false; + AppendToken(str, len); + if (!CurrentSchema().Key(CurrentContext(), str, len, copy)) return valid_ = false; + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(Key, (str, len, copy)); + return valid_ = outputHandler_.Key(str, len, copy); + } + + bool EndObject(SizeType memberCount) + { if (!valid_) return false; + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndObject, (memberCount)); + if (!CurrentSchema().EndObject(CurrentContext(), memberCount)) return valid_ = false; + RAPIDJSON_SCHEMA_HANDLE_END_(EndObject, (memberCount)); + } + + bool StartArray() + { RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartArray, (CurrentContext())); + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartArray, ()); + return valid_ = outputHandler_.StartArray(); + } + + bool EndArray(SizeType elementCount) + { if (!valid_) return false; + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndArray, (elementCount)); + if (!CurrentSchema().EndArray(CurrentContext(), elementCount)) return valid_ = false; + RAPIDJSON_SCHEMA_HANDLE_END_(EndArray, (elementCount)); + } #undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_ #undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_ #undef RAPIDJSON_SCHEMA_HANDLE_PARALLEL_ #undef RAPIDJSON_SCHEMA_HANDLE_VALUE_ - // Implementation of ISchemaStateFactory - virtual ISchemaValidator* CreateSchemaValidator(const SchemaType& root) { - return new (GetStateAllocator().Malloc(sizeof(GenericSchemaValidator))) GenericSchemaValidator(*schemaDocument_, root, + // Implementation of ISchemaStateFactory + virtual ISchemaValidator* CreateSchemaValidator(const SchemaType& root) + { return new (GetStateAllocator().Malloc(sizeof(GenericSchemaValidator))) GenericSchemaValidator(*schemaDocument_, root, #if RAPIDJSON_SCHEMA_VERBOSE - depth_ + 1, + depth_ + 1, #endif - &GetStateAllocator()); - } + &GetStateAllocator()); + } - virtual void DestroySchemaValidator(ISchemaValidator* validator) { - GenericSchemaValidator* v = static_cast(validator); - v->~GenericSchemaValidator(); - StateAllocator::Free(v); - } + virtual void DestroySchemaValidator(ISchemaValidator* validator) + { GenericSchemaValidator* v = static_cast(validator); + v->~GenericSchemaValidator(); + StateAllocator::Free(v); + } - virtual void* CreateHasher() { - return new (GetStateAllocator().Malloc(sizeof(HasherType))) HasherType(&GetStateAllocator()); - } + virtual void* CreateHasher() + { return new (GetStateAllocator().Malloc(sizeof(HasherType))) HasherType(&GetStateAllocator()); + } - virtual uint64_t GetHashCode(void* hasher) { - return static_cast(hasher)->GetHashCode(); - } + virtual uint64_t GetHashCode(void* hasher) + { return static_cast(hasher)->GetHashCode(); + } - virtual void DestroryHasher(void* hasher) { - HasherType* h = static_cast(hasher); - h->~HasherType(); - StateAllocator::Free(h); - } + virtual void DestroryHasher(void* hasher) + { HasherType* h = static_cast(hasher); + h->~HasherType(); + StateAllocator::Free(h); + } - virtual void* MallocState(size_t size) { - return GetStateAllocator().Malloc(size); - } + virtual void* MallocState(size_t size) + { return GetStateAllocator().Malloc(size); + } - virtual void FreeState(void* p) { - return StateAllocator::Free(p); - } + virtual void FreeState(void* p) + { return StateAllocator::Free(p); + } private: - typedef typename SchemaType::Context Context; - typedef GenericValue, StateAllocator> HashCodeArray; - typedef internal::Hasher HasherType; + typedef typename SchemaType::Context Context; + typedef GenericValue, StateAllocator> HashCodeArray; + typedef internal::Hasher HasherType; - GenericSchemaValidator( - const SchemaDocumentType& schemaDocument, - const SchemaType& root, + GenericSchemaValidator( + const SchemaDocumentType& schemaDocument, + const SchemaType& root, #if RAPIDJSON_SCHEMA_VERBOSE - unsigned depth, + unsigned depth, #endif - StateAllocator* allocator = 0, - size_t schemaStackCapacity = kDefaultSchemaStackCapacity, - size_t documentStackCapacity = kDefaultDocumentStackCapacity) - : - schemaDocument_(&schemaDocument), - root_(root), - outputHandler_(GetNullHandler()), - stateAllocator_(allocator), - ownStateAllocator_(0), - schemaStack_(allocator, schemaStackCapacity), - documentStack_(allocator, documentStackCapacity), - valid_(true) + StateAllocator* allocator = 0, + size_t schemaStackCapacity = kDefaultSchemaStackCapacity, + size_t documentStackCapacity = kDefaultDocumentStackCapacity) + : + schemaDocument_(&schemaDocument), + root_(root), + outputHandler_(GetNullHandler()), + stateAllocator_(allocator), + ownStateAllocator_(0), + schemaStack_(allocator, schemaStackCapacity), + documentStack_(allocator, documentStackCapacity), + valid_(true) #if RAPIDJSON_SCHEMA_VERBOSE - , depth_(depth) + , depth_(depth) #endif - { - } + { + } + + StateAllocator& GetStateAllocator() + { if (!stateAllocator_) + stateAllocator_ = ownStateAllocator_ = RAPIDJSON_NEW(StateAllocator()); + return *stateAllocator_; + } + + bool BeginValue() + { if (schemaStack_.Empty()) + PushSchema(root_); + else + { if (CurrentContext().inArray) + internal::TokenHelper, Ch>::AppendIndexToken(documentStack_, CurrentContext().arrayElementIndex); + + if (!CurrentSchema().BeginValue(CurrentContext())) + return false; - StateAllocator& GetStateAllocator() { - if (!stateAllocator_) - stateAllocator_ = ownStateAllocator_ = RAPIDJSON_NEW(StateAllocator()); - return *stateAllocator_; - } + SizeType count = CurrentContext().patternPropertiesSchemaCount; + const SchemaType** sa = CurrentContext().patternPropertiesSchemas; + typename Context::PatternValidatorType patternValidatorType = CurrentContext().valuePatternValidatorType; + bool valueUniqueness = CurrentContext().valueUniqueness; + if (CurrentContext().valueSchema) + PushSchema(*CurrentContext().valueSchema); - bool BeginValue() { - if (schemaStack_.Empty()) - PushSchema(root_); - else { - if (CurrentContext().inArray) - internal::TokenHelper, Ch>::AppendIndexToken(documentStack_, CurrentContext().arrayElementIndex); - - if (!CurrentSchema().BeginValue(CurrentContext())) - return false; - - SizeType count = CurrentContext().patternPropertiesSchemaCount; - const SchemaType** sa = CurrentContext().patternPropertiesSchemas; - typename Context::PatternValidatorType patternValidatorType = CurrentContext().valuePatternValidatorType; - bool valueUniqueness = CurrentContext().valueUniqueness; - if (CurrentContext().valueSchema) - PushSchema(*CurrentContext().valueSchema); - - if (count > 0) { - CurrentContext().objectPatternValidatorType = patternValidatorType; - ISchemaValidator**& va = CurrentContext().patternPropertiesValidators; - SizeType& validatorCount = CurrentContext().patternPropertiesValidatorCount; - va = static_cast(MallocState(sizeof(ISchemaValidator*) * count)); - for (SizeType i = 0; i < count; i++) - va[validatorCount++] = CreateSchemaValidator(*sa[i]); - } + if (count > 0) + { CurrentContext().objectPatternValidatorType = patternValidatorType; + ISchemaValidator**& va = CurrentContext().patternPropertiesValidators; + SizeType& validatorCount = CurrentContext().patternPropertiesValidatorCount; + va = static_cast(MallocState(sizeof(ISchemaValidator*) * count)); + for (SizeType i = 0; i < count; i++) + va[validatorCount++] = CreateSchemaValidator(*sa[i]); + } - CurrentContext().arrayUniqueness = valueUniqueness; - } - return true; + CurrentContext().arrayUniqueness = valueUniqueness; } + return true; + } - bool EndValue() { - if (!CurrentSchema().EndValue(CurrentContext())) - return false; + bool EndValue() + { if (!CurrentSchema().EndValue(CurrentContext())) + return false; #if RAPIDJSON_SCHEMA_VERBOSE - GenericStringBuffer sb; - schemaDocument_->GetPointer(&CurrentSchema()).Stringify(sb); + GenericStringBuffer sb; + schemaDocument_->GetPointer(&CurrentSchema()).Stringify(sb); - *documentStack_.template Push() = '\0'; - documentStack_.template Pop(1); - internal::PrintValidatorPointers(depth_, sb.GetString(), documentStack_.template Bottom()); + *documentStack_.template Push() = '\0'; + documentStack_.template Pop(1); + internal::PrintValidatorPointers(depth_, sb.GetString(), documentStack_.template Bottom()); #endif - uint64_t h = CurrentContext().arrayUniqueness ? static_cast(CurrentContext().hasher)->GetHashCode() : 0; - - PopSchema(); - - if (!schemaStack_.Empty()) { - Context& context = CurrentContext(); - if (context.valueUniqueness) { - HashCodeArray* a = static_cast(context.arrayElementHashCodes); - if (!a) - CurrentContext().arrayElementHashCodes = a = new (GetStateAllocator().Malloc(sizeof(HashCodeArray))) HashCodeArray(kArrayType); - for (typename HashCodeArray::ConstValueIterator itr = a->Begin(); itr != a->End(); ++itr) - if (itr->GetUint64() == h) - RAPIDJSON_INVALID_KEYWORD_RETURN(SchemaType::GetUniqueItemsString()); - a->PushBack(h, GetStateAllocator()); - } - } - - // Remove the last token of document pointer - while (!documentStack_.Empty() && *documentStack_.template Pop(1) != '/') - ; - - return true; - } - - void AppendToken(const Ch* str, SizeType len) { - documentStack_.template Reserve(1 + len * 2); // worst case all characters are escaped as two characters - *documentStack_.template PushUnsafe() = '/'; - for (SizeType i = 0; i < len; i++) { - if (str[i] == '~') { - *documentStack_.template PushUnsafe() = '~'; - *documentStack_.template PushUnsafe() = '0'; - } - else if (str[i] == '/') { - *documentStack_.template PushUnsafe() = '~'; - *documentStack_.template PushUnsafe() = '1'; - } - else - *documentStack_.template PushUnsafe() = str[i]; - } - } - - RAPIDJSON_FORCEINLINE void PushSchema(const SchemaType& schema) { new (schemaStack_.template Push()) Context(*this, &schema); } - - RAPIDJSON_FORCEINLINE void PopSchema() { - Context* c = schemaStack_.template Pop(1); - if (HashCodeArray* a = static_cast(c->arrayElementHashCodes)) { - a->~HashCodeArray(); - StateAllocator::Free(a); - } - c->~Context(); - } - - const SchemaType& CurrentSchema() const { return *schemaStack_.template Top()->schema; } - Context& CurrentContext() { return *schemaStack_.template Top(); } - const Context& CurrentContext() const { return *schemaStack_.template Top(); } - - static OutputHandler& GetNullHandler() { - static OutputHandler nullHandler; - return nullHandler; - } - - static const size_t kDefaultSchemaStackCapacity = 1024; - static const size_t kDefaultDocumentStackCapacity = 256; - const SchemaDocumentType* schemaDocument_; - const SchemaType& root_; - OutputHandler& outputHandler_; - StateAllocator* stateAllocator_; - StateAllocator* ownStateAllocator_; - internal::Stack schemaStack_; //!< stack to store the current path of schema (BaseSchemaType *) - internal::Stack documentStack_; //!< stack to store the current path of validating document (Ch) - bool valid_; + uint64_t h = CurrentContext().arrayUniqueness ? static_cast(CurrentContext().hasher)->GetHashCode() : 0; + + PopSchema(); + + if (!schemaStack_.Empty()) + { Context& context = CurrentContext(); + if (context.valueUniqueness) + { HashCodeArray* a = static_cast(context.arrayElementHashCodes); + if (!a) + CurrentContext().arrayElementHashCodes = a = new (GetStateAllocator().Malloc(sizeof(HashCodeArray))) HashCodeArray(kArrayType); + for (typename HashCodeArray::ConstValueIterator itr = a->Begin(); itr != a->End(); ++itr) + if (itr->GetUint64() == h) + RAPIDJSON_INVALID_KEYWORD_RETURN(SchemaType::GetUniqueItemsString()); + a->PushBack(h, GetStateAllocator()); + } + } + + // Remove the last token of document pointer + while (!documentStack_.Empty() && *documentStack_.template Pop(1) != '/') + ; + + return true; + } + + void AppendToken(const Ch* str, SizeType len) + { documentStack_.template Reserve(1 + len * 2); // worst case all characters are escaped as two characters + *documentStack_.template PushUnsafe() = '/'; + for (SizeType i = 0; i < len; i++) + { if (str[i] == '~') + { *documentStack_.template PushUnsafe() = '~'; + *documentStack_.template PushUnsafe() = '0'; + } + else if (str[i] == '/') + { *documentStack_.template PushUnsafe() = '~'; + *documentStack_.template PushUnsafe() = '1'; + } + else + *documentStack_.template PushUnsafe() = str[i]; + } + } + + RAPIDJSON_FORCEINLINE void PushSchema(const SchemaType& schema) { new (schemaStack_.template Push()) Context(*this, &schema); } + + RAPIDJSON_FORCEINLINE void PopSchema() + { Context* c = schemaStack_.template Pop(1); + if (HashCodeArray* a = static_cast(c->arrayElementHashCodes)) + { a->~HashCodeArray(); + StateAllocator::Free(a); + } + c->~Context(); + } + + const SchemaType& CurrentSchema() const { return *schemaStack_.template Top()->schema; } + Context& CurrentContext() { return *schemaStack_.template Top(); } + const Context& CurrentContext() const { return *schemaStack_.template Top(); } + + static OutputHandler& GetNullHandler() + { static OutputHandler nullHandler; + return nullHandler; + } + + static const size_t kDefaultSchemaStackCapacity = 1024; + static const size_t kDefaultDocumentStackCapacity = 256; + const SchemaDocumentType* schemaDocument_; + const SchemaType& root_; + OutputHandler& outputHandler_; + StateAllocator* stateAllocator_; + StateAllocator* ownStateAllocator_; + internal::Stack schemaStack_; //!< stack to store the current path of schema (BaseSchemaType *) + internal::Stack documentStack_; //!< stack to store the current path of validating document (Ch) + bool valid_; #if RAPIDJSON_SCHEMA_VERBOSE - unsigned depth_; + unsigned depth_; #endif }; @@ -1945,59 +1949,60 @@ typedef GenericSchemaValidator SchemaValidator; \tparam StackAllocator Allocator type for stack. */ template < - unsigned parseFlags, - typename InputStream, - typename SourceEncoding, - typename SchemaDocumentType = SchemaDocument, - typename StackAllocator = CrtAllocator> -class SchemaValidatingReader { + unsigned parseFlags, + typename InputStream, + typename SourceEncoding, + typename SchemaDocumentType = SchemaDocument, + typename StackAllocator = CrtAllocator> +class SchemaValidatingReader +{ public: - typedef typename SchemaDocumentType::PointerType PointerType; - typedef typename InputStream::Ch Ch; - - //! Constructor - /*! - \param is Input stream. - \param sd Schema document. - */ - SchemaValidatingReader(InputStream& is, const SchemaDocumentType& sd) : is_(is), sd_(sd), invalidSchemaKeyword_(), isValid_(true) {} - - template - bool operator()(Handler& handler) { - GenericReader reader; - GenericSchemaValidator validator(sd_, handler); - parseResult_ = reader.template Parse(is_, validator); - - isValid_ = validator.IsValid(); - if (isValid_) { - invalidSchemaPointer_ = PointerType(); - invalidSchemaKeyword_ = 0; - invalidDocumentPointer_ = PointerType(); - } - else { - invalidSchemaPointer_ = validator.GetInvalidSchemaPointer(); - invalidSchemaKeyword_ = validator.GetInvalidSchemaKeyword(); - invalidDocumentPointer_ = validator.GetInvalidDocumentPointer(); - } - - return parseResult_; - } - - const ParseResult& GetParseResult() const { return parseResult_; } - bool IsValid() const { return isValid_; } - const PointerType& GetInvalidSchemaPointer() const { return invalidSchemaPointer_; } - const Ch* GetInvalidSchemaKeyword() const { return invalidSchemaKeyword_; } - const PointerType& GetInvalidDocumentPointer() const { return invalidDocumentPointer_; } + typedef typename SchemaDocumentType::PointerType PointerType; + typedef typename InputStream::Ch Ch; + + //! Constructor + /*! + \param is Input stream. + \param sd Schema document. + */ + SchemaValidatingReader(InputStream& is, const SchemaDocumentType& sd) : is_(is), sd_(sd), invalidSchemaKeyword_(), isValid_(true) {} + + template + bool operator()(Handler& handler) + { GenericReader reader; + GenericSchemaValidator validator(sd_, handler); + parseResult_ = reader.template Parse(is_, validator); + + isValid_ = validator.IsValid(); + if (isValid_) + { invalidSchemaPointer_ = PointerType(); + invalidSchemaKeyword_ = 0; + invalidDocumentPointer_ = PointerType(); + } + else + { invalidSchemaPointer_ = validator.GetInvalidSchemaPointer(); + invalidSchemaKeyword_ = validator.GetInvalidSchemaKeyword(); + invalidDocumentPointer_ = validator.GetInvalidDocumentPointer(); + } + + return parseResult_; + } + + const ParseResult& GetParseResult() const { return parseResult_; } + bool IsValid() const { return isValid_; } + const PointerType& GetInvalidSchemaPointer() const { return invalidSchemaPointer_; } + const Ch* GetInvalidSchemaKeyword() const { return invalidSchemaKeyword_; } + const PointerType& GetInvalidDocumentPointer() const { return invalidDocumentPointer_; } private: - InputStream& is_; - const SchemaDocumentType& sd_; - - ParseResult parseResult_; - PointerType invalidSchemaPointer_; - const Ch* invalidSchemaKeyword_; - PointerType invalidDocumentPointer_; - bool isValid_; + InputStream& is_; + const SchemaDocumentType& sd_; + + ParseResult parseResult_; + PointerType invalidSchemaPointer_; + const Ch* invalidSchemaKeyword_; + PointerType invalidDocumentPointer_; + bool isValid_; }; RAPIDJSON_NAMESPACE_END diff --git a/rapidjson/include/rapidjson/stream.h b/rapidjson/include/rapidjson/stream.h index fef82c252ff..b2e504d417b 100644 --- a/rapidjson/include/rapidjson/stream.h +++ b/rapidjson/include/rapidjson/stream.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "rapidjson.h" @@ -70,34 +70,34 @@ concept Stream { See TEST(Reader, CustomStringStream) in readertest.cpp for example. */ template -struct StreamTraits { - //! Whether to make local copy of stream for optimization during parsing. - /*! - By default, for safety, streams do not use local copy optimization. - Stream that can be copied fast should specialize this, like StreamTraits. - */ - enum { copyOptimization = 0 }; +struct StreamTraits +{ //! Whether to make local copy of stream for optimization during parsing. + /*! + By default, for safety, streams do not use local copy optimization. + Stream that can be copied fast should specialize this, like StreamTraits. + */ + enum { copyOptimization = 0 }; }; //! Reserve n characters for writing to a stream. template -inline void PutReserve(Stream& stream, size_t count) { - (void)stream; - (void)count; +inline void PutReserve(Stream& stream, size_t count) +{ (void)stream; + (void)count; } //! Write character to a stream, presuming buffer is reserved. template -inline void PutUnsafe(Stream& stream, typename Stream::Ch c) { - stream.Put(c); +inline void PutUnsafe(Stream& stream, typename Stream::Ch c) +{ stream.Put(c); } //! Put N copies of a character to a stream. template -inline void PutN(Stream& stream, Ch c, size_t n) { - PutReserve(stream, n); - for (size_t i = 0; i < n; i++) - PutUnsafe(stream, c); +inline void PutN(Stream& stream, Ch c, size_t n) +{ PutReserve(stream, n); + for (size_t i = 0; i < n; i++) + PutUnsafe(stream, c); } /////////////////////////////////////////////////////////////////////////////// @@ -107,27 +107,27 @@ inline void PutN(Stream& stream, Ch c, size_t n) { /*! \note implements Stream concept */ template -struct GenericStringStream { - typedef typename Encoding::Ch Ch; +struct GenericStringStream +{ typedef typename Encoding::Ch Ch; - GenericStringStream(const Ch *src) : src_(src), head_(src) {} + GenericStringStream(const Ch *src) : src_(src), head_(src) {} - Ch Peek() const { return *src_; } - Ch Take() { return *src_++; } - size_t Tell() const { return static_cast(src_ - head_); } + Ch Peek() const { return *src_; } + Ch Take() { return *src_++; } + size_t Tell() const { return static_cast(src_ - head_); } - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - const Ch* src_; //!< Current read position. - const Ch* head_; //!< Original head of the string. + const Ch* src_; //!< Current read position. + const Ch* head_; //!< Original head of the string. }; template -struct StreamTraits > { - enum { copyOptimization = 1 }; +struct StreamTraits > +{ enum { copyOptimization = 1 }; }; //! String stream with UTF8 encoding. @@ -141,34 +141,34 @@ typedef GenericStringStream > StringStream; \note implements Stream concept */ template -struct GenericInsituStringStream { - typedef typename Encoding::Ch Ch; +struct GenericInsituStringStream +{ typedef typename Encoding::Ch Ch; - GenericInsituStringStream(Ch *src) : src_(src), dst_(0), head_(src) {} + GenericInsituStringStream(Ch *src) : src_(src), dst_(0), head_(src) {} - // Read - Ch Peek() { return *src_; } - Ch Take() { return *src_++; } - size_t Tell() { return static_cast(src_ - head_); } + // Read + Ch Peek() { return *src_; } + Ch Take() { return *src_++; } + size_t Tell() { return static_cast(src_ - head_); } - // Write - void Put(Ch c) { RAPIDJSON_ASSERT(dst_ != 0); *dst_++ = c; } + // Write + void Put(Ch c) { RAPIDJSON_ASSERT(dst_ != 0); *dst_++ = c; } - Ch* PutBegin() { return dst_ = src_; } - size_t PutEnd(Ch* begin) { return static_cast(dst_ - begin); } - void Flush() {} + Ch* PutBegin() { return dst_ = src_; } + size_t PutEnd(Ch* begin) { return static_cast(dst_ - begin); } + void Flush() {} - Ch* Push(size_t count) { Ch* begin = dst_; dst_ += count; return begin; } - void Pop(size_t count) { dst_ -= count; } + Ch* Push(size_t count) { Ch* begin = dst_; dst_ += count; return begin; } + void Pop(size_t count) { dst_ -= count; } - Ch* src_; - Ch* dst_; - Ch* head_; + Ch* src_; + Ch* dst_; + Ch* head_; }; template -struct StreamTraits > { - enum { copyOptimization = 1 }; +struct StreamTraits > +{ enum { copyOptimization = 1 }; }; //! Insitu string stream with UTF8 encoding. diff --git a/rapidjson/include/rapidjson/stringbuffer.h b/rapidjson/include/rapidjson/stringbuffer.h index 78f34d2098e..0db887d97a0 100644 --- a/rapidjson/include/rapidjson/stringbuffer.h +++ b/rapidjson/include/rapidjson/stringbuffer.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_STRINGBUFFER_H_ @@ -38,74 +38,75 @@ RAPIDJSON_NAMESPACE_BEGIN \note implements Stream concept */ template -class GenericStringBuffer { +class GenericStringBuffer +{ public: - typedef typename Encoding::Ch Ch; + typedef typename Encoding::Ch Ch; - GenericStringBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} + GenericStringBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} #if RAPIDJSON_HAS_CXX11_RVALUE_REFS - GenericStringBuffer(GenericStringBuffer&& rhs) : stack_(std::move(rhs.stack_)) {} - GenericStringBuffer& operator=(GenericStringBuffer&& rhs) { - if (&rhs != this) - stack_ = std::move(rhs.stack_); - return *this; - } + GenericStringBuffer(GenericStringBuffer&& rhs) : stack_(std::move(rhs.stack_)) {} + GenericStringBuffer& operator=(GenericStringBuffer&& rhs) + { if (&rhs != this) + stack_ = std::move(rhs.stack_); + return *this; + } #endif - void Put(Ch c) { *stack_.template Push() = c; } - void PutUnsafe(Ch c) { *stack_.template PushUnsafe() = c; } - void Flush() {} + void Put(Ch c) { *stack_.template Push() = c; } + void PutUnsafe(Ch c) { *stack_.template PushUnsafe() = c; } + void Flush() {} - void Clear() { stack_.Clear(); } - void ShrinkToFit() { - // Push and pop a null terminator. This is safe. - *stack_.template Push() = '\0'; - stack_.ShrinkToFit(); - stack_.template Pop(1); - } + void Clear() { stack_.Clear(); } + void ShrinkToFit() + { // Push and pop a null terminator. This is safe. + *stack_.template Push() = '\0'; + stack_.ShrinkToFit(); + stack_.template Pop(1); + } - void Reserve(size_t count) { stack_.template Reserve(count); } - Ch* Push(size_t count) { return stack_.template Push(count); } - Ch* PushUnsafe(size_t count) { return stack_.template PushUnsafe(count); } - void Pop(size_t count) { stack_.template Pop(count); } + void Reserve(size_t count) { stack_.template Reserve(count); } + Ch* Push(size_t count) { return stack_.template Push(count); } + Ch* PushUnsafe(size_t count) { return stack_.template PushUnsafe(count); } + void Pop(size_t count) { stack_.template Pop(count); } - const Ch* GetString() const { - // Push and pop a null terminator. This is safe. - *stack_.template Push() = '\0'; - stack_.template Pop(1); + const Ch* GetString() const + { // Push and pop a null terminator. This is safe. + *stack_.template Push() = '\0'; + stack_.template Pop(1); - return stack_.template Bottom(); - } + return stack_.template Bottom(); + } - size_t GetSize() const { return stack_.GetSize(); } + size_t GetSize() const { return stack_.GetSize(); } - static const size_t kDefaultCapacity = 256; - mutable internal::Stack stack_; + static const size_t kDefaultCapacity = 256; + mutable internal::Stack stack_; private: - // Prohibit copy constructor & assignment operator. - GenericStringBuffer(const GenericStringBuffer&); - GenericStringBuffer& operator=(const GenericStringBuffer&); + // Prohibit copy constructor & assignment operator. + GenericStringBuffer(const GenericStringBuffer&); + GenericStringBuffer& operator=(const GenericStringBuffer&); }; //! String buffer with UTF8 encoding typedef GenericStringBuffer > StringBuffer; template -inline void PutReserve(GenericStringBuffer& stream, size_t count) { - stream.Reserve(count); +inline void PutReserve(GenericStringBuffer& stream, size_t count) +{ stream.Reserve(count); } template -inline void PutUnsafe(GenericStringBuffer& stream, typename Encoding::Ch c) { - stream.PutUnsafe(c); +inline void PutUnsafe(GenericStringBuffer& stream, typename Encoding::Ch c) +{ stream.PutUnsafe(c); } //! Implement specialized version of PutN() with memset() for better performance. template<> -inline void PutN(GenericStringBuffer >& stream, char c, size_t n) { - std::memset(stream.stack_.Push(n), c, n * sizeof(c)); +inline void PutN(GenericStringBuffer >& stream, char c, size_t n) +{ std::memset(stream.stack_.Push(n), c, n * sizeof(c)); } RAPIDJSON_NAMESPACE_END diff --git a/rapidjson/include/rapidjson/writer.h b/rapidjson/include/rapidjson/writer.h index 94f22dd5fce..cf9efbba8ff 100644 --- a/rapidjson/include/rapidjson/writer.h +++ b/rapidjson/include/rapidjson/writer.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_WRITER_H_ @@ -49,7 +49,7 @@ RAPIDJSON_NAMESPACE_BEGIN /////////////////////////////////////////////////////////////////////////////// // WriteFlag -/*! \def RAPIDJSON_WRITE_DEFAULT_FLAGS +/*! \def RAPIDJSON_WRITE_DEFAULT_FLAGS \ingroup RAPIDJSON_CONFIG \brief User-defined kWriteDefaultFlags definition. @@ -60,11 +60,11 @@ RAPIDJSON_NAMESPACE_BEGIN #endif //! Combination of writeFlags -enum WriteFlag { - kWriteNoFlags = 0, //!< No flags are set. - kWriteValidateEncodingFlag = 1, //!< Validate encoding of JSON strings. - kWriteNanAndInfFlag = 2, //!< Allow writing of Infinity, -Infinity and NaN. - kWriteDefaultFlags = RAPIDJSON_WRITE_DEFAULT_FLAGS //!< Default write flags. Can be customized by defining RAPIDJSON_WRITE_DEFAULT_FLAGS +enum WriteFlag +{ kWriteNoFlags = 0, //!< No flags are set. + kWriteValidateEncodingFlag = 1, //!< Validate encoding of JSON strings. + kWriteNanAndInfFlag = 2, //!< Allow writing of Infinity, -Infinity and NaN. + kWriteDefaultFlags = RAPIDJSON_WRITE_DEFAULT_FLAGS //!< Default write flags. Can be customized by defining RAPIDJSON_WRITE_DEFAULT_FLAGS }; //! JSON writer @@ -73,7 +73,7 @@ enum WriteFlag { User may programmatically calls the functions of a writer to generate JSON text. - On the other side, a writer can also be passed to objects that generates events, + On the other side, a writer can also be passed to objects that generates events, for example Reader::Parse() and Document::Accept(). @@ -84,516 +84,518 @@ enum WriteFlag { \note implements Handler concept */ template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> -class Writer { +class Writer +{ public: - typedef typename SourceEncoding::Ch Ch; - - static const int kDefaultMaxDecimalPlaces = 324; - - //! Constructor - /*! \param os Output stream. - \param stackAllocator User supplied allocator. If it is null, it will create a private one. - \param levelDepth Initial capacity of stack. - */ - explicit - Writer(OutputStream& os, StackAllocator* stackAllocator = 0, size_t levelDepth = kDefaultLevelDepth) : - os_(&os), level_stack_(stackAllocator, levelDepth * sizeof(Level)), maxDecimalPlaces_(kDefaultMaxDecimalPlaces), hasRoot_(false) {} - - explicit - Writer(StackAllocator* allocator = 0, size_t levelDepth = kDefaultLevelDepth) : - os_(0), level_stack_(allocator, levelDepth * sizeof(Level)), maxDecimalPlaces_(kDefaultMaxDecimalPlaces), hasRoot_(false) {} - - //! Reset the writer with a new stream. - /*! - This function reset the writer with a new stream and default settings, - in order to make a Writer object reusable for output multiple JSONs. - - \param os New output stream. - \code - Writer writer(os1); - writer.StartObject(); - // ... - writer.EndObject(); - - writer.Reset(os2); - writer.StartObject(); - // ... - writer.EndObject(); - \endcode - */ - void Reset(OutputStream& os) { - os_ = &os; - hasRoot_ = false; - level_stack_.Clear(); - } - - //! Checks whether the output is a complete JSON. - /*! - A complete JSON has a complete root object or array. - */ - bool IsComplete() const { - return hasRoot_ && level_stack_.Empty(); - } - - int GetMaxDecimalPlaces() const { - return maxDecimalPlaces_; - } - - //! Sets the maximum number of decimal places for double output. - /*! - This setting truncates the output with specified number of decimal places. - - For example, - - \code - writer.SetMaxDecimalPlaces(3); - writer.StartArray(); - writer.Double(0.12345); // "0.123" - writer.Double(0.0001); // "0.0" - writer.Double(1.234567890123456e30); // "1.234567890123456e30" (do not truncate significand for positive exponent) - writer.Double(1.23e-4); // "0.0" (do truncate significand for negative exponent) - writer.EndArray(); - \endcode - - The default setting does not truncate any decimal places. You can restore to this setting by calling - \code - writer.SetMaxDecimalPlaces(Writer::kDefaultMaxDecimalPlaces); - \endcode - */ - void SetMaxDecimalPlaces(int maxDecimalPlaces) { - maxDecimalPlaces_ = maxDecimalPlaces; - } - - /*!@name Implementation of Handler - \see Handler - */ - //@{ - - bool Null() { Prefix(kNullType); return EndValue(WriteNull()); } - bool Bool(bool b) { Prefix(b ? kTrueType : kFalseType); return EndValue(WriteBool(b)); } - bool Int(int i) { Prefix(kNumberType); return EndValue(WriteInt(i)); } - bool Uint(unsigned u) { Prefix(kNumberType); return EndValue(WriteUint(u)); } - bool Int64(int64_t i64) { Prefix(kNumberType); return EndValue(WriteInt64(i64)); } - bool Uint64(uint64_t u64) { Prefix(kNumberType); return EndValue(WriteUint64(u64)); } - - //! Writes the given \c double value to the stream - /*! - \param d The value to be written. - \return Whether it is succeed. - */ - bool Double(double d) { Prefix(kNumberType); return EndValue(WriteDouble(d)); } - - bool RawNumber(const Ch* str, SizeType length, bool copy = false) { - (void)copy; - Prefix(kNumberType); - return EndValue(WriteString(str, length)); - } - - bool String(const Ch* str, SizeType length, bool copy = false) { - (void)copy; - Prefix(kStringType); - return EndValue(WriteString(str, length)); - } + typedef typename SourceEncoding::Ch Ch; + + static const int kDefaultMaxDecimalPlaces = 324; + + //! Constructor + /*! \param os Output stream. + \param stackAllocator User supplied allocator. If it is null, it will create a private one. + \param levelDepth Initial capacity of stack. + */ + explicit + Writer(OutputStream& os, StackAllocator* stackAllocator = 0, size_t levelDepth = kDefaultLevelDepth) : + os_(&os), level_stack_(stackAllocator, levelDepth * sizeof(Level)), maxDecimalPlaces_(kDefaultMaxDecimalPlaces), hasRoot_(false) {} + + explicit + Writer(StackAllocator* allocator = 0, size_t levelDepth = kDefaultLevelDepth) : + os_(0), level_stack_(allocator, levelDepth * sizeof(Level)), maxDecimalPlaces_(kDefaultMaxDecimalPlaces), hasRoot_(false) {} + + //! Reset the writer with a new stream. + /*! + This function reset the writer with a new stream and default settings, + in order to make a Writer object reusable for output multiple JSONs. + + \param os New output stream. + \code + Writer writer(os1); + writer.StartObject(); + // ... + writer.EndObject(); + + writer.Reset(os2); + writer.StartObject(); + // ... + writer.EndObject(); + \endcode + */ + void Reset(OutputStream& os) + { os_ = &os; + hasRoot_ = false; + level_stack_.Clear(); + } + + //! Checks whether the output is a complete JSON. + /*! + A complete JSON has a complete root object or array. + */ + bool IsComplete() const + { return hasRoot_ && level_stack_.Empty(); + } + + int GetMaxDecimalPlaces() const + { return maxDecimalPlaces_; + } + + //! Sets the maximum number of decimal places for double output. + /*! + This setting truncates the output with specified number of decimal places. + + For example, + + \code + writer.SetMaxDecimalPlaces(3); + writer.StartArray(); + writer.Double(0.12345); // "0.123" + writer.Double(0.0001); // "0.0" + writer.Double(1.234567890123456e30); // "1.234567890123456e30" (do not truncate significand for positive exponent) + writer.Double(1.23e-4); // "0.0" (do truncate significand for negative exponent) + writer.EndArray(); + \endcode + + The default setting does not truncate any decimal places. You can restore to this setting by calling + \code + writer.SetMaxDecimalPlaces(Writer::kDefaultMaxDecimalPlaces); + \endcode + */ + void SetMaxDecimalPlaces(int maxDecimalPlaces) + { maxDecimalPlaces_ = maxDecimalPlaces; + } + + /*!@name Implementation of Handler + \see Handler + */ + //@{ + + bool Null() { Prefix(kNullType); return EndValue(WriteNull()); } + bool Bool(bool b) { Prefix(b ? kTrueType : kFalseType); return EndValue(WriteBool(b)); } + bool Int(int i) { Prefix(kNumberType); return EndValue(WriteInt(i)); } + bool Uint(unsigned u) { Prefix(kNumberType); return EndValue(WriteUint(u)); } + bool Int64(int64_t i64) { Prefix(kNumberType); return EndValue(WriteInt64(i64)); } + bool Uint64(uint64_t u64) { Prefix(kNumberType); return EndValue(WriteUint64(u64)); } + + //! Writes the given \c double value to the stream + /*! + \param d The value to be written. + \return Whether it is succeed. + */ + bool Double(double d) { Prefix(kNumberType); return EndValue(WriteDouble(d)); } + + bool RawNumber(const Ch* str, SizeType length, bool copy = false) + { (void)copy; + Prefix(kNumberType); + return EndValue(WriteString(str, length)); + } + + bool String(const Ch* str, SizeType length, bool copy = false) + { (void)copy; + Prefix(kStringType); + return EndValue(WriteString(str, length)); + } #if RAPIDJSON_HAS_STDSTRING - bool String(const std::basic_string& str) { - return String(str.data(), SizeType(str.size())); - } + bool String(const std::basic_string& str) + { return String(str.data(), SizeType(str.size())); + } #endif - bool StartObject() { - Prefix(kObjectType); - new (level_stack_.template Push()) Level(false); - return WriteStartObject(); - } - - bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } - - bool EndObject(SizeType memberCount = 0) { - (void)memberCount; - RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); - RAPIDJSON_ASSERT(!level_stack_.template Top()->inArray); - level_stack_.template Pop(1); - return EndValue(WriteEndObject()); - } - - bool StartArray() { - Prefix(kArrayType); - new (level_stack_.template Push()) Level(true); - return WriteStartArray(); - } - - bool EndArray(SizeType elementCount = 0) { - (void)elementCount; - RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); - RAPIDJSON_ASSERT(level_stack_.template Top()->inArray); - level_stack_.template Pop(1); - return EndValue(WriteEndArray()); - } - //@} - - /*! @name Convenience extensions */ - //@{ - - //! Simpler but slower overload. - bool String(const Ch* str) { return String(str, internal::StrLen(str)); } - bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } - - //@} - - //! Write a raw JSON value. - /*! - For user to write a stringified JSON as a value. - - \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. - \param length Length of the json. - \param type Type of the root of json. - */ - bool RawValue(const Ch* json, size_t length, Type type) { Prefix(type); return EndValue(WriteRawValue(json, length)); } + bool StartObject() + { Prefix(kObjectType); + new (level_stack_.template Push()) Level(false); + return WriteStartObject(); + } + + bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } + + bool EndObject(SizeType memberCount = 0) + { (void)memberCount; + RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); + RAPIDJSON_ASSERT(!level_stack_.template Top()->inArray); + level_stack_.template Pop(1); + return EndValue(WriteEndObject()); + } + + bool StartArray() + { Prefix(kArrayType); + new (level_stack_.template Push()) Level(true); + return WriteStartArray(); + } + + bool EndArray(SizeType elementCount = 0) + { (void)elementCount; + RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); + RAPIDJSON_ASSERT(level_stack_.template Top()->inArray); + level_stack_.template Pop(1); + return EndValue(WriteEndArray()); + } + //@} + + /*! @name Convenience extensions */ + //@{ + + //! Simpler but slower overload. + bool String(const Ch* str) { return String(str, internal::StrLen(str)); } + bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } + + //@} + + //! Write a raw JSON value. + /*! + For user to write a stringified JSON as a value. + + \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. + \param length Length of the json. + \param type Type of the root of json. + */ + bool RawValue(const Ch* json, size_t length, Type type) { Prefix(type); return EndValue(WriteRawValue(json, length)); } protected: - //! Information for each nested level - struct Level { - Level(bool inArray_) : valueCount(0), inArray(inArray_) {} - size_t valueCount; //!< number of values in this level - bool inArray; //!< true if in array, otherwise in object - }; - - static const size_t kDefaultLevelDepth = 32; - - bool WriteNull() { - PutReserve(*os_, 4); - PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 'l'); return true; + //! Information for each nested level + struct Level +{ Level(bool inArray_) : valueCount(0), inArray(inArray_) {} + size_t valueCount; //!< number of values in this level + bool inArray; //!< true if in array, otherwise in object + }; + + static const size_t kDefaultLevelDepth = 32; + + bool WriteNull() + { PutReserve(*os_, 4); + PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 'l'); return true; + } + + bool WriteBool(bool b) + { if (b) + { PutReserve(*os_, 4); + PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'r'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'e'); } - - bool WriteBool(bool b) { - if (b) { - PutReserve(*os_, 4); - PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'r'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'e'); - } - else { - PutReserve(*os_, 5); - PutUnsafe(*os_, 'f'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 's'); PutUnsafe(*os_, 'e'); - } - return true; + else + { PutReserve(*os_, 5); + PutUnsafe(*os_, 'f'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 's'); PutUnsafe(*os_, 'e'); } + return true; + } - bool WriteInt(int i) { - char buffer[11]; - const char* end = internal::i32toa(i, buffer); - PutReserve(*os_, static_cast(end - buffer)); - for (const char* p = buffer; p != end; ++p) - PutUnsafe(*os_, static_cast(*p)); - return true; - } + bool WriteInt(int i) + { char buffer[11]; + const char* end = internal::i32toa(i, buffer); + PutReserve(*os_, static_cast(end - buffer)); + for (const char* p = buffer; p != end; ++p) + PutUnsafe(*os_, static_cast(*p)); + return true; + } - bool WriteUint(unsigned u) { - char buffer[10]; - const char* end = internal::u32toa(u, buffer); - PutReserve(*os_, static_cast(end - buffer)); - for (const char* p = buffer; p != end; ++p) - PutUnsafe(*os_, static_cast(*p)); - return true; - } + bool WriteUint(unsigned u) + { char buffer[10]; + const char* end = internal::u32toa(u, buffer); + PutReserve(*os_, static_cast(end - buffer)); + for (const char* p = buffer; p != end; ++p) + PutUnsafe(*os_, static_cast(*p)); + return true; + } - bool WriteInt64(int64_t i64) { - char buffer[21]; - const char* end = internal::i64toa(i64, buffer); - PutReserve(*os_, static_cast(end - buffer)); - for (const char* p = buffer; p != end; ++p) - PutUnsafe(*os_, static_cast(*p)); - return true; - } + bool WriteInt64(int64_t i64) + { char buffer[21]; + const char* end = internal::i64toa(i64, buffer); + PutReserve(*os_, static_cast(end - buffer)); + for (const char* p = buffer; p != end; ++p) + PutUnsafe(*os_, static_cast(*p)); + return true; + } + + bool WriteUint64(uint64_t u64) + { char buffer[20]; + char* end = internal::u64toa(u64, buffer); + PutReserve(*os_, static_cast(end - buffer)); + for (char* p = buffer; p != end; ++p) + PutUnsafe(*os_, static_cast(*p)); + return true; + } - bool WriteUint64(uint64_t u64) { - char buffer[20]; - char* end = internal::u64toa(u64, buffer); - PutReserve(*os_, static_cast(end - buffer)); - for (char* p = buffer; p != end; ++p) - PutUnsafe(*os_, static_cast(*p)); + bool WriteDouble(double d) + { if (internal::Double(d).IsNanOrInf()) + { if (!(writeFlags & kWriteNanAndInfFlag)) + return false; + if (internal::Double(d).IsNan()) + { PutReserve(*os_, 3); + PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); return true; + } + if (internal::Double(d).Sign()) + { PutReserve(*os_, 9); + PutUnsafe(*os_, '-'); + } + else + PutReserve(*os_, 8); + PutUnsafe(*os_, 'I'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'f'); + PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'y'); + return true; } - bool WriteDouble(double d) { - if (internal::Double(d).IsNanOrInf()) { - if (!(writeFlags & kWriteNanAndInfFlag)) - return false; - if (internal::Double(d).IsNan()) { - PutReserve(*os_, 3); - PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); - return true; - } - if (internal::Double(d).Sign()) { - PutReserve(*os_, 9); - PutUnsafe(*os_, '-'); - } - else - PutReserve(*os_, 8); - PutUnsafe(*os_, 'I'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'f'); - PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'y'); - return true; - } - - char buffer[25]; - char* end = internal::dtoa(d, buffer, maxDecimalPlaces_); - PutReserve(*os_, static_cast(end - buffer)); - for (char* p = buffer; p != end; ++p) - PutUnsafe(*os_, static_cast(*p)); - return true; - } + char buffer[25]; + char* end = internal::dtoa(d, buffer, maxDecimalPlaces_); + PutReserve(*os_, static_cast(end - buffer)); + for (char* p = buffer; p != end; ++p) + PutUnsafe(*os_, static_cast(*p)); + return true; + } - bool WriteString(const Ch* str, SizeType length) { - static const typename TargetEncoding::Ch hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; - static const char escape[256] = { + bool WriteString(const Ch* str, SizeType length) + { static const typename TargetEncoding::Ch hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; + static const char escape[256] = + { #define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 - //0 1 2 3 4 5 6 7 8 9 A B C D E F - 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'b', 't', 'n', 'u', 'f', 'r', 'u', 'u', // 00 - 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', // 10 - 0, 0, '"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20 - Z16, Z16, // 30~4F - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, // 50 - Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 // 60~FF + //0 1 2 3 4 5 6 7 8 9 A B C D E F + 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'b', 't', 'n', 'u', 'f', 'r', 'u', 'u', // 00 + 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', // 10 + 0, 0, '"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20 + Z16, Z16, // 30~4F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, // 50 + Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 // 60~FF #undef Z16 - }; + }; - if (TargetEncoding::supportUnicode) - PutReserve(*os_, 2 + length * 6); // "\uxxxx..." + if (TargetEncoding::supportUnicode) + PutReserve(*os_, 2 + length * 6); // "\uxxxx..." + else + PutReserve(*os_, 2 + length * 12); // "\uxxxx\uyyyy..." + + PutUnsafe(*os_, '\"'); + GenericStringStream is(str); + while (ScanWriteUnescapedString(is, length)) + { const Ch c = is.Peek(); + if (!TargetEncoding::supportUnicode && static_cast(c) >= 0x80) + { // Unicode escaping + unsigned codepoint; + if (RAPIDJSON_UNLIKELY(!SourceEncoding::Decode(is, &codepoint))) + return false; + PutUnsafe(*os_, '\\'); + PutUnsafe(*os_, 'u'); + if (codepoint <= 0xD7FF || (codepoint >= 0xE000 && codepoint <= 0xFFFF)) + { PutUnsafe(*os_, hexDigits[(codepoint >> 12) & 15]); + PutUnsafe(*os_, hexDigits[(codepoint >> 8) & 15]); + PutUnsafe(*os_, hexDigits[(codepoint >> 4) & 15]); + PutUnsafe(*os_, hexDigits[(codepoint ) & 15]); + } else - PutReserve(*os_, 2 + length * 12); // "\uxxxx\uyyyy..." - - PutUnsafe(*os_, '\"'); - GenericStringStream is(str); - while (ScanWriteUnescapedString(is, length)) { - const Ch c = is.Peek(); - if (!TargetEncoding::supportUnicode && static_cast(c) >= 0x80) { - // Unicode escaping - unsigned codepoint; - if (RAPIDJSON_UNLIKELY(!SourceEncoding::Decode(is, &codepoint))) - return false; - PutUnsafe(*os_, '\\'); - PutUnsafe(*os_, 'u'); - if (codepoint <= 0xD7FF || (codepoint >= 0xE000 && codepoint <= 0xFFFF)) { - PutUnsafe(*os_, hexDigits[(codepoint >> 12) & 15]); - PutUnsafe(*os_, hexDigits[(codepoint >> 8) & 15]); - PutUnsafe(*os_, hexDigits[(codepoint >> 4) & 15]); - PutUnsafe(*os_, hexDigits[(codepoint ) & 15]); - } - else { - RAPIDJSON_ASSERT(codepoint >= 0x010000 && codepoint <= 0x10FFFF); - // Surrogate pair - unsigned s = codepoint - 0x010000; - unsigned lead = (s >> 10) + 0xD800; - unsigned trail = (s & 0x3FF) + 0xDC00; - PutUnsafe(*os_, hexDigits[(lead >> 12) & 15]); - PutUnsafe(*os_, hexDigits[(lead >> 8) & 15]); - PutUnsafe(*os_, hexDigits[(lead >> 4) & 15]); - PutUnsafe(*os_, hexDigits[(lead ) & 15]); - PutUnsafe(*os_, '\\'); - PutUnsafe(*os_, 'u'); - PutUnsafe(*os_, hexDigits[(trail >> 12) & 15]); - PutUnsafe(*os_, hexDigits[(trail >> 8) & 15]); - PutUnsafe(*os_, hexDigits[(trail >> 4) & 15]); - PutUnsafe(*os_, hexDigits[(trail ) & 15]); - } - } - else if ((sizeof(Ch) == 1 || static_cast(c) < 256) && RAPIDJSON_UNLIKELY(escape[static_cast(c)])) { - is.Take(); - PutUnsafe(*os_, '\\'); - PutUnsafe(*os_, static_cast(escape[static_cast(c)])); - if (escape[static_cast(c)] == 'u') { - PutUnsafe(*os_, '0'); - PutUnsafe(*os_, '0'); - PutUnsafe(*os_, hexDigits[static_cast(c) >> 4]); - PutUnsafe(*os_, hexDigits[static_cast(c) & 0xF]); - } - } - else if (RAPIDJSON_UNLIKELY(!(writeFlags & kWriteValidateEncodingFlag ? - Transcoder::Validate(is, *os_) : - Transcoder::TranscodeUnsafe(is, *os_)))) - return false; + { RAPIDJSON_ASSERT(codepoint >= 0x010000 && codepoint <= 0x10FFFF); + // Surrogate pair + unsigned s = codepoint - 0x010000; + unsigned lead = (s >> 10) + 0xD800; + unsigned trail = (s & 0x3FF) + 0xDC00; + PutUnsafe(*os_, hexDigits[(lead >> 12) & 15]); + PutUnsafe(*os_, hexDigits[(lead >> 8) & 15]); + PutUnsafe(*os_, hexDigits[(lead >> 4) & 15]); + PutUnsafe(*os_, hexDigits[(lead ) & 15]); + PutUnsafe(*os_, '\\'); + PutUnsafe(*os_, 'u'); + PutUnsafe(*os_, hexDigits[(trail >> 12) & 15]); + PutUnsafe(*os_, hexDigits[(trail >> 8) & 15]); + PutUnsafe(*os_, hexDigits[(trail >> 4) & 15]); + PutUnsafe(*os_, hexDigits[(trail ) & 15]); } - PutUnsafe(*os_, '\"'); - return true; + } + else if ((sizeof(Ch) == 1 || static_cast(c) < 256) && RAPIDJSON_UNLIKELY(escape[static_cast(c)])) + { is.Take(); + PutUnsafe(*os_, '\\'); + PutUnsafe(*os_, static_cast(escape[static_cast(c)])); + if (escape[static_cast(c)] == 'u') + { PutUnsafe(*os_, '0'); + PutUnsafe(*os_, '0'); + PutUnsafe(*os_, hexDigits[static_cast(c) >> 4]); + PutUnsafe(*os_, hexDigits[static_cast(c) & 0xF]); + } + } + else if (RAPIDJSON_UNLIKELY(!(writeFlags & kWriteValidateEncodingFlag ? + Transcoder::Validate(is, *os_) : + Transcoder::TranscodeUnsafe(is, *os_)))) + return false; } - - bool ScanWriteUnescapedString(GenericStringStream& is, size_t length) { - return RAPIDJSON_LIKELY(is.Tell() < length); + PutUnsafe(*os_, '\"'); + return true; + } + + bool ScanWriteUnescapedString(GenericStringStream& is, size_t length) + { return RAPIDJSON_LIKELY(is.Tell() < length); + } + + bool WriteStartObject() { os_->Put('{'); return true; } + bool WriteEndObject() { os_->Put('}'); return true; } + bool WriteStartArray() { os_->Put('['); return true; } + bool WriteEndArray() { os_->Put(']'); return true; } + + bool WriteRawValue(const Ch* json, size_t length) + { PutReserve(*os_, length); + for (size_t i = 0; i < length; i++) + { RAPIDJSON_ASSERT(json[i] != '\0'); + PutUnsafe(*os_, json[i]); } - - bool WriteStartObject() { os_->Put('{'); return true; } - bool WriteEndObject() { os_->Put('}'); return true; } - bool WriteStartArray() { os_->Put('['); return true; } - bool WriteEndArray() { os_->Put(']'); return true; } - - bool WriteRawValue(const Ch* json, size_t length) { - PutReserve(*os_, length); - for (size_t i = 0; i < length; i++) { - RAPIDJSON_ASSERT(json[i] != '\0'); - PutUnsafe(*os_, json[i]); - } - return true; + return true; + } + + void Prefix(Type type) + { (void)type; + if (RAPIDJSON_LIKELY(level_stack_.GetSize() != 0)) // this value is not at root + { Level* level = level_stack_.template Top(); + if (level->valueCount > 0) + { if (level->inArray) + os_->Put(','); // add comma if it is not the first element in array + else // in object + os_->Put((level->valueCount % 2 == 0) ? ',' : ':'); + } + if (!level->inArray && level->valueCount % 2 == 0) + RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name + level->valueCount++; } - - void Prefix(Type type) { - (void)type; - if (RAPIDJSON_LIKELY(level_stack_.GetSize() != 0)) { // this value is not at root - Level* level = level_stack_.template Top(); - if (level->valueCount > 0) { - if (level->inArray) - os_->Put(','); // add comma if it is not the first element in array - else // in object - os_->Put((level->valueCount % 2 == 0) ? ',' : ':'); - } - if (!level->inArray && level->valueCount % 2 == 0) - RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name - level->valueCount++; - } - else { - RAPIDJSON_ASSERT(!hasRoot_); // Should only has one and only one root. - hasRoot_ = true; - } + else + { RAPIDJSON_ASSERT(!hasRoot_); // Should only has one and only one root. + hasRoot_ = true; } + } - // Flush the value if it is the top level one. - bool EndValue(bool ret) { - if (RAPIDJSON_UNLIKELY(level_stack_.Empty())) // end of json text - os_->Flush(); - return ret; - } + // Flush the value if it is the top level one. + bool EndValue(bool ret) + { if (RAPIDJSON_UNLIKELY(level_stack_.Empty())) // end of json text + os_->Flush(); + return ret; + } - OutputStream* os_; - internal::Stack level_stack_; - int maxDecimalPlaces_; - bool hasRoot_; + OutputStream* os_; + internal::Stack level_stack_; + int maxDecimalPlaces_; + bool hasRoot_; private: - // Prohibit copy constructor & assignment operator. - Writer(const Writer&); - Writer& operator=(const Writer&); + // Prohibit copy constructor & assignment operator. + Writer(const Writer&); + Writer& operator=(const Writer&); }; // Full specialization for StringStream to prevent memory copying template<> -inline bool Writer::WriteInt(int i) { - char *buffer = os_->Push(11); - const char* end = internal::i32toa(i, buffer); - os_->Pop(static_cast(11 - (end - buffer))); - return true; +inline bool Writer::WriteInt(int i) +{ char *buffer = os_->Push(11); + const char* end = internal::i32toa(i, buffer); + os_->Pop(static_cast(11 - (end - buffer))); + return true; } template<> -inline bool Writer::WriteUint(unsigned u) { - char *buffer = os_->Push(10); - const char* end = internal::u32toa(u, buffer); - os_->Pop(static_cast(10 - (end - buffer))); - return true; +inline bool Writer::WriteUint(unsigned u) +{ char *buffer = os_->Push(10); + const char* end = internal::u32toa(u, buffer); + os_->Pop(static_cast(10 - (end - buffer))); + return true; } template<> -inline bool Writer::WriteInt64(int64_t i64) { - char *buffer = os_->Push(21); - const char* end = internal::i64toa(i64, buffer); - os_->Pop(static_cast(21 - (end - buffer))); - return true; +inline bool Writer::WriteInt64(int64_t i64) +{ char *buffer = os_->Push(21); + const char* end = internal::i64toa(i64, buffer); + os_->Pop(static_cast(21 - (end - buffer))); + return true; } template<> -inline bool Writer::WriteUint64(uint64_t u) { - char *buffer = os_->Push(20); - const char* end = internal::u64toa(u, buffer); - os_->Pop(static_cast(20 - (end - buffer))); - return true; +inline bool Writer::WriteUint64(uint64_t u) +{ char *buffer = os_->Push(20); + const char* end = internal::u64toa(u, buffer); + os_->Pop(static_cast(20 - (end - buffer))); + return true; } template<> -inline bool Writer::WriteDouble(double d) { - if (internal::Double(d).IsNanOrInf()) { - // Note: This code path can only be reached if (RAPIDJSON_WRITE_DEFAULT_FLAGS & kWriteNanAndInfFlag). - if (!(kWriteDefaultFlags & kWriteNanAndInfFlag)) - return false; - if (internal::Double(d).IsNan()) { - PutReserve(*os_, 3); - PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); - return true; - } - if (internal::Double(d).Sign()) { - PutReserve(*os_, 9); - PutUnsafe(*os_, '-'); - } - else - PutReserve(*os_, 8); - PutUnsafe(*os_, 'I'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'f'); - PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'y'); - return true; +inline bool Writer::WriteDouble(double d) +{ if (internal::Double(d).IsNanOrInf()) + { // Note: This code path can only be reached if (RAPIDJSON_WRITE_DEFAULT_FLAGS & kWriteNanAndInfFlag). + if (!(kWriteDefaultFlags & kWriteNanAndInfFlag)) + return false; + if (internal::Double(d).IsNan()) + { PutReserve(*os_, 3); + PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); + return true; } - - char *buffer = os_->Push(25); - char* end = internal::dtoa(d, buffer, maxDecimalPlaces_); - os_->Pop(static_cast(25 - (end - buffer))); + if (internal::Double(d).Sign()) + { PutReserve(*os_, 9); + PutUnsafe(*os_, '-'); + } + else + PutReserve(*os_, 8); + PutUnsafe(*os_, 'I'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'f'); + PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'y'); return true; + } + + char *buffer = os_->Push(25); + char* end = internal::dtoa(d, buffer, maxDecimalPlaces_); + os_->Pop(static_cast(25 - (end - buffer))); + return true; } #if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) template<> -inline bool Writer::ScanWriteUnescapedString(StringStream& is, size_t length) { - if (length < 16) - return RAPIDJSON_LIKELY(is.Tell() < length); +inline bool Writer::ScanWriteUnescapedString(StringStream& is, size_t length) +{ if (length < 16) + return RAPIDJSON_LIKELY(is.Tell() < length); - if (!RAPIDJSON_LIKELY(is.Tell() < length)) - return false; + if (!RAPIDJSON_LIKELY(is.Tell() < length)) + return false; - const char* p = is.src_; - const char* end = is.head_ + length; - const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); - const char* endAligned = reinterpret_cast(reinterpret_cast(end) & static_cast(~15)); - if (nextAligned > end) - return true; + const char* p = is.src_; + const char* end = is.head_ + length; + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + const char* endAligned = reinterpret_cast(reinterpret_cast(end) & static_cast(~15)); + if (nextAligned > end) + return true; - while (p != nextAligned) - if (*p < 0x20 || *p == '\"' || *p == '\\') { - is.src_ = p; - return RAPIDJSON_LIKELY(is.Tell() < length); - } - else - os_->PutUnsafe(*p++); - - // The rest of string using SIMD - static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; - static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; - static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; - const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); - const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); - const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); - - for (; p != endAligned; p += 16) { - const __m128i s = _mm_load_si128(reinterpret_cast(p)); - const __m128i t1 = _mm_cmpeq_epi8(s, dq); - const __m128i t2 = _mm_cmpeq_epi8(s, bs); - const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 - const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); - unsigned short r = static_cast(_mm_movemask_epi8(x)); - if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped - SizeType len; + while (p != nextAligned) + if (*p < 0x20 || *p == '\"' || *p == '\\') + { is.src_ = p; + return RAPIDJSON_LIKELY(is.Tell() < length); + } + else + os_->PutUnsafe(*p++); + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (; p != endAligned; p += 16) + { const __m128i s = _mm_load_si128(reinterpret_cast(p)); + const __m128i t1 = _mm_cmpeq_epi8(s, dq); + const __m128i t2 = _mm_cmpeq_epi8(s, bs); + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 + const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (RAPIDJSON_UNLIKELY(r != 0)) // some of characters is escaped + { SizeType len; #ifdef _MSC_VER // Find the index of first escaped - unsigned long offset; - _BitScanForward(&offset, r); - len = offset; + unsigned long offset; + _BitScanForward(&offset, r); + len = offset; #else - len = static_cast(__builtin_ffs(r) - 1); + len = static_cast(__builtin_ffs(r) - 1); #endif - char* q = reinterpret_cast(os_->PushUnsafe(len)); - for (size_t i = 0; i < len; i++) - q[i] = p[i]; + char* q = reinterpret_cast(os_->PushUnsafe(len)); + for (size_t i = 0; i < len; i++) + q[i] = p[i]; - p += len; - break; - } - _mm_storeu_si128(reinterpret_cast<__m128i *>(os_->PushUnsafe(16)), s); + p += len; + break; } + _mm_storeu_si128(reinterpret_cast<__m128i *>(os_->PushUnsafe(16)), s); + } - is.src_ = p; - return RAPIDJSON_LIKELY(is.Tell() < length); + is.src_ = p; + return RAPIDJSON_LIKELY(is.Tell() < length); } #endif // defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) diff --git a/rapidjson/test/perftest/misctest.cpp b/rapidjson/test/perftest/misctest.cpp index aac84778429..4d473548252 100644 --- a/rapidjson/test/perftest/misctest.cpp +++ b/rapidjson/test/perftest/misctest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "perftest.h" @@ -23,7 +23,8 @@ #include "rapidjson/writer.h" #undef private -class Misc : public PerfTest { +class Misc : public PerfTest +{ }; // Copyright (c) 2008-2010 Bjoern Hoehrmann @@ -32,899 +33,922 @@ class Misc : public PerfTest { #define UTF8_ACCEPT 0 #define UTF8_REJECT 12 -static const unsigned char utf8d[] = { - // The first part of the table maps bytes to character classes that - // to reduce the size of the transition table and create bitmasks. - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, - 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, - 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, - 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, - - // The second part is a transition table that maps a combination - // of a state of the automaton and a character class to a state. - 0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12, - 12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12, - 12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12, - 12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12, - 12,36,12,12,12,12,12,12,12,12,12,12, +static const unsigned char utf8d[] = +{ // The first part of the table maps bytes to character classes that + // to reduce the size of the transition table and create bitmasks. + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, + 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, + + // The second part is a transition table that maps a combination + // of a state of the automaton and a character class to a state. + 0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12, + 12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12, + 12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12, + 12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12, + 12,36,12,12,12,12,12,12,12,12,12,12, }; -static unsigned inline decode(unsigned* state, unsigned* codep, unsigned byte) { - unsigned type = utf8d[byte]; +static unsigned inline decode(unsigned* state, unsigned* codep, unsigned byte) +{ unsigned type = utf8d[byte]; - *codep = (*state != UTF8_ACCEPT) ? - (byte & 0x3fu) | (*codep << 6) : - (0xff >> type) & (byte); + *codep = (*state != UTF8_ACCEPT) ? + (byte & 0x3fu) | (*codep << 6) : + (0xff >> type) & (byte); - *state = utf8d[256 + *state + type]; - return *state; + *state = utf8d[256 + *state + type]; + return *state; } -static bool IsUTF8(unsigned char* s) { - unsigned codepoint, state = 0; +static bool IsUTF8(unsigned char* s) +{ unsigned codepoint, state = 0; - while (*s) - decode(&state, &codepoint, *s++); + while (*s) + decode(&state, &codepoint, *s++); - return state == UTF8_ACCEPT; + return state == UTF8_ACCEPT; } -TEST_F(Misc, Hoehrmann_IsUTF8) { - for (size_t i = 0; i < kTrialCount; i++) { - EXPECT_TRUE(IsUTF8((unsigned char*)json_)); - } +TEST_F(Misc, Hoehrmann_IsUTF8) +{ for (size_t i = 0; i < kTrialCount; i++) + { EXPECT_TRUE(IsUTF8((unsigned char*)json_)); + } } //////////////////////////////////////////////////////////////////////////////// // CountDecimalDigit: Count number of decimal places -inline unsigned CountDecimalDigit_naive(unsigned n) { - unsigned count = 1; - while (n >= 10) { - n /= 10; - count++; - } - return count; -} - -inline unsigned CountDecimalDigit_enroll4(unsigned n) { - unsigned count = 1; - while (n >= 10000) { - n /= 10000u; - count += 4; - } - if (n < 10) return count; - if (n < 100) return count + 1; - if (n < 1000) return count + 2; - return count + 3; -} - -inline unsigned CountDecimalDigit64_enroll4(uint64_t n) { - unsigned count = 1; - while (n >= 10000) { - n /= 10000u; - count += 4; - } - if (n < 10) return count; - if (n < 100) return count + 1; - if (n < 1000) return count + 2; - return count + 3; -} - -inline unsigned CountDecimalDigit_fast(unsigned n) { - static const uint32_t powers_of_10[] = { - 0, - 10, - 100, - 1000, - 10000, - 100000, - 1000000, - 10000000, - 100000000, - 1000000000 - }; +inline unsigned CountDecimalDigit_naive(unsigned n) +{ unsigned count = 1; + while (n >= 10) + { n /= 10; + count++; + } + return count; +} + +inline unsigned CountDecimalDigit_enroll4(unsigned n) +{ unsigned count = 1; + while (n >= 10000) + { n /= 10000u; + count += 4; + } + if (n < 10) return count; + if (n < 100) return count + 1; + if (n < 1000) return count + 2; + return count + 3; +} + +inline unsigned CountDecimalDigit64_enroll4(uint64_t n) +{ unsigned count = 1; + while (n >= 10000) + { n /= 10000u; + count += 4; + } + if (n < 10) return count; + if (n < 100) return count + 1; + if (n < 1000) return count + 2; + return count + 3; +} + +inline unsigned CountDecimalDigit_fast(unsigned n) +{ static const uint32_t powers_of_10[] = + { 0, + 10, + 100, + 1000, + 10000, + 100000, + 1000000, + 10000000, + 100000000, + 1000000000 + }; #if defined(_M_IX86) || defined(_M_X64) - unsigned long i = 0; - _BitScanReverse(&i, n | 1); - uint32_t t = (i + 1) * 1233 >> 12; + unsigned long i = 0; + _BitScanReverse(&i, n | 1); + uint32_t t = (i + 1) * 1233 >> 12; #elif defined(__GNUC__) - uint32_t t = (32 - __builtin_clz(n | 1)) * 1233 >> 12; + uint32_t t = (32 - __builtin_clz(n | 1)) * 1233 >> 12; #else #error #endif - return t - (n < powers_of_10[t]) + 1; -} - -inline unsigned CountDecimalDigit64_fast(uint64_t n) { - static const uint64_t powers_of_10[] = { - 0, - 10, - 100, - 1000, - 10000, - 100000, - 1000000, - 10000000, - 100000000, - 1000000000, - 10000000000, - 100000000000, - 1000000000000, - 10000000000000, - 100000000000000, - 1000000000000000, - 10000000000000000, - 100000000000000000, - 1000000000000000000, - 10000000000000000000U - }; + return t - (n < powers_of_10[t]) + 1; +} + +inline unsigned CountDecimalDigit64_fast(uint64_t n) +{ static const uint64_t powers_of_10[] = + { 0, + 10, + 100, + 1000, + 10000, + 100000, + 1000000, + 10000000, + 100000000, + 1000000000, + 10000000000, + 100000000000, + 1000000000000, + 10000000000000, + 100000000000000, + 1000000000000000, + 10000000000000000, + 100000000000000000, + 1000000000000000000, + 10000000000000000000U + }; #if defined(_M_IX86) - uint64_t m = n | 1; - unsigned long i = 0; - if (_BitScanReverse(&i, m >> 32)) - i += 32; - else - _BitScanReverse(&i, m & 0xFFFFFFFF); - uint32_t t = (i + 1) * 1233 >> 12; + uint64_t m = n | 1; + unsigned long i = 0; + if (_BitScanReverse(&i, m >> 32)) + i += 32; + else + _BitScanReverse(&i, m & 0xFFFFFFFF); + uint32_t t = (i + 1) * 1233 >> 12; #elif defined(_M_X64) - unsigned long i = 0; - _BitScanReverse64(&i, n | 1); - uint32_t t = (i + 1) * 1233 >> 12; + unsigned long i = 0; + _BitScanReverse64(&i, n | 1); + uint32_t t = (i + 1) * 1233 >> 12; #elif defined(__GNUC__) - uint32_t t = (64 - __builtin_clzll(n | 1)) * 1233 >> 12; + uint32_t t = (64 - __builtin_clzll(n | 1)) * 1233 >> 12; #else #error #endif - return t - (n < powers_of_10[t]) + 1; + return t - (n < powers_of_10[t]) + 1; } #if 0 // Exhaustive, very slow -TEST_F(Misc, CountDecimalDigit_Verify) { - unsigned i = 0; - do { - if (i % (65536 * 256) == 0) - printf("%u\n", i); - ASSERT_EQ(CountDecimalDigit_enroll4(i), CountDecimalDigit_fast(i)); - i++; - } while (i != 0); +TEST_F(Misc, CountDecimalDigit_Verify) +{ unsigned i = 0; + do + { if (i % (65536 * 256) == 0) + printf("%u\n", i); + ASSERT_EQ(CountDecimalDigit_enroll4(i), CountDecimalDigit_fast(i)); + i++; + } + while (i != 0); } static const unsigned kDigits10Trial = 1000000000u; -TEST_F(Misc, CountDecimalDigit_naive) { - unsigned sum = 0; - for (unsigned i = 0; i < kDigits10Trial; i++) - sum += CountDecimalDigit_naive(i); - printf("%u\n", sum); +TEST_F(Misc, CountDecimalDigit_naive) +{ unsigned sum = 0; + for (unsigned i = 0; i < kDigits10Trial; i++) + sum += CountDecimalDigit_naive(i); + printf("%u\n", sum); } -TEST_F(Misc, CountDecimalDigit_enroll4) { - unsigned sum = 0; - for (unsigned i = 0; i < kDigits10Trial; i++) - sum += CountDecimalDigit_enroll4(i); - printf("%u\n", sum); +TEST_F(Misc, CountDecimalDigit_enroll4) +{ unsigned sum = 0; + for (unsigned i = 0; i < kDigits10Trial; i++) + sum += CountDecimalDigit_enroll4(i); + printf("%u\n", sum); } -TEST_F(Misc, CountDecimalDigit_fast) { - unsigned sum = 0; - for (unsigned i = 0; i < kDigits10Trial; i++) - sum += CountDecimalDigit_fast(i); - printf("%u\n", sum); +TEST_F(Misc, CountDecimalDigit_fast) +{ unsigned sum = 0; + for (unsigned i = 0; i < kDigits10Trial; i++) + sum += CountDecimalDigit_fast(i); + printf("%u\n", sum); } #endif -TEST_F(Misc, CountDecimalDigit64_VerifyFast) { - uint64_t i = 1, j; - do { - //printf("%" PRIu64 "\n", i); - ASSERT_EQ(CountDecimalDigit64_enroll4(i), CountDecimalDigit64_fast(i)); - j = i; - i *= 3; - } while (j < i); +TEST_F(Misc, CountDecimalDigit64_VerifyFast) +{ uint64_t i = 1, j; + do + { //printf("%" PRIu64 "\n", i); + ASSERT_EQ(CountDecimalDigit64_enroll4(i), CountDecimalDigit64_fast(i)); + j = i; + i *= 3; + } + while (j < i); } //////////////////////////////////////////////////////////////////////////////// // integer-to-string conversion // https://gist.github.com/anonymous/7179097 -static const int randval[] ={ - 936116, 369532, 453755, -72860, 209713, 268347, 435278, -360266, -416287, -182064, - -644712, 944969, 640463, -366588, 471577, -69401, -744294, -505829, 923883, 831785, - -601136, -636767, -437054, 591718, 100758, 231907, -719038, 973540, -605220, 506659, - -871653, 462533, 764843, -919138, 404305, -630931, -288711, -751454, -173726, -718208, - 432689, -281157, 360737, 659827, 19174, -376450, 769984, -858198, 439127, 734703, - -683426, 7, 386135, 186997, -643900, -744422, -604708, -629545, 42313, -933592, - -635566, 182308, 439024, -367219, -73924, -516649, 421935, -470515, 413507, -78952, - -427917, -561158, 737176, 94538, 572322, 405217, 709266, -357278, -908099, -425447, - 601119, 750712, -862285, -177869, 900102, 384877, 157859, -641680, 503738, -702558, - 278225, 463290, 268378, -212840, 580090, 347346, -473985, -950968, -114547, -839893, - -738032, -789424, 409540, 493495, 432099, 119755, 905004, -174834, 338266, 234298, - 74641, -965136, -754593, 685273, 466924, 920560, 385062, 796402, -67229, 994864, - 376974, 299869, -647540, -128724, 469890, -163167, -547803, -743363, 486463, -621028, - 612288, 27459, -514224, 126342, -66612, 803409, -777155, -336453, -284002, 472451, - 342390, -163630, 908356, -456147, -825607, 268092, -974715, 287227, 227890, -524101, - 616370, -782456, 922098, -624001, -813690, 171605, -192962, 796151, 707183, -95696, - -23163, -721260, 508892, 430715, 791331, 482048, -996102, 863274, 275406, -8279, - -556239, -902076, 268647, -818565, 260069, -798232, -172924, -566311, -806503, -885992, - 813969, -78468, 956632, 304288, 494867, -508784, 381751, 151264, 762953, 76352, - 594902, 375424, 271700, -743062, 390176, 924237, 772574, 676610, 435752, -153847, - 3959, -971937, -294181, -538049, -344620, -170136, 19120, -703157, 868152, -657961, - -818631, 219015, -872729, -940001, -956570, 880727, -345910, 942913, -942271, -788115, - 225294, 701108, -517736, -416071, 281940, 488730, 942698, 711494, 838382, -892302, - -533028, 103052, 528823, 901515, 949577, 159364, 718227, -241814, -733661, -462928, - -495829, 165170, 513580, -629188, -509571, -459083, 198437, 77198, -644612, 811276, - -422298, -860842, -52584, 920369, 686424, -530667, -243476, 49763, 345866, -411960, - -114863, 470810, -302860, 683007, -509080, 2, -174981, -772163, -48697, 447770, - -268246, 213268, 269215, 78810, -236340, -639140, -864323, 505113, -986569, -325215, - 541859, 163070, -819998, -645161, -583336, 573414, 696417, -132375, 3, -294501, - 320435, 682591, 840008, 351740, 426951, 609354, 898154, -943254, 227321, -859793, - -727993, 44137, -497965, -782239, 14955, -746080, -243366, 9837, -233083, 606507, - -995864, -615287, -994307, 602715, 770771, -315040, 610860, 446102, -307120, 710728, - -590392, -230474, -762625, -637525, 134963, -202700, -766902, -985541, 218163, 682009, - 926051, 525156, -61195, 403211, -810098, 245539, -431733, 179998, -806533, 745943, - 447597, 131973, -187130, 826019, 286107, -937230, -577419, 20254, 681802, -340500, - 323080, 266283, -667617, 309656, 416386, 611863, 759991, -534257, 523112, -634892, - -169913, -204905, -909867, -882185, -944908, 741811, -717675, 967007, -317396, 407230, - -412805, 792905, 994873, 744793, -456797, 713493, 355232, 116900, -945199, 880539, - 342505, -580824, -262273, 982968, -349497, -735488, 311767, -455191, 570918, 389734, - -958386, 10262, -99267, 155481, 304210, 204724, 704367, -144893, -233664, -671441, - 896849, 408613, 762236, 322697, 981321, 688476, 13663, -970704, -379507, 896412, - 977084, 348869, 875948, 341348, 318710, 512081, 6163, 669044, 833295, 811883, - 708756, -802534, -536057, 608413, -389625, -694603, 541106, -110037, 720322, -540581, - 645420, 32980, 62442, 510157, -981870, -87093, -325960, -500494, -718291, -67889, - 991501, 374804, 769026, -978869, 294747, 714623, 413327, -199164, 671368, 804789, - -362507, 798196, -170790, -568895, -869379, 62020, -316693, -837793, 644994, -39341, - -417504, -243068, -957756, 99072, 622234, -739992, 225668, 8863, -505910, 82483, - -559244, 241572, 1315, -36175, -54990, 376813, -11, 162647, -688204, -486163, - -54934, -197470, 744223, -762707, 732540, 996618, 351561, -445933, -898491, 486531, - 456151, 15276, 290186, -817110, -52995, 313046, -452533, -96267, 94470, -500176, - -818026, -398071, -810548, -143325, -819741, 1338, -897676, -101577, -855445, 37309, - 285742, 953804, -777927, -926962, -811217, -936744, -952245, -802300, -490188, -964953, - -552279, 329142, -570048, -505756, 682898, -381089, -14352, 175138, 152390, -582268, - -485137, 717035, 805329, 239572, -730409, 209643, -184403, -385864, 675086, 819648, - 629058, -527109, -488666, -171981, 532788, 552441, 174666, 984921, 766514, 758787, - 716309, 338801, -978004, -412163, 876079, -734212, 789557, -160491, -522719, 56644, - -991, -286038, -53983, 663740, 809812, 919889, -717502, -137704, 220511, 184396, - -825740, -588447, 430870, 124309, 135956, 558662, -307087, -788055, -451328, 812260, - 931601, 324347, -482989, -117858, -278861, 189068, -172774, 929057, 293787, 198161, - -342386, -47173, 906555, -759955, -12779, 777604, -97869, 899320, 927486, -25284, - -848550, 259450, -485856, -17820, 88, 171400, 235492, -326783, -340793, 886886, - 112428, -246280, 5979, 648444, -114982, 991013, -56489, -9497, 419706, 632820, - -341664, 393926, -848977, -22538, 257307, 773731, -905319, 491153, 734883, -868212, - -951053, 644458, -580758, 764735, 584316, 297077, 28852, -397710, -953669, 201772, - 879050, -198237, -588468, 448102, -116837, 770007, -231812, 642906, -582166, -885828, - 9, 305082, -996577, 303559, 75008, -772956, -447960, 599825, -295552, 870739, - -386278, -950300, 485359, -457081, 629461, -850276, 550496, -451755, -620841, -11766, - -950137, 832337, 28711, -273398, -507197, 91921, -271360, -705991, -753220, -388968, - 967945, 340434, -320883, -662793, -554617, -574568, 477946, -6148, -129519, 689217, - 920020, -656315, -974523, -212525, 80921, -612532, 645096, 545655, 655713, -591631, - -307385, -816688, -618823, -113713, 526430, 673063, 735916, -809095, -850417, 639004, - 432281, -388185, 270708, 860146, -39902, -786157, -258180, -246169, -966720, -264957, - 548072, -306010, -57367, -635665, 933824, 70553, -989936, -488741, 72411, -452509, - 529831, 956277, 449019, -577850, -360986, -803418, 48833, 296073, 203430, 609591, - 715483, 470964, 658106, -718254, -96424, 790163, 334739, 181070, -373578, 5, - -435088, 329841, 330939, -256602, 394355, 912412, 231910, 927278, -661933, 788539, - -769664, -893274, -96856, 298205, 901043, -608122, -527430, 183618, -553963, -35246, - -393924, 948832, -483198, 594501, 35460, -407007, 93494, -336881, -634072, 984205, - -812161, 944664, -31062, 753872, 823933, -69566, 50445, 290147, 85134, 34706, - 551902, 405202, -991246, -84642, 154341, 316432, -695101, -651588, -5030, 137564, - -294665, 332541, 528307, -90572, -344923, 523766, -758498, -968047, 339028, 494578, - 593129, -725773, 31834, -718406, -208638, 159665, -2043, 673344, -442767, 75816, - 755442, 769257, -158730, -410272, 691688, 589550, -878398, -184121, 460679, 346312, - 294163, -544602, 653308, 254167, -276979, 52073, -892684, 887653, -41222, 983065, - -68258, -408799, -99069, -674069, -863635, -32890, 622757, -743862, 40872, -4837, - -967228, 522370, -903951, -818669, 524459, 514702, 925801, 20007, -299229, 579348, - 626021, 430089, 348139, -562692, -607728, -130606, -928451, -424793, -458647, -448892, - -312230, 143337, 109746, 880042, -339658, -785614, 938995, 540916, 118429, 661351, - -402967, 404729, -40918, -976535, 743230, 713110, 440182, -381314, -499252, 74613, - 193652, 912717, 491323, 583633, 324691, 459397, 281253, 195540, -2764, -888651, - 892449, 132663, -478373, -430002, -314551, 527826, 247165, 557966, 554778, 481531, - -946634, 431685, -769059, -348371, 174046, 184597, -354867, 584422, 227390, -850397, - -542924, -849093, -737769, 325359, 736314, 269101, 767940, 674809, 81413, -447458, - 445076, 189072, 906218, 502688, -718476, -863827, -731381, 100660, 623249, 710008, - 572060, 922203, 685740, 55096, 263394, -243695, -353910, -516788, 388471, 455165, - 844103, -643772, 363976, 268875, -899450, 104470, 104029, -238874, -274659, 732969, - -676443, 953291, -916289, -861849, -242344, 958083, -479593, -970395, 799831, 277841, - -243236, -283462, -201510, 166263, -259105, -575706, 878926, 891064, 895297, 655262, - -34807, -809833, -89281, 342585, 554920, 1, 902141, -333425, 139703, 852318, - -618438, 329498, -932596, -692836, -513372, 733656, -523411, 85779, 500478, -682697, - -502836, 138776, 156341, -420037, -557964, -556378, 710993, -50383, -877159, 916334, - 132996, 583516, -603392, -111615, -12288, -780214, 476780, 123327, 137607, 519956, - 745837, 17358, -158581, -53490 +static const int randval[] = +{ 936116, 369532, 453755, -72860, 209713, 268347, 435278, -360266, -416287, -182064, + -644712, 944969, 640463, -366588, 471577, -69401, -744294, -505829, 923883, 831785, + -601136, -636767, -437054, 591718, 100758, 231907, -719038, 973540, -605220, 506659, + -871653, 462533, 764843, -919138, 404305, -630931, -288711, -751454, -173726, -718208, + 432689, -281157, 360737, 659827, 19174, -376450, 769984, -858198, 439127, 734703, + -683426, 7, 386135, 186997, -643900, -744422, -604708, -629545, 42313, -933592, + -635566, 182308, 439024, -367219, -73924, -516649, 421935, -470515, 413507, -78952, + -427917, -561158, 737176, 94538, 572322, 405217, 709266, -357278, -908099, -425447, + 601119, 750712, -862285, -177869, 900102, 384877, 157859, -641680, 503738, -702558, + 278225, 463290, 268378, -212840, 580090, 347346, -473985, -950968, -114547, -839893, + -738032, -789424, 409540, 493495, 432099, 119755, 905004, -174834, 338266, 234298, + 74641, -965136, -754593, 685273, 466924, 920560, 385062, 796402, -67229, 994864, + 376974, 299869, -647540, -128724, 469890, -163167, -547803, -743363, 486463, -621028, + 612288, 27459, -514224, 126342, -66612, 803409, -777155, -336453, -284002, 472451, + 342390, -163630, 908356, -456147, -825607, 268092, -974715, 287227, 227890, -524101, + 616370, -782456, 922098, -624001, -813690, 171605, -192962, 796151, 707183, -95696, + -23163, -721260, 508892, 430715, 791331, 482048, -996102, 863274, 275406, -8279, + -556239, -902076, 268647, -818565, 260069, -798232, -172924, -566311, -806503, -885992, + 813969, -78468, 956632, 304288, 494867, -508784, 381751, 151264, 762953, 76352, + 594902, 375424, 271700, -743062, 390176, 924237, 772574, 676610, 435752, -153847, + 3959, -971937, -294181, -538049, -344620, -170136, 19120, -703157, 868152, -657961, + -818631, 219015, -872729, -940001, -956570, 880727, -345910, 942913, -942271, -788115, + 225294, 701108, -517736, -416071, 281940, 488730, 942698, 711494, 838382, -892302, + -533028, 103052, 528823, 901515, 949577, 159364, 718227, -241814, -733661, -462928, + -495829, 165170, 513580, -629188, -509571, -459083, 198437, 77198, -644612, 811276, + -422298, -860842, -52584, 920369, 686424, -530667, -243476, 49763, 345866, -411960, + -114863, 470810, -302860, 683007, -509080, 2, -174981, -772163, -48697, 447770, + -268246, 213268, 269215, 78810, -236340, -639140, -864323, 505113, -986569, -325215, + 541859, 163070, -819998, -645161, -583336, 573414, 696417, -132375, 3, -294501, + 320435, 682591, 840008, 351740, 426951, 609354, 898154, -943254, 227321, -859793, + -727993, 44137, -497965, -782239, 14955, -746080, -243366, 9837, -233083, 606507, + -995864, -615287, -994307, 602715, 770771, -315040, 610860, 446102, -307120, 710728, + -590392, -230474, -762625, -637525, 134963, -202700, -766902, -985541, 218163, 682009, + 926051, 525156, -61195, 403211, -810098, 245539, -431733, 179998, -806533, 745943, + 447597, 131973, -187130, 826019, 286107, -937230, -577419, 20254, 681802, -340500, + 323080, 266283, -667617, 309656, 416386, 611863, 759991, -534257, 523112, -634892, + -169913, -204905, -909867, -882185, -944908, 741811, -717675, 967007, -317396, 407230, + -412805, 792905, 994873, 744793, -456797, 713493, 355232, 116900, -945199, 880539, + 342505, -580824, -262273, 982968, -349497, -735488, 311767, -455191, 570918, 389734, + -958386, 10262, -99267, 155481, 304210, 204724, 704367, -144893, -233664, -671441, + 896849, 408613, 762236, 322697, 981321, 688476, 13663, -970704, -379507, 896412, + 977084, 348869, 875948, 341348, 318710, 512081, 6163, 669044, 833295, 811883, + 708756, -802534, -536057, 608413, -389625, -694603, 541106, -110037, 720322, -540581, + 645420, 32980, 62442, 510157, -981870, -87093, -325960, -500494, -718291, -67889, + 991501, 374804, 769026, -978869, 294747, 714623, 413327, -199164, 671368, 804789, + -362507, 798196, -170790, -568895, -869379, 62020, -316693, -837793, 644994, -39341, + -417504, -243068, -957756, 99072, 622234, -739992, 225668, 8863, -505910, 82483, + -559244, 241572, 1315, -36175, -54990, 376813, -11, 162647, -688204, -486163, + -54934, -197470, 744223, -762707, 732540, 996618, 351561, -445933, -898491, 486531, + 456151, 15276, 290186, -817110, -52995, 313046, -452533, -96267, 94470, -500176, + -818026, -398071, -810548, -143325, -819741, 1338, -897676, -101577, -855445, 37309, + 285742, 953804, -777927, -926962, -811217, -936744, -952245, -802300, -490188, -964953, + -552279, 329142, -570048, -505756, 682898, -381089, -14352, 175138, 152390, -582268, + -485137, 717035, 805329, 239572, -730409, 209643, -184403, -385864, 675086, 819648, + 629058, -527109, -488666, -171981, 532788, 552441, 174666, 984921, 766514, 758787, + 716309, 338801, -978004, -412163, 876079, -734212, 789557, -160491, -522719, 56644, + -991, -286038, -53983, 663740, 809812, 919889, -717502, -137704, 220511, 184396, + -825740, -588447, 430870, 124309, 135956, 558662, -307087, -788055, -451328, 812260, + 931601, 324347, -482989, -117858, -278861, 189068, -172774, 929057, 293787, 198161, + -342386, -47173, 906555, -759955, -12779, 777604, -97869, 899320, 927486, -25284, + -848550, 259450, -485856, -17820, 88, 171400, 235492, -326783, -340793, 886886, + 112428, -246280, 5979, 648444, -114982, 991013, -56489, -9497, 419706, 632820, + -341664, 393926, -848977, -22538, 257307, 773731, -905319, 491153, 734883, -868212, + -951053, 644458, -580758, 764735, 584316, 297077, 28852, -397710, -953669, 201772, + 879050, -198237, -588468, 448102, -116837, 770007, -231812, 642906, -582166, -885828, + 9, 305082, -996577, 303559, 75008, -772956, -447960, 599825, -295552, 870739, + -386278, -950300, 485359, -457081, 629461, -850276, 550496, -451755, -620841, -11766, + -950137, 832337, 28711, -273398, -507197, 91921, -271360, -705991, -753220, -388968, + 967945, 340434, -320883, -662793, -554617, -574568, 477946, -6148, -129519, 689217, + 920020, -656315, -974523, -212525, 80921, -612532, 645096, 545655, 655713, -591631, + -307385, -816688, -618823, -113713, 526430, 673063, 735916, -809095, -850417, 639004, + 432281, -388185, 270708, 860146, -39902, -786157, -258180, -246169, -966720, -264957, + 548072, -306010, -57367, -635665, 933824, 70553, -989936, -488741, 72411, -452509, + 529831, 956277, 449019, -577850, -360986, -803418, 48833, 296073, 203430, 609591, + 715483, 470964, 658106, -718254, -96424, 790163, 334739, 181070, -373578, 5, + -435088, 329841, 330939, -256602, 394355, 912412, 231910, 927278, -661933, 788539, + -769664, -893274, -96856, 298205, 901043, -608122, -527430, 183618, -553963, -35246, + -393924, 948832, -483198, 594501, 35460, -407007, 93494, -336881, -634072, 984205, + -812161, 944664, -31062, 753872, 823933, -69566, 50445, 290147, 85134, 34706, + 551902, 405202, -991246, -84642, 154341, 316432, -695101, -651588, -5030, 137564, + -294665, 332541, 528307, -90572, -344923, 523766, -758498, -968047, 339028, 494578, + 593129, -725773, 31834, -718406, -208638, 159665, -2043, 673344, -442767, 75816, + 755442, 769257, -158730, -410272, 691688, 589550, -878398, -184121, 460679, 346312, + 294163, -544602, 653308, 254167, -276979, 52073, -892684, 887653, -41222, 983065, + -68258, -408799, -99069, -674069, -863635, -32890, 622757, -743862, 40872, -4837, + -967228, 522370, -903951, -818669, 524459, 514702, 925801, 20007, -299229, 579348, + 626021, 430089, 348139, -562692, -607728, -130606, -928451, -424793, -458647, -448892, + -312230, 143337, 109746, 880042, -339658, -785614, 938995, 540916, 118429, 661351, + -402967, 404729, -40918, -976535, 743230, 713110, 440182, -381314, -499252, 74613, + 193652, 912717, 491323, 583633, 324691, 459397, 281253, 195540, -2764, -888651, + 892449, 132663, -478373, -430002, -314551, 527826, 247165, 557966, 554778, 481531, + -946634, 431685, -769059, -348371, 174046, 184597, -354867, 584422, 227390, -850397, + -542924, -849093, -737769, 325359, 736314, 269101, 767940, 674809, 81413, -447458, + 445076, 189072, 906218, 502688, -718476, -863827, -731381, 100660, 623249, 710008, + 572060, 922203, 685740, 55096, 263394, -243695, -353910, -516788, 388471, 455165, + 844103, -643772, 363976, 268875, -899450, 104470, 104029, -238874, -274659, 732969, + -676443, 953291, -916289, -861849, -242344, 958083, -479593, -970395, 799831, 277841, + -243236, -283462, -201510, 166263, -259105, -575706, 878926, 891064, 895297, 655262, + -34807, -809833, -89281, 342585, 554920, 1, 902141, -333425, 139703, 852318, + -618438, 329498, -932596, -692836, -513372, 733656, -523411, 85779, 500478, -682697, + -502836, 138776, 156341, -420037, -557964, -556378, 710993, -50383, -877159, 916334, + 132996, 583516, -603392, -111615, -12288, -780214, 476780, 123327, 137607, 519956, + 745837, 17358, -158581, -53490 }; static const size_t randvalCount = sizeof(randval) / sizeof(randval[0]); static const size_t kItoaTrialCount = 10000; static const char digits[201] = -"0001020304050607080910111213141516171819" -"2021222324252627282930313233343536373839" -"4041424344454647484950515253545556575859" -"6061626364656667686970717273747576777879" -"8081828384858687888990919293949596979899"; + "0001020304050607080910111213141516171819" + "2021222324252627282930313233343536373839" + "4041424344454647484950515253545556575859" + "6061626364656667686970717273747576777879" + "8081828384858687888990919293949596979899"; // Prevent code being optimized out //#define OUTPUT_LENGTH(length) printf("", length) #define OUTPUT_LENGTH(length) printf("%u\n", (unsigned)length) template -class Writer1 { +class Writer1 +{ public: - Writer1() : os_() {} - Writer1(OutputStream& os) : os_(&os) {} + Writer1() : os_() {} + Writer1(OutputStream& os) : os_(&os) {} - void Reset(OutputStream& os) { - os_ = &os; - } + void Reset(OutputStream& os) + { os_ = &os; + } - bool WriteInt(int i) { - if (i < 0) { - os_->Put('-'); - i = -i; - } - return WriteUint((unsigned)i); + bool WriteInt(int i) + { if (i < 0) + { os_->Put('-'); + i = -i; } + return WriteUint((unsigned)i); + } - bool WriteUint(unsigned u) { - char buffer[10]; - char *p = buffer; - do { - *p++ = char(u % 10) + '0'; - u /= 10; - } while (u > 0); + bool WriteUint(unsigned u) + { char buffer[10]; + char *p = buffer; + do + { *p++ = char(u % 10) + '0'; + u /= 10; + } + while (u > 0); - do { - --p; - os_->Put(*p); - } while (p != buffer); - return true; + do + { --p; + os_->Put(*p); } + while (p != buffer); + return true; + } - bool WriteInt64(int64_t i64) { - if (i64 < 0) { - os_->Put('-'); - i64 = -i64; - } - WriteUint64((uint64_t)i64); - return true; + bool WriteInt64(int64_t i64) + { if (i64 < 0) + { os_->Put('-'); + i64 = -i64; } + WriteUint64((uint64_t)i64); + return true; + } - bool WriteUint64(uint64_t u64) { - char buffer[20]; - char *p = buffer; - do { - *p++ = char(u64 % 10) + '0'; - u64 /= 10; - } while (u64 > 0); + bool WriteUint64(uint64_t u64) + { char buffer[20]; + char *p = buffer; + do + { *p++ = char(u64 % 10) + '0'; + u64 /= 10; + } + while (u64 > 0); - do { - --p; - os_->Put(*p); - } while (p != buffer); - return true; + do + { --p; + os_->Put(*p); } + while (p != buffer); + return true; + } private: - OutputStream* os_; + OutputStream* os_; }; template<> -bool Writer1::WriteUint(unsigned u) { - char buffer[10]; - char* p = buffer; - do { - *p++ = char(u % 10) + '0'; - u /= 10; - } while (u > 0); - - char* d = os_->Push(p - buffer); - do { - --p; - *d++ = *p; - } while (p != buffer); - return true; +bool Writer1::WriteUint(unsigned u) +{ char buffer[10]; + char* p = buffer; + do + { *p++ = char(u % 10) + '0'; + u /= 10; + } + while (u > 0); + + char* d = os_->Push(p - buffer); + do + { --p; + *d++ = *p; + } + while (p != buffer); + return true; } // Using digits LUT to reduce divsion/modulo template -class Writer2 { +class Writer2 +{ public: - Writer2() : os_() {} - Writer2(OutputStream& os) : os_(&os) {} - - void Reset(OutputStream& os) { - os_ = &os; - } - - bool WriteInt(int i) { - if (i < 0) { - os_->Put('-'); - i = -i; - } - return WriteUint((unsigned)i); - } - - bool WriteUint(unsigned u) { - char buffer[10]; - char* p = buffer; - while (u >= 100) { - const unsigned i = (u % 100) << 1; - u /= 100; - *p++ = digits[i + 1]; - *p++ = digits[i]; - } - if (u < 10) - *p++ = char(u) + '0'; - else { - const unsigned i = u << 1; - *p++ = digits[i + 1]; - *p++ = digits[i]; - } - - do { - --p; - os_->Put(*p); - } while (p != buffer); - return true; - } - - bool WriteInt64(int64_t i64) { - if (i64 < 0) { - os_->Put('-'); - i64 = -i64; - } - WriteUint64((uint64_t)i64); - return true; - } - - bool WriteUint64(uint64_t u64) { - char buffer[20]; - char* p = buffer; - while (u64 >= 100) { - const unsigned i = static_cast(u64 % 100) << 1; - u64 /= 100; - *p++ = digits[i + 1]; - *p++ = digits[i]; - } - if (u64 < 10) - *p++ = char(u64) + '0'; - else { - const unsigned i = static_cast(u64) << 1; - *p++ = digits[i + 1]; - *p++ = digits[i]; - } - - do { - --p; - os_->Put(*p); - } while (p != buffer); - return true; + Writer2() : os_() {} + Writer2(OutputStream& os) : os_(&os) {} + + void Reset(OutputStream& os) + { os_ = &os; + } + + bool WriteInt(int i) + { if (i < 0) + { os_->Put('-'); + i = -i; + } + return WriteUint((unsigned)i); + } + + bool WriteUint(unsigned u) + { char buffer[10]; + char* p = buffer; + while (u >= 100) + { const unsigned i = (u % 100) << 1; + u /= 100; + *p++ = digits[i + 1]; + *p++ = digits[i]; + } + if (u < 10) + *p++ = char(u) + '0'; + else + { const unsigned i = u << 1; + *p++ = digits[i + 1]; + *p++ = digits[i]; } + do + { --p; + os_->Put(*p); + } + while (p != buffer); + return true; + } + + bool WriteInt64(int64_t i64) + { if (i64 < 0) + { os_->Put('-'); + i64 = -i64; + } + WriteUint64((uint64_t)i64); + return true; + } + + bool WriteUint64(uint64_t u64) + { char buffer[20]; + char* p = buffer; + while (u64 >= 100) + { const unsigned i = static_cast(u64 % 100) << 1; + u64 /= 100; + *p++ = digits[i + 1]; + *p++ = digits[i]; + } + if (u64 < 10) + *p++ = char(u64) + '0'; + else + { const unsigned i = static_cast(u64) << 1; + *p++ = digits[i + 1]; + *p++ = digits[i]; + } + + do + { --p; + os_->Put(*p); + } + while (p != buffer); + return true; + } + private: - OutputStream* os_; + OutputStream* os_; }; // First pass to count digits template -class Writer3 { +class Writer3 +{ public: - Writer3() : os_() {} - Writer3(OutputStream& os) : os_(&os) {} + Writer3() : os_() {} + Writer3(OutputStream& os) : os_(&os) {} - void Reset(OutputStream& os) { - os_ = &os; - } + void Reset(OutputStream& os) + { os_ = &os; + } - bool WriteInt(int i) { - if (i < 0) { - os_->Put('-'); - i = -i; - } - return WriteUint((unsigned)i); + bool WriteInt(int i) + { if (i < 0) + { os_->Put('-'); + i = -i; } + return WriteUint((unsigned)i); + } - bool WriteUint(unsigned u) { - char buffer[10]; - char *p = buffer; - do { - *p++ = char(u % 10) + '0'; - u /= 10; - } while (u > 0); + bool WriteUint(unsigned u) + { char buffer[10]; + char *p = buffer; + do + { *p++ = char(u % 10) + '0'; + u /= 10; + } + while (u > 0); - do { - --p; - os_->Put(*p); - } while (p != buffer); - return true; + do + { --p; + os_->Put(*p); } + while (p != buffer); + return true; + } - bool WriteInt64(int64_t i64) { - if (i64 < 0) { - os_->Put('-'); - i64 = -i64; - } - WriteUint64((uint64_t)i64); - return true; + bool WriteInt64(int64_t i64) + { if (i64 < 0) + { os_->Put('-'); + i64 = -i64; } + WriteUint64((uint64_t)i64); + return true; + } - bool WriteUint64(uint64_t u64) { - char buffer[20]; - char *p = buffer; - do { - *p++ = char(u64 % 10) + '0'; - u64 /= 10; - } while (u64 > 0); + bool WriteUint64(uint64_t u64) + { char buffer[20]; + char *p = buffer; + do + { *p++ = char(u64 % 10) + '0'; + u64 /= 10; + } + while (u64 > 0); - do { - --p; - os_->Put(*p); - } while (p != buffer); - return true; + do + { --p; + os_->Put(*p); } + while (p != buffer); + return true; + } private: - void WriteUintReverse(char* d, unsigned u) { - do { - *--d = char(u % 10) + '0'; - u /= 10; - } while (u > 0); + void WriteUintReverse(char* d, unsigned u) + { do + { *--d = char(u % 10) + '0'; + u /= 10; } + while (u > 0); + } - void WriteUint64Reverse(char* d, uint64_t u) { - do { - *--d = char(u % 10) + '0'; - u /= 10; - } while (u > 0); + void WriteUint64Reverse(char* d, uint64_t u) + { do + { *--d = char(u % 10) + '0'; + u /= 10; } + while (u > 0); + } - OutputStream* os_; + OutputStream* os_; }; template<> -inline bool Writer3::WriteUint(unsigned u) { - unsigned digit = CountDecimalDigit_fast(u); - WriteUintReverse(os_->Push(digit) + digit, u); - return true; +inline bool Writer3::WriteUint(unsigned u) +{ unsigned digit = CountDecimalDigit_fast(u); + WriteUintReverse(os_->Push(digit) + digit, u); + return true; } template<> -inline bool Writer3::WriteUint(unsigned u) { - unsigned digit = CountDecimalDigit_fast(u); - WriteUintReverse(os_->Push(digit) + digit, u); - return true; +inline bool Writer3::WriteUint(unsigned u) +{ unsigned digit = CountDecimalDigit_fast(u); + WriteUintReverse(os_->Push(digit) + digit, u); + return true; } template<> -inline bool Writer3::WriteUint64(uint64_t u) { - unsigned digit = CountDecimalDigit64_fast(u); - WriteUint64Reverse(os_->Push(digit) + digit, u); - return true; +inline bool Writer3::WriteUint64(uint64_t u) +{ unsigned digit = CountDecimalDigit64_fast(u); + WriteUint64Reverse(os_->Push(digit) + digit, u); + return true; } template<> -inline bool Writer3::WriteUint64(uint64_t u) { - unsigned digit = CountDecimalDigit64_fast(u); - WriteUint64Reverse(os_->Push(digit) + digit, u); - return true; +inline bool Writer3::WriteUint64(uint64_t u) +{ unsigned digit = CountDecimalDigit64_fast(u); + WriteUint64Reverse(os_->Push(digit) + digit, u); + return true; } // Using digits LUT to reduce divsion/modulo, two passes template -class Writer4 { +class Writer4 +{ public: - Writer4() : os_() {} - Writer4(OutputStream& os) : os_(&os) {} - - void Reset(OutputStream& os) { - os_ = &os; - } - - bool WriteInt(int i) { - if (i < 0) { - os_->Put('-'); - i = -i; - } - return WriteUint((unsigned)i); - } - - bool WriteUint(unsigned u) { - char buffer[10]; - char* p = buffer; - while (u >= 100) { - const unsigned i = (u % 100) << 1; - u /= 100; - *p++ = digits[i + 1]; - *p++ = digits[i]; - } - if (u < 10) - *p++ = char(u) + '0'; - else { - const unsigned i = u << 1; - *p++ = digits[i + 1]; - *p++ = digits[i]; - } - - do { - --p; - os_->Put(*p); - } while (p != buffer); - return true; - } - - bool WriteInt64(int64_t i64) { - if (i64 < 0) { - os_->Put('-'); - i64 = -i64; - } - WriteUint64((uint64_t)i64); - return true; - } - - bool WriteUint64(uint64_t u64) { - char buffer[20]; - char* p = buffer; - while (u64 >= 100) { - const unsigned i = static_cast(u64 % 100) << 1; - u64 /= 100; - *p++ = digits[i + 1]; - *p++ = digits[i]; - } - if (u64 < 10) - *p++ = char(u64) + '0'; - else { - const unsigned i = static_cast(u64) << 1; - *p++ = digits[i + 1]; - *p++ = digits[i]; - } - - do { - --p; - os_->Put(*p); - } while (p != buffer); - return true; + Writer4() : os_() {} + Writer4(OutputStream& os) : os_(&os) {} + + void Reset(OutputStream& os) + { os_ = &os; + } + + bool WriteInt(int i) + { if (i < 0) + { os_->Put('-'); + i = -i; + } + return WriteUint((unsigned)i); + } + + bool WriteUint(unsigned u) + { char buffer[10]; + char* p = buffer; + while (u >= 100) + { const unsigned i = (u % 100) << 1; + u /= 100; + *p++ = digits[i + 1]; + *p++ = digits[i]; + } + if (u < 10) + *p++ = char(u) + '0'; + else + { const unsigned i = u << 1; + *p++ = digits[i + 1]; + *p++ = digits[i]; + } + + do + { --p; + os_->Put(*p); + } + while (p != buffer); + return true; + } + + bool WriteInt64(int64_t i64) + { if (i64 < 0) + { os_->Put('-'); + i64 = -i64; + } + WriteUint64((uint64_t)i64); + return true; + } + + bool WriteUint64(uint64_t u64) + { char buffer[20]; + char* p = buffer; + while (u64 >= 100) + { const unsigned i = static_cast(u64 % 100) << 1; + u64 /= 100; + *p++ = digits[i + 1]; + *p++ = digits[i]; + } + if (u64 < 10) + *p++ = char(u64) + '0'; + else + { const unsigned i = static_cast(u64) << 1; + *p++ = digits[i + 1]; + *p++ = digits[i]; } + do + { --p; + os_->Put(*p); + } + while (p != buffer); + return true; + } + private: - void WriteUintReverse(char* d, unsigned u) { - while (u >= 100) { - const unsigned i = (u % 100) << 1; - u /= 100; - *--d = digits[i + 1]; - *--d = digits[i]; - } - if (u < 10) { - *--d = char(u) + '0'; - } - else { - const unsigned i = u << 1; - *--d = digits[i + 1]; - *--d = digits[i]; - } - } - - void WriteUint64Reverse(char* d, uint64_t u) { - while (u >= 100) { - const unsigned i = (u % 100) << 1; - u /= 100; - *--d = digits[i + 1]; - *--d = digits[i]; - } - if (u < 10) { - *--d = char(u) + '0'; - } - else { - const unsigned i = u << 1; - *--d = digits[i + 1]; - *--d = digits[i]; - } - } - - OutputStream* os_; + void WriteUintReverse(char* d, unsigned u) + { while (u >= 100) + { const unsigned i = (u % 100) << 1; + u /= 100; + *--d = digits[i + 1]; + *--d = digits[i]; + } + if (u < 10) + { *--d = char(u) + '0'; + } + else + { const unsigned i = u << 1; + *--d = digits[i + 1]; + *--d = digits[i]; + } + } + + void WriteUint64Reverse(char* d, uint64_t u) + { while (u >= 100) + { const unsigned i = (u % 100) << 1; + u /= 100; + *--d = digits[i + 1]; + *--d = digits[i]; + } + if (u < 10) + { *--d = char(u) + '0'; + } + else + { const unsigned i = u << 1; + *--d = digits[i + 1]; + *--d = digits[i]; + } + } + + OutputStream* os_; }; template<> -inline bool Writer4::WriteUint(unsigned u) { - unsigned digit = CountDecimalDigit_fast(u); - WriteUintReverse(os_->Push(digit) + digit, u); - return true; +inline bool Writer4::WriteUint(unsigned u) +{ unsigned digit = CountDecimalDigit_fast(u); + WriteUintReverse(os_->Push(digit) + digit, u); + return true; } template<> -inline bool Writer4::WriteUint(unsigned u) { - unsigned digit = CountDecimalDigit_fast(u); - WriteUintReverse(os_->Push(digit) + digit, u); - return true; +inline bool Writer4::WriteUint(unsigned u) +{ unsigned digit = CountDecimalDigit_fast(u); + WriteUintReverse(os_->Push(digit) + digit, u); + return true; } template<> -inline bool Writer4::WriteUint64(uint64_t u) { - unsigned digit = CountDecimalDigit64_fast(u); - WriteUint64Reverse(os_->Push(digit) + digit, u); - return true; +inline bool Writer4::WriteUint64(uint64_t u) +{ unsigned digit = CountDecimalDigit64_fast(u); + WriteUint64Reverse(os_->Push(digit) + digit, u); + return true; } template<> -inline bool Writer4::WriteUint64(uint64_t u) { - unsigned digit = CountDecimalDigit64_fast(u); - WriteUint64Reverse(os_->Push(digit) + digit, u); - return true; +inline bool Writer4::WriteUint64(uint64_t u) +{ unsigned digit = CountDecimalDigit64_fast(u); + WriteUint64Reverse(os_->Push(digit) + digit, u); + return true; } template -void itoa_Writer_StringBufferVerify() { - rapidjson::StringBuffer sb; - Writer writer(sb); - for (size_t j = 0; j < randvalCount; j++) { - char buffer[32]; - sprintf(buffer, "%d", randval[j]); - writer.WriteInt(randval[j]); - ASSERT_STREQ(buffer, sb.GetString()); - sb.Clear(); - } +void itoa_Writer_StringBufferVerify() +{ rapidjson::StringBuffer sb; + Writer writer(sb); + for (size_t j = 0; j < randvalCount; j++) + { char buffer[32]; + sprintf(buffer, "%d", randval[j]); + writer.WriteInt(randval[j]); + ASSERT_STREQ(buffer, sb.GetString()); + sb.Clear(); + } } template -void itoa_Writer_InsituStringStreamVerify() { - Writer writer; - for (size_t j = 0; j < randvalCount; j++) { - char buffer[32]; - sprintf(buffer, "%d", randval[j]); - char buffer2[32]; - rapidjson::InsituStringStream ss(buffer2); - writer.Reset(ss); - char* begin = ss.PutBegin(); - writer.WriteInt(randval[j]); - ss.Put('\0'); - ss.PutEnd(begin); - ASSERT_STREQ(buffer, buffer2); - } +void itoa_Writer_InsituStringStreamVerify() +{ Writer writer; + for (size_t j = 0; j < randvalCount; j++) + { char buffer[32]; + sprintf(buffer, "%d", randval[j]); + char buffer2[32]; + rapidjson::InsituStringStream ss(buffer2); + writer.Reset(ss); + char* begin = ss.PutBegin(); + writer.WriteInt(randval[j]); + ss.Put('\0'); + ss.PutEnd(begin); + ASSERT_STREQ(buffer, buffer2); + } } template -void itoa_Writer_StringBuffer() { - size_t length = 0; +void itoa_Writer_StringBuffer() +{ size_t length = 0; - rapidjson::StringBuffer sb; - Writer writer(sb); + rapidjson::StringBuffer sb; + Writer writer(sb); - for (size_t i = 0; i < kItoaTrialCount; i++) { - for (size_t j = 0; j < randvalCount; j++) { - writer.WriteInt(randval[j]); - length += sb.GetSize(); - sb.Clear(); - } + for (size_t i = 0; i < kItoaTrialCount; i++) + { for (size_t j = 0; j < randvalCount; j++) + { writer.WriteInt(randval[j]); + length += sb.GetSize(); + sb.Clear(); } - OUTPUT_LENGTH(length); + } + OUTPUT_LENGTH(length); } template -void itoa_Writer_InsituStringStream() { - size_t length = 0; - - char buffer[32]; - Writer writer; - for (size_t i = 0; i < kItoaTrialCount; i++) { - for (size_t j = 0; j < randvalCount; j++) { - rapidjson::InsituStringStream ss(buffer); - writer.Reset(ss); - char* begin = ss.PutBegin(); - writer.WriteInt(randval[j]); - length += ss.PutEnd(begin); - } - } - OUTPUT_LENGTH(length); +void itoa_Writer_InsituStringStream() +{ size_t length = 0; + + char buffer[32]; + Writer writer; + for (size_t i = 0; i < kItoaTrialCount; i++) + { for (size_t j = 0; j < randvalCount; j++) + { rapidjson::InsituStringStream ss(buffer); + writer.Reset(ss); + char* begin = ss.PutBegin(); + writer.WriteInt(randval[j]); + length += ss.PutEnd(begin); + } + } + OUTPUT_LENGTH(length); }; template -void itoa64_Writer_StringBufferVerify() { - rapidjson::StringBuffer sb; - Writer writer(sb); - for (size_t j = 0; j < randvalCount; j++) { - char buffer[32]; - int64_t x = randval[j] * randval[j]; - sprintf(buffer, "%" PRIi64, x); - writer.WriteInt64(x); - ASSERT_STREQ(buffer, sb.GetString()); - sb.Clear(); - } +void itoa64_Writer_StringBufferVerify() +{ rapidjson::StringBuffer sb; + Writer writer(sb); + for (size_t j = 0; j < randvalCount; j++) + { char buffer[32]; + int64_t x = randval[j] * randval[j]; + sprintf(buffer, "%" PRIi64, x); + writer.WriteInt64(x); + ASSERT_STREQ(buffer, sb.GetString()); + sb.Clear(); + } } template -void itoa64_Writer_InsituStringStreamVerify() { - Writer writer; - for (size_t j = 0; j < randvalCount; j++) { - char buffer[32]; - int64_t x = randval[j] * randval[j]; - sprintf(buffer, "%" PRIi64, x); - char buffer2[32]; - rapidjson::InsituStringStream ss(buffer2); - writer.Reset(ss); - char* begin = ss.PutBegin(); - writer.WriteInt64(x); - ss.Put('\0'); - ss.PutEnd(begin); - ASSERT_STREQ(buffer, buffer2); - } +void itoa64_Writer_InsituStringStreamVerify() +{ Writer writer; + for (size_t j = 0; j < randvalCount; j++) + { char buffer[32]; + int64_t x = randval[j] * randval[j]; + sprintf(buffer, "%" PRIi64, x); + char buffer2[32]; + rapidjson::InsituStringStream ss(buffer2); + writer.Reset(ss); + char* begin = ss.PutBegin(); + writer.WriteInt64(x); + ss.Put('\0'); + ss.PutEnd(begin); + ASSERT_STREQ(buffer, buffer2); + } } template -void itoa64_Writer_StringBuffer() { - size_t length = 0; +void itoa64_Writer_StringBuffer() +{ size_t length = 0; - rapidjson::StringBuffer sb; - Writer writer(sb); + rapidjson::StringBuffer sb; + Writer writer(sb); - for (size_t i = 0; i < kItoaTrialCount; i++) { - for (size_t j = 0; j < randvalCount; j++) { - writer.WriteInt64(randval[j] * randval[j]); - length += sb.GetSize(); - sb.Clear(); - } + for (size_t i = 0; i < kItoaTrialCount; i++) + { for (size_t j = 0; j < randvalCount; j++) + { writer.WriteInt64(randval[j] * randval[j]); + length += sb.GetSize(); + sb.Clear(); } - OUTPUT_LENGTH(length); + } + OUTPUT_LENGTH(length); } template -void itoa64_Writer_InsituStringStream() { - size_t length = 0; - - char buffer[32]; - Writer writer; - for (size_t i = 0; i < kItoaTrialCount; i++) { - for (size_t j = 0; j < randvalCount; j++) { - rapidjson::InsituStringStream ss(buffer); - writer.Reset(ss); - char* begin = ss.PutBegin(); - writer.WriteInt64(randval[j] * randval[j]); - length += ss.PutEnd(begin); - } - } - OUTPUT_LENGTH(length); +void itoa64_Writer_InsituStringStream() +{ size_t length = 0; + + char buffer[32]; + Writer writer; + for (size_t i = 0; i < kItoaTrialCount; i++) + { for (size_t j = 0; j < randvalCount; j++) + { rapidjson::InsituStringStream ss(buffer); + writer.Reset(ss); + char* begin = ss.PutBegin(); + writer.WriteInt64(randval[j] * randval[j]); + length += ss.PutEnd(begin); + } + } + OUTPUT_LENGTH(length); }; -// Full specialization for InsituStringStream to prevent memory copying +// Full specialization for InsituStringStream to prevent memory copying // (normally we will not use InsituStringStream for writing, just for testing) -namespace rapidjson { +namespace rapidjson +{ template<> -bool rapidjson::Writer::WriteInt(int i) { - char *buffer = os_->Push(11); - const char* end = internal::i32toa(i, buffer); - os_->Pop(11 - (end - buffer)); - return true; +bool rapidjson::Writer::WriteInt(int i) +{ char *buffer = os_->Push(11); + const char* end = internal::i32toa(i, buffer); + os_->Pop(11 - (end - buffer)); + return true; } template<> -bool Writer::WriteUint(unsigned u) { - char *buffer = os_->Push(10); - const char* end = internal::u32toa(u, buffer); - os_->Pop(10 - (end - buffer)); - return true; +bool Writer::WriteUint(unsigned u) +{ char *buffer = os_->Push(10); + const char* end = internal::u32toa(u, buffer); + os_->Pop(10 - (end - buffer)); + return true; } template<> -bool Writer::WriteInt64(int64_t i64) { - char *buffer = os_->Push(21); - const char* end = internal::i64toa(i64, buffer); - os_->Pop(21 - (end - buffer)); - return true; +bool Writer::WriteInt64(int64_t i64) +{ char *buffer = os_->Push(21); + const char* end = internal::i64toa(i64, buffer); + os_->Pop(21 - (end - buffer)); + return true; } template<> -bool Writer::WriteUint64(uint64_t u) { - char *buffer = os_->Push(20); - const char* end = internal::u64toa(u, buffer); - os_->Pop(20 - (end - buffer)); - return true; +bool Writer::WriteUint64(uint64_t u) +{ char *buffer = os_->Push(20); + const char* end = internal::u64toa(u, buffer); + os_->Pop(20 - (end - buffer)); + return true; } } // namespace rapidjson diff --git a/rapidjson/test/perftest/perftest.cpp b/rapidjson/test/perftest/perftest.cpp index 4e79f1f5180..c7b7337fbff 100644 --- a/rapidjson/test/perftest/perftest.cpp +++ b/rapidjson/test/perftest/perftest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,18 +7,19 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "perftest.h" -int main(int argc, char **argv) { +int main(int argc, char **argv) +{ #if _MSC_VER - _CrtSetDbgFlag ( _CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF ); - //void *testWhetherMemoryLeakDetectionWorks = malloc(1); + _CrtSetDbgFlag ( _CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF ); + //void *testWhetherMemoryLeakDetectionWorks = malloc(1); #endif - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); } diff --git a/rapidjson/test/perftest/perftest.h b/rapidjson/test/perftest/perftest.h index b098e414720..100472970ec 100644 --- a/rapidjson/test/perftest/perftest.h +++ b/rapidjson/test/perftest/perftest.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef PERFTEST_H_ @@ -62,119 +62,118 @@ #endif //! Base class for all performance tests -class PerfTest : public ::testing::Test { +class PerfTest : public ::testing::Test +{ public: - PerfTest() : filename_(), json_(), length_(), whitespace_(), whitespace_length_() {} - - virtual void SetUp() { - { - const char *paths[] = { - "data/sample.json", - "bin/data/sample.json", - "../bin/data/sample.json", - "../../bin/data/sample.json", - "../../../bin/data/sample.json" - }; - - FILE *fp = 0; - for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { - fp = fopen(filename_ = paths[i], "rb"); - if (fp) - break; - } - ASSERT_TRUE(fp != 0); - - fseek(fp, 0, SEEK_END); - length_ = (size_t)ftell(fp); - fseek(fp, 0, SEEK_SET); - json_ = (char*)malloc(length_ + 1); - ASSERT_EQ(length_, fread(json_, 1, length_, fp)); - json_[length_] = '\0'; - fclose(fp); - } - - // whitespace test - { - whitespace_length_ = 1024 * 1024; - whitespace_ = (char *)malloc(whitespace_length_ + 4); - char *p = whitespace_; - for (size_t i = 0; i < whitespace_length_; i += 4) { - *p++ = ' '; - *p++ = '\n'; - *p++ = '\r'; - *p++ = '\t'; - } - *p++ = '['; - *p++ = '0'; - *p++ = ']'; - *p++ = '\0'; - } + PerfTest() : filename_(), json_(), length_(), whitespace_(), whitespace_length_() {} + + virtual void SetUp() + { + { const char *paths[] = + { "data/sample.json", + "bin/data/sample.json", + "../bin/data/sample.json", + "../../bin/data/sample.json", + "../../../bin/data/sample.json" + }; + + FILE *fp = 0; + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) + { fp = fopen(filename_ = paths[i], "rb"); + if (fp) + break; + } + ASSERT_TRUE(fp != 0); + + fseek(fp, 0, SEEK_END); + length_ = (size_t)ftell(fp); + fseek(fp, 0, SEEK_SET); + json_ = (char*)malloc(length_ + 1); + ASSERT_EQ(length_, fread(json_, 1, length_, fp)); + json_[length_] = '\0'; + fclose(fp); + } - // types test - { - const char *typespaths[] = { - "data/types", - "bin/types", - "../bin/types", - "../../bin/types/", - "../../../bin/types" - }; - - const char* typesfilenames[] = { - "booleans.json", - "floats.json", - "guids.json", - "integers.json", - "mixed.json", - "nulls.json", - "paragraphs.json" - }; - - for (size_t j = 0; j < sizeof(typesfilenames) / sizeof(typesfilenames[0]); j++) { - types_[j] = 0; - for (size_t i = 0; i < sizeof(typespaths) / sizeof(typespaths[0]); i++) { - char filename[256]; - sprintf(filename, "%s/%s", typespaths[i], typesfilenames[j]); - if (FILE* fp = fopen(filename, "rb")) { - fseek(fp, 0, SEEK_END); - typesLength_[j] = (size_t)ftell(fp); - fseek(fp, 0, SEEK_SET); - types_[j] = (char*)malloc(typesLength_[j] + 1); - ASSERT_EQ(typesLength_[j], fread(types_[j], 1, typesLength_[j], fp)); - types_[j][typesLength_[j]] = '\0'; - fclose(fp); - break; - } - } - } - } + // whitespace test + { whitespace_length_ = 1024 * 1024; + whitespace_ = (char *)malloc(whitespace_length_ + 4); + char *p = whitespace_; + for (size_t i = 0; i < whitespace_length_; i += 4) + { *p++ = ' '; + *p++ = '\n'; + *p++ = '\r'; + *p++ = '\t'; + } + *p++ = '['; + *p++ = '0'; + *p++ = ']'; + *p++ = '\0'; } - virtual void TearDown() { - free(json_); - free(whitespace_); - json_ = 0; - whitespace_ = 0; - for (size_t i = 0; i < 7; i++) { - free(types_[i]); - types_[i] = 0; + // types test + { const char *typespaths[] = + { "data/types", + "bin/types", + "../bin/types", + "../../bin/types/", + "../../../bin/types" + }; + + const char* typesfilenames[] = + { "booleans.json", + "floats.json", + "guids.json", + "integers.json", + "mixed.json", + "nulls.json", + "paragraphs.json" + }; + + for (size_t j = 0; j < sizeof(typesfilenames) / sizeof(typesfilenames[0]); j++) + { types_[j] = 0; + for (size_t i = 0; i < sizeof(typespaths) / sizeof(typespaths[0]); i++) + { char filename[256]; + sprintf(filename, "%s/%s", typespaths[i], typesfilenames[j]); + if (FILE* fp = fopen(filename, "rb")) + { fseek(fp, 0, SEEK_END); + typesLength_[j] = (size_t)ftell(fp); + fseek(fp, 0, SEEK_SET); + types_[j] = (char*)malloc(typesLength_[j] + 1); + ASSERT_EQ(typesLength_[j], fread(types_[j], 1, typesLength_[j], fp)); + types_[j][typesLength_[j]] = '\0'; + fclose(fp); + break; + } } + } + } + } + + virtual void TearDown() + { free(json_); + free(whitespace_); + json_ = 0; + whitespace_ = 0; + for (size_t i = 0; i < 7; i++) + { free(types_[i]); + types_[i] = 0; } + } private: - PerfTest(const PerfTest&); - PerfTest& operator=(const PerfTest&); + PerfTest(const PerfTest&); + PerfTest& operator=(const PerfTest&); protected: - const char* filename_; - char *json_; - size_t length_; - char *whitespace_; - size_t whitespace_length_; - char *types_[7]; - size_t typesLength_[7]; - - static const size_t kTrialCount = 1000; + const char* filename_; + char *json_; + size_t length_; + char *whitespace_; + size_t whitespace_length_; + char *types_[7]; + size_t typesLength_[7]; + + static const size_t kTrialCount = 1000; }; #endif // __cplusplus diff --git a/rapidjson/test/perftest/platformtest.cpp b/rapidjson/test/perftest/platformtest.cpp index bb905ca73be..a222562060c 100644 --- a/rapidjson/test/perftest/platformtest.cpp +++ b/rapidjson/test/perftest/platformtest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "perftest.h" @@ -34,132 +34,133 @@ #endif #endif -class Platform : public PerfTest { +class Platform : public PerfTest +{ public: - virtual void SetUp() { - PerfTest::SetUp(); - - // temp buffer for testing - temp_ = (char *)malloc(length_ + 1); - memcpy(temp_, json_, length_); - checkSum_ = CheckSum(); - } - - char CheckSum() { - char c = 0; - for (size_t i = 0; i < length_; ++i) - c += temp_[i]; - return c; - } - - virtual void TearDown() { - PerfTest::TearDown(); - free(temp_); - } + virtual void SetUp() + { PerfTest::SetUp(); + + // temp buffer for testing + temp_ = (char *)malloc(length_ + 1); + memcpy(temp_, json_, length_); + checkSum_ = CheckSum(); + } + + char CheckSum() + { char c = 0; + for (size_t i = 0; i < length_; ++i) + c += temp_[i]; + return c; + } + + virtual void TearDown() + { PerfTest::TearDown(); + free(temp_); + } protected: - char *temp_; - char checkSum_; + char *temp_; + char checkSum_; }; -TEST_F(Platform, CheckSum) { - for (int i = 0; i < kTrialCount; i++) - EXPECT_EQ(checkSum_, CheckSum()); +TEST_F(Platform, CheckSum) +{ for (int i = 0; i < kTrialCount; i++) + EXPECT_EQ(checkSum_, CheckSum()); } -TEST_F(Platform, strlen) { - for (int i = 0; i < kTrialCount; i++) { - size_t l = strlen(json_); - EXPECT_EQ(length_, l); - } +TEST_F(Platform, strlen) +{ for (int i = 0; i < kTrialCount; i++) + { size_t l = strlen(json_); + EXPECT_EQ(length_, l); + } } -TEST_F(Platform, memcmp) { - for (int i = 0; i < kTrialCount; i++) { - EXPECT_EQ(0, memcmp(temp_, json_, length_)); - } +TEST_F(Platform, memcmp) +{ for (int i = 0; i < kTrialCount; i++) + { EXPECT_EQ(0, memcmp(temp_, json_, length_)); + } } -TEST_F(Platform, pow) { - double sum = 0; - for (int i = 0; i < kTrialCount * kTrialCount; i++) - sum += pow(10.0, i & 255); - EXPECT_GT(sum, 0.0); +TEST_F(Platform, pow) +{ double sum = 0; + for (int i = 0; i < kTrialCount * kTrialCount; i++) + sum += pow(10.0, i & 255); + EXPECT_GT(sum, 0.0); } -TEST_F(Platform, Whitespace_strlen) { - for (int i = 0; i < kTrialCount; i++) { - size_t l = strlen(whitespace_); - EXPECT_GT(l, whitespace_length_); - } +TEST_F(Platform, Whitespace_strlen) +{ for (int i = 0; i < kTrialCount; i++) + { size_t l = strlen(whitespace_); + EXPECT_GT(l, whitespace_length_); + } } -TEST_F(Platform, Whitespace_strspn) { - for (int i = 0; i < kTrialCount; i++) { - size_t l = strspn(whitespace_, " \n\r\t"); - EXPECT_EQ(whitespace_length_, l); - } +TEST_F(Platform, Whitespace_strspn) +{ for (int i = 0; i < kTrialCount; i++) + { size_t l = strspn(whitespace_, " \n\r\t"); + EXPECT_EQ(whitespace_length_, l); + } } -TEST_F(Platform, fread) { - for (int i = 0; i < kTrialCount; i++) { - FILE *fp = fopen(filename_, "rb"); - ASSERT_EQ(length_, fread(temp_, 1, length_, fp)); - EXPECT_EQ(checkSum_, CheckSum()); - fclose(fp); - } +TEST_F(Platform, fread) +{ for (int i = 0; i < kTrialCount; i++) + { FILE *fp = fopen(filename_, "rb"); + ASSERT_EQ(length_, fread(temp_, 1, length_, fp)); + EXPECT_EQ(checkSum_, CheckSum()); + fclose(fp); + } } #ifdef _MSC_VER -TEST_F(Platform, read) { - for (int i = 0; i < kTrialCount; i++) { - int fd = _open(filename_, _O_BINARY | _O_RDONLY); - ASSERT_NE(-1, fd); - ASSERT_EQ(length_, _read(fd, temp_, length_)); - EXPECT_EQ(checkSum_, CheckSum()); - _close(fd); - } +TEST_F(Platform, read) +{ for (int i = 0; i < kTrialCount; i++) + { int fd = _open(filename_, _O_BINARY | _O_RDONLY); + ASSERT_NE(-1, fd); + ASSERT_EQ(length_, _read(fd, temp_, length_)); + EXPECT_EQ(checkSum_, CheckSum()); + _close(fd); + } } #else -TEST_F(Platform, read) { - for (int i = 0; i < kTrialCount; i++) { - int fd = open(filename_, O_RDONLY); - ASSERT_NE(-1, fd); - ASSERT_EQ(length_, read(fd, temp_, length_)); - EXPECT_EQ(checkSum_, CheckSum()); - close(fd); - } +TEST_F(Platform, read) +{ for (int i = 0; i < kTrialCount; i++) + { int fd = open(filename_, O_RDONLY); + ASSERT_NE(-1, fd); + ASSERT_EQ(length_, read(fd, temp_, length_)); + EXPECT_EQ(checkSum_, CheckSum()); + close(fd); + } } #endif #ifdef _WIN32 -TEST_F(Platform, MapViewOfFile) { - for (int i = 0; i < kTrialCount; i++) { - HANDLE file = CreateFile(filename_, GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - ASSERT_NE(INVALID_HANDLE_VALUE, file); - HANDLE mapObject = CreateFileMapping(file, NULL, PAGE_READONLY, 0, length_, NULL); - ASSERT_NE(INVALID_HANDLE_VALUE, mapObject); - void *p = MapViewOfFile(mapObject, FILE_MAP_READ, 0, 0, length_); - ASSERT_TRUE(p != NULL); - EXPECT_EQ(checkSum_, CheckSum()); - ASSERT_TRUE(UnmapViewOfFile(p) == TRUE); - ASSERT_TRUE(CloseHandle(mapObject) == TRUE); - ASSERT_TRUE(CloseHandle(file) == TRUE); - } +TEST_F(Platform, MapViewOfFile) +{ for (int i = 0; i < kTrialCount; i++) + { HANDLE file = CreateFile(filename_, GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + ASSERT_NE(INVALID_HANDLE_VALUE, file); + HANDLE mapObject = CreateFileMapping(file, NULL, PAGE_READONLY, 0, length_, NULL); + ASSERT_NE(INVALID_HANDLE_VALUE, mapObject); + void *p = MapViewOfFile(mapObject, FILE_MAP_READ, 0, 0, length_); + ASSERT_TRUE(p != NULL); + EXPECT_EQ(checkSum_, CheckSum()); + ASSERT_TRUE(UnmapViewOfFile(p) == TRUE); + ASSERT_TRUE(CloseHandle(mapObject) == TRUE); + ASSERT_TRUE(CloseHandle(file) == TRUE); + } } #endif #ifdef _POSIX_MAPPED_FILES -TEST_F(Platform, mmap) { - for (int i = 0; i < kTrialCount; i++) { - int fd = open(filename_, O_RDONLY); - ASSERT_NE(-1, fd); - void *p = mmap(NULL, length_, PROT_READ, MAP_PRIVATE, fd, 0); - ASSERT_TRUE(p != NULL); - EXPECT_EQ(checkSum_, CheckSum()); - munmap(p, length_); - close(fd); - } +TEST_F(Platform, mmap) +{ for (int i = 0; i < kTrialCount; i++) + { int fd = open(filename_, O_RDONLY); + ASSERT_NE(-1, fd); + void *p = mmap(NULL, length_, PROT_READ, MAP_PRIVATE, fd, 0); + ASSERT_TRUE(p != NULL); + EXPECT_EQ(checkSum_, CheckSum()); + munmap(p, length_); + close(fd); + } } #endif diff --git a/rapidjson/test/perftest/rapidjsontest.cpp b/rapidjson/test/perftest/rapidjsontest.cpp index 675db3182a5..1f92aee496c 100644 --- a/rapidjson/test/perftest/rapidjsontest.cpp +++ b/rapidjson/test/perftest/rapidjsontest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "perftest.h" @@ -34,65 +34,66 @@ using namespace rapidjson; -class RapidJson : public PerfTest { +class RapidJson : public PerfTest +{ public: - RapidJson() : temp_(), doc_() {} + RapidJson() : temp_(), doc_() {} - virtual void SetUp() { - PerfTest::SetUp(); + virtual void SetUp() + { PerfTest::SetUp(); - // temp buffer for insitu parsing. - temp_ = (char *)malloc(length_ + 1); + // temp buffer for insitu parsing. + temp_ = (char *)malloc(length_ + 1); - // Parse as a document - EXPECT_FALSE(doc_.Parse(json_).HasParseError()); + // Parse as a document + EXPECT_FALSE(doc_.Parse(json_).HasParseError()); - for (size_t i = 0; i < 7; i++) - EXPECT_FALSE(typesDoc_[i].Parse(types_[i]).HasParseError()); - } + for (size_t i = 0; i < 7; i++) + EXPECT_FALSE(typesDoc_[i].Parse(types_[i]).HasParseError()); + } - virtual void TearDown() { - PerfTest::TearDown(); - free(temp_); - } + virtual void TearDown() + { PerfTest::TearDown(); + free(temp_); + } private: - RapidJson(const RapidJson&); - RapidJson& operator=(const RapidJson&); + RapidJson(const RapidJson&); + RapidJson& operator=(const RapidJson&); protected: - char *temp_; - Document doc_; - Document typesDoc_[7]; + char *temp_; + Document doc_; + Document typesDoc_[7]; }; -TEST_F(RapidJson, SIMD_SUFFIX(ReaderParseInsitu_DummyHandler)) { - for (size_t i = 0; i < kTrialCount; i++) { - memcpy(temp_, json_, length_ + 1); - InsituStringStream s(temp_); - BaseReaderHandler<> h; - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - } +TEST_F(RapidJson, SIMD_SUFFIX(ReaderParseInsitu_DummyHandler)) +{ for (size_t i = 0; i < kTrialCount; i++) + { memcpy(temp_, json_, length_ + 1); + InsituStringStream s(temp_); + BaseReaderHandler<> h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + } } -TEST_F(RapidJson, SIMD_SUFFIX(ReaderParseInsitu_DummyHandler_ValidateEncoding)) { - for (size_t i = 0; i < kTrialCount; i++) { - memcpy(temp_, json_, length_ + 1); - InsituStringStream s(temp_); - BaseReaderHandler<> h; - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - } +TEST_F(RapidJson, SIMD_SUFFIX(ReaderParseInsitu_DummyHandler_ValidateEncoding)) +{ for (size_t i = 0; i < kTrialCount; i++) + { memcpy(temp_, json_, length_ + 1); + InsituStringStream s(temp_); + BaseReaderHandler<> h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + } } -TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler)) { - for (size_t i = 0; i < kTrialCount; i++) { - StringStream s(json_); - BaseReaderHandler<> h; - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - } +TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler)) +{ for (size_t i = 0; i < kTrialCount; i++) + { StringStream s(json_); + BaseReaderHandler<> h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + } } #define TEST_TYPED(index, Name)\ @@ -124,155 +125,155 @@ TEST_TYPED(6, Paragraphs) #undef TEST_TYPED -TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_FullPrecision)) { - for (size_t i = 0; i < kTrialCount; i++) { - StringStream s(json_); - BaseReaderHandler<> h; - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - } -} - -TEST_F(RapidJson, SIMD_SUFFIX(ReaderParseIterative_DummyHandler)) { - for (size_t i = 0; i < kTrialCount; i++) { - StringStream s(json_); - BaseReaderHandler<> h; - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - } -} - -TEST_F(RapidJson, SIMD_SUFFIX(ReaderParseIterativeInsitu_DummyHandler)) { - for (size_t i = 0; i < kTrialCount; i++) { - memcpy(temp_, json_, length_ + 1); - InsituStringStream s(temp_); - BaseReaderHandler<> h; - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - } -} - -TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_ValidateEncoding)) { - for (size_t i = 0; i < kTrialCount; i++) { - StringStream s(json_); - BaseReaderHandler<> h; - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - } -} - -TEST_F(RapidJson, SIMD_SUFFIX(DocumentParseInsitu_MemoryPoolAllocator)) { - for (size_t i = 0; i < kTrialCount; i++) { - memcpy(temp_, json_, length_ + 1); - Document doc; - doc.ParseInsitu(temp_); - ASSERT_TRUE(doc.IsObject()); - } -} - -TEST_F(RapidJson, SIMD_SUFFIX(DocumentParseIterativeInsitu_MemoryPoolAllocator)) { - for (size_t i = 0; i < kTrialCount; i++) { - memcpy(temp_, json_, length_ + 1); - Document doc; - doc.ParseInsitu(temp_); - ASSERT_TRUE(doc.IsObject()); - } -} - -TEST_F(RapidJson, SIMD_SUFFIX(DocumentParse_MemoryPoolAllocator)) { - for (size_t i = 0; i < kTrialCount; i++) { - Document doc; - doc.Parse(json_); - ASSERT_TRUE(doc.IsObject()); - } -} - -TEST_F(RapidJson, SIMD_SUFFIX(DocumentParseLength_MemoryPoolAllocator)) { - for (size_t i = 0; i < kTrialCount; i++) { - Document doc; - doc.Parse(json_, length_); - ASSERT_TRUE(doc.IsObject()); - } +TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_FullPrecision)) +{ for (size_t i = 0; i < kTrialCount; i++) + { StringStream s(json_); + BaseReaderHandler<> h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + } +} + +TEST_F(RapidJson, SIMD_SUFFIX(ReaderParseIterative_DummyHandler)) +{ for (size_t i = 0; i < kTrialCount; i++) + { StringStream s(json_); + BaseReaderHandler<> h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + } +} + +TEST_F(RapidJson, SIMD_SUFFIX(ReaderParseIterativeInsitu_DummyHandler)) +{ for (size_t i = 0; i < kTrialCount; i++) + { memcpy(temp_, json_, length_ + 1); + InsituStringStream s(temp_); + BaseReaderHandler<> h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + } +} + +TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_ValidateEncoding)) +{ for (size_t i = 0; i < kTrialCount; i++) + { StringStream s(json_); + BaseReaderHandler<> h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + } +} + +TEST_F(RapidJson, SIMD_SUFFIX(DocumentParseInsitu_MemoryPoolAllocator)) +{ for (size_t i = 0; i < kTrialCount; i++) + { memcpy(temp_, json_, length_ + 1); + Document doc; + doc.ParseInsitu(temp_); + ASSERT_TRUE(doc.IsObject()); + } +} + +TEST_F(RapidJson, SIMD_SUFFIX(DocumentParseIterativeInsitu_MemoryPoolAllocator)) +{ for (size_t i = 0; i < kTrialCount; i++) + { memcpy(temp_, json_, length_ + 1); + Document doc; + doc.ParseInsitu(temp_); + ASSERT_TRUE(doc.IsObject()); + } +} + +TEST_F(RapidJson, SIMD_SUFFIX(DocumentParse_MemoryPoolAllocator)) +{ for (size_t i = 0; i < kTrialCount; i++) + { Document doc; + doc.Parse(json_); + ASSERT_TRUE(doc.IsObject()); + } +} + +TEST_F(RapidJson, SIMD_SUFFIX(DocumentParseLength_MemoryPoolAllocator)) +{ for (size_t i = 0; i < kTrialCount; i++) + { Document doc; + doc.Parse(json_, length_); + ASSERT_TRUE(doc.IsObject()); + } } #if RAPIDJSON_HAS_STDSTRING -TEST_F(RapidJson, SIMD_SUFFIX(DocumentParseStdString_MemoryPoolAllocator)) { - const std::string s(json_, length_); - for (size_t i = 0; i < kTrialCount; i++) { - Document doc; - doc.Parse(s); - ASSERT_TRUE(doc.IsObject()); - } +TEST_F(RapidJson, SIMD_SUFFIX(DocumentParseStdString_MemoryPoolAllocator)) +{ const std::string s(json_, length_); + for (size_t i = 0; i < kTrialCount; i++) + { Document doc; + doc.Parse(s); + ASSERT_TRUE(doc.IsObject()); + } } #endif -TEST_F(RapidJson, SIMD_SUFFIX(DocumentParseIterative_MemoryPoolAllocator)) { - for (size_t i = 0; i < kTrialCount; i++) { - Document doc; - doc.Parse(json_); - ASSERT_TRUE(doc.IsObject()); - } +TEST_F(RapidJson, SIMD_SUFFIX(DocumentParseIterative_MemoryPoolAllocator)) +{ for (size_t i = 0; i < kTrialCount; i++) + { Document doc; + doc.Parse(json_); + ASSERT_TRUE(doc.IsObject()); + } } -TEST_F(RapidJson, SIMD_SUFFIX(DocumentParse_CrtAllocator)) { - for (size_t i = 0; i < kTrialCount; i++) { - memcpy(temp_, json_, length_ + 1); - GenericDocument, CrtAllocator> doc; - doc.Parse(temp_); - ASSERT_TRUE(doc.IsObject()); - } +TEST_F(RapidJson, SIMD_SUFFIX(DocumentParse_CrtAllocator)) +{ for (size_t i = 0; i < kTrialCount; i++) + { memcpy(temp_, json_, length_ + 1); + GenericDocument, CrtAllocator> doc; + doc.Parse(temp_); + ASSERT_TRUE(doc.IsObject()); + } } -TEST_F(RapidJson, SIMD_SUFFIX(DocumentParseEncodedInputStream_MemoryStream)) { - for (size_t i = 0; i < kTrialCount; i++) { - MemoryStream ms(json_, length_); - EncodedInputStream, MemoryStream> is(ms); - Document doc; - doc.ParseStream<0, UTF8<> >(is); - ASSERT_TRUE(doc.IsObject()); - } +TEST_F(RapidJson, SIMD_SUFFIX(DocumentParseEncodedInputStream_MemoryStream)) +{ for (size_t i = 0; i < kTrialCount; i++) + { MemoryStream ms(json_, length_); + EncodedInputStream, MemoryStream> is(ms); + Document doc; + doc.ParseStream<0, UTF8<> >(is); + ASSERT_TRUE(doc.IsObject()); + } } -TEST_F(RapidJson, SIMD_SUFFIX(DocumentParseAutoUTFInputStream_MemoryStream)) { - for (size_t i = 0; i < kTrialCount; i++) { - MemoryStream ms(json_, length_); - AutoUTFInputStream is(ms); - Document doc; - doc.ParseStream<0, AutoUTF >(is); - ASSERT_TRUE(doc.IsObject()); - } +TEST_F(RapidJson, SIMD_SUFFIX(DocumentParseAutoUTFInputStream_MemoryStream)) +{ for (size_t i = 0; i < kTrialCount; i++) + { MemoryStream ms(json_, length_); + AutoUTFInputStream is(ms); + Document doc; + doc.ParseStream<0, AutoUTF >(is); + ASSERT_TRUE(doc.IsObject()); + } } template -size_t Traverse(const T& value) { - size_t count = 1; - switch(value.GetType()) { - case kObjectType: - for (typename T::ConstMemberIterator itr = value.MemberBegin(); itr != value.MemberEnd(); ++itr) { - count++; // name - count += Traverse(itr->value); - } - break; - - case kArrayType: - for (typename T::ConstValueIterator itr = value.Begin(); itr != value.End(); ++itr) - count += Traverse(*itr); - break; - - default: - // Do nothing. - break; - } - return count; -} - -TEST_F(RapidJson, DocumentTraverse) { - for (size_t i = 0; i < kTrialCount; i++) { - size_t count = Traverse(doc_); - EXPECT_EQ(4339u, count); - //if (i == 0) - // std::cout << count << std::endl; - } +size_t Traverse(const T& value) +{ size_t count = 1; + switch(value.GetType()) + { case kObjectType: + for (typename T::ConstMemberIterator itr = value.MemberBegin(); itr != value.MemberEnd(); ++itr) + { count++; // name + count += Traverse(itr->value); + } + break; + + case kArrayType: + for (typename T::ConstValueIterator itr = value.Begin(); itr != value.End(); ++itr) + count += Traverse(*itr); + break; + + default: + // Do nothing. + break; + } + return count; +} + +TEST_F(RapidJson, DocumentTraverse) +{ for (size_t i = 0; i < kTrialCount; i++) + { size_t count = Traverse(doc_); + EXPECT_EQ(4339u, count); + //if (i == 0) + // std::cout << count << std::endl; + } } #ifdef __GNUC__ @@ -280,56 +281,56 @@ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) #endif -struct ValueCounter : public BaseReaderHandler<> { - ValueCounter() : count_(1) {} // root +struct ValueCounter : public BaseReaderHandler<> +{ ValueCounter() : count_(1) {} // root - bool EndObject(SizeType memberCount) { count_ += memberCount * 2; return true; } - bool EndArray(SizeType elementCount) { count_ += elementCount; return true; } + bool EndObject(SizeType memberCount) { count_ += memberCount * 2; return true; } + bool EndArray(SizeType elementCount) { count_ += elementCount; return true; } - SizeType count_; + SizeType count_; }; #ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif -TEST_F(RapidJson, DocumentAccept) { - for (size_t i = 0; i < kTrialCount; i++) { - ValueCounter counter; - doc_.Accept(counter); - EXPECT_EQ(4339u, counter.count_); - } +TEST_F(RapidJson, DocumentAccept) +{ for (size_t i = 0; i < kTrialCount; i++) + { ValueCounter counter; + doc_.Accept(counter); + EXPECT_EQ(4339u, counter.count_); + } } -struct NullStream { - typedef char Ch; +struct NullStream +{ typedef char Ch; - NullStream() /*: length_(0)*/ {} - void Put(Ch) { /*++length_;*/ } - void Flush() {} - //size_t length_; + NullStream() /*: length_(0)*/ {} + void Put(Ch) { /*++length_;*/ } + void Flush() {} + //size_t length_; }; -TEST_F(RapidJson, Writer_NullStream) { - for (size_t i = 0; i < kTrialCount; i++) { - NullStream s; - Writer writer(s); - doc_.Accept(writer); - //if (i == 0) - // std::cout << s.length_ << std::endl; - } +TEST_F(RapidJson, Writer_NullStream) +{ for (size_t i = 0; i < kTrialCount; i++) + { NullStream s; + Writer writer(s); + doc_.Accept(writer); + //if (i == 0) + // std::cout << s.length_ << std::endl; + } } -TEST_F(RapidJson, SIMD_SUFFIX(Writer_StringBuffer)) { - for (size_t i = 0; i < kTrialCount; i++) { - StringBuffer s(0, 1024 * 1024); - Writer writer(s); - doc_.Accept(writer); - const char* str = s.GetString(); - (void)str; - //if (i == 0) - // std::cout << strlen(str) << std::endl; - } +TEST_F(RapidJson, SIMD_SUFFIX(Writer_StringBuffer)) +{ for (size_t i = 0; i < kTrialCount; i++) + { StringBuffer s(0, 1024 * 1024); + Writer writer(s); + doc_.Accept(writer); + const char* str = s.GetString(); + (void)str; + //if (i == 0) + // std::cout << strlen(str) << std::endl; + } } #define TEST_TYPED(index, Name)\ @@ -353,89 +354,89 @@ TEST_TYPED(6, Paragraphs) #undef TEST_TYPED -TEST_F(RapidJson, SIMD_SUFFIX(PrettyWriter_StringBuffer)) { - for (size_t i = 0; i < kTrialCount; i++) { - StringBuffer s(0, 2048 * 1024); - PrettyWriter writer(s); - writer.SetIndent(' ', 1); - doc_.Accept(writer); - const char* str = s.GetString(); - (void)str; - //if (i == 0) - // std::cout << strlen(str) << std::endl; - } -} - -TEST_F(RapidJson, internal_Pow10) { - double sum = 0; - for (size_t i = 0; i < kTrialCount * kTrialCount; i++) - sum += internal::Pow10(int(i & 255)); - EXPECT_GT(sum, 0.0); -} - -TEST_F(RapidJson, SkipWhitespace_Basic) { - for (size_t i = 0; i < kTrialCount; i++) { - rapidjson::StringStream s(whitespace_); - while (s.Peek() == ' ' || s.Peek() == '\n' || s.Peek() == '\r' || s.Peek() == '\t') - s.Take(); - ASSERT_EQ('[', s.Peek()); - } -} - -TEST_F(RapidJson, SIMD_SUFFIX(SkipWhitespace)) { - for (size_t i = 0; i < kTrialCount; i++) { - rapidjson::StringStream s(whitespace_); - rapidjson::SkipWhitespace(s); - ASSERT_EQ('[', s.Peek()); - } -} - -TEST_F(RapidJson, SkipWhitespace_strspn) { - for (size_t i = 0; i < kTrialCount; i++) { - const char* s = whitespace_ + std::strspn(whitespace_, " \t\r\n"); - ASSERT_EQ('[', *s); - } -} - -TEST_F(RapidJson, UTF8_Validate) { - NullStream os; - - for (size_t i = 0; i < kTrialCount; i++) { - StringStream is(json_); - bool result = true; - while (is.Peek() != '\0') - result &= UTF8<>::Validate(is, os); - EXPECT_TRUE(result); - } -} - -TEST_F(RapidJson, FileReadStream) { - for (size_t i = 0; i < kTrialCount; i++) { - FILE *fp = fopen(filename_, "rb"); - char buffer[65536]; - FileReadStream s(fp, buffer, sizeof(buffer)); - while (s.Take() != '\0') - ; - fclose(fp); - } -} - -TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_FileReadStream)) { - for (size_t i = 0; i < kTrialCount; i++) { - FILE *fp = fopen(filename_, "rb"); - char buffer[65536]; - FileReadStream s(fp, buffer, sizeof(buffer)); - BaseReaderHandler<> h; - Reader reader; - reader.Parse(s, h); - fclose(fp); - } -} - -TEST_F(RapidJson, StringBuffer) { - StringBuffer sb; - for (int i = 0; i < 32 * 1024 * 1024; i++) - sb.Put(i & 0x7f); +TEST_F(RapidJson, SIMD_SUFFIX(PrettyWriter_StringBuffer)) +{ for (size_t i = 0; i < kTrialCount; i++) + { StringBuffer s(0, 2048 * 1024); + PrettyWriter writer(s); + writer.SetIndent(' ', 1); + doc_.Accept(writer); + const char* str = s.GetString(); + (void)str; + //if (i == 0) + // std::cout << strlen(str) << std::endl; + } +} + +TEST_F(RapidJson, internal_Pow10) +{ double sum = 0; + for (size_t i = 0; i < kTrialCount * kTrialCount; i++) + sum += internal::Pow10(int(i & 255)); + EXPECT_GT(sum, 0.0); +} + +TEST_F(RapidJson, SkipWhitespace_Basic) +{ for (size_t i = 0; i < kTrialCount; i++) + { rapidjson::StringStream s(whitespace_); + while (s.Peek() == ' ' || s.Peek() == '\n' || s.Peek() == '\r' || s.Peek() == '\t') + s.Take(); + ASSERT_EQ('[', s.Peek()); + } +} + +TEST_F(RapidJson, SIMD_SUFFIX(SkipWhitespace)) +{ for (size_t i = 0; i < kTrialCount; i++) + { rapidjson::StringStream s(whitespace_); + rapidjson::SkipWhitespace(s); + ASSERT_EQ('[', s.Peek()); + } +} + +TEST_F(RapidJson, SkipWhitespace_strspn) +{ for (size_t i = 0; i < kTrialCount; i++) + { const char* s = whitespace_ + std::strspn(whitespace_, " \t\r\n"); + ASSERT_EQ('[', *s); + } +} + +TEST_F(RapidJson, UTF8_Validate) +{ NullStream os; + + for (size_t i = 0; i < kTrialCount; i++) + { StringStream is(json_); + bool result = true; + while (is.Peek() != '\0') + result &= UTF8<>::Validate(is, os); + EXPECT_TRUE(result); + } +} + +TEST_F(RapidJson, FileReadStream) +{ for (size_t i = 0; i < kTrialCount; i++) + { FILE *fp = fopen(filename_, "rb"); + char buffer[65536]; + FileReadStream s(fp, buffer, sizeof(buffer)); + while (s.Take() != '\0') + ; + fclose(fp); + } +} + +TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_FileReadStream)) +{ for (size_t i = 0; i < kTrialCount; i++) + { FILE *fp = fopen(filename_, "rb"); + char buffer[65536]; + FileReadStream s(fp, buffer, sizeof(buffer)); + BaseReaderHandler<> h; + Reader reader; + reader.Parse(s, h); + fclose(fp); + } +} + +TEST_F(RapidJson, StringBuffer) +{ StringBuffer sb; + for (int i = 0; i < 32 * 1024 * 1024; i++) + sb.Put(i & 0x7f); } #endif // TEST_RAPIDJSON diff --git a/rapidjson/test/perftest/schematest.cpp b/rapidjson/test/perftest/schematest.cpp index 468f5fe6f6a..dc88d16d842 100644 --- a/rapidjson/test/perftest/schematest.cpp +++ b/rapidjson/test/perftest/schematest.cpp @@ -12,205 +12,206 @@ using namespace rapidjson; template -static char* ReadFile(const char* filename, Allocator& allocator) { - const char *paths[] = { - "", - "bin/", - "../bin/", - "../../bin/", - "../../../bin/" - }; - char buffer[1024]; - FILE *fp = 0; - for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { - sprintf(buffer, "%s%s", paths[i], filename); - fp = fopen(buffer, "rb"); - if (fp) - break; - } - - if (!fp) - return 0; - - fseek(fp, 0, SEEK_END); - size_t length = static_cast(ftell(fp)); - fseek(fp, 0, SEEK_SET); - char* json = reinterpret_cast(allocator.Malloc(length + 1)); - size_t readLength = fread(json, 1, length, fp); - json[readLength] = '\0'; - fclose(fp); - return json; +static char* ReadFile(const char* filename, Allocator& allocator) +{ const char *paths[] = + { "", + "bin/", + "../bin/", + "../../bin/", + "../../../bin/" + }; + char buffer[1024]; + FILE *fp = 0; + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) + { sprintf(buffer, "%s%s", paths[i], filename); + fp = fopen(buffer, "rb"); + if (fp) + break; + } + + if (!fp) + return 0; + + fseek(fp, 0, SEEK_END); + size_t length = static_cast(ftell(fp)); + fseek(fp, 0, SEEK_SET); + char* json = reinterpret_cast(allocator.Malloc(length + 1)); + size_t readLength = fread(json, 1, length, fp); + json[readLength] = '\0'; + fclose(fp); + return json; } -class Schema : public PerfTest { +class Schema : public PerfTest +{ public: - Schema() {} - - virtual void SetUp() { - PerfTest::SetUp(); - - const char* filenames[] = { - "additionalItems.json", - "additionalProperties.json", - "allOf.json", - "anyOf.json", - "default.json", - "definitions.json", - "dependencies.json", - "enum.json", - "items.json", - "maximum.json", - "maxItems.json", - "maxLength.json", - "maxProperties.json", - "minimum.json", - "minItems.json", - "minLength.json", - "minProperties.json", - "multipleOf.json", - "not.json", - "oneOf.json", - "pattern.json", - "patternProperties.json", - "properties.json", - "ref.json", - "refRemote.json", - "required.json", - "type.json", - "uniqueItems.json" - }; - - char jsonBuffer[65536]; - MemoryPoolAllocator<> jsonAllocator(jsonBuffer, sizeof(jsonBuffer)); - - for (size_t i = 0; i < ARRAY_SIZE(filenames); i++) { - char filename[FILENAME_MAX]; - sprintf(filename, "jsonschema/tests/draft4/%s", filenames[i]); - char* json = ReadFile(filename, jsonAllocator); - if (!json) { - printf("json test suite file %s not found", filename); - return; - } - - Document d; - d.Parse(json); - if (d.HasParseError()) { - printf("json test suite file %s has parse error", filename); - return; - } - - for (Value::ConstValueIterator schemaItr = d.Begin(); schemaItr != d.End(); ++schemaItr) { - std::string schemaDescription = (*schemaItr)["description"].GetString(); - if (IsExcludeTestSuite(schemaDescription)) - continue; - - TestSuite* ts = new TestSuite; - ts->schema = new SchemaDocument((*schemaItr)["schema"]); - - const Value& tests = (*schemaItr)["tests"]; - for (Value::ConstValueIterator testItr = tests.Begin(); testItr != tests.End(); ++testItr) { - if (IsExcludeTest(schemaDescription + ", " + (*testItr)["description"].GetString())) - continue; - - Document* d2 = new Document; - d2->CopyFrom((*testItr)["data"], d2->GetAllocator()); - ts->tests.push_back(d2); - } - testSuites.push_back(ts); - } + Schema() {} + + virtual void SetUp() + { PerfTest::SetUp(); + + const char* filenames[] = + { "additionalItems.json", + "additionalProperties.json", + "allOf.json", + "anyOf.json", + "default.json", + "definitions.json", + "dependencies.json", + "enum.json", + "items.json", + "maximum.json", + "maxItems.json", + "maxLength.json", + "maxProperties.json", + "minimum.json", + "minItems.json", + "minLength.json", + "minProperties.json", + "multipleOf.json", + "not.json", + "oneOf.json", + "pattern.json", + "patternProperties.json", + "properties.json", + "ref.json", + "refRemote.json", + "required.json", + "type.json", + "uniqueItems.json" + }; + + char jsonBuffer[65536]; + MemoryPoolAllocator<> jsonAllocator(jsonBuffer, sizeof(jsonBuffer)); + + for (size_t i = 0; i < ARRAY_SIZE(filenames); i++) + { char filename[FILENAME_MAX]; + sprintf(filename, "jsonschema/tests/draft4/%s", filenames[i]); + char* json = ReadFile(filename, jsonAllocator); + if (!json) + { printf("json test suite file %s not found", filename); + return; + } + + Document d; + d.Parse(json); + if (d.HasParseError()) + { printf("json test suite file %s has parse error", filename); + return; + } + + for (Value::ConstValueIterator schemaItr = d.Begin(); schemaItr != d.End(); ++schemaItr) + { std::string schemaDescription = (*schemaItr)["description"].GetString(); + if (IsExcludeTestSuite(schemaDescription)) + continue; + + TestSuite* ts = new TestSuite; + ts->schema = new SchemaDocument((*schemaItr)["schema"]); + + const Value& tests = (*schemaItr)["tests"]; + for (Value::ConstValueIterator testItr = tests.Begin(); testItr != tests.End(); ++testItr) + { if (IsExcludeTest(schemaDescription + ", " + (*testItr)["description"].GetString())) + continue; + + Document* d2 = new Document; + d2->CopyFrom((*testItr)["data"], d2->GetAllocator()); + ts->tests.push_back(d2); } + testSuites.push_back(ts); + } } + } - virtual void TearDown() { - PerfTest::TearDown(); - for (TestSuiteList::const_iterator itr = testSuites.begin(); itr != testSuites.end(); ++itr) - delete *itr; - testSuites.clear(); - } + virtual void TearDown() + { PerfTest::TearDown(); + for (TestSuiteList::const_iterator itr = testSuites.begin(); itr != testSuites.end(); ++itr) + delete *itr; + testSuites.clear(); + } private: - // Using the same exclusion in https://github.com/json-schema/JSON-Schema-Test-Suite - static bool IsExcludeTestSuite(const std::string& description) { - const char* excludeTestSuites[] = { - //lost failing these tests - "remote ref", - "remote ref, containing refs itself", - "fragment within remote ref", - "ref within remote ref", - "change resolution scope", - // these below were added to get jsck in the benchmarks) - "uniqueItems validation", - "valid definition", - "invalid definition" - }; - - for (size_t i = 0; i < ARRAY_SIZE(excludeTestSuites); i++) - if (excludeTestSuites[i] == description) - return true; - return false; - } + // Using the same exclusion in https://github.com/json-schema/JSON-Schema-Test-Suite + static bool IsExcludeTestSuite(const std::string& description) + { const char* excludeTestSuites[] = + { //lost failing these tests + "remote ref", + "remote ref, containing refs itself", + "fragment within remote ref", + "ref within remote ref", + "change resolution scope", + // these below were added to get jsck in the benchmarks) + "uniqueItems validation", + "valid definition", + "invalid definition" + }; - // Using the same exclusion in https://github.com/json-schema/JSON-Schema-Test-Suite - static bool IsExcludeTest(const std::string& description) { - const char* excludeTests[] = { - //lots of validators fail these - "invalid definition, invalid definition schema", - "maxLength validation, two supplementary Unicode code points is long enough", - "minLength validation, one supplementary Unicode code point is not long enough", - //this is to get tv4 in the benchmarks - "heterogeneous enum validation, something else is invalid" - }; - - for (size_t i = 0; i < ARRAY_SIZE(excludeTests); i++) - if (excludeTests[i] == description) - return true; - return false; - } + for (size_t i = 0; i < ARRAY_SIZE(excludeTestSuites); i++) + if (excludeTestSuites[i] == description) + return true; + return false; + } + + // Using the same exclusion in https://github.com/json-schema/JSON-Schema-Test-Suite + static bool IsExcludeTest(const std::string& description) + { const char* excludeTests[] = + { //lots of validators fail these + "invalid definition, invalid definition schema", + "maxLength validation, two supplementary Unicode code points is long enough", + "minLength validation, one supplementary Unicode code point is not long enough", + //this is to get tv4 in the benchmarks + "heterogeneous enum validation, something else is invalid" + }; - Schema(const Schema&); - Schema& operator=(const Schema&); + for (size_t i = 0; i < ARRAY_SIZE(excludeTests); i++) + if (excludeTests[i] == description) + return true; + return false; + } + + Schema(const Schema&); + Schema& operator=(const Schema&); protected: - typedef std::vector DocumentList; - - struct TestSuite { - TestSuite() : schema() {} - ~TestSuite() { - delete schema; - for (DocumentList::iterator itr = tests.begin(); itr != tests.end(); ++itr) - delete *itr; - } - SchemaDocument* schema; - DocumentList tests; - }; + typedef std::vector DocumentList; + + struct TestSuite +{ TestSuite() : schema() {} + ~TestSuite() + { delete schema; + for (DocumentList::iterator itr = tests.begin(); itr != tests.end(); ++itr) + delete *itr; + } + SchemaDocument* schema; + DocumentList tests; + }; - typedef std::vector TestSuiteList; - TestSuiteList testSuites; + typedef std::vector TestSuiteList; + TestSuiteList testSuites; }; -TEST_F(Schema, TestSuite) { - char validatorBuffer[65536]; - MemoryPoolAllocator<> validatorAllocator(validatorBuffer, sizeof(validatorBuffer)); - - const int trialCount = 100000; - int testCount = 0; - clock_t start = clock(); - for (int i = 0; i < trialCount; i++) { - for (TestSuiteList::const_iterator itr = testSuites.begin(); itr != testSuites.end(); ++itr) { - const TestSuite& ts = **itr; - GenericSchemaValidator >, MemoryPoolAllocator<> > validator(*ts.schema, &validatorAllocator); - for (DocumentList::const_iterator testItr = ts.tests.begin(); testItr != ts.tests.end(); ++testItr) { - validator.Reset(); - (*testItr)->Accept(validator); - testCount++; - } - validatorAllocator.Clear(); - } +TEST_F(Schema, TestSuite) +{ char validatorBuffer[65536]; + MemoryPoolAllocator<> validatorAllocator(validatorBuffer, sizeof(validatorBuffer)); + + const int trialCount = 100000; + int testCount = 0; + clock_t start = clock(); + for (int i = 0; i < trialCount; i++) + { for (TestSuiteList::const_iterator itr = testSuites.begin(); itr != testSuites.end(); ++itr) + { const TestSuite& ts = **itr; + GenericSchemaValidator >, MemoryPoolAllocator<> > validator(*ts.schema, &validatorAllocator); + for (DocumentList::const_iterator testItr = ts.tests.begin(); testItr != ts.tests.end(); ++testItr) + { validator.Reset(); + (*testItr)->Accept(validator); + testCount++; + } + validatorAllocator.Clear(); } - clock_t end = clock(); - double duration = double(end - start) / CLOCKS_PER_SEC; - printf("%d trials in %f s -> %f trials per sec\n", trialCount, duration, trialCount / duration); - printf("%d tests per trial\n", testCount / trialCount); + } + clock_t end = clock(); + double duration = double(end - start) / CLOCKS_PER_SEC; + printf("%d trials in %f s -> %f trials per sec\n", trialCount, duration, trialCount / duration); + printf("%d tests per trial\n", testCount / trialCount); } #endif diff --git a/rapidjson/test/unittest/allocatorstest.cpp b/rapidjson/test/unittest/allocatorstest.cpp index a5958de199e..22c3113205e 100644 --- a/rapidjson/test/unittest/allocatorstest.cpp +++ b/rapidjson/test/unittest/allocatorstest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -19,84 +19,85 @@ using namespace rapidjson; template -void TestAllocator(Allocator& a) { - EXPECT_TRUE(a.Malloc(0) == 0); +void TestAllocator(Allocator& a) +{ EXPECT_TRUE(a.Malloc(0) == 0); - uint8_t* p = static_cast(a.Malloc(100)); - EXPECT_TRUE(p != 0); - for (size_t i = 0; i < 100; i++) - p[i] = static_cast(i); + uint8_t* p = static_cast(a.Malloc(100)); + EXPECT_TRUE(p != 0); + for (size_t i = 0; i < 100; i++) + p[i] = static_cast(i); - // Expand - uint8_t* q = static_cast(a.Realloc(p, 100, 200)); - EXPECT_TRUE(q != 0); - for (size_t i = 0; i < 100; i++) - EXPECT_EQ(i, q[i]); - for (size_t i = 100; i < 200; i++) - q[i] = static_cast(i); + // Expand + uint8_t* q = static_cast(a.Realloc(p, 100, 200)); + EXPECT_TRUE(q != 0); + for (size_t i = 0; i < 100; i++) + EXPECT_EQ(i, q[i]); + for (size_t i = 100; i < 200; i++) + q[i] = static_cast(i); - // Shrink - uint8_t *r = static_cast(a.Realloc(q, 200, 150)); - EXPECT_TRUE(r != 0); - for (size_t i = 0; i < 150; i++) - EXPECT_EQ(i, r[i]); + // Shrink + uint8_t *r = static_cast(a.Realloc(q, 200, 150)); + EXPECT_TRUE(r != 0); + for (size_t i = 0; i < 150; i++) + EXPECT_EQ(i, r[i]); - Allocator::Free(r); + Allocator::Free(r); - // Realloc to zero size - EXPECT_TRUE(a.Realloc(a.Malloc(1), 1, 0) == 0); + // Realloc to zero size + EXPECT_TRUE(a.Realloc(a.Malloc(1), 1, 0) == 0); } -TEST(Allocator, CrtAllocator) { - CrtAllocator a; - TestAllocator(a); +TEST(Allocator, CrtAllocator) +{ CrtAllocator a; + TestAllocator(a); } -TEST(Allocator, MemoryPoolAllocator) { - MemoryPoolAllocator<> a; - TestAllocator(a); +TEST(Allocator, MemoryPoolAllocator) +{ MemoryPoolAllocator<> a; + TestAllocator(a); - for (size_t i = 1; i < 1000; i++) { - EXPECT_TRUE(a.Malloc(i) != 0); - EXPECT_LE(a.Size(), a.Capacity()); - } + for (size_t i = 1; i < 1000; i++) + { EXPECT_TRUE(a.Malloc(i) != 0); + EXPECT_LE(a.Size(), a.Capacity()); + } } -TEST(Allocator, Alignment) { +TEST(Allocator, Alignment) +{ #if RAPIDJSON_64BIT == 1 - EXPECT_EQ(RAPIDJSON_UINT64_C2(0x00000000, 0x00000000), RAPIDJSON_ALIGN(0)); - for (uint64_t i = 1; i < 8; i++) { - EXPECT_EQ(RAPIDJSON_UINT64_C2(0x00000000, 0x00000008), RAPIDJSON_ALIGN(i)); - EXPECT_EQ(RAPIDJSON_UINT64_C2(0x00000000, 0x00000010), RAPIDJSON_ALIGN(RAPIDJSON_UINT64_C2(0x00000000, 0x00000008) + i)); - EXPECT_EQ(RAPIDJSON_UINT64_C2(0x00000001, 0x00000000), RAPIDJSON_ALIGN(RAPIDJSON_UINT64_C2(0x00000000, 0xFFFFFFF8) + i)); - EXPECT_EQ(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0xFFFFFFF8), RAPIDJSON_ALIGN(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0xFFFFFFF0) + i)); - } + EXPECT_EQ(RAPIDJSON_UINT64_C2(0x00000000, 0x00000000), RAPIDJSON_ALIGN(0)); + for (uint64_t i = 1; i < 8; i++) + { EXPECT_EQ(RAPIDJSON_UINT64_C2(0x00000000, 0x00000008), RAPIDJSON_ALIGN(i)); + EXPECT_EQ(RAPIDJSON_UINT64_C2(0x00000000, 0x00000010), RAPIDJSON_ALIGN(RAPIDJSON_UINT64_C2(0x00000000, 0x00000008) + i)); + EXPECT_EQ(RAPIDJSON_UINT64_C2(0x00000001, 0x00000000), RAPIDJSON_ALIGN(RAPIDJSON_UINT64_C2(0x00000000, 0xFFFFFFF8) + i)); + EXPECT_EQ(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0xFFFFFFF8), RAPIDJSON_ALIGN(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0xFFFFFFF0) + i)); + } #else - EXPECT_EQ(0u, RAPIDJSON_ALIGN(0u)); - for (uint32_t i = 1; i < 4; i++) { - EXPECT_EQ(4u, RAPIDJSON_ALIGN(i)); - EXPECT_EQ(8u, RAPIDJSON_ALIGN(4u + i)); - EXPECT_EQ(0xFFFFFFF8u, RAPIDJSON_ALIGN(0xFFFFFFF4u + i)); - EXPECT_EQ(0xFFFFFFFCu, RAPIDJSON_ALIGN(0xFFFFFFF8u + i)); - } + EXPECT_EQ(0u, RAPIDJSON_ALIGN(0u)); + for (uint32_t i = 1; i < 4; i++) + { EXPECT_EQ(4u, RAPIDJSON_ALIGN(i)); + EXPECT_EQ(8u, RAPIDJSON_ALIGN(4u + i)); + EXPECT_EQ(0xFFFFFFF8u, RAPIDJSON_ALIGN(0xFFFFFFF4u + i)); + EXPECT_EQ(0xFFFFFFFCu, RAPIDJSON_ALIGN(0xFFFFFFF8u + i)); + } #endif } -TEST(Allocator, Issue399) { - MemoryPoolAllocator<> a; - void* p = a.Malloc(100); - void* q = a.Realloc(p, 100, 200); - EXPECT_EQ(p, q); +TEST(Allocator, Issue399) +{ MemoryPoolAllocator<> a; + void* p = a.Malloc(100); + void* q = a.Realloc(p, 100, 200); + EXPECT_EQ(p, q); - // exhuasive testing - for (size_t j = 1; j < 32; j++) { - a.Clear(); - a.Malloc(j); // some unaligned size - p = a.Malloc(1); - for (size_t i = 1; i < 1024; i++) { - q = a.Realloc(p, i, i + 1); - EXPECT_EQ(p, q); - p = q; - } + // exhuasive testing + for (size_t j = 1; j < 32; j++) + { a.Clear(); + a.Malloc(j); // some unaligned size + p = a.Malloc(1); + for (size_t i = 1; i < 1024; i++) + { q = a.Realloc(p, i, i + 1); + EXPECT_EQ(p, q); + p = q; } + } } diff --git a/rapidjson/test/unittest/bigintegertest.cpp b/rapidjson/test/unittest/bigintegertest.cpp index a68e1444672..53cab3959d5 100644 --- a/rapidjson/test/unittest/bigintegertest.cpp +++ b/rapidjson/test/unittest/bigintegertest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -25,109 +25,109 @@ static const BigInteger kOne(1); static const BigInteger kUint64Max = BIGINTEGER_LITERAL("18446744073709551615"); static const BigInteger kTwo64 = BIGINTEGER_LITERAL("18446744073709551616"); -TEST(BigInteger, Constructor) { - EXPECT_TRUE(kZero.IsZero()); - EXPECT_TRUE(kZero == kZero); - EXPECT_TRUE(kZero == BIGINTEGER_LITERAL("0")); - EXPECT_TRUE(kZero == BIGINTEGER_LITERAL("00")); +TEST(BigInteger, Constructor) +{ EXPECT_TRUE(kZero.IsZero()); + EXPECT_TRUE(kZero == kZero); + EXPECT_TRUE(kZero == BIGINTEGER_LITERAL("0")); + EXPECT_TRUE(kZero == BIGINTEGER_LITERAL("00")); - const BigInteger a(123); - EXPECT_TRUE(a == a); - EXPECT_TRUE(a == BIGINTEGER_LITERAL("123")); - EXPECT_TRUE(a == BIGINTEGER_LITERAL("0123")); + const BigInteger a(123); + EXPECT_TRUE(a == a); + EXPECT_TRUE(a == BIGINTEGER_LITERAL("123")); + EXPECT_TRUE(a == BIGINTEGER_LITERAL("0123")); - EXPECT_EQ(2u, kTwo64.GetCount()); - EXPECT_EQ(0u, kTwo64.GetDigit(0)); - EXPECT_EQ(1u, kTwo64.GetDigit(1)); + EXPECT_EQ(2u, kTwo64.GetCount()); + EXPECT_EQ(0u, kTwo64.GetDigit(0)); + EXPECT_EQ(1u, kTwo64.GetDigit(1)); } -TEST(BigInteger, AddUint64) { - BigInteger a = kZero; - a += 0u; - EXPECT_TRUE(kZero == a); +TEST(BigInteger, AddUint64) +{ BigInteger a = kZero; + a += 0u; + EXPECT_TRUE(kZero == a); - a += 1u; - EXPECT_TRUE(kOne == a); + a += 1u; + EXPECT_TRUE(kOne == a); - a += 1u; - EXPECT_TRUE(BigInteger(2) == a); + a += 1u; + EXPECT_TRUE(BigInteger(2) == a); - EXPECT_TRUE(BigInteger(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0xFFFFFFFF)) == kUint64Max); - BigInteger b = kUint64Max; - b += 1u; - EXPECT_TRUE(kTwo64 == b); - b += RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0xFFFFFFFF); - EXPECT_TRUE(BIGINTEGER_LITERAL("36893488147419103231") == b); + EXPECT_TRUE(BigInteger(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0xFFFFFFFF)) == kUint64Max); + BigInteger b = kUint64Max; + b += 1u; + EXPECT_TRUE(kTwo64 == b); + b += RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0xFFFFFFFF); + EXPECT_TRUE(BIGINTEGER_LITERAL("36893488147419103231") == b); } -TEST(BigInteger, MultiplyUint64) { - BigInteger a = kZero; - a *= static_cast (0); - EXPECT_TRUE(kZero == a); - a *= static_cast (123); - EXPECT_TRUE(kZero == a); - - BigInteger b = kOne; - b *= static_cast(1); - EXPECT_TRUE(kOne == b); - b *= static_cast(0); - EXPECT_TRUE(kZero == b); - - BigInteger c(123); - c *= static_cast(456u); - EXPECT_TRUE(BigInteger(123u * 456u) == c); - c *= RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0xFFFFFFFF); - EXPECT_TRUE(BIGINTEGER_LITERAL("1034640981606221330982120") == c); - c *= RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0xFFFFFFFF); - EXPECT_TRUE(BIGINTEGER_LITERAL("19085757395861596536664473018420572782123800") == c); +TEST(BigInteger, MultiplyUint64) +{ BigInteger a = kZero; + a *= static_cast (0); + EXPECT_TRUE(kZero == a); + a *= static_cast (123); + EXPECT_TRUE(kZero == a); + + BigInteger b = kOne; + b *= static_cast(1); + EXPECT_TRUE(kOne == b); + b *= static_cast(0); + EXPECT_TRUE(kZero == b); + + BigInteger c(123); + c *= static_cast(456u); + EXPECT_TRUE(BigInteger(123u * 456u) == c); + c *= RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0xFFFFFFFF); + EXPECT_TRUE(BIGINTEGER_LITERAL("1034640981606221330982120") == c); + c *= RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0xFFFFFFFF); + EXPECT_TRUE(BIGINTEGER_LITERAL("19085757395861596536664473018420572782123800") == c); } -TEST(BigInteger, MultiplyUint32) { - BigInteger a = kZero; - a *= static_cast (0); - EXPECT_TRUE(kZero == a); - a *= static_cast (123); - EXPECT_TRUE(kZero == a); - - BigInteger b = kOne; - b *= static_cast(1); - EXPECT_TRUE(kOne == b); - b *= static_cast(0); - EXPECT_TRUE(kZero == b); - - BigInteger c(123); - c *= static_cast(456u); - EXPECT_TRUE(BigInteger(123u * 456u) == c); - c *= 0xFFFFFFFFu; - EXPECT_TRUE(BIGINTEGER_LITERAL("240896125641960") == c); - c *= 0xFFFFFFFFu; - EXPECT_TRUE(BIGINTEGER_LITERAL("1034640981124429079698200") == c); +TEST(BigInteger, MultiplyUint32) +{ BigInteger a = kZero; + a *= static_cast (0); + EXPECT_TRUE(kZero == a); + a *= static_cast (123); + EXPECT_TRUE(kZero == a); + + BigInteger b = kOne; + b *= static_cast(1); + EXPECT_TRUE(kOne == b); + b *= static_cast(0); + EXPECT_TRUE(kZero == b); + + BigInteger c(123); + c *= static_cast(456u); + EXPECT_TRUE(BigInteger(123u * 456u) == c); + c *= 0xFFFFFFFFu; + EXPECT_TRUE(BIGINTEGER_LITERAL("240896125641960") == c); + c *= 0xFFFFFFFFu; + EXPECT_TRUE(BIGINTEGER_LITERAL("1034640981124429079698200") == c); } -TEST(BigInteger, LeftShift) { - BigInteger a = kZero; - a <<= 1; - EXPECT_TRUE(kZero == a); - a <<= 64; - EXPECT_TRUE(kZero == a); - - a = BigInteger(123); - a <<= 0; - EXPECT_TRUE(BigInteger(123) == a); - a <<= 1; - EXPECT_TRUE(BigInteger(246) == a); - a <<= 64; - EXPECT_TRUE(BIGINTEGER_LITERAL("4537899042132549697536") == a); - a <<= 99; - EXPECT_TRUE(BIGINTEGER_LITERAL("2876235222267216943024851750785644982682875244576768") == a); +TEST(BigInteger, LeftShift) +{ BigInteger a = kZero; + a <<= 1; + EXPECT_TRUE(kZero == a); + a <<= 64; + EXPECT_TRUE(kZero == a); + + a = BigInteger(123); + a <<= 0; + EXPECT_TRUE(BigInteger(123) == a); + a <<= 1; + EXPECT_TRUE(BigInteger(246) == a); + a <<= 64; + EXPECT_TRUE(BIGINTEGER_LITERAL("4537899042132549697536") == a); + a <<= 99; + EXPECT_TRUE(BIGINTEGER_LITERAL("2876235222267216943024851750785644982682875244576768") == a); } -TEST(BigInteger, Compare) { - EXPECT_EQ(0, kZero.Compare(kZero)); - EXPECT_EQ(1, kOne.Compare(kZero)); - EXPECT_EQ(-1, kZero.Compare(kOne)); - EXPECT_EQ(0, kUint64Max.Compare(kUint64Max)); - EXPECT_EQ(0, kTwo64.Compare(kTwo64)); - EXPECT_EQ(-1, kUint64Max.Compare(kTwo64)); - EXPECT_EQ(1, kTwo64.Compare(kUint64Max)); +TEST(BigInteger, Compare) +{ EXPECT_EQ(0, kZero.Compare(kZero)); + EXPECT_EQ(1, kOne.Compare(kZero)); + EXPECT_EQ(-1, kZero.Compare(kOne)); + EXPECT_EQ(0, kUint64Max.Compare(kUint64Max)); + EXPECT_EQ(0, kTwo64.Compare(kTwo64)); + EXPECT_EQ(-1, kUint64Max.Compare(kTwo64)); + EXPECT_EQ(1, kTwo64.Compare(kUint64Max)); } diff --git a/rapidjson/test/unittest/documenttest.cpp b/rapidjson/test/unittest/documenttest.cpp index ecd4b79bc2a..ed941ee425d 100644 --- a/rapidjson/test/unittest/documenttest.cpp +++ b/rapidjson/test/unittest/documenttest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -30,369 +30,368 @@ RAPIDJSON_DIAG_OFF(missing-variable-declarations) using namespace rapidjson; template -void ParseCheck(DocumentType& doc) { - typedef typename DocumentType::ValueType ValueType; - - EXPECT_FALSE(doc.HasParseError()); - if (doc.HasParseError()) - printf("Error: %d at %zu\n", static_cast(doc.GetParseError()), doc.GetErrorOffset()); - EXPECT_TRUE(static_cast(doc)); - - EXPECT_TRUE(doc.IsObject()); - - EXPECT_TRUE(doc.HasMember("hello")); - const ValueType& hello = doc["hello"]; - EXPECT_TRUE(hello.IsString()); - EXPECT_STREQ("world", hello.GetString()); - - EXPECT_TRUE(doc.HasMember("t")); - const ValueType& t = doc["t"]; - EXPECT_TRUE(t.IsTrue()); - - EXPECT_TRUE(doc.HasMember("f")); - const ValueType& f = doc["f"]; - EXPECT_TRUE(f.IsFalse()); - - EXPECT_TRUE(doc.HasMember("n")); - const ValueType& n = doc["n"]; - EXPECT_TRUE(n.IsNull()); - - EXPECT_TRUE(doc.HasMember("i")); - const ValueType& i = doc["i"]; - EXPECT_TRUE(i.IsNumber()); - EXPECT_EQ(123, i.GetInt()); - - EXPECT_TRUE(doc.HasMember("pi")); - const ValueType& pi = doc["pi"]; - EXPECT_TRUE(pi.IsNumber()); - EXPECT_DOUBLE_EQ(3.1416, pi.GetDouble()); - - EXPECT_TRUE(doc.HasMember("a")); - const ValueType& a = doc["a"]; - EXPECT_TRUE(a.IsArray()); - EXPECT_EQ(4u, a.Size()); - for (SizeType j = 0; j < 4; j++) - EXPECT_EQ(j + 1, a[j].GetUint()); +void ParseCheck(DocumentType& doc) +{ typedef typename DocumentType::ValueType ValueType; + + EXPECT_FALSE(doc.HasParseError()); + if (doc.HasParseError()) + printf("Error: %d at %zu\n", static_cast(doc.GetParseError()), doc.GetErrorOffset()); + EXPECT_TRUE(static_cast(doc)); + + EXPECT_TRUE(doc.IsObject()); + + EXPECT_TRUE(doc.HasMember("hello")); + const ValueType& hello = doc["hello"]; + EXPECT_TRUE(hello.IsString()); + EXPECT_STREQ("world", hello.GetString()); + + EXPECT_TRUE(doc.HasMember("t")); + const ValueType& t = doc["t"]; + EXPECT_TRUE(t.IsTrue()); + + EXPECT_TRUE(doc.HasMember("f")); + const ValueType& f = doc["f"]; + EXPECT_TRUE(f.IsFalse()); + + EXPECT_TRUE(doc.HasMember("n")); + const ValueType& n = doc["n"]; + EXPECT_TRUE(n.IsNull()); + + EXPECT_TRUE(doc.HasMember("i")); + const ValueType& i = doc["i"]; + EXPECT_TRUE(i.IsNumber()); + EXPECT_EQ(123, i.GetInt()); + + EXPECT_TRUE(doc.HasMember("pi")); + const ValueType& pi = doc["pi"]; + EXPECT_TRUE(pi.IsNumber()); + EXPECT_DOUBLE_EQ(3.1416, pi.GetDouble()); + + EXPECT_TRUE(doc.HasMember("a")); + const ValueType& a = doc["a"]; + EXPECT_TRUE(a.IsArray()); + EXPECT_EQ(4u, a.Size()); + for (SizeType j = 0; j < 4; j++) + EXPECT_EQ(j + 1, a[j].GetUint()); } template -void ParseTest() { - typedef GenericDocument, Allocator, StackAllocator> DocumentType; - DocumentType doc; - - const char* json = " { \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3, 4] } "; - - doc.Parse(json); - ParseCheck(doc); - - doc.SetNull(); - StringStream s(json); - doc.template ParseStream<0>(s); - ParseCheck(doc); - - doc.SetNull(); - char *buffer = strdup(json); - doc.ParseInsitu(buffer); - ParseCheck(doc); - free(buffer); - - // Parse(const Ch*, size_t) - size_t length = strlen(json); - buffer = reinterpret_cast(malloc(length * 2)); - memcpy(buffer, json, length); - memset(buffer + length, 'X', length); +void ParseTest() +{ typedef GenericDocument, Allocator, StackAllocator> DocumentType; + DocumentType doc; + + const char* json = " { \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3, 4] } "; + + doc.Parse(json); + ParseCheck(doc); + + doc.SetNull(); + StringStream s(json); + doc.template ParseStream<0>(s); + ParseCheck(doc); + + doc.SetNull(); + char *buffer = strdup(json); + doc.ParseInsitu(buffer); + ParseCheck(doc); + free(buffer); + + // Parse(const Ch*, size_t) + size_t length = strlen(json); + buffer = reinterpret_cast(malloc(length * 2)); + memcpy(buffer, json, length); + memset(buffer + length, 'X', length); #if RAPIDJSON_HAS_STDSTRING - std::string s2(buffer, length); // backup buffer + std::string s2(buffer, length); // backup buffer #endif - doc.SetNull(); - doc.Parse(buffer, length); - free(buffer); - ParseCheck(doc); + doc.SetNull(); + doc.Parse(buffer, length); + free(buffer); + ParseCheck(doc); #if RAPIDJSON_HAS_STDSTRING - // Parse(std::string) - doc.SetNull(); - doc.Parse(s2); - ParseCheck(doc); + // Parse(std::string) + doc.SetNull(); + doc.Parse(s2); + ParseCheck(doc); #endif } -TEST(Document, Parse) { - ParseTest, CrtAllocator>(); - ParseTest, MemoryPoolAllocator<> >(); - ParseTest >(); - ParseTest(); +TEST(Document, Parse) +{ ParseTest, CrtAllocator>(); + ParseTest, MemoryPoolAllocator<> >(); + ParseTest >(); + ParseTest(); } -TEST(Document, UnchangedOnParseError) { - Document doc; - doc.SetArray().PushBack(0, doc.GetAllocator()); - - ParseResult err = doc.Parse("{]"); - EXPECT_TRUE(doc.HasParseError()); - EXPECT_EQ(err.Code(), doc.GetParseError()); - EXPECT_EQ(err.Offset(), doc.GetErrorOffset()); - EXPECT_TRUE(doc.IsArray()); - EXPECT_EQ(doc.Size(), 1u); - - err = doc.Parse("{}"); - EXPECT_FALSE(doc.HasParseError()); - EXPECT_FALSE(err.IsError()); - EXPECT_EQ(err.Code(), doc.GetParseError()); - EXPECT_EQ(err.Offset(), doc.GetErrorOffset()); - EXPECT_TRUE(doc.IsObject()); - EXPECT_EQ(doc.MemberCount(), 0u); +TEST(Document, UnchangedOnParseError) +{ Document doc; + doc.SetArray().PushBack(0, doc.GetAllocator()); + + ParseResult err = doc.Parse("{]"); + EXPECT_TRUE(doc.HasParseError()); + EXPECT_EQ(err.Code(), doc.GetParseError()); + EXPECT_EQ(err.Offset(), doc.GetErrorOffset()); + EXPECT_TRUE(doc.IsArray()); + EXPECT_EQ(doc.Size(), 1u); + + err = doc.Parse("{}"); + EXPECT_FALSE(doc.HasParseError()); + EXPECT_FALSE(err.IsError()); + EXPECT_EQ(err.Code(), doc.GetParseError()); + EXPECT_EQ(err.Offset(), doc.GetErrorOffset()); + EXPECT_TRUE(doc.IsObject()); + EXPECT_EQ(doc.MemberCount(), 0u); } -static FILE* OpenEncodedFile(const char* filename) { - const char *paths[] = { - "encodings", - "bin/encodings", - "../bin/encodings", - "../../bin/encodings", - "../../../bin/encodings" - }; - char buffer[1024]; - for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { - sprintf(buffer, "%s/%s", paths[i], filename); - FILE *fp = fopen(buffer, "rb"); - if (fp) - return fp; - } - return 0; +static FILE* OpenEncodedFile(const char* filename) +{ const char *paths[] = + { "encodings", + "bin/encodings", + "../bin/encodings", + "../../bin/encodings", + "../../../bin/encodings" + }; + char buffer[1024]; + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) + { sprintf(buffer, "%s/%s", paths[i], filename); + FILE *fp = fopen(buffer, "rb"); + if (fp) + return fp; + } + return 0; } -TEST(Document, Parse_Encoding) { - const char* json = " { \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3, 4] } "; - - typedef GenericDocument > DocumentType; - DocumentType doc; - - // Parse(const SourceEncoding::Ch*) - // doc.Parse >(json); - // EXPECT_FALSE(doc.HasParseError()); - // EXPECT_EQ(0, StrCmp(doc[L"hello"].GetString(), L"world")); - - // Parse(const SourceEncoding::Ch*, size_t) - size_t length = strlen(json); - char* buffer = reinterpret_cast(malloc(length * 2)); - memcpy(buffer, json, length); - memset(buffer + length, 'X', length); +TEST(Document, Parse_Encoding) +{ const char* json = " { \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3, 4] } "; + + typedef GenericDocument > DocumentType; + DocumentType doc; + + // Parse(const SourceEncoding::Ch*) + // doc.Parse >(json); + // EXPECT_FALSE(doc.HasParseError()); + // EXPECT_EQ(0, StrCmp(doc[L"hello"].GetString(), L"world")); + + // Parse(const SourceEncoding::Ch*, size_t) + size_t length = strlen(json); + char* buffer = reinterpret_cast(malloc(length * 2)); + memcpy(buffer, json, length); + memset(buffer + length, 'X', length); #if RAPIDJSON_HAS_STDSTRING - std::string s2(buffer, length); // backup buffer + std::string s2(buffer, length); // backup buffer #endif - doc.SetNull(); - doc.Parse >(buffer, length); - free(buffer); - EXPECT_FALSE(doc.HasParseError()); - if (doc.HasParseError()) - printf("Error: %d at %zu\n", static_cast(doc.GetParseError()), doc.GetErrorOffset()); - EXPECT_EQ(0, StrCmp(doc[L"hello"].GetString(), L"world")); + doc.SetNull(); + doc.Parse >(buffer, length); + free(buffer); + EXPECT_FALSE(doc.HasParseError()); + if (doc.HasParseError()) + printf("Error: %d at %zu\n", static_cast(doc.GetParseError()), doc.GetErrorOffset()); + EXPECT_EQ(0, StrCmp(doc[L"hello"].GetString(), L"world")); #if RAPIDJSON_HAS_STDSTRING - // Parse(std::string) - doc.SetNull(); + // Parse(std::string) + doc.SetNull(); #if defined(_MSC_VER) && _MSC_VER < 1800 - doc.Parse >(s2.c_str()); // VS2010 or below cannot handle templated function overloading. Use const char* instead. + doc.Parse >(s2.c_str()); // VS2010 or below cannot handle templated function overloading. Use const char* instead. #else - doc.Parse >(s2); + doc.Parse >(s2); #endif - EXPECT_FALSE(doc.HasParseError()); - EXPECT_EQ(0, StrCmp(doc[L"hello"].GetString(), L"world")); + EXPECT_FALSE(doc.HasParseError()); + EXPECT_EQ(0, StrCmp(doc[L"hello"].GetString(), L"world")); #endif } -TEST(Document, ParseStream_EncodedInputStream) { - // UTF8 -> UTF16 - FILE* fp = OpenEncodedFile("utf8.json"); - char buffer[256]; - FileReadStream bis(fp, buffer, sizeof(buffer)); - EncodedInputStream, FileReadStream> eis(bis); - - GenericDocument > d; - d.ParseStream<0, UTF8<> >(eis); - EXPECT_FALSE(d.HasParseError()); - - fclose(fp); - - wchar_t expected[] = L"I can eat glass and it doesn't hurt me."; - GenericValue >& v = d[L"en"]; - EXPECT_TRUE(v.IsString()); - EXPECT_EQ(sizeof(expected) / sizeof(wchar_t) - 1, v.GetStringLength()); - EXPECT_EQ(0, StrCmp(expected, v.GetString())); - - // UTF16 -> UTF8 in memory - StringBuffer bos; - typedef EncodedOutputStream, StringBuffer> OutputStream; - OutputStream eos(bos, false); // Not writing BOM - { - Writer, UTF8<> > writer(eos); - d.Accept(writer); - } - - // Condense the original file and compare. - fp = OpenEncodedFile("utf8.json"); - FileReadStream is(fp, buffer, sizeof(buffer)); - Reader reader; - StringBuffer bos2; - Writer writer2(bos2); - reader.Parse(is, writer2); - fclose(fp); - - EXPECT_EQ(bos.GetSize(), bos2.GetSize()); - EXPECT_EQ(0, memcmp(bos.GetString(), bos2.GetString(), bos2.GetSize())); -} - -TEST(Document, ParseStream_AutoUTFInputStream) { - // Any -> UTF8 - FILE* fp = OpenEncodedFile("utf32be.json"); - char buffer[256]; - FileReadStream bis(fp, buffer, sizeof(buffer)); - AutoUTFInputStream eis(bis); - - Document d; - d.ParseStream<0, AutoUTF >(eis); - EXPECT_FALSE(d.HasParseError()); - - fclose(fp); - - char expected[] = "I can eat glass and it doesn't hurt me."; - Value& v = d["en"]; - EXPECT_TRUE(v.IsString()); - EXPECT_EQ(sizeof(expected) - 1, v.GetStringLength()); - EXPECT_EQ(0, StrCmp(expected, v.GetString())); - - // UTF8 -> UTF8 in memory - StringBuffer bos; - Writer writer(bos); +TEST(Document, ParseStream_EncodedInputStream) +{ // UTF8 -> UTF16 + FILE* fp = OpenEncodedFile("utf8.json"); + char buffer[256]; + FileReadStream bis(fp, buffer, sizeof(buffer)); + EncodedInputStream, FileReadStream> eis(bis); + + GenericDocument > d; + d.ParseStream<0, UTF8<> >(eis); + EXPECT_FALSE(d.HasParseError()); + + fclose(fp); + + wchar_t expected[] = L"I can eat glass and it doesn't hurt me."; + GenericValue >& v = d[L"en"]; + EXPECT_TRUE(v.IsString()); + EXPECT_EQ(sizeof(expected) / sizeof(wchar_t) - 1, v.GetStringLength()); + EXPECT_EQ(0, StrCmp(expected, v.GetString())); + + // UTF16 -> UTF8 in memory + StringBuffer bos; + typedef EncodedOutputStream, StringBuffer> OutputStream; + OutputStream eos(bos, false); // Not writing BOM + { Writer, UTF8<> > writer(eos); d.Accept(writer); + } + + // Condense the original file and compare. + fp = OpenEncodedFile("utf8.json"); + FileReadStream is(fp, buffer, sizeof(buffer)); + Reader reader; + StringBuffer bos2; + Writer writer2(bos2); + reader.Parse(is, writer2); + fclose(fp); + + EXPECT_EQ(bos.GetSize(), bos2.GetSize()); + EXPECT_EQ(0, memcmp(bos.GetString(), bos2.GetString(), bos2.GetSize())); +} - // Condense the original file and compare. - fp = OpenEncodedFile("utf8.json"); - FileReadStream is(fp, buffer, sizeof(buffer)); - Reader reader; - StringBuffer bos2; - Writer writer2(bos2); - reader.Parse(is, writer2); - fclose(fp); - - EXPECT_EQ(bos.GetSize(), bos2.GetSize()); - EXPECT_EQ(0, memcmp(bos.GetString(), bos2.GetString(), bos2.GetSize())); +TEST(Document, ParseStream_AutoUTFInputStream) +{ // Any -> UTF8 + FILE* fp = OpenEncodedFile("utf32be.json"); + char buffer[256]; + FileReadStream bis(fp, buffer, sizeof(buffer)); + AutoUTFInputStream eis(bis); + + Document d; + d.ParseStream<0, AutoUTF >(eis); + EXPECT_FALSE(d.HasParseError()); + + fclose(fp); + + char expected[] = "I can eat glass and it doesn't hurt me."; + Value& v = d["en"]; + EXPECT_TRUE(v.IsString()); + EXPECT_EQ(sizeof(expected) - 1, v.GetStringLength()); + EXPECT_EQ(0, StrCmp(expected, v.GetString())); + + // UTF8 -> UTF8 in memory + StringBuffer bos; + Writer writer(bos); + d.Accept(writer); + + // Condense the original file and compare. + fp = OpenEncodedFile("utf8.json"); + FileReadStream is(fp, buffer, sizeof(buffer)); + Reader reader; + StringBuffer bos2; + Writer writer2(bos2); + reader.Parse(is, writer2); + fclose(fp); + + EXPECT_EQ(bos.GetSize(), bos2.GetSize()); + EXPECT_EQ(0, memcmp(bos.GetString(), bos2.GetString(), bos2.GetSize())); } -TEST(Document, Swap) { - Document d1; - Document::AllocatorType& a = d1.GetAllocator(); - - d1.SetArray().PushBack(1, a).PushBack(2, a); - - Value o; - o.SetObject().AddMember("a", 1, a); - - // Swap between Document and Value - // d1.Swap(o); // doesn't compile - o.Swap(d1); - EXPECT_TRUE(d1.IsObject()); - EXPECT_TRUE(o.IsArray()); - - // Swap between Document and Document - Document d2; - d2.SetArray().PushBack(3, a); - d1.Swap(d2); - EXPECT_TRUE(d1.IsArray()); - EXPECT_TRUE(d2.IsObject()); - EXPECT_EQ(&d2.GetAllocator(), &a); - - // reset value - Value().Swap(d1); - EXPECT_TRUE(d1.IsNull()); - - // reset document, including allocator - Document().Swap(d2); - EXPECT_TRUE(d2.IsNull()); - EXPECT_NE(&d2.GetAllocator(), &a); - - // testing std::swap compatibility - d1.SetBool(true); - using std::swap; - swap(d1, d2); - EXPECT_TRUE(d1.IsNull()); - EXPECT_TRUE(d2.IsTrue()); - - swap(o, d2); - EXPECT_TRUE(o.IsTrue()); - EXPECT_TRUE(d2.IsArray()); +TEST(Document, Swap) +{ Document d1; + Document::AllocatorType& a = d1.GetAllocator(); + + d1.SetArray().PushBack(1, a).PushBack(2, a); + + Value o; + o.SetObject().AddMember("a", 1, a); + + // Swap between Document and Value + // d1.Swap(o); // doesn't compile + o.Swap(d1); + EXPECT_TRUE(d1.IsObject()); + EXPECT_TRUE(o.IsArray()); + + // Swap between Document and Document + Document d2; + d2.SetArray().PushBack(3, a); + d1.Swap(d2); + EXPECT_TRUE(d1.IsArray()); + EXPECT_TRUE(d2.IsObject()); + EXPECT_EQ(&d2.GetAllocator(), &a); + + // reset value + Value().Swap(d1); + EXPECT_TRUE(d1.IsNull()); + + // reset document, including allocator + Document().Swap(d2); + EXPECT_TRUE(d2.IsNull()); + EXPECT_NE(&d2.GetAllocator(), &a); + + // testing std::swap compatibility + d1.SetBool(true); + using std::swap; + swap(d1, d2); + EXPECT_TRUE(d1.IsNull()); + EXPECT_TRUE(d2.IsTrue()); + + swap(o, d2); + EXPECT_TRUE(o.IsTrue()); + EXPECT_TRUE(d2.IsArray()); } // This should be slow due to assignment in inner-loop. -struct OutputStringStream : public std::ostringstream { - typedef char Ch; +struct OutputStringStream : public std::ostringstream +{ typedef char Ch; - virtual ~OutputStringStream(); + virtual ~OutputStringStream(); - void Put(char c) { - put(c); - } - void Flush() {} + void Put(char c) + { put(c); + } + void Flush() {} }; OutputStringStream::~OutputStringStream() {} -TEST(Document, AcceptWriter) { - Document doc; - doc.Parse(" { \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3, 4] } "); +TEST(Document, AcceptWriter) +{ Document doc; + doc.Parse(" { \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3, 4] } "); - OutputStringStream os; - Writer writer(os); - doc.Accept(writer); + OutputStringStream os; + Writer writer(os); + doc.Accept(writer); - EXPECT_EQ("{\"hello\":\"world\",\"t\":true,\"f\":false,\"n\":null,\"i\":123,\"pi\":3.1416,\"a\":[1,2,3,4]}", os.str()); + EXPECT_EQ("{\"hello\":\"world\",\"t\":true,\"f\":false,\"n\":null,\"i\":123,\"pi\":3.1416,\"a\":[1,2,3,4]}", os.str()); } -TEST(Document, UserBuffer) { - typedef GenericDocument, MemoryPoolAllocator<>, MemoryPoolAllocator<> > DocumentType; - char valueBuffer[4096]; - char parseBuffer[1024]; - MemoryPoolAllocator<> valueAllocator(valueBuffer, sizeof(valueBuffer)); - MemoryPoolAllocator<> parseAllocator(parseBuffer, sizeof(parseBuffer)); - DocumentType doc(&valueAllocator, sizeof(parseBuffer) / 2, &parseAllocator); - doc.Parse(" { \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3, 4] } "); - EXPECT_FALSE(doc.HasParseError()); - EXPECT_LE(valueAllocator.Size(), sizeof(valueBuffer)); - EXPECT_LE(parseAllocator.Size(), sizeof(parseBuffer)); - - // Cover MemoryPoolAllocator::Capacity() - EXPECT_LE(valueAllocator.Size(), valueAllocator.Capacity()); - EXPECT_LE(parseAllocator.Size(), parseAllocator.Capacity()); +TEST(Document, UserBuffer) +{ typedef GenericDocument, MemoryPoolAllocator<>, MemoryPoolAllocator<> > DocumentType; + char valueBuffer[4096]; + char parseBuffer[1024]; + MemoryPoolAllocator<> valueAllocator(valueBuffer, sizeof(valueBuffer)); + MemoryPoolAllocator<> parseAllocator(parseBuffer, sizeof(parseBuffer)); + DocumentType doc(&valueAllocator, sizeof(parseBuffer) / 2, &parseAllocator); + doc.Parse(" { \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3, 4] } "); + EXPECT_FALSE(doc.HasParseError()); + EXPECT_LE(valueAllocator.Size(), sizeof(valueBuffer)); + EXPECT_LE(parseAllocator.Size(), sizeof(parseBuffer)); + + // Cover MemoryPoolAllocator::Capacity() + EXPECT_LE(valueAllocator.Size(), valueAllocator.Capacity()); + EXPECT_LE(parseAllocator.Size(), parseAllocator.Capacity()); } // Issue 226: Value of string type should not point to NULL -TEST(Document, AssertAcceptInvalidNameType) { - Document doc; - doc.SetObject(); - doc.AddMember("a", 0, doc.GetAllocator()); - doc.FindMember("a")->name.SetNull(); // Change name to non-string type. - - OutputStringStream os; - Writer writer(os); - ASSERT_THROW(doc.Accept(writer), AssertException); +TEST(Document, AssertAcceptInvalidNameType) +{ Document doc; + doc.SetObject(); + doc.AddMember("a", 0, doc.GetAllocator()); + doc.FindMember("a")->name.SetNull(); // Change name to non-string type. + + OutputStringStream os; + Writer writer(os); + ASSERT_THROW(doc.Accept(writer), AssertException); } // Issue 44: SetStringRaw doesn't work with wchar_t -TEST(Document, UTF16_Document) { - GenericDocument< UTF16<> > json; - json.Parse(L"[{\"created_at\":\"Wed Oct 30 17:13:20 +0000 2012\"}]"); +TEST(Document, UTF16_Document) +{ GenericDocument< UTF16<> > json; + json.Parse(L"[{\"created_at\":\"Wed Oct 30 17:13:20 +0000 2012\"}]"); - ASSERT_TRUE(json.IsArray()); - GenericValue< UTF16<> >& v = json[0]; - ASSERT_TRUE(v.IsObject()); + ASSERT_TRUE(json.IsArray()); + GenericValue< UTF16<> >& v = json[0]; + ASSERT_TRUE(v.IsObject()); - GenericValue< UTF16<> >& s = v[L"created_at"]; - ASSERT_TRUE(s.IsString()); + GenericValue< UTF16<> >& s = v[L"created_at"]; + ASSERT_TRUE(s.IsString()); - EXPECT_EQ(0, memcmp(L"Wed Oct 30 17:13:20 +0000 2012", s.GetString(), (s.GetStringLength() + 1) * sizeof(wchar_t))); + EXPECT_EQ(0, memcmp(L"Wed Oct 30 17:13:20 +0000 2012", s.GetString(), (s.GetStringLength() + 1) * sizeof(wchar_t))); } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS @@ -401,239 +400,240 @@ TEST(Document, UTF16_Document) { #include -TEST(Document, Traits) { - static_assert(std::is_constructible::value, ""); - static_assert(std::is_default_constructible::value, ""); +TEST(Document, Traits) +{ static_assert(std::is_constructible::value, ""); + static_assert(std::is_default_constructible::value, ""); #ifndef _MSC_VER - static_assert(!std::is_copy_constructible::value, ""); + static_assert(!std::is_copy_constructible::value, ""); #endif - static_assert(std::is_move_constructible::value, ""); + static_assert(std::is_move_constructible::value, ""); - static_assert(!std::is_nothrow_constructible::value, ""); - static_assert(!std::is_nothrow_default_constructible::value, ""); + static_assert(!std::is_nothrow_constructible::value, ""); + static_assert(!std::is_nothrow_default_constructible::value, ""); #ifndef _MSC_VER - static_assert(!std::is_nothrow_copy_constructible::value, ""); - static_assert(std::is_nothrow_move_constructible::value, ""); + static_assert(!std::is_nothrow_copy_constructible::value, ""); + static_assert(std::is_nothrow_move_constructible::value, ""); #endif - static_assert(std::is_assignable::value, ""); + static_assert(std::is_assignable::value, ""); #ifndef _MSC_VER static_assert(!std::is_copy_assignable::value, ""); #endif - static_assert(std::is_move_assignable::value, ""); + static_assert(std::is_move_assignable::value, ""); #ifndef _MSC_VER - static_assert(std::is_nothrow_assignable::value, ""); + static_assert(std::is_nothrow_assignable::value, ""); #endif - static_assert(!std::is_nothrow_copy_assignable::value, ""); + static_assert(!std::is_nothrow_copy_assignable::value, ""); #ifndef _MSC_VER - static_assert(std::is_nothrow_move_assignable::value, ""); + static_assert(std::is_nothrow_move_assignable::value, ""); #endif - static_assert( std::is_destructible::value, ""); + static_assert( std::is_destructible::value, ""); #ifndef _MSC_VER - static_assert(std::is_nothrow_destructible::value, ""); + static_assert(std::is_nothrow_destructible::value, ""); #endif } #endif template -struct DocumentMove: public ::testing::Test { +struct DocumentMove: public ::testing::Test +{ }; typedef ::testing::Types< CrtAllocator, MemoryPoolAllocator<> > MoveAllocatorTypes; TYPED_TEST_CASE(DocumentMove, MoveAllocatorTypes); -TYPED_TEST(DocumentMove, MoveConstructor) { - typedef TypeParam Allocator; - typedef GenericDocument, Allocator> D; - Allocator allocator; - - D a(&allocator); - a.Parse("[\"one\", \"two\", \"three\"]"); - EXPECT_FALSE(a.HasParseError()); - EXPECT_TRUE(a.IsArray()); - EXPECT_EQ(3u, a.Size()); - EXPECT_EQ(&a.GetAllocator(), &allocator); - - // Document b(a); // does not compile (!is_copy_constructible) - D b(std::move(a)); - EXPECT_TRUE(a.IsNull()); - EXPECT_TRUE(b.IsArray()); - EXPECT_EQ(3u, b.Size()); - EXPECT_THROW(a.GetAllocator(), AssertException); - EXPECT_EQ(&b.GetAllocator(), &allocator); - - b.Parse("{\"Foo\": \"Bar\", \"Baz\": 42}"); - EXPECT_FALSE(b.HasParseError()); - EXPECT_TRUE(b.IsObject()); - EXPECT_EQ(2u, b.MemberCount()); - - // Document c = a; // does not compile (!is_copy_constructible) - D c = std::move(b); - EXPECT_TRUE(b.IsNull()); - EXPECT_TRUE(c.IsObject()); - EXPECT_EQ(2u, c.MemberCount()); - EXPECT_THROW(b.GetAllocator(), AssertException); - EXPECT_EQ(&c.GetAllocator(), &allocator); +TYPED_TEST(DocumentMove, MoveConstructor) +{ typedef TypeParam Allocator; + typedef GenericDocument, Allocator> D; + Allocator allocator; + + D a(&allocator); + a.Parse("[\"one\", \"two\", \"three\"]"); + EXPECT_FALSE(a.HasParseError()); + EXPECT_TRUE(a.IsArray()); + EXPECT_EQ(3u, a.Size()); + EXPECT_EQ(&a.GetAllocator(), &allocator); + + // Document b(a); // does not compile (!is_copy_constructible) + D b(std::move(a)); + EXPECT_TRUE(a.IsNull()); + EXPECT_TRUE(b.IsArray()); + EXPECT_EQ(3u, b.Size()); + EXPECT_THROW(a.GetAllocator(), AssertException); + EXPECT_EQ(&b.GetAllocator(), &allocator); + + b.Parse("{\"Foo\": \"Bar\", \"Baz\": 42}"); + EXPECT_FALSE(b.HasParseError()); + EXPECT_TRUE(b.IsObject()); + EXPECT_EQ(2u, b.MemberCount()); + + // Document c = a; // does not compile (!is_copy_constructible) + D c = std::move(b); + EXPECT_TRUE(b.IsNull()); + EXPECT_TRUE(c.IsObject()); + EXPECT_EQ(2u, c.MemberCount()); + EXPECT_THROW(b.GetAllocator(), AssertException); + EXPECT_EQ(&c.GetAllocator(), &allocator); } -TYPED_TEST(DocumentMove, MoveConstructorParseError) { - typedef TypeParam Allocator; - typedef GenericDocument, Allocator> D; - - ParseResult noError; - D a; - a.Parse("{ 4 = 4]"); - ParseResult error(a.GetParseError(), a.GetErrorOffset()); - EXPECT_TRUE(a.HasParseError()); - EXPECT_NE(error.Code(), noError.Code()); - EXPECT_NE(error.Offset(), noError.Offset()); - - D b(std::move(a)); - EXPECT_FALSE(a.HasParseError()); - EXPECT_TRUE(b.HasParseError()); - EXPECT_EQ(a.GetParseError(), noError.Code()); - EXPECT_EQ(b.GetParseError(), error.Code()); - EXPECT_EQ(a.GetErrorOffset(), noError.Offset()); - EXPECT_EQ(b.GetErrorOffset(), error.Offset()); - - D c(std::move(b)); - EXPECT_FALSE(b.HasParseError()); - EXPECT_TRUE(c.HasParseError()); - EXPECT_EQ(b.GetParseError(), noError.Code()); - EXPECT_EQ(c.GetParseError(), error.Code()); - EXPECT_EQ(b.GetErrorOffset(), noError.Offset()); - EXPECT_EQ(c.GetErrorOffset(), error.Offset()); +TYPED_TEST(DocumentMove, MoveConstructorParseError) +{ typedef TypeParam Allocator; + typedef GenericDocument, Allocator> D; + + ParseResult noError; + D a; + a.Parse("{ 4 = 4]"); + ParseResult error(a.GetParseError(), a.GetErrorOffset()); + EXPECT_TRUE(a.HasParseError()); + EXPECT_NE(error.Code(), noError.Code()); + EXPECT_NE(error.Offset(), noError.Offset()); + + D b(std::move(a)); + EXPECT_FALSE(a.HasParseError()); + EXPECT_TRUE(b.HasParseError()); + EXPECT_EQ(a.GetParseError(), noError.Code()); + EXPECT_EQ(b.GetParseError(), error.Code()); + EXPECT_EQ(a.GetErrorOffset(), noError.Offset()); + EXPECT_EQ(b.GetErrorOffset(), error.Offset()); + + D c(std::move(b)); + EXPECT_FALSE(b.HasParseError()); + EXPECT_TRUE(c.HasParseError()); + EXPECT_EQ(b.GetParseError(), noError.Code()); + EXPECT_EQ(c.GetParseError(), error.Code()); + EXPECT_EQ(b.GetErrorOffset(), noError.Offset()); + EXPECT_EQ(c.GetErrorOffset(), error.Offset()); } // This test does not properly use parsing, just for testing. // It must call ClearStack() explicitly to prevent memory leak. // But here we cannot as ClearStack() is private. #if 0 -TYPED_TEST(DocumentMove, MoveConstructorStack) { - typedef TypeParam Allocator; - typedef UTF8<> Encoding; - typedef GenericDocument Document; - - Document a; - size_t defaultCapacity = a.GetStackCapacity(); - - // Trick Document into getting GetStackCapacity() to return non-zero - typedef GenericReader Reader; - Reader reader(&a.GetAllocator()); - GenericStringStream is("[\"one\", \"two\", \"three\"]"); - reader.template Parse(is, a); - size_t capacity = a.GetStackCapacity(); - EXPECT_GT(capacity, 0u); - - Document b(std::move(a)); - EXPECT_EQ(a.GetStackCapacity(), defaultCapacity); - EXPECT_EQ(b.GetStackCapacity(), capacity); - - Document c = std::move(b); - EXPECT_EQ(b.GetStackCapacity(), defaultCapacity); - EXPECT_EQ(c.GetStackCapacity(), capacity); +TYPED_TEST(DocumentMove, MoveConstructorStack) +{ typedef TypeParam Allocator; + typedef UTF8<> Encoding; + typedef GenericDocument Document; + + Document a; + size_t defaultCapacity = a.GetStackCapacity(); + + // Trick Document into getting GetStackCapacity() to return non-zero + typedef GenericReader Reader; + Reader reader(&a.GetAllocator()); + GenericStringStream is("[\"one\", \"two\", \"three\"]"); + reader.template Parse(is, a); + size_t capacity = a.GetStackCapacity(); + EXPECT_GT(capacity, 0u); + + Document b(std::move(a)); + EXPECT_EQ(a.GetStackCapacity(), defaultCapacity); + EXPECT_EQ(b.GetStackCapacity(), capacity); + + Document c = std::move(b); + EXPECT_EQ(b.GetStackCapacity(), defaultCapacity); + EXPECT_EQ(c.GetStackCapacity(), capacity); } #endif -TYPED_TEST(DocumentMove, MoveAssignment) { - typedef TypeParam Allocator; - typedef GenericDocument, Allocator> D; - Allocator allocator; - - D a(&allocator); - a.Parse("[\"one\", \"two\", \"three\"]"); - EXPECT_FALSE(a.HasParseError()); - EXPECT_TRUE(a.IsArray()); - EXPECT_EQ(3u, a.Size()); - EXPECT_EQ(&a.GetAllocator(), &allocator); - - // Document b; b = a; // does not compile (!is_copy_assignable) - D b; - b = std::move(a); - EXPECT_TRUE(a.IsNull()); - EXPECT_TRUE(b.IsArray()); - EXPECT_EQ(3u, b.Size()); - EXPECT_THROW(a.GetAllocator(), AssertException); - EXPECT_EQ(&b.GetAllocator(), &allocator); - - b.Parse("{\"Foo\": \"Bar\", \"Baz\": 42}"); - EXPECT_FALSE(b.HasParseError()); - EXPECT_TRUE(b.IsObject()); - EXPECT_EQ(2u, b.MemberCount()); - - // Document c; c = a; // does not compile (see static_assert) - D c; - c = std::move(b); - EXPECT_TRUE(b.IsNull()); - EXPECT_TRUE(c.IsObject()); - EXPECT_EQ(2u, c.MemberCount()); - EXPECT_THROW(b.GetAllocator(), AssertException); - EXPECT_EQ(&c.GetAllocator(), &allocator); +TYPED_TEST(DocumentMove, MoveAssignment) +{ typedef TypeParam Allocator; + typedef GenericDocument, Allocator> D; + Allocator allocator; + + D a(&allocator); + a.Parse("[\"one\", \"two\", \"three\"]"); + EXPECT_FALSE(a.HasParseError()); + EXPECT_TRUE(a.IsArray()); + EXPECT_EQ(3u, a.Size()); + EXPECT_EQ(&a.GetAllocator(), &allocator); + + // Document b; b = a; // does not compile (!is_copy_assignable) + D b; + b = std::move(a); + EXPECT_TRUE(a.IsNull()); + EXPECT_TRUE(b.IsArray()); + EXPECT_EQ(3u, b.Size()); + EXPECT_THROW(a.GetAllocator(), AssertException); + EXPECT_EQ(&b.GetAllocator(), &allocator); + + b.Parse("{\"Foo\": \"Bar\", \"Baz\": 42}"); + EXPECT_FALSE(b.HasParseError()); + EXPECT_TRUE(b.IsObject()); + EXPECT_EQ(2u, b.MemberCount()); + + // Document c; c = a; // does not compile (see static_assert) + D c; + c = std::move(b); + EXPECT_TRUE(b.IsNull()); + EXPECT_TRUE(c.IsObject()); + EXPECT_EQ(2u, c.MemberCount()); + EXPECT_THROW(b.GetAllocator(), AssertException); + EXPECT_EQ(&c.GetAllocator(), &allocator); } -TYPED_TEST(DocumentMove, MoveAssignmentParseError) { - typedef TypeParam Allocator; - typedef GenericDocument, Allocator> D; - - ParseResult noError; - D a; - a.Parse("{ 4 = 4]"); - ParseResult error(a.GetParseError(), a.GetErrorOffset()); - EXPECT_TRUE(a.HasParseError()); - EXPECT_NE(error.Code(), noError.Code()); - EXPECT_NE(error.Offset(), noError.Offset()); - - D b; - b = std::move(a); - EXPECT_FALSE(a.HasParseError()); - EXPECT_TRUE(b.HasParseError()); - EXPECT_EQ(a.GetParseError(), noError.Code()); - EXPECT_EQ(b.GetParseError(), error.Code()); - EXPECT_EQ(a.GetErrorOffset(), noError.Offset()); - EXPECT_EQ(b.GetErrorOffset(), error.Offset()); - - D c; - c = std::move(b); - EXPECT_FALSE(b.HasParseError()); - EXPECT_TRUE(c.HasParseError()); - EXPECT_EQ(b.GetParseError(), noError.Code()); - EXPECT_EQ(c.GetParseError(), error.Code()); - EXPECT_EQ(b.GetErrorOffset(), noError.Offset()); - EXPECT_EQ(c.GetErrorOffset(), error.Offset()); +TYPED_TEST(DocumentMove, MoveAssignmentParseError) +{ typedef TypeParam Allocator; + typedef GenericDocument, Allocator> D; + + ParseResult noError; + D a; + a.Parse("{ 4 = 4]"); + ParseResult error(a.GetParseError(), a.GetErrorOffset()); + EXPECT_TRUE(a.HasParseError()); + EXPECT_NE(error.Code(), noError.Code()); + EXPECT_NE(error.Offset(), noError.Offset()); + + D b; + b = std::move(a); + EXPECT_FALSE(a.HasParseError()); + EXPECT_TRUE(b.HasParseError()); + EXPECT_EQ(a.GetParseError(), noError.Code()); + EXPECT_EQ(b.GetParseError(), error.Code()); + EXPECT_EQ(a.GetErrorOffset(), noError.Offset()); + EXPECT_EQ(b.GetErrorOffset(), error.Offset()); + + D c; + c = std::move(b); + EXPECT_FALSE(b.HasParseError()); + EXPECT_TRUE(c.HasParseError()); + EXPECT_EQ(b.GetParseError(), noError.Code()); + EXPECT_EQ(c.GetParseError(), error.Code()); + EXPECT_EQ(b.GetErrorOffset(), noError.Offset()); + EXPECT_EQ(c.GetErrorOffset(), error.Offset()); } // This test does not properly use parsing, just for testing. // It must call ClearStack() explicitly to prevent memory leak. // But here we cannot as ClearStack() is private. #if 0 -TYPED_TEST(DocumentMove, MoveAssignmentStack) { - typedef TypeParam Allocator; - typedef UTF8<> Encoding; - typedef GenericDocument D; - - D a; - size_t defaultCapacity = a.GetStackCapacity(); - - // Trick Document into getting GetStackCapacity() to return non-zero - typedef GenericReader Reader; - Reader reader(&a.GetAllocator()); - GenericStringStream is("[\"one\", \"two\", \"three\"]"); - reader.template Parse(is, a); - size_t capacity = a.GetStackCapacity(); - EXPECT_GT(capacity, 0u); - - D b; - b = std::move(a); - EXPECT_EQ(a.GetStackCapacity(), defaultCapacity); - EXPECT_EQ(b.GetStackCapacity(), capacity); - - D c; - c = std::move(b); - EXPECT_EQ(b.GetStackCapacity(), defaultCapacity); - EXPECT_EQ(c.GetStackCapacity(), capacity); +TYPED_TEST(DocumentMove, MoveAssignmentStack) +{ typedef TypeParam Allocator; + typedef UTF8<> Encoding; + typedef GenericDocument D; + + D a; + size_t defaultCapacity = a.GetStackCapacity(); + + // Trick Document into getting GetStackCapacity() to return non-zero + typedef GenericReader Reader; + Reader reader(&a.GetAllocator()); + GenericStringStream is("[\"one\", \"two\", \"three\"]"); + reader.template Parse(is, a); + size_t capacity = a.GetStackCapacity(); + EXPECT_GT(capacity, 0u); + + D b; + b = std::move(a); + EXPECT_EQ(a.GetStackCapacity(), defaultCapacity); + EXPECT_EQ(b.GetStackCapacity(), capacity); + + D c; + c = std::move(b); + EXPECT_EQ(b.GetStackCapacity(), defaultCapacity); + EXPECT_EQ(c.GetStackCapacity(), capacity); } #endif diff --git a/rapidjson/test/unittest/dtoatest.cpp b/rapidjson/test/unittest/dtoatest.cpp index afd76eb09ab..5ef930d40f0 100644 --- a/rapidjson/test/unittest/dtoatest.cpp +++ b/rapidjson/test/unittest/dtoatest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -22,72 +22,72 @@ RAPIDJSON_DIAG_OFF(type-limits) using namespace rapidjson::internal; -TEST(dtoa, normal) { - char buffer[30]; +TEST(dtoa, normal) +{ char buffer[30]; #define TEST_DTOA(d, a)\ *dtoa(d, buffer) = '\0';\ EXPECT_STREQ(a, buffer) - TEST_DTOA(0.0, "0.0"); - TEST_DTOA(-0.0, "-0.0"); - TEST_DTOA(1.0, "1.0"); - TEST_DTOA(-1.0, "-1.0"); - TEST_DTOA(1.2345, "1.2345"); - TEST_DTOA(1.2345678, "1.2345678"); - TEST_DTOA(0.123456789012, "0.123456789012"); - TEST_DTOA(1234567.8, "1234567.8"); - TEST_DTOA(-79.39773355813419, "-79.39773355813419"); - TEST_DTOA(0.000001, "0.000001"); - TEST_DTOA(0.0000001, "1e-7"); - TEST_DTOA(1e30, "1e30"); - TEST_DTOA(1.234567890123456e30, "1.234567890123456e30"); - TEST_DTOA(5e-324, "5e-324"); // Min subnormal positive double - TEST_DTOA(2.225073858507201e-308, "2.225073858507201e-308"); // Max subnormal positive double - TEST_DTOA(2.2250738585072014e-308, "2.2250738585072014e-308"); // Min normal positive double - TEST_DTOA(1.7976931348623157e308, "1.7976931348623157e308"); // Max double + TEST_DTOA(0.0, "0.0"); + TEST_DTOA(-0.0, "-0.0"); + TEST_DTOA(1.0, "1.0"); + TEST_DTOA(-1.0, "-1.0"); + TEST_DTOA(1.2345, "1.2345"); + TEST_DTOA(1.2345678, "1.2345678"); + TEST_DTOA(0.123456789012, "0.123456789012"); + TEST_DTOA(1234567.8, "1234567.8"); + TEST_DTOA(-79.39773355813419, "-79.39773355813419"); + TEST_DTOA(0.000001, "0.000001"); + TEST_DTOA(0.0000001, "1e-7"); + TEST_DTOA(1e30, "1e30"); + TEST_DTOA(1.234567890123456e30, "1.234567890123456e30"); + TEST_DTOA(5e-324, "5e-324"); // Min subnormal positive double + TEST_DTOA(2.225073858507201e-308, "2.225073858507201e-308"); // Max subnormal positive double + TEST_DTOA(2.2250738585072014e-308, "2.2250738585072014e-308"); // Min normal positive double + TEST_DTOA(1.7976931348623157e308, "1.7976931348623157e308"); // Max double #undef TEST_DTOA } -TEST(dtoa, maxDecimalPlaces) { - char buffer[30]; +TEST(dtoa, maxDecimalPlaces) +{ char buffer[30]; #define TEST_DTOA(m, d, a)\ *dtoa(d, buffer, m) = '\0';\ EXPECT_STREQ(a, buffer) - TEST_DTOA(3, 0.0, "0.0"); - TEST_DTOA(1, 0.0, "0.0"); - TEST_DTOA(3, -0.0, "-0.0"); - TEST_DTOA(3, 1.0, "1.0"); - TEST_DTOA(3, -1.0, "-1.0"); - TEST_DTOA(3, 1.2345, "1.234"); - TEST_DTOA(2, 1.2345, "1.23"); - TEST_DTOA(1, 1.2345, "1.2"); - TEST_DTOA(3, 1.2345678, "1.234"); - TEST_DTOA(3, 1.0001, "1.0"); - TEST_DTOA(2, 1.0001, "1.0"); - TEST_DTOA(1, 1.0001, "1.0"); - TEST_DTOA(3, 0.123456789012, "0.123"); - TEST_DTOA(2, 0.123456789012, "0.12"); - TEST_DTOA(1, 0.123456789012, "0.1"); - TEST_DTOA(4, 0.0001, "0.0001"); - TEST_DTOA(3, 0.0001, "0.0"); - TEST_DTOA(2, 0.0001, "0.0"); - TEST_DTOA(1, 0.0001, "0.0"); - TEST_DTOA(3, 1234567.8, "1234567.8"); - TEST_DTOA(3, 1e30, "1e30"); - TEST_DTOA(3, 5e-324, "0.0"); // Min subnormal positive double - TEST_DTOA(3, 2.225073858507201e-308, "0.0"); // Max subnormal positive double - TEST_DTOA(3, 2.2250738585072014e-308, "0.0"); // Min normal positive double - TEST_DTOA(3, 1.7976931348623157e308, "1.7976931348623157e308"); // Max double - TEST_DTOA(5, -0.14000000000000001, "-0.14"); - TEST_DTOA(4, -0.14000000000000001, "-0.14"); - TEST_DTOA(3, -0.14000000000000001, "-0.14"); - TEST_DTOA(3, -0.10000000000000001, "-0.1"); - TEST_DTOA(2, -0.10000000000000001, "-0.1"); - TEST_DTOA(1, -0.10000000000000001, "-0.1"); + TEST_DTOA(3, 0.0, "0.0"); + TEST_DTOA(1, 0.0, "0.0"); + TEST_DTOA(3, -0.0, "-0.0"); + TEST_DTOA(3, 1.0, "1.0"); + TEST_DTOA(3, -1.0, "-1.0"); + TEST_DTOA(3, 1.2345, "1.234"); + TEST_DTOA(2, 1.2345, "1.23"); + TEST_DTOA(1, 1.2345, "1.2"); + TEST_DTOA(3, 1.2345678, "1.234"); + TEST_DTOA(3, 1.0001, "1.0"); + TEST_DTOA(2, 1.0001, "1.0"); + TEST_DTOA(1, 1.0001, "1.0"); + TEST_DTOA(3, 0.123456789012, "0.123"); + TEST_DTOA(2, 0.123456789012, "0.12"); + TEST_DTOA(1, 0.123456789012, "0.1"); + TEST_DTOA(4, 0.0001, "0.0001"); + TEST_DTOA(3, 0.0001, "0.0"); + TEST_DTOA(2, 0.0001, "0.0"); + TEST_DTOA(1, 0.0001, "0.0"); + TEST_DTOA(3, 1234567.8, "1234567.8"); + TEST_DTOA(3, 1e30, "1e30"); + TEST_DTOA(3, 5e-324, "0.0"); // Min subnormal positive double + TEST_DTOA(3, 2.225073858507201e-308, "0.0"); // Max subnormal positive double + TEST_DTOA(3, 2.2250738585072014e-308, "0.0"); // Min normal positive double + TEST_DTOA(3, 1.7976931348623157e308, "1.7976931348623157e308"); // Max double + TEST_DTOA(5, -0.14000000000000001, "-0.14"); + TEST_DTOA(4, -0.14000000000000001, "-0.14"); + TEST_DTOA(3, -0.14000000000000001, "-0.14"); + TEST_DTOA(3, -0.10000000000000001, "-0.1"); + TEST_DTOA(2, -0.10000000000000001, "-0.1"); + TEST_DTOA(1, -0.10000000000000001, "-0.1"); #undef TEST_DTOA } diff --git a/rapidjson/test/unittest/encodedstreamtest.cpp b/rapidjson/test/unittest/encodedstreamtest.cpp index bc234d3ba76..5e5e24b2fd0 100644 --- a/rapidjson/test/unittest/encodedstreamtest.cpp +++ b/rapidjson/test/unittest/encodedstreamtest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -22,292 +22,284 @@ using namespace rapidjson; -class EncodedStreamTest : public ::testing::Test { +class EncodedStreamTest : public ::testing::Test +{ public: - EncodedStreamTest() : json_(), length_() {} - virtual ~EncodedStreamTest(); + EncodedStreamTest() : json_(), length_() {} + virtual ~EncodedStreamTest(); - virtual void SetUp() { - json_ = ReadFile("utf8.json", true, &length_); - } + virtual void SetUp() + { json_ = ReadFile("utf8.json", true, &length_); + } - virtual void TearDown() { - free(json_); - json_ = 0; - } + virtual void TearDown() + { free(json_); + json_ = 0; + } private: - EncodedStreamTest(const EncodedStreamTest&); - EncodedStreamTest& operator=(const EncodedStreamTest&); - + EncodedStreamTest(const EncodedStreamTest&); + EncodedStreamTest& operator=(const EncodedStreamTest&); + protected: - static FILE* Open(const char* filename) { - const char *paths[] = { - "encodings", - "bin/encodings", - "../bin/encodings", - "../../bin/encodings", - "../../../bin/encodings" - }; - char buffer[1024]; - for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { - sprintf(buffer, "%s/%s", paths[i], filename); - FILE *fp = fopen(buffer, "rb"); - if (fp) - return fp; - } - return 0; + static FILE* Open(const char* filename) + { const char *paths[] = + { "encodings", + "bin/encodings", + "../bin/encodings", + "../../bin/encodings", + "../../../bin/encodings" + }; + char buffer[1024]; + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) + { sprintf(buffer, "%s/%s", paths[i], filename); + FILE *fp = fopen(buffer, "rb"); + if (fp) + return fp; } + return 0; + } - static char *ReadFile(const char* filename, bool appendPath, size_t* outLength) { - FILE *fp = appendPath ? Open(filename) : fopen(filename, "rb"); + static char *ReadFile(const char* filename, bool appendPath, size_t* outLength) + { FILE *fp = appendPath ? Open(filename) : fopen(filename, "rb"); - if (!fp) { - *outLength = 0; - return 0; - } - - fseek(fp, 0, SEEK_END); - *outLength = static_cast(ftell(fp)); - fseek(fp, 0, SEEK_SET); - char* buffer = static_cast(malloc(*outLength + 1)); - size_t readLength = fread(buffer, 1, *outLength, fp); - buffer[readLength] = '\0'; - fclose(fp); - return buffer; + if (!fp) + { *outLength = 0; + return 0; } - template - void TestEncodedInputStream(const char* filename) { - // Test FileReadStream - { - char buffer[16]; - FILE *fp = Open(filename); - ASSERT_TRUE(fp != 0); - FileReadStream fs(fp, buffer, sizeof(buffer)); - EncodedInputStream eis(fs); - StringStream s(json_); - - while (eis.Peek() != '\0') { - unsigned expected, actual; - EXPECT_TRUE(UTF8<>::Decode(s, &expected)); - EXPECT_TRUE(MemoryEncoding::Decode(eis, &actual)); - EXPECT_EQ(expected, actual); - } - EXPECT_EQ('\0', s.Peek()); - fclose(fp); - } + fseek(fp, 0, SEEK_END); + *outLength = static_cast(ftell(fp)); + fseek(fp, 0, SEEK_SET); + char* buffer = static_cast(malloc(*outLength + 1)); + size_t readLength = fread(buffer, 1, *outLength, fp); + buffer[readLength] = '\0'; + fclose(fp); + return buffer; + } - // Test MemoryStream - { - size_t size; - char* data = ReadFile(filename, true, &size); - MemoryStream ms(data, size); - EncodedInputStream eis(ms); - StringStream s(json_); + template + void TestEncodedInputStream(const char* filename) + { // Test FileReadStream + { char buffer[16]; + FILE *fp = Open(filename); + ASSERT_TRUE(fp != 0); + FileReadStream fs(fp, buffer, sizeof(buffer)); + EncodedInputStream eis(fs); + StringStream s(json_); - while (eis.Peek() != '\0') { - unsigned expected, actual; - EXPECT_TRUE(UTF8<>::Decode(s, &expected)); - EXPECT_TRUE(MemoryEncoding::Decode(eis, &actual)); - EXPECT_EQ(expected, actual); - } - EXPECT_EQ('\0', s.Peek()); - free(data); - EXPECT_EQ(size, eis.Tell()); - } + while (eis.Peek() != '\0') + { unsigned expected, actual; + EXPECT_TRUE(UTF8<>::Decode(s, &expected)); + EXPECT_TRUE(MemoryEncoding::Decode(eis, &actual)); + EXPECT_EQ(expected, actual); + } + EXPECT_EQ('\0', s.Peek()); + fclose(fp); } - void TestAutoUTFInputStream(const char *filename, bool expectHasBOM) { - // Test FileReadStream - { - char buffer[16]; - FILE *fp = Open(filename); - ASSERT_TRUE(fp != 0); - FileReadStream fs(fp, buffer, sizeof(buffer)); - AutoUTFInputStream eis(fs); - EXPECT_EQ(expectHasBOM, eis.HasBOM()); - StringStream s(json_); - while (eis.Peek() != '\0') { - unsigned expected, actual; - EXPECT_TRUE(UTF8<>::Decode(s, &expected)); - EXPECT_TRUE(AutoUTF::Decode(eis, &actual)); - EXPECT_EQ(expected, actual); - } - EXPECT_EQ('\0', s.Peek()); - fclose(fp); - } + // Test MemoryStream + { size_t size; + char* data = ReadFile(filename, true, &size); + MemoryStream ms(data, size); + EncodedInputStream eis(ms); + StringStream s(json_); - // Test MemoryStream - { - size_t size; - char* data = ReadFile(filename, true, &size); - MemoryStream ms(data, size); - AutoUTFInputStream eis(ms); - EXPECT_EQ(expectHasBOM, eis.HasBOM()); - StringStream s(json_); + while (eis.Peek() != '\0') + { unsigned expected, actual; + EXPECT_TRUE(UTF8<>::Decode(s, &expected)); + EXPECT_TRUE(MemoryEncoding::Decode(eis, &actual)); + EXPECT_EQ(expected, actual); + } + EXPECT_EQ('\0', s.Peek()); + free(data); + EXPECT_EQ(size, eis.Tell()); + } + } - while (eis.Peek() != '\0') { - unsigned expected, actual; - EXPECT_TRUE(UTF8<>::Decode(s, &expected)); - EXPECT_TRUE(AutoUTF::Decode(eis, &actual)); - EXPECT_EQ(expected, actual); - } - EXPECT_EQ('\0', s.Peek()); - free(data); - EXPECT_EQ(size, eis.Tell()); - } + void TestAutoUTFInputStream(const char *filename, bool expectHasBOM) + { // Test FileReadStream + { char buffer[16]; + FILE *fp = Open(filename); + ASSERT_TRUE(fp != 0); + FileReadStream fs(fp, buffer, sizeof(buffer)); + AutoUTFInputStream eis(fs); + EXPECT_EQ(expectHasBOM, eis.HasBOM()); + StringStream s(json_); + while (eis.Peek() != '\0') + { unsigned expected, actual; + EXPECT_TRUE(UTF8<>::Decode(s, &expected)); + EXPECT_TRUE(AutoUTF::Decode(eis, &actual)); + EXPECT_EQ(expected, actual); + } + EXPECT_EQ('\0', s.Peek()); + fclose(fp); } - template - void TestEncodedOutputStream(const char* expectedFilename, bool putBOM) { - // Test FileWriteStream - { - char filename[L_tmpnam]; - FILE* fp = TempFile(filename); - char buffer[16]; - FileWriteStream os(fp, buffer, sizeof(buffer)); - EncodedOutputStream eos(os, putBOM); - StringStream s(json_); - while (s.Peek() != '\0') { - bool success = Transcoder, MemoryEncoding>::Transcode(s, eos); - EXPECT_TRUE(success); - } - eos.Flush(); - fclose(fp); - EXPECT_TRUE(CompareFile(filename, expectedFilename)); - remove(filename); - } + // Test MemoryStream + { size_t size; + char* data = ReadFile(filename, true, &size); + MemoryStream ms(data, size); + AutoUTFInputStream eis(ms); + EXPECT_EQ(expectHasBOM, eis.HasBOM()); + StringStream s(json_); - // Test MemoryBuffer - { - MemoryBuffer mb; - EncodedOutputStream eos(mb, putBOM); - StringStream s(json_); - while (s.Peek() != '\0') { - bool success = Transcoder, MemoryEncoding>::Transcode(s, eos); - EXPECT_TRUE(success); - } - eos.Flush(); - EXPECT_TRUE(CompareBufferFile(mb.GetBuffer(), mb.GetSize(), expectedFilename)); - } + while (eis.Peek() != '\0') + { unsigned expected, actual; + EXPECT_TRUE(UTF8<>::Decode(s, &expected)); + EXPECT_TRUE(AutoUTF::Decode(eis, &actual)); + EXPECT_EQ(expected, actual); + } + EXPECT_EQ('\0', s.Peek()); + free(data); + EXPECT_EQ(size, eis.Tell()); } + } - void TestAutoUTFOutputStream(UTFType type, bool putBOM, const char *expectedFilename) { - // Test FileWriteStream - { - char filename[L_tmpnam]; - FILE* fp = TempFile(filename); - - char buffer[16]; - FileWriteStream os(fp, buffer, sizeof(buffer)); - AutoUTFOutputStream eos(os, type, putBOM); - StringStream s(json_); - while (s.Peek() != '\0') { - bool success = Transcoder, AutoUTF >::Transcode(s, eos); - EXPECT_TRUE(success); - } - eos.Flush(); - fclose(fp); - EXPECT_TRUE(CompareFile(filename, expectedFilename)); - remove(filename); - } + template + void TestEncodedOutputStream(const char* expectedFilename, bool putBOM) + { // Test FileWriteStream + { char filename[L_tmpnam]; + FILE* fp = TempFile(filename); + char buffer[16]; + FileWriteStream os(fp, buffer, sizeof(buffer)); + EncodedOutputStream eos(os, putBOM); + StringStream s(json_); + while (s.Peek() != '\0') + { bool success = Transcoder, MemoryEncoding>::Transcode(s, eos); + EXPECT_TRUE(success); + } + eos.Flush(); + fclose(fp); + EXPECT_TRUE(CompareFile(filename, expectedFilename)); + remove(filename); + } - // Test MemoryBuffer - { - MemoryBuffer mb; - AutoUTFOutputStream eos(mb, type, putBOM); - StringStream s(json_); - while (s.Peek() != '\0') { - bool success = Transcoder, AutoUTF >::Transcode(s, eos); - EXPECT_TRUE(success); - } - eos.Flush(); - EXPECT_TRUE(CompareBufferFile(mb.GetBuffer(), mb.GetSize(), expectedFilename)); - } + // Test MemoryBuffer + { MemoryBuffer mb; + EncodedOutputStream eos(mb, putBOM); + StringStream s(json_); + while (s.Peek() != '\0') + { bool success = Transcoder, MemoryEncoding>::Transcode(s, eos); + EXPECT_TRUE(success); + } + eos.Flush(); + EXPECT_TRUE(CompareBufferFile(mb.GetBuffer(), mb.GetSize(), expectedFilename)); } + } + + void TestAutoUTFOutputStream(UTFType type, bool putBOM, const char *expectedFilename) + { // Test FileWriteStream + { char filename[L_tmpnam]; + FILE* fp = TempFile(filename); - bool CompareFile(const char* filename, const char* expectedFilename) { - size_t actualLength, expectedLength; - char* actualBuffer = ReadFile(filename, false, &actualLength); - char* expectedBuffer = ReadFile(expectedFilename, true, &expectedLength); - bool ret = (expectedLength == actualLength) && memcmp(expectedBuffer, actualBuffer, actualLength) == 0; - free(actualBuffer); - free(expectedBuffer); - return ret; + char buffer[16]; + FileWriteStream os(fp, buffer, sizeof(buffer)); + AutoUTFOutputStream eos(os, type, putBOM); + StringStream s(json_); + while (s.Peek() != '\0') + { bool success = Transcoder, AutoUTF >::Transcode(s, eos); + EXPECT_TRUE(success); + } + eos.Flush(); + fclose(fp); + EXPECT_TRUE(CompareFile(filename, expectedFilename)); + remove(filename); } - bool CompareBufferFile(const char* actualBuffer, size_t actualLength, const char* expectedFilename) { - size_t expectedLength; - char* expectedBuffer = ReadFile(expectedFilename, true, &expectedLength); - bool ret = (expectedLength == actualLength) && memcmp(expectedBuffer, actualBuffer, actualLength) == 0; - free(expectedBuffer); - return ret; + // Test MemoryBuffer + { MemoryBuffer mb; + AutoUTFOutputStream eos(mb, type, putBOM); + StringStream s(json_); + while (s.Peek() != '\0') + { bool success = Transcoder, AutoUTF >::Transcode(s, eos); + EXPECT_TRUE(success); + } + eos.Flush(); + EXPECT_TRUE(CompareBufferFile(mb.GetBuffer(), mb.GetSize(), expectedFilename)); } + } - char *json_; - size_t length_; + bool CompareFile(const char* filename, const char* expectedFilename) + { size_t actualLength, expectedLength; + char* actualBuffer = ReadFile(filename, false, &actualLength); + char* expectedBuffer = ReadFile(expectedFilename, true, &expectedLength); + bool ret = (expectedLength == actualLength) && memcmp(expectedBuffer, actualBuffer, actualLength) == 0; + free(actualBuffer); + free(expectedBuffer); + return ret; + } + + bool CompareBufferFile(const char* actualBuffer, size_t actualLength, const char* expectedFilename) + { size_t expectedLength; + char* expectedBuffer = ReadFile(expectedFilename, true, &expectedLength); + bool ret = (expectedLength == actualLength) && memcmp(expectedBuffer, actualBuffer, actualLength) == 0; + free(expectedBuffer); + return ret; + } + + char *json_; + size_t length_; }; EncodedStreamTest::~EncodedStreamTest() {} -TEST_F(EncodedStreamTest, EncodedInputStream) { - TestEncodedInputStream, UTF8<> >("utf8.json"); - TestEncodedInputStream, UTF8<> >("utf8bom.json"); - TestEncodedInputStream, UTF16<> >("utf16le.json"); - TestEncodedInputStream, UTF16<> >("utf16lebom.json"); - TestEncodedInputStream, UTF16<> >("utf16be.json"); - TestEncodedInputStream, UTF16<> >("utf16bebom.json"); - TestEncodedInputStream, UTF32<> >("utf32le.json"); - TestEncodedInputStream, UTF32<> >("utf32lebom.json"); - TestEncodedInputStream, UTF32<> >("utf32be.json"); - TestEncodedInputStream, UTF32<> >("utf32bebom.json"); +TEST_F(EncodedStreamTest, EncodedInputStream) +{ TestEncodedInputStream, UTF8<> >("utf8.json"); + TestEncodedInputStream, UTF8<> >("utf8bom.json"); + TestEncodedInputStream, UTF16<> >("utf16le.json"); + TestEncodedInputStream, UTF16<> >("utf16lebom.json"); + TestEncodedInputStream, UTF16<> >("utf16be.json"); + TestEncodedInputStream, UTF16<> >("utf16bebom.json"); + TestEncodedInputStream, UTF32<> >("utf32le.json"); + TestEncodedInputStream, UTF32<> >("utf32lebom.json"); + TestEncodedInputStream, UTF32<> >("utf32be.json"); + TestEncodedInputStream, UTF32<> >("utf32bebom.json"); } -TEST_F(EncodedStreamTest, AutoUTFInputStream) { - TestAutoUTFInputStream("utf8.json", false); - TestAutoUTFInputStream("utf8bom.json", true); - TestAutoUTFInputStream("utf16le.json", false); - TestAutoUTFInputStream("utf16lebom.json",true); - TestAutoUTFInputStream("utf16be.json", false); - TestAutoUTFInputStream("utf16bebom.json",true); - TestAutoUTFInputStream("utf32le.json", false); - TestAutoUTFInputStream("utf32lebom.json",true); - TestAutoUTFInputStream("utf32be.json", false); - TestAutoUTFInputStream("utf32bebom.json", true); +TEST_F(EncodedStreamTest, AutoUTFInputStream) +{ TestAutoUTFInputStream("utf8.json", false); + TestAutoUTFInputStream("utf8bom.json", true); + TestAutoUTFInputStream("utf16le.json", false); + TestAutoUTFInputStream("utf16lebom.json",true); + TestAutoUTFInputStream("utf16be.json", false); + TestAutoUTFInputStream("utf16bebom.json",true); + TestAutoUTFInputStream("utf32le.json", false); + TestAutoUTFInputStream("utf32lebom.json",true); + TestAutoUTFInputStream("utf32be.json", false); + TestAutoUTFInputStream("utf32bebom.json", true); - { - // Auto detection fail, use user defined UTF type - const char json[] = "{ }"; - MemoryStream ms(json, sizeof(json)); - AutoUTFInputStream eis(ms, kUTF8); - EXPECT_FALSE(eis.HasBOM()); - EXPECT_EQ(kUTF8, eis.GetType()); - } + { // Auto detection fail, use user defined UTF type + const char json[] = "{ }"; + MemoryStream ms(json, sizeof(json)); + AutoUTFInputStream eis(ms, kUTF8); + EXPECT_FALSE(eis.HasBOM()); + EXPECT_EQ(kUTF8, eis.GetType()); + } } -TEST_F(EncodedStreamTest, EncodedOutputStream) { - TestEncodedOutputStream, UTF8<> >("utf8.json", false); - TestEncodedOutputStream, UTF8<> >("utf8bom.json", true); - TestEncodedOutputStream, UTF16<> >("utf16le.json", false); - TestEncodedOutputStream, UTF16<> >("utf16lebom.json",true); - TestEncodedOutputStream, UTF16<> >("utf16be.json", false); - TestEncodedOutputStream, UTF16<> >("utf16bebom.json",true); - TestEncodedOutputStream, UTF32<> >("utf32le.json", false); - TestEncodedOutputStream, UTF32<> >("utf32lebom.json",true); - TestEncodedOutputStream, UTF32<> >("utf32be.json", false); - TestEncodedOutputStream, UTF32<> >("utf32bebom.json",true); +TEST_F(EncodedStreamTest, EncodedOutputStream) +{ TestEncodedOutputStream, UTF8<> >("utf8.json", false); + TestEncodedOutputStream, UTF8<> >("utf8bom.json", true); + TestEncodedOutputStream, UTF16<> >("utf16le.json", false); + TestEncodedOutputStream, UTF16<> >("utf16lebom.json",true); + TestEncodedOutputStream, UTF16<> >("utf16be.json", false); + TestEncodedOutputStream, UTF16<> >("utf16bebom.json",true); + TestEncodedOutputStream, UTF32<> >("utf32le.json", false); + TestEncodedOutputStream, UTF32<> >("utf32lebom.json",true); + TestEncodedOutputStream, UTF32<> >("utf32be.json", false); + TestEncodedOutputStream, UTF32<> >("utf32bebom.json",true); } -TEST_F(EncodedStreamTest, AutoUTFOutputStream) { - TestAutoUTFOutputStream(kUTF8, false, "utf8.json"); - TestAutoUTFOutputStream(kUTF8, true, "utf8bom.json"); - TestAutoUTFOutputStream(kUTF16LE, false, "utf16le.json"); - TestAutoUTFOutputStream(kUTF16LE, true, "utf16lebom.json"); - TestAutoUTFOutputStream(kUTF16BE, false, "utf16be.json"); - TestAutoUTFOutputStream(kUTF16BE, true, "utf16bebom.json"); - TestAutoUTFOutputStream(kUTF32LE, false, "utf32le.json"); - TestAutoUTFOutputStream(kUTF32LE, true, "utf32lebom.json"); - TestAutoUTFOutputStream(kUTF32BE, false, "utf32be.json"); - TestAutoUTFOutputStream(kUTF32BE, true, "utf32bebom.json"); +TEST_F(EncodedStreamTest, AutoUTFOutputStream) +{ TestAutoUTFOutputStream(kUTF8, false, "utf8.json"); + TestAutoUTFOutputStream(kUTF8, true, "utf8bom.json"); + TestAutoUTFOutputStream(kUTF16LE, false, "utf16le.json"); + TestAutoUTFOutputStream(kUTF16LE, true, "utf16lebom.json"); + TestAutoUTFOutputStream(kUTF16BE, false, "utf16be.json"); + TestAutoUTFOutputStream(kUTF16BE, true, "utf16bebom.json"); + TestAutoUTFOutputStream(kUTF32LE, false, "utf32le.json"); + TestAutoUTFOutputStream(kUTF32LE, true, "utf32lebom.json"); + TestAutoUTFOutputStream(kUTF32BE, false, "utf32be.json"); + TestAutoUTFOutputStream(kUTF32BE, true, "utf32bebom.json"); } diff --git a/rapidjson/test/unittest/encodingstest.cpp b/rapidjson/test/unittest/encodingstest.cpp index 67b0391ed06..9918289e27b 100644 --- a/rapidjson/test/unittest/encodingstest.cpp +++ b/rapidjson/test/unittest/encodingstest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -23,217 +23,217 @@ using namespace rapidjson; // Verification of encoders/decoders with Hoehrmann's UTF8 decoder // http://www.unicode.org/Public/UNIDATA/Blocks.txt -static const unsigned kCodepointRanges[] = { - 0x0000, 0x007F, // Basic Latin - 0x0080, 0x00FF, // Latin-1 Supplement - 0x0100, 0x017F, // Latin Extended-A - 0x0180, 0x024F, // Latin Extended-B - 0x0250, 0x02AF, // IPA Extensions - 0x02B0, 0x02FF, // Spacing Modifier Letters - 0x0300, 0x036F, // Combining Diacritical Marks - 0x0370, 0x03FF, // Greek and Coptic - 0x0400, 0x04FF, // Cyrillic - 0x0500, 0x052F, // Cyrillic Supplement - 0x0530, 0x058F, // Armenian - 0x0590, 0x05FF, // Hebrew - 0x0600, 0x06FF, // Arabic - 0x0700, 0x074F, // Syriac - 0x0750, 0x077F, // Arabic Supplement - 0x0780, 0x07BF, // Thaana - 0x07C0, 0x07FF, // NKo - 0x0800, 0x083F, // Samaritan - 0x0840, 0x085F, // Mandaic - 0x0900, 0x097F, // Devanagari - 0x0980, 0x09FF, // Bengali - 0x0A00, 0x0A7F, // Gurmukhi - 0x0A80, 0x0AFF, // Gujarati - 0x0B00, 0x0B7F, // Oriya - 0x0B80, 0x0BFF, // Tamil - 0x0C00, 0x0C7F, // Telugu - 0x0C80, 0x0CFF, // Kannada - 0x0D00, 0x0D7F, // Malayalam - 0x0D80, 0x0DFF, // Sinhala - 0x0E00, 0x0E7F, // Thai - 0x0E80, 0x0EFF, // Lao - 0x0F00, 0x0FFF, // Tibetan - 0x1000, 0x109F, // Myanmar - 0x10A0, 0x10FF, // Georgian - 0x1100, 0x11FF, // Hangul Jamo - 0x1200, 0x137F, // Ethiopic - 0x1380, 0x139F, // Ethiopic Supplement - 0x13A0, 0x13FF, // Cherokee - 0x1400, 0x167F, // Unified Canadian Aboriginal Syllabics - 0x1680, 0x169F, // Ogham - 0x16A0, 0x16FF, // Runic - 0x1700, 0x171F, // Tagalog - 0x1720, 0x173F, // Hanunoo - 0x1740, 0x175F, // Buhid - 0x1760, 0x177F, // Tagbanwa - 0x1780, 0x17FF, // Khmer - 0x1800, 0x18AF, // Mongolian - 0x18B0, 0x18FF, // Unified Canadian Aboriginal Syllabics Extended - 0x1900, 0x194F, // Limbu - 0x1950, 0x197F, // Tai Le - 0x1980, 0x19DF, // New Tai Lue - 0x19E0, 0x19FF, // Khmer Symbols - 0x1A00, 0x1A1F, // Buginese - 0x1A20, 0x1AAF, // Tai Tham - 0x1B00, 0x1B7F, // Balinese - 0x1B80, 0x1BBF, // Sundanese - 0x1BC0, 0x1BFF, // Batak - 0x1C00, 0x1C4F, // Lepcha - 0x1C50, 0x1C7F, // Ol Chiki - 0x1CD0, 0x1CFF, // Vedic Extensions - 0x1D00, 0x1D7F, // Phonetic Extensions - 0x1D80, 0x1DBF, // Phonetic Extensions Supplement - 0x1DC0, 0x1DFF, // Combining Diacritical Marks Supplement - 0x1E00, 0x1EFF, // Latin Extended Additional - 0x1F00, 0x1FFF, // Greek Extended - 0x2000, 0x206F, // General Punctuation - 0x2070, 0x209F, // Superscripts and Subscripts - 0x20A0, 0x20CF, // Currency Symbols - 0x20D0, 0x20FF, // Combining Diacritical Marks for Symbols - 0x2100, 0x214F, // Letterlike Symbols - 0x2150, 0x218F, // Number Forms - 0x2190, 0x21FF, // Arrows - 0x2200, 0x22FF, // Mathematical Operators - 0x2300, 0x23FF, // Miscellaneous Technical - 0x2400, 0x243F, // Control Pictures - 0x2440, 0x245F, // Optical Character Recognition - 0x2460, 0x24FF, // Enclosed Alphanumerics - 0x2500, 0x257F, // Box Drawing - 0x2580, 0x259F, // Block Elements - 0x25A0, 0x25FF, // Geometric Shapes - 0x2600, 0x26FF, // Miscellaneous Symbols - 0x2700, 0x27BF, // Dingbats - 0x27C0, 0x27EF, // Miscellaneous Mathematical Symbols-A - 0x27F0, 0x27FF, // Supplemental Arrows-A - 0x2800, 0x28FF, // Braille Patterns - 0x2900, 0x297F, // Supplemental Arrows-B - 0x2980, 0x29FF, // Miscellaneous Mathematical Symbols-B - 0x2A00, 0x2AFF, // Supplemental Mathematical Operators - 0x2B00, 0x2BFF, // Miscellaneous Symbols and Arrows - 0x2C00, 0x2C5F, // Glagolitic - 0x2C60, 0x2C7F, // Latin Extended-C - 0x2C80, 0x2CFF, // Coptic - 0x2D00, 0x2D2F, // Georgian Supplement - 0x2D30, 0x2D7F, // Tifinagh - 0x2D80, 0x2DDF, // Ethiopic Extended - 0x2DE0, 0x2DFF, // Cyrillic Extended-A - 0x2E00, 0x2E7F, // Supplemental Punctuation - 0x2E80, 0x2EFF, // CJK Radicals Supplement - 0x2F00, 0x2FDF, // Kangxi Radicals - 0x2FF0, 0x2FFF, // Ideographic Description Characters - 0x3000, 0x303F, // CJK Symbols and Punctuation - 0x3040, 0x309F, // Hiragana - 0x30A0, 0x30FF, // Katakana - 0x3100, 0x312F, // Bopomofo - 0x3130, 0x318F, // Hangul Compatibility Jamo - 0x3190, 0x319F, // Kanbun - 0x31A0, 0x31BF, // Bopomofo Extended - 0x31C0, 0x31EF, // CJK Strokes - 0x31F0, 0x31FF, // Katakana Phonetic Extensions - 0x3200, 0x32FF, // Enclosed CJK Letters and Months - 0x3300, 0x33FF, // CJK Compatibility - 0x3400, 0x4DBF, // CJK Unified Ideographs Extension A - 0x4DC0, 0x4DFF, // Yijing Hexagram Symbols - 0x4E00, 0x9FFF, // CJK Unified Ideographs - 0xA000, 0xA48F, // Yi Syllables - 0xA490, 0xA4CF, // Yi Radicals - 0xA4D0, 0xA4FF, // Lisu - 0xA500, 0xA63F, // Vai - 0xA640, 0xA69F, // Cyrillic Extended-B - 0xA6A0, 0xA6FF, // Bamum - 0xA700, 0xA71F, // Modifier Tone Letters - 0xA720, 0xA7FF, // Latin Extended-D - 0xA800, 0xA82F, // Syloti Nagri - 0xA830, 0xA83F, // Common Indic Number Forms - 0xA840, 0xA87F, // Phags-pa - 0xA880, 0xA8DF, // Saurashtra - 0xA8E0, 0xA8FF, // Devanagari Extended - 0xA900, 0xA92F, // Kayah Li - 0xA930, 0xA95F, // Rejang - 0xA960, 0xA97F, // Hangul Jamo Extended-A - 0xA980, 0xA9DF, // Javanese - 0xAA00, 0xAA5F, // Cham - 0xAA60, 0xAA7F, // Myanmar Extended-A - 0xAA80, 0xAADF, // Tai Viet - 0xAB00, 0xAB2F, // Ethiopic Extended-A - 0xABC0, 0xABFF, // Meetei Mayek - 0xAC00, 0xD7AF, // Hangul Syllables - 0xD7B0, 0xD7FF, // Hangul Jamo Extended-B - //0xD800, 0xDB7F, // High Surrogates - //0xDB80, 0xDBFF, // High Private Use Surrogates - //0xDC00, 0xDFFF, // Low Surrogates - 0xE000, 0xF8FF, // Private Use Area - 0xF900, 0xFAFF, // CJK Compatibility Ideographs - 0xFB00, 0xFB4F, // Alphabetic Presentation Forms - 0xFB50, 0xFDFF, // Arabic Presentation Forms-A - 0xFE00, 0xFE0F, // Variation Selectors - 0xFE10, 0xFE1F, // Vertical Forms - 0xFE20, 0xFE2F, // Combining Half Marks - 0xFE30, 0xFE4F, // CJK Compatibility Forms - 0xFE50, 0xFE6F, // Small Form Variants - 0xFE70, 0xFEFF, // Arabic Presentation Forms-B - 0xFF00, 0xFFEF, // Halfwidth and Fullwidth Forms - 0xFFF0, 0xFFFF, // Specials - 0x10000, 0x1007F, // Linear B Syllabary - 0x10080, 0x100FF, // Linear B Ideograms - 0x10100, 0x1013F, // Aegean Numbers - 0x10140, 0x1018F, // Ancient Greek Numbers - 0x10190, 0x101CF, // Ancient Symbols - 0x101D0, 0x101FF, // Phaistos Disc - 0x10280, 0x1029F, // Lycian - 0x102A0, 0x102DF, // Carian - 0x10300, 0x1032F, // Old Italic - 0x10330, 0x1034F, // Gothic - 0x10380, 0x1039F, // Ugaritic - 0x103A0, 0x103DF, // Old Persian - 0x10400, 0x1044F, // Deseret - 0x10450, 0x1047F, // Shavian - 0x10480, 0x104AF, // Osmanya - 0x10800, 0x1083F, // Cypriot Syllabary - 0x10840, 0x1085F, // Imperial Aramaic - 0x10900, 0x1091F, // Phoenician - 0x10920, 0x1093F, // Lydian - 0x10A00, 0x10A5F, // Kharoshthi - 0x10A60, 0x10A7F, // Old South Arabian - 0x10B00, 0x10B3F, // Avestan - 0x10B40, 0x10B5F, // Inscriptional Parthian - 0x10B60, 0x10B7F, // Inscriptional Pahlavi - 0x10C00, 0x10C4F, // Old Turkic - 0x10E60, 0x10E7F, // Rumi Numeral Symbols - 0x11000, 0x1107F, // Brahmi - 0x11080, 0x110CF, // Kaithi - 0x12000, 0x123FF, // Cuneiform - 0x12400, 0x1247F, // Cuneiform Numbers and Punctuation - 0x13000, 0x1342F, // Egyptian Hieroglyphs - 0x16800, 0x16A3F, // Bamum Supplement - 0x1B000, 0x1B0FF, // Kana Supplement - 0x1D000, 0x1D0FF, // Byzantine Musical Symbols - 0x1D100, 0x1D1FF, // Musical Symbols - 0x1D200, 0x1D24F, // Ancient Greek Musical Notation - 0x1D300, 0x1D35F, // Tai Xuan Jing Symbols - 0x1D360, 0x1D37F, // Counting Rod Numerals - 0x1D400, 0x1D7FF, // Mathematical Alphanumeric Symbols - 0x1F000, 0x1F02F, // Mahjong Tiles - 0x1F030, 0x1F09F, // Domino Tiles - 0x1F0A0, 0x1F0FF, // Playing Cards - 0x1F100, 0x1F1FF, // Enclosed Alphanumeric Supplement - 0x1F200, 0x1F2FF, // Enclosed Ideographic Supplement - 0x1F300, 0x1F5FF, // Miscellaneous Symbols And Pictographs - 0x1F600, 0x1F64F, // Emoticons - 0x1F680, 0x1F6FF, // Transport And Map Symbols - 0x1F700, 0x1F77F, // Alchemical Symbols - 0x20000, 0x2A6DF, // CJK Unified Ideographs Extension B - 0x2A700, 0x2B73F, // CJK Unified Ideographs Extension C - 0x2B740, 0x2B81F, // CJK Unified Ideographs Extension D - 0x2F800, 0x2FA1F, // CJK Compatibility Ideographs Supplement - 0xE0000, 0xE007F, // Tags - 0xE0100, 0xE01EF, // Variation Selectors Supplement - 0xF0000, 0xFFFFF, // Supplementary Private Use Area-A - 0x100000, 0x10FFFF, // Supplementary Private Use Area-B - 0xFFFFFFFF +static const unsigned kCodepointRanges[] = +{ 0x0000, 0x007F, // Basic Latin + 0x0080, 0x00FF, // Latin-1 Supplement + 0x0100, 0x017F, // Latin Extended-A + 0x0180, 0x024F, // Latin Extended-B + 0x0250, 0x02AF, // IPA Extensions + 0x02B0, 0x02FF, // Spacing Modifier Letters + 0x0300, 0x036F, // Combining Diacritical Marks + 0x0370, 0x03FF, // Greek and Coptic + 0x0400, 0x04FF, // Cyrillic + 0x0500, 0x052F, // Cyrillic Supplement + 0x0530, 0x058F, // Armenian + 0x0590, 0x05FF, // Hebrew + 0x0600, 0x06FF, // Arabic + 0x0700, 0x074F, // Syriac + 0x0750, 0x077F, // Arabic Supplement + 0x0780, 0x07BF, // Thaana + 0x07C0, 0x07FF, // NKo + 0x0800, 0x083F, // Samaritan + 0x0840, 0x085F, // Mandaic + 0x0900, 0x097F, // Devanagari + 0x0980, 0x09FF, // Bengali + 0x0A00, 0x0A7F, // Gurmukhi + 0x0A80, 0x0AFF, // Gujarati + 0x0B00, 0x0B7F, // Oriya + 0x0B80, 0x0BFF, // Tamil + 0x0C00, 0x0C7F, // Telugu + 0x0C80, 0x0CFF, // Kannada + 0x0D00, 0x0D7F, // Malayalam + 0x0D80, 0x0DFF, // Sinhala + 0x0E00, 0x0E7F, // Thai + 0x0E80, 0x0EFF, // Lao + 0x0F00, 0x0FFF, // Tibetan + 0x1000, 0x109F, // Myanmar + 0x10A0, 0x10FF, // Georgian + 0x1100, 0x11FF, // Hangul Jamo + 0x1200, 0x137F, // Ethiopic + 0x1380, 0x139F, // Ethiopic Supplement + 0x13A0, 0x13FF, // Cherokee + 0x1400, 0x167F, // Unified Canadian Aboriginal Syllabics + 0x1680, 0x169F, // Ogham + 0x16A0, 0x16FF, // Runic + 0x1700, 0x171F, // Tagalog + 0x1720, 0x173F, // Hanunoo + 0x1740, 0x175F, // Buhid + 0x1760, 0x177F, // Tagbanwa + 0x1780, 0x17FF, // Khmer + 0x1800, 0x18AF, // Mongolian + 0x18B0, 0x18FF, // Unified Canadian Aboriginal Syllabics Extended + 0x1900, 0x194F, // Limbu + 0x1950, 0x197F, // Tai Le + 0x1980, 0x19DF, // New Tai Lue + 0x19E0, 0x19FF, // Khmer Symbols + 0x1A00, 0x1A1F, // Buginese + 0x1A20, 0x1AAF, // Tai Tham + 0x1B00, 0x1B7F, // Balinese + 0x1B80, 0x1BBF, // Sundanese + 0x1BC0, 0x1BFF, // Batak + 0x1C00, 0x1C4F, // Lepcha + 0x1C50, 0x1C7F, // Ol Chiki + 0x1CD0, 0x1CFF, // Vedic Extensions + 0x1D00, 0x1D7F, // Phonetic Extensions + 0x1D80, 0x1DBF, // Phonetic Extensions Supplement + 0x1DC0, 0x1DFF, // Combining Diacritical Marks Supplement + 0x1E00, 0x1EFF, // Latin Extended Additional + 0x1F00, 0x1FFF, // Greek Extended + 0x2000, 0x206F, // General Punctuation + 0x2070, 0x209F, // Superscripts and Subscripts + 0x20A0, 0x20CF, // Currency Symbols + 0x20D0, 0x20FF, // Combining Diacritical Marks for Symbols + 0x2100, 0x214F, // Letterlike Symbols + 0x2150, 0x218F, // Number Forms + 0x2190, 0x21FF, // Arrows + 0x2200, 0x22FF, // Mathematical Operators + 0x2300, 0x23FF, // Miscellaneous Technical + 0x2400, 0x243F, // Control Pictures + 0x2440, 0x245F, // Optical Character Recognition + 0x2460, 0x24FF, // Enclosed Alphanumerics + 0x2500, 0x257F, // Box Drawing + 0x2580, 0x259F, // Block Elements + 0x25A0, 0x25FF, // Geometric Shapes + 0x2600, 0x26FF, // Miscellaneous Symbols + 0x2700, 0x27BF, // Dingbats + 0x27C0, 0x27EF, // Miscellaneous Mathematical Symbols-A + 0x27F0, 0x27FF, // Supplemental Arrows-A + 0x2800, 0x28FF, // Braille Patterns + 0x2900, 0x297F, // Supplemental Arrows-B + 0x2980, 0x29FF, // Miscellaneous Mathematical Symbols-B + 0x2A00, 0x2AFF, // Supplemental Mathematical Operators + 0x2B00, 0x2BFF, // Miscellaneous Symbols and Arrows + 0x2C00, 0x2C5F, // Glagolitic + 0x2C60, 0x2C7F, // Latin Extended-C + 0x2C80, 0x2CFF, // Coptic + 0x2D00, 0x2D2F, // Georgian Supplement + 0x2D30, 0x2D7F, // Tifinagh + 0x2D80, 0x2DDF, // Ethiopic Extended + 0x2DE0, 0x2DFF, // Cyrillic Extended-A + 0x2E00, 0x2E7F, // Supplemental Punctuation + 0x2E80, 0x2EFF, // CJK Radicals Supplement + 0x2F00, 0x2FDF, // Kangxi Radicals + 0x2FF0, 0x2FFF, // Ideographic Description Characters + 0x3000, 0x303F, // CJK Symbols and Punctuation + 0x3040, 0x309F, // Hiragana + 0x30A0, 0x30FF, // Katakana + 0x3100, 0x312F, // Bopomofo + 0x3130, 0x318F, // Hangul Compatibility Jamo + 0x3190, 0x319F, // Kanbun + 0x31A0, 0x31BF, // Bopomofo Extended + 0x31C0, 0x31EF, // CJK Strokes + 0x31F0, 0x31FF, // Katakana Phonetic Extensions + 0x3200, 0x32FF, // Enclosed CJK Letters and Months + 0x3300, 0x33FF, // CJK Compatibility + 0x3400, 0x4DBF, // CJK Unified Ideographs Extension A + 0x4DC0, 0x4DFF, // Yijing Hexagram Symbols + 0x4E00, 0x9FFF, // CJK Unified Ideographs + 0xA000, 0xA48F, // Yi Syllables + 0xA490, 0xA4CF, // Yi Radicals + 0xA4D0, 0xA4FF, // Lisu + 0xA500, 0xA63F, // Vai + 0xA640, 0xA69F, // Cyrillic Extended-B + 0xA6A0, 0xA6FF, // Bamum + 0xA700, 0xA71F, // Modifier Tone Letters + 0xA720, 0xA7FF, // Latin Extended-D + 0xA800, 0xA82F, // Syloti Nagri + 0xA830, 0xA83F, // Common Indic Number Forms + 0xA840, 0xA87F, // Phags-pa + 0xA880, 0xA8DF, // Saurashtra + 0xA8E0, 0xA8FF, // Devanagari Extended + 0xA900, 0xA92F, // Kayah Li + 0xA930, 0xA95F, // Rejang + 0xA960, 0xA97F, // Hangul Jamo Extended-A + 0xA980, 0xA9DF, // Javanese + 0xAA00, 0xAA5F, // Cham + 0xAA60, 0xAA7F, // Myanmar Extended-A + 0xAA80, 0xAADF, // Tai Viet + 0xAB00, 0xAB2F, // Ethiopic Extended-A + 0xABC0, 0xABFF, // Meetei Mayek + 0xAC00, 0xD7AF, // Hangul Syllables + 0xD7B0, 0xD7FF, // Hangul Jamo Extended-B + //0xD800, 0xDB7F, // High Surrogates + //0xDB80, 0xDBFF, // High Private Use Surrogates + //0xDC00, 0xDFFF, // Low Surrogates + 0xE000, 0xF8FF, // Private Use Area + 0xF900, 0xFAFF, // CJK Compatibility Ideographs + 0xFB00, 0xFB4F, // Alphabetic Presentation Forms + 0xFB50, 0xFDFF, // Arabic Presentation Forms-A + 0xFE00, 0xFE0F, // Variation Selectors + 0xFE10, 0xFE1F, // Vertical Forms + 0xFE20, 0xFE2F, // Combining Half Marks + 0xFE30, 0xFE4F, // CJK Compatibility Forms + 0xFE50, 0xFE6F, // Small Form Variants + 0xFE70, 0xFEFF, // Arabic Presentation Forms-B + 0xFF00, 0xFFEF, // Halfwidth and Fullwidth Forms + 0xFFF0, 0xFFFF, // Specials + 0x10000, 0x1007F, // Linear B Syllabary + 0x10080, 0x100FF, // Linear B Ideograms + 0x10100, 0x1013F, // Aegean Numbers + 0x10140, 0x1018F, // Ancient Greek Numbers + 0x10190, 0x101CF, // Ancient Symbols + 0x101D0, 0x101FF, // Phaistos Disc + 0x10280, 0x1029F, // Lycian + 0x102A0, 0x102DF, // Carian + 0x10300, 0x1032F, // Old Italic + 0x10330, 0x1034F, // Gothic + 0x10380, 0x1039F, // Ugaritic + 0x103A0, 0x103DF, // Old Persian + 0x10400, 0x1044F, // Deseret + 0x10450, 0x1047F, // Shavian + 0x10480, 0x104AF, // Osmanya + 0x10800, 0x1083F, // Cypriot Syllabary + 0x10840, 0x1085F, // Imperial Aramaic + 0x10900, 0x1091F, // Phoenician + 0x10920, 0x1093F, // Lydian + 0x10A00, 0x10A5F, // Kharoshthi + 0x10A60, 0x10A7F, // Old South Arabian + 0x10B00, 0x10B3F, // Avestan + 0x10B40, 0x10B5F, // Inscriptional Parthian + 0x10B60, 0x10B7F, // Inscriptional Pahlavi + 0x10C00, 0x10C4F, // Old Turkic + 0x10E60, 0x10E7F, // Rumi Numeral Symbols + 0x11000, 0x1107F, // Brahmi + 0x11080, 0x110CF, // Kaithi + 0x12000, 0x123FF, // Cuneiform + 0x12400, 0x1247F, // Cuneiform Numbers and Punctuation + 0x13000, 0x1342F, // Egyptian Hieroglyphs + 0x16800, 0x16A3F, // Bamum Supplement + 0x1B000, 0x1B0FF, // Kana Supplement + 0x1D000, 0x1D0FF, // Byzantine Musical Symbols + 0x1D100, 0x1D1FF, // Musical Symbols + 0x1D200, 0x1D24F, // Ancient Greek Musical Notation + 0x1D300, 0x1D35F, // Tai Xuan Jing Symbols + 0x1D360, 0x1D37F, // Counting Rod Numerals + 0x1D400, 0x1D7FF, // Mathematical Alphanumeric Symbols + 0x1F000, 0x1F02F, // Mahjong Tiles + 0x1F030, 0x1F09F, // Domino Tiles + 0x1F0A0, 0x1F0FF, // Playing Cards + 0x1F100, 0x1F1FF, // Enclosed Alphanumeric Supplement + 0x1F200, 0x1F2FF, // Enclosed Ideographic Supplement + 0x1F300, 0x1F5FF, // Miscellaneous Symbols And Pictographs + 0x1F600, 0x1F64F, // Emoticons + 0x1F680, 0x1F6FF, // Transport And Map Symbols + 0x1F700, 0x1F77F, // Alchemical Symbols + 0x20000, 0x2A6DF, // CJK Unified Ideographs Extension B + 0x2A700, 0x2B73F, // CJK Unified Ideographs Extension C + 0x2B740, 0x2B81F, // CJK Unified Ideographs Extension D + 0x2F800, 0x2FA1F, // CJK Compatibility Ideographs Supplement + 0xE0000, 0xE007F, // Tags + 0xE0100, 0xE01EF, // Variation Selectors Supplement + 0xF0000, 0xFFFFF, // Supplementary Private Use Area-A + 0x100000, 0x10FFFF, // Supplementary Private Use Area-B + 0xFFFFFFFF }; // Copyright (c) 2008-2010 Bjoern Hoehrmann @@ -241,36 +241,36 @@ static const unsigned kCodepointRanges[] = { #define UTF8_ACCEPT 0u -static const unsigned char utf8d[] = { - // The first part of the table maps bytes to character classes that - // to reduce the size of the transition table and create bitmasks. - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, - 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, - 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, - 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, +static const unsigned char utf8d[] = +{ // The first part of the table maps bytes to character classes that + // to reduce the size of the transition table and create bitmasks. + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, + 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, - // The second part is a transition table that maps a combination - // of a state of the automaton and a character class to a state. - 0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12, - 12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12, - 12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12, - 12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12, - 12,36,12,12,12,12,12,12,12,12,12,12, + // The second part is a transition table that maps a combination + // of a state of the automaton and a character class to a state. + 0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12, + 12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12, + 12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12, + 12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12, + 12,36,12,12,12,12,12,12,12,12,12,12, }; -static unsigned inline decode(unsigned* state, unsigned* codep, unsigned byte) { - unsigned type = utf8d[byte]; +static unsigned inline decode(unsigned* state, unsigned* codep, unsigned byte) +{ unsigned type = utf8d[byte]; - *codep = (*state != UTF8_ACCEPT) ? - (byte & 0x3fu) | (*codep << 6) : - (0xff >> type) & (byte); + *codep = (*state != UTF8_ACCEPT) ? + (byte & 0x3fu) | (*codep << 6) : + (0xff >> type) & (byte); - *state = utf8d[256 + *state + type]; - return *state; + *state = utf8d[256 + *state + type]; + return *state; } //static bool IsUTF8(unsigned char* s) { @@ -282,170 +282,160 @@ static unsigned inline decode(unsigned* state, unsigned* codep, unsigned byte) { // return state == UTF8_ACCEPT; //} -TEST(EncodingsTest, UTF8) { - StringBuffer os, os2; - for (const unsigned* range = kCodepointRanges; *range != 0xFFFFFFFF; range += 2) { - for (unsigned codepoint = range[0]; codepoint <= range[1]; ++codepoint) { - os.Clear(); - UTF8<>::Encode(os, codepoint); - const char* encodedStr = os.GetString(); +TEST(EncodingsTest, UTF8) +{ StringBuffer os, os2; + for (const unsigned* range = kCodepointRanges; *range != 0xFFFFFFFF; range += 2) + { for (unsigned codepoint = range[0]; codepoint <= range[1]; ++codepoint) + { os.Clear(); + UTF8<>::Encode(os, codepoint); + const char* encodedStr = os.GetString(); - // Decode with Hoehrmann - { - unsigned decodedCodepoint = 0; - unsigned state = 0; + // Decode with Hoehrmann + { unsigned decodedCodepoint = 0; + unsigned state = 0; - unsigned decodedCount = 0; - for (const char* s = encodedStr; *s; ++s) - if (!decode(&state, &decodedCodepoint, static_cast(*s))) { - EXPECT_EQ(codepoint, decodedCodepoint); - decodedCount++; - } + unsigned decodedCount = 0; + for (const char* s = encodedStr; *s; ++s) + if (!decode(&state, &decodedCodepoint, static_cast(*s))) + { EXPECT_EQ(codepoint, decodedCodepoint); + decodedCount++; + } - if (*encodedStr) { // This decoder cannot handle U+0000 - EXPECT_EQ(1u, decodedCount); // Should only contain one code point - } + if (*encodedStr) // This decoder cannot handle U+0000 + { EXPECT_EQ(1u, decodedCount); // Should only contain one code point + } - EXPECT_EQ(UTF8_ACCEPT, state); - if (UTF8_ACCEPT != state) - std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl; - } + EXPECT_EQ(UTF8_ACCEPT, state); + if (UTF8_ACCEPT != state) + std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl; + } - // Decode - { - StringStream is(encodedStr); - unsigned decodedCodepoint; - bool result = UTF8<>::Decode(is, &decodedCodepoint); - EXPECT_TRUE(result); - EXPECT_EQ(codepoint, decodedCodepoint); - if (!result || codepoint != decodedCodepoint) - std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl; - } + // Decode + { StringStream is(encodedStr); + unsigned decodedCodepoint; + bool result = UTF8<>::Decode(is, &decodedCodepoint); + EXPECT_TRUE(result); + EXPECT_EQ(codepoint, decodedCodepoint); + if (!result || codepoint != decodedCodepoint) + std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl; + } - // Validate - { - StringStream is(encodedStr); - os2.Clear(); - bool result = UTF8<>::Validate(is, os2); - EXPECT_TRUE(result); - EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString())); - } - } + // Validate + { StringStream is(encodedStr); + os2.Clear(); + bool result = UTF8<>::Validate(is, os2); + EXPECT_TRUE(result); + EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString())); + } } + } } -TEST(EncodingsTest, UTF16) { - GenericStringBuffer > os, os2; - GenericStringBuffer > utf8os; - for (const unsigned* range = kCodepointRanges; *range != 0xFFFFFFFF; range += 2) { - for (unsigned codepoint = range[0]; codepoint <= range[1]; ++codepoint) { - os.Clear(); - UTF16<>::Encode(os, codepoint); - const UTF16<>::Ch* encodedStr = os.GetString(); +TEST(EncodingsTest, UTF16) +{ GenericStringBuffer > os, os2; + GenericStringBuffer > utf8os; + for (const unsigned* range = kCodepointRanges; *range != 0xFFFFFFFF; range += 2) + { for (unsigned codepoint = range[0]; codepoint <= range[1]; ++codepoint) + { os.Clear(); + UTF16<>::Encode(os, codepoint); + const UTF16<>::Ch* encodedStr = os.GetString(); - // Encode with Hoehrmann's code - if (codepoint != 0) // cannot handle U+0000 - { - // encode with UTF8<> first - utf8os.Clear(); - UTF8<>::Encode(utf8os, codepoint); + // Encode with Hoehrmann's code + if (codepoint != 0) // cannot handle U+0000 + { // encode with UTF8<> first + utf8os.Clear(); + UTF8<>::Encode(utf8os, codepoint); - // transcode from UTF8 to UTF16 with Hoehrmann's code - unsigned decodedCodepoint = 0; - unsigned state = 0; - UTF16<>::Ch buffer[3], *p = &buffer[0]; - for (const char* s = utf8os.GetString(); *s; ++s) { - if (!decode(&state, &decodedCodepoint, static_cast(*s))) - break; - } + // transcode from UTF8 to UTF16 with Hoehrmann's code + unsigned decodedCodepoint = 0; + unsigned state = 0; + UTF16<>::Ch buffer[3], *p = &buffer[0]; + for (const char* s = utf8os.GetString(); *s; ++s) + { if (!decode(&state, &decodedCodepoint, static_cast(*s))) + break; + } - if (codepoint <= 0xFFFF) - *p++ = static_cast::Ch>(decodedCodepoint); - else { - // Encode code points above U+FFFF as surrogate pair. - *p++ = static_cast::Ch>(0xD7C0 + (decodedCodepoint >> 10)); - *p++ = static_cast::Ch>(0xDC00 + (decodedCodepoint & 0x3FF)); - } - *p++ = '\0'; + if (codepoint <= 0xFFFF) + *p++ = static_cast::Ch>(decodedCodepoint); + else + { // Encode code points above U+FFFF as surrogate pair. + *p++ = static_cast::Ch>(0xD7C0 + (decodedCodepoint >> 10)); + *p++ = static_cast::Ch>(0xDC00 + (decodedCodepoint & 0x3FF)); + } + *p++ = '\0'; - EXPECT_EQ(0, StrCmp(buffer, encodedStr)); - } + EXPECT_EQ(0, StrCmp(buffer, encodedStr)); + } - // Decode - { - GenericStringStream > is(encodedStr); - unsigned decodedCodepoint; - bool result = UTF16<>::Decode(is, &decodedCodepoint); - EXPECT_TRUE(result); - EXPECT_EQ(codepoint, decodedCodepoint); - if (!result || codepoint != decodedCodepoint) - std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl; - } + // Decode + { GenericStringStream > is(encodedStr); + unsigned decodedCodepoint; + bool result = UTF16<>::Decode(is, &decodedCodepoint); + EXPECT_TRUE(result); + EXPECT_EQ(codepoint, decodedCodepoint); + if (!result || codepoint != decodedCodepoint) + std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl; + } - // Validate - { - GenericStringStream > is(encodedStr); - os2.Clear(); - bool result = UTF16<>::Validate(is, os2); - EXPECT_TRUE(result); - EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString())); - } - } + // Validate + { GenericStringStream > is(encodedStr); + os2.Clear(); + bool result = UTF16<>::Validate(is, os2); + EXPECT_TRUE(result); + EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString())); + } } + } } -TEST(EncodingsTest, UTF32) { - GenericStringBuffer > os, os2; - for (const unsigned* range = kCodepointRanges; *range != 0xFFFFFFFF; range += 2) { - for (unsigned codepoint = range[0]; codepoint <= range[1]; ++codepoint) { - os.Clear(); - UTF32<>::Encode(os, codepoint); - const UTF32<>::Ch* encodedStr = os.GetString(); +TEST(EncodingsTest, UTF32) +{ GenericStringBuffer > os, os2; + for (const unsigned* range = kCodepointRanges; *range != 0xFFFFFFFF; range += 2) + { for (unsigned codepoint = range[0]; codepoint <= range[1]; ++codepoint) + { os.Clear(); + UTF32<>::Encode(os, codepoint); + const UTF32<>::Ch* encodedStr = os.GetString(); - // Decode - { - GenericStringStream > is(encodedStr); - unsigned decodedCodepoint; - bool result = UTF32<>::Decode(is, &decodedCodepoint); - EXPECT_TRUE(result); - EXPECT_EQ(codepoint, decodedCodepoint); - if (!result || codepoint != decodedCodepoint) - std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl; - } + // Decode + { GenericStringStream > is(encodedStr); + unsigned decodedCodepoint; + bool result = UTF32<>::Decode(is, &decodedCodepoint); + EXPECT_TRUE(result); + EXPECT_EQ(codepoint, decodedCodepoint); + if (!result || codepoint != decodedCodepoint) + std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl; + } - // Validate - { - GenericStringStream > is(encodedStr); - os2.Clear(); - bool result = UTF32<>::Validate(is, os2); - EXPECT_TRUE(result); - EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString())); - } - } + // Validate + { GenericStringStream > is(encodedStr); + os2.Clear(); + bool result = UTF32<>::Validate(is, os2); + EXPECT_TRUE(result); + EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString())); + } } + } } -TEST(EncodingsTest, ASCII) { - StringBuffer os, os2; - for (unsigned codepoint = 0; codepoint < 128; codepoint++) { - os.Clear(); - ASCII<>::Encode(os, codepoint); - const ASCII<>::Ch* encodedStr = os.GetString(); - { - StringStream is(encodedStr); - unsigned decodedCodepoint; - bool result = ASCII<>::Decode(is, &decodedCodepoint); - if (!result || codepoint != decodedCodepoint) - std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl; - } +TEST(EncodingsTest, ASCII) +{ StringBuffer os, os2; + for (unsigned codepoint = 0; codepoint < 128; codepoint++) + { os.Clear(); + ASCII<>::Encode(os, codepoint); + const ASCII<>::Ch* encodedStr = os.GetString(); + { StringStream is(encodedStr); + unsigned decodedCodepoint; + bool result = ASCII<>::Decode(is, &decodedCodepoint); + if (!result || codepoint != decodedCodepoint) + std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl; + } - // Validate - { - StringStream is(encodedStr); - os2.Clear(); - bool result = ASCII<>::Validate(is, os2); - EXPECT_TRUE(result); - EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString())); - } + // Validate + { StringStream is(encodedStr); + os2.Clear(); + bool result = ASCII<>::Validate(is, os2); + EXPECT_TRUE(result); + EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString())); } + } } diff --git a/rapidjson/test/unittest/filestreamtest.cpp b/rapidjson/test/unittest/filestreamtest.cpp index a38133fa7f6..414dbea3b40 100644 --- a/rapidjson/test/unittest/filestreamtest.cpp +++ b/rapidjson/test/unittest/filestreamtest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -19,94 +19,95 @@ using namespace rapidjson; -class FileStreamTest : public ::testing::Test { +class FileStreamTest : public ::testing::Test +{ public: - FileStreamTest() : filename_(), json_(), length_() {} - virtual ~FileStreamTest(); - - virtual void SetUp() { - const char *paths[] = { - "data/sample.json", - "bin/data/sample.json", - "../bin/data/sample.json", - "../../bin/data/sample.json", - "../../../bin/data/sample.json" - }; - FILE* fp = 0; - for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { - fp = fopen(paths[i], "rb"); - if (fp) { - filename_ = paths[i]; - break; - } - } - ASSERT_TRUE(fp != 0); - - fseek(fp, 0, SEEK_END); - length_ = static_cast(ftell(fp)); - fseek(fp, 0, SEEK_SET); - json_ = static_cast(malloc(length_ + 1)); - size_t readLength = fread(json_, 1, length_, fp); - json_[readLength] = '\0'; - fclose(fp); + FileStreamTest() : filename_(), json_(), length_() {} + virtual ~FileStreamTest(); + + virtual void SetUp() + { const char *paths[] = + { "data/sample.json", + "bin/data/sample.json", + "../bin/data/sample.json", + "../../bin/data/sample.json", + "../../../bin/data/sample.json" + }; + FILE* fp = 0; + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) + { fp = fopen(paths[i], "rb"); + if (fp) + { filename_ = paths[i]; + break; + } } + ASSERT_TRUE(fp != 0); - virtual void TearDown() { - free(json_); - json_ = 0; - } + fseek(fp, 0, SEEK_END); + length_ = static_cast(ftell(fp)); + fseek(fp, 0, SEEK_SET); + json_ = static_cast(malloc(length_ + 1)); + size_t readLength = fread(json_, 1, length_, fp); + json_[readLength] = '\0'; + fclose(fp); + } + + virtual void TearDown() + { free(json_); + json_ = 0; + } private: - FileStreamTest(const FileStreamTest&); - FileStreamTest& operator=(const FileStreamTest&); - + FileStreamTest(const FileStreamTest&); + FileStreamTest& operator=(const FileStreamTest&); + protected: - const char* filename_; - char *json_; - size_t length_; + const char* filename_; + char *json_; + size_t length_; }; FileStreamTest::~FileStreamTest() {} -TEST_F(FileStreamTest, FileReadStream) { - FILE *fp = fopen(filename_, "rb"); - ASSERT_TRUE(fp != 0); - char buffer[65536]; - FileReadStream s(fp, buffer, sizeof(buffer)); +TEST_F(FileStreamTest, FileReadStream) +{ FILE *fp = fopen(filename_, "rb"); + ASSERT_TRUE(fp != 0); + char buffer[65536]; + FileReadStream s(fp, buffer, sizeof(buffer)); - for (size_t i = 0; i < length_; i++) { - EXPECT_EQ(json_[i], s.Peek()); - EXPECT_EQ(json_[i], s.Peek()); // 2nd time should be the same - EXPECT_EQ(json_[i], s.Take()); - } + for (size_t i = 0; i < length_; i++) + { EXPECT_EQ(json_[i], s.Peek()); + EXPECT_EQ(json_[i], s.Peek()); // 2nd time should be the same + EXPECT_EQ(json_[i], s.Take()); + } - EXPECT_EQ(length_, s.Tell()); - EXPECT_EQ('\0', s.Peek()); + EXPECT_EQ(length_, s.Tell()); + EXPECT_EQ('\0', s.Peek()); - fclose(fp); + fclose(fp); } -TEST_F(FileStreamTest, FileWriteStream) { - char filename[L_tmpnam]; - FILE* fp = TempFile(filename); +TEST_F(FileStreamTest, FileWriteStream) +{ char filename[L_tmpnam]; + FILE* fp = TempFile(filename); - char buffer[65536]; - FileWriteStream os(fp, buffer, sizeof(buffer)); - for (size_t i = 0; i < length_; i++) - os.Put(json_[i]); - os.Flush(); - fclose(fp); + char buffer[65536]; + FileWriteStream os(fp, buffer, sizeof(buffer)); + for (size_t i = 0; i < length_; i++) + os.Put(json_[i]); + os.Flush(); + fclose(fp); - // Read it back to verify - fp = fopen(filename, "rb"); - FileReadStream is(fp, buffer, sizeof(buffer)); + // Read it back to verify + fp = fopen(filename, "rb"); + FileReadStream is(fp, buffer, sizeof(buffer)); - for (size_t i = 0; i < length_; i++) - EXPECT_EQ(json_[i], is.Take()); + for (size_t i = 0; i < length_; i++) + EXPECT_EQ(json_[i], is.Take()); - EXPECT_EQ(length_, is.Tell()); - fclose(fp); + EXPECT_EQ(length_, is.Tell()); + fclose(fp); - //std::cout << filename << std::endl; - remove(filename); + //std::cout << filename << std::endl; + remove(filename); } diff --git a/rapidjson/test/unittest/fwdtest.cpp b/rapidjson/test/unittest/fwdtest.cpp index 4f326846116..cb2edb46097 100644 --- a/rapidjson/test/unittest/fwdtest.cpp +++ b/rapidjson/test/unittest/fwdtest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -25,67 +25,67 @@ RAPIDJSON_DIAG_OFF(effc++) using namespace rapidjson; -struct Foo { - Foo(); - ~Foo(); +struct Foo +{ Foo(); + ~Foo(); - // encodings.h - UTF8* utf8; - UTF16* utf16; - UTF16BE* utf16be; - UTF16LE* utf16le; - UTF32* utf32; - UTF32BE* utf32be; - UTF32LE* utf32le; - ASCII* ascii; - AutoUTF* autoutf; - Transcoder, UTF8 >* transcoder; + // encodings.h + UTF8* utf8; + UTF16* utf16; + UTF16BE* utf16be; + UTF16LE* utf16le; + UTF32* utf32; + UTF32BE* utf32be; + UTF32LE* utf32le; + ASCII* ascii; + AutoUTF* autoutf; + Transcoder, UTF8 >* transcoder; - // allocators.h - CrtAllocator* crtallocator; - MemoryPoolAllocator* memorypoolallocator; + // allocators.h + CrtAllocator* crtallocator; + MemoryPoolAllocator* memorypoolallocator; - // stream.h - StringStream* stringstream; - InsituStringStream* insitustringstream; + // stream.h + StringStream* stringstream; + InsituStringStream* insitustringstream; - // stringbuffer.h - StringBuffer* stringbuffer; + // stringbuffer.h + StringBuffer* stringbuffer; - // // filereadstream.h - // FileReadStream* filereadstream; + // // filereadstream.h + // FileReadStream* filereadstream; - // // filewritestream.h - // FileWriteStream* filewritestream; + // // filewritestream.h + // FileWriteStream* filewritestream; - // memorybuffer.h - MemoryBuffer* memorybuffer; + // memorybuffer.h + MemoryBuffer* memorybuffer; - // memorystream.h - MemoryStream* memorystream; + // memorystream.h + MemoryStream* memorystream; - // reader.h - BaseReaderHandler, void>* basereaderhandler; - Reader* reader; + // reader.h + BaseReaderHandler, void>* basereaderhandler; + Reader* reader; - // writer.h - Writer, UTF8, CrtAllocator, 0>* writer; + // writer.h + Writer, UTF8, CrtAllocator, 0>* writer; - // prettywriter.h - PrettyWriter, UTF8, CrtAllocator, 0>* prettywriter; + // prettywriter.h + PrettyWriter, UTF8, CrtAllocator, 0>* prettywriter; - // document.h - Value* value; - Document* document; + // document.h + Value* value; + Document* document; - // pointer.h - Pointer* pointer; + // pointer.h + Pointer* pointer; - // schema.h - SchemaDocument* schemadocument; - SchemaValidator* schemavalidator; + // schema.h + SchemaDocument* schemadocument; + SchemaValidator* schemavalidator; - // char buffer[16]; + // char buffer[16]; }; // Using type definitions here. @@ -100,126 +100,126 @@ struct Foo { #include "rapidjson/prettywriter.h" #include "rapidjson/schema.h" // -> pointer.h -Foo::Foo() : - // encodings.h - utf8(RAPIDJSON_NEW(UTF8<>)), - utf16(RAPIDJSON_NEW(UTF16<>)), - utf16be(RAPIDJSON_NEW(UTF16BE<>)), - utf16le(RAPIDJSON_NEW(UTF16LE<>)), - utf32(RAPIDJSON_NEW(UTF32<>)), - utf32be(RAPIDJSON_NEW(UTF32BE<>)), - utf32le(RAPIDJSON_NEW(UTF32LE<>)), - ascii(RAPIDJSON_NEW(ASCII<>)), - autoutf(RAPIDJSON_NEW(AutoUTF)), - transcoder(RAPIDJSON_NEW((Transcoder, UTF8<> >))), +Foo::Foo() : + // encodings.h + utf8(RAPIDJSON_NEW(UTF8<>)), + utf16(RAPIDJSON_NEW(UTF16<>)), + utf16be(RAPIDJSON_NEW(UTF16BE<>)), + utf16le(RAPIDJSON_NEW(UTF16LE<>)), + utf32(RAPIDJSON_NEW(UTF32<>)), + utf32be(RAPIDJSON_NEW(UTF32BE<>)), + utf32le(RAPIDJSON_NEW(UTF32LE<>)), + ascii(RAPIDJSON_NEW(ASCII<>)), + autoutf(RAPIDJSON_NEW(AutoUTF)), + transcoder(RAPIDJSON_NEW((Transcoder, UTF8<> >))), - // allocators.h - crtallocator(RAPIDJSON_NEW(CrtAllocator)), - memorypoolallocator(RAPIDJSON_NEW(MemoryPoolAllocator<>)), + // allocators.h + crtallocator(RAPIDJSON_NEW(CrtAllocator)), + memorypoolallocator(RAPIDJSON_NEW(MemoryPoolAllocator<>)), - // stream.h - stringstream(RAPIDJSON_NEW(StringStream(0))), - insitustringstream(RAPIDJSON_NEW(InsituStringStream(0))), + // stream.h + stringstream(RAPIDJSON_NEW(StringStream(0))), + insitustringstream(RAPIDJSON_NEW(InsituStringStream(0))), - // stringbuffer.h - stringbuffer(RAPIDJSON_NEW(StringBuffer)), + // stringbuffer.h + stringbuffer(RAPIDJSON_NEW(StringBuffer)), - // // filereadstream.h - // filereadstream(RAPIDJSON_NEW(FileReadStream(stdout, buffer, sizeof(buffer)))), + // // filereadstream.h + // filereadstream(RAPIDJSON_NEW(FileReadStream(stdout, buffer, sizeof(buffer)))), - // // filewritestream.h - // filewritestream(RAPIDJSON_NEW(FileWriteStream(stdout, buffer, sizeof(buffer)))), + // // filewritestream.h + // filewritestream(RAPIDJSON_NEW(FileWriteStream(stdout, buffer, sizeof(buffer)))), - // memorybuffer.h - memorybuffer(RAPIDJSON_NEW(MemoryBuffer)), + // memorybuffer.h + memorybuffer(RAPIDJSON_NEW(MemoryBuffer)), - // memorystream.h - memorystream(RAPIDJSON_NEW(MemoryStream(0, 0))), + // memorystream.h + memorystream(RAPIDJSON_NEW(MemoryStream(0, 0))), - // reader.h - basereaderhandler(RAPIDJSON_NEW((BaseReaderHandler, void>))), - reader(RAPIDJSON_NEW(Reader)), + // reader.h + basereaderhandler(RAPIDJSON_NEW((BaseReaderHandler, void>))), + reader(RAPIDJSON_NEW(Reader)), - // writer.h - writer(RAPIDJSON_NEW((Writer))), + // writer.h + writer(RAPIDJSON_NEW((Writer))), - // prettywriter.h - prettywriter(RAPIDJSON_NEW((PrettyWriter))), + // prettywriter.h + prettywriter(RAPIDJSON_NEW((PrettyWriter))), - // document.h - value(RAPIDJSON_NEW(Value)), - document(RAPIDJSON_NEW(Document)), + // document.h + value(RAPIDJSON_NEW(Value)), + document(RAPIDJSON_NEW(Document)), - // pointer.h - pointer(RAPIDJSON_NEW(Pointer)), + // pointer.h + pointer(RAPIDJSON_NEW(Pointer)), - // schema.h - schemadocument(RAPIDJSON_NEW(SchemaDocument(*document))), - schemavalidator(RAPIDJSON_NEW(SchemaValidator(*schemadocument))) + // schema.h + schemadocument(RAPIDJSON_NEW(SchemaDocument(*document))), + schemavalidator(RAPIDJSON_NEW(SchemaValidator(*schemadocument))) { } -Foo::~Foo() { - // encodings.h - RAPIDJSON_DELETE(utf8); - RAPIDJSON_DELETE(utf16); - RAPIDJSON_DELETE(utf16be); - RAPIDJSON_DELETE(utf16le); - RAPIDJSON_DELETE(utf32); - RAPIDJSON_DELETE(utf32be); - RAPIDJSON_DELETE(utf32le); - RAPIDJSON_DELETE(ascii); - RAPIDJSON_DELETE(autoutf); - RAPIDJSON_DELETE(transcoder); +Foo::~Foo() +{ // encodings.h + RAPIDJSON_DELETE(utf8); + RAPIDJSON_DELETE(utf16); + RAPIDJSON_DELETE(utf16be); + RAPIDJSON_DELETE(utf16le); + RAPIDJSON_DELETE(utf32); + RAPIDJSON_DELETE(utf32be); + RAPIDJSON_DELETE(utf32le); + RAPIDJSON_DELETE(ascii); + RAPIDJSON_DELETE(autoutf); + RAPIDJSON_DELETE(transcoder); - // allocators.h - RAPIDJSON_DELETE(crtallocator); - RAPIDJSON_DELETE(memorypoolallocator); + // allocators.h + RAPIDJSON_DELETE(crtallocator); + RAPIDJSON_DELETE(memorypoolallocator); - // stream.h - RAPIDJSON_DELETE(stringstream); - RAPIDJSON_DELETE(insitustringstream); + // stream.h + RAPIDJSON_DELETE(stringstream); + RAPIDJSON_DELETE(insitustringstream); - // stringbuffer.h - RAPIDJSON_DELETE(stringbuffer); + // stringbuffer.h + RAPIDJSON_DELETE(stringbuffer); - // // filereadstream.h - // RAPIDJSON_DELETE(filereadstream); + // // filereadstream.h + // RAPIDJSON_DELETE(filereadstream); - // // filewritestream.h - // RAPIDJSON_DELETE(filewritestream); + // // filewritestream.h + // RAPIDJSON_DELETE(filewritestream); - // memorybuffer.h - RAPIDJSON_DELETE(memorybuffer); + // memorybuffer.h + RAPIDJSON_DELETE(memorybuffer); - // memorystream.h - RAPIDJSON_DELETE(memorystream); + // memorystream.h + RAPIDJSON_DELETE(memorystream); - // reader.h - RAPIDJSON_DELETE(basereaderhandler); - RAPIDJSON_DELETE(reader); + // reader.h + RAPIDJSON_DELETE(basereaderhandler); + RAPIDJSON_DELETE(reader); - // writer.h - RAPIDJSON_DELETE(writer); + // writer.h + RAPIDJSON_DELETE(writer); - // prettywriter.h - RAPIDJSON_DELETE(prettywriter); + // prettywriter.h + RAPIDJSON_DELETE(prettywriter); - // document.h - RAPIDJSON_DELETE(value); - RAPIDJSON_DELETE(document); + // document.h + RAPIDJSON_DELETE(value); + RAPIDJSON_DELETE(document); - // pointer.h - RAPIDJSON_DELETE(pointer); + // pointer.h + RAPIDJSON_DELETE(pointer); - // schema.h - RAPIDJSON_DELETE(schemadocument); - RAPIDJSON_DELETE(schemavalidator); + // schema.h + RAPIDJSON_DELETE(schemadocument); + RAPIDJSON_DELETE(schemavalidator); } -TEST(Fwd, Fwd) { - Foo f; +TEST(Fwd, Fwd) +{ Foo f; } #ifdef __GNUC__ diff --git a/rapidjson/test/unittest/istreamwrappertest.cpp b/rapidjson/test/unittest/istreamwrappertest.cpp index 9d6fbcff0de..2a7a26bed1d 100644 --- a/rapidjson/test/unittest/istreamwrappertest.cpp +++ b/rapidjson/test/unittest/istreamwrappertest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -29,118 +29,115 @@ using namespace rapidjson; using namespace std; template -static void TestStringStream() { - typedef typename StringStreamType::char_type Ch; - - { - StringStreamType iss; - BasicIStreamWrapper is(iss); - EXPECT_EQ(0, is.Tell()); - if (sizeof(Ch) == 1) { - EXPECT_EQ(0, is.Peek4()); - EXPECT_EQ(0, is.Tell()); - } - EXPECT_EQ(0, is.Peek()); - EXPECT_EQ(0, is.Take()); - EXPECT_EQ(0, is.Tell()); +static void TestStringStream() +{ typedef typename StringStreamType::char_type Ch; + + { StringStreamType iss; + BasicIStreamWrapper is(iss); + EXPECT_EQ(0, is.Tell()); + if (sizeof(Ch) == 1) + { EXPECT_EQ(0, is.Peek4()); + EXPECT_EQ(0, is.Tell()); } - - { - Ch s[] = { 'A', 'B', 'C', '\0' }; - StringStreamType iss(s); - BasicIStreamWrapper is(iss); - EXPECT_EQ(0, is.Tell()); - if (sizeof(Ch) == 1) { - EXPECT_EQ(0, is.Peek4()); // less than 4 bytes - } - for (int i = 0; i < 3; i++) { - EXPECT_EQ(static_cast(i), is.Tell()); - EXPECT_EQ('A' + i, is.Peek()); - EXPECT_EQ('A' + i, is.Peek()); - EXPECT_EQ('A' + i, is.Take()); - } - EXPECT_EQ(3, is.Tell()); - EXPECT_EQ(0, is.Peek()); - EXPECT_EQ(0, is.Take()); + EXPECT_EQ(0, is.Peek()); + EXPECT_EQ(0, is.Take()); + EXPECT_EQ(0, is.Tell()); + } + + { Ch s[] = { 'A', 'B', 'C', '\0' }; + StringStreamType iss(s); + BasicIStreamWrapper is(iss); + EXPECT_EQ(0, is.Tell()); + if (sizeof(Ch) == 1) + { EXPECT_EQ(0, is.Peek4()); // less than 4 bytes } - - { - Ch s[] = { 'A', 'B', 'C', 'D', 'E', '\0' }; - StringStreamType iss(s); - BasicIStreamWrapper is(iss); - if (sizeof(Ch) == 1) { - const Ch* c = is.Peek4(); - for (int i = 0; i < 4; i++) - EXPECT_EQ('A' + i, c[i]); - EXPECT_EQ(0, is.Tell()); - } - for (int i = 0; i < 5; i++) { - EXPECT_EQ(static_cast(i), is.Tell()); - EXPECT_EQ('A' + i, is.Peek()); - EXPECT_EQ('A' + i, is.Peek()); - EXPECT_EQ('A' + i, is.Take()); - } - EXPECT_EQ(5, is.Tell()); - EXPECT_EQ(0, is.Peek()); - EXPECT_EQ(0, is.Take()); + for (int i = 0; i < 3; i++) + { EXPECT_EQ(static_cast(i), is.Tell()); + EXPECT_EQ('A' + i, is.Peek()); + EXPECT_EQ('A' + i, is.Peek()); + EXPECT_EQ('A' + i, is.Take()); } + EXPECT_EQ(3, is.Tell()); + EXPECT_EQ(0, is.Peek()); + EXPECT_EQ(0, is.Take()); + } + + { Ch s[] = { 'A', 'B', 'C', 'D', 'E', '\0' }; + StringStreamType iss(s); + BasicIStreamWrapper is(iss); + if (sizeof(Ch) == 1) + { const Ch* c = is.Peek4(); + for (int i = 0; i < 4; i++) + EXPECT_EQ('A' + i, c[i]); + EXPECT_EQ(0, is.Tell()); + } + for (int i = 0; i < 5; i++) + { EXPECT_EQ(static_cast(i), is.Tell()); + EXPECT_EQ('A' + i, is.Peek()); + EXPECT_EQ('A' + i, is.Peek()); + EXPECT_EQ('A' + i, is.Take()); + } + EXPECT_EQ(5, is.Tell()); + EXPECT_EQ(0, is.Peek()); + EXPECT_EQ(0, is.Take()); + } } -TEST(IStreamWrapper, istringstream) { - TestStringStream(); +TEST(IStreamWrapper, istringstream) +{ TestStringStream(); } -TEST(IStreamWrapper, stringstream) { - TestStringStream(); +TEST(IStreamWrapper, stringstream) +{ TestStringStream(); } -TEST(IStreamWrapper, wistringstream) { - TestStringStream(); +TEST(IStreamWrapper, wistringstream) +{ TestStringStream(); } -TEST(IStreamWrapper, wstringstream) { - TestStringStream(); +TEST(IStreamWrapper, wstringstream) +{ TestStringStream(); } template -static bool Open(FileStreamType& fs, const char* filename) { - const char *paths[] = { - "encodings", - "bin/encodings", - "../bin/encodings", - "../../bin/encodings", - "../../../bin/encodings" - }; - char buffer[1024]; - for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { - sprintf(buffer, "%s/%s", paths[i], filename); - fs.open(buffer, ios_base::in | ios_base::binary); - if (fs.is_open()) - return true; - } - return false; +static bool Open(FileStreamType& fs, const char* filename) +{ const char *paths[] = + { "encodings", + "bin/encodings", + "../bin/encodings", + "../../bin/encodings", + "../../../bin/encodings" + }; + char buffer[1024]; + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) + { sprintf(buffer, "%s/%s", paths[i], filename); + fs.open(buffer, ios_base::in | ios_base::binary); + if (fs.is_open()) + return true; + } + return false; } -TEST(IStreamWrapper, ifstream) { - ifstream ifs; - ASSERT_TRUE(Open(ifs, "utf8bom.json")); - IStreamWrapper isw(ifs); - EncodedInputStream, IStreamWrapper> eis(isw); - Document d; - EXPECT_TRUE(!d.ParseStream(eis).HasParseError()); - EXPECT_TRUE(d.IsObject()); - EXPECT_EQ(5, d.MemberCount()); +TEST(IStreamWrapper, ifstream) +{ ifstream ifs; + ASSERT_TRUE(Open(ifs, "utf8bom.json")); + IStreamWrapper isw(ifs); + EncodedInputStream, IStreamWrapper> eis(isw); + Document d; + EXPECT_TRUE(!d.ParseStream(eis).HasParseError()); + EXPECT_TRUE(d.IsObject()); + EXPECT_EQ(5, d.MemberCount()); } -TEST(IStreamWrapper, fstream) { - fstream fs; - ASSERT_TRUE(Open(fs, "utf8bom.json")); - IStreamWrapper isw(fs); - EncodedInputStream, IStreamWrapper> eis(isw); - Document d; - EXPECT_TRUE(!d.ParseStream(eis).HasParseError()); - EXPECT_TRUE(d.IsObject()); - EXPECT_EQ(5, d.MemberCount()); +TEST(IStreamWrapper, fstream) +{ fstream fs; + ASSERT_TRUE(Open(fs, "utf8bom.json")); + IStreamWrapper isw(fs); + EncodedInputStream, IStreamWrapper> eis(isw); + Document d; + EXPECT_TRUE(!d.ParseStream(eis).HasParseError()); + EXPECT_TRUE(d.IsObject()); + EXPECT_EQ(5, d.MemberCount()); } // wifstream/wfstream only works on C++11 with codecvt_utf16 @@ -148,30 +145,30 @@ TEST(IStreamWrapper, fstream) { #if 0 #include -TEST(IStreamWrapper, wifstream) { - wifstream ifs; - ASSERT_TRUE(Open(ifs, "utf16bebom.json")); - ifs.imbue(std::locale(ifs.getloc(), - new std::codecvt_utf16)); - WIStreamWrapper isw(ifs); - GenericDocument > d; - d.ParseStream, WIStreamWrapper>(isw); - EXPECT_TRUE(!d.HasParseError()); - EXPECT_TRUE(d.IsObject()); - EXPECT_EQ(5, d.MemberCount()); +TEST(IStreamWrapper, wifstream) +{ wifstream ifs; + ASSERT_TRUE(Open(ifs, "utf16bebom.json")); + ifs.imbue(std::locale(ifs.getloc(), + new std::codecvt_utf16)); + WIStreamWrapper isw(ifs); + GenericDocument > d; + d.ParseStream, WIStreamWrapper>(isw); + EXPECT_TRUE(!d.HasParseError()); + EXPECT_TRUE(d.IsObject()); + EXPECT_EQ(5, d.MemberCount()); } -TEST(IStreamWrapper, wfstream) { - wfstream fs; - ASSERT_TRUE(Open(fs, "utf16bebom.json")); - fs.imbue(std::locale(fs.getloc(), - new std::codecvt_utf16)); - WIStreamWrapper isw(fs); - GenericDocument > d; - d.ParseStream, WIStreamWrapper>(isw); - EXPECT_TRUE(!d.HasParseError()); - EXPECT_TRUE(d.IsObject()); - EXPECT_EQ(5, d.MemberCount()); +TEST(IStreamWrapper, wfstream) +{ wfstream fs; + ASSERT_TRUE(Open(fs, "utf16bebom.json")); + fs.imbue(std::locale(fs.getloc(), + new std::codecvt_utf16)); + WIStreamWrapper isw(fs); + GenericDocument > d; + d.ParseStream, WIStreamWrapper>(isw); + EXPECT_TRUE(!d.HasParseError()); + EXPECT_TRUE(d.IsObject()); + EXPECT_EQ(5, d.MemberCount()); } #endif diff --git a/rapidjson/test/unittest/itoatest.cpp b/rapidjson/test/unittest/itoatest.cpp index b752a6a26ee..bcad44d5cea 100644 --- a/rapidjson/test/unittest/itoatest.cpp +++ b/rapidjson/test/unittest/itoatest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -23,136 +23,142 @@ RAPIDJSON_DIAG_OFF(type-limits) using namespace rapidjson::internal; template -struct Traits { +struct Traits +{ }; template <> -struct Traits { - enum { kBufferSize = 11 }; - enum { kMaxDigit = 10 }; - static uint32_t Negate(uint32_t x) { return x; } +struct Traits +{ enum { kBufferSize = 11 }; + enum { kMaxDigit = 10 }; + static uint32_t Negate(uint32_t x) { return x; } }; template <> -struct Traits { - enum { kBufferSize = 12 }; - enum { kMaxDigit = 10 }; - static int32_t Negate(int32_t x) { return -x; } +struct Traits +{ enum { kBufferSize = 12 }; + enum { kMaxDigit = 10 }; + static int32_t Negate(int32_t x) { return -x; } }; template <> -struct Traits { - enum { kBufferSize = 21 }; - enum { kMaxDigit = 20 }; - static uint64_t Negate(uint64_t x) { return x; } +struct Traits +{ enum { kBufferSize = 21 }; + enum { kMaxDigit = 20 }; + static uint64_t Negate(uint64_t x) { return x; } }; template <> -struct Traits { - enum { kBufferSize = 22 }; - enum { kMaxDigit = 20 }; - static int64_t Negate(int64_t x) { return -x; } +struct Traits +{ enum { kBufferSize = 22 }; + enum { kMaxDigit = 20 }; + static int64_t Negate(int64_t x) { return -x; } }; template -static void VerifyValue(T value, void(*f)(T, char*), char* (*g)(T, char*)) { - char buffer1[Traits::kBufferSize]; - char buffer2[Traits::kBufferSize]; +static void VerifyValue(T value, void(*f)(T, char*), char* (*g)(T, char*)) +{ char buffer1[Traits::kBufferSize]; + char buffer2[Traits::kBufferSize]; + + f(value, buffer1); + *g(value, buffer2) = '\0'; - f(value, buffer1); - *g(value, buffer2) = '\0'; - - EXPECT_STREQ(buffer1, buffer2); + EXPECT_STREQ(buffer1, buffer2); } template -static void Verify(void(*f)(T, char*), char* (*g)(T, char*)) { - // Boundary cases - VerifyValue(0, f, g); - VerifyValue(std::numeric_limits::min(), f, g); - VerifyValue(std::numeric_limits::max(), f, g); - - // 2^n - 1, 2^n, 10^n - 1, 10^n until overflow - for (uint32_t power = 2; power <= 10; power += 8) { - T i = 1, last; - do { - VerifyValue(i - 1, f, g); - VerifyValue(i, f, g); - if (std::numeric_limits::min() < 0) { - VerifyValue(Traits::Negate(i), f, g); - VerifyValue(Traits::Negate(i + 1), f, g); - } - last = i; - if (i > static_cast(std::numeric_limits::max() / static_cast(power))) - break; - i *= power; - } while (last < i); +static void Verify(void(*f)(T, char*), char* (*g)(T, char*)) +{ // Boundary cases + VerifyValue(0, f, g); + VerifyValue(std::numeric_limits::min(), f, g); + VerifyValue(std::numeric_limits::max(), f, g); + + // 2^n - 1, 2^n, 10^n - 1, 10^n until overflow + for (uint32_t power = 2; power <= 10; power += 8) + { T i = 1, last; + do + { VerifyValue(i - 1, f, g); + VerifyValue(i, f, g); + if (std::numeric_limits::min() < 0) + { VerifyValue(Traits::Negate(i), f, g); + VerifyValue(Traits::Negate(i + 1), f, g); + } + last = i; + if (i > static_cast(std::numeric_limits::max() / static_cast(power))) + break; + i *= power; } + while (last < i); + } } -static void u32toa_naive(uint32_t value, char* buffer) { - char temp[10]; - char *p = temp; - do { - *p++ = static_cast(char(value % 10) + '0'); - value /= 10; - } while (value > 0); - - do { - *buffer++ = *--p; - } while (p != temp); - - *buffer = '\0'; +static void u32toa_naive(uint32_t value, char* buffer) +{ char temp[10]; + char *p = temp; + do + { *p++ = static_cast(char(value % 10) + '0'); + value /= 10; + } + while (value > 0); + + do + { *buffer++ = *--p; + } + while (p != temp); + + *buffer = '\0'; } -static void i32toa_naive(int32_t value, char* buffer) { - uint32_t u = static_cast(value); - if (value < 0) { - *buffer++ = '-'; - u = ~u + 1; - } - u32toa_naive(u, buffer); +static void i32toa_naive(int32_t value, char* buffer) +{ uint32_t u = static_cast(value); + if (value < 0) + { *buffer++ = '-'; + u = ~u + 1; + } + u32toa_naive(u, buffer); } -static void u64toa_naive(uint64_t value, char* buffer) { - char temp[20]; - char *p = temp; - do { - *p++ = static_cast(char(value % 10) + '0'); - value /= 10; - } while (value > 0); - - do { - *buffer++ = *--p; - } while (p != temp); - - *buffer = '\0'; +static void u64toa_naive(uint64_t value, char* buffer) +{ char temp[20]; + char *p = temp; + do + { *p++ = static_cast(char(value % 10) + '0'); + value /= 10; + } + while (value > 0); + + do + { *buffer++ = *--p; + } + while (p != temp); + + *buffer = '\0'; } -static void i64toa_naive(int64_t value, char* buffer) { - uint64_t u = static_cast(value); - if (value < 0) { - *buffer++ = '-'; - u = ~u + 1; - } - u64toa_naive(u, buffer); +static void i64toa_naive(int64_t value, char* buffer) +{ uint64_t u = static_cast(value); + if (value < 0) + { *buffer++ = '-'; + u = ~u + 1; + } + u64toa_naive(u, buffer); } -TEST(itoa, u32toa) { - Verify(u32toa_naive, u32toa); +TEST(itoa, u32toa) +{ Verify(u32toa_naive, u32toa); } -TEST(itoa, i32toa) { - Verify(i32toa_naive, i32toa); +TEST(itoa, i32toa) +{ Verify(i32toa_naive, i32toa); } -TEST(itoa, u64toa) { - Verify(u64toa_naive, u64toa); +TEST(itoa, u64toa) +{ Verify(u64toa_naive, u64toa); } -TEST(itoa, i64toa) { - Verify(i64toa_naive, i64toa); +TEST(itoa, i64toa) +{ Verify(i64toa_naive, i64toa); } #ifdef __GNUC__ diff --git a/rapidjson/test/unittest/jsoncheckertest.cpp b/rapidjson/test/unittest/jsoncheckertest.cpp index bea788d26e3..8337e49488d 100644 --- a/rapidjson/test/unittest/jsoncheckertest.cpp +++ b/rapidjson/test/unittest/jsoncheckertest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -18,82 +18,82 @@ using namespace rapidjson; -static char* ReadFile(const char* filename, size_t& length) { - const char *paths[] = { - "jsonchecker", - "bin/jsonchecker", - "../bin/jsonchecker", - "../../bin/jsonchecker", - "../../../bin/jsonchecker" - }; - char buffer[1024]; - FILE *fp = 0; - for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { - sprintf(buffer, "%s/%s", paths[i], filename); - fp = fopen(buffer, "rb"); - if (fp) - break; - } - - if (!fp) - return 0; - - fseek(fp, 0, SEEK_END); - length = static_cast(ftell(fp)); - fseek(fp, 0, SEEK_SET); - char* json = static_cast(malloc(length + 1)); - size_t readLength = fread(json, 1, length, fp); - json[readLength] = '\0'; - fclose(fp); - return json; +static char* ReadFile(const char* filename, size_t& length) +{ const char *paths[] = + { "jsonchecker", + "bin/jsonchecker", + "../bin/jsonchecker", + "../../bin/jsonchecker", + "../../../bin/jsonchecker" + }; + char buffer[1024]; + FILE *fp = 0; + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) + { sprintf(buffer, "%s/%s", paths[i], filename); + fp = fopen(buffer, "rb"); + if (fp) + break; + } + + if (!fp) + return 0; + + fseek(fp, 0, SEEK_END); + length = static_cast(ftell(fp)); + fseek(fp, 0, SEEK_SET); + char* json = static_cast(malloc(length + 1)); + size_t readLength = fread(json, 1, length, fp); + json[readLength] = '\0'; + fclose(fp); + return json; } -TEST(JsonChecker, Reader) { - char filename[256]; - - // jsonchecker/failXX.json - for (int i = 1; i <= 33; i++) { - if (i == 1) // fail1.json is valid in rapidjson, which has no limitation on type of root element (RFC 7159). - continue; - if (i == 18) // fail18.json is valid in rapidjson, which has no limitation on depth of nesting. - continue; - - sprintf(filename, "fail%d.json", i); - size_t length; - char* json = ReadFile(filename, length); - if (!json) { - printf("jsonchecker file %s not found", filename); - ADD_FAILURE(); - continue; - } - - GenericDocument, CrtAllocator> document; // Use Crt allocator to check exception-safety (no memory leak) - document.Parse(json); - EXPECT_TRUE(document.HasParseError()); - - document.Parse(json); - EXPECT_TRUE(document.HasParseError()); - - free(json); +TEST(JsonChecker, Reader) +{ char filename[256]; + + // jsonchecker/failXX.json + for (int i = 1; i <= 33; i++) + { if (i == 1) // fail1.json is valid in rapidjson, which has no limitation on type of root element (RFC 7159). + continue; + if (i == 18) // fail18.json is valid in rapidjson, which has no limitation on depth of nesting. + continue; + + sprintf(filename, "fail%d.json", i); + size_t length; + char* json = ReadFile(filename, length); + if (!json) + { printf("jsonchecker file %s not found", filename); + ADD_FAILURE(); + continue; } - // passX.json - for (int i = 1; i <= 3; i++) { - sprintf(filename, "pass%d.json", i); - size_t length; - char* json = ReadFile(filename, length); - if (!json) { - printf("jsonchecker file %s not found", filename); - continue; - } + GenericDocument, CrtAllocator> document; // Use Crt allocator to check exception-safety (no memory leak) + document.Parse(json); + EXPECT_TRUE(document.HasParseError()); - GenericDocument, CrtAllocator> document; // Use Crt allocator to check exception-safety (no memory leak) - document.Parse(json); - EXPECT_FALSE(document.HasParseError()); + document.Parse(json); + EXPECT_TRUE(document.HasParseError()); - document.Parse(json); - EXPECT_FALSE(document.HasParseError()); + free(json); + } - free(json); + // passX.json + for (int i = 1; i <= 3; i++) + { sprintf(filename, "pass%d.json", i); + size_t length; + char* json = ReadFile(filename, length); + if (!json) + { printf("jsonchecker file %s not found", filename); + continue; } + + GenericDocument, CrtAllocator> document; // Use Crt allocator to check exception-safety (no memory leak) + document.Parse(json); + EXPECT_FALSE(document.HasParseError()); + + document.Parse(json); + EXPECT_FALSE(document.HasParseError()); + + free(json); + } } diff --git a/rapidjson/test/unittest/namespacetest.cpp b/rapidjson/test/unittest/namespacetest.cpp index 1814724aec6..576db4799e4 100644 --- a/rapidjson/test/unittest/namespacetest.cpp +++ b/rapidjson/test/unittest/namespacetest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,14 +7,14 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" -// test another instantiation of RapidJSON in a different namespace +// test another instantiation of RapidJSON in a different namespace #define RAPIDJSON_NAMESPACE my::rapid::json #define RAPIDJSON_NAMESPACE_BEGIN namespace my { namespace rapid { namespace json { @@ -31,40 +31,40 @@ static const char json[] = "{\"hello\":\"world\",\"t\":true,\"f\":false,\"n\":null,\"i\":123,\"pi\":3.1416,\"a\":[1,2,3,4]}"; -TEST(NamespaceTest,Using) { - using namespace RAPIDJSON_NAMESPACE; - typedef GenericDocument, CrtAllocator> DocumentType; - DocumentType doc; +TEST(NamespaceTest,Using) +{ using namespace RAPIDJSON_NAMESPACE; + typedef GenericDocument, CrtAllocator> DocumentType; + DocumentType doc; - doc.Parse(json); - EXPECT_TRUE(!doc.HasParseError()); + doc.Parse(json); + EXPECT_TRUE(!doc.HasParseError()); } -TEST(NamespaceTest,Direct) { - typedef RAPIDJSON_NAMESPACE::Document Document; - typedef RAPIDJSON_NAMESPACE::Reader Reader; - typedef RAPIDJSON_NAMESPACE::StringStream StringStream; - typedef RAPIDJSON_NAMESPACE::StringBuffer StringBuffer; - typedef RAPIDJSON_NAMESPACE::Writer WriterType; +TEST(NamespaceTest,Direct) +{ typedef RAPIDJSON_NAMESPACE::Document Document; + typedef RAPIDJSON_NAMESPACE::Reader Reader; + typedef RAPIDJSON_NAMESPACE::StringStream StringStream; + typedef RAPIDJSON_NAMESPACE::StringBuffer StringBuffer; + typedef RAPIDJSON_NAMESPACE::Writer WriterType; - StringStream s(json); - StringBuffer buffer; - WriterType writer(buffer); - buffer.ShrinkToFit(); - Reader reader; - reader.Parse(s, writer); + StringStream s(json); + StringBuffer buffer; + WriterType writer(buffer); + buffer.ShrinkToFit(); + Reader reader; + reader.Parse(s, writer); - EXPECT_STREQ(json, buffer.GetString()); - EXPECT_EQ(sizeof(json)-1, buffer.GetSize()); - EXPECT_TRUE(writer.IsComplete()); + EXPECT_STREQ(json, buffer.GetString()); + EXPECT_EQ(sizeof(json)-1, buffer.GetSize()); + EXPECT_TRUE(writer.IsComplete()); - Document doc; - doc.Parse(buffer.GetString()); - EXPECT_TRUE(!doc.HasParseError()); + Document doc; + doc.Parse(buffer.GetString()); + EXPECT_TRUE(!doc.HasParseError()); - buffer.Clear(); - writer.Reset(buffer); - doc.Accept(writer); - EXPECT_STREQ(json, buffer.GetString()); - EXPECT_TRUE(writer.IsComplete()); + buffer.Clear(); + writer.Reset(buffer); + doc.Accept(writer); + EXPECT_STREQ(json, buffer.GetString()); + EXPECT_TRUE(writer.IsComplete()); } diff --git a/rapidjson/test/unittest/ostreamwrappertest.cpp b/rapidjson/test/unittest/ostreamwrappertest.cpp index b1d1cd827fc..90d51e3f50a 100644 --- a/rapidjson/test/unittest/ostreamwrappertest.cpp +++ b/rapidjson/test/unittest/ostreamwrappertest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -24,68 +24,67 @@ using namespace rapidjson; using namespace std; template -static void TestStringStream() { - typedef typename StringStreamType::char_type Ch; +static void TestStringStream() +{ typedef typename StringStreamType::char_type Ch; - Ch s[] = { 'A', 'B', 'C', '\0' }; - StringStreamType oss(s); - BasicOStreamWrapper os(oss); - for (size_t i = 0; i < 3; i++) - os.Put(s[i]); - os.Flush(); - for (size_t i = 0; i < 3; i++) - EXPECT_EQ(s[i], oss.str()[i]); + Ch s[] = { 'A', 'B', 'C', '\0' }; + StringStreamType oss(s); + BasicOStreamWrapper os(oss); + for (size_t i = 0; i < 3; i++) + os.Put(s[i]); + os.Flush(); + for (size_t i = 0; i < 3; i++) + EXPECT_EQ(s[i], oss.str()[i]); } -TEST(OStreamWrapper, ostringstream) { - TestStringStream(); +TEST(OStreamWrapper, ostringstream) +{ TestStringStream(); } -TEST(OStreamWrapper, stringstream) { - TestStringStream(); +TEST(OStreamWrapper, stringstream) +{ TestStringStream(); } -TEST(OStreamWrapper, wostringstream) { - TestStringStream(); +TEST(OStreamWrapper, wostringstream) +{ TestStringStream(); } -TEST(OStreamWrapper, wstringstream) { - TestStringStream(); +TEST(OStreamWrapper, wstringstream) +{ TestStringStream(); } -TEST(OStreamWrapper, cout) { - OStreamWrapper os(cout); - const char* s = "Hello World!\n"; - while (*s) - os.Put(*s++); - os.Flush(); +TEST(OStreamWrapper, cout) +{ OStreamWrapper os(cout); + const char* s = "Hello World!\n"; + while (*s) + os.Put(*s++); + os.Flush(); } template -static void TestFileStream() { - char filename[L_tmpnam]; - FILE* fp = TempFile(filename); - fclose(fp); - - const char* s = "Hello World!\n"; - { - ofstream ofs(filename, ios::out | ios::binary); - BasicOStreamWrapper osw(ofs); - for (const char* p = s; *p; p++) - osw.Put(*p); - osw.Flush(); - } +static void TestFileStream() +{ char filename[L_tmpnam]; + FILE* fp = TempFile(filename); + fclose(fp); - fp = fopen(filename, "r"); + const char* s = "Hello World!\n"; + { ofstream ofs(filename, ios::out | ios::binary); + BasicOStreamWrapper osw(ofs); for (const char* p = s; *p; p++) - EXPECT_EQ(*p, static_cast(fgetc(fp))); - fclose(fp); + osw.Put(*p); + osw.Flush(); + } + + fp = fopen(filename, "r"); + for (const char* p = s; *p; p++) + EXPECT_EQ(*p, static_cast(fgetc(fp))); + fclose(fp); } -TEST(OStreamWrapper, ofstream) { - TestFileStream(); +TEST(OStreamWrapper, ofstream) +{ TestFileStream(); } -TEST(OStreamWrapper, fstream) { - TestFileStream(); +TEST(OStreamWrapper, fstream) +{ TestFileStream(); } diff --git a/rapidjson/test/unittest/pointertest.cpp b/rapidjson/test/unittest/pointertest.cpp index dbddbedee25..55174243108 100644 --- a/rapidjson/test/unittest/pointertest.cpp +++ b/rapidjson/test/unittest/pointertest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -20,424 +20,387 @@ using namespace rapidjson; static const char kJson[] = "{\n" -" \"foo\":[\"bar\", \"baz\"],\n" -" \"\" : 0,\n" -" \"a/b\" : 1,\n" -" \"c%d\" : 2,\n" -" \"e^f\" : 3,\n" -" \"g|h\" : 4,\n" -" \"i\\\\j\" : 5,\n" -" \"k\\\"l\" : 6,\n" -" \" \" : 7,\n" -" \"m~n\" : 8\n" -"}"; - -TEST(Pointer, DefaultConstructor) { - Pointer p; + " \"foo\":[\"bar\", \"baz\"],\n" + " \"\" : 0,\n" + " \"a/b\" : 1,\n" + " \"c%d\" : 2,\n" + " \"e^f\" : 3,\n" + " \"g|h\" : 4,\n" + " \"i\\\\j\" : 5,\n" + " \"k\\\"l\" : 6,\n" + " \" \" : 7,\n" + " \"m~n\" : 8\n" + "}"; + +TEST(Pointer, DefaultConstructor) +{ Pointer p; + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(0u, p.GetTokenCount()); +} + +TEST(Pointer, Parse) +{ + { Pointer p(""); EXPECT_TRUE(p.IsValid()); EXPECT_EQ(0u, p.GetTokenCount()); -} + } + + { Pointer p("/"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(1u, p.GetTokenCount()); + EXPECT_EQ(0u, p.GetTokens()[0].length); + EXPECT_STREQ("", p.GetTokens()[0].name); + EXPECT_EQ(kPointerInvalidIndex, p.GetTokens()[0].index); + } + + { Pointer p("/foo"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(1u, p.GetTokenCount()); + EXPECT_EQ(3u, p.GetTokens()[0].length); + EXPECT_STREQ("foo", p.GetTokens()[0].name); + EXPECT_EQ(kPointerInvalidIndex, p.GetTokens()[0].index); + } + +#if RAPIDJSON_HAS_STDSTRING + { Pointer p(std::string("/foo")); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(1u, p.GetTokenCount()); + EXPECT_EQ(3u, p.GetTokens()[0].length); + EXPECT_STREQ("foo", p.GetTokens()[0].name); + EXPECT_EQ(kPointerInvalidIndex, p.GetTokens()[0].index); + } +#endif + + { Pointer p("/foo/0"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(2u, p.GetTokenCount()); + EXPECT_EQ(3u, p.GetTokens()[0].length); + EXPECT_STREQ("foo", p.GetTokens()[0].name); + EXPECT_EQ(kPointerInvalidIndex, p.GetTokens()[0].index); + EXPECT_EQ(1u, p.GetTokens()[1].length); + EXPECT_STREQ("0", p.GetTokens()[1].name); + EXPECT_EQ(0u, p.GetTokens()[1].index); + } + + { // Unescape ~1 + Pointer p("/a~1b"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(1u, p.GetTokenCount()); + EXPECT_EQ(3u, p.GetTokens()[0].length); + EXPECT_STREQ("a/b", p.GetTokens()[0].name); + } + + { // Unescape ~0 + Pointer p("/m~0n"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(1u, p.GetTokenCount()); + EXPECT_EQ(3u, p.GetTokens()[0].length); + EXPECT_STREQ("m~n", p.GetTokens()[0].name); + } -TEST(Pointer, Parse) { - { - Pointer p(""); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(0u, p.GetTokenCount()); - } - - { - Pointer p("/"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(1u, p.GetTokenCount()); - EXPECT_EQ(0u, p.GetTokens()[0].length); - EXPECT_STREQ("", p.GetTokens()[0].name); - EXPECT_EQ(kPointerInvalidIndex, p.GetTokens()[0].index); - } - - { - Pointer p("/foo"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(1u, p.GetTokenCount()); - EXPECT_EQ(3u, p.GetTokens()[0].length); - EXPECT_STREQ("foo", p.GetTokens()[0].name); - EXPECT_EQ(kPointerInvalidIndex, p.GetTokens()[0].index); - } - - #if RAPIDJSON_HAS_STDSTRING - { - Pointer p(std::string("/foo")); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(1u, p.GetTokenCount()); - EXPECT_EQ(3u, p.GetTokens()[0].length); - EXPECT_STREQ("foo", p.GetTokens()[0].name); - EXPECT_EQ(kPointerInvalidIndex, p.GetTokens()[0].index); - } - #endif - - { - Pointer p("/foo/0"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(2u, p.GetTokenCount()); - EXPECT_EQ(3u, p.GetTokens()[0].length); - EXPECT_STREQ("foo", p.GetTokens()[0].name); - EXPECT_EQ(kPointerInvalidIndex, p.GetTokens()[0].index); - EXPECT_EQ(1u, p.GetTokens()[1].length); - EXPECT_STREQ("0", p.GetTokens()[1].name); - EXPECT_EQ(0u, p.GetTokens()[1].index); - } - - { - // Unescape ~1 - Pointer p("/a~1b"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(1u, p.GetTokenCount()); - EXPECT_EQ(3u, p.GetTokens()[0].length); - EXPECT_STREQ("a/b", p.GetTokens()[0].name); - } - - { - // Unescape ~0 - Pointer p("/m~0n"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(1u, p.GetTokenCount()); - EXPECT_EQ(3u, p.GetTokens()[0].length); - EXPECT_STREQ("m~n", p.GetTokens()[0].name); - } - - { - // empty name - Pointer p("/"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(1u, p.GetTokenCount()); - EXPECT_EQ(0u, p.GetTokens()[0].length); - EXPECT_STREQ("", p.GetTokens()[0].name); - } - - { - // empty and non-empty name - Pointer p("//a"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(2u, p.GetTokenCount()); - EXPECT_EQ(0u, p.GetTokens()[0].length); - EXPECT_STREQ("", p.GetTokens()[0].name); - EXPECT_EQ(1u, p.GetTokens()[1].length); - EXPECT_STREQ("a", p.GetTokens()[1].name); - } - - { - // Null characters - Pointer p("/\0\0", 3); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(1u, p.GetTokenCount()); - EXPECT_EQ(2u, p.GetTokens()[0].length); - EXPECT_EQ('\0', p.GetTokens()[0].name[0]); - EXPECT_EQ('\0', p.GetTokens()[0].name[1]); - EXPECT_EQ('\0', p.GetTokens()[0].name[2]); - } - - { - // Valid index - Pointer p("/123"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(1u, p.GetTokenCount()); - EXPECT_STREQ("123", p.GetTokens()[0].name); - EXPECT_EQ(123u, p.GetTokens()[0].index); - } - - { - // Invalid index (with leading zero) - Pointer p("/01"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(1u, p.GetTokenCount()); - EXPECT_STREQ("01", p.GetTokens()[0].name); - EXPECT_EQ(kPointerInvalidIndex, p.GetTokens()[0].index); - } - - if (sizeof(SizeType) == 4) { - // Invalid index (overflow) - Pointer p("/4294967296"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(1u, p.GetTokenCount()); - EXPECT_STREQ("4294967296", p.GetTokens()[0].name); - EXPECT_EQ(kPointerInvalidIndex, p.GetTokens()[0].index); - } - - { - // kPointerParseErrorTokenMustBeginWithSolidus - Pointer p(" "); - EXPECT_FALSE(p.IsValid()); - EXPECT_EQ(kPointerParseErrorTokenMustBeginWithSolidus, p.GetParseErrorCode()); - EXPECT_EQ(0u, p.GetParseErrorOffset()); - } - - { - // kPointerParseErrorInvalidEscape - Pointer p("/~"); - EXPECT_FALSE(p.IsValid()); - EXPECT_EQ(kPointerParseErrorInvalidEscape, p.GetParseErrorCode()); - EXPECT_EQ(2u, p.GetParseErrorOffset()); - } - - { - // kPointerParseErrorInvalidEscape - Pointer p("/~2"); - EXPECT_FALSE(p.IsValid()); - EXPECT_EQ(kPointerParseErrorInvalidEscape, p.GetParseErrorCode()); - EXPECT_EQ(2u, p.GetParseErrorOffset()); - } + { // empty name + Pointer p("/"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(1u, p.GetTokenCount()); + EXPECT_EQ(0u, p.GetTokens()[0].length); + EXPECT_STREQ("", p.GetTokens()[0].name); + } + + { // empty and non-empty name + Pointer p("//a"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(2u, p.GetTokenCount()); + EXPECT_EQ(0u, p.GetTokens()[0].length); + EXPECT_STREQ("", p.GetTokens()[0].name); + EXPECT_EQ(1u, p.GetTokens()[1].length); + EXPECT_STREQ("a", p.GetTokens()[1].name); + } + + { // Null characters + Pointer p("/\0\0", 3); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(1u, p.GetTokenCount()); + EXPECT_EQ(2u, p.GetTokens()[0].length); + EXPECT_EQ('\0', p.GetTokens()[0].name[0]); + EXPECT_EQ('\0', p.GetTokens()[0].name[1]); + EXPECT_EQ('\0', p.GetTokens()[0].name[2]); + } + + { // Valid index + Pointer p("/123"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(1u, p.GetTokenCount()); + EXPECT_STREQ("123", p.GetTokens()[0].name); + EXPECT_EQ(123u, p.GetTokens()[0].index); + } + + { // Invalid index (with leading zero) + Pointer p("/01"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(1u, p.GetTokenCount()); + EXPECT_STREQ("01", p.GetTokens()[0].name); + EXPECT_EQ(kPointerInvalidIndex, p.GetTokens()[0].index); + } + + if (sizeof(SizeType) == 4) + { // Invalid index (overflow) + Pointer p("/4294967296"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(1u, p.GetTokenCount()); + EXPECT_STREQ("4294967296", p.GetTokens()[0].name); + EXPECT_EQ(kPointerInvalidIndex, p.GetTokens()[0].index); + } + + { // kPointerParseErrorTokenMustBeginWithSolidus + Pointer p(" "); + EXPECT_FALSE(p.IsValid()); + EXPECT_EQ(kPointerParseErrorTokenMustBeginWithSolidus, p.GetParseErrorCode()); + EXPECT_EQ(0u, p.GetParseErrorOffset()); + } + + { // kPointerParseErrorInvalidEscape + Pointer p("/~"); + EXPECT_FALSE(p.IsValid()); + EXPECT_EQ(kPointerParseErrorInvalidEscape, p.GetParseErrorCode()); + EXPECT_EQ(2u, p.GetParseErrorOffset()); + } + + { // kPointerParseErrorInvalidEscape + Pointer p("/~2"); + EXPECT_FALSE(p.IsValid()); + EXPECT_EQ(kPointerParseErrorInvalidEscape, p.GetParseErrorCode()); + EXPECT_EQ(2u, p.GetParseErrorOffset()); + } } -TEST(Pointer, Parse_URIFragment) { - { - Pointer p("#"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(0u, p.GetTokenCount()); - } - - { - Pointer p("#/foo"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(1u, p.GetTokenCount()); - EXPECT_EQ(3u, p.GetTokens()[0].length); - EXPECT_STREQ("foo", p.GetTokens()[0].name); - } - - { - Pointer p("#/foo/0"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(2u, p.GetTokenCount()); - EXPECT_EQ(3u, p.GetTokens()[0].length); - EXPECT_STREQ("foo", p.GetTokens()[0].name); - EXPECT_EQ(1u, p.GetTokens()[1].length); - EXPECT_STREQ("0", p.GetTokens()[1].name); - EXPECT_EQ(0u, p.GetTokens()[1].index); - } - - { - // Unescape ~1 - Pointer p("#/a~1b"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(1u, p.GetTokenCount()); - EXPECT_EQ(3u, p.GetTokens()[0].length); - EXPECT_STREQ("a/b", p.GetTokens()[0].name); - } - - { - // Unescape ~0 - Pointer p("#/m~0n"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(1u, p.GetTokenCount()); - EXPECT_EQ(3u, p.GetTokens()[0].length); - EXPECT_STREQ("m~n", p.GetTokens()[0].name); - } - - { - // empty name - Pointer p("#/"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(1u, p.GetTokenCount()); - EXPECT_EQ(0u, p.GetTokens()[0].length); - EXPECT_STREQ("", p.GetTokens()[0].name); - } - - { - // empty and non-empty name - Pointer p("#//a"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(2u, p.GetTokenCount()); - EXPECT_EQ(0u, p.GetTokens()[0].length); - EXPECT_STREQ("", p.GetTokens()[0].name); - EXPECT_EQ(1u, p.GetTokens()[1].length); - EXPECT_STREQ("a", p.GetTokens()[1].name); - } - - { - // Null characters - Pointer p("#/%00%00"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(1u, p.GetTokenCount()); - EXPECT_EQ(2u, p.GetTokens()[0].length); - EXPECT_EQ('\0', p.GetTokens()[0].name[0]); - EXPECT_EQ('\0', p.GetTokens()[0].name[1]); - EXPECT_EQ('\0', p.GetTokens()[0].name[2]); - } - - { - // Percentage Escapes - EXPECT_STREQ("c%d", Pointer("#/c%25d").GetTokens()[0].name); - EXPECT_STREQ("e^f", Pointer("#/e%5Ef").GetTokens()[0].name); - EXPECT_STREQ("g|h", Pointer("#/g%7Ch").GetTokens()[0].name); - EXPECT_STREQ("i\\j", Pointer("#/i%5Cj").GetTokens()[0].name); - EXPECT_STREQ("k\"l", Pointer("#/k%22l").GetTokens()[0].name); - EXPECT_STREQ(" ", Pointer("#/%20").GetTokens()[0].name); - } - - { - // Valid index - Pointer p("#/123"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(1u, p.GetTokenCount()); - EXPECT_STREQ("123", p.GetTokens()[0].name); - EXPECT_EQ(123u, p.GetTokens()[0].index); - } - - { - // Invalid index (with leading zero) - Pointer p("#/01"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(1u, p.GetTokenCount()); - EXPECT_STREQ("01", p.GetTokens()[0].name); - EXPECT_EQ(kPointerInvalidIndex, p.GetTokens()[0].index); - } - - if (sizeof(SizeType) == 4) { - // Invalid index (overflow) - Pointer p("#/4294967296"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(1u, p.GetTokenCount()); - EXPECT_STREQ("4294967296", p.GetTokens()[0].name); - EXPECT_EQ(kPointerInvalidIndex, p.GetTokens()[0].index); - } - - { - // Decode UTF-8 perecent encoding to UTF-8 - Pointer p("#/%C2%A2"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(1u, p.GetTokenCount()); - EXPECT_STREQ("\xC2\xA2", p.GetTokens()[0].name); - } - - { - // Decode UTF-8 perecent encoding to UTF-16 - GenericPointer > > p(L"#/%C2%A2"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(1u, p.GetTokenCount()); - EXPECT_EQ(static_cast::Ch>(0x00A2), p.GetTokens()[0].name[0]); - EXPECT_EQ(1u, p.GetTokens()[0].length); - } - - { - // Decode UTF-8 perecent encoding to UTF-16 - GenericPointer > > p(L"#/%E2%82%AC"); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(1u, p.GetTokenCount()); - EXPECT_EQ(static_cast::Ch>(0x20AC), p.GetTokens()[0].name[0]); - EXPECT_EQ(1u, p.GetTokens()[0].length); - } - - { - // kPointerParseErrorTokenMustBeginWithSolidus - Pointer p("# "); - EXPECT_FALSE(p.IsValid()); - EXPECT_EQ(kPointerParseErrorTokenMustBeginWithSolidus, p.GetParseErrorCode()); - EXPECT_EQ(1u, p.GetParseErrorOffset()); - } - - { - // kPointerParseErrorInvalidEscape - Pointer p("#/~"); - EXPECT_FALSE(p.IsValid()); - EXPECT_EQ(kPointerParseErrorInvalidEscape, p.GetParseErrorCode()); - EXPECT_EQ(3u, p.GetParseErrorOffset()); - } - - { - // kPointerParseErrorInvalidEscape - Pointer p("#/~2"); - EXPECT_FALSE(p.IsValid()); - EXPECT_EQ(kPointerParseErrorInvalidEscape, p.GetParseErrorCode()); - EXPECT_EQ(3u, p.GetParseErrorOffset()); - } - - { - // kPointerParseErrorInvalidPercentEncoding - Pointer p("#/%"); - EXPECT_FALSE(p.IsValid()); - EXPECT_EQ(kPointerParseErrorInvalidPercentEncoding, p.GetParseErrorCode()); - EXPECT_EQ(2u, p.GetParseErrorOffset()); - } - - { - // kPointerParseErrorInvalidPercentEncoding (invalid hex) - Pointer p("#/%g0"); - EXPECT_FALSE(p.IsValid()); - EXPECT_EQ(kPointerParseErrorInvalidPercentEncoding, p.GetParseErrorCode()); - EXPECT_EQ(2u, p.GetParseErrorOffset()); - } - - { - // kPointerParseErrorInvalidPercentEncoding (invalid hex) - Pointer p("#/%0g"); - EXPECT_FALSE(p.IsValid()); - EXPECT_EQ(kPointerParseErrorInvalidPercentEncoding, p.GetParseErrorCode()); - EXPECT_EQ(2u, p.GetParseErrorOffset()); - } - - { - // kPointerParseErrorInvalidPercentEncoding (incomplete UTF-8 sequence) - Pointer p("#/%C2"); - EXPECT_FALSE(p.IsValid()); - EXPECT_EQ(kPointerParseErrorInvalidPercentEncoding, p.GetParseErrorCode()); - EXPECT_EQ(2u, p.GetParseErrorOffset()); - } - - { - // kPointerParseErrorCharacterMustPercentEncode - Pointer p("#/ "); - EXPECT_FALSE(p.IsValid()); - EXPECT_EQ(kPointerParseErrorCharacterMustPercentEncode, p.GetParseErrorCode()); - EXPECT_EQ(2u, p.GetParseErrorOffset()); - } - - { - // kPointerParseErrorCharacterMustPercentEncode - Pointer p("#/\n"); - EXPECT_FALSE(p.IsValid()); - EXPECT_EQ(kPointerParseErrorCharacterMustPercentEncode, p.GetParseErrorCode()); - EXPECT_EQ(2u, p.GetParseErrorOffset()); - } +TEST(Pointer, Parse_URIFragment) +{ + { Pointer p("#"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(0u, p.GetTokenCount()); + } + + { Pointer p("#/foo"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(1u, p.GetTokenCount()); + EXPECT_EQ(3u, p.GetTokens()[0].length); + EXPECT_STREQ("foo", p.GetTokens()[0].name); + } + + { Pointer p("#/foo/0"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(2u, p.GetTokenCount()); + EXPECT_EQ(3u, p.GetTokens()[0].length); + EXPECT_STREQ("foo", p.GetTokens()[0].name); + EXPECT_EQ(1u, p.GetTokens()[1].length); + EXPECT_STREQ("0", p.GetTokens()[1].name); + EXPECT_EQ(0u, p.GetTokens()[1].index); + } + + { // Unescape ~1 + Pointer p("#/a~1b"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(1u, p.GetTokenCount()); + EXPECT_EQ(3u, p.GetTokens()[0].length); + EXPECT_STREQ("a/b", p.GetTokens()[0].name); + } + + { // Unescape ~0 + Pointer p("#/m~0n"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(1u, p.GetTokenCount()); + EXPECT_EQ(3u, p.GetTokens()[0].length); + EXPECT_STREQ("m~n", p.GetTokens()[0].name); + } + + { // empty name + Pointer p("#/"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(1u, p.GetTokenCount()); + EXPECT_EQ(0u, p.GetTokens()[0].length); + EXPECT_STREQ("", p.GetTokens()[0].name); + } + + { // empty and non-empty name + Pointer p("#//a"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(2u, p.GetTokenCount()); + EXPECT_EQ(0u, p.GetTokens()[0].length); + EXPECT_STREQ("", p.GetTokens()[0].name); + EXPECT_EQ(1u, p.GetTokens()[1].length); + EXPECT_STREQ("a", p.GetTokens()[1].name); + } + + { // Null characters + Pointer p("#/%00%00"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(1u, p.GetTokenCount()); + EXPECT_EQ(2u, p.GetTokens()[0].length); + EXPECT_EQ('\0', p.GetTokens()[0].name[0]); + EXPECT_EQ('\0', p.GetTokens()[0].name[1]); + EXPECT_EQ('\0', p.GetTokens()[0].name[2]); + } + + { // Percentage Escapes + EXPECT_STREQ("c%d", Pointer("#/c%25d").GetTokens()[0].name); + EXPECT_STREQ("e^f", Pointer("#/e%5Ef").GetTokens()[0].name); + EXPECT_STREQ("g|h", Pointer("#/g%7Ch").GetTokens()[0].name); + EXPECT_STREQ("i\\j", Pointer("#/i%5Cj").GetTokens()[0].name); + EXPECT_STREQ("k\"l", Pointer("#/k%22l").GetTokens()[0].name); + EXPECT_STREQ(" ", Pointer("#/%20").GetTokens()[0].name); + } + + { // Valid index + Pointer p("#/123"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(1u, p.GetTokenCount()); + EXPECT_STREQ("123", p.GetTokens()[0].name); + EXPECT_EQ(123u, p.GetTokens()[0].index); + } + + { // Invalid index (with leading zero) + Pointer p("#/01"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(1u, p.GetTokenCount()); + EXPECT_STREQ("01", p.GetTokens()[0].name); + EXPECT_EQ(kPointerInvalidIndex, p.GetTokens()[0].index); + } + + if (sizeof(SizeType) == 4) + { // Invalid index (overflow) + Pointer p("#/4294967296"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(1u, p.GetTokenCount()); + EXPECT_STREQ("4294967296", p.GetTokens()[0].name); + EXPECT_EQ(kPointerInvalidIndex, p.GetTokens()[0].index); + } + + { // Decode UTF-8 perecent encoding to UTF-8 + Pointer p("#/%C2%A2"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(1u, p.GetTokenCount()); + EXPECT_STREQ("\xC2\xA2", p.GetTokens()[0].name); + } + + { // Decode UTF-8 perecent encoding to UTF-16 + GenericPointer > > p(L"#/%C2%A2"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(1u, p.GetTokenCount()); + EXPECT_EQ(static_cast::Ch>(0x00A2), p.GetTokens()[0].name[0]); + EXPECT_EQ(1u, p.GetTokens()[0].length); + } + + { // Decode UTF-8 perecent encoding to UTF-16 + GenericPointer > > p(L"#/%E2%82%AC"); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(1u, p.GetTokenCount()); + EXPECT_EQ(static_cast::Ch>(0x20AC), p.GetTokens()[0].name[0]); + EXPECT_EQ(1u, p.GetTokens()[0].length); + } + + { // kPointerParseErrorTokenMustBeginWithSolidus + Pointer p("# "); + EXPECT_FALSE(p.IsValid()); + EXPECT_EQ(kPointerParseErrorTokenMustBeginWithSolidus, p.GetParseErrorCode()); + EXPECT_EQ(1u, p.GetParseErrorOffset()); + } + + { // kPointerParseErrorInvalidEscape + Pointer p("#/~"); + EXPECT_FALSE(p.IsValid()); + EXPECT_EQ(kPointerParseErrorInvalidEscape, p.GetParseErrorCode()); + EXPECT_EQ(3u, p.GetParseErrorOffset()); + } + + { // kPointerParseErrorInvalidEscape + Pointer p("#/~2"); + EXPECT_FALSE(p.IsValid()); + EXPECT_EQ(kPointerParseErrorInvalidEscape, p.GetParseErrorCode()); + EXPECT_EQ(3u, p.GetParseErrorOffset()); + } + + { // kPointerParseErrorInvalidPercentEncoding + Pointer p("#/%"); + EXPECT_FALSE(p.IsValid()); + EXPECT_EQ(kPointerParseErrorInvalidPercentEncoding, p.GetParseErrorCode()); + EXPECT_EQ(2u, p.GetParseErrorOffset()); + } + + { // kPointerParseErrorInvalidPercentEncoding (invalid hex) + Pointer p("#/%g0"); + EXPECT_FALSE(p.IsValid()); + EXPECT_EQ(kPointerParseErrorInvalidPercentEncoding, p.GetParseErrorCode()); + EXPECT_EQ(2u, p.GetParseErrorOffset()); + } + + { // kPointerParseErrorInvalidPercentEncoding (invalid hex) + Pointer p("#/%0g"); + EXPECT_FALSE(p.IsValid()); + EXPECT_EQ(kPointerParseErrorInvalidPercentEncoding, p.GetParseErrorCode()); + EXPECT_EQ(2u, p.GetParseErrorOffset()); + } + + { // kPointerParseErrorInvalidPercentEncoding (incomplete UTF-8 sequence) + Pointer p("#/%C2"); + EXPECT_FALSE(p.IsValid()); + EXPECT_EQ(kPointerParseErrorInvalidPercentEncoding, p.GetParseErrorCode()); + EXPECT_EQ(2u, p.GetParseErrorOffset()); + } + + { // kPointerParseErrorCharacterMustPercentEncode + Pointer p("#/ "); + EXPECT_FALSE(p.IsValid()); + EXPECT_EQ(kPointerParseErrorCharacterMustPercentEncode, p.GetParseErrorCode()); + EXPECT_EQ(2u, p.GetParseErrorOffset()); + } + + { // kPointerParseErrorCharacterMustPercentEncode + Pointer p("#/\n"); + EXPECT_FALSE(p.IsValid()); + EXPECT_EQ(kPointerParseErrorCharacterMustPercentEncode, p.GetParseErrorCode()); + EXPECT_EQ(2u, p.GetParseErrorOffset()); + } } -TEST(Pointer, Stringify) { - // Test by roundtrip - const char* sources[] = { - "", - "/foo", - "/foo/0", - "/", - "/a~1b", - "/c%d", - "/e^f", - "/g|h", - "/i\\j", - "/k\"l", - "/ ", - "/m~0n", - "/\xC2\xA2", - "/\xE2\x82\xAC", - "/\xF0\x9D\x84\x9E" - }; - - for (size_t i = 0; i < sizeof(sources) / sizeof(sources[0]); i++) { - Pointer p(sources[i]); - StringBuffer s; - EXPECT_TRUE(p.Stringify(s)); - EXPECT_STREQ(sources[i], s.GetString()); - - // Stringify to URI fragment - StringBuffer s2; - EXPECT_TRUE(p.StringifyUriFragment(s2)); - Pointer p2(s2.GetString(), s2.GetSize()); - EXPECT_TRUE(p2.IsValid()); - EXPECT_TRUE(p == p2); - } - - { - // Strigify to URI fragment with an invalid UTF-8 sequence - Pointer p("/\xC2"); - StringBuffer s; - EXPECT_FALSE(p.StringifyUriFragment(s)); - } +TEST(Pointer, Stringify) +{ // Test by roundtrip + const char* sources[] = + { "", + "/foo", + "/foo/0", + "/", + "/a~1b", + "/c%d", + "/e^f", + "/g|h", + "/i\\j", + "/k\"l", + "/ ", + "/m~0n", + "/\xC2\xA2", + "/\xE2\x82\xAC", + "/\xF0\x9D\x84\x9E" + }; + + for (size_t i = 0; i < sizeof(sources) / sizeof(sources[0]); i++) + { Pointer p(sources[i]); + StringBuffer s; + EXPECT_TRUE(p.Stringify(s)); + EXPECT_STREQ(sources[i], s.GetString()); + + // Stringify to URI fragment + StringBuffer s2; + EXPECT_TRUE(p.StringifyUriFragment(s2)); + Pointer p2(s2.GetString(), s2.GetSize()); + EXPECT_TRUE(p2.IsValid()); + EXPECT_TRUE(p == p2); + } + + { // Strigify to URI fragment with an invalid UTF-8 sequence + Pointer p("/\xC2"); + StringBuffer s; + EXPECT_FALSE(p.StringifyUriFragment(s)); + } } // Construct a Pointer with static tokens, no dynamic allocation involved. @@ -449,1076 +412,1049 @@ static const Pointer::Token kTokens[] = { NAME("foo"), INDEX(0) }; // equivalent #undef NAME #undef INDEX -TEST(Pointer, ConstructorWithToken) { - Pointer p(kTokens, sizeof(kTokens) / sizeof(kTokens[0])); - EXPECT_TRUE(p.IsValid()); - EXPECT_EQ(2u, p.GetTokenCount()); - EXPECT_EQ(3u, p.GetTokens()[0].length); - EXPECT_STREQ("foo", p.GetTokens()[0].name); - EXPECT_EQ(1u, p.GetTokens()[1].length); - EXPECT_STREQ("0", p.GetTokens()[1].name); - EXPECT_EQ(0u, p.GetTokens()[1].index); +TEST(Pointer, ConstructorWithToken) +{ Pointer p(kTokens, sizeof(kTokens) / sizeof(kTokens[0])); + EXPECT_TRUE(p.IsValid()); + EXPECT_EQ(2u, p.GetTokenCount()); + EXPECT_EQ(3u, p.GetTokens()[0].length); + EXPECT_STREQ("foo", p.GetTokens()[0].name); + EXPECT_EQ(1u, p.GetTokens()[1].length); + EXPECT_STREQ("0", p.GetTokens()[1].name); + EXPECT_EQ(0u, p.GetTokens()[1].index); } -TEST(Pointer, CopyConstructor) { - { - Pointer p("/foo/0"); - Pointer q(p); - EXPECT_TRUE(q.IsValid()); - EXPECT_EQ(2u, q.GetTokenCount()); - EXPECT_EQ(3u, q.GetTokens()[0].length); - EXPECT_STREQ("foo", q.GetTokens()[0].name); - EXPECT_EQ(1u, q.GetTokens()[1].length); - EXPECT_STREQ("0", q.GetTokens()[1].name); - EXPECT_EQ(0u, q.GetTokens()[1].index); - } - - // Static tokens - { - Pointer p(kTokens, sizeof(kTokens) / sizeof(kTokens[0])); - Pointer q(p); - EXPECT_TRUE(q.IsValid()); - EXPECT_EQ(2u, q.GetTokenCount()); - EXPECT_EQ(3u, q.GetTokens()[0].length); - EXPECT_STREQ("foo", q.GetTokens()[0].name); - EXPECT_EQ(1u, q.GetTokens()[1].length); - EXPECT_STREQ("0", q.GetTokens()[1].name); - EXPECT_EQ(0u, q.GetTokens()[1].index); - } +TEST(Pointer, CopyConstructor) +{ + { Pointer p("/foo/0"); + Pointer q(p); + EXPECT_TRUE(q.IsValid()); + EXPECT_EQ(2u, q.GetTokenCount()); + EXPECT_EQ(3u, q.GetTokens()[0].length); + EXPECT_STREQ("foo", q.GetTokens()[0].name); + EXPECT_EQ(1u, q.GetTokens()[1].length); + EXPECT_STREQ("0", q.GetTokens()[1].name); + EXPECT_EQ(0u, q.GetTokens()[1].index); + } + + // Static tokens + { Pointer p(kTokens, sizeof(kTokens) / sizeof(kTokens[0])); + Pointer q(p); + EXPECT_TRUE(q.IsValid()); + EXPECT_EQ(2u, q.GetTokenCount()); + EXPECT_EQ(3u, q.GetTokens()[0].length); + EXPECT_STREQ("foo", q.GetTokens()[0].name); + EXPECT_EQ(1u, q.GetTokens()[1].length); + EXPECT_STREQ("0", q.GetTokens()[1].name); + EXPECT_EQ(0u, q.GetTokens()[1].index); + } } -TEST(Pointer, Assignment) { - { - Pointer p("/foo/0"); - Pointer q; - q = p; - EXPECT_TRUE(q.IsValid()); - EXPECT_EQ(2u, q.GetTokenCount()); - EXPECT_EQ(3u, q.GetTokens()[0].length); - EXPECT_STREQ("foo", q.GetTokens()[0].name); - EXPECT_EQ(1u, q.GetTokens()[1].length); - EXPECT_STREQ("0", q.GetTokens()[1].name); - EXPECT_EQ(0u, q.GetTokens()[1].index); - q = q; - EXPECT_TRUE(q.IsValid()); - EXPECT_EQ(2u, q.GetTokenCount()); - EXPECT_EQ(3u, q.GetTokens()[0].length); - EXPECT_STREQ("foo", q.GetTokens()[0].name); - EXPECT_EQ(1u, q.GetTokens()[1].length); - EXPECT_STREQ("0", q.GetTokens()[1].name); - EXPECT_EQ(0u, q.GetTokens()[1].index); - } - - // Static tokens - { - Pointer p(kTokens, sizeof(kTokens) / sizeof(kTokens[0])); - Pointer q; - q = p; - EXPECT_TRUE(q.IsValid()); - EXPECT_EQ(2u, q.GetTokenCount()); - EXPECT_EQ(3u, q.GetTokens()[0].length); - EXPECT_STREQ("foo", q.GetTokens()[0].name); - EXPECT_EQ(1u, q.GetTokens()[1].length); - EXPECT_STREQ("0", q.GetTokens()[1].name); - EXPECT_EQ(0u, q.GetTokens()[1].index); - } +TEST(Pointer, Assignment) +{ + { Pointer p("/foo/0"); + Pointer q; + q = p; + EXPECT_TRUE(q.IsValid()); + EXPECT_EQ(2u, q.GetTokenCount()); + EXPECT_EQ(3u, q.GetTokens()[0].length); + EXPECT_STREQ("foo", q.GetTokens()[0].name); + EXPECT_EQ(1u, q.GetTokens()[1].length); + EXPECT_STREQ("0", q.GetTokens()[1].name); + EXPECT_EQ(0u, q.GetTokens()[1].index); + q = q; + EXPECT_TRUE(q.IsValid()); + EXPECT_EQ(2u, q.GetTokenCount()); + EXPECT_EQ(3u, q.GetTokens()[0].length); + EXPECT_STREQ("foo", q.GetTokens()[0].name); + EXPECT_EQ(1u, q.GetTokens()[1].length); + EXPECT_STREQ("0", q.GetTokens()[1].name); + EXPECT_EQ(0u, q.GetTokens()[1].index); + } + + // Static tokens + { Pointer p(kTokens, sizeof(kTokens) / sizeof(kTokens[0])); + Pointer q; + q = p; + EXPECT_TRUE(q.IsValid()); + EXPECT_EQ(2u, q.GetTokenCount()); + EXPECT_EQ(3u, q.GetTokens()[0].length); + EXPECT_STREQ("foo", q.GetTokens()[0].name); + EXPECT_EQ(1u, q.GetTokens()[1].length); + EXPECT_STREQ("0", q.GetTokens()[1].name); + EXPECT_EQ(0u, q.GetTokens()[1].index); + } } -TEST(Pointer, Append) { - { - Pointer p; - Pointer q = p.Append("foo"); - EXPECT_TRUE(Pointer("/foo") == q); - q = q.Append(1234); - EXPECT_TRUE(Pointer("/foo/1234") == q); - q = q.Append(""); - EXPECT_TRUE(Pointer("/foo/1234/") == q); - } - - { - Pointer p; - Pointer q = p.Append(Value("foo").Move()); - EXPECT_TRUE(Pointer("/foo") == q); - q = q.Append(Value(1234).Move()); - EXPECT_TRUE(Pointer("/foo/1234") == q); - q = q.Append(Value(kStringType).Move()); - EXPECT_TRUE(Pointer("/foo/1234/") == q); - } +TEST(Pointer, Append) +{ + { Pointer p; + Pointer q = p.Append("foo"); + EXPECT_TRUE(Pointer("/foo") == q); + q = q.Append(1234); + EXPECT_TRUE(Pointer("/foo/1234") == q); + q = q.Append(""); + EXPECT_TRUE(Pointer("/foo/1234/") == q); + } + + { Pointer p; + Pointer q = p.Append(Value("foo").Move()); + EXPECT_TRUE(Pointer("/foo") == q); + q = q.Append(Value(1234).Move()); + EXPECT_TRUE(Pointer("/foo/1234") == q); + q = q.Append(Value(kStringType).Move()); + EXPECT_TRUE(Pointer("/foo/1234/") == q); + } #if RAPIDJSON_HAS_STDSTRING - { - Pointer p; - Pointer q = p.Append(std::string("foo")); - EXPECT_TRUE(Pointer("/foo") == q); - } + { Pointer p; + Pointer q = p.Append(std::string("foo")); + EXPECT_TRUE(Pointer("/foo") == q); + } #endif } -TEST(Pointer, Equality) { - EXPECT_TRUE(Pointer("/foo/0") == Pointer("/foo/0")); - EXPECT_FALSE(Pointer("/foo/0") == Pointer("/foo/1")); - EXPECT_FALSE(Pointer("/foo/0") == Pointer("/foo/0/1")); - EXPECT_FALSE(Pointer("/foo/0") == Pointer("a")); - EXPECT_FALSE(Pointer("a") == Pointer("a")); // Invalid always not equal +TEST(Pointer, Equality) +{ EXPECT_TRUE(Pointer("/foo/0") == Pointer("/foo/0")); + EXPECT_FALSE(Pointer("/foo/0") == Pointer("/foo/1")); + EXPECT_FALSE(Pointer("/foo/0") == Pointer("/foo/0/1")); + EXPECT_FALSE(Pointer("/foo/0") == Pointer("a")); + EXPECT_FALSE(Pointer("a") == Pointer("a")); // Invalid always not equal } -TEST(Pointer, Inequality) { - EXPECT_FALSE(Pointer("/foo/0") != Pointer("/foo/0")); - EXPECT_TRUE(Pointer("/foo/0") != Pointer("/foo/1")); - EXPECT_TRUE(Pointer("/foo/0") != Pointer("/foo/0/1")); - EXPECT_TRUE(Pointer("/foo/0") != Pointer("a")); - EXPECT_TRUE(Pointer("a") != Pointer("a")); // Invalid always not equal +TEST(Pointer, Inequality) +{ EXPECT_FALSE(Pointer("/foo/0") != Pointer("/foo/0")); + EXPECT_TRUE(Pointer("/foo/0") != Pointer("/foo/1")); + EXPECT_TRUE(Pointer("/foo/0") != Pointer("/foo/0/1")); + EXPECT_TRUE(Pointer("/foo/0") != Pointer("a")); + EXPECT_TRUE(Pointer("a") != Pointer("a")); // Invalid always not equal } -TEST(Pointer, Create) { - Document d; - { - Value* v = &Pointer("").Create(d, d.GetAllocator()); - EXPECT_EQ(&d, v); - } - { - Value* v = &Pointer("/foo").Create(d, d.GetAllocator()); - EXPECT_EQ(&d["foo"], v); - } - { - Value* v = &Pointer("/foo/0").Create(d, d.GetAllocator()); - EXPECT_EQ(&d["foo"][0], v); - } - { - Value* v = &Pointer("/foo/-").Create(d, d.GetAllocator()); - EXPECT_EQ(&d["foo"][1], v); - } - - { - Value* v = &Pointer("/foo/-/-").Create(d, d.GetAllocator()); - // "foo/-" is a newly created null value x. - // "foo/-/-" finds that x is not an array, it converts x to empty object - // and treats - as "-" member name - EXPECT_EQ(&d["foo"][2]["-"], v); - } - - { - // Document with no allocator - Value* v = &Pointer("/foo/-").Create(d); - EXPECT_EQ(&d["foo"][3], v); - } - - { - // Value (not document) must give allocator - Value* v = &Pointer("/-").Create(d["foo"], d.GetAllocator()); - EXPECT_EQ(&d["foo"][4], v); - } +TEST(Pointer, Create) +{ Document d; + { Value* v = &Pointer("").Create(d, d.GetAllocator()); + EXPECT_EQ(&d, v); + } + { Value* v = &Pointer("/foo").Create(d, d.GetAllocator()); + EXPECT_EQ(&d["foo"], v); + } + { Value* v = &Pointer("/foo/0").Create(d, d.GetAllocator()); + EXPECT_EQ(&d["foo"][0], v); + } + { Value* v = &Pointer("/foo/-").Create(d, d.GetAllocator()); + EXPECT_EQ(&d["foo"][1], v); + } + + { Value* v = &Pointer("/foo/-/-").Create(d, d.GetAllocator()); + // "foo/-" is a newly created null value x. + // "foo/-/-" finds that x is not an array, it converts x to empty object + // and treats - as "-" member name + EXPECT_EQ(&d["foo"][2]["-"], v); + } + + { // Document with no allocator + Value* v = &Pointer("/foo/-").Create(d); + EXPECT_EQ(&d["foo"][3], v); + } + + { // Value (not document) must give allocator + Value* v = &Pointer("/-").Create(d["foo"], d.GetAllocator()); + EXPECT_EQ(&d["foo"][4], v); + } } -TEST(Pointer, Get) { - Document d; - d.Parse(kJson); - - EXPECT_EQ(&d, Pointer("").Get(d)); - EXPECT_EQ(&d["foo"], Pointer("/foo").Get(d)); - EXPECT_EQ(&d["foo"][0], Pointer("/foo/0").Get(d)); - EXPECT_EQ(&d[""], Pointer("/").Get(d)); - EXPECT_EQ(&d["a/b"], Pointer("/a~1b").Get(d)); - EXPECT_EQ(&d["c%d"], Pointer("/c%d").Get(d)); - EXPECT_EQ(&d["e^f"], Pointer("/e^f").Get(d)); - EXPECT_EQ(&d["g|h"], Pointer("/g|h").Get(d)); - EXPECT_EQ(&d["i\\j"], Pointer("/i\\j").Get(d)); - EXPECT_EQ(&d["k\"l"], Pointer("/k\"l").Get(d)); - EXPECT_EQ(&d[" "], Pointer("/ ").Get(d)); - EXPECT_EQ(&d["m~n"], Pointer("/m~0n").Get(d)); - EXPECT_TRUE(Pointer("/abc").Get(d) == 0); - size_t unresolvedTokenIndex; - EXPECT_TRUE(Pointer("/foo/2").Get(d, &unresolvedTokenIndex) == 0); // Out of boundary - EXPECT_EQ(1, unresolvedTokenIndex); - EXPECT_TRUE(Pointer("/foo/a").Get(d, &unresolvedTokenIndex) == 0); // "/foo" is an array, cannot query by "a" - EXPECT_EQ(1, unresolvedTokenIndex); - EXPECT_TRUE(Pointer("/foo/0/0").Get(d, &unresolvedTokenIndex) == 0); // "/foo/0" is an string, cannot further query - EXPECT_EQ(2, unresolvedTokenIndex); - EXPECT_TRUE(Pointer("/foo/0/a").Get(d, &unresolvedTokenIndex) == 0); // "/foo/0" is an string, cannot further query - EXPECT_EQ(2, unresolvedTokenIndex); +TEST(Pointer, Get) +{ Document d; + d.Parse(kJson); + + EXPECT_EQ(&d, Pointer("").Get(d)); + EXPECT_EQ(&d["foo"], Pointer("/foo").Get(d)); + EXPECT_EQ(&d["foo"][0], Pointer("/foo/0").Get(d)); + EXPECT_EQ(&d[""], Pointer("/").Get(d)); + EXPECT_EQ(&d["a/b"], Pointer("/a~1b").Get(d)); + EXPECT_EQ(&d["c%d"], Pointer("/c%d").Get(d)); + EXPECT_EQ(&d["e^f"], Pointer("/e^f").Get(d)); + EXPECT_EQ(&d["g|h"], Pointer("/g|h").Get(d)); + EXPECT_EQ(&d["i\\j"], Pointer("/i\\j").Get(d)); + EXPECT_EQ(&d["k\"l"], Pointer("/k\"l").Get(d)); + EXPECT_EQ(&d[" "], Pointer("/ ").Get(d)); + EXPECT_EQ(&d["m~n"], Pointer("/m~0n").Get(d)); + EXPECT_TRUE(Pointer("/abc").Get(d) == 0); + size_t unresolvedTokenIndex; + EXPECT_TRUE(Pointer("/foo/2").Get(d, &unresolvedTokenIndex) == 0); // Out of boundary + EXPECT_EQ(1, unresolvedTokenIndex); + EXPECT_TRUE(Pointer("/foo/a").Get(d, &unresolvedTokenIndex) == 0); // "/foo" is an array, cannot query by "a" + EXPECT_EQ(1, unresolvedTokenIndex); + EXPECT_TRUE(Pointer("/foo/0/0").Get(d, &unresolvedTokenIndex) == 0); // "/foo/0" is an string, cannot further query + EXPECT_EQ(2, unresolvedTokenIndex); + EXPECT_TRUE(Pointer("/foo/0/a").Get(d, &unresolvedTokenIndex) == 0); // "/foo/0" is an string, cannot further query + EXPECT_EQ(2, unresolvedTokenIndex); } -TEST(Pointer, GetWithDefault) { - Document d; - d.Parse(kJson); - - // Value version - Document::AllocatorType& a = d.GetAllocator(); - const Value v("qux"); - EXPECT_TRUE(Value("bar") == Pointer("/foo/0").GetWithDefault(d, v, a)); - EXPECT_TRUE(Value("baz") == Pointer("/foo/1").GetWithDefault(d, v, a)); - EXPECT_TRUE(Value("qux") == Pointer("/foo/2").GetWithDefault(d, v, a)); - EXPECT_TRUE(Value("last") == Pointer("/foo/-").GetWithDefault(d, Value("last").Move(), a)); - EXPECT_STREQ("last", d["foo"][3].GetString()); - - EXPECT_TRUE(Pointer("/foo/null").GetWithDefault(d, Value().Move(), a).IsNull()); - EXPECT_TRUE(Pointer("/foo/null").GetWithDefault(d, "x", a).IsNull()); - - // Generic version - EXPECT_EQ(-1, Pointer("/foo/int").GetWithDefault(d, -1, a).GetInt()); - EXPECT_EQ(-1, Pointer("/foo/int").GetWithDefault(d, -2, a).GetInt()); - EXPECT_EQ(0x87654321, Pointer("/foo/uint").GetWithDefault(d, 0x87654321, a).GetUint()); - EXPECT_EQ(0x87654321, Pointer("/foo/uint").GetWithDefault(d, 0x12345678, a).GetUint()); - - const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); - EXPECT_EQ(i64, Pointer("/foo/int64").GetWithDefault(d, i64, a).GetInt64()); - EXPECT_EQ(i64, Pointer("/foo/int64").GetWithDefault(d, i64 + 1, a).GetInt64()); - - const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); - EXPECT_EQ(u64, Pointer("/foo/uint64").GetWithDefault(d, u64, a).GetUint64()); - EXPECT_EQ(u64, Pointer("/foo/uint64").GetWithDefault(d, u64 - 1, a).GetUint64()); - - EXPECT_TRUE(Pointer("/foo/true").GetWithDefault(d, true, a).IsTrue()); - EXPECT_TRUE(Pointer("/foo/true").GetWithDefault(d, false, a).IsTrue()); - - EXPECT_TRUE(Pointer("/foo/false").GetWithDefault(d, false, a).IsFalse()); - EXPECT_TRUE(Pointer("/foo/false").GetWithDefault(d, true, a).IsFalse()); - - // StringRef version - EXPECT_STREQ("Hello", Pointer("/foo/hello").GetWithDefault(d, "Hello", a).GetString()); - - // Copy string version - { - char buffer[256]; - strcpy(buffer, "World"); - EXPECT_STREQ("World", Pointer("/foo/world").GetWithDefault(d, buffer, a).GetString()); - memset(buffer, 0, sizeof(buffer)); - } - EXPECT_STREQ("World", GetValueByPointer(d, "/foo/world")->GetString()); +TEST(Pointer, GetWithDefault) +{ Document d; + d.Parse(kJson); + + // Value version + Document::AllocatorType& a = d.GetAllocator(); + const Value v("qux"); + EXPECT_TRUE(Value("bar") == Pointer("/foo/0").GetWithDefault(d, v, a)); + EXPECT_TRUE(Value("baz") == Pointer("/foo/1").GetWithDefault(d, v, a)); + EXPECT_TRUE(Value("qux") == Pointer("/foo/2").GetWithDefault(d, v, a)); + EXPECT_TRUE(Value("last") == Pointer("/foo/-").GetWithDefault(d, Value("last").Move(), a)); + EXPECT_STREQ("last", d["foo"][3].GetString()); + + EXPECT_TRUE(Pointer("/foo/null").GetWithDefault(d, Value().Move(), a).IsNull()); + EXPECT_TRUE(Pointer("/foo/null").GetWithDefault(d, "x", a).IsNull()); + + // Generic version + EXPECT_EQ(-1, Pointer("/foo/int").GetWithDefault(d, -1, a).GetInt()); + EXPECT_EQ(-1, Pointer("/foo/int").GetWithDefault(d, -2, a).GetInt()); + EXPECT_EQ(0x87654321, Pointer("/foo/uint").GetWithDefault(d, 0x87654321, a).GetUint()); + EXPECT_EQ(0x87654321, Pointer("/foo/uint").GetWithDefault(d, 0x12345678, a).GetUint()); + + const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); + EXPECT_EQ(i64, Pointer("/foo/int64").GetWithDefault(d, i64, a).GetInt64()); + EXPECT_EQ(i64, Pointer("/foo/int64").GetWithDefault(d, i64 + 1, a).GetInt64()); + + const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); + EXPECT_EQ(u64, Pointer("/foo/uint64").GetWithDefault(d, u64, a).GetUint64()); + EXPECT_EQ(u64, Pointer("/foo/uint64").GetWithDefault(d, u64 - 1, a).GetUint64()); + + EXPECT_TRUE(Pointer("/foo/true").GetWithDefault(d, true, a).IsTrue()); + EXPECT_TRUE(Pointer("/foo/true").GetWithDefault(d, false, a).IsTrue()); + + EXPECT_TRUE(Pointer("/foo/false").GetWithDefault(d, false, a).IsFalse()); + EXPECT_TRUE(Pointer("/foo/false").GetWithDefault(d, true, a).IsFalse()); + + // StringRef version + EXPECT_STREQ("Hello", Pointer("/foo/hello").GetWithDefault(d, "Hello", a).GetString()); + + // Copy string version + { char buffer[256]; + strcpy(buffer, "World"); + EXPECT_STREQ("World", Pointer("/foo/world").GetWithDefault(d, buffer, a).GetString()); + memset(buffer, 0, sizeof(buffer)); + } + EXPECT_STREQ("World", GetValueByPointer(d, "/foo/world")->GetString()); #if RAPIDJSON_HAS_STDSTRING - EXPECT_STREQ("C++", Pointer("/foo/C++").GetWithDefault(d, std::string("C++"), a).GetString()); + EXPECT_STREQ("C++", Pointer("/foo/C++").GetWithDefault(d, std::string("C++"), a).GetString()); #endif } -TEST(Pointer, GetWithDefault_NoAllocator) { - Document d; - d.Parse(kJson); - - // Value version - const Value v("qux"); - EXPECT_TRUE(Value("bar") == Pointer("/foo/0").GetWithDefault(d, v)); - EXPECT_TRUE(Value("baz") == Pointer("/foo/1").GetWithDefault(d, v)); - EXPECT_TRUE(Value("qux") == Pointer("/foo/2").GetWithDefault(d, v)); - EXPECT_TRUE(Value("last") == Pointer("/foo/-").GetWithDefault(d, Value("last").Move())); - EXPECT_STREQ("last", d["foo"][3].GetString()); - - EXPECT_TRUE(Pointer("/foo/null").GetWithDefault(d, Value().Move()).IsNull()); - EXPECT_TRUE(Pointer("/foo/null").GetWithDefault(d, "x").IsNull()); - - // Generic version - EXPECT_EQ(-1, Pointer("/foo/int").GetWithDefault(d, -1).GetInt()); - EXPECT_EQ(-1, Pointer("/foo/int").GetWithDefault(d, -2).GetInt()); - EXPECT_EQ(0x87654321, Pointer("/foo/uint").GetWithDefault(d, 0x87654321).GetUint()); - EXPECT_EQ(0x87654321, Pointer("/foo/uint").GetWithDefault(d, 0x12345678).GetUint()); - - const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); - EXPECT_EQ(i64, Pointer("/foo/int64").GetWithDefault(d, i64).GetInt64()); - EXPECT_EQ(i64, Pointer("/foo/int64").GetWithDefault(d, i64 + 1).GetInt64()); - - const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); - EXPECT_EQ(u64, Pointer("/foo/uint64").GetWithDefault(d, u64).GetUint64()); - EXPECT_EQ(u64, Pointer("/foo/uint64").GetWithDefault(d, u64 - 1).GetUint64()); - - EXPECT_TRUE(Pointer("/foo/true").GetWithDefault(d, true).IsTrue()); - EXPECT_TRUE(Pointer("/foo/true").GetWithDefault(d, false).IsTrue()); - - EXPECT_TRUE(Pointer("/foo/false").GetWithDefault(d, false).IsFalse()); - EXPECT_TRUE(Pointer("/foo/false").GetWithDefault(d, true).IsFalse()); - - // StringRef version - EXPECT_STREQ("Hello", Pointer("/foo/hello").GetWithDefault(d, "Hello").GetString()); - - // Copy string version - { - char buffer[256]; - strcpy(buffer, "World"); - EXPECT_STREQ("World", Pointer("/foo/world").GetWithDefault(d, buffer).GetString()); - memset(buffer, 0, sizeof(buffer)); - } - EXPECT_STREQ("World", GetValueByPointer(d, "/foo/world")->GetString()); +TEST(Pointer, GetWithDefault_NoAllocator) +{ Document d; + d.Parse(kJson); + + // Value version + const Value v("qux"); + EXPECT_TRUE(Value("bar") == Pointer("/foo/0").GetWithDefault(d, v)); + EXPECT_TRUE(Value("baz") == Pointer("/foo/1").GetWithDefault(d, v)); + EXPECT_TRUE(Value("qux") == Pointer("/foo/2").GetWithDefault(d, v)); + EXPECT_TRUE(Value("last") == Pointer("/foo/-").GetWithDefault(d, Value("last").Move())); + EXPECT_STREQ("last", d["foo"][3].GetString()); + + EXPECT_TRUE(Pointer("/foo/null").GetWithDefault(d, Value().Move()).IsNull()); + EXPECT_TRUE(Pointer("/foo/null").GetWithDefault(d, "x").IsNull()); + + // Generic version + EXPECT_EQ(-1, Pointer("/foo/int").GetWithDefault(d, -1).GetInt()); + EXPECT_EQ(-1, Pointer("/foo/int").GetWithDefault(d, -2).GetInt()); + EXPECT_EQ(0x87654321, Pointer("/foo/uint").GetWithDefault(d, 0x87654321).GetUint()); + EXPECT_EQ(0x87654321, Pointer("/foo/uint").GetWithDefault(d, 0x12345678).GetUint()); + + const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); + EXPECT_EQ(i64, Pointer("/foo/int64").GetWithDefault(d, i64).GetInt64()); + EXPECT_EQ(i64, Pointer("/foo/int64").GetWithDefault(d, i64 + 1).GetInt64()); + + const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); + EXPECT_EQ(u64, Pointer("/foo/uint64").GetWithDefault(d, u64).GetUint64()); + EXPECT_EQ(u64, Pointer("/foo/uint64").GetWithDefault(d, u64 - 1).GetUint64()); + + EXPECT_TRUE(Pointer("/foo/true").GetWithDefault(d, true).IsTrue()); + EXPECT_TRUE(Pointer("/foo/true").GetWithDefault(d, false).IsTrue()); + + EXPECT_TRUE(Pointer("/foo/false").GetWithDefault(d, false).IsFalse()); + EXPECT_TRUE(Pointer("/foo/false").GetWithDefault(d, true).IsFalse()); + + // StringRef version + EXPECT_STREQ("Hello", Pointer("/foo/hello").GetWithDefault(d, "Hello").GetString()); + + // Copy string version + { char buffer[256]; + strcpy(buffer, "World"); + EXPECT_STREQ("World", Pointer("/foo/world").GetWithDefault(d, buffer).GetString()); + memset(buffer, 0, sizeof(buffer)); + } + EXPECT_STREQ("World", GetValueByPointer(d, "/foo/world")->GetString()); #if RAPIDJSON_HAS_STDSTRING - EXPECT_STREQ("C++", Pointer("/foo/C++").GetWithDefault(d, std::string("C++")).GetString()); + EXPECT_STREQ("C++", Pointer("/foo/C++").GetWithDefault(d, std::string("C++")).GetString()); #endif } -TEST(Pointer, Set) { - Document d; - d.Parse(kJson); - Document::AllocatorType& a = d.GetAllocator(); - - // Value version - Pointer("/foo/0").Set(d, Value(123).Move(), a); - EXPECT_EQ(123, d["foo"][0].GetInt()); +TEST(Pointer, Set) +{ Document d; + d.Parse(kJson); + Document::AllocatorType& a = d.GetAllocator(); + + // Value version + Pointer("/foo/0").Set(d, Value(123).Move(), a); + EXPECT_EQ(123, d["foo"][0].GetInt()); - Pointer("/foo/-").Set(d, Value(456).Move(), a); - EXPECT_EQ(456, d["foo"][2].GetInt()); + Pointer("/foo/-").Set(d, Value(456).Move(), a); + EXPECT_EQ(456, d["foo"][2].GetInt()); - Pointer("/foo/null").Set(d, Value().Move(), a); - EXPECT_TRUE(GetValueByPointer(d, "/foo/null")->IsNull()); + Pointer("/foo/null").Set(d, Value().Move(), a); + EXPECT_TRUE(GetValueByPointer(d, "/foo/null")->IsNull()); - // Const Value version - const Value foo(d["foo"], a); - Pointer("/clone").Set(d, foo, a); - EXPECT_EQ(foo, *GetValueByPointer(d, "/clone")); + // Const Value version + const Value foo(d["foo"], a); + Pointer("/clone").Set(d, foo, a); + EXPECT_EQ(foo, *GetValueByPointer(d, "/clone")); - // Generic version - Pointer("/foo/int").Set(d, -1, a); - EXPECT_EQ(-1, GetValueByPointer(d, "/foo/int")->GetInt()); + // Generic version + Pointer("/foo/int").Set(d, -1, a); + EXPECT_EQ(-1, GetValueByPointer(d, "/foo/int")->GetInt()); - Pointer("/foo/uint").Set(d, 0x87654321, a); - EXPECT_EQ(0x87654321, GetValueByPointer(d, "/foo/uint")->GetUint()); + Pointer("/foo/uint").Set(d, 0x87654321, a); + EXPECT_EQ(0x87654321, GetValueByPointer(d, "/foo/uint")->GetUint()); - const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); - Pointer("/foo/int64").Set(d, i64, a); - EXPECT_EQ(i64, GetValueByPointer(d, "/foo/int64")->GetInt64()); + const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); + Pointer("/foo/int64").Set(d, i64, a); + EXPECT_EQ(i64, GetValueByPointer(d, "/foo/int64")->GetInt64()); - const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); - Pointer("/foo/uint64").Set(d, u64, a); - EXPECT_EQ(u64, GetValueByPointer(d, "/foo/uint64")->GetUint64()); + const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); + Pointer("/foo/uint64").Set(d, u64, a); + EXPECT_EQ(u64, GetValueByPointer(d, "/foo/uint64")->GetUint64()); - Pointer("/foo/true").Set(d, true, a); - EXPECT_TRUE(GetValueByPointer(d, "/foo/true")->IsTrue()); + Pointer("/foo/true").Set(d, true, a); + EXPECT_TRUE(GetValueByPointer(d, "/foo/true")->IsTrue()); - Pointer("/foo/false").Set(d, false, a); - EXPECT_TRUE(GetValueByPointer(d, "/foo/false")->IsFalse()); + Pointer("/foo/false").Set(d, false, a); + EXPECT_TRUE(GetValueByPointer(d, "/foo/false")->IsFalse()); - // StringRef version - Pointer("/foo/hello").Set(d, "Hello", a); - EXPECT_STREQ("Hello", GetValueByPointer(d, "/foo/hello")->GetString()); + // StringRef version + Pointer("/foo/hello").Set(d, "Hello", a); + EXPECT_STREQ("Hello", GetValueByPointer(d, "/foo/hello")->GetString()); - // Copy string version - { - char buffer[256]; - strcpy(buffer, "World"); - Pointer("/foo/world").Set(d, buffer, a); - memset(buffer, 0, sizeof(buffer)); - } - EXPECT_STREQ("World", GetValueByPointer(d, "/foo/world")->GetString()); + // Copy string version + { char buffer[256]; + strcpy(buffer, "World"); + Pointer("/foo/world").Set(d, buffer, a); + memset(buffer, 0, sizeof(buffer)); + } + EXPECT_STREQ("World", GetValueByPointer(d, "/foo/world")->GetString()); #if RAPIDJSON_HAS_STDSTRING - Pointer("/foo/c++").Set(d, std::string("C++"), a); - EXPECT_STREQ("C++", GetValueByPointer(d, "/foo/c++")->GetString()); + Pointer("/foo/c++").Set(d, std::string("C++"), a); + EXPECT_STREQ("C++", GetValueByPointer(d, "/foo/c++")->GetString()); #endif } -TEST(Pointer, Set_NoAllocator) { - Document d; - d.Parse(kJson); - - // Value version - Pointer("/foo/0").Set(d, Value(123).Move()); - EXPECT_EQ(123, d["foo"][0].GetInt()); +TEST(Pointer, Set_NoAllocator) +{ Document d; + d.Parse(kJson); - Pointer("/foo/-").Set(d, Value(456).Move()); - EXPECT_EQ(456, d["foo"][2].GetInt()); + // Value version + Pointer("/foo/0").Set(d, Value(123).Move()); + EXPECT_EQ(123, d["foo"][0].GetInt()); - Pointer("/foo/null").Set(d, Value().Move()); - EXPECT_TRUE(GetValueByPointer(d, "/foo/null")->IsNull()); + Pointer("/foo/-").Set(d, Value(456).Move()); + EXPECT_EQ(456, d["foo"][2].GetInt()); - // Const Value version - const Value foo(d["foo"], d.GetAllocator()); - Pointer("/clone").Set(d, foo); - EXPECT_EQ(foo, *GetValueByPointer(d, "/clone")); + Pointer("/foo/null").Set(d, Value().Move()); + EXPECT_TRUE(GetValueByPointer(d, "/foo/null")->IsNull()); - // Generic version - Pointer("/foo/int").Set(d, -1); - EXPECT_EQ(-1, GetValueByPointer(d, "/foo/int")->GetInt()); + // Const Value version + const Value foo(d["foo"], d.GetAllocator()); + Pointer("/clone").Set(d, foo); + EXPECT_EQ(foo, *GetValueByPointer(d, "/clone")); - Pointer("/foo/uint").Set(d, 0x87654321); - EXPECT_EQ(0x87654321, GetValueByPointer(d, "/foo/uint")->GetUint()); + // Generic version + Pointer("/foo/int").Set(d, -1); + EXPECT_EQ(-1, GetValueByPointer(d, "/foo/int")->GetInt()); - const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); - Pointer("/foo/int64").Set(d, i64); - EXPECT_EQ(i64, GetValueByPointer(d, "/foo/int64")->GetInt64()); + Pointer("/foo/uint").Set(d, 0x87654321); + EXPECT_EQ(0x87654321, GetValueByPointer(d, "/foo/uint")->GetUint()); - const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); - Pointer("/foo/uint64").Set(d, u64); - EXPECT_EQ(u64, GetValueByPointer(d, "/foo/uint64")->GetUint64()); + const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); + Pointer("/foo/int64").Set(d, i64); + EXPECT_EQ(i64, GetValueByPointer(d, "/foo/int64")->GetInt64()); - Pointer("/foo/true").Set(d, true); - EXPECT_TRUE(GetValueByPointer(d, "/foo/true")->IsTrue()); + const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); + Pointer("/foo/uint64").Set(d, u64); + EXPECT_EQ(u64, GetValueByPointer(d, "/foo/uint64")->GetUint64()); - Pointer("/foo/false").Set(d, false); - EXPECT_TRUE(GetValueByPointer(d, "/foo/false")->IsFalse()); + Pointer("/foo/true").Set(d, true); + EXPECT_TRUE(GetValueByPointer(d, "/foo/true")->IsTrue()); - // StringRef version - Pointer("/foo/hello").Set(d, "Hello"); - EXPECT_STREQ("Hello", GetValueByPointer(d, "/foo/hello")->GetString()); + Pointer("/foo/false").Set(d, false); + EXPECT_TRUE(GetValueByPointer(d, "/foo/false")->IsFalse()); - // Copy string version - { - char buffer[256]; - strcpy(buffer, "World"); - Pointer("/foo/world").Set(d, buffer); - memset(buffer, 0, sizeof(buffer)); - } - EXPECT_STREQ("World", GetValueByPointer(d, "/foo/world")->GetString()); + // StringRef version + Pointer("/foo/hello").Set(d, "Hello"); + EXPECT_STREQ("Hello", GetValueByPointer(d, "/foo/hello")->GetString()); + + // Copy string version + { char buffer[256]; + strcpy(buffer, "World"); + Pointer("/foo/world").Set(d, buffer); + memset(buffer, 0, sizeof(buffer)); + } + EXPECT_STREQ("World", GetValueByPointer(d, "/foo/world")->GetString()); #if RAPIDJSON_HAS_STDSTRING - Pointer("/foo/c++").Set(d, std::string("C++")); - EXPECT_STREQ("C++", GetValueByPointer(d, "/foo/c++")->GetString()); + Pointer("/foo/c++").Set(d, std::string("C++")); + EXPECT_STREQ("C++", GetValueByPointer(d, "/foo/c++")->GetString()); #endif } -TEST(Pointer, Swap) { - Document d; - d.Parse(kJson); - Document::AllocatorType& a = d.GetAllocator(); - Pointer("/foo/0").Swap(d, *Pointer("/foo/1").Get(d), a); - EXPECT_STREQ("baz", d["foo"][0].GetString()); - EXPECT_STREQ("bar", d["foo"][1].GetString()); +TEST(Pointer, Swap) +{ Document d; + d.Parse(kJson); + Document::AllocatorType& a = d.GetAllocator(); + Pointer("/foo/0").Swap(d, *Pointer("/foo/1").Get(d), a); + EXPECT_STREQ("baz", d["foo"][0].GetString()); + EXPECT_STREQ("bar", d["foo"][1].GetString()); } -TEST(Pointer, Swap_NoAllocator) { - Document d; - d.Parse(kJson); - Pointer("/foo/0").Swap(d, *Pointer("/foo/1").Get(d)); - EXPECT_STREQ("baz", d["foo"][0].GetString()); - EXPECT_STREQ("bar", d["foo"][1].GetString()); +TEST(Pointer, Swap_NoAllocator) +{ Document d; + d.Parse(kJson); + Pointer("/foo/0").Swap(d, *Pointer("/foo/1").Get(d)); + EXPECT_STREQ("baz", d["foo"][0].GetString()); + EXPECT_STREQ("bar", d["foo"][1].GetString()); } -TEST(Pointer, Erase) { - Document d; - d.Parse(kJson); - - EXPECT_FALSE(Pointer("").Erase(d)); - EXPECT_FALSE(Pointer("/nonexist").Erase(d)); - EXPECT_FALSE(Pointer("/nonexist/nonexist").Erase(d)); - EXPECT_FALSE(Pointer("/foo/nonexist").Erase(d)); - EXPECT_FALSE(Pointer("/foo/nonexist/nonexist").Erase(d)); - EXPECT_FALSE(Pointer("/foo/0/nonexist").Erase(d)); - EXPECT_FALSE(Pointer("/foo/0/nonexist/nonexist").Erase(d)); - EXPECT_FALSE(Pointer("/foo/2/nonexist").Erase(d)); - EXPECT_TRUE(Pointer("/foo/0").Erase(d)); - EXPECT_EQ(1u, d["foo"].Size()); - EXPECT_STREQ("baz", d["foo"][0].GetString()); - EXPECT_TRUE(Pointer("/foo/0").Erase(d)); - EXPECT_TRUE(d["foo"].Empty()); - EXPECT_TRUE(Pointer("/foo").Erase(d)); - EXPECT_TRUE(Pointer("/foo").Get(d) == 0); - - Pointer("/a/0/b/0").Create(d); - - EXPECT_TRUE(Pointer("/a/0/b/0").Get(d) != 0); - EXPECT_TRUE(Pointer("/a/0/b/0").Erase(d)); - EXPECT_TRUE(Pointer("/a/0/b/0").Get(d) == 0); - - EXPECT_TRUE(Pointer("/a/0/b").Get(d) != 0); - EXPECT_TRUE(Pointer("/a/0/b").Erase(d)); - EXPECT_TRUE(Pointer("/a/0/b").Get(d) == 0); - - EXPECT_TRUE(Pointer("/a/0").Get(d) != 0); - EXPECT_TRUE(Pointer("/a/0").Erase(d)); - EXPECT_TRUE(Pointer("/a/0").Get(d) == 0); - - EXPECT_TRUE(Pointer("/a").Get(d) != 0); - EXPECT_TRUE(Pointer("/a").Erase(d)); - EXPECT_TRUE(Pointer("/a").Get(d) == 0); +TEST(Pointer, Erase) +{ Document d; + d.Parse(kJson); + + EXPECT_FALSE(Pointer("").Erase(d)); + EXPECT_FALSE(Pointer("/nonexist").Erase(d)); + EXPECT_FALSE(Pointer("/nonexist/nonexist").Erase(d)); + EXPECT_FALSE(Pointer("/foo/nonexist").Erase(d)); + EXPECT_FALSE(Pointer("/foo/nonexist/nonexist").Erase(d)); + EXPECT_FALSE(Pointer("/foo/0/nonexist").Erase(d)); + EXPECT_FALSE(Pointer("/foo/0/nonexist/nonexist").Erase(d)); + EXPECT_FALSE(Pointer("/foo/2/nonexist").Erase(d)); + EXPECT_TRUE(Pointer("/foo/0").Erase(d)); + EXPECT_EQ(1u, d["foo"].Size()); + EXPECT_STREQ("baz", d["foo"][0].GetString()); + EXPECT_TRUE(Pointer("/foo/0").Erase(d)); + EXPECT_TRUE(d["foo"].Empty()); + EXPECT_TRUE(Pointer("/foo").Erase(d)); + EXPECT_TRUE(Pointer("/foo").Get(d) == 0); + + Pointer("/a/0/b/0").Create(d); + + EXPECT_TRUE(Pointer("/a/0/b/0").Get(d) != 0); + EXPECT_TRUE(Pointer("/a/0/b/0").Erase(d)); + EXPECT_TRUE(Pointer("/a/0/b/0").Get(d) == 0); + + EXPECT_TRUE(Pointer("/a/0/b").Get(d) != 0); + EXPECT_TRUE(Pointer("/a/0/b").Erase(d)); + EXPECT_TRUE(Pointer("/a/0/b").Get(d) == 0); + + EXPECT_TRUE(Pointer("/a/0").Get(d) != 0); + EXPECT_TRUE(Pointer("/a/0").Erase(d)); + EXPECT_TRUE(Pointer("/a/0").Get(d) == 0); + + EXPECT_TRUE(Pointer("/a").Get(d) != 0); + EXPECT_TRUE(Pointer("/a").Erase(d)); + EXPECT_TRUE(Pointer("/a").Get(d) == 0); } -TEST(Pointer, CreateValueByPointer) { - Document d; - Document::AllocatorType& a = d.GetAllocator(); - - { - Value& v = CreateValueByPointer(d, Pointer("/foo/0"), a); - EXPECT_EQ(&d["foo"][0], &v); - } - { - Value& v = CreateValueByPointer(d, "/foo/1", a); - EXPECT_EQ(&d["foo"][1], &v); - } +TEST(Pointer, CreateValueByPointer) +{ Document d; + Document::AllocatorType& a = d.GetAllocator(); + + { Value& v = CreateValueByPointer(d, Pointer("/foo/0"), a); + EXPECT_EQ(&d["foo"][0], &v); + } + { Value& v = CreateValueByPointer(d, "/foo/1", a); + EXPECT_EQ(&d["foo"][1], &v); + } } -TEST(Pointer, CreateValueByPointer_NoAllocator) { - Document d; - - { - Value& v = CreateValueByPointer(d, Pointer("/foo/0")); - EXPECT_EQ(&d["foo"][0], &v); - } - { - Value& v = CreateValueByPointer(d, "/foo/1"); - EXPECT_EQ(&d["foo"][1], &v); - } +TEST(Pointer, CreateValueByPointer_NoAllocator) +{ Document d; + + { Value& v = CreateValueByPointer(d, Pointer("/foo/0")); + EXPECT_EQ(&d["foo"][0], &v); + } + { Value& v = CreateValueByPointer(d, "/foo/1"); + EXPECT_EQ(&d["foo"][1], &v); + } } -TEST(Pointer, GetValueByPointer) { - Document d; - d.Parse(kJson); - - EXPECT_EQ(&d["foo"][0], GetValueByPointer(d, Pointer("/foo/0"))); - EXPECT_EQ(&d["foo"][0], GetValueByPointer(d, "/foo/0")); - - size_t unresolvedTokenIndex; - EXPECT_TRUE(GetValueByPointer(d, "/foo/2", &unresolvedTokenIndex) == 0); // Out of boundary - EXPECT_EQ(1, unresolvedTokenIndex); - EXPECT_TRUE(GetValueByPointer(d, "/foo/a", &unresolvedTokenIndex) == 0); // "/foo" is an array, cannot query by "a" - EXPECT_EQ(1, unresolvedTokenIndex); - EXPECT_TRUE(GetValueByPointer(d, "/foo/0/0", &unresolvedTokenIndex) == 0); // "/foo/0" is an string, cannot further query - EXPECT_EQ(2, unresolvedTokenIndex); - EXPECT_TRUE(GetValueByPointer(d, "/foo/0/a", &unresolvedTokenIndex) == 0); // "/foo/0" is an string, cannot further query - EXPECT_EQ(2, unresolvedTokenIndex); - - // const version - const Value& v = d; - EXPECT_EQ(&d["foo"][0], GetValueByPointer(v, Pointer("/foo/0"))); - EXPECT_EQ(&d["foo"][0], GetValueByPointer(v, "/foo/0")); - - EXPECT_TRUE(GetValueByPointer(v, "/foo/2", &unresolvedTokenIndex) == 0); // Out of boundary - EXPECT_EQ(1, unresolvedTokenIndex); - EXPECT_TRUE(GetValueByPointer(v, "/foo/a", &unresolvedTokenIndex) == 0); // "/foo" is an array, cannot query by "a" - EXPECT_EQ(1, unresolvedTokenIndex); - EXPECT_TRUE(GetValueByPointer(v, "/foo/0/0", &unresolvedTokenIndex) == 0); // "/foo/0" is an string, cannot further query - EXPECT_EQ(2, unresolvedTokenIndex); - EXPECT_TRUE(GetValueByPointer(v, "/foo/0/a", &unresolvedTokenIndex) == 0); // "/foo/0" is an string, cannot further query - EXPECT_EQ(2, unresolvedTokenIndex); +TEST(Pointer, GetValueByPointer) +{ Document d; + d.Parse(kJson); + + EXPECT_EQ(&d["foo"][0], GetValueByPointer(d, Pointer("/foo/0"))); + EXPECT_EQ(&d["foo"][0], GetValueByPointer(d, "/foo/0")); + + size_t unresolvedTokenIndex; + EXPECT_TRUE(GetValueByPointer(d, "/foo/2", &unresolvedTokenIndex) == 0); // Out of boundary + EXPECT_EQ(1, unresolvedTokenIndex); + EXPECT_TRUE(GetValueByPointer(d, "/foo/a", &unresolvedTokenIndex) == 0); // "/foo" is an array, cannot query by "a" + EXPECT_EQ(1, unresolvedTokenIndex); + EXPECT_TRUE(GetValueByPointer(d, "/foo/0/0", &unresolvedTokenIndex) == 0); // "/foo/0" is an string, cannot further query + EXPECT_EQ(2, unresolvedTokenIndex); + EXPECT_TRUE(GetValueByPointer(d, "/foo/0/a", &unresolvedTokenIndex) == 0); // "/foo/0" is an string, cannot further query + EXPECT_EQ(2, unresolvedTokenIndex); + + // const version + const Value& v = d; + EXPECT_EQ(&d["foo"][0], GetValueByPointer(v, Pointer("/foo/0"))); + EXPECT_EQ(&d["foo"][0], GetValueByPointer(v, "/foo/0")); + + EXPECT_TRUE(GetValueByPointer(v, "/foo/2", &unresolvedTokenIndex) == 0); // Out of boundary + EXPECT_EQ(1, unresolvedTokenIndex); + EXPECT_TRUE(GetValueByPointer(v, "/foo/a", &unresolvedTokenIndex) == 0); // "/foo" is an array, cannot query by "a" + EXPECT_EQ(1, unresolvedTokenIndex); + EXPECT_TRUE(GetValueByPointer(v, "/foo/0/0", &unresolvedTokenIndex) == 0); // "/foo/0" is an string, cannot further query + EXPECT_EQ(2, unresolvedTokenIndex); + EXPECT_TRUE(GetValueByPointer(v, "/foo/0/a", &unresolvedTokenIndex) == 0); // "/foo/0" is an string, cannot further query + EXPECT_EQ(2, unresolvedTokenIndex); } -TEST(Pointer, GetValueByPointerWithDefault_Pointer) { - Document d; - d.Parse(kJson); - - Document::AllocatorType& a = d.GetAllocator(); - const Value v("qux"); - EXPECT_TRUE(Value("bar") == GetValueByPointerWithDefault(d, Pointer("/foo/0"), v, a)); - EXPECT_TRUE(Value("bar") == GetValueByPointerWithDefault(d, Pointer("/foo/0"), v, a)); - EXPECT_TRUE(Value("baz") == GetValueByPointerWithDefault(d, Pointer("/foo/1"), v, a)); - EXPECT_TRUE(Value("qux") == GetValueByPointerWithDefault(d, Pointer("/foo/2"), v, a)); - EXPECT_TRUE(Value("last") == GetValueByPointerWithDefault(d, Pointer("/foo/-"), Value("last").Move(), a)); - EXPECT_STREQ("last", d["foo"][3].GetString()); - - EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/null"), Value().Move(), a).IsNull()); - EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/null"), "x", a).IsNull()); - - // Generic version - EXPECT_EQ(-1, GetValueByPointerWithDefault(d, Pointer("/foo/int"), -1, a).GetInt()); - EXPECT_EQ(-1, GetValueByPointerWithDefault(d, Pointer("/foo/int"), -2, a).GetInt()); - EXPECT_EQ(0x87654321, GetValueByPointerWithDefault(d, Pointer("/foo/uint"), 0x87654321, a).GetUint()); - EXPECT_EQ(0x87654321, GetValueByPointerWithDefault(d, Pointer("/foo/uint"), 0x12345678, a).GetUint()); - - const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); - EXPECT_EQ(i64, GetValueByPointerWithDefault(d, Pointer("/foo/int64"), i64, a).GetInt64()); - EXPECT_EQ(i64, GetValueByPointerWithDefault(d, Pointer("/foo/int64"), i64 + 1, a).GetInt64()); - - const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); - EXPECT_EQ(u64, GetValueByPointerWithDefault(d, Pointer("/foo/uint64"), u64, a).GetUint64()); - EXPECT_EQ(u64, GetValueByPointerWithDefault(d, Pointer("/foo/uint64"), u64 - 1, a).GetUint64()); - - EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/true"), true, a).IsTrue()); - EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/true"), false, a).IsTrue()); - - EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/false"), false, a).IsFalse()); - EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/false"), true, a).IsFalse()); - - // StringRef version - EXPECT_STREQ("Hello", GetValueByPointerWithDefault(d, Pointer("/foo/hello"), "Hello", a).GetString()); - - // Copy string version - { - char buffer[256]; - strcpy(buffer, "World"); - EXPECT_STREQ("World", GetValueByPointerWithDefault(d, Pointer("/foo/world"), buffer, a).GetString()); - memset(buffer, 0, sizeof(buffer)); - } - EXPECT_STREQ("World", GetValueByPointer(d, Pointer("/foo/world"))->GetString()); +TEST(Pointer, GetValueByPointerWithDefault_Pointer) +{ Document d; + d.Parse(kJson); + + Document::AllocatorType& a = d.GetAllocator(); + const Value v("qux"); + EXPECT_TRUE(Value("bar") == GetValueByPointerWithDefault(d, Pointer("/foo/0"), v, a)); + EXPECT_TRUE(Value("bar") == GetValueByPointerWithDefault(d, Pointer("/foo/0"), v, a)); + EXPECT_TRUE(Value("baz") == GetValueByPointerWithDefault(d, Pointer("/foo/1"), v, a)); + EXPECT_TRUE(Value("qux") == GetValueByPointerWithDefault(d, Pointer("/foo/2"), v, a)); + EXPECT_TRUE(Value("last") == GetValueByPointerWithDefault(d, Pointer("/foo/-"), Value("last").Move(), a)); + EXPECT_STREQ("last", d["foo"][3].GetString()); + + EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/null"), Value().Move(), a).IsNull()); + EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/null"), "x", a).IsNull()); + + // Generic version + EXPECT_EQ(-1, GetValueByPointerWithDefault(d, Pointer("/foo/int"), -1, a).GetInt()); + EXPECT_EQ(-1, GetValueByPointerWithDefault(d, Pointer("/foo/int"), -2, a).GetInt()); + EXPECT_EQ(0x87654321, GetValueByPointerWithDefault(d, Pointer("/foo/uint"), 0x87654321, a).GetUint()); + EXPECT_EQ(0x87654321, GetValueByPointerWithDefault(d, Pointer("/foo/uint"), 0x12345678, a).GetUint()); + + const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); + EXPECT_EQ(i64, GetValueByPointerWithDefault(d, Pointer("/foo/int64"), i64, a).GetInt64()); + EXPECT_EQ(i64, GetValueByPointerWithDefault(d, Pointer("/foo/int64"), i64 + 1, a).GetInt64()); + + const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); + EXPECT_EQ(u64, GetValueByPointerWithDefault(d, Pointer("/foo/uint64"), u64, a).GetUint64()); + EXPECT_EQ(u64, GetValueByPointerWithDefault(d, Pointer("/foo/uint64"), u64 - 1, a).GetUint64()); + + EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/true"), true, a).IsTrue()); + EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/true"), false, a).IsTrue()); + + EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/false"), false, a).IsFalse()); + EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/false"), true, a).IsFalse()); + + // StringRef version + EXPECT_STREQ("Hello", GetValueByPointerWithDefault(d, Pointer("/foo/hello"), "Hello", a).GetString()); + + // Copy string version + { char buffer[256]; + strcpy(buffer, "World"); + EXPECT_STREQ("World", GetValueByPointerWithDefault(d, Pointer("/foo/world"), buffer, a).GetString()); + memset(buffer, 0, sizeof(buffer)); + } + EXPECT_STREQ("World", GetValueByPointer(d, Pointer("/foo/world"))->GetString()); #if RAPIDJSON_HAS_STDSTRING - EXPECT_STREQ("C++", GetValueByPointerWithDefault(d, Pointer("/foo/C++"), std::string("C++"), a).GetString()); + EXPECT_STREQ("C++", GetValueByPointerWithDefault(d, Pointer("/foo/C++"), std::string("C++"), a).GetString()); #endif } -TEST(Pointer, GetValueByPointerWithDefault_String) { - Document d; - d.Parse(kJson); - - Document::AllocatorType& a = d.GetAllocator(); - const Value v("qux"); - EXPECT_TRUE(Value("bar") == GetValueByPointerWithDefault(d, "/foo/0", v, a)); - EXPECT_TRUE(Value("bar") == GetValueByPointerWithDefault(d, "/foo/0", v, a)); - EXPECT_TRUE(Value("baz") == GetValueByPointerWithDefault(d, "/foo/1", v, a)); - EXPECT_TRUE(Value("qux") == GetValueByPointerWithDefault(d, "/foo/2", v, a)); - EXPECT_TRUE(Value("last") == GetValueByPointerWithDefault(d, "/foo/-", Value("last").Move(), a)); - EXPECT_STREQ("last", d["foo"][3].GetString()); - - EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/null", Value().Move(), a).IsNull()); - EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/null", "x", a).IsNull()); - - // Generic version - EXPECT_EQ(-1, GetValueByPointerWithDefault(d, "/foo/int", -1, a).GetInt()); - EXPECT_EQ(-1, GetValueByPointerWithDefault(d, "/foo/int", -2, a).GetInt()); - EXPECT_EQ(0x87654321, GetValueByPointerWithDefault(d, "/foo/uint", 0x87654321, a).GetUint()); - EXPECT_EQ(0x87654321, GetValueByPointerWithDefault(d, "/foo/uint", 0x12345678, a).GetUint()); - - const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); - EXPECT_EQ(i64, GetValueByPointerWithDefault(d, "/foo/int64", i64, a).GetInt64()); - EXPECT_EQ(i64, GetValueByPointerWithDefault(d, "/foo/int64", i64 + 1, a).GetInt64()); - - const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); - EXPECT_EQ(u64, GetValueByPointerWithDefault(d, "/foo/uint64", u64, a).GetUint64()); - EXPECT_EQ(u64, GetValueByPointerWithDefault(d, "/foo/uint64", u64 - 1, a).GetUint64()); - - EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/true", true, a).IsTrue()); - EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/true", false, a).IsTrue()); - - EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/false", false, a).IsFalse()); - EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/false", true, a).IsFalse()); - - // StringRef version - EXPECT_STREQ("Hello", GetValueByPointerWithDefault(d, "/foo/hello", "Hello", a).GetString()); - - // Copy string version - { - char buffer[256]; - strcpy(buffer, "World"); - EXPECT_STREQ("World", GetValueByPointerWithDefault(d, "/foo/world", buffer, a).GetString()); - memset(buffer, 0, sizeof(buffer)); - } - EXPECT_STREQ("World", GetValueByPointer(d, "/foo/world")->GetString()); +TEST(Pointer, GetValueByPointerWithDefault_String) +{ Document d; + d.Parse(kJson); + + Document::AllocatorType& a = d.GetAllocator(); + const Value v("qux"); + EXPECT_TRUE(Value("bar") == GetValueByPointerWithDefault(d, "/foo/0", v, a)); + EXPECT_TRUE(Value("bar") == GetValueByPointerWithDefault(d, "/foo/0", v, a)); + EXPECT_TRUE(Value("baz") == GetValueByPointerWithDefault(d, "/foo/1", v, a)); + EXPECT_TRUE(Value("qux") == GetValueByPointerWithDefault(d, "/foo/2", v, a)); + EXPECT_TRUE(Value("last") == GetValueByPointerWithDefault(d, "/foo/-", Value("last").Move(), a)); + EXPECT_STREQ("last", d["foo"][3].GetString()); + + EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/null", Value().Move(), a).IsNull()); + EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/null", "x", a).IsNull()); + + // Generic version + EXPECT_EQ(-1, GetValueByPointerWithDefault(d, "/foo/int", -1, a).GetInt()); + EXPECT_EQ(-1, GetValueByPointerWithDefault(d, "/foo/int", -2, a).GetInt()); + EXPECT_EQ(0x87654321, GetValueByPointerWithDefault(d, "/foo/uint", 0x87654321, a).GetUint()); + EXPECT_EQ(0x87654321, GetValueByPointerWithDefault(d, "/foo/uint", 0x12345678, a).GetUint()); + + const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); + EXPECT_EQ(i64, GetValueByPointerWithDefault(d, "/foo/int64", i64, a).GetInt64()); + EXPECT_EQ(i64, GetValueByPointerWithDefault(d, "/foo/int64", i64 + 1, a).GetInt64()); + + const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); + EXPECT_EQ(u64, GetValueByPointerWithDefault(d, "/foo/uint64", u64, a).GetUint64()); + EXPECT_EQ(u64, GetValueByPointerWithDefault(d, "/foo/uint64", u64 - 1, a).GetUint64()); + + EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/true", true, a).IsTrue()); + EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/true", false, a).IsTrue()); + + EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/false", false, a).IsFalse()); + EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/false", true, a).IsFalse()); + + // StringRef version + EXPECT_STREQ("Hello", GetValueByPointerWithDefault(d, "/foo/hello", "Hello", a).GetString()); + + // Copy string version + { char buffer[256]; + strcpy(buffer, "World"); + EXPECT_STREQ("World", GetValueByPointerWithDefault(d, "/foo/world", buffer, a).GetString()); + memset(buffer, 0, sizeof(buffer)); + } + EXPECT_STREQ("World", GetValueByPointer(d, "/foo/world")->GetString()); #if RAPIDJSON_HAS_STDSTRING - EXPECT_STREQ("C++", GetValueByPointerWithDefault(d, "/foo/C++", std::string("C++"), a).GetString()); + EXPECT_STREQ("C++", GetValueByPointerWithDefault(d, "/foo/C++", std::string("C++"), a).GetString()); #endif } -TEST(Pointer, GetValueByPointerWithDefault_Pointer_NoAllocator) { - Document d; - d.Parse(kJson); - - const Value v("qux"); - EXPECT_TRUE(Value("bar") == GetValueByPointerWithDefault(d, Pointer("/foo/0"), v)); - EXPECT_TRUE(Value("bar") == GetValueByPointerWithDefault(d, Pointer("/foo/0"), v)); - EXPECT_TRUE(Value("baz") == GetValueByPointerWithDefault(d, Pointer("/foo/1"), v)); - EXPECT_TRUE(Value("qux") == GetValueByPointerWithDefault(d, Pointer("/foo/2"), v)); - EXPECT_TRUE(Value("last") == GetValueByPointerWithDefault(d, Pointer("/foo/-"), Value("last").Move())); - EXPECT_STREQ("last", d["foo"][3].GetString()); - - EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/null"), Value().Move()).IsNull()); - EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/null"), "x").IsNull()); - - // Generic version - EXPECT_EQ(-1, GetValueByPointerWithDefault(d, Pointer("/foo/int"), -1).GetInt()); - EXPECT_EQ(-1, GetValueByPointerWithDefault(d, Pointer("/foo/int"), -2).GetInt()); - EXPECT_EQ(0x87654321, GetValueByPointerWithDefault(d, Pointer("/foo/uint"), 0x87654321).GetUint()); - EXPECT_EQ(0x87654321, GetValueByPointerWithDefault(d, Pointer("/foo/uint"), 0x12345678).GetUint()); - - const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); - EXPECT_EQ(i64, GetValueByPointerWithDefault(d, Pointer("/foo/int64"), i64).GetInt64()); - EXPECT_EQ(i64, GetValueByPointerWithDefault(d, Pointer("/foo/int64"), i64 + 1).GetInt64()); - - const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); - EXPECT_EQ(u64, GetValueByPointerWithDefault(d, Pointer("/foo/uint64"), u64).GetUint64()); - EXPECT_EQ(u64, GetValueByPointerWithDefault(d, Pointer("/foo/uint64"), u64 - 1).GetUint64()); - - EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/true"), true).IsTrue()); - EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/true"), false).IsTrue()); - - EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/false"), false).IsFalse()); - EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/false"), true).IsFalse()); - - // StringRef version - EXPECT_STREQ("Hello", GetValueByPointerWithDefault(d, Pointer("/foo/hello"), "Hello").GetString()); - - // Copy string version - { - char buffer[256]; - strcpy(buffer, "World"); - EXPECT_STREQ("World", GetValueByPointerWithDefault(d, Pointer("/foo/world"), buffer).GetString()); - memset(buffer, 0, sizeof(buffer)); - } - EXPECT_STREQ("World", GetValueByPointer(d, Pointer("/foo/world"))->GetString()); +TEST(Pointer, GetValueByPointerWithDefault_Pointer_NoAllocator) +{ Document d; + d.Parse(kJson); + + const Value v("qux"); + EXPECT_TRUE(Value("bar") == GetValueByPointerWithDefault(d, Pointer("/foo/0"), v)); + EXPECT_TRUE(Value("bar") == GetValueByPointerWithDefault(d, Pointer("/foo/0"), v)); + EXPECT_TRUE(Value("baz") == GetValueByPointerWithDefault(d, Pointer("/foo/1"), v)); + EXPECT_TRUE(Value("qux") == GetValueByPointerWithDefault(d, Pointer("/foo/2"), v)); + EXPECT_TRUE(Value("last") == GetValueByPointerWithDefault(d, Pointer("/foo/-"), Value("last").Move())); + EXPECT_STREQ("last", d["foo"][3].GetString()); + + EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/null"), Value().Move()).IsNull()); + EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/null"), "x").IsNull()); + + // Generic version + EXPECT_EQ(-1, GetValueByPointerWithDefault(d, Pointer("/foo/int"), -1).GetInt()); + EXPECT_EQ(-1, GetValueByPointerWithDefault(d, Pointer("/foo/int"), -2).GetInt()); + EXPECT_EQ(0x87654321, GetValueByPointerWithDefault(d, Pointer("/foo/uint"), 0x87654321).GetUint()); + EXPECT_EQ(0x87654321, GetValueByPointerWithDefault(d, Pointer("/foo/uint"), 0x12345678).GetUint()); + + const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); + EXPECT_EQ(i64, GetValueByPointerWithDefault(d, Pointer("/foo/int64"), i64).GetInt64()); + EXPECT_EQ(i64, GetValueByPointerWithDefault(d, Pointer("/foo/int64"), i64 + 1).GetInt64()); + + const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); + EXPECT_EQ(u64, GetValueByPointerWithDefault(d, Pointer("/foo/uint64"), u64).GetUint64()); + EXPECT_EQ(u64, GetValueByPointerWithDefault(d, Pointer("/foo/uint64"), u64 - 1).GetUint64()); + + EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/true"), true).IsTrue()); + EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/true"), false).IsTrue()); + + EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/false"), false).IsFalse()); + EXPECT_TRUE(GetValueByPointerWithDefault(d, Pointer("/foo/false"), true).IsFalse()); + + // StringRef version + EXPECT_STREQ("Hello", GetValueByPointerWithDefault(d, Pointer("/foo/hello"), "Hello").GetString()); + + // Copy string version + { char buffer[256]; + strcpy(buffer, "World"); + EXPECT_STREQ("World", GetValueByPointerWithDefault(d, Pointer("/foo/world"), buffer).GetString()); + memset(buffer, 0, sizeof(buffer)); + } + EXPECT_STREQ("World", GetValueByPointer(d, Pointer("/foo/world"))->GetString()); #if RAPIDJSON_HAS_STDSTRING - EXPECT_STREQ("C++", GetValueByPointerWithDefault(d, Pointer("/foo/C++"), std::string("C++")).GetString()); + EXPECT_STREQ("C++", GetValueByPointerWithDefault(d, Pointer("/foo/C++"), std::string("C++")).GetString()); #endif } -TEST(Pointer, GetValueByPointerWithDefault_String_NoAllocator) { - Document d; - d.Parse(kJson); - - const Value v("qux"); - EXPECT_TRUE(Value("bar") == GetValueByPointerWithDefault(d, "/foo/0", v)); - EXPECT_TRUE(Value("bar") == GetValueByPointerWithDefault(d, "/foo/0", v)); - EXPECT_TRUE(Value("baz") == GetValueByPointerWithDefault(d, "/foo/1", v)); - EXPECT_TRUE(Value("qux") == GetValueByPointerWithDefault(d, "/foo/2", v)); - EXPECT_TRUE(Value("last") == GetValueByPointerWithDefault(d, "/foo/-", Value("last").Move())); - EXPECT_STREQ("last", d["foo"][3].GetString()); - - EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/null", Value().Move()).IsNull()); - EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/null", "x").IsNull()); - - // Generic version - EXPECT_EQ(-1, GetValueByPointerWithDefault(d, "/foo/int", -1).GetInt()); - EXPECT_EQ(-1, GetValueByPointerWithDefault(d, "/foo/int", -2).GetInt()); - EXPECT_EQ(0x87654321, GetValueByPointerWithDefault(d, "/foo/uint", 0x87654321).GetUint()); - EXPECT_EQ(0x87654321, GetValueByPointerWithDefault(d, "/foo/uint", 0x12345678).GetUint()); - - const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); - EXPECT_EQ(i64, GetValueByPointerWithDefault(d, "/foo/int64", i64).GetInt64()); - EXPECT_EQ(i64, GetValueByPointerWithDefault(d, "/foo/int64", i64 + 1).GetInt64()); - - const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); - EXPECT_EQ(u64, GetValueByPointerWithDefault(d, "/foo/uint64", u64).GetUint64()); - EXPECT_EQ(u64, GetValueByPointerWithDefault(d, "/foo/uint64", u64 - 1).GetUint64()); - - EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/true", true).IsTrue()); - EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/true", false).IsTrue()); - - EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/false", false).IsFalse()); - EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/false", true).IsFalse()); - - // StringRef version - EXPECT_STREQ("Hello", GetValueByPointerWithDefault(d, "/foo/hello", "Hello").GetString()); - - // Copy string version - { - char buffer[256]; - strcpy(buffer, "World"); - EXPECT_STREQ("World", GetValueByPointerWithDefault(d, "/foo/world", buffer).GetString()); - memset(buffer, 0, sizeof(buffer)); - } - EXPECT_STREQ("World", GetValueByPointer(d, "/foo/world")->GetString()); +TEST(Pointer, GetValueByPointerWithDefault_String_NoAllocator) +{ Document d; + d.Parse(kJson); + + const Value v("qux"); + EXPECT_TRUE(Value("bar") == GetValueByPointerWithDefault(d, "/foo/0", v)); + EXPECT_TRUE(Value("bar") == GetValueByPointerWithDefault(d, "/foo/0", v)); + EXPECT_TRUE(Value("baz") == GetValueByPointerWithDefault(d, "/foo/1", v)); + EXPECT_TRUE(Value("qux") == GetValueByPointerWithDefault(d, "/foo/2", v)); + EXPECT_TRUE(Value("last") == GetValueByPointerWithDefault(d, "/foo/-", Value("last").Move())); + EXPECT_STREQ("last", d["foo"][3].GetString()); + + EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/null", Value().Move()).IsNull()); + EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/null", "x").IsNull()); + + // Generic version + EXPECT_EQ(-1, GetValueByPointerWithDefault(d, "/foo/int", -1).GetInt()); + EXPECT_EQ(-1, GetValueByPointerWithDefault(d, "/foo/int", -2).GetInt()); + EXPECT_EQ(0x87654321, GetValueByPointerWithDefault(d, "/foo/uint", 0x87654321).GetUint()); + EXPECT_EQ(0x87654321, GetValueByPointerWithDefault(d, "/foo/uint", 0x12345678).GetUint()); + + const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); + EXPECT_EQ(i64, GetValueByPointerWithDefault(d, "/foo/int64", i64).GetInt64()); + EXPECT_EQ(i64, GetValueByPointerWithDefault(d, "/foo/int64", i64 + 1).GetInt64()); + + const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); + EXPECT_EQ(u64, GetValueByPointerWithDefault(d, "/foo/uint64", u64).GetUint64()); + EXPECT_EQ(u64, GetValueByPointerWithDefault(d, "/foo/uint64", u64 - 1).GetUint64()); + + EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/true", true).IsTrue()); + EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/true", false).IsTrue()); + + EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/false", false).IsFalse()); + EXPECT_TRUE(GetValueByPointerWithDefault(d, "/foo/false", true).IsFalse()); + + // StringRef version + EXPECT_STREQ("Hello", GetValueByPointerWithDefault(d, "/foo/hello", "Hello").GetString()); + + // Copy string version + { char buffer[256]; + strcpy(buffer, "World"); + EXPECT_STREQ("World", GetValueByPointerWithDefault(d, "/foo/world", buffer).GetString()); + memset(buffer, 0, sizeof(buffer)); + } + EXPECT_STREQ("World", GetValueByPointer(d, "/foo/world")->GetString()); #if RAPIDJSON_HAS_STDSTRING - EXPECT_STREQ("C++", GetValueByPointerWithDefault(d, Pointer("/foo/C++"), std::string("C++")).GetString()); + EXPECT_STREQ("C++", GetValueByPointerWithDefault(d, Pointer("/foo/C++"), std::string("C++")).GetString()); #endif } -TEST(Pointer, SetValueByPointer_Pointer) { - Document d; - d.Parse(kJson); - Document::AllocatorType& a = d.GetAllocator(); +TEST(Pointer, SetValueByPointer_Pointer) +{ Document d; + d.Parse(kJson); + Document::AllocatorType& a = d.GetAllocator(); - // Value version - SetValueByPointer(d, Pointer("/foo/0"), Value(123).Move(), a); - EXPECT_EQ(123, d["foo"][0].GetInt()); + // Value version + SetValueByPointer(d, Pointer("/foo/0"), Value(123).Move(), a); + EXPECT_EQ(123, d["foo"][0].GetInt()); - SetValueByPointer(d, Pointer("/foo/null"), Value().Move(), a); - EXPECT_TRUE(GetValueByPointer(d, "/foo/null")->IsNull()); + SetValueByPointer(d, Pointer("/foo/null"), Value().Move(), a); + EXPECT_TRUE(GetValueByPointer(d, "/foo/null")->IsNull()); - // Const Value version - const Value foo(d["foo"], d.GetAllocator()); - SetValueByPointer(d, Pointer("/clone"), foo, a); - EXPECT_EQ(foo, *GetValueByPointer(d, "/clone")); + // Const Value version + const Value foo(d["foo"], d.GetAllocator()); + SetValueByPointer(d, Pointer("/clone"), foo, a); + EXPECT_EQ(foo, *GetValueByPointer(d, "/clone")); - // Generic version - SetValueByPointer(d, Pointer("/foo/int"), -1, a); - EXPECT_EQ(-1, GetValueByPointer(d, "/foo/int")->GetInt()); + // Generic version + SetValueByPointer(d, Pointer("/foo/int"), -1, a); + EXPECT_EQ(-1, GetValueByPointer(d, "/foo/int")->GetInt()); - SetValueByPointer(d, Pointer("/foo/uint"), 0x87654321, a); - EXPECT_EQ(0x87654321, GetValueByPointer(d, "/foo/uint")->GetUint()); + SetValueByPointer(d, Pointer("/foo/uint"), 0x87654321, a); + EXPECT_EQ(0x87654321, GetValueByPointer(d, "/foo/uint")->GetUint()); - const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); - SetValueByPointer(d, Pointer("/foo/int64"), i64, a); - EXPECT_EQ(i64, GetValueByPointer(d, "/foo/int64")->GetInt64()); + const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); + SetValueByPointer(d, Pointer("/foo/int64"), i64, a); + EXPECT_EQ(i64, GetValueByPointer(d, "/foo/int64")->GetInt64()); - const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); - SetValueByPointer(d, Pointer("/foo/uint64"), u64, a); - EXPECT_EQ(u64, GetValueByPointer(d, "/foo/uint64")->GetUint64()); + const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); + SetValueByPointer(d, Pointer("/foo/uint64"), u64, a); + EXPECT_EQ(u64, GetValueByPointer(d, "/foo/uint64")->GetUint64()); - SetValueByPointer(d, Pointer("/foo/true"), true, a); - EXPECT_TRUE(GetValueByPointer(d, "/foo/true")->IsTrue()); + SetValueByPointer(d, Pointer("/foo/true"), true, a); + EXPECT_TRUE(GetValueByPointer(d, "/foo/true")->IsTrue()); - SetValueByPointer(d, Pointer("/foo/false"), false, a); - EXPECT_TRUE(GetValueByPointer(d, "/foo/false")->IsFalse()); + SetValueByPointer(d, Pointer("/foo/false"), false, a); + EXPECT_TRUE(GetValueByPointer(d, "/foo/false")->IsFalse()); - // StringRef version - SetValueByPointer(d, Pointer("/foo/hello"), "Hello", a); - EXPECT_STREQ("Hello", GetValueByPointer(d, "/foo/hello")->GetString()); + // StringRef version + SetValueByPointer(d, Pointer("/foo/hello"), "Hello", a); + EXPECT_STREQ("Hello", GetValueByPointer(d, "/foo/hello")->GetString()); - // Copy string version - { - char buffer[256]; - strcpy(buffer, "World"); - SetValueByPointer(d, Pointer("/foo/world"), buffer, a); - memset(buffer, 0, sizeof(buffer)); - } - EXPECT_STREQ("World", GetValueByPointer(d, "/foo/world")->GetString()); + // Copy string version + { char buffer[256]; + strcpy(buffer, "World"); + SetValueByPointer(d, Pointer("/foo/world"), buffer, a); + memset(buffer, 0, sizeof(buffer)); + } + EXPECT_STREQ("World", GetValueByPointer(d, "/foo/world")->GetString()); #if RAPIDJSON_HAS_STDSTRING - SetValueByPointer(d, Pointer("/foo/c++"), std::string("C++"), a); - EXPECT_STREQ("C++", GetValueByPointer(d, "/foo/c++")->GetString()); + SetValueByPointer(d, Pointer("/foo/c++"), std::string("C++"), a); + EXPECT_STREQ("C++", GetValueByPointer(d, "/foo/c++")->GetString()); #endif } -TEST(Pointer, SetValueByPointer_String) { - Document d; - d.Parse(kJson); - Document::AllocatorType& a = d.GetAllocator(); +TEST(Pointer, SetValueByPointer_String) +{ Document d; + d.Parse(kJson); + Document::AllocatorType& a = d.GetAllocator(); - // Value version - SetValueByPointer(d, "/foo/0", Value(123).Move(), a); - EXPECT_EQ(123, d["foo"][0].GetInt()); + // Value version + SetValueByPointer(d, "/foo/0", Value(123).Move(), a); + EXPECT_EQ(123, d["foo"][0].GetInt()); - SetValueByPointer(d, "/foo/null", Value().Move(), a); - EXPECT_TRUE(GetValueByPointer(d, "/foo/null")->IsNull()); + SetValueByPointer(d, "/foo/null", Value().Move(), a); + EXPECT_TRUE(GetValueByPointer(d, "/foo/null")->IsNull()); - // Const Value version - const Value foo(d["foo"], d.GetAllocator()); - SetValueByPointer(d, "/clone", foo, a); - EXPECT_EQ(foo, *GetValueByPointer(d, "/clone")); + // Const Value version + const Value foo(d["foo"], d.GetAllocator()); + SetValueByPointer(d, "/clone", foo, a); + EXPECT_EQ(foo, *GetValueByPointer(d, "/clone")); - // Generic version - SetValueByPointer(d, "/foo/int", -1, a); - EXPECT_EQ(-1, GetValueByPointer(d, "/foo/int")->GetInt()); + // Generic version + SetValueByPointer(d, "/foo/int", -1, a); + EXPECT_EQ(-1, GetValueByPointer(d, "/foo/int")->GetInt()); - SetValueByPointer(d, "/foo/uint", 0x87654321, a); - EXPECT_EQ(0x87654321, GetValueByPointer(d, "/foo/uint")->GetUint()); + SetValueByPointer(d, "/foo/uint", 0x87654321, a); + EXPECT_EQ(0x87654321, GetValueByPointer(d, "/foo/uint")->GetUint()); - const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); - SetValueByPointer(d, "/foo/int64", i64, a); - EXPECT_EQ(i64, GetValueByPointer(d, "/foo/int64")->GetInt64()); + const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); + SetValueByPointer(d, "/foo/int64", i64, a); + EXPECT_EQ(i64, GetValueByPointer(d, "/foo/int64")->GetInt64()); - const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); - SetValueByPointer(d, "/foo/uint64", u64, a); - EXPECT_EQ(u64, GetValueByPointer(d, "/foo/uint64")->GetUint64()); + const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); + SetValueByPointer(d, "/foo/uint64", u64, a); + EXPECT_EQ(u64, GetValueByPointer(d, "/foo/uint64")->GetUint64()); - SetValueByPointer(d, "/foo/true", true, a); - EXPECT_TRUE(GetValueByPointer(d, "/foo/true")->IsTrue()); + SetValueByPointer(d, "/foo/true", true, a); + EXPECT_TRUE(GetValueByPointer(d, "/foo/true")->IsTrue()); - SetValueByPointer(d, "/foo/false", false, a); - EXPECT_TRUE(GetValueByPointer(d, "/foo/false")->IsFalse()); + SetValueByPointer(d, "/foo/false", false, a); + EXPECT_TRUE(GetValueByPointer(d, "/foo/false")->IsFalse()); - // StringRef version - SetValueByPointer(d, "/foo/hello", "Hello", a); - EXPECT_STREQ("Hello", GetValueByPointer(d, "/foo/hello")->GetString()); + // StringRef version + SetValueByPointer(d, "/foo/hello", "Hello", a); + EXPECT_STREQ("Hello", GetValueByPointer(d, "/foo/hello")->GetString()); - // Copy string version - { - char buffer[256]; - strcpy(buffer, "World"); - SetValueByPointer(d, "/foo/world", buffer, a); - memset(buffer, 0, sizeof(buffer)); - } - EXPECT_STREQ("World", GetValueByPointer(d, "/foo/world")->GetString()); + // Copy string version + { char buffer[256]; + strcpy(buffer, "World"); + SetValueByPointer(d, "/foo/world", buffer, a); + memset(buffer, 0, sizeof(buffer)); + } + EXPECT_STREQ("World", GetValueByPointer(d, "/foo/world")->GetString()); #if RAPIDJSON_HAS_STDSTRING - SetValueByPointer(d, "/foo/c++", std::string("C++"), a); - EXPECT_STREQ("C++", GetValueByPointer(d, "/foo/c++")->GetString()); + SetValueByPointer(d, "/foo/c++", std::string("C++"), a); + EXPECT_STREQ("C++", GetValueByPointer(d, "/foo/c++")->GetString()); #endif } -TEST(Pointer, SetValueByPointer_Pointer_NoAllocator) { - Document d; - d.Parse(kJson); +TEST(Pointer, SetValueByPointer_Pointer_NoAllocator) +{ Document d; + d.Parse(kJson); - // Value version - SetValueByPointer(d, Pointer("/foo/0"), Value(123).Move()); - EXPECT_EQ(123, d["foo"][0].GetInt()); + // Value version + SetValueByPointer(d, Pointer("/foo/0"), Value(123).Move()); + EXPECT_EQ(123, d["foo"][0].GetInt()); - SetValueByPointer(d, Pointer("/foo/null"), Value().Move()); - EXPECT_TRUE(GetValueByPointer(d, "/foo/null")->IsNull()); + SetValueByPointer(d, Pointer("/foo/null"), Value().Move()); + EXPECT_TRUE(GetValueByPointer(d, "/foo/null")->IsNull()); - // Const Value version - const Value foo(d["foo"], d.GetAllocator()); - SetValueByPointer(d, Pointer("/clone"), foo); - EXPECT_EQ(foo, *GetValueByPointer(d, "/clone")); + // Const Value version + const Value foo(d["foo"], d.GetAllocator()); + SetValueByPointer(d, Pointer("/clone"), foo); + EXPECT_EQ(foo, *GetValueByPointer(d, "/clone")); - // Generic version - SetValueByPointer(d, Pointer("/foo/int"), -1); - EXPECT_EQ(-1, GetValueByPointer(d, "/foo/int")->GetInt()); + // Generic version + SetValueByPointer(d, Pointer("/foo/int"), -1); + EXPECT_EQ(-1, GetValueByPointer(d, "/foo/int")->GetInt()); - SetValueByPointer(d, Pointer("/foo/uint"), 0x87654321); - EXPECT_EQ(0x87654321, GetValueByPointer(d, "/foo/uint")->GetUint()); + SetValueByPointer(d, Pointer("/foo/uint"), 0x87654321); + EXPECT_EQ(0x87654321, GetValueByPointer(d, "/foo/uint")->GetUint()); - const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); - SetValueByPointer(d, Pointer("/foo/int64"), i64); - EXPECT_EQ(i64, GetValueByPointer(d, "/foo/int64")->GetInt64()); + const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); + SetValueByPointer(d, Pointer("/foo/int64"), i64); + EXPECT_EQ(i64, GetValueByPointer(d, "/foo/int64")->GetInt64()); - const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); - SetValueByPointer(d, Pointer("/foo/uint64"), u64); - EXPECT_EQ(u64, GetValueByPointer(d, "/foo/uint64")->GetUint64()); + const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); + SetValueByPointer(d, Pointer("/foo/uint64"), u64); + EXPECT_EQ(u64, GetValueByPointer(d, "/foo/uint64")->GetUint64()); - SetValueByPointer(d, Pointer("/foo/true"), true); - EXPECT_TRUE(GetValueByPointer(d, "/foo/true")->IsTrue()); + SetValueByPointer(d, Pointer("/foo/true"), true); + EXPECT_TRUE(GetValueByPointer(d, "/foo/true")->IsTrue()); - SetValueByPointer(d, Pointer("/foo/false"), false); - EXPECT_TRUE(GetValueByPointer(d, "/foo/false")->IsFalse()); + SetValueByPointer(d, Pointer("/foo/false"), false); + EXPECT_TRUE(GetValueByPointer(d, "/foo/false")->IsFalse()); - // StringRef version - SetValueByPointer(d, Pointer("/foo/hello"), "Hello"); - EXPECT_STREQ("Hello", GetValueByPointer(d, "/foo/hello")->GetString()); + // StringRef version + SetValueByPointer(d, Pointer("/foo/hello"), "Hello"); + EXPECT_STREQ("Hello", GetValueByPointer(d, "/foo/hello")->GetString()); - // Copy string version - { - char buffer[256]; - strcpy(buffer, "World"); - SetValueByPointer(d, Pointer("/foo/world"), buffer); - memset(buffer, 0, sizeof(buffer)); - } - EXPECT_STREQ("World", GetValueByPointer(d, "/foo/world")->GetString()); + // Copy string version + { char buffer[256]; + strcpy(buffer, "World"); + SetValueByPointer(d, Pointer("/foo/world"), buffer); + memset(buffer, 0, sizeof(buffer)); + } + EXPECT_STREQ("World", GetValueByPointer(d, "/foo/world")->GetString()); #if RAPIDJSON_HAS_STDSTRING - SetValueByPointer(d, Pointer("/foo/c++"), std::string("C++")); - EXPECT_STREQ("C++", GetValueByPointer(d, "/foo/c++")->GetString()); + SetValueByPointer(d, Pointer("/foo/c++"), std::string("C++")); + EXPECT_STREQ("C++", GetValueByPointer(d, "/foo/c++")->GetString()); #endif } -TEST(Pointer, SetValueByPointer_String_NoAllocator) { - Document d; - d.Parse(kJson); +TEST(Pointer, SetValueByPointer_String_NoAllocator) +{ Document d; + d.Parse(kJson); - // Value version - SetValueByPointer(d, "/foo/0", Value(123).Move()); - EXPECT_EQ(123, d["foo"][0].GetInt()); + // Value version + SetValueByPointer(d, "/foo/0", Value(123).Move()); + EXPECT_EQ(123, d["foo"][0].GetInt()); - SetValueByPointer(d, "/foo/null", Value().Move()); - EXPECT_TRUE(GetValueByPointer(d, "/foo/null")->IsNull()); + SetValueByPointer(d, "/foo/null", Value().Move()); + EXPECT_TRUE(GetValueByPointer(d, "/foo/null")->IsNull()); - // Const Value version - const Value foo(d["foo"], d.GetAllocator()); - SetValueByPointer(d, "/clone", foo); - EXPECT_EQ(foo, *GetValueByPointer(d, "/clone")); + // Const Value version + const Value foo(d["foo"], d.GetAllocator()); + SetValueByPointer(d, "/clone", foo); + EXPECT_EQ(foo, *GetValueByPointer(d, "/clone")); - // Generic version - SetValueByPointer(d, "/foo/int", -1); - EXPECT_EQ(-1, GetValueByPointer(d, "/foo/int")->GetInt()); + // Generic version + SetValueByPointer(d, "/foo/int", -1); + EXPECT_EQ(-1, GetValueByPointer(d, "/foo/int")->GetInt()); - SetValueByPointer(d, "/foo/uint", 0x87654321); - EXPECT_EQ(0x87654321, GetValueByPointer(d, "/foo/uint")->GetUint()); + SetValueByPointer(d, "/foo/uint", 0x87654321); + EXPECT_EQ(0x87654321, GetValueByPointer(d, "/foo/uint")->GetUint()); - const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); - SetValueByPointer(d, "/foo/int64", i64); - EXPECT_EQ(i64, GetValueByPointer(d, "/foo/int64")->GetInt64()); + const int64_t i64 = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0)); + SetValueByPointer(d, "/foo/int64", i64); + EXPECT_EQ(i64, GetValueByPointer(d, "/foo/int64")->GetInt64()); - const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); - SetValueByPointer(d, "/foo/uint64", u64); - EXPECT_EQ(u64, GetValueByPointer(d, "/foo/uint64")->GetUint64()); + const uint64_t u64 = RAPIDJSON_UINT64_C2(0xFFFFFFFFF, 0xFFFFFFFFF); + SetValueByPointer(d, "/foo/uint64", u64); + EXPECT_EQ(u64, GetValueByPointer(d, "/foo/uint64")->GetUint64()); - SetValueByPointer(d, "/foo/true", true); - EXPECT_TRUE(GetValueByPointer(d, "/foo/true")->IsTrue()); + SetValueByPointer(d, "/foo/true", true); + EXPECT_TRUE(GetValueByPointer(d, "/foo/true")->IsTrue()); - SetValueByPointer(d, "/foo/false", false); - EXPECT_TRUE(GetValueByPointer(d, "/foo/false")->IsFalse()); + SetValueByPointer(d, "/foo/false", false); + EXPECT_TRUE(GetValueByPointer(d, "/foo/false")->IsFalse()); - // StringRef version - SetValueByPointer(d, "/foo/hello", "Hello"); - EXPECT_STREQ("Hello", GetValueByPointer(d, "/foo/hello")->GetString()); + // StringRef version + SetValueByPointer(d, "/foo/hello", "Hello"); + EXPECT_STREQ("Hello", GetValueByPointer(d, "/foo/hello")->GetString()); - // Copy string version - { - char buffer[256]; - strcpy(buffer, "World"); - SetValueByPointer(d, "/foo/world", buffer); - memset(buffer, 0, sizeof(buffer)); - } - EXPECT_STREQ("World", GetValueByPointer(d, "/foo/world")->GetString()); + // Copy string version + { char buffer[256]; + strcpy(buffer, "World"); + SetValueByPointer(d, "/foo/world", buffer); + memset(buffer, 0, sizeof(buffer)); + } + EXPECT_STREQ("World", GetValueByPointer(d, "/foo/world")->GetString()); #if RAPIDJSON_HAS_STDSTRING - SetValueByPointer(d, "/foo/c++", std::string("C++")); - EXPECT_STREQ("C++", GetValueByPointer(d, "/foo/c++")->GetString()); + SetValueByPointer(d, "/foo/c++", std::string("C++")); + EXPECT_STREQ("C++", GetValueByPointer(d, "/foo/c++")->GetString()); #endif } -TEST(Pointer, SwapValueByPointer) { - Document d; - d.Parse(kJson); - Document::AllocatorType& a = d.GetAllocator(); - SwapValueByPointer(d, Pointer("/foo/0"), *GetValueByPointer(d, "/foo/1"), a); - EXPECT_STREQ("baz", d["foo"][0].GetString()); - EXPECT_STREQ("bar", d["foo"][1].GetString()); - - SwapValueByPointer(d, "/foo/0", *GetValueByPointer(d, "/foo/1"), a); - EXPECT_STREQ("bar", d["foo"][0].GetString()); - EXPECT_STREQ("baz", d["foo"][1].GetString()); +TEST(Pointer, SwapValueByPointer) +{ Document d; + d.Parse(kJson); + Document::AllocatorType& a = d.GetAllocator(); + SwapValueByPointer(d, Pointer("/foo/0"), *GetValueByPointer(d, "/foo/1"), a); + EXPECT_STREQ("baz", d["foo"][0].GetString()); + EXPECT_STREQ("bar", d["foo"][1].GetString()); + + SwapValueByPointer(d, "/foo/0", *GetValueByPointer(d, "/foo/1"), a); + EXPECT_STREQ("bar", d["foo"][0].GetString()); + EXPECT_STREQ("baz", d["foo"][1].GetString()); } -TEST(Pointer, SwapValueByPointer_NoAllocator) { - Document d; - d.Parse(kJson); - SwapValueByPointer(d, Pointer("/foo/0"), *GetValueByPointer(d, "/foo/1")); - EXPECT_STREQ("baz", d["foo"][0].GetString()); - EXPECT_STREQ("bar", d["foo"][1].GetString()); +TEST(Pointer, SwapValueByPointer_NoAllocator) +{ Document d; + d.Parse(kJson); + SwapValueByPointer(d, Pointer("/foo/0"), *GetValueByPointer(d, "/foo/1")); + EXPECT_STREQ("baz", d["foo"][0].GetString()); + EXPECT_STREQ("bar", d["foo"][1].GetString()); - SwapValueByPointer(d, "/foo/0", *GetValueByPointer(d, "/foo/1")); - EXPECT_STREQ("bar", d["foo"][0].GetString()); - EXPECT_STREQ("baz", d["foo"][1].GetString()); + SwapValueByPointer(d, "/foo/0", *GetValueByPointer(d, "/foo/1")); + EXPECT_STREQ("bar", d["foo"][0].GetString()); + EXPECT_STREQ("baz", d["foo"][1].GetString()); } -TEST(Pointer, EraseValueByPointer_Pointer) { - Document d; - d.Parse(kJson); - - EXPECT_FALSE(EraseValueByPointer(d, Pointer(""))); - EXPECT_FALSE(Pointer("/foo/nonexist").Erase(d)); - EXPECT_TRUE(EraseValueByPointer(d, Pointer("/foo/0"))); - EXPECT_EQ(1u, d["foo"].Size()); - EXPECT_STREQ("baz", d["foo"][0].GetString()); - EXPECT_TRUE(EraseValueByPointer(d, Pointer("/foo/0"))); - EXPECT_TRUE(d["foo"].Empty()); - EXPECT_TRUE(EraseValueByPointer(d, Pointer("/foo"))); - EXPECT_TRUE(Pointer("/foo").Get(d) == 0); +TEST(Pointer, EraseValueByPointer_Pointer) +{ Document d; + d.Parse(kJson); + + EXPECT_FALSE(EraseValueByPointer(d, Pointer(""))); + EXPECT_FALSE(Pointer("/foo/nonexist").Erase(d)); + EXPECT_TRUE(EraseValueByPointer(d, Pointer("/foo/0"))); + EXPECT_EQ(1u, d["foo"].Size()); + EXPECT_STREQ("baz", d["foo"][0].GetString()); + EXPECT_TRUE(EraseValueByPointer(d, Pointer("/foo/0"))); + EXPECT_TRUE(d["foo"].Empty()); + EXPECT_TRUE(EraseValueByPointer(d, Pointer("/foo"))); + EXPECT_TRUE(Pointer("/foo").Get(d) == 0); } -TEST(Pointer, EraseValueByPointer_String) { - Document d; - d.Parse(kJson); - - EXPECT_FALSE(EraseValueByPointer(d, "")); - EXPECT_FALSE(Pointer("/foo/nonexist").Erase(d)); - EXPECT_TRUE(EraseValueByPointer(d, "/foo/0")); - EXPECT_EQ(1u, d["foo"].Size()); - EXPECT_STREQ("baz", d["foo"][0].GetString()); - EXPECT_TRUE(EraseValueByPointer(d, "/foo/0")); - EXPECT_TRUE(d["foo"].Empty()); - EXPECT_TRUE(EraseValueByPointer(d, "/foo")); - EXPECT_TRUE(Pointer("/foo").Get(d) == 0); +TEST(Pointer, EraseValueByPointer_String) +{ Document d; + d.Parse(kJson); + + EXPECT_FALSE(EraseValueByPointer(d, "")); + EXPECT_FALSE(Pointer("/foo/nonexist").Erase(d)); + EXPECT_TRUE(EraseValueByPointer(d, "/foo/0")); + EXPECT_EQ(1u, d["foo"].Size()); + EXPECT_STREQ("baz", d["foo"][0].GetString()); + EXPECT_TRUE(EraseValueByPointer(d, "/foo/0")); + EXPECT_TRUE(d["foo"].Empty()); + EXPECT_TRUE(EraseValueByPointer(d, "/foo")); + EXPECT_TRUE(Pointer("/foo").Get(d) == 0); } -TEST(Pointer, Ambiguity) { - { - Document d; - d.Parse("{\"0\" : [123]}"); - EXPECT_EQ(123, Pointer("/0/0").Get(d)->GetInt()); - Pointer("/0/a").Set(d, 456); // Change array [123] to object {456} - EXPECT_EQ(456, Pointer("/0/a").Get(d)->GetInt()); - } - - { - Document d; - EXPECT_FALSE(d.Parse("[{\"0\": 123}]").HasParseError()); - EXPECT_EQ(123, Pointer("/0/0").Get(d)->GetInt()); - Pointer("/0/1").Set(d, 456); // 1 is treated as "1" to index object - EXPECT_EQ(123, Pointer("/0/0").Get(d)->GetInt()); - EXPECT_EQ(456, Pointer("/0/1").Get(d)->GetInt()); - } +TEST(Pointer, Ambiguity) +{ + { Document d; + d.Parse("{\"0\" : [123]}"); + EXPECT_EQ(123, Pointer("/0/0").Get(d)->GetInt()); + Pointer("/0/a").Set(d, 456); // Change array [123] to object {456} + EXPECT_EQ(456, Pointer("/0/a").Get(d)->GetInt()); + } + + { Document d; + EXPECT_FALSE(d.Parse("[{\"0\": 123}]").HasParseError()); + EXPECT_EQ(123, Pointer("/0/0").Get(d)->GetInt()); + Pointer("/0/1").Set(d, 456); // 1 is treated as "1" to index object + EXPECT_EQ(123, Pointer("/0/0").Get(d)->GetInt()); + EXPECT_EQ(456, Pointer("/0/1").Get(d)->GetInt()); + } } // https://github.com/miloyip/rapidjson/issues/483 -namespace myjson { +namespace myjson +{ class MyAllocator { public: - static const bool kNeedFree = true; - void * Malloc(size_t _size) { return malloc(_size); } - void * Realloc(void *_org_p, size_t _org_size, size_t _new_size) { (void)_org_size; return realloc(_org_p, _new_size); } - static void Free(void *_p) { return free(_p); } + static const bool kNeedFree = true; + void * Malloc(size_t _size) { return malloc(_size); } + void * Realloc(void *_org_p, size_t _org_size, size_t _new_size) { (void)_org_size; return realloc(_org_p, _new_size); } + static void Free(void *_p) { return free(_p); } }; typedef rapidjson::GenericDocument< - rapidjson::UTF8<>, - rapidjson::MemoryPoolAllocator< MyAllocator >, - MyAllocator - > Document; +rapidjson::UTF8<>, + rapidjson::MemoryPoolAllocator< MyAllocator >, + MyAllocator + > Document; typedef rapidjson::GenericPointer< - ::myjson::Document::ValueType, - MyAllocator - > Pointer; +::myjson::Document::ValueType, +MyAllocator +> Pointer; typedef ::myjson::Document::ValueType Value; } -TEST(Pointer, Issue483) { - std::string mystr, path; - myjson::Document document; - myjson::Value value(rapidjson::kStringType); - value.SetString(mystr.c_str(), static_cast(mystr.length()), document.GetAllocator()); - myjson::Pointer(path.c_str()).Set(document, value, document.GetAllocator()); +TEST(Pointer, Issue483) +{ std::string mystr, path; + myjson::Document document; + myjson::Value value(rapidjson::kStringType); + value.SetString(mystr.c_str(), static_cast(mystr.length()), document.GetAllocator()); + myjson::Pointer(path.c_str()).Set(document, value, document.GetAllocator()); } diff --git a/rapidjson/test/unittest/prettywritertest.cpp b/rapidjson/test/unittest/prettywritertest.cpp index a372f7986f8..db190213d0c 100644 --- a/rapidjson/test/unittest/prettywritertest.cpp +++ b/rapidjson/test/unittest/prettywritertest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -22,182 +22,183 @@ using namespace rapidjson; static const char kJson[] = "{\"hello\":\"world\",\"t\":true,\"f\":false,\"n\":null,\"i\":123,\"pi\":3.1416,\"a\":[1,2,3,-1],\"u64\":1234567890123456789,\"i64\":-1234567890123456789}"; static const char kPrettyJson[] = -"{\n" -" \"hello\": \"world\",\n" -" \"t\": true,\n" -" \"f\": false,\n" -" \"n\": null,\n" -" \"i\": 123,\n" -" \"pi\": 3.1416,\n" -" \"a\": [\n" -" 1,\n" -" 2,\n" -" 3,\n" -" -1\n" -" ],\n" -" \"u64\": 1234567890123456789,\n" -" \"i64\": -1234567890123456789\n" -"}"; + "{\n" + " \"hello\": \"world\",\n" + " \"t\": true,\n" + " \"f\": false,\n" + " \"n\": null,\n" + " \"i\": 123,\n" + " \"pi\": 3.1416,\n" + " \"a\": [\n" + " 1,\n" + " 2,\n" + " 3,\n" + " -1\n" + " ],\n" + " \"u64\": 1234567890123456789,\n" + " \"i64\": -1234567890123456789\n" + "}"; static const char kPrettyJson_FormatOptions_SLA[] = -"{\n" -" \"hello\": \"world\",\n" -" \"t\": true,\n" -" \"f\": false,\n" -" \"n\": null,\n" -" \"i\": 123,\n" -" \"pi\": 3.1416,\n" -" \"a\": [1, 2, 3, -1],\n" -" \"u64\": 1234567890123456789,\n" -" \"i64\": -1234567890123456789\n" -"}"; - -TEST(PrettyWriter, Basic) { - StringBuffer buffer; - PrettyWriter writer(buffer); - Reader reader; - StringStream s(kJson); - reader.Parse(s, writer); - EXPECT_STREQ(kPrettyJson, buffer.GetString()); + "{\n" + " \"hello\": \"world\",\n" + " \"t\": true,\n" + " \"f\": false,\n" + " \"n\": null,\n" + " \"i\": 123,\n" + " \"pi\": 3.1416,\n" + " \"a\": [1, 2, 3, -1],\n" + " \"u64\": 1234567890123456789,\n" + " \"i64\": -1234567890123456789\n" + "}"; + +TEST(PrettyWriter, Basic) +{ StringBuffer buffer; + PrettyWriter writer(buffer); + Reader reader; + StringStream s(kJson); + reader.Parse(s, writer); + EXPECT_STREQ(kPrettyJson, buffer.GetString()); } -TEST(PrettyWriter, FormatOptions) { - StringBuffer buffer; - PrettyWriter writer(buffer); - writer.SetFormatOptions(kFormatSingleLineArray); - Reader reader; - StringStream s(kJson); - reader.Parse(s, writer); - EXPECT_STREQ(kPrettyJson_FormatOptions_SLA, buffer.GetString()); +TEST(PrettyWriter, FormatOptions) +{ StringBuffer buffer; + PrettyWriter writer(buffer); + writer.SetFormatOptions(kFormatSingleLineArray); + Reader reader; + StringStream s(kJson); + reader.Parse(s, writer); + EXPECT_STREQ(kPrettyJson_FormatOptions_SLA, buffer.GetString()); } -TEST(PrettyWriter, SetIndent) { - StringBuffer buffer; - PrettyWriter writer(buffer); - writer.SetIndent('\t', 1); - Reader reader; - StringStream s(kJson); - reader.Parse(s, writer); - EXPECT_STREQ( - "{\n" - "\t\"hello\": \"world\",\n" - "\t\"t\": true,\n" - "\t\"f\": false,\n" - "\t\"n\": null,\n" - "\t\"i\": 123,\n" - "\t\"pi\": 3.1416,\n" - "\t\"a\": [\n" - "\t\t1,\n" - "\t\t2,\n" - "\t\t3,\n" - "\t\t-1\n" - "\t],\n" - "\t\"u64\": 1234567890123456789,\n" - "\t\"i64\": -1234567890123456789\n" - "}", - buffer.GetString()); +TEST(PrettyWriter, SetIndent) +{ StringBuffer buffer; + PrettyWriter writer(buffer); + writer.SetIndent('\t', 1); + Reader reader; + StringStream s(kJson); + reader.Parse(s, writer); + EXPECT_STREQ( + "{\n" + "\t\"hello\": \"world\",\n" + "\t\"t\": true,\n" + "\t\"f\": false,\n" + "\t\"n\": null,\n" + "\t\"i\": 123,\n" + "\t\"pi\": 3.1416,\n" + "\t\"a\": [\n" + "\t\t1,\n" + "\t\t2,\n" + "\t\t3,\n" + "\t\t-1\n" + "\t],\n" + "\t\"u64\": 1234567890123456789,\n" + "\t\"i64\": -1234567890123456789\n" + "}", + buffer.GetString()); } -TEST(PrettyWriter, String) { - StringBuffer buffer; - PrettyWriter writer(buffer); - EXPECT_TRUE(writer.StartArray()); - EXPECT_TRUE(writer.String("Hello\n")); - EXPECT_TRUE(writer.EndArray()); - EXPECT_STREQ("[\n \"Hello\\n\"\n]", buffer.GetString()); +TEST(PrettyWriter, String) +{ StringBuffer buffer; + PrettyWriter writer(buffer); + EXPECT_TRUE(writer.StartArray()); + EXPECT_TRUE(writer.String("Hello\n")); + EXPECT_TRUE(writer.EndArray()); + EXPECT_STREQ("[\n \"Hello\\n\"\n]", buffer.GetString()); } #if RAPIDJSON_HAS_STDSTRING -TEST(PrettyWriter, String_STDSTRING) { - StringBuffer buffer; - PrettyWriter writer(buffer); - EXPECT_TRUE(writer.StartArray()); - EXPECT_TRUE(writer.String(std::string("Hello\n"))); - EXPECT_TRUE(writer.EndArray()); - EXPECT_STREQ("[\n \"Hello\\n\"\n]", buffer.GetString()); +TEST(PrettyWriter, String_STDSTRING) +{ StringBuffer buffer; + PrettyWriter writer(buffer); + EXPECT_TRUE(writer.StartArray()); + EXPECT_TRUE(writer.String(std::string("Hello\n"))); + EXPECT_TRUE(writer.EndArray()); + EXPECT_STREQ("[\n \"Hello\\n\"\n]", buffer.GetString()); } #endif #include -class OStreamWrapper { +class OStreamWrapper +{ public: - typedef char Ch; + typedef char Ch; - OStreamWrapper(std::ostream& os) : os_(os) {} + OStreamWrapper(std::ostream& os) : os_(os) {} - Ch Peek() const { assert(false); return '\0'; } - Ch Take() { assert(false); return '\0'; } - size_t Tell() const { return 0; } + Ch Peek() const { assert(false); return '\0'; } + Ch Take() { assert(false); return '\0'; } + size_t Tell() const { return 0; } - Ch* PutBegin() { assert(false); return 0; } - void Put(Ch c) { os_.put(c); } - void Flush() { os_.flush(); } - size_t PutEnd(Ch*) { assert(false); return 0; } + Ch* PutBegin() { assert(false); return 0; } + void Put(Ch c) { os_.put(c); } + void Flush() { os_.flush(); } + size_t PutEnd(Ch*) { assert(false); return 0; } private: - OStreamWrapper(const OStreamWrapper&); - OStreamWrapper& operator=(const OStreamWrapper&); + OStreamWrapper(const OStreamWrapper&); + OStreamWrapper& operator=(const OStreamWrapper&); - std::ostream& os_; + std::ostream& os_; }; // For covering PutN() generic version -TEST(PrettyWriter, OStreamWrapper) { - StringStream s(kJson); - - std::stringstream ss; - OStreamWrapper os(ss); - - PrettyWriter writer(os); - - Reader reader; - reader.Parse(s, writer); - - std::string actual = ss.str(); - EXPECT_STREQ(kPrettyJson, actual.c_str()); +TEST(PrettyWriter, OStreamWrapper) +{ StringStream s(kJson); + + std::stringstream ss; + OStreamWrapper os(ss); + + PrettyWriter writer(os); + + Reader reader; + reader.Parse(s, writer); + + std::string actual = ss.str(); + EXPECT_STREQ(kPrettyJson, actual.c_str()); } // For covering FileWriteStream::PutN() -TEST(PrettyWriter, FileWriteStream) { - char filename[L_tmpnam]; - FILE* fp = TempFile(filename); - char buffer[16]; - FileWriteStream os(fp, buffer, sizeof(buffer)); - PrettyWriter writer(os); - Reader reader; - StringStream s(kJson); - reader.Parse(s, writer); - fclose(fp); - - fp = fopen(filename, "rb"); - fseek(fp, 0, SEEK_END); - size_t size = static_cast(ftell(fp)); - fseek(fp, 0, SEEK_SET); - char* json = static_cast(malloc(size + 1)); - size_t readLength = fread(json, 1, size, fp); - json[readLength] = '\0'; - fclose(fp); - remove(filename); - EXPECT_STREQ(kPrettyJson, json); - free(json); +TEST(PrettyWriter, FileWriteStream) +{ char filename[L_tmpnam]; + FILE* fp = TempFile(filename); + char buffer[16]; + FileWriteStream os(fp, buffer, sizeof(buffer)); + PrettyWriter writer(os); + Reader reader; + StringStream s(kJson); + reader.Parse(s, writer); + fclose(fp); + + fp = fopen(filename, "rb"); + fseek(fp, 0, SEEK_END); + size_t size = static_cast(ftell(fp)); + fseek(fp, 0, SEEK_SET); + char* json = static_cast(malloc(size + 1)); + size_t readLength = fread(json, 1, size, fp); + json[readLength] = '\0'; + fclose(fp); + remove(filename); + EXPECT_STREQ(kPrettyJson, json); + free(json); } -TEST(PrettyWriter, RawValue) { - StringBuffer buffer; - PrettyWriter writer(buffer); - writer.StartObject(); - writer.Key("a"); - writer.Int(1); - writer.Key("raw"); - const char json[] = "[\"Hello\\nWorld\", 123.456]"; - writer.RawValue(json, strlen(json), kArrayType); - writer.EndObject(); - EXPECT_TRUE(writer.IsComplete()); - EXPECT_STREQ( - "{\n" - " \"a\": 1,\n" - " \"raw\": [\"Hello\\nWorld\", 123.456]\n" // no indentation within raw value - "}", - buffer.GetString()); +TEST(PrettyWriter, RawValue) +{ StringBuffer buffer; + PrettyWriter writer(buffer); + writer.StartObject(); + writer.Key("a"); + writer.Int(1); + writer.Key("raw"); + const char json[] = "[\"Hello\\nWorld\", 123.456]"; + writer.RawValue(json, strlen(json), kArrayType); + writer.EndObject(); + EXPECT_TRUE(writer.IsComplete()); + EXPECT_STREQ( + "{\n" + " \"a\": 1,\n" + " \"raw\": [\"Hello\\nWorld\", 123.456]\n" // no indentation within raw value + "}", + buffer.GetString()); } diff --git a/rapidjson/test/unittest/readertest.cpp b/rapidjson/test/unittest/readertest.cpp index 64a1f9c3cf4..92ff952622d 100644 --- a/rapidjson/test/unittest/readertest.cpp +++ b/rapidjson/test/unittest/readertest.cpp @@ -39,78 +39,79 @@ RAPIDJSON_DIAG_OFF(c++98-compat-pedantic) #endif template -struct ParseBoolHandler : BaseReaderHandler, ParseBoolHandler > { - ParseBoolHandler() : step_(0) {} - bool Default() { ADD_FAILURE(); return false; } - // gcc 4.8.x generates warning in EXPECT_EQ(bool, bool) on this gtest version. - // Workaround with EXPECT_TRUE(). - bool Bool(bool b) { /*EXPECT_EQ(expect, b); */EXPECT_TRUE(expect == b); ++step_; return true; } - - unsigned step_; +struct ParseBoolHandler : BaseReaderHandler, ParseBoolHandler > +{ ParseBoolHandler() : step_(0) {} + bool Default() { ADD_FAILURE(); return false; } + // gcc 4.8.x generates warning in EXPECT_EQ(bool, bool) on this gtest version. + // Workaround with EXPECT_TRUE(). + bool Bool(bool b) { /*EXPECT_EQ(expect, b); */EXPECT_TRUE(expect == b); ++step_; return true; } + + unsigned step_; }; -TEST(Reader, ParseTrue) { - StringStream s("true"); - ParseBoolHandler h; - Reader reader; - reader.Parse(s, h); - EXPECT_EQ(1u, h.step_); +TEST(Reader, ParseTrue) +{ StringStream s("true"); + ParseBoolHandler h; + Reader reader; + reader.Parse(s, h); + EXPECT_EQ(1u, h.step_); } -TEST(Reader, ParseFalse) { - StringStream s("false"); - ParseBoolHandler h; - Reader reader; - reader.Parse(s, h); - EXPECT_EQ(1u, h.step_); +TEST(Reader, ParseFalse) +{ StringStream s("false"); + ParseBoolHandler h; + Reader reader; + reader.Parse(s, h); + EXPECT_EQ(1u, h.step_); } -struct ParseIntHandler : BaseReaderHandler, ParseIntHandler> { - ParseIntHandler() : step_(0), actual_() {} - bool Default() { ADD_FAILURE(); return false; } - bool Int(int i) { actual_ = i; step_++; return true; } +struct ParseIntHandler : BaseReaderHandler, ParseIntHandler> +{ ParseIntHandler() : step_(0), actual_() {} + bool Default() { ADD_FAILURE(); return false; } + bool Int(int i) { actual_ = i; step_++; return true; } - unsigned step_; - int actual_; + unsigned step_; + int actual_; }; -struct ParseUintHandler : BaseReaderHandler, ParseUintHandler> { - ParseUintHandler() : step_(0), actual_() {} - bool Default() { ADD_FAILURE(); return false; } - bool Uint(unsigned i) { actual_ = i; step_++; return true; } +struct ParseUintHandler : BaseReaderHandler, ParseUintHandler> +{ ParseUintHandler() : step_(0), actual_() {} + bool Default() { ADD_FAILURE(); return false; } + bool Uint(unsigned i) { actual_ = i; step_++; return true; } - unsigned step_; - unsigned actual_; + unsigned step_; + unsigned actual_; }; -struct ParseInt64Handler : BaseReaderHandler, ParseInt64Handler> { - ParseInt64Handler() : step_(0), actual_() {} - bool Default() { ADD_FAILURE(); return false; } - bool Int64(int64_t i) { actual_ = i; step_++; return true; } +struct ParseInt64Handler : BaseReaderHandler, ParseInt64Handler> +{ ParseInt64Handler() : step_(0), actual_() {} + bool Default() { ADD_FAILURE(); return false; } + bool Int64(int64_t i) { actual_ = i; step_++; return true; } - unsigned step_; - int64_t actual_; + unsigned step_; + int64_t actual_; }; -struct ParseUint64Handler : BaseReaderHandler, ParseUint64Handler> { - ParseUint64Handler() : step_(0), actual_() {} - bool Default() { ADD_FAILURE(); return false; } - bool Uint64(uint64_t i) { actual_ = i; step_++; return true; } +struct ParseUint64Handler : BaseReaderHandler, ParseUint64Handler> +{ ParseUint64Handler() : step_(0), actual_() {} + bool Default() { ADD_FAILURE(); return false; } + bool Uint64(uint64_t i) { actual_ = i; step_++; return true; } - unsigned step_; - uint64_t actual_; + unsigned step_; + uint64_t actual_; }; -struct ParseDoubleHandler : BaseReaderHandler, ParseDoubleHandler> { - ParseDoubleHandler() : step_(0), actual_() {} - bool Default() { ADD_FAILURE(); return false; } - bool Double(double d) { actual_ = d; step_++; return true; } +struct ParseDoubleHandler : BaseReaderHandler, ParseDoubleHandler> +{ ParseDoubleHandler() : step_(0), actual_() {} + bool Default() { ADD_FAILURE(); return false; } + bool Double(double d) { actual_ = d; step_++; return true; } - unsigned step_; - double actual_; + unsigned step_; + double actual_; }; -TEST(Reader, ParseNumber_Integer) { +TEST(Reader, ParseNumber_Integer) +{ #define TEST_INTEGER(Handler, str, x) \ { \ StringStream s(str); \ @@ -121,71 +122,70 @@ TEST(Reader, ParseNumber_Integer) { EXPECT_EQ(x, h.actual_); \ } - TEST_INTEGER(ParseUintHandler, "0", 0u); - TEST_INTEGER(ParseUintHandler, "123", 123u); - TEST_INTEGER(ParseUintHandler, "2147483648", 2147483648u); // 2^31 - 1 (cannot be stored in int) - TEST_INTEGER(ParseUintHandler, "4294967295", 4294967295u); + TEST_INTEGER(ParseUintHandler, "0", 0u); + TEST_INTEGER(ParseUintHandler, "123", 123u); + TEST_INTEGER(ParseUintHandler, "2147483648", 2147483648u); // 2^31 - 1 (cannot be stored in int) + TEST_INTEGER(ParseUintHandler, "4294967295", 4294967295u); - TEST_INTEGER(ParseIntHandler, "-123", -123); - TEST_INTEGER(ParseIntHandler, "-2147483648", static_cast(0x80000000)); // -2^31 (min of int) + TEST_INTEGER(ParseIntHandler, "-123", -123); + TEST_INTEGER(ParseIntHandler, "-2147483648", static_cast(0x80000000)); // -2^31 (min of int) - TEST_INTEGER(ParseUint64Handler, "4294967296", RAPIDJSON_UINT64_C2(1, 0)); // 2^32 (max of unsigned + 1, force to use uint64_t) - TEST_INTEGER(ParseUint64Handler, "18446744073709551615", RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0xFFFFFFFF)); // 2^64 - 1 (max of uint64_t) + TEST_INTEGER(ParseUint64Handler, "4294967296", RAPIDJSON_UINT64_C2(1, 0)); // 2^32 (max of unsigned + 1, force to use uint64_t) + TEST_INTEGER(ParseUint64Handler, "18446744073709551615", RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0xFFFFFFFF)); // 2^64 - 1 (max of uint64_t) - TEST_INTEGER(ParseInt64Handler, "-2147483649", static_cast(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x7FFFFFFF))); // -2^31 -1 (min of int - 1, force to use int64_t) - TEST_INTEGER(ParseInt64Handler, "-9223372036854775808", static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0x00000000))); // -2^63 (min of int64_t) + TEST_INTEGER(ParseInt64Handler, "-2147483649", static_cast(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x7FFFFFFF))); // -2^31 -1 (min of int - 1, force to use int64_t) + TEST_INTEGER(ParseInt64Handler, "-9223372036854775808", static_cast(RAPIDJSON_UINT64_C2(0x80000000, 0x00000000))); // -2^63 (min of int64_t) - // Random test for uint32_t/int32_t - { - union { - uint32_t u; - int32_t i; - }u; - Random r; + // Random test for uint32_t/int32_t + { union + { uint32_t u; + int32_t i; + } u; + Random r; - for (unsigned i = 0; i < 100000; i++) { - u.u = r(); + for (unsigned i = 0; i < 100000; i++) + { u.u = r(); - char buffer[32]; - *internal::u32toa(u.u, buffer) = '\0'; - TEST_INTEGER(ParseUintHandler, buffer, u.u); + char buffer[32]; + *internal::u32toa(u.u, buffer) = '\0'; + TEST_INTEGER(ParseUintHandler, buffer, u.u); - if (u.i < 0) { - *internal::i32toa(u.i, buffer) = '\0'; - TEST_INTEGER(ParseIntHandler, buffer, u.i); - } - } + if (u.i < 0) + { *internal::i32toa(u.i, buffer) = '\0'; + TEST_INTEGER(ParseIntHandler, buffer, u.i); + } } + } - // Random test for uint64_t/int64_t - { - union { - uint64_t u; - int64_t i; - }u; - Random r; - - for (unsigned i = 0; i < 100000; i++) { - u.u = uint64_t(r()) << 32; - u.u |= r(); - - char buffer[32]; - if (u.u > uint64_t(4294967295u)) { - *internal::u64toa(u.u, buffer) = '\0'; - TEST_INTEGER(ParseUint64Handler, buffer, u.u); - } - - if (u.i < -int64_t(2147483648u)) { - *internal::i64toa(u.i, buffer) = '\0'; - TEST_INTEGER(ParseInt64Handler, buffer, u.i); - } - } + // Random test for uint64_t/int64_t + { union + { uint64_t u; + int64_t i; + } u; + Random r; + + for (unsigned i = 0; i < 100000; i++) + { u.u = uint64_t(r()) << 32; + u.u |= r(); + + char buffer[32]; + if (u.u > uint64_t(4294967295u)) + { *internal::u64toa(u.u, buffer) = '\0'; + TEST_INTEGER(ParseUint64Handler, buffer, u.u); + } + + if (u.i < -int64_t(2147483648u)) + { *internal::i64toa(u.i, buffer) = '\0'; + TEST_INTEGER(ParseInt64Handler, buffer, u.i); + } } + } #undef TEST_INTEGER } template -static void TestParseDouble() { +static void TestParseDouble() +{ #define TEST_DOUBLE(fullPrecision, str, x) \ { \ StringStream s(str); \ @@ -205,223 +205,222 @@ static void TestParseDouble() { } \ } - TEST_DOUBLE(fullPrecision, "0.0", 0.0); - TEST_DOUBLE(fullPrecision, "-0.0", -0.0); // For checking issue #289 - TEST_DOUBLE(fullPrecision, "1.0", 1.0); - TEST_DOUBLE(fullPrecision, "-1.0", -1.0); - TEST_DOUBLE(fullPrecision, "1.5", 1.5); - TEST_DOUBLE(fullPrecision, "-1.5", -1.5); - TEST_DOUBLE(fullPrecision, "3.1416", 3.1416); - TEST_DOUBLE(fullPrecision, "1E10", 1E10); - TEST_DOUBLE(fullPrecision, "1e10", 1e10); - TEST_DOUBLE(fullPrecision, "1E+10", 1E+10); - TEST_DOUBLE(fullPrecision, "1E-10", 1E-10); - TEST_DOUBLE(fullPrecision, "-1E10", -1E10); - TEST_DOUBLE(fullPrecision, "-1e10", -1e10); - TEST_DOUBLE(fullPrecision, "-1E+10", -1E+10); - TEST_DOUBLE(fullPrecision, "-1E-10", -1E-10); - TEST_DOUBLE(fullPrecision, "1.234E+10", 1.234E+10); - TEST_DOUBLE(fullPrecision, "1.234E-10", 1.234E-10); - TEST_DOUBLE(fullPrecision, "1.79769e+308", 1.79769e+308); - TEST_DOUBLE(fullPrecision, "2.22507e-308", 2.22507e-308); - TEST_DOUBLE(fullPrecision, "-1.79769e+308", -1.79769e+308); - TEST_DOUBLE(fullPrecision, "-2.22507e-308", -2.22507e-308); - TEST_DOUBLE(fullPrecision, "4.9406564584124654e-324", 4.9406564584124654e-324); // minimum denormal - TEST_DOUBLE(fullPrecision, "2.2250738585072009e-308", 2.2250738585072009e-308); // Max subnormal double - TEST_DOUBLE(fullPrecision, "2.2250738585072014e-308", 2.2250738585072014e-308); // Min normal positive double - TEST_DOUBLE(fullPrecision, "1.7976931348623157e+308", 1.7976931348623157e+308); // Max double - TEST_DOUBLE(fullPrecision, "1e-10000", 0.0); // must underflow - TEST_DOUBLE(fullPrecision, "18446744073709551616", 18446744073709551616.0); // 2^64 (max of uint64_t + 1, force to use double) - TEST_DOUBLE(fullPrecision, "-9223372036854775809", -9223372036854775809.0); // -2^63 - 1(min of int64_t + 1, force to use double) - TEST_DOUBLE(fullPrecision, "0.9868011474609375", 0.9868011474609375); // https://github.com/miloyip/rapidjson/issues/120 - TEST_DOUBLE(fullPrecision, "123e34", 123e34); // Fast Path Cases In Disguise - TEST_DOUBLE(fullPrecision, "45913141877270640000.0", 45913141877270640000.0); - TEST_DOUBLE(fullPrecision, "2.2250738585072011e-308", 2.2250738585072011e-308); // http://www.exploringbinary.com/php-hangs-on-numeric-value-2-2250738585072011e-308/ - TEST_DOUBLE(fullPrecision, "1e-00011111111111", 0.0); // Issue #313 - TEST_DOUBLE(fullPrecision, "-1e-00011111111111", -0.0); - TEST_DOUBLE(fullPrecision, "1e-214748363", 0.0); // Maximum supported negative exponent - TEST_DOUBLE(fullPrecision, "1e-214748364", 0.0); - TEST_DOUBLE(fullPrecision, "1e-21474836311", 0.0); - TEST_DOUBLE(fullPrecision, "0.017976931348623157e+310", 1.7976931348623157e+308); // Max double in another form - - // Since - // abs((2^-1022 - 2^-1074) - 2.2250738585072012e-308) = 3.109754131239141401123495768877590405345064751974375599... �� 10^-324 - // abs((2^-1022) - 2.2250738585072012e-308) = 1.830902327173324040642192159804623318305533274168872044... �� 10 ^ -324 - // So 2.2250738585072012e-308 should round to 2^-1022 = 2.2250738585072014e-308 - TEST_DOUBLE(fullPrecision, "2.2250738585072012e-308", 2.2250738585072014e-308); // http://www.exploringbinary.com/java-hangs-when-converting-2-2250738585072012e-308/ - - // More closer to normal/subnormal boundary - // boundary = 2^-1022 - 2^-1075 = 2.225073858507201136057409796709131975934819546351645648... �� 10^-308 - TEST_DOUBLE(fullPrecision, "2.22507385850720113605740979670913197593481954635164564e-308", 2.2250738585072009e-308); - TEST_DOUBLE(fullPrecision, "2.22507385850720113605740979670913197593481954635164565e-308", 2.2250738585072014e-308); - - // 1.0 is in (1.0 - 2^-54, 1.0 + 2^-53) - // 1.0 - 2^-54 = 0.999999999999999944488848768742172978818416595458984375 - TEST_DOUBLE(fullPrecision, "0.999999999999999944488848768742172978818416595458984375", 1.0); // round to even - TEST_DOUBLE(fullPrecision, "0.999999999999999944488848768742172978818416595458984374", 0.99999999999999989); // previous double - TEST_DOUBLE(fullPrecision, "0.999999999999999944488848768742172978818416595458984376", 1.0); // next double - // 1.0 + 2^-53 = 1.00000000000000011102230246251565404236316680908203125 - TEST_DOUBLE(fullPrecision, "1.00000000000000011102230246251565404236316680908203125", 1.0); // round to even - TEST_DOUBLE(fullPrecision, "1.00000000000000011102230246251565404236316680908203124", 1.0); // previous double - TEST_DOUBLE(fullPrecision, "1.00000000000000011102230246251565404236316680908203126", 1.00000000000000022); // next double - - // Numbers from https://github.com/floitsch/double-conversion/blob/master/test/cctest/test-strtod.cc - - TEST_DOUBLE(fullPrecision, "72057594037927928.0", 72057594037927928.0); - TEST_DOUBLE(fullPrecision, "72057594037927936.0", 72057594037927936.0); - TEST_DOUBLE(fullPrecision, "72057594037927932.0", 72057594037927936.0); - TEST_DOUBLE(fullPrecision, "7205759403792793199999e-5", 72057594037927928.0); - TEST_DOUBLE(fullPrecision, "7205759403792793200001e-5", 72057594037927936.0); - - TEST_DOUBLE(fullPrecision, "9223372036854774784.0", 9223372036854774784.0); - TEST_DOUBLE(fullPrecision, "9223372036854775808.0", 9223372036854775808.0); - TEST_DOUBLE(fullPrecision, "9223372036854775296.0", 9223372036854775808.0); - TEST_DOUBLE(fullPrecision, "922337203685477529599999e-5", 9223372036854774784.0); - TEST_DOUBLE(fullPrecision, "922337203685477529600001e-5", 9223372036854775808.0); - - TEST_DOUBLE(fullPrecision, "10141204801825834086073718800384", 10141204801825834086073718800384.0); - TEST_DOUBLE(fullPrecision, "10141204801825835211973625643008", 10141204801825835211973625643008.0); - TEST_DOUBLE(fullPrecision, "10141204801825834649023672221696", 10141204801825835211973625643008.0); - TEST_DOUBLE(fullPrecision, "1014120480182583464902367222169599999e-5", 10141204801825834086073718800384.0); - TEST_DOUBLE(fullPrecision, "1014120480182583464902367222169600001e-5", 10141204801825835211973625643008.0); - - TEST_DOUBLE(fullPrecision, "5708990770823838890407843763683279797179383808", 5708990770823838890407843763683279797179383808.0); - TEST_DOUBLE(fullPrecision, "5708990770823839524233143877797980545530986496", 5708990770823839524233143877797980545530986496.0); - TEST_DOUBLE(fullPrecision, "5708990770823839207320493820740630171355185152", 5708990770823839524233143877797980545530986496.0); - TEST_DOUBLE(fullPrecision, "5708990770823839207320493820740630171355185151999e-3", 5708990770823838890407843763683279797179383808.0); - TEST_DOUBLE(fullPrecision, "5708990770823839207320493820740630171355185152001e-3", 5708990770823839524233143877797980545530986496.0); - - { - char n1e308[310]; // '1' followed by 308 '0' - n1e308[0] = '1'; - for (int i = 1; i < 309; i++) - n1e308[i] = '0'; - n1e308[309] = '\0'; - TEST_DOUBLE(fullPrecision, n1e308, 1E308); - } + TEST_DOUBLE(fullPrecision, "0.0", 0.0); + TEST_DOUBLE(fullPrecision, "-0.0", -0.0); // For checking issue #289 + TEST_DOUBLE(fullPrecision, "1.0", 1.0); + TEST_DOUBLE(fullPrecision, "-1.0", -1.0); + TEST_DOUBLE(fullPrecision, "1.5", 1.5); + TEST_DOUBLE(fullPrecision, "-1.5", -1.5); + TEST_DOUBLE(fullPrecision, "3.1416", 3.1416); + TEST_DOUBLE(fullPrecision, "1E10", 1E10); + TEST_DOUBLE(fullPrecision, "1e10", 1e10); + TEST_DOUBLE(fullPrecision, "1E+10", 1E+10); + TEST_DOUBLE(fullPrecision, "1E-10", 1E-10); + TEST_DOUBLE(fullPrecision, "-1E10", -1E10); + TEST_DOUBLE(fullPrecision, "-1e10", -1e10); + TEST_DOUBLE(fullPrecision, "-1E+10", -1E+10); + TEST_DOUBLE(fullPrecision, "-1E-10", -1E-10); + TEST_DOUBLE(fullPrecision, "1.234E+10", 1.234E+10); + TEST_DOUBLE(fullPrecision, "1.234E-10", 1.234E-10); + TEST_DOUBLE(fullPrecision, "1.79769e+308", 1.79769e+308); + TEST_DOUBLE(fullPrecision, "2.22507e-308", 2.22507e-308); + TEST_DOUBLE(fullPrecision, "-1.79769e+308", -1.79769e+308); + TEST_DOUBLE(fullPrecision, "-2.22507e-308", -2.22507e-308); + TEST_DOUBLE(fullPrecision, "4.9406564584124654e-324", 4.9406564584124654e-324); // minimum denormal + TEST_DOUBLE(fullPrecision, "2.2250738585072009e-308", 2.2250738585072009e-308); // Max subnormal double + TEST_DOUBLE(fullPrecision, "2.2250738585072014e-308", 2.2250738585072014e-308); // Min normal positive double + TEST_DOUBLE(fullPrecision, "1.7976931348623157e+308", 1.7976931348623157e+308); // Max double + TEST_DOUBLE(fullPrecision, "1e-10000", 0.0); // must underflow + TEST_DOUBLE(fullPrecision, "18446744073709551616", 18446744073709551616.0); // 2^64 (max of uint64_t + 1, force to use double) + TEST_DOUBLE(fullPrecision, "-9223372036854775809", -9223372036854775809.0); // -2^63 - 1(min of int64_t + 1, force to use double) + TEST_DOUBLE(fullPrecision, "0.9868011474609375", 0.9868011474609375); // https://github.com/miloyip/rapidjson/issues/120 + TEST_DOUBLE(fullPrecision, "123e34", 123e34); // Fast Path Cases In Disguise + TEST_DOUBLE(fullPrecision, "45913141877270640000.0", 45913141877270640000.0); + TEST_DOUBLE(fullPrecision, "2.2250738585072011e-308", 2.2250738585072011e-308); // http://www.exploringbinary.com/php-hangs-on-numeric-value-2-2250738585072011e-308/ + TEST_DOUBLE(fullPrecision, "1e-00011111111111", 0.0); // Issue #313 + TEST_DOUBLE(fullPrecision, "-1e-00011111111111", -0.0); + TEST_DOUBLE(fullPrecision, "1e-214748363", 0.0); // Maximum supported negative exponent + TEST_DOUBLE(fullPrecision, "1e-214748364", 0.0); + TEST_DOUBLE(fullPrecision, "1e-21474836311", 0.0); + TEST_DOUBLE(fullPrecision, "0.017976931348623157e+310", 1.7976931348623157e+308); // Max double in another form + + // Since + // abs((2^-1022 - 2^-1074) - 2.2250738585072012e-308) = 3.109754131239141401123495768877590405345064751974375599... �� 10^-324 + // abs((2^-1022) - 2.2250738585072012e-308) = 1.830902327173324040642192159804623318305533274168872044... �� 10 ^ -324 + // So 2.2250738585072012e-308 should round to 2^-1022 = 2.2250738585072014e-308 + TEST_DOUBLE(fullPrecision, "2.2250738585072012e-308", 2.2250738585072014e-308); // http://www.exploringbinary.com/java-hangs-when-converting-2-2250738585072012e-308/ + + // More closer to normal/subnormal boundary + // boundary = 2^-1022 - 2^-1075 = 2.225073858507201136057409796709131975934819546351645648... �� 10^-308 + TEST_DOUBLE(fullPrecision, "2.22507385850720113605740979670913197593481954635164564e-308", 2.2250738585072009e-308); + TEST_DOUBLE(fullPrecision, "2.22507385850720113605740979670913197593481954635164565e-308", 2.2250738585072014e-308); + + // 1.0 is in (1.0 - 2^-54, 1.0 + 2^-53) + // 1.0 - 2^-54 = 0.999999999999999944488848768742172978818416595458984375 + TEST_DOUBLE(fullPrecision, "0.999999999999999944488848768742172978818416595458984375", 1.0); // round to even + TEST_DOUBLE(fullPrecision, "0.999999999999999944488848768742172978818416595458984374", 0.99999999999999989); // previous double + TEST_DOUBLE(fullPrecision, "0.999999999999999944488848768742172978818416595458984376", 1.0); // next double + // 1.0 + 2^-53 = 1.00000000000000011102230246251565404236316680908203125 + TEST_DOUBLE(fullPrecision, "1.00000000000000011102230246251565404236316680908203125", 1.0); // round to even + TEST_DOUBLE(fullPrecision, "1.00000000000000011102230246251565404236316680908203124", 1.0); // previous double + TEST_DOUBLE(fullPrecision, "1.00000000000000011102230246251565404236316680908203126", 1.00000000000000022); // next double + + // Numbers from https://github.com/floitsch/double-conversion/blob/master/test/cctest/test-strtod.cc + + TEST_DOUBLE(fullPrecision, "72057594037927928.0", 72057594037927928.0); + TEST_DOUBLE(fullPrecision, "72057594037927936.0", 72057594037927936.0); + TEST_DOUBLE(fullPrecision, "72057594037927932.0", 72057594037927936.0); + TEST_DOUBLE(fullPrecision, "7205759403792793199999e-5", 72057594037927928.0); + TEST_DOUBLE(fullPrecision, "7205759403792793200001e-5", 72057594037927936.0); + + TEST_DOUBLE(fullPrecision, "9223372036854774784.0", 9223372036854774784.0); + TEST_DOUBLE(fullPrecision, "9223372036854775808.0", 9223372036854775808.0); + TEST_DOUBLE(fullPrecision, "9223372036854775296.0", 9223372036854775808.0); + TEST_DOUBLE(fullPrecision, "922337203685477529599999e-5", 9223372036854774784.0); + TEST_DOUBLE(fullPrecision, "922337203685477529600001e-5", 9223372036854775808.0); + + TEST_DOUBLE(fullPrecision, "10141204801825834086073718800384", 10141204801825834086073718800384.0); + TEST_DOUBLE(fullPrecision, "10141204801825835211973625643008", 10141204801825835211973625643008.0); + TEST_DOUBLE(fullPrecision, "10141204801825834649023672221696", 10141204801825835211973625643008.0); + TEST_DOUBLE(fullPrecision, "1014120480182583464902367222169599999e-5", 10141204801825834086073718800384.0); + TEST_DOUBLE(fullPrecision, "1014120480182583464902367222169600001e-5", 10141204801825835211973625643008.0); + + TEST_DOUBLE(fullPrecision, "5708990770823838890407843763683279797179383808", 5708990770823838890407843763683279797179383808.0); + TEST_DOUBLE(fullPrecision, "5708990770823839524233143877797980545530986496", 5708990770823839524233143877797980545530986496.0); + TEST_DOUBLE(fullPrecision, "5708990770823839207320493820740630171355185152", 5708990770823839524233143877797980545530986496.0); + TEST_DOUBLE(fullPrecision, "5708990770823839207320493820740630171355185151999e-3", 5708990770823838890407843763683279797179383808.0); + TEST_DOUBLE(fullPrecision, "5708990770823839207320493820740630171355185152001e-3", 5708990770823839524233143877797980545530986496.0); + + { char n1e308[310]; // '1' followed by 308 '0' + n1e308[0] = '1'; + for (int i = 1; i < 309; i++) + n1e308[i] = '0'; + n1e308[309] = '\0'; + TEST_DOUBLE(fullPrecision, n1e308, 1E308); + } + + // Cover trimming + TEST_DOUBLE(fullPrecision, + "2.22507385850720113605740979670913197593481954635164564802342610972482222202107694551652952390813508" + "7914149158913039621106870086438694594645527657207407820621743379988141063267329253552286881372149012" + "9811224514518898490572223072852551331557550159143974763979834118019993239625482890171070818506906306" + "6665599493827577257201576306269066333264756530000924588831643303777979186961204949739037782970490505" + "1080609940730262937128958950003583799967207254304360284078895771796150945516748243471030702609144621" + "5722898802581825451803257070188608721131280795122334262883686223215037756666225039825343359745688844" + "2390026549819838548794829220689472168983109969836584681402285424333066033985088644580400103493397042" + "7567186443383770486037861622771738545623065874679014086723327636718751234567890123456789012345678901" + "e-308", + 2.2250738585072014e-308); + + { static const unsigned count = 100; // Tested with 1000000 locally + Random r; + Reader reader; // Reusing reader to prevent heap allocation - // Cover trimming - TEST_DOUBLE(fullPrecision, -"2.22507385850720113605740979670913197593481954635164564802342610972482222202107694551652952390813508" -"7914149158913039621106870086438694594645527657207407820621743379988141063267329253552286881372149012" -"9811224514518898490572223072852551331557550159143974763979834118019993239625482890171070818506906306" -"6665599493827577257201576306269066333264756530000924588831643303777979186961204949739037782970490505" -"1080609940730262937128958950003583799967207254304360284078895771796150945516748243471030702609144621" -"5722898802581825451803257070188608721131280795122334262883686223215037756666225039825343359745688844" -"2390026549819838548794829220689472168983109969836584681402285424333066033985088644580400103493397042" -"7567186443383770486037861622771738545623065874679014086723327636718751234567890123456789012345678901" -"e-308", - 2.2250738585072014e-308); - - { - static const unsigned count = 100; // Tested with 1000000 locally - Random r; - Reader reader; // Reusing reader to prevent heap allocation - - // Exhaustively test different exponents with random significant - for (uint64_t exp = 0; exp < 2047; exp++) { - ; - for (unsigned i = 0; i < count; i++) { - // Need to call r() in two statements for cross-platform coherent sequence. - uint64_t u = (exp << 52) | uint64_t(r() & 0x000FFFFF) << 32; - u |= uint64_t(r()); - internal::Double d = internal::Double(u); - - char buffer[32]; - *internal::dtoa(d.Value(), buffer) = '\0'; - - StringStream s(buffer); - ParseDoubleHandler h; - ASSERT_EQ(kParseErrorNone, reader.Parse(s, h).Code()); - EXPECT_EQ(1u, h.step_); - internal::Double a(h.actual_); - if (fullPrecision) { - EXPECT_EQ(d.Uint64Value(), a.Uint64Value()); - if (d.Uint64Value() != a.Uint64Value()) - printf(" String: %s\n Actual: %.17g\nExpected: %.17g\n", buffer, h.actual_, d.Value()); - } - else { - EXPECT_EQ(d.Sign(), a.Sign()); // for 0.0 != -0.0 - EXPECT_DOUBLE_EQ(d.Value(), h.actual_); - } - } - } - } + // Exhaustively test different exponents with random significant + for (uint64_t exp = 0; exp < 2047; exp++) + { ; + for (unsigned i = 0; i < count; i++) + { // Need to call r() in two statements for cross-platform coherent sequence. + uint64_t u = (exp << 52) | uint64_t(r() & 0x000FFFFF) << 32; + u |= uint64_t(r()); + internal::Double d = internal::Double(u); - // Issue #340 - TEST_DOUBLE(fullPrecision, "7.450580596923828e-9", 7.450580596923828e-9); - { - internal::Double d(1.0); - for (int i = 0; i < 324; i++) { - char buffer[32]; - *internal::dtoa(d.Value(), buffer) = '\0'; - - StringStream s(buffer); - ParseDoubleHandler h; - Reader reader; - ASSERT_EQ(kParseErrorNone, reader.Parse(s, h).Code()); - EXPECT_EQ(1u, h.step_); - internal::Double a(h.actual_); - if (fullPrecision) { - EXPECT_EQ(d.Uint64Value(), a.Uint64Value()); - if (d.Uint64Value() != a.Uint64Value()) - printf(" String: %s\n Actual: %.17g\nExpected: %.17g\n", buffer, h.actual_, d.Value()); - } - else { - EXPECT_EQ(d.Sign(), a.Sign()); // for 0.0 != -0.0 - EXPECT_DOUBLE_EQ(d.Value(), h.actual_); - } - - - d = d.Value() * 0.5; + char buffer[32]; + *internal::dtoa(d.Value(), buffer) = '\0'; + + StringStream s(buffer); + ParseDoubleHandler h; + ASSERT_EQ(kParseErrorNone, reader.Parse(s, h).Code()); + EXPECT_EQ(1u, h.step_); + internal::Double a(h.actual_); + if (fullPrecision) + { EXPECT_EQ(d.Uint64Value(), a.Uint64Value()); + if (d.Uint64Value() != a.Uint64Value()) + printf(" String: %s\n Actual: %.17g\nExpected: %.17g\n", buffer, h.actual_, d.Value()); + } + else + { EXPECT_EQ(d.Sign(), a.Sign()); // for 0.0 != -0.0 + EXPECT_DOUBLE_EQ(d.Value(), h.actual_); } + } } + } + + // Issue #340 + TEST_DOUBLE(fullPrecision, "7.450580596923828e-9", 7.450580596923828e-9); + { internal::Double d(1.0); + for (int i = 0; i < 324; i++) + { char buffer[32]; + *internal::dtoa(d.Value(), buffer) = '\0'; + + StringStream s(buffer); + ParseDoubleHandler h; + Reader reader; + ASSERT_EQ(kParseErrorNone, reader.Parse(s, h).Code()); + EXPECT_EQ(1u, h.step_); + internal::Double a(h.actual_); + if (fullPrecision) + { EXPECT_EQ(d.Uint64Value(), a.Uint64Value()); + if (d.Uint64Value() != a.Uint64Value()) + printf(" String: %s\n Actual: %.17g\nExpected: %.17g\n", buffer, h.actual_, d.Value()); + } + else + { EXPECT_EQ(d.Sign(), a.Sign()); // for 0.0 != -0.0 + EXPECT_DOUBLE_EQ(d.Value(), h.actual_); + } + + + d = d.Value() * 0.5; + } + } #undef TEST_DOUBLE } -TEST(Reader, ParseNumber_NormalPrecisionDouble) { - TestParseDouble(); +TEST(Reader, ParseNumber_NormalPrecisionDouble) +{ TestParseDouble(); } -TEST(Reader, ParseNumber_FullPrecisionDouble) { - TestParseDouble(); +TEST(Reader, ParseNumber_FullPrecisionDouble) +{ TestParseDouble(); } -TEST(Reader, ParseNumber_NormalPrecisionError) { - static unsigned count = 1000000; - Random r; +TEST(Reader, ParseNumber_NormalPrecisionError) +{ static unsigned count = 1000000; + Random r; - double ulpSum = 0.0; - double ulpMax = 0.0; - for (unsigned i = 0; i < count; i++) { - internal::Double e, a; - do { - // Need to call r() in two statements for cross-platform coherent sequence. - uint64_t u = uint64_t(r()) << 32; - u |= uint64_t(r()); - e = u; - } while (e.IsNan() || e.IsInf() || !e.IsNormal()); + double ulpSum = 0.0; + double ulpMax = 0.0; + for (unsigned i = 0; i < count; i++) + { internal::Double e, a; + do + { // Need to call r() in two statements for cross-platform coherent sequence. + uint64_t u = uint64_t(r()) << 32; + u |= uint64_t(r()); + e = u; + } + while (e.IsNan() || e.IsInf() || !e.IsNormal()); - char buffer[32]; - *internal::dtoa(e.Value(), buffer) = '\0'; + char buffer[32]; + *internal::dtoa(e.Value(), buffer) = '\0'; - StringStream s(buffer); - ParseDoubleHandler h; - Reader reader; - ASSERT_EQ(kParseErrorNone, reader.Parse(s, h).Code()); - EXPECT_EQ(1u, h.step_); + StringStream s(buffer); + ParseDoubleHandler h; + Reader reader; + ASSERT_EQ(kParseErrorNone, reader.Parse(s, h).Code()); + EXPECT_EQ(1u, h.step_); - a = h.actual_; - uint64_t bias1 = e.ToBias(); - uint64_t bias2 = a.ToBias(); - double ulp = static_cast(bias1 >= bias2 ? bias1 - bias2 : bias2 - bias1); - ulpMax = std::max(ulpMax, ulp); - ulpSum += ulp; - } - printf("ULP Average = %g, Max = %g \n", ulpSum / count, ulpMax); + a = h.actual_; + uint64_t bias1 = e.ToBias(); + uint64_t bias2 = a.ToBias(); + double ulp = static_cast(bias1 >= bias2 ? bias1 - bias2 : bias2 - bias1); + ulpMax = std::max(ulpMax, ulp); + ulpSum += ulp; + } + printf("ULP Average = %g, Max = %g \n", ulpSum / count, ulpMax); } -TEST(Reader, ParseNumber_Error) { +TEST(Reader, ParseNumber_Error) +{ #define TEST_NUMBER_ERROR(errorCode, str, errorOffset, streamPos) \ { \ char buffer[1001]; \ @@ -435,56 +434,56 @@ TEST(Reader, ParseNumber_Error) { EXPECT_EQ(streamPos, s.Tell());\ } - // Number too big to be stored in double. - { - char n1e309[311]; // '1' followed by 309 '0' - n1e309[0] = '1'; - for (int i = 1; i < 310; i++) - n1e309[i] = '0'; - n1e309[310] = '\0'; - TEST_NUMBER_ERROR(kParseErrorNumberTooBig, n1e309, 0, 309); - } - TEST_NUMBER_ERROR(kParseErrorNumberTooBig, "1e309", 0, 5); + // Number too big to be stored in double. + { char n1e309[311]; // '1' followed by 309 '0' + n1e309[0] = '1'; + for (int i = 1; i < 310; i++) + n1e309[i] = '0'; + n1e309[310] = '\0'; + TEST_NUMBER_ERROR(kParseErrorNumberTooBig, n1e309, 0, 309); + } + TEST_NUMBER_ERROR(kParseErrorNumberTooBig, "1e309", 0, 5); - // Miss fraction part in number. - TEST_NUMBER_ERROR(kParseErrorNumberMissFraction, "1.", 2, 2); - TEST_NUMBER_ERROR(kParseErrorNumberMissFraction, "1.a", 2, 2); + // Miss fraction part in number. + TEST_NUMBER_ERROR(kParseErrorNumberMissFraction, "1.", 2, 2); + TEST_NUMBER_ERROR(kParseErrorNumberMissFraction, "1.a", 2, 2); - // Miss exponent in number. - TEST_NUMBER_ERROR(kParseErrorNumberMissExponent, "1e", 2, 2); - TEST_NUMBER_ERROR(kParseErrorNumberMissExponent, "1e_", 2, 2); + // Miss exponent in number. + TEST_NUMBER_ERROR(kParseErrorNumberMissExponent, "1e", 2, 2); + TEST_NUMBER_ERROR(kParseErrorNumberMissExponent, "1e_", 2, 2); #undef TEST_NUMBER_ERROR } template -struct ParseStringHandler : BaseReaderHandler > { - ParseStringHandler() : str_(0), length_(0), copy_() {} - ~ParseStringHandler() { EXPECT_TRUE(str_ != 0); if (copy_) free(const_cast(str_)); } - - ParseStringHandler(const ParseStringHandler&); - ParseStringHandler& operator=(const ParseStringHandler&); - - bool Default() { ADD_FAILURE(); return false; } - bool String(const typename Encoding::Ch* str, size_t length, bool copy) { - EXPECT_EQ(0, str_); - if (copy) { - str_ = static_cast(malloc((length + 1) * sizeof(typename Encoding::Ch))); - memcpy(const_cast(str_), str, (length + 1) * sizeof(typename Encoding::Ch)); - } - else - str_ = str; - length_ = length; - copy_ = copy; - return true; +struct ParseStringHandler : BaseReaderHandler > +{ ParseStringHandler() : str_(0), length_(0), copy_() {} + ~ParseStringHandler() { EXPECT_TRUE(str_ != 0); if (copy_) free(const_cast(str_)); } + + ParseStringHandler(const ParseStringHandler&); + ParseStringHandler& operator=(const ParseStringHandler&); + + bool Default() { ADD_FAILURE(); return false; } + bool String(const typename Encoding::Ch* str, size_t length, bool copy) + { EXPECT_EQ(0, str_); + if (copy) + { str_ = static_cast(malloc((length + 1) * sizeof(typename Encoding::Ch))); + memcpy(const_cast(str_), str, (length + 1) * sizeof(typename Encoding::Ch)); } - - const typename Encoding::Ch* str_; - size_t length_; - bool copy_; + else + str_ = str; + length_ = length; + copy_ = copy; + return true; + } + + const typename Encoding::Ch* str_; + size_t length_; + bool copy_; }; -TEST(Reader, ParseString) { +TEST(Reader, ParseString) +{ #define TEST_STRING(Encoding, e, x) \ { \ Encoding::Ch* buffer = StrDup(x); \ @@ -503,10 +502,10 @@ TEST(Reader, ParseString) { EXPECT_EQ(StrLen(e), h2.length_); \ } - // String constant L"\xXX" can only specify character code in bytes, which is not endianness-neutral. - // And old compiler does not support u"" and U"" string literal. So here specify string literal by array of Ch. - // In addition, GCC 4.8 generates -Wnarrowing warnings when character code >= 128 are assigned to signed integer types. - // Therefore, utype is added for declaring unsigned array, and then cast it to Encoding::Ch. + // String constant L"\xXX" can only specify character code in bytes, which is not endianness-neutral. + // And old compiler does not support u"" and U"" string literal. So here specify string literal by array of Ch. + // In addition, GCC 4.8 generates -Wnarrowing warnings when character code >= 128 are assigned to signed integer types. + // Therefore, utype is added for declaring unsigned array, and then cast it to Encoding::Ch. #define ARRAY(...) { __VA_ARGS__ } #define TEST_STRINGARRAY(Encoding, utype, array, x) \ { \ @@ -524,92 +523,92 @@ TEST(Reader, ParseString) { TEST_STRING(Encoding, e, x); \ } - TEST_STRING(UTF8<>, "", "\"\""); - TEST_STRING(UTF8<>, "Hello", "\"Hello\""); - TEST_STRING(UTF8<>, "Hello\nWorld", "\"Hello\\nWorld\""); - TEST_STRING(UTF8<>, "\"\\/\b\f\n\r\t", "\"\\\"\\\\/\\b\\f\\n\\r\\t\""); - TEST_STRING(UTF8<>, "\x24", "\"\\u0024\""); // Dollar sign U+0024 - TEST_STRING(UTF8<>, "\xC2\xA2", "\"\\u00A2\""); // Cents sign U+00A2 - TEST_STRING(UTF8<>, "\xE2\x82\xAC", "\"\\u20AC\""); // Euro sign U+20AC - TEST_STRING(UTF8<>, "\xF0\x9D\x84\x9E", "\"\\uD834\\uDD1E\""); // G clef sign U+1D11E - - // UTF16 - TEST_STRING(UTF16<>, L"", L"\"\""); - TEST_STRING(UTF16<>, L"Hello", L"\"Hello\""); - TEST_STRING(UTF16<>, L"Hello\nWorld", L"\"Hello\\nWorld\""); - TEST_STRING(UTF16<>, L"\"\\/\b\f\n\r\t", L"\"\\\"\\\\/\\b\\f\\n\\r\\t\""); - TEST_STRINGARRAY(UTF16<>, wchar_t, ARRAY(0x0024, 0x0000), L"\"\\u0024\""); - TEST_STRINGARRAY(UTF16<>, wchar_t, ARRAY(0x00A2, 0x0000), L"\"\\u00A2\""); // Cents sign U+00A2 - TEST_STRINGARRAY(UTF16<>, wchar_t, ARRAY(0x20AC, 0x0000), L"\"\\u20AC\""); // Euro sign U+20AC - TEST_STRINGARRAY(UTF16<>, wchar_t, ARRAY(0xD834, 0xDD1E, 0x0000), L"\"\\uD834\\uDD1E\""); // G clef sign U+1D11E - - // UTF32 - TEST_STRINGARRAY2(UTF32<>, unsigned, ARRAY('\0'), ARRAY('\"', '\"', '\0')); - TEST_STRINGARRAY2(UTF32<>, unsigned, ARRAY('H', 'e', 'l', 'l', 'o', '\0'), ARRAY('\"', 'H', 'e', 'l', 'l', 'o', '\"', '\0')); - TEST_STRINGARRAY2(UTF32<>, unsigned, ARRAY('H', 'e', 'l', 'l', 'o', '\n', 'W', 'o', 'r', 'l', 'd', '\0'), ARRAY('\"', 'H', 'e', 'l', 'l', 'o', '\\', 'n', 'W', 'o', 'r', 'l', 'd', '\"', '\0')); - TEST_STRINGARRAY2(UTF32<>, unsigned, ARRAY('\"', '\\', '/', '\b', '\f', '\n', '\r', '\t', '\0'), ARRAY('\"', '\\', '\"', '\\', '\\', '/', '\\', 'b', '\\', 'f', '\\', 'n', '\\', 'r', '\\', 't', '\"', '\0')); - TEST_STRINGARRAY2(UTF32<>, unsigned, ARRAY(0x00024, 0x0000), ARRAY('\"', '\\', 'u', '0', '0', '2', '4', '\"', '\0')); - TEST_STRINGARRAY2(UTF32<>, unsigned, ARRAY(0x000A2, 0x0000), ARRAY('\"', '\\', 'u', '0', '0', 'A', '2', '\"', '\0')); // Cents sign U+00A2 - TEST_STRINGARRAY2(UTF32<>, unsigned, ARRAY(0x020AC, 0x0000), ARRAY('\"', '\\', 'u', '2', '0', 'A', 'C', '\"', '\0')); // Euro sign U+20AC - TEST_STRINGARRAY2(UTF32<>, unsigned, ARRAY(0x1D11E, 0x0000), ARRAY('\"', '\\', 'u', 'D', '8', '3', '4', '\\', 'u', 'D', 'D', '1', 'E', '\"', '\0')); // G clef sign U+1D11E + TEST_STRING(UTF8<>, "", "\"\""); + TEST_STRING(UTF8<>, "Hello", "\"Hello\""); + TEST_STRING(UTF8<>, "Hello\nWorld", "\"Hello\\nWorld\""); + TEST_STRING(UTF8<>, "\"\\/\b\f\n\r\t", "\"\\\"\\\\/\\b\\f\\n\\r\\t\""); + TEST_STRING(UTF8<>, "\x24", "\"\\u0024\""); // Dollar sign U+0024 + TEST_STRING(UTF8<>, "\xC2\xA2", "\"\\u00A2\""); // Cents sign U+00A2 + TEST_STRING(UTF8<>, "\xE2\x82\xAC", "\"\\u20AC\""); // Euro sign U+20AC + TEST_STRING(UTF8<>, "\xF0\x9D\x84\x9E", "\"\\uD834\\uDD1E\""); // G clef sign U+1D11E + + // UTF16 + TEST_STRING(UTF16<>, L"", L"\"\""); + TEST_STRING(UTF16<>, L"Hello", L"\"Hello\""); + TEST_STRING(UTF16<>, L"Hello\nWorld", L"\"Hello\\nWorld\""); + TEST_STRING(UTF16<>, L"\"\\/\b\f\n\r\t", L"\"\\\"\\\\/\\b\\f\\n\\r\\t\""); + TEST_STRINGARRAY(UTF16<>, wchar_t, ARRAY(0x0024, 0x0000), L"\"\\u0024\""); + TEST_STRINGARRAY(UTF16<>, wchar_t, ARRAY(0x00A2, 0x0000), L"\"\\u00A2\""); // Cents sign U+00A2 + TEST_STRINGARRAY(UTF16<>, wchar_t, ARRAY(0x20AC, 0x0000), L"\"\\u20AC\""); // Euro sign U+20AC + TEST_STRINGARRAY(UTF16<>, wchar_t, ARRAY(0xD834, 0xDD1E, 0x0000), L"\"\\uD834\\uDD1E\""); // G clef sign U+1D11E + + // UTF32 + TEST_STRINGARRAY2(UTF32<>, unsigned, ARRAY('\0'), ARRAY('\"', '\"', '\0')); + TEST_STRINGARRAY2(UTF32<>, unsigned, ARRAY('H', 'e', 'l', 'l', 'o', '\0'), ARRAY('\"', 'H', 'e', 'l', 'l', 'o', '\"', '\0')); + TEST_STRINGARRAY2(UTF32<>, unsigned, ARRAY('H', 'e', 'l', 'l', 'o', '\n', 'W', 'o', 'r', 'l', 'd', '\0'), ARRAY('\"', 'H', 'e', 'l', 'l', 'o', '\\', 'n', 'W', 'o', 'r', 'l', 'd', '\"', '\0')); + TEST_STRINGARRAY2(UTF32<>, unsigned, ARRAY('\"', '\\', '/', '\b', '\f', '\n', '\r', '\t', '\0'), ARRAY('\"', '\\', '\"', '\\', '\\', '/', '\\', 'b', '\\', 'f', '\\', 'n', '\\', 'r', '\\', 't', '\"', '\0')); + TEST_STRINGARRAY2(UTF32<>, unsigned, ARRAY(0x00024, 0x0000), ARRAY('\"', '\\', 'u', '0', '0', '2', '4', '\"', '\0')); + TEST_STRINGARRAY2(UTF32<>, unsigned, ARRAY(0x000A2, 0x0000), ARRAY('\"', '\\', 'u', '0', '0', 'A', '2', '\"', '\0')); // Cents sign U+00A2 + TEST_STRINGARRAY2(UTF32<>, unsigned, ARRAY(0x020AC, 0x0000), ARRAY('\"', '\\', 'u', '2', '0', 'A', 'C', '\"', '\0')); // Euro sign U+20AC + TEST_STRINGARRAY2(UTF32<>, unsigned, ARRAY(0x1D11E, 0x0000), ARRAY('\"', '\\', 'u', 'D', '8', '3', '4', '\\', 'u', 'D', 'D', '1', 'E', '\"', '\0')); // G clef sign U+1D11E #undef TEST_STRINGARRAY #undef ARRAY #undef TEST_STRING - // Support of null character in string - { - StringStream s("\"Hello\\u0000World\""); - const char e[] = "Hello\0World"; - ParseStringHandler > h; - Reader reader; - reader.Parse(s, h); - EXPECT_EQ(0, memcmp(e, h.str_, h.length_ + 1)); - EXPECT_EQ(11u, h.length_); - } + // Support of null character in string + { StringStream s("\"Hello\\u0000World\""); + const char e[] = "Hello\0World"; + ParseStringHandler > h; + Reader reader; + reader.Parse(s, h); + EXPECT_EQ(0, memcmp(e, h.str_, h.length_ + 1)); + EXPECT_EQ(11u, h.length_); + } } -TEST(Reader, ParseString_Transcoding) { - const char* x = "\"Hello\""; - const wchar_t* e = L"Hello"; - GenericStringStream > is(x); - GenericReader, UTF16<> > reader; - ParseStringHandler > h; - reader.Parse(is, h); - EXPECT_EQ(0, StrCmp::Ch>(e, h.str_)); - EXPECT_EQ(StrLen(e), h.length_); +TEST(Reader, ParseString_Transcoding) +{ const char* x = "\"Hello\""; + const wchar_t* e = L"Hello"; + GenericStringStream > is(x); + GenericReader, UTF16<> > reader; + ParseStringHandler > h; + reader.Parse(is, h); + EXPECT_EQ(0, StrCmp::Ch>(e, h.str_)); + EXPECT_EQ(StrLen(e), h.length_); } -TEST(Reader, ParseString_TranscodingWithValidation) { - const char* x = "\"Hello\""; - const wchar_t* e = L"Hello"; - GenericStringStream > is(x); - GenericReader, UTF16<> > reader; - ParseStringHandler > h; - reader.Parse(is, h); - EXPECT_EQ(0, StrCmp::Ch>(e, h.str_)); - EXPECT_EQ(StrLen(e), h.length_); +TEST(Reader, ParseString_TranscodingWithValidation) +{ const char* x = "\"Hello\""; + const wchar_t* e = L"Hello"; + GenericStringStream > is(x); + GenericReader, UTF16<> > reader; + ParseStringHandler > h; + reader.Parse(is, h); + EXPECT_EQ(0, StrCmp::Ch>(e, h.str_)); + EXPECT_EQ(StrLen(e), h.length_); } -TEST(Reader, ParseString_NonDestructive) { - StringStream s("\"Hello\\nWorld\""); - ParseStringHandler > h; - Reader reader; - reader.Parse(s, h); - EXPECT_EQ(0, StrCmp("Hello\nWorld", h.str_)); - EXPECT_EQ(11u, h.length_); +TEST(Reader, ParseString_NonDestructive) +{ StringStream s("\"Hello\\nWorld\""); + ParseStringHandler > h; + Reader reader; + reader.Parse(s, h); + EXPECT_EQ(0, StrCmp("Hello\nWorld", h.str_)); + EXPECT_EQ(11u, h.length_); } template -ParseErrorCode TestString(const typename Encoding::Ch* str) { - GenericStringStream s(str); - BaseReaderHandler h; - GenericReader reader; - reader.template Parse(s, h); - return reader.GetParseErrorCode(); +ParseErrorCode TestString(const typename Encoding::Ch* str) +{ GenericStringStream s(str); + BaseReaderHandler h; + GenericReader reader; + reader.template Parse(s, h); + return reader.GetParseErrorCode(); } -TEST(Reader, ParseString_Error) { +TEST(Reader, ParseString_Error) +{ #define TEST_STRING_ERROR(errorCode, str, errorOffset, streamPos)\ {\ GenericStringStream > s(str);\ @@ -635,134 +634,133 @@ TEST(Reader, ParseString_Error) { EXPECT_EQ(kParseErrorStringInvalidEncoding, reader.GetParseErrorCode());\ } - // Invalid escape character in string. - TEST_STRING_ERROR(kParseErrorStringEscapeInvalid, "[\"\\a\"]", 2, 3); + // Invalid escape character in string. + TEST_STRING_ERROR(kParseErrorStringEscapeInvalid, "[\"\\a\"]", 2, 3); - // Incorrect hex digit after \\u escape in string. - TEST_STRING_ERROR(kParseErrorStringUnicodeEscapeInvalidHex, "[\"\\uABCG\"]", 2, 7); + // Incorrect hex digit after \\u escape in string. + TEST_STRING_ERROR(kParseErrorStringUnicodeEscapeInvalidHex, "[\"\\uABCG\"]", 2, 7); - // Quotation in \\u escape in string (Issue #288) - TEST_STRING_ERROR(kParseErrorStringUnicodeEscapeInvalidHex, "[\"\\uaaa\"]", 2, 7); - TEST_STRING_ERROR(kParseErrorStringUnicodeEscapeInvalidHex, "[\"\\uD800\\uFFF\"]", 2, 13); + // Quotation in \\u escape in string (Issue #288) + TEST_STRING_ERROR(kParseErrorStringUnicodeEscapeInvalidHex, "[\"\\uaaa\"]", 2, 7); + TEST_STRING_ERROR(kParseErrorStringUnicodeEscapeInvalidHex, "[\"\\uD800\\uFFF\"]", 2, 13); - // The surrogate pair in string is invalid. - TEST_STRING_ERROR(kParseErrorStringUnicodeSurrogateInvalid, "[\"\\uD800X\"]", 2, 8); - TEST_STRING_ERROR(kParseErrorStringUnicodeSurrogateInvalid, "[\"\\uD800\\uFFFF\"]", 2, 14); + // The surrogate pair in string is invalid. + TEST_STRING_ERROR(kParseErrorStringUnicodeSurrogateInvalid, "[\"\\uD800X\"]", 2, 8); + TEST_STRING_ERROR(kParseErrorStringUnicodeSurrogateInvalid, "[\"\\uD800\\uFFFF\"]", 2, 14); - // Missing a closing quotation mark in string. - TEST_STRING_ERROR(kParseErrorStringMissQuotationMark, "[\"Test]", 7, 7); + // Missing a closing quotation mark in string. + TEST_STRING_ERROR(kParseErrorStringMissQuotationMark, "[\"Test]", 7, 7); - // http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt + // http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt - // 3 Malformed sequences + // 3 Malformed sequences - // 3.1 Unexpected continuation bytes - { - char e[] = { '[', '\"', 0, '\"', ']', '\0' }; - for (unsigned char c = 0x80u; c <= 0xBFu; c++) { - e[2] = static_cast(c); - ParseErrorCode error = TestString >(e); - EXPECT_EQ(kParseErrorStringInvalidEncoding, error); - if (error != kParseErrorStringInvalidEncoding) - std::cout << static_cast(c) << std::endl; - } + // 3.1 Unexpected continuation bytes + { char e[] = { '[', '\"', 0, '\"', ']', '\0' }; + for (unsigned char c = 0x80u; c <= 0xBFu; c++) + { e[2] = static_cast(c); + ParseErrorCode error = TestString >(e); + EXPECT_EQ(kParseErrorStringInvalidEncoding, error); + if (error != kParseErrorStringInvalidEncoding) + std::cout << static_cast(c) << std::endl; } - - // 3.2 Lonely start characters, 3.5 Impossible bytes - { - char e[] = { '[', '\"', 0, ' ', '\"', ']', '\0' }; - for (unsigned c = 0xC0u; c <= 0xFFu; c++) { - e[2] = static_cast(c); - int streamPos; - if (c <= 0xC1u) - streamPos = 3; // 0xC0 - 0xC1 - else if (c <= 0xDFu) - streamPos = 4; // 0xC2 - 0xDF - else if (c <= 0xEFu) - streamPos = 5; // 0xE0 - 0xEF - else if (c <= 0xF4u) - streamPos = 6; // 0xF0 - 0xF4 - else - streamPos = 3; // 0xF5 - 0xFF - TEST_STRING_ERROR(kParseErrorStringInvalidEncoding, e, 2, streamPos); - } + } + + // 3.2 Lonely start characters, 3.5 Impossible bytes + { char e[] = { '[', '\"', 0, ' ', '\"', ']', '\0' }; + for (unsigned c = 0xC0u; c <= 0xFFu; c++) + { e[2] = static_cast(c); + int streamPos; + if (c <= 0xC1u) + streamPos = 3; // 0xC0 - 0xC1 + else if (c <= 0xDFu) + streamPos = 4; // 0xC2 - 0xDF + else if (c <= 0xEFu) + streamPos = 5; // 0xE0 - 0xEF + else if (c <= 0xF4u) + streamPos = 6; // 0xF0 - 0xF4 + else + streamPos = 3; // 0xF5 - 0xFF + TEST_STRING_ERROR(kParseErrorStringInvalidEncoding, e, 2, streamPos); } + } - // 4 Overlong sequences + // 4 Overlong sequences - // 4.1 Examples of an overlong ASCII character - TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xC0u, 0xAFu, '\"', ']', '\0')); - TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xE0u, 0x80u, 0xAFu, '\"', ']', '\0')); - TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xF0u, 0x80u, 0x80u, 0xAFu, '\"', ']', '\0')); + // 4.1 Examples of an overlong ASCII character + TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xC0u, 0xAFu, '\"', ']', '\0')); + TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xE0u, 0x80u, 0xAFu, '\"', ']', '\0')); + TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xF0u, 0x80u, 0x80u, 0xAFu, '\"', ']', '\0')); - // 4.2 Maximum overlong sequences - TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xC1u, 0xBFu, '\"', ']', '\0')); - TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xE0u, 0x9Fu, 0xBFu, '\"', ']', '\0')); - TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xF0u, 0x8Fu, 0xBFu, 0xBFu, '\"', ']', '\0')); + // 4.2 Maximum overlong sequences + TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xC1u, 0xBFu, '\"', ']', '\0')); + TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xE0u, 0x9Fu, 0xBFu, '\"', ']', '\0')); + TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xF0u, 0x8Fu, 0xBFu, 0xBFu, '\"', ']', '\0')); - // 4.3 Overlong representation of the NUL character - TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xC0u, 0x80u, '\"', ']', '\0')); - TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xE0u, 0x80u, 0x80u, '\"', ']', '\0')); - TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xF0u, 0x80u, 0x80u, 0x80u, '\"', ']', '\0')); + // 4.3 Overlong representation of the NUL character + TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xC0u, 0x80u, '\"', ']', '\0')); + TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xE0u, 0x80u, 0x80u, '\"', ']', '\0')); + TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xF0u, 0x80u, 0x80u, 0x80u, '\"', ']', '\0')); - // 5 Illegal code positions + // 5 Illegal code positions - // 5.1 Single UTF-16 surrogates - TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xEDu, 0xA0u, 0x80u, '\"', ']', '\0')); - TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xEDu, 0xADu, 0xBFu, '\"', ']', '\0')); - TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xEDu, 0xAEu, 0x80u, '\"', ']', '\0')); - TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xEDu, 0xAFu, 0xBFu, '\"', ']', '\0')); - TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xEDu, 0xB0u, 0x80u, '\"', ']', '\0')); - TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xEDu, 0xBEu, 0x80u, '\"', ']', '\0')); - TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xEDu, 0xBFu, 0xBFu, '\"', ']', '\0')); + // 5.1 Single UTF-16 surrogates + TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xEDu, 0xA0u, 0x80u, '\"', ']', '\0')); + TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xEDu, 0xADu, 0xBFu, '\"', ']', '\0')); + TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xEDu, 0xAEu, 0x80u, '\"', ']', '\0')); + TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xEDu, 0xAFu, 0xBFu, '\"', ']', '\0')); + TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xEDu, 0xB0u, 0x80u, '\"', ']', '\0')); + TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xEDu, 0xBEu, 0x80u, '\"', ']', '\0')); + TEST_STRINGENCODING_ERROR(UTF8<>, UTF16<>, unsigned char, ARRAY('[', '\"', 0xEDu, 0xBFu, 0xBFu, '\"', ']', '\0')); - // Malform UTF-16 sequences - TEST_STRINGENCODING_ERROR(UTF16<>, UTF8<>, wchar_t, ARRAY('[', '\"', 0xDC00, 0xDC00, '\"', ']', '\0')); - TEST_STRINGENCODING_ERROR(UTF16<>, UTF8<>, wchar_t, ARRAY('[', '\"', 0xD800, 0xD800, '\"', ']', '\0')); + // Malform UTF-16 sequences + TEST_STRINGENCODING_ERROR(UTF16<>, UTF8<>, wchar_t, ARRAY('[', '\"', 0xDC00, 0xDC00, '\"', ']', '\0')); + TEST_STRINGENCODING_ERROR(UTF16<>, UTF8<>, wchar_t, ARRAY('[', '\"', 0xD800, 0xD800, '\"', ']', '\0')); - // Malform UTF-32 sequence - TEST_STRINGENCODING_ERROR(UTF32<>, UTF8<>, unsigned, ARRAY('[', '\"', 0x110000, '\"', ']', '\0')); + // Malform UTF-32 sequence + TEST_STRINGENCODING_ERROR(UTF32<>, UTF8<>, unsigned, ARRAY('[', '\"', 0x110000, '\"', ']', '\0')); - // Malform ASCII sequence - TEST_STRINGENCODING_ERROR(ASCII<>, UTF8<>, char, ARRAY('[', '\"', char(0x80u), '\"', ']', '\0')); + // Malform ASCII sequence + TEST_STRINGENCODING_ERROR(ASCII<>, UTF8<>, char, ARRAY('[', '\"', char(0x80u), '\"', ']', '\0')); #undef ARRAY #undef TEST_STRINGARRAY_ERROR } template -struct ParseArrayHandler : BaseReaderHandler, ParseArrayHandler > { - ParseArrayHandler() : step_(0) {} +struct ParseArrayHandler : BaseReaderHandler, ParseArrayHandler > +{ ParseArrayHandler() : step_(0) {} - bool Default() { ADD_FAILURE(); return false; } - bool Uint(unsigned i) { EXPECT_EQ(step_, i); step_++; return true; } - bool StartArray() { EXPECT_EQ(0u, step_); step_++; return true; } - bool EndArray(SizeType) { step_++; return true; } + bool Default() { ADD_FAILURE(); return false; } + bool Uint(unsigned i) { EXPECT_EQ(step_, i); step_++; return true; } + bool StartArray() { EXPECT_EQ(0u, step_); step_++; return true; } + bool EndArray(SizeType) { step_++; return true; } - unsigned step_; + unsigned step_; }; -TEST(Reader, ParseEmptyArray) { - char *json = StrDup("[ ] "); - InsituStringStream s(json); - ParseArrayHandler<0> h; - Reader reader; - reader.Parse(s, h); - EXPECT_EQ(2u, h.step_); - free(json); +TEST(Reader, ParseEmptyArray) +{ char *json = StrDup("[ ] "); + InsituStringStream s(json); + ParseArrayHandler<0> h; + Reader reader; + reader.Parse(s, h); + EXPECT_EQ(2u, h.step_); + free(json); } -TEST(Reader, ParseArray) { - char *json = StrDup("[1, 2, 3, 4]"); - InsituStringStream s(json); - ParseArrayHandler<4> h; - Reader reader; - reader.Parse(s, h); - EXPECT_EQ(6u, h.step_); - free(json); +TEST(Reader, ParseArray) +{ char *json = StrDup("[1, 2, 3, 4]"); + InsituStringStream s(json); + ParseArrayHandler<4> h; + Reader reader; + reader.Parse(s, h); + EXPECT_EQ(6u, h.step_); + free(json); } -TEST(Reader, ParseArray_Error) { +TEST(Reader, ParseArray_Error) +{ #define TEST_ARRAY_ERROR(errorCode, str, errorOffset) \ { \ int streamPos = errorOffset; \ @@ -777,158 +775,156 @@ TEST(Reader, ParseArray_Error) { EXPECT_EQ(streamPos, s.Tell());\ } - // Missing a comma or ']' after an array element. - TEST_ARRAY_ERROR(kParseErrorArrayMissCommaOrSquareBracket, "[1", 2); - TEST_ARRAY_ERROR(kParseErrorArrayMissCommaOrSquareBracket, "[1}", 2); - TEST_ARRAY_ERROR(kParseErrorArrayMissCommaOrSquareBracket, "[1 2]", 3); + // Missing a comma or ']' after an array element. + TEST_ARRAY_ERROR(kParseErrorArrayMissCommaOrSquareBracket, "[1", 2); + TEST_ARRAY_ERROR(kParseErrorArrayMissCommaOrSquareBracket, "[1}", 2); + TEST_ARRAY_ERROR(kParseErrorArrayMissCommaOrSquareBracket, "[1 2]", 3); - // Array cannot have a trailing comma (without kParseTrailingCommasFlag); - // a value must follow a comma - TEST_ARRAY_ERROR(kParseErrorValueInvalid, "[1,]", 3); + // Array cannot have a trailing comma (without kParseTrailingCommasFlag); + // a value must follow a comma + TEST_ARRAY_ERROR(kParseErrorValueInvalid, "[1,]", 3); #undef TEST_ARRAY_ERROR } -struct ParseObjectHandler : BaseReaderHandler, ParseObjectHandler> { - ParseObjectHandler() : step_(0) {} +struct ParseObjectHandler : BaseReaderHandler, ParseObjectHandler> +{ ParseObjectHandler() : step_(0) {} - bool Default() { ADD_FAILURE(); return false; } - bool Null() { EXPECT_EQ(8u, step_); step_++; return true; } - bool Bool(bool b) { - switch(step_) { - case 4: EXPECT_TRUE(b); step_++; return true; - case 6: EXPECT_FALSE(b); step_++; return true; - default: ADD_FAILURE(); return false; - } + bool Default() { ADD_FAILURE(); return false; } + bool Null() { EXPECT_EQ(8u, step_); step_++; return true; } + bool Bool(bool b) + { switch(step_) + { case 4: EXPECT_TRUE(b); step_++; return true; + case 6: EXPECT_FALSE(b); step_++; return true; + default: ADD_FAILURE(); return false; } - bool Int(int i) { - switch(step_) { - case 10: EXPECT_EQ(123, i); step_++; return true; - case 15: EXPECT_EQ(1, i); step_++; return true; - case 16: EXPECT_EQ(2, i); step_++; return true; - case 17: EXPECT_EQ(3, i); step_++; return true; - default: ADD_FAILURE(); return false; - } + } + bool Int(int i) + { switch(step_) + { case 10: EXPECT_EQ(123, i); step_++; return true; + case 15: EXPECT_EQ(1, i); step_++; return true; + case 16: EXPECT_EQ(2, i); step_++; return true; + case 17: EXPECT_EQ(3, i); step_++; return true; + default: ADD_FAILURE(); return false; } - bool Uint(unsigned i) { return Int(static_cast(i)); } - bool Double(double d) { EXPECT_EQ(12u, step_); EXPECT_DOUBLE_EQ(3.1416, d); step_++; return true; } - bool String(const char* str, size_t, bool) { - switch(step_) { - case 1: EXPECT_STREQ("hello", str); step_++; return true; - case 2: EXPECT_STREQ("world", str); step_++; return true; - case 3: EXPECT_STREQ("t", str); step_++; return true; - case 5: EXPECT_STREQ("f", str); step_++; return true; - case 7: EXPECT_STREQ("n", str); step_++; return true; - case 9: EXPECT_STREQ("i", str); step_++; return true; - case 11: EXPECT_STREQ("pi", str); step_++; return true; - case 13: EXPECT_STREQ("a", str); step_++; return true; - default: ADD_FAILURE(); return false; - } + } + bool Uint(unsigned i) { return Int(static_cast(i)); } + bool Double(double d) { EXPECT_EQ(12u, step_); EXPECT_DOUBLE_EQ(3.1416, d); step_++; return true; } + bool String(const char* str, size_t, bool) + { switch(step_) + { case 1: EXPECT_STREQ("hello", str); step_++; return true; + case 2: EXPECT_STREQ("world", str); step_++; return true; + case 3: EXPECT_STREQ("t", str); step_++; return true; + case 5: EXPECT_STREQ("f", str); step_++; return true; + case 7: EXPECT_STREQ("n", str); step_++; return true; + case 9: EXPECT_STREQ("i", str); step_++; return true; + case 11: EXPECT_STREQ("pi", str); step_++; return true; + case 13: EXPECT_STREQ("a", str); step_++; return true; + default: ADD_FAILURE(); return false; } - bool StartObject() { EXPECT_EQ(0u, step_); step_++; return true; } - bool EndObject(SizeType memberCount) { EXPECT_EQ(19u, step_); EXPECT_EQ(7u, memberCount); step_++; return true; } - bool StartArray() { EXPECT_EQ(14u, step_); step_++; return true; } - bool EndArray(SizeType elementCount) { EXPECT_EQ(18u, step_); EXPECT_EQ(3u, elementCount); step_++; return true; } + } + bool StartObject() { EXPECT_EQ(0u, step_); step_++; return true; } + bool EndObject(SizeType memberCount) { EXPECT_EQ(19u, step_); EXPECT_EQ(7u, memberCount); step_++; return true; } + bool StartArray() { EXPECT_EQ(14u, step_); step_++; return true; } + bool EndArray(SizeType elementCount) { EXPECT_EQ(18u, step_); EXPECT_EQ(3u, elementCount); step_++; return true; } - unsigned step_; + unsigned step_; }; -TEST(Reader, ParseObject) { - const char* json = "{ \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] } "; - - // Insitu - { - char* json2 = StrDup(json); - InsituStringStream s(json2); - ParseObjectHandler h; - Reader reader; - reader.Parse(s, h); - EXPECT_EQ(20u, h.step_); - free(json2); - } +TEST(Reader, ParseObject) +{ const char* json = "{ \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] } "; - // Normal - { - StringStream s(json); - ParseObjectHandler h; - Reader reader; - reader.Parse(s, h); - EXPECT_EQ(20u, h.step_); - } + // Insitu + { char* json2 = StrDup(json); + InsituStringStream s(json2); + ParseObjectHandler h; + Reader reader; + reader.Parse(s, h); + EXPECT_EQ(20u, h.step_); + free(json2); + } + + // Normal + { StringStream s(json); + ParseObjectHandler h; + Reader reader; + reader.Parse(s, h); + EXPECT_EQ(20u, h.step_); + } } -struct ParseEmptyObjectHandler : BaseReaderHandler, ParseEmptyObjectHandler> { - ParseEmptyObjectHandler() : step_(0) {} +struct ParseEmptyObjectHandler : BaseReaderHandler, ParseEmptyObjectHandler> +{ ParseEmptyObjectHandler() : step_(0) {} - bool Default() { ADD_FAILURE(); return false; } - bool StartObject() { EXPECT_EQ(0u, step_); step_++; return true; } - bool EndObject(SizeType) { EXPECT_EQ(1u, step_); step_++; return true; } + bool Default() { ADD_FAILURE(); return false; } + bool StartObject() { EXPECT_EQ(0u, step_); step_++; return true; } + bool EndObject(SizeType) { EXPECT_EQ(1u, step_); step_++; return true; } - unsigned step_; + unsigned step_; }; -TEST(Reader, Parse_EmptyObject) { - StringStream s("{ } "); - ParseEmptyObjectHandler h; - Reader reader; - reader.Parse(s, h); - EXPECT_EQ(2u, h.step_); +TEST(Reader, Parse_EmptyObject) +{ StringStream s("{ } "); + ParseEmptyObjectHandler h; + Reader reader; + reader.Parse(s, h); + EXPECT_EQ(2u, h.step_); } -struct ParseMultipleRootHandler : BaseReaderHandler, ParseMultipleRootHandler> { - ParseMultipleRootHandler() : step_(0) {} +struct ParseMultipleRootHandler : BaseReaderHandler, ParseMultipleRootHandler> +{ ParseMultipleRootHandler() : step_(0) {} - bool Default() { ADD_FAILURE(); return false; } - bool StartObject() { EXPECT_EQ(0u, step_); step_++; return true; } - bool EndObject(SizeType) { EXPECT_EQ(1u, step_); step_++; return true; } - bool StartArray() { EXPECT_EQ(2u, step_); step_++; return true; } - bool EndArray(SizeType) { EXPECT_EQ(3u, step_); step_++; return true; } + bool Default() { ADD_FAILURE(); return false; } + bool StartObject() { EXPECT_EQ(0u, step_); step_++; return true; } + bool EndObject(SizeType) { EXPECT_EQ(1u, step_); step_++; return true; } + bool StartArray() { EXPECT_EQ(2u, step_); step_++; return true; } + bool EndArray(SizeType) { EXPECT_EQ(3u, step_); step_++; return true; } - unsigned step_; + unsigned step_; }; template -void TestMultipleRoot() { - StringStream s("{}[] a"); - ParseMultipleRootHandler h; - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - EXPECT_EQ(2u, h.step_); - EXPECT_TRUE(reader.Parse(s, h)); - EXPECT_EQ(4u, h.step_); - EXPECT_EQ(' ', s.Take()); - EXPECT_EQ('a', s.Take()); +void TestMultipleRoot() +{ StringStream s("{}[] a"); + ParseMultipleRootHandler h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + EXPECT_EQ(2u, h.step_); + EXPECT_TRUE(reader.Parse(s, h)); + EXPECT_EQ(4u, h.step_); + EXPECT_EQ(' ', s.Take()); + EXPECT_EQ('a', s.Take()); } -TEST(Reader, Parse_MultipleRoot) { - TestMultipleRoot(); +TEST(Reader, Parse_MultipleRoot) +{ TestMultipleRoot(); } -TEST(Reader, ParseIterative_MultipleRoot) { - TestMultipleRoot(); +TEST(Reader, ParseIterative_MultipleRoot) +{ TestMultipleRoot(); } template -void TestInsituMultipleRoot() { - char* buffer = strdup("{}[] a"); - InsituStringStream s(buffer); - ParseMultipleRootHandler h; - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - EXPECT_EQ(2u, h.step_); - EXPECT_TRUE(reader.Parse(s, h)); - EXPECT_EQ(4u, h.step_); - EXPECT_EQ(' ', s.Take()); - EXPECT_EQ('a', s.Take()); - free(buffer); +void TestInsituMultipleRoot() +{ char* buffer = strdup("{}[] a"); + InsituStringStream s(buffer); + ParseMultipleRootHandler h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + EXPECT_EQ(2u, h.step_); + EXPECT_TRUE(reader.Parse(s, h)); + EXPECT_EQ(4u, h.step_); + EXPECT_EQ(' ', s.Take()); + EXPECT_EQ('a', s.Take()); + free(buffer); } -TEST(Reader, ParseInsitu_MultipleRoot) { - TestInsituMultipleRoot(); +TEST(Reader, ParseInsitu_MultipleRoot) +{ TestInsituMultipleRoot(); } -TEST(Reader, ParseInsituIterative_MultipleRoot) { - TestInsituMultipleRoot(); +TEST(Reader, ParseInsituIterative_MultipleRoot) +{ TestInsituMultipleRoot(); } #define TEST_ERROR(errorCode, str, errorOffset) \ @@ -945,163 +941,165 @@ TEST(Reader, ParseInsituIterative_MultipleRoot) { EXPECT_EQ(streamPos, s.Tell());\ } -TEST(Reader, ParseDocument_Error) { - // The document is empty. - TEST_ERROR(kParseErrorDocumentEmpty, "", 0); - TEST_ERROR(kParseErrorDocumentEmpty, " ", 1); - TEST_ERROR(kParseErrorDocumentEmpty, " \n", 2); - - // The document root must not follow by other values. - TEST_ERROR(kParseErrorDocumentRootNotSingular, "[] 0", 3); - TEST_ERROR(kParseErrorDocumentRootNotSingular, "{} 0", 3); - TEST_ERROR(kParseErrorDocumentRootNotSingular, "null []", 5); - TEST_ERROR(kParseErrorDocumentRootNotSingular, "0 {}", 2); -} - -TEST(Reader, ParseValue_Error) { - // Invalid value. - TEST_ERROR(kParseErrorValueInvalid, "nulL", 3); - TEST_ERROR(kParseErrorValueInvalid, "truE", 3); - TEST_ERROR(kParseErrorValueInvalid, "falsE", 4); - TEST_ERROR(kParseErrorValueInvalid, "a]", 0); - TEST_ERROR(kParseErrorValueInvalid, ".1", 0); -} - -TEST(Reader, ParseObject_Error) { - // Missing a name for object member. - TEST_ERROR(kParseErrorObjectMissName, "{1}", 1); - TEST_ERROR(kParseErrorObjectMissName, "{:1}", 1); - TEST_ERROR(kParseErrorObjectMissName, "{null:1}", 1); - TEST_ERROR(kParseErrorObjectMissName, "{true:1}", 1); - TEST_ERROR(kParseErrorObjectMissName, "{false:1}", 1); - TEST_ERROR(kParseErrorObjectMissName, "{1:1}", 1); - TEST_ERROR(kParseErrorObjectMissName, "{[]:1}", 1); - TEST_ERROR(kParseErrorObjectMissName, "{{}:1}", 1); - TEST_ERROR(kParseErrorObjectMissName, "{xyz:1}", 1); - - // Missing a colon after a name of object member. - TEST_ERROR(kParseErrorObjectMissColon, "{\"a\" 1}", 5); - TEST_ERROR(kParseErrorObjectMissColon, "{\"a\",1}", 4); - - // Must be a comma or '}' after an object member - TEST_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, "{\"a\":1]", 6); - - // Object cannot have a trailing comma (without kParseTrailingCommasFlag); - // an object member name must follow a comma - TEST_ERROR(kParseErrorObjectMissName, "{\"a\":1,}", 7); - - // This tests that MemoryStream is checking the length in Peek(). - { - MemoryStream ms("{\"a\"", 1); - BaseReaderHandler<> h; - Reader reader; - EXPECT_FALSE(reader.Parse(ms, h)); - EXPECT_EQ(kParseErrorObjectMissName, reader.GetParseErrorCode()); - } +TEST(Reader, ParseDocument_Error) +{ // The document is empty. + TEST_ERROR(kParseErrorDocumentEmpty, "", 0); + TEST_ERROR(kParseErrorDocumentEmpty, " ", 1); + TEST_ERROR(kParseErrorDocumentEmpty, " \n", 2); + + // The document root must not follow by other values. + TEST_ERROR(kParseErrorDocumentRootNotSingular, "[] 0", 3); + TEST_ERROR(kParseErrorDocumentRootNotSingular, "{} 0", 3); + TEST_ERROR(kParseErrorDocumentRootNotSingular, "null []", 5); + TEST_ERROR(kParseErrorDocumentRootNotSingular, "0 {}", 2); +} + +TEST(Reader, ParseValue_Error) +{ // Invalid value. + TEST_ERROR(kParseErrorValueInvalid, "nulL", 3); + TEST_ERROR(kParseErrorValueInvalid, "truE", 3); + TEST_ERROR(kParseErrorValueInvalid, "falsE", 4); + TEST_ERROR(kParseErrorValueInvalid, "a]", 0); + TEST_ERROR(kParseErrorValueInvalid, ".1", 0); +} + +TEST(Reader, ParseObject_Error) +{ // Missing a name for object member. + TEST_ERROR(kParseErrorObjectMissName, "{1}", 1); + TEST_ERROR(kParseErrorObjectMissName, "{:1}", 1); + TEST_ERROR(kParseErrorObjectMissName, "{null:1}", 1); + TEST_ERROR(kParseErrorObjectMissName, "{true:1}", 1); + TEST_ERROR(kParseErrorObjectMissName, "{false:1}", 1); + TEST_ERROR(kParseErrorObjectMissName, "{1:1}", 1); + TEST_ERROR(kParseErrorObjectMissName, "{[]:1}", 1); + TEST_ERROR(kParseErrorObjectMissName, "{{}:1}", 1); + TEST_ERROR(kParseErrorObjectMissName, "{xyz:1}", 1); + + // Missing a colon after a name of object member. + TEST_ERROR(kParseErrorObjectMissColon, "{\"a\" 1}", 5); + TEST_ERROR(kParseErrorObjectMissColon, "{\"a\",1}", 4); + + // Must be a comma or '}' after an object member + TEST_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, "{\"a\":1]", 6); + + // Object cannot have a trailing comma (without kParseTrailingCommasFlag); + // an object member name must follow a comma + TEST_ERROR(kParseErrorObjectMissName, "{\"a\":1,}", 7); + + // This tests that MemoryStream is checking the length in Peek(). + { MemoryStream ms("{\"a\"", 1); + BaseReaderHandler<> h; + Reader reader; + EXPECT_FALSE(reader.Parse(ms, h)); + EXPECT_EQ(kParseErrorObjectMissName, reader.GetParseErrorCode()); + } } #undef TEST_ERROR -TEST(Reader, SkipWhitespace) { - StringStream ss(" A \t\tB\n \n\nC\r\r \rD \t\n\r E"); - const char* expected = "ABCDE"; - for (size_t i = 0; i < 5; i++) { - SkipWhitespace(ss); - EXPECT_EQ(expected[i], ss.Take()); - } +TEST(Reader, SkipWhitespace) +{ StringStream ss(" A \t\tB\n \n\nC\r\r \rD \t\n\r E"); + const char* expected = "ABCDE"; + for (size_t i = 0; i < 5; i++) + { SkipWhitespace(ss); + EXPECT_EQ(expected[i], ss.Take()); + } } // Test implementing a stream without copy stream optimization. // Clone from GenericStringStream except that copy constructor is disabled. template -class CustomStringStream { +class CustomStringStream +{ public: - typedef typename Encoding::Ch Ch; + typedef typename Encoding::Ch Ch; - CustomStringStream(const Ch *src) : src_(src), head_(src) {} + CustomStringStream(const Ch *src) : src_(src), head_(src) {} - Ch Peek() const { return *src_; } - Ch Take() { return *src_++; } - size_t Tell() const { return static_cast(src_ - head_); } + Ch Peek() const { return *src_; } + Ch Take() { return *src_++; } + size_t Tell() const { return static_cast(src_ - head_); } - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } private: - // Prohibit copy constructor & assignment operator. - CustomStringStream(const CustomStringStream&); - CustomStringStream& operator=(const CustomStringStream&); + // Prohibit copy constructor & assignment operator. + CustomStringStream(const CustomStringStream&); + CustomStringStream& operator=(const CustomStringStream&); - const Ch* src_; //!< Current read position. - const Ch* head_; //!< Original head of the string. + const Ch* src_; //!< Current read position. + const Ch* head_; //!< Original head of the string. }; // If the following code is compiled, it should generate compilation error as predicted. // Because CustomStringStream<> is not copyable via making copy constructor private. #if 0 -namespace rapidjson { +namespace rapidjson +{ template -struct StreamTraits > { - enum { copyOptimization = 1 }; +struct StreamTraits > +{ enum { copyOptimization = 1 }; }; } // namespace rapidjson #endif -TEST(Reader, CustomStringStream) { - const char* json = "{ \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] } "; - CustomStringStream > s(json); - ParseObjectHandler h; - Reader reader; - reader.Parse(s, h); - EXPECT_EQ(20u, h.step_); +TEST(Reader, CustomStringStream) +{ const char* json = "{ \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] } "; + CustomStringStream > s(json); + ParseObjectHandler h; + Reader reader; + reader.Parse(s, h); + EXPECT_EQ(20u, h.step_); } #include -class IStreamWrapper { +class IStreamWrapper +{ public: - typedef char Ch; + typedef char Ch; - IStreamWrapper(std::istream& is) : is_(is) {} + IStreamWrapper(std::istream& is) : is_(is) {} - Ch Peek() const { - int c = is_.peek(); - return c == std::char_traits::eof() ? '\0' : static_cast(c); - } + Ch Peek() const + { int c = is_.peek(); + return c == std::char_traits::eof() ? '\0' : static_cast(c); + } - Ch Take() { - int c = is_.get(); - return c == std::char_traits::eof() ? '\0' : static_cast(c); - } + Ch Take() + { int c = is_.get(); + return c == std::char_traits::eof() ? '\0' : static_cast(c); + } - size_t Tell() const { return static_cast(is_.tellg()); } + size_t Tell() const { return static_cast(is_.tellg()); } - Ch* PutBegin() { assert(false); return 0; } - void Put(Ch) { assert(false); } - void Flush() { assert(false); } - size_t PutEnd(Ch*) { assert(false); return 0; } + Ch* PutBegin() { assert(false); return 0; } + void Put(Ch) { assert(false); } + void Flush() { assert(false); } + size_t PutEnd(Ch*) { assert(false); return 0; } private: - IStreamWrapper(const IStreamWrapper&); - IStreamWrapper& operator=(const IStreamWrapper&); + IStreamWrapper(const IStreamWrapper&); + IStreamWrapper& operator=(const IStreamWrapper&); - std::istream& is_; + std::istream& is_; }; -TEST(Reader, Parse_IStreamWrapper_StringStream) { - const char* json = "[1,2,3,4]"; +TEST(Reader, Parse_IStreamWrapper_StringStream) +{ const char* json = "[1,2,3,4]"; - std::stringstream ss(json); - IStreamWrapper is(ss); + std::stringstream ss(json); + IStreamWrapper is(ss); - Reader reader; - ParseArrayHandler<4> h; - reader.Parse(is, h); - EXPECT_FALSE(reader.HasParseError()); + Reader reader; + ParseArrayHandler<4> h; + reader.Parse(is, h); + EXPECT_FALSE(reader.HasParseError()); } // Test iterative parsing. @@ -1119,267 +1117,265 @@ TEST(Reader, Parse_IStreamWrapper_StringStream) { EXPECT_EQ(streamPos, json.Tell()); \ } -TEST(Reader, IterativeParsing_ErrorHandling) { - TESTERRORHANDLING("{\"a\": a}", kParseErrorValueInvalid, 6u); - - TESTERRORHANDLING("", kParseErrorDocumentEmpty, 0u); - TESTERRORHANDLING("{}{}", kParseErrorDocumentRootNotSingular, 2u); - - TESTERRORHANDLING("{1}", kParseErrorObjectMissName, 1u); - TESTERRORHANDLING("{\"a\", 1}", kParseErrorObjectMissColon, 4u); - TESTERRORHANDLING("{\"a\"}", kParseErrorObjectMissColon, 4u); - TESTERRORHANDLING("{\"a\": 1", kParseErrorObjectMissCommaOrCurlyBracket, 7u); - TESTERRORHANDLING("[1 2 3]", kParseErrorArrayMissCommaOrSquareBracket, 3u); - TESTERRORHANDLING("{\"a: 1", kParseErrorStringMissQuotationMark, 6u); - TESTERRORHANDLING("{\"a\":}", kParseErrorValueInvalid, 5u); - TESTERRORHANDLING("{\"a\":]", kParseErrorValueInvalid, 5u); - TESTERRORHANDLING("[1,2,}", kParseErrorValueInvalid, 5u); - TESTERRORHANDLING("[}]", kParseErrorValueInvalid, 1u); - TESTERRORHANDLING("[,]", kParseErrorValueInvalid, 1u); - TESTERRORHANDLING("[1,,]", kParseErrorValueInvalid, 3u); - - // Trailing commas are not allowed without kParseTrailingCommasFlag - TESTERRORHANDLING("{\"a\": 1,}", kParseErrorObjectMissName, 8u); - TESTERRORHANDLING("[1,2,3,]", kParseErrorValueInvalid, 7u); - - // Any JSON value can be a valid root element in RFC7159. - TESTERRORHANDLING("\"ab", kParseErrorStringMissQuotationMark, 3u); - TESTERRORHANDLING("truE", kParseErrorValueInvalid, 3u); - TESTERRORHANDLING("False", kParseErrorValueInvalid, 0u); - TESTERRORHANDLING("true, false", kParseErrorDocumentRootNotSingular, 4u); - TESTERRORHANDLING("false, false", kParseErrorDocumentRootNotSingular, 5u); - TESTERRORHANDLING("nulL", kParseErrorValueInvalid, 3u); - TESTERRORHANDLING("null , null", kParseErrorDocumentRootNotSingular, 5u); - TESTERRORHANDLING("1a", kParseErrorDocumentRootNotSingular, 1u); +TEST(Reader, IterativeParsing_ErrorHandling) +{ TESTERRORHANDLING("{\"a\": a}", kParseErrorValueInvalid, 6u); + + TESTERRORHANDLING("", kParseErrorDocumentEmpty, 0u); + TESTERRORHANDLING("{}{}", kParseErrorDocumentRootNotSingular, 2u); + + TESTERRORHANDLING("{1}", kParseErrorObjectMissName, 1u); + TESTERRORHANDLING("{\"a\", 1}", kParseErrorObjectMissColon, 4u); + TESTERRORHANDLING("{\"a\"}", kParseErrorObjectMissColon, 4u); + TESTERRORHANDLING("{\"a\": 1", kParseErrorObjectMissCommaOrCurlyBracket, 7u); + TESTERRORHANDLING("[1 2 3]", kParseErrorArrayMissCommaOrSquareBracket, 3u); + TESTERRORHANDLING("{\"a: 1", kParseErrorStringMissQuotationMark, 6u); + TESTERRORHANDLING("{\"a\":}", kParseErrorValueInvalid, 5u); + TESTERRORHANDLING("{\"a\":]", kParseErrorValueInvalid, 5u); + TESTERRORHANDLING("[1,2,}", kParseErrorValueInvalid, 5u); + TESTERRORHANDLING("[}]", kParseErrorValueInvalid, 1u); + TESTERRORHANDLING("[,]", kParseErrorValueInvalid, 1u); + TESTERRORHANDLING("[1,,]", kParseErrorValueInvalid, 3u); + + // Trailing commas are not allowed without kParseTrailingCommasFlag + TESTERRORHANDLING("{\"a\": 1,}", kParseErrorObjectMissName, 8u); + TESTERRORHANDLING("[1,2,3,]", kParseErrorValueInvalid, 7u); + + // Any JSON value can be a valid root element in RFC7159. + TESTERRORHANDLING("\"ab", kParseErrorStringMissQuotationMark, 3u); + TESTERRORHANDLING("truE", kParseErrorValueInvalid, 3u); + TESTERRORHANDLING("False", kParseErrorValueInvalid, 0u); + TESTERRORHANDLING("true, false", kParseErrorDocumentRootNotSingular, 4u); + TESTERRORHANDLING("false, false", kParseErrorDocumentRootNotSingular, 5u); + TESTERRORHANDLING("nulL", kParseErrorValueInvalid, 3u); + TESTERRORHANDLING("null , null", kParseErrorDocumentRootNotSingular, 5u); + TESTERRORHANDLING("1a", kParseErrorDocumentRootNotSingular, 1u); } template > -struct IterativeParsingReaderHandler { - typedef typename Encoding::Ch Ch; - - const static int LOG_NULL = -1; - const static int LOG_BOOL = -2; - const static int LOG_INT = -3; - const static int LOG_UINT = -4; - const static int LOG_INT64 = -5; - const static int LOG_UINT64 = -6; - const static int LOG_DOUBLE = -7; - const static int LOG_STRING = -8; - const static int LOG_STARTOBJECT = -9; - const static int LOG_KEY = -10; - const static int LOG_ENDOBJECT = -11; - const static int LOG_STARTARRAY = -12; - const static int LOG_ENDARRAY = -13; - - const static size_t LogCapacity = 256; - int Logs[LogCapacity]; - size_t LogCount; - - IterativeParsingReaderHandler() : LogCount(0) { - } +struct IterativeParsingReaderHandler +{ typedef typename Encoding::Ch Ch; - bool Null() { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_NULL; return true; } + const static int LOG_NULL = -1; + const static int LOG_BOOL = -2; + const static int LOG_INT = -3; + const static int LOG_UINT = -4; + const static int LOG_INT64 = -5; + const static int LOG_UINT64 = -6; + const static int LOG_DOUBLE = -7; + const static int LOG_STRING = -8; + const static int LOG_STARTOBJECT = -9; + const static int LOG_KEY = -10; + const static int LOG_ENDOBJECT = -11; + const static int LOG_STARTARRAY = -12; + const static int LOG_ENDARRAY = -13; - bool Bool(bool) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_BOOL; return true; } + const static size_t LogCapacity = 256; + int Logs[LogCapacity]; + size_t LogCount; - bool Int(int) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_INT; return true; } + IterativeParsingReaderHandler() : LogCount(0) + { + } - bool Uint(unsigned) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_INT; return true; } + bool Null() { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_NULL; return true; } - bool Int64(int64_t) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_INT64; return true; } + bool Bool(bool) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_BOOL; return true; } - bool Uint64(uint64_t) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_UINT64; return true; } + bool Int(int) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_INT; return true; } - bool Double(double) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_DOUBLE; return true; } + bool Uint(unsigned) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_INT; return true; } - bool RawNumber(const Ch*, SizeType, bool) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_STRING; return true; } + bool Int64(int64_t) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_INT64; return true; } - bool String(const Ch*, SizeType, bool) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_STRING; return true; } + bool Uint64(uint64_t) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_UINT64; return true; } - bool StartObject() { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_STARTOBJECT; return true; } + bool Double(double) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_DOUBLE; return true; } - bool Key (const Ch*, SizeType, bool) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_KEY; return true; } + bool RawNumber(const Ch*, SizeType, bool) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_STRING; return true; } - bool EndObject(SizeType c) { - RAPIDJSON_ASSERT(LogCount < LogCapacity); - Logs[LogCount++] = LOG_ENDOBJECT; - Logs[LogCount++] = static_cast(c); - return true; - } + bool String(const Ch*, SizeType, bool) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_STRING; return true; } - bool StartArray() { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_STARTARRAY; return true; } + bool StartObject() { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_STARTOBJECT; return true; } - bool EndArray(SizeType c) { - RAPIDJSON_ASSERT(LogCount < LogCapacity); - Logs[LogCount++] = LOG_ENDARRAY; - Logs[LogCount++] = static_cast(c); - return true; - } + bool Key (const Ch*, SizeType, bool) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_KEY; return true; } + + bool EndObject(SizeType c) + { RAPIDJSON_ASSERT(LogCount < LogCapacity); + Logs[LogCount++] = LOG_ENDOBJECT; + Logs[LogCount++] = static_cast(c); + return true; + } + + bool StartArray() { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_STARTARRAY; return true; } + + bool EndArray(SizeType c) + { RAPIDJSON_ASSERT(LogCount < LogCapacity); + Logs[LogCount++] = LOG_ENDARRAY; + Logs[LogCount++] = static_cast(c); + return true; + } }; -TEST(Reader, IterativeParsing_General) { - { - StringStream is("[1, {\"k\": [1, 2]}, null, false, true, \"string\", 1.2]"); - Reader reader; - IterativeParsingReaderHandler<> handler; - - ParseResult r = reader.Parse(is, handler); - - EXPECT_FALSE(r.IsError()); - EXPECT_FALSE(reader.HasParseError()); - - int e[] = { - handler.LOG_STARTARRAY, - handler.LOG_INT, - handler.LOG_STARTOBJECT, - handler.LOG_KEY, - handler.LOG_STARTARRAY, - handler.LOG_INT, - handler.LOG_INT, - handler.LOG_ENDARRAY, 2, - handler.LOG_ENDOBJECT, 1, - handler.LOG_NULL, - handler.LOG_BOOL, - handler.LOG_BOOL, - handler.LOG_STRING, - handler.LOG_DOUBLE, - handler.LOG_ENDARRAY, 7 - }; - - EXPECT_EQ(sizeof(e) / sizeof(int), handler.LogCount); - - for (size_t i = 0; i < handler.LogCount; ++i) { - EXPECT_EQ(e[i], handler.Logs[i]) << "i = " << i; - } +TEST(Reader, IterativeParsing_General) +{ + { StringStream is("[1, {\"k\": [1, 2]}, null, false, true, \"string\", 1.2]"); + Reader reader; + IterativeParsingReaderHandler<> handler; + + ParseResult r = reader.Parse(is, handler); + + EXPECT_FALSE(r.IsError()); + EXPECT_FALSE(reader.HasParseError()); + + int e[] = + { handler.LOG_STARTARRAY, + handler.LOG_INT, + handler.LOG_STARTOBJECT, + handler.LOG_KEY, + handler.LOG_STARTARRAY, + handler.LOG_INT, + handler.LOG_INT, + handler.LOG_ENDARRAY, 2, + handler.LOG_ENDOBJECT, 1, + handler.LOG_NULL, + handler.LOG_BOOL, + handler.LOG_BOOL, + handler.LOG_STRING, + handler.LOG_DOUBLE, + handler.LOG_ENDARRAY, 7 + }; + + EXPECT_EQ(sizeof(e) / sizeof(int), handler.LogCount); + + for (size_t i = 0; i < handler.LogCount; ++i) + { EXPECT_EQ(e[i], handler.Logs[i]) << "i = " << i; } + } } -TEST(Reader, IterativeParsing_Count) { - { - StringStream is("[{}, {\"k\": 1}, [1], []]"); - Reader reader; - IterativeParsingReaderHandler<> handler; - - ParseResult r = reader.Parse(is, handler); - - EXPECT_FALSE(r.IsError()); - EXPECT_FALSE(reader.HasParseError()); - - int e[] = { - handler.LOG_STARTARRAY, - handler.LOG_STARTOBJECT, - handler.LOG_ENDOBJECT, 0, - handler.LOG_STARTOBJECT, - handler.LOG_KEY, - handler.LOG_INT, - handler.LOG_ENDOBJECT, 1, - handler.LOG_STARTARRAY, - handler.LOG_INT, - handler.LOG_ENDARRAY, 1, - handler.LOG_STARTARRAY, - handler.LOG_ENDARRAY, 0, - handler.LOG_ENDARRAY, 4 - }; - - EXPECT_EQ(sizeof(e) / sizeof(int), handler.LogCount); - - for (size_t i = 0; i < handler.LogCount; ++i) { - EXPECT_EQ(e[i], handler.Logs[i]) << "i = " << i; - } +TEST(Reader, IterativeParsing_Count) +{ + { StringStream is("[{}, {\"k\": 1}, [1], []]"); + Reader reader; + IterativeParsingReaderHandler<> handler; + + ParseResult r = reader.Parse(is, handler); + + EXPECT_FALSE(r.IsError()); + EXPECT_FALSE(reader.HasParseError()); + + int e[] = + { handler.LOG_STARTARRAY, + handler.LOG_STARTOBJECT, + handler.LOG_ENDOBJECT, 0, + handler.LOG_STARTOBJECT, + handler.LOG_KEY, + handler.LOG_INT, + handler.LOG_ENDOBJECT, 1, + handler.LOG_STARTARRAY, + handler.LOG_INT, + handler.LOG_ENDARRAY, 1, + handler.LOG_STARTARRAY, + handler.LOG_ENDARRAY, 0, + handler.LOG_ENDARRAY, 4 + }; + + EXPECT_EQ(sizeof(e) / sizeof(int), handler.LogCount); + + for (size_t i = 0; i < handler.LogCount; ++i) + { EXPECT_EQ(e[i], handler.Logs[i]) << "i = " << i; } + } } // Test iterative parsing on kParseErrorTermination. -struct HandlerTerminateAtStartObject : public IterativeParsingReaderHandler<> { - bool StartObject() { return false; } +struct HandlerTerminateAtStartObject : public IterativeParsingReaderHandler<> +{ bool StartObject() { return false; } }; -struct HandlerTerminateAtStartArray : public IterativeParsingReaderHandler<> { - bool StartArray() { return false; } +struct HandlerTerminateAtStartArray : public IterativeParsingReaderHandler<> +{ bool StartArray() { return false; } }; -struct HandlerTerminateAtEndObject : public IterativeParsingReaderHandler<> { - bool EndObject(SizeType) { return false; } +struct HandlerTerminateAtEndObject : public IterativeParsingReaderHandler<> +{ bool EndObject(SizeType) { return false; } }; -struct HandlerTerminateAtEndArray : public IterativeParsingReaderHandler<> { - bool EndArray(SizeType) { return false; } +struct HandlerTerminateAtEndArray : public IterativeParsingReaderHandler<> +{ bool EndArray(SizeType) { return false; } }; -TEST(Reader, IterativeParsing_ShortCircuit) { - { - HandlerTerminateAtStartObject handler; - Reader reader; - StringStream is("[1, {}]"); +TEST(Reader, IterativeParsing_ShortCircuit) +{ + { HandlerTerminateAtStartObject handler; + Reader reader; + StringStream is("[1, {}]"); - ParseResult r = reader.Parse(is, handler); + ParseResult r = reader.Parse(is, handler); - EXPECT_TRUE(reader.HasParseError()); - EXPECT_EQ(kParseErrorTermination, r.Code()); - EXPECT_EQ(4u, r.Offset()); - } + EXPECT_TRUE(reader.HasParseError()); + EXPECT_EQ(kParseErrorTermination, r.Code()); + EXPECT_EQ(4u, r.Offset()); + } - { - HandlerTerminateAtStartArray handler; - Reader reader; - StringStream is("{\"a\": []}"); + { HandlerTerminateAtStartArray handler; + Reader reader; + StringStream is("{\"a\": []}"); - ParseResult r = reader.Parse(is, handler); + ParseResult r = reader.Parse(is, handler); - EXPECT_TRUE(reader.HasParseError()); - EXPECT_EQ(kParseErrorTermination, r.Code()); - EXPECT_EQ(6u, r.Offset()); - } + EXPECT_TRUE(reader.HasParseError()); + EXPECT_EQ(kParseErrorTermination, r.Code()); + EXPECT_EQ(6u, r.Offset()); + } - { - HandlerTerminateAtEndObject handler; - Reader reader; - StringStream is("[1, {}]"); + { HandlerTerminateAtEndObject handler; + Reader reader; + StringStream is("[1, {}]"); - ParseResult r = reader.Parse(is, handler); + ParseResult r = reader.Parse(is, handler); - EXPECT_TRUE(reader.HasParseError()); - EXPECT_EQ(kParseErrorTermination, r.Code()); - EXPECT_EQ(5u, r.Offset()); - } + EXPECT_TRUE(reader.HasParseError()); + EXPECT_EQ(kParseErrorTermination, r.Code()); + EXPECT_EQ(5u, r.Offset()); + } - { - HandlerTerminateAtEndArray handler; - Reader reader; - StringStream is("{\"a\": []}"); + { HandlerTerminateAtEndArray handler; + Reader reader; + StringStream is("{\"a\": []}"); - ParseResult r = reader.Parse(is, handler); + ParseResult r = reader.Parse(is, handler); - EXPECT_TRUE(reader.HasParseError()); - EXPECT_EQ(kParseErrorTermination, r.Code()); - EXPECT_EQ(7u, r.Offset()); - } + EXPECT_TRUE(reader.HasParseError()); + EXPECT_EQ(kParseErrorTermination, r.Code()); + EXPECT_EQ(7u, r.Offset()); + } } // For covering BaseReaderHandler default functions -TEST(Reader, BaseReaderHandler_Default) { - BaseReaderHandler<> h; - Reader reader; - StringStream is("[null, true, -1, 1, -1234567890123456789, 1234567890123456789, 3.14, \"s\", { \"a\" : 1 }]"); - EXPECT_TRUE(reader.Parse(is, h)); +TEST(Reader, BaseReaderHandler_Default) +{ BaseReaderHandler<> h; + Reader reader; + StringStream is("[null, true, -1, 1, -1234567890123456789, 1234567890123456789, 3.14, \"s\", { \"a\" : 1 }]"); + EXPECT_TRUE(reader.Parse(is, h)); } template -struct TerminateHandler { - bool Null() { return e != 0; } - bool Bool(bool) { return e != 1; } - bool Int(int) { return e != 2; } - bool Uint(unsigned) { return e != 3; } - bool Int64(int64_t) { return e != 4; } - bool Uint64(uint64_t) { return e != 5; } - bool Double(double) { return e != 6; } - bool RawNumber(const char*, SizeType, bool) { return e != 7; } - bool String(const char*, SizeType, bool) { return e != 8; } - bool StartObject() { return e != 9; } - bool Key(const char*, SizeType, bool) { return e != 10; } - bool EndObject(SizeType) { return e != 11; } - bool StartArray() { return e != 12; } - bool EndArray(SizeType) { return e != 13; } +struct TerminateHandler +{ bool Null() { return e != 0; } + bool Bool(bool) { return e != 1; } + bool Int(int) { return e != 2; } + bool Uint(unsigned) { return e != 3; } + bool Int64(int64_t) { return e != 4; } + bool Uint64(uint64_t) { return e != 5; } + bool Double(double) { return e != 6; } + bool RawNumber(const char*, SizeType, bool) { return e != 7; } + bool String(const char*, SizeType, bool) { return e != 8; } + bool StartObject() { return e != 9; } + bool Key(const char*, SizeType, bool) { return e != 10; } + bool EndObject(SizeType) { return e != 11; } + bool StartArray() { return e != 12; } + bool EndArray(SizeType) { return e != 13; } }; #define TEST_TERMINATION(e, json)\ @@ -1391,28 +1387,28 @@ struct TerminateHandler { EXPECT_EQ(kParseErrorTermination, reader.GetParseErrorCode());\ } -TEST(Reader, ParseTerminationByHandler) { - TEST_TERMINATION(0, "[null"); - TEST_TERMINATION(1, "[true"); - TEST_TERMINATION(1, "[false"); - TEST_TERMINATION(2, "[-1"); - TEST_TERMINATION(3, "[1"); - TEST_TERMINATION(4, "[-1234567890123456789"); - TEST_TERMINATION(5, "[1234567890123456789"); - TEST_TERMINATION(6, "[0.5]"); - // RawNumber() is never called - TEST_TERMINATION(8, "[\"a\""); - TEST_TERMINATION(9, "[{"); - TEST_TERMINATION(10, "[{\"a\""); - TEST_TERMINATION(11, "[{}"); - TEST_TERMINATION(11, "[{\"a\":1}"); // non-empty object - TEST_TERMINATION(12, "{\"a\":["); - TEST_TERMINATION(13, "{\"a\":[]"); - TEST_TERMINATION(13, "{\"a\":[1]"); // non-empty array -} - -TEST(Reader, ParseComments) { - const char* json = +TEST(Reader, ParseTerminationByHandler) +{ TEST_TERMINATION(0, "[null"); + TEST_TERMINATION(1, "[true"); + TEST_TERMINATION(1, "[false"); + TEST_TERMINATION(2, "[-1"); + TEST_TERMINATION(3, "[1"); + TEST_TERMINATION(4, "[-1234567890123456789"); + TEST_TERMINATION(5, "[1234567890123456789"); + TEST_TERMINATION(6, "[0.5]"); + // RawNumber() is never called + TEST_TERMINATION(8, "[\"a\""); + TEST_TERMINATION(9, "[{"); + TEST_TERMINATION(10, "[{\"a\""); + TEST_TERMINATION(11, "[{}"); + TEST_TERMINATION(11, "[{\"a\":1}"); // non-empty object + TEST_TERMINATION(12, "{\"a\":["); + TEST_TERMINATION(13, "{\"a\":[]"); + TEST_TERMINATION(13, "{\"a\":[1]"); // non-empty array +} + +TEST(Reader, ParseComments) +{ const char* json = "// Here is a one-line comment.\n" "{// And here's another one\n" " /*And here's an in-line one.*/\"hello\" : \"world\"," @@ -1422,363 +1418,347 @@ TEST(Reader, ParseComments) { " \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3]" "}/*And the last one to be sure */"; - StringStream s(json); - ParseObjectHandler h; - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - EXPECT_EQ(20u, h.step_); + StringStream s(json); + ParseObjectHandler h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + EXPECT_EQ(20u, h.step_); } -TEST(Reader, ParseEmptyInlineComment) { - const char* json = "{/**/\"hello\" : \"world\", \"t\" : true, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }"; +TEST(Reader, ParseEmptyInlineComment) +{ const char* json = "{/**/\"hello\" : \"world\", \"t\" : true, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }"; - StringStream s(json); - ParseObjectHandler h; - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - EXPECT_EQ(20u, h.step_); + StringStream s(json); + ParseObjectHandler h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + EXPECT_EQ(20u, h.step_); } -TEST(Reader, ParseEmptyOnelineComment) { - const char* json = "{//\n\"hello\" : \"world\", \"t\" : true, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }"; +TEST(Reader, ParseEmptyOnelineComment) +{ const char* json = "{//\n\"hello\" : \"world\", \"t\" : true, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }"; - StringStream s(json); - ParseObjectHandler h; - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - EXPECT_EQ(20u, h.step_); + StringStream s(json); + ParseObjectHandler h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + EXPECT_EQ(20u, h.step_); } -TEST(Reader, ParseMultipleCommentsInARow) { - const char* json = +TEST(Reader, ParseMultipleCommentsInARow) +{ const char* json = "{/* first comment *//* second */\n" "/* third */ /*fourth*/// last one\n" "\"hello\" : \"world\", \"t\" : true, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }"; - StringStream s(json); - ParseObjectHandler h; - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - EXPECT_EQ(20u, h.step_); + StringStream s(json); + ParseObjectHandler h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + EXPECT_EQ(20u, h.step_); } -TEST(Reader, InlineCommentsAreDisabledByDefault) { - { - const char* json = "{/* Inline comment. */\"hello\" : \"world\", \"t\" : true, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }"; - - StringStream s(json); - ParseObjectHandler h; - Reader reader; - EXPECT_FALSE(reader.Parse(s, h)); - } - - { - const char* json = - "{\"hello\" : /* Multiline comment starts here\n" - " continues here\n" - " and ends here */\"world\", \"t\" :true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }"; - - StringStream s(json); - ParseObjectHandler h; - Reader reader; - EXPECT_FALSE(reader.Parse(s, h)); - } -} - -TEST(Reader, OnelineCommentsAreDisabledByDefault) { - const char* json = "{// One-line comment\n\"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }"; +TEST(Reader, InlineCommentsAreDisabledByDefault) +{ + { const char* json = "{/* Inline comment. */\"hello\" : \"world\", \"t\" : true, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }"; StringStream s(json); ParseObjectHandler h; Reader reader; EXPECT_FALSE(reader.Parse(s, h)); -} + } -TEST(Reader, EofAfterOneLineComment) { - const char* json = "{\"hello\" : \"world\" // EOF is here -->\0 \n}"; + { const char* json = + "{\"hello\" : /* Multiline comment starts here\n" + " continues here\n" + " and ends here */\"world\", \"t\" :true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }"; StringStream s(json); ParseObjectHandler h; Reader reader; - EXPECT_FALSE(reader.Parse(s, h)); - EXPECT_EQ(kParseErrorObjectMissCommaOrCurlyBracket, reader.GetParseErrorCode()); -} - -TEST(Reader, IncompleteMultilineComment) { - const char* json = "{\"hello\" : \"world\" /* EOF is here -->\0 */}"; + EXPECT_FALSE(reader.Parse(s, h)); + } +} + +TEST(Reader, OnelineCommentsAreDisabledByDefault) +{ const char* json = "{// One-line comment\n\"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }"; + + StringStream s(json); + ParseObjectHandler h; + Reader reader; + EXPECT_FALSE(reader.Parse(s, h)); +} + +TEST(Reader, EofAfterOneLineComment) +{ const char* json = "{\"hello\" : \"world\" // EOF is here -->\0 \n}"; + + StringStream s(json); + ParseObjectHandler h; + Reader reader; + EXPECT_FALSE(reader.Parse(s, h)); + EXPECT_EQ(kParseErrorObjectMissCommaOrCurlyBracket, reader.GetParseErrorCode()); +} + +TEST(Reader, IncompleteMultilineComment) +{ const char* json = "{\"hello\" : \"world\" /* EOF is here -->\0 */}"; + + StringStream s(json); + ParseObjectHandler h; + Reader reader; + EXPECT_FALSE(reader.Parse(s, h)); + EXPECT_EQ(kParseErrorUnspecificSyntaxError, reader.GetParseErrorCode()); +} + +TEST(Reader, IncompleteMultilineComment2) +{ const char* json = "{\"hello\" : \"world\" /* *\0 */}"; + + StringStream s(json); + ParseObjectHandler h; + Reader reader; + EXPECT_FALSE(reader.Parse(s, h)); + EXPECT_EQ(kParseErrorUnspecificSyntaxError, reader.GetParseErrorCode()); +} + +TEST(Reader, UnrecognizedComment) +{ const char* json = "{\"hello\" : \"world\" /! }"; + + StringStream s(json); + ParseObjectHandler h; + Reader reader; + EXPECT_FALSE(reader.Parse(s, h)); + EXPECT_EQ(kParseErrorUnspecificSyntaxError, reader.GetParseErrorCode()); +} + +struct NumbersAsStringsHandler +{ bool Null() { return true; } + bool Bool(bool) { return true; } + bool Int(int) { return true; } + bool Uint(unsigned) { return true; } + bool Int64(int64_t) { return true; } + bool Uint64(uint64_t) { return true; } + bool Double(double) { return true; } + // 'str' is not null-terminated + bool RawNumber(const char* str, SizeType length, bool) + { EXPECT_TRUE(str != 0); + EXPECT_TRUE(expected_len_ == length); + EXPECT_TRUE(strncmp(str, expected_, length) == 0); + return true; + } + bool String(const char*, SizeType, bool) { return true; } + bool StartObject() { return true; } + bool Key(const char*, SizeType, bool) { return true; } + bool EndObject(SizeType) { return true; } + bool StartArray() { return true; } + bool EndArray(SizeType) { return true; } + + NumbersAsStringsHandler(const char* expected) + : expected_(expected) + , expected_len_(strlen(expected)) {} + + const char* expected_; + size_t expected_len_; +}; +TEST(Reader, NumbersAsStrings) +{ + { const char* json = "{ \"pi\": 3.1416 } "; StringStream s(json); - ParseObjectHandler h; + NumbersAsStringsHandler h("3.1416"); + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + } + { char* json = StrDup("{ \"pi\": 3.1416 } "); + InsituStringStream s(json); + NumbersAsStringsHandler h("3.1416"); + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + free(json); + } + { const char* json = "{ \"gigabyte\": 1.0e9 } "; + StringStream s(json); + NumbersAsStringsHandler h("1.0e9"); Reader reader; - EXPECT_FALSE(reader.Parse(s, h)); - EXPECT_EQ(kParseErrorUnspecificSyntaxError, reader.GetParseErrorCode()); + EXPECT_TRUE(reader.Parse(s, h)); + } + { char* json = StrDup("{ \"gigabyte\": 1.0e9 } "); + InsituStringStream s(json); + NumbersAsStringsHandler h("1.0e9"); + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + free(json); + } + { const char* json = "{ \"pi\": 314.159e-2 } "; + StringStream s(json); + NumbersAsStringsHandler h("314.159e-2"); + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + } + { char* json = StrDup("{ \"gigabyte\": 314.159e-2 } "); + InsituStringStream s(json); + NumbersAsStringsHandler h("314.159e-2"); + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + free(json); + } + { const char* json = "{ \"negative\": -1.54321 } "; + StringStream s(json); + NumbersAsStringsHandler h("-1.54321"); + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + } + { char* json = StrDup("{ \"negative\": -1.54321 } "); + InsituStringStream s(json); + NumbersAsStringsHandler h("-1.54321"); + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + free(json); + } + { const char* json = "{ \"pi\": 314.159e-2 } "; + std::stringstream ss(json); + IStreamWrapper s(ss); + NumbersAsStringsHandler h("314.159e-2"); + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + } } -TEST(Reader, IncompleteMultilineComment2) { - const char* json = "{\"hello\" : \"world\" /* *\0 */}"; - +template +void TestTrailingCommas() +{ + { StringStream s("[1,2,3,]"); + ParseArrayHandler<3> h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + EXPECT_EQ(5u, h.step_); + } + { const char* json = "{ \"hello\" : \"world\", \"t\" : true , \"f\" : false," + "\"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3],}"; StringStream s(json); ParseObjectHandler h; Reader reader; - EXPECT_FALSE(reader.Parse(s, h)); - EXPECT_EQ(kParseErrorUnspecificSyntaxError, reader.GetParseErrorCode()); -} - -TEST(Reader, UnrecognizedComment) { - const char* json = "{\"hello\" : \"world\" /! }"; - + EXPECT_TRUE(reader.Parse(s, h)); + EXPECT_EQ(20u, h.step_); + } + { // whitespace around trailing commas + const char* json = "{ \"hello\" : \"world\", \"t\" : true , \"f\" : false," + "\"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3\n,\n]\n,\n} "; StringStream s(json); ParseObjectHandler h; Reader reader; - EXPECT_FALSE(reader.Parse(s, h)); - EXPECT_EQ(kParseErrorUnspecificSyntaxError, reader.GetParseErrorCode()); -} - -struct NumbersAsStringsHandler { - bool Null() { return true; } - bool Bool(bool) { return true; } - bool Int(int) { return true; } - bool Uint(unsigned) { return true; } - bool Int64(int64_t) { return true; } - bool Uint64(uint64_t) { return true; } - bool Double(double) { return true; } - // 'str' is not null-terminated - bool RawNumber(const char* str, SizeType length, bool) { - EXPECT_TRUE(str != 0); - EXPECT_TRUE(expected_len_ == length); - EXPECT_TRUE(strncmp(str, expected_, length) == 0); - return true; - } - bool String(const char*, SizeType, bool) { return true; } - bool StartObject() { return true; } - bool Key(const char*, SizeType, bool) { return true; } - bool EndObject(SizeType) { return true; } - bool StartArray() { return true; } - bool EndArray(SizeType) { return true; } - - NumbersAsStringsHandler(const char* expected) - : expected_(expected) - , expected_len_(strlen(expected)) {} - - const char* expected_; - size_t expected_len_; -}; - -TEST(Reader, NumbersAsStrings) { - { - const char* json = "{ \"pi\": 3.1416 } "; - StringStream s(json); - NumbersAsStringsHandler h("3.1416"); - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - } - { - char* json = StrDup("{ \"pi\": 3.1416 } "); - InsituStringStream s(json); - NumbersAsStringsHandler h("3.1416"); - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - free(json); - } - { - const char* json = "{ \"gigabyte\": 1.0e9 } "; - StringStream s(json); - NumbersAsStringsHandler h("1.0e9"); - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - } - { - char* json = StrDup("{ \"gigabyte\": 1.0e9 } "); - InsituStringStream s(json); - NumbersAsStringsHandler h("1.0e9"); - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - free(json); - } - { - const char* json = "{ \"pi\": 314.159e-2 } "; - StringStream s(json); - NumbersAsStringsHandler h("314.159e-2"); - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - } - { - char* json = StrDup("{ \"gigabyte\": 314.159e-2 } "); - InsituStringStream s(json); - NumbersAsStringsHandler h("314.159e-2"); - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - free(json); - } - { - const char* json = "{ \"negative\": -1.54321 } "; - StringStream s(json); - NumbersAsStringsHandler h("-1.54321"); - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - } - { - char* json = StrDup("{ \"negative\": -1.54321 } "); - InsituStringStream s(json); - NumbersAsStringsHandler h("-1.54321"); - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - free(json); - } - { - const char* json = "{ \"pi\": 314.159e-2 } "; - std::stringstream ss(json); - IStreamWrapper s(ss); - NumbersAsStringsHandler h("314.159e-2"); - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - } -} - -template -void TestTrailingCommas() { - { - StringStream s("[1,2,3,]"); - ParseArrayHandler<3> h; - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - EXPECT_EQ(5u, h.step_); - } - { - const char* json = "{ \"hello\" : \"world\", \"t\" : true , \"f\" : false," - "\"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3],}"; - StringStream s(json); - ParseObjectHandler h; - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - EXPECT_EQ(20u, h.step_); - } - { - // whitespace around trailing commas - const char* json = "{ \"hello\" : \"world\", \"t\" : true , \"f\" : false," - "\"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3\n,\n]\n,\n} "; - StringStream s(json); - ParseObjectHandler h; - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - EXPECT_EQ(20u, h.step_); - } - { - // comments around trailing commas - const char* json = "{ \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null," - "\"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3/*test*/,/*test*/]/*test*/,/*test*/}"; - StringStream s(json); - ParseObjectHandler h; - Reader reader; - EXPECT_TRUE(reader.Parse(s, h)); - EXPECT_EQ(20u, h.step_); - } + EXPECT_TRUE(reader.Parse(s, h)); + EXPECT_EQ(20u, h.step_); + } + { // comments around trailing commas + const char* json = "{ \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null," + "\"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3/*test*/,/*test*/]/*test*/,/*test*/}"; + StringStream s(json); + ParseObjectHandler h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + EXPECT_EQ(20u, h.step_); + } } -TEST(Reader, TrailingCommas) { - TestTrailingCommas(); +TEST(Reader, TrailingCommas) +{ TestTrailingCommas(); } -TEST(Reader, TrailingCommasIterative) { - TestTrailingCommas(); +TEST(Reader, TrailingCommasIterative) +{ TestTrailingCommas(); } template -void TestMultipleTrailingCommaErrors() { - // only a single trailing comma is allowed. - { - StringStream s("[1,2,3,,]"); - ParseArrayHandler<3> h; - Reader reader; - ParseResult r = reader.Parse(s, h); - EXPECT_TRUE(reader.HasParseError()); - EXPECT_EQ(kParseErrorValueInvalid, r.Code()); - EXPECT_EQ(7u, r.Offset()); - } - { - const char* json = "{ \"hello\" : \"world\", \"t\" : true , \"f\" : false," - "\"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3,],,}"; - StringStream s(json); - ParseObjectHandler h; - Reader reader; - ParseResult r = reader.Parse(s, h); - EXPECT_TRUE(reader.HasParseError()); - EXPECT_EQ(kParseErrorObjectMissName, r.Code()); - EXPECT_EQ(95u, r.Offset()); - } +void TestMultipleTrailingCommaErrors() +{ // only a single trailing comma is allowed. + { StringStream s("[1,2,3,,]"); + ParseArrayHandler<3> h; + Reader reader; + ParseResult r = reader.Parse(s, h); + EXPECT_TRUE(reader.HasParseError()); + EXPECT_EQ(kParseErrorValueInvalid, r.Code()); + EXPECT_EQ(7u, r.Offset()); + } + { const char* json = "{ \"hello\" : \"world\", \"t\" : true , \"f\" : false," + "\"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3,],,}"; + StringStream s(json); + ParseObjectHandler h; + Reader reader; + ParseResult r = reader.Parse(s, h); + EXPECT_TRUE(reader.HasParseError()); + EXPECT_EQ(kParseErrorObjectMissName, r.Code()); + EXPECT_EQ(95u, r.Offset()); + } } -TEST(Reader, MultipleTrailingCommaErrors) { - TestMultipleTrailingCommaErrors(); +TEST(Reader, MultipleTrailingCommaErrors) +{ TestMultipleTrailingCommaErrors(); } -TEST(Reader, MultipleTrailingCommaErrorsIterative) { - TestMultipleTrailingCommaErrors(); +TEST(Reader, MultipleTrailingCommaErrorsIterative) +{ TestMultipleTrailingCommaErrors(); } template -void TestEmptyExceptForCommaErrors() { - // not allowed even with trailing commas enabled; the - // trailing comma must follow a value. - { - StringStream s("[,]"); - ParseArrayHandler<3> h; - Reader reader; - ParseResult r = reader.Parse(s, h); - EXPECT_TRUE(reader.HasParseError()); - EXPECT_EQ(kParseErrorValueInvalid, r.Code()); - EXPECT_EQ(1u, r.Offset()); - } - { - StringStream s("{,}"); - ParseObjectHandler h; - Reader reader; - ParseResult r = reader.Parse(s, h); - EXPECT_TRUE(reader.HasParseError()); - EXPECT_EQ(kParseErrorObjectMissName, r.Code()); - EXPECT_EQ(1u, r.Offset()); - } +void TestEmptyExceptForCommaErrors() +{ // not allowed even with trailing commas enabled; the + // trailing comma must follow a value. + { StringStream s("[,]"); + ParseArrayHandler<3> h; + Reader reader; + ParseResult r = reader.Parse(s, h); + EXPECT_TRUE(reader.HasParseError()); + EXPECT_EQ(kParseErrorValueInvalid, r.Code()); + EXPECT_EQ(1u, r.Offset()); + } + { StringStream s("{,}"); + ParseObjectHandler h; + Reader reader; + ParseResult r = reader.Parse(s, h); + EXPECT_TRUE(reader.HasParseError()); + EXPECT_EQ(kParseErrorObjectMissName, r.Code()); + EXPECT_EQ(1u, r.Offset()); + } } -TEST(Reader, EmptyExceptForCommaErrors) { - TestEmptyExceptForCommaErrors(); +TEST(Reader, EmptyExceptForCommaErrors) +{ TestEmptyExceptForCommaErrors(); } -TEST(Reader, EmptyExceptForCommaErrorsIterative) { - TestEmptyExceptForCommaErrors(); +TEST(Reader, EmptyExceptForCommaErrorsIterative) +{ TestEmptyExceptForCommaErrors(); } template -void TestTrailingCommaHandlerTermination() { - { - HandlerTerminateAtEndArray h; - Reader reader; - StringStream s("[1,2,3,]"); - ParseResult r = reader.Parse(s, h); - EXPECT_TRUE(reader.HasParseError()); - EXPECT_EQ(kParseErrorTermination, r.Code()); - EXPECT_EQ(7u, r.Offset()); - } - { - HandlerTerminateAtEndObject h; - Reader reader; - StringStream s("{\"t\": true, \"f\": false,}"); - ParseResult r = reader.Parse(s, h); - EXPECT_TRUE(reader.HasParseError()); - EXPECT_EQ(kParseErrorTermination, r.Code()); - EXPECT_EQ(23u, r.Offset()); - } +void TestTrailingCommaHandlerTermination() +{ + { HandlerTerminateAtEndArray h; + Reader reader; + StringStream s("[1,2,3,]"); + ParseResult r = reader.Parse(s, h); + EXPECT_TRUE(reader.HasParseError()); + EXPECT_EQ(kParseErrorTermination, r.Code()); + EXPECT_EQ(7u, r.Offset()); + } + { HandlerTerminateAtEndObject h; + Reader reader; + StringStream s("{\"t\": true, \"f\": false,}"); + ParseResult r = reader.Parse(s, h); + EXPECT_TRUE(reader.HasParseError()); + EXPECT_EQ(kParseErrorTermination, r.Code()); + EXPECT_EQ(23u, r.Offset()); + } } -TEST(Reader, TrailingCommaHandlerTermination) { - TestTrailingCommaHandlerTermination(); +TEST(Reader, TrailingCommaHandlerTermination) +{ TestTrailingCommaHandlerTermination(); } -TEST(Reader, TrailingCommaHandlerTerminationIterative) { - TestTrailingCommaHandlerTermination(); +TEST(Reader, TrailingCommaHandlerTerminationIterative) +{ TestTrailingCommaHandlerTermination(); } -TEST(Reader, ParseNanAndInfinity) { +TEST(Reader, ParseNanAndInfinity) +{ #define TEST_NAN_INF(str, x) \ { \ { \ @@ -1823,19 +1803,19 @@ TEST(Reader, ParseNanAndInfinity) { EXPECT_EQ(streamPos, s.Tell());\ } - double nan = std::numeric_limits::quiet_NaN(); - double inf = std::numeric_limits::infinity(); - - TEST_NAN_INF("NaN", nan); - TEST_NAN_INF("-NaN", nan); - TEST_NAN_INF("Inf", inf); - TEST_NAN_INF("Infinity", inf); - TEST_NAN_INF("-Inf", -inf); - TEST_NAN_INF("-Infinity", -inf); - TEST_NAN_INF_ERROR(kParseErrorValueInvalid, "nan", 1); - TEST_NAN_INF_ERROR(kParseErrorValueInvalid, "-nan", 1); - TEST_NAN_INF_ERROR(kParseErrorValueInvalid, "NAN", 1); - TEST_NAN_INF_ERROR(kParseErrorValueInvalid, "-Infinty", 6); + double nan = std::numeric_limits::quiet_NaN(); + double inf = std::numeric_limits::infinity(); + + TEST_NAN_INF("NaN", nan); + TEST_NAN_INF("-NaN", nan); + TEST_NAN_INF("Inf", inf); + TEST_NAN_INF("Infinity", inf); + TEST_NAN_INF("-Inf", -inf); + TEST_NAN_INF("-Infinity", -inf); + TEST_NAN_INF_ERROR(kParseErrorValueInvalid, "nan", 1); + TEST_NAN_INF_ERROR(kParseErrorValueInvalid, "-nan", 1); + TEST_NAN_INF_ERROR(kParseErrorValueInvalid, "NAN", 1); + TEST_NAN_INF_ERROR(kParseErrorValueInvalid, "-Infinty", 6); #undef TEST_NAN_INF_ERROR #undef TEST_NAN_INF diff --git a/rapidjson/test/unittest/regextest.cpp b/rapidjson/test/unittest/regextest.cpp index 4fb5b222e44..2df2e9eafda 100644 --- a/rapidjson/test/unittest/regextest.cpp +++ b/rapidjson/test/unittest/regextest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -17,576 +17,577 @@ using namespace rapidjson::internal; -TEST(Regex, Single) { - Regex re("a"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("a")); - EXPECT_FALSE(re.Match("")); - EXPECT_FALSE(re.Match("b")); -} - -TEST(Regex, Concatenation) { - Regex re("abc"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("abc")); - EXPECT_FALSE(re.Match("")); - EXPECT_FALSE(re.Match("a")); - EXPECT_FALSE(re.Match("b")); - EXPECT_FALSE(re.Match("ab")); - EXPECT_FALSE(re.Match("abcd")); -} - -TEST(Regex, Alternation1) { - Regex re("abab|abbb"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("abab")); - EXPECT_TRUE(re.Match("abbb")); - EXPECT_FALSE(re.Match("")); - EXPECT_FALSE(re.Match("ab")); - EXPECT_FALSE(re.Match("ababa")); - EXPECT_FALSE(re.Match("abb")); - EXPECT_FALSE(re.Match("abbbb")); -} - -TEST(Regex, Alternation2) { - Regex re("a|b|c"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("a")); - EXPECT_TRUE(re.Match("b")); - EXPECT_TRUE(re.Match("c")); - EXPECT_FALSE(re.Match("")); - EXPECT_FALSE(re.Match("aa")); - EXPECT_FALSE(re.Match("ab")); -} - -TEST(Regex, Parenthesis1) { - Regex re("(ab)c"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("abc")); - EXPECT_FALSE(re.Match("")); - EXPECT_FALSE(re.Match("a")); - EXPECT_FALSE(re.Match("b")); - EXPECT_FALSE(re.Match("ab")); - EXPECT_FALSE(re.Match("abcd")); -} - -TEST(Regex, Parenthesis2) { - Regex re("a(bc)"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("abc")); - EXPECT_FALSE(re.Match("")); - EXPECT_FALSE(re.Match("a")); - EXPECT_FALSE(re.Match("b")); - EXPECT_FALSE(re.Match("ab")); - EXPECT_FALSE(re.Match("abcd")); -} - -TEST(Regex, Parenthesis3) { - Regex re("(a|b)(c|d)"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("ac")); - EXPECT_TRUE(re.Match("ad")); - EXPECT_TRUE(re.Match("bc")); - EXPECT_TRUE(re.Match("bd")); - EXPECT_FALSE(re.Match("")); - EXPECT_FALSE(re.Match("ab")); - EXPECT_FALSE(re.Match("cd")); -} - -TEST(Regex, ZeroOrOne1) { - Regex re("a?"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("")); - EXPECT_TRUE(re.Match("a")); - EXPECT_FALSE(re.Match("aa")); -} - -TEST(Regex, ZeroOrOne2) { - Regex re("a?b"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("b")); - EXPECT_TRUE(re.Match("ab")); - EXPECT_FALSE(re.Match("a")); - EXPECT_FALSE(re.Match("aa")); - EXPECT_FALSE(re.Match("bb")); - EXPECT_FALSE(re.Match("ba")); -} - -TEST(Regex, ZeroOrOne3) { - Regex re("ab?"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("a")); - EXPECT_TRUE(re.Match("ab")); - EXPECT_FALSE(re.Match("b")); - EXPECT_FALSE(re.Match("aa")); - EXPECT_FALSE(re.Match("bb")); - EXPECT_FALSE(re.Match("ba")); -} - -TEST(Regex, ZeroOrOne4) { - Regex re("a?b?"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("")); - EXPECT_TRUE(re.Match("a")); - EXPECT_TRUE(re.Match("b")); - EXPECT_TRUE(re.Match("ab")); - EXPECT_FALSE(re.Match("aa")); - EXPECT_FALSE(re.Match("bb")); - EXPECT_FALSE(re.Match("ba")); - EXPECT_FALSE(re.Match("abc")); -} - -TEST(Regex, ZeroOrOne5) { - Regex re("a(ab)?b"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("ab")); - EXPECT_TRUE(re.Match("aabb")); - EXPECT_FALSE(re.Match("aab")); - EXPECT_FALSE(re.Match("abb")); -} - -TEST(Regex, ZeroOrMore1) { - Regex re("a*"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("")); - EXPECT_TRUE(re.Match("a")); - EXPECT_TRUE(re.Match("aa")); - EXPECT_FALSE(re.Match("b")); - EXPECT_FALSE(re.Match("ab")); -} - -TEST(Regex, ZeroOrMore2) { - Regex re("a*b"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("b")); - EXPECT_TRUE(re.Match("ab")); - EXPECT_TRUE(re.Match("aab")); - EXPECT_FALSE(re.Match("")); - EXPECT_FALSE(re.Match("bb")); -} - -TEST(Regex, ZeroOrMore3) { - Regex re("a*b*"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("")); - EXPECT_TRUE(re.Match("a")); - EXPECT_TRUE(re.Match("aa")); - EXPECT_TRUE(re.Match("b")); - EXPECT_TRUE(re.Match("bb")); - EXPECT_TRUE(re.Match("ab")); - EXPECT_TRUE(re.Match("aabb")); - EXPECT_FALSE(re.Match("ba")); -} - -TEST(Regex, ZeroOrMore4) { - Regex re("a(ab)*b"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("ab")); - EXPECT_TRUE(re.Match("aabb")); - EXPECT_TRUE(re.Match("aababb")); - EXPECT_FALSE(re.Match("")); - EXPECT_FALSE(re.Match("aa")); -} - -TEST(Regex, OneOrMore1) { - Regex re("a+"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("a")); - EXPECT_TRUE(re.Match("aa")); - EXPECT_FALSE(re.Match("")); - EXPECT_FALSE(re.Match("b")); - EXPECT_FALSE(re.Match("ab")); -} - -TEST(Regex, OneOrMore2) { - Regex re("a+b"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("ab")); - EXPECT_TRUE(re.Match("aab")); - EXPECT_FALSE(re.Match("")); - EXPECT_FALSE(re.Match("b")); -} - -TEST(Regex, OneOrMore3) { - Regex re("a+b+"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("ab")); - EXPECT_TRUE(re.Match("aab")); - EXPECT_TRUE(re.Match("abb")); - EXPECT_TRUE(re.Match("aabb")); - EXPECT_FALSE(re.Match("")); - EXPECT_FALSE(re.Match("b")); - EXPECT_FALSE(re.Match("ba")); -} - -TEST(Regex, OneOrMore4) { - Regex re("a(ab)+b"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("aabb")); - EXPECT_TRUE(re.Match("aababb")); - EXPECT_FALSE(re.Match("")); - EXPECT_FALSE(re.Match("ab")); -} - -TEST(Regex, QuantifierExact1) { - Regex re("ab{3}c"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("abbbc")); - EXPECT_FALSE(re.Match("ac")); - EXPECT_FALSE(re.Match("abc")); - EXPECT_FALSE(re.Match("abbc")); - EXPECT_FALSE(re.Match("abbbbc")); -} - -TEST(Regex, QuantifierExact2) { - Regex re("a(bc){3}d"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("abcbcbcd")); - EXPECT_FALSE(re.Match("ad")); - EXPECT_FALSE(re.Match("abcd")); - EXPECT_FALSE(re.Match("abcbcd")); - EXPECT_FALSE(re.Match("abcbcbcbcd")); -} - -TEST(Regex, QuantifierExact3) { - Regex re("a(b|c){3}d"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("abbbd")); - EXPECT_TRUE(re.Match("acccd")); - EXPECT_TRUE(re.Match("abcbd")); - EXPECT_FALSE(re.Match("ad")); - EXPECT_FALSE(re.Match("abbd")); - EXPECT_FALSE(re.Match("accccd")); - EXPECT_FALSE(re.Match("abbbbd")); -} - -TEST(Regex, QuantifierMin1) { - Regex re("ab{3,}c"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("abbbc")); - EXPECT_TRUE(re.Match("abbbbc")); - EXPECT_TRUE(re.Match("abbbbbc")); - EXPECT_FALSE(re.Match("ac")); - EXPECT_FALSE(re.Match("abc")); - EXPECT_FALSE(re.Match("abbc")); -} - -TEST(Regex, QuantifierMin2) { - Regex re("a(bc){3,}d"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("abcbcbcd")); - EXPECT_TRUE(re.Match("abcbcbcbcd")); - EXPECT_FALSE(re.Match("ad")); - EXPECT_FALSE(re.Match("abcd")); - EXPECT_FALSE(re.Match("abcbcd")); -} - -TEST(Regex, QuantifierMin3) { - Regex re("a(b|c){3,}d"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("abbbd")); - EXPECT_TRUE(re.Match("acccd")); - EXPECT_TRUE(re.Match("abcbd")); - EXPECT_TRUE(re.Match("accccd")); - EXPECT_TRUE(re.Match("abbbbd")); - EXPECT_FALSE(re.Match("ad")); - EXPECT_FALSE(re.Match("abbd")); -} - -TEST(Regex, QuantifierMinMax1) { - Regex re("ab{3,5}c"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("abbbc")); - EXPECT_TRUE(re.Match("abbbbc")); - EXPECT_TRUE(re.Match("abbbbbc")); - EXPECT_FALSE(re.Match("ac")); - EXPECT_FALSE(re.Match("abc")); - EXPECT_FALSE(re.Match("abbc")); - EXPECT_FALSE(re.Match("abbbbbbc")); -} - -TEST(Regex, QuantifierMinMax2) { - Regex re("a(bc){3,5}d"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("abcbcbcd")); - EXPECT_TRUE(re.Match("abcbcbcbcd")); - EXPECT_TRUE(re.Match("abcbcbcbcbcd")); - EXPECT_FALSE(re.Match("ad")); - EXPECT_FALSE(re.Match("abcd")); - EXPECT_FALSE(re.Match("abcbcd")); - EXPECT_FALSE(re.Match("abcbcbcbcbcbcd")); -} - -TEST(Regex, QuantifierMinMax3) { - Regex re("a(b|c){3,5}d"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("abbbd")); - EXPECT_TRUE(re.Match("acccd")); - EXPECT_TRUE(re.Match("abcbd")); - EXPECT_TRUE(re.Match("accccd")); - EXPECT_TRUE(re.Match("abbbbd")); - EXPECT_TRUE(re.Match("acccccd")); - EXPECT_TRUE(re.Match("abbbbbd")); - EXPECT_FALSE(re.Match("ad")); - EXPECT_FALSE(re.Match("abbd")); - EXPECT_FALSE(re.Match("accccccd")); - EXPECT_FALSE(re.Match("abbbbbbd")); +TEST(Regex, Single) +{ Regex re("a"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("a")); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("b")); +} + +TEST(Regex, Concatenation) +{ Regex re("abc"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("abc")); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("a")); + EXPECT_FALSE(re.Match("b")); + EXPECT_FALSE(re.Match("ab")); + EXPECT_FALSE(re.Match("abcd")); +} + +TEST(Regex, Alternation1) +{ Regex re("abab|abbb"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("abab")); + EXPECT_TRUE(re.Match("abbb")); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("ab")); + EXPECT_FALSE(re.Match("ababa")); + EXPECT_FALSE(re.Match("abb")); + EXPECT_FALSE(re.Match("abbbb")); +} + +TEST(Regex, Alternation2) +{ Regex re("a|b|c"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("a")); + EXPECT_TRUE(re.Match("b")); + EXPECT_TRUE(re.Match("c")); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("aa")); + EXPECT_FALSE(re.Match("ab")); +} + +TEST(Regex, Parenthesis1) +{ Regex re("(ab)c"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("abc")); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("a")); + EXPECT_FALSE(re.Match("b")); + EXPECT_FALSE(re.Match("ab")); + EXPECT_FALSE(re.Match("abcd")); +} + +TEST(Regex, Parenthesis2) +{ Regex re("a(bc)"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("abc")); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("a")); + EXPECT_FALSE(re.Match("b")); + EXPECT_FALSE(re.Match("ab")); + EXPECT_FALSE(re.Match("abcd")); +} + +TEST(Regex, Parenthesis3) +{ Regex re("(a|b)(c|d)"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("ac")); + EXPECT_TRUE(re.Match("ad")); + EXPECT_TRUE(re.Match("bc")); + EXPECT_TRUE(re.Match("bd")); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("ab")); + EXPECT_FALSE(re.Match("cd")); +} + +TEST(Regex, ZeroOrOne1) +{ Regex re("a?"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("")); + EXPECT_TRUE(re.Match("a")); + EXPECT_FALSE(re.Match("aa")); +} + +TEST(Regex, ZeroOrOne2) +{ Regex re("a?b"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("b")); + EXPECT_TRUE(re.Match("ab")); + EXPECT_FALSE(re.Match("a")); + EXPECT_FALSE(re.Match("aa")); + EXPECT_FALSE(re.Match("bb")); + EXPECT_FALSE(re.Match("ba")); +} + +TEST(Regex, ZeroOrOne3) +{ Regex re("ab?"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("a")); + EXPECT_TRUE(re.Match("ab")); + EXPECT_FALSE(re.Match("b")); + EXPECT_FALSE(re.Match("aa")); + EXPECT_FALSE(re.Match("bb")); + EXPECT_FALSE(re.Match("ba")); +} + +TEST(Regex, ZeroOrOne4) +{ Regex re("a?b?"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("")); + EXPECT_TRUE(re.Match("a")); + EXPECT_TRUE(re.Match("b")); + EXPECT_TRUE(re.Match("ab")); + EXPECT_FALSE(re.Match("aa")); + EXPECT_FALSE(re.Match("bb")); + EXPECT_FALSE(re.Match("ba")); + EXPECT_FALSE(re.Match("abc")); +} + +TEST(Regex, ZeroOrOne5) +{ Regex re("a(ab)?b"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("ab")); + EXPECT_TRUE(re.Match("aabb")); + EXPECT_FALSE(re.Match("aab")); + EXPECT_FALSE(re.Match("abb")); +} + +TEST(Regex, ZeroOrMore1) +{ Regex re("a*"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("")); + EXPECT_TRUE(re.Match("a")); + EXPECT_TRUE(re.Match("aa")); + EXPECT_FALSE(re.Match("b")); + EXPECT_FALSE(re.Match("ab")); +} + +TEST(Regex, ZeroOrMore2) +{ Regex re("a*b"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("b")); + EXPECT_TRUE(re.Match("ab")); + EXPECT_TRUE(re.Match("aab")); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("bb")); +} + +TEST(Regex, ZeroOrMore3) +{ Regex re("a*b*"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("")); + EXPECT_TRUE(re.Match("a")); + EXPECT_TRUE(re.Match("aa")); + EXPECT_TRUE(re.Match("b")); + EXPECT_TRUE(re.Match("bb")); + EXPECT_TRUE(re.Match("ab")); + EXPECT_TRUE(re.Match("aabb")); + EXPECT_FALSE(re.Match("ba")); +} + +TEST(Regex, ZeroOrMore4) +{ Regex re("a(ab)*b"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("ab")); + EXPECT_TRUE(re.Match("aabb")); + EXPECT_TRUE(re.Match("aababb")); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("aa")); +} + +TEST(Regex, OneOrMore1) +{ Regex re("a+"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("a")); + EXPECT_TRUE(re.Match("aa")); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("b")); + EXPECT_FALSE(re.Match("ab")); +} + +TEST(Regex, OneOrMore2) +{ Regex re("a+b"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("ab")); + EXPECT_TRUE(re.Match("aab")); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("b")); +} + +TEST(Regex, OneOrMore3) +{ Regex re("a+b+"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("ab")); + EXPECT_TRUE(re.Match("aab")); + EXPECT_TRUE(re.Match("abb")); + EXPECT_TRUE(re.Match("aabb")); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("b")); + EXPECT_FALSE(re.Match("ba")); +} + +TEST(Regex, OneOrMore4) +{ Regex re("a(ab)+b"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("aabb")); + EXPECT_TRUE(re.Match("aababb")); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("ab")); +} + +TEST(Regex, QuantifierExact1) +{ Regex re("ab{3}c"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("abbbc")); + EXPECT_FALSE(re.Match("ac")); + EXPECT_FALSE(re.Match("abc")); + EXPECT_FALSE(re.Match("abbc")); + EXPECT_FALSE(re.Match("abbbbc")); +} + +TEST(Regex, QuantifierExact2) +{ Regex re("a(bc){3}d"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("abcbcbcd")); + EXPECT_FALSE(re.Match("ad")); + EXPECT_FALSE(re.Match("abcd")); + EXPECT_FALSE(re.Match("abcbcd")); + EXPECT_FALSE(re.Match("abcbcbcbcd")); +} + +TEST(Regex, QuantifierExact3) +{ Regex re("a(b|c){3}d"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("abbbd")); + EXPECT_TRUE(re.Match("acccd")); + EXPECT_TRUE(re.Match("abcbd")); + EXPECT_FALSE(re.Match("ad")); + EXPECT_FALSE(re.Match("abbd")); + EXPECT_FALSE(re.Match("accccd")); + EXPECT_FALSE(re.Match("abbbbd")); +} + +TEST(Regex, QuantifierMin1) +{ Regex re("ab{3,}c"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("abbbc")); + EXPECT_TRUE(re.Match("abbbbc")); + EXPECT_TRUE(re.Match("abbbbbc")); + EXPECT_FALSE(re.Match("ac")); + EXPECT_FALSE(re.Match("abc")); + EXPECT_FALSE(re.Match("abbc")); +} + +TEST(Regex, QuantifierMin2) +{ Regex re("a(bc){3,}d"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("abcbcbcd")); + EXPECT_TRUE(re.Match("abcbcbcbcd")); + EXPECT_FALSE(re.Match("ad")); + EXPECT_FALSE(re.Match("abcd")); + EXPECT_FALSE(re.Match("abcbcd")); +} + +TEST(Regex, QuantifierMin3) +{ Regex re("a(b|c){3,}d"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("abbbd")); + EXPECT_TRUE(re.Match("acccd")); + EXPECT_TRUE(re.Match("abcbd")); + EXPECT_TRUE(re.Match("accccd")); + EXPECT_TRUE(re.Match("abbbbd")); + EXPECT_FALSE(re.Match("ad")); + EXPECT_FALSE(re.Match("abbd")); +} + +TEST(Regex, QuantifierMinMax1) +{ Regex re("ab{3,5}c"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("abbbc")); + EXPECT_TRUE(re.Match("abbbbc")); + EXPECT_TRUE(re.Match("abbbbbc")); + EXPECT_FALSE(re.Match("ac")); + EXPECT_FALSE(re.Match("abc")); + EXPECT_FALSE(re.Match("abbc")); + EXPECT_FALSE(re.Match("abbbbbbc")); +} + +TEST(Regex, QuantifierMinMax2) +{ Regex re("a(bc){3,5}d"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("abcbcbcd")); + EXPECT_TRUE(re.Match("abcbcbcbcd")); + EXPECT_TRUE(re.Match("abcbcbcbcbcd")); + EXPECT_FALSE(re.Match("ad")); + EXPECT_FALSE(re.Match("abcd")); + EXPECT_FALSE(re.Match("abcbcd")); + EXPECT_FALSE(re.Match("abcbcbcbcbcbcd")); +} + +TEST(Regex, QuantifierMinMax3) +{ Regex re("a(b|c){3,5}d"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("abbbd")); + EXPECT_TRUE(re.Match("acccd")); + EXPECT_TRUE(re.Match("abcbd")); + EXPECT_TRUE(re.Match("accccd")); + EXPECT_TRUE(re.Match("abbbbd")); + EXPECT_TRUE(re.Match("acccccd")); + EXPECT_TRUE(re.Match("abbbbbd")); + EXPECT_FALSE(re.Match("ad")); + EXPECT_FALSE(re.Match("abbd")); + EXPECT_FALSE(re.Match("accccccd")); + EXPECT_FALSE(re.Match("abbbbbbd")); } // Issue538 -TEST(Regex, QuantifierMinMax4) { - Regex re("a(b|c){0,3}d"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("ad")); - EXPECT_TRUE(re.Match("abd")); - EXPECT_TRUE(re.Match("acd")); - EXPECT_TRUE(re.Match("abbd")); - EXPECT_TRUE(re.Match("accd")); - EXPECT_TRUE(re.Match("abcd")); - EXPECT_TRUE(re.Match("abbbd")); - EXPECT_TRUE(re.Match("acccd")); - EXPECT_FALSE(re.Match("abbbbd")); - EXPECT_FALSE(re.Match("add")); - EXPECT_FALSE(re.Match("accccd")); - EXPECT_FALSE(re.Match("abcbcd")); +TEST(Regex, QuantifierMinMax4) +{ Regex re("a(b|c){0,3}d"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("ad")); + EXPECT_TRUE(re.Match("abd")); + EXPECT_TRUE(re.Match("acd")); + EXPECT_TRUE(re.Match("abbd")); + EXPECT_TRUE(re.Match("accd")); + EXPECT_TRUE(re.Match("abcd")); + EXPECT_TRUE(re.Match("abbbd")); + EXPECT_TRUE(re.Match("acccd")); + EXPECT_FALSE(re.Match("abbbbd")); + EXPECT_FALSE(re.Match("add")); + EXPECT_FALSE(re.Match("accccd")); + EXPECT_FALSE(re.Match("abcbcd")); } // Issue538 -TEST(Regex, QuantifierMinMax5) { - Regex re("a(b|c){0,}d"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("ad")); - EXPECT_TRUE(re.Match("abd")); - EXPECT_TRUE(re.Match("acd")); - EXPECT_TRUE(re.Match("abbd")); - EXPECT_TRUE(re.Match("accd")); - EXPECT_TRUE(re.Match("abcd")); - EXPECT_TRUE(re.Match("abbbd")); - EXPECT_TRUE(re.Match("acccd")); - EXPECT_TRUE(re.Match("abbbbd")); - EXPECT_TRUE(re.Match("accccd")); - EXPECT_TRUE(re.Match("abcbcd")); - EXPECT_FALSE(re.Match("add")); - EXPECT_FALSE(re.Match("aad")); +TEST(Regex, QuantifierMinMax5) +{ Regex re("a(b|c){0,}d"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("ad")); + EXPECT_TRUE(re.Match("abd")); + EXPECT_TRUE(re.Match("acd")); + EXPECT_TRUE(re.Match("abbd")); + EXPECT_TRUE(re.Match("accd")); + EXPECT_TRUE(re.Match("abcd")); + EXPECT_TRUE(re.Match("abbbd")); + EXPECT_TRUE(re.Match("acccd")); + EXPECT_TRUE(re.Match("abbbbd")); + EXPECT_TRUE(re.Match("accccd")); + EXPECT_TRUE(re.Match("abcbcd")); + EXPECT_FALSE(re.Match("add")); + EXPECT_FALSE(re.Match("aad")); } #define EURO "\xE2\x82\xAC" // "\xE2\x82\xAC" is UTF-8 sequence of Euro sign U+20AC -TEST(Regex, Unicode) { - Regex re("a" EURO "+b"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("a" EURO "b")); - EXPECT_TRUE(re.Match("a" EURO EURO "b")); - EXPECT_FALSE(re.Match("a?b")); - EXPECT_FALSE(re.Match("a" EURO "\xAC" "b")); // unaware of UTF-8 will match -} - -TEST(Regex, AnyCharacter) { - Regex re("."); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("a")); - EXPECT_TRUE(re.Match("b")); - EXPECT_TRUE(re.Match(EURO)); - EXPECT_FALSE(re.Match("")); - EXPECT_FALSE(re.Match("aa")); -} - -TEST(Regex, CharacterRange1) { - Regex re("[abc]"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("a")); - EXPECT_TRUE(re.Match("b")); - EXPECT_TRUE(re.Match("c")); - EXPECT_FALSE(re.Match("")); - EXPECT_FALSE(re.Match("`")); - EXPECT_FALSE(re.Match("d")); - EXPECT_FALSE(re.Match("aa")); -} - -TEST(Regex, CharacterRange2) { - Regex re("[^abc]"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("`")); - EXPECT_TRUE(re.Match("d")); - EXPECT_FALSE(re.Match("a")); - EXPECT_FALSE(re.Match("b")); - EXPECT_FALSE(re.Match("c")); - EXPECT_FALSE(re.Match("")); - EXPECT_FALSE(re.Match("aa")); -} - -TEST(Regex, CharacterRange3) { - Regex re("[a-c]"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("a")); - EXPECT_TRUE(re.Match("b")); - EXPECT_TRUE(re.Match("c")); - EXPECT_FALSE(re.Match("")); - EXPECT_FALSE(re.Match("`")); - EXPECT_FALSE(re.Match("d")); - EXPECT_FALSE(re.Match("aa")); -} - -TEST(Regex, CharacterRange4) { - Regex re("[^a-c]"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("`")); - EXPECT_TRUE(re.Match("d")); - EXPECT_FALSE(re.Match("a")); - EXPECT_FALSE(re.Match("b")); - EXPECT_FALSE(re.Match("c")); - EXPECT_FALSE(re.Match("")); - EXPECT_FALSE(re.Match("aa")); -} - -TEST(Regex, CharacterRange5) { - Regex re("[-]"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("-")); - EXPECT_FALSE(re.Match("")); - EXPECT_FALSE(re.Match("a")); -} - -TEST(Regex, CharacterRange6) { - Regex re("[a-]"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("a")); - EXPECT_TRUE(re.Match("-")); - EXPECT_FALSE(re.Match("")); - EXPECT_FALSE(re.Match("`")); - EXPECT_FALSE(re.Match("b")); -} - -TEST(Regex, CharacterRange7) { - Regex re("[-a]"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("a")); - EXPECT_TRUE(re.Match("-")); - EXPECT_FALSE(re.Match("")); - EXPECT_FALSE(re.Match("`")); - EXPECT_FALSE(re.Match("b")); -} - -TEST(Regex, CharacterRange8) { - Regex re("[a-zA-Z0-9]*"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("Milo")); - EXPECT_TRUE(re.Match("MT19937")); - EXPECT_TRUE(re.Match("43")); - EXPECT_FALSE(re.Match("a_b")); - EXPECT_FALSE(re.Match("!")); -} - -TEST(Regex, Search) { - Regex re("abc"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Search("abc")); - EXPECT_TRUE(re.Search("_abc")); - EXPECT_TRUE(re.Search("abc_")); - EXPECT_TRUE(re.Search("_abc_")); - EXPECT_TRUE(re.Search("__abc__")); - EXPECT_TRUE(re.Search("abcabc")); - EXPECT_FALSE(re.Search("a")); - EXPECT_FALSE(re.Search("ab")); - EXPECT_FALSE(re.Search("bc")); - EXPECT_FALSE(re.Search("cba")); -} - -TEST(Regex, Search_BeginAnchor) { - Regex re("^abc"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Search("abc")); - EXPECT_TRUE(re.Search("abc_")); - EXPECT_TRUE(re.Search("abcabc")); - EXPECT_FALSE(re.Search("_abc")); - EXPECT_FALSE(re.Search("_abc_")); - EXPECT_FALSE(re.Search("a")); - EXPECT_FALSE(re.Search("ab")); - EXPECT_FALSE(re.Search("bc")); - EXPECT_FALSE(re.Search("cba")); -} - -TEST(Regex, Search_EndAnchor) { - Regex re("abc$"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Search("abc")); - EXPECT_TRUE(re.Search("_abc")); - EXPECT_TRUE(re.Search("abcabc")); - EXPECT_FALSE(re.Search("abc_")); - EXPECT_FALSE(re.Search("_abc_")); - EXPECT_FALSE(re.Search("a")); - EXPECT_FALSE(re.Search("ab")); - EXPECT_FALSE(re.Search("bc")); - EXPECT_FALSE(re.Search("cba")); -} - -TEST(Regex, Search_BothAnchor) { - Regex re("^abc$"); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Search("abc")); - EXPECT_FALSE(re.Search("")); - EXPECT_FALSE(re.Search("a")); - EXPECT_FALSE(re.Search("b")); - EXPECT_FALSE(re.Search("ab")); - EXPECT_FALSE(re.Search("abcd")); -} - -TEST(Regex, Escape) { - const char* s = "\\^\\$\\|\\(\\)\\?\\*\\+\\.\\[\\]\\{\\}\\\\\\f\\n\\r\\t\\v[\\b][\\[][\\]]"; - Regex re(s); - ASSERT_TRUE(re.IsValid()); - EXPECT_TRUE(re.Match("^$|()?*+.[]{}\\\x0C\n\r\t\x0B\b[]")); - EXPECT_FALSE(re.Match(s)); // Not escaping -} - -TEST(Regex, Invalid) { +TEST(Regex, Unicode) +{ Regex re("a" EURO "+b"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("a" EURO "b")); + EXPECT_TRUE(re.Match("a" EURO EURO "b")); + EXPECT_FALSE(re.Match("a?b")); + EXPECT_FALSE(re.Match("a" EURO "\xAC" "b")); // unaware of UTF-8 will match +} + +TEST(Regex, AnyCharacter) +{ Regex re("."); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("a")); + EXPECT_TRUE(re.Match("b")); + EXPECT_TRUE(re.Match(EURO)); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("aa")); +} + +TEST(Regex, CharacterRange1) +{ Regex re("[abc]"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("a")); + EXPECT_TRUE(re.Match("b")); + EXPECT_TRUE(re.Match("c")); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("`")); + EXPECT_FALSE(re.Match("d")); + EXPECT_FALSE(re.Match("aa")); +} + +TEST(Regex, CharacterRange2) +{ Regex re("[^abc]"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("`")); + EXPECT_TRUE(re.Match("d")); + EXPECT_FALSE(re.Match("a")); + EXPECT_FALSE(re.Match("b")); + EXPECT_FALSE(re.Match("c")); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("aa")); +} + +TEST(Regex, CharacterRange3) +{ Regex re("[a-c]"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("a")); + EXPECT_TRUE(re.Match("b")); + EXPECT_TRUE(re.Match("c")); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("`")); + EXPECT_FALSE(re.Match("d")); + EXPECT_FALSE(re.Match("aa")); +} + +TEST(Regex, CharacterRange4) +{ Regex re("[^a-c]"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("`")); + EXPECT_TRUE(re.Match("d")); + EXPECT_FALSE(re.Match("a")); + EXPECT_FALSE(re.Match("b")); + EXPECT_FALSE(re.Match("c")); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("aa")); +} + +TEST(Regex, CharacterRange5) +{ Regex re("[-]"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("-")); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("a")); +} + +TEST(Regex, CharacterRange6) +{ Regex re("[a-]"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("a")); + EXPECT_TRUE(re.Match("-")); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("`")); + EXPECT_FALSE(re.Match("b")); +} + +TEST(Regex, CharacterRange7) +{ Regex re("[-a]"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("a")); + EXPECT_TRUE(re.Match("-")); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("`")); + EXPECT_FALSE(re.Match("b")); +} + +TEST(Regex, CharacterRange8) +{ Regex re("[a-zA-Z0-9]*"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("Milo")); + EXPECT_TRUE(re.Match("MT19937")); + EXPECT_TRUE(re.Match("43")); + EXPECT_FALSE(re.Match("a_b")); + EXPECT_FALSE(re.Match("!")); +} + +TEST(Regex, Search) +{ Regex re("abc"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Search("abc")); + EXPECT_TRUE(re.Search("_abc")); + EXPECT_TRUE(re.Search("abc_")); + EXPECT_TRUE(re.Search("_abc_")); + EXPECT_TRUE(re.Search("__abc__")); + EXPECT_TRUE(re.Search("abcabc")); + EXPECT_FALSE(re.Search("a")); + EXPECT_FALSE(re.Search("ab")); + EXPECT_FALSE(re.Search("bc")); + EXPECT_FALSE(re.Search("cba")); +} + +TEST(Regex, Search_BeginAnchor) +{ Regex re("^abc"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Search("abc")); + EXPECT_TRUE(re.Search("abc_")); + EXPECT_TRUE(re.Search("abcabc")); + EXPECT_FALSE(re.Search("_abc")); + EXPECT_FALSE(re.Search("_abc_")); + EXPECT_FALSE(re.Search("a")); + EXPECT_FALSE(re.Search("ab")); + EXPECT_FALSE(re.Search("bc")); + EXPECT_FALSE(re.Search("cba")); +} + +TEST(Regex, Search_EndAnchor) +{ Regex re("abc$"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Search("abc")); + EXPECT_TRUE(re.Search("_abc")); + EXPECT_TRUE(re.Search("abcabc")); + EXPECT_FALSE(re.Search("abc_")); + EXPECT_FALSE(re.Search("_abc_")); + EXPECT_FALSE(re.Search("a")); + EXPECT_FALSE(re.Search("ab")); + EXPECT_FALSE(re.Search("bc")); + EXPECT_FALSE(re.Search("cba")); +} + +TEST(Regex, Search_BothAnchor) +{ Regex re("^abc$"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Search("abc")); + EXPECT_FALSE(re.Search("")); + EXPECT_FALSE(re.Search("a")); + EXPECT_FALSE(re.Search("b")); + EXPECT_FALSE(re.Search("ab")); + EXPECT_FALSE(re.Search("abcd")); +} + +TEST(Regex, Escape) +{ const char* s = "\\^\\$\\|\\(\\)\\?\\*\\+\\.\\[\\]\\{\\}\\\\\\f\\n\\r\\t\\v[\\b][\\[][\\]]"; + Regex re(s); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("^$|()?*+.[]{}\\\x0C\n\r\t\x0B\b[]")); + EXPECT_FALSE(re.Match(s)); // Not escaping +} + +TEST(Regex, Invalid) +{ #define TEST_INVALID(s) \ {\ Regex re(s);\ EXPECT_FALSE(re.IsValid());\ } - TEST_INVALID(""); - TEST_INVALID("a|"); - TEST_INVALID("()"); - TEST_INVALID(")"); - TEST_INVALID("(a))"); - TEST_INVALID("(a|)"); - TEST_INVALID("(a||b)"); - TEST_INVALID("(|b)"); - TEST_INVALID("?"); - TEST_INVALID("*"); - TEST_INVALID("+"); - TEST_INVALID("{"); - TEST_INVALID("{}"); - TEST_INVALID("a{a}"); - TEST_INVALID("a{0}"); - TEST_INVALID("a{-1}"); - TEST_INVALID("a{}"); - // TEST_INVALID("a{0,}"); // Support now - TEST_INVALID("a{,0}"); - TEST_INVALID("a{1,0}"); - TEST_INVALID("a{-1,0}"); - TEST_INVALID("a{-1,1}"); - TEST_INVALID("a{4294967296}"); // overflow of unsigned - TEST_INVALID("a{1a}"); - TEST_INVALID("["); - TEST_INVALID("[]"); - TEST_INVALID("[^]"); - TEST_INVALID("[\\a]"); - TEST_INVALID("\\a"); + TEST_INVALID(""); + TEST_INVALID("a|"); + TEST_INVALID("()"); + TEST_INVALID(")"); + TEST_INVALID("(a))"); + TEST_INVALID("(a|)"); + TEST_INVALID("(a||b)"); + TEST_INVALID("(|b)"); + TEST_INVALID("?"); + TEST_INVALID("*"); + TEST_INVALID("+"); + TEST_INVALID("{"); + TEST_INVALID("{}"); + TEST_INVALID("a{a}"); + TEST_INVALID("a{0}"); + TEST_INVALID("a{-1}"); + TEST_INVALID("a{}"); + // TEST_INVALID("a{0,}"); // Support now + TEST_INVALID("a{,0}"); + TEST_INVALID("a{1,0}"); + TEST_INVALID("a{-1,0}"); + TEST_INVALID("a{-1,1}"); + TEST_INVALID("a{4294967296}"); // overflow of unsigned + TEST_INVALID("a{1a}"); + TEST_INVALID("["); + TEST_INVALID("[]"); + TEST_INVALID("[^]"); + TEST_INVALID("[\\a]"); + TEST_INVALID("\\a"); #undef TEST_INVALID } -TEST(Regex, Issue538) { - Regex re("^[0-9]+(\\\\.[0-9]+){0,2}"); - EXPECT_TRUE(re.IsValid()); +TEST(Regex, Issue538) +{ Regex re("^[0-9]+(\\\\.[0-9]+){0,2}"); + EXPECT_TRUE(re.IsValid()); } -TEST(Regex, Issue583) { - Regex re("[0-9]{99999}"); - ASSERT_TRUE(re.IsValid()); +TEST(Regex, Issue583) +{ Regex re("[0-9]{99999}"); + ASSERT_TRUE(re.IsValid()); } #undef EURO diff --git a/rapidjson/test/unittest/schematest.cpp b/rapidjson/test/unittest/schematest.cpp index d75b1e593e0..e2d4d499450 100644 --- a/rapidjson/test/unittest/schematest.cpp +++ b/rapidjson/test/unittest/schematest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -40,64 +40,64 @@ using namespace rapidjson; EXPECT_TRUE(expected == (h1.GetHashCode() == h2.GetHashCode()));\ } -TEST(SchemaValidator, Hasher) { - TEST_HASHER("null", "null", true); - - TEST_HASHER("true", "true", true); - TEST_HASHER("false", "false", true); - TEST_HASHER("true", "false", false); - TEST_HASHER("false", "true", false); - TEST_HASHER("true", "null", false); - TEST_HASHER("false", "null", false); - - TEST_HASHER("1", "1", true); - TEST_HASHER("2147483648", "2147483648", true); // 2^31 can only be fit in unsigned - TEST_HASHER("-2147483649", "-2147483649", true); // -2^31 - 1 can only be fit in int64_t - TEST_HASHER("2147483648", "2147483648", true); // 2^31 can only be fit in unsigned - TEST_HASHER("4294967296", "4294967296", true); // 2^32 can only be fit in int64_t - TEST_HASHER("9223372036854775808", "9223372036854775808", true); // 2^63 can only be fit in uint64_t - TEST_HASHER("1.5", "1.5", true); - TEST_HASHER("1", "1.0", true); - TEST_HASHER("1", "-1", false); - TEST_HASHER("0.0", "-0.0", false); - TEST_HASHER("1", "true", false); - TEST_HASHER("0", "false", false); - TEST_HASHER("0", "null", false); - - TEST_HASHER("\"\"", "\"\"", true); - TEST_HASHER("\"\"", "\"\\u0000\"", false); - TEST_HASHER("\"Hello\"", "\"Hello\"", true); - TEST_HASHER("\"Hello\"", "\"World\"", false); - TEST_HASHER("\"Hello\"", "null", false); - TEST_HASHER("\"Hello\\u0000\"", "\"Hello\"", false); - TEST_HASHER("\"\"", "null", false); - TEST_HASHER("\"\"", "true", false); - TEST_HASHER("\"\"", "false", false); - - TEST_HASHER("[]", "[ ]", true); - TEST_HASHER("[1, true, false]", "[1, true, false]", true); - TEST_HASHER("[1, true, false]", "[1, true]", false); - TEST_HASHER("[1, 2]", "[2, 1]", false); - TEST_HASHER("[[1], 2]", "[[1, 2]]", false); - TEST_HASHER("[1, 2]", "[1, [2]]", false); - TEST_HASHER("[]", "null", false); - TEST_HASHER("[]", "true", false); - TEST_HASHER("[]", "false", false); - TEST_HASHER("[]", "0", false); - TEST_HASHER("[]", "0.0", false); - TEST_HASHER("[]", "\"\"", false); - - TEST_HASHER("{}", "{ }", true); - TEST_HASHER("{\"a\":1}", "{\"a\":1}", true); - TEST_HASHER("{\"a\":1}", "{\"b\":1}", false); - TEST_HASHER("{\"a\":1}", "{\"a\":2}", false); - TEST_HASHER("{\"a\":1, \"b\":2}", "{\"b\":2, \"a\":1}", true); // Member order insensitive - TEST_HASHER("{}", "null", false); - TEST_HASHER("{}", "false", false); - TEST_HASHER("{}", "true", false); - TEST_HASHER("{}", "0", false); - TEST_HASHER("{}", "0.0", false); - TEST_HASHER("{}", "\"\"", false); +TEST(SchemaValidator, Hasher) +{ TEST_HASHER("null", "null", true); + + TEST_HASHER("true", "true", true); + TEST_HASHER("false", "false", true); + TEST_HASHER("true", "false", false); + TEST_HASHER("false", "true", false); + TEST_HASHER("true", "null", false); + TEST_HASHER("false", "null", false); + + TEST_HASHER("1", "1", true); + TEST_HASHER("2147483648", "2147483648", true); // 2^31 can only be fit in unsigned + TEST_HASHER("-2147483649", "-2147483649", true); // -2^31 - 1 can only be fit in int64_t + TEST_HASHER("2147483648", "2147483648", true); // 2^31 can only be fit in unsigned + TEST_HASHER("4294967296", "4294967296", true); // 2^32 can only be fit in int64_t + TEST_HASHER("9223372036854775808", "9223372036854775808", true); // 2^63 can only be fit in uint64_t + TEST_HASHER("1.5", "1.5", true); + TEST_HASHER("1", "1.0", true); + TEST_HASHER("1", "-1", false); + TEST_HASHER("0.0", "-0.0", false); + TEST_HASHER("1", "true", false); + TEST_HASHER("0", "false", false); + TEST_HASHER("0", "null", false); + + TEST_HASHER("\"\"", "\"\"", true); + TEST_HASHER("\"\"", "\"\\u0000\"", false); + TEST_HASHER("\"Hello\"", "\"Hello\"", true); + TEST_HASHER("\"Hello\"", "\"World\"", false); + TEST_HASHER("\"Hello\"", "null", false); + TEST_HASHER("\"Hello\\u0000\"", "\"Hello\"", false); + TEST_HASHER("\"\"", "null", false); + TEST_HASHER("\"\"", "true", false); + TEST_HASHER("\"\"", "false", false); + + TEST_HASHER("[]", "[ ]", true); + TEST_HASHER("[1, true, false]", "[1, true, false]", true); + TEST_HASHER("[1, true, false]", "[1, true]", false); + TEST_HASHER("[1, 2]", "[2, 1]", false); + TEST_HASHER("[[1], 2]", "[[1, 2]]", false); + TEST_HASHER("[1, 2]", "[1, [2]]", false); + TEST_HASHER("[]", "null", false); + TEST_HASHER("[]", "true", false); + TEST_HASHER("[]", "false", false); + TEST_HASHER("[]", "0", false); + TEST_HASHER("[]", "0.0", false); + TEST_HASHER("[]", "\"\"", false); + + TEST_HASHER("{}", "{ }", true); + TEST_HASHER("{\"a\":1}", "{\"a\":1}", true); + TEST_HASHER("{\"a\":1}", "{\"b\":1}", false); + TEST_HASHER("{\"a\":1}", "{\"a\":2}", false); + TEST_HASHER("{\"a\":1, \"b\":2}", "{\"b\":2, \"a\":1}", true); // Member order insensitive + TEST_HASHER("{}", "null", false); + TEST_HASHER("{}", "false", false); + TEST_HASHER("{}", "true", false); + TEST_HASHER("{}", "0", false); + TEST_HASHER("{}", "0.0", false); + TEST_HASHER("{}", "\"\"", false); } // Test cases following http://spacetelescope.github.io/understanding-json-schema @@ -150,798 +150,797 @@ TEST(SchemaValidator, Hasher) { }\ } -TEST(SchemaValidator, Typeless) { - Document sd; - sd.Parse("{}"); - SchemaDocument s(sd); - - VALIDATE(s, "42", true); - VALIDATE(s, "\"I'm a string\"", true); - VALIDATE(s, "{ \"an\": [ \"arbitrarily\", \"nested\" ], \"data\": \"structure\" }", true); -} - -TEST(SchemaValidator, MultiType) { - Document sd; - sd.Parse("{ \"type\": [\"number\", \"string\"] }"); - SchemaDocument s(sd); +TEST(SchemaValidator, Typeless) +{ Document sd; + sd.Parse("{}"); + SchemaDocument s(sd); - VALIDATE(s, "42", true); - VALIDATE(s, "\"Life, the universe, and everything\"", true); - INVALIDATE(s, "[\"Life\", \"the universe\", \"and everything\"]", "", "type", ""); + VALIDATE(s, "42", true); + VALIDATE(s, "\"I'm a string\"", true); + VALIDATE(s, "{ \"an\": [ \"arbitrarily\", \"nested\" ], \"data\": \"structure\" }", true); } -TEST(SchemaValidator, Enum_Typed) { - Document sd; - sd.Parse("{ \"type\": \"string\", \"enum\" : [\"red\", \"amber\", \"green\"] }"); - SchemaDocument s(sd); +TEST(SchemaValidator, MultiType) +{ Document sd; + sd.Parse("{ \"type\": [\"number\", \"string\"] }"); + SchemaDocument s(sd); - VALIDATE(s, "\"red\"", true); - INVALIDATE(s, "\"blue\"", "", "enum", ""); + VALIDATE(s, "42", true); + VALIDATE(s, "\"Life, the universe, and everything\"", true); + INVALIDATE(s, "[\"Life\", \"the universe\", \"and everything\"]", "", "type", ""); } -TEST(SchemaValidator, Enum_Typless) { - Document sd; - sd.Parse("{ \"enum\": [\"red\", \"amber\", \"green\", null, 42] }"); - SchemaDocument s(sd); +TEST(SchemaValidator, Enum_Typed) +{ Document sd; + sd.Parse("{ \"type\": \"string\", \"enum\" : [\"red\", \"amber\", \"green\"] }"); + SchemaDocument s(sd); - VALIDATE(s, "\"red\"", true); - VALIDATE(s, "null", true); - VALIDATE(s, "42", true); - INVALIDATE(s, "0", "", "enum", ""); + VALIDATE(s, "\"red\"", true); + INVALIDATE(s, "\"blue\"", "", "enum", ""); } -TEST(SchemaValidator, Enum_InvalidType) { - Document sd; - sd.Parse("{ \"type\": \"string\", \"enum\": [\"red\", \"amber\", \"green\", null] }"); - SchemaDocument s(sd); +TEST(SchemaValidator, Enum_Typless) +{ Document sd; + sd.Parse("{ \"enum\": [\"red\", \"amber\", \"green\", null, 42] }"); + SchemaDocument s(sd); - VALIDATE(s, "\"red\"", true); - INVALIDATE(s, "null", "", "type", ""); + VALIDATE(s, "\"red\"", true); + VALIDATE(s, "null", true); + VALIDATE(s, "42", true); + INVALIDATE(s, "0", "", "enum", ""); } -TEST(SchemaValidator, AllOf) { - { - Document sd; - sd.Parse("{\"allOf\": [{ \"type\": \"string\" }, { \"type\": \"string\", \"maxLength\": 5 }]}"); - SchemaDocument s(sd); +TEST(SchemaValidator, Enum_InvalidType) +{ Document sd; + sd.Parse("{ \"type\": \"string\", \"enum\": [\"red\", \"amber\", \"green\", null] }"); + SchemaDocument s(sd); - VALIDATE(s, "\"ok\"", true); - INVALIDATE(s, "\"too long\"", "", "allOf", ""); - } - { - Document sd; - sd.Parse("{\"allOf\": [{ \"type\": \"string\" }, { \"type\": \"number\" } ] }"); - SchemaDocument s(sd); - - VALIDATE(s, "\"No way\"", false); - INVALIDATE(s, "-1", "", "allOf", ""); - } + VALIDATE(s, "\"red\"", true); + INVALIDATE(s, "null", "", "type", ""); } -TEST(SchemaValidator, AnyOf) { - Document sd; - sd.Parse("{\"anyOf\": [{ \"type\": \"string\" }, { \"type\": \"number\" } ] }"); +TEST(SchemaValidator, AllOf) +{ + { Document sd; + sd.Parse("{\"allOf\": [{ \"type\": \"string\" }, { \"type\": \"string\", \"maxLength\": 5 }]}"); SchemaDocument s(sd); - VALIDATE(s, "\"Yes\"", true); - VALIDATE(s, "42", true); - INVALIDATE(s, "{ \"Not a\": \"string or number\" }", "", "anyOf", ""); -} - -TEST(SchemaValidator, OneOf) { - Document sd; - sd.Parse("{\"oneOf\": [{ \"type\": \"number\", \"multipleOf\": 5 }, { \"type\": \"number\", \"multipleOf\": 3 } ] }"); + VALIDATE(s, "\"ok\"", true); + INVALIDATE(s, "\"too long\"", "", "allOf", ""); + } + { Document sd; + sd.Parse("{\"allOf\": [{ \"type\": \"string\" }, { \"type\": \"number\" } ] }"); SchemaDocument s(sd); - VALIDATE(s, "10", true); - VALIDATE(s, "9", true); - INVALIDATE(s, "2", "", "oneOf", ""); - INVALIDATE(s, "15", "", "oneOf", ""); + VALIDATE(s, "\"No way\"", false); + INVALIDATE(s, "-1", "", "allOf", ""); + } } -TEST(SchemaValidator, Not) { - Document sd; - sd.Parse("{\"not\":{ \"type\": \"string\"}}"); - SchemaDocument s(sd); - - VALIDATE(s, "42", true); - VALIDATE(s, "{ \"key\": \"value\" }", true); - INVALIDATE(s, "\"I am a string\"", "", "not", ""); -} - -TEST(SchemaValidator, Ref) { - Document sd; - sd.Parse( - "{" - " \"$schema\": \"http://json-schema.org/draft-04/schema#\"," - "" - " \"definitions\": {" - " \"address\": {" - " \"type\": \"object\"," - " \"properties\": {" - " \"street_address\": { \"type\": \"string\" }," - " \"city\": { \"type\": \"string\" }," - " \"state\": { \"type\": \"string\" }" - " }," - " \"required\": [\"street_address\", \"city\", \"state\"]" - " }" - " }," - " \"type\": \"object\"," - " \"properties\": {" - " \"billing_address\": { \"$ref\": \"#/definitions/address\" }," - " \"shipping_address\": { \"$ref\": \"#/definitions/address\" }" - " }" - "}"); - SchemaDocument s(sd); +TEST(SchemaValidator, AnyOf) +{ Document sd; + sd.Parse("{\"anyOf\": [{ \"type\": \"string\" }, { \"type\": \"number\" } ] }"); + SchemaDocument s(sd); - VALIDATE(s, "{\"shipping_address\": {\"street_address\": \"1600 Pennsylvania Avenue NW\", \"city\": \"Washington\", \"state\": \"DC\"}, \"billing_address\": {\"street_address\": \"1st Street SE\", \"city\": \"Washington\", \"state\": \"DC\"} }", true); -} - -TEST(SchemaValidator, Ref_AllOf) { - Document sd; - sd.Parse( - "{" - " \"$schema\": \"http://json-schema.org/draft-04/schema#\"," - "" - " \"definitions\": {" - " \"address\": {" - " \"type\": \"object\"," - " \"properties\": {" - " \"street_address\": { \"type\": \"string\" }," - " \"city\": { \"type\": \"string\" }," - " \"state\": { \"type\": \"string\" }" - " }," - " \"required\": [\"street_address\", \"city\", \"state\"]" - " }" - " }," - " \"type\": \"object\"," - " \"properties\": {" - " \"billing_address\": { \"$ref\": \"#/definitions/address\" }," - " \"shipping_address\": {" - " \"allOf\": [" - " { \"$ref\": \"#/definitions/address\" }," - " { \"properties\":" - " { \"type\": { \"enum\": [ \"residential\", \"business\" ] } }," - " \"required\": [\"type\"]" - " }" - " ]" - " }" - " }" - "}"); - SchemaDocument s(sd); - - INVALIDATE(s, "{\"shipping_address\": {\"street_address\": \"1600 Pennsylvania Avenue NW\", \"city\": \"Washington\", \"state\": \"DC\"} }", "/properties/shipping_address", "allOf", "/shipping_address"); - VALIDATE(s, "{\"shipping_address\": {\"street_address\": \"1600 Pennsylvania Avenue NW\", \"city\": \"Washington\", \"state\": \"DC\", \"type\": \"business\"} }", true); + VALIDATE(s, "\"Yes\"", true); + VALIDATE(s, "42", true); + INVALIDATE(s, "{ \"Not a\": \"string or number\" }", "", "anyOf", ""); } -TEST(SchemaValidator, String) { - Document sd; - sd.Parse("{\"type\":\"string\"}"); - SchemaDocument s(sd); +TEST(SchemaValidator, OneOf) +{ Document sd; + sd.Parse("{\"oneOf\": [{ \"type\": \"number\", \"multipleOf\": 5 }, { \"type\": \"number\", \"multipleOf\": 3 } ] }"); + SchemaDocument s(sd); - VALIDATE(s, "\"I'm a string\"", true); - INVALIDATE(s, "42", "", "type", ""); - INVALIDATE(s, "2147483648", "", "type", ""); // 2^31 can only be fit in unsigned - INVALIDATE(s, "-2147483649", "", "type", ""); // -2^31 - 1 can only be fit in int64_t - INVALIDATE(s, "4294967296", "", "type", ""); // 2^32 can only be fit in int64_t - INVALIDATE(s, "3.1415926", "", "type", ""); + VALIDATE(s, "10", true); + VALIDATE(s, "9", true); + INVALIDATE(s, "2", "", "oneOf", ""); + INVALIDATE(s, "15", "", "oneOf", ""); } -TEST(SchemaValidator, String_LengthRange) { - Document sd; - sd.Parse("{\"type\":\"string\",\"minLength\":2,\"maxLength\":3}"); - SchemaDocument s(sd); +TEST(SchemaValidator, Not) +{ Document sd; + sd.Parse("{\"not\":{ \"type\": \"string\"}}"); + SchemaDocument s(sd); - INVALIDATE(s, "\"A\"", "", "minLength", ""); - VALIDATE(s, "\"AB\"", true); - VALIDATE(s, "\"ABC\"", true); - INVALIDATE(s, "\"ABCD\"", "", "maxLength", ""); + VALIDATE(s, "42", true); + VALIDATE(s, "{ \"key\": \"value\" }", true); + INVALIDATE(s, "\"I am a string\"", "", "not", ""); } -#if RAPIDJSON_SCHEMA_HAS_REGEX -TEST(SchemaValidator, String_Pattern) { - Document sd; - sd.Parse("{\"type\":\"string\",\"pattern\":\"^(\\\\([0-9]{3}\\\\))?[0-9]{3}-[0-9]{4}$\"}"); - SchemaDocument s(sd); +TEST(SchemaValidator, Ref) +{ Document sd; + sd.Parse( + "{" + " \"$schema\": \"http://json-schema.org/draft-04/schema#\"," + "" + " \"definitions\": {" + " \"address\": {" + " \"type\": \"object\"," + " \"properties\": {" + " \"street_address\": { \"type\": \"string\" }," + " \"city\": { \"type\": \"string\" }," + " \"state\": { \"type\": \"string\" }" + " }," + " \"required\": [\"street_address\", \"city\", \"state\"]" + " }" + " }," + " \"type\": \"object\"," + " \"properties\": {" + " \"billing_address\": { \"$ref\": \"#/definitions/address\" }," + " \"shipping_address\": { \"$ref\": \"#/definitions/address\" }" + " }" + "}"); + SchemaDocument s(sd); - VALIDATE(s, "\"555-1212\"", true); - VALIDATE(s, "\"(888)555-1212\"", true); - INVALIDATE(s, "\"(888)555-1212 ext. 532\"", "", "pattern", ""); - INVALIDATE(s, "\"(800)FLOWERS\"", "", "pattern", ""); + VALIDATE(s, "{\"shipping_address\": {\"street_address\": \"1600 Pennsylvania Avenue NW\", \"city\": \"Washington\", \"state\": \"DC\"}, \"billing_address\": {\"street_address\": \"1st Street SE\", \"city\": \"Washington\", \"state\": \"DC\"} }", true); } -TEST(SchemaValidator, String_Pattern_Invalid) { - Document sd; - sd.Parse("{\"type\":\"string\",\"pattern\":\"a{0}\"}"); // TODO: report regex is invalid somehow - SchemaDocument s(sd); +TEST(SchemaValidator, Ref_AllOf) +{ Document sd; + sd.Parse( + "{" + " \"$schema\": \"http://json-schema.org/draft-04/schema#\"," + "" + " \"definitions\": {" + " \"address\": {" + " \"type\": \"object\"," + " \"properties\": {" + " \"street_address\": { \"type\": \"string\" }," + " \"city\": { \"type\": \"string\" }," + " \"state\": { \"type\": \"string\" }" + " }," + " \"required\": [\"street_address\", \"city\", \"state\"]" + " }" + " }," + " \"type\": \"object\"," + " \"properties\": {" + " \"billing_address\": { \"$ref\": \"#/definitions/address\" }," + " \"shipping_address\": {" + " \"allOf\": [" + " { \"$ref\": \"#/definitions/address\" }," + " { \"properties\":" + " { \"type\": { \"enum\": [ \"residential\", \"business\" ] } }," + " \"required\": [\"type\"]" + " }" + " ]" + " }" + " }" + "}"); + SchemaDocument s(sd); - VALIDATE(s, "\"\"", true); - VALIDATE(s, "\"a\"", true); - VALIDATE(s, "\"aa\"", true); + INVALIDATE(s, "{\"shipping_address\": {\"street_address\": \"1600 Pennsylvania Avenue NW\", \"city\": \"Washington\", \"state\": \"DC\"} }", "/properties/shipping_address", "allOf", "/shipping_address"); + VALIDATE(s, "{\"shipping_address\": {\"street_address\": \"1600 Pennsylvania Avenue NW\", \"city\": \"Washington\", \"state\": \"DC\", \"type\": \"business\"} }", true); } -#endif -TEST(SchemaValidator, Integer) { - Document sd; - sd.Parse("{\"type\":\"integer\"}"); - SchemaDocument s(sd); +TEST(SchemaValidator, String) +{ Document sd; + sd.Parse("{\"type\":\"string\"}"); + SchemaDocument s(sd); - VALIDATE(s, "42", true); - VALIDATE(s, "-1", true); - VALIDATE(s, "2147483648", true); // 2^31 can only be fit in unsigned - VALIDATE(s, "-2147483649", true); // -2^31 - 1 can only be fit in int64_t - VALIDATE(s, "2147483648", true); // 2^31 can only be fit in unsigned - VALIDATE(s, "4294967296", true); // 2^32 can only be fit in int64_t - INVALIDATE(s, "3.1415926", "", "type", ""); - INVALIDATE(s, "\"42\"", "", "type", ""); + VALIDATE(s, "\"I'm a string\"", true); + INVALIDATE(s, "42", "", "type", ""); + INVALIDATE(s, "2147483648", "", "type", ""); // 2^31 can only be fit in unsigned + INVALIDATE(s, "-2147483649", "", "type", ""); // -2^31 - 1 can only be fit in int64_t + INVALIDATE(s, "4294967296", "", "type", ""); // 2^32 can only be fit in int64_t + INVALIDATE(s, "3.1415926", "", "type", ""); } -TEST(SchemaValidator, Integer_Range) { - Document sd; - sd.Parse("{\"type\":\"integer\",\"minimum\":0,\"maximum\":100,\"exclusiveMaximum\":true}"); - SchemaDocument s(sd); +TEST(SchemaValidator, String_LengthRange) +{ Document sd; + sd.Parse("{\"type\":\"string\",\"minLength\":2,\"maxLength\":3}"); + SchemaDocument s(sd); - INVALIDATE(s, "-1", "", "minimum", ""); - VALIDATE(s, "0", true); - VALIDATE(s, "10", true); - VALIDATE(s, "99", true); - INVALIDATE(s, "100", "", "maximum", ""); - INVALIDATE(s, "101", "", "maximum", ""); + INVALIDATE(s, "\"A\"", "", "minLength", ""); + VALIDATE(s, "\"AB\"", true); + VALIDATE(s, "\"ABC\"", true); + INVALIDATE(s, "\"ABCD\"", "", "maxLength", ""); } -TEST(SchemaValidator, Integer_Range64Boundary) { - Document sd; - sd.Parse("{\"type\":\"integer\",\"minimum\":-9223372036854775807,\"maximum\":9223372036854775806}"); - SchemaDocument s(sd); - - INVALIDATE(s, "-9223372036854775808", "", "minimum", ""); - VALIDATE(s, "-9223372036854775807", true); - VALIDATE(s, "-2147483648", true); // int min - VALIDATE(s, "0", true); - VALIDATE(s, "2147483647", true); // int max - VALIDATE(s, "2147483648", true); // unsigned first - VALIDATE(s, "4294967295", true); // unsigned max - VALIDATE(s, "9223372036854775806", true); - INVALIDATE(s, "9223372036854775807", "", "maximum", ""); - INVALIDATE(s, "18446744073709551615", "", "maximum", ""); // uint64_t max -} - -TEST(SchemaValidator, Integer_RangeU64Boundary) { - Document sd; - sd.Parse("{\"type\":\"integer\",\"minimum\":9223372036854775808,\"maximum\":18446744073709551614}"); - SchemaDocument s(sd); - - INVALIDATE(s, "-9223372036854775808", "", "minimum", ""); - INVALIDATE(s, "9223372036854775807", "", "minimum", ""); - INVALIDATE(s, "-2147483648", "", "minimum", ""); // int min - INVALIDATE(s, "0", "", "minimum", ""); - INVALIDATE(s, "2147483647", "", "minimum", ""); // int max - INVALIDATE(s, "2147483648", "", "minimum", ""); // unsigned first - INVALIDATE(s, "4294967295", "", "minimum", ""); // unsigned max - VALIDATE(s, "9223372036854775808", true); - VALIDATE(s, "18446744073709551614", true); - INVALIDATE(s, "18446744073709551615", "", "maximum", ""); -} - -TEST(SchemaValidator, Integer_Range64BoundaryExclusive) { - Document sd; - sd.Parse("{\"type\":\"integer\",\"minimum\":-9223372036854775808,\"maximum\":18446744073709551615,\"exclusiveMinimum\":true,\"exclusiveMaximum\":true}"); - SchemaDocument s(sd); +#if RAPIDJSON_SCHEMA_HAS_REGEX +TEST(SchemaValidator, String_Pattern) +{ Document sd; + sd.Parse("{\"type\":\"string\",\"pattern\":\"^(\\\\([0-9]{3}\\\\))?[0-9]{3}-[0-9]{4}$\"}"); + SchemaDocument s(sd); - INVALIDATE(s, "-9223372036854775808", "", "minimum", ""); - VALIDATE(s, "-9223372036854775807", true); - VALIDATE(s, "18446744073709551614", true); - INVALIDATE(s, "18446744073709551615", "", "maximum", ""); + VALIDATE(s, "\"555-1212\"", true); + VALIDATE(s, "\"(888)555-1212\"", true); + INVALIDATE(s, "\"(888)555-1212 ext. 532\"", "", "pattern", ""); + INVALIDATE(s, "\"(800)FLOWERS\"", "", "pattern", ""); } -TEST(SchemaValidator, Integer_MultipleOf) { - Document sd; - sd.Parse("{\"type\":\"integer\",\"multipleOf\":10}"); - SchemaDocument s(sd); +TEST(SchemaValidator, String_Pattern_Invalid) +{ Document sd; + sd.Parse("{\"type\":\"string\",\"pattern\":\"a{0}\"}"); // TODO: report regex is invalid somehow + SchemaDocument s(sd); - VALIDATE(s, "0", true); - VALIDATE(s, "10", true); - VALIDATE(s, "-10", true); - VALIDATE(s, "20", true); - INVALIDATE(s, "23", "", "multipleOf", ""); - INVALIDATE(s, "-23", "", "multipleOf", ""); + VALIDATE(s, "\"\"", true); + VALIDATE(s, "\"a\"", true); + VALIDATE(s, "\"aa\"", true); } +#endif -TEST(SchemaValidator, Integer_MultipleOf64Boundary) { - Document sd; - sd.Parse("{\"type\":\"integer\",\"multipleOf\":18446744073709551615}"); - SchemaDocument s(sd); - - VALIDATE(s, "0", true); - VALIDATE(s, "18446744073709551615", true); - INVALIDATE(s, "18446744073709551614", "", "multipleOf", ""); -} +TEST(SchemaValidator, Integer) +{ Document sd; + sd.Parse("{\"type\":\"integer\"}"); + SchemaDocument s(sd); + + VALIDATE(s, "42", true); + VALIDATE(s, "-1", true); + VALIDATE(s, "2147483648", true); // 2^31 can only be fit in unsigned + VALIDATE(s, "-2147483649", true); // -2^31 - 1 can only be fit in int64_t + VALIDATE(s, "2147483648", true); // 2^31 can only be fit in unsigned + VALIDATE(s, "4294967296", true); // 2^32 can only be fit in int64_t + INVALIDATE(s, "3.1415926", "", "type", ""); + INVALIDATE(s, "\"42\"", "", "type", ""); +} + +TEST(SchemaValidator, Integer_Range) +{ Document sd; + sd.Parse("{\"type\":\"integer\",\"minimum\":0,\"maximum\":100,\"exclusiveMaximum\":true}"); + SchemaDocument s(sd); + + INVALIDATE(s, "-1", "", "minimum", ""); + VALIDATE(s, "0", true); + VALIDATE(s, "10", true); + VALIDATE(s, "99", true); + INVALIDATE(s, "100", "", "maximum", ""); + INVALIDATE(s, "101", "", "maximum", ""); +} + +TEST(SchemaValidator, Integer_Range64Boundary) +{ Document sd; + sd.Parse("{\"type\":\"integer\",\"minimum\":-9223372036854775807,\"maximum\":9223372036854775806}"); + SchemaDocument s(sd); + + INVALIDATE(s, "-9223372036854775808", "", "minimum", ""); + VALIDATE(s, "-9223372036854775807", true); + VALIDATE(s, "-2147483648", true); // int min + VALIDATE(s, "0", true); + VALIDATE(s, "2147483647", true); // int max + VALIDATE(s, "2147483648", true); // unsigned first + VALIDATE(s, "4294967295", true); // unsigned max + VALIDATE(s, "9223372036854775806", true); + INVALIDATE(s, "9223372036854775807", "", "maximum", ""); + INVALIDATE(s, "18446744073709551615", "", "maximum", ""); // uint64_t max +} + +TEST(SchemaValidator, Integer_RangeU64Boundary) +{ Document sd; + sd.Parse("{\"type\":\"integer\",\"minimum\":9223372036854775808,\"maximum\":18446744073709551614}"); + SchemaDocument s(sd); + + INVALIDATE(s, "-9223372036854775808", "", "minimum", ""); + INVALIDATE(s, "9223372036854775807", "", "minimum", ""); + INVALIDATE(s, "-2147483648", "", "minimum", ""); // int min + INVALIDATE(s, "0", "", "minimum", ""); + INVALIDATE(s, "2147483647", "", "minimum", ""); // int max + INVALIDATE(s, "2147483648", "", "minimum", ""); // unsigned first + INVALIDATE(s, "4294967295", "", "minimum", ""); // unsigned max + VALIDATE(s, "9223372036854775808", true); + VALIDATE(s, "18446744073709551614", true); + INVALIDATE(s, "18446744073709551615", "", "maximum", ""); +} + +TEST(SchemaValidator, Integer_Range64BoundaryExclusive) +{ Document sd; + sd.Parse("{\"type\":\"integer\",\"minimum\":-9223372036854775808,\"maximum\":18446744073709551615,\"exclusiveMinimum\":true,\"exclusiveMaximum\":true}"); + SchemaDocument s(sd); + + INVALIDATE(s, "-9223372036854775808", "", "minimum", ""); + VALIDATE(s, "-9223372036854775807", true); + VALIDATE(s, "18446744073709551614", true); + INVALIDATE(s, "18446744073709551615", "", "maximum", ""); +} + +TEST(SchemaValidator, Integer_MultipleOf) +{ Document sd; + sd.Parse("{\"type\":\"integer\",\"multipleOf\":10}"); + SchemaDocument s(sd); + + VALIDATE(s, "0", true); + VALIDATE(s, "10", true); + VALIDATE(s, "-10", true); + VALIDATE(s, "20", true); + INVALIDATE(s, "23", "", "multipleOf", ""); + INVALIDATE(s, "-23", "", "multipleOf", ""); +} + +TEST(SchemaValidator, Integer_MultipleOf64Boundary) +{ Document sd; + sd.Parse("{\"type\":\"integer\",\"multipleOf\":18446744073709551615}"); + SchemaDocument s(sd); + + VALIDATE(s, "0", true); + VALIDATE(s, "18446744073709551615", true); + INVALIDATE(s, "18446744073709551614", "", "multipleOf", ""); +} + +TEST(SchemaValidator, Number_Range) +{ Document sd; + sd.Parse("{\"type\":\"number\",\"minimum\":0,\"maximum\":100,\"exclusiveMaximum\":true}"); + SchemaDocument s(sd); + + INVALIDATE(s, "-1", "", "minimum", ""); + VALIDATE(s, "0", true); + VALIDATE(s, "0.1", true); + VALIDATE(s, "10", true); + VALIDATE(s, "99", true); + VALIDATE(s, "99.9", true); + INVALIDATE(s, "100", "", "maximum", ""); + INVALIDATE(s, "100.0", "", "maximum", ""); + INVALIDATE(s, "101.5", "", "maximum", ""); +} + +TEST(SchemaValidator, Number_RangeInt) +{ Document sd; + sd.Parse("{\"type\":\"number\",\"minimum\":-100,\"maximum\":-1,\"exclusiveMaximum\":true}"); + SchemaDocument s(sd); + + INVALIDATE(s, "-101", "", "minimum", ""); + INVALIDATE(s, "-100.1", "", "minimum", ""); + VALIDATE(s, "-100", true); + VALIDATE(s, "-2", true); + INVALIDATE(s, "-1", "", "maximum", ""); + INVALIDATE(s, "-0.9", "", "maximum", ""); + INVALIDATE(s, "0", "", "maximum", ""); + INVALIDATE(s, "2147483647", "", "maximum", ""); // int max + INVALIDATE(s, "2147483648", "", "maximum", ""); // unsigned first + INVALIDATE(s, "4294967295", "", "maximum", ""); // unsigned max + INVALIDATE(s, "9223372036854775808", "", "maximum", ""); + INVALIDATE(s, "18446744073709551614", "", "maximum", ""); + INVALIDATE(s, "18446744073709551615", "", "maximum", ""); +} + +TEST(SchemaValidator, Number_RangeDouble) +{ Document sd; + sd.Parse("{\"type\":\"number\",\"minimum\":0.1,\"maximum\":100.1,\"exclusiveMaximum\":true}"); + SchemaDocument s(sd); + + INVALIDATE(s, "-9223372036854775808", "", "minimum", ""); + INVALIDATE(s, "-2147483648", "", "minimum", ""); // int min + INVALIDATE(s, "-1", "", "minimum", ""); + VALIDATE(s, "0.1", true); + VALIDATE(s, "10", true); + VALIDATE(s, "99", true); + VALIDATE(s, "100", true); + INVALIDATE(s, "101", "", "maximum", ""); + INVALIDATE(s, "101.5", "", "maximum", ""); + INVALIDATE(s, "18446744073709551614", "", "maximum", ""); + INVALIDATE(s, "18446744073709551615", "", "maximum", ""); + INVALIDATE(s, "2147483647", "", "maximum", ""); // int max + INVALIDATE(s, "2147483648", "", "maximum", ""); // unsigned first + INVALIDATE(s, "4294967295", "", "maximum", ""); // unsigned max + INVALIDATE(s, "9223372036854775808", "", "maximum", ""); + INVALIDATE(s, "18446744073709551614", "", "maximum", ""); + INVALIDATE(s, "18446744073709551615", "", "maximum", ""); +} + +TEST(SchemaValidator, Number_RangeDoubleU64Boundary) +{ Document sd; + sd.Parse("{\"type\":\"number\",\"minimum\":9223372036854775808.0,\"maximum\":18446744073709550000.0}"); + SchemaDocument s(sd); + + INVALIDATE(s, "-9223372036854775808", "", "minimum", ""); + INVALIDATE(s, "-2147483648", "", "minimum", ""); // int min + INVALIDATE(s, "0", "", "minimum", ""); + INVALIDATE(s, "2147483647", "", "minimum", ""); // int max + INVALIDATE(s, "2147483648", "", "minimum", ""); // unsigned first + INVALIDATE(s, "4294967295", "", "minimum", ""); // unsigned max + VALIDATE(s, "9223372036854775808", true); + VALIDATE(s, "18446744073709540000", true); + INVALIDATE(s, "18446744073709551615", "", "maximum", ""); +} + +TEST(SchemaValidator, Number_MultipleOf) +{ Document sd; + sd.Parse("{\"type\":\"number\",\"multipleOf\":10.0}"); + SchemaDocument s(sd); + + VALIDATE(s, "0", true); + VALIDATE(s, "10", true); + VALIDATE(s, "-10", true); + VALIDATE(s, "20", true); + INVALIDATE(s, "23", "", "multipleOf", ""); + INVALIDATE(s, "-2147483648", "", "multipleOf", ""); // int min + VALIDATE(s, "-2147483640", true); + INVALIDATE(s, "2147483647", "", "multipleOf", ""); // int max + INVALIDATE(s, "2147483648", "", "multipleOf", ""); // unsigned first + VALIDATE(s, "2147483650", true); + INVALIDATE(s, "4294967295", "", "multipleOf", ""); // unsigned max + VALIDATE(s, "4294967300", true); +} + +TEST(SchemaValidator, Number_MultipleOfOne) +{ Document sd; + sd.Parse("{\"type\":\"number\",\"multipleOf\":1}"); + SchemaDocument s(sd); + + VALIDATE(s, "42", true); + VALIDATE(s, "42.0", true); + INVALIDATE(s, "3.1415926", "", "multipleOf", ""); +} + +TEST(SchemaValidator, Object) +{ Document sd; + sd.Parse("{\"type\":\"object\"}"); + SchemaDocument s(sd); + + VALIDATE(s, "{\"key\":\"value\",\"another_key\":\"another_value\"}", true); + VALIDATE(s, "{\"Sun\":1.9891e30,\"Jupiter\":1.8986e27,\"Saturn\":5.6846e26,\"Neptune\":10.243e25,\"Uranus\":8.6810e25,\"Earth\":5.9736e24,\"Venus\":4.8685e24,\"Mars\":6.4185e23,\"Mercury\":3.3022e23,\"Moon\":7.349e22,\"Pluto\":1.25e22}", true); + INVALIDATE(s, "[\"An\", \"array\", \"not\", \"an\", \"object\"]", "", "type", ""); + INVALIDATE(s, "\"Not an object\"", "", "type", ""); +} + +TEST(SchemaValidator, Object_Properties) +{ Document sd; + sd.Parse( + "{" + " \"type\": \"object\"," + " \"properties\" : {" + " \"number\": { \"type\": \"number\" }," + " \"street_name\" : { \"type\": \"string\" }," + " \"street_type\" : { \"type\": \"string\", \"enum\" : [\"Street\", \"Avenue\", \"Boulevard\"] }" + " }" + "}"); -TEST(SchemaValidator, Number_Range) { - Document sd; - sd.Parse("{\"type\":\"number\",\"minimum\":0,\"maximum\":100,\"exclusiveMaximum\":true}"); - SchemaDocument s(sd); + SchemaDocument s(sd); - INVALIDATE(s, "-1", "", "minimum", ""); - VALIDATE(s, "0", true); - VALIDATE(s, "0.1", true); - VALIDATE(s, "10", true); - VALIDATE(s, "99", true); - VALIDATE(s, "99.9", true); - INVALIDATE(s, "100", "", "maximum", ""); - INVALIDATE(s, "100.0", "", "maximum", ""); - INVALIDATE(s, "101.5", "", "maximum", ""); + VALIDATE(s, "{ \"number\": 1600, \"street_name\": \"Pennsylvania\", \"street_type\": \"Avenue\" }", true); + INVALIDATE(s, "{ \"number\": \"1600\", \"street_name\": \"Pennsylvania\", \"street_type\": \"Avenue\" }", "/properties/number", "type", "/number"); + VALIDATE(s, "{ \"number\": 1600, \"street_name\": \"Pennsylvania\" }", true); + VALIDATE(s, "{}", true); + VALIDATE(s, "{ \"number\": 1600, \"street_name\": \"Pennsylvania\", \"street_type\": \"Avenue\", \"direction\": \"NW\" }", true); } -TEST(SchemaValidator, Number_RangeInt) { - Document sd; - sd.Parse("{\"type\":\"number\",\"minimum\":-100,\"maximum\":-1,\"exclusiveMaximum\":true}"); - SchemaDocument s(sd); - - INVALIDATE(s, "-101", "", "minimum", ""); - INVALIDATE(s, "-100.1", "", "minimum", ""); - VALIDATE(s, "-100", true); - VALIDATE(s, "-2", true); - INVALIDATE(s, "-1", "", "maximum", ""); - INVALIDATE(s, "-0.9", "", "maximum", ""); - INVALIDATE(s, "0", "", "maximum", ""); - INVALIDATE(s, "2147483647", "", "maximum", ""); // int max - INVALIDATE(s, "2147483648", "", "maximum", ""); // unsigned first - INVALIDATE(s, "4294967295", "", "maximum", ""); // unsigned max - INVALIDATE(s, "9223372036854775808", "", "maximum", ""); - INVALIDATE(s, "18446744073709551614", "", "maximum", ""); - INVALIDATE(s, "18446744073709551615", "", "maximum", ""); -} - -TEST(SchemaValidator, Number_RangeDouble) { - Document sd; - sd.Parse("{\"type\":\"number\",\"minimum\":0.1,\"maximum\":100.1,\"exclusiveMaximum\":true}"); - SchemaDocument s(sd); +TEST(SchemaValidator, Object_AdditionalPropertiesBoolean) +{ Document sd; + sd.Parse( + "{" + " \"type\": \"object\"," + " \"properties\" : {" + " \"number\": { \"type\": \"number\" }," + " \"street_name\" : { \"type\": \"string\" }," + " \"street_type\" : { \"type\": \"string\"," + " \"enum\" : [\"Street\", \"Avenue\", \"Boulevard\"]" + " }" + " }," + " \"additionalProperties\": false" + "}"); - INVALIDATE(s, "-9223372036854775808", "", "minimum", ""); - INVALIDATE(s, "-2147483648", "", "minimum", ""); // int min - INVALIDATE(s, "-1", "", "minimum", ""); - VALIDATE(s, "0.1", true); - VALIDATE(s, "10", true); - VALIDATE(s, "99", true); - VALIDATE(s, "100", true); - INVALIDATE(s, "101", "", "maximum", ""); - INVALIDATE(s, "101.5", "", "maximum", ""); - INVALIDATE(s, "18446744073709551614", "", "maximum", ""); - INVALIDATE(s, "18446744073709551615", "", "maximum", ""); - INVALIDATE(s, "2147483647", "", "maximum", ""); // int max - INVALIDATE(s, "2147483648", "", "maximum", ""); // unsigned first - INVALIDATE(s, "4294967295", "", "maximum", ""); // unsigned max - INVALIDATE(s, "9223372036854775808", "", "maximum", ""); - INVALIDATE(s, "18446744073709551614", "", "maximum", ""); - INVALIDATE(s, "18446744073709551615", "", "maximum", ""); -} - -TEST(SchemaValidator, Number_RangeDoubleU64Boundary) { - Document sd; - sd.Parse("{\"type\":\"number\",\"minimum\":9223372036854775808.0,\"maximum\":18446744073709550000.0}"); - SchemaDocument s(sd); + SchemaDocument s(sd); - INVALIDATE(s, "-9223372036854775808", "", "minimum", ""); - INVALIDATE(s, "-2147483648", "", "minimum", ""); // int min - INVALIDATE(s, "0", "", "minimum", ""); - INVALIDATE(s, "2147483647", "", "minimum", ""); // int max - INVALIDATE(s, "2147483648", "", "minimum", ""); // unsigned first - INVALIDATE(s, "4294967295", "", "minimum", ""); // unsigned max - VALIDATE(s, "9223372036854775808", true); - VALIDATE(s, "18446744073709540000", true); - INVALIDATE(s, "18446744073709551615", "", "maximum", ""); + VALIDATE(s, "{ \"number\": 1600, \"street_name\": \"Pennsylvania\", \"street_type\": \"Avenue\" }", true); + INVALIDATE(s, "{ \"number\": 1600, \"street_name\": \"Pennsylvania\", \"street_type\": \"Avenue\", \"direction\": \"NW\" }", "", "additionalProperties", "/direction"); } -TEST(SchemaValidator, Number_MultipleOf) { - Document sd; - sd.Parse("{\"type\":\"number\",\"multipleOf\":10.0}"); - SchemaDocument s(sd); - - VALIDATE(s, "0", true); - VALIDATE(s, "10", true); - VALIDATE(s, "-10", true); - VALIDATE(s, "20", true); - INVALIDATE(s, "23", "", "multipleOf", ""); - INVALIDATE(s, "-2147483648", "", "multipleOf", ""); // int min - VALIDATE(s, "-2147483640", true); - INVALIDATE(s, "2147483647", "", "multipleOf", ""); // int max - INVALIDATE(s, "2147483648", "", "multipleOf", ""); // unsigned first - VALIDATE(s, "2147483650", true); - INVALIDATE(s, "4294967295", "", "multipleOf", ""); // unsigned max - VALIDATE(s, "4294967300", true); -} - -TEST(SchemaValidator, Number_MultipleOfOne) { - Document sd; - sd.Parse("{\"type\":\"number\",\"multipleOf\":1}"); - SchemaDocument s(sd); +TEST(SchemaValidator, Object_AdditionalPropertiesObject) +{ Document sd; + sd.Parse( + "{" + " \"type\": \"object\"," + " \"properties\" : {" + " \"number\": { \"type\": \"number\" }," + " \"street_name\" : { \"type\": \"string\" }," + " \"street_type\" : { \"type\": \"string\"," + " \"enum\" : [\"Street\", \"Avenue\", \"Boulevard\"]" + " }" + " }," + " \"additionalProperties\": { \"type\": \"string\" }" + "}"); + SchemaDocument s(sd); - VALIDATE(s, "42", true); - VALIDATE(s, "42.0", true); - INVALIDATE(s, "3.1415926", "", "multipleOf", ""); + VALIDATE(s, "{ \"number\": 1600, \"street_name\": \"Pennsylvania\", \"street_type\": \"Avenue\" }", true); + VALIDATE(s, "{ \"number\": 1600, \"street_name\": \"Pennsylvania\", \"street_type\": \"Avenue\", \"direction\": \"NW\" }", true); + INVALIDATE(s, "{ \"number\": 1600, \"street_name\": \"Pennsylvania\", \"street_type\": \"Avenue\", \"office_number\": 201 }", "/additionalProperties", "type", "/office_number"); } -TEST(SchemaValidator, Object) { - Document sd; - sd.Parse("{\"type\":\"object\"}"); - SchemaDocument s(sd); +TEST(SchemaValidator, Object_Required) +{ Document sd; + sd.Parse( + "{" + " \"type\": \"object\"," + " \"properties\" : {" + " \"name\": { \"type\": \"string\" }," + " \"email\" : { \"type\": \"string\" }," + " \"address\" : { \"type\": \"string\" }," + " \"telephone\" : { \"type\": \"string\" }" + " }," + " \"required\":[\"name\", \"email\"]" + "}"); + SchemaDocument s(sd); - VALIDATE(s, "{\"key\":\"value\",\"another_key\":\"another_value\"}", true); - VALIDATE(s, "{\"Sun\":1.9891e30,\"Jupiter\":1.8986e27,\"Saturn\":5.6846e26,\"Neptune\":10.243e25,\"Uranus\":8.6810e25,\"Earth\":5.9736e24,\"Venus\":4.8685e24,\"Mars\":6.4185e23,\"Mercury\":3.3022e23,\"Moon\":7.349e22,\"Pluto\":1.25e22}", true); - INVALIDATE(s, "[\"An\", \"array\", \"not\", \"an\", \"object\"]", "", "type", ""); - INVALIDATE(s, "\"Not an object\"", "", "type", ""); + VALIDATE(s, "{ \"name\": \"William Shakespeare\", \"email\" : \"bill@stratford-upon-avon.co.uk\" }", true); + VALIDATE(s, "{ \"name\": \"William Shakespeare\", \"email\" : \"bill@stratford-upon-avon.co.uk\", \"address\" : \"Henley Street, Stratford-upon-Avon, Warwickshire, England\", \"authorship\" : \"in question\"}", true); + INVALIDATE(s, "{ \"name\": \"William Shakespeare\", \"address\" : \"Henley Street, Stratford-upon-Avon, Warwickshire, England\" }", "", "required", ""); } -TEST(SchemaValidator, Object_Properties) { - Document sd; - sd.Parse( - "{" - " \"type\": \"object\"," - " \"properties\" : {" - " \"number\": { \"type\": \"number\" }," - " \"street_name\" : { \"type\": \"string\" }," - " \"street_type\" : { \"type\": \"string\", \"enum\" : [\"Street\", \"Avenue\", \"Boulevard\"] }" - " }" - "}"); - SchemaDocument s(sd); +TEST(SchemaValidator, Object_PropertiesRange) +{ Document sd; + sd.Parse("{\"type\":\"object\", \"minProperties\":2, \"maxProperties\":3}"); + SchemaDocument s(sd); - VALIDATE(s, "{ \"number\": 1600, \"street_name\": \"Pennsylvania\", \"street_type\": \"Avenue\" }", true); - INVALIDATE(s, "{ \"number\": \"1600\", \"street_name\": \"Pennsylvania\", \"street_type\": \"Avenue\" }", "/properties/number", "type", "/number"); - VALIDATE(s, "{ \"number\": 1600, \"street_name\": \"Pennsylvania\" }", true); - VALIDATE(s, "{}", true); - VALIDATE(s, "{ \"number\": 1600, \"street_name\": \"Pennsylvania\", \"street_type\": \"Avenue\", \"direction\": \"NW\" }", true); -} - -TEST(SchemaValidator, Object_AdditionalPropertiesBoolean) { - Document sd; - sd.Parse( - "{" - " \"type\": \"object\"," - " \"properties\" : {" - " \"number\": { \"type\": \"number\" }," - " \"street_name\" : { \"type\": \"string\" }," - " \"street_type\" : { \"type\": \"string\"," - " \"enum\" : [\"Street\", \"Avenue\", \"Boulevard\"]" - " }" - " }," - " \"additionalProperties\": false" - "}"); + INVALIDATE(s, "{}", "", "minProperties", ""); + INVALIDATE(s, "{\"a\":0}", "", "minProperties", ""); + VALIDATE(s, "{\"a\":0,\"b\":1}", true); + VALIDATE(s, "{\"a\":0,\"b\":1,\"c\":2}", true); + INVALIDATE(s, "{\"a\":0,\"b\":1,\"c\":2,\"d\":3}", "", "maxProperties", ""); +} - SchemaDocument s(sd); +TEST(SchemaValidator, Object_PropertyDependencies) +{ Document sd; + sd.Parse( + "{" + " \"type\": \"object\"," + " \"properties\": {" + " \"name\": { \"type\": \"string\" }," + " \"credit_card\": { \"type\": \"number\" }," + " \"billing_address\": { \"type\": \"string\" }" + " }," + " \"required\": [\"name\"]," + " \"dependencies\": {" + " \"credit_card\": [\"billing_address\"]" + " }" + "}"); + SchemaDocument s(sd); - VALIDATE(s, "{ \"number\": 1600, \"street_name\": \"Pennsylvania\", \"street_type\": \"Avenue\" }", true); - INVALIDATE(s, "{ \"number\": 1600, \"street_name\": \"Pennsylvania\", \"street_type\": \"Avenue\", \"direction\": \"NW\" }", "", "additionalProperties", "/direction"); -} - -TEST(SchemaValidator, Object_AdditionalPropertiesObject) { - Document sd; - sd.Parse( - "{" - " \"type\": \"object\"," - " \"properties\" : {" - " \"number\": { \"type\": \"number\" }," - " \"street_name\" : { \"type\": \"string\" }," - " \"street_type\" : { \"type\": \"string\"," - " \"enum\" : [\"Street\", \"Avenue\", \"Boulevard\"]" - " }" - " }," - " \"additionalProperties\": { \"type\": \"string\" }" - "}"); - SchemaDocument s(sd); + VALIDATE(s, "{ \"name\": \"John Doe\", \"credit_card\": 5555555555555555, \"billing_address\": \"555 Debtor's Lane\" }", true); + INVALIDATE(s, "{ \"name\": \"John Doe\", \"credit_card\": 5555555555555555 }", "", "dependencies", ""); + VALIDATE(s, "{ \"name\": \"John Doe\"}", true); + VALIDATE(s, "{ \"name\": \"John Doe\", \"billing_address\": \"555 Debtor's Lane\" }", true); +} - VALIDATE(s, "{ \"number\": 1600, \"street_name\": \"Pennsylvania\", \"street_type\": \"Avenue\" }", true); - VALIDATE(s, "{ \"number\": 1600, \"street_name\": \"Pennsylvania\", \"street_type\": \"Avenue\", \"direction\": \"NW\" }", true); - INVALIDATE(s, "{ \"number\": 1600, \"street_name\": \"Pennsylvania\", \"street_type\": \"Avenue\", \"office_number\": 201 }", "/additionalProperties", "type", "/office_number"); -} - -TEST(SchemaValidator, Object_Required) { - Document sd; - sd.Parse( - "{" - " \"type\": \"object\"," - " \"properties\" : {" - " \"name\": { \"type\": \"string\" }," - " \"email\" : { \"type\": \"string\" }," - " \"address\" : { \"type\": \"string\" }," - " \"telephone\" : { \"type\": \"string\" }" - " }," - " \"required\":[\"name\", \"email\"]" - "}"); - SchemaDocument s(sd); +TEST(SchemaValidator, Object_SchemaDependencies) +{ Document sd; + sd.Parse( + "{" + " \"type\": \"object\"," + " \"properties\" : {" + " \"name\": { \"type\": \"string\" }," + " \"credit_card\" : { \"type\": \"number\" }" + " }," + " \"required\" : [\"name\"]," + " \"dependencies\" : {" + " \"credit_card\": {" + " \"properties\": {" + " \"billing_address\": { \"type\": \"string\" }" + " }," + " \"required\" : [\"billing_address\"]" + " }" + " }" + "}"); + SchemaDocument s(sd); - VALIDATE(s, "{ \"name\": \"William Shakespeare\", \"email\" : \"bill@stratford-upon-avon.co.uk\" }", true); - VALIDATE(s, "{ \"name\": \"William Shakespeare\", \"email\" : \"bill@stratford-upon-avon.co.uk\", \"address\" : \"Henley Street, Stratford-upon-Avon, Warwickshire, England\", \"authorship\" : \"in question\"}", true); - INVALIDATE(s, "{ \"name\": \"William Shakespeare\", \"address\" : \"Henley Street, Stratford-upon-Avon, Warwickshire, England\" }", "", "required", ""); + VALIDATE(s, "{\"name\": \"John Doe\", \"credit_card\" : 5555555555555555,\"billing_address\" : \"555 Debtor's Lane\"}", true); + INVALIDATE(s, "{\"name\": \"John Doe\", \"credit_card\" : 5555555555555555 }", "", "dependencies", ""); + VALIDATE(s, "{\"name\": \"John Doe\", \"billing_address\" : \"555 Debtor's Lane\"}", true); } +#if RAPIDJSON_SCHEMA_HAS_REGEX +TEST(SchemaValidator, Object_PatternProperties) +{ Document sd; + sd.Parse( + "{" + " \"type\": \"object\"," + " \"patternProperties\": {" + " \"^S_\": { \"type\": \"string\" }," + " \"^I_\": { \"type\": \"integer\" }" + " }" + "}"); + SchemaDocument s(sd); -TEST(SchemaValidator, Object_PropertiesRange) { - Document sd; - sd.Parse("{\"type\":\"object\", \"minProperties\":2, \"maxProperties\":3}"); - SchemaDocument s(sd); - - INVALIDATE(s, "{}", "", "minProperties", ""); - INVALIDATE(s, "{\"a\":0}", "", "minProperties", ""); - VALIDATE(s, "{\"a\":0,\"b\":1}", true); - VALIDATE(s, "{\"a\":0,\"b\":1,\"c\":2}", true); - INVALIDATE(s, "{\"a\":0,\"b\":1,\"c\":2,\"d\":3}", "", "maxProperties", ""); -} - -TEST(SchemaValidator, Object_PropertyDependencies) { - Document sd; - sd.Parse( - "{" - " \"type\": \"object\"," - " \"properties\": {" - " \"name\": { \"type\": \"string\" }," - " \"credit_card\": { \"type\": \"number\" }," - " \"billing_address\": { \"type\": \"string\" }" - " }," - " \"required\": [\"name\"]," - " \"dependencies\": {" - " \"credit_card\": [\"billing_address\"]" - " }" - "}"); - SchemaDocument s(sd); + VALIDATE(s, "{ \"S_25\": \"This is a string\" }", true); + VALIDATE(s, "{ \"I_0\": 42 }", true); + INVALIDATE(s, "{ \"S_0\": 42 }", "", "patternProperties", "/S_0"); + INVALIDATE(s, "{ \"I_42\": \"This is a string\" }", "", "patternProperties", "/I_42"); + VALIDATE(s, "{ \"keyword\": \"value\" }", true); +} - VALIDATE(s, "{ \"name\": \"John Doe\", \"credit_card\": 5555555555555555, \"billing_address\": \"555 Debtor's Lane\" }", true); - INVALIDATE(s, "{ \"name\": \"John Doe\", \"credit_card\": 5555555555555555 }", "", "dependencies", ""); - VALIDATE(s, "{ \"name\": \"John Doe\"}", true); - VALIDATE(s, "{ \"name\": \"John Doe\", \"billing_address\": \"555 Debtor's Lane\" }", true); -} - -TEST(SchemaValidator, Object_SchemaDependencies) { - Document sd; - sd.Parse( - "{" - " \"type\": \"object\"," - " \"properties\" : {" - " \"name\": { \"type\": \"string\" }," - " \"credit_card\" : { \"type\": \"number\" }" - " }," - " \"required\" : [\"name\"]," - " \"dependencies\" : {" - " \"credit_card\": {" - " \"properties\": {" - " \"billing_address\": { \"type\": \"string\" }" - " }," - " \"required\" : [\"billing_address\"]" - " }" - " }" - "}"); - SchemaDocument s(sd); +TEST(SchemaValidator, Object_PatternProperties_AdditionalProperties) +{ Document sd; + sd.Parse( + "{" + " \"type\": \"object\"," + " \"properties\": {" + " \"builtin\": { \"type\": \"number\" }" + " }," + " \"patternProperties\": {" + " \"^S_\": { \"type\": \"string\" }," + " \"^I_\": { \"type\": \"integer\" }" + " }," + " \"additionalProperties\": { \"type\": \"string\" }" + "}"); + SchemaDocument s(sd); - VALIDATE(s, "{\"name\": \"John Doe\", \"credit_card\" : 5555555555555555,\"billing_address\" : \"555 Debtor's Lane\"}", true); - INVALIDATE(s, "{\"name\": \"John Doe\", \"credit_card\" : 5555555555555555 }", "", "dependencies", ""); - VALIDATE(s, "{\"name\": \"John Doe\", \"billing_address\" : \"555 Debtor's Lane\"}", true); + VALIDATE(s, "{ \"builtin\": 42 }", true); + VALIDATE(s, "{ \"keyword\": \"value\" }", true); + INVALIDATE(s, "{ \"keyword\": 42 }", "/additionalProperties", "type", "/keyword"); } +#endif -#if RAPIDJSON_SCHEMA_HAS_REGEX -TEST(SchemaValidator, Object_PatternProperties) { - Document sd; - sd.Parse( - "{" - " \"type\": \"object\"," - " \"patternProperties\": {" - " \"^S_\": { \"type\": \"string\" }," - " \"^I_\": { \"type\": \"integer\" }" - " }" - "}"); - SchemaDocument s(sd); - - VALIDATE(s, "{ \"S_25\": \"This is a string\" }", true); - VALIDATE(s, "{ \"I_0\": 42 }", true); - INVALIDATE(s, "{ \"S_0\": 42 }", "", "patternProperties", "/S_0"); - INVALIDATE(s, "{ \"I_42\": \"This is a string\" }", "", "patternProperties", "/I_42"); - VALIDATE(s, "{ \"keyword\": \"value\" }", true); -} - -TEST(SchemaValidator, Object_PatternProperties_AdditionalProperties) { - Document sd; - sd.Parse( - "{" - " \"type\": \"object\"," - " \"properties\": {" - " \"builtin\": { \"type\": \"number\" }" - " }," - " \"patternProperties\": {" - " \"^S_\": { \"type\": \"string\" }," - " \"^I_\": { \"type\": \"integer\" }" - " }," - " \"additionalProperties\": { \"type\": \"string\" }" - "}"); - SchemaDocument s(sd); +TEST(SchemaValidator, Array) +{ Document sd; + sd.Parse("{\"type\":\"array\"}"); + SchemaDocument s(sd); - VALIDATE(s, "{ \"builtin\": 42 }", true); - VALIDATE(s, "{ \"keyword\": \"value\" }", true); - INVALIDATE(s, "{ \"keyword\": 42 }", "/additionalProperties", "type", "/keyword"); + VALIDATE(s, "[1, 2, 3, 4, 5]", true); + VALIDATE(s, "[3, \"different\", { \"types\" : \"of values\" }]", true); + INVALIDATE(s, "{\"Not\": \"an array\"}", "", "type", ""); } -#endif -TEST(SchemaValidator, Array) { - Document sd; - sd.Parse("{\"type\":\"array\"}"); - SchemaDocument s(sd); +TEST(SchemaValidator, Array_ItemsList) +{ Document sd; + sd.Parse( + "{" + " \"type\": \"array\"," + " \"items\" : {" + " \"type\": \"number\"" + " }" + "}"); + SchemaDocument s(sd); - VALIDATE(s, "[1, 2, 3, 4, 5]", true); - VALIDATE(s, "[3, \"different\", { \"types\" : \"of values\" }]", true); - INVALIDATE(s, "{\"Not\": \"an array\"}", "", "type", ""); + VALIDATE(s, "[1, 2, 3, 4, 5]", true); + INVALIDATE(s, "[1, 2, \"3\", 4, 5]", "/items", "type", "/2"); + VALIDATE(s, "[]", true); } -TEST(SchemaValidator, Array_ItemsList) { - Document sd; - sd.Parse( - "{" - " \"type\": \"array\"," - " \"items\" : {" - " \"type\": \"number\"" - " }" - "}"); - SchemaDocument s(sd); +TEST(SchemaValidator, Array_ItemsTuple) +{ Document sd; + sd.Parse( + "{" + " \"type\": \"array\"," + " \"items\": [" + " {" + " \"type\": \"number\"" + " }," + " {" + " \"type\": \"string\"" + " }," + " {" + " \"type\": \"string\"," + " \"enum\": [\"Street\", \"Avenue\", \"Boulevard\"]" + " }," + " {" + " \"type\": \"string\"," + " \"enum\": [\"NW\", \"NE\", \"SW\", \"SE\"]" + " }" + " ]" + "}"); + SchemaDocument s(sd); - VALIDATE(s, "[1, 2, 3, 4, 5]", true); - INVALIDATE(s, "[1, 2, \"3\", 4, 5]", "/items", "type", "/2"); - VALIDATE(s, "[]", true); -} - -TEST(SchemaValidator, Array_ItemsTuple) { - Document sd; - sd.Parse( - "{" - " \"type\": \"array\"," - " \"items\": [" - " {" - " \"type\": \"number\"" - " }," - " {" - " \"type\": \"string\"" - " }," - " {" - " \"type\": \"string\"," - " \"enum\": [\"Street\", \"Avenue\", \"Boulevard\"]" - " }," - " {" - " \"type\": \"string\"," - " \"enum\": [\"NW\", \"NE\", \"SW\", \"SE\"]" - " }" - " ]" - "}"); - SchemaDocument s(sd); + VALIDATE(s, "[1600, \"Pennsylvania\", \"Avenue\", \"NW\"]", true); + INVALIDATE(s, "[24, \"Sussex\", \"Drive\"]", "/items/2", "enum", "/2"); + INVALIDATE(s, "[\"Palais de l'Elysee\"]", "/items/0", "type", "/0"); + VALIDATE(s, "[10, \"Downing\", \"Street\"]", true); + VALIDATE(s, "[1600, \"Pennsylvania\", \"Avenue\", \"NW\", \"Washington\"]", true); +} - VALIDATE(s, "[1600, \"Pennsylvania\", \"Avenue\", \"NW\"]", true); - INVALIDATE(s, "[24, \"Sussex\", \"Drive\"]", "/items/2", "enum", "/2"); - INVALIDATE(s, "[\"Palais de l'Elysee\"]", "/items/0", "type", "/0"); - VALIDATE(s, "[10, \"Downing\", \"Street\"]", true); - VALIDATE(s, "[1600, \"Pennsylvania\", \"Avenue\", \"NW\", \"Washington\"]", true); -} - -TEST(SchemaValidator, Array_AdditionalItmes) { - Document sd; - sd.Parse( - "{" - " \"type\": \"array\"," - " \"items\": [" - " {" - " \"type\": \"number\"" - " }," - " {" - " \"type\": \"string\"" - " }," - " {" - " \"type\": \"string\"," - " \"enum\": [\"Street\", \"Avenue\", \"Boulevard\"]" - " }," - " {" - " \"type\": \"string\"," - " \"enum\": [\"NW\", \"NE\", \"SW\", \"SE\"]" - " }" - " ]," - " \"additionalItems\": false" - "}"); - SchemaDocument s(sd); +TEST(SchemaValidator, Array_AdditionalItmes) +{ Document sd; + sd.Parse( + "{" + " \"type\": \"array\"," + " \"items\": [" + " {" + " \"type\": \"number\"" + " }," + " {" + " \"type\": \"string\"" + " }," + " {" + " \"type\": \"string\"," + " \"enum\": [\"Street\", \"Avenue\", \"Boulevard\"]" + " }," + " {" + " \"type\": \"string\"," + " \"enum\": [\"NW\", \"NE\", \"SW\", \"SE\"]" + " }" + " ]," + " \"additionalItems\": false" + "}"); + SchemaDocument s(sd); - VALIDATE(s, "[1600, \"Pennsylvania\", \"Avenue\", \"NW\"]", true); - VALIDATE(s, "[1600, \"Pennsylvania\", \"Avenue\"]", true); - INVALIDATE(s, "[1600, \"Pennsylvania\", \"Avenue\", \"NW\", \"Washington\"]", "", "items", "/4"); + VALIDATE(s, "[1600, \"Pennsylvania\", \"Avenue\", \"NW\"]", true); + VALIDATE(s, "[1600, \"Pennsylvania\", \"Avenue\"]", true); + INVALIDATE(s, "[1600, \"Pennsylvania\", \"Avenue\", \"NW\", \"Washington\"]", "", "items", "/4"); } -TEST(SchemaValidator, Array_ItemsRange) { - Document sd; - sd.Parse("{\"type\": \"array\",\"minItems\": 2,\"maxItems\" : 3}"); - SchemaDocument s(sd); +TEST(SchemaValidator, Array_ItemsRange) +{ Document sd; + sd.Parse("{\"type\": \"array\",\"minItems\": 2,\"maxItems\" : 3}"); + SchemaDocument s(sd); - INVALIDATE(s, "[]", "", "minItems", ""); - INVALIDATE(s, "[1]", "", "minItems", ""); - VALIDATE(s, "[1, 2]", true); - VALIDATE(s, "[1, 2, 3]", true); - INVALIDATE(s, "[1, 2, 3, 4]", "", "maxItems", ""); + INVALIDATE(s, "[]", "", "minItems", ""); + INVALIDATE(s, "[1]", "", "minItems", ""); + VALIDATE(s, "[1, 2]", true); + VALIDATE(s, "[1, 2, 3]", true); + INVALIDATE(s, "[1, 2, 3, 4]", "", "maxItems", ""); } -TEST(SchemaValidator, Array_UniqueItems) { - Document sd; - sd.Parse("{\"type\": \"array\", \"uniqueItems\": true}"); - SchemaDocument s(sd); +TEST(SchemaValidator, Array_UniqueItems) +{ Document sd; + sd.Parse("{\"type\": \"array\", \"uniqueItems\": true}"); + SchemaDocument s(sd); - VALIDATE(s, "[1, 2, 3, 4, 5]", true); - INVALIDATE(s, "[1, 2, 3, 3, 4]", "", "uniqueItems", "/3"); - VALIDATE(s, "[]", true); + VALIDATE(s, "[1, 2, 3, 4, 5]", true); + INVALIDATE(s, "[1, 2, 3, 3, 4]", "", "uniqueItems", "/3"); + VALIDATE(s, "[]", true); } -TEST(SchemaValidator, Boolean) { - Document sd; - sd.Parse("{\"type\":\"boolean\"}"); - SchemaDocument s(sd); +TEST(SchemaValidator, Boolean) +{ Document sd; + sd.Parse("{\"type\":\"boolean\"}"); + SchemaDocument s(sd); - VALIDATE(s, "true", true); - VALIDATE(s, "false", true); - INVALIDATE(s, "\"true\"", "", "type", ""); - INVALIDATE(s, "0", "", "type", ""); + VALIDATE(s, "true", true); + VALIDATE(s, "false", true); + INVALIDATE(s, "\"true\"", "", "type", ""); + INVALIDATE(s, "0", "", "type", ""); } -TEST(SchemaValidator, Null) { - Document sd; - sd.Parse("{\"type\":\"null\"}"); - SchemaDocument s(sd); +TEST(SchemaValidator, Null) +{ Document sd; + sd.Parse("{\"type\":\"null\"}"); + SchemaDocument s(sd); - VALIDATE(s, "null", true); - INVALIDATE(s, "false", "", "type", ""); - INVALIDATE(s, "0", "", "type", ""); - INVALIDATE(s, "\"\"", "", "type", ""); + VALIDATE(s, "null", true); + INVALIDATE(s, "false", "", "type", ""); + INVALIDATE(s, "0", "", "type", ""); + INVALIDATE(s, "\"\"", "", "type", ""); } // Additional tests -TEST(SchemaValidator, ObjectInArray) { - Document sd; - sd.Parse("{\"type\":\"array\", \"items\": { \"type\":\"string\" }}"); - SchemaDocument s(sd); +TEST(SchemaValidator, ObjectInArray) +{ Document sd; + sd.Parse("{\"type\":\"array\", \"items\": { \"type\":\"string\" }}"); + SchemaDocument s(sd); - VALIDATE(s, "[\"a\"]", true); - INVALIDATE(s, "[1]", "/items", "type", "/0"); - INVALIDATE(s, "[{}]", "/items", "type", "/0"); -} - -TEST(SchemaValidator, MultiTypeInObject) { - Document sd; - sd.Parse( - "{" - " \"type\":\"object\"," - " \"properties\": {" - " \"tel\" : {" - " \"type\":[\"integer\", \"string\"]" - " }" - " }" - "}"); - SchemaDocument s(sd); + VALIDATE(s, "[\"a\"]", true); + INVALIDATE(s, "[1]", "/items", "type", "/0"); + INVALIDATE(s, "[{}]", "/items", "type", "/0"); +} - VALIDATE(s, "{ \"tel\": 999 }", true); - VALIDATE(s, "{ \"tel\": \"123-456\" }", true); - INVALIDATE(s, "{ \"tel\": true }", "/properties/tel", "type", "/tel"); -} - -TEST(SchemaValidator, MultiTypeWithObject) { - Document sd; - sd.Parse( - "{" - " \"type\": [\"object\",\"string\"]," - " \"properties\": {" - " \"tel\" : {" - " \"type\": \"integer\"" - " }" - " }" - "}"); - SchemaDocument s(sd); +TEST(SchemaValidator, MultiTypeInObject) +{ Document sd; + sd.Parse( + "{" + " \"type\":\"object\"," + " \"properties\": {" + " \"tel\" : {" + " \"type\":[\"integer\", \"string\"]" + " }" + " }" + "}"); + SchemaDocument s(sd); + + VALIDATE(s, "{ \"tel\": 999 }", true); + VALIDATE(s, "{ \"tel\": \"123-456\" }", true); + INVALIDATE(s, "{ \"tel\": true }", "/properties/tel", "type", "/tel"); +} + +TEST(SchemaValidator, MultiTypeWithObject) +{ Document sd; + sd.Parse( + "{" + " \"type\": [\"object\",\"string\"]," + " \"properties\": {" + " \"tel\" : {" + " \"type\": \"integer\"" + " }" + " }" + "}"); + SchemaDocument s(sd); - VALIDATE(s, "\"Hello\"", true); - VALIDATE(s, "{ \"tel\": 999 }", true); - INVALIDATE(s, "{ \"tel\": \"fail\" }", "/properties/tel", "type", "/tel"); + VALIDATE(s, "\"Hello\"", true); + VALIDATE(s, "{ \"tel\": 999 }", true); + INVALIDATE(s, "{ \"tel\": \"fail\" }", "/properties/tel", "type", "/tel"); } -TEST(SchemaValidator, AllOf_Nested) { - Document sd; - sd.Parse( +TEST(SchemaValidator, AllOf_Nested) +{ Document sd; + sd.Parse( "{" " \"allOf\": [" " { \"type\": \"string\", \"minLength\": 2 }," @@ -949,363 +948,363 @@ TEST(SchemaValidator, AllOf_Nested) { " { \"allOf\": [ { \"enum\" : [\"ok\", \"okay\", \"OK\", \"o\"] }, { \"enum\" : [\"ok\", \"OK\", \"o\"]} ] }" " ]" "}"); - SchemaDocument s(sd); + SchemaDocument s(sd); - VALIDATE(s, "\"ok\"", true); - VALIDATE(s, "\"OK\"", true); - INVALIDATE(s, "\"okay\"", "", "allOf", ""); - INVALIDATE(s, "\"o\"", "", "allOf", ""); - INVALIDATE(s, "\"n\"", "", "allOf", ""); - INVALIDATE(s, "\"too long\"", "", "allOf", ""); - INVALIDATE(s, "123", "", "allOf", ""); -} - -TEST(SchemaValidator, EscapedPointer) { - Document sd; - sd.Parse( - "{" - " \"type\": \"object\"," - " \"properties\": {" - " \"~/\": { \"type\": \"number\" }" - " }" - "}"); - SchemaDocument s(sd); - INVALIDATE(s, "{\"~/\":true}", "/properties/~0~1", "type", "/~0~1"); + VALIDATE(s, "\"ok\"", true); + VALIDATE(s, "\"OK\"", true); + INVALIDATE(s, "\"okay\"", "", "allOf", ""); + INVALIDATE(s, "\"o\"", "", "allOf", ""); + INVALIDATE(s, "\"n\"", "", "allOf", ""); + INVALIDATE(s, "\"too long\"", "", "allOf", ""); + INVALIDATE(s, "123", "", "allOf", ""); } -template -static char* ReadFile(const char* filename, Allocator& allocator) { - const char *paths[] = { - "", - "bin/", - "../bin/", - "../../bin/", - "../../../bin/" - }; - char buffer[1024]; - FILE *fp = 0; - for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { - sprintf(buffer, "%s%s", paths[i], filename); - fp = fopen(buffer, "rb"); - if (fp) - break; - } +TEST(SchemaValidator, EscapedPointer) +{ Document sd; + sd.Parse( + "{" + " \"type\": \"object\"," + " \"properties\": {" + " \"~/\": { \"type\": \"number\" }" + " }" + "}"); + SchemaDocument s(sd); + INVALIDATE(s, "{\"~/\":true}", "/properties/~0~1", "type", "/~0~1"); +} - if (!fp) - return 0; - - fseek(fp, 0, SEEK_END); - size_t length = static_cast(ftell(fp)); - fseek(fp, 0, SEEK_SET); - char* json = reinterpret_cast(allocator.Malloc(length + 1)); - size_t readLength = fread(json, 1, length, fp); - json[readLength] = '\0'; - fclose(fp); - return json; -} - -TEST(SchemaValidator, ValidateMetaSchema) { - CrtAllocator allocator; - char* json = ReadFile("draft-04/schema", allocator); - Document d; - d.Parse(json); - ASSERT_FALSE(d.HasParseError()); - SchemaDocument sd(d); - SchemaValidator validator(sd); - if (!d.Accept(validator)) { - StringBuffer sb; - validator.GetInvalidSchemaPointer().StringifyUriFragment(sb); - printf("Invalid schema: %s\n", sb.GetString()); - printf("Invalid keyword: %s\n", validator.GetInvalidSchemaKeyword()); - sb.Clear(); - validator.GetInvalidDocumentPointer().StringifyUriFragment(sb); - printf("Invalid document: %s\n", sb.GetString()); - ADD_FAILURE(); - } - CrtAllocator::Free(json); -} - -TEST(SchemaValidator, ValidateMetaSchema_UTF16) { - typedef GenericDocument > D; - typedef GenericSchemaDocument SD; - typedef GenericSchemaValidator SV; - - CrtAllocator allocator; - char* json = ReadFile("draft-04/schema", allocator); - - D d; - StringStream ss(json); - d.ParseStream<0, UTF8<> >(ss); - ASSERT_FALSE(d.HasParseError()); - SD sd(d); - SV validator(sd); - if (!d.Accept(validator)) { - GenericStringBuffer > sb; - validator.GetInvalidSchemaPointer().StringifyUriFragment(sb); - wprintf(L"Invalid schema: %ls\n", sb.GetString()); - wprintf(L"Invalid keyword: %ls\n", validator.GetInvalidSchemaKeyword()); - sb.Clear(); - validator.GetInvalidDocumentPointer().StringifyUriFragment(sb); - wprintf(L"Invalid document: %ls\n", sb.GetString()); - ADD_FAILURE(); - } - CrtAllocator::Free(json); +template +static char* ReadFile(const char* filename, Allocator& allocator) +{ const char *paths[] = + { "", + "bin/", + "../bin/", + "../../bin/", + "../../../bin/" + }; + char buffer[1024]; + FILE *fp = 0; + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) + { sprintf(buffer, "%s%s", paths[i], filename); + fp = fopen(buffer, "rb"); + if (fp) + break; + } + + if (!fp) + return 0; + + fseek(fp, 0, SEEK_END); + size_t length = static_cast(ftell(fp)); + fseek(fp, 0, SEEK_SET); + char* json = reinterpret_cast(allocator.Malloc(length + 1)); + size_t readLength = fread(json, 1, length, fp); + json[readLength] = '\0'; + fclose(fp); + return json; +} + +TEST(SchemaValidator, ValidateMetaSchema) +{ CrtAllocator allocator; + char* json = ReadFile("draft-04/schema", allocator); + Document d; + d.Parse(json); + ASSERT_FALSE(d.HasParseError()); + SchemaDocument sd(d); + SchemaValidator validator(sd); + if (!d.Accept(validator)) + { StringBuffer sb; + validator.GetInvalidSchemaPointer().StringifyUriFragment(sb); + printf("Invalid schema: %s\n", sb.GetString()); + printf("Invalid keyword: %s\n", validator.GetInvalidSchemaKeyword()); + sb.Clear(); + validator.GetInvalidDocumentPointer().StringifyUriFragment(sb); + printf("Invalid document: %s\n", sb.GetString()); + ADD_FAILURE(); + } + CrtAllocator::Free(json); +} + +TEST(SchemaValidator, ValidateMetaSchema_UTF16) +{ typedef GenericDocument > D; + typedef GenericSchemaDocument SD; + typedef GenericSchemaValidator SV; + + CrtAllocator allocator; + char* json = ReadFile("draft-04/schema", allocator); + + D d; + StringStream ss(json); + d.ParseStream<0, UTF8<> >(ss); + ASSERT_FALSE(d.HasParseError()); + SD sd(d); + SV validator(sd); + if (!d.Accept(validator)) + { GenericStringBuffer > sb; + validator.GetInvalidSchemaPointer().StringifyUriFragment(sb); + wprintf(L"Invalid schema: %ls\n", sb.GetString()); + wprintf(L"Invalid keyword: %ls\n", validator.GetInvalidSchemaKeyword()); + sb.Clear(); + validator.GetInvalidDocumentPointer().StringifyUriFragment(sb); + wprintf(L"Invalid document: %ls\n", sb.GetString()); + ADD_FAILURE(); + } + CrtAllocator::Free(json); } template -class RemoteSchemaDocumentProvider : public IGenericRemoteSchemaDocumentProvider { +class RemoteSchemaDocumentProvider : public IGenericRemoteSchemaDocumentProvider +{ public: - RemoteSchemaDocumentProvider() : - documentAllocator_(documentBuffer_, sizeof(documentBuffer_)), - schemaAllocator_(schemaBuffer_, sizeof(schemaBuffer_)) - { - const char* filenames[kCount] = { - "jsonschema/remotes/integer.json", - "jsonschema/remotes/subSchemas.json", - "jsonschema/remotes/folder/folderInteger.json", - "draft-04/schema" - }; - - for (size_t i = 0; i < kCount; i++) { - sd_[i] = 0; - - char jsonBuffer[8192]; - MemoryPoolAllocator<> jsonAllocator(jsonBuffer, sizeof(jsonBuffer)); - char* json = ReadFile(filenames[i], jsonAllocator); - if (!json) { - printf("json remote file %s not found", filenames[i]); - ADD_FAILURE(); - } - else { - char stackBuffer[4096]; - MemoryPoolAllocator<> stackAllocator(stackBuffer, sizeof(stackBuffer)); - DocumentType d(&documentAllocator_, 1024, &stackAllocator); - d.Parse(json); - sd_[i] = new SchemaDocumentType(d, 0, &schemaAllocator_); - MemoryPoolAllocator<>::Free(json); - } - }; - } + RemoteSchemaDocumentProvider() : + documentAllocator_(documentBuffer_, sizeof(documentBuffer_)), + schemaAllocator_(schemaBuffer_, sizeof(schemaBuffer_)) + { const char* filenames[kCount] = + { "jsonschema/remotes/integer.json", + "jsonschema/remotes/subSchemas.json", + "jsonschema/remotes/folder/folderInteger.json", + "draft-04/schema" + }; - ~RemoteSchemaDocumentProvider() { - for (size_t i = 0; i < kCount; i++) - delete sd_[i]; - } + for (size_t i = 0; i < kCount; i++) + { sd_[i] = 0; - virtual const SchemaDocumentType* GetRemoteDocument(const char* uri, SizeType length) { - const char* uris[kCount] = { - "http://localhost:1234/integer.json", - "http://localhost:1234/subSchemas.json", - "http://localhost:1234/folder/folderInteger.json", - "http://json-schema.org/draft-04/schema" - }; - - for (size_t i = 0; i < kCount; i++) - if (strncmp(uri, uris[i], length) == 0) - return sd_[i]; - return 0; - } + char jsonBuffer[8192]; + MemoryPoolAllocator<> jsonAllocator(jsonBuffer, sizeof(jsonBuffer)); + char* json = ReadFile(filenames[i], jsonAllocator); + if (!json) + { printf("json remote file %s not found", filenames[i]); + ADD_FAILURE(); + } + else + { char stackBuffer[4096]; + MemoryPoolAllocator<> stackAllocator(stackBuffer, sizeof(stackBuffer)); + DocumentType d(&documentAllocator_, 1024, &stackAllocator); + d.Parse(json); + sd_[i] = new SchemaDocumentType(d, 0, &schemaAllocator_); + MemoryPoolAllocator<>::Free(json); + } + }; + } + + ~RemoteSchemaDocumentProvider() + { for (size_t i = 0; i < kCount; i++) + delete sd_[i]; + } + + virtual const SchemaDocumentType* GetRemoteDocument(const char* uri, SizeType length) + { const char* uris[kCount] = + { "http://localhost:1234/integer.json", + "http://localhost:1234/subSchemas.json", + "http://localhost:1234/folder/folderInteger.json", + "http://json-schema.org/draft-04/schema" + }; + + for (size_t i = 0; i < kCount; i++) + if (strncmp(uri, uris[i], length) == 0) + return sd_[i]; + return 0; + } private: - typedef GenericDocument, MemoryPoolAllocator<> > DocumentType; + typedef GenericDocument, MemoryPoolAllocator<> > DocumentType; - RemoteSchemaDocumentProvider(const RemoteSchemaDocumentProvider&); - RemoteSchemaDocumentProvider& operator=(const RemoteSchemaDocumentProvider&); + RemoteSchemaDocumentProvider(const RemoteSchemaDocumentProvider&); + RemoteSchemaDocumentProvider& operator=(const RemoteSchemaDocumentProvider&); - static const size_t kCount = 4; - SchemaDocumentType* sd_[kCount]; - typename DocumentType::AllocatorType documentAllocator_; - typename SchemaDocumentType::AllocatorType schemaAllocator_; - char documentBuffer_[16384]; - char schemaBuffer_[128 * 1024]; + static const size_t kCount = 4; + SchemaDocumentType* sd_[kCount]; + typename DocumentType::AllocatorType documentAllocator_; + typename SchemaDocumentType::AllocatorType schemaAllocator_; + char documentBuffer_[16384]; + char schemaBuffer_[128 * 1024]; }; -TEST(SchemaValidator, TestSuite) { - const char* filenames[] = { - "additionalItems.json", - "additionalProperties.json", - "allOf.json", - "anyOf.json", - "default.json", - "definitions.json", - "dependencies.json", - "enum.json", - "items.json", - "maximum.json", - "maxItems.json", - "maxLength.json", - "maxProperties.json", - "minimum.json", - "minItems.json", - "minLength.json", - "minProperties.json", - "multipleOf.json", - "not.json", - "oneOf.json", - "pattern.json", - "patternProperties.json", - "properties.json", - "ref.json", - "refRemote.json", - "required.json", - "type.json", - "uniqueItems.json" - }; - - const char* onlyRunDescription = 0; - //const char* onlyRunDescription = "a string is a string"; - - unsigned testCount = 0; - unsigned passCount = 0; - - typedef GenericSchemaDocument > SchemaDocumentType; - RemoteSchemaDocumentProvider provider; - - char jsonBuffer[65536]; - char documentBuffer[65536]; - char documentStackBuffer[65536]; - char schemaBuffer[65536]; - char validatorBuffer[65536]; - MemoryPoolAllocator<> jsonAllocator(jsonBuffer, sizeof(jsonBuffer)); - MemoryPoolAllocator<> documentAllocator(documentBuffer, sizeof(documentBuffer)); - MemoryPoolAllocator<> documentStackAllocator(documentStackBuffer, sizeof(documentStackBuffer)); - MemoryPoolAllocator<> schemaAllocator(schemaBuffer, sizeof(schemaBuffer)); - MemoryPoolAllocator<> validatorAllocator(validatorBuffer, sizeof(validatorBuffer)); - - for (size_t i = 0; i < sizeof(filenames) / sizeof(filenames[0]); i++) { - char filename[FILENAME_MAX]; - sprintf(filename, "jsonschema/tests/draft4/%s", filenames[i]); - char* json = ReadFile(filename, jsonAllocator); - if (!json) { - printf("json test suite file %s not found", filename); - ADD_FAILURE(); - } - else { - GenericDocument, MemoryPoolAllocator<>, MemoryPoolAllocator<> > d(&documentAllocator, 1024, &documentStackAllocator); - d.Parse(json); - if (d.HasParseError()) { - printf("json test suite file %s has parse error", filename); - ADD_FAILURE(); - } - else { - for (Value::ConstValueIterator schemaItr = d.Begin(); schemaItr != d.End(); ++schemaItr) { - { - SchemaDocumentType schema((*schemaItr)["schema"], &provider, &schemaAllocator); - GenericSchemaValidator >, MemoryPoolAllocator<> > validator(schema, &validatorAllocator); - const char* description1 = (*schemaItr)["description"].GetString(); - const Value& tests = (*schemaItr)["tests"]; - for (Value::ConstValueIterator testItr = tests.Begin(); testItr != tests.End(); ++testItr) { - const char* description2 = (*testItr)["description"].GetString(); - if (!onlyRunDescription || strcmp(description2, onlyRunDescription) == 0) { - const Value& data = (*testItr)["data"]; - bool expected = (*testItr)["valid"].GetBool(); - testCount++; - validator.Reset(); - bool actual = data.Accept(validator); - if (expected != actual) - printf("Fail: %30s \"%s\" \"%s\"\n", filename, description1, description2); - else - passCount++; - } - } - //printf("%zu %zu %zu\n", documentAllocator.Size(), schemaAllocator.Size(), validatorAllocator.Size()); - } - schemaAllocator.Clear(); - validatorAllocator.Clear(); - } +TEST(SchemaValidator, TestSuite) +{ const char* filenames[] = + { "additionalItems.json", + "additionalProperties.json", + "allOf.json", + "anyOf.json", + "default.json", + "definitions.json", + "dependencies.json", + "enum.json", + "items.json", + "maximum.json", + "maxItems.json", + "maxLength.json", + "maxProperties.json", + "minimum.json", + "minItems.json", + "minLength.json", + "minProperties.json", + "multipleOf.json", + "not.json", + "oneOf.json", + "pattern.json", + "patternProperties.json", + "properties.json", + "ref.json", + "refRemote.json", + "required.json", + "type.json", + "uniqueItems.json" + }; + + const char* onlyRunDescription = 0; + //const char* onlyRunDescription = "a string is a string"; + + unsigned testCount = 0; + unsigned passCount = 0; + + typedef GenericSchemaDocument > SchemaDocumentType; + RemoteSchemaDocumentProvider provider; + + char jsonBuffer[65536]; + char documentBuffer[65536]; + char documentStackBuffer[65536]; + char schemaBuffer[65536]; + char validatorBuffer[65536]; + MemoryPoolAllocator<> jsonAllocator(jsonBuffer, sizeof(jsonBuffer)); + MemoryPoolAllocator<> documentAllocator(documentBuffer, sizeof(documentBuffer)); + MemoryPoolAllocator<> documentStackAllocator(documentStackBuffer, sizeof(documentStackBuffer)); + MemoryPoolAllocator<> schemaAllocator(schemaBuffer, sizeof(schemaBuffer)); + MemoryPoolAllocator<> validatorAllocator(validatorBuffer, sizeof(validatorBuffer)); + + for (size_t i = 0; i < sizeof(filenames) / sizeof(filenames[0]); i++) + { char filename[FILENAME_MAX]; + sprintf(filename, "jsonschema/tests/draft4/%s", filenames[i]); + char* json = ReadFile(filename, jsonAllocator); + if (!json) + { printf("json test suite file %s not found", filename); + ADD_FAILURE(); + } + else + { GenericDocument, MemoryPoolAllocator<>, MemoryPoolAllocator<> > d(&documentAllocator, 1024, &documentStackAllocator); + d.Parse(json); + if (d.HasParseError()) + { printf("json test suite file %s has parse error", filename); + ADD_FAILURE(); + } + else + { for (Value::ConstValueIterator schemaItr = d.Begin(); schemaItr != d.End(); ++schemaItr) + { + { SchemaDocumentType schema((*schemaItr)["schema"], &provider, &schemaAllocator); + GenericSchemaValidator >, MemoryPoolAllocator<> > validator(schema, &validatorAllocator); + const char* description1 = (*schemaItr)["description"].GetString(); + const Value& tests = (*schemaItr)["tests"]; + for (Value::ConstValueIterator testItr = tests.Begin(); testItr != tests.End(); ++testItr) + { const char* description2 = (*testItr)["description"].GetString(); + if (!onlyRunDescription || strcmp(description2, onlyRunDescription) == 0) + { const Value& data = (*testItr)["data"]; + bool expected = (*testItr)["valid"].GetBool(); + testCount++; + validator.Reset(); + bool actual = data.Accept(validator); + if (expected != actual) + printf("Fail: %30s \"%s\" \"%s\"\n", filename, description1, description2); + else + passCount++; + } } + //printf("%zu %zu %zu\n", documentAllocator.Size(), schemaAllocator.Size(), validatorAllocator.Size()); + } + schemaAllocator.Clear(); + validatorAllocator.Clear(); } - documentAllocator.Clear(); - MemoryPoolAllocator<>::Free(json); - jsonAllocator.Clear(); + } } - printf("%d / %d passed (%2d%%)\n", passCount, testCount, passCount * 100 / testCount); - // if (passCount != testCount) - // ADD_FAILURE(); -} - -TEST(SchemaValidatingReader, Simple) { - Document sd; - sd.Parse("{ \"type\": \"string\", \"enum\" : [\"red\", \"amber\", \"green\"] }"); - SchemaDocument s(sd); - - Document d; - StringStream ss("\"red\""); - SchemaValidatingReader > reader(ss, s); - d.Populate(reader); - EXPECT_TRUE(reader.GetParseResult()); - EXPECT_TRUE(reader.IsValid()); - EXPECT_TRUE(d.IsString()); - EXPECT_STREQ("red", d.GetString()); -} - -TEST(SchemaValidatingReader, Invalid) { - Document sd; - sd.Parse("{\"type\":\"string\",\"minLength\":2,\"maxLength\":3}"); - SchemaDocument s(sd); - - Document d; - StringStream ss("\"ABCD\""); - SchemaValidatingReader > reader(ss, s); - d.Populate(reader); - EXPECT_FALSE(reader.GetParseResult()); - EXPECT_FALSE(reader.IsValid()); - EXPECT_EQ(kParseErrorTermination, reader.GetParseResult().Code()); - EXPECT_STREQ("maxLength", reader.GetInvalidSchemaKeyword()); - EXPECT_TRUE(reader.GetInvalidSchemaPointer() == SchemaDocument::PointerType("")); - EXPECT_TRUE(reader.GetInvalidDocumentPointer() == SchemaDocument::PointerType("")); - EXPECT_TRUE(d.IsNull()); -} - -TEST(SchemaValidatingWriter, Simple) { - Document sd; - sd.Parse("{\"type\":\"string\",\"minLength\":2,\"maxLength\":3}"); - SchemaDocument s(sd); - - Document d; - StringBuffer sb; - Writer writer(sb); - GenericSchemaValidator > validator(s, writer); - - d.Parse("\"red\""); - EXPECT_TRUE(d.Accept(validator)); - EXPECT_TRUE(validator.IsValid()); - EXPECT_STREQ("\"red\"", sb.GetString()); - - sb.Clear(); - validator.Reset(); - d.Parse("\"ABCD\""); - EXPECT_FALSE(d.Accept(validator)); - EXPECT_FALSE(validator.IsValid()); - EXPECT_TRUE(validator.GetInvalidSchemaPointer() == SchemaDocument::PointerType("")); - EXPECT_TRUE(validator.GetInvalidDocumentPointer() == SchemaDocument::PointerType("")); + documentAllocator.Clear(); + MemoryPoolAllocator<>::Free(json); + jsonAllocator.Clear(); + } + printf("%d / %d passed (%2d%%)\n", passCount, testCount, passCount * 100 / testCount); + // if (passCount != testCount) + // ADD_FAILURE(); +} + +TEST(SchemaValidatingReader, Simple) +{ Document sd; + sd.Parse("{ \"type\": \"string\", \"enum\" : [\"red\", \"amber\", \"green\"] }"); + SchemaDocument s(sd); + + Document d; + StringStream ss("\"red\""); + SchemaValidatingReader > reader(ss, s); + d.Populate(reader); + EXPECT_TRUE(reader.GetParseResult()); + EXPECT_TRUE(reader.IsValid()); + EXPECT_TRUE(d.IsString()); + EXPECT_STREQ("red", d.GetString()); +} + +TEST(SchemaValidatingReader, Invalid) +{ Document sd; + sd.Parse("{\"type\":\"string\",\"minLength\":2,\"maxLength\":3}"); + SchemaDocument s(sd); + + Document d; + StringStream ss("\"ABCD\""); + SchemaValidatingReader > reader(ss, s); + d.Populate(reader); + EXPECT_FALSE(reader.GetParseResult()); + EXPECT_FALSE(reader.IsValid()); + EXPECT_EQ(kParseErrorTermination, reader.GetParseResult().Code()); + EXPECT_STREQ("maxLength", reader.GetInvalidSchemaKeyword()); + EXPECT_TRUE(reader.GetInvalidSchemaPointer() == SchemaDocument::PointerType("")); + EXPECT_TRUE(reader.GetInvalidDocumentPointer() == SchemaDocument::PointerType("")); + EXPECT_TRUE(d.IsNull()); +} + +TEST(SchemaValidatingWriter, Simple) +{ Document sd; + sd.Parse("{\"type\":\"string\",\"minLength\":2,\"maxLength\":3}"); + SchemaDocument s(sd); + + Document d; + StringBuffer sb; + Writer writer(sb); + GenericSchemaValidator > validator(s, writer); + + d.Parse("\"red\""); + EXPECT_TRUE(d.Accept(validator)); + EXPECT_TRUE(validator.IsValid()); + EXPECT_STREQ("\"red\"", sb.GetString()); + + sb.Clear(); + validator.Reset(); + d.Parse("\"ABCD\""); + EXPECT_FALSE(d.Accept(validator)); + EXPECT_FALSE(validator.IsValid()); + EXPECT_TRUE(validator.GetInvalidSchemaPointer() == SchemaDocument::PointerType("")); + EXPECT_TRUE(validator.GetInvalidDocumentPointer() == SchemaDocument::PointerType("")); } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS -static SchemaDocument ReturnSchemaDocument() { - Document sd; - sd.Parse("{ \"type\": [\"number\", \"string\"] }"); - SchemaDocument s(sd); - return s; +static SchemaDocument ReturnSchemaDocument() +{ Document sd; + sd.Parse("{ \"type\": [\"number\", \"string\"] }"); + SchemaDocument s(sd); + return s; } -TEST(Schema, Issue552) { - SchemaDocument s = ReturnSchemaDocument(); - VALIDATE(s, "42", true); - VALIDATE(s, "\"Life, the universe, and everything\"", true); - INVALIDATE(s, "[\"Life\", \"the universe\", \"and everything\"]", "", "type", ""); +TEST(Schema, Issue552) +{ SchemaDocument s = ReturnSchemaDocument(); + VALIDATE(s, "42", true); + VALIDATE(s, "\"Life, the universe, and everything\"", true); + INVALIDATE(s, "[\"Life\", \"the universe\", \"and everything\"]", "", "type", ""); } #endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS -TEST(SchemaValidator, Issue608) { - Document sd; - sd.Parse("{\"required\": [\"a\", \"b\"] }"); - SchemaDocument s(sd); +TEST(SchemaValidator, Issue608) +{ Document sd; + sd.Parse("{\"required\": [\"a\", \"b\"] }"); + SchemaDocument s(sd); - VALIDATE(s, "{\"a\" : null, \"b\": null}", true); - INVALIDATE(s, "{\"a\" : null, \"a\" : null}", "", "required", ""); + VALIDATE(s, "{\"a\" : null, \"b\": null}", true); + INVALIDATE(s, "{\"a\" : null, \"a\" : null}", "", "required", ""); } #ifdef __clang__ diff --git a/rapidjson/test/unittest/simdtest.cpp b/rapidjson/test/unittest/simdtest.cpp index b01b559f42b..e00819de940 100644 --- a/rapidjson/test/unittest/simdtest.cpp +++ b/rapidjson/test/unittest/simdtest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. // Since Travis CI installs old Valgrind 3.7.0, which fails with some SSE4.2 @@ -46,168 +46,168 @@ using namespace rapidjson_simd; #endif template -void TestSkipWhitespace() { - for (size_t step = 1; step < 32; step++) { - char buffer[1025]; - for (size_t i = 0; i < 1024; i++) - buffer[i] = " \t\r\n"[i % 4]; - for (size_t i = 0; i < 1024; i += step) - buffer[i] = 'X'; - buffer[1024] = '\0'; - - StreamType s(buffer); - size_t i = 0; - for (;;) { - SkipWhitespace(s); - if (s.Peek() == '\0') - break; - EXPECT_EQ(i, s.Tell()); - EXPECT_EQ('X', s.Take()); - i += step; - } +void TestSkipWhitespace() +{ for (size_t step = 1; step < 32; step++) + { char buffer[1025]; + for (size_t i = 0; i < 1024; i++) + buffer[i] = " \t\r\n"[i % 4]; + for (size_t i = 0; i < 1024; i += step) + buffer[i] = 'X'; + buffer[1024] = '\0'; + + StreamType s(buffer); + size_t i = 0; + for (;;) + { SkipWhitespace(s); + if (s.Peek() == '\0') + break; + EXPECT_EQ(i, s.Tell()); + EXPECT_EQ('X', s.Take()); + i += step; } + } } -TEST(SIMD, SIMD_SUFFIX(SkipWhitespace)) { - TestSkipWhitespace(); - TestSkipWhitespace(); +TEST(SIMD, SIMD_SUFFIX(SkipWhitespace)) +{ TestSkipWhitespace(); + TestSkipWhitespace(); } -TEST(SIMD, SIMD_SUFFIX(SkipWhitespace_EncodedMemoryStream)) { - for (size_t step = 1; step < 32; step++) { - char buffer[1024]; - for (size_t i = 0; i < 1024; i++) - buffer[i] = " \t\r\n"[i % 4]; - for (size_t i = 0; i < 1024; i += step) - buffer[i] = 'X'; - - MemoryStream ms(buffer, 1024); - EncodedInputStream, MemoryStream> s(ms); - size_t i = 0; - for (;;) { - SkipWhitespace(s); - if (s.Peek() == '\0') - break; - //EXPECT_EQ(i, s.Tell()); - EXPECT_EQ('X', s.Take()); - i += step; - } +TEST(SIMD, SIMD_SUFFIX(SkipWhitespace_EncodedMemoryStream)) +{ for (size_t step = 1; step < 32; step++) + { char buffer[1024]; + for (size_t i = 0; i < 1024; i++) + buffer[i] = " \t\r\n"[i % 4]; + for (size_t i = 0; i < 1024; i += step) + buffer[i] = 'X'; + + MemoryStream ms(buffer, 1024); + EncodedInputStream, MemoryStream> s(ms); + size_t i = 0; + for (;;) + { SkipWhitespace(s); + if (s.Peek() == '\0') + break; + //EXPECT_EQ(i, s.Tell()); + EXPECT_EQ('X', s.Take()); + i += step; } + } } -struct ScanCopyUnescapedStringHandler : BaseReaderHandler, ScanCopyUnescapedStringHandler> { - bool String(const char* str, size_t length, bool) { - memcpy(buffer, str, length + 1); - return true; - } - char buffer[1024 + 5 + 32]; +struct ScanCopyUnescapedStringHandler : BaseReaderHandler, ScanCopyUnescapedStringHandler> +{ bool String(const char* str, size_t length, bool) + { memcpy(buffer, str, length + 1); + return true; + } + char buffer[1024 + 5 + 32]; }; template -void TestScanCopyUnescapedString() { - char buffer[1024 + 5 + 32]; - char backup[1024 + 5 + 32]; - - // Test "ABCDABCD...\\" - for (size_t offset = 0; offset < 32; offset++) { - for (size_t step = 0; step < 1024; step++) { - char* json = buffer + offset; - char *p = json; - *p++ = '\"'; - for (size_t i = 0; i < step; i++) - *p++ = "ABCD"[i % 4]; - *p++ = '\\'; - *p++ = '\\'; - *p++ = '\"'; - *p++ = '\0'; - strcpy(backup, json); // insitu parsing will overwrite buffer, so need to backup first - - StreamType s(json); - Reader reader; - ScanCopyUnescapedStringHandler h; - reader.Parse(s, h); - EXPECT_TRUE(memcmp(h.buffer, backup + 1, step) == 0); - EXPECT_EQ('\\', h.buffer[step]); // escaped - EXPECT_EQ('\0', h.buffer[step + 1]); - } +void TestScanCopyUnescapedString() +{ char buffer[1024 + 5 + 32]; + char backup[1024 + 5 + 32]; + + // Test "ABCDABCD...\\" + for (size_t offset = 0; offset < 32; offset++) + { for (size_t step = 0; step < 1024; step++) + { char* json = buffer + offset; + char *p = json; + *p++ = '\"'; + for (size_t i = 0; i < step; i++) + *p++ = "ABCD"[i % 4]; + *p++ = '\\'; + *p++ = '\\'; + *p++ = '\"'; + *p++ = '\0'; + strcpy(backup, json); // insitu parsing will overwrite buffer, so need to backup first + + StreamType s(json); + Reader reader; + ScanCopyUnescapedStringHandler h; + reader.Parse(s, h); + EXPECT_TRUE(memcmp(h.buffer, backup + 1, step) == 0); + EXPECT_EQ('\\', h.buffer[step]); // escaped + EXPECT_EQ('\0', h.buffer[step + 1]); } - - // Test "\\ABCDABCD..." - for (size_t offset = 0; offset < 32; offset++) { - for (size_t step = 0; step < 1024; step++) { - char* json = buffer + offset; - char *p = json; - *p++ = '\"'; - *p++ = '\\'; - *p++ = '\\'; - for (size_t i = 0; i < step; i++) - *p++ = "ABCD"[i % 4]; - *p++ = '\"'; - *p++ = '\0'; - strcpy(backup, json); // insitu parsing will overwrite buffer, so need to backup first - - StreamType s(json); - Reader reader; - ScanCopyUnescapedStringHandler h; - reader.Parse(s, h); - EXPECT_TRUE(memcmp(h.buffer + 1, backup + 3, step) == 0); - EXPECT_EQ('\\', h.buffer[0]); // escaped - EXPECT_EQ('\0', h.buffer[step + 1]); - } + } + + // Test "\\ABCDABCD..." + for (size_t offset = 0; offset < 32; offset++) + { for (size_t step = 0; step < 1024; step++) + { char* json = buffer + offset; + char *p = json; + *p++ = '\"'; + *p++ = '\\'; + *p++ = '\\'; + for (size_t i = 0; i < step; i++) + *p++ = "ABCD"[i % 4]; + *p++ = '\"'; + *p++ = '\0'; + strcpy(backup, json); // insitu parsing will overwrite buffer, so need to backup first + + StreamType s(json); + Reader reader; + ScanCopyUnescapedStringHandler h; + reader.Parse(s, h); + EXPECT_TRUE(memcmp(h.buffer + 1, backup + 3, step) == 0); + EXPECT_EQ('\\', h.buffer[0]); // escaped + EXPECT_EQ('\0', h.buffer[step + 1]); } + } } -TEST(SIMD, SIMD_SUFFIX(ScanCopyUnescapedString)) { - TestScanCopyUnescapedString(); - TestScanCopyUnescapedString(); +TEST(SIMD, SIMD_SUFFIX(ScanCopyUnescapedString)) +{ TestScanCopyUnescapedString(); + TestScanCopyUnescapedString(); } -TEST(SIMD, SIMD_SUFFIX(ScanWriteUnescapedString)) { - char buffer[2048 + 1 + 32]; - for (size_t offset = 0; offset < 32; offset++) { - for (size_t step = 0; step < 1024; step++) { - char* s = buffer + offset; - char* p = s; - for (size_t i = 0; i < step; i++) - *p++ = "ABCD"[i % 4]; - char escape = "\0\n\\\""[step % 4]; - *p++ = escape; - for (size_t i = 0; i < step; i++) - *p++ = "ABCD"[i % 4]; - - StringBuffer sb; - Writer writer(sb); - writer.String(s, SizeType(step * 2 + 1)); - const char* q = sb.GetString(); - EXPECT_EQ('\"', *q++); - for (size_t i = 0; i < step; i++) - EXPECT_EQ("ABCD"[i % 4], *q++); - if (escape == '\0') { - EXPECT_EQ('\\', *q++); - EXPECT_EQ('u', *q++); - EXPECT_EQ('0', *q++); - EXPECT_EQ('0', *q++); - EXPECT_EQ('0', *q++); - EXPECT_EQ('0', *q++); - } - else if (escape == '\n') { - EXPECT_EQ('\\', *q++); - EXPECT_EQ('n', *q++); - } - else if (escape == '\\') { - EXPECT_EQ('\\', *q++); - EXPECT_EQ('\\', *q++); - } - else if (escape == '\"') { - EXPECT_EQ('\\', *q++); - EXPECT_EQ('\"', *q++); - } - for (size_t i = 0; i < step; i++) - EXPECT_EQ("ABCD"[i % 4], *q++); - EXPECT_EQ('\"', *q++); - EXPECT_EQ('\0', *q++); - } +TEST(SIMD, SIMD_SUFFIX(ScanWriteUnescapedString)) +{ char buffer[2048 + 1 + 32]; + for (size_t offset = 0; offset < 32; offset++) + { for (size_t step = 0; step < 1024; step++) + { char* s = buffer + offset; + char* p = s; + for (size_t i = 0; i < step; i++) + *p++ = "ABCD"[i % 4]; + char escape = "\0\n\\\""[step % 4]; + *p++ = escape; + for (size_t i = 0; i < step; i++) + *p++ = "ABCD"[i % 4]; + + StringBuffer sb; + Writer writer(sb); + writer.String(s, SizeType(step * 2 + 1)); + const char* q = sb.GetString(); + EXPECT_EQ('\"', *q++); + for (size_t i = 0; i < step; i++) + EXPECT_EQ("ABCD"[i % 4], *q++); + if (escape == '\0') + { EXPECT_EQ('\\', *q++); + EXPECT_EQ('u', *q++); + EXPECT_EQ('0', *q++); + EXPECT_EQ('0', *q++); + EXPECT_EQ('0', *q++); + EXPECT_EQ('0', *q++); + } + else if (escape == '\n') + { EXPECT_EQ('\\', *q++); + EXPECT_EQ('n', *q++); + } + else if (escape == '\\') + { EXPECT_EQ('\\', *q++); + EXPECT_EQ('\\', *q++); + } + else if (escape == '\"') + { EXPECT_EQ('\\', *q++); + EXPECT_EQ('\"', *q++); + } + for (size_t i = 0; i < step; i++) + EXPECT_EQ("ABCD"[i % 4], *q++); + EXPECT_EQ('\"', *q++); + EXPECT_EQ('\0', *q++); } + } } #ifdef __GNUC__ diff --git a/rapidjson/test/unittest/strfunctest.cpp b/rapidjson/test/unittest/strfunctest.cpp index cc1bb22f0e9..9196b401b15 100644 --- a/rapidjson/test/unittest/strfunctest.cpp +++ b/rapidjson/test/unittest/strfunctest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -18,13 +18,13 @@ using namespace rapidjson; using namespace rapidjson::internal; -TEST(StrFunc, CountStringCodePoint) { - SizeType count; - EXPECT_TRUE(CountStringCodePoint >("", 0, &count)); - EXPECT_EQ(0u, count); - EXPECT_TRUE(CountStringCodePoint >("Hello", 5, &count)); - EXPECT_EQ(5u, count); - EXPECT_TRUE(CountStringCodePoint >("\xC2\xA2\xE2\x82\xAC\xF0\x9D\x84\x9E", 9, &count)); // cents euro G-clef - EXPECT_EQ(3u, count); - EXPECT_FALSE(CountStringCodePoint >("\xC2\xA2\xE2\x82\xAC\xF0\x9D\x84\x9E\x80", 10, &count)); +TEST(StrFunc, CountStringCodePoint) +{ SizeType count; + EXPECT_TRUE(CountStringCodePoint >("", 0, &count)); + EXPECT_EQ(0u, count); + EXPECT_TRUE(CountStringCodePoint >("Hello", 5, &count)); + EXPECT_EQ(5u, count); + EXPECT_TRUE(CountStringCodePoint >("\xC2\xA2\xE2\x82\xAC\xF0\x9D\x84\x9E", 9, &count)); // cents euro G-clef + EXPECT_EQ(3u, count); + EXPECT_FALSE(CountStringCodePoint >("\xC2\xA2\xE2\x82\xAC\xF0\x9D\x84\x9E\x80", 10, &count)); } diff --git a/rapidjson/test/unittest/stringbuffertest.cpp b/rapidjson/test/unittest/stringbuffertest.cpp index ded513cddcf..ede26592682 100644 --- a/rapidjson/test/unittest/stringbuffertest.cpp +++ b/rapidjson/test/unittest/stringbuffertest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -23,60 +23,60 @@ RAPIDJSON_DIAG_OFF(c++98-compat) using namespace rapidjson; -TEST(StringBuffer, InitialSize) { - StringBuffer buffer; - EXPECT_EQ(0u, buffer.GetSize()); - EXPECT_STREQ("", buffer.GetString()); +TEST(StringBuffer, InitialSize) +{ StringBuffer buffer; + EXPECT_EQ(0u, buffer.GetSize()); + EXPECT_STREQ("", buffer.GetString()); } -TEST(StringBuffer, Put) { - StringBuffer buffer; - buffer.Put('A'); +TEST(StringBuffer, Put) +{ StringBuffer buffer; + buffer.Put('A'); - EXPECT_EQ(1u, buffer.GetSize()); - EXPECT_STREQ("A", buffer.GetString()); + EXPECT_EQ(1u, buffer.GetSize()); + EXPECT_STREQ("A", buffer.GetString()); } -TEST(StringBuffer, PutN_Issue672) { - GenericStringBuffer, MemoryPoolAllocator<> > buffer; - EXPECT_EQ(0, buffer.GetSize()); - rapidjson::PutN(buffer, ' ', 1); - EXPECT_EQ(1, buffer.GetSize()); +TEST(StringBuffer, PutN_Issue672) +{ GenericStringBuffer, MemoryPoolAllocator<> > buffer; + EXPECT_EQ(0, buffer.GetSize()); + rapidjson::PutN(buffer, ' ', 1); + EXPECT_EQ(1, buffer.GetSize()); } -TEST(StringBuffer, Clear) { - StringBuffer buffer; - buffer.Put('A'); - buffer.Put('B'); - buffer.Put('C'); - buffer.Clear(); +TEST(StringBuffer, Clear) +{ StringBuffer buffer; + buffer.Put('A'); + buffer.Put('B'); + buffer.Put('C'); + buffer.Clear(); - EXPECT_EQ(0u, buffer.GetSize()); - EXPECT_STREQ("", buffer.GetString()); + EXPECT_EQ(0u, buffer.GetSize()); + EXPECT_STREQ("", buffer.GetString()); } -TEST(StringBuffer, Push) { - StringBuffer buffer; - buffer.Push(5); +TEST(StringBuffer, Push) +{ StringBuffer buffer; + buffer.Push(5); - EXPECT_EQ(5u, buffer.GetSize()); + EXPECT_EQ(5u, buffer.GetSize()); - // Causes sudden expansion to make the stack's capacity equal to size - buffer.Push(65536u); - EXPECT_EQ(5u + 65536u, buffer.GetSize()); + // Causes sudden expansion to make the stack's capacity equal to size + buffer.Push(65536u); + EXPECT_EQ(5u + 65536u, buffer.GetSize()); } -TEST(StringBuffer, Pop) { - StringBuffer buffer; - buffer.Put('A'); - buffer.Put('B'); - buffer.Put('C'); - buffer.Put('D'); - buffer.Put('E'); - buffer.Pop(3); - - EXPECT_EQ(2u, buffer.GetSize()); - EXPECT_STREQ("AB", buffer.GetString()); +TEST(StringBuffer, Pop) +{ StringBuffer buffer; + buffer.Put('A'); + buffer.Put('B'); + buffer.Put('C'); + buffer.Put('D'); + buffer.Put('E'); + buffer.Pop(3); + + EXPECT_EQ(2u, buffer.GetSize()); + EXPECT_STREQ("AB", buffer.GetString()); } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS @@ -85,82 +85,82 @@ TEST(StringBuffer, Pop) { #include -TEST(StringBuffer, Traits) { - static_assert( std::is_constructible::value, ""); - static_assert( std::is_default_constructible::value, ""); +TEST(StringBuffer, Traits) +{ static_assert( std::is_constructible::value, ""); + static_assert( std::is_default_constructible::value, ""); #ifndef _MSC_VER - static_assert(!std::is_copy_constructible::value, ""); + static_assert(!std::is_copy_constructible::value, ""); #endif - static_assert( std::is_move_constructible::value, ""); + static_assert( std::is_move_constructible::value, ""); - static_assert(!std::is_nothrow_constructible::value, ""); - static_assert(!std::is_nothrow_default_constructible::value, ""); + static_assert(!std::is_nothrow_constructible::value, ""); + static_assert(!std::is_nothrow_default_constructible::value, ""); #if !defined(_MSC_VER) || _MSC_VER >= 1800 - static_assert(!std::is_nothrow_copy_constructible::value, ""); - static_assert(!std::is_nothrow_move_constructible::value, ""); + static_assert(!std::is_nothrow_copy_constructible::value, ""); + static_assert(!std::is_nothrow_move_constructible::value, ""); #endif - static_assert( std::is_assignable::value, ""); + static_assert( std::is_assignable::value, ""); #ifndef _MSC_VER - static_assert(!std::is_copy_assignable::value, ""); + static_assert(!std::is_copy_assignable::value, ""); #endif - static_assert( std::is_move_assignable::value, ""); + static_assert( std::is_move_assignable::value, ""); #if !defined(_MSC_VER) || _MSC_VER >= 1800 - static_assert(!std::is_nothrow_assignable::value, ""); + static_assert(!std::is_nothrow_assignable::value, ""); #endif - static_assert(!std::is_nothrow_copy_assignable::value, ""); - static_assert(!std::is_nothrow_move_assignable::value, ""); + static_assert(!std::is_nothrow_copy_assignable::value, ""); + static_assert(!std::is_nothrow_move_assignable::value, ""); - static_assert( std::is_destructible::value, ""); + static_assert( std::is_destructible::value, ""); #ifndef _MSC_VER - static_assert(std::is_nothrow_destructible::value, ""); + static_assert(std::is_nothrow_destructible::value, ""); #endif } #endif -TEST(StringBuffer, MoveConstructor) { - StringBuffer x; - x.Put('A'); - x.Put('B'); - x.Put('C'); - x.Put('D'); - - EXPECT_EQ(4u, x.GetSize()); - EXPECT_STREQ("ABCD", x.GetString()); - - // StringBuffer y(x); // does not compile (!is_copy_constructible) - StringBuffer y(std::move(x)); - EXPECT_EQ(0u, x.GetSize()); - EXPECT_EQ(4u, y.GetSize()); - EXPECT_STREQ("ABCD", y.GetString()); - - // StringBuffer z = y; // does not compile (!is_copy_assignable) - StringBuffer z = std::move(y); - EXPECT_EQ(0u, y.GetSize()); - EXPECT_EQ(4u, z.GetSize()); - EXPECT_STREQ("ABCD", z.GetString()); +TEST(StringBuffer, MoveConstructor) +{ StringBuffer x; + x.Put('A'); + x.Put('B'); + x.Put('C'); + x.Put('D'); + + EXPECT_EQ(4u, x.GetSize()); + EXPECT_STREQ("ABCD", x.GetString()); + + // StringBuffer y(x); // does not compile (!is_copy_constructible) + StringBuffer y(std::move(x)); + EXPECT_EQ(0u, x.GetSize()); + EXPECT_EQ(4u, y.GetSize()); + EXPECT_STREQ("ABCD", y.GetString()); + + // StringBuffer z = y; // does not compile (!is_copy_assignable) + StringBuffer z = std::move(y); + EXPECT_EQ(0u, y.GetSize()); + EXPECT_EQ(4u, z.GetSize()); + EXPECT_STREQ("ABCD", z.GetString()); } -TEST(StringBuffer, MoveAssignment) { - StringBuffer x; - x.Put('A'); - x.Put('B'); - x.Put('C'); - x.Put('D'); - - EXPECT_EQ(4u, x.GetSize()); - EXPECT_STREQ("ABCD", x.GetString()); - - StringBuffer y; - // y = x; // does not compile (!is_copy_assignable) - y = std::move(x); - EXPECT_EQ(0u, x.GetSize()); - EXPECT_EQ(4u, y.GetSize()); - EXPECT_STREQ("ABCD", y.GetString()); +TEST(StringBuffer, MoveAssignment) +{ StringBuffer x; + x.Put('A'); + x.Put('B'); + x.Put('C'); + x.Put('D'); + + EXPECT_EQ(4u, x.GetSize()); + EXPECT_STREQ("ABCD", x.GetString()); + + StringBuffer y; + // y = x; // does not compile (!is_copy_assignable) + y = std::move(x); + EXPECT_EQ(0u, x.GetSize()); + EXPECT_EQ(4u, y.GetSize()); + EXPECT_STREQ("ABCD", y.GetString()); } #endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS diff --git a/rapidjson/test/unittest/strtodtest.cpp b/rapidjson/test/unittest/strtodtest.cpp index cde836c7eb7..0bf6c90b8d3 100644 --- a/rapidjson/test/unittest/strtodtest.cpp +++ b/rapidjson/test/unittest/strtodtest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -25,106 +25,106 @@ RAPIDJSON_DIAG_OFF(unreachable-code) using namespace rapidjson::internal; -TEST(Strtod, CheckApproximationCase) { - static const int kSignificandSize = 52; - static const int kExponentBias = 0x3FF; - static const uint64_t kExponentMask = RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000); - static const uint64_t kSignificandMask = RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF); - static const uint64_t kHiddenBit = RAPIDJSON_UINT64_C2(0x00100000, 0x00000000); - - // http://www.exploringbinary.com/using-integers-to-check-a-floating-point-approximation/ - // Let b = 0x1.465a72e467d88p-149 - // = 5741268244528520 x 2^-201 - union { - double d; - uint64_t u; - }u; - u.u = 0x465a72e467d88 | ((static_cast(-149 + kExponentBias)) << kSignificandSize); - const double b = u.d; - const uint64_t bInt = (u.u & kSignificandMask) | kHiddenBit; - const int bExp = static_cast(((u.u & kExponentMask) >> kSignificandSize) - kExponentBias - kSignificandSize); - EXPECT_DOUBLE_EQ(1.7864e-45, b); - EXPECT_EQ(RAPIDJSON_UINT64_C2(0x001465a7, 0x2e467d88), bInt); - EXPECT_EQ(-201, bExp); - - // Let d = 17864 x 10-49 - const char dInt[] = "17864"; - const int dExp = -49; - - // Let h = 2^(bExp-1) - const int hExp = bExp - 1; - EXPECT_EQ(-202, hExp); - - int dS_Exp2 = 0; - int dS_Exp5 = 0; - int bS_Exp2 = 0; - int bS_Exp5 = 0; - int hS_Exp2 = 0; - int hS_Exp5 = 0; - - // Adjust for decimal exponent - if (dExp >= 0) { - dS_Exp2 += dExp; - dS_Exp5 += dExp; - } - else { - bS_Exp2 -= dExp; - bS_Exp5 -= dExp; - hS_Exp2 -= dExp; - hS_Exp5 -= dExp; - } - - // Adjust for binary exponent - if (bExp >= 0) - bS_Exp2 += bExp; - else { - dS_Exp2 -= bExp; - hS_Exp2 -= bExp; - } - - // Adjust for half ulp exponent - if (hExp >= 0) - hS_Exp2 += hExp; - else { - dS_Exp2 -= hExp; - bS_Exp2 -= hExp; - } - - // Remove common power of two factor from all three scaled values - int common_Exp2 = std::min(dS_Exp2, std::min(bS_Exp2, hS_Exp2)); - dS_Exp2 -= common_Exp2; - bS_Exp2 -= common_Exp2; - hS_Exp2 -= common_Exp2; - - EXPECT_EQ(153, dS_Exp2); - EXPECT_EQ(0, dS_Exp5); - EXPECT_EQ(1, bS_Exp2); - EXPECT_EQ(49, bS_Exp5); - EXPECT_EQ(0, hS_Exp2); - EXPECT_EQ(49, hS_Exp5); - - BigInteger dS = BIGINTEGER_LITERAL(dInt); - dS.MultiplyPow5(static_cast(dS_Exp5)) <<= static_cast(dS_Exp2); - - BigInteger bS(bInt); - bS.MultiplyPow5(static_cast(bS_Exp5)) <<= static_cast(bS_Exp2); - - BigInteger hS(1); - hS.MultiplyPow5(static_cast(hS_Exp5)) <<= static_cast(hS_Exp2); - - EXPECT_TRUE(BIGINTEGER_LITERAL("203970822259994138521801764465966248930731085529088") == dS); - EXPECT_TRUE(BIGINTEGER_LITERAL("203970822259994122305215569213032722473144531250000") == bS); - EXPECT_TRUE(BIGINTEGER_LITERAL("17763568394002504646778106689453125") == hS); - - EXPECT_EQ(1, dS.Compare(bS)); - - BigInteger delta(0); - EXPECT_FALSE(dS.Difference(bS, &delta)); - EXPECT_TRUE(BIGINTEGER_LITERAL("16216586195252933526457586554279088") == delta); - EXPECT_TRUE(bS.Difference(dS, &delta)); - EXPECT_TRUE(BIGINTEGER_LITERAL("16216586195252933526457586554279088") == delta); - - EXPECT_EQ(-1, delta.Compare(hS)); +TEST(Strtod, CheckApproximationCase) +{ static const int kSignificandSize = 52; + static const int kExponentBias = 0x3FF; + static const uint64_t kExponentMask = RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000); + static const uint64_t kSignificandMask = RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF); + static const uint64_t kHiddenBit = RAPIDJSON_UINT64_C2(0x00100000, 0x00000000); + + // http://www.exploringbinary.com/using-integers-to-check-a-floating-point-approximation/ + // Let b = 0x1.465a72e467d88p-149 + // = 5741268244528520 x 2^-201 + union + { double d; + uint64_t u; + } u; + u.u = 0x465a72e467d88 | ((static_cast(-149 + kExponentBias)) << kSignificandSize); + const double b = u.d; + const uint64_t bInt = (u.u & kSignificandMask) | kHiddenBit; + const int bExp = static_cast(((u.u & kExponentMask) >> kSignificandSize) - kExponentBias - kSignificandSize); + EXPECT_DOUBLE_EQ(1.7864e-45, b); + EXPECT_EQ(RAPIDJSON_UINT64_C2(0x001465a7, 0x2e467d88), bInt); + EXPECT_EQ(-201, bExp); + + // Let d = 17864 x 10-49 + const char dInt[] = "17864"; + const int dExp = -49; + + // Let h = 2^(bExp-1) + const int hExp = bExp - 1; + EXPECT_EQ(-202, hExp); + + int dS_Exp2 = 0; + int dS_Exp5 = 0; + int bS_Exp2 = 0; + int bS_Exp5 = 0; + int hS_Exp2 = 0; + int hS_Exp5 = 0; + + // Adjust for decimal exponent + if (dExp >= 0) + { dS_Exp2 += dExp; + dS_Exp5 += dExp; + } + else + { bS_Exp2 -= dExp; + bS_Exp5 -= dExp; + hS_Exp2 -= dExp; + hS_Exp5 -= dExp; + } + + // Adjust for binary exponent + if (bExp >= 0) + bS_Exp2 += bExp; + else + { dS_Exp2 -= bExp; + hS_Exp2 -= bExp; + } + + // Adjust for half ulp exponent + if (hExp >= 0) + hS_Exp2 += hExp; + else + { dS_Exp2 -= hExp; + bS_Exp2 -= hExp; + } + + // Remove common power of two factor from all three scaled values + int common_Exp2 = std::min(dS_Exp2, std::min(bS_Exp2, hS_Exp2)); + dS_Exp2 -= common_Exp2; + bS_Exp2 -= common_Exp2; + hS_Exp2 -= common_Exp2; + + EXPECT_EQ(153, dS_Exp2); + EXPECT_EQ(0, dS_Exp5); + EXPECT_EQ(1, bS_Exp2); + EXPECT_EQ(49, bS_Exp5); + EXPECT_EQ(0, hS_Exp2); + EXPECT_EQ(49, hS_Exp5); + + BigInteger dS = BIGINTEGER_LITERAL(dInt); + dS.MultiplyPow5(static_cast(dS_Exp5)) <<= static_cast(dS_Exp2); + + BigInteger bS(bInt); + bS.MultiplyPow5(static_cast(bS_Exp5)) <<= static_cast(bS_Exp2); + + BigInteger hS(1); + hS.MultiplyPow5(static_cast(hS_Exp5)) <<= static_cast(hS_Exp2); + + EXPECT_TRUE(BIGINTEGER_LITERAL("203970822259994138521801764465966248930731085529088") == dS); + EXPECT_TRUE(BIGINTEGER_LITERAL("203970822259994122305215569213032722473144531250000") == bS); + EXPECT_TRUE(BIGINTEGER_LITERAL("17763568394002504646778106689453125") == hS); + + EXPECT_EQ(1, dS.Compare(bS)); + + BigInteger delta(0); + EXPECT_FALSE(dS.Difference(bS, &delta)); + EXPECT_TRUE(BIGINTEGER_LITERAL("16216586195252933526457586554279088") == delta); + EXPECT_TRUE(bS.Difference(dS, &delta)); + EXPECT_TRUE(BIGINTEGER_LITERAL("16216586195252933526457586554279088") == delta); + + EXPECT_EQ(-1, delta.Compare(hS)); } #ifdef __clang__ diff --git a/rapidjson/test/unittest/unittest.cpp b/rapidjson/test/unittest/unittest.cpp index b754563ea2b..2cf0bcbc0c6 100644 --- a/rapidjson/test/unittest/unittest.cpp +++ b/rapidjson/test/unittest/unittest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -28,24 +28,24 @@ AssertException::~AssertException() throw() {} #pragma GCC diagnostic pop #endif -int main(int argc, char **argv) { - ::testing::InitGoogleTest(&argc, argv); +int main(int argc, char **argv) +{ ::testing::InitGoogleTest(&argc, argv); - std::cout << "RapidJSON v" << RAPIDJSON_VERSION_STRING << std::endl; + std::cout << "RapidJSON v" << RAPIDJSON_VERSION_STRING << std::endl; #ifdef _MSC_VER - _CrtMemState memoryState = { 0 }; - (void)memoryState; - _CrtMemCheckpoint(&memoryState); - //_CrtSetBreakAlloc(X); - //void *testWhetherMemoryLeakDetectionWorks = malloc(1); + _CrtMemState memoryState = { 0 }; + (void)memoryState; + _CrtMemCheckpoint(&memoryState); + //_CrtSetBreakAlloc(X); + //void *testWhetherMemoryLeakDetectionWorks = malloc(1); #endif - int ret = RUN_ALL_TESTS(); + int ret = RUN_ALL_TESTS(); #ifdef _MSC_VER - // Current gtest constantly leak 2 blocks at exit - _CrtMemDumpAllObjectsSince(&memoryState); + // Current gtest constantly leak 2 blocks at exit + _CrtMemDumpAllObjectsSince(&memoryState); #endif - return ret; + return ret; } diff --git a/rapidjson/test/unittest/unittest.h b/rapidjson/test/unittest/unittest.h index e125bf88dc3..b2bc7760877 100644 --- a/rapidjson/test/unittest/unittest.h +++ b/rapidjson/test/unittest/unittest.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef UNITTEST_H_ @@ -57,40 +57,41 @@ #endif template -inline unsigned StrLen(const Ch* s) { - const Ch* p = s; - while (*p) p++; - return unsigned(p - s); +inline unsigned StrLen(const Ch* s) +{ const Ch* p = s; + while (*p) p++; + return unsigned(p - s); } template -inline int StrCmp(const Ch* s1, const Ch* s2) { - while(*s1 && (*s1 == *s2)) { s1++; s2++; } - return static_cast(*s1) < static_cast(*s2) ? -1 : static_cast(*s1) > static_cast(*s2); +inline int StrCmp(const Ch* s1, const Ch* s2) +{ while(*s1 && (*s1 == *s2)) { s1++; s2++; } + return static_cast(*s1) < static_cast(*s2) ? -1 : static_cast(*s1) > static_cast(*s2); } template -inline Ch* StrDup(const Ch* str) { - size_t bufferSize = sizeof(Ch) * (StrLen(str) + 1); - Ch* buffer = static_cast(malloc(bufferSize)); - memcpy(buffer, str, bufferSize); - return buffer; +inline Ch* StrDup(const Ch* str) +{ size_t bufferSize = sizeof(Ch) * (StrLen(str) + 1); + Ch* buffer = static_cast(malloc(bufferSize)); + memcpy(buffer, str, bufferSize); + return buffer; } -inline FILE* TempFile(char *filename) { +inline FILE* TempFile(char *filename) +{ #ifdef _MSC_VER - filename = tmpnam(filename); - - // For Visual Studio, tmpnam() adds a backslash in front. Remove it. - if (filename[0] == '\\') - for (int i = 0; filename[i] != '\0'; i++) - filename[i] = filename[i + 1]; - - return fopen(filename, "wb"); + filename = tmpnam(filename); + + // For Visual Studio, tmpnam() adds a backslash in front. Remove it. + if (filename[0] == '\\') + for (int i = 0; filename[i] != '\0'; i++) + filename[i] = filename[i + 1]; + + return fopen(filename, "wb"); #else - strcpy(filename, "/tmp/fileXXXXXX"); - int fd = mkstemp(filename); - return fdopen(fd, "w"); + strcpy(filename, "/tmp/fileXXXXXX"); + int fd = mkstemp(filename); + return fdopen(fd, "w"); #endif } @@ -106,11 +107,12 @@ inline FILE* TempFile(char *filename) { #endif #endif -class AssertException : public std::logic_error { +class AssertException : public std::logic_error +{ public: - AssertException(const char* w) : std::logic_error(w) {} - AssertException(const AssertException& rhs) : std::logic_error(rhs) {} - virtual ~AssertException() throw(); + AssertException(const char* w) : std::logic_error(w) {} + AssertException(const AssertException& rhs) : std::logic_error(rhs) {} + virtual ~AssertException() throw(); }; #ifdef __clang__ @@ -119,17 +121,18 @@ class AssertException : public std::logic_error { #define RAPIDJSON_ASSERT(x) if (!(x)) throw AssertException(RAPIDJSON_STRINGIFY(x)) -class Random { +class Random +{ public: - Random(unsigned seed = 0) : mSeed(seed) {} + Random(unsigned seed = 0) : mSeed(seed) {} - unsigned operator()() { - mSeed = 214013 * mSeed + 2531011; - return mSeed; - } + unsigned operator()() + { mSeed = 214013 * mSeed + 2531011; + return mSeed; + } private: - unsigned mSeed; + unsigned mSeed; }; #endif // UNITTEST_H_ diff --git a/rapidjson/test/unittest/valuetest.cpp b/rapidjson/test/unittest/valuetest.cpp index fefc001d45d..a1302f0be1a 100644 --- a/rapidjson/test/unittest/valuetest.cpp +++ b/rapidjson/test/unittest/valuetest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -23,24 +23,25 @@ RAPIDJSON_DIAG_OFF(c++98-compat) using namespace rapidjson; -TEST(Value, Size) { - if (sizeof(SizeType) == 4) { +TEST(Value, Size) +{ if (sizeof(SizeType) == 4) + { #if RAPIDJSON_48BITPOINTER_OPTIMIZATION - EXPECT_EQ(16, sizeof(Value)); + EXPECT_EQ(16, sizeof(Value)); #elif RAPIDJSON_64BIT - EXPECT_EQ(24, sizeof(Value)); + EXPECT_EQ(24, sizeof(Value)); #else - EXPECT_EQ(16, sizeof(Value)); + EXPECT_EQ(16, sizeof(Value)); #endif - } + } } -TEST(Value, DefaultConstructor) { - Value x; - EXPECT_EQ(kNullType, x.GetType()); - EXPECT_TRUE(x.IsNull()); +TEST(Value, DefaultConstructor) +{ Value x; + EXPECT_EQ(kNullType, x.GetType()); + EXPECT_TRUE(x.IsNull()); - //std::cout << "sizeof(Value): " << sizeof(x) << std::endl; + //std::cout << "sizeof(Value): " << sizeof(x) << std::endl; } // Should not pass compilation @@ -55,1628 +56,1610 @@ TEST(Value, DefaultConstructor) { #include -TEST(Value, Traits) { - typedef GenericValue, CrtAllocator> Value; - static_assert(std::is_constructible::value, ""); - static_assert(std::is_default_constructible::value, ""); +TEST(Value, Traits) +{ typedef GenericValue, CrtAllocator> Value; + static_assert(std::is_constructible::value, ""); + static_assert(std::is_default_constructible::value, ""); #ifndef _MSC_VER - static_assert(!std::is_copy_constructible::value, ""); + static_assert(!std::is_copy_constructible::value, ""); #endif - static_assert(std::is_move_constructible::value, ""); + static_assert(std::is_move_constructible::value, ""); #ifndef _MSC_VER - static_assert(std::is_nothrow_constructible::value, ""); - static_assert(std::is_nothrow_default_constructible::value, ""); - static_assert(!std::is_nothrow_copy_constructible::value, ""); - static_assert(std::is_nothrow_move_constructible::value, ""); + static_assert(std::is_nothrow_constructible::value, ""); + static_assert(std::is_nothrow_default_constructible::value, ""); + static_assert(!std::is_nothrow_copy_constructible::value, ""); + static_assert(std::is_nothrow_move_constructible::value, ""); #endif - static_assert(std::is_assignable::value, ""); + static_assert(std::is_assignable::value, ""); #ifndef _MSC_VER - static_assert(!std::is_copy_assignable::value, ""); + static_assert(!std::is_copy_assignable::value, ""); #endif - static_assert(std::is_move_assignable::value, ""); + static_assert(std::is_move_assignable::value, ""); #ifndef _MSC_VER - static_assert(std::is_nothrow_assignable::value, ""); + static_assert(std::is_nothrow_assignable::value, ""); #endif - static_assert(!std::is_nothrow_copy_assignable::value, ""); + static_assert(!std::is_nothrow_copy_assignable::value, ""); #ifndef _MSC_VER - static_assert(std::is_nothrow_move_assignable::value, ""); + static_assert(std::is_nothrow_move_assignable::value, ""); #endif - static_assert(std::is_destructible::value, ""); + static_assert(std::is_destructible::value, ""); #ifndef _MSC_VER - static_assert(std::is_nothrow_destructible::value, ""); + static_assert(std::is_nothrow_destructible::value, ""); #endif } #endif -TEST(Value, MoveConstructor) { - typedef GenericValue, CrtAllocator> V; - V::AllocatorType allocator; - - V x((V(kArrayType))); - x.Reserve(4u, allocator); - x.PushBack(1, allocator).PushBack(2, allocator).PushBack(3, allocator).PushBack(4, allocator); - EXPECT_TRUE(x.IsArray()); - EXPECT_EQ(4u, x.Size()); - - // Value y(x); // does not compile (!is_copy_constructible) - V y(std::move(x)); - EXPECT_TRUE(x.IsNull()); - EXPECT_TRUE(y.IsArray()); - EXPECT_EQ(4u, y.Size()); - - // Value z = y; // does not compile (!is_copy_assignable) - V z = std::move(y); - EXPECT_TRUE(y.IsNull()); - EXPECT_TRUE(z.IsArray()); - EXPECT_EQ(4u, z.Size()); +TEST(Value, MoveConstructor) +{ typedef GenericValue, CrtAllocator> V; + V::AllocatorType allocator; + + V x((V(kArrayType))); + x.Reserve(4u, allocator); + x.PushBack(1, allocator).PushBack(2, allocator).PushBack(3, allocator).PushBack(4, allocator); + EXPECT_TRUE(x.IsArray()); + EXPECT_EQ(4u, x.Size()); + + // Value y(x); // does not compile (!is_copy_constructible) + V y(std::move(x)); + EXPECT_TRUE(x.IsNull()); + EXPECT_TRUE(y.IsArray()); + EXPECT_EQ(4u, y.Size()); + + // Value z = y; // does not compile (!is_copy_assignable) + V z = std::move(y); + EXPECT_TRUE(y.IsNull()); + EXPECT_TRUE(z.IsArray()); + EXPECT_EQ(4u, z.Size()); } #endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS -TEST(Value, AssignmentOperator) { - Value x(1234); - Value y; - y = x; - EXPECT_TRUE(x.IsNull()); // move semantic - EXPECT_EQ(1234, y.GetInt()); +TEST(Value, AssignmentOperator) +{ Value x(1234); + Value y; + y = x; + EXPECT_TRUE(x.IsNull()); // move semantic + EXPECT_EQ(1234, y.GetInt()); - y = 5678; - EXPECT_TRUE(y.IsInt()); - EXPECT_EQ(5678, y.GetInt()); + y = 5678; + EXPECT_TRUE(y.IsInt()); + EXPECT_EQ(5678, y.GetInt()); - x = "Hello"; - EXPECT_TRUE(x.IsString()); - EXPECT_STREQ(x.GetString(),"Hello"); + x = "Hello"; + EXPECT_TRUE(x.IsString()); + EXPECT_STREQ(x.GetString(),"Hello"); - y = StringRef(x.GetString(),x.GetStringLength()); - EXPECT_TRUE(y.IsString()); - EXPECT_EQ(y.GetString(),x.GetString()); - EXPECT_EQ(y.GetStringLength(),x.GetStringLength()); + y = StringRef(x.GetString(),x.GetStringLength()); + EXPECT_TRUE(y.IsString()); + EXPECT_EQ(y.GetString(),x.GetString()); + EXPECT_EQ(y.GetStringLength(),x.GetStringLength()); - static char mstr[] = "mutable"; - // y = mstr; // should not compile - y = StringRef(mstr); - EXPECT_TRUE(y.IsString()); - EXPECT_EQ(y.GetString(),mstr); + static char mstr[] = "mutable"; + // y = mstr; // should not compile + y = StringRef(mstr); + EXPECT_TRUE(y.IsString()); + EXPECT_EQ(y.GetString(),mstr); #if RAPIDJSON_HAS_CXX11_RVALUE_REFS - // C++11 move assignment - x = Value("World"); - EXPECT_TRUE(x.IsString()); - EXPECT_STREQ("World", x.GetString()); - - x = std::move(y); - EXPECT_TRUE(y.IsNull()); - EXPECT_TRUE(x.IsString()); - EXPECT_EQ(x.GetString(), mstr); - - y = std::move(Value().SetInt(1234)); - EXPECT_TRUE(y.IsInt()); - EXPECT_EQ(1234, y); + // C++11 move assignment + x = Value("World"); + EXPECT_TRUE(x.IsString()); + EXPECT_STREQ("World", x.GetString()); + + x = std::move(y); + EXPECT_TRUE(y.IsNull()); + EXPECT_TRUE(x.IsString()); + EXPECT_EQ(x.GetString(), mstr); + + y = std::move(Value().SetInt(1234)); + EXPECT_TRUE(y.IsInt()); + EXPECT_EQ(1234, y); #endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS } -template -void TestEqual(const A& a, const B& b) { - EXPECT_TRUE (a == b); - EXPECT_FALSE(a != b); - EXPECT_TRUE (b == a); - EXPECT_FALSE(b != a); +template +void TestEqual(const A& a, const B& b) +{ EXPECT_TRUE (a == b); + EXPECT_FALSE(a != b); + EXPECT_TRUE (b == a); + EXPECT_FALSE(b != a); } -template -void TestUnequal(const A& a, const B& b) { - EXPECT_FALSE(a == b); - EXPECT_TRUE (a != b); - EXPECT_FALSE(b == a); - EXPECT_TRUE (b != a); +template +void TestUnequal(const A& a, const B& b) +{ EXPECT_FALSE(a == b); + EXPECT_TRUE (a != b); + EXPECT_FALSE(b == a); + EXPECT_TRUE (b != a); } -TEST(Value, EqualtoOperator) { - Value::AllocatorType allocator; - Value x(kObjectType); - x.AddMember("hello", "world", allocator) - .AddMember("t", Value(true).Move(), allocator) - .AddMember("f", Value(false).Move(), allocator) - .AddMember("n", Value(kNullType).Move(), allocator) - .AddMember("i", 123, allocator) - .AddMember("pi", 3.14, allocator) - .AddMember("a", Value(kArrayType).Move().PushBack(1, allocator).PushBack(2, allocator).PushBack(3, allocator), allocator); - - // Test templated operator==() and operator!=() - TestEqual(x["hello"], "world"); - const char* cc = "world"; - TestEqual(x["hello"], cc); - char* c = strdup("world"); - TestEqual(x["hello"], c); - free(c); - - TestEqual(x["t"], true); - TestEqual(x["f"], false); - TestEqual(x["i"], 123); - TestEqual(x["pi"], 3.14); - - // Test operator==() (including different allocators) - CrtAllocator crtAllocator; - GenericValue, CrtAllocator> y; - GenericDocument, CrtAllocator> z(&crtAllocator); - y.CopyFrom(x, crtAllocator); - z.CopyFrom(y, z.GetAllocator()); - TestEqual(x, y); - TestEqual(y, z); - TestEqual(z, x); - - // Swapping member order should be fine. - EXPECT_TRUE(y.RemoveMember("t")); - TestUnequal(x, y); - TestUnequal(z, y); - EXPECT_TRUE(z.RemoveMember("t")); - TestUnequal(x, z); - TestEqual(y, z); - y.AddMember("t", false, crtAllocator); - z.AddMember("t", false, z.GetAllocator()); - TestUnequal(x, y); - TestUnequal(z, x); - y["t"] = true; - z["t"] = true; - TestEqual(x, y); - TestEqual(y, z); - TestEqual(z, x); - - // Swapping element order is not OK - x["a"][0].Swap(x["a"][1]); - TestUnequal(x, y); - x["a"][0].Swap(x["a"][1]); - TestEqual(x, y); - - // Array of different size - x["a"].PushBack(4, allocator); - TestUnequal(x, y); - x["a"].PopBack(); - TestEqual(x, y); - - // Issue #129: compare Uint64 - x.SetUint64(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0xFFFFFFF0)); - y.SetUint64(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0xFFFFFFFF)); - TestUnequal(x, y); +TEST(Value, EqualtoOperator) +{ Value::AllocatorType allocator; + Value x(kObjectType); + x.AddMember("hello", "world", allocator) + .AddMember("t", Value(true).Move(), allocator) + .AddMember("f", Value(false).Move(), allocator) + .AddMember("n", Value(kNullType).Move(), allocator) + .AddMember("i", 123, allocator) + .AddMember("pi", 3.14, allocator) + .AddMember("a", Value(kArrayType).Move().PushBack(1, allocator).PushBack(2, allocator).PushBack(3, allocator), allocator); + + // Test templated operator==() and operator!=() + TestEqual(x["hello"], "world"); + const char* cc = "world"; + TestEqual(x["hello"], cc); + char* c = strdup("world"); + TestEqual(x["hello"], c); + free(c); + + TestEqual(x["t"], true); + TestEqual(x["f"], false); + TestEqual(x["i"], 123); + TestEqual(x["pi"], 3.14); + + // Test operator==() (including different allocators) + CrtAllocator crtAllocator; + GenericValue, CrtAllocator> y; + GenericDocument, CrtAllocator> z(&crtAllocator); + y.CopyFrom(x, crtAllocator); + z.CopyFrom(y, z.GetAllocator()); + TestEqual(x, y); + TestEqual(y, z); + TestEqual(z, x); + + // Swapping member order should be fine. + EXPECT_TRUE(y.RemoveMember("t")); + TestUnequal(x, y); + TestUnequal(z, y); + EXPECT_TRUE(z.RemoveMember("t")); + TestUnequal(x, z); + TestEqual(y, z); + y.AddMember("t", false, crtAllocator); + z.AddMember("t", false, z.GetAllocator()); + TestUnequal(x, y); + TestUnequal(z, x); + y["t"] = true; + z["t"] = true; + TestEqual(x, y); + TestEqual(y, z); + TestEqual(z, x); + + // Swapping element order is not OK + x["a"][0].Swap(x["a"][1]); + TestUnequal(x, y); + x["a"][0].Swap(x["a"][1]); + TestEqual(x, y); + + // Array of different size + x["a"].PushBack(4, allocator); + TestUnequal(x, y); + x["a"].PopBack(); + TestEqual(x, y); + + // Issue #129: compare Uint64 + x.SetUint64(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0xFFFFFFF0)); + y.SetUint64(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0xFFFFFFFF)); + TestUnequal(x, y); } template -void TestCopyFrom() { - typename Value::AllocatorType a; - Value v1(1234); - Value v2(v1, a); // deep copy constructor - EXPECT_TRUE(v1.GetType() == v2.GetType()); - EXPECT_EQ(v1.GetInt(), v2.GetInt()); - - v1.SetString("foo"); - v2.CopyFrom(v1, a); - EXPECT_TRUE(v1.GetType() == v2.GetType()); - EXPECT_STREQ(v1.GetString(), v2.GetString()); - EXPECT_EQ(v1.GetString(), v2.GetString()); // string NOT copied - - v1.SetString("bar", a); // copy string - v2.CopyFrom(v1, a); - EXPECT_TRUE(v1.GetType() == v2.GetType()); - EXPECT_STREQ(v1.GetString(), v2.GetString()); - EXPECT_NE(v1.GetString(), v2.GetString()); // string copied - - - v1.SetArray().PushBack(1234, a); - v2.CopyFrom(v1, a); - EXPECT_TRUE(v2.IsArray()); - EXPECT_EQ(v1.Size(), v2.Size()); - - v1.PushBack(Value().SetString("foo", a), a); // push string copy - EXPECT_TRUE(v1.Size() != v2.Size()); - v2.CopyFrom(v1, a); - EXPECT_TRUE(v1.Size() == v2.Size()); - EXPECT_STREQ(v1[1].GetString(), v2[1].GetString()); - EXPECT_NE(v1[1].GetString(), v2[1].GetString()); // string got copied +void TestCopyFrom() +{ typename Value::AllocatorType a; + Value v1(1234); + Value v2(v1, a); // deep copy constructor + EXPECT_TRUE(v1.GetType() == v2.GetType()); + EXPECT_EQ(v1.GetInt(), v2.GetInt()); + + v1.SetString("foo"); + v2.CopyFrom(v1, a); + EXPECT_TRUE(v1.GetType() == v2.GetType()); + EXPECT_STREQ(v1.GetString(), v2.GetString()); + EXPECT_EQ(v1.GetString(), v2.GetString()); // string NOT copied + + v1.SetString("bar", a); // copy string + v2.CopyFrom(v1, a); + EXPECT_TRUE(v1.GetType() == v2.GetType()); + EXPECT_STREQ(v1.GetString(), v2.GetString()); + EXPECT_NE(v1.GetString(), v2.GetString()); // string copied + + + v1.SetArray().PushBack(1234, a); + v2.CopyFrom(v1, a); + EXPECT_TRUE(v2.IsArray()); + EXPECT_EQ(v1.Size(), v2.Size()); + + v1.PushBack(Value().SetString("foo", a), a); // push string copy + EXPECT_TRUE(v1.Size() != v2.Size()); + v2.CopyFrom(v1, a); + EXPECT_TRUE(v1.Size() == v2.Size()); + EXPECT_STREQ(v1[1].GetString(), v2[1].GetString()); + EXPECT_NE(v1[1].GetString(), v2[1].GetString()); // string got copied } -TEST(Value, CopyFrom) { - TestCopyFrom(); - TestCopyFrom, CrtAllocator> >(); +TEST(Value, CopyFrom) +{ TestCopyFrom(); + TestCopyFrom, CrtAllocator> >(); } -TEST(Value, Swap) { - Value v1(1234); - Value v2(kObjectType); +TEST(Value, Swap) +{ Value v1(1234); + Value v2(kObjectType); - EXPECT_EQ(&v1, &v1.Swap(v2)); - EXPECT_TRUE(v1.IsObject()); - EXPECT_TRUE(v2.IsInt()); - EXPECT_EQ(1234, v2.GetInt()); + EXPECT_EQ(&v1, &v1.Swap(v2)); + EXPECT_TRUE(v1.IsObject()); + EXPECT_TRUE(v2.IsInt()); + EXPECT_EQ(1234, v2.GetInt()); - // testing std::swap compatibility - using std::swap; - swap(v1, v2); - EXPECT_TRUE(v1.IsInt()); - EXPECT_TRUE(v2.IsObject()); + // testing std::swap compatibility + using std::swap; + swap(v1, v2); + EXPECT_TRUE(v1.IsInt()); + EXPECT_TRUE(v2.IsObject()); } -TEST(Value, Null) { - // Default constructor - Value x; - EXPECT_EQ(kNullType, x.GetType()); - EXPECT_TRUE(x.IsNull()); - - EXPECT_FALSE(x.IsTrue()); - EXPECT_FALSE(x.IsFalse()); - EXPECT_FALSE(x.IsNumber()); - EXPECT_FALSE(x.IsString()); - EXPECT_FALSE(x.IsObject()); - EXPECT_FALSE(x.IsArray()); - - // Constructor with type - Value y(kNullType); - EXPECT_TRUE(y.IsNull()); - - // SetNull(); - Value z(true); - z.SetNull(); - EXPECT_TRUE(z.IsNull()); +TEST(Value, Null) +{ // Default constructor + Value x; + EXPECT_EQ(kNullType, x.GetType()); + EXPECT_TRUE(x.IsNull()); + + EXPECT_FALSE(x.IsTrue()); + EXPECT_FALSE(x.IsFalse()); + EXPECT_FALSE(x.IsNumber()); + EXPECT_FALSE(x.IsString()); + EXPECT_FALSE(x.IsObject()); + EXPECT_FALSE(x.IsArray()); + + // Constructor with type + Value y(kNullType); + EXPECT_TRUE(y.IsNull()); + + // SetNull(); + Value z(true); + z.SetNull(); + EXPECT_TRUE(z.IsNull()); } -TEST(Value, True) { - // Constructor with bool - Value x(true); - EXPECT_EQ(kTrueType, x.GetType()); - EXPECT_TRUE(x.GetBool()); - EXPECT_TRUE(x.IsBool()); - EXPECT_TRUE(x.IsTrue()); - - EXPECT_FALSE(x.IsNull()); - EXPECT_FALSE(x.IsFalse()); - EXPECT_FALSE(x.IsNumber()); - EXPECT_FALSE(x.IsString()); - EXPECT_FALSE(x.IsObject()); - EXPECT_FALSE(x.IsArray()); - - // Constructor with type - Value y(kTrueType); - EXPECT_TRUE(y.IsTrue()); - - // SetBool() - Value z; - z.SetBool(true); - EXPECT_TRUE(z.IsTrue()); - - // Templated functions - EXPECT_TRUE(z.Is()); - EXPECT_TRUE(z.Get()); - EXPECT_FALSE(z.Set(false).Get()); - EXPECT_TRUE(z.Set(true).Get()); +TEST(Value, True) +{ // Constructor with bool + Value x(true); + EXPECT_EQ(kTrueType, x.GetType()); + EXPECT_TRUE(x.GetBool()); + EXPECT_TRUE(x.IsBool()); + EXPECT_TRUE(x.IsTrue()); + + EXPECT_FALSE(x.IsNull()); + EXPECT_FALSE(x.IsFalse()); + EXPECT_FALSE(x.IsNumber()); + EXPECT_FALSE(x.IsString()); + EXPECT_FALSE(x.IsObject()); + EXPECT_FALSE(x.IsArray()); + + // Constructor with type + Value y(kTrueType); + EXPECT_TRUE(y.IsTrue()); + + // SetBool() + Value z; + z.SetBool(true); + EXPECT_TRUE(z.IsTrue()); + + // Templated functions + EXPECT_TRUE(z.Is()); + EXPECT_TRUE(z.Get()); + EXPECT_FALSE(z.Set(false).Get()); + EXPECT_TRUE(z.Set(true).Get()); } -TEST(Value, False) { - // Constructor with bool - Value x(false); - EXPECT_EQ(kFalseType, x.GetType()); - EXPECT_TRUE(x.IsBool()); - EXPECT_TRUE(x.IsFalse()); - - EXPECT_FALSE(x.IsNull()); - EXPECT_FALSE(x.IsTrue()); - EXPECT_FALSE(x.GetBool()); - //EXPECT_FALSE((bool)x); - EXPECT_FALSE(x.IsNumber()); - EXPECT_FALSE(x.IsString()); - EXPECT_FALSE(x.IsObject()); - EXPECT_FALSE(x.IsArray()); - - // Constructor with type - Value y(kFalseType); - EXPECT_TRUE(y.IsFalse()); - - // SetBool() - Value z; - z.SetBool(false); - EXPECT_TRUE(z.IsFalse()); +TEST(Value, False) +{ // Constructor with bool + Value x(false); + EXPECT_EQ(kFalseType, x.GetType()); + EXPECT_TRUE(x.IsBool()); + EXPECT_TRUE(x.IsFalse()); + + EXPECT_FALSE(x.IsNull()); + EXPECT_FALSE(x.IsTrue()); + EXPECT_FALSE(x.GetBool()); + //EXPECT_FALSE((bool)x); + EXPECT_FALSE(x.IsNumber()); + EXPECT_FALSE(x.IsString()); + EXPECT_FALSE(x.IsObject()); + EXPECT_FALSE(x.IsArray()); + + // Constructor with type + Value y(kFalseType); + EXPECT_TRUE(y.IsFalse()); + + // SetBool() + Value z; + z.SetBool(false); + EXPECT_TRUE(z.IsFalse()); } -TEST(Value, Int) { - // Constructor with int - Value x(1234); - EXPECT_EQ(kNumberType, x.GetType()); - EXPECT_EQ(1234, x.GetInt()); - EXPECT_EQ(1234u, x.GetUint()); - EXPECT_EQ(1234, x.GetInt64()); - EXPECT_EQ(1234u, x.GetUint64()); - EXPECT_NEAR(1234.0, x.GetDouble(), 0.0); - //EXPECT_EQ(1234, (int)x); - //EXPECT_EQ(1234, (unsigned)x); - //EXPECT_EQ(1234, (int64_t)x); - //EXPECT_EQ(1234, (uint64_t)x); - //EXPECT_EQ(1234, (double)x); - EXPECT_TRUE(x.IsNumber()); - EXPECT_TRUE(x.IsInt()); - EXPECT_TRUE(x.IsUint()); - EXPECT_TRUE(x.IsInt64()); - EXPECT_TRUE(x.IsUint64()); - - EXPECT_FALSE(x.IsDouble()); - EXPECT_FALSE(x.IsFloat()); - EXPECT_FALSE(x.IsNull()); - EXPECT_FALSE(x.IsBool()); - EXPECT_FALSE(x.IsFalse()); - EXPECT_FALSE(x.IsTrue()); - EXPECT_FALSE(x.IsString()); - EXPECT_FALSE(x.IsObject()); - EXPECT_FALSE(x.IsArray()); - - Value nx(-1234); - EXPECT_EQ(-1234, nx.GetInt()); - EXPECT_EQ(-1234, nx.GetInt64()); - EXPECT_TRUE(nx.IsInt()); - EXPECT_TRUE(nx.IsInt64()); - EXPECT_FALSE(nx.IsUint()); - EXPECT_FALSE(nx.IsUint64()); - - // Constructor with type - Value y(kNumberType); - EXPECT_TRUE(y.IsNumber()); - EXPECT_TRUE(y.IsInt()); - EXPECT_EQ(0, y.GetInt()); - - // SetInt() - Value z; - z.SetInt(1234); - EXPECT_EQ(1234, z.GetInt()); - - // operator=(int) - z = 5678; - EXPECT_EQ(5678, z.GetInt()); - - // Templated functions - EXPECT_TRUE(z.Is()); - EXPECT_EQ(5678, z.Get()); - EXPECT_EQ(5679, z.Set(5679).Get()); - EXPECT_EQ(5680, z.Set(5680).Get()); +TEST(Value, Int) +{ // Constructor with int + Value x(1234); + EXPECT_EQ(kNumberType, x.GetType()); + EXPECT_EQ(1234, x.GetInt()); + EXPECT_EQ(1234u, x.GetUint()); + EXPECT_EQ(1234, x.GetInt64()); + EXPECT_EQ(1234u, x.GetUint64()); + EXPECT_NEAR(1234.0, x.GetDouble(), 0.0); + //EXPECT_EQ(1234, (int)x); + //EXPECT_EQ(1234, (unsigned)x); + //EXPECT_EQ(1234, (int64_t)x); + //EXPECT_EQ(1234, (uint64_t)x); + //EXPECT_EQ(1234, (double)x); + EXPECT_TRUE(x.IsNumber()); + EXPECT_TRUE(x.IsInt()); + EXPECT_TRUE(x.IsUint()); + EXPECT_TRUE(x.IsInt64()); + EXPECT_TRUE(x.IsUint64()); + + EXPECT_FALSE(x.IsDouble()); + EXPECT_FALSE(x.IsFloat()); + EXPECT_FALSE(x.IsNull()); + EXPECT_FALSE(x.IsBool()); + EXPECT_FALSE(x.IsFalse()); + EXPECT_FALSE(x.IsTrue()); + EXPECT_FALSE(x.IsString()); + EXPECT_FALSE(x.IsObject()); + EXPECT_FALSE(x.IsArray()); + + Value nx(-1234); + EXPECT_EQ(-1234, nx.GetInt()); + EXPECT_EQ(-1234, nx.GetInt64()); + EXPECT_TRUE(nx.IsInt()); + EXPECT_TRUE(nx.IsInt64()); + EXPECT_FALSE(nx.IsUint()); + EXPECT_FALSE(nx.IsUint64()); + + // Constructor with type + Value y(kNumberType); + EXPECT_TRUE(y.IsNumber()); + EXPECT_TRUE(y.IsInt()); + EXPECT_EQ(0, y.GetInt()); + + // SetInt() + Value z; + z.SetInt(1234); + EXPECT_EQ(1234, z.GetInt()); + + // operator=(int) + z = 5678; + EXPECT_EQ(5678, z.GetInt()); + + // Templated functions + EXPECT_TRUE(z.Is()); + EXPECT_EQ(5678, z.Get()); + EXPECT_EQ(5679, z.Set(5679).Get()); + EXPECT_EQ(5680, z.Set(5680).Get()); } -TEST(Value, Uint) { - // Constructor with int - Value x(1234u); - EXPECT_EQ(kNumberType, x.GetType()); - EXPECT_EQ(1234, x.GetInt()); - EXPECT_EQ(1234u, x.GetUint()); - EXPECT_EQ(1234, x.GetInt64()); - EXPECT_EQ(1234u, x.GetUint64()); - EXPECT_TRUE(x.IsNumber()); - EXPECT_TRUE(x.IsInt()); - EXPECT_TRUE(x.IsUint()); - EXPECT_TRUE(x.IsInt64()); - EXPECT_TRUE(x.IsUint64()); - EXPECT_NEAR(1234.0, x.GetDouble(), 0.0); // Number can always be cast as double but !IsDouble(). - - EXPECT_FALSE(x.IsDouble()); - EXPECT_FALSE(x.IsFloat()); - EXPECT_FALSE(x.IsNull()); - EXPECT_FALSE(x.IsBool()); - EXPECT_FALSE(x.IsFalse()); - EXPECT_FALSE(x.IsTrue()); - EXPECT_FALSE(x.IsString()); - EXPECT_FALSE(x.IsObject()); - EXPECT_FALSE(x.IsArray()); - - // SetUint() - Value z; - z.SetUint(1234); - EXPECT_EQ(1234u, z.GetUint()); - - // operator=(unsigned) - z = 5678u; - EXPECT_EQ(5678u, z.GetUint()); - - z = 2147483648u; // 2^31, cannot cast as int - EXPECT_EQ(2147483648u, z.GetUint()); - EXPECT_FALSE(z.IsInt()); - EXPECT_TRUE(z.IsInt64()); // Issue 41: Incorrect parsing of unsigned int number types - - // Templated functions - EXPECT_TRUE(z.Is()); - EXPECT_EQ(2147483648u, z.Get()); - EXPECT_EQ(2147483649u, z.Set(2147483649u).Get()); - EXPECT_EQ(2147483650u, z.Set(2147483650u).Get()); +TEST(Value, Uint) +{ // Constructor with int + Value x(1234u); + EXPECT_EQ(kNumberType, x.GetType()); + EXPECT_EQ(1234, x.GetInt()); + EXPECT_EQ(1234u, x.GetUint()); + EXPECT_EQ(1234, x.GetInt64()); + EXPECT_EQ(1234u, x.GetUint64()); + EXPECT_TRUE(x.IsNumber()); + EXPECT_TRUE(x.IsInt()); + EXPECT_TRUE(x.IsUint()); + EXPECT_TRUE(x.IsInt64()); + EXPECT_TRUE(x.IsUint64()); + EXPECT_NEAR(1234.0, x.GetDouble(), 0.0); // Number can always be cast as double but !IsDouble(). + + EXPECT_FALSE(x.IsDouble()); + EXPECT_FALSE(x.IsFloat()); + EXPECT_FALSE(x.IsNull()); + EXPECT_FALSE(x.IsBool()); + EXPECT_FALSE(x.IsFalse()); + EXPECT_FALSE(x.IsTrue()); + EXPECT_FALSE(x.IsString()); + EXPECT_FALSE(x.IsObject()); + EXPECT_FALSE(x.IsArray()); + + // SetUint() + Value z; + z.SetUint(1234); + EXPECT_EQ(1234u, z.GetUint()); + + // operator=(unsigned) + z = 5678u; + EXPECT_EQ(5678u, z.GetUint()); + + z = 2147483648u; // 2^31, cannot cast as int + EXPECT_EQ(2147483648u, z.GetUint()); + EXPECT_FALSE(z.IsInt()); + EXPECT_TRUE(z.IsInt64()); // Issue 41: Incorrect parsing of unsigned int number types + + // Templated functions + EXPECT_TRUE(z.Is()); + EXPECT_EQ(2147483648u, z.Get()); + EXPECT_EQ(2147483649u, z.Set(2147483649u).Get()); + EXPECT_EQ(2147483650u, z.Set(2147483650u).Get()); } -TEST(Value, Int64) { - // Constructor with int - Value x(int64_t(1234)); - EXPECT_EQ(kNumberType, x.GetType()); - EXPECT_EQ(1234, x.GetInt()); - EXPECT_EQ(1234u, x.GetUint()); - EXPECT_EQ(1234, x.GetInt64()); - EXPECT_EQ(1234u, x.GetUint64()); - EXPECT_TRUE(x.IsNumber()); - EXPECT_TRUE(x.IsInt()); - EXPECT_TRUE(x.IsUint()); - EXPECT_TRUE(x.IsInt64()); - EXPECT_TRUE(x.IsUint64()); - - EXPECT_FALSE(x.IsDouble()); - EXPECT_FALSE(x.IsFloat()); - EXPECT_FALSE(x.IsNull()); - EXPECT_FALSE(x.IsBool()); - EXPECT_FALSE(x.IsFalse()); - EXPECT_FALSE(x.IsTrue()); - EXPECT_FALSE(x.IsString()); - EXPECT_FALSE(x.IsObject()); - EXPECT_FALSE(x.IsArray()); - - Value nx(int64_t(-1234)); - EXPECT_EQ(-1234, nx.GetInt()); - EXPECT_EQ(-1234, nx.GetInt64()); - EXPECT_TRUE(nx.IsInt()); - EXPECT_TRUE(nx.IsInt64()); - EXPECT_FALSE(nx.IsUint()); - EXPECT_FALSE(nx.IsUint64()); - - // SetInt64() - Value z; - z.SetInt64(1234); - EXPECT_EQ(1234, z.GetInt64()); - - z.SetInt64(2147483648u); // 2^31, cannot cast as int - EXPECT_FALSE(z.IsInt()); - EXPECT_TRUE(z.IsUint()); - EXPECT_NEAR(2147483648.0, z.GetDouble(), 0.0); - - z.SetInt64(int64_t(4294967295u) + 1); // 2^32, cannot cast as uint - EXPECT_FALSE(z.IsInt()); - EXPECT_FALSE(z.IsUint()); - EXPECT_NEAR(4294967296.0, z.GetDouble(), 0.0); - - z.SetInt64(-int64_t(2147483648u) - 1); // -2^31-1, cannot cast as int - EXPECT_FALSE(z.IsInt()); - EXPECT_NEAR(-2147483649.0, z.GetDouble(), 0.0); - - int64_t i = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 00000000)); - z.SetInt64(i); - EXPECT_DOUBLE_EQ(-9223372036854775808.0, z.GetDouble()); - - // Templated functions - EXPECT_TRUE(z.Is()); - EXPECT_EQ(i, z.Get()); +TEST(Value, Int64) +{ // Constructor with int + Value x(int64_t(1234)); + EXPECT_EQ(kNumberType, x.GetType()); + EXPECT_EQ(1234, x.GetInt()); + EXPECT_EQ(1234u, x.GetUint()); + EXPECT_EQ(1234, x.GetInt64()); + EXPECT_EQ(1234u, x.GetUint64()); + EXPECT_TRUE(x.IsNumber()); + EXPECT_TRUE(x.IsInt()); + EXPECT_TRUE(x.IsUint()); + EXPECT_TRUE(x.IsInt64()); + EXPECT_TRUE(x.IsUint64()); + + EXPECT_FALSE(x.IsDouble()); + EXPECT_FALSE(x.IsFloat()); + EXPECT_FALSE(x.IsNull()); + EXPECT_FALSE(x.IsBool()); + EXPECT_FALSE(x.IsFalse()); + EXPECT_FALSE(x.IsTrue()); + EXPECT_FALSE(x.IsString()); + EXPECT_FALSE(x.IsObject()); + EXPECT_FALSE(x.IsArray()); + + Value nx(int64_t(-1234)); + EXPECT_EQ(-1234, nx.GetInt()); + EXPECT_EQ(-1234, nx.GetInt64()); + EXPECT_TRUE(nx.IsInt()); + EXPECT_TRUE(nx.IsInt64()); + EXPECT_FALSE(nx.IsUint()); + EXPECT_FALSE(nx.IsUint64()); + + // SetInt64() + Value z; + z.SetInt64(1234); + EXPECT_EQ(1234, z.GetInt64()); + + z.SetInt64(2147483648u); // 2^31, cannot cast as int + EXPECT_FALSE(z.IsInt()); + EXPECT_TRUE(z.IsUint()); + EXPECT_NEAR(2147483648.0, z.GetDouble(), 0.0); + + z.SetInt64(int64_t(4294967295u) + 1); // 2^32, cannot cast as uint + EXPECT_FALSE(z.IsInt()); + EXPECT_FALSE(z.IsUint()); + EXPECT_NEAR(4294967296.0, z.GetDouble(), 0.0); + + z.SetInt64(-int64_t(2147483648u) - 1); // -2^31-1, cannot cast as int + EXPECT_FALSE(z.IsInt()); + EXPECT_NEAR(-2147483649.0, z.GetDouble(), 0.0); + + int64_t i = static_cast(RAPIDJSON_UINT64_C2(0x80000000, 00000000)); + z.SetInt64(i); + EXPECT_DOUBLE_EQ(-9223372036854775808.0, z.GetDouble()); + + // Templated functions + EXPECT_TRUE(z.Is()); + EXPECT_EQ(i, z.Get()); #if 0 // signed integer underflow is undefined behaviour - EXPECT_EQ(i - 1, z.Set(i - 1).Get()); - EXPECT_EQ(i - 2, z.Set(i - 2).Get()); + EXPECT_EQ(i - 1, z.Set(i - 1).Get()); + EXPECT_EQ(i - 2, z.Set(i - 2).Get()); #endif } -TEST(Value, Uint64) { - // Constructor with int - Value x(uint64_t(1234)); - EXPECT_EQ(kNumberType, x.GetType()); - EXPECT_EQ(1234, x.GetInt()); - EXPECT_EQ(1234u, x.GetUint()); - EXPECT_EQ(1234, x.GetInt64()); - EXPECT_EQ(1234u, x.GetUint64()); - EXPECT_TRUE(x.IsNumber()); - EXPECT_TRUE(x.IsInt()); - EXPECT_TRUE(x.IsUint()); - EXPECT_TRUE(x.IsInt64()); - EXPECT_TRUE(x.IsUint64()); - - EXPECT_FALSE(x.IsDouble()); - EXPECT_FALSE(x.IsFloat()); - EXPECT_FALSE(x.IsNull()); - EXPECT_FALSE(x.IsBool()); - EXPECT_FALSE(x.IsFalse()); - EXPECT_FALSE(x.IsTrue()); - EXPECT_FALSE(x.IsString()); - EXPECT_FALSE(x.IsObject()); - EXPECT_FALSE(x.IsArray()); - - // SetUint64() - Value z; - z.SetUint64(1234); - EXPECT_EQ(1234u, z.GetUint64()); - - z.SetUint64(uint64_t(2147483648u)); // 2^31, cannot cast as int - EXPECT_FALSE(z.IsInt()); - EXPECT_TRUE(z.IsUint()); - EXPECT_TRUE(z.IsInt64()); - - z.SetUint64(uint64_t(4294967295u) + 1); // 2^32, cannot cast as uint - EXPECT_FALSE(z.IsInt()); - EXPECT_FALSE(z.IsUint()); - EXPECT_TRUE(z.IsInt64()); - - uint64_t u = RAPIDJSON_UINT64_C2(0x80000000, 0x00000000); - z.SetUint64(u); // 2^63 cannot cast as int64 - EXPECT_FALSE(z.IsInt64()); - EXPECT_EQ(u, z.GetUint64()); // Issue 48 - EXPECT_DOUBLE_EQ(9223372036854775808.0, z.GetDouble()); - - // Templated functions - EXPECT_TRUE(z.Is()); - EXPECT_EQ(u, z.Get()); - EXPECT_EQ(u + 1, z.Set(u + 1).Get()); - EXPECT_EQ(u + 2, z.Set(u + 2).Get()); +TEST(Value, Uint64) +{ // Constructor with int + Value x(uint64_t(1234)); + EXPECT_EQ(kNumberType, x.GetType()); + EXPECT_EQ(1234, x.GetInt()); + EXPECT_EQ(1234u, x.GetUint()); + EXPECT_EQ(1234, x.GetInt64()); + EXPECT_EQ(1234u, x.GetUint64()); + EXPECT_TRUE(x.IsNumber()); + EXPECT_TRUE(x.IsInt()); + EXPECT_TRUE(x.IsUint()); + EXPECT_TRUE(x.IsInt64()); + EXPECT_TRUE(x.IsUint64()); + + EXPECT_FALSE(x.IsDouble()); + EXPECT_FALSE(x.IsFloat()); + EXPECT_FALSE(x.IsNull()); + EXPECT_FALSE(x.IsBool()); + EXPECT_FALSE(x.IsFalse()); + EXPECT_FALSE(x.IsTrue()); + EXPECT_FALSE(x.IsString()); + EXPECT_FALSE(x.IsObject()); + EXPECT_FALSE(x.IsArray()); + + // SetUint64() + Value z; + z.SetUint64(1234); + EXPECT_EQ(1234u, z.GetUint64()); + + z.SetUint64(uint64_t(2147483648u)); // 2^31, cannot cast as int + EXPECT_FALSE(z.IsInt()); + EXPECT_TRUE(z.IsUint()); + EXPECT_TRUE(z.IsInt64()); + + z.SetUint64(uint64_t(4294967295u) + 1); // 2^32, cannot cast as uint + EXPECT_FALSE(z.IsInt()); + EXPECT_FALSE(z.IsUint()); + EXPECT_TRUE(z.IsInt64()); + + uint64_t u = RAPIDJSON_UINT64_C2(0x80000000, 0x00000000); + z.SetUint64(u); // 2^63 cannot cast as int64 + EXPECT_FALSE(z.IsInt64()); + EXPECT_EQ(u, z.GetUint64()); // Issue 48 + EXPECT_DOUBLE_EQ(9223372036854775808.0, z.GetDouble()); + + // Templated functions + EXPECT_TRUE(z.Is()); + EXPECT_EQ(u, z.Get()); + EXPECT_EQ(u + 1, z.Set(u + 1).Get()); + EXPECT_EQ(u + 2, z.Set(u + 2).Get()); } -TEST(Value, Double) { - // Constructor with double - Value x(12.34); - EXPECT_EQ(kNumberType, x.GetType()); - EXPECT_NEAR(12.34, x.GetDouble(), 0.0); - EXPECT_TRUE(x.IsNumber()); - EXPECT_TRUE(x.IsDouble()); - - EXPECT_FALSE(x.IsInt()); - EXPECT_FALSE(x.IsNull()); - EXPECT_FALSE(x.IsBool()); - EXPECT_FALSE(x.IsFalse()); - EXPECT_FALSE(x.IsTrue()); - EXPECT_FALSE(x.IsString()); - EXPECT_FALSE(x.IsObject()); - EXPECT_FALSE(x.IsArray()); - - // SetDouble() - Value z; - z.SetDouble(12.34); - EXPECT_NEAR(12.34, z.GetDouble(), 0.0); - - z = 56.78; - EXPECT_NEAR(56.78, z.GetDouble(), 0.0); - - // Templated functions - EXPECT_TRUE(z.Is()); - EXPECT_EQ(56.78, z.Get()); - EXPECT_EQ(57.78, z.Set(57.78).Get()); - EXPECT_EQ(58.78, z.Set(58.78).Get()); +TEST(Value, Double) +{ // Constructor with double + Value x(12.34); + EXPECT_EQ(kNumberType, x.GetType()); + EXPECT_NEAR(12.34, x.GetDouble(), 0.0); + EXPECT_TRUE(x.IsNumber()); + EXPECT_TRUE(x.IsDouble()); + + EXPECT_FALSE(x.IsInt()); + EXPECT_FALSE(x.IsNull()); + EXPECT_FALSE(x.IsBool()); + EXPECT_FALSE(x.IsFalse()); + EXPECT_FALSE(x.IsTrue()); + EXPECT_FALSE(x.IsString()); + EXPECT_FALSE(x.IsObject()); + EXPECT_FALSE(x.IsArray()); + + // SetDouble() + Value z; + z.SetDouble(12.34); + EXPECT_NEAR(12.34, z.GetDouble(), 0.0); + + z = 56.78; + EXPECT_NEAR(56.78, z.GetDouble(), 0.0); + + // Templated functions + EXPECT_TRUE(z.Is()); + EXPECT_EQ(56.78, z.Get()); + EXPECT_EQ(57.78, z.Set(57.78).Get()); + EXPECT_EQ(58.78, z.Set(58.78).Get()); } -TEST(Value, Float) { - // Constructor with double - Value x(12.34f); - EXPECT_EQ(kNumberType, x.GetType()); - EXPECT_NEAR(12.34f, x.GetFloat(), 0.0); - EXPECT_TRUE(x.IsNumber()); - EXPECT_TRUE(x.IsDouble()); - EXPECT_TRUE(x.IsFloat()); - - EXPECT_FALSE(x.IsInt()); - EXPECT_FALSE(x.IsNull()); - EXPECT_FALSE(x.IsBool()); - EXPECT_FALSE(x.IsFalse()); - EXPECT_FALSE(x.IsTrue()); - EXPECT_FALSE(x.IsString()); - EXPECT_FALSE(x.IsObject()); - EXPECT_FALSE(x.IsArray()); - - // SetFloat() - Value z; - z.SetFloat(12.34f); - EXPECT_NEAR(12.34f, z.GetFloat(), 0.0f); - - // Issue 573 - z.SetInt(0); - EXPECT_EQ(0.0f, z.GetFloat()); - - z = 56.78f; - EXPECT_NEAR(56.78f, z.GetFloat(), 0.0f); - - // Templated functions - EXPECT_TRUE(z.Is()); - EXPECT_EQ(56.78f, z.Get()); - EXPECT_EQ(57.78f, z.Set(57.78f).Get()); - EXPECT_EQ(58.78f, z.Set(58.78f).Get()); +TEST(Value, Float) +{ // Constructor with double + Value x(12.34f); + EXPECT_EQ(kNumberType, x.GetType()); + EXPECT_NEAR(12.34f, x.GetFloat(), 0.0); + EXPECT_TRUE(x.IsNumber()); + EXPECT_TRUE(x.IsDouble()); + EXPECT_TRUE(x.IsFloat()); + + EXPECT_FALSE(x.IsInt()); + EXPECT_FALSE(x.IsNull()); + EXPECT_FALSE(x.IsBool()); + EXPECT_FALSE(x.IsFalse()); + EXPECT_FALSE(x.IsTrue()); + EXPECT_FALSE(x.IsString()); + EXPECT_FALSE(x.IsObject()); + EXPECT_FALSE(x.IsArray()); + + // SetFloat() + Value z; + z.SetFloat(12.34f); + EXPECT_NEAR(12.34f, z.GetFloat(), 0.0f); + + // Issue 573 + z.SetInt(0); + EXPECT_EQ(0.0f, z.GetFloat()); + + z = 56.78f; + EXPECT_NEAR(56.78f, z.GetFloat(), 0.0f); + + // Templated functions + EXPECT_TRUE(z.Is()); + EXPECT_EQ(56.78f, z.Get()); + EXPECT_EQ(57.78f, z.Set(57.78f).Get()); + EXPECT_EQ(58.78f, z.Set(58.78f).Get()); } -TEST(Value, IsLosslessDouble) { - EXPECT_TRUE(Value(0.0).IsLosslessDouble()); - EXPECT_TRUE(Value(12.34).IsLosslessDouble()); - EXPECT_TRUE(Value(-123).IsLosslessDouble()); - EXPECT_TRUE(Value(2147483648u).IsLosslessDouble()); - EXPECT_TRUE(Value(-static_cast(RAPIDJSON_UINT64_C2(0x40000000, 0x00000000))).IsLosslessDouble()); +TEST(Value, IsLosslessDouble) +{ EXPECT_TRUE(Value(0.0).IsLosslessDouble()); + EXPECT_TRUE(Value(12.34).IsLosslessDouble()); + EXPECT_TRUE(Value(-123).IsLosslessDouble()); + EXPECT_TRUE(Value(2147483648u).IsLosslessDouble()); + EXPECT_TRUE(Value(-static_cast(RAPIDJSON_UINT64_C2(0x40000000, 0x00000000))).IsLosslessDouble()); #if !(defined(_MSC_VER) && _MSC_VER < 1800) // VC2010 has problem - EXPECT_TRUE(Value(RAPIDJSON_UINT64_C2(0xA0000000, 0x00000000)).IsLosslessDouble()); + EXPECT_TRUE(Value(RAPIDJSON_UINT64_C2(0xA0000000, 0x00000000)).IsLosslessDouble()); #endif - EXPECT_FALSE(Value(static_cast(RAPIDJSON_UINT64_C2(0x7FFFFFFF, 0xFFFFFFFF))).IsLosslessDouble()); // INT64_MAX - EXPECT_FALSE(Value(-static_cast(RAPIDJSON_UINT64_C2(0x7FFFFFFF, 0xFFFFFFFF))).IsLosslessDouble()); // -INT64_MAX - EXPECT_TRUE(Value(-static_cast(RAPIDJSON_UINT64_C2(0x7FFFFFFF, 0xFFFFFFFF)) - 1).IsLosslessDouble()); // INT64_MIN - EXPECT_FALSE(Value(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0xFFFFFFFF)).IsLosslessDouble()); // UINT64_MAX - - EXPECT_TRUE(Value(3.4028234e38f).IsLosslessDouble()); // FLT_MAX - EXPECT_TRUE(Value(-3.4028234e38f).IsLosslessDouble()); // -FLT_MAX - EXPECT_TRUE(Value(1.17549435e-38f).IsLosslessDouble()); // FLT_MIN - EXPECT_TRUE(Value(-1.17549435e-38f).IsLosslessDouble()); // -FLT_MIN - EXPECT_TRUE(Value(1.7976931348623157e+308).IsLosslessDouble()); // DBL_MAX - EXPECT_TRUE(Value(-1.7976931348623157e+308).IsLosslessDouble()); // -DBL_MAX - EXPECT_TRUE(Value(2.2250738585072014e-308).IsLosslessDouble()); // DBL_MIN - EXPECT_TRUE(Value(-2.2250738585072014e-308).IsLosslessDouble()); // -DBL_MIN + EXPECT_FALSE(Value(static_cast(RAPIDJSON_UINT64_C2(0x7FFFFFFF, 0xFFFFFFFF))).IsLosslessDouble()); // INT64_MAX + EXPECT_FALSE(Value(-static_cast(RAPIDJSON_UINT64_C2(0x7FFFFFFF, 0xFFFFFFFF))).IsLosslessDouble()); // -INT64_MAX + EXPECT_TRUE(Value(-static_cast(RAPIDJSON_UINT64_C2(0x7FFFFFFF, 0xFFFFFFFF)) - 1).IsLosslessDouble()); // INT64_MIN + EXPECT_FALSE(Value(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0xFFFFFFFF)).IsLosslessDouble()); // UINT64_MAX + + EXPECT_TRUE(Value(3.4028234e38f).IsLosslessDouble()); // FLT_MAX + EXPECT_TRUE(Value(-3.4028234e38f).IsLosslessDouble()); // -FLT_MAX + EXPECT_TRUE(Value(1.17549435e-38f).IsLosslessDouble()); // FLT_MIN + EXPECT_TRUE(Value(-1.17549435e-38f).IsLosslessDouble()); // -FLT_MIN + EXPECT_TRUE(Value(1.7976931348623157e+308).IsLosslessDouble()); // DBL_MAX + EXPECT_TRUE(Value(-1.7976931348623157e+308).IsLosslessDouble()); // -DBL_MAX + EXPECT_TRUE(Value(2.2250738585072014e-308).IsLosslessDouble()); // DBL_MIN + EXPECT_TRUE(Value(-2.2250738585072014e-308).IsLosslessDouble()); // -DBL_MIN } -TEST(Value, IsLosslessFloat) { - EXPECT_TRUE(Value(12.25).IsLosslessFloat()); - EXPECT_TRUE(Value(-123).IsLosslessFloat()); - EXPECT_TRUE(Value(2147483648u).IsLosslessFloat()); - EXPECT_TRUE(Value(3.4028234e38f).IsLosslessFloat()); - EXPECT_TRUE(Value(-3.4028234e38f).IsLosslessFloat()); - EXPECT_FALSE(Value(3.4028235e38).IsLosslessFloat()); - EXPECT_FALSE(Value(0.3).IsLosslessFloat()); +TEST(Value, IsLosslessFloat) +{ EXPECT_TRUE(Value(12.25).IsLosslessFloat()); + EXPECT_TRUE(Value(-123).IsLosslessFloat()); + EXPECT_TRUE(Value(2147483648u).IsLosslessFloat()); + EXPECT_TRUE(Value(3.4028234e38f).IsLosslessFloat()); + EXPECT_TRUE(Value(-3.4028234e38f).IsLosslessFloat()); + EXPECT_FALSE(Value(3.4028235e38).IsLosslessFloat()); + EXPECT_FALSE(Value(0.3).IsLosslessFloat()); } -TEST(Value, String) { - // Construction with const string - Value x("Hello", 5); // literal - EXPECT_EQ(kStringType, x.GetType()); - EXPECT_TRUE(x.IsString()); - EXPECT_STREQ("Hello", x.GetString()); - EXPECT_EQ(5u, x.GetStringLength()); - - EXPECT_FALSE(x.IsNumber()); - EXPECT_FALSE(x.IsNull()); - EXPECT_FALSE(x.IsBool()); - EXPECT_FALSE(x.IsFalse()); - EXPECT_FALSE(x.IsTrue()); - EXPECT_FALSE(x.IsObject()); - EXPECT_FALSE(x.IsArray()); - - static const char cstr[] = "World"; // const array - Value(cstr).Swap(x); - EXPECT_TRUE(x.IsString()); - EXPECT_EQ(x.GetString(), cstr); - EXPECT_EQ(x.GetStringLength(), sizeof(cstr)-1); - - static char mstr[] = "Howdy"; // non-const array - // Value(mstr).Swap(x); // should not compile - Value(StringRef(mstr)).Swap(x); - EXPECT_TRUE(x.IsString()); - EXPECT_EQ(x.GetString(), mstr); - EXPECT_EQ(x.GetStringLength(), sizeof(mstr)-1); - strncpy(mstr,"Hello", sizeof(mstr)); - EXPECT_STREQ(x.GetString(), "Hello"); - - const char* pstr = cstr; - //Value(pstr).Swap(x); // should not compile - Value(StringRef(pstr)).Swap(x); - EXPECT_TRUE(x.IsString()); - EXPECT_EQ(x.GetString(), cstr); - EXPECT_EQ(x.GetStringLength(), sizeof(cstr)-1); - - char* mpstr = mstr; - Value(StringRef(mpstr,sizeof(mstr)-1)).Swap(x); - EXPECT_TRUE(x.IsString()); - EXPECT_EQ(x.GetString(), mstr); - EXPECT_EQ(x.GetStringLength(), 5u); - EXPECT_STREQ(x.GetString(), "Hello"); - - // Constructor with copy string - MemoryPoolAllocator<> allocator; - Value c(x.GetString(), x.GetStringLength(), allocator); - EXPECT_NE(x.GetString(), c.GetString()); - EXPECT_EQ(x.GetStringLength(), c.GetStringLength()); - EXPECT_STREQ(x.GetString(), c.GetString()); - //x.SetString("World"); - x.SetString("World", 5); - EXPECT_STREQ("Hello", c.GetString()); - EXPECT_EQ(5u, c.GetStringLength()); - - // Constructor with type - Value y(kStringType); - EXPECT_TRUE(y.IsString()); - EXPECT_STREQ("", y.GetString()); // Empty string should be "" instead of 0 (issue 226) - EXPECT_EQ(0u, y.GetStringLength()); - - // SetConsttring() - Value z; - z.SetString("Hello"); - EXPECT_TRUE(x.IsString()); - z.SetString("Hello", 5); - EXPECT_STREQ("Hello", z.GetString()); - EXPECT_STREQ("Hello", z.GetString()); - EXPECT_EQ(5u, z.GetStringLength()); - - z.SetString("Hello"); - EXPECT_TRUE(z.IsString()); - EXPECT_STREQ("Hello", z.GetString()); - - //z.SetString(mstr); // should not compile - //z.SetString(pstr); // should not compile - z.SetString(StringRef(mstr)); - EXPECT_TRUE(z.IsString()); - EXPECT_STREQ(z.GetString(), mstr); - - z.SetString(cstr); - EXPECT_TRUE(z.IsString()); - EXPECT_EQ(cstr, z.GetString()); - - z = cstr; - EXPECT_TRUE(z.IsString()); - EXPECT_EQ(cstr, z.GetString()); - - // SetString() - char s[] = "World"; - Value w; - w.SetString(s, static_cast(strlen(s)), allocator); - s[0] = '\0'; - EXPECT_STREQ("World", w.GetString()); - EXPECT_EQ(5u, w.GetStringLength()); - - // templated functions - EXPECT_TRUE(z.Is()); - EXPECT_STREQ(cstr, z.Get()); - EXPECT_STREQ("Apple", z.Set("Apple").Get()); +TEST(Value, String) +{ // Construction with const string + Value x("Hello", 5); // literal + EXPECT_EQ(kStringType, x.GetType()); + EXPECT_TRUE(x.IsString()); + EXPECT_STREQ("Hello", x.GetString()); + EXPECT_EQ(5u, x.GetStringLength()); + + EXPECT_FALSE(x.IsNumber()); + EXPECT_FALSE(x.IsNull()); + EXPECT_FALSE(x.IsBool()); + EXPECT_FALSE(x.IsFalse()); + EXPECT_FALSE(x.IsTrue()); + EXPECT_FALSE(x.IsObject()); + EXPECT_FALSE(x.IsArray()); + + static const char cstr[] = "World"; // const array + Value(cstr).Swap(x); + EXPECT_TRUE(x.IsString()); + EXPECT_EQ(x.GetString(), cstr); + EXPECT_EQ(x.GetStringLength(), sizeof(cstr)-1); + + static char mstr[] = "Howdy"; // non-const array + // Value(mstr).Swap(x); // should not compile + Value(StringRef(mstr)).Swap(x); + EXPECT_TRUE(x.IsString()); + EXPECT_EQ(x.GetString(), mstr); + EXPECT_EQ(x.GetStringLength(), sizeof(mstr)-1); + strncpy(mstr,"Hello", sizeof(mstr)); + EXPECT_STREQ(x.GetString(), "Hello"); + + const char* pstr = cstr; + //Value(pstr).Swap(x); // should not compile + Value(StringRef(pstr)).Swap(x); + EXPECT_TRUE(x.IsString()); + EXPECT_EQ(x.GetString(), cstr); + EXPECT_EQ(x.GetStringLength(), sizeof(cstr)-1); + + char* mpstr = mstr; + Value(StringRef(mpstr,sizeof(mstr)-1)).Swap(x); + EXPECT_TRUE(x.IsString()); + EXPECT_EQ(x.GetString(), mstr); + EXPECT_EQ(x.GetStringLength(), 5u); + EXPECT_STREQ(x.GetString(), "Hello"); + + // Constructor with copy string + MemoryPoolAllocator<> allocator; + Value c(x.GetString(), x.GetStringLength(), allocator); + EXPECT_NE(x.GetString(), c.GetString()); + EXPECT_EQ(x.GetStringLength(), c.GetStringLength()); + EXPECT_STREQ(x.GetString(), c.GetString()); + //x.SetString("World"); + x.SetString("World", 5); + EXPECT_STREQ("Hello", c.GetString()); + EXPECT_EQ(5u, c.GetStringLength()); + + // Constructor with type + Value y(kStringType); + EXPECT_TRUE(y.IsString()); + EXPECT_STREQ("", y.GetString()); // Empty string should be "" instead of 0 (issue 226) + EXPECT_EQ(0u, y.GetStringLength()); + + // SetConsttring() + Value z; + z.SetString("Hello"); + EXPECT_TRUE(x.IsString()); + z.SetString("Hello", 5); + EXPECT_STREQ("Hello", z.GetString()); + EXPECT_STREQ("Hello", z.GetString()); + EXPECT_EQ(5u, z.GetStringLength()); + + z.SetString("Hello"); + EXPECT_TRUE(z.IsString()); + EXPECT_STREQ("Hello", z.GetString()); + + //z.SetString(mstr); // should not compile + //z.SetString(pstr); // should not compile + z.SetString(StringRef(mstr)); + EXPECT_TRUE(z.IsString()); + EXPECT_STREQ(z.GetString(), mstr); + + z.SetString(cstr); + EXPECT_TRUE(z.IsString()); + EXPECT_EQ(cstr, z.GetString()); + + z = cstr; + EXPECT_TRUE(z.IsString()); + EXPECT_EQ(cstr, z.GetString()); + + // SetString() + char s[] = "World"; + Value w; + w.SetString(s, static_cast(strlen(s)), allocator); + s[0] = '\0'; + EXPECT_STREQ("World", w.GetString()); + EXPECT_EQ(5u, w.GetStringLength()); + + // templated functions + EXPECT_TRUE(z.Is()); + EXPECT_STREQ(cstr, z.Get()); + EXPECT_STREQ("Apple", z.Set("Apple").Get()); #if RAPIDJSON_HAS_STDSTRING - { - std::string str = "Hello World"; - str[5] = '\0'; - EXPECT_STREQ(str.data(),"Hello"); // embedded '\0' - EXPECT_EQ(str.size(), 11u); - - // no copy - Value vs0(StringRef(str)); - EXPECT_TRUE(vs0.IsString()); - EXPECT_EQ(vs0.GetString(), str.data()); - EXPECT_EQ(vs0.GetStringLength(), str.size()); - TestEqual(vs0, str); - - // do copy - Value vs1(str, allocator); - EXPECT_TRUE(vs1.IsString()); - EXPECT_NE(vs1.GetString(), str.data()); - EXPECT_NE(vs1.GetString(), str); // not equal due to embedded '\0' - EXPECT_EQ(vs1.GetStringLength(), str.size()); - TestEqual(vs1, str); - - // SetString - str = "World"; - vs0.SetNull().SetString(str, allocator); - EXPECT_TRUE(vs0.IsString()); - EXPECT_STREQ(vs0.GetString(), str.c_str()); - EXPECT_EQ(vs0.GetStringLength(), str.size()); - TestEqual(str, vs0); - TestUnequal(str, vs1); - - // vs1 = str; // should not compile - vs1 = StringRef(str); - TestEqual(str, vs1); - TestEqual(vs0, vs1); - - // Templated function. - EXPECT_TRUE(vs0.Is()); - EXPECT_EQ(str, vs0.Get()); - vs0.Set(std::string("Apple"), allocator); - EXPECT_EQ(std::string("Apple"), vs0.Get()); - vs0.Set(std::string("Orange"), allocator); - EXPECT_EQ(std::string("Orange"), vs0.Get()); - } + { std::string str = "Hello World"; + str[5] = '\0'; + EXPECT_STREQ(str.data(),"Hello"); // embedded '\0' + EXPECT_EQ(str.size(), 11u); + + // no copy + Value vs0(StringRef(str)); + EXPECT_TRUE(vs0.IsString()); + EXPECT_EQ(vs0.GetString(), str.data()); + EXPECT_EQ(vs0.GetStringLength(), str.size()); + TestEqual(vs0, str); + + // do copy + Value vs1(str, allocator); + EXPECT_TRUE(vs1.IsString()); + EXPECT_NE(vs1.GetString(), str.data()); + EXPECT_NE(vs1.GetString(), str); // not equal due to embedded '\0' + EXPECT_EQ(vs1.GetStringLength(), str.size()); + TestEqual(vs1, str); + + // SetString + str = "World"; + vs0.SetNull().SetString(str, allocator); + EXPECT_TRUE(vs0.IsString()); + EXPECT_STREQ(vs0.GetString(), str.c_str()); + EXPECT_EQ(vs0.GetStringLength(), str.size()); + TestEqual(str, vs0); + TestUnequal(str, vs1); + + // vs1 = str; // should not compile + vs1 = StringRef(str); + TestEqual(str, vs1); + TestEqual(vs0, vs1); + + // Templated function. + EXPECT_TRUE(vs0.Is()); + EXPECT_EQ(str, vs0.Get()); + vs0.Set(std::string("Apple"), allocator); + EXPECT_EQ(std::string("Apple"), vs0.Get()); + vs0.Set(std::string("Orange"), allocator); + EXPECT_EQ(std::string("Orange"), vs0.Get()); + } #endif // RAPIDJSON_HAS_STDSTRING } // Issue 226: Value of string type should not point to NULL -TEST(Value, SetStringNullException) { - Value v; - EXPECT_THROW(v.SetString(0, 0), AssertException); +TEST(Value, SetStringNullException) +{ Value v; + EXPECT_THROW(v.SetString(0, 0), AssertException); } template -static void TestArray(T& x, Allocator& allocator) { - const T& y = x; - - // PushBack() - Value v; - x.PushBack(v, allocator); - v.SetBool(true); - x.PushBack(v, allocator); - v.SetBool(false); - x.PushBack(v, allocator); - v.SetInt(123); - x.PushBack(v, allocator); - //x.PushBack((const char*)"foo", allocator); // should not compile - x.PushBack("foo", allocator); - - EXPECT_FALSE(x.Empty()); - EXPECT_EQ(5u, x.Size()); - EXPECT_FALSE(y.Empty()); - EXPECT_EQ(5u, y.Size()); - EXPECT_TRUE(x[SizeType(0)].IsNull()); - EXPECT_TRUE(x[1].IsTrue()); - EXPECT_TRUE(x[2].IsFalse()); - EXPECT_TRUE(x[3].IsInt()); - EXPECT_EQ(123, x[3].GetInt()); - EXPECT_TRUE(y[SizeType(0)].IsNull()); - EXPECT_TRUE(y[1].IsTrue()); - EXPECT_TRUE(y[2].IsFalse()); - EXPECT_TRUE(y[3].IsInt()); - EXPECT_EQ(123, y[3].GetInt()); - EXPECT_TRUE(y[4].IsString()); - EXPECT_STREQ("foo", y[4].GetString()); +static void TestArray(T& x, Allocator& allocator) +{ const T& y = x; + + // PushBack() + Value v; + x.PushBack(v, allocator); + v.SetBool(true); + x.PushBack(v, allocator); + v.SetBool(false); + x.PushBack(v, allocator); + v.SetInt(123); + x.PushBack(v, allocator); + //x.PushBack((const char*)"foo", allocator); // should not compile + x.PushBack("foo", allocator); + + EXPECT_FALSE(x.Empty()); + EXPECT_EQ(5u, x.Size()); + EXPECT_FALSE(y.Empty()); + EXPECT_EQ(5u, y.Size()); + EXPECT_TRUE(x[SizeType(0)].IsNull()); + EXPECT_TRUE(x[1].IsTrue()); + EXPECT_TRUE(x[2].IsFalse()); + EXPECT_TRUE(x[3].IsInt()); + EXPECT_EQ(123, x[3].GetInt()); + EXPECT_TRUE(y[SizeType(0)].IsNull()); + EXPECT_TRUE(y[1].IsTrue()); + EXPECT_TRUE(y[2].IsFalse()); + EXPECT_TRUE(y[3].IsInt()); + EXPECT_EQ(123, y[3].GetInt()); + EXPECT_TRUE(y[4].IsString()); + EXPECT_STREQ("foo", y[4].GetString()); #if RAPIDJSON_HAS_CXX11_RVALUE_REFS - // PushBack(GenericValue&&, Allocator&); - { - Value y2(kArrayType); - y2.PushBack(Value(true), allocator); - y2.PushBack(std::move(Value(kArrayType).PushBack(Value(1), allocator).PushBack("foo", allocator)), allocator); - EXPECT_EQ(2u, y2.Size()); - EXPECT_TRUE(y2[0].IsTrue()); - EXPECT_TRUE(y2[1].IsArray()); - EXPECT_EQ(2u, y2[1].Size()); - EXPECT_TRUE(y2[1][0].IsInt()); - EXPECT_TRUE(y2[1][1].IsString()); - } + // PushBack(GenericValue&&, Allocator&); + { Value y2(kArrayType); + y2.PushBack(Value(true), allocator); + y2.PushBack(std::move(Value(kArrayType).PushBack(Value(1), allocator).PushBack("foo", allocator)), allocator); + EXPECT_EQ(2u, y2.Size()); + EXPECT_TRUE(y2[0].IsTrue()); + EXPECT_TRUE(y2[1].IsArray()); + EXPECT_EQ(2u, y2[1].Size()); + EXPECT_TRUE(y2[1][0].IsInt()); + EXPECT_TRUE(y2[1][1].IsString()); + } #endif - // iterator - typename T::ValueIterator itr = x.Begin(); - EXPECT_TRUE(itr != x.End()); - EXPECT_TRUE(itr->IsNull()); - ++itr; - EXPECT_TRUE(itr != x.End()); - EXPECT_TRUE(itr->IsTrue()); - ++itr; - EXPECT_TRUE(itr != x.End()); - EXPECT_TRUE(itr->IsFalse()); - ++itr; - EXPECT_TRUE(itr != x.End()); - EXPECT_TRUE(itr->IsInt()); - EXPECT_EQ(123, itr->GetInt()); - ++itr; - EXPECT_TRUE(itr != x.End()); - EXPECT_TRUE(itr->IsString()); - EXPECT_STREQ("foo", itr->GetString()); - - // const iterator - typename T::ConstValueIterator citr = y.Begin(); - EXPECT_TRUE(citr != y.End()); - EXPECT_TRUE(citr->IsNull()); - ++citr; - EXPECT_TRUE(citr != y.End()); - EXPECT_TRUE(citr->IsTrue()); - ++citr; - EXPECT_TRUE(citr != y.End()); - EXPECT_TRUE(citr->IsFalse()); - ++citr; - EXPECT_TRUE(citr != y.End()); - EXPECT_TRUE(citr->IsInt()); - EXPECT_EQ(123, citr->GetInt()); - ++citr; - EXPECT_TRUE(citr != y.End()); - EXPECT_TRUE(citr->IsString()); - EXPECT_STREQ("foo", citr->GetString()); - - // PopBack() - x.PopBack(); - EXPECT_EQ(4u, x.Size()); - EXPECT_TRUE(y[SizeType(0)].IsNull()); - EXPECT_TRUE(y[1].IsTrue()); - EXPECT_TRUE(y[2].IsFalse()); - EXPECT_TRUE(y[3].IsInt()); - - // Clear() - x.Clear(); - EXPECT_TRUE(x.Empty()); - EXPECT_EQ(0u, x.Size()); - EXPECT_TRUE(y.Empty()); - EXPECT_EQ(0u, y.Size()); - - // Erase(ValueIterator) - - // Use array of array to ensure removed elements' destructor is called. - // [[0],[1],[2],...] - for (int i = 0; i < 10; i++) + // iterator + typename T::ValueIterator itr = x.Begin(); + EXPECT_TRUE(itr != x.End()); + EXPECT_TRUE(itr->IsNull()); + ++itr; + EXPECT_TRUE(itr != x.End()); + EXPECT_TRUE(itr->IsTrue()); + ++itr; + EXPECT_TRUE(itr != x.End()); + EXPECT_TRUE(itr->IsFalse()); + ++itr; + EXPECT_TRUE(itr != x.End()); + EXPECT_TRUE(itr->IsInt()); + EXPECT_EQ(123, itr->GetInt()); + ++itr; + EXPECT_TRUE(itr != x.End()); + EXPECT_TRUE(itr->IsString()); + EXPECT_STREQ("foo", itr->GetString()); + + // const iterator + typename T::ConstValueIterator citr = y.Begin(); + EXPECT_TRUE(citr != y.End()); + EXPECT_TRUE(citr->IsNull()); + ++citr; + EXPECT_TRUE(citr != y.End()); + EXPECT_TRUE(citr->IsTrue()); + ++citr; + EXPECT_TRUE(citr != y.End()); + EXPECT_TRUE(citr->IsFalse()); + ++citr; + EXPECT_TRUE(citr != y.End()); + EXPECT_TRUE(citr->IsInt()); + EXPECT_EQ(123, citr->GetInt()); + ++citr; + EXPECT_TRUE(citr != y.End()); + EXPECT_TRUE(citr->IsString()); + EXPECT_STREQ("foo", citr->GetString()); + + // PopBack() + x.PopBack(); + EXPECT_EQ(4u, x.Size()); + EXPECT_TRUE(y[SizeType(0)].IsNull()); + EXPECT_TRUE(y[1].IsTrue()); + EXPECT_TRUE(y[2].IsFalse()); + EXPECT_TRUE(y[3].IsInt()); + + // Clear() + x.Clear(); + EXPECT_TRUE(x.Empty()); + EXPECT_EQ(0u, x.Size()); + EXPECT_TRUE(y.Empty()); + EXPECT_EQ(0u, y.Size()); + + // Erase(ValueIterator) + + // Use array of array to ensure removed elements' destructor is called. + // [[0],[1],[2],...] + for (int i = 0; i < 10; i++) + x.PushBack(Value(kArrayType).PushBack(i, allocator).Move(), allocator); + + // Erase the first + itr = x.Erase(x.Begin()); + EXPECT_EQ(x.Begin(), itr); + EXPECT_EQ(9u, x.Size()); + for (int i = 0; i < 9; i++) + EXPECT_EQ(i + 1, x[static_cast(i)][0].GetInt()); + + // Ease the last + itr = x.Erase(x.End() - 1); + EXPECT_EQ(x.End(), itr); + EXPECT_EQ(8u, x.Size()); + for (int i = 0; i < 8; i++) + EXPECT_EQ(i + 1, x[static_cast(i)][0].GetInt()); + + // Erase the middle + itr = x.Erase(x.Begin() + 4); + EXPECT_EQ(x.Begin() + 4, itr); + EXPECT_EQ(7u, x.Size()); + for (int i = 0; i < 4; i++) + EXPECT_EQ(i + 1, x[static_cast(i)][0].GetInt()); + for (int i = 4; i < 7; i++) + EXPECT_EQ(i + 2, x[static_cast(i)][0].GetInt()); + + // Erase(ValueIterator, ValueIterator) + // Exhaustive test with all 0 <= first < n, first <= last <= n cases + const unsigned n = 10; + for (unsigned first = 0; first < n; first++) + { for (unsigned last = first; last <= n; last++) + { x.Clear(); + for (unsigned i = 0; i < n; i++) x.PushBack(Value(kArrayType).PushBack(i, allocator).Move(), allocator); - // Erase the first - itr = x.Erase(x.Begin()); - EXPECT_EQ(x.Begin(), itr); - EXPECT_EQ(9u, x.Size()); - for (int i = 0; i < 9; i++) - EXPECT_EQ(i + 1, x[static_cast(i)][0].GetInt()); - - // Ease the last - itr = x.Erase(x.End() - 1); - EXPECT_EQ(x.End(), itr); - EXPECT_EQ(8u, x.Size()); - for (int i = 0; i < 8; i++) - EXPECT_EQ(i + 1, x[static_cast(i)][0].GetInt()); - - // Erase the middle - itr = x.Erase(x.Begin() + 4); - EXPECT_EQ(x.Begin() + 4, itr); - EXPECT_EQ(7u, x.Size()); - for (int i = 0; i < 4; i++) - EXPECT_EQ(i + 1, x[static_cast(i)][0].GetInt()); - for (int i = 4; i < 7; i++) - EXPECT_EQ(i + 2, x[static_cast(i)][0].GetInt()); - - // Erase(ValueIterator, ValueIterator) - // Exhaustive test with all 0 <= first < n, first <= last <= n cases - const unsigned n = 10; - for (unsigned first = 0; first < n; first++) { - for (unsigned last = first; last <= n; last++) { - x.Clear(); - for (unsigned i = 0; i < n; i++) - x.PushBack(Value(kArrayType).PushBack(i, allocator).Move(), allocator); - - itr = x.Erase(x.Begin() + first, x.Begin() + last); - if (last == n) - EXPECT_EQ(x.End(), itr); - else - EXPECT_EQ(x.Begin() + first, itr); - - size_t removeCount = last - first; - EXPECT_EQ(n - removeCount, x.Size()); - for (unsigned i = 0; i < first; i++) - EXPECT_EQ(i, x[i][0].GetUint()); - for (unsigned i = first; i < n - removeCount; i++) - EXPECT_EQ(i + removeCount, x[static_cast(i)][0].GetUint()); - } + itr = x.Erase(x.Begin() + first, x.Begin() + last); + if (last == n) + EXPECT_EQ(x.End(), itr); + else + EXPECT_EQ(x.Begin() + first, itr); + + size_t removeCount = last - first; + EXPECT_EQ(n - removeCount, x.Size()); + for (unsigned i = 0; i < first; i++) + EXPECT_EQ(i, x[i][0].GetUint()); + for (unsigned i = first; i < n - removeCount; i++) + EXPECT_EQ(i + removeCount, x[static_cast(i)][0].GetUint()); } + } } -TEST(Value, Array) { - Value x(kArrayType); - const Value& y = x; - Value::AllocatorType allocator; +TEST(Value, Array) +{ Value x(kArrayType); + const Value& y = x; + Value::AllocatorType allocator; + + EXPECT_EQ(kArrayType, x.GetType()); + EXPECT_TRUE(x.IsArray()); + EXPECT_TRUE(x.Empty()); + EXPECT_EQ(0u, x.Size()); + EXPECT_TRUE(y.IsArray()); + EXPECT_TRUE(y.Empty()); + EXPECT_EQ(0u, y.Size()); + + EXPECT_FALSE(x.IsNull()); + EXPECT_FALSE(x.IsBool()); + EXPECT_FALSE(x.IsFalse()); + EXPECT_FALSE(x.IsTrue()); + EXPECT_FALSE(x.IsString()); + EXPECT_FALSE(x.IsObject()); + + TestArray(x, allocator); + + // Working in gcc without C++11, but VS2013 cannot compile. To be diagnosed. + // http://en.wikipedia.org/wiki/Erase-remove_idiom + x.Clear(); + for (int i = 0; i < 10; i++) + if (i % 2 == 0) + x.PushBack(i, allocator); + else + x.PushBack(Value(kNullType).Move(), allocator); + + const Value null(kNullType); + x.Erase(std::remove(x.Begin(), x.End(), null), x.End()); + EXPECT_EQ(5u, x.Size()); + for (int i = 0; i < 5; i++) + EXPECT_EQ(i * 2, x[static_cast(i)]); + + // SetArray() + Value z; + z.SetArray(); + EXPECT_TRUE(z.IsArray()); + EXPECT_TRUE(z.Empty()); +} - EXPECT_EQ(kArrayType, x.GetType()); - EXPECT_TRUE(x.IsArray()); - EXPECT_TRUE(x.Empty()); - EXPECT_EQ(0u, x.Size()); - EXPECT_TRUE(y.IsArray()); - EXPECT_TRUE(y.Empty()); - EXPECT_EQ(0u, y.Size()); +TEST(Value, ArrayHelper) +{ Value::AllocatorType allocator; + { Value x(kArrayType); + Value::Array a = x.GetArray(); + TestArray(a, allocator); + } - EXPECT_FALSE(x.IsNull()); - EXPECT_FALSE(x.IsBool()); - EXPECT_FALSE(x.IsFalse()); - EXPECT_FALSE(x.IsTrue()); - EXPECT_FALSE(x.IsString()); - EXPECT_FALSE(x.IsObject()); + { Value x(kArrayType); + Value::Array a = x.GetArray(); + a.PushBack(1, allocator); - TestArray(x, allocator); + Value::Array a2(a); // copy constructor + EXPECT_EQ(1, a2.Size()); - // Working in gcc without C++11, but VS2013 cannot compile. To be diagnosed. - // http://en.wikipedia.org/wiki/Erase-remove_idiom - x.Clear(); - for (int i = 0; i < 10; i++) - if (i % 2 == 0) - x.PushBack(i, allocator); - else - x.PushBack(Value(kNullType).Move(), allocator); - - const Value null(kNullType); - x.Erase(std::remove(x.Begin(), x.End(), null), x.End()); - EXPECT_EQ(5u, x.Size()); - for (int i = 0; i < 5; i++) - EXPECT_EQ(i * 2, x[static_cast(i)]); - - // SetArray() - Value z; - z.SetArray(); - EXPECT_TRUE(z.IsArray()); - EXPECT_TRUE(z.Empty()); -} + Value::Array a3 = a; + EXPECT_EQ(1, a3.Size()); -TEST(Value, ArrayHelper) { - Value::AllocatorType allocator; - { - Value x(kArrayType); - Value::Array a = x.GetArray(); - TestArray(a, allocator); - } + Value::ConstArray y = static_cast(x).GetArray(); + (void)y; + // y.PushBack(1, allocator); // should not compile - { - Value x(kArrayType); - Value::Array a = x.GetArray(); - a.PushBack(1, allocator); - - Value::Array a2(a); // copy constructor - EXPECT_EQ(1, a2.Size()); - - Value::Array a3 = a; - EXPECT_EQ(1, a3.Size()); - - Value::ConstArray y = static_cast(x).GetArray(); - (void)y; - // y.PushBack(1, allocator); // should not compile - - // Templated functions - x.Clear(); - EXPECT_TRUE(x.Is()); - EXPECT_TRUE(x.Is()); - a.PushBack(1, allocator); - EXPECT_EQ(1, x.Get()[0].GetInt()); - EXPECT_EQ(1, x.Get()[0].GetInt()); - - Value x2; - x2.Set(a); - EXPECT_TRUE(x.IsArray()); // IsArray() is invariant after moving. - EXPECT_EQ(1, x2.Get()[0].GetInt()); - } + // Templated functions + x.Clear(); + EXPECT_TRUE(x.Is()); + EXPECT_TRUE(x.Is()); + a.PushBack(1, allocator); + EXPECT_EQ(1, x.Get()[0].GetInt()); + EXPECT_EQ(1, x.Get()[0].GetInt()); - { - Value y(kArrayType); - y.PushBack(123, allocator); + Value x2; + x2.Set(a); + EXPECT_TRUE(x.IsArray()); // IsArray() is invariant after moving. + EXPECT_EQ(1, x2.Get()[0].GetInt()); + } - Value x(y.GetArray()); // Construct value form array. - EXPECT_TRUE(x.IsArray()); - EXPECT_EQ(123, x[0].GetInt()); - EXPECT_TRUE(y.IsArray()); // Invariant - EXPECT_TRUE(y.Empty()); - } + { Value y(kArrayType); + y.PushBack(123, allocator); + + Value x(y.GetArray()); // Construct value form array. + EXPECT_TRUE(x.IsArray()); + EXPECT_EQ(123, x[0].GetInt()); + EXPECT_TRUE(y.IsArray()); // Invariant + EXPECT_TRUE(y.Empty()); + } - { - Value x(kArrayType); - Value y(kArrayType); - y.PushBack(123, allocator); - x.PushBack(y.GetArray(), allocator); // Implicit constructor to convert Array to GenericValue + { Value x(kArrayType); + Value y(kArrayType); + y.PushBack(123, allocator); + x.PushBack(y.GetArray(), allocator); // Implicit constructor to convert Array to GenericValue - EXPECT_EQ(1, x.Size()); - EXPECT_EQ(123, x[0][0].GetInt()); - EXPECT_TRUE(y.IsArray()); - EXPECT_TRUE(y.Empty()); - } + EXPECT_EQ(1, x.Size()); + EXPECT_EQ(123, x[0][0].GetInt()); + EXPECT_TRUE(y.IsArray()); + EXPECT_TRUE(y.Empty()); + } } #if RAPIDJSON_HAS_CXX11_RANGE_FOR -TEST(Value, ArrayHelperRangeFor) { - Value::AllocatorType allocator; - Value x(kArrayType); - - for (int i = 0; i < 10; i++) - x.PushBack(i, allocator); - - { - int i = 0; - for (auto& v : x.GetArray()) { - EXPECT_EQ(i, v.GetInt()); - i++; - } - EXPECT_EQ(i, 10); +TEST(Value, ArrayHelperRangeFor) +{ Value::AllocatorType allocator; + Value x(kArrayType); + + for (int i = 0; i < 10; i++) + x.PushBack(i, allocator); + + { int i = 0; + for (auto& v : x.GetArray()) + { EXPECT_EQ(i, v.GetInt()); + i++; } - { - int i = 0; - for (const auto& v : const_cast(x).GetArray()) { - EXPECT_EQ(i, v.GetInt()); - i++; - } - EXPECT_EQ(i, 10); + EXPECT_EQ(i, 10); + } + { int i = 0; + for (const auto& v : const_cast(x).GetArray()) + { EXPECT_EQ(i, v.GetInt()); + i++; } + EXPECT_EQ(i, 10); + } - // Array a = x.GetArray(); - // Array ca = const_cast(x).GetArray(); + // Array a = x.GetArray(); + // Array ca = const_cast(x).GetArray(); } #endif template -static void TestObject(T& x, Allocator& allocator) { - const T& y = x; // const version - - // AddMember() - x.AddMember("A", "Apple", allocator); - EXPECT_FALSE(x.ObjectEmpty()); - EXPECT_EQ(1u, x.MemberCount()); - - Value value("Banana", 6); - x.AddMember("B", "Banana", allocator); - EXPECT_EQ(2u, x.MemberCount()); - - // AddMember(StringRefType, T, Allocator) - { - Value o(kObjectType); - o.AddMember("true", true, allocator); - o.AddMember("false", false, allocator); - o.AddMember("int", -1, allocator); - o.AddMember("uint", 1u, allocator); - o.AddMember("int64", int64_t(-4294967296), allocator); - o.AddMember("uint64", uint64_t(4294967296), allocator); - o.AddMember("double", 3.14, allocator); - o.AddMember("string", "Jelly", allocator); - - EXPECT_TRUE(o["true"].GetBool()); - EXPECT_FALSE(o["false"].GetBool()); - EXPECT_EQ(-1, o["int"].GetInt()); - EXPECT_EQ(1u, o["uint"].GetUint()); - EXPECT_EQ(int64_t(-4294967296), o["int64"].GetInt64()); - EXPECT_EQ(uint64_t(4294967296), o["uint64"].GetUint64()); - EXPECT_STREQ("Jelly",o["string"].GetString()); - EXPECT_EQ(8u, o.MemberCount()); - } - - // AddMember(Value&, T, Allocator) - { - Value o(kObjectType); - - Value n("s"); - o.AddMember(n, "string", allocator); - EXPECT_EQ(1u, o.MemberCount()); - - Value count("#"); - o.AddMember(count, o.MemberCount(), allocator); - EXPECT_EQ(2u, o.MemberCount()); - } +static void TestObject(T& x, Allocator& allocator) +{ const T& y = x; // const version + + // AddMember() + x.AddMember("A", "Apple", allocator); + EXPECT_FALSE(x.ObjectEmpty()); + EXPECT_EQ(1u, x.MemberCount()); + + Value value("Banana", 6); + x.AddMember("B", "Banana", allocator); + EXPECT_EQ(2u, x.MemberCount()); + + // AddMember(StringRefType, T, Allocator) + { Value o(kObjectType); + o.AddMember("true", true, allocator); + o.AddMember("false", false, allocator); + o.AddMember("int", -1, allocator); + o.AddMember("uint", 1u, allocator); + o.AddMember("int64", int64_t(-4294967296), allocator); + o.AddMember("uint64", uint64_t(4294967296), allocator); + o.AddMember("double", 3.14, allocator); + o.AddMember("string", "Jelly", allocator); + + EXPECT_TRUE(o["true"].GetBool()); + EXPECT_FALSE(o["false"].GetBool()); + EXPECT_EQ(-1, o["int"].GetInt()); + EXPECT_EQ(1u, o["uint"].GetUint()); + EXPECT_EQ(int64_t(-4294967296), o["int64"].GetInt64()); + EXPECT_EQ(uint64_t(4294967296), o["uint64"].GetUint64()); + EXPECT_STREQ("Jelly",o["string"].GetString()); + EXPECT_EQ(8u, o.MemberCount()); + } + + // AddMember(Value&, T, Allocator) + { Value o(kObjectType); + + Value n("s"); + o.AddMember(n, "string", allocator); + EXPECT_EQ(1u, o.MemberCount()); + + Value count("#"); + o.AddMember(count, o.MemberCount(), allocator); + EXPECT_EQ(2u, o.MemberCount()); + } #if RAPIDJSON_HAS_STDSTRING - { - // AddMember(StringRefType, const std::string&, Allocator) - Value o(kObjectType); - o.AddMember("b", std::string("Banana"), allocator); - EXPECT_STREQ("Banana", o["b"].GetString()); - - // RemoveMember(const std::string&) - o.RemoveMember(std::string("b")); - EXPECT_TRUE(o.ObjectEmpty()); - } + { // AddMember(StringRefType, const std::string&, Allocator) + Value o(kObjectType); + o.AddMember("b", std::string("Banana"), allocator); + EXPECT_STREQ("Banana", o["b"].GetString()); + + // RemoveMember(const std::string&) + o.RemoveMember(std::string("b")); + EXPECT_TRUE(o.ObjectEmpty()); + } #endif #if RAPIDJSON_HAS_CXX11_RVALUE_REFS - // AddMember(GenericValue&&, ...) variants - { - Value o(kObjectType); - o.AddMember(Value("true"), Value(true), allocator); - o.AddMember(Value("false"), Value(false).Move(), allocator); // value is lvalue ref - o.AddMember(Value("int").Move(), Value(-1), allocator); // name is lvalue ref - o.AddMember("uint", std::move(Value().SetUint(1u)), allocator); // name is literal, value is rvalue - EXPECT_TRUE(o["true"].GetBool()); - EXPECT_FALSE(o["false"].GetBool()); - EXPECT_EQ(-1, o["int"].GetInt()); - EXPECT_EQ(1u, o["uint"].GetUint()); - EXPECT_EQ(4u, o.MemberCount()); - } + // AddMember(GenericValue&&, ...) variants + { Value o(kObjectType); + o.AddMember(Value("true"), Value(true), allocator); + o.AddMember(Value("false"), Value(false).Move(), allocator); // value is lvalue ref + o.AddMember(Value("int").Move(), Value(-1), allocator); // name is lvalue ref + o.AddMember("uint", std::move(Value().SetUint(1u)), allocator); // name is literal, value is rvalue + EXPECT_TRUE(o["true"].GetBool()); + EXPECT_FALSE(o["false"].GetBool()); + EXPECT_EQ(-1, o["int"].GetInt()); + EXPECT_EQ(1u, o["uint"].GetUint()); + EXPECT_EQ(4u, o.MemberCount()); + } #endif - // Tests a member with null character - Value name; - const Value C0D("C\0D", 3); - name.SetString(C0D.GetString(), 3); - value.SetString("CherryD", 7); - x.AddMember(name, value, allocator); + // Tests a member with null character + Value name; + const Value C0D("C\0D", 3); + name.SetString(C0D.GetString(), 3); + value.SetString("CherryD", 7); + x.AddMember(name, value, allocator); - // HasMember() - EXPECT_TRUE(x.HasMember("A")); - EXPECT_TRUE(x.HasMember("B")); - EXPECT_TRUE(y.HasMember("A")); - EXPECT_TRUE(y.HasMember("B")); + // HasMember() + EXPECT_TRUE(x.HasMember("A")); + EXPECT_TRUE(x.HasMember("B")); + EXPECT_TRUE(y.HasMember("A")); + EXPECT_TRUE(y.HasMember("B")); #if RAPIDJSON_HAS_STDSTRING - EXPECT_TRUE(x.HasMember(std::string("A"))); + EXPECT_TRUE(x.HasMember(std::string("A"))); #endif - name.SetString("C\0D"); - EXPECT_TRUE(x.HasMember(name)); - EXPECT_TRUE(y.HasMember(name)); - - GenericValue, CrtAllocator> othername("A"); - EXPECT_TRUE(x.HasMember(othername)); - EXPECT_TRUE(y.HasMember(othername)); - othername.SetString("C\0D"); - EXPECT_TRUE(x.HasMember(othername)); - EXPECT_TRUE(y.HasMember(othername)); - - // operator[] - EXPECT_STREQ("Apple", x["A"].GetString()); - EXPECT_STREQ("Banana", x["B"].GetString()); - EXPECT_STREQ("CherryD", x[C0D].GetString()); - EXPECT_STREQ("CherryD", x[othername].GetString()); - EXPECT_THROW(x["nonexist"], AssertException); - - // const operator[] - EXPECT_STREQ("Apple", y["A"].GetString()); - EXPECT_STREQ("Banana", y["B"].GetString()); - EXPECT_STREQ("CherryD", y[C0D].GetString()); + name.SetString("C\0D"); + EXPECT_TRUE(x.HasMember(name)); + EXPECT_TRUE(y.HasMember(name)); + + GenericValue, CrtAllocator> othername("A"); + EXPECT_TRUE(x.HasMember(othername)); + EXPECT_TRUE(y.HasMember(othername)); + othername.SetString("C\0D"); + EXPECT_TRUE(x.HasMember(othername)); + EXPECT_TRUE(y.HasMember(othername)); + + // operator[] + EXPECT_STREQ("Apple", x["A"].GetString()); + EXPECT_STREQ("Banana", x["B"].GetString()); + EXPECT_STREQ("CherryD", x[C0D].GetString()); + EXPECT_STREQ("CherryD", x[othername].GetString()); + EXPECT_THROW(x["nonexist"], AssertException); + + // const operator[] + EXPECT_STREQ("Apple", y["A"].GetString()); + EXPECT_STREQ("Banana", y["B"].GetString()); + EXPECT_STREQ("CherryD", y[C0D].GetString()); #if RAPIDJSON_HAS_STDSTRING - EXPECT_STREQ("Apple", x["A"].GetString()); - EXPECT_STREQ("Apple", y[std::string("A")].GetString()); + EXPECT_STREQ("Apple", x["A"].GetString()); + EXPECT_STREQ("Apple", y[std::string("A")].GetString()); #endif - // member iterator - Value::MemberIterator itr = x.MemberBegin(); - EXPECT_TRUE(itr != x.MemberEnd()); - EXPECT_STREQ("A", itr->name.GetString()); - EXPECT_STREQ("Apple", itr->value.GetString()); - ++itr; - EXPECT_TRUE(itr != x.MemberEnd()); - EXPECT_STREQ("B", itr->name.GetString()); - EXPECT_STREQ("Banana", itr->value.GetString()); - ++itr; - EXPECT_TRUE(itr != x.MemberEnd()); - EXPECT_TRUE(memcmp(itr->name.GetString(), "C\0D", 4) == 0); - EXPECT_STREQ("CherryD", itr->value.GetString()); - ++itr; - EXPECT_FALSE(itr != x.MemberEnd()); - - // const member iterator - Value::ConstMemberIterator citr = y.MemberBegin(); - EXPECT_TRUE(citr != y.MemberEnd()); - EXPECT_STREQ("A", citr->name.GetString()); - EXPECT_STREQ("Apple", citr->value.GetString()); - ++citr; - EXPECT_TRUE(citr != y.MemberEnd()); - EXPECT_STREQ("B", citr->name.GetString()); - EXPECT_STREQ("Banana", citr->value.GetString()); - ++citr; - EXPECT_TRUE(citr != y.MemberEnd()); - EXPECT_TRUE(memcmp(citr->name.GetString(), "C\0D", 4) == 0); - EXPECT_STREQ("CherryD", citr->value.GetString()); - ++citr; - EXPECT_FALSE(citr != y.MemberEnd()); - - // member iterator conversions/relations - itr = x.MemberBegin(); - citr = x.MemberBegin(); // const conversion - TestEqual(itr, citr); - EXPECT_TRUE(itr < x.MemberEnd()); - EXPECT_FALSE(itr > y.MemberEnd()); - EXPECT_TRUE(citr < x.MemberEnd()); - EXPECT_FALSE(citr > y.MemberEnd()); - ++citr; - TestUnequal(itr, citr); - EXPECT_FALSE(itr < itr); - EXPECT_TRUE(itr < citr); - EXPECT_FALSE(itr > itr); - EXPECT_TRUE(citr > itr); - EXPECT_EQ(1, citr - x.MemberBegin()); - EXPECT_EQ(0, itr - y.MemberBegin()); - itr += citr - x.MemberBegin(); - EXPECT_EQ(1, itr - y.MemberBegin()); - TestEqual(citr, itr); - EXPECT_TRUE(itr <= citr); - EXPECT_TRUE(citr <= itr); - itr++; - EXPECT_TRUE(itr >= citr); - EXPECT_FALSE(citr >= itr); - - // RemoveMember() - EXPECT_TRUE(x.RemoveMember("A")); - EXPECT_FALSE(x.HasMember("A")); - - EXPECT_TRUE(x.RemoveMember("B")); - EXPECT_FALSE(x.HasMember("B")); - - EXPECT_FALSE(x.RemoveMember("nonexist")); - - EXPECT_TRUE(x.RemoveMember(othername)); - EXPECT_FALSE(x.HasMember(name)); - - EXPECT_TRUE(x.MemberBegin() == x.MemberEnd()); - - // EraseMember(ConstMemberIterator) - - // Use array members to ensure removed elements' destructor is called. - // { "a": [0], "b": [1],[2],...] - const char keys[][2] = { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j" }; - for (int i = 0; i < 10; i++) + // member iterator + Value::MemberIterator itr = x.MemberBegin(); + EXPECT_TRUE(itr != x.MemberEnd()); + EXPECT_STREQ("A", itr->name.GetString()); + EXPECT_STREQ("Apple", itr->value.GetString()); + ++itr; + EXPECT_TRUE(itr != x.MemberEnd()); + EXPECT_STREQ("B", itr->name.GetString()); + EXPECT_STREQ("Banana", itr->value.GetString()); + ++itr; + EXPECT_TRUE(itr != x.MemberEnd()); + EXPECT_TRUE(memcmp(itr->name.GetString(), "C\0D", 4) == 0); + EXPECT_STREQ("CherryD", itr->value.GetString()); + ++itr; + EXPECT_FALSE(itr != x.MemberEnd()); + + // const member iterator + Value::ConstMemberIterator citr = y.MemberBegin(); + EXPECT_TRUE(citr != y.MemberEnd()); + EXPECT_STREQ("A", citr->name.GetString()); + EXPECT_STREQ("Apple", citr->value.GetString()); + ++citr; + EXPECT_TRUE(citr != y.MemberEnd()); + EXPECT_STREQ("B", citr->name.GetString()); + EXPECT_STREQ("Banana", citr->value.GetString()); + ++citr; + EXPECT_TRUE(citr != y.MemberEnd()); + EXPECT_TRUE(memcmp(citr->name.GetString(), "C\0D", 4) == 0); + EXPECT_STREQ("CherryD", citr->value.GetString()); + ++citr; + EXPECT_FALSE(citr != y.MemberEnd()); + + // member iterator conversions/relations + itr = x.MemberBegin(); + citr = x.MemberBegin(); // const conversion + TestEqual(itr, citr); + EXPECT_TRUE(itr < x.MemberEnd()); + EXPECT_FALSE(itr > y.MemberEnd()); + EXPECT_TRUE(citr < x.MemberEnd()); + EXPECT_FALSE(citr > y.MemberEnd()); + ++citr; + TestUnequal(itr, citr); + EXPECT_FALSE(itr < itr); + EXPECT_TRUE(itr < citr); + EXPECT_FALSE(itr > itr); + EXPECT_TRUE(citr > itr); + EXPECT_EQ(1, citr - x.MemberBegin()); + EXPECT_EQ(0, itr - y.MemberBegin()); + itr += citr - x.MemberBegin(); + EXPECT_EQ(1, itr - y.MemberBegin()); + TestEqual(citr, itr); + EXPECT_TRUE(itr <= citr); + EXPECT_TRUE(citr <= itr); + itr++; + EXPECT_TRUE(itr >= citr); + EXPECT_FALSE(citr >= itr); + + // RemoveMember() + EXPECT_TRUE(x.RemoveMember("A")); + EXPECT_FALSE(x.HasMember("A")); + + EXPECT_TRUE(x.RemoveMember("B")); + EXPECT_FALSE(x.HasMember("B")); + + EXPECT_FALSE(x.RemoveMember("nonexist")); + + EXPECT_TRUE(x.RemoveMember(othername)); + EXPECT_FALSE(x.HasMember(name)); + + EXPECT_TRUE(x.MemberBegin() == x.MemberEnd()); + + // EraseMember(ConstMemberIterator) + + // Use array members to ensure removed elements' destructor is called. + // { "a": [0], "b": [1],[2],...] + const char keys[][2] = { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j" }; + for (int i = 0; i < 10; i++) + x.AddMember(keys[i], Value(kArrayType).PushBack(i, allocator), allocator); + + // MemberCount, iterator difference + EXPECT_EQ(x.MemberCount(), SizeType(x.MemberEnd() - x.MemberBegin())); + + // Erase the first + itr = x.EraseMember(x.MemberBegin()); + EXPECT_FALSE(x.HasMember(keys[0])); + EXPECT_EQ(x.MemberBegin(), itr); + EXPECT_EQ(9u, x.MemberCount()); + for (; itr != x.MemberEnd(); ++itr) + { size_t i = static_cast((itr - x.MemberBegin())) + 1; + EXPECT_STREQ(itr->name.GetString(), keys[i]); + EXPECT_EQ(i, itr->value[0].GetInt()); + } + + // Erase the last + itr = x.EraseMember(x.MemberEnd() - 1); + EXPECT_FALSE(x.HasMember(keys[9])); + EXPECT_EQ(x.MemberEnd(), itr); + EXPECT_EQ(8u, x.MemberCount()); + for (; itr != x.MemberEnd(); ++itr) + { size_t i = static_cast(itr - x.MemberBegin()) + 1; + EXPECT_STREQ(itr->name.GetString(), keys[i]); + EXPECT_EQ(i, itr->value[0].GetInt()); + } + + // Erase the middle + itr = x.EraseMember(x.MemberBegin() + 4); + EXPECT_FALSE(x.HasMember(keys[5])); + EXPECT_EQ(x.MemberBegin() + 4, itr); + EXPECT_EQ(7u, x.MemberCount()); + for (; itr != x.MemberEnd(); ++itr) + { size_t i = static_cast(itr - x.MemberBegin()); + i += (i < 4) ? 1 : 2; + EXPECT_STREQ(itr->name.GetString(), keys[i]); + EXPECT_EQ(i, itr->value[0].GetInt()); + } + + // EraseMember(ConstMemberIterator, ConstMemberIterator) + // Exhaustive test with all 0 <= first < n, first <= last <= n cases + const unsigned n = 10; + for (unsigned first = 0; first < n; first++) + { for (unsigned last = first; last <= n; last++) + { x.RemoveAllMembers(); + for (unsigned i = 0; i < n; i++) x.AddMember(keys[i], Value(kArrayType).PushBack(i, allocator), allocator); - // MemberCount, iterator difference - EXPECT_EQ(x.MemberCount(), SizeType(x.MemberEnd() - x.MemberBegin())); - - // Erase the first - itr = x.EraseMember(x.MemberBegin()); - EXPECT_FALSE(x.HasMember(keys[0])); - EXPECT_EQ(x.MemberBegin(), itr); - EXPECT_EQ(9u, x.MemberCount()); - for (; itr != x.MemberEnd(); ++itr) { - size_t i = static_cast((itr - x.MemberBegin())) + 1; - EXPECT_STREQ(itr->name.GetString(), keys[i]); - EXPECT_EQ(i, itr->value[0].GetInt()); + itr = x.EraseMember(x.MemberBegin() + static_cast(first), x.MemberBegin() + static_cast(last)); + if (last == n) + EXPECT_EQ(x.MemberEnd(), itr); + else + EXPECT_EQ(x.MemberBegin() + static_cast(first), itr); + + size_t removeCount = last - first; + EXPECT_EQ(n - removeCount, x.MemberCount()); + for (unsigned i = 0; i < first; i++) + EXPECT_EQ(i, x[keys[i]][0].GetUint()); + for (unsigned i = first; i < n - removeCount; i++) + EXPECT_EQ(i + removeCount, x[keys[i+removeCount]][0].GetUint()); } + } - // Erase the last - itr = x.EraseMember(x.MemberEnd() - 1); - EXPECT_FALSE(x.HasMember(keys[9])); - EXPECT_EQ(x.MemberEnd(), itr); - EXPECT_EQ(8u, x.MemberCount()); - for (; itr != x.MemberEnd(); ++itr) { - size_t i = static_cast(itr - x.MemberBegin()) + 1; - EXPECT_STREQ(itr->name.GetString(), keys[i]); - EXPECT_EQ(i, itr->value[0].GetInt()); - } + // RemoveAllMembers() + x.RemoveAllMembers(); + EXPECT_TRUE(x.ObjectEmpty()); + EXPECT_EQ(0u, x.MemberCount()); +} - // Erase the middle - itr = x.EraseMember(x.MemberBegin() + 4); - EXPECT_FALSE(x.HasMember(keys[5])); - EXPECT_EQ(x.MemberBegin() + 4, itr); - EXPECT_EQ(7u, x.MemberCount()); - for (; itr != x.MemberEnd(); ++itr) { - size_t i = static_cast(itr - x.MemberBegin()); - i += (i < 4) ? 1 : 2; - EXPECT_STREQ(itr->name.GetString(), keys[i]); - EXPECT_EQ(i, itr->value[0].GetInt()); - } +TEST(Value, Object) +{ Value x(kObjectType); + const Value& y = x; // const version + Value::AllocatorType allocator; + + EXPECT_EQ(kObjectType, x.GetType()); + EXPECT_TRUE(x.IsObject()); + EXPECT_TRUE(x.ObjectEmpty()); + EXPECT_EQ(0u, x.MemberCount()); + EXPECT_EQ(kObjectType, y.GetType()); + EXPECT_TRUE(y.IsObject()); + EXPECT_TRUE(y.ObjectEmpty()); + EXPECT_EQ(0u, y.MemberCount()); + + TestObject(x, allocator); + + // SetObject() + Value z; + z.SetObject(); + EXPECT_TRUE(z.IsObject()); +} - // EraseMember(ConstMemberIterator, ConstMemberIterator) - // Exhaustive test with all 0 <= first < n, first <= last <= n cases - const unsigned n = 10; - for (unsigned first = 0; first < n; first++) { - for (unsigned last = first; last <= n; last++) { - x.RemoveAllMembers(); - for (unsigned i = 0; i < n; i++) - x.AddMember(keys[i], Value(kArrayType).PushBack(i, allocator), allocator); - - itr = x.EraseMember(x.MemberBegin() + static_cast(first), x.MemberBegin() + static_cast(last)); - if (last == n) - EXPECT_EQ(x.MemberEnd(), itr); - else - EXPECT_EQ(x.MemberBegin() + static_cast(first), itr); - - size_t removeCount = last - first; - EXPECT_EQ(n - removeCount, x.MemberCount()); - for (unsigned i = 0; i < first; i++) - EXPECT_EQ(i, x[keys[i]][0].GetUint()); - for (unsigned i = first; i < n - removeCount; i++) - EXPECT_EQ(i + removeCount, x[keys[i+removeCount]][0].GetUint()); - } - } +TEST(Value, ObjectHelper) +{ Value::AllocatorType allocator; + { Value x(kObjectType); + Value::Object o = x.GetObject(); + TestObject(o, allocator); + } - // RemoveAllMembers() - x.RemoveAllMembers(); - EXPECT_TRUE(x.ObjectEmpty()); - EXPECT_EQ(0u, x.MemberCount()); -} + { Value x(kObjectType); + Value::Object o = x.GetObject(); + o.AddMember("1", 1, allocator); -TEST(Value, Object) { - Value x(kObjectType); - const Value& y = x; // const version - Value::AllocatorType allocator; - - EXPECT_EQ(kObjectType, x.GetType()); - EXPECT_TRUE(x.IsObject()); - EXPECT_TRUE(x.ObjectEmpty()); - EXPECT_EQ(0u, x.MemberCount()); - EXPECT_EQ(kObjectType, y.GetType()); - EXPECT_TRUE(y.IsObject()); - EXPECT_TRUE(y.ObjectEmpty()); - EXPECT_EQ(0u, y.MemberCount()); - - TestObject(x, allocator); - - // SetObject() - Value z; - z.SetObject(); - EXPECT_TRUE(z.IsObject()); -} + Value::Object o2(o); // copy constructor + EXPECT_EQ(1, o2.MemberCount()); -TEST(Value, ObjectHelper) { - Value::AllocatorType allocator; - { - Value x(kObjectType); - Value::Object o = x.GetObject(); - TestObject(o, allocator); - } + Value::Object o3 = o; + EXPECT_EQ(1, o3.MemberCount()); - { - Value x(kObjectType); - Value::Object o = x.GetObject(); - o.AddMember("1", 1, allocator); - - Value::Object o2(o); // copy constructor - EXPECT_EQ(1, o2.MemberCount()); - - Value::Object o3 = o; - EXPECT_EQ(1, o3.MemberCount()); - - Value::ConstObject y = static_cast(x).GetObject(); - (void)y; - // y.AddMember("1", 1, allocator); // should not compile - - // Templated functions - x.RemoveAllMembers(); - EXPECT_TRUE(x.Is()); - EXPECT_TRUE(x.Is()); - o.AddMember("1", 1, allocator); - EXPECT_EQ(1, x.Get()["1"].GetInt()); - EXPECT_EQ(1, x.Get()["1"].GetInt()); - - Value x2; - x2.Set(o); - EXPECT_TRUE(x.IsObject()); // IsObject() is invariant after moving - EXPECT_EQ(1, x2.Get()["1"].GetInt()); - } + Value::ConstObject y = static_cast(x).GetObject(); + (void)y; + // y.AddMember("1", 1, allocator); // should not compile - { - Value x(kObjectType); - x.AddMember("a", "apple", allocator); - Value y(x.GetObject()); - EXPECT_STREQ("apple", y["a"].GetString()); - EXPECT_TRUE(x.IsObject()); // Invariant - } - - { - Value x(kObjectType); - x.AddMember("a", "apple", allocator); - Value y(kObjectType); - y.AddMember("fruits", x.GetObject(), allocator); - EXPECT_STREQ("apple", y["fruits"]["a"].GetString()); - EXPECT_TRUE(x.IsObject()); // Invariant - } + // Templated functions + x.RemoveAllMembers(); + EXPECT_TRUE(x.Is()); + EXPECT_TRUE(x.Is()); + o.AddMember("1", 1, allocator); + EXPECT_EQ(1, x.Get()["1"].GetInt()); + EXPECT_EQ(1, x.Get()["1"].GetInt()); + + Value x2; + x2.Set(o); + EXPECT_TRUE(x.IsObject()); // IsObject() is invariant after moving + EXPECT_EQ(1, x2.Get()["1"].GetInt()); + } + + { Value x(kObjectType); + x.AddMember("a", "apple", allocator); + Value y(x.GetObject()); + EXPECT_STREQ("apple", y["a"].GetString()); + EXPECT_TRUE(x.IsObject()); // Invariant + } + + { Value x(kObjectType); + x.AddMember("a", "apple", allocator); + Value y(kObjectType); + y.AddMember("fruits", x.GetObject(), allocator); + EXPECT_STREQ("apple", y["fruits"]["a"].GetString()); + EXPECT_TRUE(x.IsObject()); // Invariant + } } #if RAPIDJSON_HAS_CXX11_RANGE_FOR -TEST(Value, ObjectHelperRangeFor) { - Value::AllocatorType allocator; - Value x(kObjectType); - - for (int i = 0; i < 10; i++) { - char name[10]; - Value n(name, static_cast(sprintf(name, "%d", i)), allocator); - x.AddMember(n, i, allocator); +TEST(Value, ObjectHelperRangeFor) +{ Value::AllocatorType allocator; + Value x(kObjectType); + + for (int i = 0; i < 10; i++) + { char name[10]; + Value n(name, static_cast(sprintf(name, "%d", i)), allocator); + x.AddMember(n, i, allocator); + } + + { int i = 0; + for (auto& m : x.GetObject()) + { char name[10]; + sprintf(name, "%d", i); + EXPECT_STREQ(name, m.name.GetString()); + EXPECT_EQ(i, m.value.GetInt()); + i++; } - - { - int i = 0; - for (auto& m : x.GetObject()) { - char name[10]; - sprintf(name, "%d", i); - EXPECT_STREQ(name, m.name.GetString()); - EXPECT_EQ(i, m.value.GetInt()); - i++; - } - EXPECT_EQ(i, 10); - } - { - int i = 0; - for (const auto& m : const_cast(x).GetObject()) { - char name[10]; - sprintf(name, "%d", i); - EXPECT_STREQ(name, m.name.GetString()); - EXPECT_EQ(i, m.value.GetInt()); - i++; - } - EXPECT_EQ(i, 10); + EXPECT_EQ(i, 10); + } + { int i = 0; + for (const auto& m : const_cast(x).GetObject()) + { char name[10]; + sprintf(name, "%d", i); + EXPECT_STREQ(name, m.name.GetString()); + EXPECT_EQ(i, m.value.GetInt()); + i++; } + EXPECT_EQ(i, 10); + } - // Object a = x.GetObject(); - // Object ca = const_cast(x).GetObject(); + // Object a = x.GetObject(); + // Object ca = const_cast(x).GetObject(); } #endif -TEST(Value, EraseMember_String) { - Value::AllocatorType allocator; - Value x(kObjectType); - x.AddMember("A", "Apple", allocator); - x.AddMember("B", "Banana", allocator); +TEST(Value, EraseMember_String) +{ Value::AllocatorType allocator; + Value x(kObjectType); + x.AddMember("A", "Apple", allocator); + x.AddMember("B", "Banana", allocator); - EXPECT_TRUE(x.EraseMember("B")); - EXPECT_FALSE(x.HasMember("B")); + EXPECT_TRUE(x.EraseMember("B")); + EXPECT_FALSE(x.HasMember("B")); - EXPECT_FALSE(x.EraseMember("nonexist")); + EXPECT_FALSE(x.EraseMember("nonexist")); - GenericValue, CrtAllocator> othername("A"); - EXPECT_TRUE(x.EraseMember(othername)); - EXPECT_FALSE(x.HasMember("A")); + GenericValue, CrtAllocator> othername("A"); + EXPECT_TRUE(x.EraseMember(othername)); + EXPECT_FALSE(x.HasMember("A")); - EXPECT_TRUE(x.MemberBegin() == x.MemberEnd()); + EXPECT_TRUE(x.MemberBegin() == x.MemberEnd()); } -TEST(Value, BigNestedArray) { - MemoryPoolAllocator<> allocator; - Value x(kArrayType); - static const SizeType n = 200; - - for (SizeType i = 0; i < n; i++) { - Value y(kArrayType); - for (SizeType j = 0; j < n; j++) { - Value number(static_cast(i * n + j)); - y.PushBack(number, allocator); - } - x.PushBack(y, allocator); +TEST(Value, BigNestedArray) +{ MemoryPoolAllocator<> allocator; + Value x(kArrayType); + static const SizeType n = 200; + + for (SizeType i = 0; i < n; i++) + { Value y(kArrayType); + for (SizeType j = 0; j < n; j++) + { Value number(static_cast(i * n + j)); + y.PushBack(number, allocator); } + x.PushBack(y, allocator); + } - for (SizeType i = 0; i < n; i++) - for (SizeType j = 0; j < n; j++) { - EXPECT_TRUE(x[i][j].IsInt()); - EXPECT_EQ(static_cast(i * n + j), x[i][j].GetInt()); - } + for (SizeType i = 0; i < n; i++) + for (SizeType j = 0; j < n; j++) + { EXPECT_TRUE(x[i][j].IsInt()); + EXPECT_EQ(static_cast(i * n + j), x[i][j].GetInt()); + } } -TEST(Value, BigNestedObject) { - MemoryPoolAllocator<> allocator; - Value x(kObjectType); - static const SizeType n = 200; - - for (SizeType i = 0; i < n; i++) { - char name1[10]; - sprintf(name1, "%d", i); +TEST(Value, BigNestedObject) +{ MemoryPoolAllocator<> allocator; + Value x(kObjectType); + static const SizeType n = 200; - // Value name(name1); // should not compile - Value name(name1, static_cast(strlen(name1)), allocator); - Value object(kObjectType); + for (SizeType i = 0; i < n; i++) + { char name1[10]; + sprintf(name1, "%d", i); - for (SizeType j = 0; j < n; j++) { - char name2[10]; - sprintf(name2, "%d", j); + // Value name(name1); // should not compile + Value name(name1, static_cast(strlen(name1)), allocator); + Value object(kObjectType); - Value name3(name2, static_cast(strlen(name2)), allocator); - Value number(static_cast(i * n + j)); - object.AddMember(name3, number, allocator); - } + for (SizeType j = 0; j < n; j++) + { char name2[10]; + sprintf(name2, "%d", j); - // x.AddMember(name1, object, allocator); // should not compile - x.AddMember(name, object, allocator); + Value name3(name2, static_cast(strlen(name2)), allocator); + Value number(static_cast(i * n + j)); + object.AddMember(name3, number, allocator); } - for (SizeType i = 0; i < n; i++) { - char name1[10]; - sprintf(name1, "%d", i); - - for (SizeType j = 0; j < n; j++) { - char name2[10]; - sprintf(name2, "%d", j); - x[name1]; - EXPECT_EQ(static_cast(i * n + j), x[name1][name2].GetInt()); - } + // x.AddMember(name1, object, allocator); // should not compile + x.AddMember(name, object, allocator); + } + + for (SizeType i = 0; i < n; i++) + { char name1[10]; + sprintf(name1, "%d", i); + + for (SizeType j = 0; j < n; j++) + { char name2[10]; + sprintf(name2, "%d", j); + x[name1]; + EXPECT_EQ(static_cast(i * n + j), x[name1][name2].GetInt()); } + } } // Issue 18: Error removing last element of object // http://code.google.com/p/rapidjson/issues/detail?id=18 -TEST(Value, RemoveLastElement) { - rapidjson::Document doc; - rapidjson::Document::AllocatorType& allocator = doc.GetAllocator(); - rapidjson::Value objVal(rapidjson::kObjectType); - objVal.AddMember("var1", 123, allocator); - objVal.AddMember("var2", "444", allocator); - objVal.AddMember("var3", 555, allocator); - EXPECT_TRUE(objVal.HasMember("var3")); - objVal.RemoveMember("var3"); // Assertion here in r61 - EXPECT_FALSE(objVal.HasMember("var3")); +TEST(Value, RemoveLastElement) +{ rapidjson::Document doc; + rapidjson::Document::AllocatorType& allocator = doc.GetAllocator(); + rapidjson::Value objVal(rapidjson::kObjectType); + objVal.AddMember("var1", 123, allocator); + objVal.AddMember("var2", "444", allocator); + objVal.AddMember("var3", 555, allocator); + EXPECT_TRUE(objVal.HasMember("var3")); + objVal.RemoveMember("var3"); // Assertion here in r61 + EXPECT_FALSE(objVal.HasMember("var3")); } // Issue 38: Segmentation fault with CrtAllocator -TEST(Document, CrtAllocator) { - typedef GenericValue, CrtAllocator> V; +TEST(Document, CrtAllocator) +{ typedef GenericValue, CrtAllocator> V; - V::AllocatorType allocator; - V o(kObjectType); - o.AddMember("x", 1, allocator); // Should not call destructor on uninitialized name/value of newly allocated members. + V::AllocatorType allocator; + V o(kObjectType); + o.AddMember("x", 1, allocator); // Should not call destructor on uninitialized name/value of newly allocated members. - V a(kArrayType); - a.PushBack(1, allocator); // Should not call destructor on uninitialized Value of newly allocated elements. + V a(kArrayType); + a.PushBack(1, allocator); // Should not call destructor on uninitialized Value of newly allocated elements. } -static void TestShortStringOptimization(const char* str) { - const rapidjson::SizeType len = static_cast(strlen(str)); - - rapidjson::Document doc; - rapidjson::Value val; - val.SetString(str, len, doc.GetAllocator()); - - EXPECT_EQ(val.GetStringLength(), len); - EXPECT_STREQ(val.GetString(), str); +static void TestShortStringOptimization(const char* str) +{ const rapidjson::SizeType len = static_cast(strlen(str)); + + rapidjson::Document doc; + rapidjson::Value val; + val.SetString(str, len, doc.GetAllocator()); + + EXPECT_EQ(val.GetStringLength(), len); + EXPECT_STREQ(val.GetString(), str); } -TEST(Value, AllocateShortString) { - TestShortStringOptimization(""); // edge case: empty string - TestShortStringOptimization("12345678"); // regular case for short strings: 8 chars - TestShortStringOptimization("12345678901"); // edge case: 11 chars in 32-bit mode (=> short string) - TestShortStringOptimization("123456789012"); // edge case: 12 chars in 32-bit mode (=> regular string) - TestShortStringOptimization("123456789012345"); // edge case: 15 chars in 64-bit mode (=> short string) - TestShortStringOptimization("1234567890123456"); // edge case: 16 chars in 64-bit mode (=> regular string) +TEST(Value, AllocateShortString) +{ TestShortStringOptimization(""); // edge case: empty string + TestShortStringOptimization("12345678"); // regular case for short strings: 8 chars + TestShortStringOptimization("12345678901"); // edge case: 11 chars in 32-bit mode (=> short string) + TestShortStringOptimization("123456789012"); // edge case: 12 chars in 32-bit mode (=> regular string) + TestShortStringOptimization("123456789012345"); // edge case: 15 chars in 64-bit mode (=> short string) + TestShortStringOptimization("1234567890123456"); // edge case: 16 chars in 64-bit mode (=> regular string) } template -struct TerminateHandler { - bool Null() { return e != 0; } - bool Bool(bool) { return e != 1; } - bool Int(int) { return e != 2; } - bool Uint(unsigned) { return e != 3; } - bool Int64(int64_t) { return e != 4; } - bool Uint64(uint64_t) { return e != 5; } - bool Double(double) { return e != 6; } - bool RawNumber(const char*, SizeType, bool) { return e != 7; } - bool String(const char*, SizeType, bool) { return e != 8; } - bool StartObject() { return e != 9; } - bool Key(const char*, SizeType, bool) { return e != 10; } - bool EndObject(SizeType) { return e != 11; } - bool StartArray() { return e != 12; } - bool EndArray(SizeType) { return e != 13; } +struct TerminateHandler +{ bool Null() { return e != 0; } + bool Bool(bool) { return e != 1; } + bool Int(int) { return e != 2; } + bool Uint(unsigned) { return e != 3; } + bool Int64(int64_t) { return e != 4; } + bool Uint64(uint64_t) { return e != 5; } + bool Double(double) { return e != 6; } + bool RawNumber(const char*, SizeType, bool) { return e != 7; } + bool String(const char*, SizeType, bool) { return e != 8; } + bool StartObject() { return e != 9; } + bool Key(const char*, SizeType, bool) { return e != 10; } + bool EndObject(SizeType) { return e != 11; } + bool StartArray() { return e != 12; } + bool EndArray(SizeType) { return e != 13; } }; #define TEST_TERMINATION(e, json)\ @@ -1688,103 +1671,103 @@ struct TerminateHandler { EXPECT_FALSE(d.Accept(h));\ } -TEST(Value, AcceptTerminationByHandler) { - TEST_TERMINATION(0, "[null]"); - TEST_TERMINATION(1, "[true]"); - TEST_TERMINATION(1, "[false]"); - TEST_TERMINATION(2, "[-1]"); - TEST_TERMINATION(3, "[2147483648]"); - TEST_TERMINATION(4, "[-1234567890123456789]"); - TEST_TERMINATION(5, "[9223372036854775808]"); - TEST_TERMINATION(6, "[0.5]"); - // RawNumber() is never called - TEST_TERMINATION(8, "[\"a\"]"); - TEST_TERMINATION(9, "[{}]"); - TEST_TERMINATION(10, "[{\"a\":1}]"); - TEST_TERMINATION(11, "[{}]"); - TEST_TERMINATION(12, "{\"a\":[]}"); - TEST_TERMINATION(13, "{\"a\":[]}"); +TEST(Value, AcceptTerminationByHandler) +{ TEST_TERMINATION(0, "[null]"); + TEST_TERMINATION(1, "[true]"); + TEST_TERMINATION(1, "[false]"); + TEST_TERMINATION(2, "[-1]"); + TEST_TERMINATION(3, "[2147483648]"); + TEST_TERMINATION(4, "[-1234567890123456789]"); + TEST_TERMINATION(5, "[9223372036854775808]"); + TEST_TERMINATION(6, "[0.5]"); + // RawNumber() is never called + TEST_TERMINATION(8, "[\"a\"]"); + TEST_TERMINATION(9, "[{}]"); + TEST_TERMINATION(10, "[{\"a\":1}]"); + TEST_TERMINATION(11, "[{}]"); + TEST_TERMINATION(12, "{\"a\":[]}"); + TEST_TERMINATION(13, "{\"a\":[]}"); } -struct ValueIntComparer { - bool operator()(const Value& lhs, const Value& rhs) const { - return lhs.GetInt() < rhs.GetInt(); - } +struct ValueIntComparer +{ bool operator()(const Value& lhs, const Value& rhs) const + { return lhs.GetInt() < rhs.GetInt(); + } }; #if RAPIDJSON_HAS_CXX11_RVALUE_REFS -TEST(Value, Sorting) { - Value::AllocatorType allocator; - Value a(kArrayType); - a.PushBack(5, allocator); - a.PushBack(1, allocator); - a.PushBack(3, allocator); - std::sort(a.Begin(), a.End(), ValueIntComparer()); - EXPECT_EQ(1, a[0].GetInt()); - EXPECT_EQ(3, a[1].GetInt()); - EXPECT_EQ(5, a[2].GetInt()); +TEST(Value, Sorting) +{ Value::AllocatorType allocator; + Value a(kArrayType); + a.PushBack(5, allocator); + a.PushBack(1, allocator); + a.PushBack(3, allocator); + std::sort(a.Begin(), a.End(), ValueIntComparer()); + EXPECT_EQ(1, a[0].GetInt()); + EXPECT_EQ(3, a[1].GetInt()); + EXPECT_EQ(5, a[2].GetInt()); } #endif // http://stackoverflow.com/questions/35222230/ -static void MergeDuplicateKey(Value& v, Value::AllocatorType& a) { - if (v.IsObject()) { - // Convert all key:value into key:[value] - for (Value::MemberIterator itr = v.MemberBegin(); itr != v.MemberEnd(); ++itr) - itr->value = Value(kArrayType).Move().PushBack(itr->value, a); - - // Merge arrays if key is duplicated - for (Value::MemberIterator itr = v.MemberBegin(); itr != v.MemberEnd();) { - Value::MemberIterator itr2 = v.FindMember(itr->name); - if (itr != itr2) { - itr2->value.PushBack(itr->value[0], a); - itr = v.EraseMember(itr); - } - else - ++itr; - } - - // Convert key:[values] back to key:value if there is only one value - for (Value::MemberIterator itr = v.MemberBegin(); itr != v.MemberEnd(); ++itr) { - if (itr->value.Size() == 1) - itr->value = itr->value[0]; - MergeDuplicateKey(itr->value, a); // Recursion on the value - } +static void MergeDuplicateKey(Value& v, Value::AllocatorType& a) +{ if (v.IsObject()) + { // Convert all key:value into key:[value] + for (Value::MemberIterator itr = v.MemberBegin(); itr != v.MemberEnd(); ++itr) + itr->value = Value(kArrayType).Move().PushBack(itr->value, a); + + // Merge arrays if key is duplicated + for (Value::MemberIterator itr = v.MemberBegin(); itr != v.MemberEnd();) + { Value::MemberIterator itr2 = v.FindMember(itr->name); + if (itr != itr2) + { itr2->value.PushBack(itr->value[0], a); + itr = v.EraseMember(itr); + } + else + ++itr; + } + + // Convert key:[values] back to key:value if there is only one value + for (Value::MemberIterator itr = v.MemberBegin(); itr != v.MemberEnd(); ++itr) + { if (itr->value.Size() == 1) + itr->value = itr->value[0]; + MergeDuplicateKey(itr->value, a); // Recursion on the value } - else if (v.IsArray()) - for (Value::ValueIterator itr = v.Begin(); itr != v.End(); ++itr) - MergeDuplicateKey(*itr, a); + } + else if (v.IsArray()) + for (Value::ValueIterator itr = v.Begin(); itr != v.End(); ++itr) + MergeDuplicateKey(*itr, a); } -TEST(Value, MergeDuplicateKey) { - Document d; - d.Parse( - "{" - " \"key1\": {" - " \"a\": \"asdf\"," - " \"b\": \"foo\"," - " \"b\": \"bar\"," - " \"c\": \"fdas\"" - " }" - "}"); - - Document d2; - d2.Parse( - "{" - " \"key1\": {" - " \"a\": \"asdf\"," - " \"b\": [" - " \"foo\"," - " \"bar\"" - " ]," - " \"c\": \"fdas\"" - " }" - "}"); - - EXPECT_NE(d2, d); - MergeDuplicateKey(d, d.GetAllocator()); - EXPECT_EQ(d2, d); +TEST(Value, MergeDuplicateKey) +{ Document d; + d.Parse( + "{" + " \"key1\": {" + " \"a\": \"asdf\"," + " \"b\": \"foo\"," + " \"b\": \"bar\"," + " \"c\": \"fdas\"" + " }" + "}"); + + Document d2; + d2.Parse( + "{" + " \"key1\": {" + " \"a\": \"asdf\"," + " \"b\": [" + " \"foo\"," + " \"bar\"" + " ]," + " \"c\": \"fdas\"" + " }" + "}"); + + EXPECT_NE(d2, d); + MergeDuplicateKey(d, d.GetAllocator()); + EXPECT_EQ(d2, d); } #ifdef __clang__ diff --git a/rapidjson/test/unittest/writertest.cpp b/rapidjson/test/unittest/writertest.cpp index 29f7626092c..cfad40d5035 100644 --- a/rapidjson/test/unittest/writertest.cpp +++ b/rapidjson/test/unittest/writertest.cpp @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "unittest.h" @@ -22,16 +22,16 @@ using namespace rapidjson; -TEST(Writer, Compact) { - StringStream s("{ \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] } "); - StringBuffer buffer; - Writer writer(buffer); - buffer.ShrinkToFit(); - Reader reader; - reader.Parse<0>(s, writer); - EXPECT_STREQ("{\"hello\":\"world\",\"t\":true,\"f\":false,\"n\":null,\"i\":123,\"pi\":3.1416,\"a\":[1,2,3]}", buffer.GetString()); - EXPECT_EQ(77u, buffer.GetSize()); - EXPECT_TRUE(writer.IsComplete()); +TEST(Writer, Compact) +{ StringStream s("{ \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] } "); + StringBuffer buffer; + Writer writer(buffer); + buffer.ShrinkToFit(); + Reader reader; + reader.Parse<0>(s, writer); + EXPECT_STREQ("{\"hello\":\"world\",\"t\":true,\"f\":false,\"n\":null,\"i\":123,\"pi\":3.1416,\"a\":[1,2,3]}", buffer.GetString()); + EXPECT_EQ(77u, buffer.GetSize()); + EXPECT_TRUE(writer.IsComplete()); } // json -> parse -> writer -> json @@ -46,225 +46,227 @@ TEST(Writer, Compact) { EXPECT_TRUE(writer.IsComplete()); \ } -TEST(Writer, Root) { - TEST_ROUNDTRIP("null"); - TEST_ROUNDTRIP("true"); - TEST_ROUNDTRIP("false"); - TEST_ROUNDTRIP("0"); - TEST_ROUNDTRIP("\"foo\""); - TEST_ROUNDTRIP("[]"); - TEST_ROUNDTRIP("{}"); +TEST(Writer, Root) +{ TEST_ROUNDTRIP("null"); + TEST_ROUNDTRIP("true"); + TEST_ROUNDTRIP("false"); + TEST_ROUNDTRIP("0"); + TEST_ROUNDTRIP("\"foo\""); + TEST_ROUNDTRIP("[]"); + TEST_ROUNDTRIP("{}"); } -TEST(Writer, Int) { - TEST_ROUNDTRIP("[-1]"); - TEST_ROUNDTRIP("[-123]"); - TEST_ROUNDTRIP("[-2147483648]"); +TEST(Writer, Int) +{ TEST_ROUNDTRIP("[-1]"); + TEST_ROUNDTRIP("[-123]"); + TEST_ROUNDTRIP("[-2147483648]"); } -TEST(Writer, UInt) { - TEST_ROUNDTRIP("[0]"); - TEST_ROUNDTRIP("[1]"); - TEST_ROUNDTRIP("[123]"); - TEST_ROUNDTRIP("[2147483647]"); - TEST_ROUNDTRIP("[4294967295]"); +TEST(Writer, UInt) +{ TEST_ROUNDTRIP("[0]"); + TEST_ROUNDTRIP("[1]"); + TEST_ROUNDTRIP("[123]"); + TEST_ROUNDTRIP("[2147483647]"); + TEST_ROUNDTRIP("[4294967295]"); } -TEST(Writer, Int64) { - TEST_ROUNDTRIP("[-1234567890123456789]"); - TEST_ROUNDTRIP("[-9223372036854775808]"); +TEST(Writer, Int64) +{ TEST_ROUNDTRIP("[-1234567890123456789]"); + TEST_ROUNDTRIP("[-9223372036854775808]"); } -TEST(Writer, Uint64) { - TEST_ROUNDTRIP("[1234567890123456789]"); - TEST_ROUNDTRIP("[9223372036854775807]"); +TEST(Writer, Uint64) +{ TEST_ROUNDTRIP("[1234567890123456789]"); + TEST_ROUNDTRIP("[9223372036854775807]"); } -TEST(Writer, String) { - TEST_ROUNDTRIP("[\"Hello\"]"); - TEST_ROUNDTRIP("[\"Hello\\u0000World\"]"); - TEST_ROUNDTRIP("[\"\\\"\\\\/\\b\\f\\n\\r\\t\"]"); +TEST(Writer, String) +{ TEST_ROUNDTRIP("[\"Hello\"]"); + TEST_ROUNDTRIP("[\"Hello\\u0000World\"]"); + TEST_ROUNDTRIP("[\"\\\"\\\\/\\b\\f\\n\\r\\t\"]"); #if RAPIDJSON_HAS_STDSTRING - { - StringBuffer buffer; - Writer writer(buffer); - writer.String(std::string("Hello\n")); - EXPECT_STREQ("\"Hello\\n\"", buffer.GetString()); - } + { StringBuffer buffer; + Writer writer(buffer); + writer.String(std::string("Hello\n")); + EXPECT_STREQ("\"Hello\\n\"", buffer.GetString()); + } #endif } -TEST(Writer, ScanWriteUnescapedString) { - const char json[] = "[\" \\\"0123456789ABCDEF\"]"; - // ^ scanning stops here. - char buffer2[sizeof(json) + 32]; - - // Use different offset to test different alignments - for (int i = 0; i < 32; i++) { - char* p = buffer2 + i; - memcpy(p, json, sizeof(json)); - TEST_ROUNDTRIP(p); - } +TEST(Writer, ScanWriteUnescapedString) +{ const char json[] = "[\" \\\"0123456789ABCDEF\"]"; + // ^ scanning stops here. + char buffer2[sizeof(json) + 32]; + + // Use different offset to test different alignments + for (int i = 0; i < 32; i++) + { char* p = buffer2 + i; + memcpy(p, json, sizeof(json)); + TEST_ROUNDTRIP(p); + } } -TEST(Writer, Double) { - TEST_ROUNDTRIP("[1.2345,1.2345678,0.123456789012,1234567.8]"); - TEST_ROUNDTRIP("0.0"); - TEST_ROUNDTRIP("-0.0"); // Issue #289 - TEST_ROUNDTRIP("1e30"); - TEST_ROUNDTRIP("1.0"); - TEST_ROUNDTRIP("5e-324"); // Min subnormal positive double - TEST_ROUNDTRIP("2.225073858507201e-308"); // Max subnormal positive double - TEST_ROUNDTRIP("2.2250738585072014e-308"); // Min normal positive double - TEST_ROUNDTRIP("1.7976931348623157e308"); // Max double +TEST(Writer, Double) +{ TEST_ROUNDTRIP("[1.2345,1.2345678,0.123456789012,1234567.8]"); + TEST_ROUNDTRIP("0.0"); + TEST_ROUNDTRIP("-0.0"); // Issue #289 + TEST_ROUNDTRIP("1e30"); + TEST_ROUNDTRIP("1.0"); + TEST_ROUNDTRIP("5e-324"); // Min subnormal positive double + TEST_ROUNDTRIP("2.225073858507201e-308"); // Max subnormal positive double + TEST_ROUNDTRIP("2.2250738585072014e-308"); // Min normal positive double + TEST_ROUNDTRIP("1.7976931348623157e308"); // Max double } // UTF8 -> TargetEncoding -> UTF8 template -void TestTranscode(const char* json) { - StringStream s(json); - GenericStringBuffer buffer; - Writer, UTF8<>, TargetEncoding> writer(buffer); - Reader reader; - reader.Parse(s, writer); - - StringBuffer buffer2; - Writer writer2(buffer2); - GenericReader > reader2; - GenericStringStream s2(buffer.GetString()); - reader2.Parse(s2, writer2); - - EXPECT_STREQ(json, buffer2.GetString()); +void TestTranscode(const char* json) +{ StringStream s(json); + GenericStringBuffer buffer; + Writer, UTF8<>, TargetEncoding> writer(buffer); + Reader reader; + reader.Parse(s, writer); + + StringBuffer buffer2; + Writer writer2(buffer2); + GenericReader > reader2; + GenericStringStream s2(buffer.GetString()); + reader2.Parse(s2, writer2); + + EXPECT_STREQ(json, buffer2.GetString()); } -TEST(Writer, Transcode) { - const char json[] = "{\"hello\":\"world\",\"t\":true,\"f\":false,\"n\":null,\"i\":123,\"pi\":3.1416,\"a\":[1,2,3],\"dollar\":\"\x24\",\"cents\":\"\xC2\xA2\",\"euro\":\"\xE2\x82\xAC\",\"gclef\":\"\xF0\x9D\x84\x9E\"}"; +TEST(Writer, Transcode) +{ const char json[] = "{\"hello\":\"world\",\"t\":true,\"f\":false,\"n\":null,\"i\":123,\"pi\":3.1416,\"a\":[1,2,3],\"dollar\":\"\x24\",\"cents\":\"\xC2\xA2\",\"euro\":\"\xE2\x82\xAC\",\"gclef\":\"\xF0\x9D\x84\x9E\"}"; - // UTF8 -> UTF16 -> UTF8 - TestTranscode >(json); + // UTF8 -> UTF16 -> UTF8 + TestTranscode >(json); - // UTF8 -> ASCII -> UTF8 - TestTranscode >(json); + // UTF8 -> ASCII -> UTF8 + TestTranscode >(json); - // UTF8 -> UTF16 -> UTF8 - TestTranscode >(json); + // UTF8 -> UTF16 -> UTF8 + TestTranscode >(json); - // UTF8 -> UTF32 -> UTF8 - TestTranscode >(json); + // UTF8 -> UTF32 -> UTF8 + TestTranscode >(json); - // UTF8 -> AutoUTF -> UTF8 - UTFType types[] = { kUTF8, kUTF16LE , kUTF16BE, kUTF32LE , kUTF32BE }; - for (size_t i = 0; i < 5; i++) { - StringStream s(json); - MemoryBuffer buffer; - AutoUTFOutputStream os(buffer, types[i], true); - Writer, UTF8<>, AutoUTF > writer(os); - Reader reader; - reader.Parse(s, writer); + // UTF8 -> AutoUTF -> UTF8 + UTFType types[] = { kUTF8, kUTF16LE , kUTF16BE, kUTF32LE , kUTF32BE }; + for (size_t i = 0; i < 5; i++) + { StringStream s(json); + MemoryBuffer buffer; + AutoUTFOutputStream os(buffer, types[i], true); + Writer, UTF8<>, AutoUTF > writer(os); + Reader reader; + reader.Parse(s, writer); - StringBuffer buffer2; - Writer writer2(buffer2); - GenericReader, UTF8<> > reader2; - MemoryStream s2(buffer.GetBuffer(), buffer.GetSize()); - AutoUTFInputStream is(s2); - reader2.Parse(is, writer2); + StringBuffer buffer2; + Writer writer2(buffer2); + GenericReader, UTF8<> > reader2; + MemoryStream s2(buffer.GetBuffer(), buffer.GetSize()); + AutoUTFInputStream is(s2); + reader2.Parse(is, writer2); - EXPECT_STREQ(json, buffer2.GetString()); - } + EXPECT_STREQ(json, buffer2.GetString()); + } } #include -class OStreamWrapper { +class OStreamWrapper +{ public: - typedef char Ch; + typedef char Ch; - OStreamWrapper(std::ostream& os) : os_(os) {} + OStreamWrapper(std::ostream& os) : os_(os) {} - Ch Peek() const { assert(false); return '\0'; } - Ch Take() { assert(false); return '\0'; } - size_t Tell() const { return 0; } + Ch Peek() const { assert(false); return '\0'; } + Ch Take() { assert(false); return '\0'; } + size_t Tell() const { return 0; } - Ch* PutBegin() { assert(false); return 0; } - void Put(Ch c) { os_.put(c); } - void Flush() { os_.flush(); } - size_t PutEnd(Ch*) { assert(false); return 0; } + Ch* PutBegin() { assert(false); return 0; } + void Put(Ch c) { os_.put(c); } + void Flush() { os_.flush(); } + size_t PutEnd(Ch*) { assert(false); return 0; } private: - OStreamWrapper(const OStreamWrapper&); - OStreamWrapper& operator=(const OStreamWrapper&); + OStreamWrapper(const OStreamWrapper&); + OStreamWrapper& operator=(const OStreamWrapper&); - std::ostream& os_; + std::ostream& os_; }; -TEST(Writer, OStreamWrapper) { - StringStream s("{ \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3], \"u64\": 1234567890123456789, \"i64\":-1234567890123456789 } "); - - std::stringstream ss; - OStreamWrapper os(ss); - - Writer writer(os); +TEST(Writer, OStreamWrapper) +{ StringStream s("{ \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3], \"u64\": 1234567890123456789, \"i64\":-1234567890123456789 } "); - Reader reader; - reader.Parse<0>(s, writer); - - std::string actual = ss.str(); - EXPECT_STREQ("{\"hello\":\"world\",\"t\":true,\"f\":false,\"n\":null,\"i\":123,\"pi\":3.1416,\"a\":[1,2,3],\"u64\":1234567890123456789,\"i64\":-1234567890123456789}", actual.c_str()); + std::stringstream ss; + OStreamWrapper os(ss); + + Writer writer(os); + + Reader reader; + reader.Parse<0>(s, writer); + + std::string actual = ss.str(); + EXPECT_STREQ("{\"hello\":\"world\",\"t\":true,\"f\":false,\"n\":null,\"i\":123,\"pi\":3.1416,\"a\":[1,2,3],\"u64\":1234567890123456789,\"i64\":-1234567890123456789}", actual.c_str()); } -TEST(Writer, AssertRootMayBeAnyValue) { +TEST(Writer, AssertRootMayBeAnyValue) +{ #define T(x)\ {\ StringBuffer buffer;\ Writer writer(buffer);\ EXPECT_TRUE(x);\ } - T(writer.Bool(false)); - T(writer.Bool(true)); - T(writer.Null()); - T(writer.Int(0)); - T(writer.Uint(0)); - T(writer.Int64(0)); - T(writer.Uint64(0)); - T(writer.Double(0)); - T(writer.String("foo")); + T(writer.Bool(false)); + T(writer.Bool(true)); + T(writer.Null()); + T(writer.Int(0)); + T(writer.Uint(0)); + T(writer.Int64(0)); + T(writer.Uint64(0)); + T(writer.Double(0)); + T(writer.String("foo")); #undef T } -TEST(Writer, AssertIncorrectObjectLevel) { - StringBuffer buffer; - Writer writer(buffer); - writer.StartObject(); - writer.EndObject(); - ASSERT_THROW(writer.EndObject(), AssertException); +TEST(Writer, AssertIncorrectObjectLevel) +{ StringBuffer buffer; + Writer writer(buffer); + writer.StartObject(); + writer.EndObject(); + ASSERT_THROW(writer.EndObject(), AssertException); } -TEST(Writer, AssertIncorrectArrayLevel) { - StringBuffer buffer; - Writer writer(buffer); - writer.StartArray(); - writer.EndArray(); - ASSERT_THROW(writer.EndArray(), AssertException); +TEST(Writer, AssertIncorrectArrayLevel) +{ StringBuffer buffer; + Writer writer(buffer); + writer.StartArray(); + writer.EndArray(); + ASSERT_THROW(writer.EndArray(), AssertException); } -TEST(Writer, AssertIncorrectEndObject) { - StringBuffer buffer; - Writer writer(buffer); - writer.StartObject(); - ASSERT_THROW(writer.EndArray(), AssertException); +TEST(Writer, AssertIncorrectEndObject) +{ StringBuffer buffer; + Writer writer(buffer); + writer.StartObject(); + ASSERT_THROW(writer.EndArray(), AssertException); } -TEST(Writer, AssertIncorrectEndArray) { - StringBuffer buffer; - Writer writer(buffer); - writer.StartObject(); - ASSERT_THROW(writer.EndArray(), AssertException); +TEST(Writer, AssertIncorrectEndArray) +{ StringBuffer buffer; + Writer writer(buffer); + writer.StartObject(); + ASSERT_THROW(writer.EndArray(), AssertException); } -TEST(Writer, AssertObjectKeyNotString) { +TEST(Writer, AssertObjectKeyNotString) +{ #define T(x)\ {\ StringBuffer buffer;\ @@ -272,70 +274,71 @@ TEST(Writer, AssertObjectKeyNotString) { writer.StartObject();\ ASSERT_THROW(x, AssertException); \ } - T(writer.Bool(false)); - T(writer.Bool(true)); - T(writer.Null()); - T(writer.Int(0)); - T(writer.Uint(0)); - T(writer.Int64(0)); - T(writer.Uint64(0)); - T(writer.Double(0)); - T(writer.StartObject()); - T(writer.StartArray()); + T(writer.Bool(false)); + T(writer.Bool(true)); + T(writer.Null()); + T(writer.Int(0)); + T(writer.Uint(0)); + T(writer.Int64(0)); + T(writer.Uint64(0)); + T(writer.Double(0)); + T(writer.StartObject()); + T(writer.StartArray()); #undef T } -TEST(Writer, AssertMultipleRoot) { - StringBuffer buffer; - Writer writer(buffer); +TEST(Writer, AssertMultipleRoot) +{ StringBuffer buffer; + Writer writer(buffer); - writer.StartObject(); - writer.EndObject(); - ASSERT_THROW(writer.StartObject(), AssertException); + writer.StartObject(); + writer.EndObject(); + ASSERT_THROW(writer.StartObject(), AssertException); - writer.Reset(buffer); - writer.Null(); - ASSERT_THROW(writer.Int(0), AssertException); + writer.Reset(buffer); + writer.Null(); + ASSERT_THROW(writer.Int(0), AssertException); - writer.Reset(buffer); - writer.String("foo"); - ASSERT_THROW(writer.StartArray(), AssertException); + writer.Reset(buffer); + writer.String("foo"); + ASSERT_THROW(writer.StartArray(), AssertException); - writer.Reset(buffer); - writer.StartArray(); - writer.EndArray(); - //ASSERT_THROW(writer.Double(3.14), AssertException); + writer.Reset(buffer); + writer.StartArray(); + writer.EndArray(); + //ASSERT_THROW(writer.Double(3.14), AssertException); } -TEST(Writer, RootObjectIsComplete) { - StringBuffer buffer; - Writer writer(buffer); - EXPECT_FALSE(writer.IsComplete()); - writer.StartObject(); - EXPECT_FALSE(writer.IsComplete()); - writer.String("foo"); - EXPECT_FALSE(writer.IsComplete()); - writer.Int(1); - EXPECT_FALSE(writer.IsComplete()); - writer.EndObject(); - EXPECT_TRUE(writer.IsComplete()); +TEST(Writer, RootObjectIsComplete) +{ StringBuffer buffer; + Writer writer(buffer); + EXPECT_FALSE(writer.IsComplete()); + writer.StartObject(); + EXPECT_FALSE(writer.IsComplete()); + writer.String("foo"); + EXPECT_FALSE(writer.IsComplete()); + writer.Int(1); + EXPECT_FALSE(writer.IsComplete()); + writer.EndObject(); + EXPECT_TRUE(writer.IsComplete()); } -TEST(Writer, RootArrayIsComplete) { - StringBuffer buffer; - Writer writer(buffer); - EXPECT_FALSE(writer.IsComplete()); - writer.StartArray(); - EXPECT_FALSE(writer.IsComplete()); - writer.String("foo"); - EXPECT_FALSE(writer.IsComplete()); - writer.Int(1); - EXPECT_FALSE(writer.IsComplete()); - writer.EndArray(); - EXPECT_TRUE(writer.IsComplete()); +TEST(Writer, RootArrayIsComplete) +{ StringBuffer buffer; + Writer writer(buffer); + EXPECT_FALSE(writer.IsComplete()); + writer.StartArray(); + EXPECT_FALSE(writer.IsComplete()); + writer.String("foo"); + EXPECT_FALSE(writer.IsComplete()); + writer.Int(1); + EXPECT_FALSE(writer.IsComplete()); + writer.EndArray(); + EXPECT_TRUE(writer.IsComplete()); } -TEST(Writer, RootValueIsComplete) { +TEST(Writer, RootValueIsComplete) +{ #define T(x)\ {\ StringBuffer buffer;\ @@ -344,154 +347,141 @@ TEST(Writer, RootValueIsComplete) { x; \ EXPECT_TRUE(writer.IsComplete()); \ } - T(writer.Null()); - T(writer.Bool(true)); - T(writer.Bool(false)); - T(writer.Int(0)); - T(writer.Uint(0)); - T(writer.Int64(0)); - T(writer.Uint64(0)); - T(writer.Double(0)); - T(writer.String("")); + T(writer.Null()); + T(writer.Bool(true)); + T(writer.Bool(false)); + T(writer.Int(0)); + T(writer.Uint(0)); + T(writer.Int64(0)); + T(writer.Uint64(0)); + T(writer.Double(0)); + T(writer.String("")); #undef T } -TEST(Writer, InvalidEncoding) { - // Fail in decoding invalid UTF-8 sequence http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt - { - GenericStringBuffer > buffer; - Writer >, UTF8<>, UTF16<> > writer(buffer); - writer.StartArray(); - EXPECT_FALSE(writer.String("\xfe")); - EXPECT_FALSE(writer.String("\xff")); - EXPECT_FALSE(writer.String("\xfe\xfe\xff\xff")); - writer.EndArray(); - } - - // Fail in encoding - { - StringBuffer buffer; - Writer > writer(buffer); - static const UTF32<>::Ch s[] = { 0x110000, 0 }; // Out of U+0000 to U+10FFFF - EXPECT_FALSE(writer.String(s)); - } - - // Fail in unicode escaping in ASCII output - { - StringBuffer buffer; - Writer, ASCII<> > writer(buffer); - static const UTF32<>::Ch s[] = { 0x110000, 0 }; // Out of U+0000 to U+10FFFF - EXPECT_FALSE(writer.String(s)); - } +TEST(Writer, InvalidEncoding) +{ // Fail in decoding invalid UTF-8 sequence http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt + { GenericStringBuffer > buffer; + Writer >, UTF8<>, UTF16<> > writer(buffer); + writer.StartArray(); + EXPECT_FALSE(writer.String("\xfe")); + EXPECT_FALSE(writer.String("\xff")); + EXPECT_FALSE(writer.String("\xfe\xfe\xff\xff")); + writer.EndArray(); + } + + // Fail in encoding + { StringBuffer buffer; + Writer > writer(buffer); + static const UTF32<>::Ch s[] = { 0x110000, 0 }; // Out of U+0000 to U+10FFFF + EXPECT_FALSE(writer.String(s)); + } + + // Fail in unicode escaping in ASCII output + { StringBuffer buffer; + Writer, ASCII<> > writer(buffer); + static const UTF32<>::Ch s[] = { 0x110000, 0 }; // Out of U+0000 to U+10FFFF + EXPECT_FALSE(writer.String(s)); + } } -TEST(Writer, ValidateEncoding) { - { - StringBuffer buffer; - Writer, UTF8<>, CrtAllocator, kWriteValidateEncodingFlag> writer(buffer); - writer.StartArray(); - EXPECT_TRUE(writer.String("\x24")); // Dollar sign U+0024 - EXPECT_TRUE(writer.String("\xC2\xA2")); // Cents sign U+00A2 - EXPECT_TRUE(writer.String("\xE2\x82\xAC")); // Euro sign U+20AC - EXPECT_TRUE(writer.String("\xF0\x9D\x84\x9E")); // G clef sign U+1D11E - writer.EndArray(); - EXPECT_STREQ("[\"\x24\",\"\xC2\xA2\",\"\xE2\x82\xAC\",\"\xF0\x9D\x84\x9E\"]", buffer.GetString()); - } +TEST(Writer, ValidateEncoding) +{ + { StringBuffer buffer; + Writer, UTF8<>, CrtAllocator, kWriteValidateEncodingFlag> writer(buffer); + writer.StartArray(); + EXPECT_TRUE(writer.String("\x24")); // Dollar sign U+0024 + EXPECT_TRUE(writer.String("\xC2\xA2")); // Cents sign U+00A2 + EXPECT_TRUE(writer.String("\xE2\x82\xAC")); // Euro sign U+20AC + EXPECT_TRUE(writer.String("\xF0\x9D\x84\x9E")); // G clef sign U+1D11E + writer.EndArray(); + EXPECT_STREQ("[\"\x24\",\"\xC2\xA2\",\"\xE2\x82\xAC\",\"\xF0\x9D\x84\x9E\"]", buffer.GetString()); + } - // Fail in decoding invalid UTF-8 sequence http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt - { - StringBuffer buffer; - Writer, UTF8<>, CrtAllocator, kWriteValidateEncodingFlag> writer(buffer); - writer.StartArray(); - EXPECT_FALSE(writer.String("\xfe")); - EXPECT_FALSE(writer.String("\xff")); - EXPECT_FALSE(writer.String("\xfe\xfe\xff\xff")); - writer.EndArray(); - } + // Fail in decoding invalid UTF-8 sequence http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt + { StringBuffer buffer; + Writer, UTF8<>, CrtAllocator, kWriteValidateEncodingFlag> writer(buffer); + writer.StartArray(); + EXPECT_FALSE(writer.String("\xfe")); + EXPECT_FALSE(writer.String("\xff")); + EXPECT_FALSE(writer.String("\xfe\xfe\xff\xff")); + writer.EndArray(); + } } -TEST(Writer, InvalidEventSequence) { - // {] - { - StringBuffer buffer; - Writer writer(buffer); - writer.StartObject(); - EXPECT_THROW(writer.EndArray(), AssertException); - EXPECT_FALSE(writer.IsComplete()); - } +TEST(Writer, InvalidEventSequence) +{ // {] + { StringBuffer buffer; + Writer writer(buffer); + writer.StartObject(); + EXPECT_THROW(writer.EndArray(), AssertException); + EXPECT_FALSE(writer.IsComplete()); + } - // [} - { - StringBuffer buffer; - Writer writer(buffer); - writer.StartArray(); - EXPECT_THROW(writer.EndObject(), AssertException); - EXPECT_FALSE(writer.IsComplete()); - } + // [} + { StringBuffer buffer; + Writer writer(buffer); + writer.StartArray(); + EXPECT_THROW(writer.EndObject(), AssertException); + EXPECT_FALSE(writer.IsComplete()); + } - // { 1: - { - StringBuffer buffer; - Writer writer(buffer); - writer.StartObject(); - EXPECT_THROW(writer.Int(1), AssertException); - EXPECT_FALSE(writer.IsComplete()); - } + // { 1: + { StringBuffer buffer; + Writer writer(buffer); + writer.StartObject(); + EXPECT_THROW(writer.Int(1), AssertException); + EXPECT_FALSE(writer.IsComplete()); + } } -TEST(Writer, NaN) { - double nan = std::numeric_limits::quiet_NaN(); - - EXPECT_TRUE(internal::Double(nan).IsNan()); - StringBuffer buffer; - { - Writer writer(buffer); - EXPECT_FALSE(writer.Double(nan)); - } - { - Writer, UTF8<>, CrtAllocator, kWriteNanAndInfFlag> writer(buffer); - EXPECT_TRUE(writer.Double(nan)); - EXPECT_STREQ("NaN", buffer.GetString()); - } - GenericStringBuffer > buffer2; - Writer > > writer2(buffer2); - EXPECT_FALSE(writer2.Double(nan)); +TEST(Writer, NaN) +{ double nan = std::numeric_limits::quiet_NaN(); + + EXPECT_TRUE(internal::Double(nan).IsNan()); + StringBuffer buffer; + { Writer writer(buffer); + EXPECT_FALSE(writer.Double(nan)); + } + { Writer, UTF8<>, CrtAllocator, kWriteNanAndInfFlag> writer(buffer); + EXPECT_TRUE(writer.Double(nan)); + EXPECT_STREQ("NaN", buffer.GetString()); + } + GenericStringBuffer > buffer2; + Writer > > writer2(buffer2); + EXPECT_FALSE(writer2.Double(nan)); } -TEST(Writer, Inf) { - double inf = std::numeric_limits::infinity(); - - EXPECT_TRUE(internal::Double(inf).IsInf()); - StringBuffer buffer; - { - Writer writer(buffer); - EXPECT_FALSE(writer.Double(inf)); - } - { - Writer writer(buffer); - EXPECT_FALSE(writer.Double(-inf)); - } - { - Writer, UTF8<>, CrtAllocator, kWriteNanAndInfFlag> writer(buffer); - EXPECT_TRUE(writer.Double(inf)); - } - { - Writer, UTF8<>, CrtAllocator, kWriteNanAndInfFlag> writer(buffer); - EXPECT_TRUE(writer.Double(-inf)); - } - EXPECT_STREQ("Infinity-Infinity", buffer.GetString()); +TEST(Writer, Inf) +{ double inf = std::numeric_limits::infinity(); + + EXPECT_TRUE(internal::Double(inf).IsInf()); + StringBuffer buffer; + { Writer writer(buffer); + EXPECT_FALSE(writer.Double(inf)); + } + { Writer writer(buffer); + EXPECT_FALSE(writer.Double(-inf)); + } + { Writer, UTF8<>, CrtAllocator, kWriteNanAndInfFlag> writer(buffer); + EXPECT_TRUE(writer.Double(inf)); + } + { Writer, UTF8<>, CrtAllocator, kWriteNanAndInfFlag> writer(buffer); + EXPECT_TRUE(writer.Double(-inf)); + } + EXPECT_STREQ("Infinity-Infinity", buffer.GetString()); } -TEST(Writer, RawValue) { - StringBuffer buffer; - Writer writer(buffer); - writer.StartObject(); - writer.Key("a"); - writer.Int(1); - writer.Key("raw"); - const char json[] = "[\"Hello\\nWorld\", 123.456]"; - writer.RawValue(json, strlen(json), kArrayType); - writer.EndObject(); - EXPECT_TRUE(writer.IsComplete()); - EXPECT_STREQ("{\"a\":1,\"raw\":[\"Hello\\nWorld\", 123.456]}", buffer.GetString()); +TEST(Writer, RawValue) +{ StringBuffer buffer; + Writer writer(buffer); + writer.StartObject(); + writer.Key("a"); + writer.Int(1); + writer.Key("raw"); + const char json[] = "[\"Hello\\nWorld\", 123.456]"; + writer.RawValue(json, strlen(json), kArrayType); + writer.EndObject(); + EXPECT_TRUE(writer.IsComplete()); + EXPECT_STREQ("{\"a\":1,\"raw\":[\"Hello\\nWorld\", 123.456]}", buffer.GetString()); } diff --git a/vowpalwabbit/OjaNewton.cc b/vowpalwabbit/OjaNewton.cc index 4eb4c7a5171..1e4724a98f2 100644 --- a/vowpalwabbit/OjaNewton.cc +++ b/vowpalwabbit/OjaNewton.cc @@ -14,551 +14,544 @@ using namespace LEARNER; #define NORM2 (m+1) -struct update_data { - struct OjaNewton *ON; - float g; - float sketch_cnt; - float norm2_x; - float *Zx; - float *AZx; - float *delta; - float bdelta; - float prediction; +struct update_data +{ struct OjaNewton *ON; + float g; + float sketch_cnt; + float norm2_x; + float *Zx; + float *AZx; + float *delta; + float bdelta; + float prediction; }; -struct OjaNewton { - vw* all; - int m; - int epoch_size; - float alpha; - int cnt; - int t; - - float *ev; - float *b; - float *D; - float **A; - float **K; - - float *zv; - float *vv; - float *tmp; - - example **buffer; - float *weight_buffer; - struct update_data data; - - float learning_rate_cnt; - bool normalize; - bool random_init; - - void initialize_Z() //TODO: use weight_paramters::set_default for initialization - { weight_parameters& weights = all->weights; - if (normalize) { // initialize normalization part - for (weight_parameters::iterator iter = weights.begin(); iter != weights.end(); ++iter) - (&(*iter))[NORM2] = 0.1f; - } - if(!random_init) { - // simple initialization - weight_parameters::iterator iter = weights.begin() + 1; - for (int i = 1; i <= m; ++i, ++iter) - { weights_iterator_iterator j = iter.begin() + i; - *j = 1.f; - } - } - else { - // more complicated initialization: orthgonal basis of a random matrix - - const double PI2 = 2.f * 3.1415927f; - for (weight_parameters::iterator i = weights.begin(); i != weights.end(); ++i) - for (weights_iterator_iterator j = i.begin() + 1; j != i.end(m + 1); ++j) - { float r1 = frand48(); - float r2 = frand48(); - *j = sqrt(-2.f * log(r1)) * (float)cos(PI2 * r2); - } - - // Gram-Schmidt - for (int j = 1; j <= m; j++) { - for (int k = 1; k <= j - 1; k++) { - double tmp = 0; - - weight_parameters::iterator w = weights.begin(); - for (; w != weights.end(); ++w) - tmp += (&(*w))[j] * (&(*w))[k]; - - w = weights.begin(); - for (; w != weights.end(); ++w) - (&(*w))[j] -= (float)tmp * (&(*w))[k]; - } - double norm = 0; - weight_parameters::iterator w = weights.begin(); - for (; w != weights.end(); ++w) - norm += (&(*w))[j] * (&(*w))[j]; - norm = sqrt(norm); - w = weights.begin(); - for (; w != weights.end(); ++w) - (&(*w))[j] /= (float)norm; - } - } +struct OjaNewton +{ vw* all; + int m; + int epoch_size; + float alpha; + int cnt; + int t; + + float *ev; + float *b; + float *D; + float **A; + float **K; + + float *zv; + float *vv; + float *tmp; + + example **buffer; + float *weight_buffer; + struct update_data data; + + float learning_rate_cnt; + bool normalize; + bool random_init; + + void initialize_Z() //TODO: use weight_paramters::set_default for initialization + { weight_parameters& weights = all->weights; + if (normalize) // initialize normalization part + { for (weight_parameters::iterator iter = weights.begin(); iter != weights.end(); ++iter) + (&(*iter))[NORM2] = 0.1f; } - - void compute_AZx() - { - for (int i = 1; i <= m; i++) { - data.AZx[i] = 0; - for (int j = 1; j <= i; j++) { - data.AZx[i] += A[i][j] * data.Zx[j]; - } - } + if(!random_init) + { // simple initialization + weight_parameters::iterator iter = weights.begin() + 1; + for (int i = 1; i <= m; ++i, ++iter) + { weights_iterator_iterator j = iter.begin() + i; + *j = 1.f; + } } - - void update_eigenvalues() - { - for (int i = 1; i <= m; i++) { - float gamma = fmin(learning_rate_cnt / t, 1.f); - float tmp = data.AZx[i] * data.sketch_cnt; - - if (t == 1) { - ev[i] = gamma * tmp * tmp; - } - else { - ev[i] = (1 - gamma) * t * ev[i] / (t - 1) + gamma * t * tmp * tmp; - } + else + { // more complicated initialization: orthgonal basis of a random matrix + + const double PI2 = 2.f * 3.1415927f; + for (weight_parameters::iterator i = weights.begin(); i != weights.end(); ++i) + for (weights_iterator_iterator j = i.begin() + 1; j != i.end(m + 1); ++j) + { float r1 = frand48(); + float r2 = frand48(); + *j = sqrt(-2.f * log(r1)) * (float)cos(PI2 * r2); } - } - void compute_delta() - { - data.bdelta = 0; - for (int i = 1; i <= m; i++) { - float gamma = fmin(learning_rate_cnt / t, 1.f); - - // if different learning rates are used - /*data.delta[i] = gamma * data.AZx[i] * data.sketch_cnt; - for (int j = 1; j < i; j++) { - data.delta[i] -= A[i][j] * data.delta[j]; - } - data.delta[i] /= A[i][i];*/ - - // if a same learning rate is used - data.delta[i] = gamma * data.Zx[i] * data.sketch_cnt; - - data.bdelta += data.delta[i] * b[i]; - } - } + // Gram-Schmidt + for (int j = 1; j <= m; j++) + { for (int k = 1; k <= j - 1; k++) + { double tmp = 0; - void update_K() - { - float tmp = data.norm2_x * data.sketch_cnt * data.sketch_cnt; - for (int i = 1; i <= m; i++) { - for (int j = 1; j <= m; j++) { - K[i][j] += data.delta[i] * data.Zx[j] * data.sketch_cnt; - K[i][j] += data.delta[j] * data.Zx[i] * data.sketch_cnt; - K[i][j] += data.delta[i] * data.delta[j] * tmp; - } - } - } + weight_parameters::iterator w = weights.begin(); + for (; w != weights.end(); ++w) + tmp += (&(*w))[j] * (&(*w))[k]; - void update_A() - { - for (int i = 1; i <= m; i++) { - - for (int j = 1; j < i; j++) { - zv[j] = 0; - for (int k = 1; k <= i; k++) { - zv[j] += A[i][k] * K[k][j]; - } - } - - for (int j = 1; j < i; j++) { - vv[j] = 0; - for (int k = 1; k <= j; k++) { - vv[j] += A[j][k] * zv[k]; - } - } - - for (int j = 1; j < i; j++) { - for (int k = j; k < i; k++) { - A[i][j] -= vv[k] * A[k][j]; - } - } - - float norm = 0; - for (int j = 1; j <= i; j++) { - float temp = 0; - for (int k = 1; k <= i; k++) { - temp += K[j][k] * A[i][k]; - } - norm += A[i][j]*temp; - } - norm = sqrtf(norm); - - for (int j = 1; j <= i; j++) { - A[i][j] /= norm; - } + w = weights.begin(); + for (; w != weights.end(); ++w) + (&(*w))[j] -= (float)tmp * (&(*w))[k]; } + double norm = 0; + weight_parameters::iterator w = weights.begin(); + for (; w != weights.end(); ++w) + norm += (&(*w))[j] * (&(*w))[j]; + norm = sqrt(norm); + w = weights.begin(); + for (; w != weights.end(); ++w) + (&(*w))[j] /= (float)norm; + } } - - void update_b() - { - for (int j = 1; j <= m; j++) { - float tmp = 0; - for (int i = j; i <= m; i++) { - tmp += ev[i] * data.AZx[i] * A[i][j] / (alpha * (alpha + ev[i])); - } - b[j] += tmp * data.g; - } + } + + void compute_AZx() + { for (int i = 1; i <= m; i++) + { data.AZx[i] = 0; + for (int j = 1; j <= i; j++) + { data.AZx[i] += A[i][j] * data.Zx[j]; + } } + } + + void update_eigenvalues() + { for (int i = 1; i <= m; i++) + { float gamma = fmin(learning_rate_cnt / t, 1.f); + float tmp = data.AZx[i] * data.sketch_cnt; + + if (t == 1) + { ev[i] = gamma * tmp * tmp; + } + else + { ev[i] = (1 - gamma) * t * ev[i] / (t - 1) + gamma * t * tmp * tmp; + } + } + } - void update_D() - { - for (int j = 1; j <= m; j++) { - float scale = fabs(A[j][j]); - for (int i = j+1; i <= m; i++) - scale = fmin(fabs(A[i][j]), scale); - if (scale < 1e-10) continue; - for (int i = 1; i <= m; i++) { - A[i][j] /= scale; - K[j][i] *= scale; - K[i][j] *= scale; - } - b[j] /= scale; - D[j] *= scale; - //printf("D[%d] = %f\n", j, D[j]); - } + void compute_delta() + { data.bdelta = 0; + for (int i = 1; i <= m; i++) + { float gamma = fmin(learning_rate_cnt / t, 1.f); + + // if different learning rates are used + /*data.delta[i] = gamma * data.AZx[i] * data.sketch_cnt; + for (int j = 1; j < i; j++) { + data.delta[i] -= A[i][j] * data.delta[j]; + } + data.delta[i] /= A[i][i];*/ + + // if a same learning rate is used + data.delta[i] = gamma * data.Zx[i] * data.sketch_cnt; + + data.bdelta += data.delta[i] * b[i]; + } + } + + void update_K() + { float tmp = data.norm2_x * data.sketch_cnt * data.sketch_cnt; + for (int i = 1; i <= m; i++) + { for (int j = 1; j <= m; j++) + { K[i][j] += data.delta[i] * data.Zx[j] * data.sketch_cnt; + K[i][j] += data.delta[j] * data.Zx[i] * data.sketch_cnt; + K[i][j] += data.delta[i] * data.delta[j] * tmp; + } } + } - void check() + void update_A() + { for (int i = 1; i <= m; i++) { - double max_norm = 0; - for (int i = 1; i <= m; i++) - for (int j = i; j <= m ;j++) - max_norm = fmax(max_norm, fabs(K[i][j])); - //printf("|K| = %f\n", max_norm); - if (max_norm < 1e7) return; - - // implicit -> explicit representation - // printf("begin conversion: t = %d, norm(K) = %f\n", t, max_norm); - - // first step: K <- AKA' - - // K <- AK - for (int j = 1; j <= m; j++) { - memset(tmp, 0, sizeof(double) * (m+1)); - - for (int i = 1; i <= m; i++) { - for (int h = 1; h <= m; h++) { - tmp[i] += A[i][h] * K[h][j]; - } - } - - for (int i = 1; i <= m; i++) - K[i][j] = tmp[i]; - } - // K <- KA' - for (int i = 1; i <= m; i++) { - memset(tmp, 0, sizeof(double) * (m+1)); - - for (int j = 1; j <= m; j++) - for (int h = 1; h <= m; h++) - tmp[j] += K[i][h] * A[j][h]; - - for (int j = 1; j <= m; j++) { - K[i][j] = tmp[j]; - } + + for (int j = 1; j < i; j++) + { zv[j] = 0; + for (int k = 1; k <= i; k++) + { zv[j] += A[i][k] * K[k][j]; } - - //second step: w[0] <- w[0] + (DZ)'b, b <- 0. - - uint32_t length = 1 << all->num_bits; - //size_t stride_shift = all->stride_shift; - weight_parameters& weights = all->weights; - for (weight_parameters::iterator i = weights.begin(); i != weights.end(); ++i) - { weights_iterator_iterator w_j = i.begin() + 1; - for (int j = 1; j <= m; ++j, ++w_j) - *i += *w_j * b[j] * D[j]; - } - memset(b, 0, sizeof(double) * (m+1)); - - //third step: Z <- ADZ, A, D <- Identity - - //double norm = 0; - weight_parameters::iterator iter = weights.begin(); - for (uint32_t i = 0; i < length; ++i, ++iter) { - memset(tmp, 0, sizeof(float) * (m+1)); - for (int j = 1; j <= m; j++) - { weights_iterator_iterator w_k = iter.begin() + 1; - for (int h = 1; h <= m; ++h, ++w_k) - tmp[j] += A[j][h] * D[h] * (*w_k); - } - weights_iterator_iterator w_j = iter.begin() + 1; - for (int j = 1; j <= m; ++j, ++w_j) { - //norm = max(norm, fabs(tmp[j])); - *w_j = tmp[j]; - } + } + + for (int j = 1; j < i; j++) + { vv[j] = 0; + for (int k = 1; k <= j; k++) + { vv[j] += A[j][k] * zv[k]; } - //printf("|Z| = %f\n", norm); + } - for (int i = 1; i <= m; i++) { - memset(A[i], 0, sizeof(double) * (m+1)); - D[i] = 1; - A[i][i] = 1; + for (int j = 1; j < i; j++) + { for (int k = j; k < i; k++) + { A[i][j] -= vv[k] * A[k][j]; } - } -}; + } -void keep_example(vw& all, OjaNewton& ON, example& ec) { - output_and_account_example(all, ec); -} + float norm = 0; + for (int j = 1; j <= i; j++) + { float temp = 0; + for (int k = 1; k <= i; k++) + { temp += K[j][k] * A[i][k]; + } + norm += A[i][j]*temp; + } + norm = sqrtf(norm); -void finish(OjaNewton& ON) { - free(ON.ev); - free(ON.b); - free(ON.D); - free(ON.buffer); - free(ON.weight_buffer); - free(ON.zv); - free(ON.vv); - free(ON.tmp); - - for (int i = 1; i <= ON.m; i++) { - free(ON.A[i]); - free(ON.K[i]); + for (int j = 1; j <= i; j++) + { A[i][j] /= norm; + } } - free(ON.A); - free(ON.K); - - free(ON.data.Zx); - free(ON.data.AZx); - free(ON.data.delta); -} + } + + void update_b() + { for (int j = 1; j <= m; j++) + { float tmp = 0; + for (int i = j; i <= m; i++) + { tmp += ev[i] * data.AZx[i] * A[i][j] / (alpha * (alpha + ev[i])); + } + b[j] += tmp * data.g; + } + } + + void update_D() + { for (int j = 1; j <= m; j++) + { float scale = fabs(A[j][j]); + for (int i = j+1; i <= m; i++) + scale = fmin(fabs(A[i][j]), scale); + if (scale < 1e-10) continue; + for (int i = 1; i <= m; i++) + { A[i][j] /= scale; + K[j][i] *= scale; + K[i][j] *= scale; + } + b[j] /= scale; + D[j] *= scale; + //printf("D[%d] = %f\n", j, D[j]); + } + } -void make_pred(update_data& data, float x, float& wref) { - int m = data.ON->m; - float* w = &wref; + void check() + { double max_norm = 0; + for (int i = 1; i <= m; i++) + for (int j = i; j <= m ; j++) + max_norm = fmax(max_norm, fabs(K[i][j])); + //printf("|K| = %f\n", max_norm); + if (max_norm < 1e7) return; - if (data.ON->normalize) { - x /= sqrt(w[NORM2]); - } + // implicit -> explicit representation + // printf("begin conversion: t = %d, norm(K) = %f\n", t, max_norm); - data.prediction += w[0] * x; - for (int i = 1; i <= m; i++) { - data.prediction += w[i] * x * data.ON->D[i] * data.ON->b[i]; - } -} + // first step: K <- AKA' -void predict(OjaNewton& ON, base_learner&, example& ec) { - ON.data.prediction = 0; - GD::foreach_feature(*ON.all, ec, ON.data); - ec.partial_prediction = (float)ON.data.prediction; - ec.pred.scalar = GD::finalize_prediction(ON.all->sd, ec.partial_prediction); -} + // K <- AK + for (int j = 1; j <= m; j++) + { memset(tmp, 0, sizeof(double) * (m+1)); -void update_Z_and_wbar(update_data& data, float x, float& wref) { - float* w = &wref; - int m = data.ON->m; - if (data.ON->normalize) x /= sqrt(w[NORM2]); - float s = data.sketch_cnt * x; + for (int i = 1; i <= m; i++) + { for (int h = 1; h <= m; h++) + { tmp[i] += A[i][h] * K[h][j]; + } + } - for (int i = 1; i <= m; i++) { - w[i] += data.delta[i] * s / data.ON->D[i]; + for (int i = 1; i <= m; i++) + K[i][j] = tmp[i]; } - w[0] -= s * data.bdelta; -} + // K <- KA' + for (int i = 1; i <= m; i++) + { memset(tmp, 0, sizeof(double) * (m+1)); -void compute_Zx_and_norm(update_data& data, float x, float& wref) { - float* w = &wref; - int m = data.ON->m; - if (data.ON->normalize) x /= sqrt(w[NORM2]); + for (int j = 1; j <= m; j++) + for (int h = 1; h <= m; h++) + tmp[j] += K[i][h] * A[j][h]; - for (int i = 1; i <= m; i++) { - data.Zx[i] += w[i] * x * data.ON->D[i]; + for (int j = 1; j <= m; j++) + { K[i][j] = tmp[j]; + } } - data.norm2_x += x * x; -} -void update_wbar_and_Zx(update_data& data, float x, float& wref) { - float* w = &wref; - int m = data.ON->m; - if (data.ON->normalize) x /= sqrt(w[NORM2]); + //second step: w[0] <- w[0] + (DZ)'b, b <- 0. - float g = data.g * x; + uint32_t length = 1 << all->num_bits; + //size_t stride_shift = all->stride_shift; + weight_parameters& weights = all->weights; + for (weight_parameters::iterator i = weights.begin(); i != weights.end(); ++i) + { weights_iterator_iterator w_j = i.begin() + 1; + for (int j = 1; j <= m; ++j, ++w_j) + *i += *w_j * b[j] * D[j]; + } + memset(b, 0, sizeof(double) * (m+1)); + + //third step: Z <- ADZ, A, D <- Identity + + //double norm = 0; + weight_parameters::iterator iter = weights.begin(); + for (uint32_t i = 0; i < length; ++i, ++iter) + { memset(tmp, 0, sizeof(float) * (m+1)); + for (int j = 1; j <= m; j++) + { weights_iterator_iterator w_k = iter.begin() + 1; + for (int h = 1; h <= m; ++h, ++w_k) + tmp[j] += A[j][h] * D[h] * (*w_k); + } + weights_iterator_iterator w_j = iter.begin() + 1; + for (int j = 1; j <= m; ++j, ++w_j) + { //norm = max(norm, fabs(tmp[j])); + *w_j = tmp[j]; + } + } + //printf("|Z| = %f\n", norm); - for (int i = 1; i <= m; i++) { - data.Zx[i] += w[i] * x * data.ON->D[i]; + for (int i = 1; i <= m; i++) + { memset(A[i], 0, sizeof(double) * (m+1)); + D[i] = 1; + A[i][i] = 1; } - w[0] -= g / data.ON->alpha; -} + } +}; +void keep_example(vw& all, OjaNewton& ON, example& ec) +{ output_and_account_example(all, ec); +} -void update_normalization(update_data& data, float x, float& wref) { - float* w = &wref; - int m = data.ON->m; - - w[NORM2] += x * x * data.g * data.g; +void finish(OjaNewton& ON) +{ free(ON.ev); + free(ON.b); + free(ON.D); + free(ON.buffer); + free(ON.weight_buffer); + free(ON.zv); + free(ON.vv); + free(ON.tmp); + + for (int i = 1; i <= ON.m; i++) + { free(ON.A[i]); + free(ON.K[i]); + } + free(ON.A); + free(ON.K); + + free(ON.data.Zx); + free(ON.data.AZx); + free(ON.data.delta); } -void learn(OjaNewton& ON, base_learner& base, example& ec) { - assert(ec.in_use); +void make_pred(update_data& data, float x, float& wref) +{ int m = data.ON->m; + float* w = &wref; - // predict - predict(ON, base, ec); + if (data.ON->normalize) + { x /= sqrt(w[NORM2]); + } - update_data& data = ON.data; - data.g = ON.all->loss->first_derivative(ON.all->sd, ec.pred.scalar, ec.l.simple.label)*ec.l.simple.weight; - data.g /= 2; // for half square loss - - if(ON.normalize) GD::foreach_feature(*ON.all, ec, data); + data.prediction += w[0] * x; + for (int i = 1; i <= m; i++) + { data.prediction += w[i] * x * data.ON->D[i] * data.ON->b[i]; + } +} - ON.buffer[ON.cnt] = &ec; - ON.weight_buffer[ON.cnt++] = data.g / 2; +void predict(OjaNewton& ON, base_learner&, example& ec) +{ ON.data.prediction = 0; + GD::foreach_feature(*ON.all, ec, ON.data); + ec.partial_prediction = (float)ON.data.prediction; + ec.pred.scalar = GD::finalize_prediction(ON.all->sd, ec.partial_prediction); +} - if (ON.cnt == ON.epoch_size) { - for (int k = 0; k < ON.epoch_size; k++, ON.t++) { - example& ex = *(ON.buffer[k]); - data.sketch_cnt = ON.weight_buffer[k]; +void update_Z_and_wbar(update_data& data, float x, float& wref) +{ float* w = &wref; + int m = data.ON->m; + if (data.ON->normalize) x /= sqrt(w[NORM2]); + float s = data.sketch_cnt * x; - data.norm2_x = 0; - memset(data.Zx, 0, sizeof(float)* (ON.m+1)); - GD::foreach_feature(*ON.all, ex, data); - ON.compute_AZx(); + for (int i = 1; i <= m; i++) + { w[i] += data.delta[i] * s / data.ON->D[i]; + } + w[0] -= s * data.bdelta; +} - ON.update_eigenvalues(); - ON.compute_delta(); +void compute_Zx_and_norm(update_data& data, float x, float& wref) +{ float* w = &wref; + int m = data.ON->m; + if (data.ON->normalize) x /= sqrt(w[NORM2]); - ON.update_K(); + for (int i = 1; i <= m; i++) + { data.Zx[i] += w[i] * x * data.ON->D[i]; + } + data.norm2_x += x * x; +} - GD::foreach_feature(*ON.all, ex, data); - } +void update_wbar_and_Zx(update_data& data, float x, float& wref) +{ float* w = &wref; + int m = data.ON->m; + if (data.ON->normalize) x /= sqrt(w[NORM2]); - ON.update_A(); - //ON.update_D(); - } + float g = data.g * x; + + for (int i = 1; i <= m; i++) + { data.Zx[i] += w[i] * x * data.ON->D[i]; + } + w[0] -= g / data.ON->alpha; +} - memset(data.Zx, 0, sizeof(float)* (ON.m+1)); - GD::foreach_feature(*ON.all, ec, data); - ON.compute_AZx(); - ON.update_b(); - ON.check(); +void update_normalization(update_data& data, float x, float& wref) +{ float* w = &wref; + int m = data.ON->m; - if (ON.cnt == ON.epoch_size) { - ON.cnt = 0; - for (int k = 0; k < ON.epoch_size; k++) { - VW::finish_example(*ON.all, ON.buffer[k]); - } - } + w[NORM2] += x * x * data.g * data.g; } +void learn(OjaNewton& ON, base_learner& base, example& ec) +{ assert(ec.in_use); -void save_load(OjaNewton& ON, io_buf& model_file, bool read, bool text) { - vw* all = ON.all; - if (read) { - initialize_regressor(*all); - ON.initialize_Z(); - } + // predict + predict(ON, base, ec); - if (model_file.files.size() > 0) { - bool resume = all->save_resume; - stringstream msg; - msg << ":"<< resume <<"\n"; - bin_text_read_write_fixed(model_file, (char *)&resume, sizeof (resume), "", read, msg, text); + update_data& data = ON.data; + data.g = ON.all->loss->first_derivative(ON.all->sd, ec.pred.scalar, ec.l.simple.label)*ec.l.simple.weight; + data.g /= 2; // for half square loss + if(ON.normalize) GD::foreach_feature(*ON.all, ec, data); - if (resume) - GD::save_load_online_state(*all, model_file, read, text); - else - GD::save_load_regressor(*all, model_file, read, text); - } -} - -base_learner* OjaNewton_setup(vw& all) { - if (missing_option(all, false, "OjaNewton", "Online Newton with Oja's Sketch")) - return nullptr; + ON.buffer[ON.cnt] = &ec; + ON.weight_buffer[ON.cnt++] = data.g / 2; - new_options(all, "OjaNewton options") - ("sketch_size", po::value(), "size of sketch") - ("epoch_size", po::value(), "size of epoch") - ("alpha", po::value(), "mutiplicative constant for indentiy") - ("alpha_inverse", po::value(), "one over alpha, similar to learning rate") - ("learning_rate_cnt", po::value(), "constant for the learning rate 1/t") - ("normalize", po::value(), "normalize the features or not") - ("random_init", po::value(), "randomize initialization of Oja or not"); - add_options(all); + if (ON.cnt == ON.epoch_size) + { for (int k = 0; k < ON.epoch_size; k++, ON.t++) + { example& ex = *(ON.buffer[k]); + data.sketch_cnt = ON.weight_buffer[k]; - po::variables_map& vm = all.vm; + data.norm2_x = 0; + memset(data.Zx, 0, sizeof(float)* (ON.m+1)); + GD::foreach_feature(*ON.all, ex, data); + ON.compute_AZx(); - OjaNewton& ON = calloc_or_throw(); - ON.all = &all; + ON.update_eigenvalues(); + ON.compute_delta(); - if (vm.count("sketch_size")) - ON.m = vm["sketch_size"].as(); - else - ON.m = 10; + ON.update_K(); - if (vm.count("epoch_size")) - ON.epoch_size = vm["epoch_size"].as(); - else - ON.epoch_size = 1; + GD::foreach_feature(*ON.all, ex, data); + } - if (vm.count("alpha")) - ON.alpha = vm["alpha"].as(); - else - ON.alpha = 1.f; + ON.update_A(); + //ON.update_D(); + } - if (vm.count("alpha_inverse")) - ON.alpha = 1.f / vm["alpha_inverse"].as(); + memset(data.Zx, 0, sizeof(float)* (ON.m+1)); + GD::foreach_feature(*ON.all, ec, data); + ON.compute_AZx(); - if (vm.count("learning_rate_cnt")) - ON.learning_rate_cnt = vm["learning_rate_cnt"].as(); - else - ON.learning_rate_cnt = 2; + ON.update_b(); + ON.check(); - if (vm.count("normalize")) - ON.normalize = vm["normalize"].as(); - else - ON.normalize = true; - - if (vm.count("random_init")) - ON.random_init = vm["random_init"].as(); - else - ON.random_init = true; - - ON.cnt = 0; - ON.t = 1; - - ON.ev = calloc_or_throw(ON.m+1); - ON.b = calloc_or_throw(ON.m+1); - ON.D = calloc_or_throw(ON.m+1); - ON.A = calloc_or_throw(ON.m+1); - ON.K = calloc_or_throw(ON.m+1); - for (int i = 1; i <= ON.m; i++) { - ON.A[i] = calloc_or_throw(ON.m+1); - ON.K[i] = calloc_or_throw(ON.m+1); - ON.A[i][i] = 1; - ON.K[i][i] = 1; - ON.D[i] = 1; + if (ON.cnt == ON.epoch_size) + { ON.cnt = 0; + for (int k = 0; k < ON.epoch_size; k++) + { VW::finish_example(*ON.all, ON.buffer[k]); } + } +} + - ON.buffer = calloc_or_throw(ON.epoch_size); - ON.weight_buffer = calloc_or_throw(ON.epoch_size); - - ON.zv = calloc_or_throw(ON.m+1); - ON.vv = calloc_or_throw(ON.m+1); - ON.tmp = calloc_or_throw(ON.m+1); +void save_load(OjaNewton& ON, io_buf& model_file, bool read, bool text) +{ vw* all = ON.all; + if (read) + { initialize_regressor(*all); + ON.initialize_Z(); + } - ON.data.ON = &ON; - ON.data.Zx = calloc_or_throw(ON.m+1); - ON.data.AZx = calloc_or_throw(ON.m+1); - ON.data.delta = calloc_or_throw(ON.m+1); + if (model_file.files.size() > 0) + { bool resume = all->save_resume; + stringstream msg; + msg << ":"<< resume <<"\n"; + bin_text_read_write_fixed(model_file, (char *)&resume, sizeof (resume), "", read, msg, text); - all.weights.stride_shift((uint32_t)ceil(log2(ON.m + 2))); - learner& l = init_learner(&ON, learn, 1 << all.weights.stride_shift()); + if (resume) + GD::save_load_online_state(*all, model_file, read, text); + else + GD::save_load_regressor(*all, model_file, read, text); + } +} - l.set_predict(predict); - l.set_save_load(save_load); - l.set_finish_example(keep_example); - l.set_finish(finish); - return make_base(l); +base_learner* OjaNewton_setup(vw& all) +{ if (missing_option(all, false, "OjaNewton", "Online Newton with Oja's Sketch")) + return nullptr; + + new_options(all, "OjaNewton options") + ("sketch_size", po::value(), "size of sketch") + ("epoch_size", po::value(), "size of epoch") + ("alpha", po::value(), "mutiplicative constant for indentiy") + ("alpha_inverse", po::value(), "one over alpha, similar to learning rate") + ("learning_rate_cnt", po::value(), "constant for the learning rate 1/t") + ("normalize", po::value(), "normalize the features or not") + ("random_init", po::value(), "randomize initialization of Oja or not"); + add_options(all); + + po::variables_map& vm = all.vm; + + OjaNewton& ON = calloc_or_throw(); + ON.all = &all; + + if (vm.count("sketch_size")) + ON.m = vm["sketch_size"].as(); + else + ON.m = 10; + + if (vm.count("epoch_size")) + ON.epoch_size = vm["epoch_size"].as(); + else + ON.epoch_size = 1; + + if (vm.count("alpha")) + ON.alpha = vm["alpha"].as(); + else + ON.alpha = 1.f; + + if (vm.count("alpha_inverse")) + ON.alpha = 1.f / vm["alpha_inverse"].as(); + + if (vm.count("learning_rate_cnt")) + ON.learning_rate_cnt = vm["learning_rate_cnt"].as(); + else + ON.learning_rate_cnt = 2; + + if (vm.count("normalize")) + ON.normalize = vm["normalize"].as(); + else + ON.normalize = true; + + if (vm.count("random_init")) + ON.random_init = vm["random_init"].as(); + else + ON.random_init = true; + + ON.cnt = 0; + ON.t = 1; + + ON.ev = calloc_or_throw(ON.m+1); + ON.b = calloc_or_throw(ON.m+1); + ON.D = calloc_or_throw(ON.m+1); + ON.A = calloc_or_throw(ON.m+1); + ON.K = calloc_or_throw(ON.m+1); + for (int i = 1; i <= ON.m; i++) + { ON.A[i] = calloc_or_throw(ON.m+1); + ON.K[i] = calloc_or_throw(ON.m+1); + ON.A[i][i] = 1; + ON.K[i][i] = 1; + ON.D[i] = 1; + } + + ON.buffer = calloc_or_throw(ON.epoch_size); + ON.weight_buffer = calloc_or_throw(ON.epoch_size); + + ON.zv = calloc_or_throw(ON.m+1); + ON.vv = calloc_or_throw(ON.m+1); + ON.tmp = calloc_or_throw(ON.m+1); + + ON.data.ON = &ON; + ON.data.Zx = calloc_or_throw(ON.m+1); + ON.data.AZx = calloc_or_throw(ON.m+1); + ON.data.delta = calloc_or_throw(ON.m+1); + + all.weights.stride_shift((uint32_t)ceil(log2(ON.m + 2))); + + learner& l = init_learner(&ON, learn, 1 << all.weights.stride_shift()); + + l.set_predict(predict); + l.set_save_load(save_load); + l.set_finish_example(keep_example); + l.set_finish(finish); + return make_base(l); } diff --git a/vowpalwabbit/accumulate.cc b/vowpalwabbit/accumulate.cc index 010282c7ade..4293c0b9d40 100644 --- a/vowpalwabbit/accumulate.cc +++ b/vowpalwabbit/accumulate.cc @@ -22,16 +22,16 @@ void add_float(float& c1, const float& c2) { c1 += c2; } void accumulate(vw& all, weight_parameters& weights, size_t offset) { uint32_t length = 1 << all.num_bits; //This is size of gradient float* local_grad = new float[length]; - + weight_parameters::iterator iter = weights.begin(); for (uint32_t i = 0; iter != weights.end(); ++i, ++iter) - local_grad[i] = (&(*iter))[offset]; - + local_grad[i] = (&(*iter))[offset]; + all_reduce(all, local_grad, length); //TODO: modify to not use first() - + iter = weights.begin(); for (uint32_t i = 0; iter != weights.end(); ++i, ++iter) - (&(*iter))[offset] = local_grad[i]; + (&(*iter))[offset] = local_grad[i]; delete[] local_grad; } @@ -49,14 +49,14 @@ void accumulate_avg(vw& all, weight_parameters& weights, size_t offset) weight_parameters::iterator iter = weights.begin(); for (uint32_t i = 0; iter != weights.end(); ++i, ++iter) - local_grad[i] = (&(*iter))[offset]; - + local_grad[i] = (&(*iter))[offset]; + all_reduce(all, local_grad, length); //TODO: modify to not use first() iter = weights.begin(); for (uint32_t i = 0; iter != weights.end(); ++i, ++iter) - (&(*iter))[offset] = local_grad[i]/numnodes; - + (&(*iter))[offset] = local_grad[i]/numnodes; + delete[] local_grad; } @@ -84,27 +84,27 @@ void accumulate_weighted_avg(vw& all, weight_parameters& weights) weight_parameters::iterator iter = weights.begin(); for (uint32_t i = 0; iter != weights.end(); ++i, ++iter) - local_weights[i] = (&(*iter))[1]; + local_weights[i] = (&(*iter))[1]; //First compute weights for averaging - all_reduce(all, local_weights, length); + all_reduce(all, local_weights, length); - iter = weights.begin(); + iter = weights.begin(); for (uint32_t i =0 ; iter != weights.end(); ++i, ++iter) - if (local_weights[i] > 0) - { float ratio = (&(*iter))[1] / local_weights[i]; - local_weights[i] = *iter * ratio; - *iter *= ratio; - (&(*iter))[1] *= ratio; //A crude max - if (all.normalized_updates) - (&(*iter))[all.normalized_idx] *= ratio; //A crude max - } - else - { local_weights[i] = 0; - *iter = 0; - } - - all_reduce(all, weights.first(), length*weights.stride_shift()); + if (local_weights[i] > 0) + { float ratio = (&(*iter))[1] / local_weights[i]; + local_weights[i] = *iter * ratio; + *iter *= ratio; + (&(*iter))[1] *= ratio; //A crude max + if (all.normalized_updates) + (&(*iter))[all.normalized_idx] *= ratio; //A crude max + } + else + { local_weights[i] = 0; + *iter = 0; + } + + all_reduce(all, weights.first(), length*weights.stride_shift()); delete[] local_weights; } diff --git a/vowpalwabbit/action_score.cc b/vowpalwabbit/action_score.cc index 5f6b4c37785..4c4f52054f1 100644 --- a/vowpalwabbit/action_score.cc +++ b/vowpalwabbit/action_score.cc @@ -23,9 +23,8 @@ void print_action_score(int f, v_array& a_s, v_array&) } void delete_action_scores(void* v) -{ - v_array* cs = (v_array*)v; - cs->delete_v(); +{ v_array* cs = (v_array*)v; + cs->delete_v(); } } diff --git a/vowpalwabbit/action_score.h b/vowpalwabbit/action_score.h index 7846577677d..1d424615497 100644 --- a/vowpalwabbit/action_score.h +++ b/vowpalwabbit/action_score.h @@ -2,35 +2,35 @@ namespace ACTION_SCORE { - struct action_score - { uint32_t action; - float score; - }; +struct action_score +{ uint32_t action; + float score; +}; - typedef v_array action_scores; +typedef v_array action_scores; - inline int cmp(size_t a, size_t b) - { if (a == b) return 0; - if (a > b) return 1; - return -1; - } - - inline int score_comp(const void* p1, const void* p2) - { action_score* s1 = (action_score*)p1; - action_score* s2 = (action_score*)p2; - // Most sorting algos do not guarantee the output order of elements that compare equal. - // Tie-breaking on the index ensures that the result is deterministic across platforms. - // However, this forces a strict ordering, rather than a weak ordering, which carries a performance cost. - if(s2->score == s1->score) return cmp(s1->action, s2->action); - else if(s2->score >= s1->score) return -1; - else return 1; - } +inline int cmp(size_t a, size_t b) +{ if (a == b) return 0; + if (a > b) return 1; + return -1; +} + +inline int score_comp(const void* p1, const void* p2) +{ action_score* s1 = (action_score*)p1; + action_score* s2 = (action_score*)p2; + // Most sorting algos do not guarantee the output order of elements that compare equal. + // Tie-breaking on the index ensures that the result is deterministic across platforms. + // However, this forces a strict ordering, rather than a weak ordering, which carries a performance cost. + if(s2->score == s1->score) return cmp(s1->action, s2->action); + else if(s2->score >= s1->score) return -1; + else return 1; +} + +inline int reverse_order(const void* p1, const void* p2) +{ return score_comp(p2,p1); +} - inline int reverse_order(const void* p1, const void* p2) - { return score_comp(p2,p1); - } - - void print_action_score(int f, v_array& a_s, v_array&); +void print_action_score(int f, v_array& a_s, v_array&); - void delete_action_scores(void* v); +void delete_action_scores(void* v); } diff --git a/vowpalwabbit/active.cc b/vowpalwabbit/active.cc index ca75d99b9d4..cedd79ba809 100644 --- a/vowpalwabbit/active.cc +++ b/vowpalwabbit/active.cc @@ -55,10 +55,9 @@ void predict_or_learn_simulation(active& a, base_learner& base, example& ec) base.learn(ec); } else - { - ec.l.simple.label = FLT_MAX; - ec.weight = 0.f; - } + { ec.l.simple.label = FLT_MAX; + ec.weight = 0.f; + } } } diff --git a/vowpalwabbit/allreduce.h b/vowpalwabbit/allreduce.h index 61a953be8c8..8bbfa76ff69 100644 --- a/vowpalwabbit/allreduce.h +++ b/vowpalwabbit/allreduce.h @@ -178,7 +178,7 @@ class AllReduceSockets : public AllReduce void all_reduce_init(); template void pass_up(char* buffer, size_t left_read_pos, size_t right_read_pos, size_t& parent_sent_pos) - { size_t my_bufsize = (std::min)(ar_buf_size, (std::min)(left_read_pos, right_read_pos) / sizeof(T) * sizeof(T) - parent_sent_pos); + { size_t my_bufsize = (std::min)(ar_buf_size, (std::min)(left_read_pos, right_read_pos) / sizeof(T) * sizeof(T) - parent_sent_pos); if (my_bufsize > 0) { //going to pass up this chunk of data to the parent diff --git a/vowpalwabbit/array_parameters.h b/vowpalwabbit/array_parameters.h index 7b29cffd56e..315dad5fc84 100644 --- a/vowpalwabbit/array_parameters.h +++ b/vowpalwabbit/array_parameters.h @@ -15,34 +15,32 @@ typedef float weight; class weight_parameters; -template +template class weights_iterator_iterator { private: - T* _cur; + T* _cur; public: - weights_iterator_iterator(T* cur) - : _cur(cur) - { } - - T& operator*() { return *_cur; } + weights_iterator_iterator(T* cur) + : _cur(cur) + { } + + T& operator*() { return *_cur; } - weights_iterator_iterator& operator++() - { - ++_cur; - return *this; - } + weights_iterator_iterator& operator++() + { ++_cur; + return *this; + } - weights_iterator_iterator operator+(size_t index) { return weights_iterator_iterator(_cur + index); } + weights_iterator_iterator operator+(size_t index) { return weights_iterator_iterator(_cur + index); } - weights_iterator_iterator& operator+=(size_t index) - { - _cur += index; - return *this; - } + weights_iterator_iterator& operator+=(size_t index) + { _cur += index; + return *this; + } - bool operator==(const weights_iterator_iterator& rhs) const { return _cur == rhs._cur; } - bool operator!=(const weights_iterator_iterator& rhs) const { return _cur != rhs._cur; } + bool operator==(const weights_iterator_iterator& rhs) const { return _cur == rhs._cur; } + bool operator!=(const weights_iterator_iterator& rhs) const { return _cur != rhs._cur; } }; @@ -50,165 +48,161 @@ template class weights_iterator { private: - T* _current; - uint32_t _stride; + T* _current; + uint32_t _stride; public: - typedef std::forward_iterator_tag iterator_category; - typedef T value_type; - typedef ptrdiff_t difference_type; - typedef T* pointer; - typedef T& reference; - - typedef weights_iterator_iterator w_iter; - - weights_iterator(T* current, uint32_t stride) - : _current(current), _stride(stride) - { } - - T& operator*() { return *_current; } - - weights_iterator& operator++() - { - _current += _stride; - return *this; - } - - weights_iterator operator+(size_t index) { return weights_iterator(_current + (index*_stride), _stride); } - - weights_iterator& operator+=(size_t index) - { _current += (index*_stride); - return *this; - } - - bool operator==(const weights_iterator& rhs) const { return _current == rhs._current; } - bool operator!=(const weights_iterator& rhs) const { return _current != rhs._current; } - - //to iterate within a bucket - w_iter begin() { return w_iter(_current); } - w_iter end(size_t offset) { return w_iter(_current + offset); } + typedef std::forward_iterator_tag iterator_category; + typedef T value_type; + typedef ptrdiff_t difference_type; + typedef T* pointer; + typedef T& reference; + + typedef weights_iterator_iterator w_iter; + + weights_iterator(T* current, uint32_t stride) + : _current(current), _stride(stride) + { } + + T& operator*() { return *_current; } + + weights_iterator& operator++() + { _current += _stride; + return *this; + } + + weights_iterator operator+(size_t index) { return weights_iterator(_current + (index*_stride), _stride); } + + weights_iterator& operator+=(size_t index) + { _current += (index*_stride); + return *this; + } + + bool operator==(const weights_iterator& rhs) const { return _current == rhs._current; } + bool operator!=(const weights_iterator& rhs) const { return _current != rhs._current; } + + //to iterate within a bucket + w_iter begin() { return w_iter(_current); } + w_iter end(size_t offset) { return w_iter(_current + offset); } }; -class weight_parameters +class weight_parameters { private: - weight* _begin; - uint64_t _weight_mask; // (stride*(1 << num_bits) -1) - uint32_t _stride_shift; - bool _seeded; // whether the instance is sharing model state with others + weight* _begin; + uint64_t _weight_mask; // (stride*(1 << num_bits) -1) + uint32_t _stride_shift; + bool _seeded; // whether the instance is sharing model state with others public: - typedef weights_iterator iterator; - typedef weights_iterator const_iterator; - - weight_parameters(size_t length, uint32_t stride_shift=0) - : _begin(calloc_mergable_or_throw(length << stride_shift)), - _weight_mask((length << stride_shift) - 1), - _stride_shift(stride_shift), - _seeded(false) - { } - - weight_parameters() - : _begin(nullptr), _weight_mask(0), _stride_shift(0), _seeded(false) - {} - - bool not_null() { return (_weight_mask > 0 && _begin != nullptr);} - - weight_parameters(const weight_parameters &other) { shallow_copy(other); } - weight_parameters(weight_parameters &&) = delete; - - weight* first() { return _begin; } //TODO: Temporary fix for allreduce. - - //iterator with stride - iterator begin() { return iterator(_begin, (1<<_stride_shift)); } - iterator end() { return iterator(_begin + _weight_mask + 1, (1 << _stride_shift)); } - - iterator change_begin() { return iterator(_begin, 1); } - //const iterator - const_iterator cbegin() { return const_iterator(_begin, (1 << _stride_shift)); } - const_iterator cend() { return const_iterator(_begin + _weight_mask + 1, (1 << _stride_shift)); } - - inline weight& operator[](size_t i) const { return _begin[i & _weight_mask]; } - void shallow_copy(const weight_parameters& input) - { _begin = input._begin; - _weight_mask = input._weight_mask; - _stride_shift = input._stride_shift; - _seeded = true; - } - - template - inline void set_default() - { - for (iterator iter = begin(); iter != end(); ++iter) - T(iter); - } - - template //for random initialization of weights (with stride) - inline void set_default() - { uint32_t stride = 1 << _stride_shift; - iterator iter = begin(); - for (size_t i = 0; iter != end(); ++iter, i += stride) - T(iter, i); - } - - template //for random initialization of the entire weight_vector - inline void set_default() - { uint32_t stride = 1 << _stride_shift; - iterator iter = begin(); - for (size_t i = 0; iter != end(); ++iter, i += stride) - T(iter, i, stride); - } - - template - void set_default(T t) - { uint32_t stride = 1 << _stride_shift; - iterator iter = begin(); - for (size_t i = 0; iter != end(); ++iter, i+= stride) - t(iter, i); - } - - - void set_zero(size_t offset) - { - for (iterator iter = begin(); iter != end(); ++iter) - (&(*iter))[offset] = 0; - } - - uint64_t mask() - { return _weight_mask; - } - - uint64_t seeded() - { return _seeded; - } - - uint32_t stride_shift() - { return _stride_shift; - } - - void stride_shift(uint32_t stride_shift) - { _stride_shift = stride_shift; - } - - #ifndef _WIN32 - void share(size_t length) - { - float* shared_weights = (float*)mmap(0, (length << _stride_shift) * sizeof(float), - PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); - size_t float_count = length << _stride_shift; - weight* dest = shared_weights; - memcpy(dest, _begin, float_count*sizeof(float)); - free(_begin); - _begin = dest; - } - #endif - - ~weight_parameters() - { if (_begin != nullptr && !_seeded) // don't free weight vector if it is shared with another instance - { free(_begin); - _begin = nullptr; - } - } + typedef weights_iterator iterator; + typedef weights_iterator const_iterator; + + weight_parameters(size_t length, uint32_t stride_shift=0) + : _begin(calloc_mergable_or_throw(length << stride_shift)), + _weight_mask((length << stride_shift) - 1), + _stride_shift(stride_shift), + _seeded(false) + { } + + weight_parameters() + : _begin(nullptr), _weight_mask(0), _stride_shift(0), _seeded(false) + {} + + bool not_null() { return (_weight_mask > 0 && _begin != nullptr);} + + weight_parameters(const weight_parameters &other) { shallow_copy(other); } + weight_parameters(weight_parameters &&) = delete; + + weight* first() { return _begin; } //TODO: Temporary fix for allreduce. + + //iterator with stride + iterator begin() { return iterator(_begin, (1<<_stride_shift)); } + iterator end() { return iterator(_begin + _weight_mask + 1, (1 << _stride_shift)); } + + iterator change_begin() { return iterator(_begin, 1); } + //const iterator + const_iterator cbegin() { return const_iterator(_begin, (1 << _stride_shift)); } + const_iterator cend() { return const_iterator(_begin + _weight_mask + 1, (1 << _stride_shift)); } + + inline weight& operator[](size_t i) const { return _begin[i & _weight_mask]; } + void shallow_copy(const weight_parameters& input) + { _begin = input._begin; + _weight_mask = input._weight_mask; + _stride_shift = input._stride_shift; + _seeded = true; + } + + template + inline void set_default() + { for (iterator iter = begin(); iter != end(); ++iter) + T(iter); + } + + template //for random initialization of weights (with stride) + inline void set_default() + { uint32_t stride = 1 << _stride_shift; + iterator iter = begin(); + for (size_t i = 0; iter != end(); ++iter, i += stride) + T(iter, i); + } + + template //for random initialization of the entire weight_vector + inline void set_default() + { uint32_t stride = 1 << _stride_shift; + iterator iter = begin(); + for (size_t i = 0; iter != end(); ++iter, i += stride) + T(iter, i, stride); + } + + template + void set_default(T t) + { uint32_t stride = 1 << _stride_shift; + iterator iter = begin(); + for (size_t i = 0; iter != end(); ++iter, i+= stride) + t(iter, i); + } + + + void set_zero(size_t offset) + { for (iterator iter = begin(); iter != end(); ++iter) + (&(*iter))[offset] = 0; + } + + uint64_t mask() + { return _weight_mask; + } + + uint64_t seeded() + { return _seeded; + } + + uint32_t stride_shift() + { return _stride_shift; + } + + void stride_shift(uint32_t stride_shift) + { _stride_shift = stride_shift; + } + +#ifndef _WIN32 + void share(size_t length) + { float* shared_weights = (float*)mmap(0, (length << _stride_shift) * sizeof(float), + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + size_t float_count = length << _stride_shift; + weight* dest = shared_weights; + memcpy(dest, _begin, float_count*sizeof(float)); + free(_begin); + _begin = dest; + } +#endif + + ~weight_parameters() + { if (_begin != nullptr && !_seeded) // don't free weight vector if it is shared with another instance + { free(_begin); + _begin = nullptr; + } + } }; diff --git a/vowpalwabbit/audit_regressor.cc b/vowpalwabbit/audit_regressor.cc index 30a1b94f21a..3bc242336ae 100644 --- a/vowpalwabbit/audit_regressor.cc +++ b/vowpalwabbit/audit_regressor.cc @@ -10,257 +10,242 @@ license as described in the file LICENSE. #include "vw.h" using namespace std; struct audit_regressor_data -{ - vw* all; - size_t increment; - size_t cur_class; - size_t total_class_cnt; - vector* ns_pre; - io_buf* out_file; - size_t loaded_regressor_values; - size_t values_audited; +{ vw* all; + size_t increment; + size_t cur_class; + size_t total_class_cnt; + vector* ns_pre; + io_buf* out_file; + size_t loaded_regressor_values; + size_t values_audited; }; inline void audit_regressor_interaction(audit_regressor_data& dat, const audit_strings* f) { // same as audit_interaction in gd.cc - if (f == nullptr) - { dat.ns_pre->pop_back(); - return; - } - - string ns_pre; - if (!dat.ns_pre->empty()) - ns_pre += '*'; - - if (f->first != "" && ((f->first) != " ")) - { - ns_pre.append(f->first); - ns_pre += '^'; - } - if (f->second != "") - { - ns_pre.append(f->second); - dat.ns_pre->push_back(ns_pre); - } + if (f == nullptr) + { dat.ns_pre->pop_back(); + return; + } + + string ns_pre; + if (!dat.ns_pre->empty()) + ns_pre += '*'; + + if (f->first != "" && ((f->first) != " ")) + { ns_pre.append(f->first); + ns_pre += '^'; + } + if (f->second != "") + { ns_pre.append(f->second); + dat.ns_pre->push_back(ns_pre); + } } inline void audit_regressor_feature(audit_regressor_data& dat, const float /*ft_weight*/, const uint64_t ft_idx) -{ - weight_parameters& weights = dat.all->weights; +{ weight_parameters& weights = dat.all->weights; - if (weights[ft_idx] != 0) - ++dat.values_audited; - else return; + if (weights[ft_idx] != 0) + ++dat.values_audited; + else return; - string ns_pre; - for (vector::const_iterator s = dat.ns_pre->begin(); s != dat.ns_pre->end(); ++s) ns_pre += *s; + string ns_pre; + for (vector::const_iterator s = dat.ns_pre->begin(); s != dat.ns_pre->end(); ++s) ns_pre += *s; - ostringstream tempstream; - tempstream << ':' << ((ft_idx & weights.mask()) >> weights.stride_shift()) << ':' << weights[ft_idx]; + ostringstream tempstream; + tempstream << ':' << ((ft_idx & weights.mask()) >> weights.stride_shift()) << ':' << weights[ft_idx]; - string temp = ns_pre+tempstream.str() + '\n'; - if (dat.total_class_cnt > 1) // add class prefix for multiclass problems - temp = to_string(dat.cur_class) + ':' + temp; + string temp = ns_pre+tempstream.str() + '\n'; + if (dat.total_class_cnt > 1) // add class prefix for multiclass problems + temp = to_string(dat.cur_class) + ':' + temp; - bin_write_fixed(*dat.out_file, temp.c_str(), (uint32_t)temp.size()); + bin_write_fixed(*dat.out_file, temp.c_str(), (uint32_t)temp.size()); - weights[ft_idx] = 0.; //mark value audited + weights[ft_idx] = 0.; //mark value audited } // This is a learner which does nothing with examples. //void learn(audit_regressor_data&, LEARNER::base_learner&, example&) {} void audit_regressor(audit_regressor_data& rd, LEARNER::base_learner& base, example& ec) -{ - - vw& all = *rd.all; +{ - if (all.lda > 0) - { - ostringstream tempstream; - weight_parameters& weights = all.weights; - for (unsigned char* i = ec.indices.begin(); i != ec.indices.end(); i++) - { - features& fs = ec.feature_space[*i]; - for (size_t j = 0; j < fs.size(); ++j) - { tempstream << '\t' << fs.space_names[j].get()->first << '^' << fs.space_names[j].get()->second << ':' << ((fs.indicies[j] >> weights.stride_shift()) & all.parse_mask); - for (size_t k = 0; k < all.lda; k++) - { - weight& w = weights[(fs.indicies[j]+k)]; - tempstream << ':' << w; - w = 0.; - } - tempstream << endl; - } + vw& all = *rd.all; + + if (all.lda > 0) + { ostringstream tempstream; + weight_parameters& weights = all.weights; + for (unsigned char* i = ec.indices.begin(); i != ec.indices.end(); i++) + { features& fs = ec.feature_space[*i]; + for (size_t j = 0; j < fs.size(); ++j) + { tempstream << '\t' << fs.space_names[j].get()->first << '^' << fs.space_names[j].get()->second << ':' << ((fs.indicies[j] >> weights.stride_shift()) & all.parse_mask); + for (size_t k = 0; k < all.lda; k++) + { weight& w = weights[(fs.indicies[j]+k)]; + tempstream << ':' << w; + w = 0.; } + tempstream << endl; + } + } - bin_write_fixed(*rd.out_file, tempstream.str().c_str(), (uint32_t)tempstream.str().size()); + bin_write_fixed(*rd.out_file, tempstream.str().c_str(), (uint32_t)tempstream.str().size()); - } else { + } + else + { - rd.cur_class = 0; - uint64_t old_offset = ec.ft_offset; + rd.cur_class = 0; + uint64_t old_offset = ec.ft_offset; - while ( rd.cur_class < rd.total_class_cnt ) - { + while ( rd.cur_class < rd.total_class_cnt ) + { - for (unsigned char* i = ec.indices.begin(); i != ec.indices.end(); ++i) - { features& fs = ec.feature_space[(size_t)*i]; - if (fs.space_names.size() > 0) - for (size_t j = 0; j < fs.size(); ++j) - { - audit_regressor_interaction(rd, fs.space_names[j].get()); - audit_regressor_feature(rd, fs.values[j], (uint32_t)fs.indicies[j] + ec.ft_offset); - audit_regressor_interaction(rd, NULL); - } - else - for (size_t j = 0; j < fs.size(); ++j) - audit_regressor_feature(rd, fs.values[j], (uint32_t)fs.indicies[j] + ec.ft_offset); - } + for (unsigned char* i = ec.indices.begin(); i != ec.indices.end(); ++i) + { features& fs = ec.feature_space[(size_t)*i]; + if (fs.space_names.size() > 0) + for (size_t j = 0; j < fs.size(); ++j) + { audit_regressor_interaction(rd, fs.space_names[j].get()); + audit_regressor_feature(rd, fs.values[j], (uint32_t)fs.indicies[j] + ec.ft_offset); + audit_regressor_interaction(rd, NULL); + } + else + for (size_t j = 0; j < fs.size(); ++j) + audit_regressor_feature(rd, fs.values[j], (uint32_t)fs.indicies[j] + ec.ft_offset); + } - INTERACTIONS::generate_interactions(*rd.all, ec, rd); + INTERACTIONS::generate_interactions(*rd.all, ec, rd); - ec.ft_offset += rd.increment; - ++rd.cur_class; - } - - ec.ft_offset = old_offset; // make sure example is not changed. + ec.ft_offset += rd.increment; + ++rd.cur_class; } + + ec.ft_offset = old_offset; // make sure example is not changed. + } } void end_examples(audit_regressor_data& d) -{ - d.out_file->flush(); // close_file() should do this for me ... - d.out_file->close_file(); - delete (d.out_file); - d.out_file = NULL; - delete d.ns_pre; - d.ns_pre = NULL; +{ d.out_file->flush(); // close_file() should do this for me ... + d.out_file->close_file(); + delete (d.out_file); + d.out_file = NULL; + delete d.ns_pre; + d.ns_pre = NULL; } inline void print_ex(size_t ex_processed, size_t vals_found, size_t progress) -{ - std::cerr << std::left - << std::setw(shared_data::col_example_counter) << ex_processed - << " " << std::right - << std::setw(9) << vals_found - << " " << std::right - << std::setw(12) << progress << '%' - << std::endl; +{ std::cerr << std::left + << std::setw(shared_data::col_example_counter) << ex_processed + << " " << std::right + << std::setw(9) << vals_found + << " " << std::right + << std::setw(12) << progress << '%' + << std::endl; } void finish_example(vw& all, audit_regressor_data& dd, example& ec) -{ - bool printed = false; - if (ec.example_counter+1 >= all.sd->dump_interval && !all.quiet) - { - print_ex(ec.example_counter+1, dd.values_audited, dd.values_audited*100/dd.loaded_regressor_values); - all.sd->weighted_examples = (double)(ec.example_counter+1); //used in update_dump_interval - all.sd->update_dump_interval(all.progress_add, all.progress_arg); - printed = true; - } - - if (dd.values_audited == dd.loaded_regressor_values) - { // all regressor values were audited - if (!printed) - print_ex(ec.example_counter+1, dd.values_audited, 100); - set_done(all); - } - - VW::finish_example(all, &ec); +{ bool printed = false; + if (ec.example_counter+1 >= all.sd->dump_interval && !all.quiet) + { print_ex(ec.example_counter+1, dd.values_audited, dd.values_audited*100/dd.loaded_regressor_values); + all.sd->weighted_examples = (double)(ec.example_counter+1); //used in update_dump_interval + all.sd->update_dump_interval(all.progress_add, all.progress_arg); + printed = true; + } + + if (dd.values_audited == dd.loaded_regressor_values) + { // all regressor values were audited + if (!printed) + print_ex(ec.example_counter+1, dd.values_audited, 100); + set_done(all); + } + + VW::finish_example(all, &ec); } void finish(audit_regressor_data& dat) -{ - if (dat.values_audited < dat.loaded_regressor_values) - cerr << "Note: for some reason audit couldn't find all regressor values in dataset (" << - dat.values_audited << " of " << dat.loaded_regressor_values << " found)." << endl; +{ if (dat.values_audited < dat.loaded_regressor_values) + cerr << "Note: for some reason audit couldn't find all regressor values in dataset (" << + dat.values_audited << " of " << dat.loaded_regressor_values << " found)." << endl; } void init_driver(audit_regressor_data& dat) -{ // checks a few settings that might be applied after audit_regressor_setup() is called +{ // checks a few settings that might be applied after audit_regressor_setup() is called - po::variables_map& vm = dat.all->vm; - if ( (vm.count("cache_file") || vm.count("cache") ) && !vm.count("kill_cache") ) - THROW("audit_regressor is incompatible with a cache file. Use it in single pass mode only."); + po::variables_map& vm = dat.all->vm; + if ( (vm.count("cache_file") || vm.count("cache") ) && !vm.count("kill_cache") ) + THROW("audit_regressor is incompatible with a cache file. Use it in single pass mode only."); - dat.all->sd->dump_interval = 1.; // regressor could initialize these if saved with --save_resume - dat.all->sd->example_number = 0; + dat.all->sd->dump_interval = 1.; // regressor could initialize these if saved with --save_resume + dat.all->sd->example_number = 0; - dat.increment = dat.all->l->increment/dat.all->l->weights; - dat.total_class_cnt = dat.all->l->weights; + dat.increment = dat.all->l->increment/dat.all->l->weights; + dat.total_class_cnt = dat.all->l->weights; - if (dat.all->vm.count("csoaa")) - { - size_t n = dat.all->vm["csoaa"].as(); - if (n != dat.total_class_cnt) - { - dat.total_class_cnt = n; - dat.increment = dat.all->l->increment/n; - } + if (dat.all->vm.count("csoaa")) + { size_t n = dat.all->vm["csoaa"].as(); + if (n != dat.total_class_cnt) + { dat.total_class_cnt = n; + dat.increment = dat.all->l->increment/n; } + } - // count non-null feature values in regressor - weight_parameters& w = dat.all->weights; - for (weight_parameters::iterator iter = w.change_begin(); iter != w.end(); iter += dat.increment) //TODO:modify - if (*iter != 0) dat.loaded_regressor_values++; + // count non-null feature values in regressor + weight_parameters& w = dat.all->weights; + for (weight_parameters::iterator iter = w.change_begin(); iter != w.end(); iter += dat.increment) //TODO:modify + if (*iter != 0) dat.loaded_regressor_values++; - if (dat.loaded_regressor_values == 0) - THROW("regressor has no non-zero weights. Nothing to audit."); + if (dat.loaded_regressor_values == 0) + THROW("regressor has no non-zero weights. Nothing to audit."); - if (!dat.all->quiet) - { - std::cerr << "Regressor contains " << dat.loaded_regressor_values << " values\n"; - std::cerr << std::left - << std::setw(shared_data::col_example_counter) << "example" - << " " - << std::setw(shared_data::col_example_weight) << "values" - << " " - << std::setw(shared_data::col_current_label) << "total" - << std::endl; - std::cerr << std::left - << std::setw(shared_data::col_example_counter) << "counter" - << " " - << std::setw(shared_data::col_example_weight) << "audited" - << " " - << std::setw(shared_data::col_current_label) << "progress" - << std::endl; - } + if (!dat.all->quiet) + { std::cerr << "Regressor contains " << dat.loaded_regressor_values << " values\n"; + std::cerr << std::left + << std::setw(shared_data::col_example_counter) << "example" + << " " + << std::setw(shared_data::col_example_weight) << "values" + << " " + << std::setw(shared_data::col_current_label) << "total" + << std::endl; + std::cerr << std::left + << std::setw(shared_data::col_example_counter) << "counter" + << " " + << std::setw(shared_data::col_example_weight) << "audited" + << " " + << std::setw(shared_data::col_current_label) << "progress" + << std::endl; + } } LEARNER::base_learner* audit_regressor_setup(vw& all) -{ - if (missing_option(all, "audit_regressor", "stores feature names and their regressor values. Same dataset must be used for both regressor training and this mode.")) return nullptr; +{ if (missing_option(all, "audit_regressor", "stores feature names and their regressor values. Same dataset must be used for both regressor training and this mode.")) return nullptr; - po::variables_map& vm = all.vm; + po::variables_map& vm = all.vm; - string out_file = vm["audit_regressor"].as(); - if (out_file.empty()) - THROW("audit_regressor argument (output filename) is missing."); + string out_file = vm["audit_regressor"].as(); + if (out_file.empty()) + THROW("audit_regressor argument (output filename) is missing."); - if (all.numpasses > 1) - THROW("audit_regressor can't be used with --passes > 1."); + if (all.numpasses > 1) + THROW("audit_regressor can't be used with --passes > 1."); - all.audit = true; + all.audit = true; - audit_regressor_data& dat = calloc_or_throw(); - dat.all = &all; - dat.ns_pre = new vector(); // explicitly invoking vector's constructor - dat.out_file = new io_buf(); - dat.out_file->open_file( out_file.c_str(), all.stdin_off, io_buf::WRITE ); + audit_regressor_data& dat = calloc_or_throw(); + dat.all = &all; + dat.ns_pre = new vector(); // explicitly invoking vector's constructor + dat.out_file = new io_buf(); + dat.out_file->open_file( out_file.c_str(), all.stdin_off, io_buf::WRITE ); - LEARNER::learner& ret = LEARNER::init_learner(&dat, setup_base(all), audit_regressor, audit_regressor, 1); - ret.set_end_examples(end_examples); - ret.set_finish_example(finish_example); - ret.set_finish(finish); - ret.set_init_driver(init_driver); + LEARNER::learner& ret = LEARNER::init_learner(&dat, setup_base(all), audit_regressor, audit_regressor, 1); + ret.set_end_examples(end_examples); + ret.set_finish_example(finish_example); + ret.set_finish(finish); + ret.set_init_driver(init_driver); - return LEARNER::make_base(ret); + return LEARNER::make_base(ret); } diff --git a/vowpalwabbit/autolink.cc b/vowpalwabbit/autolink.cc index a36988353f0..af9beb6af34 100644 --- a/vowpalwabbit/autolink.cc +++ b/vowpalwabbit/autolink.cc @@ -16,9 +16,8 @@ void predict_or_learn(autolink& b, LEARNER::base_learner& base, example& ec) features& fs = ec.feature_space[autolink_namespace]; for (size_t i = 0; i < b.d; i++) if (base_pred != 0.) - { - fs.push_back(base_pred, autoconstant + (i << b.stride_shift)); - base_pred *= ec.pred.scalar; + { fs.push_back(base_pred, autoconstant + (i << b.stride_shift)); + base_pred *= ec.pred.scalar; } ec.total_sum_feat_sq += fs.sum_feat_sq; diff --git a/vowpalwabbit/bfgs.cc b/vowpalwabbit/bfgs.cc index a2ba456def6..1d604b42a89 100644 --- a/vowpalwabbit/bfgs.cc +++ b/vowpalwabbit/bfgs.cc @@ -106,12 +106,12 @@ const char* curv_message = "Zero or negative curvature detected.\n" void zero_derivative(vw& all) { //set derivative to 0. - all.weights.set_zero(W_GT); + all.weights.set_zero(W_GT); } void zero_preconditioner(vw& all) { //set derivative to 0. - all.weights.set_zero(W_COND); + all.weights.set_zero(W_COND); } void reset_state(vw& all, bfgs& b, bool zero) @@ -190,16 +190,15 @@ double regularizer_direction_magnitude(vw& all, bfgs& b, float regularizer) return ret; weight_parameters& weights = all.weights; - + if (b.regularizers == nullptr) - for (weight_parameters::iterator iter = weights.begin(); iter != weights.end(); ++iter) - ret += regularizer* (&(*iter))[W_DIR] * (&(*iter))[W_DIR]; - + for (weight_parameters::iterator iter = weights.begin(); iter != weights.end(); ++iter) + ret += regularizer* (&(*iter))[W_DIR] * (&(*iter))[W_DIR]; + else - { - weight_parameters::iterator iter = weights.begin(); - for (uint32_t i = 0; iter != weights.end(); ++i, ++iter) - ret += b.regularizers[2 * i] * (&(*iter))[W_DIR] * (&(*iter))[W_DIR]; + { weight_parameters::iterator iter = weights.begin(); + for (uint32_t i = 0; iter != weights.end(); ++i, ++iter) + ret += b.regularizers[2 * i] * (&(*iter))[W_DIR] * (&(*iter))[W_DIR]; } return ret; @@ -210,7 +209,7 @@ float direction_magnitude(vw& all) weight_parameters& weights = all.weights; double ret = 0.; for (weight_parameters::iterator iter = weights.begin(); iter != weights.end(); ++iter) - ret += (&(*iter))[W_DIR] * (&(*iter))[W_DIR]; + ret += (&(*iter))[W_DIR] * (&(*iter))[W_DIR]; return (float)ret; } @@ -226,11 +225,11 @@ void bfgs_iter_start(vw& all, bfgs& b, float* mem, int& lastj, double importance for(; w != weights.end(); mem+=b.mem_stride, ++w) { if (b.m>0) mem[(MEM_XT+origin)%b.mem_stride] = (&(*w))[W_XT]; - mem[(MEM_GT + origin) % b.mem_stride] = (&(*w))[W_GT]; - g1_Hg1 += ((&(*w))[W_GT]) * ((&(*w))[W_GT]) * ((&(*w))[W_COND]); - g1_g1 += ((&(*w))[W_GT]) * ((&(*w))[W_GT]); - (&(*w))[W_DIR] = -(&(*w))[W_COND] * ((&(*w))[W_GT]); - ((&(*w))[W_GT]) = 0; + mem[(MEM_GT + origin) % b.mem_stride] = (&(*w))[W_GT]; + g1_Hg1 += ((&(*w))[W_GT]) * ((&(*w))[W_GT]) * ((&(*w))[W_COND]); + g1_g1 += ((&(*w))[W_GT]) * ((&(*w))[W_GT]); + (&(*w))[W_DIR] = -(&(*w))[W_COND] * ((&(*w))[W_GT]); + ((&(*w))[W_GT]) = 0; } lastj = 0; if (!all.quiet) @@ -240,8 +239,7 @@ void bfgs_iter_start(vw& all, bfgs& b, float* mem, int& lastj, double importance } void bfgs_iter_middle(vw& all, bfgs& b, float* mem, double* rho, double* alpha, int& lastj, int &origin) -{ - weight_parameters& weights = all.weights; +{ weight_parameters& weights = all.weights; float* mem0 = mem; @@ -253,26 +251,26 @@ void bfgs_iter_middle(vw& all, bfgs& b, float* mem, double* rho, double* alpha, weight_parameters::iterator w = weights.begin(); for(; w != weights.end(); mem+=b.mem_stride, ++w) - { y = (&(*w))[W_GT]-mem[(MEM_GT+origin)%b.mem_stride]; - g_Hy += ((&(*w))[W_GT]) * ((&(*w))[W_COND]) * y; - g_Hg += mem[(MEM_GT+origin)%b.mem_stride] * ((&(*w))[W_COND]) * mem[(MEM_GT+origin)%b.mem_stride]; - } - + { y = (&(*w))[W_GT]-mem[(MEM_GT+origin)%b.mem_stride]; + g_Hy += ((&(*w))[W_GT]) * ((&(*w))[W_COND]) * y; + g_Hg += mem[(MEM_GT+origin)%b.mem_stride] * ((&(*w))[W_COND]) * mem[(MEM_GT+origin)%b.mem_stride]; + } + float beta = (float) (g_Hy/g_Hg); - + if (beta<0.f || nanpattern(beta)) beta = 0.f; - + mem = mem0; w = weights.begin(); - + for (; w != weights.end(); mem += b.mem_stride, ++w) - { mem[(MEM_GT+origin)%b.mem_stride] = (&(*w))[W_GT]; - - (&(*w))[W_DIR] *= beta; - (&(*w))[W_DIR] -= ((&(*w))[W_COND])*((&(*w))[W_GT]); - (&(*w))[W_GT] = 0; - } + { mem[(MEM_GT+origin)%b.mem_stride] = (&(*w))[W_GT]; + + (&(*w))[W_DIR] *= beta; + (&(*w))[W_DIR] -= ((&(*w))[W_COND])*((&(*w))[W_GT]); + (&(*w))[W_GT] = 0; + } if (!all.quiet) fprintf(stderr, "%f\t", beta); return; @@ -291,10 +289,10 @@ void bfgs_iter_middle(vw& all, bfgs& b, float* mem, double* rho, double* alpha, for (; w != weights.end(); mem += b.mem_stride, ++w) { mem[(MEM_YT+origin)%b.mem_stride] = (&(*w))[W_GT] - mem[(MEM_GT+origin)%b.mem_stride]; mem[(MEM_ST + origin) % b.mem_stride] = (&(*w))[W_XT] - mem[(MEM_XT + origin) % b.mem_stride]; - (&(*w))[W_DIR] = (&(*w))[W_GT]; + (&(*w))[W_DIR] = (&(*w))[W_GT]; y_s += mem[(MEM_YT+origin)%b.mem_stride]*mem[(MEM_ST+origin)%b.mem_stride]; y_Hy += mem[(MEM_YT+origin)%b.mem_stride]*mem[(MEM_YT+origin)%b.mem_stride]* ((&(*w))[W_COND]); - s_q += mem[(MEM_ST + origin) % b.mem_stride] * ((&(*w))[W_GT]); + s_q += mem[(MEM_ST + origin) % b.mem_stride] * ((&(*w))[W_GT]); } if (y_s <= 0. || y_Hy <= 0.) @@ -330,8 +328,8 @@ void bfgs_iter_middle(vw& all, bfgs& b, float* mem, double* rho, double* alpha, { coef_j = alpha[j] - rho[j] * y_r; y_r = 0.; mem = mem0; - w = weights.begin(); - for (; w != weights.end(); mem += b.mem_stride, ++w) + w = weights.begin(); + for (; w != weights.end(); mem += b.mem_stride, ++w) { (&(*w))[W_DIR] += (float)coef_j*mem[(2*j+MEM_ST+origin)%b.mem_stride]; y_r += mem[(2*j-2+MEM_YT+origin)%b.mem_stride]*((&(*w))[W_DIR]); } @@ -363,8 +361,7 @@ void bfgs_iter_middle(vw& all, bfgs& b, float* mem, double* rho, double* alpha, } double wolfe_eval(vw& all, bfgs& b, float* mem, double loss_sum, double previous_loss_sum, double step_size, double importance_weight_sum, int &origin, double& wolfe1) -{ - weight_parameters& weights = all.weights; +{ weight_parameters& weights = all.weights; double g0_d = 0.; double g1_d = 0.; @@ -396,14 +393,14 @@ double add_regularization(vw& all, bfgs& b, float regularization) weight_parameters::iterator w = weights.begin(); if (b.regularizers == nullptr) - { for(; w != weights.end(); ++w) - { (&(*w))[W_GT] += regularization*(*w); - ret += 0.5*regularization*(*w)*(*w); - } + { for(; w != weights.end(); ++w) + { (&(*w))[W_GT] += regularization*(*w); + ret += 0.5*regularization*(*w)*(*w); } + } else { uint32_t i = 0; - for (; w != weights.end(); ++i, ++w) + for (; w != weights.end(); ++i, ++w) { weight delta_weight = *w - b.regularizers[2*i+1]; (&(*w))[W_GT] += b.regularizers[2*i]*delta_weight; ret += 0.5*b.regularizers[2*i]*delta_weight*delta_weight; @@ -414,40 +411,39 @@ double add_regularization(vw& all, bfgs& b, float regularization) } void finalize_preconditioner(vw& all, bfgs& b, float regularization) -{ - weight_parameters& weights = all.weights; +{ weight_parameters& weights = all.weights; float max_hessian = 0.f; weight_parameters::iterator w = weights.begin(); uint32_t i = 0; if (b.regularizers == nullptr) for(; w != weights.end(); ++w) - { (&(*w))[W_COND] += regularization; + { (&(*w))[W_COND] += regularization; if ((&(*w))[W_COND] > max_hessian) - max_hessian = (&(*w))[W_COND]; - if ((&(*w))[W_COND] > 0) - (&(*w))[W_COND] = 1.f / (&(*w))[W_COND]; + max_hessian = (&(*w))[W_COND]; + if ((&(*w))[W_COND] > 0) + (&(*w))[W_COND] = 1.f / (&(*w))[W_COND]; } else - for (; w != weights.end(); ++w, ++i) - { (&(*w))[W_COND] += b.regularizers[2 * i]; - if ((&(*w))[W_COND] > max_hessian) - max_hessian = (&(*w))[W_COND]; - if ((&(*w))[W_COND] > 0) - (&(*w))[W_COND] = 1.f / (&(*w))[W_COND]; + for (; w != weights.end(); ++w, ++i) + { (&(*w))[W_COND] += b.regularizers[2 * i]; + if ((&(*w))[W_COND] > max_hessian) + max_hessian = (&(*w))[W_COND]; + if ((&(*w))[W_COND] > 0) + (&(*w))[W_COND] = 1.f / (&(*w))[W_COND]; } float max_precond = (max_hessian==0.f) ? 0.f : max_precond_ratio / max_hessian; - + w = weights.begin(); for (; w != weights.end(); ++w) { if (infpattern(*w) || *w >max_precond) - (&(*w))[W_COND] = max_precond; + (&(*w))[W_COND] = max_precond; } } void preconditioner_to_regularizer(vw& all, bfgs& b, float regularization) { uint32_t length = 1 << all.num_bits; -weight_parameters& weights = all.weights; + weight_parameters& weights = all.weights; weight_parameters::iterator w = weights.begin(); uint32_t i = 0; if (b.regularizers == nullptr) @@ -456,16 +452,16 @@ weight_parameters& weights = all.weights; if (b.regularizers == nullptr) THROW("Failed to allocate weight array: try decreasing -b "); - for (; w != weights.end(); ++w, ++i) + for (; w != weights.end(); ++w, ++i) { b.regularizers[2*i] = regularization; - if ((&(*w))[W_COND] > 0.f) - b.regularizers[2 * i] += 1.f / (&(*w))[W_COND]; + if ((&(*w))[W_COND] > 0.f) + b.regularizers[2 * i] += 1.f / (&(*w))[W_COND]; } } else - for (; w != weights.end(); ++w, ++i) - {if ((&(*w))[W_COND] > 0.f) - b.regularizers[2 * i] += 1.f / (&(*w))[W_COND]; + for (; w != weights.end(); ++w, ++i) + { if ((&(*w))[W_COND] > 0.f) + b.regularizers[2 * i] += 1.f / (&(*w))[W_COND]; } w = weights.begin(); @@ -480,15 +476,14 @@ void regularizer_to_weight(vw& all, bfgs& b) uint32_t i = 0; if (b.regularizers != nullptr) { for(; w != weights.end(); ++i, ++w) - { (&(*w))[W_COND] = b.regularizers[2*i]; + { (&(*w))[W_COND] = b.regularizers[2*i]; *w = b.regularizers[2*i+1]; } } } void zero_state(vw& all) -{ - all.weights.set_zero(W_GT); +{ all.weights.set_zero(W_GT); all.weights.set_zero(W_DIR); all.weights.set_zero(W_COND); } @@ -499,7 +494,7 @@ double derivative_in_direction(vw& all, bfgs& b, float* mem, int &origin) weight_parameters::iterator w = weights.begin(); for(; w != weights.end(); mem+=b.mem_stride, ++w) - ret += mem[(MEM_GT + origin) % b.mem_stride] * (&(*w))[W_DIR]; + ret += mem[(MEM_GT + origin) % b.mem_stride] * (&(*w))[W_DIR]; return ret; } @@ -513,22 +508,22 @@ void update_weight(vw& all, float step_size) int process_pass(vw& all, bfgs& b) { int status = LEARN_OK; - finalize_preconditioner(all, b, all.l2_lambda); + finalize_preconditioner(all, b, all.l2_lambda); /********************************************************************/ /* A) FIRST PASS FINISHED: INITIALIZE FIRST LINE SEARCH *************/ /********************************************************************/ - if (b.first_pass) - { if(all.all_reduce != nullptr) - { accumulate(all, all.weights, W_COND); //Accumulate preconditioner - float temp = (float)b.importance_weight_sum; - b.importance_weight_sum = accumulate_scalar(all, temp); - } - //finalize_preconditioner(all, b, all.l2_lambda); - if(all.all_reduce != nullptr) - { float temp = (float)b.loss_sum; - b.loss_sum = accumulate_scalar(all, temp); //Accumulate loss_sums - accumulate(all, all.weights, 1); //Accumulate gradients from all nodes - } + if (b.first_pass) + { if(all.all_reduce != nullptr) + { accumulate(all, all.weights, W_COND); //Accumulate preconditioner + float temp = (float)b.importance_weight_sum; + b.importance_weight_sum = accumulate_scalar(all, temp); + } + //finalize_preconditioner(all, b, all.l2_lambda); + if(all.all_reduce != nullptr) + { float temp = (float)b.loss_sum; + b.loss_sum = accumulate_scalar(all, temp); //Accumulate loss_sums + accumulate(all, all.weights, 1); //Accumulate gradients from all nodes + } if (all.l2_lambda > 0.) b.loss_sum += add_regularization(all, b, all.l2_lambda); if (!all.quiet) @@ -805,8 +800,7 @@ void finish(bfgs& b) } void save_load_regularizer(vw& all, bfgs& b, io_buf& model_file, bool read, bool text) -{ - int c = 0; +{ int c = 0; uint32_t length = 2*(1 << all.num_bits); uint32_t i = 0; size_t brw = 1; @@ -845,7 +839,7 @@ void save_load_regularizer(vw& all, bfgs& b, io_buf& model_file, bool read, bool i++; } while ((!read && i < length) || (read && brw >0)); - + if (read) regularizer_to_weight(all, b); } @@ -894,10 +888,10 @@ void save_load(bfgs& b, io_buf& model_file, bool read, bool text) bool reg_vector = (b.output_regularizer && !read) || (all->per_feature_regularizer_input.length() > 0 && read); if (model_file.files.size() > 0) - { stringstream msg; - msg << ":"<< reg_vector <<"\n"; - bin_text_read_write_fixed(model_file,(char *)®_vector, sizeof (reg_vector), - "", read, msg, text); + { stringstream msg; + msg << ":"<< reg_vector <<"\n"; + bin_text_read_write_fixed(model_file,(char *)®_vector, sizeof (reg_vector), + "", read, msg, text); if (reg_vector) save_load_regularizer(*all, b, model_file, read, text); diff --git a/vowpalwabbit/bs.cc b/vowpalwabbit/bs.cc index 678202c95e1..c7178acc1d5 100644 --- a/vowpalwabbit/bs.cc +++ b/vowpalwabbit/bs.cc @@ -143,7 +143,7 @@ void output_example(vw& all, bs& d, example& ec) if(all.final_prediction_sink.size() != 0)//get confidence interval only when printing out predictions { d.lb = FLT_MAX; d.ub = -FLT_MAX; - for (double v : *d.pred_vec) + for (double v : *d.pred_vec) { if(v > d.ub) d.ub = (float)v; if(v < d.lb) diff --git a/vowpalwabbit/cache.cc b/vowpalwabbit/cache.cc index c88b11984b0..0c3faa05ed0 100644 --- a/vowpalwabbit/cache.cc +++ b/vowpalwabbit/cache.cc @@ -49,7 +49,7 @@ size_t read_cached_tag(io_buf& cache, example* ae) ae->tag.erase(); push_many(ae->tag, c, tag_size); - return tag_size+sizeof(tag_size); + return tag_size+sizeof(tag_size); } struct one_float { float f; } @@ -102,23 +102,23 @@ int read_cached_features(vw* all, v_array& examples) uint64_t last = 0; for (; c!= end;) - { feature_index i = 0; - c = run_len_decode(c,i); - feature_value v = 1.f; - if (i & neg_1) - v = -1.; - else if (i & general) - { v = ((one_float *)c)->f; - c += sizeof(float); - } - uint64_t diff = i >> 2; - int64_t s_diff = ZigZagDecode(diff); - if (s_diff < 0) - ae->sorted = false; - i = last + s_diff; - last = i; - ours.push_back(v,i); + { feature_index i = 0; + c = run_len_decode(c,i); + feature_value v = 1.f; + if (i & neg_1) + v = -1.; + else if (i & general) + { v = ((one_float *)c)->f; + c += sizeof(float); } + uint64_t diff = i >> 2; + int64_t s_diff = ZigZagDecode(diff); + if (s_diff < 0) + ae->sorted = false; + i = last + s_diff; + last = i; + ours.push_back(v,i); + } all->p->input->set(c); } @@ -154,8 +154,7 @@ void output_features(io_buf& cache, unsigned char index, features& fs, uint64_t uint64_t last = 0; for (features::iterator& f : fs) - { - feature_index fi = f.index() & mask; + { feature_index fi = f.index() & mask; int64_t s_diff = (fi - last); uint64_t diff = ZigZagEncode(s_diff) << 2; last = fi; @@ -165,8 +164,7 @@ void output_features(io_buf& cache, unsigned char index, features& fs, uint64_t else if (f.value() == -1.) c = run_len_encode(c, diff | neg_1); else - { - c = run_len_encode(c, diff | general); + { c = run_len_encode(c, diff | general); memcpy(c, &f.value(), sizeof(feature_value)); c += sizeof(feature_value); } diff --git a/vowpalwabbit/cb.cc b/vowpalwabbit/cb.cc index 6ff7dd16541..dc2fa056ee0 100644 --- a/vowpalwabbit/cb.cc +++ b/vowpalwabbit/cb.cc @@ -15,14 +15,14 @@ using namespace std; namespace CB { - bool is_test_label(CB::label& ld) - { if (ld.costs.size() == 0) - return true; - for (size_t i=0; i 0.) - return false; +bool is_test_label(CB::label& ld) +{ if (ld.costs.size() == 0) return true; - } + for (size_t i=0; i 0.) + return false; + return true; +} char* bufread_label(CB::label* ld, char* c, io_buf& cache) { size_t num = *(size_t *)c; diff --git a/vowpalwabbit/cb.h b/vowpalwabbit/cb.h index 3a8cbd3d616..6cbc1105217 100644 --- a/vowpalwabbit/cb.h +++ b/vowpalwabbit/cb.h @@ -21,7 +21,7 @@ struct label { v_array costs; }; - bool is_test_label(CB::label& ld); +bool is_test_label(CB::label& ld); extern label_parser cb_label;//for learning bool example_is_test(example& ec); diff --git a/vowpalwabbit/cb_adf.cc b/vowpalwabbit/cb_adf.cc index 1ce6aa8e33b..d07903257ae 100644 --- a/vowpalwabbit/cb_adf.cc +++ b/vowpalwabbit/cb_adf.cc @@ -35,12 +35,12 @@ struct cb_adf action_scores a_s;//temporary storage for mtr - uint64_t offset; + uint64_t offset; bool predict; bool rank_all; }; - CB::cb_class get_observed_cost(v_array& examples) +CB::cb_class get_observed_cost(v_array& examples) { CB::label ld; ld.costs = v_init(); int index = -1; @@ -79,14 +79,12 @@ void learn_IPS(cb_adf& mydata, base_learner& base, v_array& examples) } void learn_DR(cb_adf& mydata, base_learner& base, v_array& examples) -{ - gen_cs_example_dr(mydata.gen_cs, examples, mydata.cs_labels); +{ gen_cs_example_dr(mydata.gen_cs, examples, mydata.cs_labels); call_cs_ldf(base, examples, mydata.cb_labels, mydata.cs_labels, mydata.prepped_cs_labels, mydata.offset); } - void learn_DM(cb_adf& mydata, base_learner& base, v_array& examples) -{ - gen_cs_example_dm(examples, mydata.cs_labels); +void learn_DM(cb_adf& mydata, base_learner& base, v_array& examples) +{ gen_cs_example_dm(examples, mydata.cs_labels); call_cs_ldf(base, examples, mydata.cb_labels, mydata.cs_labels, mydata.prepped_cs_labels, mydata.offset); } @@ -136,43 +134,42 @@ bool test_adf_sequence(cb_adf& data) template void do_actual_learning(cb_adf& data, base_learner& base) -{ - data.gen_cs.known_cost = get_observed_cost(data.ec_seq);//need to set for test case +{ data.gen_cs.known_cost = get_observed_cost(data.ec_seq);//need to set for test case if (is_learn && !test_adf_sequence(data)) - { /* v_array temp_scores; - temp_scores = v_init(); - do_actual_learning(data,base); - for (size_t i = 0; i < data.ec_seq[0]->pred.a_s.size(); i++) - temp_scores.push_back(data.ec_seq[0]->pred.a_s[i].score);*/ - switch (data.gen_cs.cb_type) - { case CB_TYPE_IPS: - learn_IPS(data, base, data.ec_seq); - break; - case CB_TYPE_DR: - learn_DR(data, base, data.ec_seq); - break; - case CB_TYPE_DM: - learn_DM(data, base, data.ec_seq); - break; - case CB_TYPE_MTR: - if (data.predict) - learn_MTR(data, base, data.ec_seq); - else - learn_MTR(data, base, data.ec_seq); - break; - default: - THROW("Unknown cb_type specified for contextual bandit learning: " << data.gen_cs.cb_type); - } - - /* for (size_t i = 0; i < temp_scores.size(); i++) - if (temp_scores[i] != data.ec_seq[0]->pred.a_s[i].score) - cout << "problem! " << temp_scores[i] << " != " << data.ec_seq[0]->pred.a_s[i].score << " for " << data.ec_seq[0]->pred.a_s[i].action << endl; - temp_scores.delete_v();*/ - } - else - { gen_cs_test_example(data.ec_seq, data.cs_labels);//create test labels. - call_cs_ldf(base, data.ec_seq, data.cb_labels, data.cs_labels, data.prepped_cs_labels, data.offset); + { /* v_array temp_scores; + temp_scores = v_init(); + do_actual_learning(data,base); + for (size_t i = 0; i < data.ec_seq[0]->pred.a_s.size(); i++) + temp_scores.push_back(data.ec_seq[0]->pred.a_s[i].score);*/ + switch (data.gen_cs.cb_type) + { case CB_TYPE_IPS: + learn_IPS(data, base, data.ec_seq); + break; + case CB_TYPE_DR: + learn_DR(data, base, data.ec_seq); + break; + case CB_TYPE_DM: + learn_DM(data, base, data.ec_seq); + break; + case CB_TYPE_MTR: + if (data.predict) + learn_MTR(data, base, data.ec_seq); + else + learn_MTR(data, base, data.ec_seq); + break; + default: + THROW("Unknown cb_type specified for contextual bandit learning: " << data.gen_cs.cb_type); } + + /* for (size_t i = 0; i < temp_scores.size(); i++) + if (temp_scores[i] != data.ec_seq[0]->pred.a_s[i].score) + cout << "problem! " << temp_scores[i] << " != " << data.ec_seq[0]->pred.a_s[i].score << " for " << data.ec_seq[0]->pred.a_s[i].action << endl; + temp_scores.delete_v();*/ + } + else + { gen_cs_test_example(data.ec_seq, data.cs_labels);//create test labels. + call_cs_ldf(base, data.ec_seq, data.cb_labels, data.cs_labels, data.prepped_cs_labels, data.offset); + } } void global_print_newline(vw& all) @@ -424,7 +421,7 @@ base_learner* cb_adf_setup(vw& all) all.label_type = label_type::cb; learner& l = init_learner(&ld, base, CB_ADF::predict_or_learn, CB_ADF::predict_or_learn, problem_multiplier, - prediction_type::action_scores); + prediction_type::action_scores); l.set_finish_example(CB_ADF::finish_multiline_example); ld.gen_cs.scorer = all.scorer; diff --git a/vowpalwabbit/cb_adf.h b/vowpalwabbit/cb_adf.h index 9e23e6f4809..e21e831292b 100644 --- a/vowpalwabbit/cb_adf.h +++ b/vowpalwabbit/cb_adf.h @@ -2,7 +2,8 @@ LEARNER::base_learner* cb_adf_setup(vw& all); -namespace CB_ADF { - CB::cb_class get_observed_cost(v_array& examples); - void global_print_newline(vw& all); +namespace CB_ADF +{ +CB::cb_class get_observed_cost(v_array& examples); +void global_print_newline(vw& all); } diff --git a/vowpalwabbit/cb_algs.cc b/vowpalwabbit/cb_algs.cc index 289ffe4dcf8..815c96dc32f 100644 --- a/vowpalwabbit/cb_algs.cc +++ b/vowpalwabbit/cb_algs.cc @@ -16,10 +16,10 @@ using namespace std; using namespace CB; using namespace GEN_CS; -namespace CB_ALGS { -struct cb +namespace CB_ALGS { - cb_to_cs cbcs; +struct cb +{ cb_to_cs cbcs; COST_SENSITIVE::label cb_cs_ld; }; @@ -105,8 +105,7 @@ void output_example(vw& all, cb& data, example& ec, CB::label& ld) } void finish(cb& data) -{ - cb_to_cs& c = data.cbcs; +{ cb_to_cs& c = data.cbcs; data.cb_cs_ld.costs.delete_v(); COST_SENSITIVE::cs_label.delete_label(&c.pred_scores); } @@ -181,11 +180,11 @@ base_learner* cb_algs_setup(vw& all) base_learner* base = setup_base(all); if (eval) { all.p->lp = CB_EVAL::cb_eval; - all.label_type = label_type::cb_eval; + all.label_type = label_type::cb_eval; } else { all.p->lp = CB::cb_label; - all.label_type = label_type::cb; + all.label_type = label_type::cb; } learner* l; diff --git a/vowpalwabbit/cb_algs.h b/vowpalwabbit/cb_algs.h index d93cc720c71..ffe7050e896 100644 --- a/vowpalwabbit/cb_algs.h +++ b/vowpalwabbit/cb_algs.h @@ -43,23 +43,21 @@ float get_cost_pred(LEARNER::base_learner* scorer, CB::cb_class* known_cost, exa return pred; } -inline float get_unbiased_cost(CB::cb_class* observation, uint32_t action, float offset = 0.) -{ - if (action == observation->action) +inline float get_unbiased_cost(CB::cb_class* observation, uint32_t action, float offset = 0.) +{ if (action == observation->action) return (observation->cost - offset) / observation->probability; return 0.; } inline float get_unbiased_cost(CB::cb_class* observation, COST_SENSITIVE::label& scores, uint32_t action) -{ - for (auto& cl : scores.costs) +{ for (auto& cl : scores.costs) if (cl.class_index == action) return get_unbiased_cost(observation, action, cl.x) + cl.x; return get_unbiased_cost(observation, action); } inline bool example_is_newline_not_header(example& ec) - { return (example_is_newline(ec) && !CB::ec_is_example_header(ec)); } +{ return (example_is_newline(ec) && !CB::ec_is_example_header(ec)); } } diff --git a/vowpalwabbit/cb_explore.cc b/vowpalwabbit/cb_explore.cc index d02a420f5d4..dfb81b84071 100644 --- a/vowpalwabbit/cb_explore.cc +++ b/vowpalwabbit/cb_explore.cc @@ -11,289 +11,283 @@ using namespace std; using namespace CB_ALGS; //All exploration algorithms return a vector of probabilities, to be used by GenericExplorer downstream -namespace CB_EXPLORE{ +namespace CB_EXPLORE +{ - struct cb_explore - { - cb_to_cs cbcs; - v_array preds; - v_array cover_probs; +struct cb_explore +{ cb_to_cs cbcs; + v_array preds; + v_array cover_probs; - CB::label cb_label; - COST_SENSITIVE::label cs_label; - COST_SENSITIVE::label second_cs_label; + CB::label cb_label; + COST_SENSITIVE::label cs_label; + COST_SENSITIVE::label second_cs_label; - base_learner* cs; + base_learner* cs; - size_t tau; - float epsilon; - size_t bag_size; - size_t cover_size; + size_t tau; + float epsilon; + size_t bag_size; + size_t cover_size; - size_t counter; + size_t counter; - }; +}; - template - void predict_or_learn_first(cb_explore& data, base_learner& base, example& ec) - { //Explore tau times, then act according to optimal. - v_array probs = ec.pred.a_s; +template +void predict_or_learn_first(cb_explore& data, base_learner& base, example& ec) +{ //Explore tau times, then act according to optimal. + v_array probs = ec.pred.a_s; - if (is_learn && ec.l.cb.costs[0].probability < 1) - base.learn(ec); - else - base.predict(ec); - - probs.erase(); - if(data.tau > 0) - { - float prob = 1.f/(float)data.cbcs.num_actions; - for(uint32_t i = 0;i < data.cbcs.num_actions;i++) - probs.push_back({i,prob}); - data.tau--; - } - else - { - uint32_t chosen = ec.pred.multiclass-1; - for(uint32_t i = 0;i < data.cbcs.num_actions;i++) - probs.push_back({i,0.}); - probs[chosen].score = 1.0; - } + if (is_learn && ec.l.cb.costs[0].probability < 1) + base.learn(ec); + else + base.predict(ec); - ec.pred.a_s = probs; + probs.erase(); + if(data.tau > 0) + { float prob = 1.f/(float)data.cbcs.num_actions; + for(uint32_t i = 0; i < data.cbcs.num_actions; i++) + probs.push_back({i,prob}); + data.tau--; + } + else + { uint32_t chosen = ec.pred.multiclass-1; + for(uint32_t i = 0; i < data.cbcs.num_actions; i++) + probs.push_back({i,0.}); + probs[chosen].score = 1.0; } - template - void predict_or_learn_greedy(cb_explore& data, base_learner& base, example& ec) - { //Explore uniform random an epsilon fraction of the time. + ec.pred.a_s = probs; +} - v_array probs = ec.pred.a_s; - probs.erase(); +template +void predict_or_learn_greedy(cb_explore& data, base_learner& base, example& ec) +{ //Explore uniform random an epsilon fraction of the time. - if (is_learn) - base.learn(ec); - else - base.predict(ec); + v_array probs = ec.pred.a_s; + probs.erase(); - float prob = data.epsilon/(float)data.cbcs.num_actions; - for(uint32_t i = 0;i < data.cbcs.num_actions;i++) - probs.push_back({i,prob}); - uint32_t chosen = ec.pred.multiclass-1; - probs[chosen].score += (1-data.epsilon); + if (is_learn) + base.learn(ec); + else + base.predict(ec); - ec.pred.a_s = probs; - } + float prob = data.epsilon/(float)data.cbcs.num_actions; + for(uint32_t i = 0; i < data.cbcs.num_actions; i++) + probs.push_back({i,prob}); + uint32_t chosen = ec.pred.multiclass-1; + probs[chosen].score += (1-data.epsilon); - template - void predict_or_learn_bag(cb_explore& data, base_learner& base, example& ec) - { //Randomize over predictions from a base set of predictors + ec.pred.a_s = probs; +} - v_array probs = ec.pred.a_s; - probs.erase(); +template +void predict_or_learn_bag(cb_explore& data, base_learner& base, example& ec) +{ //Randomize over predictions from a base set of predictors - for(uint32_t i = 0;i < data.cbcs.num_actions;i++) - probs.push_back({i,0.}); - float prob = 1.f/(float)data.bag_size; - for(size_t i = 0;i < data.bag_size;i++) { - uint32_t count = BS::weight_gen(); - if (is_learn && count > 0) - base.learn(ec,i); - else - base.predict(ec, i); - uint32_t chosen = ec.pred.multiclass-1; - probs[chosen].score += prob; - if (is_learn) - for (uint32_t j = 1; j < count; j++) - base.learn(ec,i); - } + v_array probs = ec.pred.a_s; + probs.erase(); - ec.pred.a_s = probs; + for(uint32_t i = 0; i < data.cbcs.num_actions; i++) + probs.push_back({i,0.}); + float prob = 1.f/(float)data.bag_size; + for(size_t i = 0; i < data.bag_size; i++) + { uint32_t count = BS::weight_gen(); + if (is_learn && count > 0) + base.learn(ec,i); + else + base.predict(ec, i); + uint32_t chosen = ec.pred.multiclass-1; + probs[chosen].score += prob; + if (is_learn) + for (uint32_t j = 1; j < count; j++) + base.learn(ec,i); } - void safety(v_array& distribution, float min_prob, bool zeros) - { //input: a probability distribution - //output: a probability distribution with all events having probability > min_prob. This includes events with probability 0 if zeros = true - min_prob /= distribution.size(); - float touched_mass = 0.; - float untouched_mass = 0.; + ec.pred.a_s = probs; +} + +void safety(v_array& distribution, float min_prob, bool zeros) +{ //input: a probability distribution + //output: a probability distribution with all events having probability > min_prob. This includes events with probability 0 if zeros = true + min_prob /= distribution.size(); + float touched_mass = 0.; + float untouched_mass = 0.; + for (uint32_t i = 0; i < distribution.size(); i++) + if ((distribution[i].score > 0 || (distribution[i].score ==0 && zeros)) && distribution[i].score <= min_prob) + { touched_mass += min_prob; + distribution[i].score = min_prob; + } + else + untouched_mass += distribution[i].score; + + if (touched_mass > 0.) + { if (touched_mass > 0.999) + THROW("Cannot safety this distribution"); + float ratio = (1.f - touched_mass) / untouched_mass; for (uint32_t i = 0; i < distribution.size(); i++) - if ((distribution[i].score > 0 || (distribution[i].score ==0 && zeros)) && distribution[i].score <= min_prob) - { touched_mass += min_prob; - distribution[i].score = min_prob; - } - else - untouched_mass += distribution[i].score; - - if (touched_mass > 0.) - { - if (touched_mass > 0.999) - THROW("Cannot safety this distribution"); - float ratio = (1.f - touched_mass) / untouched_mass; - for (uint32_t i = 0; i < distribution.size(); i++) - if (distribution[i].score > min_prob) - distribution[i].score = distribution[i].score * ratio; - } + if (distribution[i].score > min_prob) + distribution[i].score = distribution[i].score * ratio; } +} - void get_cover_probabilities(cb_explore& data, base_learner& base, example& ec, v_array& probs) - { - float additive_probability = 1.f / (float)data.cover_size; - data.preds.erase(); +void get_cover_probabilities(cb_explore& data, base_learner& base, example& ec, v_array& probs) +{ float additive_probability = 1.f / (float)data.cover_size; + data.preds.erase(); - for(uint32_t i = 0;i < data.cbcs.num_actions;i++) - probs.push_back({i,0.}); + for(uint32_t i = 0; i < data.cbcs.num_actions; i++) + probs.push_back({i,0.}); - for (size_t i = 0; i < data.cover_size; i++) - { //get predicted cost-sensitive predictions - if (i == 0) - data.cs->predict(ec, i); - else - data.cs->predict(ec, i + 1); - uint32_t pred = ec.pred.multiclass; - probs[pred - 1].score += additive_probability; - data.preds.push_back((uint32_t)pred); - } - uint32_t num_actions = data.cbcs.num_actions; - float epsilon = data.epsilon; + for (size_t i = 0; i < data.cover_size; i++) + { //get predicted cost-sensitive predictions + if (i == 0) + data.cs->predict(ec, i); + else + data.cs->predict(ec, i + 1); + uint32_t pred = ec.pred.multiclass; + probs[pred - 1].score += additive_probability; + data.preds.push_back((uint32_t)pred); + } + uint32_t num_actions = data.cbcs.num_actions; + float epsilon = data.epsilon; - float min_prob = epsilon * min(1.f / num_actions, 1.f / (float)sqrt(data.counter * num_actions)); + float min_prob = epsilon * min(1.f / num_actions, 1.f / (float)sqrt(data.counter * num_actions)); - safety(probs, min_prob*num_actions, false); + safety(probs, min_prob*num_actions, false); - data.counter++; - } + data.counter++; +} - template - void predict_or_learn_cover(cb_explore& data, base_learner& base, example& ec) - { //Randomize over predictions from a base set of predictors - //Use cost sensitive oracle to cover actions to form distribution. +template +void predict_or_learn_cover(cb_explore& data, base_learner& base, example& ec) +{ //Randomize over predictions from a base set of predictors + //Use cost sensitive oracle to cover actions to form distribution. - uint32_t num_actions = data.cbcs.num_actions; + uint32_t num_actions = data.cbcs.num_actions; - v_array probs = ec.pred.a_s; - probs.erase(); - data.cs_label.costs.erase(); + v_array probs = ec.pred.a_s; + probs.erase(); + data.cs_label.costs.erase(); - for (uint32_t j = 0; j < num_actions; j++) - data.cs_label.costs.push_back({FLT_MAX,j+1,0.,0.}); - - float epsilon = data.epsilon; - size_t cover_size = data.cover_size; - size_t counter = data.counter; - v_array& probabilities = data.cover_probs; - v_array& predictions = data.preds; - - float additive_probability = 1.f / (float)cover_size; - - float min_prob = epsilon * min(1.f / num_actions, 1.f / (float)sqrt(counter * num_actions)); - - data.cb_label = ec.l.cb; - - ec.l.cs = data.cs_label; - get_cover_probabilities(data, base, ec, probs); - - if (is_learn) { - ec.l.cb = data.cb_label; - base.learn(ec); - - //Now update oracles - - //1. Compute loss vector - data.cs_label.costs.erase(); - float norm = min_prob * num_actions; - ec.l.cb = data.cb_label; - data.cbcs.known_cost = get_observed_cost(data.cb_label); - gen_cs_example(data.cbcs, ec, data.cb_label, data.cs_label); - for(uint32_t i = 0;i < num_actions;i++) - probabilities[i] = 0; - - ec.l.cs = data.second_cs_label; - //2. Update functions - for (size_t i = 0; i < cover_size; i++) - { //Create costs of each action based on online cover - for (uint32_t j = 0; j < num_actions; j++) - { float pseudo_cost = data.cs_label.costs[j].x - epsilon * min_prob / (max(probabilities[j], min_prob) / norm) + 1; - data.second_cs_label.costs[j].class_index = j+1; - data.second_cs_label.costs[j].x = pseudo_cost; - //cout<learn(ec,i+1); - if (probabilities[predictions[i] - 1] < min_prob) - norm += max(0, additive_probability - (min_prob - probabilities[predictions[i] - 1])); - else - norm += additive_probability; - probabilities[predictions[i] - 1] += additive_probability; - } - } + for (uint32_t j = 0; j < num_actions; j++) + data.cs_label.costs.push_back({FLT_MAX,j+1,0.,0.}); - ec.l.cb = data.cb_label; - ec.pred.a_s = probs; - } + float epsilon = data.epsilon; + size_t cover_size = data.cover_size; + size_t counter = data.counter; + v_array& probabilities = data.cover_probs; + v_array& predictions = data.preds; - void finish(cb_explore& data) - { data.preds.delete_v(); - data.cover_probs.delete_v(); - cb_to_cs& c = data.cbcs; - COST_SENSITIVE::cs_label.delete_label(&c.pred_scores); - COST_SENSITIVE::cs_label.delete_label(&data.cs_label); - COST_SENSITIVE::cs_label.delete_label(&data.second_cs_label); - } + float additive_probability = 1.f / (float)cover_size; - void print_update_cb_explore(vw& all, bool is_test, example& ec, stringstream& pred_string) - { if (all.sd->weighted_examples >= all.sd->dump_interval && !all.quiet && !all.bfgs) - { - stringstream label_string; - if (is_test) - label_string << " unknown"; - else - label_string << ec.l.cb.costs[0].action; - all.sd->print_update(all.holdout_set_off, all.current_pass, label_string.str(), pred_string.str(), ec.num_features, all.progress_add, all.progress_arg); - } - } + float min_prob = epsilon * min(1.f / num_actions, 1.f / (float)sqrt(counter * num_actions)); - void output_example(vw& all, cb_explore& data, example& ec, CB::label& ld) - { float loss = 0.; + data.cb_label = ec.l.cb; - cb_to_cs& c = data.cbcs; + ec.l.cs = data.cs_label; + get_cover_probabilities(data, base, ec, probs); - if ((c.known_cost = get_observed_cost(ld)) != nullptr) - for(uint32_t i = 0;i < ec.pred.a_s.size();i++) - loss += get_unbiased_cost(c.known_cost, c.pred_scores, i)*ec.pred.a_s[i].score; + if (is_learn) + { ec.l.cb = data.cb_label; + base.learn(ec); - all.sd->update(ec.test_only, loss, 1.f, ec.num_features); + //Now update oracles - char temp_str[20]; - stringstream ss, sso; - float maxprob = 0.; - uint32_t maxid; - //cout< maxprob) { - maxprob = ec.pred.a_s[i].score; - maxid = i+1; + //1. Compute loss vector + data.cs_label.costs.erase(); + float norm = min_prob * num_actions; + ec.l.cb = data.cb_label; + data.cbcs.known_cost = get_observed_cost(data.cb_label); + gen_cs_example(data.cbcs, ec, data.cb_label, data.cs_label); + for(uint32_t i = 0; i < num_actions; i++) + probabilities[i] = 0; + + ec.l.cs = data.second_cs_label; + //2. Update functions + for (size_t i = 0; i < cover_size; i++) + { //Create costs of each action based on online cover + for (uint32_t j = 0; j < num_actions; j++) + { float pseudo_cost = data.cs_label.costs[j].x - epsilon * min_prob / (max(probabilities[j], min_prob) / norm) + 1; + data.second_cs_label.costs[j].class_index = j+1; + data.second_cs_label.costs[j].x = pseudo_cost; + //cout<learn(ec,i+1); + if (probabilities[predictions[i] - 1] < min_prob) + norm += max(0, additive_probability - (min_prob - probabilities[predictions[i] - 1])); + else + norm += additive_probability; + probabilities[predictions[i] - 1] += additive_probability; } + } - sprintf(temp_str, "%d:%f", maxid, maxprob); - sso << temp_str; - //cout<weighted_examples >= all.sd->dump_interval && !all.quiet && !all.bfgs) + { stringstream label_string; + if (is_test) + label_string << " unknown"; + else + label_string << ec.l.cb.costs[0].action; + all.sd->print_update(all.holdout_set_off, all.current_pass, label_string.str(), pred_string.str(), ec.num_features, all.progress_add, all.progress_arg); } +} + +void output_example(vw& all, cb_explore& data, example& ec, CB::label& ld) +{ float loss = 0.; + + cb_to_cs& c = data.cbcs; + + if ((c.known_cost = get_observed_cost(ld)) != nullptr) + for(uint32_t i = 0; i < ec.pred.a_s.size(); i++) + loss += get_unbiased_cost(c.known_cost, c.pred_scores, i)*ec.pred.a_s[i].score; - void finish_example(vw& all, cb_explore& c, example& ec) - { - output_example(all, c, ec, ec.l.cb); - VW::finish_example(all, &ec); + all.sd->update(ec.test_only, loss, 1.f, ec.num_features); + + char temp_str[20]; + stringstream ss, sso; + float maxprob = 0.; + uint32_t maxid; + //cout< maxprob) + { maxprob = ec.pred.a_s[i].score; + maxid = i+1; + } } + + sprintf(temp_str, "%d:%f", maxid, maxprob); + sso << temp_str; + //cout<(all, "cb_explore", "Online explore-exploit for a action contextual bandit problem")) return nullptr; new_options(all, "CB_EXPLORE options") - ("first", po::value(), "tau-first exploration") - ("epsilon",po::value() ,"epsilon-greedy exploration") - ("bag",po::value() ,"bagging-based exploration") - ("cover",po::value() ,"Online cover based exploration"); + ("first", po::value(), "tau-first exploration") + ("epsilon",po::value() ,"epsilon-greedy exploration") + ("bag",po::value() ,"bagging-based exploration") + ("cover",po::value() ,"Online cover based exploration"); add_options(all); po::variables_map& vm = all.vm; @@ -315,11 +309,11 @@ base_learner* cb_explore_setup(vw& all) uint32_t num_actions = data.cbcs.num_actions; if (count(all.args.begin(), all.args.end(),"--cb") == 0) - { all.args.push_back("--cb"); - stringstream ss; - ss << vm["cb_explore"].as(); - all.args.push_back(ss.str()); - } + { all.args.push_back("--cb"); + stringstream ss; + ss << vm["cb_explore"].as(); + all.args.push_back(ss.str()); + } char type_string[30]; @@ -335,44 +329,44 @@ base_learner* cb_explore_setup(vw& all) learner* l; if (vm.count("cover")) - { data.cover_size = (uint32_t)vm["cover"].as(); - data.cs = all.cost_sensitive; - data.second_cs_label.costs.resize(num_actions); - data.second_cs_label.costs.end() = data.second_cs_label.costs.begin()+num_actions; - data.epsilon = 0.05f; - sprintf(type_string, "%lu", data.cover_size); - *all.file_options << " --cover " << type_string; - - if (vm.count("epsilon")) - data.epsilon = vm["epsilon"].as(); - data.cover_probs = v_init(); - data.cover_probs.resize(num_actions); - data.preds = v_init(); - data.preds.resize(data.cover_size); - sprintf(type_string, "%f", data.epsilon); - *all.file_options << " --epsilon " << type_string; - l = &init_learner(&data, base, predict_or_learn_cover, predict_or_learn_cover, data.cover_size + 1, prediction_type::action_probs); - } + { data.cover_size = (uint32_t)vm["cover"].as(); + data.cs = all.cost_sensitive; + data.second_cs_label.costs.resize(num_actions); + data.second_cs_label.costs.end() = data.second_cs_label.costs.begin()+num_actions; + data.epsilon = 0.05f; + sprintf(type_string, "%lu", data.cover_size); + *all.file_options << " --cover " << type_string; + + if (vm.count("epsilon")) + data.epsilon = vm["epsilon"].as(); + data.cover_probs = v_init(); + data.cover_probs.resize(num_actions); + data.preds = v_init(); + data.preds.resize(data.cover_size); + sprintf(type_string, "%f", data.epsilon); + *all.file_options << " --epsilon " << type_string; + l = &init_learner(&data, base, predict_or_learn_cover, predict_or_learn_cover, data.cover_size + 1, prediction_type::action_probs); + } else if (vm.count("bag")) - { data.bag_size = (uint32_t)vm["bag"].as(); - sprintf(type_string, "%lu", data.bag_size); - *all.file_options << " --bag "<, predict_or_learn_bag, data.bag_size, prediction_type::action_probs); - } + { data.bag_size = (uint32_t)vm["bag"].as(); + sprintf(type_string, "%lu", data.bag_size); + *all.file_options << " --bag "<, predict_or_learn_bag, data.bag_size, prediction_type::action_probs); + } else if (vm.count("first") ) - { data.tau = (uint32_t)vm["first"].as(); - sprintf(type_string, "%lu", data.tau); - *all.file_options << " --first "<, predict_or_learn_first, 1, prediction_type::action_probs); - } + { data.tau = (uint32_t)vm["first"].as(); + sprintf(type_string, "%lu", data.tau); + *all.file_options << " --first "<, predict_or_learn_first, 1, prediction_type::action_probs); + } else - { data.epsilon = 0.05f; - if (vm.count("epsilon")) - data.epsilon = vm["epsilon"].as(); - sprintf(type_string, "%f", data.epsilon); - *all.file_options << " --epsilon "<, predict_or_learn_greedy, 1, prediction_type::action_probs); - } + { data.epsilon = 0.05f; + if (vm.count("epsilon")) + data.epsilon = vm["epsilon"].as(); + sprintf(type_string, "%f", data.epsilon); + *all.file_options << " --epsilon "<, predict_or_learn_greedy, 1, prediction_type::action_probs); + } data.cbcs.scorer = all.scorer; l->set_finish(finish); l->set_finish_example(finish_example); diff --git a/vowpalwabbit/cb_explore.h b/vowpalwabbit/cb_explore.h old mode 100755 new mode 100644 index af958448aa7..57f65cc5968 --- a/vowpalwabbit/cb_explore.h +++ b/vowpalwabbit/cb_explore.h @@ -7,11 +7,12 @@ license as described in the file LICENSE. namespace LEARNER { template struct learner; - typedef learner base_learner; +typedef learner base_learner; } LEARNER::base_learner* cb_explore_setup(vw& all); -namespace CB_EXPLORE{ - void safety(v_array& distribution, float min_prob, bool zeros); +namespace CB_EXPLORE +{ +void safety(v_array& distribution, float min_prob, bool zeros); } diff --git a/vowpalwabbit/cb_explore_adf.cc b/vowpalwabbit/cb_explore_adf.cc index 5f3a30079c1..51b4a61aaa9 100644 --- a/vowpalwabbit/cb_explore_adf.cc +++ b/vowpalwabbit/cb_explore_adf.cc @@ -9,7 +9,7 @@ using namespace LEARNER; using namespace ACTION_SCORE; using namespace std; using namespace CB_ALGS; -//All exploration algorithms return a vector of id, probability tuples, sorted in order of scores. The probabilities are the probability with which each action should be replaced to the top of the list. +//All exploration algorithms return a vector of id, probability tuples, sorted in order of scores. The probabilities are the probability with which each action should be replaced to the top of the list. //tau first #define EXPLORE_FIRST 0 @@ -22,454 +22,423 @@ using namespace CB_ALGS; //cover #define COVER 4 -namespace CB_EXPLORE_ADF{ +namespace CB_EXPLORE_ADF +{ - struct cb_explore_adf - { - v_array ec_seq; - v_array action_probs; +struct cb_explore_adf +{ v_array ec_seq; + v_array action_probs; - size_t explore_type; + size_t explore_type; - size_t tau; - float epsilon; - size_t bag_size; - size_t cover_size; - float lambda; - uint64_t offset; + size_t tau; + float epsilon; + size_t bag_size; + size_t cover_size; + float lambda; + uint64_t offset; - bool need_to_clear; - vw* all; - LEARNER::base_learner* cs_ldf_learner; + bool need_to_clear; + vw* all; + LEARNER::base_learner* cs_ldf_learner; - GEN_CS::cb_to_cs_adf gen_cs; - COST_SENSITIVE::label cs_labels; - v_array cb_labels; + GEN_CS::cb_to_cs_adf gen_cs; + COST_SENSITIVE::label cs_labels; + v_array cb_labels; - CB::label action_label; - CB::label empty_label; + CB::label action_label; + CB::label empty_label; - COST_SENSITIVE::label cs_labels_2; + COST_SENSITIVE::label cs_labels_2; - v_array prepped_cs_labels; - }; + v_array prepped_cs_labels; +}; - template void swap(T& ele1, T& ele2) - { - T temp = ele2; - ele2 = ele1; - ele1 = temp; - } - template - void multiline_learn_or_predict(base_learner& base, v_array& examples, uint64_t offset, uint32_t id = 0) - { for (example* ec : examples) - { - uint64_t old_offset = ec->ft_offset; - ec->ft_offset = offset; - if (is_learn) - base.learn(*ec, id); - else - base.predict(*ec, id); - ec->ft_offset = old_offset; - } - } - - example* test_adf_sequence(v_array& ec_seq) - { - uint32_t count = 0; - example* ret = nullptr; - for (size_t k = 0; k < ec_seq.size(); k++) - { - example *ec = ec_seq[k]; - - if (ec->l.cb.costs.size() > 1) - THROW("cb_adf: badly formatted example, only one cost can be known."); - - if (ec->l.cb.costs.size() == 1 && ec->l.cb.costs[0].cost != FLT_MAX) - { - ret = ec; - count += 1; - } - - if (CB::ec_is_example_header(*ec)) - if (k != 0) - THROW("warning: example headers at position " << k << ": can only have in initial position!"); - } - if (count == 0 || count == 1) - return ret; +template void swap(T& ele1, T& ele2) +{ T temp = ele2; + ele2 = ele1; + ele1 = temp; +} +template +void multiline_learn_or_predict(base_learner& base, v_array& examples, uint64_t offset, uint32_t id = 0) +{ for (example* ec : examples) + { uint64_t old_offset = ec->ft_offset; + ec->ft_offset = offset; + if (is_learn) + base.learn(*ec, id); else - THROW("cb_adf: badly formatted example, only one line can have a cost"); + base.predict(*ec, id); + ec->ft_offset = old_offset; } - - template - void predict_or_learn_first(cb_explore_adf& data, base_learner& base, v_array& examples) - { //Explore tau times, then act according to optimal. - if (is_learn && data.gen_cs.known_cost.probability < 1 && test_adf_sequence(data.ec_seq) != nullptr) - multiline_learn_or_predict(base, examples, data.offset); - else - multiline_learn_or_predict(base, examples, data.offset); +} - v_array& preds = examples[0]->pred.a_s; - uint32_t num_actions = (uint32_t)preds.size(); +example* test_adf_sequence(v_array& ec_seq) +{ uint32_t count = 0; + example* ret = nullptr; + for (size_t k = 0; k < ec_seq.size(); k++) + { example *ec = ec_seq[k]; - if (data.tau) { - float prob = 1.f / (float)num_actions; - for (size_t i = 0; i < num_actions; i++) - preds[i].score = prob; - data.tau--; - } - else { - for (size_t i = 1; i < num_actions; i++) - preds[i].score = 0.; - preds[0].score = 1.0; + if (ec->l.cb.costs.size() > 1) + THROW("cb_adf: badly formatted example, only one cost can be known."); + + if (ec->l.cb.costs.size() == 1 && ec->l.cb.costs[0].cost != FLT_MAX) + { ret = ec; + count += 1; } - CB_EXPLORE::safety(preds, data.epsilon, true); + + if (CB::ec_is_example_header(*ec)) + if (k != 0) + THROW("warning: example headers at position " << k << ": can only have in initial position!"); } - - template - void predict_or_learn_greedy(cb_explore_adf& data, base_learner& base, v_array& examples) - { //Explore uniform random an epsilon fraction of the time. - if (is_learn && test_adf_sequence(data.ec_seq) != nullptr) - multiline_learn_or_predict(base, examples, data.offset); - else - multiline_learn_or_predict(base, examples, data.offset); + if (count == 0 || count == 1) + return ret; + else + THROW("cb_adf: badly formatted example, only one line can have a cost"); +} - v_array& preds = examples[0]->pred.a_s; - uint32_t num_actions = (uint32_t)preds.size(); - float prob = data.epsilon/(float)num_actions; +template +void predict_or_learn_first(cb_explore_adf& data, base_learner& base, v_array& examples) +{ //Explore tau times, then act according to optimal. + if (is_learn && data.gen_cs.known_cost.probability < 1 && test_adf_sequence(data.ec_seq) != nullptr) + multiline_learn_or_predict(base, examples, data.offset); + else + multiline_learn_or_predict(base, examples, data.offset); + + v_array& preds = examples[0]->pred.a_s; + uint32_t num_actions = (uint32_t)preds.size(); + + if (data.tau) + { float prob = 1.f / (float)num_actions; for (size_t i = 0; i < num_actions; i++) preds[i].score = prob; - preds[0].score += 1.f - data.epsilon; + data.tau--; + } + else + { for (size_t i = 1; i < num_actions; i++) + preds[i].score = 0.; + preds[0].score = 1.0; } + CB_EXPLORE::safety(preds, data.epsilon, true); +} - template - void predict_or_learn_bag(cb_explore_adf& data, base_learner& base, v_array& examples) - { //Randomize over predictions from a base set of predictors - v_array& preds = examples[0]->pred.a_s; - uint32_t num_actions = (uint32_t)(examples.size() - 1); - if (CB::ec_is_example_header(*examples[0])) - num_actions--; - - data.action_probs.resize(num_actions); - data.action_probs.erase(); - for (uint32_t i = 0; i < num_actions; i++) - data.action_probs.push_back({ i,0. }); - float prob = 1.f / (float)data.bag_size; - bool test_sequence = test_adf_sequence(data.ec_seq) == nullptr; - for (uint32_t i = 0; i < data.bag_size; i++) - { - uint32_t count = BS::weight_gen(); - if (is_learn && count > 0 && !test_sequence) - multiline_learn_or_predict(base, examples, data.offset, i); - else - multiline_learn_or_predict(base, examples, data.offset, i); - assert(preds.size() == num_actions); - data.action_probs[preds[0].action].score += prob; - if (is_learn && !test_sequence) - for (uint32_t j = 1; j < count; j++) - multiline_learn_or_predict(base, examples, data.offset, i); - } +template +void predict_or_learn_greedy(cb_explore_adf& data, base_learner& base, v_array& examples) +{ //Explore uniform random an epsilon fraction of the time. + if (is_learn && test_adf_sequence(data.ec_seq) != nullptr) + multiline_learn_or_predict(base, examples, data.offset); + else + multiline_learn_or_predict(base, examples, data.offset); + + v_array& preds = examples[0]->pred.a_s; + uint32_t num_actions = (uint32_t)preds.size(); + float prob = data.epsilon/(float)num_actions; + for (size_t i = 0; i < num_actions; i++) + preds[i].score = prob; + preds[0].score += 1.f - data.epsilon; +} - CB_EXPLORE::safety(data.action_probs, data.epsilon, true); - qsort((void*) data.action_probs.begin(), data.action_probs.size(), sizeof(action_score), reverse_order); +template +void predict_or_learn_bag(cb_explore_adf& data, base_learner& base, v_array& examples) +{ //Randomize over predictions from a base set of predictors + v_array& preds = examples[0]->pred.a_s; + uint32_t num_actions = (uint32_t)(examples.size() - 1); + if (CB::ec_is_example_header(*examples[0])) + num_actions--; + + data.action_probs.resize(num_actions); + data.action_probs.erase(); + for (uint32_t i = 0; i < num_actions; i++) + data.action_probs.push_back({ i,0. }); + float prob = 1.f / (float)data.bag_size; + bool test_sequence = test_adf_sequence(data.ec_seq) == nullptr; + for (uint32_t i = 0; i < data.bag_size; i++) + { uint32_t count = BS::weight_gen(); + if (is_learn && count > 0 && !test_sequence) + multiline_learn_or_predict(base, examples, data.offset, i); + else + multiline_learn_or_predict(base, examples, data.offset, i); + assert(preds.size() == num_actions); + data.action_probs[preds[0].action].score += prob; + if (is_learn && !test_sequence) + for (uint32_t j = 1; j < count; j++) + multiline_learn_or_predict(base, examples, data.offset, i); + } + + CB_EXPLORE::safety(data.action_probs, data.epsilon, true); + qsort((void*) data.action_probs.begin(), data.action_probs.size(), sizeof(action_score), reverse_order); - for (size_t i = 0; i < num_actions; i++) - preds[i] = data.action_probs[i]; + for (size_t i = 0; i < num_actions; i++) + preds[i] = data.action_probs[i]; +} + +template +void predict_or_learn_cover(cb_explore_adf& data, base_learner& base, v_array& examples) +{ //Randomize over predictions from a base set of predictors + //Use cost sensitive oracle to cover actions to form distribution. + if (is_learn) + { GEN_CS::gen_cs_example(data.gen_cs, examples, data.cs_labels); + multiline_learn_or_predict(base, examples, data.offset); + } + else + { GEN_CS::gen_cs_example_ips(examples, data.cs_labels); + multiline_learn_or_predict(base, examples, data.offset); } - template - void predict_or_learn_cover(cb_explore_adf& data, base_learner& base, v_array& examples) - { //Randomize over predictions from a base set of predictors - //Use cost sensitive oracle to cover actions to form distribution. + v_array& preds = examples[0]->pred.a_s; + uint32_t num_actions = (uint32_t)preds.size(); + + float additive_probability = 1.f / (float)data.cover_size; + float min_prob = data.epsilon / num_actions; + v_array& probs = data.action_probs; + probs.erase(); + for(uint32_t i = 0; i < num_actions; i++) + probs.push_back({i,0.}); + + probs[preds[0].action].score += additive_probability; + + uint32_t shared = CB::ec_is_example_header(*examples[0]) ? 1 : 0; + + float norm = min_prob * num_actions + (additive_probability - min_prob); + for (size_t i = 1; i < data.cover_size; i++) + { //Create costs of each action based on online cover if (is_learn) - { - GEN_CS::gen_cs_example(data.gen_cs, examples, data.cs_labels); - multiline_learn_or_predict(base, examples, data.offset); + { data.cs_labels_2.costs.erase(); + if (shared > 0) + data.cs_labels_2.costs.push_back(data.cs_labels.costs[0]); + for (uint32_t j = 0; j < num_actions; j++) + { float pseudo_cost = data.cs_labels.costs[j+shared].x - data.epsilon * min_prob / (max(probs[j].score, min_prob) / norm); + data.cs_labels_2.costs.push_back({pseudo_cost,j,0.,0.}); } + GEN_CS::call_cs_ldf(*(data.cs_ldf_learner), examples, data.cb_labels, data.cs_labels_2, data.prepped_cs_labels, data.offset, i+1); + } else - { - GEN_CS::gen_cs_example_ips(examples, data.cs_labels); - multiline_learn_or_predict(base, examples, data.offset); - } + GEN_CS::call_cs_ldf(*(data.cs_ldf_learner), examples, data.cb_labels, data.cs_labels, data.prepped_cs_labels, data.offset, i+1); - v_array& preds = examples[0]->pred.a_s; - uint32_t num_actions = (uint32_t)preds.size(); - - float additive_probability = 1.f / (float)data.cover_size; - float min_prob = data.epsilon / num_actions; - v_array& probs = data.action_probs; - probs.erase(); - for(uint32_t i = 0;i < num_actions;i++) - probs.push_back({i,0.}); - - probs[preds[0].action].score += additive_probability; - - uint32_t shared = CB::ec_is_example_header(*examples[0]) ? 1 : 0; - - float norm = min_prob * num_actions + (additive_probability - min_prob); - for (size_t i = 1; i < data.cover_size; i++) - { //Create costs of each action based on online cover - if (is_learn) - { - data.cs_labels_2.costs.erase(); - if (shared > 0) - data.cs_labels_2.costs.push_back(data.cs_labels.costs[0]); - for (uint32_t j = 0; j < num_actions; j++) - { float pseudo_cost = data.cs_labels.costs[j+shared].x - data.epsilon * min_prob / (max(probs[j].score, min_prob) / norm); - data.cs_labels_2.costs.push_back({pseudo_cost,j,0.,0.}); - } - GEN_CS::call_cs_ldf(*(data.cs_ldf_learner), examples, data.cb_labels, data.cs_labels_2, data.prepped_cs_labels, data.offset, i+1); - } - else - GEN_CS::call_cs_ldf(*(data.cs_ldf_learner), examples, data.cb_labels, data.cs_labels, data.prepped_cs_labels, data.offset, i+1); - - uint32_t action = preds[0].action; - if (probs[action].score < min_prob) - norm += max(0, additive_probability - (min_prob - probs[action].score)); - else - norm += additive_probability; - probs[action].score += additive_probability; - } - - CB_EXPLORE::safety(data.action_probs, data.epsilon, true); - - qsort((void*) probs.begin(), probs.size(), sizeof(action_score), reverse_order); - for (size_t i = 0; i < num_actions; i++) - preds[i] = probs[i]; - } - - template - void predict_or_learn_softmax(cb_explore_adf& data, base_learner& base, v_array& examples) - { - if (is_learn && test_adf_sequence(data.ec_seq) != nullptr) - multiline_learn_or_predict(base, examples, data.offset); + uint32_t action = preds[0].action; + if (probs[action].score < min_prob) + norm += max(0, additive_probability - (min_prob - probs[action].score)); else - multiline_learn_or_predict(base, examples, data.offset); - - v_array& preds = examples[0]->pred.a_s; - uint32_t num_actions = (uint32_t)preds.size(); - float norm = 0.; - float max_score = preds[0].score; - - for (size_t i = 0; i < num_actions; i++) { - float prob = exp(data.lambda*(preds[i].score - max_score)); - preds[i].score = prob; - norm += prob; - } - for (size_t i = 0; i < num_actions; i++) - preds[i].score /= norm; - CB_EXPLORE::safety(preds, data.epsilon, true); - } - - void end_examples(cb_explore_adf& data) - { - if (data.need_to_clear) - data.ec_seq.erase(); + norm += additive_probability; + probs[action].score += additive_probability; } - void finish(cb_explore_adf& data) - { - data.ec_seq.delete_v(); - data.action_probs.delete_v(); - data.cs_labels.costs.delete_v(); - data.cs_labels_2.costs.delete_v(); - data.cb_labels.delete_v(); - for(size_t i = 0; i < data.prepped_cs_labels.size(); i++) - data.prepped_cs_labels[i].costs.delete_v(); - data.prepped_cs_labels.delete_v(); - data.gen_cs.pred_scores.costs.delete_v(); + CB_EXPLORE::safety(data.action_probs, data.epsilon, true); + + qsort((void*) probs.begin(), probs.size(), sizeof(action_score), reverse_order); + for (size_t i = 0; i < num_actions; i++) + preds[i] = probs[i]; +} + +template +void predict_or_learn_softmax(cb_explore_adf& data, base_learner& base, v_array& examples) +{ if (is_learn && test_adf_sequence(data.ec_seq) != nullptr) + multiline_learn_or_predict(base, examples, data.offset); + else + multiline_learn_or_predict(base, examples, data.offset); + + v_array& preds = examples[0]->pred.a_s; + uint32_t num_actions = (uint32_t)preds.size(); + float norm = 0.; + float max_score = preds[0].score; + + for (size_t i = 0; i < num_actions; i++) + { float prob = exp(data.lambda*(preds[i].score - max_score)); + preds[i].score = prob; + norm += prob; } + for (size_t i = 0; i < num_actions; i++) + preds[i].score /= norm; + CB_EXPLORE::safety(preds, data.epsilon, true); +} +void end_examples(cb_explore_adf& data) +{ if (data.need_to_clear) + data.ec_seq.erase(); +} - //Semantics: Currently we compute the IPS loss no matter what flags - //are specified. We print the first action and probability, based on - //ordering by scores in the final output. +void finish(cb_explore_adf& data) +{ data.ec_seq.delete_v(); + data.action_probs.delete_v(); + data.cs_labels.costs.delete_v(); + data.cs_labels_2.costs.delete_v(); + data.cb_labels.delete_v(); + for(size_t i = 0; i < data.prepped_cs_labels.size(); i++) + data.prepped_cs_labels[i].costs.delete_v(); + data.prepped_cs_labels.delete_v(); + data.gen_cs.pred_scores.costs.delete_v(); +} - void output_example(vw& all, cb_explore_adf& c, example& ec, v_array* ec_seq) - { - if (CB_ALGS::example_is_newline_not_header(ec)) return; - size_t num_features = 0; +//Semantics: Currently we compute the IPS loss no matter what flags +//are specified. We print the first action and probability, based on +//ordering by scores in the final output. - float loss = 0.; - ACTION_SCORE::action_scores preds = (*ec_seq)[0]->pred.a_s; +void output_example(vw& all, cb_explore_adf& c, example& ec, v_array* ec_seq) +{ if (CB_ALGS::example_is_newline_not_header(ec)) return; - for (size_t i = 0; i < (*ec_seq).size(); i++) - if (!CB::ec_is_example_header(*(*ec_seq)[i])) { - num_features += (*ec_seq)[i]->num_features; - } + size_t num_features = 0; - all.sd->total_features += num_features; + float loss = 0.; + ACTION_SCORE::action_scores preds = (*ec_seq)[0]->pred.a_s; - bool is_test = false; - if (c.gen_cs.known_cost.probability > 0) { - for (uint32_t i = 0; i < preds.size(); i++) { - float l = get_unbiased_cost(&c.gen_cs.known_cost, preds[i].action); - loss += l*preds[i].score; - } - all.sd->sum_loss += loss; - all.sd->sum_loss_since_last_dump += loss; + for (size_t i = 0; i < (*ec_seq).size(); i++) + if (!CB::ec_is_example_header(*(*ec_seq)[i])) + { num_features += (*ec_seq)[i]->num_features; } - else - is_test = true; - for (int sink : all.final_prediction_sink) - print_action_score(sink, ec.pred.a_s, ec.tag); + all.sd->total_features += num_features; - if (all.raw_prediction > 0) - { - string outputString; - stringstream outputStringStream(outputString); - v_array costs = ec.l.cb.costs; - - for (size_t i = 0; i < costs.size(); i++) - { - if (i > 0) outputStringStream << ' '; - outputStringStream << costs[i].action << ':' << costs[i].partial_prediction; - } - all.print_text(all.raw_prediction, outputStringStream.str(), ec.tag); + bool is_test = false; + if (c.gen_cs.known_cost.probability > 0) + { for (uint32_t i = 0; i < preds.size(); i++) + { float l = get_unbiased_cost(&c.gen_cs.known_cost, preds[i].action); + loss += l*preds[i].score; } - - CB::print_update(all, is_test, ec, ec_seq, true); + all.sd->sum_loss += loss; + all.sd->sum_loss_since_last_dump += loss; } + else + is_test = true; + + for (int sink : all.final_prediction_sink) + print_action_score(sink, ec.pred.a_s, ec.tag); + + if (all.raw_prediction > 0) + { string outputString; + stringstream outputStringStream(outputString); + v_array costs = ec.l.cb.costs; - void output_example_seq(vw& all, cb_explore_adf& data) - { - if (data.ec_seq.size() > 0) - { - all.sd->weighted_examples += 1; - all.sd->example_number++; - output_example(all, data, **(data.ec_seq.begin()), &(data.ec_seq)); - if (all.raw_prediction > 0) - all.print_text(all.raw_prediction, "", data.ec_seq[0]->tag); + for (size_t i = 0; i < costs.size(); i++) + { if (i > 0) outputStringStream << ' '; + outputStringStream << costs[i].action << ':' << costs[i].partial_prediction; } + all.print_text(all.raw_prediction, outputStringStream.str(), ec.tag); } + CB::print_update(all, is_test, ec, ec_seq, true); +} - void clear_seq_and_finish_examples(vw& all, cb_explore_adf& data) - { - if (data.ec_seq.size() > 0) - for (example* ecc : data.ec_seq) - if (ecc->in_use) - VW::finish_example(all, ecc); - data.ec_seq.erase(); +void output_example_seq(vw& all, cb_explore_adf& data) +{ if (data.ec_seq.size() > 0) + { all.sd->weighted_examples += 1; + all.sd->example_number++; + output_example(all, data, **(data.ec_seq.begin()), &(data.ec_seq)); + if (all.raw_prediction > 0) + all.print_text(all.raw_prediction, "", data.ec_seq[0]->tag); } +} - void finish_multiline_example(vw& all, cb_explore_adf& data, example& ec) - { - if (data.need_to_clear) - { - if (data.ec_seq.size() > 0) - { - output_example_seq(all, data); - CB_ADF::global_print_newline(all); - } - clear_seq_and_finish_examples(all, data); - data.need_to_clear = false; + +void clear_seq_and_finish_examples(vw& all, cb_explore_adf& data) +{ if (data.ec_seq.size() > 0) + for (example* ecc : data.ec_seq) + if (ecc->in_use) + VW::finish_example(all, ecc); + data.ec_seq.erase(); +} + +void finish_multiline_example(vw& all, cb_explore_adf& data, example& ec) +{ if (data.need_to_clear) + { if (data.ec_seq.size() > 0) + { output_example_seq(all, data); + CB_ADF::global_print_newline(all); } + clear_seq_and_finish_examples(all, data); + data.need_to_clear = false; } +} - template - void do_actual_learning(cb_explore_adf& data, base_learner& base) - { - example* label_example=test_adf_sequence(data.ec_seq); - data.gen_cs.known_cost = CB_ADF::get_observed_cost(data.ec_seq); - - if (label_example == nullptr || !is_learn) - { - if (label_example != nullptr)//extract label - { - data.action_label = label_example->l.cb; - label_example->l.cb = data.empty_label; - } - switch (data.explore_type) - { - case EXPLORE_FIRST: - predict_or_learn_first(data, base, data.ec_seq); - break; - case EPS_GREEDY: - predict_or_learn_greedy(data, base, data.ec_seq); - break; - case SOFTMAX: - predict_or_learn_softmax(data, base, data.ec_seq); - break; - case BAG_EXPLORE: - predict_or_learn_bag(data, base, data.ec_seq); - break; - case COVER: - predict_or_learn_cover(data, base, data.ec_seq); - break; - default: - THROW("Unknown explorer type specified for contextual bandit learning: " << data.explore_type); - } - if (label_example != nullptr) //restore label - label_example->l.cb = data.action_label; - } - else - { - /* v_array temp_probs; - temp_probs = v_init(); - do_actual_learning(data,base); - for (size_t i = 0; i < data.ec_seq[0]->pred.a_s.size(); i++) - temp_probs.push_back(data.ec_seq[0]->pred.a_s[i].score);*/ - - switch (data.explore_type) - { - case EXPLORE_FIRST: - predict_or_learn_first(data, base, data.ec_seq); - break; - case EPS_GREEDY: - predict_or_learn_greedy(data, base, data.ec_seq); - break; - case SOFTMAX: - predict_or_learn_softmax(data, base, data.ec_seq); - break; - case BAG_EXPLORE: - predict_or_learn_bag(data, base, data.ec_seq); - break; - case COVER: - predict_or_learn_cover(data, base, data.ec_seq); - break; - default: - THROW("Unknown explorer type specified for contextual bandit learning: " << data.explore_type); - } - - /* for (size_t i = 0; i < temp_probs.size(); i++) - if (temp_probs[i] != data.ec_seq[0]->pred.a_s[i].score) - cout << "problem! " << temp_probs[i] << " != " << data.ec_seq[0]->pred.a_s[i].score << " for " << data.ec_seq[0]->pred.a_s[i].action << endl; - temp_probs.delete_v();*/ - } - } +template +void do_actual_learning(cb_explore_adf& data, base_learner& base) +{ example* label_example=test_adf_sequence(data.ec_seq); + data.gen_cs.known_cost = CB_ADF::get_observed_cost(data.ec_seq); - template - void predict_or_learn(cb_explore_adf& data, base_learner& base, example &ec) - { - vw* all = data.all; - //data.base = &base; - data.offset = ec.ft_offset; - bool is_test_ec = CB::example_is_test(ec); - bool need_to_break = VW::is_ring_example(*all, &ec) && (data.ec_seq.size() >= all->p->ring_size - 2); - - if ((CB_ALGS::example_is_newline_not_header(ec) && is_test_ec) || need_to_break) - { - data.ec_seq.push_back(&ec); - if (data.ec_seq.size() == 1) - cout << "Something is wrong---an example with no choice. Do you have all 0 features? Or multiple empty lines?" << endl; - else - do_actual_learning(data, base); - // using flag to clear, because ec_seq is used in finish_example - data.need_to_clear = true; + if (label_example == nullptr || !is_learn) + { if (label_example != nullptr)//extract label + { data.action_label = label_example->l.cb; + label_example->l.cb = data.empty_label; } + switch (data.explore_type) + { case EXPLORE_FIRST: + predict_or_learn_first(data, base, data.ec_seq); + break; + case EPS_GREEDY: + predict_or_learn_greedy(data, base, data.ec_seq); + break; + case SOFTMAX: + predict_or_learn_softmax(data, base, data.ec_seq); + break; + case BAG_EXPLORE: + predict_or_learn_bag(data, base, data.ec_seq); + break; + case COVER: + predict_or_learn_cover(data, base, data.ec_seq); + break; + default: + THROW("Unknown explorer type specified for contextual bandit learning: " << data.explore_type); + } + if (label_example != nullptr) //restore label + label_example->l.cb = data.action_label; + } + else + { /* v_array temp_probs; + temp_probs = v_init(); + do_actual_learning(data,base); + for (size_t i = 0; i < data.ec_seq[0]->pred.a_s.size(); i++) + temp_probs.push_back(data.ec_seq[0]->pred.a_s[i].score);*/ + + switch (data.explore_type) + { case EXPLORE_FIRST: + predict_or_learn_first(data, base, data.ec_seq); + break; + case EPS_GREEDY: + predict_or_learn_greedy(data, base, data.ec_seq); + break; + case SOFTMAX: + predict_or_learn_softmax(data, base, data.ec_seq); + break; + case BAG_EXPLORE: + predict_or_learn_bag(data, base, data.ec_seq); + break; + case COVER: + predict_or_learn_cover(data, base, data.ec_seq); + break; + default: + THROW("Unknown explorer type specified for contextual bandit learning: " << data.explore_type); + } + + /* for (size_t i = 0; i < temp_probs.size(); i++) + if (temp_probs[i] != data.ec_seq[0]->pred.a_s[i].score) + cout << "problem! " << temp_probs[i] << " != " << data.ec_seq[0]->pred.a_s[i].score << " for " << data.ec_seq[0]->pred.a_s[i].action << endl; + temp_probs.delete_v();*/ + } +} + +template +void predict_or_learn(cb_explore_adf& data, base_learner& base, example &ec) +{ vw* all = data.all; + //data.base = &base; + data.offset = ec.ft_offset; + bool is_test_ec = CB::example_is_test(ec); + bool need_to_break = VW::is_ring_example(*all, &ec) && (data.ec_seq.size() >= all->p->ring_size - 2); + + if ((CB_ALGS::example_is_newline_not_header(ec) && is_test_ec) || need_to_break) + { data.ec_seq.push_back(&ec); + if (data.ec_seq.size() == 1) + cout << "Something is wrong---an example with no choice. Do you have all 0 features? Or multiple empty lines?" << endl; else - { - if (data.need_to_clear) // should only happen if we're NOT driving - { - data.ec_seq.erase(); - data.need_to_clear = false; - } - data.ec_seq.push_back(&ec); + do_actual_learning(data, base); + // using flag to clear, because ec_seq is used in finish_example + data.need_to_clear = true; + } + else + { if (data.need_to_clear) // should only happen if we're NOT driving + { data.ec_seq.erase(); + data.need_to_clear = false; } + data.ec_seq.push_back(&ec); } +} } @@ -482,12 +451,12 @@ base_learner* cb_explore_adf_setup(vw& all) if (missing_option(all, true, "cb_explore_adf", "Online explore-exploit for a contextual bandit problem with multiline action dependent features")) return nullptr; new_options(all, "CB_EXPLORE_ADF options") - ("first", po::value(), "tau-first exploration") - ("epsilon", po::value(), "epsilon-greedy exploration") - ("bag", po::value(), "bagging-based exploration") - ("cover",po::value() ,"Online cover based exploration") - ("softmax", "softmax exploration") - ("lambda", po::value(), "parameter for softmax"); + ("first", po::value(), "tau-first exploration") + ("epsilon", po::value(), "epsilon-greedy exploration") + ("bag", po::value(), "bagging-based exploration") + ("cover",po::value() ,"Online cover based exploration") + ("softmax", "softmax exploration") + ("lambda", po::value(), "parameter for softmax"); add_options(all); po::variables_map& vm = all.vm; @@ -496,46 +465,42 @@ base_learner* cb_explore_adf_setup(vw& all) data.all = &all; if (count(all.args.begin(), all.args.end(), "--cb_adf") == 0) all.args.push_back("--cb_adf"); - + all.delete_prediction = delete_action_scores; size_t problem_multiplier = 1; char type_string[10]; if (vm.count("epsilon")) - { - data.epsilon = vm["epsilon"].as(); - sprintf(type_string, "%f", data.epsilon); - *all.file_options << " --epsilon "<(); + sprintf(type_string, "%f", data.epsilon); + *all.file_options << " --epsilon "<(); - data.explore_type = COVER; - problem_multiplier = data.cover_size+1; - sprintf(type_string, "%lu", data.cover_size); - *all.file_options << " --cover " << type_string; - - if (!vm.count("epsilon")) - data.epsilon = 0.05f; - } + { data.cover_size = (uint32_t)vm["cover"].as(); + data.explore_type = COVER; + problem_multiplier = data.cover_size+1; + sprintf(type_string, "%lu", data.cover_size); + *all.file_options << " --cover " << type_string; + + if (!vm.count("epsilon")) + data.epsilon = 0.05f; + } else if (vm.count("bag")) - { - data.bag_size = (uint32_t)vm["bag"].as(); - data.explore_type = BAG_EXPLORE; - problem_multiplier = data.bag_size; - sprintf(type_string, "%lu", data.bag_size); - *all.file_options << " --bag "<(); + data.explore_type = BAG_EXPLORE; + problem_multiplier = data.bag_size; + sprintf(type_string, "%lu", data.bag_size); + *all.file_options << " --bag "<(); - data.explore_type = EXPLORE_FIRST; - sprintf(type_string, "%lu", data.tau); - *all.file_options << " --first "<(); + data.explore_type = EXPLORE_FIRST; + sprintf(type_string, "%lu", data.tau); + *all.file_options << " --first "<(); data.explore_type = SOFTMAX; @@ -545,10 +510,9 @@ base_learner* cb_explore_adf_setup(vw& all) else if (vm.count("epsilon")) data.explore_type = EPS_GREEDY; else //epsilon - { - data.epsilon = 0.05f; - data.explore_type = EPS_GREEDY; - } + { data.epsilon = 0.05f; + data.explore_type = EPS_GREEDY; + } base_learner* base = setup_base(all); all.p->lp = CB::cb_label; @@ -561,26 +525,25 @@ base_learner* cb_explore_adf_setup(vw& all) data.cs_ldf_learner = all.cost_sensitive; data.gen_cs.cb_type = CB_TYPE_IPS; if (all.vm.count("cb_type")) - { std::string type_string; - type_string = all.vm["cb_type"].as(); - - if (type_string.compare("dr") == 0) - data.gen_cs.cb_type = CB_TYPE_DR; - else if (type_string.compare("ips") == 0) - data.gen_cs.cb_type = CB_TYPE_IPS; - else if (type_string.compare("mtr") == 0) - if (vm.count("cover")) - { - cout << "warning: cover and mtr are not simultaneously supported yet, defaulting to ips" << endl; - data.gen_cs.cb_type = CB_TYPE_IPS; - } - else - data.gen_cs.cb_type = CB_TYPE_MTR; + { std::string type_string; + type_string = all.vm["cb_type"].as(); + + if (type_string.compare("dr") == 0) + data.gen_cs.cb_type = CB_TYPE_DR; + else if (type_string.compare("ips") == 0) + data.gen_cs.cb_type = CB_TYPE_IPS; + else if (type_string.compare("mtr") == 0) + if (vm.count("cover")) + { cout << "warning: cover and mtr are not simultaneously supported yet, defaulting to ips" << endl; + data.gen_cs.cb_type = CB_TYPE_IPS; + } else - std::cerr << "warning: cb_type must be in {'ips','dr'}; resetting to ips." << std::endl; - } + data.gen_cs.cb_type = CB_TYPE_MTR; + else + std::cerr << "warning: cb_type must be in {'ips','dr'}; resetting to ips." << std::endl; + } - l.set_finish_example(CB_EXPLORE_ADF::finish_multiline_example); + l.set_finish_example(CB_EXPLORE_ADF::finish_multiline_example); l.set_finish(CB_EXPLORE_ADF::finish); l.set_end_examples(CB_EXPLORE_ADF::end_examples); return make_base(l); diff --git a/vowpalwabbit/cb_explore_adf.h b/vowpalwabbit/cb_explore_adf.h index 717825df705..f4a9eec9e0e 100644 --- a/vowpalwabbit/cb_explore_adf.h +++ b/vowpalwabbit/cb_explore_adf.h @@ -7,6 +7,7 @@ license as described in the file LICENSE. LEARNER::base_learner* cb_explore_adf_setup(vw& all); -namespace CB_EXPLORE_ADF{ - example* test_adf_sequence(v_array& ec_seq); +namespace CB_EXPLORE_ADF +{ +example* test_adf_sequence(v_array& ec_seq); } diff --git a/vowpalwabbit/cbify.cc b/vowpalwabbit/cbify.cc index 6748e8e69e3..c010034c0ac 100644 --- a/vowpalwabbit/cbify.cc +++ b/vowpalwabbit/cbify.cc @@ -16,7 +16,7 @@ struct cbify; //Scorer class for use by the exploration library class vw_scorer : public IScorer { - public: +public: vector Score_Actions(example& ctx); }; @@ -41,16 +41,14 @@ struct cbify }; vector vw_scorer::Score_Actions(example& ctx) -{ - vector probs_vec; - for(uint32_t i = 0;i < ctx.pred.a_s.size();i++) +{ vector probs_vec; + for(uint32_t i = 0; i < ctx.pred.a_s.size(); i++) probs_vec.push_back(ctx.pred.a_s[i].score); return probs_vec; } float loss(uint32_t label, uint32_t final_prediction) -{ - if (label != final_prediction) +{ if (label != final_prediction) return 1.; else return 0.; @@ -70,8 +68,7 @@ void finish(cbify& data) template void predict_or_learn(cbify& data, base_learner& base, example& ec) -{ - //Store the multiclass input label +{ //Store the multiclass input label MULTICLASS::label_t ld = ec.l.multi; data.cb_label.costs.erase(); ec.l.cb = data.cb_label; @@ -115,7 +112,7 @@ base_learner* cbify_setup(vw& all) data.scorer = new vw_scorer(); data.a_s = v_init(); //data.probs = v_init(); - data.generic_explorer = new GenericExplorer(*data.scorer, (u32)num_actions); + data.generic_explorer = new GenericExplorer(*data.scorer, (u32)num_actions); if (count(all.args.begin(), all.args.end(),"--cb_explore") == 0) { all.args.push_back("--cb_explore"); @@ -124,7 +121,7 @@ base_learner* cbify_setup(vw& all) all.args.push_back(ss.str()); } base_learner* base = setup_base(all); - + all.delete_prediction = nullptr; learner* l; l = &init_multiclass_learner(&data, base, predict_or_learn, predict_or_learn, all.p, 1); diff --git a/vowpalwabbit/confidence.cc b/vowpalwabbit/confidence.cc index ed28ba42c63..26816852f73 100644 --- a/vowpalwabbit/confidence.cc +++ b/vowpalwabbit/confidence.cc @@ -14,7 +14,7 @@ void predict_or_learn_with_confidence(confidence& c, base_learner& base, example float sensitivity = 0.f; if (!is_confidence_after_training) sensitivity = base.sensitivity(ec); - + if (is_learn) base.learn(ec); else @@ -73,23 +73,23 @@ base_learner* confidence_setup(vw& all) if(missing_option(all, false, "confidence", "Get confidence for binary predictions")) return nullptr; if(!all.training) - { - cout << "Confidence does not work in test mode because learning algorithm state is needed. Use --save_resume when saving the model and avoid --test_only" << endl; - return nullptr; - } - + { cout << "Confidence does not work in test mode because learning algorithm state is needed. Use --save_resume when saving the model and avoid --test_only" << endl; + return nullptr; + } + confidence& data = calloc_or_throw(); data.all=&all; void (*learn_with_confidence_ptr)(confidence&, base_learner&, example&) = nullptr; void (*predict_with_confidence_ptr)(confidence&, base_learner&, example&) = nullptr; - if(vm.count("confidence_after_training")){ - learn_with_confidence_ptr = predict_or_learn_with_confidence; - predict_with_confidence_ptr = predict_or_learn_with_confidence; - }else{ - learn_with_confidence_ptr = predict_or_learn_with_confidence; - predict_with_confidence_ptr = predict_or_learn_with_confidence; + if(vm.count("confidence_after_training")) + { learn_with_confidence_ptr = predict_or_learn_with_confidence; + predict_with_confidence_ptr = predict_or_learn_with_confidence; + } + else + { learn_with_confidence_ptr = predict_or_learn_with_confidence; + predict_with_confidence_ptr = predict_or_learn_with_confidence; } //Create new learner diff --git a/vowpalwabbit/cost_sensitive.cc b/vowpalwabbit/cost_sensitive.cc index 6f9c51f9764..fd9c105df85 100644 --- a/vowpalwabbit/cost_sensitive.cc +++ b/vowpalwabbit/cost_sensitive.cc @@ -117,27 +117,27 @@ void parse_label(parser* p, shared_data*sd, void* v, v_array& words) ld->costs.erase(); // handle shared and label first - if (words.size() == 1) { - float fx; + if (words.size() == 1) + { float fx; name_value(words[0], p->parse_name, fx); bool eq_shared = substring_eq(p->parse_name[0], "***shared***"); bool eq_label = substring_eq(p->parse_name[0], "***label***"); - if (! sd->ldict) { - eq_shared |= substring_eq(p->parse_name[0], "shared"); + if (! sd->ldict) + { eq_shared |= substring_eq(p->parse_name[0], "shared"); eq_label |= substring_eq(p->parse_name[0], "label"); } - if (eq_shared || eq_label) { - if (eq_shared) { - if (p->parse_name.size() != 1) cerr << "shared feature vectors should not have costs on: " << words[0] << endl; - else { - wclass f = { -FLT_MAX, 0, 0., 0.}; + if (eq_shared || eq_label) + { if (eq_shared) + { if (p->parse_name.size() != 1) cerr << "shared feature vectors should not have costs on: " << words[0] << endl; + else + { wclass f = { -FLT_MAX, 0, 0., 0.}; ld->costs.push_back(f); } } - if (eq_label) { - if (p->parse_name.size() != 2) cerr << "label feature vectors should have exactly one cost on: " << words[0] << endl; - else { - wclass f = { float_of_substring(p->parse_name[1]), 0, 0., 0.}; + if (eq_label) + { if (p->parse_name.size() != 2) cerr << "label feature vectors should have exactly one cost on: " << words[0] << endl; + else + { wclass f = { float_of_substring(p->parse_name[1]), 0, 0., 0.}; ld->costs.push_back(f); } } diff --git a/vowpalwabbit/csoaa.cc b/vowpalwabbit/csoaa.cc index 0956f20e3dc..3df3f0e69a5 100644 --- a/vowpalwabbit/csoaa.cc +++ b/vowpalwabbit/csoaa.cc @@ -77,12 +77,12 @@ void predict_or_learn(csoaa& c, base_learner& base, example& ec) { uint64_t second_best = 0; float second_best_cost = FLT_MAX; for (size_t i=0; isize() - pt_start; i++) - { float val = ec.passthrough->values[pt_start + i]; - if ((val > ec.partial_prediction) && (val < second_best_cost)) - { second_best_cost = val; - second_best = ec.passthrough->indicies[pt_start + i]; - } + { float val = ec.passthrough->values[pt_start + i]; + if ((val > ec.partial_prediction) && (val < second_best_cost)) + { second_best_cost = val; + second_best = ec.passthrough->indicies[pt_start + i]; } + } if (second_best_cost < FLT_MAX) { float margin = second_best_cost - ec.partial_prediction; add_passthrough_feature(ec, constant*2, margin); @@ -234,8 +234,7 @@ void make_single_prediction(ldf& data, base_learner& base, example& ec) } bool check_ldf_sequence(ldf& data, size_t start_K) -{ - bool isTest = COST_SENSITIVE::example_is_test(*data.ec_seq[start_K]); +{ bool isTest = COST_SENSITIVE::example_is_test(*data.ec_seq[start_K]); for (size_t k=start_K; kft_offset; ec->ft_offset = data.ft_offset; base.learn(*ec); - ec->ft_offset = old_offset; + ec->ft_offset = old_offset; LabelDict::del_example_namespace_from_memory(data.label_features, *ec, costs[0].class_index); ec->weight = old_weight; @@ -362,16 +361,15 @@ void do_actual_learning(ldf& data, base_learner& base) /////////////////////// handle label definitions if (ec_seq_is_label_definition(data.ec_seq)) - { - for (size_t i=0; ifeature_space[data.ec_seq[i]->indices[0]]; - - v_array& costs = data.ec_seq[i]->l.cs.costs; - for (size_t j=0; jfeature_space[data.ec_seq[i]->indices[0]]; + + v_array& costs = data.ec_seq[i]->l.cs.costs; + for (size_t j=0; j 0) { data.ec_seq[0]->pred.a_s = data.stored_preds[0]; } - for (size_t k=start_K; kpred.a_s = data.stored_preds[k]; data.ec_seq[0]->pred.a_s.push_back(data.a_s[k-start_K]); } @@ -435,11 +433,11 @@ void do_actual_learning(ldf& data, base_learner& base) else { // Mark the predicted subexample with its class_index, all other with 0 for (size_t k=start_K; kpred.multiclass = data.ec_seq[k]->l.cs.costs[0].class_index; else data.ec_seq[k]->pred.multiclass = 0; - } + } } /////////////////////// remove header if (start_K > 0) @@ -455,13 +453,13 @@ void do_actual_learning(ldf& data, base_learner& base) // so we need to take score = -partial_prediction, // thus probability(correct_class) = 1 / (1+exp(-(-partial_prediction))) float prob = 1.f / (1.f + exp(data.ec_seq[k]->partial_prediction)); - data.ec_seq[k]->pred.prob = prob; + data.ec_seq[k]->pred.prob = prob; sum_prob += prob; } // make sure that the probabilities sum up (exactly) to one for (size_t k=start_K; kpred.prob /= sum_prob; - } + { data.ec_seq[k]->pred.prob /= sum_prob; + } } } @@ -721,22 +719,22 @@ void predict_or_learn(ldf& data, base_learner& base, example &ec) base_learner* csldf_setup(vw& all) { if (missing_option(all, "csoaa_ldf", "Use one-against-all multiclass learning with label dependent features. Specify singleline or multiline.") - && missing_option(all, "wap_ldf", "Use weighted all-pairs multiclass learning with label dependent features. Specify singleline or multiline.")) - return nullptr; - new_options(all, "LDF Options") - ("ldf_override", po::value(), "Override singleline or multiline from csoaa_ldf or wap_ldf, eg if stored in file") - ("csoaa_rank", "Return actions sorted by score order") - ("probabilities", "predict probabilites of all classes"); - add_options(all); + && missing_option(all, "wap_ldf", "Use weighted all-pairs multiclass learning with label dependent features. Specify singleline or multiline.")) + return nullptr; + new_options(all, "LDF Options") + ("ldf_override", po::value(), "Override singleline or multiline from csoaa_ldf or wap_ldf, eg if stored in file") + ("csoaa_rank", "Return actions sorted by score order") + ("probabilities", "predict probabilites of all classes"); + add_options(all); - po::variables_map& vm = all.vm; - ldf& ld = calloc_or_throw(); + po::variables_map& vm = all.vm; + ldf& ld = calloc_or_throw(); - ld.all = &all; - ld.need_to_clear = true; - ld.first_pass = true; + ld.all = &all; + ld.need_to_clear = true; + ld.first_pass = true; - string ldf_arg; + string ldf_arg; if( vm.count("csoaa_ldf") ) { ldf_arg = vm["csoaa_ldf"].as(); @@ -745,20 +743,20 @@ base_learner* csldf_setup(vw& all) { ldf_arg = vm["wap_ldf"].as(); ld.is_wap = true; } - if ( vm.count("ldf_override") ) + if ( vm.count("ldf_override") ) ldf_arg = vm["ldf_override"].as(); if (vm.count("csoaa_rank")) { ld.rank = true; - *all.file_options << " --csoaa_rank"; - all.delete_prediction = delete_action_scores; - } + *all.file_options << " --csoaa_rank"; + all.delete_prediction = delete_action_scores; + } - all.p->lp = COST_SENSITIVE::cs_label; - all.label_type = label_type::cs; + all.p->lp = COST_SENSITIVE::cs_label; + all.label_type = label_type::cs; - ld.treat_as_classifier = false; - ld.is_singleline = false; -if (ldf_arg.compare("multiline") == 0 || ldf_arg.compare("m") == 0) + ld.treat_as_classifier = false; + ld.is_singleline = false; + if (ldf_arg.compare("multiline") == 0 || ldf_arg.compare("m") == 0) { ld.treat_as_classifier = false; } else if (ldf_arg.compare("multiline-classifier") == 0 || ldf_arg.compare("mc") == 0) @@ -793,22 +791,22 @@ if (ldf_arg.compare("multiline") == 0 || ldf_arg.compare("m") == 0) { ld.is_probabilities = false; } - all.p->emptylines_separate_examples = true; // TODO: check this to be sure!!! !ld.is_singleline; + all.p->emptylines_separate_examples = true; // TODO: check this to be sure!!! !ld.is_singleline; - /*if (all.add_constant) { - all.add_constant = false; - }*/ - features fs; - ld.label_features.init(256, fs, LabelDict::size_t_eq); - ld.label_features.get(1, 94717244); // TODO: figure this out - prediction_type::prediction_type_t pred_type; + /*if (all.add_constant) { + all.add_constant = false; + }*/ + features fs; + ld.label_features.init(256, fs, LabelDict::size_t_eq); + ld.label_features.get(1, 94717244); // TODO: figure this out + prediction_type::prediction_type_t pred_type; - if (ld.rank) - pred_type = prediction_type::multiclass; - else if (ld.is_probabilities) - pred_type = prediction_type::prob; - else - pred_type = prediction_type::action_scores; + if (ld.rank) + pred_type = prediction_type::multiclass; + else if (ld.is_probabilities) + pred_type = prediction_type::prob; + else + pred_type = prediction_type::action_scores; ld.read_example_this_loop = 0; ld.need_to_clear = false; diff --git a/vowpalwabbit/ect.cc b/vowpalwabbit/ect.cc index 762751f6b4f..06de5d3776f 100644 --- a/vowpalwabbit/ect.cc +++ b/vowpalwabbit/ect.cc @@ -345,7 +345,7 @@ base_learner* ect_setup(vw& all) l.set_finish(finish); if (all.vm["link"].as().compare("logistic") == 0) - class_boundary = 0.5; // as --link=logistic maps predictions in [0;1] + class_boundary = 0.5; // as --link=logistic maps predictions in [0;1] return make_base(l); } diff --git a/vowpalwabbit/example.cc b/vowpalwabbit/example.cc index 92bfed034cd..94ad7c567a1 100644 --- a/vowpalwabbit/example.cc +++ b/vowpalwabbit/example.cc @@ -7,17 +7,14 @@ license as described in the file LICENSE. #include "gd.h" float collision_cleanup(features& fs) -{ - uint64_t last_index = (uint64_t)-1; +{ uint64_t last_index = (uint64_t)-1; float sum_sq = 0.f; features::iterator pos = fs.begin(); for (features::iterator& f : fs) - { - if (last_index == f.index()) + { if (last_index == f.index()) pos.value() += f.value(); else - { - sum_sq += pos.value() * pos.value(); + { sum_sq += pos.value() * pos.value(); ++pos; pos.value() = f.value(); pos.index() = f.index(); @@ -35,7 +32,7 @@ float collision_cleanup(features& fs) namespace VW { - void copy_example_label(example* dst, example* src, size_t, void(*copy_label)(void*,void*)) +void copy_example_label(example* dst, example* src, size_t, void(*copy_label)(void*,void*)) { if (copy_label) copy_label(&dst->l, &src->l); // TODO: we really need to delete_label on dst :( else @@ -150,11 +147,11 @@ flat_example* flatten_sort_example(vw& all, example *ec) void free_flatten_example(flat_example* fec) { //note: The label memory should be freed by by freeing the original example. if (fec) - { fec->fs.delete_v(); - if (fec->tag_len > 0) - free(fec->tag); - free(fec); - } + { fec->fs.delete_v(); + if (fec->tag_len > 0) + free(fec->tag); + free(fec); + } } namespace VW diff --git a/vowpalwabbit/example.h b/vowpalwabbit/example.h index ad2b67ab972..ce039df212e 100644 --- a/vowpalwabbit/example.h +++ b/vowpalwabbit/example.h @@ -42,22 +42,19 @@ inline void delete_scalars(void* v) } typedef union -{ - float scalar; - v_array scalars;//a sequence of scalar predictions - ACTION_SCORE::action_scores a_s;//a sequence of classes with scores. Also used for probabilities. - uint32_t multiclass; - MULTILABEL::labels multilabels; - float prob; // for --probabilities --csoaa_ldf=mc +{ float scalar; + v_array scalars;//a sequence of scalar predictions + ACTION_SCORE::action_scores a_s;//a sequence of classes with scores. Also used for probabilities. + uint32_t multiclass; + MULTILABEL::labels multilabels; + float prob; // for --probabilities --csoaa_ldf=mc } polyprediction; typedef unsigned char namespace_index; struct example // core example datatype. -{ - class iterator - { - features* _feature_space; +{ class iterator + { features* _feature_space; namespace_index* _index; public: iterator(features* feature_space, namespace_index* index) @@ -73,7 +70,7 @@ struct example // core example datatype. return *this; } - namespace_index index(){ return *_index; } + namespace_index index() { return *_index; } bool operator==(const iterator& rhs) { return _index == rhs._index; } bool operator!=(const iterator& rhs) { return _index != rhs._index; } diff --git a/vowpalwabbit/explore_eval.cc b/vowpalwabbit/explore_eval.cc index adc0243a29c..0a226c67427 100644 --- a/vowpalwabbit/explore_eval.cc +++ b/vowpalwabbit/explore_eval.cc @@ -16,232 +16,209 @@ using namespace MultiWorldTesting; using namespace MultiWorldTesting::SingleAction; using namespace std; -namespace EXPLORE_EVAL { - - struct explore_eval - { - CB::cb_class known_cost; - v_array ec_seq; - bool need_to_clear; - vw* all; - uint64_t offset; - CB::label action_label; - CB::label empty_label; - size_t example_counter; - - size_t update_count; - size_t violations; - float multiplier; - - bool fixed_multiplier; - }; - - template - void multiline_learn_or_predict(base_learner& base, v_array& examples, uint64_t offset, uint32_t id = 0) - { for (example* ec : examples) - { - uint64_t old_offset = ec->ft_offset; - ec->ft_offset = offset; - if (is_learn) - base.learn(*ec, id); - else - base.predict(*ec, id); - ec->ft_offset = old_offset; - } - } - - void end_examples(explore_eval& data) - { - if (data.need_to_clear) - data.ec_seq.erase(); +namespace EXPLORE_EVAL +{ + +struct explore_eval +{ CB::cb_class known_cost; + v_array ec_seq; + bool need_to_clear; + vw* all; + uint64_t offset; + CB::label action_label; + CB::label empty_label; + size_t example_counter; + + size_t update_count; + size_t violations; + float multiplier; + + bool fixed_multiplier; +}; + +template +void multiline_learn_or_predict(base_learner& base, v_array& examples, uint64_t offset, uint32_t id = 0) +{ for (example* ec : examples) + { uint64_t old_offset = ec->ft_offset; + ec->ft_offset = offset; + if (is_learn) + base.learn(*ec, id); + else + base.predict(*ec, id); + ec->ft_offset = old_offset; } +} +void end_examples(explore_eval& data) +{ if (data.need_to_clear) + data.ec_seq.erase(); +} - void finish(explore_eval& data) - { - data.ec_seq.delete_v(); - if (!data.all->quiet) - { - cout << "update count = " << data.update_count << endl; - if (data.violations > 0) - cout << "violation count = " << data.violations << endl; - if (!data.fixed_multiplier) - cout << "final multiplier = " << data.multiplier << endl; - } + +void finish(explore_eval& data) +{ data.ec_seq.delete_v(); + if (!data.all->quiet) + { cout << "update count = " << data.update_count << endl; + if (data.violations > 0) + cout << "violation count = " << data.violations << endl; + if (!data.fixed_multiplier) + cout << "final multiplier = " << data.multiplier << endl; } +} - //Semantics: Currently we compute the IPS loss no matter what flags - //are specified. We print the first action and probability, based on - //ordering by scores in the final output. +//Semantics: Currently we compute the IPS loss no matter what flags +//are specified. We print the first action and probability, based on +//ordering by scores in the final output. - void output_example(vw& all, explore_eval& c, example& ec, v_array* ec_seq) - { - if (example_is_newline_not_header(ec)) return; +void output_example(vw& all, explore_eval& c, example& ec, v_array* ec_seq) +{ if (example_is_newline_not_header(ec)) return; - size_t num_features = 0; + size_t num_features = 0; - float loss = 0.; - ACTION_SCORE::action_scores preds = (*ec_seq)[0]->pred.a_s; + float loss = 0.; + ACTION_SCORE::action_scores preds = (*ec_seq)[0]->pred.a_s; - for (size_t i = 0; i < (*ec_seq).size(); i++) - if (!CB::ec_is_example_header(*(*ec_seq)[i])) { - num_features += (*ec_seq)[i]->num_features; - } + for (size_t i = 0; i < (*ec_seq).size(); i++) + if (!CB::ec_is_example_header(*(*ec_seq)[i])) + { num_features += (*ec_seq)[i]->num_features; + } - all.sd->total_features += num_features; + all.sd->total_features += num_features; - bool is_test = false; - if (c.known_cost.probability > 0) { - for (uint32_t i = 0; i < preds.size(); i++) { - float l = get_unbiased_cost(&c.known_cost, preds[i].action); - loss += l*preds[i].score; - } - all.sd->sum_loss += loss; - all.sd->sum_loss_since_last_dump += loss; + bool is_test = false; + if (c.known_cost.probability > 0) + { for (uint32_t i = 0; i < preds.size(); i++) + { float l = get_unbiased_cost(&c.known_cost, preds[i].action); + loss += l*preds[i].score; } - else - is_test = true; + all.sd->sum_loss += loss; + all.sd->sum_loss_since_last_dump += loss; + } + else + is_test = true; - for (int sink : all.final_prediction_sink) - print_action_score(sink, ec.pred.a_s, ec.tag); + for (int sink : all.final_prediction_sink) + print_action_score(sink, ec.pred.a_s, ec.tag); - if (all.raw_prediction > 0) - { - string outputString; - stringstream outputStringStream(outputString); - v_array costs = ec.l.cb.costs; - - for (size_t i = 0; i < costs.size(); i++) - { - if (i > 0) outputStringStream << ' '; - outputStringStream << costs[i].action << ':' << costs[i].partial_prediction; - } - all.print_text(all.raw_prediction, outputStringStream.str(), ec.tag); + if (all.raw_prediction > 0) + { string outputString; + stringstream outputStringStream(outputString); + v_array costs = ec.l.cb.costs; + + for (size_t i = 0; i < costs.size(); i++) + { if (i > 0) outputStringStream << ' '; + outputStringStream << costs[i].action << ':' << costs[i].partial_prediction; } + all.print_text(all.raw_prediction, outputStringStream.str(), ec.tag); + } + + CB::print_update(all, is_test, ec, ec_seq, true); +} - CB::print_update(all, is_test, ec, ec_seq, true); +void output_example_seq(vw& all, explore_eval& data) +{ if (data.ec_seq.size() > 0) + { all.sd->weighted_examples += 1; + all.sd->example_number++; + output_example(all, data, **(data.ec_seq.begin()), &(data.ec_seq)); + if (all.raw_prediction > 0) + all.print_text(all.raw_prediction, "", data.ec_seq[0]->tag); } +} + - void output_example_seq(vw& all, explore_eval& data) - { - if (data.ec_seq.size() > 0) - { - all.sd->weighted_examples += 1; - all.sd->example_number++; - output_example(all, data, **(data.ec_seq.begin()), &(data.ec_seq)); - if (all.raw_prediction > 0) - all.print_text(all.raw_prediction, "", data.ec_seq[0]->tag); +void clear_seq_and_finish_examples(vw& all, explore_eval& data) +{ if (data.ec_seq.size() > 0) + for (example* ecc : data.ec_seq) + if (ecc->in_use) + VW::finish_example(all, ecc); + data.ec_seq.erase(); +} + +void finish_multiline_example(vw& all, explore_eval& data, example& ec) +{ if (data.need_to_clear) + { if (data.ec_seq.size() > 0) + { output_example_seq(all, data); + CB_ADF::global_print_newline(all); } + clear_seq_and_finish_examples(all, data); + data.need_to_clear = false; } +} +template +void do_actual_learning(explore_eval& data, base_learner& base) +{ example* label_example=CB_EXPLORE_ADF::test_adf_sequence(data.ec_seq); - void clear_seq_and_finish_examples(vw& all, explore_eval& data) - { - if (data.ec_seq.size() > 0) - for (example* ecc : data.ec_seq) - if (ecc->in_use) - VW::finish_example(all, ecc); - data.ec_seq.erase(); + if (label_example != nullptr)//extract label + { data.action_label = label_example->l.cb; + label_example->l.cb = data.empty_label; } + multiline_learn_or_predict(base, data.ec_seq, data.offset); - void finish_multiline_example(vw& all, explore_eval& data, example& ec) - { - if (data.need_to_clear) - { - if (data.ec_seq.size() > 0) - { - output_example_seq(all, data); - CB_ADF::global_print_newline(all); - } - clear_seq_and_finish_examples(all, data); - data.need_to_clear = false; - } - } - - template - void do_actual_learning(explore_eval& data, base_learner& base) - { - example* label_example=CB_EXPLORE_ADF::test_adf_sequence(data.ec_seq); - - if (label_example != nullptr)//extract label - { - data.action_label = label_example->l.cb; - label_example->l.cb = data.empty_label; - } - multiline_learn_or_predict(base, data.ec_seq, data.offset); - - if (label_example != nullptr) //restore label - label_example->l.cb = data.action_label; - - data.known_cost = CB_ADF::get_observed_cost(data.ec_seq); - if (label_example != nullptr && is_learn) - { - ACTION_SCORE::action_scores& a_s = data.ec_seq[0]->pred.a_s; - - float action_probability = 0; - for (size_t i =0 ; i < a_s.size(); i++) - if (data.known_cost.action == a_s[i].action) - action_probability = a_s[i].score; - - float threshold = action_probability / data.known_cost.probability; - - if (!data.fixed_multiplier) - data.multiplier = min(data.multiplier, 1/threshold); - - threshold *= data.multiplier; - - if (threshold > 1. + 1e-6) - data.violations++; - - if (frand48() < threshold) - { - example* ec_found = nullptr; - for (example*& ec : data.ec_seq) - { - if (ec->l.cb.costs.size() == 1 && + if (label_example != nullptr) //restore label + label_example->l.cb = data.action_label; + + data.known_cost = CB_ADF::get_observed_cost(data.ec_seq); + if (label_example != nullptr && is_learn) + { ACTION_SCORE::action_scores& a_s = data.ec_seq[0]->pred.a_s; + + float action_probability = 0; + for (size_t i =0 ; i < a_s.size(); i++) + if (data.known_cost.action == a_s[i].action) + action_probability = a_s[i].score; + + float threshold = action_probability / data.known_cost.probability; + + if (!data.fixed_multiplier) + data.multiplier = min(data.multiplier, 1/threshold); + + threshold *= data.multiplier; + + if (threshold > 1. + 1e-6) + data.violations++; + + if (frand48() < threshold) + { example* ec_found = nullptr; + for (example*& ec : data.ec_seq) + { if (ec->l.cb.costs.size() == 1 && ec->l.cb.costs[0].cost != FLT_MAX && ec->l.cb.costs[0].probability > 0) - { - ec_found = ec; - } + { ec_found = ec; } - ec_found->l.cb.costs[0].probability = action_probability; - - multiline_learn_or_predict(base, data.ec_seq, data.offset); - ec_found->l.cb.costs[0].probability = data.known_cost.probability; - data.update_count++; - } } - } + ec_found->l.cb.costs[0].probability = action_probability; - template - void predict_or_learn(explore_eval& data, base_learner& base, example &ec) - { - vw* all = data.all; - //data.base = &base; - data.offset = ec.ft_offset; - bool is_test_ec = CB::example_is_test(ec); - bool need_to_break = VW::is_ring_example(*all, &ec) && (data.ec_seq.size() >= all->p->ring_size - 2); - - if ((CB_ALGS::example_is_newline_not_header(ec) && is_test_ec) || need_to_break) - { - data.ec_seq.push_back(&ec); - do_actual_learning(data, base); - // using flag to clear, because ec_seq is used in finish_example - data.need_to_clear = true; + multiline_learn_or_predict(base, data.ec_seq, data.offset); + ec_found->l.cb.costs[0].probability = data.known_cost.probability; + data.update_count++; } - else - { - if (data.need_to_clear) // should only happen if we're NOT driving - { - data.ec_seq.erase(); - data.need_to_clear = false; - } - data.ec_seq.push_back(&ec); + } +} + +template +void predict_or_learn(explore_eval& data, base_learner& base, example &ec) +{ vw* all = data.all; + //data.base = &base; + data.offset = ec.ft_offset; + bool is_test_ec = CB::example_is_test(ec); + bool need_to_break = VW::is_ring_example(*all, &ec) && (data.ec_seq.size() >= all->p->ring_size - 2); + + if ((CB_ALGS::example_is_newline_not_header(ec) && is_test_ec) || need_to_break) + { data.ec_seq.push_back(&ec); + do_actual_learning(data, base); + // using flag to clear, because ec_seq is used in finish_example + data.need_to_clear = true; + } + else + { if (data.need_to_clear) // should only happen if we're NOT driving + { data.ec_seq.erase(); + data.need_to_clear = false; } + data.ec_seq.push_back(&ec); } } +} using namespace EXPLORE_EVAL; @@ -250,7 +227,7 @@ base_learner* explore_eval_setup(vw& all) if (missing_option(all, true, "explore_eval", "Evaluate explore_eval adf policies")) return nullptr; new_options(all, "Explore evaluation options") - ("multiplier", po::value(), "rejection sampling multiplier < 1"); + ("multiplier", po::value(), "rejection sampling multiplier < 1"); add_options(all); @@ -259,16 +236,15 @@ base_learner* explore_eval_setup(vw& all) data.all = &all; if (all.vm.count("multiplier") > 0) - { - data.multiplier = all.vm["multiplier"].as(); - data.fixed_multiplier = true; - } - else + { data.multiplier = all.vm["multiplier"].as(); + data.fixed_multiplier = true; + } + else data.multiplier = 1; if (count(all.args.begin(), all.args.end(), "--cb_explore_adf") == 0) all.args.push_back("--cb_explore_adf"); - + all.delete_prediction = nullptr; base_learner* base = setup_base(all); @@ -277,7 +253,7 @@ base_learner* explore_eval_setup(vw& all) learner& l = init_learner(&data, base, predict_or_learn, predict_or_learn, 1, prediction_type::action_probs); - l.set_finish_example(finish_multiline_example); + l.set_finish_example(finish_multiline_example); l.set_finish(finish); l.set_end_examples(end_examples); return make_base(l); diff --git a/vowpalwabbit/expreplay.h b/vowpalwabbit/expreplay.h index d83463700b2..f006eda2f43 100644 --- a/vowpalwabbit/expreplay.h +++ b/vowpalwabbit/expreplay.h @@ -68,7 +68,7 @@ void finish(expreplay& er) template LEARNER::base_learner* expreplay_setup(vw& all) - { std::string replay_string = "replay_"; replay_string += er_level; +{ std::string replay_string = "replay_"; replay_string += er_level; if (missing_option(all, replay_string.c_str(), "use experience replay at a specified level [b=classification/regression, m=multiclass, c=cost sensitive] with specified buffer size")) return nullptr; diff --git a/vowpalwabbit/ezexample.h b/vowpalwabbit/ezexample.h index 9b3432bc2b6..1effd8c85dd 100644 --- a/vowpalwabbit/ezexample.h +++ b/vowpalwabbit/ezexample.h @@ -146,12 +146,12 @@ class ezexample } else { if (ns_exists[(int)current_ns]) - { ec->total_sum_feat_sq -= ec->feature_space[(int)current_ns].sum_feat_sq; - ec->feature_space[(int)current_ns].erase(); - ec->num_features -= ec->feature_space[(int)current_ns].size(); + { ec->total_sum_feat_sq -= ec->feature_space[(int)current_ns].sum_feat_sq; + ec->feature_space[(int)current_ns].erase(); + ec->num_features -= ec->feature_space[(int)current_ns].size(); - ns_exists[(int)current_ns] = false; - } + ns_exists[(int)current_ns] = false; + } current_seed = past_seeds.back(); past_seeds.pop_back(); @@ -209,7 +209,7 @@ class ezexample for (std::vector::iterator i = vw_ref->pairs.begin(); i != vw_ref->pairs.end(); i++) { quadratic_features_num - += ec->feature_space[(int)(*i)[0]].size() * ec->feature_space[(int)(*i)[1]].size(); + += ec->feature_space[(int)(*i)[0]].size() * ec->feature_space[(int)(*i)[1]].size(); quadratic_features_sqr += ec->feature_space[(int)(*i)[0]].sum_feat_sq * ec->feature_space[(int)(*i)[1]].sum_feat_sq; } diff --git a/vowpalwabbit/feature_group.h b/vowpalwabbit/feature_group.h index 272410ab320..0cf17f1dac9 100644 --- a/vowpalwabbit/feature_group.h +++ b/vowpalwabbit/feature_group.h @@ -5,15 +5,15 @@ typedef uint64_t feature_index; typedef std::pair audit_strings; typedef std::shared_ptr audit_strings_ptr; -struct feature {//sparse feature definition for the library interface - float x; +struct feature //sparse feature definition for the library interface +{ float x; uint64_t weight_index; feature(float _x, uint64_t _index): x(_x), weight_index(_index) {} feature() {feature(0.f,0);} }; -struct feature_slice{ //a helper struct for functions using the set {v,i,space_name} - feature_value x; +struct feature_slice //a helper struct for functions using the set {v,i,space_name} +{ feature_value x; feature_index weight_index; audit_strings space_name; }; @@ -130,7 +130,7 @@ class features_value_index_iterator : public features_value_iterator } features_value_index_iterator& operator=(const features_value_index_iterator& other) - { features_value_iterator::operator=(other); + { features_value_iterator::operator=(other); _begin_index = other._begin_index; return *this; } @@ -204,8 +204,8 @@ class features_value_index_audit_iterator : public features_value_index_iterator }; /// the core definition of a set of features. -struct features { - v_array values; // Always needed. +struct features +{ v_array values; // Always needed. v_array indicies; //Optional for sparse data. v_array space_names; //Optional for audit mode. @@ -216,7 +216,8 @@ struct features { typedef features_value_index_audit_iterator iterator_all; /// defines a "range" usable by C++ 11 for loops - class features_value_index_audit_range { + class features_value_index_audit_range + { private: features* _outer; public: @@ -253,88 +254,75 @@ struct features { iterator end() { return iterator(values.end(), indicies.end()); } void erase() - { - sum_feat_sq = 0.f; + { sum_feat_sq = 0.f; values.erase(); indicies.erase(); space_names.erase(); } void truncate_to(const features_value_iterator& pos) - { - ssize_t i = pos._begin - values.begin(); + { ssize_t i = pos._begin - values.begin(); values.end() = pos._begin; if (indicies.end() != indicies.begin()) indicies.end() = indicies.begin() + i; if (space_names.begin() != space_names.end()) - { - free_space_names(i); + { free_space_names(i); space_names.end() = space_names.begin() + i; } } void truncate_to(size_t i) - { - values.end() = values.begin() + i; + { values.end() = values.begin() + i; if (indicies.end() != indicies.begin()) indicies.end() = indicies.begin() + i; if (space_names.begin() != space_names.end()) { free_space_names(i); - space_names.end() = space_names.begin() + i; + space_names.end() = space_names.begin() + i; } } void delete_v() - { - values.delete_v(); + { values.delete_v(); indicies.delete_v(); space_names.delete_v(); } void push_back(feature_value v, feature_index i) - { - values.push_back(v); + { values.push_back(v); indicies.push_back(i); sum_feat_sq += v*v; } bool sort(uint64_t parse_mask) - { - if (indicies.size() == 0) + { if (indicies.size() == 0) return false; if (space_names.size() != 0) - { - v_array slice = v_init(); - for (size_t i = 0; i < indicies.size(); i++) - { - feature_slice temp = { values[i], indicies[i] & parse_mask, *space_names[i].get()}; - slice.push_back(temp); - } - qsort(slice.begin(), slice.size(), sizeof(feature_slice), order_features); - for (size_t i = 0; i < slice.size(); i++) - { - values[i] = slice[i].x; - indicies[i] = slice[i].weight_index; - *space_names[i].get() = slice[i].space_name; - } - slice.delete_v(); + { v_array slice = v_init(); + for (size_t i = 0; i < indicies.size(); i++) + { feature_slice temp = { values[i], indicies[i] & parse_mask, *space_names[i].get()}; + slice.push_back(temp); + } + qsort(slice.begin(), slice.size(), sizeof(feature_slice), order_features); + for (size_t i = 0; i < slice.size(); i++) + { values[i] = slice[i].x; + indicies[i] = slice[i].weight_index; + *space_names[i].get() = slice[i].space_name; } + slice.delete_v(); + } else - { - v_array slice = v_init(); - for (size_t i = 0; i < indicies.size(); i++) - { - feature temp = { values[i], indicies[i] & parse_mask}; - slice.push_back(temp); - } - qsort(slice.begin(), slice.size(), sizeof(feature), order_features); - for (size_t i = 0; i < slice.size(); i++) - { - values[i] = slice[i].x; - indicies[i] = slice[i].weight_index; - } - slice.delete_v(); + { v_array slice = v_init(); + for (size_t i = 0; i < indicies.size(); i++) + { feature temp = { values[i], indicies[i] & parse_mask}; + slice.push_back(temp); + } + qsort(slice.begin(), slice.size(), sizeof(feature), order_features); + for (size_t i = 0; i < slice.size(); i++) + { values[i] = slice[i].x; + indicies[i] = slice[i].weight_index; } + slice.delete_v(); + } return true; } diff --git a/vowpalwabbit/ftrl.cc b/vowpalwabbit/ftrl.cc index 81b22fefb31..f4e015c2af1 100644 --- a/vowpalwabbit/ftrl.cc +++ b/vowpalwabbit/ftrl.cc @@ -54,9 +54,9 @@ inline void predict_with_confidence(uncertainty& d, const float fx, float& fw) } float sensitivity(ftrl& b, base_learner& base, example& ec) -{ uncertainty uncetain(b); - GD::foreach_feature(*(b.all), ec, uncetain); - return uncetain.score; +{ uncertainty uncetain(b); + GD::foreach_feature(*(b.all), ec, uncetain); + return uncetain.score; } template void predict(ftrl& b, base_learner&, example& ec) @@ -80,13 +80,13 @@ void multipredict(ftrl& b, base_learner&, example& ec, size_t count, size_t step for (size_t c=0; c(), "Learning rate for FTRL optimization") diff --git a/vowpalwabbit/gd.cc b/vowpalwabbit/gd.cc index 4e9153bb48d..48d96052a35 100644 --- a/vowpalwabbit/gd.cc +++ b/vowpalwabbit/gd.cc @@ -176,12 +176,9 @@ struct audit_results inline void audit_interaction(audit_results& dat, const audit_strings* f) -{ - if (f == nullptr) - { - if (!dat.ns_pre.empty()) - { - dat.ns_pre.pop_back(); +{ if (f == nullptr) + { if (!dat.ns_pre.empty()) + { dat.ns_pre.pop_back(); } return; @@ -192,25 +189,21 @@ inline void audit_interaction(audit_results& dat, const audit_strings* f) ns_pre += '*'; if (f->first != "" && ((f->first) != " ")) - { - ns_pre.append(f->first); + { ns_pre.append(f->first); ns_pre += '^'; } - + if (f->second != "") - { - ns_pre.append(f->second); + { ns_pre.append(f->second); } if (!ns_pre.empty()) - { - dat.ns_pre.push_back(ns_pre); + { dat.ns_pre.push_back(ns_pre); } } inline void audit_feature(audit_results& dat, const float ft_weight, const uint64_t ft_idx) -{ - weight_parameters& weights = dat.all.weights; +{ weight_parameters& weights = dat.all.weights; uint64_t index = ft_idx & weights.mask(); size_t stride_shift = dat.all.weights.stride_shift(); @@ -270,12 +263,11 @@ void print_features(vw& all, example& ec) for (features& fs : ec) { if (fs.space_names.size() > 0) for (features::iterator_all& f : fs.values_indices_audit()) - { - audit_interaction(dat, f.audit().get()); - audit_feature(dat, f.value(), f.index() + ec.ft_offset); - audit_interaction(dat, NULL); - } - else + { audit_interaction(dat, f.audit().get()); + audit_feature(dat, f.value(), f.index() + ec.ft_offset); + audit_interaction(dat, NULL); + } + else for (features::iterator& f : fs) audit_feature(dat, f.value(), f.index() + ec.ft_offset); } @@ -348,12 +340,12 @@ void predict(gd& g, base_learner&, example& ec) } inline void vec_add_trunc_multipredict(multipredict_info& mp, const float fx, uint64_t fi) -{ weight_parameters w = mp.weights; +{ weight_parameters w = mp.weights; size_t index = fi; for (size_t c=0; c @@ -386,16 +378,16 @@ struct power_data float neg_norm_power; }; - template +template inline float compute_rate_decay(power_data& s, float& fw) { weight* w = &fw; float rate_decay = 1.f; if(adaptive) - { if (sqrt_rate) - rate_decay = InvSqrt(w[adaptive]); - else - rate_decay = powf(w[adaptive],s.minus_power_t); - } + { if (sqrt_rate) + rate_decay = InvSqrt(w[adaptive]); + else + rate_decay = powf(w[adaptive],s.minus_power_t); + } if(normalized) { if (sqrt_rate) { float inv_norm = 1.f / w[normalized]; @@ -419,7 +411,7 @@ struct norm_data const float x_min = 1.084202e-19f; const float x2_min = x_min*x_min; - const float x2_max = FLT_MAX; +const float x2_max = FLT_MAX; template inline void pred_per_update_feature(norm_data& nd, float x, float& fw) @@ -575,19 +567,16 @@ void save_load_regressor(vw& all, io_buf& model_file, bool read, bool text) uint64_t i = 0; uint32_t old_i = 0; size_t brw = 1; - + if(all.print_invert) //write readable model with feature names - { - weight_parameters::iterator v = weights.begin(); - stringstream msg; + { weight_parameters::iterator v = weights.begin(); + stringstream msg; typedef std::map< std::string, size_t> str_int_map; - + for(str_int_map::iterator it = all.name_index_map.begin(); it != all.name_index_map.end(); ++it) - { - v = weights.begin() + it->second; + { v = weights.begin() + it->second; if(*v != 0.) - { - msg << it->first; + { msg << it->first; brw = bin_text_write_fixed(model_file, (char*)it->first.c_str(), sizeof(*it->first.c_str()), msg, true); @@ -601,44 +590,42 @@ void save_load_regressor(vw& all, io_buf& model_file, bool read, bool text) { brw = 1; weight_parameters::iterator v = weights.begin(); if (read) - { - if (all.num_bits < 31)//backwards compatible - { brw = bin_read_fixed(model_file, (char*)&old_i, sizeof(old_i), ""); - i = old_i; - } - else - brw = bin_read_fixed(model_file, (char*)&i, sizeof(i), ""); + { if (all.num_bits < 31)//backwards compatible + { brw = bin_read_fixed(model_file, (char*)&old_i, sizeof(old_i), ""); + i = old_i; + } + else + brw = bin_read_fixed(model_file, (char*)&i, sizeof(i), ""); if (brw > 0) { if (i >= length) { THROW("Model content is corrupted, weight vector index " << i << " must be less than total vector length " << length); } - v += i; - brw += bin_read_fixed(model_file, (char*)&(*v), sizeof(*v), ""); + v += i; + brw += bin_read_fixed(model_file, (char*)&(*v), sizeof(*v), ""); } } else// write binary or text { - v += i; + v += i; if (*v != 0.) - { stringstream msg; - msg << i; - if (all.num_bits < 31) - { - old_i = (uint32_t)i; - brw = bin_text_write_fixed(model_file, (char *)&old_i, sizeof(old_i), msg, text); - } - else - brw = bin_text_write_fixed(model_file, (char *)&i, sizeof(i), msg, text); - - msg << ":"<< *v << "\n"; - brw += bin_text_write_fixed(model_file, (char *)&(*v), sizeof(*v), msg, text); + { stringstream msg; + msg << i; + if (all.num_bits < 31) + { old_i = (uint32_t)i; + brw = bin_text_write_fixed(model_file, (char *)&old_i, sizeof(old_i), msg, text); } + else + brw = bin_text_write_fixed(model_file, (char *)&i, sizeof(i), msg, text); + + msg << ":"<< *v << "\n"; + brw += bin_text_write_fixed(model_file, (char *)&(*v), sizeof(*v), msg, text); + } } - if (!read) - ++i; + if (!read) + ++i; } while ((!read && i < length) || (read && brw >0)); } @@ -741,52 +728,49 @@ void save_load_online_state(vw& all, io_buf& model_file, bool read, bool text, g { brw = 1; weight_parameters::iterator v = weights.begin(); if (read) - { - brw = bin_read_fixed(model_file, (char*)&i, sizeof(i), ""); + { brw = bin_read_fixed(model_file, (char*)&i, sizeof(i), ""); if (brw > 0) { if (i >= length) { THROW("Model content is corrupted, weight vector index " << i << " must be less than total vector length " << length); } - v += i; + v += i; if (g == NULL || (! g->adaptive && ! g->normalized)) brw += bin_read_fixed(model_file, (char*)&(*v), sizeof(*v), ""); else if ((g->adaptive && !g->normalized) || (!g->adaptive && g->normalized)) - brw += bin_read_fixed(model_file, (char*)&(*v), sizeof(*v) * 2, ""); + brw += bin_read_fixed(model_file, (char*)&(*v), sizeof(*v) * 2, ""); else //adaptive and normalized - brw += bin_read_fixed(model_file, (char*)&(*v), sizeof(*v) * 3, ""); + brw += bin_read_fixed(model_file, (char*)&(*v), sizeof(*v) * 3, ""); /* if (!all.training) v[1] = v[2] = 0.;*/ } } else // write binary or text - { - v += i; + { v += i; if (*v != 0.) - { - msg << i; + { msg << i; brw = bin_text_write_fixed(model_file, (char *)&i, sizeof(i), msg, text); if (g == nullptr || (! g->adaptive && ! g->normalized)) - { msg << ":" << *v << "\n"; - brw += bin_text_write_fixed(model_file, (char *)&(*v), sizeof(*v), + { msg << ":" << *v << "\n"; + brw += bin_text_write_fixed(model_file, (char *)&(*v), sizeof(*v), msg, text); } else if ((g->adaptive && !g->normalized) || (!g->adaptive && g->normalized)) { //either adaptive or normalized msg << ":"<< *v << " "<< (&(*v))[1]<< "\n"; - brw += bin_text_write_fixed(model_file, (char *)&(*v), 2 * sizeof(*v), - msg, text); + brw += bin_text_write_fixed(model_file, (char *)&(*v), 2 * sizeof(*v), + msg, text); } else - { //adaptive and normalized - msg << ":"<< *v << " "<< (&(*v))[1] << " "<< (&(*v))[2]<< "\n"; - brw += bin_text_write_fixed(model_file, (char *)&(*v), 3 * sizeof(*v), - msg, text); + { //adaptive and normalized + msg << ":"<< *v << " "<< (&(*v))[1] << " "<< (&(*v))[2]<< "\n"; + brw += bin_text_write_fixed(model_file, (char *)&(*v), 3 * sizeof(*v), + msg, text); } } } - if (!read) - ++i; + if (!read) + ++i; } while ((!read && i < length) || (read && brw >0)); } @@ -794,12 +778,12 @@ void save_load_online_state(vw& all, io_buf& model_file, bool read, bool text, g struct initial_t { private: - weight _initial; + weight _initial; public: - initial_t(weight initial) : _initial(initial){} - void operator()(weight_parameters::iterator& iter, size_t /*index*/) - { (&(*iter))[1] = _initial; - } + initial_t(weight initial) : _initial(initial) {} + void operator()(weight_parameters::iterator& iter, size_t /*index*/) + { (&(*iter))[1] = _initial; + } }; void save_load(gd& g, io_buf& model_file, bool read, bool text) @@ -807,13 +791,13 @@ void save_load(gd& g, io_buf& model_file, bool read, bool text) if(read) { initialize_regressor(all); - if (all.adaptive && all.initial_t > 0){ - initial_t init(all.initial_t); - all.weights.set_default(init); //for adaptive update, we interpret initial_t as previously seeing initial_t fake datapoints, all with squared gradient=1 - //NOTE: this is not invariant to the scaling of the data (i.e. when combined with normalized). Since scaling the data scales the gradient, this should ideally be - //feature_range*initial_t, or something like that. We could potentially fix this by just adding this base quantity times the current range to the sum of gradients - //stored in memory at each update, and always start sum of gradients to 0, at the price of additional additions and multiplications during the update... - } + if (all.adaptive && all.initial_t > 0) + { initial_t init(all.initial_t); + all.weights.set_default(init); //for adaptive update, we interpret initial_t as previously seeing initial_t fake datapoints, all with squared gradient=1 + //NOTE: this is not invariant to the scaling of the data (i.e. when combined with normalized). Since scaling the data scales the gradient, this should ideally be + //feature_range*initial_t, or something like that. We could potentially fix this by just adding this base quantity times the current range to the sum of gradients + //stored in memory at each update, and always start sum of gradients to 0, at the price of additional additions and multiplications during the update... + } if (g.initial_constant != 0.0) VW::set_weight(all, constant, 0, g.initial_constant); } @@ -953,11 +937,10 @@ base_learner* setup(vw& all) } } else - { - all.adaptive = all.training; - all.invariant_updates = all.training; - all.normalized_updates = all.training; - } + { all.adaptive = all.training; + all.invariant_updates = all.training; + all.normalized_updates = all.training; + } if (pow((double)all.eta_decay_rate, (double)all.numpasses) < 0.0001 ) cerr << "Warning: the learning rate for the last pass is multiplied by: " << pow((double)all.eta_decay_rate, (double)all.numpasses) diff --git a/vowpalwabbit/gd.h b/vowpalwabbit/gd.h index 70c5731edac..15292235574 100644 --- a/vowpalwabbit/gd.h +++ b/vowpalwabbit/gd.h @@ -29,37 +29,34 @@ struct multipredict_info { size_t count; size_t step; polyprediction* pred; weig inline void vec_add_multipredict(multipredict_info& mp, const float fx, uint64_t fi) { if ((-1e-10 < fx) && (fx < 1e-10)) return; weight_parameters& w = mp.weights; - uint64_t mask = w.mask(); + uint64_t mask = w.mask(); polyprediction* p = mp.pred; fi &= mask; uint64_t top = fi + (uint64_t)((mp.count-1) * mp.step); uint64_t i = 0; if (top <= mask) - { - i += fi; - for (; i <= top; i+= mp.step, ++p) - p->scalar += fx * w[i]; //TODO: figure out how to use weight_parameters::iterator (not using change_begin()) + { i += fi; + for (; i <= top; i+= mp.step, ++p) + p->scalar += fx * w[i]; //TODO: figure out how to use weight_parameters::iterator (not using change_begin()) } else // TODO: this could be faster by unrolling into two loops for (size_t c=0; cscalar += fx * w[fi]; + p->scalar += fx * w[fi]; } } // iterate through one namespace (or its part), callback function T(some_data_R, feature_value_x, feature_weight) template inline void foreach_feature(weight_parameters& weights, features& fs, R& dat, uint64_t offset = 0, float mult = 1.) -{ - for (features::iterator& f : fs) +{ for (features::iterator& f : fs) T(dat, mult*f.value(), weights[(f.index() + offset)]); } // iterate through one namespace (or its part), callback function T(some_data_R, feature_value_x, feature_index) template void foreach_feature(weight_parameters& /*weights*/, features& fs, R&dat, uint64_t offset = 0, float mult = 1.) -{ - for (features::iterator& f : fs) +{ for (features::iterator& f : fs) T(dat, mult*f.value(), f.index() + offset); } @@ -69,7 +66,7 @@ template inline void foreach_feature(vw& all, example& ec, R& dat) { uint64_t offset = ec.ft_offset; -for (features& f : ec) + for (features& f : ec) foreach_feature(all.weights, f, dat, offset); INTERACTIONS::generate_interactions(all, ec, dat); @@ -92,8 +89,7 @@ inline float inline_predict(vw& all, example& ec) inline float sign(float w) { if (w < 0.) return -1.; else return 1.; } inline float trunc_weight(const float w, const float gravity) -{ - return (gravity < fabsf(w)) ? w - sign(w) * gravity : 0.f; +{ return (gravity < fabsf(w)) ? w - sign(w) * gravity : 0.f; } } diff --git a/vowpalwabbit/gd_mf.cc b/vowpalwabbit/gd_mf.cc index 570795db22a..91c30b0a218 100644 --- a/vowpalwabbit/gd_mf.cc +++ b/vowpalwabbit/gd_mf.cc @@ -50,8 +50,7 @@ void mf_print_offset_features(gdmf& d, example& ec, size_t offset) if (ec.feature_space[(unsigned char)i[0]].size() > 0 && ec.feature_space[(unsigned char)i[1]].size() > 0) { /* print out nsk^feature:hash:value:weight:nsk^feature^:hash:value:weight:prod_weights */ for (size_t k = 1; k <= d.rank; k++) - { - for (features::iterator_all& f1 : ec.feature_space[(unsigned char)i[0]].values_indices_audit()) + { for (features::iterator_all& f1 : ec.feature_space[(unsigned char)i[0]].values_indices_audit()) for (features::iterator_all& f2 : ec.feature_space[(unsigned char)i[1]].values_indices_audit()) { cout << '\t' << f1.audit().get()->first << k << '^' << f1.audit().get()->second << ':' << ((f1.index()+k)&mask) <<"(" << ((f1.index() + offset +k) & mask) << ")" << ':' << f1.value(); @@ -193,9 +192,8 @@ void mf_train(gdmf& d, example& ec) } void set_rand(weight_parameters::iterator& iter, uint64_t index, uint32_t stride) -{ - for (weights_iterator_iterator w = iter.begin(); w != iter.end(stride); ++w, ++index) - *w = (float)(0.1 * merand48(index)); +{ for (weights_iterator_iterator w = iter.begin(); w != iter.end(stride); ++w, ++index) + *w = (float)(0.1 * merand48(index)); } void save_load(gdmf& d, io_buf& model_file, bool read, bool text) @@ -203,14 +201,14 @@ void save_load(gdmf& d, io_buf& model_file, bool read, bool text) uint64_t length = (uint64_t)1 << all->num_bits; if(read) { initialize_regressor(*all); - if (all->random_weights) - all->weights.set_default(); + if (all->random_weights) + all->weights.set_default(); } if (model_file.files.size() > 0) { uint64_t i = 0; - size_t brw = 1; - weight_parameters& w = all->weights; + size_t brw = 1; + weight_parameters& w = all->weights; do { brw = 0; size_t K = d.rank*2+1; @@ -218,23 +216,22 @@ void save_load(gdmf& d, io_buf& model_file, bool read, bool text) msg << i << " "; brw += bin_text_read_write_fixed(model_file,(char *)&i, sizeof (i), "", read, msg, text); - if (brw != 0) - { weight_parameters::iterator iter = w.begin()+ i; - for (weights_iterator_iterator v = iter.begin(); v != iter.end(K); ++v) - { msg << &(*v) << " "; - brw += bin_text_read_write_fixed(model_file, (char *)&(*v), sizeof(*v), - "", read, msg, text); - } - } - if (text) - { - msg << "\n"; - brw += bin_text_read_write_fixed(model_file, nullptr, 0, - "", read, msg,text); + if (brw != 0) + { weight_parameters::iterator iter = w.begin()+ i; + for (weights_iterator_iterator v = iter.begin(); v != iter.end(K); ++v) + { msg << &(*v) << " "; + brw += bin_text_read_write_fixed(model_file, (char *)&(*v), sizeof(*v), + "", read, msg, text); } + } + if (text) + { msg << "\n"; + brw += bin_text_read_write_fixed(model_file, nullptr, 0, + "", read, msg,text); + } - if (!read) - ++i; + if (!read) + ++i; } while ((!read && i < length) || (read && brw >0)); } diff --git a/vowpalwabbit/gen_cs_example.cc b/vowpalwabbit/gen_cs_example.cc old mode 100755 new mode 100644 index b7fd5308547..81bf8920d2c --- a/vowpalwabbit/gen_cs_example.cc +++ b/vowpalwabbit/gen_cs_example.cc @@ -10,9 +10,10 @@ #include "vw_exception.h" #include "gen_cs_example.h" -namespace GEN_CS { +namespace GEN_CS +{ using namespace LEARNER; - using namespace CB_ALGS; +using namespace CB_ALGS; using namespace CB; inline bool observed_cost(cb_class* cl) @@ -21,8 +22,7 @@ inline bool observed_cost(cb_class* cl) } cb_class* get_observed_cost(CB::label& ld) -{ - for (auto& cl : ld.costs) +{ for (auto& cl : ld.costs) if (observed_cost(&cl)) return &cl; return nullptr; @@ -30,8 +30,7 @@ cb_class* get_observed_cost(CB::label& ld) //Multiline version void gen_cs_example_ips(v_array examples, COST_SENSITIVE::label& cs_labels) -{ - cs_labels.costs.erase(); +{ cs_labels.costs.erase(); bool shared = CB::ec_is_example_header(*examples[0]); for (uint32_t i = 0; i < examples.size()-1; i++) { CB::label ld = examples[i]->l.cb; @@ -45,15 +44,14 @@ void gen_cs_example_ips(v_array examples, COST_SENSITIVE::label& cs_la } if (shared)//take care of shared examples - { cs_labels.costs[0].class_index = 0; - cs_labels.costs[0].x = -FLT_MAX; - } + { cs_labels.costs[0].class_index = 0; + cs_labels.costs[0].x = -FLT_MAX; + } } - //Multiline version +//Multiline version void gen_cs_example_dm(v_array examples, COST_SENSITIVE::label& cs_labels) -{ - cs_labels.costs.erase(); +{ cs_labels.costs.erase(); bool shared = CB::ec_is_example_header(*examples[0]); for (uint32_t i = 0; i < examples.size()-1; i++) { CB::label ld = examples[i]->l.cb; @@ -67,28 +65,26 @@ void gen_cs_example_dm(v_array examples, COST_SENSITIVE::label& cs_lab } if (shared)//take care of shared examples - { cs_labels.costs[0].class_index = 0; - cs_labels.costs[0].x = -FLT_MAX; - } + { cs_labels.costs[0].class_index = 0; + cs_labels.costs[0].x = -FLT_MAX; + } } //Multiline version void gen_cs_test_example(v_array examples, COST_SENSITIVE::label& cs_labels) -{ - cs_labels.costs.erase(); +{ cs_labels.costs.erase(); bool shared = CB::ec_is_example_header(*examples[0]); for (uint32_t i = 0; i < examples.size()-1; i++) - { - COST_SENSITIVE::wclass wc = {FLT_MAX,i,0.,0.}; + { COST_SENSITIVE::wclass wc = {FLT_MAX,i,0.,0.}; if (shared && i > 0) wc.class_index = (uint32_t)i-1; cs_labels.costs.push_back(wc); } if (shared)//take care of shared examples - { cs_labels.costs[0].class_index = 0; - cs_labels.costs[0].x = -FLT_MAX; - } + { cs_labels.costs[0].class_index = 0; + cs_labels.costs[0].x = -FLT_MAX; + } } //single line version @@ -97,76 +93,71 @@ void gen_cs_example_ips(cb_to_cs& c, CB::label& ld, COST_SENSITIVE::label& cs_ld //generate cost-sensitive example cs_ld.costs.erase(); if (ld.costs.size() == 1 && !is_test_label(ld)) //this is a typical example where we can perform all actions - { //in this case generate cost-sensitive example with all actions - for (uint32_t i = 1; i <= c.num_actions; i++) - { - COST_SENSITIVE::wclass wc = {0.,i,0.,0.}; - if (c.known_cost != nullptr && i == c.known_cost->action) - { - wc.x = c.known_cost->cost / c.known_cost->probability; //use importance weighted cost for observed action, 0 otherwise - //ips can be thought as the doubly robust method with a fixed regressor that predicts 0 costs for everything - //update the loss of this regressor - c.nb_ex_regressors++; - c.avg_loss_regressors += (1.0f / c.nb_ex_regressors)*((c.known_cost->cost)*(c.known_cost->cost) - c.avg_loss_regressors); - c.last_pred_reg = 0; - c.last_correct_cost = c.known_cost->cost; - } - - cs_ld.costs.push_back(wc); - } + { //in this case generate cost-sensitive example with all actions + for (uint32_t i = 1; i <= c.num_actions; i++) + { COST_SENSITIVE::wclass wc = {0.,i,0.,0.}; + if (c.known_cost != nullptr && i == c.known_cost->action) + { wc.x = c.known_cost->cost / c.known_cost->probability; //use importance weighted cost for observed action, 0 otherwise + //ips can be thought as the doubly robust method with a fixed regressor that predicts 0 costs for everything + //update the loss of this regressor + c.nb_ex_regressors++; + c.avg_loss_regressors += (1.0f / c.nb_ex_regressors)*((c.known_cost->cost)*(c.known_cost->cost) - c.avg_loss_regressors); + c.last_pred_reg = 0; + c.last_correct_cost = c.known_cost->cost; + } + + cs_ld.costs.push_back(wc); } + } else //this is an example where we can only perform a subset of the actions - { //in this case generate cost-sensitive example with only allowed actions - for (auto& cl : ld.costs) - { - COST_SENSITIVE::wclass wc = {0., cl.action, 0., 0.}; - if (c.known_cost != nullptr && cl.action == c.known_cost->action) - { - wc.x = c.known_cost->cost / c.known_cost->probability; //use importance weighted cost for observed action, 0 otherwise - - //ips can be thought as the doubly robust method with a fixed regressor that predicts 0 costs for everything - //update the loss of this regressor - c.nb_ex_regressors++; - c.avg_loss_regressors += (1.0f / c.nb_ex_regressors)*((c.known_cost->cost)*(c.known_cost->cost) - c.avg_loss_regressors); - c.last_pred_reg = 0; - c.last_correct_cost = c.known_cost->cost; - } - - cs_ld.costs.push_back(wc); - } + { //in this case generate cost-sensitive example with only allowed actions + for (auto& cl : ld.costs) + { COST_SENSITIVE::wclass wc = {0., cl.action, 0., 0.}; + if (c.known_cost != nullptr && cl.action == c.known_cost->action) + { wc.x = c.known_cost->cost / c.known_cost->probability; //use importance weighted cost for observed action, 0 otherwise + + //ips can be thought as the doubly robust method with a fixed regressor that predicts 0 costs for everything + //update the loss of this regressor + c.nb_ex_regressors++; + c.avg_loss_regressors += (1.0f / c.nb_ex_regressors)*((c.known_cost->cost)*(c.known_cost->cost) - c.avg_loss_regressors); + c.last_pred_reg = 0; + c.last_correct_cost = c.known_cost->cost; + } + + cs_ld.costs.push_back(wc); } + } } - - void gen_cs_example_mtr(cb_to_cs_adf& c, v_array& ec_seq, COST_SENSITIVE::label& cs_labels) - { - bool shared = CB::ec_is_example_header(*(ec_seq[0])); - c.action_sum += ec_seq.size()-2; //-1 for shared -1 for end example - if (!shared) - c.action_sum += 1; - c.event_sum++; - - c.mtr_ec_seq.erase(); - cs_labels.costs.erase(); - for (size_t i = 0; i < ec_seq.size(); i++) - { CB::label ld = ec_seq[i]->l.cb; - - COST_SENSITIVE::wclass wc = {0, 0}; - - bool keep_example = false; - if (shared && i == 0) - { wc.x = -FLT_MAX; - keep_example = true; - } - else if (ld.costs.size() == 1 && ld.costs[0].cost != FLT_MAX) - { wc.x = ld.costs[0].cost; - c.mtr_example = (uint32_t)i; - keep_example = true; - } - if (keep_example) - { c.mtr_ec_seq.push_back(ec_seq[i]); - cs_labels.costs.push_back(wc); - } - } - c.mtr_ec_seq.push_back(ec_seq[ec_seq.size()-1]);//must include the end-of-line example + +void gen_cs_example_mtr(cb_to_cs_adf& c, v_array& ec_seq, COST_SENSITIVE::label& cs_labels) +{ bool shared = CB::ec_is_example_header(*(ec_seq[0])); + c.action_sum += ec_seq.size()-2; //-1 for shared -1 for end example + if (!shared) + c.action_sum += 1; + c.event_sum++; + + c.mtr_ec_seq.erase(); + cs_labels.costs.erase(); + for (size_t i = 0; i < ec_seq.size(); i++) + { CB::label ld = ec_seq[i]->l.cb; + + COST_SENSITIVE::wclass wc = {0, 0}; + + bool keep_example = false; + if (shared && i == 0) + { wc.x = -FLT_MAX; + keep_example = true; + } + else if (ld.costs.size() == 1 && ld.costs[0].cost != FLT_MAX) + { wc.x = ld.costs[0].cost; + c.mtr_example = (uint32_t)i; + keep_example = true; + } + if (keep_example) + { c.mtr_ec_seq.push_back(ec_seq[i]); + cs_labels.costs.push_back(wc); + } } + c.mtr_ec_seq.push_back(ec_seq[ec_seq.size()-1]);//must include the end-of-line example +} } diff --git a/vowpalwabbit/gen_cs_example.h b/vowpalwabbit/gen_cs_example.h old mode 100755 new mode 100644 index 8df1a5f5fd0..7b1f081c7db --- a/vowpalwabbit/gen_cs_example.h +++ b/vowpalwabbit/gen_cs_example.h @@ -10,12 +10,12 @@ #include "cb_algs.h" #include "vw_exception.h" -namespace GEN_CS{ +namespace GEN_CS +{ struct cb_to_cs -{ - size_t cb_type; - uint32_t num_actions; +{ size_t cb_type; + uint32_t num_actions; COST_SENSITIVE::label pred_scores; LEARNER::base_learner* scorer; float avg_loss_regressors; @@ -27,13 +27,12 @@ struct cb_to_cs }; struct cb_to_cs_adf -{ - size_t cb_type; +{ size_t cb_type; //for MTR uint64_t action_sum; uint64_t event_sum; - uint32_t mtr_example; + uint32_t mtr_example; v_array mtr_ec_seq;//shared + the one example + an end example. //for DR @@ -46,7 +45,7 @@ CB::cb_class* get_observed_cost(CB::label& ld); void gen_cs_example_ips(cb_to_cs& c, CB::label& ld, COST_SENSITIVE::label& cs_ld); -template +template void gen_cs_example_dm(cb_to_cs& c, example& ec, COST_SENSITIVE::label& cs_ld) { //this implements the direct estimation method, where costs are directly specified by the learned regressor. CB::label ld = ec.l.cb; @@ -58,65 +57,58 @@ void gen_cs_example_dm(cb_to_cs& c, example& ec, COST_SENSITIVE::label& cs_ld) c.pred_scores.costs.erase(); if (ld.costs.size() == 1 && !is_test_label(ld)) //this is a typical example where we can perform all actions - { //in this case generate cost-sensitive example with all actions - for (uint32_t i = 1; i <= c.num_actions; i++) - { - COST_SENSITIVE::wclass wc = {0., i, 0., 0.}; - //get cost prediction for this action - wc.x = CB_ALGS::get_cost_pred(c.scorer, c.known_cost, ec, i, 0); - if (wc.x < min) - { - min = wc.x; - argmin = i; - } - - c.pred_scores.costs.push_back(wc); - - if (c.known_cost != nullptr && c.known_cost->action == i) - { - c.nb_ex_regressors++; - c.avg_loss_regressors += (1.0f / c.nb_ex_regressors)*((c.known_cost->cost - wc.x)*(c.known_cost->cost - wc.x) - c.avg_loss_regressors); - c.last_pred_reg = wc.x; - c.last_correct_cost = c.known_cost->cost; - } - - cs_ld.costs.push_back(wc); - } + { //in this case generate cost-sensitive example with all actions + for (uint32_t i = 1; i <= c.num_actions; i++) + { COST_SENSITIVE::wclass wc = {0., i, 0., 0.}; + //get cost prediction for this action + wc.x = CB_ALGS::get_cost_pred(c.scorer, c.known_cost, ec, i, 0); + if (wc.x < min) + { min = wc.x; + argmin = i; + } + + c.pred_scores.costs.push_back(wc); + + if (c.known_cost != nullptr && c.known_cost->action == i) + { c.nb_ex_regressors++; + c.avg_loss_regressors += (1.0f / c.nb_ex_regressors)*((c.known_cost->cost - wc.x)*(c.known_cost->cost - wc.x) - c.avg_loss_regressors); + c.last_pred_reg = wc.x; + c.last_correct_cost = c.known_cost->cost; + } + + cs_ld.costs.push_back(wc); } + } else //this is an example where we can only perform a subset of the actions - { //in this case generate cost-sensitive example with only allowed actions - for (auto& cl : ld.costs) - { - COST_SENSITIVE::wclass wc = {0., cl.action, 0., 0.}; - - //get cost prediction for this action - wc.x = CB_ALGS::get_cost_pred(c.scorer, c.known_cost, ec, cl.action, 0); - if (wc.x < min || (wc.x == min && cl.action < argmin)) - { - min = wc.x; - argmin = cl.action; - } - c.pred_scores.costs.push_back(wc); - - if (c.known_cost != nullptr && c.known_cost->action == cl.action) - { - c.nb_ex_regressors++; - c.avg_loss_regressors += (1.0f / c.nb_ex_regressors)*((c.known_cost->cost - wc.x)*(c.known_cost->cost - wc.x) - c.avg_loss_regressors); - c.last_pred_reg = wc.x; - c.last_correct_cost = c.known_cost->cost; - } - - cs_ld.costs.push_back(wc); - } + { //in this case generate cost-sensitive example with only allowed actions + for (auto& cl : ld.costs) + { COST_SENSITIVE::wclass wc = {0., cl.action, 0., 0.}; + + //get cost prediction for this action + wc.x = CB_ALGS::get_cost_pred(c.scorer, c.known_cost, ec, cl.action, 0); + if (wc.x < min || (wc.x == min && cl.action < argmin)) + { min = wc.x; + argmin = cl.action; + } + c.pred_scores.costs.push_back(wc); + + if (c.known_cost != nullptr && c.known_cost->action == cl.action) + { c.nb_ex_regressors++; + c.avg_loss_regressors += (1.0f / c.nb_ex_regressors)*((c.known_cost->cost - wc.x)*(c.known_cost->cost - wc.x) - c.avg_loss_regressors); + c.last_pred_reg = wc.x; + c.last_correct_cost = c.known_cost->cost; + } + + cs_ld.costs.push_back(wc); } + } ec.pred.multiclass = argmin; } -template +template void gen_cs_label(cb_to_cs& c, example& ec, COST_SENSITIVE::label& cs_ld, uint32_t action) -{ - COST_SENSITIVE::wclass wc = {0., action, 0., 0.}; +{ COST_SENSITIVE::wclass wc = {0., action, 0., 0.}; //get cost prediction for this action wc.x = CB_ALGS::get_cost_pred(c.scorer, c.known_cost, ec, action, c.num_actions); @@ -124,33 +116,32 @@ void gen_cs_label(cb_to_cs& c, example& ec, COST_SENSITIVE::label& cs_ld, uint32 c.pred_scores.costs.push_back(wc); //add correction if we observed cost for this action and regressor is wrong if (c.known_cost != nullptr && c.known_cost->action == action) - { - c.nb_ex_regressors++; - c.avg_loss_regressors += (1.0f / c.nb_ex_regressors)*((c.known_cost->cost - wc.x)*(c.known_cost->cost - wc.x) - c.avg_loss_regressors); - c.last_pred_reg = wc.x; - c.last_correct_cost = c.known_cost->cost; - wc.x += (c.known_cost->cost - wc.x) / c.known_cost->probability; - } + { c.nb_ex_regressors++; + c.avg_loss_regressors += (1.0f / c.nb_ex_regressors)*((c.known_cost->cost - wc.x)*(c.known_cost->cost - wc.x) - c.avg_loss_regressors); + c.last_pred_reg = wc.x; + c.last_correct_cost = c.known_cost->cost; + wc.x += (c.known_cost->cost - wc.x) / c.known_cost->probability; + } cs_ld.costs.push_back(wc); } -template +template void gen_cs_example_dr(cb_to_cs& c, example& ec, CB::label& ld, COST_SENSITIVE::label& cs_ld) { //this implements the doubly robust method cs_ld.costs.erase(); c.pred_scores.costs.erase(); if (ld.costs.size() == 0)//a test example for (uint32_t i = 1; i <= c.num_actions; i++) - { //Explicit declaration for a weak compiler. - COST_SENSITIVE::wclass c = { FLT_MAX, i, 0., 0. }; - cs_ld.costs.push_back(c); - } + { //Explicit declaration for a weak compiler. + COST_SENSITIVE::wclass c = { FLT_MAX, i, 0., 0. }; + cs_ld.costs.push_back(c); + } else if (ld.costs.size() == 1 && !is_test_label(ld)) //this is a typical example where we can perform all actions //in this case generate cost-sensitive example with all actions - for (uint32_t i = 1; i <= c.num_actions; i++) { - gen_cs_label(c, ec, cs_ld, i); + for (uint32_t i = 1; i <= c.num_actions; i++) + { gen_cs_label(c, ec, cs_ld, i); } else //this is an example where we can only perform a subset of the actions //in this case generate cost-sensitive example with only allowed actions @@ -160,10 +151,8 @@ void gen_cs_example_dr(cb_to_cs& c, example& ec, CB::label& ld, COST_SENSITIVE:: template void gen_cs_example(cb_to_cs& c, example& ec, CB::label& ld, COST_SENSITIVE::label& cs_ld) -{ - switch (c.cb_type) - { - case CB_TYPE_IPS: +{ switch (c.cb_type) + { case CB_TYPE_IPS: gen_cs_example_ips(c, ld, cs_ld); break; case CB_TYPE_DM: @@ -174,19 +163,19 @@ void gen_cs_example(cb_to_cs& c, example& ec, CB::label& ld, COST_SENSITIVE::lab break; default: THROW("Unknown cb_type specified for contextual bandit learning: " << c.cb_type); - } + } } - void gen_cs_test_example(v_array examples, COST_SENSITIVE::label& cs_labels); +void gen_cs_test_example(v_array examples, COST_SENSITIVE::label& cs_labels); - void gen_cs_example_ips(v_array examples, COST_SENSITIVE::label& cs_labels); +void gen_cs_example_ips(v_array examples, COST_SENSITIVE::label& cs_labels); - void gen_cs_example_dm(v_array examples, COST_SENSITIVE::label& cs_labels); - - void gen_cs_example_mtr(cb_to_cs_adf& c, v_array& ec_seq, COST_SENSITIVE::label& cs_labels); +void gen_cs_example_dm(v_array examples, COST_SENSITIVE::label& cs_labels); - template - void gen_cs_example_dr(cb_to_cs_adf& c, v_array examples, COST_SENSITIVE::label& cs_labels) +void gen_cs_example_mtr(cb_to_cs_adf& c, v_array& ec_seq, COST_SENSITIVE::label& cs_labels); + +template +void gen_cs_example_dr(cb_to_cs_adf& c, v_array examples, COST_SENSITIVE::label& cs_labels) { //size_t mysize = examples.size(); c.pred_scores.costs.erase(); bool shared = CB::ec_is_example_header(*examples[0]); @@ -195,7 +184,7 @@ void gen_cs_example(cb_to_cs& c, example& ec, CB::label& ld, COST_SENSITIVE::lab cs_labels.costs.erase(); for (size_t i = 0; i < examples.size(); i++) - { if (CB_ALGS::example_is_newline_not_header(*examples[i])) continue; + { if (CB_ALGS::example_is_newline_not_header(*examples[i])) continue; COST_SENSITIVE::wclass wc = {0.,0,0.,0.}; @@ -229,65 +218,60 @@ void gen_cs_example(cb_to_cs& c, example& ec, CB::label& ld, COST_SENSITIVE::lab } } - template - void gen_cs_example(cb_to_cs_adf& c, v_array& ec_seq, COST_SENSITIVE::label& cs_labels) - { - switch (c.cb_type) - { - case CB_TYPE_IPS: - gen_cs_example_ips(ec_seq, cs_labels); - break; - case CB_TYPE_DR: - gen_cs_example_dr(c, ec_seq, cs_labels); - break; - case CB_TYPE_MTR: - gen_cs_example_mtr(c, ec_seq, cs_labels); - break; - default: - THROW("Unknown cb_type specified for contextual bandit learning: " << c.cb_type); - } - } - - template - void call_cs_ldf(LEARNER::base_learner& base, v_array& examples, v_array& cb_labels, - COST_SENSITIVE::label& cs_labels, v_array& prepped_cs_labels, uint64_t offset, size_t id = 0) - { - cb_labels.erase(); - if (prepped_cs_labels.size() < cs_labels.costs.size()+1) - { - prepped_cs_labels.resize(cs_labels.costs.size()+1); - prepped_cs_labels.end() = prepped_cs_labels.end_array; - } - - // 1st: save cb_label (into mydata) and store cs_label for each example, which will be passed into base.learn. - size_t index = 0; - for (example* ec : examples) - { cb_labels.push_back(ec->l.cb); - prepped_cs_labels[index].costs.erase(); - if (index != examples.size()-1) - prepped_cs_labels[index].costs.push_back(cs_labels.costs[index]); - else - prepped_cs_labels[index].costs.push_back({FLT_MAX,0,0.,0.}); - ec->l.cs = prepped_cs_labels[index++]; - } - - // 2nd: predict for each ex - // // call base.predict for each vw exmaple in the sequence - for (example* ec : examples) - { - uint64_t old_offset = ec->ft_offset; - ec->ft_offset = offset; - if (is_learn) - base.learn(*ec, id); - else - base.predict(*ec, id); - ec->ft_offset = old_offset; - - } - // 3rd: restore cb_label for each example - // (**ec).l.cb = array.element. - size_t i = 0; - for (example* ec : examples) - ec->l.cb = cb_labels[i++]; - } +template +void gen_cs_example(cb_to_cs_adf& c, v_array& ec_seq, COST_SENSITIVE::label& cs_labels) +{ switch (c.cb_type) + { case CB_TYPE_IPS: + gen_cs_example_ips(ec_seq, cs_labels); + break; + case CB_TYPE_DR: + gen_cs_example_dr(c, ec_seq, cs_labels); + break; + case CB_TYPE_MTR: + gen_cs_example_mtr(c, ec_seq, cs_labels); + break; + default: + THROW("Unknown cb_type specified for contextual bandit learning: " << c.cb_type); + } +} + +template +void call_cs_ldf(LEARNER::base_learner& base, v_array& examples, v_array& cb_labels, + COST_SENSITIVE::label& cs_labels, v_array& prepped_cs_labels, uint64_t offset, size_t id = 0) +{ cb_labels.erase(); + if (prepped_cs_labels.size() < cs_labels.costs.size()+1) + { prepped_cs_labels.resize(cs_labels.costs.size()+1); + prepped_cs_labels.end() = prepped_cs_labels.end_array; + } + + // 1st: save cb_label (into mydata) and store cs_label for each example, which will be passed into base.learn. + size_t index = 0; + for (example* ec : examples) + { cb_labels.push_back(ec->l.cb); + prepped_cs_labels[index].costs.erase(); + if (index != examples.size()-1) + prepped_cs_labels[index].costs.push_back(cs_labels.costs[index]); + else + prepped_cs_labels[index].costs.push_back({FLT_MAX,0,0.,0.}); + ec->l.cs = prepped_cs_labels[index++]; + } + + // 2nd: predict for each ex + // // call base.predict for each vw exmaple in the sequence + for (example* ec : examples) + { uint64_t old_offset = ec->ft_offset; + ec->ft_offset = offset; + if (is_learn) + base.learn(*ec, id); + else + base.predict(*ec, id); + ec->ft_offset = old_offset; + + } + // 3rd: restore cb_label for each example + // (**ec).l.cb = array.element. + size_t i = 0; + for (example* ec : examples) + ec->l.cb = cb_labels[i++]; +} } diff --git a/vowpalwabbit/global_data.h b/vowpalwabbit/global_data.h index 35401f1ebd1..065ecb40cb8 100644 --- a/vowpalwabbit/global_data.h +++ b/vowpalwabbit/global_data.h @@ -108,9 +108,9 @@ struct version_struct void from_string(const char* str) { #ifdef _WIN32 - sscanf_s(str, "%d.%d.%d", &major, &minor, &rev); + sscanf_s(str, "%d.%d.%d", &major, &minor, &rev); #else - std::sscanf(str,"%d.%d.%d",&major,&minor,&rev); + std::sscanf(str,"%d.%d.%d",&major,&minor,&rev); #endif } }; @@ -179,8 +179,8 @@ class namedlabels { uint64_t hash = uniform_hash((unsigned char*)s.begin, s.end-s.begin, 378401); uint64_t v = name2id.get(s, hash); if (v == 0) - { std::cerr << "warning: missing named label '"; - for (char*c = s.begin; c != s.end; c++) std::cerr << *c; + { std::cerr << "warning: missing named label '"; + for (char*c = s.begin; c != s.end; c++) std::cerr << *c; std::cerr << '\'' << std::endl; } return v; @@ -394,13 +394,13 @@ class AllReduce; // avoid name clash namespace label_type { enum label_type_t - { simple, - cb, // contextual-bandit - cb_eval, // contextual-bandit evaluation - cs, // cost-sensitive - multi, - mc - }; +{ simple, + cb, // contextual-bandit + cb_eval, // contextual-bandit evaluation + cs, // cost-sensitive + multi, + mc +}; } struct vw @@ -494,7 +494,7 @@ struct vw std::vector namespace_dictionaries[256]; // each namespace has a list of dictionaries attached to it std::vector loaded_dictionaries; // which dictionaries have we loaded from a file to memory? - void(*delete_prediction)(void*);bool audit;//should I print lots of debugging information? + void(*delete_prediction)(void*); bool audit; //should I print lots of debugging information? bool quiet;//Should I suppress progress-printing of updates? bool training;//Should I train if lable data is available? bool active; @@ -558,7 +558,7 @@ struct vw std::map< std::string, size_t> name_index_map; label_type::label_type_t label_type; - + vw(); }; diff --git a/vowpalwabbit/interact.cc b/vowpalwabbit/interact.cc index 11b15c4cf75..f150ca78504 100644 --- a/vowpalwabbit/interact.cc +++ b/vowpalwabbit/interact.cc @@ -65,10 +65,10 @@ void multiply(features& f_dest, features& f_src2, interact& in) } if(cur_id1 == cur_id2) - { f_dest.push_back(f_src1.values[i1]*f_src2.values[i2], f_src1.indicies[i1]); - i1++; - i2++; - } + { f_dest.push_back(f_src1.values[i1]*f_src2.values[i2], f_src1.indicies[i1]); + i1++; + i2++; + } else if (cur_id1 < cur_id2) i1++; else @@ -82,13 +82,13 @@ void predict_or_learn(interact& in, LEARNER::base_learner& base, example& ec) features& f2 = ec.feature_space[in.n2]; if (!contains_valid_namespaces(f1, f2, in)) - { if (is_learn) - base.learn(ec); - else - base.predict(ec); + { if (is_learn) + base.learn(ec); + else + base.predict(ec); - return; - } + return; + } in.num_features = ec.num_features; in.total_sum_feat_sq = ec.total_sum_feat_sq; @@ -139,9 +139,9 @@ LEARNER::base_learner* interact_setup(vw& all) return nullptr; string s = all.vm["interact"].as(); if(s.length() != 2) - { cerr<<"Need two namespace arguments to interact: " << s << " won't do EXITING\n"; + { cerr<<"Need two namespace arguments to interact: " << s << " won't do EXITING\n"; return nullptr; - } + } interact& data = calloc_or_throw(); diff --git a/vowpalwabbit/interactions.cc b/vowpalwabbit/interactions.cc index f6a5de152ee..9585aff2d41 100644 --- a/vowpalwabbit/interactions.cc +++ b/vowpalwabbit/interactions.cc @@ -57,7 +57,7 @@ v_array expand_interactions(const vector& vec, const size_t re { const size_t len = i.length(); if (required_length > 0 && len != required_length) // got strict requirement of interaction length and it was failed. - { THROW(err_msg); } + { THROW(err_msg); } else if (len < 2) // regardles of required_length value this check is always performed THROW("error, feature interactions must involve at least two namespaces" << err_msg); diff --git a/vowpalwabbit/interactions.h b/vowpalwabbit/interactions.h index b2b1c05b275..ae910ed5bfb 100644 --- a/vowpalwabbit/interactions.h +++ b/vowpalwabbit/interactions.h @@ -1,4 +1,4 @@ - #pragma once +#pragma once #include "global_data.h" #include "constant.h" @@ -23,7 +23,7 @@ const uint64_t valid_ns_size = printable_end - printable_start - 1; // -1 to ski // exand all wildcard namespaces in vector // req_length must be 0 if interactions of any length are allowed, otherwise contains required length // err_msg will be printed plus exception will be thrown if req_length != 0 and mismatch interaction length. - v_array expand_interactions(const std::vector &vec, const size_t required_length, const std::string &err_msg); +v_array expand_interactions(const std::vector &vec, const size_t required_length, const std::string &err_msg); // remove duplicate interactions and sort namespaces in them (if required) void sort_and_filter_duplicate_interactions(v_array &vec, bool filter_duplicates, size_t &removed_cnt, size_t &sorted_cnt); @@ -55,14 +55,12 @@ void eval_count_of_generated_ft(vw& all, example& ec, size_t& new_features_cnt, template inline void call_T(R& dat, weight_parameters& weights, const float ft_value, const uint64_t ft_idx) -{ - T(dat, ft_value, weights[ft_idx]); +{ T(dat, ft_value, weights[ft_idx]); } template inline void call_T(R& dat, weight_parameters& /*weights*/, const float ft_value, const uint64_t ft_idx) -{ - T(dat, ft_value, ft_idx); +{ T(dat, ft_value, ft_idx); } // state data used in non-recursive feature generation algorithm @@ -91,19 +89,15 @@ inline float INTERACTION_VALUE(float value1, float value2) { return value1*value template inline void inner_kernel(R& dat, features::iterator_all& begin, features::iterator_all& end, const uint64_t offset, weight_parameters& weights, feature_value ft_value, feature_index halfhash) -{ - if (audit) - { - for (; begin != end; ++begin) - { - audit_func(dat, begin.audit().get()); +{ if (audit) + { for (; begin != end; ++begin) + { audit_func(dat, begin.audit().get()); call_T(dat, weights, INTERACTION_VALUE(ft_value, begin.value()), (begin.index() ^ halfhash) + offset); audit_func(dat, nullptr); } } else - { - for (; begin != end; ++begin) + { for (; begin != end; ++begin) call_T(dat, weights, INTERACTION_VALUE(ft_value, begin.value()), (begin.index() ^ halfhash) + offset); } } @@ -113,10 +107,9 @@ inline void inner_kernel(R& dat, features::iterator_all& begin, features::iterat // and passes each of them to given function T() // it must be in header file to avoid compilation problems - template // nullptr func can't be used as template param in old compilers - inline void generate_interactions(vw& all, example& ec, R& dat) // default value removed to eliminate ambiguity in old complers - { - features* features_data = ec.feature_space; +template // nullptr func can't be used as template param in old compilers +inline void generate_interactions(vw& all, example& ec, R& dat) // default value removed to eliminate ambiguity in old complers +{ features* features_data = ec.feature_space; // often used values const uint64_t offset = ec.ft_offset; @@ -145,223 +138,213 @@ inline void inner_kernel(R& dat, features::iterator_all& begin, features::iterat const size_t len = ns.size(); if (len == 2) //special case of pairs - { - features& first = features_data[ns[0]]; - if (first.nonempty()) - { - features& second = features_data[ns[1]]; - if (second.nonempty()) - { - const bool same_namespace = ( !all.permutations && ( ns[0] == ns[1] ) ); - - for(size_t i = 0; i < first.indicies.size(); ++i) - { feature_index halfhash = FNV_prime * (uint64_t)first.indicies[i]; - if (audit) audit_func(dat, first.space_names[i].get()); - // next index differs for permutations and simple combinations - feature_value ft_value = first.values[i]; - features::features_value_index_audit_range range = second.values_indices_audit(); - features::iterator_all begin = range.begin(); - if (same_namespace) - begin += (PROCESS_SELF_INTERACTIONS(ft_value)) ? i : i + 1; - - features::iterator_all end = range.end(); - inner_kernel(dat, begin, end, offset, weights, ft_value, halfhash); - - if (audit) audit_func(dat, nullptr); - } // end for(fst) - } // end if (data[snd] size > 0) - } // end if (data[fst] size > 0) - } - else - if (len == 3) // special case for triples - { features& first = features_data[ns[0]]; - if (first.nonempty()) - { - features& second = features_data[ns[1]]; - if (second.nonempty()) - { - features& third = features_data[ns[2]]; - if (third.nonempty()) - { - // don't compare 1 and 3 as interaction is sorted - const bool same_namespace1 = ( !all.permutations && ( ns[0] == ns[1] ) ); - const bool same_namespace2 = ( !all.permutations && ( ns[1] == ns[2] ) ); - - for(size_t i = 0; i < first.indicies.size(); ++i) - { - if(audit) audit_func(dat, first.space_names[i].get()); - const uint64_t halfhash1 = FNV_prime * (uint64_t)first.indicies[i]; - const float& first_ft_value = first.values[i]; - size_t j=0; - if (same_namespace1) // next index differs for permutations and simple combinations - j = (PROCESS_SELF_INTERACTIONS(first_ft_value)) ? i : i+1; - - for (; j < second.indicies.size(); ++j) - { //f3 x k*(f2 x k*f1) - if(audit) audit_func(dat, second.space_names[j].get()); - feature_index halfhash = FNV_prime * (halfhash1 ^ (uint64_t)second.indicies[j]); - feature_value ft_value = INTERACTION_VALUE(first_ft_value, second.values[j]); - - features::features_value_index_audit_range range = third.values_indices_audit(); - features::iterator_all begin = range.begin(); - if (same_namespace2) //next index differs for permutations and simple combinations - begin += (PROCESS_SELF_INTERACTIONS(ft_value)) ? j : j + 1; - - features::iterator_all end = range.end(); - inner_kernel(dat, begin, end, offset, weights, ft_value, halfhash); - } // end for (snd) - if(audit) audit_func(dat, nullptr); - } // end for (fst) - - } // end if (data[thr] size > 0) - } // end if (data[snd] size > 0) - } // end if (data[fst] size > 0) + { features& first = features_data[ns[0]]; + if (first.nonempty()) + { features& second = features_data[ns[1]]; + if (second.nonempty()) + { const bool same_namespace = ( !all.permutations && ( ns[0] == ns[1] ) ); + + for(size_t i = 0; i < first.indicies.size(); ++i) + { feature_index halfhash = FNV_prime * (uint64_t)first.indicies[i]; + if (audit) audit_func(dat, first.space_names[i].get()); + // next index differs for permutations and simple combinations + feature_value ft_value = first.values[i]; + features::features_value_index_audit_range range = second.values_indices_audit(); + features::iterator_all begin = range.begin(); + if (same_namespace) + begin += (PROCESS_SELF_INTERACTIONS(ft_value)) ? i : i + 1; - } - else // generic case: quatriples, etc. + features::iterator_all end = range.end(); + inner_kernel(dat, begin, end, offset, weights, ft_value, halfhash); + + if (audit) audit_func(dat, nullptr); + } // end for(fst) + } // end if (data[snd] size > 0) + } // end if (data[fst] size > 0) + } + else if (len == 3) // special case for triples + { features& first = features_data[ns[0]]; + if (first.nonempty()) + { features& second = features_data[ns[1]]; + if (second.nonempty()) + { features& third = features_data[ns[2]]; + if (third.nonempty()) + { // don't compare 1 and 3 as interaction is sorted + const bool same_namespace1 = ( !all.permutations && ( ns[0] == ns[1] ) ); + const bool same_namespace2 = ( !all.permutations && ( ns[1] == ns[2] ) ); + + for(size_t i = 0; i < first.indicies.size(); ++i) + { if(audit) audit_func(dat, first.space_names[i].get()); + const uint64_t halfhash1 = FNV_prime * (uint64_t)first.indicies[i]; + const float& first_ft_value = first.values[i]; + size_t j=0; + if (same_namespace1) // next index differs for permutations and simple combinations + j = (PROCESS_SELF_INTERACTIONS(first_ft_value)) ? i : i+1; + + for (; j < second.indicies.size(); ++j) + { //f3 x k*(f2 x k*f1) + if(audit) audit_func(dat, second.space_names[j].get()); + feature_index halfhash = FNV_prime * (halfhash1 ^ (uint64_t)second.indicies[j]); + feature_value ft_value = INTERACTION_VALUE(first_ft_value, second.values[j]); + + features::features_value_index_audit_range range = third.values_indices_audit(); + features::iterator_all begin = range.begin(); + if (same_namespace2) //next index differs for permutations and simple combinations + begin += (PROCESS_SELF_INTERACTIONS(ft_value)) ? j : j + 1; + + features::iterator_all end = range.end(); + inner_kernel(dat, begin, end, offset, weights, ft_value, halfhash); + } // end for (snd) + if(audit) audit_func(dat, nullptr); + } // end for (fst) + + } // end if (data[thr] size > 0) + } // end if (data[snd] size > 0) + } // end if (data[fst] size > 0) + + } + else // generic case: quatriples, etc. #endif - { - - bool must_skip_interaction = false; - // preparing state data - feature_gen_data* fgd = state_data.begin(); - feature_gen_data* fgd2; // for further use - for (namespace_index n : ns) - { features& ft = features_data[(int32_t)n]; - const size_t ft_cnt = ft.indicies.size(); - - if (ft_cnt == 0) - { must_skip_interaction = true; - break; - } - - if (fgd == state_data.end()) - { state_data.push_back(empty_ns_data); - fgd = state_data.end()-1; // reassign as memory could be realloced - } - - fgd->loop_end = ft_cnt-1; // saving number of features for each namespace - fgd->ft_arr = &ft; - ++fgd; - } - - // if any of interacting namespace has 0 features - whole interaction is skipped - if (must_skip_interaction) continue; //no_data_to_interact - - if (!all.permutations) // adjust state_data for simple combinations - { // if permutations mode is disabeled then namespaces in ns are already sorted and thus grouped - // (in fact, currently they are sorted even for enabled permutations mode) - // let's go throw the list and calculate number of features to skip in namespaces which - // repeated more than once to generate only simple combinations of features - - size_t margin = 0; // number of features to ignore if namespace has been seen before - - // iterate list backward as margin grows in this order - - for (fgd = state_data.end()-1; fgd > state_data.begin(); --fgd) - { fgd2 = fgd-1; - fgd->self_interaction = (fgd->ft_arr == fgd2->ft_arr); //state_data.begin().self_interaction is always false - if (fgd->self_interaction) - { size_t& loop_end = fgd2->loop_end; - - if (!PROCESS_SELF_INTERACTIONS((*fgd2->ft_arr).values[loop_end-margin])) - { ++margin; // otherwise margin can't be increased - if ( (must_skip_interaction = (loop_end < margin)) ) break; - } - - if (margin != 0) - loop_end -= margin; // skip some features and increase margin + { + + bool must_skip_interaction = false; + // preparing state data + feature_gen_data* fgd = state_data.begin(); + feature_gen_data* fgd2; // for further use + for (namespace_index n : ns) + { features& ft = features_data[(int32_t)n]; + const size_t ft_cnt = ft.indicies.size(); + + if (ft_cnt == 0) + { must_skip_interaction = true; + break; + } + + if (fgd == state_data.end()) + { state_data.push_back(empty_ns_data); + fgd = state_data.end()-1; // reassign as memory could be realloced + } + + fgd->loop_end = ft_cnt-1; // saving number of features for each namespace + fgd->ft_arr = &ft; + ++fgd; + } + + // if any of interacting namespace has 0 features - whole interaction is skipped + if (must_skip_interaction) continue; //no_data_to_interact + + if (!all.permutations) // adjust state_data for simple combinations + { // if permutations mode is disabeled then namespaces in ns are already sorted and thus grouped + // (in fact, currently they are sorted even for enabled permutations mode) + // let's go throw the list and calculate number of features to skip in namespaces which + // repeated more than once to generate only simple combinations of features + + size_t margin = 0; // number of features to ignore if namespace has been seen before + + // iterate list backward as margin grows in this order + + for (fgd = state_data.end()-1; fgd > state_data.begin(); --fgd) + { fgd2 = fgd-1; + fgd->self_interaction = (fgd->ft_arr == fgd2->ft_arr); //state_data.begin().self_interaction is always false + if (fgd->self_interaction) + { size_t& loop_end = fgd2->loop_end; + + if (!PROCESS_SELF_INTERACTIONS((*fgd2->ft_arr).values[loop_end-margin])) + { ++margin; // otherwise margin can't be increased + if ( (must_skip_interaction = (loop_end < margin)) ) break; } - else if (margin != 0) margin = 0; + + if (margin != 0) + loop_end -= margin; // skip some features and increase margin + } + else if (margin != 0) margin = 0; + } + + // if impossible_without_permutations == true then we faced with case like interaction 'aaaa' + // where namespace 'a' contains less than 4 unique features. It's impossible to make simple + // combination of length 4 without repetitions from 3 or less elements. + if (must_skip_interaction) continue; // impossible_without_permutations + } // end of state_data adjustment + + + fgd = state_data.begin(); // always equal to first ns + fgd2 = state_data.end()-1; // always equal to last ns + fgd->loop_idx = 0; // loop_idx contains current feature id for curently processed namespace. + + // beware: micro-optimization. + /* start & end are always point to features in last namespace of interaction. + for 'all.permutations == true' they are constant.*/ + size_t start_i = 0; + + feature_gen_data* cur_data = fgd; + // end of micro-optimization block + + // generic feature generation cycle for interactions of any length + bool do_it = true; + while (do_it) + { if (cur_data < fgd2) // can go further threw the list of namespaces in interaction + { feature_gen_data* next_data = cur_data+1; + size_t feature = cur_data->loop_idx; + features& fs = *(cur_data->ft_arr); + + if (next_data->self_interaction) + { // if next namespace is same, we should start with loop_idx + 1 to avoid feature interaction with itself + // unless feature has value x and x != x*x. E.g. x != 0 and x != 1. Features with x == 0 are already + // filtered out in parce_args.cc::maybeFeature(). + + next_data->loop_idx = (PROCESS_SELF_INTERACTIONS(fs.values[feature])) ? cur_data->loop_idx : cur_data->loop_idx + 1; } + else + next_data->loop_idx = 0; - // if impossible_without_permutations == true then we faced with case like interaction 'aaaa' - // where namespace 'a' contains less than 4 unique features. It's impossible to make simple - // combination of length 4 without repetitions from 3 or less elements. - if (must_skip_interaction) continue; // impossible_without_permutations - } // end of state_data adjustment - - - fgd = state_data.begin(); // always equal to first ns - fgd2 = state_data.end()-1; // always equal to last ns - fgd->loop_idx = 0; // loop_idx contains current feature id for curently processed namespace. - - // beware: micro-optimization. - /* start & end are always point to features in last namespace of interaction. - for 'all.permutations == true' they are constant.*/ - size_t start_i = 0; - - feature_gen_data* cur_data = fgd; - // end of micro-optimization block - - // generic feature generation cycle for interactions of any length - bool do_it = true; - while (do_it) - { - if (cur_data < fgd2) // can go further threw the list of namespaces in interaction - { feature_gen_data* next_data = cur_data+1; - size_t feature = cur_data->loop_idx; - features& fs = *(cur_data->ft_arr); - - if (next_data->self_interaction) - { // if next namespace is same, we should start with loop_idx + 1 to avoid feature interaction with itself - // unless feature has value x and x != x*x. E.g. x != 0 and x != 1. Features with x == 0 are already - // filtered out in parce_args.cc::maybeFeature(). - - next_data->loop_idx = (PROCESS_SELF_INTERACTIONS(fs.values[feature])) ? cur_data->loop_idx : cur_data->loop_idx + 1; - } - else - next_data->loop_idx = 0; - - if(audit) audit_func(dat, fs.space_names[feature].get()); - - if (cur_data == fgd) // first namespace - { next_data->hash = FNV_prime * (uint64_t)fs.indicies[feature]; - next_data->x = fs.values[feature]; // data->x == 1. - } - else - { // feature2 xor (16777619*feature1) - next_data->hash = FNV_prime * (cur_data->hash ^ (uint64_t)fs.indicies[feature]); - next_data->x = INTERACTION_VALUE(fs.values[feature], cur_data->x); - } - - ++cur_data; - } + if(audit) audit_func(dat, fs.space_names[feature].get()); + + if (cur_data == fgd) // first namespace + { next_data->hash = FNV_prime * (uint64_t)fs.indicies[feature]; + next_data->x = fs.values[feature]; // data->x == 1. + } else - { - // last namespace - iterate its features and go back - if (!all.permutations) // start value is not a constant in this case - start_i = fgd2->loop_idx; + { // feature2 xor (16777619*feature1) + next_data->hash = FNV_prime * (cur_data->hash ^ (uint64_t)fs.indicies[feature]); + next_data->x = INTERACTION_VALUE(fs.values[feature], cur_data->x); + } - features& fs = *(fgd2->ft_arr); + ++cur_data; + } + else + { // last namespace - iterate its features and go back + if (!all.permutations) // start value is not a constant in this case + start_i = fgd2->loop_idx; - feature_value ft_value = fgd2->x; - feature_index halfhash = fgd2->hash; + features& fs = *(fgd2->ft_arr); - features::features_value_index_audit_range range = fs.values_indices_audit(); - features::iterator_all begin = range.begin(); - begin += start_i; - features::iterator_all end = range.begin(); - end += fgd2->loop_end + 1; - inner_kernel(dat, begin, end, offset, weights, ft_value, halfhash); + feature_value ft_value = fgd2->x; + feature_index halfhash = fgd2->hash; - // trying to go back increasing loop_idx of each namespace by the way + features::features_value_index_audit_range range = fs.values_indices_audit(); + features::iterator_all begin = range.begin(); + begin += start_i; + features::iterator_all end = range.begin(); + end += fgd2->loop_end + 1; + inner_kernel(dat, begin, end, offset, weights, ft_value, halfhash); - bool go_further = true; + // trying to go back increasing loop_idx of each namespace by the way - do - { --cur_data; - go_further = (++cur_data->loop_idx > cur_data->loop_end); //increment loop_idx - if (audit) audit_func(dat, nullptr); - } - while (go_further && cur_data != fgd); + bool go_further = true; - do_it = !(cur_data == fgd && go_further); - //if do_it==false - we've reached 0 namespace but its 'cur_data.loop_idx > cur_data.loop_end' -> exit the while loop - } // if last namespace - } // while do_it - } + do + { --cur_data; + go_further = (++cur_data->loop_idx > cur_data->loop_end); //increment loop_idx + if (audit) audit_func(dat, nullptr); + } + while (go_further && cur_data != fgd); + + do_it = !(cur_data == fgd && go_further); + //if do_it==false - we've reached 0 namespace but its 'cur_data.loop_idx > cur_data.loop_end' -> exit the while loop + } // if last namespace + } // while do_it + } } // foreach interaction in all.interactions state_data.delete_v(); diff --git a/vowpalwabbit/io_buf.cc b/vowpalwabbit/io_buf.cc index ea22ec00670..8a0e876f6d7 100644 --- a/vowpalwabbit/io_buf.cc +++ b/vowpalwabbit/io_buf.cc @@ -89,7 +89,7 @@ void buf_write(io_buf &o, char* &pointer, size_t n) else // Time to dump the file { if (o.head != o.space.begin()) o.flush(); - else // Array is short, so increase size. + else // Array is short, so increase size. { o.space.resize(2*(o.space.end_array - o.space.begin())); o.space.end() = o.space.begin(); o.head = o.space.begin(); diff --git a/vowpalwabbit/io_buf.h b/vowpalwabbit/io_buf.h index b1080a2dc16..d17d0bdd45c 100644 --- a/vowpalwabbit/io_buf.h +++ b/vowpalwabbit/io_buf.h @@ -164,7 +164,7 @@ class io_buf } // read more bytes from file up to the remaining allocated space ssize_t num_read = read_file(f, space.end(), space.end_array - space.end()); - if (num_read >= 0) + if (num_read >= 0) { // if some bytes were actually loaded, update the end of loaded values space.end() = space.end() + num_read; return num_read; @@ -306,8 +306,8 @@ inline size_t bin_text_read_write_fixed(io_buf& io, char* data, size_t len, } inline size_t bin_text_read_write_fixed_validated(io_buf& io, char* data, size_t len, - const char* read_message, bool read, - std::stringstream& msg, bool text) + const char* read_message, bool read, + std::stringstream& msg, bool text) { size_t nbytes = bin_text_read_write_fixed(io, data, len, read_message, read, msg, text); if (read && len > 0) // only validate bytes read/write if expected length > 0 { if (nbytes == 0) diff --git a/vowpalwabbit/kernel_svm.cc b/vowpalwabbit/kernel_svm.cc index 6e893b28b19..f3fe5440da6 100644 --- a/vowpalwabbit/kernel_svm.cc +++ b/vowpalwabbit/kernel_svm.cc @@ -210,18 +210,18 @@ int save_load_flat_example(io_buf& model_file, bool read, flat_example*& fec) if(!brw) return 2; } if(fec->fs.size() > 0) - { features& fs = fec->fs; - size_t len = fs.size(); - fs.values = v_init(); - fs.values.resize(len); - brw = bin_read_fixed(model_file, (char*) fs.values.begin(), len*sizeof(feature_value), ""); if(!brw) return 3; - fs.values.end() = fs.values.begin()+len; - - len = fs.indicies.size(); - fs.indicies = v_init(); - fs.indicies.resize(len); - brw = bin_read_fixed(model_file, (char*) fs.indicies.begin(), len*sizeof(feature_index), ""); if(!brw) return 3; - fs.indicies.end() = fs.indicies.begin()+len; + { features& fs = fec->fs; + size_t len = fs.size(); + fs.values = v_init(); + fs.values.resize(len); + brw = bin_read_fixed(model_file, (char*) fs.values.begin(), len*sizeof(feature_value), ""); if(!brw) return 3; + fs.values.end() = fs.values.begin()+len; + + len = fs.indicies.size(); + fs.indicies = v_init(); + fs.indicies.resize(len); + brw = bin_read_fixed(model_file, (char*) fs.indicies.begin(), len*sizeof(feature_index), ""); if(!brw) return 3; + fs.indicies.end() = fs.indicies.begin()+len; } } else return 1; @@ -238,9 +238,9 @@ int save_load_flat_example(io_buf& model_file, bool read, flat_example*& fec) } } if(fec->fs.size() > 0) - { brw = bin_write_fixed(model_file, (char*) fec->fs.values.begin(), (uint32_t)fec->fs.size()*sizeof(feature_value)); if(!brw) return 3; - brw = bin_write_fixed(model_file, (char*) fec->fs.indicies.begin(), (uint32_t)fec->fs.indicies.size()*sizeof(feature_index)); if(!brw) return 3; - } + { brw = bin_write_fixed(model_file, (char*) fec->fs.values.begin(), (uint32_t)fec->fs.size()*sizeof(feature_value)); if(!brw) return 3; + brw = bin_write_fixed(model_file, (char*) fec->fs.indicies.begin(), (uint32_t)fec->fs.indicies.size()*sizeof(feature_index)); if(!brw) return 3; + } } else return 1; } @@ -297,8 +297,7 @@ void save_load(svm_params& params, io_buf& model_file, bool read, bool text) } float linear_kernel(const flat_example* fec1, const flat_example* fec2) -{ - float dotprod = 0; +{ float dotprod = 0; features& fs_1 = (features&)fec1->fs; features& fs_2 = (features&)fec2->fs; @@ -307,21 +306,21 @@ float linear_kernel(const flat_example* fec1, const flat_example* fec2) int numint = 0; for (size_t idx1 = 0, idx2 = 0; idx1 < fs_1.size() && idx2 < fs_2.size() ; idx1++) - { uint64_t ec1pos = fs_1.indicies[idx1]; - uint64_t ec2pos = fs_2.indicies[idx2]; - //cerr<x<<" "<x< ec2pos && ++idx2 < fs_2.size()) - ec2pos = fs_2.indicies[idx2]; - - if(ec1pos == ec2pos) - { //cerr<x<<" "<x<x<<" "<x< ec2pos && ++idx2 < fs_2.size()) + ec2pos = fs_2.indicies[idx2]; + + if(ec1pos == ec2pos) + { //cerr<x<<" "<x< 0) @@ -686,9 +685,9 @@ void train(svm_params& params) } } else - { size_t rand_pos = (size_t)floorf(frand48() * model->num_support); - update(params, rand_pos); - } + { size_t rand_pos = (size_t)floorf(frand48() * model->num_support); + update(params, rand_pos); + } } //cerr<support_vec[0]->example_counter<training && ec.example_counter % 100 == 0) trim_cache(params); @@ -872,7 +871,7 @@ LEARNER::base_learner* kernel_svm_setup(vw &all) else params.kernel_type = SVM_KER_LIN; - //params.all->weights->mask((uint64_t)LONG_MAX); + //params.all->weights->mask((uint64_t)LONG_MAX); params.all->weights.stride_shift(0); learner& l = init_learner(¶ms, learn, 1); diff --git a/vowpalwabbit/label_dictionary.cc b/vowpalwabbit/label_dictionary.cc index cc76097f25a..ed1c8d08150 100644 --- a/vowpalwabbit/label_dictionary.cc +++ b/vowpalwabbit/label_dictionary.cc @@ -6,42 +6,40 @@ namespace LabelDict { size_t hash_lab(size_t lab) { return 328051 + 94389193 * lab; } - void del_example_namespace(example& ec, char ns, features& fs) - { - // print_update is called after this del_example_namespace, - // so we need to keep the ec.num_features correct, - // so shared features are included in the reported number of "current features" - //ec.num_features -= numf; - features& del_target = ec.feature_space[(size_t)ns]; - assert(del_target.size() >= fs.size()); - assert(ec.indices.size() > 0); - if (ec.indices.last() == ns && ec.feature_space[(size_t)ns].size() == fs.size()) - ec.indices.pop(); - ec.total_sum_feat_sq -= fs.sum_feat_sq; - //ec.num_features -= fs.size(); - del_target.truncate_to(del_target.size() - fs.size()); - del_target.sum_feat_sq -= fs.sum_feat_sq; - } +void del_example_namespace(example& ec, char ns, features& fs) +{ // print_update is called after this del_example_namespace, + // so we need to keep the ec.num_features correct, + // so shared features are included in the reported number of "current features" + //ec.num_features -= numf; + features& del_target = ec.feature_space[(size_t)ns]; + assert(del_target.size() >= fs.size()); + assert(ec.indices.size() > 0); + if (ec.indices.last() == ns && ec.feature_space[(size_t)ns].size() == fs.size()) + ec.indices.pop(); + ec.total_sum_feat_sq -= fs.sum_feat_sq; + //ec.num_features -= fs.size(); + del_target.truncate_to(del_target.size() - fs.size()); + del_target.sum_feat_sq -= fs.sum_feat_sq; +} void add_example_namespace(example& ec, char ns, features& fs) { bool has_ns = false; for (size_t i=0; i 0; features& add_fs = ec.feature_space[(size_t)ns]; - for (size_t i = 0; i < fs.size(); ++i) - { - add_fs.push_back(fs.values[i], fs.indicies[i]); - if (audit) - add_fs.space_names.push_back(fs.space_names[i]); - } + for (size_t i = 0; i < fs.size(); ++i) + { add_fs.push_back(fs.values[i], fs.indicies[i]); + if (audit) + add_fs.space_names.push_back(fs.space_names[i]); + } ec.total_sum_feat_sq += fs.sum_feat_sq; ec.num_features += fs.size(); @@ -60,9 +58,9 @@ void del_example_namespaces_from_example(example& target, example& source) namespace_index* idx = source.indices.end(); idx--; for (; idx>=source.indices.begin(); idx--) - { if (*idx == constant_namespace) continue; - del_example_namespace(target, (char)*idx, source.feature_space[*idx]); - } + { if (*idx == constant_namespace) continue; + del_example_namespace(target, (char)*idx, source.feature_space[*idx]); + } } void add_example_namespace_from_memory(label_feature_map& lfm, example& ec, size_t lab) diff --git a/vowpalwabbit/lda_core.cc b/vowpalwabbit/lda_core.cc index 413983d2363..95a6be1101a 100644 --- a/vowpalwabbit/lda_core.cc +++ b/vowpalwabbit/lda_core.cc @@ -209,23 +209,19 @@ inline const v4si v4sil(const uint32_t x) { return _mm_set1_epi32(x); } #ifdef WIN32 inline __m128 operator+(const __m128 a, const __m128 b) -{ - return _mm_add_ps(a, b); +{ return _mm_add_ps(a, b); } inline __m128 operator-(const __m128 a, const __m128 b) -{ - return _mm_sub_ps(a, b); +{ return _mm_sub_ps(a, b); } inline __m128 operator*(const __m128 a, const __m128 b) -{ - return _mm_mul_ps(a, b); +{ return _mm_mul_ps(a, b); } inline __m128 operator/(const __m128 a, const __m128 b) -{ - return _mm_div_ps(a, b); +{ return _mm_div_ps(a, b); } #endif @@ -621,18 +617,18 @@ float lda_loop(lda &l, v_array &Elogtheta, float *v, weight_parameters& w size_t word_count = 0; doc_length = 0; for (features& fs : *ec) - { for (features::iterator& f : fs) - { weight_parameters::iterator u_for_w = weights.change_begin() + (f.index() & weights.mask()) + l.topics + 1; - float c_w = find_cw(l, u_for_w, v); - xc_w = c_w * f.value(); - score += -f.value() * log(c_w); - size_t max_k = l.topics; - for (size_t k = 0; k < max_k; k++, ++u_for_w) - new_gamma[k] += xc_w * *u_for_w; - word_count++; - doc_length += f.value(); - } + { for (features::iterator& f : fs) + { weight_parameters::iterator u_for_w = weights.change_begin() + (f.index() & weights.mask()) + l.topics + 1; + float c_w = find_cw(l, u_for_w, v); + xc_w = c_w * f.value(); + score += -f.value() * log(c_w); + size_t max_k = l.topics; + for (size_t k = 0; k < max_k; k++, ++u_for_w) + new_gamma[k] += xc_w * *u_for_w; + word_count++; + doc_length += f.value(); } + } for (size_t k = 0; k < l.topics; k++) new_gamma[k] = new_gamma[k] * v[k] + l.lda_alpha; } @@ -661,23 +657,23 @@ size_t next_pow2(size_t x) struct initial_weights { private: - weight _initial; - weight _initial_random; - bool _random; - uint32_t _lda; + weight _initial; + weight _initial_random; + bool _random; + uint32_t _lda; public: - initial_weights(weight initial, weight initial_random, bool random, uint32_t lda ) - : _initial(initial), _initial_random(initial_random), _random(random), _lda(lda) - {} - void operator()(weight_parameters::iterator& iter, uint64_t index) - { if (_random) - { for (weights_iterator_iterator k = iter.begin(); k != iter.end(_lda); ++k, ++index) - { *k = (float)(-log(merand48(index) + 1e-6) + 1.0f); - *k *= _initial_random; - } - } - (&(*iter))[_lda] = _initial; - } + initial_weights(weight initial, weight initial_random, bool random, uint32_t lda ) + : _initial(initial), _initial_random(initial_random), _random(random), _lda(lda) + {} + void operator()(weight_parameters::iterator& iter, uint64_t index) + { if (_random) + { for (weights_iterator_iterator k = iter.begin(); k != iter.end(_lda); ++k, ++index) + { *k = (float)(-log(merand48(index) + 1e-6) + 1.0f); + *k *= _initial_random; + } + } + (&(*iter))[_lda] = _initial; + } }; @@ -694,59 +690,56 @@ void save_load(lda &l, io_buf &model_file, bool read, bool text) { uint64_t i = 0; stringstream msg; size_t brw = 1; - weight_parameters& weights = all->weights; - do + weight_parameters& weights = all->weights; + do { brw = 0; - if (!read && text) - msg << i << " "; - - if (!read || l.all->model_file_ver >= VERSION_FILE_WITH_HEADER_ID) - brw += bin_text_read_write_fixed(model_file, (char *)&i, sizeof(i), "", read, msg, text); - else - { - // support 32bit build models - uint32_t j; - brw += bin_text_read_write_fixed(model_file, (char *)&j, sizeof(j), "", read, msg, text); - i = j; - } - - if (brw != 0) - { weight_parameters::iterator iter = weights.begin() + i; - for (weights_iterator_iterator v = iter.begin(); v != iter.end(all->lda); ++v) - { if (!read && text) - msg << *v + l.lda_rho << " "; - brw += bin_text_read_write_fixed(model_file, (char *)&(*v), sizeof(*v), "", read, msg, text); - } - } - if (text) - { - if (!read) - msg << "\n"; - brw += bin_text_read_write_fixed(model_file, nullptr, 0, "", read, msg, text); + if (!read && text) + msg << i << " "; + + if (!read || l.all->model_file_ver >= VERSION_FILE_WITH_HEADER_ID) + brw += bin_text_read_write_fixed(model_file, (char *)&i, sizeof(i), "", read, msg, text); + else + { // support 32bit build models + uint32_t j; + brw += bin_text_read_write_fixed(model_file, (char *)&j, sizeof(j), "", read, msg, text); + i = j; + } + + if (brw != 0) + { weight_parameters::iterator iter = weights.begin() + i; + for (weights_iterator_iterator v = iter.begin(); v != iter.end(all->lda); ++v) + { if (!read && text) + msg << *v + l.lda_rho << " "; + brw += bin_text_read_write_fixed(model_file, (char *)&(*v), sizeof(*v), "", read, msg, text); } - if (!read) - ++i; + } + if (text) + { if (!read) + msg << "\n"; + brw += bin_text_read_write_fixed(model_file, nullptr, 0, "", read, msg, text); + } + if (!read) + ++i; } while ((!read && i < length) || (read && brw > 0)); } } void return_example(vw& all, example& ec) -{ - label_data ld = ec.l.simple; - +{ label_data ld = ec.l.simple; + all.sd->update(ec.test_only, ec.loss, ec.weight, ec.num_features); if (ld.label != FLT_MAX && !ec.test_only) all.sd->weighted_labels += ld.label * ec.weight; all.sd->weighted_unlabeled_examples += ld.label == FLT_MAX ? ec.weight : 0; - + for (int f: all.final_prediction_sink) MWT::print_scalars(f, ec.pred.scalars, ec.tag); - + if (all.sd->weighted_examples >= all.sd->dump_interval && !all.quiet) all.sd->print_update(all.holdout_set_off, all.current_pass, ec.l.simple.label, 0.f, - ec.num_features, all.progress_add, all.progress_arg); + ec.num_features, all.progress_add, all.progress_arg); VW::finish_example(all,&ec); } @@ -758,15 +751,14 @@ void learn_batch(lda &l) // the for loops down there. Since it seems that there's not much to // do in this case, we just return. for (size_t d = 0; d < l.examples.size(); d++) - { - l.examples[d]->pred.scalars.erase(); - l.examples[d]->pred.scalars.resize(l.topics); - memset(l.examples[d]->pred.scalars.begin(), 0, l.topics * sizeof(float)); - l.examples[d]->pred.scalars.end() = l.examples[d]->pred.scalars.begin() + l.topics; - - l.examples[d]->pred.scalars.erase(); - return_example(*l.all, *l.examples[d]); - } + { l.examples[d]->pred.scalars.erase(); + l.examples[d]->pred.scalars.resize(l.topics); + memset(l.examples[d]->pred.scalars.begin(), 0, l.topics * sizeof(float)); + l.examples[d]->pred.scalars.end() = l.examples[d]->pred.scalars.begin() + l.topics; + + l.examples[d]->pred.scalars.erase(); + return_example(*l.all, *l.examples[d]); + } l.examples.erase(); return; } @@ -778,14 +770,14 @@ void learn_batch(lda &l) { for (size_t k = 0; k < l.all->lda; k++) l.total_lambda.push_back(0.f); - weight_parameters& weights = l.all->weights; - size_t stride = 1 << weights.stride_shift(); - weight_parameters::iterator iter = weights.begin(); - for (size_t i = 0; i <= weights.mask(); i += stride, ++iter) - { weights_iterator_iterator k_iter = iter.begin(); - for (size_t k = 0; k < l.all->lda; k++, ++k_iter) - l.total_lambda[k] += *k_iter; - } + weight_parameters& weights = l.all->weights; + size_t stride = 1 << weights.stride_shift(); + weight_parameters::iterator iter = weights.begin(); + for (size_t i = 0; i <= weights.mask(); i += stride, ++iter) + { weights_iterator_iterator k_iter = iter.begin(); + for (size_t k = 0; k < l.all->lda; k++, ++k_iter) + l.total_lambda[k] += *k_iter; + } } l.example_t++; @@ -816,18 +808,18 @@ void learn_batch(lda &l) continue; last_weight_index = s->f.weight_index; //float *weights_for_w = &(weights[s->f.weight_index]); - weight_parameters::iterator weights_for_w = weights.change_begin() + (s->f.weight_index & weights.mask()); + weight_parameters::iterator weights_for_w = weights.change_begin() + (s->f.weight_index & weights.mask()); float decay_component = l.decay_levels.end()[-2] - l.decay_levels.end()[(int)(-1 - l.example_t + *(weights_for_w + l.all->lda))]; float decay = fmin(1.0f, correctedExp(decay_component)); - weight_parameters::iterator u_for_w = weights_for_w + l.all->lda + 1; + weight_parameters::iterator u_for_w = weights_for_w + l.all->lda + 1; *(weights_for_w + l.all->lda) = (float)l.example_t; for (size_t k = 0; k < l.all->lda; k++, ++weights_for_w, ++u_for_w) { *weights_for_w *= decay; *u_for_w = *weights_for_w + l.lda_rho; } - u_for_w = weights.change_begin() + (s->f.weight_index & weights.mask()) +l.all->lda + 1; + u_for_w = weights.change_begin() + (s->f.weight_index & weights.mask()) +l.all->lda + 1; l.expdigammify_2(*l.all, u_for_w, l.digammas.begin()); } @@ -845,40 +837,35 @@ void learn_batch(lda &l) // -t there's no need to update weights (especially since it's a noop) if (eta != 0) - { - for (index_feature *s = &l.sorted_features[0]; s <= &l.sorted_features.back();) - { - index_feature *next = s + 1; - while (next <= &l.sorted_features.back() && next->f.weight_index == s->f.weight_index) - next++; - - //float *word_weights = &(weights[s->f.weight_index]); - weight_parameters::iterator word_weights = weights.change_begin() + (s->f.weight_index & weights.mask()); - for (size_t k = 0; k < l.all->lda; k++, ++word_weights) - { - float new_value = minuseta * *word_weights; - *word_weights = new_value; - } - - for (; s != next; s++) - { - float *v_s = &(l.v[s->document * l.all->lda]); - weight_parameters::iterator u_for_w = weights.change_begin() + (s->f.weight_index & weights.mask()) + l.all->lda + 1; - float c_w = eta * find_cw(l, u_for_w, v_s) * s->f.x; - word_weights = weights.change_begin() + (s->f.weight_index & weights.mask()); - for (size_t k = 0; k < l.all->lda; k++, ++u_for_w, ++word_weights) - { - float new_value = *u_for_w * v_s[k] * c_w; - l.total_new[k] += new_value; - *word_weights += new_value; - } - } - } - - for (size_t k = 0; k < l.all->lda; k++) - { l.total_lambda[k] *= minuseta; - l.total_lambda[k] += l.total_new[k]; - } + { for (index_feature *s = &l.sorted_features[0]; s <= &l.sorted_features.back();) + { index_feature *next = s + 1; + while (next <= &l.sorted_features.back() && next->f.weight_index == s->f.weight_index) + next++; + + //float *word_weights = &(weights[s->f.weight_index]); + weight_parameters::iterator word_weights = weights.change_begin() + (s->f.weight_index & weights.mask()); + for (size_t k = 0; k < l.all->lda; k++, ++word_weights) + { float new_value = minuseta * *word_weights; + *word_weights = new_value; + } + + for (; s != next; s++) + { float *v_s = &(l.v[s->document * l.all->lda]); + weight_parameters::iterator u_for_w = weights.change_begin() + (s->f.weight_index & weights.mask()) + l.all->lda + 1; + float c_w = eta * find_cw(l, u_for_w, v_s) * s->f.x; + word_weights = weights.change_begin() + (s->f.weight_index & weights.mask()); + for (size_t k = 0; k < l.all->lda; k++, ++u_for_w, ++word_weights) + { float new_value = *u_for_w * v_s[k] * c_w; + l.total_new[k] += new_value; + *word_weights += new_value; + } + } + } + + for (size_t k = 0; k < l.all->lda; k++) + { l.total_lambda[k] *= minuseta; + l.total_lambda[k] += l.total_new[k]; + } } l.sorted_features.resize(0); @@ -887,8 +874,7 @@ void learn_batch(lda &l) } void learn(lda &l, LEARNER::base_learner &, example &ec) -{ - uint32_t num_ex = (uint32_t)l.examples.size(); +{ uint32_t num_ex = (uint32_t)l.examples.size(); l.examples.push_back(&ec); l.doc_lengths.push_back(0); for (features& fs : ec) @@ -903,12 +889,11 @@ void learn(lda &l, LEARNER::base_learner &, example &ec) } void learn_with_metrics(lda &l, LEARNER::base_learner &base, example &ec) -{ - if (l.all->passes_complete == 0) +{ if (l.all->passes_complete == 0) { // build feature to example map - auto weight_mask = l.all->weights.mask(); + auto weight_mask = l.all->weights.mask(); auto stride_shift = l.all->weights.stride_shift(); - + for (features& fs : ec) { for (features::iterator& f : fs) { uint64_t idx = (f.index() & weight_mask) >> stride_shift; @@ -926,244 +911,222 @@ void predict(lda &l, LEARNER::base_learner &base, example &ec) { learn(l, base, void predict_with_metrics(lda &l, LEARNER::base_learner &base, example &ec) { learn_with_metrics(l, base, ec); } struct word_doc_frequency -{ - // feature/word index - uint64_t idx; - // document count - uint32_t count; +{ // feature/word index + uint64_t idx; + // document count + uint32_t count; }; // cooccurence of 2 features/words struct feature_pair -{ - // feature/word 1 - uint64_t f1; - // feature/word 2 - uint64_t f2; +{ // feature/word 1 + uint64_t f1; + // feature/word 2 + uint64_t f2; - feature_pair(uint64_t _f1, uint64_t _f2) : f1(_f1), f2(_f2) - {} + feature_pair(uint64_t _f1, uint64_t _f2) : f1(_f1), f2(_f2) + {} }; void get_top_weights(vw* all, int top_words_count, int topic, std::vector& output) -{ - weight_parameters& weights = all->weights; - uint64_t length = (uint64_t)1 << all->num_bits; - - // get top features for this topic - auto cmp = [](feature left, feature right) { return left.x > right.x; }; - std::priority_queue, decltype(cmp)> top_features(cmp); - weight_parameters::iterator iter = weights.begin(); - for (uint64_t i = 0; i < min(top_words_count, length); i++, ++iter) - top_features.push({(&(*iter))[topic], i}); - - for (uint64_t i = top_words_count; i < length; i++, ++iter) - { - weight v = (&(*iter))[topic]; - if (v > top_features.top().x) - { top_features.pop(); - top_features.push({v, i}); - } - } - - // extract idx and sort descending - output.resize(top_features.size()); - for (int i = (int)top_features.size() - 1; i >= 0; i--) - { - output[i] = top_features.top(); - top_features.pop(); - } +{ weight_parameters& weights = all->weights; + uint64_t length = (uint64_t)1 << all->num_bits; + + // get top features for this topic + auto cmp = [](feature left, feature right) { return left.x > right.x; }; + std::priority_queue, decltype(cmp)> top_features(cmp); + weight_parameters::iterator iter = weights.begin(); + for (uint64_t i = 0; i < min(top_words_count, length); i++, ++iter) + top_features.push({(&(*iter))[topic], i}); + + for (uint64_t i = top_words_count; i < length; i++, ++iter) + { weight v = (&(*iter))[topic]; + if (v > top_features.top().x) + { top_features.pop(); + top_features.push({v, i}); + } + } + + // extract idx and sort descending + output.resize(top_features.size()); + for (int i = (int)top_features.size() - 1; i >= 0; i--) + { output[i] = top_features.top(); + top_features.pop(); + } } void compute_coherence_metrics(lda &l) -{ - weight_parameters& weights = l.all->weights; - uint64_t length = (uint64_t)1 << l.all->num_bits; - - std::vector> topics_word_pairs; - topics_word_pairs.resize(l.topics); - - int top_words_count = 10; // parameterize and check - - // TODO: remove or make output file available through parameters - /* - FILE* vocab = fopen("C:\\Data\\MinMaxWordFreq_1200_0.3\\vocab.tsv", "w"); - for (int f = 0; f < length;f++) - { - fprintf(vocab, "%d\t%d\t%d\n", - f, - l.feature_counts[f], - l.feature_to_example_map[f].size()); - } - fclose(vocab); - - FILE* docAlloc = fopen("C:\\Data\\MinMaxWordFreq_1200_0.3\\VW-DocumentTopicAllocations.txt", "w"); - // using jagged array to enable LINQ - auto K = l.all->lda; - - uint64_t stride_shift = l.all->reg.stride_shift; - for (uint64_t i = 0; i < length; i++) - { - auto offset = i << stride_shift; - - // over topics - for (uint64_t k = 0; k < K; k++) - { - weight *v = &(l.all->reg.weight_vector[offset + k]); - fprintf(docAlloc, "%f ", *v + l.lda_rho); - } - fprintf(docAlloc, "\n"); - } - fclose(docAlloc); - */ - - for (size_t topic = 0; topic < l.topics;topic++) - { - // get top features for this topic - auto cmp = [](feature& left, feature& right) { return left.x > right.x; }; - std::priority_queue, decltype(cmp)> top_features(cmp); - weight_parameters::iterator iter = weights.begin(); - for (uint64_t i = 0; i < min(top_words_count, length); i++, ++iter) - top_features.push(feature((&(*iter))[topic], i)); - - iter = weights.begin() + top_words_count; - for (uint64_t i = top_words_count; i < length; i++, ++iter) - {if ((&(*iter))[topic] > top_features.top().x) - { top_features.pop(); - top_features.push(feature((&(*iter))[topic], i)); - } - } - - // extract idx and sort descending - vector top_features_idx; - top_features_idx.resize(top_features.size()); - for (int i = (int)top_features.size() - 1; i >= 0; i--) - { - top_features_idx[i] = top_features.top().weight_index; - top_features.pop(); - } - - auto& word_pairs = topics_word_pairs[topic]; - for (size_t i = 0; i < top_features_idx.size(); i++) - for (size_t j = i + 1; j < top_features_idx.size(); j++) - word_pairs.push_back(feature_pair(top_features_idx[i], top_features_idx[j])); - } - - // compress word pairs and create record for storing frequency - std::map> coWordsDFSet; - for (auto& vec : topics_word_pairs) - { - for (auto& wp : vec) - { - auto f1 = wp.f1; - auto f2 = wp.f2; - auto wdf = coWordsDFSet.find(f1); - - if (wdf != coWordsDFSet.end()) - { - // http://stackoverflow.com/questions/5377434/does-stdmapiterator-return-a-copy-of-value-or-a-value-itself - //if (wdf->second.find(f2) == wdf->second.end()) - - if (std::find_if(wdf->second.begin(), wdf->second.end(), [&f2](const word_doc_frequency& v) { return v.idx == f2; }) != wdf->second.end()) - { - wdf->second.push_back({ f2, 0 }); - //printf(" add %d %d\n", f1, f2); - } - } - else - { - std::vector vec = { { f2, 0 } }; - coWordsDFSet.insert(std::make_pair(f1, vec)); - //printf(" insert %d %d\n", f1, f2); - } - } - } - - // this.GetWordPairsDocumentFrequency(coWordsDFSet); - for (auto& pair : coWordsDFSet) - { - auto& examples_for_f1 = l.feature_to_example_map[pair.first]; - for (auto& wdf : pair.second) - { - auto& examples_for_f2 = l.feature_to_example_map[wdf.idx]; - - // assumes examples_for_f1 and examples_for_f2 are orderd - size_t i = 0; - size_t j = 0; - while (i < examples_for_f1.size() && j < examples_for_f2.size()) - { - if (examples_for_f1[i] == examples_for_f2[j]) - { - wdf.count++; - i++; - j++; - } - else if (examples_for_f2[j] < examples_for_f1[i]) - j++; - else - i++; - } - } - } - - float epsilon = 1e-6f; // TODO - float avg_coherence = 0; - for (size_t topic = 0; topic < l.topics;topic++) - { - float coherence = 0; - - for (auto& pairs : topics_word_pairs[topic]) - { - auto f1 = pairs.f1; - if (l.feature_counts[f1] == 0) - continue; - - auto f2 = pairs.f2; - auto& co_feature = coWordsDFSet[f1]; - auto co_feature_df = std::find_if(co_feature.begin(), co_feature.end(), [&f2](const word_doc_frequency& v) { return v.idx == f2; }); - - if (co_feature_df != co_feature.end()) - { - // printf("(%d:%d + eps)/(%d:%d)\n", f2, co_feature_df->count, f1, l.feature_counts[f1]); - coherence += logf((co_feature_df->count + epsilon) / l.feature_counts[f1]); - } - } - - printf("Topic %3d coherence: %f\n", (int)topic, coherence); - - // TODO: expose per topic coherence - - // TODO: good vs. bad topics - avg_coherence += coherence; - } - - avg_coherence /= l.topics; - - printf("Avg topic coherence: %f\n", avg_coherence); +{ weight_parameters& weights = l.all->weights; + uint64_t length = (uint64_t)1 << l.all->num_bits; + + std::vector> topics_word_pairs; + topics_word_pairs.resize(l.topics); + + int top_words_count = 10; // parameterize and check + + // TODO: remove or make output file available through parameters + /* + FILE* vocab = fopen("C:\\Data\\MinMaxWordFreq_1200_0.3\\vocab.tsv", "w"); + for (int f = 0; f < length;f++) + { + fprintf(vocab, "%d\t%d\t%d\n", + f, + l.feature_counts[f], + l.feature_to_example_map[f].size()); + } + fclose(vocab); + + FILE* docAlloc = fopen("C:\\Data\\MinMaxWordFreq_1200_0.3\\VW-DocumentTopicAllocations.txt", "w"); + // using jagged array to enable LINQ + auto K = l.all->lda; + + uint64_t stride_shift = l.all->reg.stride_shift; + for (uint64_t i = 0; i < length; i++) + { + auto offset = i << stride_shift; + + // over topics + for (uint64_t k = 0; k < K; k++) + { + weight *v = &(l.all->reg.weight_vector[offset + k]); + fprintf(docAlloc, "%f ", *v + l.lda_rho); + } + fprintf(docAlloc, "\n"); + } + fclose(docAlloc); + */ + + for (size_t topic = 0; topic < l.topics; topic++) + { // get top features for this topic + auto cmp = [](feature& left, feature& right) { return left.x > right.x; }; + std::priority_queue, decltype(cmp)> top_features(cmp); + weight_parameters::iterator iter = weights.begin(); + for (uint64_t i = 0; i < min(top_words_count, length); i++, ++iter) + top_features.push(feature((&(*iter))[topic], i)); + + iter = weights.begin() + top_words_count; + for (uint64_t i = top_words_count; i < length; i++, ++iter) + { if ((&(*iter))[topic] > top_features.top().x) + { top_features.pop(); + top_features.push(feature((&(*iter))[topic], i)); + } + } + + // extract idx and sort descending + vector top_features_idx; + top_features_idx.resize(top_features.size()); + for (int i = (int)top_features.size() - 1; i >= 0; i--) + { top_features_idx[i] = top_features.top().weight_index; + top_features.pop(); + } + + auto& word_pairs = topics_word_pairs[topic]; + for (size_t i = 0; i < top_features_idx.size(); i++) + for (size_t j = i + 1; j < top_features_idx.size(); j++) + word_pairs.push_back(feature_pair(top_features_idx[i], top_features_idx[j])); + } + + // compress word pairs and create record for storing frequency + std::map> coWordsDFSet; + for (auto& vec : topics_word_pairs) + { for (auto& wp : vec) + { auto f1 = wp.f1; + auto f2 = wp.f2; + auto wdf = coWordsDFSet.find(f1); + + if (wdf != coWordsDFSet.end()) + { // http://stackoverflow.com/questions/5377434/does-stdmapiterator-return-a-copy-of-value-or-a-value-itself + //if (wdf->second.find(f2) == wdf->second.end()) + + if (std::find_if(wdf->second.begin(), wdf->second.end(), [&f2](const word_doc_frequency& v) { return v.idx == f2; }) != wdf->second.end()) + { wdf->second.push_back({ f2, 0 }); + //printf(" add %d %d\n", f1, f2); + } + } + else + { std::vector vec = { { f2, 0 } }; + coWordsDFSet.insert(std::make_pair(f1, vec)); + //printf(" insert %d %d\n", f1, f2); + } + } + } + + // this.GetWordPairsDocumentFrequency(coWordsDFSet); + for (auto& pair : coWordsDFSet) + { auto& examples_for_f1 = l.feature_to_example_map[pair.first]; + for (auto& wdf : pair.second) + { auto& examples_for_f2 = l.feature_to_example_map[wdf.idx]; + + // assumes examples_for_f1 and examples_for_f2 are orderd + size_t i = 0; + size_t j = 0; + while (i < examples_for_f1.size() && j < examples_for_f2.size()) + { if (examples_for_f1[i] == examples_for_f2[j]) + { wdf.count++; + i++; + j++; + } + else if (examples_for_f2[j] < examples_for_f1[i]) + j++; + else + i++; + } + } + } + + float epsilon = 1e-6f; // TODO + float avg_coherence = 0; + for (size_t topic = 0; topic < l.topics; topic++) + { float coherence = 0; + + for (auto& pairs : topics_word_pairs[topic]) + { auto f1 = pairs.f1; + if (l.feature_counts[f1] == 0) + continue; + + auto f2 = pairs.f2; + auto& co_feature = coWordsDFSet[f1]; + auto co_feature_df = std::find_if(co_feature.begin(), co_feature.end(), [&f2](const word_doc_frequency& v) { return v.idx == f2; }); + + if (co_feature_df != co_feature.end()) + { // printf("(%d:%d + eps)/(%d:%d)\n", f2, co_feature_df->count, f1, l.feature_counts[f1]); + coherence += logf((co_feature_df->count + epsilon) / l.feature_counts[f1]); + } + } + + printf("Topic %3d coherence: %f\n", (int)topic, coherence); + + // TODO: expose per topic coherence + + // TODO: good vs. bad topics + avg_coherence += coherence; + } + + avg_coherence /= l.topics; + + printf("Avg topic coherence: %f\n", avg_coherence); } void end_pass(lda &l) -{ - if (l.examples.size()) - learn_batch(l); - - if (l.compute_coherence_metrics && l.all->passes_complete == l.all->numpasses) - { - compute_coherence_metrics(l); - // FASTPASS return; - } +{ if (l.examples.size()) + learn_batch(l); + + if (l.compute_coherence_metrics && l.all->passes_complete == l.all->numpasses) + { compute_coherence_metrics(l); + // FASTPASS return; + } } void end_examples(lda &l) -{ weight_parameters& weights = l.all->weights; - for (weight_parameters::iterator iter = weights.begin(); iter != weights.end(); ++iter) - { float decay_component = +{ weight_parameters& weights = l.all->weights; + for (weight_parameters::iterator iter = weights.begin(); iter != weights.end(); ++iter) + { float decay_component = l.decay_levels.last() - l.decay_levels.end()[(int)(-1 - l.example_t + (&(*iter))[l.all->lda])]; float decay = fmin(1.f, correctedExp(decay_component)); - for (weights_iterator_iterator k = iter.begin(); k != iter.end(l.all->lda); ++k) + for (weights_iterator_iterator k = iter.begin(); k != iter.end(l.all->lda); ++k) *k *= decay; - } + } } void finish_example(vw&, lda&, example &) {} @@ -1200,13 +1163,13 @@ LEARNER::base_learner *lda_setup(vw &all) { if (missing_option(all, "lda", "Run lda with topics")) return nullptr; new_options(all, "Lda options") - ("lda_alpha", po::value()->default_value(0.1f),"Prior on sparsity of per-document topic weights") - ("lda_rho", po::value()->default_value(0.1f), "Prior on sparsity of topic distributions") - ("lda_D", po::value()->default_value(10000.), "Number of documents") - ("lda_epsilon", po::value()->default_value(0.001f), "Loop convergence threshold") - ("minibatch", po::value()->default_value(1), "Minibatch size, for LDA") - ("math-mode", po::value()->default_value(USE_SIMD), "Math mode: simd, accuracy, fast-approx") - ("metrics", po::value()->default_value(false), "Compute metrics"); + ("lda_alpha", po::value()->default_value(0.1f),"Prior on sparsity of per-document topic weights") + ("lda_rho", po::value()->default_value(0.1f), "Prior on sparsity of topic distributions") + ("lda_D", po::value()->default_value(10000.), "Number of documents") + ("lda_epsilon", po::value()->default_value(0.001f), "Loop convergence threshold") + ("minibatch", po::value()->default_value(1), "Minibatch size, for LDA") + ("math-mode", po::value()->default_value(USE_SIMD), "Math mode: simd, accuracy, fast-approx") + ("metrics", po::value()->default_value(false), "Compute metrics"); add_options(all); po::variables_map &vm = all.vm; @@ -1227,9 +1190,9 @@ LEARNER::base_learner *lda_setup(vw &all) ld.example_t = all.initial_t; ld.mmode = vm["math-mode"].as(); ld.compute_coherence_metrics = vm["metrics"].as(); - if (ld.compute_coherence_metrics) { - ld.feature_counts.resize((uint32_t)1 << all.num_bits); - ld.feature_to_example_map.resize((uint32_t)1 << all.num_bits); + if (ld.compute_coherence_metrics) + { ld.feature_counts.resize((uint32_t)1 << all.num_bits); + ld.feature_to_example_map.resize((uint32_t)1 << all.num_bits); } float temp = ceilf(logf((float)(all.lda * 2 + 1)) / logf(2.f)); diff --git a/vowpalwabbit/learner.cc b/vowpalwabbit/learner.cc index 7e81a9c3e79..26e098408b7 100644 --- a/vowpalwabbit/learner.cc +++ b/vowpalwabbit/learner.cc @@ -13,30 +13,27 @@ void dispatch_example(vw& all, example& ec) namespace prediction_type { -#define CASE(type) case type: return #type; +#define CASE(type) case type: return #type; - const char* to_string(prediction_type_t prediction_type) - { - switch (prediction_type) - { - CASE(scalar) - CASE(scalars) - CASE(action_scores) - CASE(action_probs) - CASE(multiclass) - CASE(multilabels) - CASE(prob) - CASE(multiclassprobs) - default: return ""; - } - } +const char* to_string(prediction_type_t prediction_type) +{ switch (prediction_type) + { CASE(scalar) + CASE(scalars) + CASE(action_scores) + CASE(action_probs) + CASE(multiclass) + CASE(multilabels) + CASE(prob) + CASE(multiclassprobs) + default: return ""; + } +} } namespace LEARNER { void process_example(vw& all, example* ec) -{ - if (ec->indices.size() > 1) // 1+ nonconstant feature. (most common case first) +{ if (ec->indices.size() > 1) // 1+ nonconstant feature. (most common case first) dispatch_example(all, *ec); else if (ec->end_pass) { all.l->end_pass(); @@ -66,10 +63,10 @@ template void generic_driver(vw& all, T context while ( all.early_terminate == false ) if ((ec = VW::get_example(all.p)) != nullptr) f(context, ec); - else + else break; if (all.early_terminate) //drain any extra examples from parser. - while ((ec = VW::get_example(all.p)) != nullptr) + while ((ec = VW::get_example(all.p)) != nullptr) VW::finish_example(all, ec); all.l->end_examples(); } diff --git a/vowpalwabbit/learner.h b/vowpalwabbit/learner.h index ef192252f59..e1654cb64fc 100644 --- a/vowpalwabbit/learner.h +++ b/vowpalwabbit/learner.h @@ -15,19 +15,18 @@ license as described in the file LICENSE. namespace prediction_type { - enum prediction_type_t - { - scalar, - scalars, - action_scores, - action_probs, - multiclass, - multilabels, - prob, - multiclassprobs - }; - - const char* to_string(prediction_type_t prediction_type); +enum prediction_type_t +{ scalar, + scalars, + action_scores, + action_probs, + multiclass, + multilabels, + prob, + multiclassprobs +}; + +const char* to_string(prediction_type_t prediction_type); } namespace LEARNER @@ -92,7 +91,7 @@ typedef void (*tend_example)(vw& all, void* d, example& ec); template learner& init_learner(T*, void (*)(T&, base_learner&, example&), size_t, prediction_type::prediction_type_t pred = prediction_type::scalar); template learner& init_learner(T*, base_learner*, void(*learn)(T&, base_learner&, example&), - void(*predict)(T&, base_learner&, example&), size_t ws = 1); + void(*predict)(T&, base_learner&, example&), size_t ws = 1); template learner& init_learner(T*, base_learner*, void (*learn)(T&, base_learner&, example&), void (*predict)(T&, base_learner&, example&), size_t ws, prediction_type::prediction_type_t); @@ -184,7 +183,7 @@ struct learner { finisher_fd = tuple_dbf(learn_fd.data,learn_fd.base, (tfunc)f); } inline void finish() { if (finisher_fd.data) - {finisher_fd.func(finisher_fd.data); free(finisher_fd.data); } + {finisher_fd.func(finisher_fd.data); free(finisher_fd.data); } if (finisher_fd.base) { finisher_fd.base->finish(); free(finisher_fd.base); @@ -218,9 +217,9 @@ struct learner { finish_example_fd.data = learn_fd.data; finish_example_fd.finish_example_f = (tend_example)f; } - + friend learner& init_learner<>(T*, base_learner*, void(*l)(T&, base_learner&, example&), - void(*pred)(T&, base_learner&, example&), size_t); + void(*pred)(T&, base_learner&, example&), size_t); friend learner& init_learner<>(T*, void (*learn)(T&, base_learner&, example&), size_t, prediction_type::prediction_type_t); friend learner& init_learner<>(T*, base_learner*, void (*l)(T&, base_learner&, example&), @@ -256,17 +255,16 @@ learner& init_learner(T* dat, void (*learn)(T&, base_learner&, example&), template learner& init_learner(T* dat, base_learner* base, - void(*learn)(T&, base_learner&, example&), - void(*predict)(T&, base_learner&, example&), size_t ws) -{ - return init_learner(dat, base, learn, predict, ws, base->pred_type); + void(*learn)(T&, base_learner&, example&), + void(*predict)(T&, base_learner&, example&), size_t ws) +{ return init_learner(dat, base, learn, predict, ws, base->pred_type); } template learner& init_learner(T* dat, base_learner* base, void (*learn)(T&, base_learner&, example&), void (*predict)(T&, base_learner&, example&), size_t ws, - prediction_type::prediction_type_t pred_type) + prediction_type::prediction_type_t pred_type) { //the reduction constructor, with separate learn and predict functions learner& ret = calloc_or_throw >(); ret = *(learner*)base; @@ -292,7 +290,7 @@ template learner& init_multiclass_learner(T* dat, base_learner* base, void (*learn)(T&, base_learner&, example&), void (*predict)(T&, base_learner&, example&), parser* p, size_t ws, - prediction_type::prediction_type_t pred_type = prediction_type::multiclass) + prediction_type::prediction_type_t pred_type = prediction_type::multiclass) { learner& l = init_learner(dat,base,learn,predict,ws,pred_type); l.set_finish_example(MULTICLASS::finish_example); p->lp = MULTICLASS::mc_label; diff --git a/vowpalwabbit/log_multi.cc b/vowpalwabbit/log_multi.cc index c4d8f79f608..edeb97ebe8c 100644 --- a/vowpalwabbit/log_multi.cc +++ b/vowpalwabbit/log_multi.cc @@ -307,7 +307,7 @@ void learn(log_multi& b, base_learner& base, example& ec) b.nodes[cn].min_count++; update_min_count(b, cn); ec.pred.multiclass = start_pred; - ec.l.multi = mc; + ec.l.multi = mc; } } @@ -360,13 +360,13 @@ void finish(log_multi& b) void save_load_tree(log_multi& b, io_buf& model_file, bool read, bool text) { if (model_file.files.size() > 0) - { stringstream msg; - msg << "k = " << b.k; - bin_text_read_write_fixed(model_file,(char*)&b.max_predictors, sizeof(b.k), "", read, msg, text); + { stringstream msg; + msg << "k = " << b.k; + bin_text_read_write_fixed(model_file,(char*)&b.max_predictors, sizeof(b.k), "", read, msg, text); - msg << "nodes = " << b.nodes.size() << " "; - uint32_t temp = (uint32_t)b.nodes.size(); - bin_text_read_write_fixed(model_file,(char*)&temp, sizeof(temp), "", read, msg, text); + msg << "nodes = " << b.nodes.size() << " "; + uint32_t temp = (uint32_t)b.nodes.size(); + bin_text_read_write_fixed(model_file,(char*)&temp, sizeof(temp), "", read, msg, text); if (read) for (uint32_t j = 1; j < temp; j++) b.nodes.push_back(init_node()); @@ -405,7 +405,7 @@ void save_load_tree(log_multi& b, io_buf& model_file, bool read, bool text) bin_text_read_write_fixed(model_file,(char*)&n.internal, sizeof(n.internal), "", read, msg, text); if (n.internal) - { msg << " base_predictor = " << n.base_predictor; + { msg << " base_predictor = " << n.base_predictor; bin_text_read_write_fixed(model_file,(char*)&n.base_predictor, sizeof(n.base_predictor), "", read, msg, text); msg << " left = " << n.left; @@ -424,11 +424,11 @@ void save_load_tree(log_multi& b, io_buf& model_file, bool read, bool text) bin_text_read_write_fixed(model_file,(char*)&n.n, sizeof(n.n), "", read, msg, text); } else - { msg << " max_count = " << n.max_count; - bin_text_read_write_fixed(model_file,(char*)&n.max_count, sizeof(n.max_count), "", read, msg, text); - msg << " max_count_label = "<< n.max_count_label <<"\n"; - bin_text_read_write_fixed(model_file,(char*)&n.max_count_label, sizeof(n.max_count_label), "", read, msg, text); - } + { msg << " max_count = " << n.max_count; + bin_text_read_write_fixed(model_file,(char*)&n.max_count, sizeof(n.max_count), "", read, msg, text); + msg << " max_count_label = "<< n.max_count_label <<"\n"; + bin_text_read_write_fixed(model_file,(char*)&n.max_count_label, sizeof(n.max_count_label), "", read, msg, text); + } for (size_t k = 0; k < n.preds.size(); k++) { node_pred& p = n.preds[k]; diff --git a/vowpalwabbit/loss_functions.cc b/vowpalwabbit/loss_functions.cc index 7733c8991b0..f9aee690741 100644 --- a/vowpalwabbit/loss_functions.cc +++ b/vowpalwabbit/loss_functions.cc @@ -296,50 +296,42 @@ class poisson_loss : public loss_function float getLoss(shared_data*, float prediction, float label) { if (label < 0.f) cout << "You are using label " << label << " but loss function expects label >= 0!" << endl; - float exp_prediction = expf(prediction); + float exp_prediction = expf(prediction); // deviance is used instead of log-likelihood - return 2 * (label * (logf(label + 1e-6f) - prediction) - (label - exp_prediction)); + return 2 * (label * (logf(label + 1e-6f) - prediction) - (label - exp_prediction)); } float getUpdate(float prediction, float label,float update_scale, float pred_per_update) - { - float exp_prediction = expf(prediction); + { float exp_prediction = expf(prediction); if (label > 0) - { - return label * update_scale - log1p(exp_prediction*expm1(label * update_scale * pred_per_update)/label)/pred_per_update; + { return label * update_scale - log1p(exp_prediction*expm1(label * update_scale * pred_per_update)/label)/pred_per_update; } else - { - return - log1p(exp_prediction * update_scale * pred_per_update)/pred_per_update; + { return - log1p(exp_prediction * update_scale * pred_per_update)/pred_per_update; } } float getUnsafeUpdate(float prediction, float label,float update_scale) - { - float exp_prediction = expf(prediction); - return (label - exp_prediction) * update_scale; + { float exp_prediction = expf(prediction); + return (label - exp_prediction) * update_scale; } float getRevertingWeight(shared_data* sd, float prediction, float eta_t) - { - THROW("Active learning not supported by poisson loss"); + { THROW("Active learning not supported by poisson loss"); } float getSquareGrad(float prediction, float label) - { - float exp_prediction = expf(prediction); + { float exp_prediction = expf(prediction); return (exp_prediction - label) * (exp_prediction - label); } float first_derivative(shared_data*, float prediction, float label) - { - float exp_prediction = expf(prediction); + { float exp_prediction = expf(prediction); return (exp_prediction - label); } float second_derivative(shared_data*, float prediction, float label) - { - float exp_prediction = expf(prediction); + { float exp_prediction = expf(prediction); return exp_prediction; } }; @@ -362,10 +354,8 @@ loss_function* getLossFunction(vw& all, string funcName, float function_paramete { return new quantileloss(function_parameter); } else if(funcName.compare("poisson") == 0) - { - if (all.set_minmax != noop_mm) - { - all.sd->min_label = -50; + { if (all.set_minmax != noop_mm) + { all.sd->min_label = -50; all.sd->max_label = 50; } return new poisson_loss(); diff --git a/vowpalwabbit/lrq.cc b/vowpalwabbit/lrq.cc index 75ad2702626..3070e8946d9 100644 --- a/vowpalwabbit/lrq.cc +++ b/vowpalwabbit/lrq.cc @@ -84,49 +84,48 @@ void predict_or_learn(LRQstate& lrq, base_learner& base, example& ec) features& left_fs = ec.feature_space[left]; for (unsigned int lfn = 0; lfn < lrq.orig_size[left]; ++lfn) - { - float lfx = left_fs.values[lfn]; - uint64_t lindex = left_fs.indicies[lfn] + ec.ft_offset; - weight_parameters& w = all.weights; - for (unsigned int n = 1; n <= k; ++n) - { if (! do_dropout || cheesyrbit (lrq.seed)) - { uint64_t lwindex = (uint64_t)(lindex + (n << all.weights.stride_shift())); - weight_parameters::iterator lw = w.change_begin() + (lwindex & w.mask()); - - // perturb away from saddle point at (0, 0) - if (is_learn && ! example_is_test (ec) && *lw == 0) - *lw = cheesyrand (lwindex); //not sure if lw needs a weight mask? - - features& right_fs = ec.feature_space[right]; - for (unsigned int rfn = 0; - rfn < lrq.orig_size[right]; - ++rfn) - { // NB: ec.ft_offset added by base learner - float rfx = right_fs.values[rfn]; - uint64_t rindex = right_fs.indicies[rfn]; - uint64_t rwindex = (uint64_t)(rindex + (n << all.weights.stride_shift())); - - right_fs.push_back(scale **lw * lfx * rfx, rwindex); - - if (all.audit || all.hash_inv) - { std::stringstream new_feature_buffer; - new_feature_buffer << right << '^' - << right_fs.space_names[rfn].get()->second << '^' - << n; + { float lfx = left_fs.values[lfn]; + uint64_t lindex = left_fs.indicies[lfn] + ec.ft_offset; + weight_parameters& w = all.weights; + for (unsigned int n = 1; n <= k; ++n) + { if (! do_dropout || cheesyrbit (lrq.seed)) + { uint64_t lwindex = (uint64_t)(lindex + (n << all.weights.stride_shift())); + weight_parameters::iterator lw = w.change_begin() + (lwindex & w.mask()); + + // perturb away from saddle point at (0, 0) + if (is_learn && ! example_is_test (ec) && *lw == 0) + *lw = cheesyrand (lwindex); //not sure if lw needs a weight mask? + + features& right_fs = ec.feature_space[right]; + for (unsigned int rfn = 0; + rfn < lrq.orig_size[right]; + ++rfn) + { // NB: ec.ft_offset added by base learner + float rfx = right_fs.values[rfn]; + uint64_t rindex = right_fs.indicies[rfn]; + uint64_t rwindex = (uint64_t)(rindex + (n << all.weights.stride_shift())); + + right_fs.push_back(scale **lw * lfx * rfx, rwindex); + + if (all.audit || all.hash_inv) + { std::stringstream new_feature_buffer; + new_feature_buffer << right << '^' + << right_fs.space_names[rfn].get()->second << '^' + << n; #ifdef _WIN32 - char* new_space = _strdup("lrq"); - char* new_feature = _strdup(new_feature_buffer.str().c_str()); + char* new_space = _strdup("lrq"); + char* new_feature = _strdup(new_feature_buffer.str().c_str()); #else - char* new_space = strdup("lrq"); - char* new_feature = strdup(new_feature_buffer.str().c_str()); + char* new_space = strdup("lrq"); + char* new_feature = strdup(new_feature_buffer.str().c_str()); #endif - right_fs.space_names.push_back(audit_strings_ptr(new audit_strings(new_space,new_feature))); - } - } - } + right_fs.space_names.push_back(audit_strings_ptr(new audit_strings(new_space,new_feature))); + } } + } } + } } if (is_learn) @@ -136,20 +135,20 @@ void predict_or_learn(LRQstate& lrq, base_learner& base, example& ec) // Restore example if (iter == 0) - { first_prediction = ec.pred.scalar; - first_loss = ec.loss; - first_uncertainty = ec.confidence; - } + { first_prediction = ec.pred.scalar; + first_loss = ec.loss; + first_uncertainty = ec.confidence; + } else - { ec.pred.scalar = first_prediction; - ec.loss = first_loss; - ec.confidence = first_uncertainty; - } + { ec.pred.scalar = first_prediction; + ec.loss = first_loss; + ec.confidence = first_uncertainty; + } for (string const& i : lrq.lrpairs) - { unsigned char right = i[(which+1)%2]; - ec.feature_space[right].truncate_to(lrq.orig_size[right]); - } + { unsigned char right = i[(which+1)%2]; + ec.feature_space[right].truncate_to(lrq.orig_size[right]); + } } } @@ -216,7 +215,7 @@ base_learner* lrq_setup(vw& all) all.wpp = all.wpp * (uint64_t)(1 + maxk); learner& l = init_learner(&lrq, setup_base(all), predict_or_learn, - predict_or_learn, 1 + maxk); + predict_or_learn, 1 + maxk); l.set_end_pass(reset_seed); l.set_finish(finish); diff --git a/vowpalwabbit/lrqfa.cc b/vowpalwabbit/lrqfa.cc index d2e38825e95..c04bab9c669 100644 --- a/vowpalwabbit/lrqfa.cc +++ b/vowpalwabbit/lrqfa.cc @@ -52,44 +52,44 @@ void predict_or_learn(LRQFAstate& lrq, base_learner& base, example& ec) unsigned int lfd_id = lrq.field_id[left]; unsigned int rfd_id = lrq.field_id[right]; for (unsigned int lfn = 0; lfn < lrq.orig_size[left]; ++lfn) - { features& fs = ec.feature_space[left]; - float lfx = fs.values[lfn]; - uint64_t lindex = fs.indicies[lfn]; - weight_parameters::iterator iter = w.begin(); - for (unsigned int n = 1; n <= k; ++n) - { uint64_t lwindex = (uint64_t)(lindex + ((rfd_id*k+n) << all.weights.stride_shift())); // a feature has k weights in each field - (&(*iter))[lindex] += ((rfd_id*k + n) & w.mask()); //TODO: get ride of mask() - // perturb away from saddle point at (0, 0) - if (is_learn && !example_is_test(ec) && (&(*iter))[lindex] == 0) - { (&(*iter))[lindex] = cheesyrand(lwindex) * 0.5f / sqrtk; - } - - for (unsigned int rfn = 0; rfn < lrq.orig_size[right]; ++rfn) - { features& rfs = ec.feature_space[right]; - // feature* rf = ec.atomics[right].begin + rfn; - // NB: ec.ft_offset added by base learner - float rfx = rfs.values[rfn]; - uint64_t rindex = rfs.indicies[rfn]; - uint64_t rwindex = (uint64_t)(rindex + ((lfd_id*k+n) << all.weights.stride_shift())); - - rfs.push_back((&(*iter))[lindex] * lfx * rfx, rwindex); - if (all.audit || all.hash_inv) - { std::stringstream new_feature_buffer; - new_feature_buffer << right << '^' - << rfs.space_names[rfn].get()->second << '^' - << n; + { features& fs = ec.feature_space[left]; + float lfx = fs.values[lfn]; + uint64_t lindex = fs.indicies[lfn]; + weight_parameters::iterator iter = w.begin(); + for (unsigned int n = 1; n <= k; ++n) + { uint64_t lwindex = (uint64_t)(lindex + ((rfd_id*k+n) << all.weights.stride_shift())); // a feature has k weights in each field + (&(*iter))[lindex] += ((rfd_id*k + n) & w.mask()); //TODO: get ride of mask() + // perturb away from saddle point at (0, 0) + if (is_learn && !example_is_test(ec) && (&(*iter))[lindex] == 0) + { (&(*iter))[lindex] = cheesyrand(lwindex) * 0.5f / sqrtk; + } + + for (unsigned int rfn = 0; rfn < lrq.orig_size[right]; ++rfn) + { features& rfs = ec.feature_space[right]; + // feature* rf = ec.atomics[right].begin + rfn; + // NB: ec.ft_offset added by base learner + float rfx = rfs.values[rfn]; + uint64_t rindex = rfs.indicies[rfn]; + uint64_t rwindex = (uint64_t)(rindex + ((lfd_id*k+n) << all.weights.stride_shift())); + + rfs.push_back((&(*iter))[lindex] * lfx * rfx, rwindex); + if (all.audit || all.hash_inv) + { std::stringstream new_feature_buffer; + new_feature_buffer << right << '^' + << rfs.space_names[rfn].get()->second << '^' + << n; #ifdef _WIN32 - char* new_space = _strdup("lrqfa"); - char* new_feature = _strdup(new_feature_buffer.str().c_str()); + char* new_space = _strdup("lrqfa"); + char* new_feature = _strdup(new_feature_buffer.str().c_str()); #else - char* new_space = strdup("lrqfa"); - char* new_feature = strdup(new_feature_buffer.str().c_str()); + char* new_space = strdup("lrqfa"); + char* new_feature = strdup(new_feature_buffer.str().c_str()); #endif - rfs.space_names.push_back(audit_strings_ptr(new audit_strings(new_space,new_feature))); - } - } + rfs.space_names.push_back(audit_strings_ptr(new audit_strings(new_space,new_feature))); } + } } + } } } @@ -113,13 +113,12 @@ void predict_or_learn(LRQFAstate& lrq, base_learner& base, example& ec) features& rfs = ec.feature_space[right]; rfs.values.end() = rfs.values.begin() + lrq.orig_size[right]; - if (all.audit || all.hash_inv) - { - for (size_t j = lrq.orig_size[right]; j < rfs.space_names.size(); ++j) + if (all.audit || all.hash_inv) + { for (size_t j = lrq.orig_size[right]; j < rfs.space_names.size(); ++j) rfs.space_names[j].~audit_strings_ptr(); - rfs.space_names.end() = rfs.space_names.begin() + lrq.orig_size[right]; - } + rfs.space_names.end() = rfs.space_names.begin() + lrq.orig_size[right]; + } } } } diff --git a/vowpalwabbit/marginal.cc b/vowpalwabbit/marginal.cc index 580cf65113a..0ba44b4c1cf 100644 --- a/vowpalwabbit/marginal.cc +++ b/vowpalwabbit/marginal.cc @@ -3,13 +3,13 @@ using namespace std; -namespace MARGINAL { +namespace MARGINAL +{ - typedef pair marginal; +typedef pair marginal; struct data -{ - float initial_numerator; +{ float initial_numerator; float initial_denominator; float decay; bool id_features[256]; @@ -20,118 +20,103 @@ struct data template void predict_or_learn(data& sm, LEARNER::base_learner& base, example& ec) -{ - uint64_t mask = sm.all->weights.mask(); +{ uint64_t mask = sm.all->weights.mask(); for (example::iterator i = ec.begin(); i!= ec.end(); ++i) - { - namespace_index n = i.index(); - if (sm.id_features[n]) - { - std::swap(sm.temp[n],*i); - features& f = *i; - f.erase(); - for (features::iterator j = sm.temp[n].begin(); j != sm.temp[n].end(); ++j) - { - float first_value = j.value(); - uint64_t first_index = j.index() & mask; - if (++j == sm.temp[n].end()) - { - cout << "warning: id feature namespace has " << sm.temp[n].size() << " features. Should be a multiple of 2" << endl; - break; - } - float second_value = j.value(); - uint64_t second_index = j.index() & mask; - if (first_value != 1. || second_value != 1.) - { - cout << "warning: bad id features, must have value 1." << endl; - continue; - } - uint64_t key = second_index + ec.ft_offset; - if (sm.marginals.find(key) == sm.marginals.end())//need to initialize things. - sm.marginals.insert(make_pair(key,make_pair(sm.initial_numerator, sm.initial_denominator))); - f.push_back((sm.marginals[key].first / sm.marginals[key].second), first_index); - if (!sm.temp[n].space_names.empty()) - f.space_names.push_back(sm.temp[n].space_names[2*(f.size()-1)]); - } - } + { namespace_index n = i.index(); + if (sm.id_features[n]) + { std::swap(sm.temp[n],*i); + features& f = *i; + f.erase(); + for (features::iterator j = sm.temp[n].begin(); j != sm.temp[n].end(); ++j) + { float first_value = j.value(); + uint64_t first_index = j.index() & mask; + if (++j == sm.temp[n].end()) + { cout << "warning: id feature namespace has " << sm.temp[n].size() << " features. Should be a multiple of 2" << endl; + break; + } + float second_value = j.value(); + uint64_t second_index = j.index() & mask; + if (first_value != 1. || second_value != 1.) + { cout << "warning: bad id features, must have value 1." << endl; + continue; + } + uint64_t key = second_index + ec.ft_offset; + if (sm.marginals.find(key) == sm.marginals.end())//need to initialize things. + sm.marginals.insert(make_pair(key,make_pair(sm.initial_numerator, sm.initial_denominator))); + f.push_back((sm.marginals[key].first / sm.marginals[key].second), first_index); + if (!sm.temp[n].space_names.empty()) + f.space_names.push_back(sm.temp[n].space_names[2*(f.size()-1)]); + } } + } if (is_learn) base.learn(ec); else base.predict(ec); for (example::iterator i = ec.begin(); i!= ec.end(); ++i) - { - namespace_index n = i.index(); - if (sm.id_features[n]) - { - if (is_learn) - for (features::iterator j = sm.temp[n].begin(); j != sm.temp[n].end(); ++j) - { - if (++j == sm.temp[n].end()) - break; - uint64_t second_index = j.index() & mask; - uint64_t key = second_index + ec.ft_offset; - marginal& m = sm.marginals[key]; - m.first = m.first * (1. - sm.decay) + ec.l.simple.label * ec.weight; - m.second = m.second * (1. - sm.decay) + ec.weight; - } - std::swap(sm.temp[n],*i); - } + { namespace_index n = i.index(); + if (sm.id_features[n]) + { if (is_learn) + for (features::iterator j = sm.temp[n].begin(); j != sm.temp[n].end(); ++j) + { if (++j == sm.temp[n].end()) + break; + uint64_t second_index = j.index() & mask; + uint64_t key = second_index + ec.ft_offset; + marginal& m = sm.marginals[key]; + m.first = m.first * (1. - sm.decay) + ec.l.simple.label * ec.weight; + m.second = m.second * (1. - sm.decay) + ec.weight; + } + std::swap(sm.temp[n],*i); } + } } - - void finish(data& sm) - { sm.marginals.~unordered_map(); - for (size_t i =0; i < 256; i++) - sm.temp[i].delete_v(); + +void finish(data& sm) +{ sm.marginals.~unordered_map(); + for (size_t i =0; i < 256; i++) + sm.temp[i].delete_v(); +} + +void save_load(data& sm, io_buf& io, bool read, bool text) +{ uint64_t stride_shift = sm.all->weights.stride_shift(); + if (io.files.size() == 0) + return; + stringstream msg; + uint64_t total_size; + if (!read) + { total_size = (uint64_t)sm.marginals.size(); + msg << "marginals size = " << total_size << "\n"; } + bin_text_read_write_fixed_validated(io, (char*)&total_size, sizeof(total_size), "", read, msg, text); - void save_load(data& sm, io_buf& io, bool read, bool text) - { - uint64_t stride_shift = sm.all->weights.stride_shift(); - if (io.files.size() == 0) - return; - stringstream msg; - uint64_t total_size; + auto iter = sm.marginals.begin(); + for (size_t i = 0; i < total_size; ++i) + { uint64_t index; if (!read) - { - total_size = (uint64_t)sm.marginals.size(); - msg << "marginals size = " << total_size << "\n"; - } - bin_text_read_write_fixed_validated(io, (char*)&total_size, sizeof(total_size), "", read, msg, text); - - auto iter = sm.marginals.begin(); - for (size_t i = 0; i < total_size; ++i) - { - uint64_t index; - if (!read) - { - index = iter->first >> stride_shift; - msg << index << ":"; - } - bin_text_read_write_fixed(io, (char*)&index, sizeof(index), "", read, msg, text); - double numerator; - if (!read) - { - numerator = iter->second.first; - msg << numerator << ":"; - } - bin_text_read_write_fixed(io, (char*)&numerator, sizeof(numerator), "", read, msg, text); - double denominator; - if (!read) - { - denominator = iter->second.second; - msg << denominator << "\n"; - } - bin_text_read_write_fixed(io, (char*)&denominator, sizeof(denominator), "", read, msg, text); - if (read) - sm.marginals.insert(make_pair(index << stride_shift,make_pair(numerator,denominator))); - else - ++iter; - } + { index = iter->first >> stride_shift; + msg << index << ":"; + } + bin_text_read_write_fixed(io, (char*)&index, sizeof(index), "", read, msg, text); + double numerator; + if (!read) + { numerator = iter->second.first; + msg << numerator << ":"; + } + bin_text_read_write_fixed(io, (char*)&numerator, sizeof(numerator), "", read, msg, text); + double denominator; + if (!read) + { denominator = iter->second.second; + msg << denominator << "\n"; + } + bin_text_read_write_fixed(io, (char*)&denominator, sizeof(denominator), "", read, msg, text); + if (read) + sm.marginals.insert(make_pair(index << stride_shift,make_pair(numerator,denominator))); + else + ++iter; } } +} using namespace MARGINAL; @@ -139,9 +124,9 @@ LEARNER::base_learner* marginal_setup(vw& all) { if (missing_option(all, "marginal", "substitute marginal label estimates for ids")) return nullptr; new_options(all) - ("initial_denominator", po::value()->default_value(1.f), "initial denominator") - ("initial_numerator", po::value()->default_value(0.5f), "initial numerator") - ("decay", po::value()->default_value(0.f), "decay multiplier per event (1e-3 for example)"); + ("initial_denominator", po::value()->default_value(1.f), "initial denominator") + ("initial_numerator", po::value()->default_value(0.5f), "initial numerator") + ("decay", po::value()->default_value(0.f), "decay multiplier per event (1e-3 for example)"); add_options(all); data& d = calloc_or_throw(); @@ -155,11 +140,11 @@ LEARNER::base_learner* marginal_setup(vw& all) if (s.find((char)u) != string::npos) d.id_features[u] = true; new(&d.marginals)unordered_map(); - + LEARNER::learner& ret = init_learner(&d, setup_base(all), predict_or_learn, predict_or_learn); ret.set_finish(finish); ret.set_save_load(save_load); - + return make_base(ret); } diff --git a/vowpalwabbit/memory.h b/vowpalwabbit/memory.h index e0ddbc79d4a..a003aff2b9c 100644 --- a/vowpalwabbit/memory.h +++ b/vowpalwabbit/memory.h @@ -29,8 +29,7 @@ T* calloc_mergable_or_throw(size_t nmemb) size_t length = nmemb * sizeof(T); void* data; if (0 != posix_memalign(&data, sysconf(_SC_PAGE_SIZE), length)) - { - const char* msg = "internal error: memory allocation failed!\n"; + { const char* msg = "internal error: memory allocation failed!\n"; fputs(msg, stderr); THROW(msg); } @@ -51,8 +50,8 @@ T* calloc_mergable_or_throw(size_t nmemb) // you can enable ksmd with sudo "echo 1 > /sys/kernel/mm/ksm/run" // mark address space as a candidate for merging - if (0 != madvise(data, length, MADV_MERGEABLE)) { - const char* msg = "internal warning: marking memory as ksm mergeable failed!\n"; + if (0 != madvise(data, length, MADV_MERGEABLE)) + { const char* msg = "internal warning: marking memory as ksm mergeable failed!\n"; fputs(msg, stderr); } return (T*)data; diff --git a/vowpalwabbit/mf.cc b/vowpalwabbit/mf.cc index 92f4cba2b9c..307fe958092 100644 --- a/vowpalwabbit/mf.cc +++ b/vowpalwabbit/mf.cc @@ -61,14 +61,12 @@ void predict(mf& data, base_learner& base, example& ec) // add interaction terms to prediction for (string& i : data.pairs) - { - int left_ns = (int) i[0]; + { int left_ns = (int) i[0]; int right_ns = (int) i[1]; if (ec.feature_space[left_ns].size() > 0 && ec.feature_space[right_ns].size() > 0) { for (size_t k = 1; k <= data.rank; k++) - { - ec.indices[0] = left_ns; + { ec.indices[0] = left_ns; // compute l^k * x_l using base learner base.predict(ec, k); @@ -132,8 +130,7 @@ void learn(mf& data, base_learner& base, example& ec) data.temp_features.deep_copy_from(ec.feature_space[left_ns]); for (size_t k = 1; k <= data.rank; k++) - { - features& fs = ec.feature_space[left_ns]; + { features& fs = ec.feature_space[left_ns]; // multiply features in left namespace by r^k * x_r for (size_t i= 0; i < fs.size(); ++i) fs.values[i] *= data.sub_predictions[2*k]; @@ -157,8 +154,7 @@ void learn(mf& data, base_learner& base, example& ec) data.temp_features.deep_copy_from(ec.feature_space[right_ns]); for (size_t k = 1; k <= data.rank; k++) - { - features& fs = ec.feature_space[right_ns]; + { features& fs = ec.feature_space[right_ns]; // multiply features in right namespace by l^k * x_l for (size_t i = 0; i < fs.size(); ++i) fs.values[i] *= data.sub_predictions[2*k-1]; diff --git a/vowpalwabbit/multiclass.cc b/vowpalwabbit/multiclass.cc index b7fef47c9cd..2551b2c241d 100644 --- a/vowpalwabbit/multiclass.cc +++ b/vowpalwabbit/multiclass.cc @@ -89,56 +89,55 @@ label_parser mc_label = {default_label, parse_label, sizeof(label_t) }; - void print_label_pred(vw& all, example& ec, uint32_t prediction) - { substring ss_label = all.sd->ldict->get(ec.l.multi.label); - substring ss_pred = all.sd->ldict->get(prediction); - all.sd->print_update(all.holdout_set_off, all.current_pass, - !ss_label.begin ? "unknown" : string(ss_label.begin, ss_label.end - ss_label.begin), - !ss_pred.begin ? "unknown" : string(ss_pred.begin, ss_pred.end - ss_pred.begin), - ec.num_features, all.progress_add, all.progress_arg); - } +void print_label_pred(vw& all, example& ec, uint32_t prediction) +{ substring ss_label = all.sd->ldict->get(ec.l.multi.label); + substring ss_pred = all.sd->ldict->get(prediction); + all.sd->print_update(all.holdout_set_off, all.current_pass, + !ss_label.begin ? "unknown" : string(ss_label.begin, ss_label.end - ss_label.begin), + !ss_pred.begin ? "unknown" : string(ss_pred.begin, ss_pred.end - ss_pred.begin), + ec.num_features, all.progress_add, all.progress_arg); +} - void print_probability(vw& all, example& ec, uint32_t prediction) - { - char temp_str[10]; - sprintf_s(temp_str, 10, "%d(%2.0f%%)", prediction, 100 * ec.pred.scalars[prediction - 1]); - - char label_str[512]; - sprintf_s(label_str, 512, "%u", ec.l.multi.label); - - all.sd->print_update(all.holdout_set_off, all.current_pass, label_str, temp_str, - ec.num_features, all.progress_add, all.progress_arg); - } +void print_probability(vw& all, example& ec, uint32_t prediction) +{ char temp_str[10]; + sprintf_s(temp_str, 10, "%d(%2.0f%%)", prediction, 100 * ec.pred.scalars[prediction - 1]); - void print_score(vw& all, example& ec, uint32_t prediction) - { char temp_str[10]; - sprintf_s(temp_str, 10, "%d", prediction); - - char label_str[512]; - sprintf_s(label_str, 512, "%u", ec.l.multi.label); - - all.sd->print_update(all.holdout_set_off, all.current_pass, label_str, temp_str, - ec.num_features, all.progress_add, all.progress_arg); - } - - void direct_print_update(vw& all, example& ec, uint32_t prediction) - { all.sd->print_update(all.holdout_set_off, all.current_pass, ec.l.multi.label, prediction, - ec.num_features, all.progress_add, all.progress_arg); - } + char label_str[512]; + sprintf_s(label_str, 512, "%u", ec.l.multi.label); + + all.sd->print_update(all.holdout_set_off, all.current_pass, label_str, temp_str, + ec.num_features, all.progress_add, all.progress_arg); +} + +void print_score(vw& all, example& ec, uint32_t prediction) +{ char temp_str[10]; + sprintf_s(temp_str, 10, "%d", prediction); - template - void print_update(vw& all, example &ec, uint32_t prediction) - { if (all.sd->weighted_examples >= all.sd->dump_interval && !all.quiet && !all.bfgs) - { if (! all.sd->ldict) - T(all, ec, prediction); - else - print_label_pred(all, ec, ec.pred.multiclass); - } + char label_str[512]; + sprintf_s(label_str, 512, "%u", ec.l.multi.label); + + all.sd->print_update(all.holdout_set_off, all.current_pass, label_str, temp_str, + ec.num_features, all.progress_add, all.progress_arg); +} + +void direct_print_update(vw& all, example& ec, uint32_t prediction) +{ all.sd->print_update(all.holdout_set_off, all.current_pass, ec.l.multi.label, prediction, + ec.num_features, all.progress_add, all.progress_arg); +} + +template +void print_update(vw& all, example &ec, uint32_t prediction) +{ if (all.sd->weighted_examples >= all.sd->dump_interval && !all.quiet && !all.bfgs) + { if (! all.sd->ldict) + T(all, ec, prediction); + else + print_label_pred(all, ec, ec.pred.multiclass); } +} + +void print_update_with_probability(vw& all, example& ec, uint32_t pred) {print_update(all, ec, pred);} +void print_update_with_score(vw& all, example& ec, uint32_t pred) {print_update(all, ec, pred);} - void print_update_with_probability(vw& all, example& ec, uint32_t pred){print_update(all, ec, pred);} - void print_update_with_score(vw& all, example& ec, uint32_t pred){print_update(all, ec, pred);} - void finish_example(vw& all, example& ec) { float loss = 0; if (ec.l.multi.label != (uint32_t)ec.pred.multiclass) diff --git a/vowpalwabbit/multilabel.cc b/vowpalwabbit/multilabel.cc index cb0683c6b27..faeee3958c3 100644 --- a/vowpalwabbit/multilabel.cc +++ b/vowpalwabbit/multilabel.cc @@ -161,20 +161,20 @@ void output_example(vw& all, example& ec) } all.sd->update(ec.test_only, loss, 1.f, ec.num_features); - + for (int sink : all.final_prediction_sink) if (sink >= 0) - { std::stringstream ss; - - for (size_t i = 0; i < ec.pred.multilabels.label_v.size(); i++) - { if (i > 0) - ss << ','; - ss << ec.pred.multilabels.label_v[i]; - } - ss << ' '; - all.print_text(sink, ss.str(), ec.tag); + { std::stringstream ss; + + for (size_t i = 0; i < ec.pred.multilabels.label_v.size(); i++) + { if (i > 0) + ss << ','; + ss << ec.pred.multilabels.label_v[i]; } - + ss << ' '; + all.print_text(sink, ss.str(), ec.tag); + } + print_update(all, is_test_label(ec.l.multilabels), ec); } diff --git a/vowpalwabbit/mwt.cc b/vowpalwabbit/mwt.cc index 26451c5a513..297cd195a50 100644 --- a/vowpalwabbit/mwt.cc +++ b/vowpalwabbit/mwt.cc @@ -13,218 +13,205 @@ using namespace std; using namespace LEARNER; using namespace CB_ALGS; -namespace MWT { - struct policy_data - { double cost; - uint32_t action; - bool seen; - }; - - struct mwt - { bool namespaces[256]; // the set of namespaces to evaluate. - v_array evals; // accrued losses of features. - CB::cb_class* observation; - v_array policies; - double total; - uint32_t num_classes; - bool learn; - - v_array indices;// excluded namespaces - features feature_space[256]; - vw* all; - }; - - inline bool observed_cost(CB::cb_class* cl) - { //cost observed for this action if it has non zero probability and cost != FLT_MAX - if (cl != nullptr) - if (cl->cost != FLT_MAX && cl->probability > .0) - return true; - return false; - } +namespace MWT +{ +struct policy_data +{ double cost; + uint32_t action; + bool seen; +}; + +struct mwt +{ bool namespaces[256]; // the set of namespaces to evaluate. + v_array evals; // accrued losses of features. + CB::cb_class* observation; + v_array policies; + double total; + uint32_t num_classes; + bool learn; + + v_array indices;// excluded namespaces + features feature_space[256]; + vw* all; +}; + +inline bool observed_cost(CB::cb_class* cl) +{ //cost observed for this action if it has non zero probability and cost != FLT_MAX + if (cl != nullptr) + if (cl->cost != FLT_MAX && cl->probability > .0) + return true; + return false; +} - CB::cb_class* get_observed_cost(CB::label& ld) - { for (auto& cl : ld.costs) - if( observed_cost(&cl) ) - return &cl; - return nullptr; - } +CB::cb_class* get_observed_cost(CB::label& ld) +{ for (auto& cl : ld.costs) + if( observed_cost(&cl) ) + return &cl; + return nullptr; +} - void value_policy(mwt& c, float val, uint64_t index)//estimate the value of a single feature. - { - if (val < 0 || floor(val) != val) - cout << "error " << val << " is not a valid action " << endl; +void value_policy(mwt& c, float val, uint64_t index)//estimate the value of a single feature. +{ if (val < 0 || floor(val) != val) + cout << "error " << val << " is not a valid action " << endl; - uint32_t value = (uint32_t) val; - uint64_t new_index = ((index & c.all->weights.mask()) >> c.all->weights.stride_shift()); + uint32_t value = (uint32_t) val; + uint64_t new_index = ((index & c.all->weights.mask()) >> c.all->weights.stride_shift()); - if (!c.evals[new_index].seen) - { - c.evals[new_index].seen = true; - c.policies.push_back(new_index); + if (!c.evals[new_index].seen) + { c.evals[new_index].seen = true; + c.policies.push_back(new_index); + } + + c.evals[new_index].action = value; +} + +template +void predict_or_learn(mwt& c, base_learner& base, example& ec) +{ c.observation = get_observed_cost(ec.l.cb); + + if (c.observation != nullptr) + { c.total++; + //For each nonzero feature in observed namespaces, check it's value. + for (unsigned char ns : ec.indices) + if (c.namespaces[ns]) + GD::foreach_feature(c.all->weights, ec.feature_space[ns], c); + for (uint64_t policy : c.policies) + { c.evals[policy].cost += get_unbiased_cost(c.observation, c.evals[policy].action); + c.evals[policy].action = 0; + } + } + if (exclude || learn) + { c.indices.erase(); + for (unsigned char ns : ec.indices) + if (c.namespaces[ns]) + { c.indices.push_back(ns); + if (learn) + { c.feature_space[ns].erase(); + uint32_t stride_shift = c.all->weights.stride_shift(); + for ( features::iterator& f : ec.feature_space[ns]) + { uint64_t new_index=((f.index()& c.all->weights.mask()) >> stride_shift)*c.num_classes +(uint64_t)f.value(); + c.feature_space[ns].push_back(1, new_index << stride_shift); + } + } + std::swap(c.feature_space[ns], ec.feature_space[ns]); } + } + + v_array preds = ec.pred.scalars; - c.evals[new_index].action = value; + if (learn) + { if (is_learn) + base.learn(ec); + else + base.predict(ec); } - template - void predict_or_learn(mwt& c, base_learner& base, example& ec) - { - c.observation = get_observed_cost(ec.l.cb); - - if (c.observation != nullptr) - { - c.total++; - //For each nonzero feature in observed namespaces, check it's value. - for (unsigned char ns : ec.indices) - if (c.namespaces[ns]) - GD::foreach_feature(c.all->weights, ec.feature_space[ns], c); - for (uint64_t policy : c.policies) - { - c.evals[policy].cost += get_unbiased_cost(c.observation, c.evals[policy].action); - c.evals[policy].action = 0; - } - } - if (exclude || learn) - { - c.indices.erase(); - for (unsigned char ns : ec.indices) - if (c.namespaces[ns]) - { - c.indices.push_back(ns); - if (learn) - { - c.feature_space[ns].erase(); - uint32_t stride_shift = c.all->weights.stride_shift(); - for ( features::iterator& f : ec.feature_space[ns]) - { uint64_t new_index=((f.index()& c.all->weights.mask()) >> stride_shift)*c.num_classes +(uint64_t)f.value(); - c.feature_space[ns].push_back(1, new_index << stride_shift); - } - } - std::swap(c.feature_space[ns], ec.feature_space[ns]); - } - } + if (exclude || learn) + while (c.indices.size() > 0) + { unsigned char ns = c.indices.pop(); + swap(c.feature_space[ns], ec.feature_space[ns]); + } - v_array preds = ec.pred.scalars; + //modify the predictions to use a vector with a score for each evaluated feature. + preds.erase(); + if (learn) + preds.push_back((float)ec.pred.multiclass); + for(uint64_t index : c.policies) + preds.push_back((float)c.evals[index].cost / (float)c.total); - if (learn) - { - if (is_learn) - base.learn(ec); - else - base.predict(ec); - } + ec.pred.scalars = preds; +} - if (exclude || learn) - while (c.indices.size() > 0) - { - unsigned char ns = c.indices.pop(); - swap(c.feature_space[ns], ec.feature_space[ns]); - } - - //modify the predictions to use a vector with a score for each evaluated feature. - preds.erase(); - if (learn) - preds.push_back((float)ec.pred.multiclass); - for(uint64_t index : c.policies) - preds.push_back((float)c.evals[index].cost / (float)c.total); - - ec.pred.scalars = preds; - } +void print_scalars(int f, v_array& scalars, v_array& tag) +{ if (f >= 0) + { std::stringstream ss; - void print_scalars(int f, v_array& scalars, v_array& tag) - { if (f >= 0) - { std::stringstream ss; - - for (size_t i = 0; i < scalars.size(); i++) - { if (i > 0) - ss << ' '; - ss << scalars[i]; - } - ss << '\n'; - ssize_t len = ss.str().size(); - ssize_t t = io_buf::write_file_or_socket(f, ss.str().c_str(), (unsigned int)len); - if (t != len) - cerr << "write error: " << strerror(errno) << endl; - } + for (size_t i = 0; i < scalars.size(); i++) + { if (i > 0) + ss << ' '; + ss << scalars[i]; + } + ss << '\n'; + ssize_t len = ss.str().size(); + ssize_t t = io_buf::write_file_or_socket(f, ss.str().c_str(), (unsigned int)len); + if (t != len) + cerr << "write error: " << strerror(errno) << endl; } +} - void finish_example(vw& all, mwt& c, example& ec) - { - float loss = 0.; - if (c.learn) - { - if (c.observation != nullptr) - loss = get_unbiased_cost(c.observation, (uint32_t)ec.pred.scalars[0]); - } - all.sd->update(ec.test_only, loss, 1.f, ec.num_features); - - for (int sink : all.final_prediction_sink) - print_scalars(sink, ec.pred.scalars, ec.tag); - - if (c.learn) - { - v_array temp = ec.pred.scalars; - ec.pred.multiclass = (uint32_t)temp[0]; - CB::print_update(all, c.observation != nullptr, ec, nullptr, false); - ec.pred.scalars = temp; - } - VW::finish_example(all, &ec); +void finish_example(vw& all, mwt& c, example& ec) +{ float loss = 0.; + if (c.learn) + { if (c.observation != nullptr) + loss = get_unbiased_cost(c.observation, (uint32_t)ec.pred.scalars[0]); } + all.sd->update(ec.test_only, loss, 1.f, ec.num_features); - void finish(mwt& c) - { c.evals.delete_v(); - c.policies.delete_v(); - for (size_t i = 0; i < 256; i++) - c.feature_space[i].delete_v(); - c.indices.delete_v(); + for (int sink : all.final_prediction_sink) + print_scalars(sink, ec.pred.scalars, ec.tag); + + if (c.learn) + { v_array temp = ec.pred.scalars; + ec.pred.multiclass = (uint32_t)temp[0]; + CB::print_update(all, c.observation != nullptr, ec, nullptr, false); + ec.pred.scalars = temp; } + VW::finish_example(all, &ec); +} - void save_load(mwt& c, io_buf& model_file, bool read, bool text) - { - if (model_file.files.size() == 0) - return; +void finish(mwt& c) +{ c.evals.delete_v(); + c.policies.delete_v(); + for (size_t i = 0; i < 256; i++) + c.feature_space[i].delete_v(); + c.indices.delete_v(); +} - stringstream msg; +void save_load(mwt& c, io_buf& model_file, bool read, bool text) +{ if (model_file.files.size() == 0) + return; - // total - msg << "total: " << c.total; - bin_text_read_write_fixed_validated(model_file, (char*)&c.total, sizeof(c.total), "", read, msg, text); + stringstream msg; - // policies - size_t policies_size = c.policies.size(); - bin_text_read_write_fixed_validated(model_file, (char*)&policies_size, sizeof(policies_size), "", read, msg, text); + // total + msg << "total: " << c.total; + bin_text_read_write_fixed_validated(model_file, (char*)&c.total, sizeof(c.total), "", read, msg, text); - if (read) - { c.policies.resize(policies_size); - c.policies.end() = c.policies.begin() + policies_size; - } - else - { msg << "policies: "; - for (feature_index& policy : c.policies) - msg << policy << " "; - } + // policies + size_t policies_size = c.policies.size(); + bin_text_read_write_fixed_validated(model_file, (char*)&policies_size, sizeof(policies_size), "", read, msg, text); - bin_text_read_write_fixed_validated(model_file, (char*)c.policies.begin(), policies_size * sizeof(feature_index), - "", read, msg, text); - - // c.evals is already initialized nicely to the same size as the regressor. + if (read) + { c.policies.resize(policies_size); + c.policies.end() = c.policies.begin() + policies_size; + } + else + { msg << "policies: "; for (feature_index& policy : c.policies) - { policy_data& pd = c.evals[policy]; - if (read) - msg << "evals: "<< policy << ":" << pd.action << ":" << pd.cost << " "; - bin_text_read_write_fixed_validated(model_file, (char*)&c.evals[policy], sizeof(policy_data), - "", read, msg, text); - } + msg << policy << " "; + } + + bin_text_read_write_fixed_validated(model_file, (char*)c.policies.begin(), policies_size * sizeof(feature_index), + "", read, msg, text); + + // c.evals is already initialized nicely to the same size as the regressor. + for (feature_index& policy : c.policies) + { policy_data& pd = c.evals[policy]; + if (read) + msg << "evals: "<< policy << ":" << pd.action << ":" << pd.cost << " "; + bin_text_read_write_fixed_validated(model_file, (char*)&c.evals[policy], sizeof(policy_data), + "", read, msg, text); } } +} using namespace MWT; base_learner* mwt_setup(vw& all) { if (missing_option(all, "multiworld_test", "Evaluate features as a policies")) return nullptr; new_options(all, "MWT options") - ("learn", po::value(), "Do Contextual Bandit learning on classes.") - ("exclude_eval", "Discard mwt policy features before learning"); + ("learn", po::value(), "Do Contextual Bandit learning on classes.") + ("exclude_eval", "Discard mwt policy features before learning"); add_options(all); mwt& c = calloc_or_throw(); @@ -242,17 +229,16 @@ base_learner* mwt_setup(vw& all) all.label_type = label_type::cb; if (all.vm.count("learn")) - { - c.num_classes = all.vm["learn"].as(); - c.learn = true; - - if (count(all.args.begin(), all.args.end(),"--cb") == 0) - { all.args.push_back("--cb"); - stringstream ss; - ss << c.num_classes; - all.args.push_back(ss.str()); - } + { c.num_classes = all.vm["learn"].as(); + c.learn = true; + + if (count(all.args.begin(), all.args.end(),"--cb") == 0) + { all.args.push_back("--cb"); + stringstream ss; + ss << c.num_classes; + all.args.push_back(ss.str()); } + } learner* l; if (c.learn) diff --git a/vowpalwabbit/mwt.h b/vowpalwabbit/mwt.h index a3b18b27a09..96ad199540c 100644 --- a/vowpalwabbit/mwt.h +++ b/vowpalwabbit/mwt.h @@ -6,7 +6,8 @@ license as described in the file LICENSE. #pragma once LEARNER::base_learner* mwt_setup(vw& all); -namespace MWT{ +namespace MWT +{ void delete_scalars(void* v); void print_scalars(int f, v_array& scalars, v_array& tag); } diff --git a/vowpalwabbit/nn.cc b/vowpalwabbit/nn.cc index 581aeea466b..28ad2340247 100644 --- a/vowpalwabbit/nn.cc +++ b/vowpalwabbit/nn.cc @@ -76,15 +76,15 @@ void finish_setup (nn& n, vw& all) features& fs = n.output_layer.feature_space[nn_output_namespace]; for (unsigned int i = 0; i < n.k; ++i) - { fs.push_back(1., nn_index); - nn_index += (uint64_t)n.increment; - } + { fs.push_back(1., nn_index); + nn_index += (uint64_t)n.increment; + } n.output_layer.num_features += n.k; if (! n.inpass) - { fs.push_back(1.,nn_index); - ++n.output_layer.num_features; - } + { fs.push_back(1.,nn_index); + ++n.output_layer.num_features; + } n.output_layer.in_use = true; @@ -301,7 +301,7 @@ void predict_or_learn_multi(nn& n, base_learner& base, example& ec) ec.l.simple.label = GD::finalize_prediction (n.all->sd, hidden_units[i].scalar - gradhw); ec.pred.scalar = hidden_units[i].scalar; - if (ec.l.simple.label != hidden_units[i].scalar) + if (ec.l.simple.label != hidden_units[i].scalar) base.update(ec, i); } } diff --git a/vowpalwabbit/oaa.cc b/vowpalwabbit/oaa.cc index 508c211e947..d9c6f68b56c 100644 --- a/vowpalwabbit/oaa.cc +++ b/vowpalwabbit/oaa.cc @@ -79,7 +79,7 @@ void predict_or_learn(oaa& o, LEARNER::base_learner& base, example& ec) { for (uint32_t i=1; i<=o.k; i++) { ec.l.simple = { (mc_label_data.label == i) ? 1.f : -1.f, 0.f, 0.f }; ec.pred.scalar = o.pred[i-1].scalar; - base.update(ec, i-1); + base.update(ec, i-1); } } @@ -90,24 +90,21 @@ void predict_or_learn(oaa& o, LEARNER::base_learner& base, example& ec) } if (scores) - { - scores_array.erase(); + { scores_array.erase(); for (uint32_t i=0; i 0) - multiclass_log_loss = -log(correct_class_prob) * ec.l.multi.weight; - if (ec.test_only) - all.sd->holdout_multiclass_log_loss += multiclass_log_loss; - else - all.sd->multiclass_log_loss += multiclass_log_loss; - } + { if (ec.l.multi.label <= o.k) // prevent segmentation fault if labeĺ==(uint32_t)-1 + correct_class_prob = ec.pred.scalars[ec.l.multi.label-1]; + if (correct_class_prob > 0) + multiclass_log_loss = -log(correct_class_prob) * ec.l.multi.weight; + if (ec.test_only) + all.sd->holdout_multiclass_log_loss += multiclass_log_loss; + else + all.sd->multiclass_log_loss += multiclass_log_loss; + } // === Compute `prediction` and zero_one_loss // We have already computed `prediction` in predict_or_learn, // but we cannot store it in ec.pred union because we store ec.pred.probs there. @@ -231,24 +227,21 @@ LEARNER::base_learner* oaa_setup(vw& all) LEARNER::learner* l; if( all.vm.count("probabilities") || all.vm.count("scores") ) - { - all.delete_prediction = delete_scalars; + { all.delete_prediction = delete_scalars; if (all.vm.count("probabilities")) - { - if (!all.vm.count("loss_function") || all.vm["loss_function"].as() != "logistic" ) - cerr << "WARNING: --probabilities should be used only with --loss_function=logistic" << endl; - // the three boolean template parameters are: is_learn, print_all and scores - l = &LEARNER::init_multiclass_learner(data_ptr, setup_base(all), predict_or_learn, - predict_or_learn, all.p, data.k, prediction_type::scalars); - all.sd->report_multiclass_log_loss = true; - l->set_finish_example(finish_example_scores); - } + { if (!all.vm.count("loss_function") || all.vm["loss_function"].as() != "logistic" ) + cerr << "WARNING: --probabilities should be used only with --loss_function=logistic" << endl; + // the three boolean template parameters are: is_learn, print_all and scores + l = &LEARNER::init_multiclass_learner(data_ptr, setup_base(all), predict_or_learn, + predict_or_learn, all.p, data.k, prediction_type::scalars); + all.sd->report_multiclass_log_loss = true; + l->set_finish_example(finish_example_scores); + } else - { - l = &LEARNER::init_multiclass_learner(data_ptr, setup_base(all), predict_or_learn, - predict_or_learn, all.p, data.k, prediction_type::scalars); - l->set_finish_example(finish_example_scores); - } + { l = &LEARNER::init_multiclass_learner(data_ptr, setup_base(all), predict_or_learn, + predict_or_learn, all.p, data.k, prediction_type::scalars); + l->set_finish_example(finish_example_scores); + } } else if (all.raw_prediction > 0) l = &LEARNER::init_multiclass_learner(data_ptr, setup_base(all), predict_or_learn, diff --git a/vowpalwabbit/parse_args.cc b/vowpalwabbit/parse_args.cc index 61800b96b8e..3ba04fd12fc 100644 --- a/vowpalwabbit/parse_args.cc +++ b/vowpalwabbit/parse_args.cc @@ -540,10 +540,9 @@ void parse_feature_tweaks(vw& all) //feature manipulation string hash_function("strings"); if(vm.count("hash")) - { - hash_function = vm["hash"].as(); - *all.file_options << " --hash " << hash_function; - } + { hash_function = vm["hash"].as(); + *all.file_options << " --hash " << hash_function; + } all.p->hasher = getHasher(hash_function); if (vm.count("spelling")) @@ -1260,9 +1259,8 @@ void parse_modules(vw& all, io_buf& model) } void parse_sources(vw& all, io_buf& model, bool skipModelLoad) -{ - if (!skipModelLoad) - load_input_model(all, model); +{ if (!skipModelLoad) + load_input_model(all, model); parse_source(all); @@ -1340,18 +1338,17 @@ void free_args(int argc, char* argv[]) } vw* initialize(string s, io_buf* model, bool skipModelLoad) -{ - int argc = 0; +{ int argc = 0; char** argv = get_argv_from_string(s,argc); vw* ret = nullptr; - + try { ret = initialize(argc, argv, model, skipModelLoad); } catch(...) { free_args(argc, argv); throw; } - + free_args(argc, argv); return ret; } @@ -1378,8 +1375,8 @@ vw* initialize(int argc, char* argv[], io_buf* model, bool skipModelLoad) } catch (std::exception& e) { std::cerr << "Error: " << e.what() << "\n"; - finish(all); - throw; + finish(all); + throw; } catch (...) { finish(all); @@ -1494,7 +1491,7 @@ void finish(vw& all, bool delete_all) { all.l->finish(); free_it(all.l); } - + free_parser(all); finalize_source(all.p); all.p->parse_name.erase(); diff --git a/vowpalwabbit/parse_example.cc b/vowpalwabbit/parse_example.cc index a92a6fc4af1..9e6b7593bc3 100644 --- a/vowpalwabbit/parse_example.cc +++ b/vowpalwabbit/parse_example.cc @@ -59,19 +59,19 @@ int read_features_json(vw* all, v_array& examples) // there is no way to determine upfront if a shared example exists // thus even if there are no features for the shared example, still an empty example is returned. - if (examples.size() > 1) + if (examples.size() > 1) { // insert new line example at the end - example& ae = VW::get_unused_example(all); + example& ae = VW::get_unused_example(all); char empty = '\0'; substring example = { &empty, &empty }; substring_to_example(all, &ae, example); - - examples.push_back(&ae); + + examples.push_back(&ae); } return 1; } - + template class TC_parser { @@ -142,66 +142,64 @@ class TC_parser inline void maybeFeature() { if(*reading_head == ' ' || *reading_head == '\t' || *reading_head == '|'|| reading_head == endLine || *reading_head == '\r' ) - { // maybeFeature --> ø - } + { // maybeFeature --> ø + } else - { // maybeFeature --> 'String' FeatureValue - substring feature_name=read_name(); - v = cur_channel_v * featureValue(); - uint64_t word_hash; - if (feature_name.end != feature_name.begin) - word_hash = (p->hasher(feature_name, channel_hash)); - else - word_hash = channel_hash + anon++; - if(v == 0) return; //dont add 0 valued features to list of features - features& fs = ae->feature_space[index]; - fs.push_back(v, word_hash); - if(audit) - { v_array feature_v = v_init(); - push_many(feature_v, feature_name.begin, feature_name.end - feature_name.begin); - feature_v.push_back('\0'); - fs.space_names.push_back(audit_strings_ptr(new audit_strings(base, feature_v.begin()))); - feature_v.delete_v(); + { // maybeFeature --> 'String' FeatureValue + substring feature_name=read_name(); + v = cur_channel_v * featureValue(); + uint64_t word_hash; + if (feature_name.end != feature_name.begin) + word_hash = (p->hasher(feature_name, channel_hash)); + else + word_hash = channel_hash + anon++; + if(v == 0) return; //dont add 0 valued features to list of features + features& fs = ae->feature_space[index]; + fs.push_back(v, word_hash); + if(audit) + { v_array feature_v = v_init(); + push_many(feature_v, feature_name.begin, feature_name.end - feature_name.begin); + feature_v.push_back('\0'); + fs.space_names.push_back(audit_strings_ptr(new audit_strings(base, feature_v.begin()))); + feature_v.delete_v(); + } + if ((affix_features[index] > 0) && (feature_name.end != feature_name.begin)) + { features& affix_fs = ae->feature_space[affix_namespace]; + if (affix_fs.size() == 0) + ae->indices.push_back(affix_namespace); + uint64_t affix = affix_features[index]; + while (affix > 0) + { bool is_prefix = affix & 0x1; + uint64_t len = (affix >> 1) & 0x7; + substring affix_name = { feature_name.begin, feature_name.end }; + if (affix_name.end > affix_name.begin + len) + { if (is_prefix) + affix_name.end = affix_name.begin + len; + else + affix_name.begin = affix_name.end - len; } - if ((affix_features[index] > 0) && (feature_name.end != feature_name.begin)) - { - features& affix_fs = ae->feature_space[affix_namespace]; - if (affix_fs.size() == 0) - ae->indices.push_back(affix_namespace); - uint64_t affix = affix_features[index]; - while (affix > 0) - { bool is_prefix = affix & 0x1; - uint64_t len = (affix >> 1) & 0x7; - substring affix_name = { feature_name.begin, feature_name.end }; - if (affix_name.end > affix_name.begin + len) - { if (is_prefix) - affix_name.end = affix_name.begin + len; - else - affix_name.begin = affix_name.end - len; - } - word_hash = p->hasher(affix_name,(uint64_t)channel_hash) * (affix_constant + (affix & 0xF) * quadratic_constant); - affix_fs.push_back(v, word_hash); - if (audit) - { v_array affix_v = v_init(); - if (index != ' ') affix_v.push_back(index); - affix_v.push_back(is_prefix ? '+' : '-'); - affix_v.push_back('0' + (char)len); - affix_v.push_back('='); - push_many(affix_v, affix_name.begin, affix_name.end - affix_name.begin); - affix_v.push_back('\0'); - affix_fs.space_names.push_back(audit_strings_ptr(new audit_strings("affix",affix_v.begin()))); - } - affix >>= 4; - } + word_hash = p->hasher(affix_name,(uint64_t)channel_hash) * (affix_constant + (affix & 0xF) * quadratic_constant); + affix_fs.push_back(v, word_hash); + if (audit) + { v_array affix_v = v_init(); + if (index != ' ') affix_v.push_back(index); + affix_v.push_back(is_prefix ? '+' : '-'); + affix_v.push_back('0' + (char)len); + affix_v.push_back('='); + push_many(affix_v, affix_name.begin, affix_name.end - affix_name.begin); + affix_v.push_back('\0'); + affix_fs.space_names.push_back(audit_strings_ptr(new audit_strings("affix",affix_v.begin()))); } - if (spelling_features[index]) - { - features& spell_fs = ae->feature_space[spelling_namespace]; - if (spell_fs.size() == 0) - ae->indices.push_back(spelling_namespace); - //v_array spelling; - spelling.erase(); - for (char*c = feature_name.begin; c!=feature_name.end; ++c) + affix >>= 4; + } + } + if (spelling_features[index]) + { features& spell_fs = ae->feature_space[spelling_namespace]; + if (spell_fs.size() == 0) + ae->indices.push_back(spelling_namespace); + //v_array spelling; + spelling.erase(); + for (char*c = feature_name.begin; c!=feature_name.end; ++c) { char d = 0; if ((*c >= '0') && (*c <= '9')) d = '0'; else if ((*c >= 'a') && (*c <= 'z')) d = 'a'; @@ -228,25 +226,25 @@ class TC_parser uint64_t hash = uniform_hash(feature_name.begin, feature_name.end-feature_name.begin, quadratic_constant); features* feats = map->get(feature_name, hash); if ((feats != nullptr) && (feats->values.size() > 0)) - { features& dict_fs = ae->feature_space[dictionary_namespace]; - if (dict_fs.size() == 0) - ae->indices.push_back(dictionary_namespace); - push_many(dict_fs.values, feats->values.begin(), feats->values.size()); - push_many(dict_fs.indicies, feats->indicies.begin(), feats->indicies.size()); - dict_fs.sum_feat_sq += feats->sum_feat_sq; - if (audit) - for (size_t i = 0; i < feats->indicies.size(); ++i) - { uint64_t id = feats->indicies[i]; - stringstream ss; - ss << index << '_'; - for (char* fc=feature_name.begin; fc!=feature_name.end; ++fc) ss << *fc; - ss << '=' << id; - dict_fs.space_names.push_back(audit_strings_ptr(new audit_strings("dictionary", ss.str()))); - } - } + { features& dict_fs = ae->feature_space[dictionary_namespace]; + if (dict_fs.size() == 0) + ae->indices.push_back(dictionary_namespace); + push_many(dict_fs.values, feats->values.begin(), feats->values.size()); + push_many(dict_fs.indicies, feats->indicies.begin(), feats->indicies.size()); + dict_fs.sum_feat_sq += feats->sum_feat_sq; + if (audit) + for (size_t i = 0; i < feats->indicies.size(); ++i) + { uint64_t id = feats->indicies[i]; + stringstream ss; + ss << index << '_'; + for (char* fc=feature_name.begin; fc!=feature_name.end; ++fc) ss << *fc; + ss << '=' << id; + dict_fs.space_names.push_back(audit_strings_ptr(new audit_strings("dictionary", ss.str()))); + } + } } } - } + } } inline void nameSpaceInfoValue() diff --git a/vowpalwabbit/parse_example.h b/vowpalwabbit/parse_example.h index ea2a26ae1af..814387bd8ec 100644 --- a/vowpalwabbit/parse_example.h +++ b/vowpalwabbit/parse_example.h @@ -11,9 +11,8 @@ license as described in the file LICENSE. //example processing typedef enum -{ - StringFeatures, - JsonFeatures +{ StringFeatures, + JsonFeatures } FeatureInputType; void substring_to_example(vw* all, example* ae, substring example); diff --git a/vowpalwabbit/parse_example_json.cc b/vowpalwabbit/parse_example_json.cc index 7c4ed3d2492..2e431f25f5a 100644 --- a/vowpalwabbit/parse_example_json.cc +++ b/vowpalwabbit/parse_example_json.cc @@ -16,235 +16,203 @@ using namespace std; using namespace rapidjson; void Namespace::AddFeature(feature_value v, feature_index i) -{ - // filter out 0-values - if (v == 0) - return; +{ // filter out 0-values + if (v == 0) + return; - ftrs->push_back(v, i); - feature_count++; + ftrs->push_back(v, i); + feature_count++; } void Namespace::AddFeature(vw* all, const char* str) -{ - ftrs->push_back( - 1., - VW::hash_feature(*all, str, namespace_hash)); - feature_count++; +{ ftrs->push_back( + 1., + VW::hash_feature(*all, str, namespace_hash)); + feature_count++; } Context::Context() -{ - namespace_path = v_init(); - current_state = &default_state; +{ namespace_path = v_init(); + current_state = &default_state; } -void Context::init(vw* pall) -{ - all = pall; - key = " "; - key_length = 1; - previous_state = nullptr; - label_object_state.init(pall); +void Context::init(vw* pall) +{ all = pall; + key = " "; + key_length = 1; + previous_state = nullptr; + label_object_state.init(pall); } Context::~Context() -{ - namespace_path.delete_v(); +{ namespace_path.delete_v(); } void Context::PushNamespace(const char* ns, BaseState* return_state) -{ - Namespace n; - n.feature_group = ns[0]; - n.namespace_hash = VW::hash_space(*all, ns); - n.ftrs = ex->feature_space + ns[0]; - n.feature_count = 0; - n.return_state = return_state; +{ Namespace n; + n.feature_group = ns[0]; + n.namespace_hash = VW::hash_space(*all, ns); + n.ftrs = ex->feature_space + ns[0]; + n.feature_count = 0; + n.return_state = return_state; - namespace_path.push_back(n); + namespace_path.push_back(n); } Namespace& Context::CurrentNamespace() { return *(namespace_path._end - 1); } bool Context::TransitionState(BaseState* next_state) -{ - if (next_state == nullptr) - return false; +{ if (next_state == nullptr) + return false; - previous_state = current_state; - current_state = next_state; + previous_state = current_state; + current_state = next_state; - return true; + return true; } BaseState::BaseState(const char* pname) : name(pname) { } BaseState* BaseState::Null(Context& ctx) -{ - ctx.error << "Unexpected token: null"; - return nullptr; +{ ctx.error << "Unexpected token: null"; + return nullptr; } BaseState* BaseState::Bool(Context& ctx, bool b) -{ - ctx.error << "Unexpected token: bool (" << (b ? "true":"false") << ")"; - return nullptr; +{ ctx.error << "Unexpected token: bool (" << (b ? "true":"false") << ")"; + return nullptr; } BaseState* BaseState::Float(Context& ctx, float v) -{ - ctx.error << "Unexpected token: float (" << v << ")"; - return nullptr; +{ ctx.error << "Unexpected token: float (" << v << ")"; + return nullptr; } BaseState* BaseState::Uint(Context& ctx, unsigned v) -{ - ctx.error << "Unexpected token: uint (" << v << ")"; - return nullptr; +{ ctx.error << "Unexpected token: uint (" << v << ")"; + return nullptr; } BaseState* BaseState::String(Context& ctx, const char* str, SizeType len, bool) -{ - ctx.error << "Unexpected token: string('" << str << "' len: " << len <<")"; - return nullptr; +{ ctx.error << "Unexpected token: string('" << str << "' len: " << len <<")"; + return nullptr; } - + BaseState* BaseState::StartObject(Context& ctx) -{ - ctx.error << "Unexpected token: {"; - return nullptr; +{ ctx.error << "Unexpected token: {"; + return nullptr; } BaseState* BaseState::Key(Context& ctx, const char* str, SizeType len, bool copy) -{ - ctx.error << "Unexpected token: key('" << str << "' len: " << len << ")"; - return nullptr; +{ ctx.error << "Unexpected token: key('" << str << "' len: " << len << ")"; + return nullptr; } BaseState* BaseState::EndObject(Context& ctx, SizeType) -{ - ctx.error << "Unexpected token: }"; - return nullptr; +{ ctx.error << "Unexpected token: }"; + return nullptr; } BaseState* BaseState::StartArray(Context& ctx) -{ - ctx.error << "Unexpected token: ["; - return nullptr; +{ ctx.error << "Unexpected token: ["; + return nullptr; } BaseState* BaseState::EndArray(Context& ctx, SizeType) -{ - ctx.error << "Unexpected token: ]"; - return nullptr; +{ ctx.error << "Unexpected token: ]"; + return nullptr; } // LabelObjectState LabelObjectState::LabelObjectState() : BaseState("LabelObject") -{ +{ } void LabelObjectState::init(vw* all) -{ - found = found_cb = false; +{ found = found_cb = false; - cb_label = {0.,0,0.,0.}; + cb_label = {0.,0,0.,0.}; } BaseState* LabelObjectState::StartObject(Context& ctx) -{ - ctx.all->p->lp.default_label(&ctx.ex->l); +{ ctx.all->p->lp.default_label(&ctx.ex->l); - // don't allow { { { } } } - if (ctx.previous_state == this) - { - ctx.error << "invalid label object. nested objected."; - return nullptr; - } + // don't allow { { { } } } + if (ctx.previous_state == this) + { ctx.error << "invalid label object. nested objected."; + return nullptr; + } - // keep previous state - return_state = ctx.previous_state; + // keep previous state + return_state = ctx.previous_state; - return this; + return this; } BaseState* LabelObjectState::Key(Context& ctx, const char* str, SizeType len, bool copy) -{ - ctx.key = str; - ctx.key_length = len; - return this; +{ ctx.key = str; + ctx.key_length = len; + return this; } // portability fun #ifndef _WIN32 -#define _stricmp strcasecmp +#define _stricmp strcasecmp #endif BaseState* LabelObjectState::Float(Context& ctx, float v) -{ - // simple - if (!_stricmp(ctx.key, "Label")) - { - ctx.ex->l.simple.label = v; - found = true; - } - else if (!_stricmp(ctx.key, "Initial")) - { - ctx.ex->l.simple.initial = v; - found = true; - } - else if (!_stricmp(ctx.key, "Weight")) - { - ctx.ex->l.simple.weight = v; - found = true; - } - // CB - else if (!_stricmp(ctx.key, "Action")) - { - cb_label.action = (uint32_t)v; - found_cb = true; - } - else if (!_stricmp(ctx.key, "Cost")) - { - cb_label.cost = v; - found_cb = true; - } - else if (!_stricmp(ctx.key, "Probability")) - { - cb_label.probability = v; - found_cb = true; - } - else - { - ctx.error << "Unsupported label property: '" << ctx.key << "' len: " << ctx.key_length; - return nullptr; - } - - return this; +{ // simple + if (!_stricmp(ctx.key, "Label")) + { ctx.ex->l.simple.label = v; + found = true; + } + else if (!_stricmp(ctx.key, "Initial")) + { ctx.ex->l.simple.initial = v; + found = true; + } + else if (!_stricmp(ctx.key, "Weight")) + { ctx.ex->l.simple.weight = v; + found = true; + } + // CB + else if (!_stricmp(ctx.key, "Action")) + { cb_label.action = (uint32_t)v; + found_cb = true; + } + else if (!_stricmp(ctx.key, "Cost")) + { cb_label.cost = v; + found_cb = true; + } + else if (!_stricmp(ctx.key, "Probability")) + { cb_label.probability = v; + found_cb = true; + } + else + { ctx.error << "Unsupported label property: '" << ctx.key << "' len: " << ctx.key_length; + return nullptr; + } + + return this; } BaseState* LabelObjectState::Uint(Context& ctx, unsigned v) { return Float(ctx, (float)v); } BaseState* LabelObjectState::EndObject(Context& ctx, SizeType) -{ - if (found_cb) - { - CB::label* ld = (CB::label*)&ctx.ex->l; - ld->costs.push_back(cb_label); +{ if (found_cb) + { CB::label* ld = (CB::label*)&ctx.ex->l; + ld->costs.push_back(cb_label); - found_cb = false; - cb_label = {0.,0,0.,0.}; - } - else if (found) - { - count_label(ctx.ex->l.simple.label); + found_cb = false; + cb_label = {0.,0,0.,0.}; + } + else if (found) + { count_label(ctx.ex->l.simple.label); - found = false; - } + found = false; + } - return return_state; + return return_state; } LabelSinglePropertyState::LabelSinglePropertyState() : BaseState("LabelSingleProperty") @@ -252,37 +220,34 @@ LabelSinglePropertyState::LabelSinglePropertyState() : BaseState("LabelSinglePro // forward _label BaseState* LabelSinglePropertyState::Float(Context& ctx, float v) -{ - // skip "_label_" - ctx.key += 7; - ctx.key_length -= 7; +{ // skip "_label_" + ctx.key += 7; + ctx.key_length -= 7; - if (ctx.label_object_state.Float(ctx, v) == nullptr) - return nullptr; + if (ctx.label_object_state.Float(ctx, v) == nullptr) + return nullptr; - return ctx.previous_state; + return ctx.previous_state; } BaseState* LabelSinglePropertyState::Uint(Context& ctx, unsigned v) -{ - // skip "_label_" - ctx.key += 7; - ctx.key_length -= 7; +{ // skip "_label_" + ctx.key += 7; + ctx.key_length -= 7; - if (ctx.label_object_state.Uint(ctx, v) == nullptr) - return nullptr; + if (ctx.label_object_state.Uint(ctx, v) == nullptr) + return nullptr; - return ctx.previous_state; + return ctx.previous_state; } // LabelIndexState LabelIndexState::LabelIndexState() : BaseState("LabelIndex"), index(-1) -{ } +{ } BaseState* LabelIndexState::Uint(Context& ctx, unsigned int v) -{ - index = v; - return ctx.previous_state; +{ index = v; + return ctx.previous_state; } // LabelState @@ -292,67 +257,61 @@ LabelState::LabelState() : BaseState("Label") BaseState* LabelState::StartObject(Context& ctx) { return ctx.label_object_state.StartObject(ctx); } BaseState* LabelState::String(Context& ctx, const char* str, SizeType len, bool copy) -{ - // only to be used with copy=false - assert(!copy); +{ // only to be used with copy=false + assert(!copy); - VW::parse_example_label(*ctx.all, *ctx.ex, str); - return ctx.previous_state; + VW::parse_example_label(*ctx.all, *ctx.ex, str); + return ctx.previous_state; } BaseState* LabelState::Float(Context& ctx, float v) -{ - // TODO: once we introduce label types, check here - ctx.ex->l.simple.label = v; - return ctx.previous_state; +{ // TODO: once we introduce label types, check here + ctx.ex->l.simple.label = v; + return ctx.previous_state; } BaseState* LabelState::Uint(Context& ctx, unsigned v) -{ - // TODO: once we introduce label types, check here - ctx.ex->l.simple.label = (float)v; - return ctx.previous_state; +{ // TODO: once we introduce label types, check here + ctx.ex->l.simple.label = (float)v; + return ctx.previous_state; } -// "_text":"a b c" +// "_text":"a b c" TextState::TextState() : BaseState("text") { } BaseState* TextState::String(Context& ctx, const char* str, SizeType length, bool copy) -{ - // only to be used with copy=false - assert(!copy); - - auto& ns = ctx.CurrentNamespace(); - - // split into individual features - const char* start = str; - const char* end = str + length; - for (char* p = (char*)str;p != end;p++) - { - switch (*p) - { - // split on space and tab - case ' ': - case '\t': - *p = '\0'; - if (p - start > 0) - ns.AddFeature(ctx.all, start); - - start = p + 1; - break; - // escape chars - case ':': - case '|': - *p = '_'; - break; - } - } - - if (start < end) - ns.AddFeature(ctx.all, start); - - return ctx.previous_state; +{ // only to be used with copy=false + assert(!copy); + + auto& ns = ctx.CurrentNamespace(); + + // split into individual features + const char* start = str; + const char* end = str + length; + for (char* p = (char*)str; p != end; p++) + { switch (*p) + { // split on space and tab + case ' ': + case '\t': + *p = '\0'; + if (p - start > 0) + ns.AddFeature(ctx.all, start); + + start = p + 1; + break; + // escape chars + case ':': + case '|': + *p = '_'; + break; + } + } + + if (start < end) + ns.AddFeature(ctx.all, start); + + return ctx.previous_state; } // MultiState @@ -360,44 +319,40 @@ MultiState::MultiState() : BaseState("Multi") { } BaseState* MultiState::StartArray(Context& ctx) -{ - // mark shared example - // TODO: how to check if we're in CB mode (--cb?) - // if (ctx.all->p->lp == CB::cb_label) // not sure how to compare - { - CB::label* ld = (CB::label*)&ctx.ex->l; - CB::cb_class f; +{ // mark shared example + // TODO: how to check if we're in CB mode (--cb?) + // if (ctx.all->p->lp == CB::cb_label) // not sure how to compare + { CB::label* ld = (CB::label*)&ctx.ex->l; + CB::cb_class f; - f.partial_prediction = 0.; - f.action = (uint32_t)uniform_hash("shared", 6, 0); - f.cost = FLT_MAX; - f.probability = -1.f; + f.partial_prediction = 0.; + f.action = (uint32_t)uniform_hash("shared", 6, 0); + f.cost = FLT_MAX; + f.probability = -1.f; - ld->costs.push_back(f); - } + ld->costs.push_back(f); + } - return this; + return this; } BaseState* MultiState::StartObject(Context& ctx) -{ - // allocate new example - ctx.ex = &(*ctx.example_factory)(ctx.example_factory_context); - ctx.all->p->lp.default_label(&ctx.ex->l); - ctx.examples->push_back(ctx.ex); +{ // allocate new example + ctx.ex = &(*ctx.example_factory)(ctx.example_factory_context); + ctx.all->p->lp.default_label(&ctx.ex->l); + ctx.examples->push_back(ctx.ex); - // setup default namespace - ctx.PushNamespace(" ", this); + // setup default namespace + ctx.PushNamespace(" ", this); - return &ctx.default_state; + return &ctx.default_state; } BaseState* MultiState::EndArray(Context& ctx, SizeType) -{ - // return to shared example - ctx.ex = (*ctx.examples)[0]; +{ // return to shared example + ctx.ex = (*ctx.examples)[0]; - return &ctx.default_state; + return &ctx.default_state; } // ArrayState @@ -405,25 +360,22 @@ ArrayState::ArrayState() : BaseState("Array"), return_state(nullptr) { } BaseState* ArrayState::StartArray(Context &ctx) -{ - if (ctx.previous_state == this) - { - ctx.error << "Nested arrays are not supported"; - return nullptr; - } +{ if (ctx.previous_state == this) + { ctx.error << "Nested arrays are not supported"; + return nullptr; + } - array_hash = ctx.CurrentNamespace().namespace_hash; - return_state = ctx.previous_state; + array_hash = ctx.CurrentNamespace().namespace_hash; + return_state = ctx.previous_state; - return this; + return this; } BaseState* ArrayState::Float(Context& ctx, float f) -{ - ctx.CurrentNamespace().AddFeature(f, array_hash); - array_hash++; +{ ctx.CurrentNamespace().AddFeature(f, array_hash); + array_hash++; - return this; + return this; } BaseState* ArrayState::Uint(Context& ctx, unsigned f) { return Float(ctx, (float)f); } @@ -438,230 +390,206 @@ BaseState* IgnoreState::Uint(Context& ctx, unsigned) { return ctx.previous_state // DefaultState BaseState* DefaultState::Ignore(Context& ctx, SizeType length) -{ - // fast ignore - // skip key + \0 + " - char* head = ctx.stream->src_ + length + 2; - - if (*head != ':') - { - ctx.error << "Expected ':' found '" << *head << "'"; - return nullptr; - } - head++; - - // scan for ,} - // support { { ... } } - int depth = 0, sq_depth =0 ; - bool stop = false; - while (!stop) - { - switch (*head) - { - case '"': - { - // skip strings - bool stopInner = false; - while (!stopInner) - { - head++; - switch (*head) - { - case '\\': - head++; - break; - case '"': - stopInner = true; - break; - } - } - break; - } - case '{': - depth++; - break; - case '}': - if (depth == 0 && sq_depth == 0) - stop = true; - else - depth--; - break; - case '[': - sq_depth++; - break; - case ']': - if (depth == 0 && sq_depth == 0) - stop = true; - else - sq_depth--; - break; - case ',': - if (depth == 0 && sq_depth == 0) - stop = true; - break; - } - head++; - } - - // skip key + \0 + ": - char* value = ctx.stream->src_ + length + 3; - *value = '0'; - value++; - memset(value, ' ', head - value - 1); - - return &ctx.ignore_state; +{ // fast ignore + // skip key + \0 + " + char* head = ctx.stream->src_ + length + 2; + + if (*head != ':') + { ctx.error << "Expected ':' found '" << *head << "'"; + return nullptr; + } + head++; + + // scan for ,} + // support { { ... } } + int depth = 0, sq_depth =0 ; + bool stop = false; + while (!stop) + { switch (*head) + { case '"': + { // skip strings + bool stopInner = false; + while (!stopInner) + { head++; + switch (*head) + { case '\\': + head++; + break; + case '"': + stopInner = true; + break; + } + } + break; + } + case '{': + depth++; + break; + case '}': + if (depth == 0 && sq_depth == 0) + stop = true; + else + depth--; + break; + case '[': + sq_depth++; + break; + case ']': + if (depth == 0 && sq_depth == 0) + stop = true; + else + sq_depth--; + break; + case ',': + if (depth == 0 && sq_depth == 0) + stop = true; + break; + } + head++; + } + + // skip key + \0 + ": + char* value = ctx.stream->src_ + length + 3; + *value = '0'; + value++; + memset(value, ' ', head - value - 1); + + return &ctx.ignore_state; } void DefaultState::InsertNamespace(Context& ctx) -{ - auto& ns = ctx.CurrentNamespace(); - if (ns.feature_count > 0) - { - auto feature_group = ns.feature_group; - // avoid duplicate insertion - for (unsigned char ns : ctx.ex->indices) - if (ns == feature_group) - return; +{ auto& ns = ctx.CurrentNamespace(); + if (ns.feature_count > 0) + { auto feature_group = ns.feature_group; + // avoid duplicate insertion + for (unsigned char ns : ctx.ex->indices) + if (ns == feature_group) + return; - ctx.ex->indices.push_back(feature_group); - } + ctx.ex->indices.push_back(feature_group); + } } DefaultState::DefaultState() : BaseState("Default") { } BaseState* DefaultState::Key(Context& ctx, const char* str, SizeType length, bool copy) -{ - // only to be used with copy=false - assert(!copy); - - ctx.key = str; - ctx.key_length = length; - - if (length > 0 && str[0] == '_') - { - // match _label* - if (ctx.key_length >= 6 && !strncmp(str, "_label", 6)) - { - if (ctx.key_length >= 7 && ctx.key[6] == '_') - return &ctx.label_single_property_state; - else if (ctx.key_length == 6) - return &ctx.label_state; - else if (ctx.key_length == 11 && !_stricmp(str, "_labelIndex")) - return &ctx.label_index_state; - else - { - ctx.error << "Unsupported key '" << str << "' len: " << length; - return nullptr; - } - } - - if (ctx.key_length == 5 && !strcmp(str, "_text")) - return &ctx.text_state; - - // TODO: _multi in _multi... - if (ctx.key_length == 6 && !strcmp(str, "_multi")) - return &ctx.multi_state; - - return Ignore(ctx, length); - } - - return this; +{ // only to be used with copy=false + assert(!copy); + + ctx.key = str; + ctx.key_length = length; + + if (length > 0 && str[0] == '_') + { // match _label* + if (ctx.key_length >= 6 && !strncmp(str, "_label", 6)) + { if (ctx.key_length >= 7 && ctx.key[6] == '_') + return &ctx.label_single_property_state; + else if (ctx.key_length == 6) + return &ctx.label_state; + else if (ctx.key_length == 11 && !_stricmp(str, "_labelIndex")) + return &ctx.label_index_state; + else + { ctx.error << "Unsupported key '" << str << "' len: " << length; + return nullptr; + } + } + + if (ctx.key_length == 5 && !strcmp(str, "_text")) + return &ctx.text_state; + + // TODO: _multi in _multi... + if (ctx.key_length == 6 && !strcmp(str, "_multi")) + return &ctx.multi_state; + + return Ignore(ctx, length); + } + + return this; } BaseState* DefaultState::String(Context& ctx, const char* str, SizeType length, bool copy) -{ - assert(!copy); +{ assert(!copy); - // string escape - const char* end = str + length; - for (char* p = (char*)str;p != end;p++) - { - switch (*p) - { - case ' ': - case '\t': - case '|': - case ':': - *p = '_'; - } - } + // string escape + const char* end = str + length; + for (char* p = (char*)str; p != end; p++) + { switch (*p) + { case ' ': + case '\t': + case '|': + case ':': + *p = '_'; + } + } - char* prepend = (char*)str - ctx.key_length; - memmove(prepend, ctx.key, ctx.key_length); + char* prepend = (char*)str - ctx.key_length; + memmove(prepend, ctx.key, ctx.key_length); - ctx.CurrentNamespace().AddFeature(ctx.all, prepend); + ctx.CurrentNamespace().AddFeature(ctx.all, prepend); - return this; + return this; } BaseState* DefaultState::Bool(Context& ctx, bool b) -{ - if (b) - ctx.CurrentNamespace().AddFeature(ctx.all, ctx.key); +{ if (b) + ctx.CurrentNamespace().AddFeature(ctx.all, ctx.key); - return this; + return this; } BaseState* DefaultState::StartObject(Context& ctx) -{ - ctx.PushNamespace(ctx.key, this); - return this; +{ ctx.PushNamespace(ctx.key, this); + return this; } BaseState* DefaultState::EndObject(Context& ctx, SizeType memberCount) -{ - InsertNamespace(ctx); - BaseState* return_state = ctx.namespace_path.pop().return_state; - - if (ctx.namespace_path.empty()) - { - int label_index = ctx.label_index_state.index; - // we're at the end of the example - if (label_index >= 0) - { - // skip shared example - label_index++; - if (label_index >= (int)ctx.examples->size()) - { - ctx.error << "_label_index out of bounds: " << (label_index - 1) << " examples available: " << ctx.examples->size() - 1; - return nullptr; - } - - // apply labelIndex - ctx.ex = (*ctx.examples)[label_index]; - } - - // inject label - ctx.label_object_state.EndObject(ctx, memberCount); - } - - return return_state; +{ InsertNamespace(ctx); + BaseState* return_state = ctx.namespace_path.pop().return_state; + + if (ctx.namespace_path.empty()) + { int label_index = ctx.label_index_state.index; + // we're at the end of the example + if (label_index >= 0) + { // skip shared example + label_index++; + if (label_index >= (int)ctx.examples->size()) + { ctx.error << "_label_index out of bounds: " << (label_index - 1) << " examples available: " << ctx.examples->size() - 1; + return nullptr; + } + + // apply labelIndex + ctx.ex = (*ctx.examples)[label_index]; + } + + // inject label + ctx.label_object_state.EndObject(ctx, memberCount); + } + + return return_state; } BaseState* DefaultState::Float(Context& ctx, float f) -{ - auto& ns = ctx.CurrentNamespace(); - ns.AddFeature(f, VW::hash_feature(*ctx.all, ctx.key, ns.namespace_hash)); +{ auto& ns = ctx.CurrentNamespace(); + ns.AddFeature(f, VW::hash_feature(*ctx.all, ctx.key, ns.namespace_hash)); - return this; + return this; } BaseState* DefaultState::Uint(Context& ctx, unsigned f) { return Float(ctx, (float)f); } BaseState* DefaultState::StartArray(Context& ctx) { return ctx.array_state.StartArray(ctx); } -// VWReaderHandler +// VWReaderHandler void VWReaderHandler::init(vw* all, v_array* examples, InsituStringStream* stream, VW::example_factory_t example_factory, void* example_factory_context) -{ - ctx.init(all); - ctx.examples = examples; - ctx.ex = (*examples)[0]; - all->p->lp.default_label(&ctx.ex->l); +{ ctx.init(all); + ctx.examples = examples; + ctx.ex = (*examples)[0]; + all->p->lp.default_label(&ctx.ex->l); - ctx.stream = stream; - ctx.example_factory = example_factory; - ctx.example_factory_context = example_factory_context; + ctx.stream = stream; + ctx.example_factory = example_factory; + ctx.example_factory_context = example_factory_context; } // virtual dispatch to current state @@ -690,22 +618,21 @@ BaseState* VWReaderHandler::current_state() { return ctx.current_state; } namespace VW { - void read_line_json(vw& all, v_array& examples, char* line, example_factory_t example_factory, void* ex_factory_context) - { - // destructive parsing - InsituStringStream ss(line); - - VWReaderHandler& handler = all.p->jsonp->handler; - handler.init(&all, &examples, &ss, example_factory, ex_factory_context); - - ParseResult result = all.p->jsonp->reader.Parse(ss, handler); - if (!result.IsError()) - return; - - BaseState* current_state = handler.current_state(); - - THROW("JSON parser error at " << result.Offset() << ": " << GetParseError_En(result.Code()) << ". " - "Handler: " << handler.error().str() << - "State: " << (current_state ? current_state->name : "null")); - } +void read_line_json(vw& all, v_array& examples, char* line, example_factory_t example_factory, void* ex_factory_context) +{ // destructive parsing + InsituStringStream ss(line); + + VWReaderHandler& handler = all.p->jsonp->handler; + handler.init(&all, &examples, &ss, example_factory, ex_factory_context); + + ParseResult result = all.p->jsonp->reader.Parse(ss, handler); + if (!result.IsError()) + return; + + BaseState* current_state = handler.current_state(); + + THROW("JSON parser error at " << result.Offset() << ": " << GetParseError_En(result.Code()) << ". " + "Handler: " << handler.error().str() << + "State: " << (current_state ? current_state->name : "null")); +} } diff --git a/vowpalwabbit/parse_example_json.h b/vowpalwabbit/parse_example_json.h index cf9b68d1a57..16c2ea830a5 100644 --- a/vowpalwabbit/parse_example_json.h +++ b/vowpalwabbit/parse_example_json.h @@ -20,224 +20,212 @@ struct BaseState; struct Context; struct Namespace -{ - char feature_group; - feature_index namespace_hash; - features* ftrs; - size_t feature_count; - BaseState* return_state; +{ char feature_group; + feature_index namespace_hash; + features* ftrs; + size_t feature_count; + BaseState* return_state; - void AddFeature(feature_value v, feature_index i); + void AddFeature(feature_value v, feature_index i); - void AddFeature(vw* all, const char* str); + void AddFeature(vw* all, const char* str); }; struct BaseState -{ - const char* name; - - BaseState(const char* pname); - - virtual BaseState* Null(Context& ctx); - virtual BaseState* Bool(Context& ctx, bool b); - virtual BaseState* Float(Context& ctx, float v); - virtual BaseState* Uint(Context& ctx, unsigned v); - virtual BaseState* String(Context& ctx, const char* str, rapidjson::SizeType len, bool); - virtual BaseState* StartObject(Context& ctx); - virtual BaseState* Key(Context& ctx, const char* str, rapidjson::SizeType len, bool copy); - virtual BaseState* EndObject(Context& ctx, rapidjson::SizeType); - virtual BaseState* StartArray(Context& ctx); - virtual BaseState* EndArray(Context& ctx, rapidjson::SizeType); +{ const char* name; + + BaseState(const char* pname); + + virtual BaseState* Null(Context& ctx); + virtual BaseState* Bool(Context& ctx, bool b); + virtual BaseState* Float(Context& ctx, float v); + virtual BaseState* Uint(Context& ctx, unsigned v); + virtual BaseState* String(Context& ctx, const char* str, rapidjson::SizeType len, bool); + virtual BaseState* StartObject(Context& ctx); + virtual BaseState* Key(Context& ctx, const char* str, rapidjson::SizeType len, bool copy); + virtual BaseState* EndObject(Context& ctx, rapidjson::SizeType); + virtual BaseState* StartArray(Context& ctx); + virtual BaseState* EndArray(Context& ctx, rapidjson::SizeType); }; class LabelObjectState : public BaseState { private: - BaseState* return_state; - CB::cb_class cb_label; - bool found; - bool found_cb; + BaseState* return_state; + CB::cb_class cb_label; + bool found; + bool found_cb; public: - LabelObjectState(); + LabelObjectState(); - void init(vw* all); + void init(vw* all); - BaseState* StartObject(Context& ctx); - BaseState* Key(Context& ctx, const char* str, rapidjson::SizeType len, bool copy); - BaseState* Float(Context& ctx, float v); - BaseState* Uint(Context& ctx, unsigned v); - BaseState* EndObject(Context& ctx, rapidjson::SizeType); + BaseState* StartObject(Context& ctx); + BaseState* Key(Context& ctx, const char* str, rapidjson::SizeType len, bool copy); + BaseState* Float(Context& ctx, float v); + BaseState* Uint(Context& ctx, unsigned v); + BaseState* EndObject(Context& ctx, rapidjson::SizeType); }; // "_label_*": struct LabelSinglePropertyState : BaseState -{ - LabelSinglePropertyState(); +{ LabelSinglePropertyState(); - // forward _label - BaseState* Float(Context& ctx, float v); + // forward _label + BaseState* Float(Context& ctx, float v); - BaseState* Uint(Context& ctx, unsigned v); + BaseState* Uint(Context& ctx, unsigned v); }; struct LabelIndexState : BaseState -{ - int index; +{ int index; - LabelIndexState(); + LabelIndexState(); - BaseState* Uint(Context& ctx, unsigned int v); + BaseState* Uint(Context& ctx, unsigned int v); }; // "_label":"1" // Note: doesn't support labelIndex struct LabelState : BaseState -{ - LabelState(); +{ LabelState(); - BaseState* StartObject(Context& ctx); - BaseState* String(Context& ctx, const char* str, rapidjson::SizeType len, bool copy); - BaseState* Float(Context& ctx, float v); - BaseState* Uint(Context& ctx, unsigned v); + BaseState* StartObject(Context& ctx); + BaseState* String(Context& ctx, const char* str, rapidjson::SizeType len, bool copy); + BaseState* Float(Context& ctx, float v); + BaseState* Uint(Context& ctx, unsigned v); }; struct TextState : BaseState -{ - TextState(); +{ TextState(); - BaseState* String(Context& ctx, const char* str, rapidjson::SizeType length, bool copy); + BaseState* String(Context& ctx, const char* str, rapidjson::SizeType length, bool copy); }; struct MultiState : BaseState -{ - MultiState(); +{ MultiState(); - BaseState* StartArray(Context& ctx); - BaseState* StartObject(Context& ctx); - BaseState* EndArray(Context& ctx, rapidjson::SizeType); + BaseState* StartArray(Context& ctx); + BaseState* StartObject(Context& ctx); + BaseState* EndArray(Context& ctx, rapidjson::SizeType); }; // "...":[Numbers only] class ArrayState : public BaseState -{ - feature_index array_hash; - BaseState* return_state; +{ feature_index array_hash; + BaseState* return_state; public: - ArrayState(); + ArrayState(); - BaseState* StartArray(Context &ctx); - BaseState* Float(Context& ctx, float f); - BaseState* Uint(Context& ctx, unsigned f); - BaseState* EndArray(Context& ctx, rapidjson::SizeType elementCount); + BaseState* StartArray(Context &ctx); + BaseState* Float(Context& ctx, float f); + BaseState* Uint(Context& ctx, unsigned f); + BaseState* EndArray(Context& ctx, rapidjson::SizeType elementCount); }; // only 0 is valid as DefaultState::Ignore injected that into the source stream struct IgnoreState : BaseState -{ - IgnoreState(); +{ IgnoreState(); - BaseState* Uint(Context& ctx, unsigned); + BaseState* Uint(Context& ctx, unsigned); }; class DefaultState : public BaseState { private: - BaseState* Ignore(Context& ctx, rapidjson::SizeType length); + BaseState* Ignore(Context& ctx, rapidjson::SizeType length); - void InsertNamespace(Context& ctx); + void InsertNamespace(Context& ctx); public: - DefaultState(); - - BaseState* Key(Context& ctx, const char* str, rapidjson::SizeType length, bool copy); - BaseState* String(Context& ctx, const char* str, rapidjson::SizeType length, bool copy); - BaseState* Bool(Context& ctx, bool b); - BaseState* StartObject(Context& ctx); - BaseState* EndObject(Context& ctx, rapidjson::SizeType memberCount); - BaseState* Float(Context& ctx, float f); - BaseState* Uint(Context& ctx, unsigned f); - BaseState* StartArray(Context& ctx); + DefaultState(); + + BaseState* Key(Context& ctx, const char* str, rapidjson::SizeType length, bool copy); + BaseState* String(Context& ctx, const char* str, rapidjson::SizeType length, bool copy); + BaseState* Bool(Context& ctx, bool b); + BaseState* StartObject(Context& ctx); + BaseState* EndObject(Context& ctx, rapidjson::SizeType memberCount); + BaseState* Float(Context& ctx, float f); + BaseState* Uint(Context& ctx, unsigned f); + BaseState* StartArray(Context& ctx); }; struct Context -{ - vw* all; - std::stringstream error; +{ vw* all; + std::stringstream error; - // last "": encountered - const char* key; - rapidjson::SizeType key_length; + // last "": encountered + const char* key; + rapidjson::SizeType key_length; - BaseState* current_state; - BaseState* previous_state; + BaseState* current_state; + BaseState* previous_state; - // the path of namespaces - v_array namespace_path; + // the path of namespaces + v_array namespace_path; - v_array* examples; - example* ex; - rapidjson::InsituStringStream* stream; + v_array* examples; + example* ex; + rapidjson::InsituStringStream* stream; - VW::example_factory_t example_factory; - void* example_factory_context; + VW::example_factory_t example_factory; + void* example_factory_context; - // states - DefaultState default_state; - LabelState label_state; - LabelObjectState label_object_state; - LabelSinglePropertyState label_single_property_state; - LabelIndexState label_index_state; - TextState text_state; - MultiState multi_state; - IgnoreState ignore_state; - ArrayState array_state; + // states + DefaultState default_state; + LabelState label_state; + LabelObjectState label_object_state; + LabelSinglePropertyState label_single_property_state; + LabelIndexState label_index_state; + TextState text_state; + MultiState multi_state; + IgnoreState ignore_state; + ArrayState array_state; - Context(); - ~Context(); - void init(vw* all); + Context(); + ~Context(); + void init(vw* all); - void PushNamespace(const char* ns, BaseState* return_state); + void PushNamespace(const char* ns, BaseState* return_state); - Namespace& CurrentNamespace(); + Namespace& CurrentNamespace(); - bool TransitionState(BaseState* next_state); + bool TransitionState(BaseState* next_state); }; struct VWReaderHandler : public rapidjson::BaseReaderHandler, VWReaderHandler> -{ - Context ctx; +{ Context ctx; - void init(vw* all, v_array* examples, rapidjson::InsituStringStream* stream, VW::example_factory_t example_factory, void* example_factory_context); + void init(vw* all, v_array* examples, rapidjson::InsituStringStream* stream, VW::example_factory_t example_factory, void* example_factory_context); - // virtual dispatch to current state - bool Bool(bool v); - bool Int(int v); - bool Uint(unsigned v); - bool Int64(int64_t v); - bool Uint64(uint64_t v); - bool Double(double v); - bool String(const Ch* str, rapidjson::SizeType len, bool copy); - bool StartObject(); - bool Key(const Ch* str, rapidjson::SizeType len, bool copy); - bool EndObject(rapidjson::SizeType count); - bool StartArray(); - bool EndArray(rapidjson::SizeType count); + // virtual dispatch to current state + bool Bool(bool v); + bool Int(int v); + bool Uint(unsigned v); + bool Int64(int64_t v); + bool Uint64(uint64_t v); + bool Double(double v); + bool String(const Ch* str, rapidjson::SizeType len, bool copy); + bool StartObject(); + bool Key(const Ch* str, rapidjson::SizeType len, bool copy); + bool EndObject(rapidjson::SizeType count); + bool StartArray(); + bool EndArray(rapidjson::SizeType count); - bool Null(); - bool Default(); + bool Null(); + bool Default(); - // alternative to above if we want to re-use the VW float parser... - bool RawNumber(const Ch* str, rapidjson::SizeType length, bool copy); + // alternative to above if we want to re-use the VW float parser... + bool RawNumber(const Ch* str, rapidjson::SizeType length, bool copy); - std::stringstream& error(); + std::stringstream& error(); - BaseState* current_state(); + BaseState* current_state(); }; struct json_parser -{ - rapidjson::Reader reader; - VWReaderHandler handler; +{ rapidjson::Reader reader; + VWReaderHandler handler; }; diff --git a/vowpalwabbit/parse_primitives.h b/vowpalwabbit/parse_primitives.h index f600026b886..35e93cc88c7 100644 --- a/vowpalwabbit/parse_primitives.h +++ b/vowpalwabbit/parse_primitives.h @@ -45,7 +45,8 @@ inline void print_substring(substring s) // can't type as it forces C++/CLI part to include rapidjson, which leads to name clashes... struct example; -namespace VW { +namespace VW +{ typedef example& (*example_factory_t)(void*); } diff --git a/vowpalwabbit/parse_regressor.cc b/vowpalwabbit/parse_regressor.cc index 14d674c10a9..9f6c609bd19 100644 --- a/vowpalwabbit/parse_regressor.cc +++ b/vowpalwabbit/parse_regressor.cc @@ -27,21 +27,21 @@ using namespace std; struct initial_weight -{private: - weight _initial; +{ +private: + weight _initial; public: - initial_weight(weight initial) : _initial(initial){} - void operator()(weight_parameters::iterator& iter, uint64_t /*index*/) - { - *iter = _initial; - } + initial_weight(weight initial) : _initial(initial) {} + void operator()(weight_parameters::iterator& iter, uint64_t /*index*/) + { *iter = _initial; + } }; void random_positive(weight_parameters::iterator& iter, uint64_t ind) -{*iter = (float)(0.1 * merand48(ind)); +{ *iter = (float)(0.1 * merand48(ind)); } void random_weights(weight_parameters::iterator& iter, uint64_t ind) -{*iter = (float)(merand48(ind) - 0.5); +{ *iter = (float)(merand48(ind) - 0.5); } void initialize_regressor(vw& all) { // Regressor is already initialized. @@ -49,21 +49,20 @@ void initialize_regressor(vw& all) return; size_t length = ((size_t)1) << all.num_bits; try - { new(&all.weights) weight_parameters(length, all.weights.stride_shift()); } + { new(&all.weights) weight_parameters(length, all.weights.stride_shift()); } catch (VW::vw_exception anExc) - { THROW(" Failed to allocate weight array with " << all.num_bits << " bits: try decreasing -b "); - } + { THROW(" Failed to allocate weight array with " << all.num_bits << " bits: try decreasing -b "); + } if (!all.weights.not_null()) - { THROW(" Failed to allocate weight array with " << all.num_bits << " bits: try decreasing -b "); } + { THROW(" Failed to allocate weight array with " << all.num_bits << " bits: try decreasing -b "); } else if (all.initial_weight != 0.) - { - initial_weight init(all.initial_weight); - all.weights.set_default(init); + { initial_weight init(all.initial_weight); + all.weights.set_default(init); } else if (all.random_positive_weights) - all.weights.set_default(); + all.weights.set_default(); else if (all.random_weights) - all.weights.set_default(); + all.weights.set_default(); } const size_t default_buf_size = 512; @@ -100,8 +99,7 @@ inline void safe_memcpy(char *& __dest, size_t& __dest_size, const void *__src, } void save_load_header(vw& all, io_buf& model_file, bool read, bool text) -{ - char* buff2 = (char*) malloc(default_buf_size); +{ char* buff2 = (char*) malloc(default_buf_size); size_t buf2_size = default_buf_size; try @@ -112,11 +110,11 @@ void save_load_header(vw& all, io_buf& model_file, bool read, bool text) stringstream msg; msg << "Version " << version.to_string() << "\n"; memcpy(buff2, version.to_string().c_str(), min(v_length, buf2_size)); - if (read) - { v_length = (uint32_t)buf2_size; - if (v_length > 0) // all.model_file_ver = buff2; uses scanf which doesn't accept a maximum buffer length, but just expects valid zero terminated string - buff2[min(v_length, default_buf_size) - 1] = '\0'; - } + if (read) + { v_length = (uint32_t)buf2_size; + if (v_length > 0) // all.model_file_ver = buff2; uses scanf which doesn't accept a maximum buffer length, but just expects valid zero terminated string + buff2[min(v_length, default_buf_size) - 1] = '\0'; + } bytes_read_write += bin_text_read_write(model_file, buff2, v_length, "", read, msg, text); all.model_file_ver = buff2; //stored in all to check save_resume fix in gd @@ -126,46 +124,46 @@ void save_load_header(vw& all, io_buf& model_file, bool read, bool text) model_file.verify_hash = true; if (all.model_file_ver >= VERSION_FILE_WITH_HEADER_ID) - { v_length = (uint32_t)all.id.length() + 1; + { v_length = (uint32_t)all.id.length() + 1; - msg << "Id" << all.id << "\n"; - memcpy(buff2, all.id.c_str(), min(v_length, default_buf_size)); - if (read) - v_length = default_buf_size; - bytes_read_write += bin_text_read_write(model_file, buff2, v_length, - "", read, msg, text); - all.id = buff2; - - if (read && find(all.args.begin(), all.args.end(), "--id") == all.args.end() && !all.id.empty()) - { all.args.push_back("--id"); - all.args.push_back(all.id); - } + msg << "Id" << all.id << "\n"; + memcpy(buff2, all.id.c_str(), min(v_length, default_buf_size)); + if (read) + v_length = default_buf_size; + bytes_read_write += bin_text_read_write(model_file, buff2, v_length, + "", read, msg, text); + all.id = buff2; + + if (read && find(all.args.begin(), all.args.end(), "--id") == all.args.end() && !all.id.empty()) + { all.args.push_back("--id"); + all.args.push_back(all.id); } + } char model = 'm'; bytes_read_write += bin_text_read_write_fixed_validated(model_file, &model, 1, - "file is not a model file", read, - msg, text); + "file is not a model file", read, + msg, text); msg << "Min label:" << all.sd->min_label << "\n"; bytes_read_write += bin_text_read_write_fixed_validated(model_file, (char*)&all.sd->min_label, sizeof(all.sd->min_label), - "", read, msg, text); + "", read, msg, text); msg << "Max label:" << all.sd->max_label << "\n"; bytes_read_write += bin_text_read_write_fixed_validated(model_file, (char*)&all.sd->max_label, sizeof(all.sd->max_label), - "", read, msg, text); + "", read, msg, text); msg << "bits:" << all.num_bits << "\n"; uint32_t local_num_bits = all.num_bits; bytes_read_write += bin_text_read_write_fixed_validated(model_file, (char *)&local_num_bits, sizeof(local_num_bits), - "", read, msg, text); + "", read, msg, text); if (read && find(all.args.begin(), all.args.end(), "--bit_precision") == all.args.end()) - { all.args.push_back("--bit_precision"); - all.args.push_back(boost::lexical_cast(local_num_bits)); - } + { all.args.push_back("--bit_precision"); + all.args.push_back(boost::lexical_cast(local_num_bits)); + } VW::validate_default_bits(all, local_num_bits); @@ -180,47 +178,47 @@ void save_load_header(vw& all, io_buf& model_file, bool read, bool text) msg << pair_len << " pairs: "; bytes_read_write += bin_text_read_write_fixed_validated(model_file, (char *)&pair_len, sizeof(pair_len), - "", read, msg, text); + "", read, msg, text); // TODO: validate pairs? for (size_t i = 0; i < pair_len; i++) - { char pair[3] = { 0, 0, 0 }; + { char pair[3] = { 0, 0, 0 }; - if (!read) - { memcpy(pair, all.pairs[i].c_str(), 2); - msg << all.pairs[i] << " "; - } + if (!read) + { memcpy(pair, all.pairs[i].c_str(), 2); + msg << all.pairs[i] << " "; + } - bytes_read_write += bin_text_read_write_fixed_validated(model_file, pair, 2, - "", read, msg, text); - if (read) - { string temp(pair); - if (count(all.pairs.begin(), all.pairs.end(), temp) == 0) - all.pairs.push_back(temp); - } + bytes_read_write += bin_text_read_write_fixed_validated(model_file, pair, 2, + "", read, msg, text); + if (read) + { string temp(pair); + if (count(all.pairs.begin(), all.pairs.end(), temp) == 0) + all.pairs.push_back(temp); } + } msg << "\n"; bytes_read_write += bin_text_read_write_fixed_validated(model_file, nullptr, 0, - "", read, msg, text); + "", read, msg, text); uint32_t triple_len = (uint32_t)all.triples.size(); msg << triple_len << " triples: "; bytes_read_write += bin_text_read_write_fixed_validated(model_file, (char *)&triple_len, sizeof(triple_len), - "", read, msg, text); + "", read, msg, text); // TODO: validate triples? for (size_t i = 0; i < triple_len; i++) { char triple[4] = { 0, 0, 0, 0 }; if (!read) - { msg << all.triples[i] << " "; - memcpy(triple, all.triples[i].c_str(), 3); - } + { msg << all.triples[i] << " "; + memcpy(triple, all.triples[i].c_str(), 3); + } bytes_read_write += bin_text_read_write_fixed_validated(model_file, triple, 3, - "", read, msg, text); + "", read, msg, text); if (read) { string temp(triple); if (count(all.triples.begin(), all.triples.end(), temp) == 0) @@ -230,7 +228,7 @@ void save_load_header(vw& all, io_buf& model_file, bool read, bool text) msg << "\n"; bytes_read_write += bin_text_read_write_fixed_validated(model_file, nullptr, 0, - "", read, msg, text); + "", read, msg, text); if (all.model_file_ver >= VERSION_FILE_WITH_INTERACTIONS) // && < VERSION_FILE_WITH_INTERACTIONS_IN_FO (previous if) { // the only version that saves interacions among pairs and triples @@ -238,7 +236,7 @@ void save_load_header(vw& all, io_buf& model_file, bool read, bool text) msg << len << " interactions: "; bytes_read_write += bin_text_read_write_fixed_validated(model_file, (char *)&len, sizeof(len), - "", read, msg, text); + "", read, msg, text); for (size_t i = 0; i < len; i++) { uint32_t inter_len = 0; @@ -247,27 +245,26 @@ void save_load_header(vw& all, io_buf& model_file, bool read, bool text) msg << "len: " << inter_len << " "; } bytes_read_write += bin_text_read_write_fixed_validated(model_file, (char *)&inter_len, sizeof(inter_len), - "", read, msg, text); + "", read, msg, text); if (read) - { v_string s = v_init(); - s.resize(inter_len); - s.end() += inter_len; - all.interactions.push_back(s); - } + { v_string s = v_init(); + s.resize(inter_len); + s.end() += inter_len; + all.interactions.push_back(s); + } else - { - msg << "interaction: "; - msg.write((char*)all.interactions[i].begin(), inter_len); - } + { msg << "interaction: "; + msg.write((char*)all.interactions[i].begin(), inter_len); + } bytes_read_write += bin_text_read_write_fixed_validated(model_file, (char*)all.interactions[i].begin(), inter_len, - "", read, msg, text); + "", read, msg, text); } - msg << "\n"; - bytes_read_write += bin_text_read_write_fixed_validated(model_file, nullptr, 0, - "", read, msg, text); + msg << "\n"; + bytes_read_write += bin_text_read_write_fixed_validated(model_file, nullptr, 0, + "", read, msg, text); } else // < VERSION_FILE_WITH_INTERACTIONS { //pairs and triples may be restored but not reflected in interactions @@ -283,7 +280,7 @@ void save_load_header(vw& all, io_buf& model_file, bool read, bool text) uint32_t rank = 0; msg << "rank:" << rank << "\n"; bytes_read_write += bin_text_read_write_fixed_validated(model_file, (char*)&rank, sizeof(rank), - "", read, msg, text); + "", read, msg, text); if (rank != 0) { if (std::find(all.args.begin(), all.args.end(), "--rank") == all.args.end()) { all.args.push_back("--rank"); @@ -299,50 +296,50 @@ void save_load_header(vw& all, io_buf& model_file, bool read, bool text) msg << "lda:" << all.lda <<"\n"; bytes_read_write += bin_text_read_write_fixed_validated(model_file, (char*)&all.lda, sizeof(all.lda), - "", read, msg, text); + "", read, msg, text); // TODO: validate ngram_len? uint32_t ngram_len = (uint32_t)all.ngram_strings.size(); msg << ngram_len << " ngram:"; bytes_read_write += bin_text_read_write_fixed_validated(model_file, (char *)&ngram_len, sizeof(ngram_len), - "", read, msg, text); + "", read, msg, text); for (size_t i = 0; i < ngram_len; i++) { // have '\0' at the end for sure char ngram[4] = { 0, 0, 0, 0 }; if (!read) - { msg << all.ngram_strings[i] << " "; - memcpy(ngram, all.ngram_strings[i].c_str(), min(3, all.ngram_strings[i].size())); - } + { msg << all.ngram_strings[i] << " "; + memcpy(ngram, all.ngram_strings[i].c_str(), min(3, all.ngram_strings[i].size())); + } bytes_read_write += bin_text_read_write_fixed_validated(model_file, ngram, 3, - "", read, msg, text); + "", read, msg, text); if (read) - { string temp(ngram); - all.ngram_strings.push_back(temp); + { string temp(ngram); + all.ngram_strings.push_back(temp); - all.args.push_back("--ngram"); - all.args.push_back(boost::lexical_cast(temp)); - } + all.args.push_back("--ngram"); + all.args.push_back(boost::lexical_cast(temp)); + } } msg <<"\n"; bytes_read_write += bin_text_read_write_fixed_validated(model_file, nullptr, 0, - "", read, msg, text); + "", read, msg, text); // TODO: validate skips? uint32_t skip_len = (uint32_t)all.skip_strings.size(); msg << skip_len << " skip:"; bytes_read_write += bin_text_read_write_fixed_validated(model_file, (char *)&skip_len, sizeof(skip_len), - "", read, msg, text); + "", read, msg, text); for (size_t i = 0; i < skip_len; i++) { char skip[4] = { 0, 0, 0, 0 }; if (!read) - { msg << all.skip_strings[i] << " "; - memcpy(skip, all.skip_strings[i].c_str(), min(3, all.skip_strings[i].size())); - } + { msg << all.skip_strings[i] << " "; + memcpy(skip, all.skip_strings[i].c_str(), min(3, all.skip_strings[i].size())); + } bytes_read_write += bin_text_read_write_fixed_validated(model_file, skip, 3, - "", read, msg, text); + "", read, msg, text); if (read) { string temp(skip); all.skip_strings.push_back(temp); @@ -353,7 +350,7 @@ void save_load_header(vw& all, io_buf& model_file, bool read, bool text) } msg << "\n"; bytes_read_write += bin_text_read_write_fixed_validated(model_file, nullptr, 0, - "", read, msg, text); + "", read, msg, text); if (read) { uint32_t len; @@ -365,14 +362,14 @@ void save_load_header(vw& all, io_buf& model_file, bool read, bool text) all.file_options->str(buff2); } else - { msg << "options:"<< all.file_options->str() << "\n"; - - uint32_t len = (uint32_t)all.file_options->str().length(); - if (len > 0) - safe_memcpy(buff2, buf2_size, all.file_options->str().c_str(), len + 1); - *(buff2 + len) = 0; - bytes_read_write += bin_text_read_write(model_file, buff2, len + 1, //len+1 to write a \0 - "", read, msg, text); + { msg << "options:"<< all.file_options->str() << "\n"; + + uint32_t len = (uint32_t)all.file_options->str().length(); + if (len > 0) + safe_memcpy(buff2, buf2_size, all.file_options->str().c_str(), len + 1); + *(buff2 + len) = 0; + bytes_read_write += bin_text_read_write(model_file, buff2, len + 1, //len+1 to write a \0 + "", read, msg, text); } @@ -402,8 +399,7 @@ void save_load_header(vw& all, io_buf& model_file, bool read, bool text) } } catch (...) - { - free(buff2); + { free(buff2); throw; } @@ -507,8 +503,8 @@ void parse_mask_regressor_args(vw& all) io_temp.close_file(); // Re-zero the weights, in case weights of initial regressor use different indices - weight_parameters& weights = all.weights; - weights.set_zero(0); + weight_parameters& weights = all.weights; + weights.set_zero(0); } else { // If no initial regressor, just clear out the options loaded from the header. diff --git a/vowpalwabbit/parser.cc b/vowpalwabbit/parser.cc index e0fe344be8e..041443c4510 100644 --- a/vowpalwabbit/parser.cc +++ b/vowpalwabbit/parser.cc @@ -163,36 +163,36 @@ uint32_t cache_numbits(io_buf* buf, int filepointer) { v_array t = v_init(); try - { size_t v_length; - buf->read_file(filepointer, (char*)&v_length, sizeof(v_length)); - if (v_length > 61) - THROW("cache version too long, cache file is probably invalid"); - - if (v_length == 0) - THROW("cache version too short, cache file is probably invalid"); - - t.erase(); - if (t.size() < v_length) - t.resize(v_length); - - buf->read_file(filepointer,t.begin(),v_length); - version_struct v_tmp(t.begin()); - if ( v_tmp != version ) - { cout << "cache has possibly incompatible version, rebuilding" << endl; - t.delete_v(); - return 0; - } - - char temp; - if (buf->read_file(filepointer, &temp, 1) < 1) - THROW("failed to read"); - - if (temp != 'c') - THROW("data file is not a cache file"); + { size_t v_length; + buf->read_file(filepointer, (char*)&v_length, sizeof(v_length)); + if (v_length > 61) + THROW("cache version too long, cache file is probably invalid"); + + if (v_length == 0) + THROW("cache version too short, cache file is probably invalid"); + + t.erase(); + if (t.size() < v_length) + t.resize(v_length); + + buf->read_file(filepointer,t.begin(),v_length); + version_struct v_tmp(t.begin()); + if ( v_tmp != version ) + { cout << "cache has possibly incompatible version, rebuilding" << endl; + t.delete_v(); + return 0; } + + char temp; + if (buf->read_file(filepointer, &temp, 1) < 1) + THROW("failed to read"); + + if (temp != 'c') + THROW("data file is not a cache file"); + } catch(...) - { t.delete_v(); - } + { t.delete_v(); + } t.delete_v(); @@ -215,7 +215,7 @@ void reset_source(vw& all, size_t numbits) { io_buf* input = all.p->input; input->current = 0; if (all.p->write_cache) - { all.p->output->flush(); + { all.p->output->flush(); all.p->write_cache = false; all.p->output->close_file(); remove(all.p->output->finalname.begin()); @@ -289,7 +289,7 @@ void finalize_source(parser* p) p->output->close_files(); delete p->output; if (p->jsonp) - delete p->jsonp; + delete p->jsonp; } void make_write_cache(vw& all, string &newname, bool quiet) @@ -575,8 +575,8 @@ void enable_sources(vw& all, bool quiet, size_t passes) all.p->jsonp = new json_parser; } else - all.p->reader = read_features_string; - + all.p->reader = read_features_string; + all.p->resettable = all.p->write_cache; } } @@ -601,20 +601,20 @@ void addgrams(vw& all, size_t ngram, size_t skip_gram, features& fs, { if (ngram == 0 && gram_mask.last() < initial_length) { size_t last = initial_length - gram_mask.last(); for(size_t i = 0; i < last; i++) - { uint64_t new_index = fs.indicies[i]; + { uint64_t new_index = fs.indicies[i]; + for (size_t n = 1; n < gram_mask.size(); n++) + new_index = new_index*quadratic_constant + fs.indicies[i+gram_mask[n]]; + + fs.push_back(1.,new_index); + if (fs.space_names.size() > 0) + { string feature_name(fs.space_names[i].get()->second); for (size_t n = 1; n < gram_mask.size(); n++) - new_index = new_index*quadratic_constant + fs.indicies[i+gram_mask[n]]; - - fs.push_back(1.,new_index); - if (fs.space_names.size() > 0) - { string feature_name(fs.space_names[i].get()->second); - for (size_t n = 1; n < gram_mask.size(); n++) - { feature_name += string("^"); - feature_name += string(fs.space_names[i+gram_mask[n]].get()->second); - } - fs.space_names.push_back(audit_strings_ptr(new audit_strings(fs.space_names[i].get()->first, feature_name))); - } + { feature_name += string("^"); + feature_name += string(fs.space_names[i+gram_mask[n]].get()->second); + } + fs.space_names.push_back(audit_strings_ptr(new audit_strings(fs.space_names[i].get()->first, feature_name))); } + } } if (ngram > 0) { gram_mask.push_back(gram_mask.last()+1+skips); @@ -657,10 +657,10 @@ void end_pass_example(vw& all, example* ae) void feature_limit(vw& all, example* ex) { for(namespace_index index : ex->indices) if (all.limit[index] < ex->feature_space[index].size()) - { features& fs = ex->feature_space[index]; - fs.sort(all.parse_mask); - unique_features(fs, all.limit[index]); - } + { features& fs = ex->feature_space[index]; + fs.sort(all.parse_mask); + unique_features(fs, all.limit[index]); + } } namespace VW @@ -682,21 +682,18 @@ example& get_unused_example(vw* all) } void setup_examples(vw& all, v_array& examples) -{ - for (example* ae : examples) - setup_example(all, ae); +{ for (example* ae : examples) + setup_example(all, ae); } void setup_example(vw& all, example* ae) -{ - if (all.p->sort_features && ae->sorted == false) - unique_sort_features(all.parse_mask, ae); +{ if (all.p->sort_features && ae->sorted == false) + unique_sort_features(all.parse_mask, ae); - if (all.p->write_cache) - { - all.p->lp.cache_label(&ae->l, *(all.p->output)); - cache_features(*(all.p->output), ae, all.parse_mask); - } + if (all.p->write_cache) + { all.p->lp.cache_label(&ae->l, *(all.p->output)); + cache_features(*(all.p->output), ae, all.parse_mask); + } ae->partial_prediction = 0.; ae->num_features = 0; @@ -713,7 +710,7 @@ void setup_example(vw& all, example* ae) all.p->in_pass_counter++; ae->weight = all.p->lp.get_weight(&ae->l); - + if (all.ignore_some) for (unsigned char* i = ae->indices.begin(); i != ae->indices.end(); i++) if (all.ignore[*i]) @@ -775,8 +772,7 @@ example* read_example(vw& all, char* example_line) example* read_example(vw& all, string example_line) { return read_example(all, (char*)example_line.c_str()); } void add_constant_feature(vw& vw, example*ec) -{ - ec->indices.push_back(constant_namespace); +{ ec->indices.push_back(constant_namespace); ec->feature_space[constant_namespace].push_back(1,constant); ec->total_sum_feat_sq++; ec->num_features++; @@ -820,11 +816,11 @@ primitive_feature_space* export_example(vw& all, example* ec, size_t& len) int f_count = 0; for (features::iterator& f : ec->feature_space[i]) - { feature t = {f.value(), f.index()}; - t.weight_index >>= all.weights.stride_shift(); - fs_ptr[fs_count].fs[f_count] = t; - f_count++; - } + { feature t = {f.value(), f.index()}; + t.weight_index >>= all.weights.stride_shift(); + fs_ptr[fs_count].fs[f_count] = t; + f_count++; + } fs_count++; } return fs_ptr; @@ -847,8 +843,7 @@ void parse_example_label(vw& all, example&ec, string label) } void empty_example(vw& all, example& ec) -{ - for (features& fs : ec) +{ for (features& fs : ec) fs.erase(); ec.indices.erase(); @@ -890,7 +885,7 @@ void *main_parse_loop(void *in) v_array examples = v_init(); while(!all->p->done) { examples.push_back(&VW::get_unused_example(all)); // need at least 1 example - if (!all->do_reset_source && example_number != all->pass_length && all->max_examples > example_number + if (!all->do_reset_source && example_number != all->pass_length && all->max_examples > example_number && all->p->reader(all, examples) > 0) { VW::setup_examples(*all, examples); example_number+=examples.size(); @@ -918,7 +913,7 @@ void *main_parse_loop(void *in) all->p->end_parsed_examples+=examples_available; condition_variable_signal_all(&all->p->example_available); mutex_unlock(&all->p->examples_lock); - examples.erase(); + examples.erase(); } examples.delete_v(); return 0L; @@ -968,8 +963,8 @@ float get_prediction(example* ec) float get_cost_sensitive_prediction(example* ec) { return (float)ec->pred.multiclass; } -v_array& get_cost_sensitive_prediction_confidence_scores(example* ec) { - return ec->pred.scalars; +v_array& get_cost_sensitive_prediction_confidence_scores(example* ec) +{ return ec->pred.scalars; } uint32_t* get_multilabel_predictions(example* ec, size_t& len) @@ -1021,7 +1016,7 @@ void initialize_parser_datastructures(vw& all) namespace VW { void start_parser(vw& all) -{ +{ #ifndef _WIN32 pthread_create(&all.parse_thread, nullptr, main_parse_loop, &all); #else @@ -1038,8 +1033,7 @@ void free_parser(vw& all) all.p->gram_mask.delete_v(); if (all.p->examples != nullptr) - { - for (size_t i = 0; i < all.p->ring_size; i++) + { for (size_t i = 0; i < all.p->ring_size; i++) VW::dealloc_example(all.p->lp.delete_label, all.p->examples[i], all.delete_prediction); free(all.p->examples); diff --git a/vowpalwabbit/rand48.cc b/vowpalwabbit/rand48.cc index 0bc3d360458..4fb7e795559 100644 --- a/vowpalwabbit/rand48.cc +++ b/vowpalwabbit/rand48.cc @@ -14,9 +14,9 @@ uint64_t c = 2147483647; int bias = 127 << 23; -float merand48(uint64_t& initial) { - initial = a * initial + c; - int32_t temp = ((initial >> 25) & 0x7FFFFF) | bias; +float merand48(uint64_t& initial) +{ initial = a * initial + c; + int32_t temp = ((initial >> 25) & 0x7FFFFF) | bias; return bits_to_float(temp) - 1; } diff --git a/vowpalwabbit/recall_tree.cc b/vowpalwabbit/recall_tree.cc index d94059494fb..2d0d1d8370c 100644 --- a/vowpalwabbit/recall_tree.cc +++ b/vowpalwabbit/recall_tree.cc @@ -15,17 +15,18 @@ license as described in the file LICENSE.node using namespace std; using namespace LEARNER; -namespace recall_tree_ns { +namespace recall_tree_ns +{ -struct node_pred { - uint32_t label; +struct node_pred +{ uint32_t label; double label_count; node_pred (uint32_t a) : label (a), label_count (0) { } }; -struct node { - uint32_t parent; +struct node +{ uint32_t parent; float recall_lbest; bool internal; @@ -41,22 +42,22 @@ struct node { v_array preds; node () : parent (0), - recall_lbest (0), - internal (false), - depth (0), - base_router (0), - left (0), - right (0), - n (0), - entropy (0), - passes (1), - preds (v_init()) - { - } + recall_lbest (0), + internal (false), + depth (0), + base_router (0), + left (0), + right (0), + n (0), + entropy (0), + passes (1), + preds (v_init()) + { + } }; -struct recall_tree { - vw* all; +struct recall_tree +{ vw* all; uint32_t k; bool node_only; @@ -71,8 +72,7 @@ struct recall_tree { }; float to_prob (float x) -{ - static const float alpha = 2.0f; +{ static const float alpha = 2.0f; // http://stackoverflow.com/questions/2789481/problem-calling-stdmax return (std::max) (0.f, (std::min) (1.f, 0.5f * (1.0f + alpha * x))); } @@ -81,33 +81,30 @@ void init_tree (recall_tree& b, uint32_t root, uint32_t depth, uint32_t& routers_used) -{ - if (depth <= b.max_depth) - { - uint32_t left_child; - uint32_t right_child; - left_child = (uint32_t)b.nodes.size (); - b.nodes.push_back (node ()); - right_child = (uint32_t)b.nodes.size (); - b.nodes.push_back (node ()); - b.nodes[root].base_router = routers_used++; - - b.nodes[root].internal = true; - b.nodes[root].left = left_child; - b.nodes[left_child].parent = root; - b.nodes[left_child].depth = depth; - b.nodes[root].right = right_child; - b.nodes[right_child].parent = root; - b.nodes[right_child].depth = depth; - - init_tree (b, left_child, depth + 1, routers_used); - init_tree (b, right_child, depth + 1, routers_used); - } +{ if (depth <= b.max_depth) + { uint32_t left_child; + uint32_t right_child; + left_child = (uint32_t)b.nodes.size (); + b.nodes.push_back (node ()); + right_child = (uint32_t)b.nodes.size (); + b.nodes.push_back (node ()); + b.nodes[root].base_router = routers_used++; + + b.nodes[root].internal = true; + b.nodes[root].left = left_child; + b.nodes[left_child].parent = root; + b.nodes[left_child].depth = depth; + b.nodes[root].right = right_child; + b.nodes[right_child].parent = root; + b.nodes[right_child].depth = depth; + + init_tree (b, left_child, depth + 1, routers_used); + init_tree (b, right_child, depth + 1, routers_used); + } } void init_tree (recall_tree& b) -{ - uint32_t routers_used = 0; +{ uint32_t routers_used = 0; b.nodes.push_back (node ()); init_tree (b, 0, 1, routers_used); @@ -115,8 +112,7 @@ void init_tree (recall_tree& b) } node_pred* find (recall_tree& b, uint32_t cn, example& ec) -{ - node_pred* ls; +{ node_pred* ls; for (ls = b.nodes[cn].preds.begin (); ls != b.nodes[cn].preds.end () && ls->label != ec.l.multi.label; @@ -126,11 +122,10 @@ node_pred* find (recall_tree& b, uint32_t cn, example& ec) } node_pred* find_or_create (recall_tree& b, uint32_t cn, example& ec) -{ - node_pred* ls = find (b, cn, ec); +{ node_pred* ls = find (b, cn, ec); - if (ls == b.nodes[cn].preds.end ()) { - node_pred newls (ec.l.multi.label); + if (ls == b.nodes[cn].preds.end ()) + { node_pred newls (ec.l.multi.label); b.nodes[cn].preds.push_back (newls); ls = b.nodes[cn].preds.end () - 1; } @@ -139,8 +134,7 @@ node_pred* find_or_create (recall_tree& b, uint32_t cn, example& ec) } void compute_recall_lbest (recall_tree& b, node* n) -{ - if (n->n <= 0) +{ if (n->n <= 0) return; double mass_at_k = 0; @@ -149,9 +143,8 @@ void compute_recall_lbest (recall_tree& b, node* n) ls != n->preds.end () && ls < n->preds.begin () + b.max_candidates; ++ls) - { - mass_at_k += ls->label_count; - } + { mass_at_k += ls->label_count; + } float f = (float)mass_at_k / (float)n->n; float stdf = sqrt (f * (1.f - f) / (float)n->n); @@ -160,17 +153,15 @@ void compute_recall_lbest (recall_tree& b, node* n) // http://stackoverflow.com/questions/2789481/problem-calling-stdmax n->recall_lbest = (std::max) (0.f, f - sqrt (b.bern_hyper) * stdf - - b.bern_hyper * diamf); + - b.bern_hyper * diamf); } double plogp (double c, double n) -{ - return (c == 0) ? 0 : (c / n) * log (c / n); +{ return (c == 0) ? 0 : (c / n) * log (c / n); } double updated_entropy (recall_tree& b, uint32_t cn, example& ec) -{ - node_pred* ls = find (b, cn, ec); +{ node_pred* ls = find (b, cn, ec); // entropy = -\sum_k (c_k/n) Log[c_k/n] // c_0 <- c_0 + 1, n <- n + 1 @@ -198,16 +189,15 @@ double updated_entropy (recall_tree& b, uint32_t cn, example& ec) } void insert_example_at_node (recall_tree& b, uint32_t cn, example& ec) -{ - node_pred* ls = find_or_create (b, cn, ec); +{ node_pred* ls = find_or_create (b, cn, ec); b.nodes[cn].entropy = updated_entropy (b, cn, ec); ls->label_count += ec.l.multi.weight; while (ls != b.nodes[cn].preds.begin () && - ls[-1].label_count < ls[0].label_count) { - std::swap (ls[-1], ls[0]); + ls[-1].label_count < ls[0].label_count) + { std::swap (ls[-1], ls[0]); --ls; } @@ -219,33 +209,28 @@ void insert_example_at_node (recall_tree& b, uint32_t cn, example& ec) // TODO: handle if features already in this namespace void add_node_id_feature (recall_tree& b, uint32_t cn, example& ec) -{ - vw* all = b.all; +{ vw* all = b.all; uint64_t mask = all->weights.mask(); size_t ss = all->weights.stride_shift(); ec.indices.push_back (node_id_namespace); features& fs = ec.feature_space[node_id_namespace]; if (b.node_only) - { - fs.push_back (1., ((868771 * cn) << ss) & mask); - } + { fs.push_back (1., ((868771 * cn) << ss) & mask); + } else - { - while (cn > 0) - { - fs.push_back (1., ((868771 * cn) << ss) & mask); - cn = b.nodes[cn].parent; - } + { while (cn > 0) + { fs.push_back (1., ((868771 * cn) << ss) & mask); + cn = b.nodes[cn].parent; } + } // TODO: audit ? // TODO: if namespace already exists ? } void remove_node_id_feature (recall_tree& b, uint32_t cn, example& ec) -{ - features& fs = ec.feature_space[node_id_namespace]; +{ features& fs = ec.feature_space[node_id_namespace]; fs.erase (); ec.indices.pop (); } @@ -254,8 +239,7 @@ uint32_t oas_predict (recall_tree& b, base_learner& base, uint32_t cn, example& ec) -{ - MULTICLASS::label_t mc = ec.l.multi; +{ MULTICLASS::label_t mc = ec.l.multi; uint32_t save_pred = ec.pred.multiclass; uint32_t amaxscore = 0; @@ -268,14 +252,12 @@ uint32_t oas_predict (recall_tree& b, ls != b.nodes[cn].preds.end () && ls < b.nodes[cn].preds.begin () + b.max_candidates; ++ls) - { - base.predict (ec, b.max_routers + ls->label - 1); - if (amaxscore == 0 || ec.partial_prediction > maxscore) - { - maxscore = ec.partial_prediction; - amaxscore = ls->label; - } + { base.predict (ec, b.max_routers + ls->label - 1); + if (amaxscore == 0 || ec.partial_prediction > maxscore) + { maxscore = ec.partial_prediction; + amaxscore = ls->label; } + } remove_node_id_feature (b, cn, ec); @@ -286,27 +268,23 @@ uint32_t oas_predict (recall_tree& b, } bool is_candidate (recall_tree& b, uint32_t cn, example& ec) -{ - for (node_pred* ls = b.nodes[cn].preds.begin (); - ls != b.nodes[cn].preds.end () +{ for (node_pred* ls = b.nodes[cn].preds.begin (); + ls != b.nodes[cn].preds.end () && ls < b.nodes[cn].preds.begin () + b.max_candidates; ++ls) - { - if (ls->label == ec.l.multi.label) - return true; - } + { if (ls->label == ec.l.multi.label) + return true; + } return false; } inline uint32_t descend (node& n, float prediction) -{ - return prediction < 0 ? n.left : n.right; +{ return prediction < 0 ? n.left : n.right; } struct predict_type -{ - uint32_t node_id; +{ uint32_t node_id; uint32_t class_prediction; predict_type (uint32_t a, uint32_t b) : node_id (a), class_prediction (b) { } @@ -315,8 +293,7 @@ struct predict_type bool stop_recurse_check (recall_tree& b, uint32_t parent, uint32_t child) -{ - return b.bern_hyper > 0 && +{ return b.bern_hyper > 0 && b.nodes[parent].recall_lbest >= b.nodes[child].recall_lbest; } @@ -324,22 +301,20 @@ predict_type predict_from (recall_tree& b, base_learner& base, example& ec, uint32_t cn) -{ - MULTICLASS::label_t mc = ec.l.multi; +{ MULTICLASS::label_t mc = ec.l.multi; uint32_t save_pred = ec.pred.multiclass; ec.l.simple = {FLT_MAX, 0.f, 0.f}; while (b.nodes[cn].internal) - { - base.predict (ec, b.nodes[cn].base_router); - uint32_t newcn = descend (b.nodes[cn], ec.partial_prediction); - bool cond = stop_recurse_check (b, cn, newcn); + { base.predict (ec, b.nodes[cn].base_router); + uint32_t newcn = descend (b.nodes[cn], ec.partial_prediction); + bool cond = stop_recurse_check (b, cn, newcn); - if (cond) - break; + if (cond) + break; - cn = newcn; - } + cn = newcn; + } ec.l.multi = mc; ec.pred.multiclass = save_pred; @@ -348,8 +323,7 @@ predict_type predict_from (recall_tree& b, } void predict (recall_tree& b, base_learner& base, example& ec) -{ - predict_type pred = predict_from (b, base, ec, 0); +{ predict_type pred = predict_from (b, base, ec, 0); ec.pred.multiclass = pred.class_prediction; } @@ -358,8 +332,7 @@ float train_node (recall_tree& b, base_learner& base, example& ec, uint32_t cn) -{ - MULTICLASS::label_t mc = ec.l.multi; +{ MULTICLASS::label_t mc = ec.l.multi; uint32_t save_pred = ec.pred.multiclass; // minimize entropy @@ -393,68 +366,61 @@ float train_node (recall_tree& b, void learn (recall_tree& b, base_learner& base, example& ec) -{ - predict (b, base, ec); +{ predict (b, base, ec); if (b.all->training && ec.l.multi.label != (uint32_t)-1) // if training the tree - { - uint32_t cn = 0; + { uint32_t cn = 0; - while (b.nodes[cn].internal) - { - float which = train_node (b, base, ec, cn); + while (b.nodes[cn].internal) + { float which = train_node (b, base, ec, cn); - if (b.randomized_routing) - which = (frand48 () > to_prob (which) ? -1.f : 1.f); + if (b.randomized_routing) + which = (frand48 () > to_prob (which) ? -1.f : 1.f); - uint32_t newcn = descend (b.nodes[cn], which); - bool cond = stop_recurse_check (b, cn, newcn); - insert_example_at_node (b, cn, ec); + uint32_t newcn = descend (b.nodes[cn], which); + bool cond = stop_recurse_check (b, cn, newcn); + insert_example_at_node (b, cn, ec); - if (cond) - { - insert_example_at_node (b, newcn, ec); - break; - } + if (cond) + { insert_example_at_node (b, newcn, ec); + break; + } - cn = newcn; - } + cn = newcn; + } - if (! b.nodes[cn].internal) - insert_example_at_node (b, cn, ec); + if (! b.nodes[cn].internal) + insert_example_at_node (b, cn, ec); - if (is_candidate (b, cn, ec)) - { - MULTICLASS::label_t mc = ec.l.multi; - uint32_t save_pred = ec.pred.multiclass; + if (is_candidate (b, cn, ec)) + { MULTICLASS::label_t mc = ec.l.multi; + uint32_t save_pred = ec.pred.multiclass; - add_node_id_feature (b, cn, ec); + add_node_id_feature (b, cn, ec); - ec.l.simple = { 1.f, 1.f, 0.f }; - base.learn (ec, b.max_routers + mc.label - 1); - ec.l.simple = { -1.f, 1.f, 0.f }; + ec.l.simple = { 1.f, 1.f, 0.f }; + base.learn (ec, b.max_routers + mc.label - 1); + ec.l.simple = { -1.f, 1.f, 0.f }; - for (node_pred* ls = b.nodes[cn].preds.begin (); - ls != b.nodes[cn].preds.end () - && ls < b.nodes[cn].preds.begin () + b.max_candidates; - ++ls) - { - if (ls->label != mc.label) - base.learn (ec, b.max_routers + ls->label - 1); - } + for (node_pred* ls = b.nodes[cn].preds.begin (); + ls != b.nodes[cn].preds.end () + && ls < b.nodes[cn].preds.begin () + b.max_candidates; + ++ls) + { if (ls->label != mc.label) + base.learn (ec, b.max_routers + ls->label - 1); + } - remove_node_id_feature (b, cn, ec); + remove_node_id_feature (b, cn, ec); - ec.l.multi = mc; - ec.pred.multiclass = save_pred; - } + ec.l.multi = mc; + ec.pred.multiclass = save_pred; } + } } void finish (recall_tree& b) -{ - for (size_t i = 0; i < b.nodes.size (); ++i) +{ for (size_t i = 0; i < b.nodes.size (); ++i) b.nodes[i].preds.delete_v (); b.nodes.delete_v (); } @@ -489,75 +455,65 @@ void finish (recall_tree& b) while (0); void save_load_tree(recall_tree& b, io_buf& model_file, bool read, bool text) -{ - if (model_file.files.size() > 0) - { - stringstream msg; +{ if (model_file.files.size() > 0) + { stringstream msg; + + writeit (b.k, "k"); + writeit (b.node_only, "node_only"); + writeitvar (b.nodes.size (), "nodes", n_nodes); + + if (read) + { b.nodes.erase (); + for (uint32_t j = 0; j < n_nodes; ++j) + { b.nodes.push_back (node ()); + } + } + + writeit (b.max_candidates, "max_candidates"); + writeit (b.max_depth, "max_depth"); + + for (uint32_t j = 0; j < n_nodes; ++j) + { node* cn = &b.nodes[j]; - writeit (b.k, "k"); - writeit (b.node_only, "node_only"); - writeitvar (b.nodes.size (), "nodes", n_nodes); + writeit (cn->parent, "parent"); + writeit (cn->recall_lbest, "recall_lbest"); + writeit (cn->internal, "internal"); + writeit (cn->depth, "depth"); + writeit (cn->base_router, "base_router"); + writeit (cn->left, "left"); + writeit (cn->right, "right"); + writeit (cn->n, "n"); + writeit (cn->entropy, "entropy"); + writeit (cn->passes, "passes"); + + writeitvar (cn->preds.size (), "n_preds", n_preds); if (read) - { - b.nodes.erase (); - for (uint32_t j = 0; j < n_nodes; ++j) - { - b.nodes.push_back (node ()); - } + { cn->preds.erase (); + + for (uint32_t k = 0; k < n_preds; ++k) + { cn->preds.push_back (node_pred (0)); } + } - writeit (b.max_candidates, "max_candidates"); - writeit (b.max_depth, "max_depth"); + for (uint32_t k = 0; k < n_preds; ++k) + { node_pred* pred = &cn->preds[k]; - for (uint32_t j = 0; j < n_nodes; ++j) - { - node* cn = &b.nodes[j]; - - writeit (cn->parent, "parent"); - writeit (cn->recall_lbest, "recall_lbest"); - writeit (cn->internal, "internal"); - writeit (cn->depth, "depth"); - writeit (cn->base_router, "base_router"); - writeit (cn->left, "left"); - writeit (cn->right, "right"); - writeit (cn->n, "n"); - writeit (cn->entropy, "entropy"); - writeit (cn->passes, "passes"); - - writeitvar (cn->preds.size (), "n_preds", n_preds); - - if (read) - { - cn->preds.erase (); - - for (uint32_t k = 0; k < n_preds; ++k) - { - cn->preds.push_back (node_pred (0)); - } - } - - for (uint32_t k = 0; k < n_preds; ++k) - { - node_pred* pred = &cn->preds[k]; - - writeit (pred->label, "label"); - writeit (pred->label_count, "label_count"); - } - - if (read) - { - compute_recall_lbest (b, cn); - } - } + writeit (pred->label, "label"); + writeit (pred->label_count, "label_count"); + } + + if (read) + { compute_recall_lbest (b, cn); + } } + } } } // namespace base_learner* recall_tree_setup(vw& all) -{ - using namespace recall_tree_ns; +{ using namespace recall_tree_ns; if (missing_option(all, "recall_tree", @@ -565,30 +521,30 @@ base_learner* recall_tree_setup(vw& all) return nullptr; new_options(all, "recall tree options") - ("max_candidates", po::value(), "maximum number of labels per leaf in the tree") - ("bern_hyper", po::value()->default_value(1), "recall tree depth penalty") - ("max_depth", po::value(), "maximum depth of the tree, default log_2 (#classes)") - ("node_only", po::value()->default_value(false), "only use node features, not full path features") - ("randomized_routing", po::value()->default_value (false), "randomized routing"); + ("max_candidates", po::value(), "maximum number of labels per leaf in the tree") + ("bern_hyper", po::value()->default_value(1), "recall tree depth penalty") + ("max_depth", po::value(), "maximum depth of the tree, default log_2 (#classes)") + ("node_only", po::value()->default_value(false), "only use node features, not full path features") + ("randomized_routing", po::value()->default_value (false), "randomized routing"); add_options(all); po::variables_map& vm = all.vm; recall_tree& tree = calloc_or_throw (); - tree.bern_hyper = vm["bern_hyper"].as (); + tree.bern_hyper = vm["bern_hyper"].as (); tree.all = &all; tree.k = (uint32_t)vm["recall_tree"].as(); tree.node_only = vm["node_only"].as (); *(all.file_options) << " --node_only " << tree.node_only; tree.max_candidates = vm.count ("max_candidates") > 0 - ? vm["max_candidates"].as() - : (std::min) (tree.k, 4 * (uint32_t) (ceil (log (tree.k) / log (2.0)))); + ? vm["max_candidates"].as() + : (std::min) (tree.k, 4 * (uint32_t) (ceil (log (tree.k) / log (2.0)))); *(all.file_options) << " --max_candidates " << tree.max_candidates; tree.max_depth = - vm.count ("max_depth") > 0 - ? vm["max_depth"].as() - : (uint32_t) std::ceil (std::log (tree.k) / std::log (2.0)); + vm.count ("max_depth") > 0 + ? vm["max_depth"].as() + : (uint32_t) std::ceil (std::log (tree.k) / std::log (2.0)); *(all.file_options) << " --max_depth " << tree.max_depth; tree.randomized_routing = vm["randomized_routing"].as (); *(all.file_options) << " --randomized_routing " << tree.randomized_routing; @@ -604,7 +560,7 @@ base_learner* recall_tree_setup(vw& all) << (all.training ? (tree.randomized_routing ? "randomized" : "deterministic") : "n/a testonly") << std::endl; - learner& l = + learner& l = init_multiclass_learner (&tree, setup_base (all), learn, diff --git a/vowpalwabbit/scorer.cc b/vowpalwabbit/scorer.cc index 01f6132eeab..e38f2f3a786 100644 --- a/vowpalwabbit/scorer.cc +++ b/vowpalwabbit/scorer.cc @@ -71,8 +71,7 @@ LEARNER::base_learner* scorer_setup(vw& all) multipredict_f = multipredict; } else if (link.compare("poisson") == 0) - { - *all.file_options << " --link=poisson "; + { *all.file_options << " --link=poisson "; l = &init_learner(&s, base, predict_or_learn, predict_or_learn); multipredict_f = multipredict; } diff --git a/vowpalwabbit/search.cc b/vowpalwabbit/search.cc index 0e86367f214..3f4b2e513c5 100644 --- a/vowpalwabbit/search.cc +++ b/vowpalwabbit/search.cc @@ -95,7 +95,7 @@ struct action_repr features *repr; action_repr(action _a, features* _repr) : a(_a) { if(_repr!=nullptr) - { repr = new features(); + { repr = new features(); repr->deep_copy_from(*_repr); } else @@ -437,9 +437,9 @@ void print_update(search_private& priv) priv.beta); if (PRINT_CLOCK_TIME) - { size_t num_sec = (size_t)(((float)(clock() - priv.start_clock_time)) / CLOCKS_PER_SEC); - cerr <<" "<< num_sec << "sec"; - } + { size_t num_sec = (size_t)(((float)(clock() - priv.start_clock_time)) / CLOCKS_PER_SEC); + cerr <<" "<< num_sec << "sec"; + } if (use_heldout_loss) fprintf(stderr, " h"); @@ -451,14 +451,13 @@ void print_update(search_private& priv) void add_new_feature(search_private& priv, float val, uint64_t idx) { uint64_t mask = priv.all->weights.mask(); -size_t ss = priv.all->weights.stride_shift(); + size_t ss = priv.all->weights.stride_shift(); uint64_t idx2 = ((idx & mask) >> ss) & mask; features& fs = priv.dat_new_feature_ec->feature_space[priv.dat_new_feature_namespace]; fs.push_back(val * priv.dat_new_feature_value, ((priv.dat_new_feature_idx + idx2) << ss) ); cdbg << "adding: " << fs.indicies.last() << ':' << fs.values.last() << endl; if (priv.all->audit) - { - stringstream temp; + { stringstream temp; temp << "fid=" << ((idx & mask) >> ss) << "_" << priv.dat_new_feature_audit_ss.str(); fs.space_names.push_back(audit_strings_ptr(new audit_strings(*priv.dat_new_feature_feature_space, temp.str()))); } @@ -467,9 +466,9 @@ size_t ss = priv.all->weights.stride_shift(); void del_features_in_top_namespace(search_private& priv, example& ec, size_t ns) { if ((ec.indices.size() == 0) || (ec.indices.last() != ns)) { if (ec.indices.size() == 0) - { THROW("internal error (bug): expecting top namespace to be '" << ns << "' but it was empty"); } + { THROW("internal error (bug): expecting top namespace to be '" << ns << "' but it was empty"); } else - { THROW("internal error (bug): expecting top namespace to be '" << ns << "' but it was " << (size_t)ec.indices.last()); } + { THROW("internal error (bug): expecting top namespace to be '" << ns << "' but it was " << (size_t)ec.indices.last()); } } features& fs = ec.feature_space[ns]; ec.indices.decr(); @@ -501,9 +500,9 @@ void add_neighbor_features(search_private& priv) //cerr << "n=" << n << " offset=" << offset << endl; if ((offset < 0) && (n < (uint64_t)(-offset))) // add feature - add_new_feature(priv, 1., 925871901 << priv.all->weights.stride_shift()); + add_new_feature(priv, 1., 925871901 << priv.all->weights.stride_shift()); else if (n + offset >= priv.ec_seq.size()) // add feature - add_new_feature(priv, 1., 3824917 << priv.all->weights.stride_shift()); + add_new_feature(priv, 1., 3824917 << priv.all->weights.stride_shift()); else // this is actually a neighbor { example& other = *priv.ec_seq[n + offset]; GD::foreach_feature(all.weights, other.feature_space[ns], priv, me.ft_offset); @@ -513,10 +512,10 @@ void add_neighbor_features(search_private& priv) features& fs = me.feature_space[neighbor_namespace]; size_t sz = fs.size(); if ((sz > 0) && (fs.sum_feat_sq > 0.)) - { me.indices.push_back(neighbor_namespace); - me.total_sum_feat_sq += fs.sum_feat_sq; - me.num_features += sz; - } + { me.indices.push_back(neighbor_namespace); + me.total_sum_feat_sq += fs.sum_feat_sq; + me.num_features += sz; + } else fs.erase(); } @@ -651,12 +650,12 @@ void add_example_conditioning(search_private& priv, example& ec, size_t conditio priv.dat_new_feature_audit_ss.clear(); priv.dat_new_feature_audit_ss << "passthrough_repr_" << i << '_' << k; } - + priv.dat_new_feature_ec = &ec; priv.dat_new_feature_idx = fid; priv.dat_new_feature_namespace = conditioning_namespace; priv.dat_new_feature_value = fs.values[k]; - add_new_feature(priv, 1., 4398201 << priv.all->weights.stride_shift()); + add_new_feature(priv, 1., 4398201 << priv.all->weights.stride_shift()); } } cdbg << "END adding passthrough features" << endl; @@ -664,10 +663,10 @@ void add_example_conditioning(search_private& priv, example& ec, size_t conditio features& con_fs = ec.feature_space[conditioning_namespace]; if ((con_fs.size() > 0) && (con_fs.sum_feat_sq > 0.)) - { ec.indices.push_back(conditioning_namespace); - ec.total_sum_feat_sq += con_fs.sum_feat_sq; - ec.num_features += con_fs.size(); - } + { ec.indices.push_back(conditioning_namespace); + ec.total_sum_feat_sq += con_fs.sum_feat_sq; + ec.num_features += con_fs.size(); + } else con_fs.erase(); } @@ -938,8 +937,7 @@ action single_prediction_notLDF(search_private& priv, example& ec, int policy, c this_cache->push_back( action_cache(min_cost, cl, cl==act, cost) ); } if (this_cache) - { - assert( priv.memo_foreach_action.size() == priv.meta_t + priv.t - 1 ); + { assert( priv.memo_foreach_action.size() == priv.meta_t + priv.t - 1 ); priv.memo_foreach_action.push_back(this_cache); cdbg << "memo_foreach_action[" << priv.meta_t + priv.t -1 << "] = " << this_cache << endl; } @@ -1005,11 +1003,11 @@ action single_prediction_LDF(search_private& priv, example* ecs, size_t ec_cnt, ecs[a].ft_offset = priv.offset; priv.base_learner->predict(ecs[a], policy); ecs[a].ft_offset = old_offset; - + priv.empty_example->in_use = true; priv.empty_example->ft_offset = priv.offset; priv.base_learner->predict(*priv.empty_example); - + cdbg << "partial_prediction[" << a << "] = " << ecs[a].partial_prediction << endl; if (override_action != (action)-1) @@ -1190,10 +1188,10 @@ void generate_training_example(search_private& priv, polylabel& losses, float we lab.costs[0].x = losses.cs.costs[a-start_K].x; //cerr << "cost[" << a << "] = " << losses[a] << " - " << min_loss << " = " << lab.costs[0].x << endl; ec.in_use = true; - uint64_t old_offset = ec.ft_offset; - ec.ft_offset = priv.offset; + uint64_t old_offset = ec.ft_offset; + ec.ft_offset = priv.offset; priv.base_learner->learn(ec, learner); - ec.ft_offset = old_offset; + ec.ft_offset = old_offset; cdbg << "generate_training_example called learn on action a=" << a << ", costs.size=" << lab.costs.size() << " ec=" << &ec << endl; priv.total_examples_generated++; @@ -1439,7 +1437,7 @@ action search_predict(search_private& priv, example* ecs, size_t ec_cnt, ptag my ecs[0].passthrough = &priv.last_action_repr; } a = priv.is_ldf ? single_prediction_LDF(priv, ecs, ec_cnt, learner, a_cost, need_fea ? a : (action)-1) - : single_prediction_notLDF(priv, *ecs, learner, allowed_actions, allowed_actions_cnt, allowed_actions_cost, a_cost, need_fea ? a : (action)-1); + : single_prediction_notLDF(priv, *ecs, learner, allowed_actions, allowed_actions_cnt, allowed_actions_cost, a_cost, need_fea ? a : (action)-1); cdbg << "passthrough = ["; for (size_t kk=0; kkacset.use_passthrough_repr) { assert((mytag >= priv->ptag_to_action.size()) || (priv->ptag_to_action[mytag].repr == nullptr)); push_at(priv->ptag_to_action, action_repr(a, &(priv->last_action_repr)), mytag); - } else + } + else push_at(priv->ptag_to_action, action_repr(a, (features*)nullptr), mytag); cdbg << "push_at " << mytag << endl; } diff --git a/vowpalwabbit/search.h b/vowpalwabbit/search.h index 7f1fa7d5636..43e974d2dd0 100644 --- a/vowpalwabbit/search.h +++ b/vowpalwabbit/search.h @@ -26,7 +26,7 @@ struct search; class BaseTask { public: - BaseTask(search* _sch, std::vector& _ec) : sch(_sch), ec(_ec) { _foreach_action = nullptr; _post_prediction = nullptr; _maybe_override_prediction = nullptr; _with_output_string = nullptr; _final_run = false; } + BaseTask(search* _sch, std::vector& _ec) : sch(_sch), ec(_ec) { _foreach_action = nullptr; _post_prediction = nullptr; _maybe_override_prediction = nullptr; _with_output_string = nullptr; _final_run = false; } inline BaseTask& foreach_action(void (*f)(search&,size_t,float,action,bool,float)) { _foreach_action = f; return *this; } inline BaseTask& post_prediction(void (*f)(search&,size_t,action,float)) { _post_prediction = f; return *this; } inline BaseTask& maybe_override_prediction(bool (*f)(search&,size_t,action&,float&)) { _maybe_override_prediction = f; return *this; } @@ -324,14 +324,14 @@ template void check_option(T& ret, vw&all, po::variables_map& vm, const *all.file_options << " --" << opt_name << " " << ret; } else if (strlen(required_error_string)>0) - { std::cerr << required_error_string << std::endl; + { std::cerr << required_error_string << std::endl; if (! vm.count("help")) THROW(required_error_string); } } void check_option(bool& ret, vw&all, po::variables_map& vm, const char* opt_name, bool default_to_cmdline, const char* mismatch_error_string); - bool string_equal(std::string a, std::string b); +bool string_equal(std::string a, std::string b); bool float_equal(float a, float b); bool uint32_equal(uint32_t a, uint32_t b); bool size_equal(size_t a, size_t b); diff --git a/vowpalwabbit/search_dep_parser.cc b/vowpalwabbit/search_dep_parser.cc index b0904c559e0..fa5f45a43c6 100644 --- a/vowpalwabbit/search_dep_parser.cc +++ b/vowpalwabbit/search_dep_parser.cc @@ -56,7 +56,7 @@ void initialize(Search::search& sch, size_t& /*num_actions*/, po::variables_map& ("transition_system", po::value(&(data->transition_system))->default_value(1), "1: arc-hybrid 2: arc-eager") ("one_learner", "Using one learner instead of three learners for labeled parser") ("cost_to_go", "Estimating cost-to-go matrix based on dynamic oracle rathan than rolling-out") - ("old_style_labels", "Use old hack of label information"); + ("old_style_labels", "Use old hack of label information"); add_options(all); check_option(data->root_label, all, vm, "root_label", false, size_equal, @@ -127,17 +127,15 @@ void finish(Search::search& sch) } void inline add_feature(example& ex, uint64_t idx, unsigned char ns, uint64_t mask, uint64_t multiplier, bool audit=false) -{ - ex.feature_space[(int)ns].push_back(1.0f, (idx * multiplier) & mask); +{ ex.feature_space[(int)ns].push_back(1.0f, (idx * multiplier) & mask); } void add_all_features(example& ex, example& src, unsigned char tgt_ns, uint64_t mask, uint64_t multiplier, uint64_t offset, bool audit=false) -{ - features& tgt_fs = ex.feature_space[tgt_ns]; +{ features& tgt_fs = ex.feature_space[tgt_ns]; for (namespace_index ns : src.indices) if(ns != constant_namespace) // ignore constant_namespace - for (feature_index i : src.feature_space[ns].indicies) - tgt_fs.push_back(1.0f, ((i / multiplier + offset) * multiplier) & mask ); + for (feature_index i : src.feature_space[ns].indicies) + tgt_fs.push_back(1.0f, ((i / multiplier + offset) * multiplier) & mask ); } void inline reset_ex(example *ex) @@ -197,7 +195,7 @@ size_t transition_eager(Search::search& sch, uint64_t a_id, uint32_t idx, uint32 else if (a_id == REDUCE_RIGHT) { uint32_t hd = stack.last(); stack.push_back(idx); - uint32_t last = idx; + uint32_t last = idx; heads[last] = hd; children[5][hd] = children[4][hd]; children[4][hd] = last; @@ -220,8 +218,7 @@ size_t transition_eager(Search::search& sch, uint64_t a_id, uint32_t idx, uint32 return idx; } else if (a_id == REDUCE) - { - assert(! stack.empty()); + { assert(! stack.empty()); stack.pop(); return idx; } @@ -285,15 +282,14 @@ void extract_features(Search::search& sch, uint32_t idx, vector &ec) uint64_t additional_offset = val_namespace*offset_const; for(size_t j=0; j< 10; j++) - { - additional_offset += j* 1023; + { additional_offset += j* 1023; add_feature(ex, temp[j]+ additional_offset , val_namespace, mask, multiplier); } size_t count=0; for (features fs : *data->ex) - { fs.sum_feat_sq = (float) fs.size(); - count+= fs.size(); - } + { fs.sum_feat_sq = (float) fs.size(); + count+= fs.size(); + } size_t new_count; float new_weight; @@ -312,15 +308,15 @@ void get_valid_actions(Search::search &sch, v_array & valid_action, ui { if(idx<=n) // SHIFT valid_action.push_back( SHIFT ); if(stack_depth >=2) // RIGHT - valid_action.push_back( REDUCE_RIGHT ); + valid_action.push_back( REDUCE_RIGHT ); if(stack_depth >=1 && state!=0 && idx<=n) // LEFT valid_action.push_back( REDUCE_LEFT ); - } + } else if(sys == arc_eager) // assume root is in N+1 { temp.erase(); for(size_t i=0; i<=4; i++) temp.push_back(1); - if (idx>n) + if (idx>n) { temp[SHIFT] = 0; temp[REDUCE_RIGHT] = 0; } @@ -330,15 +326,16 @@ void get_valid_actions(Search::search &sch, v_array & valid_action, ui else if (idx<=n+1 && heads[stack.last()] == my_null) temp[REDUCE] = 0; - if(stack_depth ==0) + if(stack_depth ==0) { temp[REDUCE_LEFT] = 0; temp[REDUCE_RIGHT] = 0; - } else + } + else { if(heads[stack.last()]!=my_null) temp[REDUCE_LEFT] = 0; - if(idx <=n && heads[idx] != my_null) temp[REDUCE_RIGHT] = 0; + if(idx <=n && heads[idx] != my_null) temp[REDUCE_RIGHT] = 0; } - for(uint32_t i=1; i<=4; i++){ - if(temp[i]) valid_action.push_back(i); + for(uint32_t i=1; i<=4; i++) + { if(temp[i]) valid_action.push_back(i); } } } @@ -365,7 +362,7 @@ void get_eager_action_cost(Search::search &sch, uint32_t idx, uint64_t n) } if(idx <=n && (gold_heads[idx] == stack[i])) { if(stack[i]!=0) action_loss[SHIFT] += 1; - if(stack[i]!=last) action_loss[REDUCE_RIGHT]+=1; + if(stack[i]!=last) action_loss[REDUCE_RIGHT]+=1; } } for(size_t i = idx; i<=n+1; i++) @@ -387,7 +384,7 @@ void get_hybrid_action_cost(Search::search &sch, size_t idx, uint64_t n) { task_data *data = sch.get_task_data(); v_array &action_loss = data->action_loss, &stack = data->stack, &gold_heads=data->gold_heads; size_t size = stack.size(); - size_t last = (size==0) ? 0 : stack.last(); + size_t last = (size==0) ? 0 : stack.last(); for(size_t i = 1; i<= 3; i++) action_loss[i] = 0; @@ -409,7 +406,7 @@ void get_hybrid_action_cost(Search::search &sch, size_t idx, uint64_t n) if(gold_heads[last] >= idx) action_loss[REDUCE_RIGHT] +=1; - + for(size_t i = idx; i<=n; i++) if(gold_heads[i] == (uint32_t)last) action_loss[REDUCE_RIGHT] +=1; @@ -440,7 +437,7 @@ void get_cost_to_go_losses(Search::search &sch, v_array>& go if(is_valid(i, valid_actions)) gold_action_losses.push_back(make_pair(i, (float)action_loss[i])); if(sys==arc_eager && is_valid(REDUCE, valid_actions)) - gold_action_losses.push_back(make_pair(REDUCE, (float)action_loss[REDUCE])); + gold_action_losses.push_back(make_pair(REDUCE, (float)action_loss[REDUCE])); } } @@ -460,7 +457,7 @@ void get_gold_actions(Search::search &sch, uint32_t idx, uint64_t n, v_array &actions if(is_valid(SHIFT, actions)) actions_onelearner.push_back(SHIFT); if(sys==arc_eager && is_valid(REDUCE, actions)) actions_onelearner.push_back(2+2*num_label); if(left_label!=my_null && is_valid(REDUCE_RIGHT, actions)) actions_onelearner.push_back(1+right_label); - if(left_label!=my_null && is_valid(REDUCE_LEFT, actions)) actions_onelearner.push_back(1+left_label+num_label); + if(left_label!=my_null && is_valid(REDUCE_LEFT, actions)) actions_onelearner.push_back(1+left_label+num_label); if(left_label==my_null && is_valid(REDUCE_RIGHT, actions)) - for(uint32_t i=0; i< num_label; i++) - if(i!=data->root_label-1) - actions_onelearner.push_back(i+2); + for(uint32_t i=0; i< num_label; i++) + if(i!=data->root_label-1) + actions_onelearner.push_back(i+2); if(left_label==my_null && is_valid(REDUCE_LEFT, actions)) - for(uint32_t i=0; iroot_label-1) - actions_onelearner.push_back((uint32_t)(i+2+num_label)); -} + for(uint32_t i=0; iroot_label-1) + actions_onelearner.push_back((uint32_t)(i+2+num_label)); +} void setup(Search::search& sch, vector& ec) { task_data *data = sch.get_task_data(); @@ -547,8 +544,8 @@ void run(Search::search& sch, vector& ec) stack.push_back((data->root_label==0&&sys==arc_hybrid)?0:1); for(size_t i=0; i<6; i++) for(size_t j=0; jchildren[i][j] = 0; - for(size_t i=0; ichildren[i][j] = 0; + for(size_t i=0; i& ec) bool computedFeatures = false; if(sch.predictNeedsExample()) { extract_features(sch, idx, ec); - computedFeatures = true; + computedFeatures = true; } get_valid_actions(sch, valid_actions, idx, n, (uint64_t) stack.size(), stack.empty() ? 0 : stack.last()); if(sys == arc_hybrid) @@ -574,7 +571,7 @@ void run(Search::search& sch, vector& ec) // get gold tag labels left_label = stack.empty()?my_null:gold_tags[stack.last()]; if(sys==arc_hybrid) right_label = stack.empty()?my_null:gold_tags[stack.last()]; - else if(sys==arc_eager) right_label = idx<=n? gold_tags[idx] : (uint32_t) data->root_label; + else if(sys==arc_eager) right_label = idx<=n? gold_tags[idx] : (uint32_t) data->root_label; else THROW("unknown transition system"); uint32_t a_id = 0, t_id = 0; @@ -589,7 +586,7 @@ void run(Search::search& sch, vector& ec) .predict(); } else - { get_gold_actions(sch, idx, n, gold_actions); + { get_gold_actions(sch, idx, n, gold_actions); convert_to_onelearner_actions(sch, gold_actions, gold_action_temp, left_label, right_label); convert_to_onelearner_actions(sch, valid_actions, valid_action_temp, my_null, my_null); a_id = P.set_tag((ptag) count) @@ -639,14 +636,14 @@ void run(Search::search& sch, vector& ec) count++; if (a_id != SHIFT && a_id != REDUCE ) - { if ((!computedFeatures) && sch.predictNeedsExample()) + { if ((!computedFeatures) && sch.predictNeedsExample()) extract_features(sch, idx, ec); if(cost_to_go) { gold_action_losses.erase(); for(size_t i=1; i<= data->num_label; i++) gold_action_losses.push_back(make_pair((action)i, i != (a_id==REDUCE_LEFT?left_label:right_label))); - t_id = P.set_tag((ptag) count) + t_id = P.set_tag((ptag) count) .set_input(*(data->ex)) .set_allowed(gold_action_losses) .set_condition_range(count-1, sch.get_history_length(), 'p') @@ -660,20 +657,20 @@ void run(Search::search& sch, vector& ec) .erase_alloweds() .set_condition_range(count-1, sch.get_history_length(), 'p') .set_learner_id(a_id-1) - .predict(); + .predict(); } } } count++; if(sys==arc_hybrid) - idx = (uint32_t)transition_hybrid(sch, a_id, idx, t_id, n); + idx = (uint32_t)transition_hybrid(sch, a_id, idx, t_id, n); else if(sys==arc_eager) - idx = (uint32_t)transition_eager(sch, a_id, idx, t_id, n); + idx = (uint32_t)transition_eager(sch, a_id, idx, t_id, n); } if(sys==arc_hybrid) - { heads[stack.last()] = 0; - tags[stack.last()] = (uint32_t)data->root_label; - sch.loss((gold_heads[stack.last()] != heads[stack.last()])); + { heads[stack.last()] = 0; + tags[stack.last()] = (uint32_t)data->root_label; + sch.loss((gold_heads[stack.last()] != heads[stack.last()])); } if (sch.output().good()) for(size_t i=1; i<=n; i++) diff --git a/vowpalwabbit/search_dep_parser.h b/vowpalwabbit/search_dep_parser.h index 5608cd4293e..e66370c7a05 100644 --- a/vowpalwabbit/search_dep_parser.h +++ b/vowpalwabbit/search_dep_parser.h @@ -10,7 +10,7 @@ namespace DepParserTask { void initialize(Search::search&, size_t&, po::variables_map&); void finish(Search::search&); - void run(Search::search&, std::vector&); - void setup(Search::search&, std::vector&); +void run(Search::search&, std::vector&); +void setup(Search::search&, std::vector&); extern Search::search_task task; } diff --git a/vowpalwabbit/search_entityrelationtask.h b/vowpalwabbit/search_entityrelationtask.h index 2f39b146c29..71db4779efa 100644 --- a/vowpalwabbit/search_entityrelationtask.h +++ b/vowpalwabbit/search_entityrelationtask.h @@ -10,6 +10,6 @@ namespace EntityRelationTask { void initialize(Search::search&, size_t&, po::variables_map&); void finish(Search::search&); - void run(Search::search&, std::vector&); +void run(Search::search&, std::vector&); extern Search::search_task task; } diff --git a/vowpalwabbit/search_graph.cc b/vowpalwabbit/search_graph.cc index 0152fe836ae..909464fdc93 100644 --- a/vowpalwabbit/search_graph.cc +++ b/vowpalwabbit/search_graph.cc @@ -233,9 +233,9 @@ void add_edge_features_group_fn(task_data&D, float fv, uint64_t fx) { example*node = D.cur_node; uint64_t fx2 = fx / (uint64_t)D.multiplier; for (size_t k=0; kfeature_space[neighbor_namespace].push_back(fv * D.neighbor_predictions[k], (uint64_t)(( fx2 + 348919043 * k ) * D.multiplier) & (uint64_t)D.mask); - } + { if (D.neighbor_predictions[k] == 0.) continue; + node->feature_space[neighbor_namespace].push_back(fv * D.neighbor_predictions[k], (uint64_t)(( fx2 + 348919043 * k ) * D.multiplier) & (uint64_t)D.mask); + } } void add_edge_features_single_fn(task_data&D, float fv, uint64_t fx) diff --git a/vowpalwabbit/search_graph.h b/vowpalwabbit/search_graph.h index 11a25480b72..b3bd6448d52 100644 --- a/vowpalwabbit/search_graph.h +++ b/vowpalwabbit/search_graph.h @@ -10,8 +10,8 @@ namespace GraphTask { void initialize(Search::search&, size_t&, po::variables_map&); void finish(Search::search&); - void setup(Search::search&, std::vector&); - void run(Search::search&, std::vector&); - void takedown(Search::search&, std::vector&); +void setup(Search::search&, std::vector&); +void run(Search::search&, std::vector&); +void takedown(Search::search&, std::vector&); extern Search::search_task task; } diff --git a/vowpalwabbit/search_hooktask.h b/vowpalwabbit/search_hooktask.h index 5ebcb245c3f..0157db86b4b 100644 --- a/vowpalwabbit/search_hooktask.h +++ b/vowpalwabbit/search_hooktask.h @@ -10,9 +10,9 @@ namespace HookTask { void initialize(Search::search&, size_t&, po::variables_map&); void finish(Search::search&); - void run(Search::search&, std::vector&); - void run_setup(Search::search&, std::vector&); - void run_takedown(Search::search&, std::vector&); +void run(Search::search&, std::vector&); +void run_setup(Search::search&, std::vector&); +void run_takedown(Search::search&, std::vector&); extern Search::search_task task; struct task_data diff --git a/vowpalwabbit/search_multiclasstask.h b/vowpalwabbit/search_multiclasstask.h index 6993c8b3aef..d0b1ed6f64a 100644 --- a/vowpalwabbit/search_multiclasstask.h +++ b/vowpalwabbit/search_multiclasstask.h @@ -10,6 +10,6 @@ namespace MulticlassTask { void initialize(Search::search&, size_t&, po::variables_map&); void finish(Search::search&); - void run(Search::search&, std::vector&); +void run(Search::search&, std::vector&); extern Search::search_task task; } diff --git a/vowpalwabbit/search_sequencetask.cc b/vowpalwabbit/search_sequencetask.cc index ef99cc88096..13265060f91 100644 --- a/vowpalwabbit/search_sequencetask.cc +++ b/vowpalwabbit/search_sequencetask.cc @@ -23,7 +23,7 @@ void initialize(Search::search& sch, size_t& /*num_actions*/, po::variables_map& 0); } - void run(Search::search& sch, vector& ec) +void run(Search::search& sch, vector& ec) { Search::predictor P(sch, (ptag)0); for (size_t i=0; il.multi.label; diff --git a/vowpalwabbit/search_sequencetask.h b/vowpalwabbit/search_sequencetask.h index 0be6d026a9d..c303f458c29 100644 --- a/vowpalwabbit/search_sequencetask.h +++ b/vowpalwabbit/search_sequencetask.h @@ -9,7 +9,7 @@ license as described in the file LICENSE. namespace SequenceTask { void initialize(Search::search&, size_t&, po::variables_map&); - void run(Search::search&, std::vector&); +void run(Search::search&, std::vector&); extern Search::search_task task; } @@ -17,23 +17,23 @@ namespace SequenceSpanTask { void initialize(Search::search&, size_t&, po::variables_map&); void finish(Search::search&); - void run(Search::search&, std::vector&); - void setup(Search::search&, std::vector&); - void takedown(Search::search&, std::vector&); +void run(Search::search&, std::vector&); +void setup(Search::search&, std::vector&); +void takedown(Search::search&, std::vector&); extern Search::search_task task; } namespace SequenceTaskCostToGo { void initialize(Search::search&, size_t&, po::variables_map&); - void run(Search::search&, std::vector&); +void run(Search::search&, std::vector&); extern Search::search_task task; } namespace ArgmaxTask { void initialize(Search::search&, size_t&, po::variables_map&); - void run(Search::search&, std::vector&); +void run(Search::search&, std::vector&); void finish(Search::search&); extern Search::search_task task; } @@ -42,6 +42,6 @@ namespace SequenceTask_DemoLDF { void initialize(Search::search&, size_t&, po::variables_map&); void finish(Search::search&); - void run(Search::search&, std::vector&); +void run(Search::search&, std::vector&); extern Search::search_task task; } diff --git a/vowpalwabbit/simple_label.cc b/vowpalwabbit/simple_label.cc index a8e1657c14a..0596c211b88 100644 --- a/vowpalwabbit/simple_label.cc +++ b/vowpalwabbit/simple_label.cc @@ -101,9 +101,9 @@ label_parser simple_label = {default_simple_label, parse_simple_label, void print_update(vw& all, example& ec) { if (all.sd->weighted_examples >= all.sd->dump_interval && !all.quiet && !all.bfgs) - { all.sd->print_update(all.holdout_set_off, all.current_pass, ec.l.simple.label, ec.pred.scalar, - ec.num_features, all.progress_add, all.progress_arg); - } + { all.sd->print_update(all.holdout_set_off, all.current_pass, ec.l.simple.label, ec.pred.scalar, + ec.num_features, all.progress_add, all.progress_arg); + } } void output_and_account_example(vw& all, example& ec) diff --git a/vowpalwabbit/stagewise_poly.cc b/vowpalwabbit/stagewise_poly.cc index 27083669bf6..5895e1e5302 100644 --- a/vowpalwabbit/stagewise_poly.cc +++ b/vowpalwabbit/stagewise_poly.cc @@ -66,13 +66,11 @@ struct stagewise_poly inline uint64_t stride_shift(const stagewise_poly &poly, uint64_t idx) -{ - return idx << poly.all->weights.stride_shift(); +{ return idx << poly.all->weights.stride_shift(); } inline uint64_t stride_un_shift(const stagewise_poly &poly, uint64_t idx) -{ - return idx >> poly.all->weights.stride_shift(); +{ return idx >> poly.all->weights.stride_shift(); } inline uint64_t do_ft_offset(const stagewise_poly &poly, uint64_t idx) @@ -87,8 +85,7 @@ inline uint64_t un_ft_offset(const stagewise_poly &poly, uint64_t idx) return idx; else { while (idx < poly.synth_ec.ft_offset) - { - idx += poly.all->length() << poly.all->weights.stride_shift(); + { idx += poly.all->length() << poly.all->weights.stride_shift(); } return idx - poly.synth_ec.ft_offset; } @@ -272,17 +269,17 @@ void sort_data_update_support(stagewise_poly &poly) for (uint64_t i = 0; i != poly.all->length(); ++i) { uint64_t wid = stride_shift(poly, i); if (!parent_get(poly, wid) && wid != constant_feat_masked(poly)) - { float weightsal = (fabsf(poly.all->weights[wid]) - * poly.all->weights[poly.all->normalized_idx + (wid)]) - /* - * here's some depth penalization code. It was found to not improve - * statistical performance, and meanwhile it is verified as giving - * a nontrivial computational hit, thus commented out. - * - * - poly.magic_argument - * sqrtf(min_depths_get(poly, stride_shift(poly, i)) * 1.0 / poly.num_examples) - */ - ; + { float weightsal = (fabsf(poly.all->weights[wid]) + * poly.all->weights[poly.all->normalized_idx + (wid)]) + /* + * here's some depth penalization code. It was found to not improve + * statistical performance, and meanwhile it is verified as giving + * a nontrivial computational hit, thus commented out. + * + * - poly.magic_argument + * sqrtf(min_depths_get(poly, stride_shift(poly, i)) * 1.0 / poly.num_examples) + */ + ; if (weightsal > tolerance) { assert(heap_end >= poly.sd); assert(heap_end <= poly.sd + num_new_features); @@ -382,17 +379,16 @@ void synthetic_reset(stagewise_poly &poly, example &ec) } void synthetic_decycle(stagewise_poly &poly) -{ - features& fs = poly.synth_ec.feature_space[tree_atomics]; +{ features& fs = poly.synth_ec.feature_space[tree_atomics]; for (size_t i = 0; i < fs.size(); ++i) - { assert(cycle_get(poly, fs.indicies[i])); - cycle_toggle(poly, fs.indicies[i]); - } + { assert(cycle_get(poly, fs.indicies[i])); + cycle_toggle(poly, fs.indicies[i]); + } } void synthetic_create_rec(stagewise_poly &poly, float v, uint64_t findex) { //Note: need to un_ft_shift since gd::foreach_feature bakes in the offset. - uint64_t wid_atomic = wid_mask(poly, un_ft_offset(poly, findex)); + uint64_t wid_atomic = wid_mask(poly, un_ft_offset(poly, findex)); uint64_t wid_cur = child_wid(poly, wid_atomic, poly.synth_rec_f.weight_index); assert(wid_atomic % stride_shift(poly, 1) == 0); @@ -426,7 +422,7 @@ void synthetic_create_rec(stagewise_poly &poly, float v, uint64_t findex) ++poly.depths[poly.cur_depth]; #endif //DEBUG - feature temp ={v * poly.synth_rec_f.x, wid_cur}; + feature temp = {v * poly.synth_rec_f.x, wid_cur}; poly.synth_ec.feature_space[tree_atomics].push_back(temp.x, temp.weight_index); poly.synth_ec.num_features++; @@ -616,10 +612,9 @@ void finish(stagewise_poly &poly) void save_load(stagewise_poly &poly, io_buf &model_file, bool read, bool text) { if (model_file.files.size() > 0) - { - stringstream msg; - bin_text_read_write_fixed(model_file, (char *) poly.depthsbits, (uint32_t)depthsbits_sizeof(poly), "", read, msg, text); - } + { stringstream msg; + bin_text_read_write_fixed(model_file, (char *) poly.depthsbits, (uint32_t)depthsbits_sizeof(poly), "", read, msg, text); + } //unfortunately, following can't go here since save_load called before gd::save_load and thus //weight vector state uninitialiazed. //#ifdef DEBUG diff --git a/vowpalwabbit/unique_sort.cc b/vowpalwabbit/unique_sort.cc index 09b6b3e822d..bbed0d4a8fb 100644 --- a/vowpalwabbit/unique_sort.cc +++ b/vowpalwabbit/unique_sort.cc @@ -17,8 +17,7 @@ void unique_features(features& fs, int max) for (features::iterator_all i = ++range.begin(); i != end; ++i) if (i.index() != last_index.index()) if (i != ++last_index) - { - last_index.value() = i.value(); + { last_index.value() = i.value(); last_index.index() = i.index(); if (!fs.space_names.empty()) last_index.audit() = i.audit(); @@ -29,8 +28,7 @@ void unique_features(features& fs, int max) } void unique_sort_features(uint64_t parse_mask, example* ae) -{ - for (features& fs : *ae) +{ for (features& fs : *ae) if (fs.sort(parse_mask)) unique_features(fs); diff --git a/vowpalwabbit/v_array.h b/vowpalwabbit/v_array.h index 70782ceb573..0243fbbac24 100644 --- a/vowpalwabbit/v_array.h +++ b/vowpalwabbit/v_array.h @@ -74,19 +74,19 @@ template struct v_array void erase() { if (++erase_count & erase_point) - { resize(_end-_begin); - erase_count = 0; - } + { resize(_end-_begin); + erase_count = 0; + } for (T*item = _begin; item != _end; ++item) - item->~T(); + item->~T(); _end = _begin; } void delete_v() - { if (_begin != nullptr) { - for (T*item = _begin; item != _end; ++item) - item->~T(); - free(_begin); - } + { if (_begin != nullptr) + { for (T*item = _begin; item != _end; ++item) + item->~T(); + free(_begin); + } _begin = _end = end_array = nullptr; } void push_back(const T& new_ele) diff --git a/vowpalwabbit/vw.h b/vowpalwabbit/vw.h index 9f8a5979d44..52901c940b6 100644 --- a/vowpalwabbit/vw.h +++ b/vowpalwabbit/vw.h @@ -26,13 +26,13 @@ namespace VW (1) Some commandline parameters do not make sense as a library. (2) The code is not yet reentrant. */ - vw* initialize(std::string s, io_buf* model=nullptr, bool skipModelLoad=false); +vw* initialize(std::string s, io_buf* model=nullptr, bool skipModelLoad=false); vw* initialize(int argc, char* argv[], io_buf* model=nullptr, bool skipModelLoad = false); - vw* seed_vw_model(vw* vw_model, std::string extra_args); +vw* seed_vw_model(vw* vw_model, std::string extra_args); - void cmd_string_replace_value( std::stringstream*& ss, std::string flag_to_replace, std::string new_value ); +void cmd_string_replace_value( std::stringstream*& ss, std::string flag_to_replace, std::string new_value ); - char** get_argv_from_string(std::string s, int& argc); +char** get_argv_from_string(std::string s, int& argc); const char* are_features_compatible(vw& vw1, vw& vw2); /* @@ -61,7 +61,7 @@ example* read_example(vw& all, std::string example_line); //The more complex way to create an example. //after you create and fill feature_spaces, get an example with everything filled in. - example* import_example(vw& all, std::string label, primitive_feature_space* features, size_t len); +example* import_example(vw& all, std::string label, primitive_feature_space* features, size_t len); // callers must free memory using release_example // this interface must be used with care as finish_example is a no-op for these examples. @@ -105,32 +105,32 @@ void clear_example_data(example&); // don't clear the label primitive_feature_space* export_example(vw& all, example* e, size_t& len); void releaseFeatureSpace(primitive_feature_space* features, size_t len); - void save_predictor(vw& all, std::string reg_name); +void save_predictor(vw& all, std::string reg_name); void save_predictor(vw& all, io_buf& buf); // inlines //First create the hash of a namespace. - inline uint32_t hash_space(vw& all, std::string s) +inline uint32_t hash_space(vw& all, std::string s) { substring ss; ss.begin = (char*)s.c_str(); ss.end = ss.begin + s.length(); return (uint32_t)all.p->hasher(ss,hash_base); } - inline uint32_t hash_space_static(std::string s, std::string hash) +inline uint32_t hash_space_static(std::string s, std::string hash) { substring ss; ss.begin = (char*)s.c_str(); ss.end = ss.begin + s.length(); return (uint32_t)getHasher(hash)(ss, hash_base); } //Then use it as the seed for hashing features. - inline uint32_t hash_feature(vw& all, std::string s, uint64_t u) +inline uint32_t hash_feature(vw& all, std::string s, uint64_t u) { substring ss; ss.begin = (char*)s.c_str(); ss.end = ss.begin + s.length(); return (uint32_t)(all.p->hasher(ss,u) & all.parse_mask); } - inline uint32_t hash_feature_static(std::string s, unsigned long u, std::string h, uint32_t num_bits) +inline uint32_t hash_feature_static(std::string s, unsigned long u, std::string h, uint32_t num_bits) { substring ss; ss.begin = (char*)s.c_str(); ss.end = ss.begin + s.length(); @@ -146,20 +146,17 @@ inline uint32_t hash_feature_cstr(vw& all, char* fstr, unsigned long u) } inline float get_weight(vw& all, uint32_t index, uint32_t offset) -{ - return all.weights[(index << all.weights.stride_shift()) + offset]; +{ return all.weights[(index << all.weights.stride_shift()) + offset]; } inline void set_weight(vw& all, uint32_t index, uint32_t offset, float value) -{ - all.weights[(index << all.weights.stride_shift()) + offset] = value; +{ all.weights[(index << all.weights.stride_shift()) + offset] = value; } inline uint32_t num_weights(vw& all) { return (uint32_t)all.length();} inline uint32_t get_stride(vw& all) -{ - return (uint32_t)(1 << all.weights.stride_shift()); +{ return (uint32_t)(1 << all.weights.stride_shift()); } } diff --git a/vowpalwabbit/vw_exception.cc b/vowpalwabbit/vw_exception.cc index 1e62a2a3c0b..d104284bc48 100644 --- a/vowpalwabbit/vw_exception.cc +++ b/vowpalwabbit/vw_exception.cc @@ -50,28 +50,24 @@ void vw_trace(const char* filename, int linenumber, const char* fmt, ...) } struct StopWatchData -{ - LARGE_INTEGER frequency_; - LARGE_INTEGER startTime_; +{ LARGE_INTEGER frequency_; + LARGE_INTEGER startTime_; }; StopWatch::StopWatch() : data(new StopWatchData()) -{ - if (!::QueryPerformanceFrequency(&data->frequency_)) throw "Error with QueryPerformanceFrequency"; - ::QueryPerformanceCounter(&data->startTime_); +{ if (!::QueryPerformanceFrequency(&data->frequency_)) throw "Error with QueryPerformanceFrequency"; + ::QueryPerformanceCounter(&data->startTime_); } StopWatch::~StopWatch() -{ - delete data; +{ delete data; } double StopWatch::MilliSeconds() const -{ - LARGE_INTEGER now; - ::QueryPerformanceCounter(&now); +{ LARGE_INTEGER now; + ::QueryPerformanceCounter(&now); - return double(now.QuadPart - data->startTime_.QuadPart) / (double(data->frequency_.QuadPart) / 1000); + return double(now.QuadPart - data->startTime_.QuadPart) / (double(data->frequency_.QuadPart) / 1000); } bool launchDebugger() diff --git a/vowpalwabbit/vw_exception.h b/vowpalwabbit/vw_exception.h index 173c40ea64b..1c82210e901 100644 --- a/vowpalwabbit/vw_exception.h +++ b/vowpalwabbit/vw_exception.h @@ -49,14 +49,14 @@ void vw_trace(const char* filename, int linenumber, const char* fmt, ...); struct StopWatchData; -class StopWatch { - StopWatchData* data; +class StopWatch +{ StopWatchData* data; public: - StopWatch(); - ~StopWatch(); + StopWatch(); + ~StopWatch(); - double MilliSeconds() const; + double MilliSeconds() const; }; // Equivalent to System::Diagnostics::Debugger::Launch(); diff --git a/vowpalwabbit/vw_validate.cc b/vowpalwabbit/vw_validate.cc index 085b9093873..6b1747fbdcd 100644 --- a/vowpalwabbit/vw_validate.cc +++ b/vowpalwabbit/vw_validate.cc @@ -9,12 +9,11 @@ license as described in the file LICENSE. namespace VW { - void validate_version(vw& all) - { - if (all.model_file_ver < LAST_COMPATIBLE_VERSION || all.model_file_ver > PACKAGE_VERSION) - THROW("Model has possibly incompatible version! " << all.model_file_ver.to_string()); - } - +void validate_version(vw& all) +{ if (all.model_file_ver < LAST_COMPATIBLE_VERSION || all.model_file_ver > PACKAGE_VERSION) + THROW("Model has possibly incompatible version! " << all.model_file_ver.to_string()); +} + void validate_min_max_label(vw& all) { if (all.sd->max_label < all.sd->min_label) THROW("Max label cannot be less than min label."); @@ -27,6 +26,6 @@ void validate_default_bits(vw& all, uint32_t local_num_bits) void validate_num_bits(vw& all) { if (all.num_bits > sizeof(size_t) * 8 - 3) - THROW("Only " << sizeof(size_t) * 8 - 3 << " or fewer bits allowed. If this is a serious limit, speak up."); + THROW("Only " << sizeof(size_t) * 8 - 3 << " or fewer bits allowed. If this is a serious limit, speak up."); } } diff --git a/vowpalwabbit/vwdll.cpp b/vowpalwabbit/vwdll.cpp index 104cdc831cb..44ac0e32ae2 100644 --- a/vowpalwabbit/vwdll.cpp +++ b/vowpalwabbit/vwdll.cpp @@ -26,253 +26,252 @@ // in the hashing to avoid allocating an intermediate string. extern "C" -{ - using namespace std; +{ using namespace std; #ifdef USE_CODECVT VW_DLL_MEMBER VW_HANDLE VW_CALLING_CONV VW_Initialize(const char16_t * pstrArgs) - { std::wstring_convert, char16_t> convert; - std::string sa(convert.to_bytes(pstrArgs)); - return VW_InitializeA(sa.c_str()); - } +{ std::wstring_convert, char16_t> convert; + std::string sa(convert.to_bytes(pstrArgs)); + return VW_InitializeA(sa.c_str()); +} #endif - VW_DLL_MEMBER VW_HANDLE VW_CALLING_CONV VW_InitializeA(const char * pstrArgs) - { string s(pstrArgs); - vw* all = VW::initialize(s); - return static_cast(all); - } +VW_DLL_MEMBER VW_HANDLE VW_CALLING_CONV VW_InitializeA(const char * pstrArgs) +{ string s(pstrArgs); + vw* all = VW::initialize(s); + return static_cast(all); +} - VW_DLL_MEMBER void VW_CALLING_CONV VW_Finish_Passes(VW_HANDLE handle) - { vw * pointer = static_cast(handle); - if (pointer->numpasses > 1) - { adjust_used_index(*pointer); - pointer->do_reset_source = true; - VW::start_parser(*pointer); - LEARNER::generic_driver(*pointer); - VW::end_parser(*pointer); - } +VW_DLL_MEMBER void VW_CALLING_CONV VW_Finish_Passes(VW_HANDLE handle) +{ vw * pointer = static_cast(handle); + if (pointer->numpasses > 1) + { adjust_used_index(*pointer); + pointer->do_reset_source = true; + VW::start_parser(*pointer); + LEARNER::generic_driver(*pointer); + VW::end_parser(*pointer); } +} - VW_DLL_MEMBER void VW_CALLING_CONV VW_Finish(VW_HANDLE handle) - { vw * pointer = static_cast(handle); - release_parser_datastructures(*pointer); - VW::finish(*pointer); - } +VW_DLL_MEMBER void VW_CALLING_CONV VW_Finish(VW_HANDLE handle) +{ vw * pointer = static_cast(handle); + release_parser_datastructures(*pointer); + VW::finish(*pointer); +} - VW_DLL_MEMBER VW_EXAMPLE VW_CALLING_CONV VW_ImportExample(VW_HANDLE handle, const char * label, VW_FEATURE_SPACE* features, size_t len) - { vw * pointer = static_cast(handle); - VW::primitive_feature_space * f = reinterpret_cast( features ); - return static_cast(VW::import_example(*pointer, label, f, len)); - } +VW_DLL_MEMBER VW_EXAMPLE VW_CALLING_CONV VW_ImportExample(VW_HANDLE handle, const char * label, VW_FEATURE_SPACE* features, size_t len) +{ vw * pointer = static_cast(handle); + VW::primitive_feature_space * f = reinterpret_cast( features ); + return static_cast(VW::import_example(*pointer, label, f, len)); +} - VW_DLL_MEMBER VW_FEATURE_SPACE VW_CALLING_CONV VW_ExportExample(VW_HANDLE handle, VW_EXAMPLE e, size_t * plen) - { vw* pointer = static_cast(handle); - example* ex = static_cast(e); - return static_cast(VW::export_example(*pointer, ex, *plen)); - } +VW_DLL_MEMBER VW_FEATURE_SPACE VW_CALLING_CONV VW_ExportExample(VW_HANDLE handle, VW_EXAMPLE e, size_t * plen) +{ vw* pointer = static_cast(handle); + example* ex = static_cast(e); + return static_cast(VW::export_example(*pointer, ex, *plen)); +} - VW_DLL_MEMBER void VW_CALLING_CONV VW_ReleaseFeatureSpace(VW_FEATURE_SPACE* features, size_t len) - { VW::primitive_feature_space * f = reinterpret_cast( features ); - VW::releaseFeatureSpace(f, len); - } +VW_DLL_MEMBER void VW_CALLING_CONV VW_ReleaseFeatureSpace(VW_FEATURE_SPACE* features, size_t len) +{ VW::primitive_feature_space * f = reinterpret_cast( features ); + VW::releaseFeatureSpace(f, len); +} #ifdef USE_CODECVT - VW_DLL_MEMBER VW_EXAMPLE VW_CALLING_CONV VW_ReadExample(VW_HANDLE handle, const char16_t * line) - { std::wstring_convert, char16_t> convert; - std::string sa(convert.to_bytes(line)); - return VW_ReadExampleA(handle, sa.c_str()); - } +VW_DLL_MEMBER VW_EXAMPLE VW_CALLING_CONV VW_ReadExample(VW_HANDLE handle, const char16_t * line) +{ std::wstring_convert, char16_t> convert; + std::string sa(convert.to_bytes(line)); + return VW_ReadExampleA(handle, sa.c_str()); +} #endif - VW_DLL_MEMBER VW_EXAMPLE VW_CALLING_CONV VW_ReadExampleA(VW_HANDLE handle, const char * line) - { vw * pointer = static_cast(handle); - // BUGBUG: I really dislike this const_cast. should VW really change the input string? - return static_cast(VW::read_example(*pointer, const_cast(line))); - } +VW_DLL_MEMBER VW_EXAMPLE VW_CALLING_CONV VW_ReadExampleA(VW_HANDLE handle, const char * line) +{ vw * pointer = static_cast(handle); + // BUGBUG: I really dislike this const_cast. should VW really change the input string? + return static_cast(VW::read_example(*pointer, const_cast(line))); +} - VW_DLL_MEMBER void VW_CALLING_CONV VW_StartParser(VW_HANDLE handle) - { vw * pointer = static_cast(handle); - VW::start_parser(*pointer); - } +VW_DLL_MEMBER void VW_CALLING_CONV VW_StartParser(VW_HANDLE handle) +{ vw * pointer = static_cast(handle); + VW::start_parser(*pointer); +} - VW_DLL_MEMBER void VW_CALLING_CONV VW_EndParser(VW_HANDLE handle) - { vw * pointer = static_cast(handle); - VW::end_parser(*pointer); - } +VW_DLL_MEMBER void VW_CALLING_CONV VW_EndParser(VW_HANDLE handle) +{ vw * pointer = static_cast(handle); + VW::end_parser(*pointer); +} - VW_DLL_MEMBER VW_EXAMPLE VW_CALLING_CONV VW_GetExample(VW_HANDLE handle) - { vw * pointer = static_cast(handle); - parser * parser_pointer = static_cast(pointer->p); - return static_cast(VW::get_example(parser_pointer)); - } +VW_DLL_MEMBER VW_EXAMPLE VW_CALLING_CONV VW_GetExample(VW_HANDLE handle) +{ vw * pointer = static_cast(handle); + parser * parser_pointer = static_cast(pointer->p); + return static_cast(VW::get_example(parser_pointer)); +} - VW_DLL_MEMBER float VW_CALLING_CONV VW_GetLabel(VW_EXAMPLE e) - { return VW::get_label(static_cast(e)); - } +VW_DLL_MEMBER float VW_CALLING_CONV VW_GetLabel(VW_EXAMPLE e) +{ return VW::get_label(static_cast(e)); +} - VW_DLL_MEMBER float VW_CALLING_CONV VW_GetTopicPrediction(VW_EXAMPLE e, size_t i) - { return VW::get_topic_prediction(static_cast(e), i); - } +VW_DLL_MEMBER float VW_CALLING_CONV VW_GetTopicPrediction(VW_EXAMPLE e, size_t i) +{ return VW::get_topic_prediction(static_cast(e), i); +} - VW_DLL_MEMBER float VW_CALLING_CONV VW_GetImportance(VW_EXAMPLE e) - { return VW::get_importance(static_cast(e)); - } +VW_DLL_MEMBER float VW_CALLING_CONV VW_GetImportance(VW_EXAMPLE e) +{ return VW::get_importance(static_cast(e)); +} - VW_DLL_MEMBER float VW_CALLING_CONV VW_GetInitial(VW_EXAMPLE e) - { return VW::get_initial(static_cast(e)); - } +VW_DLL_MEMBER float VW_CALLING_CONV VW_GetInitial(VW_EXAMPLE e) +{ return VW::get_initial(static_cast(e)); +} - VW_DLL_MEMBER float VW_CALLING_CONV VW_GetPrediction(VW_EXAMPLE e) - { return VW::get_prediction(static_cast(e)); - } +VW_DLL_MEMBER float VW_CALLING_CONV VW_GetPrediction(VW_EXAMPLE e) +{ return VW::get_prediction(static_cast(e)); +} - VW_DLL_MEMBER float VW_CALLING_CONV VW_GetCostSensitivePrediction(VW_EXAMPLE e) - { return VW::get_cost_sensitive_prediction(static_cast(e)); - } +VW_DLL_MEMBER float VW_CALLING_CONV VW_GetCostSensitivePrediction(VW_EXAMPLE e) +{ return VW::get_cost_sensitive_prediction(static_cast(e)); +} - VW_DLL_MEMBER void* VW_CALLING_CONV VW_GetMultilabelPredictions(VW_EXAMPLE e, size_t* plen) - { return VW::get_multilabel_predictions(static_cast(e), *plen); - } +VW_DLL_MEMBER void* VW_CALLING_CONV VW_GetMultilabelPredictions(VW_EXAMPLE e, size_t* plen) +{ return VW::get_multilabel_predictions(static_cast(e), *plen); +} - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_GetTagLength(VW_EXAMPLE e) - { return VW::get_tag_length(static_cast(e)); - } +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_GetTagLength(VW_EXAMPLE e) +{ return VW::get_tag_length(static_cast(e)); +} - VW_DLL_MEMBER const char* VW_CALLING_CONV VW_GetTag(VW_EXAMPLE e) - { return VW::get_tag(static_cast(e)); - } +VW_DLL_MEMBER const char* VW_CALLING_CONV VW_GetTag(VW_EXAMPLE e) +{ return VW::get_tag(static_cast(e)); +} - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_GetFeatureNumber(VW_EXAMPLE e) - { return VW::get_feature_number(static_cast(e)); - } +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_GetFeatureNumber(VW_EXAMPLE e) +{ return VW::get_feature_number(static_cast(e)); +} - VW_DLL_MEMBER VW_FEATURE VW_CALLING_CONV VW_GetFeatures(VW_HANDLE handle, VW_EXAMPLE e, size_t* plen) - { vw* pointer = static_cast(handle); - return VW::get_features(*pointer, static_cast(e), *plen); - } +VW_DLL_MEMBER VW_FEATURE VW_CALLING_CONV VW_GetFeatures(VW_HANDLE handle, VW_EXAMPLE e, size_t* plen) +{ vw* pointer = static_cast(handle); + return VW::get_features(*pointer, static_cast(e), *plen); +} - VW_DLL_MEMBER void VW_CALLING_CONV VW_ReturnFeatures(VW_FEATURE f) - { VW::return_features(static_cast(f)); - } - VW_DLL_MEMBER void VW_CALLING_CONV VW_FinishExample(VW_HANDLE handle, VW_EXAMPLE e) - { vw * pointer = static_cast(handle); - VW::finish_example(*pointer, static_cast(e)); - } +VW_DLL_MEMBER void VW_CALLING_CONV VW_ReturnFeatures(VW_FEATURE f) +{ VW::return_features(static_cast(f)); +} +VW_DLL_MEMBER void VW_CALLING_CONV VW_FinishExample(VW_HANDLE handle, VW_EXAMPLE e) +{ vw * pointer = static_cast(handle); + VW::finish_example(*pointer, static_cast(e)); +} #ifdef USE_CODECVT - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashSpace(VW_HANDLE handle, const char16_t * s) - { std::wstring_convert, char16_t> convert; - std::string sa(convert.to_bytes(s)); - return VW_HashSpaceA(handle,sa.c_str()); - } +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashSpace(VW_HANDLE handle, const char16_t * s) +{ std::wstring_convert, char16_t> convert; + std::string sa(convert.to_bytes(s)); + return VW_HashSpaceA(handle,sa.c_str()); +} - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashSpaceStatic(const char16_t * s, const char16_t * h) - { std::wstring_convert, char16_t> convert; - std::string sa(convert.to_bytes(s)); - std::string ha(convert.to_bytes(h)); +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashSpaceStatic(const char16_t * s, const char16_t * h) +{ std::wstring_convert, char16_t> convert; + std::string sa(convert.to_bytes(s)); + std::string ha(convert.to_bytes(h)); - return VW_HashSpaceStaticA(sa.c_str(), ha.c_str()); - } + return VW_HashSpaceStaticA(sa.c_str(), ha.c_str()); +} #endif - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashSpaceA(VW_HANDLE handle, const char * s) - { vw * pointer = static_cast(handle); - string str(s); - return VW::hash_space(*pointer, str); - } +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashSpaceA(VW_HANDLE handle, const char * s) +{ vw * pointer = static_cast(handle); + string str(s); + return VW::hash_space(*pointer, str); +} - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashSpaceStaticA(const char * s, const char* h = "strings") - { string str(s); - string hash(h); - return VW::hash_space_static(str, hash); - } +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashSpaceStaticA(const char * s, const char* h = "strings") +{ string str(s); + string hash(h); + return VW::hash_space_static(str, hash); +} #ifdef USE_CODECVT - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashFeature(VW_HANDLE handle, const char16_t * s, unsigned long u) - { std::wstring_convert, char16_t> convert; - std::string sa(convert.to_bytes(s)); - return VW_HashFeatureA(handle,sa.c_str(),u); - } +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashFeature(VW_HANDLE handle, const char16_t * s, unsigned long u) +{ std::wstring_convert, char16_t> convert; + std::string sa(convert.to_bytes(s)); + return VW_HashFeatureA(handle,sa.c_str(),u); +} - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashFeatureStatic(const char16_t * s, unsigned long u, const char16_t * h, unsigned int num_bits) - { std::wstring_convert, char16_t> convert; - std::string sa(convert.to_bytes(s)); - std::string ha(convert.to_bytes(h)); - return VW_HashFeatureStaticA(sa.c_str(), u, ha.c_str(), num_bits); - } +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashFeatureStatic(const char16_t * s, unsigned long u, const char16_t * h, unsigned int num_bits) +{ std::wstring_convert, char16_t> convert; + std::string sa(convert.to_bytes(s)); + std::string ha(convert.to_bytes(h)); + return VW_HashFeatureStaticA(sa.c_str(), u, ha.c_str(), num_bits); +} #endif - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashFeatureA(VW_HANDLE handle, const char * s, unsigned long u) - { vw * pointer = static_cast(handle); - string str(s); - return VW::hash_feature(*pointer, str, u); - } - - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashFeatureStaticA(const char * s, unsigned long u, const char * h = "strings", unsigned int num_bits = 18) - { string str(s); - string hash(h); - return VW::hash_feature_static(str, u, hash, num_bits); - } +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashFeatureA(VW_HANDLE handle, const char * s, unsigned long u) +{ vw * pointer = static_cast(handle); + string str(s); + return VW::hash_feature(*pointer, str, u); +} - VW_DLL_MEMBER void VW_CALLING_CONV VW_AddLabel(VW_EXAMPLE e, float label, float weight, float base) - { example* ex = static_cast(e); - return VW::add_label(ex, label, weight, base); - } +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashFeatureStaticA(const char * s, unsigned long u, const char * h = "strings", unsigned int num_bits = 18) +{ string str(s); + string hash(h); + return VW::hash_feature_static(str, u, hash, num_bits); +} - VW_DLL_MEMBER void VW_CALLING_CONV VW_AddStringLabel(VW_HANDLE handle, VW_EXAMPLE e, const char* label) - { vw * pointer = static_cast(handle); - example* ex = static_cast(e); - VW::parse_example_label(*pointer, *ex, label); - } +VW_DLL_MEMBER void VW_CALLING_CONV VW_AddLabel(VW_EXAMPLE e, float label, float weight, float base) +{ example* ex = static_cast(e); + return VW::add_label(ex, label, weight, base); +} - VW_DLL_MEMBER float VW_CALLING_CONV VW_Learn(VW_HANDLE handle, VW_EXAMPLE e) - { vw * pointer = static_cast(handle); - example * ex = static_cast(e); - pointer->learn(ex); - return VW::get_prediction(ex); - } +VW_DLL_MEMBER void VW_CALLING_CONV VW_AddStringLabel(VW_HANDLE handle, VW_EXAMPLE e, const char* label) +{ vw * pointer = static_cast(handle); + example* ex = static_cast(e); + VW::parse_example_label(*pointer, *ex, label); +} - VW_DLL_MEMBER float VW_CALLING_CONV VW_Predict(VW_HANDLE handle, VW_EXAMPLE e) - { vw * pointer = static_cast(handle); - example * ex = static_cast(e); - pointer->l->predict(*ex); - //BUG: The below method may return garbage as it assumes a certain structure for ex->ld - //which may not be the actual one used (e.g., for cost-sensitive multi-class learning) - return VW::get_prediction(ex); - } +VW_DLL_MEMBER float VW_CALLING_CONV VW_Learn(VW_HANDLE handle, VW_EXAMPLE e) +{ vw * pointer = static_cast(handle); + example * ex = static_cast(e); + pointer->learn(ex); + return VW::get_prediction(ex); +} - VW_DLL_MEMBER float VW_CALLING_CONV VW_PredictCostSensitive(VW_HANDLE handle, VW_EXAMPLE e) - { vw * pointer = static_cast(handle); - example * ex = static_cast(e); - pointer->l->predict(*ex); - return VW::get_cost_sensitive_prediction(ex); - } +VW_DLL_MEMBER float VW_CALLING_CONV VW_Predict(VW_HANDLE handle, VW_EXAMPLE e) +{ vw * pointer = static_cast(handle); + example * ex = static_cast(e); + pointer->l->predict(*ex); + //BUG: The below method may return garbage as it assumes a certain structure for ex->ld + //which may not be the actual one used (e.g., for cost-sensitive multi-class learning) + return VW::get_prediction(ex); +} - VW_DLL_MEMBER float VW_CALLING_CONV VW_Get_Weight(VW_HANDLE handle, size_t index, size_t offset) - { vw* pointer = static_cast(handle); - return VW::get_weight(*pointer, (uint32_t) index, (uint32_t) offset); - } +VW_DLL_MEMBER float VW_CALLING_CONV VW_PredictCostSensitive(VW_HANDLE handle, VW_EXAMPLE e) +{ vw * pointer = static_cast(handle); + example * ex = static_cast(e); + pointer->l->predict(*ex); + return VW::get_cost_sensitive_prediction(ex); +} - VW_DLL_MEMBER void VW_CALLING_CONV VW_Set_Weight(VW_HANDLE handle, size_t index, size_t offset, float value) - { vw* pointer = static_cast(handle); - return VW::set_weight(*pointer, (uint32_t) index, (uint32_t)offset, value); - } +VW_DLL_MEMBER float VW_CALLING_CONV VW_Get_Weight(VW_HANDLE handle, size_t index, size_t offset) +{ vw* pointer = static_cast(handle); + return VW::get_weight(*pointer, (uint32_t) index, (uint32_t) offset); +} - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_Num_Weights(VW_HANDLE handle) - { vw* pointer = static_cast(handle); - return VW::num_weights(*pointer); - } +VW_DLL_MEMBER void VW_CALLING_CONV VW_Set_Weight(VW_HANDLE handle, size_t index, size_t offset, float value) +{ vw* pointer = static_cast(handle); + return VW::set_weight(*pointer, (uint32_t) index, (uint32_t)offset, value); +} - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_Get_Stride(VW_HANDLE handle) - { vw* pointer = static_cast(handle); - return VW::get_stride(*pointer); - } +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_Num_Weights(VW_HANDLE handle) +{ vw* pointer = static_cast(handle); + return VW::num_weights(*pointer); +} - VW_DLL_MEMBER void VW_CALLING_CONV VW_SaveModel(VW_HANDLE handle) - { vw* pointer = static_cast(handle); +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_Get_Stride(VW_HANDLE handle) +{ vw* pointer = static_cast(handle); + return VW::get_stride(*pointer); +} - string name = pointer->final_regressor_name; - if (name.empty()) - { return; - } +VW_DLL_MEMBER void VW_CALLING_CONV VW_SaveModel(VW_HANDLE handle) +{ vw* pointer = static_cast(handle); - return VW::save_predictor(*pointer, name); + string name = pointer->final_regressor_name; + if (name.empty()) + { return; } + + return VW::save_predictor(*pointer, name); +} } diff --git a/vowpalwabbit/vwdll.h b/vowpalwabbit/vwdll.h index 4516af41764..fe44190917d 100644 --- a/vowpalwabbit/vwdll.h +++ b/vowpalwabbit/vwdll.h @@ -52,74 +52,74 @@ extern "C" #define VW_TYPE_SAFE_NULL NULL #endif - typedef void * VW_HANDLE; - typedef void * VW_EXAMPLE; - typedef void * VW_LABEL; - typedef void * VW_FEATURE_SPACE; - typedef void * VW_FEATURE; +typedef void * VW_HANDLE; +typedef void * VW_EXAMPLE; +typedef void * VW_LABEL; +typedef void * VW_FEATURE_SPACE; +typedef void * VW_FEATURE; - const VW_HANDLE INVALID_VW_HANDLE = VW_TYPE_SAFE_NULL; - const VW_HANDLE INVALID_VW_EXAMPLE = VW_TYPE_SAFE_NULL; +const VW_HANDLE INVALID_VW_HANDLE = VW_TYPE_SAFE_NULL; +const VW_HANDLE INVALID_VW_EXAMPLE = VW_TYPE_SAFE_NULL; #ifdef USE_CODECVT - VW_DLL_MEMBER VW_HANDLE VW_CALLING_CONV VW_Initialize(const char16_t * pstrArgs); +VW_DLL_MEMBER VW_HANDLE VW_CALLING_CONV VW_Initialize(const char16_t * pstrArgs); #endif - VW_DLL_MEMBER VW_HANDLE VW_CALLING_CONV VW_InitializeA(const char * pstrArgs); +VW_DLL_MEMBER VW_HANDLE VW_CALLING_CONV VW_InitializeA(const char * pstrArgs); - VW_DLL_MEMBER void VW_CALLING_CONV VW_Finish_Passes(VW_HANDLE handle); - VW_DLL_MEMBER void VW_CALLING_CONV VW_Finish(VW_HANDLE handle); +VW_DLL_MEMBER void VW_CALLING_CONV VW_Finish_Passes(VW_HANDLE handle); +VW_DLL_MEMBER void VW_CALLING_CONV VW_Finish(VW_HANDLE handle); - VW_DLL_MEMBER VW_EXAMPLE VW_CALLING_CONV VW_ImportExample(VW_HANDLE handle, const char * label, VW_FEATURE_SPACE * features, size_t len); +VW_DLL_MEMBER VW_EXAMPLE VW_CALLING_CONV VW_ImportExample(VW_HANDLE handle, const char * label, VW_FEATURE_SPACE * features, size_t len); - VW_DLL_MEMBER VW_FEATURE_SPACE VW_CALLING_CONV VW_ExportExample(VW_HANDLE handle, VW_EXAMPLE e, size_t* plen); - VW_DLL_MEMBER void VW_CALLING_CONV VW_ReleaseFeatureSpace(VW_FEATURE_SPACE * features, size_t len); +VW_DLL_MEMBER VW_FEATURE_SPACE VW_CALLING_CONV VW_ExportExample(VW_HANDLE handle, VW_EXAMPLE e, size_t* plen); +VW_DLL_MEMBER void VW_CALLING_CONV VW_ReleaseFeatureSpace(VW_FEATURE_SPACE * features, size_t len); #ifdef USE_CODECVT - VW_DLL_MEMBER VW_EXAMPLE VW_CALLING_CONV VW_ReadExample(VW_HANDLE handle, const char16_t * line); +VW_DLL_MEMBER VW_EXAMPLE VW_CALLING_CONV VW_ReadExample(VW_HANDLE handle, const char16_t * line); #endif - VW_DLL_MEMBER VW_EXAMPLE VW_CALLING_CONV VW_ReadExampleA(VW_HANDLE handle, const char * line); - - VW_DLL_MEMBER void VW_CALLING_CONV VW_StartParser(VW_HANDLE handle); - VW_DLL_MEMBER void VW_CALLING_CONV VW_EndParser(VW_HANDLE handle); - - VW_DLL_MEMBER VW_EXAMPLE VW_CALLING_CONV VW_GetExample(VW_HANDLE handle); - VW_DLL_MEMBER void VW_CALLING_CONV VW_FinishExample(VW_HANDLE handle, VW_EXAMPLE e); - VW_DLL_MEMBER float VW_CALLING_CONV VW_GetLabel(VW_EXAMPLE e); - VW_DLL_MEMBER float VW_CALLING_CONV VW_GetImportance(VW_EXAMPLE e); - VW_DLL_MEMBER float VW_CALLING_CONV VW_GetInitial(VW_EXAMPLE e); - VW_DLL_MEMBER float VW_CALLING_CONV VW_GetPrediction(VW_EXAMPLE e); - VW_DLL_MEMBER float VW_CALLING_CONV VW_GetCostSensitivePrediction(VW_EXAMPLE e); - VW_DLL_MEMBER void* VW_CALLING_CONV VW_GetMultilabelPredictions(VW_EXAMPLE e, size_t* plen); - VW_DLL_MEMBER float VW_CALLING_CONV VW_GetTopicPrediction(VW_EXAMPLE e, size_t i); - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_GetTagLength(VW_EXAMPLE e); - VW_DLL_MEMBER const char* VW_CALLING_CONV VW_GetTag(VW_EXAMPLE e); - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_GetFeatureNumber(VW_EXAMPLE e); - VW_DLL_MEMBER VW_FEATURE VW_CALLING_CONV VW_GetFeatures(VW_HANDLE handle, VW_EXAMPLE e, size_t* plen); - VW_DLL_MEMBER void VW_CALLING_CONV VW_ReturnFeatures(VW_FEATURE f); +VW_DLL_MEMBER VW_EXAMPLE VW_CALLING_CONV VW_ReadExampleA(VW_HANDLE handle, const char * line); + +VW_DLL_MEMBER void VW_CALLING_CONV VW_StartParser(VW_HANDLE handle); +VW_DLL_MEMBER void VW_CALLING_CONV VW_EndParser(VW_HANDLE handle); + +VW_DLL_MEMBER VW_EXAMPLE VW_CALLING_CONV VW_GetExample(VW_HANDLE handle); +VW_DLL_MEMBER void VW_CALLING_CONV VW_FinishExample(VW_HANDLE handle, VW_EXAMPLE e); +VW_DLL_MEMBER float VW_CALLING_CONV VW_GetLabel(VW_EXAMPLE e); +VW_DLL_MEMBER float VW_CALLING_CONV VW_GetImportance(VW_EXAMPLE e); +VW_DLL_MEMBER float VW_CALLING_CONV VW_GetInitial(VW_EXAMPLE e); +VW_DLL_MEMBER float VW_CALLING_CONV VW_GetPrediction(VW_EXAMPLE e); +VW_DLL_MEMBER float VW_CALLING_CONV VW_GetCostSensitivePrediction(VW_EXAMPLE e); +VW_DLL_MEMBER void* VW_CALLING_CONV VW_GetMultilabelPredictions(VW_EXAMPLE e, size_t* plen); +VW_DLL_MEMBER float VW_CALLING_CONV VW_GetTopicPrediction(VW_EXAMPLE e, size_t i); +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_GetTagLength(VW_EXAMPLE e); +VW_DLL_MEMBER const char* VW_CALLING_CONV VW_GetTag(VW_EXAMPLE e); +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_GetFeatureNumber(VW_EXAMPLE e); +VW_DLL_MEMBER VW_FEATURE VW_CALLING_CONV VW_GetFeatures(VW_HANDLE handle, VW_EXAMPLE e, size_t* plen); +VW_DLL_MEMBER void VW_CALLING_CONV VW_ReturnFeatures(VW_FEATURE f); #ifdef USE_CODECVT - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashSpace(VW_HANDLE handle, const char16_t * s); - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashSpaceStatic(const char16_t * s, const char16_t * h); +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashSpace(VW_HANDLE handle, const char16_t * s); +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashSpaceStatic(const char16_t * s, const char16_t * h); #endif - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashSpaceA(VW_HANDLE handle, const char * s); - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashSpaceStaticA(const char * s, const char* h); +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashSpaceA(VW_HANDLE handle, const char * s); +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashSpaceStaticA(const char * s, const char* h); #ifdef USE_CODECVT - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashFeature(VW_HANDLE handle, const char16_t * s, unsigned long u); - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashFeatureStatic(const char16_t * s, unsigned long u, const char16_t * h, unsigned int num_bits); +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashFeature(VW_HANDLE handle, const char16_t * s, unsigned long u); +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashFeatureStatic(const char16_t * s, unsigned long u, const char16_t * h, unsigned int num_bits); #endif - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashFeatureA(VW_HANDLE handle, const char * s, unsigned long u); - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashFeatureStaticA(const char * s, unsigned long u, const char * h, unsigned int num_bits); +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashFeatureA(VW_HANDLE handle, const char * s, unsigned long u); +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_HashFeatureStaticA(const char * s, unsigned long u, const char * h, unsigned int num_bits); - VW_DLL_MEMBER float VW_CALLING_CONV VW_Learn(VW_HANDLE handle, VW_EXAMPLE e); - VW_DLL_MEMBER float VW_CALLING_CONV VW_Predict(VW_HANDLE handle, VW_EXAMPLE e); - VW_DLL_MEMBER float VW_CALLING_CONV VW_PredictCostSensitive(VW_HANDLE handle, VW_EXAMPLE e); - VW_DLL_MEMBER void VW_CALLING_CONV VW_AddLabel(VW_EXAMPLE e, float label, float weight, float base); - VW_DLL_MEMBER void VW_CALLING_CONV VW_AddStringLabel(VW_HANDLE handle, VW_EXAMPLE e, const char* label); +VW_DLL_MEMBER float VW_CALLING_CONV VW_Learn(VW_HANDLE handle, VW_EXAMPLE e); +VW_DLL_MEMBER float VW_CALLING_CONV VW_Predict(VW_HANDLE handle, VW_EXAMPLE e); +VW_DLL_MEMBER float VW_CALLING_CONV VW_PredictCostSensitive(VW_HANDLE handle, VW_EXAMPLE e); +VW_DLL_MEMBER void VW_CALLING_CONV VW_AddLabel(VW_EXAMPLE e, float label, float weight, float base); +VW_DLL_MEMBER void VW_CALLING_CONV VW_AddStringLabel(VW_HANDLE handle, VW_EXAMPLE e, const char* label); - VW_DLL_MEMBER float VW_CALLING_CONV VW_Get_Weight(VW_HANDLE handle, size_t index, size_t offset); - VW_DLL_MEMBER void VW_CALLING_CONV VW_Set_Weight(VW_HANDLE handle, size_t index, size_t offset, float value); - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_Num_Weights(VW_HANDLE handle); - VW_DLL_MEMBER size_t VW_CALLING_CONV VW_Get_Stride(VW_HANDLE handle); +VW_DLL_MEMBER float VW_CALLING_CONV VW_Get_Weight(VW_HANDLE handle, size_t index, size_t offset); +VW_DLL_MEMBER void VW_CALLING_CONV VW_Set_Weight(VW_HANDLE handle, size_t index, size_t offset, float value); +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_Num_Weights(VW_HANDLE handle); +VW_DLL_MEMBER size_t VW_CALLING_CONV VW_Get_Stride(VW_HANDLE handle); - VW_DLL_MEMBER void VW_CALLING_CONV VW_SaveModel(VW_HANDLE handle); +VW_DLL_MEMBER void VW_CALLING_CONV VW_SaveModel(VW_HANDLE handle); #ifdef __cplusplus }