From 8d3f4b8c260f617fa37ada9d3b9328e5d5dd3af4 Mon Sep 17 00:00:00 2001 From: Mathieu Stefani Date: Fri, 25 Sep 2015 15:18:11 -0400 Subject: [PATCH] Upstreamed a bunch of changes to RTBKit --- jml/arch/exception.h | 3 +- jml/arch/exception_handler.cc | 12 +- jml/arch/node_exception_tracing.cc | 2 - jml/boosting/boosted_stumps.cc | 2 - jml/boosting/decision_tree_generator.cc | 46 +- jml/boosting/dense_features.cc | 2 - jml/boosting/feature_info.cc | 2 +- jml/boosting/stump_training_core.h | 42 +- jml/boosting/training_data.cc | 4 +- jml/jml-build | 1 - jml/judy/judy.mk | 3 +- jml/utils/exc_assert.cc | 10 +- jml/utils/exc_assert.h | 8 +- jml/utils/exc_check.cc | 12 +- jml/utils/exc_check.h | 8 +- jml/utils/filter_streams.cc | 49 +- jml/utils/filter_streams.h | 46 +- jml/utils/json_parsing.cc | 104 +- jml/utils/json_parsing.h | 4 +- jml/utils/lightweight_hash.h | 315 +- jml/utils/string_functions.cc | 4 +- jml/utils/testing/filter_streams_test.cc | 27 +- jml/utils/testing/lightweight_hash_test.cc | 25 +- jml/utils/utils.mk | 2 - rtbkit/common/bids.h | 2 +- .../core/agent_configuration/agent_config.cc | 15 +- .../core/agent_configuration/agent_config.h | 5 + .../agent_configuration.mk | 3 +- rtbkit/core/agent_configuration/fees.cc | 231 ++ rtbkit/core/agent_configuration/fees.h | 92 + .../agent_configuration/include_exclude.h | 26 +- .../rtb_agent_config_validator_test.cc | 237 ++ .../rtb_agent_configuration_testing.mk | 7 + .../testing/rtb_fees_test.cc | 209 ++ rtbkit/core/banker/local_banker.cc | 2 +- rtbkit/core/banker/null_banker.cc | 7 +- rtbkit/core/banker/null_banker.h | 3 +- .../core/post_auction/post_auction_runner.cc | 21 +- rtbkit/core/router/augmentation_loop.cc | 41 +- rtbkit/core/router/augmentation_loop.h | 20 +- rtbkit/core/router/filters/static_filters.cc | 4 +- rtbkit/core/router/filters/static_filters.h | 1 + rtbkit/core/router/router.cc | 121 +- rtbkit/core/router/router.h | 36 +- rtbkit/core/router/router_runner.cc | 18 +- rtbkit/core/router/router_stack.cc | 1 + .../availability_agent/availability_agent.cc | 2 +- .../availability_agent/availability_check.h | 2 +- rtbkit/examples/bid_request_endpoint.cc | 11 +- rtbkit/examples/integration_endpoints.cc | 11 +- rtbkit/examples/multi_agent.cc | 10 +- rtbkit/examples/rtbkit_integration_test.cc | 2 + .../adserver/http_adserver_connector.h | 4 +- rtbkit/plugins/analytics/analytics_runner.cc | 42 +- .../bid_request/openrtb_bid_request_parser.cc | 26 +- .../bid_request/openrtb_bid_request_parser.h | 2 +- .../bidder_interface/http_bidder_interface.cc | 81 +- rtbkit/plugins/bidding_agent/bidding_agent.cc | 17 +- rtbkit/plugins/bidding_agent/bidding_agent.h | 3 +- .../exchange/bidswitch_exchange_connector.cc | 12 + .../plugins/exchange/http_auction_handler.cc | 2 +- .../plugins/exchange/http_auction_handler.h | 2 +- .../exchange/http_exchange_connector.h | 35 +- .../exchange/openrtb_exchange_connector.cc | 1 + .../exchange/rtbkit_exchange_connector.cc | 10 + .../testing/adx_exchange_connector_test.cc | 1 + .../bidswitch_exchange_connector_adx_test.cc | 1 + .../bidswitch_exchange_connector_test.cc | 1 + .../testing/gumgum_exchange_connector_test.cc | 1 + .../testing/mopub_exchange_connector_test.cc | 1 + .../testing/nexage_exchange_connector_test.cc | 1 + .../openrtb_exchange_connector_test.cc | 1 + .../testing/rtbkit_exchange_connector_test.cc | 1 + .../rubicon_exchange_connector_test.cc | 1 + .../testing/smaato_exchange_connector_test.cc | 1 + rtbkit/testing/bid_stack.h | 9 +- rtbkit/testing/bidder_test.cc | 1 + rtbkit/testing/exchange_parsing_from_file.cc | 1 + soa/js/js_utils.h | 4 +- soa/jsoncpp/json_reader.cpp | 4 + soa/launcher/launcher.h | 12 +- soa/logger/kvp_logger_interface.cc | 11 +- soa/logger/kvp_logger_mongodb.cc | 3 +- soa/logger/logger.mk | 4 +- soa/logger/logger_metrics_interface.cc | 8 +- soa/logger/logger_metrics_interface.h | 229 +- soa/logger/logger_metrics_mongo.cc | 50 +- soa/logger/logger_metrics_mongo.h | 50 +- soa/logger/logger_metrics_term.cc | 52 +- soa/logger/logger_metrics_term.h | 43 +- soa/logger/logger_metrics_void.h | 54 +- soa/logger/testing/logger_metrics_config.json | 2 +- soa/logger/testing/logger_metrics_test.cc | 58 +- soa/logger/testing/logger_testing.mk | 2 +- soa/service/async_writer_source.cc | 166 +- soa/service/async_writer_source.h | 82 +- soa/service/carbon_connector.cc | 10 +- soa/service/carbon_connector.h | 4 +- soa/service/chunked_http_endpoint.cc | 2 +- soa/service/chunked_http_endpoint.h | 6 +- soa/service/connection_handler.cc | 2 +- soa/service/endpoint.cc | 21 +- soa/service/endpoint.h | 1 - soa/service/epoll_loop.cc | 14 +- soa/service/epoll_loop.h | 8 +- soa/service/epoller.h | 4 +- soa/service/fs_utils.cc | 12 - soa/service/fs_utils.h | 26 - soa/service/http_endpoint.cc | 2 +- soa/service/http_header.cc | 3 - soa/service/http_named_endpoint.cc | 2 +- soa/service/http_rest_proxy.cc | 6 +- soa/service/http_rest_proxy.h | 2 + soa/service/logs.h | 2 +- soa/service/loop_monitor.cc | 34 +- soa/service/message_loop.cc | 31 +- soa/service/message_loop.h | 20 +- soa/service/redis.cc | 45 +- soa/service/s3.cc | 4 +- soa/service/s3.h | 3 +- soa/service/service.mk | 1 - soa/service/service_base.cc | 6 +- soa/service/service_base.h | 10 +- soa/service/stat_aggregator.cc | 4 +- soa/service/stat_aggregator.h | 2 +- soa/service/stats_events.h | 2 +- soa/service/tcp_client.cc | 6 +- soa/service/testing/http_client_bench.cc | 16 +- soa/service/testing/http_client_test.cc | 1 + soa/service/testing/http_header_test.cc | 7 - soa/service/testing/mongo_temporary_server.cc | 20 +- .../testing/py/mongo_temp_server_wrapping.cc | 2 +- soa/service/testing/redis_temporary_server.h | 7 +- soa/service/testing/runner_stress_test.cc | 6 +- soa/service/testing/service_testing.mk | 9 +- soa/service/testing/test_http_services.cc | 1 - soa/types/basic_value_descriptions.h | 4 +- soa/types/date.cc | 2 +- soa/types/id.cc | 49 +- soa/types/id.h | 7 +- soa/types/json_parsing.h | 11 +- soa/types/json_printing.cc | 326 +- soa/types/json_printing.h | 231 +- soa/types/testing/id_test.cc | 43 +- soa/types/testing/string_test.cc | 19 - soa/types/types.mk | 3 +- soa/utils/fixtures.h | 9 - soa/utils/mongo_init.h | 33 + soa/valgrind.supp | 20 + types/dtoa.c | 2787 ----------------- types/dtoa.h | 106 - 151 files changed, 2514 insertions(+), 4465 deletions(-) delete mode 160000 jml/jml-build create mode 100644 rtbkit/core/agent_configuration/fees.cc create mode 100644 rtbkit/core/agent_configuration/fees.h create mode 100644 rtbkit/core/agent_configuration/testing/rtb_agent_config_validator_test.cc create mode 100644 rtbkit/core/agent_configuration/testing/rtb_agent_configuration_testing.mk create mode 100644 rtbkit/core/agent_configuration/testing/rtb_fees_test.cc create mode 100644 soa/utils/mongo_init.h delete mode 100644 types/dtoa.c delete mode 100644 types/dtoa.h diff --git a/jml/arch/exception.h b/jml/arch/exception.h index 60328683c..493da1ecc 100644 --- a/jml/arch/exception.h +++ b/jml/arch/exception.h @@ -25,7 +25,6 @@ #include #include -#include "jml/compiler/compiler.h" #include "stdarg.h" #include "exception_handler.h" @@ -34,7 +33,7 @@ namespace ML { class Exception : public std::exception { public: Exception(const std::string & msg); - Exception(const char * msg, ...) JML_FORMAT_STRING(2, 3); + Exception(const char * msg, ...); Exception(const char * msg, va_list ap); Exception(int errnum, const std::string & msg, const char * function = 0); virtual ~Exception() throw(); diff --git a/jml/arch/exception_handler.cc b/jml/arch/exception_handler.cc index 8687c52b2..75dffc9ad 100644 --- a/jml/arch/exception_handler.cc +++ b/jml/arch/exception_handler.cc @@ -4,7 +4,6 @@ */ -#include #include #include #include @@ -137,14 +136,6 @@ void default_exception_tracer(void * object, const std::type_info * tinfo) heapDemangled = char_demangle(tinfo->name()); demangled = heapDemangled; } - const char *nodeName = ""; - struct utsname utsName; - if (::uname(&utsName) == 0) { - nodeName = utsName.nodename; - } - else { - cerr << "error calling uname\n"; - } auto pid = getpid(); auto tid = gettid(); @@ -154,9 +145,8 @@ void default_exception_tracer(void * object, const std::type_info * tinfo) "---------------------------\n" "time: %s\n" "type: %s\n" - "node: %s\n" "pid: %d; tid: %d\n", - datetime, demangled, nodeName, pid, tid); + datetime, demangled, pid, tid); if (heapDemangled) { free(heapDemangled); } diff --git a/jml/arch/node_exception_tracing.cc b/jml/arch/node_exception_tracing.cc index 2f4d31adf..48da53e65 100644 --- a/jml/arch/node_exception_tracing.cc +++ b/jml/arch/node_exception_tracing.cc @@ -15,14 +15,12 @@ __thread BacktraceInfo * current_backtrace = nullptr; namespace { -#if 0 void cleanup_current_backtrace(void * arg) { BacktraceInfo * p = (BacktraceInfo *)arg; delete p; p = nullptr; } -#endif void ensure_current_backtrace() { diff --git a/jml/boosting/boosted_stumps.cc b/jml/boosting/boosted_stumps.cc index bb535856a..5f5a4a258 100644 --- a/jml/boosting/boosted_stumps.cc +++ b/jml/boosting/boosted_stumps.cc @@ -515,7 +515,6 @@ namespace { static const std::string BOOSTED_STUMPS_MAGIC = "BOOSTED_STUMPS"; static const compact_size_t BOOSTED_STUMPS_VERSION = 4; -#if 0 void serialize_dist(const distribution & dist, DB::Store_Writer & store) { @@ -558,7 +557,6 @@ void reconstitute_dist(distribution & dist, store >> dist[i]; } } -#endif } // file scope diff --git a/jml/boosting/decision_tree_generator.cc b/jml/boosting/decision_tree_generator.cc index 05193ef36..19719b663 100644 --- a/jml/boosting/decision_tree_generator.cc +++ b/jml/boosting/decision_tree_generator.cc @@ -21,7 +21,6 @@ #include #include #include -#include using namespace std; @@ -306,23 +305,6 @@ struct Tree_Accum { if (z < best_z) { - if (!isfinite(arg)) { // will never happen - static std::mutex mutex; - std::unique_lock guard(mutex); - - cerr << "Best arg had non-finite split" << endl; - cerr << "feature = " << fs.print(feature) << endl; - cerr << "info = " << fs.info(feature) << endl; - cerr << "z = " << z << endl; - cerr << "arg = " << arg << endl; - - cerr << "Beating previous best" << endl; - cerr << "feature = " << fs.print(best_feature) << endl; - cerr << "info = " << fs.info(best_feature) << endl; - cerr << "z = " << best_z << endl; - cerr << "arg = " << best_arg << endl; - } - if (tracer || print_feat) tracer("tree accum", 4) << w.print() << endl; // A better one. This replaces whatever we had accumulated so @@ -339,18 +321,6 @@ struct Tree_Accum { float add(const Feature & feature, const W & w, float arg, double missing) { - // If the decision tree generator is having a really tough time - // separating the classes, and it's a bucketed feature, than it - // may send back a -INF for arg (which means split on missing or - // not missing), even if there is no missing feature, due to - // numerical issues. Since the decision tree can't handle a split - // point of -INIFINITY, we return that we don't want this split - // point so that it will continue looking for something better. - - if (!isfinite(arg)) { - return Z::none; - } - float z = calc_z.non_missing(w, missing); return add_z(feature, w, arg, z); } @@ -904,19 +874,17 @@ train_recursive(Thread_Context & context, } if (split.feature() == MISSING_FEATURE) { - //cerr << "in_class = " << in_class << endl; - //cerr << "weights = " << endl; - //for (unsigned i = 0; i < weights.size(); ++i) - // cerr << weights[i][0] << " " << weights[i][1] << endl; - cerr << "in_class.total() = " << in_class.total() << endl; - cerr << "in_class non zero = " << (in_class != 0.0).count() << endl; - cerr << "in_class.size() = " << in_class.size() << endl; - +/* + cerr << "in_class = " << in_class << endl; + cerr << "weights = " << endl; + for (unsigned i = 0; i < weights.size(); ++i) + cerr << weights[i][0] << " " << weights[i][1] << endl; cerr << "example count = " << data.example_count() << endl; cerr << "class_weights = " << class_weights << endl; cerr << "total_weight = " << total_weight << endl; - + */ cerr << "WARNING: no feature found in decision tree split" << endl; + cerr << "warning : didn't print a sometimes awfully long print in decision_tree_generator.cc" << endl; Tree::Leaf * result = tree.new_leaf(); *result = leaf; return result; diff --git a/jml/boosting/dense_features.cc b/jml/boosting/dense_features.cc index b282962a7..b5372fc95 100644 --- a/jml/boosting/dense_features.cc +++ b/jml/boosting/dense_features.cc @@ -806,7 +806,6 @@ get_sizes(Parse_Context & context, return boost::make_tuple(row_count, var_count, header); } -#if 0 boost::tuple get_sizes(const std::string & filename, vector & row_start_ofs) @@ -815,7 +814,6 @@ get_sizes(const std::string & filename, return get_sizes(context, row_start_ofs); } -#endif } // file scope diff --git a/jml/boosting/feature_info.cc b/jml/boosting/feature_info.cc index 89f0c9fdc..571e4da1d 100644 --- a/jml/boosting/feature_info.cc +++ b/jml/boosting/feature_info.cc @@ -290,7 +290,7 @@ void Mutable_Feature_Info::parse(Parse_Context & context) set_categorical(make_sp(new Mutable_Categorical_Info()), STRING); else if (context.match_literal("REAL")) type_ = REAL; - else if (context.match_literal("INUTILE")) + else if (context.match_literal("UNUTILE")) type_ = INUTILE; else context.exception("Feature_Info::parse(): unknown type"); break; diff --git a/jml/boosting/stump_training_core.h b/jml/boosting/stump_training_core.h index 1ad3197ce..c1a712004 100644 --- a/jml/boosting/stump_training_core.h +++ b/jml/boosting/stump_training_core.h @@ -719,10 +719,10 @@ struct Stump_Trainer { double missing; if (!results.start(feature, w, missing)) return Z::worst; - float Zvalue = results.add(feature, w, 0.5, missing); + float Z = results.add(feature, w, 0.5, missing); results.finish(feature); - return Zvalue; + return Z; } /** Test a presence variable. */ @@ -751,10 +751,10 @@ struct Stump_Trainer { double missing; if (!results.start(feature, w, missing)) return Z::worst; - float Zvalue = results.add_presence(feature, w, 0.5, missing); + float Z = results.add_presence(feature, w, 0.5, missing); results.finish(feature); - return Zvalue; + return Z; } template @@ -815,14 +815,14 @@ struct Stump_Trainer { /* One candidate split point is -INF, which lets us split only based upon missing or not. */ - float Zvalue = Z::worst; + float Z = Z::worst; if (i != 0) { - Zvalue = results.add(feature, w, -INFINITY, missing); + Z = results.add(feature, w, -INFINITY, missing); if (debug) cerr << "added split " << -INFINITY << " with " << missing - << " missing and score " << Zvalue << endl; + << " missing and score " << Z << endl; } float prev = index[i].value(); @@ -853,15 +853,15 @@ struct Stump_Trainer { /* Add this split point. */ float arg = prev; float new_Z = results.add(feature, w, arg, missing); - Zvalue = std::min(Zvalue, new_Z); + Z = std::min(Z, new_Z); - if (debug && new_Z == Zvalue) { + if (debug && new_Z == Z) { cerr << "i = " << i << endl; cerr << "added split " << data.feature_space()->print(feature, arg) << " with " << missing << " missing and score " << new_Z - << (new_Z == Zvalue ? " *** BEST ***" : "") + << (new_Z == Z ? " *** BEST ***" : "") << endl; cerr << "nex = " << nex << endl; @@ -878,7 +878,7 @@ struct Stump_Trainer { results.finish(feature); - return Zvalue; + return Z; } template @@ -953,11 +953,11 @@ struct Stump_Trainer { } // TODO: not missing - float Zvalue = Z::worst; + float Z = Z::worst; #if 0 /* One candidate split point is -INF, which lets us split only based upon missing or not. */ - float Zvalue = results.add(feature, w, -INFINITY, missing); + float Z = results.add(feature, w, -INFINITY, missing); #endif float prev = index[i].value(); @@ -1020,14 +1020,14 @@ struct Stump_Trainer { prev, i1, index[i].value(), i2, dist) << endl; } - Zvalue = std::min(Zvalue, new_Z); + Z = std::min(Z, new_Z); prev = index[i].value(); } results.finish(feature); - return Zvalue; + return Z; } template @@ -1103,7 +1103,7 @@ struct Stump_Trainer { /* One candidate split point is -INF, which lets us split only based upon missing or not. */ - float Zvalue = Z::worst; + float Z = Z::worst; if (missing > 0.0) results.add(feature, w, -INFINITY, missing); @@ -1113,7 +1113,7 @@ struct Stump_Trainer { cerr << "missing = " << missing << endl; cerr << "nb = " << nb << endl; cerr << "added default split " << -INFINITY << " with " - << missing << " missing and score " << Zvalue + << missing << " missing and score " << Z << endl; } @@ -1151,12 +1151,12 @@ struct Stump_Trainer { cerr << "i = " << i << endl; cerr << "added split " << arg << " with " << missing << " missing and score " << new_Z - << (new_Z < Zvalue ? " *** BEST ***" : "") + << (new_Z < Z ? " *** BEST ***" : "") << endl; - if (new_Z < Zvalue) best_arg = arg; + if (new_Z < Z) best_arg = arg; } - Zvalue = std::min(Zvalue, new_Z); + Z = std::min(Z, new_Z); } @@ -1165,7 +1165,7 @@ struct Stump_Trainer { } results.finish(feature); - return Zvalue; + return Z; } }; diff --git a/jml/boosting/training_data.cc b/jml/boosting/training_data.cc index 7033f37cc..e7bc16ee9 100644 --- a/jml/boosting/training_data.cc +++ b/jml/boosting/training_data.cc @@ -311,13 +311,13 @@ preindex(const Feature & label, const std::vector & features) void Training_Data::preindex(const Feature & label) { return; - throw Exception("STUB: %s", __PRETTY_FUNCTION__); + throw Exception("STUB", __PRETTY_FUNCTION__); } void Training_Data::preindex_features() { return; - throw Exception("STUB: %s", __PRETTY_FUNCTION__); + throw Exception("STUB", __PRETTY_FUNCTION__); } vector > diff --git a/jml/jml-build b/jml/jml-build deleted file mode 160000 index 6b111d735..000000000 --- a/jml/jml-build +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6b111d735fcb2bb28ab8b66a05d22f9fa53c885b diff --git a/jml/judy/judy.mk b/jml/judy/judy.mk index 24c162513..576c65422 100644 --- a/jml/judy/judy.mk +++ b/jml/judy/judy.mk @@ -25,7 +25,6 @@ LIBJUDY_SOURCES := \ LIBJUDY_LINK := -# gcc 4.9 compilation requirements -$(eval $(call set_compile_option,$(LIBJUDY_SOURCES),-fno-strict-aliasing -Wno-array-bounds)) +$(eval $(call set_compile_option,$(LIBJUDY_SOURCES),-fno-strict-aliasing)) $(eval $(call library,judy,$(LIBJUDY_SOURCES),$(LIBJUDY_LINK))) diff --git a/jml/utils/exc_assert.cc b/jml/utils/exc_assert.cc index 96a83c7ce..66a2d3044 100644 --- a/jml/utils/exc_assert.cc +++ b/jml/utils/exc_assert.cc @@ -16,13 +16,19 @@ Assertion_Failure(const std::string & msg) { } +Assertion_Failure:: +Assertion_Failure(const char * msg, ...) + : Exception(msg) +{ +} + Assertion_Failure:: Assertion_Failure(const char * assertion, const char * function, const char * file, int line) - : Exception("assertion failure: %s at %s:%d in %s", - assertion, file, line, function) + : Exception(format("assertion failure: %s at %s:%d in %s", + assertion, file, line, function)) { } diff --git a/jml/utils/exc_assert.h b/jml/utils/exc_assert.h index 19be919ee..be60f8136 100644 --- a/jml/utils/exc_assert.h +++ b/jml/utils/exc_assert.h @@ -17,13 +17,7 @@ namespace ML { struct Assertion_Failure: public Exception { Assertion_Failure(const std::string & msg); - - template - Assertion_Failure(const char * msg, Args &&... args) - : Exception(msg, std::forward(args)...) - { - } - + Assertion_Failure(const char * msg, ...); Assertion_Failure(const char * assertion, const char * function, const char * file, diff --git a/jml/utils/exc_check.cc b/jml/utils/exc_check.cc index 4f950bd02..d92398577 100644 --- a/jml/utils/exc_check.cc +++ b/jml/utils/exc_check.cc @@ -16,11 +16,17 @@ Check_Failure(const std::string & msg) { } +Check_Failure:: +Check_Failure(const char * msg, ...) + : Exception(msg) +{ +} + Check_Failure:: Check_Failure(const char * assertion, - const char * function, - const char * file, - int line) + const char * function, + const char * file, + int line) : Exception(format("%s at %s:%d in %s", assertion, file, line, function)) { diff --git a/jml/utils/exc_check.h b/jml/utils/exc_check.h index 5cd185663..1c271d40a 100644 --- a/jml/utils/exc_check.h +++ b/jml/utils/exc_check.h @@ -23,13 +23,7 @@ namespace ML { struct Check_Failure: public Exception { Check_Failure(const std::string & msg); - - template - Check_Failure(const char * msg, Args &&... args) - : Exception(msg, std::forward(args)...) - { - } - + Check_Failure(const char * msg, ...); Check_Failure(const char * assertion, const char * function, const char * file, diff --git a/jml/utils/filter_streams.cc b/jml/utils/filter_streams.cc index 71c2fdf3f..75ce7c67c 100644 --- a/jml/utils/filter_streams.cc +++ b/jml/utils/filter_streams.cc @@ -124,51 +124,6 @@ filter_ostream:: } } -filter_ostream & -filter_ostream:: -put(char_type data) -{ - try { - std::ostream::put(data); - } - catch (...) { - clearDeferredFailure(); - throw; - } - - return *this; -} - -filter_ostream & -filter_ostream:: -write(const char_type * data, std::streamsize count) -{ - try { - std::ostream::write(data, count); - } - catch (...) { - clearDeferredFailure(); - throw; - } - - return *this; -} - -filter_ostream & -filter_ostream:: -flush() -{ - try { - std::ostream::flush(); - } - catch (...) { - clearDeferredFailure(); - throw; - } - - return *this; -} - filter_ostream & filter_ostream:: operator = (filter_ostream && other) @@ -481,8 +436,8 @@ close() stream.reset(); sink.reset(); options.clear(); - if (hasDeferredFailure()) { - clearDeferredFailure(); + if (deferredFailure) { + deferredFailure = false; exceptions(ios::badbit | ios::failbit); setstate(ios::badbit); } diff --git a/jml/utils/filter_streams.h b/jml/utils/filter_streams.h index 33c450e43..b6a83f99f 100644 --- a/jml/utils/filter_streams.h +++ b/jml/utils/filter_streams.h @@ -28,11 +28,10 @@ #include #include #include -#include - namespace ML { + /*****************************************************************************/ /* FILTER OSTREAM */ /*****************************************************************************/ @@ -110,38 +109,6 @@ class filter_ostream : public std::ostream { void close(); - /* The next four overrides only take care of handling the deferred - exception flag, when an exception occurs. They otherwise execute their - std::ostream equivalent. */ - filter_ostream & operator << (std::ostream & (*f) (std::ostream &)) - { - try { - f(*this); - } - catch (...) { - this->clearDeferredFailure(); - throw; - } - return *this; - } - - template filter_ostream & operator << (T && data) - { - try { - std::ostream & stdStream(*this); - stdStream << std::forward(data); - } - catch (...) { - this->clearDeferredFailure(); - throw; - } - return *this; - } - - filter_ostream & put(char_type data); - filter_ostream & write(const char_type * data, std::streamsize count); - filter_ostream & flush(); - std::string status() const; /* notifies that an exception occurred in the streambuf */ @@ -150,18 +117,7 @@ class filter_ostream : public std::ostream { deferredFailure = true; } - bool hasDeferredFailure() - const - { - return deferredFailure; - } - private: - void clearDeferredFailure() - { - deferredFailure = false; - } - std::unique_ptr stream; std::unique_ptr sink; std::atomic deferredFailure; diff --git a/jml/utils/json_parsing.cc b/jml/utils/json_parsing.cc index 8f4563d32..730d3f422 100644 --- a/jml/utils/json_parsing.cc +++ b/jml/utils/json_parsing.cc @@ -194,6 +194,106 @@ std::string expectJsonStringAsciiPermissive(Parse_Context & context, char sub) return result; } +std::string expectJsonString(Parse_Context & context) +{ + skipJsonWhitespace(context); + context.expect_literal('"'); + + char internalBuffer[4096]; + + char * buffer = internalBuffer; + size_t bufferSize = 4096; + size_t pos = 0; + + auto encode = [&] (int code, unsigned pos, uint32_t mask, uint32_t head) -> char { + return ((code >> (6 * pos)) & mask) | head; + }; + + // Try multiple times to make it fit + while (!context.match_literal('"')) { + + int c = *context++; + if (c == '\\') { + c = *context++; + switch (c) { + case 't': c = '\t'; break; + case 'n': c = '\n'; break; + case 'r': c = '\r'; break; + case 'f': c = '\f'; break; + case 'b': c = '\b'; break; + case '/': c = '/'; break; + case '\\':c = '\\'; break; + case '"': c = '"'; break; + case 'u': { + c = context.expect_hex4(); + break; + } + default: + context.exception("invalid escaped char"); + } + } + + if ((pos + 6) == bufferSize) { + size_t newBufferSize = bufferSize * 8; + char * newBuffer = new char[newBufferSize]; + + std::copy(buffer, buffer + bufferSize, newBuffer); + + if (buffer != internalBuffer) + delete[] buffer; + + buffer = newBuffer; + bufferSize = newBufferSize; + } + + if (c <= 0x7f) { + buffer[pos++] = (char) c; + } + + else if (c <= 0x7FF) { + buffer[pos++] = encode(c, 1, 0x1F, 0xC0); + buffer[pos++] = encode(c, 0, 0x3F, 0x80); + } + + else if (c <= 0xFFFF) { + buffer[pos++] = encode(c, 2, 0x0F, 0xE0); + buffer[pos++] = encode(c, 1, 0x3F, 0x80); + buffer[pos++] = encode(c, 0, 0x3F, 0x80); + } + + else if (c <= 0x1FFFFF) { + buffer[pos++] = encode(c, 3, 0x07, 0xF0); + buffer[pos++] = encode(c, 2, 0x3F, 0x80); + buffer[pos++] = encode(c, 1, 0x3F, 0x80); + buffer[pos++] = encode(c, 0, 0x3F, 0x80); + } + + else if (c <= 0x3FFFFFFF) { + buffer[pos++] = encode(c, 4, 0x03, 0xF8); + buffer[pos++] = encode(c, 3, 0x3F, 0x80); + buffer[pos++] = encode(c, 2, 0x3F, 0x80); + buffer[pos++] = encode(c, 1, 0x3F, 0x80); + buffer[pos++] = encode(c, 0, 0x3F, 0x80); + } + + else { + buffer[pos++] = encode(c, 5, 0x01, 0xFC); + buffer[pos++] = encode(c, 4, 0x3F, 0x80); + buffer[pos++] = encode(c, 3, 0x3F, 0x80); + buffer[pos++] = encode(c, 2, 0x3F, 0x80); + buffer[pos++] = encode(c, 1, 0x3F, 0x80); + buffer[pos++] = encode(c, 0, 0x3F, 0x80); + } + } + + string result(buffer, buffer + pos); + + if (buffer != internalBuffer) + delete[] buffer; + + return result; +} + ssize_t expectJsonStringAscii(Parse_Context & context, char * buffer, size_t maxLength) { skipJsonWhitespace(context); @@ -361,7 +461,7 @@ expectJsonObject(Parse_Context & context, for (;;) { skipJsonWhitespace(context); - string key = expectJsonStringAscii(context); + string key = expectJsonString(context); skipJsonWhitespace(context); @@ -437,7 +537,7 @@ matchJsonObject(Parse_Context & context, for (;;) { skipJsonWhitespace(context); - string key = expectJsonStringAscii(context); + string key = expectJsonString(context); skipJsonWhitespace(context); if (!context.match_literal(':')) return false; diff --git a/jml/utils/json_parsing.h b/jml/utils/json_parsing.h index 32acce8f1..2d7eb2350 100644 --- a/jml/utils/json_parsing.h +++ b/jml/utils/json_parsing.h @@ -26,6 +26,8 @@ std::string jsonEscape(const std::string & str); void jsonEscape(const std::string & str, std::ostream & out); +std::string expectJsonString(Parse_Context & context); + /* * If non-ascii characters are found an exception is thrown */ @@ -114,7 +116,7 @@ expectJson(Parse_Context & context) { context.skip_whitespace(); if (*context == '"') - return expectJsonStringAscii(context); + return expectJsonString(context); else if (context.match_literal("null")) return Json::Value(); else if (context.match_literal("true")) diff --git a/jml/utils/lightweight_hash.h b/jml/utils/lightweight_hash.h index 210cc26ef..472f2d9fb 100644 --- a/jml/utils/lightweight_hash.h +++ b/jml/utils/lightweight_hash.h @@ -28,11 +28,6 @@ namespace ML { /* LIGHTWEIGHT HASH ITERATOR */ /*****************************************************************************/ -/** Iterator for a lightweight hash. Note that bucket indexes go from -1 to - capacity; index -1 is reserved for the bucket containing the element that - corresponds to the "guard" value in the key. -*/ - template class Lightweight_Hash_Iterator : public boost::iterator_facade, @@ -44,11 +39,11 @@ class Lightweight_Hash_Iterator boost::bidirectional_traversal_tag> Base; public: Lightweight_Hash_Iterator() - : hash(0), index(-1) + : hash(0), index(0) { } - Lightweight_Hash_Iterator(Hash * hash, ssize_t index) + Lightweight_Hash_Iterator(Hash * hash, int index) : hash(hash), index(index) { if (index != hash->capacity()) @@ -73,7 +68,7 @@ class Lightweight_Hash_Iterator // capacity, which means at the end. public: Hash * hash; - ssize_t index; + int index; friend class boost::iterator_core_access; @@ -102,12 +97,12 @@ class Lightweight_Hash_Iterator void decrement() { - if (index == -1) + if (index == 0) throw Exception("decrement past the start"); --index; - if (index != -1) index = hash->backup_to_valid(index); + if (index != 0) index = hash->backup_to_valid(index); } - + template friend class Lightweight_Hash_Iterator; }; @@ -128,19 +123,18 @@ operator << (std::ostream & stream, template > struct MemStorage { MemStorage() - : capacity_(0), vals_(0), guardBucketIsFull_(0) + : capacity_(0), vals_(0) { } MemStorage(size_t capacity) - : capacity_(0), vals_(0), guardBucketIsFull_(0) + : capacity_(0), vals_(0) { reserve(capacity); } MemStorage(MemStorage && other) - : capacity_(other.capacity_), vals_(other.vals_), - guardBucketIsFull_(other.guardBucketIsFull_) + : capacity_(other.capacity_), vals_(other.vals_) { other.capacity_ = 0; other.vals_ = 0; @@ -160,25 +154,21 @@ struct MemStorage { int capacity_; Bucket * vals_; - uint8_t guardBucketIsFull_; void swap(MemStorage & other) { std::swap(capacity_, other.capacity_); std::swap(vals_, other.vals_); - std::swap(guardBucketIsFull_, other.guardBucketIsFull_); } - ssize_t capacity() const { return capacity_; } + size_t capacity() const JML_PURE_FN { return capacity_; } void reserve(size_t newCapacity) { if (vals_) throw ML::Exception("can't double initialize storage"); - // We allocate one extra bucket and shift things so that index -1 has - // a valid bucket - vals_ = allocator.allocate(newCapacity + 1) + 1; + vals_ = allocator.allocate(newCapacity); capacity_ = newCapacity; } @@ -186,27 +176,27 @@ struct MemStorage { { if (!vals_) return; try { - allocator.deallocate(vals_ - 1, capacity_ + 1); + allocator.deallocate(vals_, capacity_); } catch (...) {} vals_ = 0; capacity_ = 0; } - Bucket * operator + (ssize_t index) + Bucket * operator + (size_t index) { //ExcAssert(vals_); //ExcAssertLess(index, capacity_); return vals_ + index; } - Bucket & operator [] (ssize_t index) + Bucket & operator [] (size_t index) { //ExcAssert(vals_); //ExcAssertLess(index, capacity_); return vals_[index]; } - const Bucket & operator [] (ssize_t index) const + const Bucket & operator [] (size_t index) const { //ExcAssert(vals_); //ExcAssertLess(index, capacity_); @@ -231,19 +221,18 @@ allocator; template > struct LogMemStorage { LogMemStorage() - : vals_(0), bits_(0), guardBucketIsFull_(0) + : vals_(0), bits_(0) { } LogMemStorage(size_t capacity) - : vals_(0), bits_(0), guardBucketIsFull_(0) + : vals_(0), bits_(0) { reserve(capacity); } LogMemStorage(LogMemStorage && other) - : bits_(other.bits_), vals_(other.vals_), - guardBucketIsFull_(other.guardBucketIsFull_) + : bits_(other.bits_), vals_(other.vals_) { other.bits_ = 0; other.vals_ = 0; @@ -263,16 +252,14 @@ struct LogMemStorage { Bucket * vals_; uint8_t bits_; - uint8_t guardBucketIsFull_; void swap(LogMemStorage & other) { std::swap(bits_, other.bits_); std::swap(vals_, other.vals_); - std::swap(guardBucketIsFull_, other.guardBucketIsFull_); } - ssize_t capacity() const + size_t capacity() const JML_PURE_FN { return size_t(bits_ != 0) * (1ULL << (bits_ - 1)); } @@ -285,7 +272,7 @@ struct LogMemStorage { if (newCapacity == 0) return; bits_ = ML::highest_bit((newCapacity - 1), -1) + 2; - vals_ = allocator.allocate(capacity() + 1) + 1; + vals_ = allocator.allocate(capacity()); ExcAssertGreaterEqual(capacity(), newCapacity); } @@ -294,38 +281,27 @@ struct LogMemStorage { { if (!vals_) return; try { - allocator.deallocate(vals_ - 1, capacity() + 1); + allocator.deallocate(vals_, capacity()); } catch (...) {} vals_ = 0; bits_ = 0; } - bool guardBucketIsFull() const - { - return guardBucketIsFull_; - } - - void setGuardBucketIsOccupied() - { - ExcAssert(!guardBucketIsFull_); - guardBucketIsFull_ = true; - } - - Bucket * operator + (ssize_t index) + Bucket * operator + (size_t index) { //ExcAssert(vals_); //ExcAssertLess(index, capacity_); return vals_ + index; } - Bucket & operator [] (ssize_t index) + Bucket & operator [] (size_t index) { //ExcAssert(vals_); //ExcAssertLess(index, capacity_); return vals_[index]; } - const Bucket & operator [] (ssize_t index) const + const Bucket & operator [] (size_t index) const { //ExcAssert(vals_); //ExcAssertLess(index, capacity_); @@ -361,11 +337,11 @@ struct Lightweight_Hash_Base { { if (capacity == 0) storage_.reserve(std::distance(first, last) * 2); - - ssize_t cp = this->capacity(); + + size_t cp = this->capacity(); if (cp == 0) return; - for (ssize_t i = -1; i < cp; ++i) + for (unsigned i = 0; i < cp; ++i) Ops::initEmptyBucket(storage_ + i); for (; first != last; ++first) @@ -376,14 +352,14 @@ struct Lightweight_Hash_Base { size_t capacity) : storage_(capacity), size_(0) { - ssize_t cp = this->capacity(); + size_t cp = this->capacity(); - for (ssize_t i = -1; i < cp; ++i) + for (unsigned i = 0; i < cp; ++i) Ops::initEmptyBucket(storage_ + i); - ssize_t ocp = other.capacity(); - for (ssize_t i = -1; i < ocp; ++i) - if (Ops::bucketIsFull(other.storage_, i)) + size_t ocp = other.capacity(); + for (unsigned i = 0; i < ocp; ++i) + if (Ops::bucketIsFull(other.storage_[i])) must_insert(other.storage_[i]); } @@ -394,15 +370,15 @@ struct Lightweight_Hash_Base { ExcAssertEqual(capacity(), other.capacity()); - for (ssize_t i = -1; i < capacity(); ++i) { - if (Ops::bucketIsFull(other.storage_, i)) + for (unsigned i = 0; i < capacity(); ++i) { + if (Ops::bucketIsFull(other.storage_[i])) Ops::initBucket(storage_ + i, other.storage_[i]); else Ops::initEmptyBucket(storage_ + i); } } Lightweight_Hash_Base(Lightweight_Hash_Base && other) - : storage_(std::move(other.storage_)), size_(other.size_) + : storage_(other.storage_), size_(other.size_) { other.size_ = 0; } @@ -434,15 +410,15 @@ struct Lightweight_Hash_Base { size_t size() const { return size_; } bool empty() const { return size_ == 0; } - ssize_t capacity() const { return storage_.capacity(); } + size_t capacity() const { return storage_.capacity(); } void clear() { - ssize_t cp = capacity(); + size_t cp = capacity(); // Empty buckets - for (ssize_t i = -1; i < cp; ++i) { - if (Ops::bucketIsFull(storage_, i)) { + for (unsigned i = 0; i < cp; ++i) { + if (Ops::bucketIsFull(storage_[i])) { try { Ops::emptyBucket(storage_ + i); } catch (...) {} @@ -454,16 +430,16 @@ struct Lightweight_Hash_Base { bool count(const Key & key) const { - ssize_t bucket = this->find_full_bucket(key); - return bucket != NO_BUCKET; + int bucket = this->find_full_bucket(key); + return bucket != -1; } void destroy() { - ssize_t cp = capacity(); + size_t cp = capacity(); // Run destructors - for (ssize_t i = -1; i < cp; ++i) { + for (unsigned i = 0; i < cp; ++i) { try { Ops::destroyBucket(storage_ + i); } catch (...) {} @@ -490,7 +466,7 @@ struct Lightweight_Hash_Base { using namespace std; stream << "Lightweight_Hash: size " << size_ << " capacity " << capacity() << endl; - for (ssize_t i = -1; i < capacity(); ++i) { + for (unsigned i = 0; i < capacity(); ++i) { stream << " bucket " << i << ": hash " << Ops::hashKey(storage_[i], capacity(), storage_) << " bucket " << storage_[i] << endl; @@ -506,59 +482,39 @@ struct Lightweight_Hash_Base { Storage storage_; int size_; - std::pair + std::pair find_or_insert(const Bucket & toInsert) { - using namespace std; - //cerr << "find_or_insert " << toInsert << endl; Key key = Ops::getKey(toInsert); - //cerr << "key = " << key << endl; - //cerr << "Ops::hashKey() = " << Ops::hashKey(key, capacity(), storage_) - // << endl; - - ssize_t bucket = find_bucket(key); - - //cerr << "found bucket " << bucket << endl; - - if (bucket != NO_BUCKET && Ops::bucketIsFull(storage_, bucket)) + int bucket = find_bucket(key); + if (bucket != -1 && Ops::bucketIsFull(storage_[bucket])) return std::make_pair(bucket, false); return std::make_pair(insert_new(bucket, toInsert), true); } - ssize_t must_insert(const Bucket & toInsert) + int must_insert(const Bucket & toInsert) { - using namespace std; - //cerr << "must_insert " << toInsert << endl; Key key = Ops::getKey(toInsert); - ssize_t bucket = find_bucket(key); - if (bucket != NO_BUCKET && Ops::bucketIsFull(storage_, bucket)) + int bucket = find_bucket(key); + if (bucket != -1 && Ops::bucketIsFull(storage_[bucket])) throw ML::Exception("must_insert of value already there"); return insert_new(bucket, toInsert); } - enum { - GUARD_BUCKET = -1, - NO_BUCKET = -2 - }; - public: //static uint64_t numCalls, numHops; protected: //__attribute__((__noinline__)) - ssize_t find_bucket(const Key & key) const + int find_bucket(const Key & key) const { - using namespace std; - - if (Ops::isGuardValue(key)) { - //cerr << "is guard value" << endl; - return GUARD_BUCKET; - } + if (Ops::isGuardValue(key)) + throw Exception("searching for or inserting guard value"); - ssize_t cap = capacity(); + size_t cap = capacity(); - if (cap == 0) return NO_BUCKET; - ssize_t bucket = Ops::hashKey(key, cap, storage_); + if (cap == 0) return -1; + int bucket = Ops::hashKey(key, cap, storage_); //using namespace std; //cerr << "find_bucket: key " << key << " bucket " << bucket @@ -566,10 +522,10 @@ struct Lightweight_Hash_Base { // << endl; bool wrapped = false; - ssize_t i; - //ssize_t hops = 0; + int i; + //int hops = 0; for (i = bucket; - Ops::bucketIsFull(storage_, i) && (i != bucket || !wrapped); + Ops::bucketIsFull(storage_[i]) && (i != bucket || !wrapped); /* no inc */ /*++hops*/) { if (Ops::bucketHasKey(storage_[i], key)) { //ML::atomic_inc(numCalls); @@ -577,13 +533,10 @@ struct Lightweight_Hash_Base { return i; } ++i; - // NOTE: we don't wrap around the guard bucket. It can't be used. if (i == cap) { i = 0; wrapped = true; } } - ExcAssertNotEqual(i, GUARD_BUCKET); - - if (!Ops::bucketIsFull(storage_, i)) { + if (!Ops::bucketIsFull(storage_[i])) { //ML::atomic_inc(numCalls); //ML::atomic_add(numHops, hops); return i; @@ -594,13 +547,13 @@ struct Lightweight_Hash_Base { dump(std::cerr); throw Exception("find_bucket: inconsistency"); } - return NO_BUCKET; + return -1; } - ssize_t find_full_bucket(const Key & key) const + int find_full_bucket(const Key & key) const { - ssize_t bucket = find_bucket(key); - if (bucket == NO_BUCKET || !Ops::bucketIsFull(storage_, bucket)) return NO_BUCKET; + int bucket = find_bucket(key); + if (bucket == -1 || !Ops::bucketIsFull(storage_[bucket])) return -1; if (!Ops::bucketHasKey(storage_[bucket], key)) { #if 0 using namespace std; @@ -617,86 +570,63 @@ struct Lightweight_Hash_Base { } //__attribute__((__noinline__)) - ssize_t insert_new(ssize_t bucket, const Bucket & toInsert) + int insert_new(int bucket, const Bucket & toInsert) { - //using namespace std; - //cerr << "insert_new " << bucket << " " << toInsert << endl; - Key key = Ops::getKey(toInsert); - if (bucket == GUARD_BUCKET) { - ExcAssert(Ops::isGuardValue(key)); - - if (capacity() != 0) { - Ops::fillBucket(storage_, bucket, toInsert); - //using namespace std; - //cerr << "incrementing size from " << size_ << endl; - ++size_; - - return bucket; - } - } - else { - ExcAssert(!Ops::isGuardValue(key)); - } + if (Ops::isGuardValue(key)) + throw Exception("searching for or inserting guard value"); if (needs_expansion()) { // expand reserve(std::max(4, capacity() * 2)); bucket = find_bucket(key); - if (bucket == NO_BUCKET || Ops::bucketIsFull(storage_, bucket)) + if (bucket == -1 || Ops::bucketIsFull(storage_[bucket])) throw Exception("logic error: bucket appeared after reserve"); } - Ops::fillBucket(storage_, bucket, toInsert); - //using namespace std; - //cerr << "incrementing size from " << size_ << endl; + Ops::fillBucket(storage_ + bucket, toInsert); ++size_; return bucket; } - ssize_t advance_to_valid(ssize_t index) const + int advance_to_valid(int index) const { - using namespace std; - //cerr << "advancing to valid from index " << index << endl; - - if (index < -1 || index >= capacity()) { + if (index < 0 || index >= capacity()) { //dump(std::cerr); std::cerr << "index = " << index << std::endl; throw Exception("advance_to_valid: already at end"); } - ssize_t cap = capacity(); + size_t cap = capacity(); // Scan through until we find a valid bucket - while (index < cap && !Ops::bucketIsFull(storage_, index)) + while (index < cap && !Ops::bucketIsFull(storage_[index])) ++index; - //cerr << "advancing to valid: final index " << index << endl; - return index; } - ssize_t backup_to_valid(ssize_t index) const + int backup_to_valid(int index) const { - if (index < -1 || index >= capacity()) + if (index < 0 || index >= capacity()) throw Exception("backup_to_valid: already outside range"); // Scan through until we find a valid bucket - while (index >= -1 && !Ops::bucketIsFull(storage_, index)) + while (index >= 0 && !Ops::bucketIsFull(storage_[index])) --index; - if (index < -1) + if (index < 0) throw Exception("backup_to_valid: none found"); return index; } - const Bucket & dereference(ssize_t bucket) const + const Bucket & dereference(int bucket) const { - if (bucket < -1 || bucket > capacity()) + if (bucket < 0 || bucket > capacity()) throw Exception("dereferencing invalid iterator"); - if (!Ops::bucketIsFull(storage_, bucket)) { + if (!Ops::bucketIsFull(storage_[bucket])) { using namespace std; cerr << "bucket = " << bucket << endl; dump(cerr); @@ -731,18 +661,11 @@ struct PairOps { new (bucket) Bucket(value); } - template - static void fillBucket(Storage & storage, ssize_t index, const Bucket & value) + static void fillBucket(Bucket * bucket, const Bucket & value) { - using namespace std; - //cerr << "index = " << index << endl; - Bucket * bucket = storage + index; - //cerr << "bucket = " << bucket << endl; bucket->second = value.second; ML::memory_barrier(); bucket->first = value.first; - if (index == -1) - storage.setGuardBucketIsOccupied(); //*bucket = value; } @@ -757,44 +680,41 @@ struct PairOps { bucket->~Bucket(); } - template - static bool bucketIsFull(const Storage & storage, ssize_t index) + static bool bucketIsFull(Bucket bucket) JML_PURE_FN { - if (index == -1) - return storage.guardBucketIsFull(); - return storage[index].first; + return bucket.first; } - static bool isGuardValue(Key key) + static bool isGuardValue(Key key) JML_CONST_FN { return key == 0; } - static bool bucketHasKey(Bucket bucket, Key key) + static bool bucketHasKey(Bucket bucket, Key key) JML_PURE_FN { return bucket.first == key; } - static Key getKey(Bucket bucket) + static Key getKey(Bucket bucket) JML_CONST_FN { return bucket.first; } template - static size_t hashKey(Bucket bucket, ssize_t capacity, + static size_t hashKey(Bucket bucket, int capacity, const Storage & storage) { return hashKey(getKey(bucket), capacity, storage); } template - static size_t hashKey(Key key, ssize_t capacity, + static size_t hashKey(Key key, int capacity, const Storage & storage) { return Hash()(key) % capacity; } - static size_t hashKey(Key key, ssize_t capacity, + static size_t hashKey(Key key, int capacity, const LogMemStorage & storage) { uint64_t mask = (1ULL << ((storage.bits_ - 1))) - 1; @@ -862,12 +782,11 @@ struct Lightweight_Hash using Base::empty; using Base::capacity; using Base::clear; - using Base::NO_BUCKET; iterator begin() { if (empty()) return end(); - return iterator(this, -1); + return iterator(this, 0); } iterator end() @@ -878,7 +797,7 @@ struct Lightweight_Hash const_iterator begin() const { if (empty()) return end(); - return const_iterator(this, -1); + return const_iterator(this, 0); } const_iterator end() const @@ -888,21 +807,21 @@ struct Lightweight_Hash iterator find(const Key & key) { - ssize_t bucket = this->find_full_bucket(key); - if (bucket == NO_BUCKET) return end(); + int bucket = this->find_full_bucket(key); + if (bucket == -1) return end(); return iterator(this, bucket); } const_iterator find(const Key & key) const { - ssize_t bucket = this->find_full_bucket(key); - if (bucket == NO_BUCKET) return end(); + int bucket = this->find_full_bucket(key); + if (bucket == -1) return end(); return const_iterator(this, bucket); } Value & operator [] (const Key & key) { - ssize_t bucket = this->find_or_insert(Bucket(key, Value())).first; + int bucket = this->find_or_insert(Bucket(key, Value())).first; assert(this->storage_[bucket].first == key); return this->storage_[bucket].second; } @@ -910,7 +829,7 @@ struct Lightweight_Hash std::pair insert(const Bucket & val) { - std::pair r = this->find_or_insert(val); + std::pair r = this->find_or_insert(val); try { return make_pair(iterator(this, r.first), r.second); } catch (...) { @@ -922,9 +841,9 @@ struct Lightweight_Hash cerr << "capacity = " << capacity() << endl; cerr << "val = (" << val.first << "," << val.second << ")" << endl; - ssize_t cap = capacity(); + size_t cap = capacity(); auto key = Ops::getKey(val); - ssize_t bucket = Ops::hashKey(key, cap, this->storage_); + int bucket = Ops::hashKey(key, cap, this->storage_); cerr << "bucket = " << bucket << endl; throw; @@ -938,7 +857,7 @@ struct Lightweight_Hash friend class Lightweight_Hash_Iterator; using Base::dereference; - ConstKeyBucket & dereference(ssize_t bucket) + ConstKeyBucket & dereference(int bucket) { if (bucket < 0 || bucket > this->capacity()) throw Exception("dereferencing invalid iterator"); @@ -951,13 +870,12 @@ struct Lightweight_Hash return reinterpret_cast(this->storage_[bucket]); } -public: void dump(std::ostream & stream) const { using namespace std; stream << "Lightweight_Hash: size " << this->size_ << " capacity " << this->capacity() << endl; - for (ssize_t i = -1; i < this->capacity(); ++i) { + for (unsigned i = 0; i < this->capacity(); ++i) { stream << " bucket " << i << ": hash " << Ops::hashKey(this->storage_[i], this->capacity(), this->storage_) @@ -990,15 +908,6 @@ struct ScalarOps { new (bucket) Bucket(value); } - template - static void fillBucket(Storage & storage, ssize_t index, const Bucket & value) - { - Bucket * bucket = storage + index; - *bucket = value; - if (index == -1) - storage.setGuardBucketIsOccupied(); - } - static void fillBucket(Bucket * bucket, const Bucket & value) { *bucket = value; @@ -1015,12 +924,9 @@ struct ScalarOps { bucket->~Bucket(); } - template - static bool bucketIsFull(const Storage & storage, ssize_t index) + static bool bucketIsFull(Bucket bucket) { - if (index == -1) - return storage.guardBucketIsFull(); - return storage[index] != guard; + return bucket != guard; } static bool isGuardValue(Key key) @@ -1039,12 +945,12 @@ struct ScalarOps { } template - static size_t hashKey(Key key, ssize_t capacity, const Storage & storage) + static size_t hashKey(Key key, int capacity, const Storage & storage) { return Hash()(key) % capacity; } - static size_t hashKey(Key key, ssize_t capacity, + static size_t hashKey(Key key, int capacity, const LogMemStorage & storage) { uint64_t mask = (1ULL << ((storage.bits_ - 1))) - 1; @@ -1054,7 +960,7 @@ struct ScalarOps { template const Key -ScalarOps::guard(-2 /*NO_BUCKET*/); +ScalarOps::guard(-1); template, class Bucket = Key, @@ -1118,12 +1024,11 @@ struct Lightweight_Hash_Set using Base::empty; using Base::capacity; using Base::clear; - using Base::NO_BUCKET; const_iterator begin() const { if (empty()) return end(); - return const_iterator(this, -1); + return const_iterator(this, 0); } const_iterator end() const @@ -1133,15 +1038,15 @@ struct Lightweight_Hash_Set const_iterator find(const Key & key) const { - ssize_t bucket = this->find_full_bucket(key); - if (bucket == NO_BUCKET) return end(); + int bucket = this->find_full_bucket(key); + if (bucket == -1) return end(); return const_iterator(this, bucket); } std::pair insert(const Key & val) { - std::pair r = this->find_or_insert(val); + std::pair r = this->find_or_insert(val); return make_pair(const_iterator(this, r.first), r.second); } diff --git a/jml/utils/string_functions.cc b/jml/utils/string_functions.cc index c40635dc6..312b806ba 100644 --- a/jml/utils/string_functions.cc +++ b/jml/utils/string_functions.cc @@ -20,9 +20,7 @@ String manipulation functions. */ -#define __STDC_FORMAT_MACROS 1 #include "string_functions.h" -#include #include #include #include "jml/arch/exception.h" @@ -193,7 +191,7 @@ antoi(const char * start, const char * end, int base) if (digit > base) { intptr_t offset = ptr - start; throw ML::Exception("digit '%c' (%d) exceeds base '%d'" - " at offset '%" PRIdPTR "'", + " at offset '%d'", *ptr, digit, base, offset); } result = result * base + digit; diff --git a/jml/utils/testing/filter_streams_test.cc b/jml/utils/testing/filter_streams_test.cc index 0b5e929a6..f4882abf7 100644 --- a/jml/utils/testing/filter_streams_test.cc +++ b/jml/utils/testing/filter_streams_test.cc @@ -409,9 +409,6 @@ struct ExceptionSource { std::streamsize write(const char_type* s, std::streamsize n) { if (throwType_ == ThrowType::ThrowOnWrite) { - if (onException_) { - onException_(); - } throw ML::Exception("throwing when writing"); } return n; @@ -420,9 +417,6 @@ struct ExceptionSource { std::streamsize read(char_type* s, std::streamsize n) { if (throwType_ == ThrowType::ThrowOnRead) { - if (onException_) { - onException_(); - } throw ML::Exception("throwing when reading"); } char randomdata[n]; @@ -499,11 +493,11 @@ struct RegisterExcHandlers { BOOST_AUTO_TEST_CASE(test_filter_stream_exceptions_read) { - JML_TRACE_EXCEPTIONS(false); ML::filter_istream stream("throw-on-read://exception-zone"); string data; auto action = [&]() { + JML_TRACE_EXCEPTIONS(false); stream >> data; }; @@ -525,10 +519,6 @@ BOOST_AUTO_TEST_CASE(test_filter_stream_exceptions_write) }; BOOST_CHECK_THROW(action(), ML::Exception); - - /* ensure that no deferred exceptions stays reported for exceptions - occurring during writes */ - BOOST_CHECK_EQUAL(stream.hasDeferredFailure(), false); } BOOST_AUTO_TEST_CASE(test_filter_stream_exceptions_close) @@ -568,20 +558,5 @@ BOOST_AUTO_TEST_CASE(test_filter_stream_exceptions_destruction_istream) action(); } -#endif -#if 1 -/* ensure that the overloads of operator << are working correctly for most - * common cases */ -BOOST_AUTO_TEST_CASE(test_filter_stream_op_overloads) -{ - ML::filter_ostream stream("mem://operator<<_overload"); - - stream << "coucou\n"; - stream << endl; - stream << ends; - stream << "coucou" << endl; - - stream.close(); -} #endif diff --git a/jml/utils/testing/lightweight_hash_test.cc b/jml/utils/testing/lightweight_hash_test.cc index 0d943cde4..6577a5ee1 100644 --- a/jml/utils/testing/lightweight_hash_test.cc +++ b/jml/utils/testing/lightweight_hash_test.cc @@ -58,26 +58,15 @@ BOOST_AUTO_TEST_CASE(test1) BOOST_CHECK_EQUAL(h.size(), 2); BOOST_CHECK(h.capacity() >= 2); - h[0] = 0; - - BOOST_CHECK_EQUAL(h[0], 0); - BOOST_CHECK_EQUAL(h[1], 1); - BOOST_CHECK_EQUAL(h[2], 2); - BOOST_CHECK_EQUAL(h.size(), 3); - BOOST_CHECK_GE(h.capacity(), 2); - h.reserve(1024); - BOOST_CHECK_EQUAL(h[0], 0); BOOST_CHECK_EQUAL(h[1], 1); BOOST_CHECK_EQUAL(h[2], 2); - BOOST_CHECK_EQUAL(h.size(), 3); + BOOST_CHECK_EQUAL(h.size(), 2); BOOST_CHECK(h.capacity() >= 2); - BOOST_CHECK_EQUAL(++++++h.begin(), h.end()); + BOOST_CHECK_EQUAL(++++h.begin(), h.end()); } -#if 1 - // TODO: use live counting object to check that everything works OK struct Entry { @@ -92,6 +81,12 @@ std::ostream & operator << (std::ostream & stream, Entry entry) return stream << "Entry"; } +BOOST_AUTO_TEST_CASE(test2) +{ + Lightweight_Hash h; + BOOST_CHECK_THROW(h[(void *)0].p1, ML::Exception); +} + BOOST_AUTO_TEST_CASE(test3) { int nobj = 100; @@ -119,8 +114,6 @@ BOOST_AUTO_TEST_CASE(test3) for (unsigned j = 0; j < nobj; ++j) free(objects[j]); - - cerr << "done test 3" << endl; } BOOST_AUTO_TEST_CASE(test3_log) @@ -198,5 +191,3 @@ BOOST_AUTO_TEST_CASE(test_set) BOOST_CHECK_EQUAL_COLLECTIONS(objects.begin(), objects.end(), obj3.begin(), obj3.end()); } - -#endif diff --git a/jml/utils/utils.mk b/jml/utils/utils.mk index e86064cf1..bcc13da52 100644 --- a/jml/utils/utils.mk +++ b/jml/utils/utils.mk @@ -27,8 +27,6 @@ $(eval $(call library,utils,$(LIBUTILS_SOURCES),$(LIBUTILS_LINK))) $(eval $(call program,lz4cli,,lz4cli.c lz4.c lz4hc.c xxhash.c)) -# gcc 4.7 -$(eval $(call set_compile_option,hash.cc,-fpermissive)) LIBWORKER_TASK_SOURCES := worker_task.cc LIBWORKER_TASK_LINK := ACE arch pthread diff --git a/rtbkit/common/bids.h b/rtbkit/common/bids.h index 7506cec28..3137f9c65 100644 --- a/rtbkit/common/bids.h +++ b/rtbkit/common/bids.h @@ -87,7 +87,7 @@ struct Bid //extra data come with this bid Json::Value ext; - bool isNullBid() const { return price.isZero(); } + bool isNullBid() const { return price.isZero() || price.isNegative(); } void bid(int creativeIndex, Amount price, double priority = 0.0); diff --git a/rtbkit/core/agent_configuration/agent_config.cc b/rtbkit/core/agent_configuration/agent_config.cc index 3c3cb817a..813626f30 100644 --- a/rtbkit/core/agent_configuration/agent_config.cc +++ b/rtbkit/core/agent_configuration/agent_config.cc @@ -28,8 +28,8 @@ namespace RTBKIT { Creative:: Creative(int width, int height, std::string name, int id, const std::string dealId) - : format(width, height), name(name), id(id), dealId(dealId) - , type(Type::Image) + : format(width, height), name(name), id(id), dealId(dealId), + fees(NullFees::createNullFees()), type(Type::Image) { } @@ -81,6 +81,12 @@ fromJson(const Json::Value & val) locationFilter.fromJson(val["locationFilter"], "locationFilter"); exchangeFilter.fromJson(val["exchangeFilter"], "exchangeFilter"); + if (val.isMember("fees")) { + fees = Fees::createFees(val["fees"]); + } else { + fees = NullFees::createNullFees(); + } + if (val.isMember("segmentFilter")){ Json::Value segs = val["segmentFilter"]; for (auto jt = segs.begin(), jend = segs.end(); jt != jend; ++jt) { @@ -137,6 +143,10 @@ toJson() const if (!dealId.empty()) result["dealId"] = dealId; + if (fees) { + result["fees"] = fees->toJson(); + } + if (type == Type::Video) { result["duration"] = duration; result["bitrate"] = bitrate; @@ -157,6 +167,7 @@ Creative:: compatible(const AdSpot & adspot) const { return ((format.width == 0 && format.height == 0) + || adspot.formats.empty() // if no format was specified in bid request || adspot.formats.compatible(format)); } diff --git a/rtbkit/core/agent_configuration/agent_config.h b/rtbkit/core/agent_configuration/agent_config.h index 288f694a8..8e0f6678c 100644 --- a/rtbkit/core/agent_configuration/agent_config.h +++ b/rtbkit/core/agent_configuration/agent_config.h @@ -15,6 +15,7 @@ #include #include "rtbkit/common/bid_request.h" #include "include_exclude.h" +#include "fees.h" #include "rtbkit/common/account_key.h" #include "rtbkit/core/agent_configuration/latlonrad.h" @@ -96,6 +97,9 @@ struct Creative { // Needed for PMP filter std::string dealId; + // Fees + shared_ptr fees; + struct SegmentInfo { SegmentInfo() : excludeIfNotPresent(false){} @@ -135,6 +139,7 @@ struct Creative { Type type; std::string typeString() const; + }; diff --git a/rtbkit/core/agent_configuration/agent_configuration.mk b/rtbkit/core/agent_configuration/agent_configuration.mk index 53ed586b6..58cc578e5 100644 --- a/rtbkit/core/agent_configuration/agent_configuration.mk +++ b/rtbkit/core/agent_configuration/agent_configuration.mk @@ -7,6 +7,7 @@ LIBAGENT_CONFIGURATION_SOURCES := \ agent_configuration_listener.cc \ agent_configuration_service.cc \ latlonrad.cc \ + fees.cc \ LIBAGENT_CONFIGURATION_LINK := \ rtb zeromq boost_thread opstats gc services utils monitor @@ -15,5 +16,5 @@ $(eval $(call library,agent_configuration,$(LIBAGENT_CONFIGURATION_SOURCES),$(LI $(eval $(call program,agent_configuration_service_runner,agent_configuration boost_program_options opstats)) -#$(eval $(call include_sub_make,rtb_router_testing,testing,rtb_router_testing.mk)) +$(eval $(call include_sub_make,rtb_agent_configuration_testing,testing,rtb_agent_configuration_testing.mk)) diff --git a/rtbkit/core/agent_configuration/fees.cc b/rtbkit/core/agent_configuration/fees.cc new file mode 100644 index 000000000..dd73445f2 --- /dev/null +++ b/rtbkit/core/agent_configuration/fees.cc @@ -0,0 +1,231 @@ +/* fees.cc -*- C++ -*- + JS Bejeau, 13 June 2015 + Copyright (c) 2012 Datacratic. All rights reserved. + +*/ + +#include "fees.h" + +using namespace std; + +namespace RTBKIT { +namespace { + + unordered_map FeesRegistery; + +} // namespace anonymous + +/***************************/ +// FEES +/***************************/ +Fees:: +Fees() { +} + +Fees:: +~Fees() { +} + +void +Fees:: +RegisterFees(string name, FeesFactory factory) { + FeesRegistery[name] = factory; + +} + +shared_ptr +Fees:: +createFees(Json::Value const &json) { + auto name = json.get("type", "null").asString(); + if(FeesRegistery.find(name) == FeesRegistery.end()) { + throw ML::Exception("Unknown type '%s'", name.c_str()); + }else { + return FeesRegistery[name](json); + } + + throw ML::Exception("Unknown type '%s'", name.c_str()); +} + +/***************************/ +// NULL FEES +/***************************/ + +NullFees:: +NullFees(Json::Value const &) { +} + +NullFees:: +~NullFees(){ +} + +shared_ptr +NullFees:: +createNullFees() { + Json::Value json; + shared_ptr ret(new NullFees(json)); + return ret; +} + + +Amount +NullFees:: +applyFees(Amount bidPrice) const { + return bidPrice; +} + +Json::Value +NullFees:: +toJson() const { + + Json::Value result; + result["type"] = "none"; + return result; +} + + +/***************************/ +// FLAT FEES +/***************************/ + +FlatFees:: +FlatFees(Json::Value const & json) { + + if (json.isMember("params") && json["params"].isObject()) { + auto params = json["params"]; + if (params.isMember("a") && params["a"].isString()) { + a = Amount::parse(params["a"].asString()); + if (a.isNegative()) { + throw ML::Exception("Flat fees should have one parameter :" + " a (Positive or Null)"); + } + }else { + throw ML::Exception("Flat fees should have one parameter : a (Amount string)"); + } + + } else { + throw ML::Exception("Flat fees should have parameters object : params"); + } +} + +FlatFees:: +~FlatFees(){ +} + +Amount +FlatFees:: +applyFees(Amount bidPrice) const { + auto bidPriceFinal = bidPrice - a; + if (bidPriceFinal.isNonNegative()) { + if (bidPriceFinal <= bidPrice) { + return bidPriceFinal; + }else { + return bidPrice; + } + } else { + return MicroUSD(0); + } +} + +Json::Value +FlatFees:: +toJson() const { + + Json::Value result; + Json::Value params; + result["type"] = "flat"; + params["a"] = a.toString(); + result["params"] = params; + return result; +} + +/***************************/ +// LINEAR FEES +/***************************/ + +LinearFees:: +LinearFees(Json::Value const & json) { + + if (json.isMember("params") && json["params"].isObject()) { + auto params = json["params"]; + if (params.isMember("a") && params["a"].isNumeric() && + params.isMember("b") && params["b"].isNumeric()) { + a = params["a"].asDouble(); + b = params["b"].asDouble(); + + // "a" and "b" validation + // Bid Price Final must be always inferior than Original Bid Price + // on the entire biddable range : 0-40000 $ + + // "b" is the origin ordinate and Original Bid Price minimum is 0 + // so "b" must be inferior or equal to 0 + if (b > 0) { + throw ML::Exception("Linear fees should be able to bid" + " on all biddable price range"); + } + + // Known "b", if we want to bid on the total range of price + // "a", the maximum slope should give a Final Bid Price at 40000 + if (a > (1.0 - (b/40000))) { + throw ML::Exception("Linear fees should be able to bid" + " on all biddable price range"); + } + + }else { + throw ML::Exception("Linear fees should have 2 parameters : a & b (double)"); + } + + } else { + throw ML::Exception("Linear fees should have parameters object : params"); + } +} + +LinearFees:: +~LinearFees(){ +} + +Amount +LinearFees:: +applyFees(Amount bidPrice) const { + + auto bidPriceFinal = bidPrice * a + MicroUSD(b); + if (bidPriceFinal.isNonNegative()) { + if (bidPriceFinal <= bidPrice) { + return bidPriceFinal; + }else { + return bidPrice; + } + } else { + return MicroUSD(0); + } +} + +Json::Value +LinearFees:: +toJson() const { + + Json::Value result; + Json::Value params; + result["type"] = "linear"; + params["a"] = a; + params["b"] = b; + result["params"] = params; + return result; +} + +/***************************/ +// REGISTER FEES +/***************************/ +namespace { + +struct InitFees { + InitFees(){ + RTBKIT::Fees::RegisterFees("none", NullFees::RegisterSpecificFees); + RTBKIT::Fees::RegisterFees("flat", FlatFees::RegisterSpecificFees); + RTBKIT::Fees::RegisterFees("linear", LinearFees::RegisterSpecificFees); + } +}initfees; + +} // anonymous namespace + + +} // namespace RTBKIT diff --git a/rtbkit/core/agent_configuration/fees.h b/rtbkit/core/agent_configuration/fees.h new file mode 100644 index 000000000..08aacabf1 --- /dev/null +++ b/rtbkit/core/agent_configuration/fees.h @@ -0,0 +1,92 @@ +/* fees.h -*- C++ -*- + JS Bejeau, 13 June 2015 + Copyright (c) 2012 Datacratic. All rights reserved. + +*/ + +#include "soa/service/service_base.h" +#include "rtbkit/common/plugin_interface.h" +#include "rtbkit/common/currency.h" +#include "soa/jsoncpp/json.h" + +using namespace std; + +namespace RTBKIT { + +/***************************/ +// FEES +/***************************/ +struct Fees { + Fees(); + ~Fees(); + + typedef function (Json::Value const &)> FeesFactory; + static void RegisterFees(string name, FeesFactory factory); + + static shared_ptr createFees(Json::Value const & json); + + virtual Amount applyFees(Amount) const = 0; + virtual Json::Value toJson() const = 0; +}; + +/***************************/ +// NULL FEES +/***************************/ +struct NullFees : public Fees { + NullFees(Json::Value const &); + ~NullFees(); + + static shared_ptr RegisterSpecificFees(Json::Value const & json) { + shared_ptr ret(new NullFees(json)); + return ret; + }; + + static shared_ptr createNullFees(); + + virtual Amount applyFees(Amount) const ; + virtual Json::Value toJson() const ; + +}; + + +/***************************/ +// FLAT FEES +/***************************/ +struct FlatFees : public Fees { + FlatFees(Json::Value const &); + ~FlatFees(); + + static shared_ptr RegisterSpecificFees(Json::Value const & json) { + shared_ptr ret(new FlatFees(json)); + return ret; + }; + + virtual Amount applyFees(Amount) const ; + virtual Json::Value toJson() const ; + +private : + Amount a; + +}; + +/***************************/ +// LINEAR FEES +/***************************/ +struct LinearFees : public Fees { + LinearFees(Json::Value const &); + ~LinearFees(); + + static shared_ptr RegisterSpecificFees(Json::Value const & json) { + shared_ptr ret(new LinearFees(json)); + return ret; + }; + + virtual Amount applyFees(Amount) const ; + virtual Json::Value toJson() const ; + +private : + double a; + double b; +}; + +} // namespace RTBKIT diff --git a/rtbkit/core/agent_configuration/include_exclude.h b/rtbkit/core/agent_configuration/include_exclude.h index ce1be1852..4144bdf25 100644 --- a/rtbkit/core/agent_configuration/include_exclude.h +++ b/rtbkit/core/agent_configuration/include_exclude.h @@ -450,17 +450,23 @@ struct IncludeExclude { name.c_str(), jt.memberName().c_str()); const Json::Value & val = *jt; - for (unsigned i = 0; i != val.size(); ++i) { - try { - T t; - jsonParse(val[i], t); - if (jt.memberName() == "include") - result.include.push_back(t); - else result.exclude.push_back(t); - } catch (...) { - throw ML::Exception("error parsing include/exclude %s in %s", - val[i].toString().c_str(), name.c_str()); + if(!val.isNull() && val.isArray()) { + + for (unsigned i = 0; i != val.size(); ++i) { + try { + T t; + jsonParse(val[i], t); + if (jt.memberName() == "include") + result.include.push_back(t); + else result.exclude.push_back(t); + } catch (...) { + throw ML::Exception("error parsing include/exclude %s in %s", + val[i].toString().c_str(), name.c_str()); + } } + }else { + throw ML::Exception("error parsing include/exclude : " + "include/exclude must be an array"); } } diff --git a/rtbkit/core/agent_configuration/testing/rtb_agent_config_validator_test.cc b/rtbkit/core/agent_configuration/testing/rtb_agent_config_validator_test.cc new file mode 100644 index 000000000..62a56cb8f --- /dev/null +++ b/rtbkit/core/agent_configuration/testing/rtb_agent_config_validator_test.cc @@ -0,0 +1,237 @@ +/* rtb_agent_config_validator.cc +*/ + +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK + +#include +#include "rtbkit/core/agent_configuration/agent_config.h" +#include + +using namespace std; +using namespace RTBKIT; + +BOOST_AUTO_TEST_CASE( test_agent_config_fees ) +{ + AgentConfig config; + std::string payload; + + // Normal case : No fees / NO EXCEPTION expected + payload = R"JSON( { + "account" : ["hello", "worlds"], + "creatives": [ + { + "name": "MaCreative", + "height": 250, + "width": 300, + "languageFilter": { + "include": [] + }, + "id": 5 + }]} + )JSON"; + + + config.parse(payload); + + // Normal case : NullFees : NO EXCEPTION expected + payload = R"JSON( { + "account" : ["hello", "worlds"], + "creatives": [ + { + "name": "MaCreative", + "height": 250, + "width": 300, + "languageFilter": { + "include": [] + }, + "id": 5, + "fees": { + "type": "none" + } + }]} + )JSON"; + + + config.parse(payload); + + // Normal case : Flat Fees : NO EXCEPTION expected + payload = R"JSON( { + "account" : ["hello", "worlds"], + "creatives": [ + { + "name": "MaCreative", + "height": 250, + "width": 300, + "languageFilter": { + "include": [] + }, + "id": 5, + "fees": { + "type": "flat", + "params" : { + "a" : "10USD/1M" + } + } + }]} + )JSON"; + + + config.parse(payload); + + // Normal case : NO EXCEPTION expected + payload = R"JSON( { + "account" : ["hello", "worlds"], + "creatives": [ + { + "name": "MaCreative", + "height": 250, + "width": 300, + "languageFilter": { + "include": [] + }, + "id": 5, + "fees": { + "type": "linear", + "params": { + "a": 0.1, + "b": -0.2 + } + } + }]} + )JSON"; + + + config.parse(payload); +} + +BOOST_AUTO_TEST_CASE( test_agent_config_language_filter ) +{ + AgentConfig config; + std::string payload; + + // Normal case : Empty array : NO EXCEPTION expected + payload = R"JSON( { + "account" : ["hello", "worlds"], + "creatives": [ + { + "name": "MaCreative", + "height": 250, + "width": 300, + "languageFilter": { + "include": [] + }, + "id": 5 + }]} + )JSON"; + + + config.parse(payload); + + // Normal case : NO EXCEPTION expected + payload = R"JSON( { + "account" : ["hello", "worlds"], + "creatives": [ + { + "name": "MaCreative", + "height": 250, + "width": 300, + "id": 5 + }]} + )JSON"; + + + config.parse(payload); + + // Include is null + payload = R"JSON( { + "account" : ["hello", "worlds"], + "creatives": [ + { + "name": "MaCreative", + "height": 250, + "width": 300, + "languageFilter": { + "include": null + }, + "id": 5 + }]} + )JSON"; + + BOOST_CHECK_THROW(config.parse(payload),ML::Exception); + + // Normal case : Include + payload = R"JSON( { + "account" : ["hello", "worlds"], + "creatives": [ + { + "name": "MaCreative", + "height": 250, + "width": 300, + "languageFilter": { + "include": ["zh"] + }, + "id": 5 + }]} + )JSON"; + + + config.parse(payload); + BOOST_CHECK_EQUAL(config.creatives[0].languageFilter.include[0], "zh"); + + // Normal case : Exclude + payload = R"JSON( { + "account" : ["hello", "worlds"], + "creatives": [ + { + "name": "MaCreative", + "height": 250, + "width": 300, + "languageFilter": { + "exclude": ["de"] + }, + "id": 5 + }]} + )JSON"; + + + config.parse(payload); + BOOST_CHECK_EQUAL(config.creatives[0].languageFilter.exclude[0], "de"); + + // Include is not an array + payload = R"JSON( { + "account" : ["hello", "worlds"], + "creatives": [ + { + "name": "MaCreative", + "height": 250, + "width": 300, + "languageFilter": { + "include": "zh" + }, + "id": 5 + }]} + )JSON"; + + + BOOST_CHECK_THROW(config.parse(payload),ML::Exception); + + + // Exclude is not an array + payload = R"JSON( { + "account" : ["hello", "worlds"], + "creatives": [ + { + "name": "MaCreative", + "height": 250, + "width": 300, + "languageFilter": { + "exclude": "zh" + }, + "id": 5 + }]} + )JSON"; + + + BOOST_CHECK_THROW(config.parse(payload),ML::Exception); +} diff --git a/rtbkit/core/agent_configuration/testing/rtb_agent_configuration_testing.mk b/rtbkit/core/agent_configuration/testing/rtb_agent_configuration_testing.mk new file mode 100644 index 000000000..e9a1aaa49 --- /dev/null +++ b/rtbkit/core/agent_configuration/testing/rtb_agent_configuration_testing.mk @@ -0,0 +1,7 @@ +# RTB Agent Configuration testing makefile +# JS Bejeau, 25 June 2015 + +$(eval $(call test,rtb_agent_config_validator_test,agent_configuration,boost)) +$(eval $(call test,rtb_fees_test,agent_configuration,boost)) + + diff --git a/rtbkit/core/agent_configuration/testing/rtb_fees_test.cc b/rtbkit/core/agent_configuration/testing/rtb_fees_test.cc new file mode 100644 index 000000000..ab3f32180 --- /dev/null +++ b/rtbkit/core/agent_configuration/testing/rtb_fees_test.cc @@ -0,0 +1,209 @@ +/* rtb_agent_config_validator.cc +*/ + +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK + +#include +#include "rtbkit/core/agent_configuration/agent_config.h" +#include + +using namespace std; +using namespace RTBKIT; + +BOOST_AUTO_TEST_CASE( test_fees ) +{ + Amount bidPrice = MicroUSD(5); + + std::string payload; + Json::Value json; + std::shared_ptr fees; + + /***********************************************************/ + // Null Fees + payload = R"JSON( { + "type": "none" + } + )JSON"; + + json = Json::parse(payload); + fees = Fees::createFees(json); + + BOOST_CHECK_EQUAL(fees->applyFees(bidPrice), bidPrice); + + + payload = R"JSON( { + } + )JSON"; + + json = Json::parse(payload); + BOOST_CHECK_THROW(Fees::createFees(json),ML::Exception); + + /***********************************************************/ + // Flat Fees + payload = R"JSON( { + "type": "flat", + "params": { + "a": "1USD/1M" + } + } + )JSON"; + + json = Json::parse(payload); + fees = Fees::createFees(json); + + BOOST_CHECK_EQUAL(fees->applyFees(bidPrice), bidPrice - MicroUSD(1)); + + payload = R"JSON( { + "type": "flat", + "params": { + "a": "6USD/1M" + } + } + )JSON"; + + json = Json::parse(payload); + fees = Fees::createFees(json); + + BOOST_CHECK_EQUAL(fees->applyFees(bidPrice), MicroUSD(0)); + + // Check error handling + payload = R"JSON( { + "type": "flat", + "params": 1 + } + )JSON"; + + json = Json::parse(payload); + BOOST_CHECK_THROW(Fees::createFees(json),ML::Exception); + + payload = R"JSON( { + "type": "flat", + "params": { + "b" : 1 + } + } + )JSON"; + + json = Json::parse(payload); + BOOST_CHECK_THROW(Fees::createFees(json),ML::Exception); + + payload = R"JSON( { + "type": "flat", + "params": { + "a" : 1.2 + } + } + )JSON"; + + json = Json::parse(payload); + BOOST_CHECK_THROW(Fees::createFees(json),ML::Exception); + + payload = R"JSON( { + "type": "flat", + "params": { + "a": "-6USD/1M" + } + } + )JSON"; + + json = Json::parse(payload); + BOOST_CHECK_THROW(Fees::createFees(json),ML::Exception); + + /***********************************************************/ + // Linear Fees + payload = R"JSON( { + "type": "linear", + "params": { + "a": 1, + "b" : -2 + } + } + )JSON"; + + json = Json::parse(payload); + fees = Fees::createFees(json); + + BOOST_CHECK_EQUAL(fees->applyFees(bidPrice), bidPrice * 1 + MicroUSD(-2)); + + payload = R"JSON( { + "type": "linear", + "params": { + "a": 1.25, + "b" : -10000 + } + } + )JSON"; + + json = Json::parse(payload); + fees = Fees::createFees(json); + + BOOST_CHECK_EQUAL(fees->applyFees(bidPrice), MicroUSD(0)); + + // Check error handling + payload = R"JSON( { + "type": "linear", + "params": 1 + } + )JSON"; + + json = Json::parse(payload); + BOOST_CHECK_THROW(Fees::createFees(json),ML::Exception); + + payload = R"JSON( { + "type": "linear", + "params": { + "b" : 1 + } + } + )JSON"; + + json = Json::parse(payload); + BOOST_CHECK_THROW(Fees::createFees(json),ML::Exception); + + payload = R"JSON( { + "type": "linear", + "params": { + "a" : 1 + } + } + )JSON"; + + json = Json::parse(payload); + BOOST_CHECK_THROW(Fees::createFees(json),ML::Exception); + + payload = R"JSON( { + "type": "linear", + "params": { + "a" : "1" + } + } + )JSON"; + + json = Json::parse(payload); + BOOST_CHECK_THROW(Fees::createFees(json),ML::Exception); + + payload = R"JSON( { + "type": "linear", + "params": { + "a" : 1, + "b" : 10 + } + } + )JSON"; + + json = Json::parse(payload); + BOOST_CHECK_THROW(Fees::createFees(json),ML::Exception); + + payload = R"JSON( { + "type": "linear", + "params": { + "a" : 2, + "b" : -10 + } + } + )JSON"; + + json = Json::parse(payload); + BOOST_CHECK_THROW(Fees::createFees(json),ML::Exception); +} diff --git a/rtbkit/core/banker/local_banker.cc b/rtbkit/core/banker/local_banker.cc index ec340bea0..49a3f9f09 100644 --- a/rtbkit/core/banker/local_banker.cc +++ b/rtbkit/core/banker/local_banker.cc @@ -508,7 +508,7 @@ getProviderIndicators() const Date now = Date::now(); std::lock_guard guard(syncMtx); - bool syncOk = now < lastSync.plusSeconds(syncRate*2) || + bool syncOk = now < lastSync.plusSeconds(syncRate*4) || now < lastReauth.plusSeconds(reauthRate*2); MonitorIndicator ind; diff --git a/rtbkit/core/banker/null_banker.cc b/rtbkit/core/banker/null_banker.cc index 838670cb8..8cdbb05cd 100644 --- a/rtbkit/core/banker/null_banker.cc +++ b/rtbkit/core/banker/null_banker.cc @@ -55,8 +55,8 @@ addBudgetSync(const std::string & topLevelCampaign, /* NULL BANKER */ /*****************************************************************************/ -NullBanker::NullBanker(bool authorize) - : authorize_(authorize) +NullBanker::NullBanker(bool authorize, const std::string & serviceName) + : authorize_(authorize), serviceName_(serviceName) { } //---------------------------------------------------------------------- @@ -92,8 +92,9 @@ NullBanker:: getProviderIndicators() const { MonitorIndicator ind; + ind.serviceName = serviceName_; ind.status = true; - ind.message = "OK"; + ind.message = "NullBanker: OK"; return ind; } diff --git a/rtbkit/core/banker/null_banker.h b/rtbkit/core/banker/null_banker.h index 81250b29a..5f9a71cff 100644 --- a/rtbkit/core/banker/null_banker.h +++ b/rtbkit/core/banker/null_banker.h @@ -44,7 +44,7 @@ struct NullBudgetController: public BudgetController { class NullBanker: public Banker { public: - NullBanker(bool authorize = false); + NullBanker(bool authorize = false, const std::string & servicName = ""); virtual bool authorizeBid(const AccountKey & account, const std::string & item, @@ -107,6 +107,7 @@ class NullBanker: public Banker { friend class Datacratic::JS::NullBankerJS; bool authorize_; + std::string serviceName_; }; diff --git a/rtbkit/core/post_auction/post_auction_runner.cc b/rtbkit/core/post_auction/post_auction_runner.cc index cc588ab53..c5851f78f 100644 --- a/rtbkit/core/post_auction/post_auction_runner.cc +++ b/rtbkit/core/post_auction/post_auction_runner.cc @@ -10,6 +10,7 @@ #include "rtbkit/core/banker/slave_banker.h" #include "rtbkit/core/banker/local_banker.h" #include "rtbkit/core/banker/split_banker.h" +#include "rtbkit/core/banker/null_banker.h" #include "soa/service/service_utils.h" #include "soa/service/process_stats.h" #include "soa/utils/print_utils.h" @@ -130,7 +131,6 @@ init() LOG(print) << "winLoss pipe timeout is " << winLossPipeTimeout << std::endl; LOG(print) << "campaignEvent pipe timeout is " << campaignEventPipeTimeout << std::endl; - slaveBanker = bankerArgs.makeBanker(proxies, postAuctionLoop->serviceName() + ".slaveBanker"); if (localBankerUri != "") { localBanker = make_shared(proxies, POST_AUCTION, postAuctionLoop->serviceName()); localBanker->init(localBankerUri); @@ -146,14 +146,25 @@ init() } } } - postAuctionLoop->addSource("local-banker", *localBanker); + + slaveBanker = bankerArgs.makeBanker(proxies, postAuctionLoop->serviceName() + ".slaveBanker"); banker = make_shared(slaveBanker, localBanker, campaignSet); - } else if (localBanker && bankerChoice == "local") { postAuctionLoop->addSource("local-banker", *localBanker); + postAuctionLoop->addSource("slave-banker", *slaveBanker); + + } else if (localBanker && bankerChoice == "local") { banker = localBanker; + postAuctionLoop->addSource("local-banker", *localBanker); + + } else if (bankerChoice == "null") { + banker = make_shared(true, postAuctionLoop->serviceName()); + } else { + slaveBanker = bankerArgs.makeBanker(proxies, postAuctionLoop->serviceName() + ".slaveBanker"); banker = slaveBanker; + postAuctionLoop->addSource("slave-banker", *slaveBanker); } + postAuctionLoop->setBanker(banker); if (analyticsOn) { const auto & analyticsUri = proxies->params["analytics-uri"].asString(); @@ -164,8 +175,6 @@ init() LOG(print) << "analytics-uri is not in the config" << endl; } - postAuctionLoop->addSource("slave-banker", *slaveBanker); - postAuctionLoop->setBanker(banker); postAuctionLoop->bindTcp(); if (!forwardAuctionsUri.empty()) @@ -184,7 +193,7 @@ PostAuctionRunner:: shutdown() { postAuctionLoop->shutdown(); - slaveBanker->shutdown(); + if (slaveBanker) slaveBanker->shutdown(); if (localBanker) localBanker->shutdown(); } diff --git a/rtbkit/core/router/augmentation_loop.cc b/rtbkit/core/router/augmentation_loop.cc index 509fc5a5c..e51921310 100644 --- a/rtbkit/core/router/augmentation_loop.cc +++ b/rtbkit/core/router/augmentation_loop.cc @@ -95,9 +95,9 @@ init() doDisconnection(addr); }; - inbox.onEvent = [&] (const std::shared_ptr& entry) + inbox.onEvent = [&] (std::shared_ptr&& entry) { - doAugmentation(entry); + doAugmentation(std::move(entry)); }; addSource("AugmentationLoop::inbox", inbox); @@ -181,7 +181,7 @@ recordStats() { size_t inFlights = 0; for (const auto& instance : it->second->instances) - inFlights += instance.numInFlight; + inFlights += instance->numInFlight; recordLevel(inFlights, "augmentor.%s.numInFlight", it->first); } @@ -343,11 +343,11 @@ augment(const std::shared_ptr & info, } } -AugmentorInstanceInfo* +std::shared_ptr AugmentationLoop:: pickInstance(AugmentorInfo& aug) { - AugmentorInstanceInfo* instance = nullptr; + std::shared_ptr instance; int minInFlights = std::numeric_limits::max(); stringstream ss; @@ -355,11 +355,12 @@ pickInstance(AugmentorInfo& aug) for (auto it = aug.instances.begin(), end = aug.instances.end(); it != end; ++it) { - if (it->numInFlight >= minInFlights) continue; - if (it->numInFlight >= it->maxInFlight) continue; + auto & ptr = *it; + if (ptr->numInFlight >= minInFlights) continue; + if (ptr->numInFlight >= ptr->maxInFlight) continue; - instance = &(*it); - minInFlights = it->numInFlight; + instance = ptr; + minInFlights = ptr->numInFlight; } if (instance) instance->numInFlight++; @@ -369,7 +370,7 @@ pickInstance(AugmentorInfo& aug) void AugmentationLoop:: -doAugmentation(const std::shared_ptr & entry) +doAugmentation(std::shared_ptr&& entry) { Date now = Date::now(); @@ -389,7 +390,7 @@ doAugmentation(const std::shared_ptr & entry) { auto & aug = *augmentors[*it]; - const AugmentorInstanceInfo* instance = pickInstance(aug); + auto instance = pickInstance(aug); if (!instance) { recordHit("augmentor.%s.skippedTooManyInFlight", *it); continue; @@ -397,6 +398,8 @@ doAugmentation(const std::shared_ptr & entry) recordHit("augmentor.%s.instances.%s.request", *it, instance->addr); set agents = entry->augmentorAgents[*it]; + + entry->instances[*it] = instance; std::ostringstream availableAgentsStr; ML::DB::Store_Writer writer(availableAgentsStr); @@ -416,7 +419,7 @@ doAugmentation(const std::shared_ptr & entry) } if (sentToAugmentor) - augmenting.insert(entry->info->auction->id, entry, entry->timeout); + augmenting.insert(entry->info->auction->id, std::move(entry), entry->timeout); else entry->onFinished(entry->info); recordLevel(Date::now().secondsSince(now), "requestTimeMs"); @@ -454,7 +457,7 @@ doConfig(const std::vector & message) recordHit("augmentor.%s.configured", name); } - info->instances.emplace_back(addr, maxInFlight); + info->instances.push_back(std::make_shared(addr, maxInFlight)); recordHit("augmentor.%s.instances.%s.configured", name, addr); @@ -484,10 +487,11 @@ doDisconnection(const std::string & addr, const std::string & aug) end = augmentor.instances.end(); it != end; ++it) { - if (it->addr != addr) continue; + auto & ptr = *it; + if (ptr->addr != addr) continue; recordHit("augmentor.%s.instances.%s.disconnected", - augmentor.name, it->addr); + augmentor.name, ptr->addr); augmentor.instances.erase(it); break; @@ -592,6 +596,13 @@ void AugmentationLoop:: augmentationExpired(const Id & id, const Entry & entry) { + for (const auto & instance: entry.instances) { + // If the instance still exsits (it is still alive), we decrement + // the inFlight count + auto info = instance.second.lock(); + if (info) info->numInFlight--; + } + entry.onFinished(entry.info); } diff --git a/rtbkit/core/router/augmentation_loop.h b/rtbkit/core/router/augmentation_loop.h index fccb12dbd..b4d4bad9c 100644 --- a/rtbkit/core/router/augmentation_loop.h +++ b/rtbkit/core/router/augmentation_loop.h @@ -47,17 +47,17 @@ struct AugmentorInfo { AugmentorInfo(const std::string& name = "") : name(name) {} std::string name; ///< What the augmentation is called - std::vector instances; + std::vector> instances; - AugmentorInstanceInfo* findInstance(const std::string& addr) + std::shared_ptr findInstance(const std::string& addr) { for (auto it = instances.begin(), end = instances.end(); it != end; ++it) { - if (it->addr == addr) return &(*it); + auto & info = *it; + if (info->addr == addr) return info; } return nullptr; - } }; @@ -113,6 +113,14 @@ struct AugmentationLoop : public ServiceBase, public MessageLoop { struct Entry { std::shared_ptr info; std::set outstanding; + // We need to keep a list of current outstanding instances in our entry + // to be able to decrement the inFlight count when expiring an entry + // (after a timeout) + // + // Note that we are keeping a weak_ptr in the case where the instance + // either disconnects or crashes. Keeping a weak_ptr prevents us from + // possibly keeping a dangling pointer + std::map> instances; std::map > augmentorAgents; OnFinished onFinished; Date timeout; @@ -159,8 +167,8 @@ struct AugmentationLoop : public ServiceBase, public MessageLoop { void handleAugmentorMessage(const std::vector & message); - AugmentorInstanceInfo* pickInstance(AugmentorInfo& aug); - void doAugmentation(const std::shared_ptr & entry); + std::shared_ptr pickInstance(AugmentorInfo& aug); + void doAugmentation(std::shared_ptr&& entry); void recordStats(); diff --git a/rtbkit/core/router/filters/static_filters.cc b/rtbkit/core/router/filters/static_filters.cc index 0b02a5491..15b9770ca 100644 --- a/rtbkit/core/router/filters/static_filters.cc +++ b/rtbkit/core/router/filters/static_filters.cc @@ -144,6 +144,7 @@ setConfig(unsigned cfgIndex, const AgentConfig& config, bool value) } auto& entry = data[getKey(part)]; + entry.uid = config.externalId; if (entry.hashOn == UserPartition::NONE) { entry.modulus = part.modulus; entry.hashOn = part.hashOn; @@ -219,7 +220,8 @@ getValue(const BidRequest& br, const FilterEntry& entry) const if (str.empty() || str == "null") return make_pair(false, 0); - return make_pair(true, calcHash(str) % entry.modulus); + auto h = calcHash(str) + entry.uid; + return make_pair(true, h % entry.modulus); } diff --git a/rtbkit/core/router/filters/static_filters.h b/rtbkit/core/router/filters/static_filters.h index 0d119b883..508c8a203 100644 --- a/rtbkit/core/router/filters/static_filters.h +++ b/rtbkit/core/router/filters/static_filters.h @@ -77,6 +77,7 @@ struct UserPartitionFilter : public FilterBaseT IntervalFilter filter; ConfigSet excludeIfEmpty; int modulus; + int uid; UserPartition::HashOn hashOn; }; diff --git a/rtbkit/core/router/router.cc b/rtbkit/core/router/router.cc index 59c2973a9..baab953ea 100644 --- a/rtbkit/core/router/router.cc +++ b/rtbkit/core/router/router.cc @@ -5,7 +5,6 @@ RTB router code. */ -#include #include #include "router.h" #include "soa/service/zmq_utils.h" @@ -118,8 +117,7 @@ Router(ServiceBase & parent, Amount maxBidAmount, int secondsUntilSlowMode, Amount slowModeAuthorizedMoneyLimit, - Seconds augmentationWindow, - Json::Value filtersConfig) + Seconds augmentationWindow) : ServiceBase(serviceName, parent), shutdown_(false), postAuctionEndpoint(*this), @@ -157,8 +155,7 @@ Router(ServiceBase & parent, monitorProviderClient(getZmqContext()), maxBidAmount(maxBidAmount), slowModeTolerance(MonitorClient::DefaultTolerance), - augmentationWindow(augmentationWindow), - filtersConfig(filtersConfig) + augmentationWindow(augmentationWindow) { monitorProviderClient.addProvider(this); } @@ -174,8 +171,7 @@ Router(std::shared_ptr services, Amount maxBidAmount, int secondsUntilSlowMode, Amount slowModeAuthorizedMoneyLimit, - Seconds augmentationWindow, - Json::Value filtersConfig) + Seconds augmentationWindow) : ServiceBase(serviceName, services), shutdown_(false), postAuctionEndpoint(*this), @@ -213,8 +209,7 @@ Router(std::shared_ptr services, monitorProviderClient(getZmqContext()), maxBidAmount(maxBidAmount), slowModeTolerance(MonitorClient::DefaultTolerance), - augmentationWindow(augmentationWindow), - filtersConfig(filtersConfig) + augmentationWindow(augmentationWindow) { monitorProviderClient.addProvider(this); @@ -238,22 +233,57 @@ initAnalytics(const string & baseUrl, const int numConnections) void Router:: -init() +initExchanges(const Json::Value & config) { + for (auto & exchange: config) { + initExchange(exchange); + } +} + +void +Router:: +initExchange(const std::string & type, + const Json::Value & config) { - ExcAssert(!initialized); + auto exchange = ExchangeConnector::create(type, *this, type); + exchange->configure(config); - registerServiceProvider(serviceName(), { "rtbRequestRouter" }); + std::shared_ptr item(exchange.release()); + addExchangeNoConnect(item); - filters.init(this); + exchangeBuffer.push(item); +} + +void +Router:: +initExchange(const Json::Value & exchangeConfig) +{ + std::string exchangeType = exchangeConfig["exchangeType"].asString(); + initExchange(exchangeType, exchangeConfig); +} + +void +Router:: +initFilters(const Json::Value & config) { - if (filtersConfig != Json::Value::null){ - if (!filtersConfig.isArray()){ + if (config != Json::Value::null) { + if (!config.isArray()) { throw Exception("couldn't parse formats other then array"); } - filters.initWithFiltersFromJson(filtersConfig); - } - else + filters.initWithFiltersFromJson(config); + } else { filters.initWithDefaultFilters(); + } +} + +void +Router:: +init() +{ + ExcAssert(!initialized); + + registerServiceProvider(serviceName(), { "rtbRequestRouter" }); + + filters.init(this); banker.reset(new NullBanker()); @@ -415,6 +445,11 @@ start(boost::function onStop) if (onStop) onStop(); }; + for ( auto & exchange : exchanges) { + exchange->start(); + connectExchange(*exchange); + } + bidder->start(); logger.start(); analytics.start(); @@ -961,14 +996,8 @@ logUsageMetrics(double period) } } - set agentAccounts; for (const auto & item : agents) { auto & info = item.second; - const AccountKey & account = info.config->account; - if (!agentAccounts.insert(account).second) { - continue; - } - auto & last = lastAgentUsageMetrics[item.first]; AgentUsageMetrics newMetrics(info.stats->intoFilters, @@ -2047,13 +2076,13 @@ doBidImpl(const BidMessage &message, const std::vector &originalMes cerr << "creative not compatible with spot: " << endl; cerr << "auction: " << auctionInfo.auction->requestStr << endl; - cerr << "config: " << config.toJson() << endl; + cerr << "config: " << config.toJson().toStringNoNewLine() << endl; cerr << "bid: " << bidsString << endl; - cerr << "spot: " << imp[i].toJson() << endl; + cerr << "spot: " << imp[i].toJson().toStringNoNewLine() << endl; cerr << "spot num: " << spotIndex << endl; cerr << "bid num: " << i << endl; cerr << "creative num: " << bid.creativeIndex << endl; - cerr << "creative: " << creative.toJson() << endl; + cerr << "creative: " << creative.toJson().toStringNoNewLine() << endl; #endif returnInvalidBid(agent, bidsString, auctionInfo.auction, "creativeNotCompatibleWithSpot", @@ -2101,7 +2130,7 @@ doBidImpl(const BidMessage &message, const std::vector &originalMes slowModePeriodicSpentReached = true; bidder->sendBidDroppedMessage(agentConfig, agent, auctionInfo.auction); recordHit("slowMode.droppedBid"); - recordHit("accounts.%s.DROPPED", config.account.toString('.')); + recordHit("accounts.%s.IGNORED", config.account.toString('.')); continue; } } @@ -2564,11 +2593,15 @@ doConfig(const std::string & agent, RouterProfiler profiler(dutyCycleCurrent.nsConfig); if (!config) { - cerr << "agent " << agent << " lost configuration" << endl; - filters.removeConfig(agent); auto it = agents.find(agent); - ExcAssert(it != std::end(agents)); - agents.erase(it); + // It might happen that we don't find the agent if for example we received + // an empty configuration because the agent crashed prior to sending its initial + // configuration to the ACS. + if (it != std::end(agents)) { + cerr << "agent " << agent << " lost configuration" << endl; + filters.removeConfig(agent); + agents.erase(it); + } } else { AgentInfo & info = agents[agent]; logMessage("CONFIG", agent, boost::trim_copy(config->toJson().toString())); @@ -2969,30 +3002,6 @@ getProviderIndicators() return ind; } -void -Router:: -startExchange(const std::string & type, - const Json::Value & config) -{ - auto exchange = ExchangeConnector::create(type, *this, type); - exchange->configure(config); - exchange->start(); - - std::shared_ptr item(exchange.release()); - addExchange(item); - - exchangeBuffer.push(item); - -} - -void -Router:: -startExchange(const Json::Value & exchangeConfig) -{ - std::string exchangeType = exchangeConfig["exchangeType"].asString(); - startExchange(exchangeType, exchangeConfig); -} - } // namespace RTBKIT diff --git a/rtbkit/core/router/router.h b/rtbkit/core/router/router.h index 519d87a25..a12f5b7a0 100644 --- a/rtbkit/core/router/router.h +++ b/rtbkit/core/router/router.h @@ -113,8 +113,7 @@ struct Router : public ServiceBase, Amount maxBidAmount = USD_CPM(40), int secondsUntilSlowMode = MonitorClient::DefaultCheckTimeout, Amount slowModeAuthorizedMoneyLimit = USD_CPM(100), - Seconds augmentationWindow = std::chrono::milliseconds(5), - Json::Value filtersConfig = Json::Value::null); + Seconds augmentationWindow = std::chrono::milliseconds(5)); Router(std::shared_ptr services = std::make_shared(), const std::string & serviceName = "router", @@ -126,8 +125,7 @@ struct Router : public ServiceBase, Amount maxBidAmount = USD_CPM(40), int secondsUntilSlowMode = MonitorClient::DefaultCheckTimeout, Amount slowModeAuthorizedMoneyLimit = USD_CPM(100), - Seconds augmentationWindow = std::chrono::milliseconds(5), - Json::Value filtersConfig = Json::Value::null); + Seconds augmentationWindow = std::chrono::milliseconds(5)); ~Router(); @@ -149,8 +147,11 @@ struct Router : public ServiceBase, /** Initialize analytics if it is used. */ void initAnalytics(const std::string & baseUrl, const int numConnections); - /** Initialize filters from json configuration*/ - void initWithFiltersFromJson(const Json::Value & json); + /** Initialize exchages from json configuration. */ + void initExchanges(const Json::Value & config); + + /** Initialize filters from json configuration. */ + void initFilters(const Json::Value & config = Json::Value::null); /** Initialize all of the internal data structures and configuration. */ void init(); @@ -266,16 +267,22 @@ struct Router : public ServiceBase, connectExchange(*exchange); } - /** Start up a new exchange and connect it to the router. The exchange - will read its configuration from the given JSON blob. - */ - void startExchange(const std::string & exchangeType, + void addExchangeNoConnect(std::shared_ptr const & exchange) + { + loopMonitor.addCallback( + "exchanges." + exchange->exchangeName(), + exchange->getLoadSampleFn()); + + Guard guard(lock); + exchanges.push_back(exchange); + } + + /** Start up a new exchange from type and configuration from the given JSON blob. */ + void initExchange(const std::string & exchangeType, const Json::Value & exchangeConfig); - /** Start up a new exchange and connect it to the router. The exchange - will read its configuration and type from the given JSON blob. - */ - void startExchange(const Json::Value & exchangeConfig); + /** Init a new exchange from configuration and type from the given JSON blob. */ + void initExchange(const Json::Value & exchangeConfig); /** Inject an auction into the router. auction: the auction object @@ -811,7 +818,6 @@ struct Router : public ServiceBase, double slowModeTolerance; Seconds augmentationWindow; - Json::Value filtersConfig; }; diff --git a/rtbkit/core/router/router_runner.cc b/rtbkit/core/router/router_runner.cc index 374566498..8d1c6756a 100644 --- a/rtbkit/core/router/router_runner.cc +++ b/rtbkit/core/router/router_runner.cc @@ -20,6 +20,7 @@ #include "rtbkit/core/banker/slave_banker.h" #include "rtbkit/core/banker/local_banker.h" #include "rtbkit/core/banker/split_banker.h" +#include "rtbkit/core/banker/null_banker.h" #include "soa/service/process_stats.h" #include "jml/arch/timers.h" #include "jml/utils/file_functions.h" @@ -172,7 +173,7 @@ init() enableBidProbability, logAuctions, logBids, USD_CPM(maxBidPrice), - slowModeTimeout, amountSlowModeMoneyLimit, augmentationWindow, filtersConfig); + slowModeTimeout, amountSlowModeMoneyLimit, augmentationWindow); router->slowModeTolerance = slowModeTolerance; router->initBidderInterface(bidderConfig); if (dableSlowMode) { @@ -188,7 +189,6 @@ init() } router->init(); - slaveBanker = bankerArgs.makeBanker(proxies, router->serviceName() + ".slaveBanker"); if (localBankerUri != "") { localBanker = make_shared(proxies, ROUTER, router->serviceName()); localBanker->init(localBankerUri); @@ -205,14 +205,20 @@ init() } } } + slaveBanker = bankerArgs.makeBanker(proxies, router->serviceName() + ".slaveBanker"); banker = make_shared(slaveBanker, localBanker, campaignSet); } else if (localBanker && bankerChoice == "local") { banker = localBanker; + } else if (bankerChoice == "null") { + banker = make_shared(true, router->serviceName()); } else { + slaveBanker = bankerArgs.makeBanker(proxies, router->serviceName() + ".slaveBanker"); banker = slaveBanker; } router->setBanker(banker); + router->initExchanges(exchangeConfig); + router->initFilters(filtersConfig); router->bindTcp(); } @@ -220,13 +226,9 @@ void RouterRunner:: start() { - slaveBanker->start(); + if (slaveBanker) slaveBanker->start(); if (localBanker) localBanker->start(); router->start(); - - // Start all exchanges - for (auto & exchange: exchangeConfig) - router->startExchange(exchange); } void @@ -234,7 +236,7 @@ RouterRunner:: shutdown() { router->shutdown(); - slaveBanker->shutdown(); + if (slaveBanker) slaveBanker->shutdown(); if (localBanker) localBanker->shutdown(); } diff --git a/rtbkit/core/router/router_stack.cc b/rtbkit/core/router/router_stack.cc index 6833dc78a..369caaec7 100644 --- a/rtbkit/core/router/router_stack.cc +++ b/rtbkit/core/router/router_stack.cc @@ -70,6 +70,7 @@ init() postAuctionLoop.bindTcp(); router.init(); + router.initFilters(); router.setBanker(makeSlaveBanker("router")); router.bindTcp(); diff --git a/rtbkit/examples/availability_agent/availability_agent.cc b/rtbkit/examples/availability_agent/availability_agent.cc index 96cde846a..0ff45cd9e 100644 --- a/rtbkit/examples/availability_agent/availability_agent.cc +++ b/rtbkit/examples/availability_agent/availability_agent.cc @@ -56,7 +56,7 @@ start() serviceRunning = true; checker.reset(new AvailabilityCheck(ringBufferSize)); - checker->onEvent = [&](const string& ev, EventType type, float val) { + checker->onEvent = [&](const string& ev, StatEventType type, float val) { recordEvent(ev.c_str(), type, val); if (ev == "newRequests") ML::atomic_inc(stats.newRequests); diff --git a/rtbkit/examples/availability_agent/availability_check.h b/rtbkit/examples/availability_agent/availability_check.h index d838168b3..e00e9022f 100644 --- a/rtbkit/examples/availability_agent/availability_check.h +++ b/rtbkit/examples/availability_agent/availability_check.h @@ -50,7 +50,7 @@ struct AvailabilityCheck /** Notify any attached logger that an event took place. */ - std::function onEvent; + std::function onEvent; private: size_t size; diff --git a/rtbkit/examples/bid_request_endpoint.cc b/rtbkit/examples/bid_request_endpoint.cc index 1df5862b1..b4806b8f3 100644 --- a/rtbkit/examples/bid_request_endpoint.cc +++ b/rtbkit/examples/bid_request_endpoint.cc @@ -79,16 +79,15 @@ int main(int argc, char ** argv) { // bid. router.setBanker(std::make_shared(true)); + // Configure exchange connectors + Json::Value exchangeConfiguration = loadJsonFromFile(exchangeConfigurationFile); + router.initExchanges(exchangeConfiguration); + router.initFilters(); + // Start the router up router.bindTcp(); router.start(); - // Configure exchange connectors - Json::Value exchangeConfiguration = loadJsonFromFile(exchangeConfigurationFile); - for(auto & exchange : exchangeConfiguration) { - router.startExchange(exchange); - } - router.forAllExchanges([&](std::shared_ptr const & item) { item->enableUntil(Date::positiveInfinity()); if(!recordFile.empty()) { diff --git a/rtbkit/examples/integration_endpoints.cc b/rtbkit/examples/integration_endpoints.cc index 44a88cc7d..7a18c627d 100644 --- a/rtbkit/examples/integration_endpoints.cc +++ b/rtbkit/examples/integration_endpoints.cc @@ -116,16 +116,15 @@ int main(int argc, char ** argv) { // bid. router.setBanker(std::make_shared(true)); + // Configure exchange connectors + Json::Value exchangeConfiguration = loadJsonFromFile(exchangeConfigurationFile); + router.initExchanges(exchangeConfiguration); + router.initFilters(); + // Start the router up router.bindTcp(); router.start(); - // Configure exchange connectors - Json::Value exchangeConfiguration = loadJsonFromFile(exchangeConfigurationFile); - for(auto & exchange : exchangeConfiguration) { - router.startExchange(exchange); - } - router.forAllExchanges([&](std::shared_ptr const & item) { item->enableUntil(Date::positiveInfinity()); if(!recordFile.empty()) { diff --git a/rtbkit/examples/multi_agent.cc b/rtbkit/examples/multi_agent.cc index 0d61de487..1b98c2270 100644 --- a/rtbkit/examples/multi_agent.cc +++ b/rtbkit/examples/multi_agent.cc @@ -91,15 +91,15 @@ int main(int argc, char ** argv) // bid. router.setBanker(std::make_shared(true)); + // Configure exchange connectors + Json::Value exchangeConfiguration = loadJsonFromFile(exchangeConfigurationFile); + router.initExchanges(exchangeConfiguration); + router.initFilters(); + // Start the router up router.bindTcp(); router.start(); - // Configure exchange connectors - Json::Value exchangeConfiguration = loadJsonFromFile(exchangeConfigurationFile); - for(auto & exchange : exchangeConfiguration) { - router.startExchange(exchange); - } router.forAllExchanges([&](std::shared_ptr const & item) { item->enableUntil(Date::positiveInfinity()); diff --git a/rtbkit/examples/rtbkit_integration_test.cc b/rtbkit/examples/rtbkit_integration_test.cc index c72eb924c..536971bd0 100644 --- a/rtbkit/examples/rtbkit_integration_test.cc +++ b/rtbkit/examples/rtbkit_integration_test.cc @@ -321,7 +321,9 @@ struct Components } router1.addExchange(*exchangeConnectors[0]); + router1.initFilters(); router2.addExchange(*exchangeConnectors[1]); + router2.initFilters(); // Setup an ad server connector that also acts as a midlle men between // the exchange's wins and the post auction loop. diff --git a/rtbkit/plugins/adserver/http_adserver_connector.h b/rtbkit/plugins/adserver/http_adserver_connector.h index 1d5838dab..90f8a4e6b 100644 --- a/rtbkit/plugins/adserver/http_adserver_connector.h +++ b/rtbkit/plugins/adserver/http_adserver_connector.h @@ -75,12 +75,12 @@ struct HttpAdServerHttpEndpoint : public Datacratic::HttpEndpoint { /* carbon logging */ typedef std::function)> OnEvent; OnEvent onEvent; - void doEvent(const char * eventName, EventType type = ET_COUNT, + void doEvent(const char * eventName, StatEventType type = ET_COUNT, float value = 1.0, const char * units = "", std::initializer_list extra = DefaultOutcomePercentiles) const diff --git a/rtbkit/plugins/analytics/analytics_runner.cc b/rtbkit/plugins/analytics/analytics_runner.cc index 9bdeae3f5..1a15558d5 100644 --- a/rtbkit/plugins/analytics/analytics_runner.cc +++ b/rtbkit/plugins/analytics/analytics_runner.cc @@ -20,26 +20,28 @@ int main(int argc, char ** argv) { bool enableAllChannels = false; ChannelFilter channels( - { {"AUCTION", false}, - {"ADSERVER_ERROR",false}, - {"BID", false}, - {"CLICK", false}, - {"CONFIG", false}, - {"CONVERSION", false}, - {"ERROR", false}, - {"EXCHANGE_ERROR",false}, - {"MATCHEDCLICK", false}, - {"MATCHEDLOSS", false}, - {"MATCHEDWIN", false}, - {"NOBUDGET", false}, - {"PAERROR", false}, - {"SUBMITTED", false}, - {"TOOLATE", false}, - {"UNMATCHEDWIN", false}, - {"UNMATCHEDLOSS", false}, - {"UNMATCHEDCLICK",false}, - {"USAGE", false}, - {"WIN", false} + { {"AUCTION", false}, + {"ADSERVER_ERROR", false}, + {"BID", false}, + {"CLICK", false}, + {"CONFIG", false}, + {"CONVERSION", false}, + {"ERROR", false}, + {"EXCHANGE_ERROR", false}, + {"MATCHEDCLICK", false}, + {"MATCHEDLOSS", false}, + {"MATCHEDWIN", false}, + {"MATCHEDCONVERSION", false}, + {"NOBUDGET", false}, + {"PAERROR", false}, + {"SUBMITTED", false}, + {"TOOLATE", false}, + {"UNMATCHEDWIN", false}, + {"UNMATCHEDLOSS", false}, + {"UNMATCHEDCLICK", false}, + {"UNMATCHEDCONVERSION", false}, + {"USAGE", false}, + {"WIN", false} } ); options_description configuration_options("Configuration options"); diff --git a/rtbkit/plugins/bid_request/openrtb_bid_request_parser.cc b/rtbkit/plugins/bid_request/openrtb_bid_request_parser.cc index 87522a151..0e68accbc 100644 --- a/rtbkit/plugins/bid_request/openrtb_bid_request_parser.cc +++ b/rtbkit/plugins/bid_request/openrtb_bid_request_parser.cc @@ -279,25 +279,25 @@ void OpenRTBBidRequestParser:: onImpression(OpenRTB::Impression & impression) { - ctx.spot = std::move(std::unique_ptr(new AdSpot(std::move(impression)))); + ctx.spot = AdSpot(std::move(impression)); /* if(!ctx.spot->banner && !ctx.spot->video) LOG(openrtbBidRequestError) << "br.imp must included either a video or a banner object." << endl; */ // Possible to have a video and a banner object. - if(ctx.spot->banner) { - this->onBanner(*ctx.spot->banner); + if(ctx.spot.banner) { + this->onBanner(*ctx.spot.banner); } - if(ctx.spot->video) { - if(ctx.spot->banner && ctx.spot->banner->id == Id("0")) + if(ctx.spot.video) { + if(ctx.spot.banner && ctx.spot.banner->id == Id("0")) LOG(OpenRTBBidRequestLogs::trace) << "It's recommended to include br.imp.banner.id when subordinate to video object." << endl; - this->onVideo(*ctx.spot->video); + this->onVideo(*ctx.spot.video); } // TODO Support tagFilters / mime filers - ctx.br->imp.emplace_back(*ctx.spot); + ctx.br->imp.emplace_back(std::move(ctx.spot)); } void @@ -316,7 +316,7 @@ onBanner(OpenRTB::Banner & banner) { LOG(OpenRTBBidRequestLogs::error) << "Mismatch between number of width and heights illegal." << endl; for(unsigned int i = 0; i < banner.w.size(); ++i) { - ctx.spot->formats.push_back(Format(banner.w[i], banner.h[i])); + ctx.spot.formats.push_back(Format(banner.w[i], banner.h[i])); } // Add api to the segments in order to filter on it @@ -325,7 +325,7 @@ onBanner(OpenRTB::Banner & banner) { if (framework != apiFrameworks.end()) ctx.br->segments.add("api-banner", framework->second , 1.0); } - ctx.spot->position = banner.pos; + ctx.spot.position = banner.pos; } void @@ -360,7 +360,7 @@ onVideo(OpenRTB::Video & video) { THROW(OpenRTBBidRequestLogs::error) << "br.imp.video.maxduration can't be smaller than br.imp.video.minduration." << endl; } - ctx.spot->position = video.pos; + ctx.spot.position = video.pos; // Add api to the segments in order to filter on it for(auto & api : video.api) { @@ -368,7 +368,7 @@ onVideo(OpenRTB::Video & video) { if (framework != apiFrameworks.end()) ctx.br->segments.add("api-video", framework->second, 1.0); } - ctx.spot->formats.push_back(Format(video.w.value(), video.h.value())); + ctx.spot.formats.push_back(Format(video.w.value(), video.h.value())); } void @@ -703,7 +703,7 @@ onVideo(OpenRTB::Video & video) { THROW(OpenRTBBidRequestLogs::error22) << "br.imp.video.maxduration can't be smaller than br.imp.video.minduration." << endl; } - ctx.spot->position = video.pos; + ctx.spot.position = video.pos; // Add api to the segments in order to filter on it for(auto & api : video.api) { @@ -711,7 +711,7 @@ onVideo(OpenRTB::Video & video) { if (framework != apiFrameworks.end()) ctx.br->segments.add("api-video", framework->second, 1.0); } - ctx.spot->formats.push_back(Format(video.w.value(), video.h.value())); + ctx.spot.formats.push_back(Format(video.w.value(), video.h.value())); } diff --git a/rtbkit/plugins/bid_request/openrtb_bid_request_parser.h b/rtbkit/plugins/bid_request/openrtb_bid_request_parser.h index daa7f19fe..7cdcb781f 100644 --- a/rtbkit/plugins/bid_request/openrtb_bid_request_parser.h +++ b/rtbkit/plugins/bid_request/openrtb_bid_request_parser.h @@ -49,7 +49,7 @@ struct OpenRTBBidRequestParser struct OpenRTBParsingContext { std::unique_ptr br; - std::unique_ptr spot; + AdSpot spot; } ctx; virtual ~OpenRTBBidRequestParser(){}; diff --git a/rtbkit/plugins/bidder_interface/http_bidder_interface.cc b/rtbkit/plugins/bidder_interface/http_bidder_interface.cc index 4d8b9e2dd..12949a52f 100644 --- a/rtbkit/plugins/bidder_interface/http_bidder_interface.cc +++ b/rtbkit/plugins/bidder_interface/http_bidder_interface.cc @@ -199,7 +199,15 @@ void HttpBidderInterface::sendAuctionMessage(std::shared_ptr const & au }; BidRequest & originalRequest = *auction->request; - std::shared_ptr parser = OpenRTBBidRequestParser::openRTBBidRequestParserFactory("2.1"); + + std::string openRtbVersion; + if (!originalRequest.protocolVersion.empty()) + openRtbVersion = originalRequest.protocolVersion; + else + openRtbVersion = "2.1"; + + std::shared_ptr parser = OpenRTBBidRequestParser::openRTBBidRequestParserFactory(openRtbVersion); + OpenRTB::BidRequest openRtbRequest = parser->toBidRequest(originalRequest); bool ok = prepareRequest(openRtbRequest, originalRequest, auction, bidders); @@ -207,10 +215,27 @@ void HttpBidderInterface::sendAuctionMessage(std::shared_ptr const & au if (!ok) { return; } - StructuredJsonPrintingContext context; - desc.printJson(&openRtbRequest, context); - auto requestStr = context.output.toString(); + string requestStr; + if (routerFormat == FMT_DATACRATIC) { + Json::Value jReq(Json::objectValue); + jReq["id"] = openRtbRequest.id.toString(); + + DefaultDescription impDesc; + for (auto & imp : openRtbRequest.imp) { + StructuredJsonPrintingContext ctx; + impDesc.printJson(&imp, ctx); + jReq["imp"].append(ctx.output); + } + + jReq["ext"]["datacratic"] = openRtbRequest.ext["datacratic"]; + jReq["ext"]["rtbkit"] = openRtbRequest.ext["rtbkit"]; + requestStr = jReq.toString(); + } else { + StructuredJsonPrintingContext context; + desc.printJson(&openRtbRequest, context); + requestStr = context.output.toString(); + } Date sentResponseTime = Date::now(); /* We need to capture by copy inside the lambda otherwise we might get @@ -371,7 +396,7 @@ void HttpBidderInterface::sendAuctionMessage(std::shared_ptr const & au HttpRequest::Content reqContent { requestStr, "application/json" }; - RestParams headers { { "x-openrtb-version", "2.1" } }; + RestParams headers { { "x-openrtb-version", openRtbVersion } }; // std::cerr << "Sending HTTP POST to: " << routerHost << " " << routerPath << std::endl; // std::cerr << "Content " << reqContent.str << std::endl; @@ -383,12 +408,40 @@ void HttpBidderInterface::sendLossMessage( const std::shared_ptr& agentConfig, std::string const & agent, std::string const & id) { + auto callbacks = std::make_shared( + [=](const HttpRequest &, HttpClientError errorCode, + int statusCode, const std::string &, std::string &&body) + { + if (errorCode != HttpClientError::None) { + LOG(error) << "Error requesting " + << adserverHost << ":" << adserverEventPort + << " (" << httpErrorString(errorCode) << ")" << std::endl; + recordError("network"); + } + }); + + Json::Value content; + + if (adserverEventFormat == FMT_DATACRATIC) { + content["id"] = id; + + Json::Value entry; + { + entry["cid"] = agent; + entry["type"] = "loss"; + entry["id"] = id; + } + content["events"].append(entry); + + } else ExcAssert(false); + + HttpRequest::Content reqContent { content, "application/json" }; + httpClientAdserverEvents->post(adserverEventPath, callbacks, reqContent, {} /* queryParams */); } void HttpBidderInterface::sendWinLossMessage( const std::shared_ptr& agentConfig, MatchedWinLoss const & event) { - if (event.type == MatchedWinLoss::Loss) return; auto callbacks = std::make_shared( [=](const HttpRequest &, HttpClientError errorCode, @@ -425,7 +478,8 @@ void HttpBidderInterface::sendWinLossMessage( { entry["impid"] = event.impId.toString(); entry["type"] = event.type == MatchedWinLoss::Loss ? "loss" : "win"; - entry["price"] = (double) getAmountIn(event.winPrice); + entry["bidPrice"] = (double) getAmountIn(event.response.price.maxPrice); + entry["winPrice"] = event.type == MatchedWinLoss::Loss ? 0.0 : (double) getAmountIn(event.winPrice); entry["cid"] = event.response.account[1]; entry["ext"]["datacratic"]["meta"] = Json::parse(event.response.meta.rawString()); @@ -444,7 +498,6 @@ void HttpBidderInterface::sendWinLossMessage( HttpRequest::Content reqContent { content, "application/json" }; httpClientAdserverWins->post(adserverWinPath, callbacks, reqContent, { } /* queryParams */); - } @@ -520,7 +573,7 @@ void HttpBidderInterface::sendBidErrorMessage( Json::Value content; content["id"] = auction->id.toString(); - content["crid"] = agentConfig->account[1]; + content["cid"] = agent; content["type"] = type; if (!reason.empty()) content["reason"] = reason; @@ -533,7 +586,7 @@ void HttpBidderInterface::sendBidDroppedMessage( std::string const & agent, std::shared_ptr const & auction) { if (adserverEventFormat == FMT_DATACRATIC) - sendBidErrorMessage(agentConfig, agent, auction, "DROPPED"); + sendBidErrorMessage(agentConfig, agent, auction, "ERROR", "DROPPED"); } void HttpBidderInterface::sendBidInvalidMessage( @@ -542,7 +595,7 @@ void HttpBidderInterface::sendBidInvalidMessage( std::shared_ptr const & auction) { if (adserverEventFormat == FMT_DATACRATIC) - sendBidErrorMessage(agentConfig, agent, auction, "INVALID"); + sendBidErrorMessage(agentConfig, agent, auction, "ERROR", "INVALID"); } void HttpBidderInterface::sendNoBudgetMessage( @@ -550,7 +603,7 @@ void HttpBidderInterface::sendNoBudgetMessage( std::string const & agent, std::shared_ptr const & auction) { if (adserverEventFormat == FMT_DATACRATIC) - sendBidErrorMessage(agentConfig, agent, auction, "NOBUDGET"); + sendBidErrorMessage(agentConfig, agent, auction, "ERROR", "NOBUDGET"); } void HttpBidderInterface::sendTooLateMessage( @@ -558,7 +611,7 @@ void HttpBidderInterface::sendTooLateMessage( std::string const & agent, std::shared_ptr const & auction) { if (adserverEventFormat == FMT_DATACRATIC) - sendBidErrorMessage(agentConfig, agent, auction, "TOOLATE"); + sendBidErrorMessage(agentConfig, agent, auction, "ERROR", "TOOLATE"); } void HttpBidderInterface::sendMessage( @@ -655,6 +708,8 @@ bool HttpBidderInterface::prepareStandardRequest(OpenRTB::BidRequest &request, const std::map &bidders) const { tagRequest(request, bidders); + request.ext["exchange"] = originalRequest.exchange; + // Take any augmentation data and fill in the ext field of the bid request with the data, // under the rtbkit "namespace" const auto& augmentations = auction->augmentations; diff --git a/rtbkit/plugins/bidding_agent/bidding_agent.cc b/rtbkit/plugins/bidding_agent/bidding_agent.cc index b6f55838f..c25c0f36b 100644 --- a/rtbkit/plugins/bidding_agent/bidding_agent.cc +++ b/rtbkit/plugins/bidding_agent/bidding_agent.cc @@ -471,8 +471,18 @@ handleDelivery(const std::vector& msg, DeliveryCbFn& callback) void BiddingAgent:: -doBid(Id id, const Bids & bids, const Json::Value & jsonMeta, const WinCostModel & wcm) +doBid(Id id, Bids bids, const Json::Value & jsonMeta, const WinCostModel & wcm) { + for (Bid& bid : bids) { + if (bid.creativeIndex >= 0) { + if (!bid.isNullBid()) { + recordLevel(bid.price.value, "bidPrice." + bid.price.getCurrencyStr()); + } + + bid.price = agent_config.creatives[bid.creativeIndex].fees->applyFees(bid.price); + } + } + Json::FastWriter jsonWriter; string response = jsonWriter.write(bids.toJson()); @@ -517,7 +527,7 @@ doBid(Id id, const Bids & bids, const Json::Value & jsonMeta, const WinCostModel if (bid.isNullBid()) recordHit("noBid"); else { recordHit("bids"); - recordLevel(bid.price.value, "bidPrice." + bid.price.getCurrencyStr()); + recordLevel(bid.price.value, "bidPriceAugmented." + bid.price.getCurrencyStr()); } } } @@ -573,6 +583,9 @@ doConfigJson(Json::Value jsonConfig) boost::trim(newConfig); sendConfig(newConfig); + + agent_config = AgentConfig::createFromJson(jsonConfig); + } void diff --git a/rtbkit/plugins/bidding_agent/bidding_agent.h b/rtbkit/plugins/bidding_agent/bidding_agent.h index 7d4daaa0a..9eae96a10 100644 --- a/rtbkit/plugins/bidding_agent/bidding_agent.h +++ b/rtbkit/plugins/bidding_agent/bidding_agent.h @@ -87,7 +87,7 @@ struct BiddingAgent : public ServiceBase, public MessageLoop { \param meta A json blob that will be returned as is in the bid result. \param wcm win cost model for this bid. */ - void doBid(Id id, const Bids& bids, + void doBid(Id id, Bids bids, const Json::Value& meta = Json::Value(), const WinCostModel& wmc = WinCostModel()); @@ -286,6 +286,7 @@ struct BiddingAgent : public ServiceBase, public MessageLoop { */ std::mutex configLock; std::string config; // The agent's configuration. + AgentConfig agent_config; void sendConfig(const std::string& newConfig = ""); diff --git a/rtbkit/plugins/exchange/bidswitch_exchange_connector.cc b/rtbkit/plugins/exchange/bidswitch_exchange_connector.cc index 35ac5f49a..34f3bd75a 100644 --- a/rtbkit/plugins/exchange/bidswitch_exchange_connector.cc +++ b/rtbkit/plugins/exchange/bidswitch_exchange_connector.cc @@ -362,6 +362,18 @@ parseBidRequest(HttpAuctionHandler & connection, ML::Parse_Context context("Bid Request", payload.c_str(), payload.size()); res.reset(OpenRTBBidRequestParser::openRTBBidRequestParserFactory("2.2")->parseBidRequest(context, exchangeName(), exchangeName())); + //Parsing "ssp" filed + if (res!=nullptr){ + std::string exchange; + if (res->ext.isMember("ssp")) { + exchange =res->ext["ssp"].asString(); + } + else { + exchange = exchangeName(); + } + res->exchange = std::move(exchange); + } + return res; } diff --git a/rtbkit/plugins/exchange/http_auction_handler.cc b/rtbkit/plugins/exchange/http_auction_handler.cc index fc918acaa..bd14e27b1 100644 --- a/rtbkit/plugins/exchange/http_auction_handler.cc +++ b/rtbkit/plugins/exchange/http_auction_handler.cc @@ -260,7 +260,7 @@ onCleanup() void HttpAuctionHandler:: doEvent(const char * eventName, - EventType type, + StatEventType type, float value, const char * units, std::initializer_list extra) diff --git a/rtbkit/plugins/exchange/http_auction_handler.h b/rtbkit/plugins/exchange/http_auction_handler.h index d0586aded..d55e899a9 100644 --- a/rtbkit/plugins/exchange/http_auction_handler.h +++ b/rtbkit/plugins/exchange/http_auction_handler.h @@ -99,7 +99,7 @@ struct HttpAuctionHandler table. */ void doEvent(const char * eventName, - EventType type = ET_COUNT, + StatEventType type = ET_COUNT, float value = 1.0, const char * units = "", std::initializer_list extra = DefaultOutcomePercentiles); diff --git a/rtbkit/plugins/exchange/http_exchange_connector.h b/rtbkit/plugins/exchange/http_exchange_connector.h index c6bb0f67a..a887d45ce 100644 --- a/rtbkit/plugins/exchange/http_exchange_connector.h +++ b/rtbkit/plugins/exchange/http_exchange_connector.h @@ -99,22 +99,39 @@ struct HttpExchangeConnector { // This gets captured by value and effectively becomes a state variable // for the mutable lambda. Not very clean but it works. - std::vector lastSample; + std::vector lastSample; + auto lastTime = Date::now(); return [=] (double elapsed) mutable -> double { - auto sample = totalSleepSeconds(); + auto sample = getResourceUsage(); - if (lastSample.size() < sample.size()) - lastSample.resize(sample.size(), 0.0); + // get how much time elapsed since last time + auto now = Date::now(); + auto dt = now.secondsSince(lastTime); + lastTime = now; - double maxLoad = 0.0; - for (size_t i = 0; i < sample.size(); ++i) { - double load = 1.0 - ((sample[i] - lastSample[i]) / elapsed); - maxLoad = std::max(load, maxLoad); + //first time? + if(sample.size() != lastSample.size()) { + lastSample = std::move(sample); + return 0.0; } + double sum = 0.0; + for (auto i = 0; i < sample.size(); i++) { + auto sec = double(sample[i].ru_utime.tv_sec - lastSample[i].ru_utime.tv_sec); + auto usec = double(sample[i].ru_utime.tv_usec - lastSample[i].ru_utime.tv_usec); + + auto load = sec + usec * 0.000001; + if (load >= dt) { + sum += 1.0; + } else { + sum += load/dt; + } + } + + double value = sum / sample.size(); lastSample = std::move(sample); - return maxLoad; + return value; }; } diff --git a/rtbkit/plugins/exchange/openrtb_exchange_connector.cc b/rtbkit/plugins/exchange/openrtb_exchange_connector.cc index 57a7862d4..1f43c95c3 100644 --- a/rtbkit/plugins/exchange/openrtb_exchange_connector.cc +++ b/rtbkit/plugins/exchange/openrtb_exchange_connector.cc @@ -155,6 +155,7 @@ parseBidRequest(HttpAuctionHandler & connection, result.reset(OpenRTBBidRequestParser::openRTBBidRequestParserFactory(openRtbVersion)->parseBidRequest(context, exchangeName(), exchangeName())); + result->protocolVersion = openRtbVersion; } catch(ML::Exception const & e) { this->recordHit("error.parsingBidRequest"); diff --git a/rtbkit/plugins/exchange/rtbkit_exchange_connector.cc b/rtbkit/plugins/exchange/rtbkit_exchange_connector.cc index 2179ac4a5..261f28964 100644 --- a/rtbkit/plugins/exchange/rtbkit_exchange_connector.cc +++ b/rtbkit/plugins/exchange/rtbkit_exchange_connector.cc @@ -40,6 +40,16 @@ parseBidRequest(HttpAuctionHandler &connection, if (request != nullptr) { + + std::string exchange; + if (request->ext.isMember("exchange")) { + exchange = request->ext["exchange"].asString(); + } + else { + exchange = exchangeName(); + } + request->exchange = std::move(exchange); + auto failure = ScopeFailure([&]() noexcept { request.reset(); }); for (const auto &imp: request->imp) { diff --git a/rtbkit/plugins/exchange/testing/adx_exchange_connector_test.cc b/rtbkit/plugins/exchange/testing/adx_exchange_connector_test.cc index 0f4811a60..ec905c985 100644 --- a/rtbkit/plugins/exchange/testing/adx_exchange_connector_test.cc +++ b/rtbkit/plugins/exchange/testing/adx_exchange_connector_test.cc @@ -87,6 +87,7 @@ BOOST_AUTO_TEST_CASE( test_adx ) // Tell the router about the new exchange connector router.addExchange(connector); + router.initFilters(); // This is our bidding agent, that actually calculates the bid price TestAgent agent(proxies, "BOB"); diff --git a/rtbkit/plugins/exchange/testing/bidswitch_exchange_connector_adx_test.cc b/rtbkit/plugins/exchange/testing/bidswitch_exchange_connector_adx_test.cc index 1d62f5995..105579c36 100644 --- a/rtbkit/plugins/exchange/testing/bidswitch_exchange_connector_adx_test.cc +++ b/rtbkit/plugins/exchange/testing/bidswitch_exchange_connector_adx_test.cc @@ -83,6 +83,7 @@ BOOST_AUTO_TEST_CASE( test_bidswitch ) // Tell the router about the new exchange connector router.addExchange(connector); + router.initFilters(); // This is our bidding agent, that actually calculates the bid price TestAgent agent(proxies, "BOB"); diff --git a/rtbkit/plugins/exchange/testing/bidswitch_exchange_connector_test.cc b/rtbkit/plugins/exchange/testing/bidswitch_exchange_connector_test.cc index 9d7800108..89ff9bff6 100644 --- a/rtbkit/plugins/exchange/testing/bidswitch_exchange_connector_test.cc +++ b/rtbkit/plugins/exchange/testing/bidswitch_exchange_connector_test.cc @@ -83,6 +83,7 @@ BOOST_AUTO_TEST_CASE( test_bidswitch ) // Tell the router about the new exchange connector router.addExchange(connector); + router.initFilters(); // This is our bidding agent, that actually calculates the bid price TestAgent agent(proxies, "BOB"); diff --git a/rtbkit/plugins/exchange/testing/gumgum_exchange_connector_test.cc b/rtbkit/plugins/exchange/testing/gumgum_exchange_connector_test.cc index dfc8abd79..89e98bc7c 100644 --- a/rtbkit/plugins/exchange/testing/gumgum_exchange_connector_test.cc +++ b/rtbkit/plugins/exchange/testing/gumgum_exchange_connector_test.cc @@ -84,6 +84,7 @@ BOOST_AUTO_TEST_CASE( test_gumgum ) // Tell the router about the new exchange connector router.addExchange(connector); + router.initFilters(); // This is our bidding agent, that actually calculates the bid price TestAgent agent(proxies, "agent"); diff --git a/rtbkit/plugins/exchange/testing/mopub_exchange_connector_test.cc b/rtbkit/plugins/exchange/testing/mopub_exchange_connector_test.cc index f20960572..b109398e1 100644 --- a/rtbkit/plugins/exchange/testing/mopub_exchange_connector_test.cc +++ b/rtbkit/plugins/exchange/testing/mopub_exchange_connector_test.cc @@ -86,6 +86,7 @@ BOOST_AUTO_TEST_CASE( test_mopub ) // Tell the router about the new exchange connector router.addExchange(connector); + router.initFilters(); // This is our bidding agent, that actually calculates the bid price TestAgent agent(proxies, "BOB"); diff --git a/rtbkit/plugins/exchange/testing/nexage_exchange_connector_test.cc b/rtbkit/plugins/exchange/testing/nexage_exchange_connector_test.cc index cdcecfbdb..d6063c112 100644 --- a/rtbkit/plugins/exchange/testing/nexage_exchange_connector_test.cc +++ b/rtbkit/plugins/exchange/testing/nexage_exchange_connector_test.cc @@ -83,6 +83,7 @@ BOOST_AUTO_TEST_CASE( test_nexage ) // Tell the router about the new exchange connector router.addExchange(connector); + router.initFilters(); // This is our bidding agent, that actually calculates the bid price TestAgent agent(proxies, "BOB"); diff --git a/rtbkit/plugins/exchange/testing/openrtb_exchange_connector_test.cc b/rtbkit/plugins/exchange/testing/openrtb_exchange_connector_test.cc index 318a03bb5..d60079eb9 100644 --- a/rtbkit/plugins/exchange/testing/openrtb_exchange_connector_test.cc +++ b/rtbkit/plugins/exchange/testing/openrtb_exchange_connector_test.cc @@ -37,6 +37,7 @@ BOOST_AUTO_TEST_CASE( test_openrtb_error_codes ) // Tell the router about the new exchange connector router.addExchange(connector); + router.initFilters(); ML::sleep(1.0); diff --git a/rtbkit/plugins/exchange/testing/rtbkit_exchange_connector_test.cc b/rtbkit/plugins/exchange/testing/rtbkit_exchange_connector_test.cc index 360106844..70cce70d0 100644 --- a/rtbkit/plugins/exchange/testing/rtbkit_exchange_connector_test.cc +++ b/rtbkit/plugins/exchange/testing/rtbkit_exchange_connector_test.cc @@ -46,6 +46,7 @@ struct TestContext { connector->enableUntil(Date::positiveInfinity()); router->addExchange(connector); + router->initFilters(); ML::sleep(1.0); diff --git a/rtbkit/plugins/exchange/testing/rubicon_exchange_connector_test.cc b/rtbkit/plugins/exchange/testing/rubicon_exchange_connector_test.cc index e69dec596..b216bae26 100644 --- a/rtbkit/plugins/exchange/testing/rubicon_exchange_connector_test.cc +++ b/rtbkit/plugins/exchange/testing/rubicon_exchange_connector_test.cc @@ -242,6 +242,7 @@ BOOST_AUTO_TEST_CASE( test_rubicon ) // Tell the router about the new exchange connector router.addExchange(connector); + router.initFilters(); // This is our bidding agent, that actually calculates the bid price TestAgent agent(proxies, "agent"); diff --git a/rtbkit/plugins/exchange/testing/smaato_exchange_connector_test.cc b/rtbkit/plugins/exchange/testing/smaato_exchange_connector_test.cc index eeaffa2a5..065029626 100644 --- a/rtbkit/plugins/exchange/testing/smaato_exchange_connector_test.cc +++ b/rtbkit/plugins/exchange/testing/smaato_exchange_connector_test.cc @@ -85,6 +85,7 @@ BOOST_AUTO_TEST_CASE( test_smaato ) // Tell the router about the new exchange connector router.addExchange(connector); + router.initFilters(); // This is our bidding agent, that actually calculates the bid price TestAgent agent(proxies, "BOB"); diff --git a/rtbkit/testing/bid_stack.h b/rtbkit/testing/bid_stack.h index 150adaeac..f12ab788b 100644 --- a/rtbkit/testing/bid_stack.h +++ b/rtbkit/testing/bid_stack.h @@ -83,15 +83,14 @@ struct BidStack { } services.router->setBanker(services.banker); + + services.router->initExchanges(routerConfig); + services.router->initFilters(); + // Start the router up services.router->bindTcp(); services.router->start(); - // Configure exchange connectors - for(auto & exchange : routerConfig) { - services.router->startExchange(exchange); - } - std::string mock = "{\"workers\":["; services.router->forAllExchanges([&](std::shared_ptr const & item) { diff --git a/rtbkit/testing/bidder_test.cc b/rtbkit/testing/bidder_test.cc index 5c2fef694..b9f693211 100644 --- a/rtbkit/testing/bidder_test.cc +++ b/rtbkit/testing/bidder_test.cc @@ -343,6 +343,7 @@ BOOST_AUTO_TEST_CASE( test_http_bidder_multiple_impressions_tagging ) dummyExchange->start(); dummyExchange->enableUntil(Date::positiveInfinity()); router->addExchange(dummyExchange); + router->initFilters(); Json::Value upstreamRouterConfig; upstreamRouterConfig[0]["exchangeType"] = "openrtb"; diff --git a/rtbkit/testing/exchange_parsing_from_file.cc b/rtbkit/testing/exchange_parsing_from_file.cc index 6929a7223..f299e4367 100644 --- a/rtbkit/testing/exchange_parsing_from_file.cc +++ b/rtbkit/testing/exchange_parsing_from_file.cc @@ -78,6 +78,7 @@ run() // Tell the router about the new exchange connector router.addExchange(connector1); + router.initFilters(); ML::sleep(1.0); // prepare request diff --git a/soa/js/js_utils.h b/soa/js/js_utils.h index 6810eb572..90da1e943 100644 --- a/soa/js/js_utils.h +++ b/soa/js/js_utils.h @@ -89,12 +89,12 @@ v8::Handle injectBacktrace(v8::Handle value); exceptions. */ #define HANDLE_JS_EXCEPTIONS \ catch (...) { \ - return Datacratic::JS::translateCurrentException(); \ + return translateCurrentException(); \ } #define HANDLE_JS_EXCEPTIONS_SETTER \ catch (...) { \ - Datacratic::JS::translateCurrentException(); \ + translateCurrentException(); \ return; \ } diff --git a/soa/jsoncpp/json_reader.cpp b/soa/jsoncpp/json_reader.cpp index 524d2b7c6..c19b0d506 100644 --- a/soa/jsoncpp/json_reader.cpp +++ b/soa/jsoncpp/json_reader.cpp @@ -156,6 +156,10 @@ Json::Value parse(std::istream & ifs) Json::Value parseFromFile(const std::string& filename) { std::ifstream ifs(filename); + if (!ifs.is_open()) { + std::string msg = "unable to open JSON file: " + filename; + throw Exception(msg); + } return parse(ifs); } diff --git a/soa/launcher/launcher.h b/soa/launcher/launcher.h index 5b2fdca79..5959bbbda 100644 --- a/soa/launcher/launcher.h +++ b/soa/launcher/launcher.h @@ -37,7 +37,7 @@ struct Launcher { struct Task { - Task() : pid(-1), log(false), delay(45.0) { + Task() : pid(-1), log(false), delay(45.0), once(false) { } std::string const & getName() const { @@ -52,8 +52,10 @@ struct Launcher } void restart(std::string const & node) { - stop(); - start(node); + if (!once) { + stop(); + start(node); + } } void start(std::string const & node) { @@ -150,6 +152,9 @@ struct Launcher else if(i.memberName() == "delay") { result.delay = i->asDouble(); } + else if(i.memberName() == "once") { + result.once = i->asBool(); + } else if(i.memberName() == "arg") { auto & json = *i; if(!json.empty() && !json.isArray()) { @@ -275,6 +280,7 @@ struct Launcher std::vector arg; bool log; double delay; + bool once; }; struct Node diff --git a/soa/logger/kvp_logger_interface.cc b/soa/logger/kvp_logger_interface.cc index b44eeed67..51a71b02f 100644 --- a/soa/logger/kvp_logger_interface.cc +++ b/soa/logger/kvp_logger_interface.cc @@ -1,5 +1,4 @@ #include "kvp_logger_interface.h" -#include "kvp_logger_mongodb.h" #include "kvp_logger_void.h" #include @@ -8,9 +7,7 @@ namespace Datacratic{ std::shared_ptr IKvpLogger ::kvpLoggerFactory(const std::string& type, const KvpLoggerParams& params){ - if(type == "mongodb"){ - return std::shared_ptr(new KvpLoggerMongoDb(params)); - }else if(type == "void"){ + if (type == "void"){ return std::shared_ptr(new KvpLoggerVoid()); }else if(type == "metricsLogger"){ @@ -27,11 +24,7 @@ ::kvpLoggerFactory(const std::string& configKey){ json_parser::read_json(getenv("CONFIG"), pt); pt = pt.get_child(configKey); string type = pt.get("type"); - if(type == "mongodb"){ - - }else if(type == "void"){ - - }else if(type == "mongodbMetrics"){ + if(type == "void"){ } throw ML::Exception("Unknown KvpLogger [" + type + "]"); diff --git a/soa/logger/kvp_logger_mongodb.cc b/soa/logger/kvp_logger_mongodb.cc index 0afa1b9b8..ebfd22814 100644 --- a/soa/logger/kvp_logger_mongodb.cc +++ b/soa/logger/kvp_logger_mongodb.cc @@ -12,8 +12,7 @@ ::makeInitFct(const string& hostAndPort, const string& db, { function init = [&] (){ cerr << hostAndPort << endl; - mongo::HostAndPort mongoHostAndPort(hostAndPort); - conn.connect(mongoHostAndPort); + conn.connect(hostAndPort); string err; if(!conn.auth(db, user, pwd, err)){ throw ML::Exception("MongoDB connection failed with msg [" diff --git a/soa/logger/logger.mk b/soa/logger/logger.mk index f29504e49..b8dbfdd36 100644 --- a/soa/logger/logger.mk +++ b/soa/logger/logger.mk @@ -17,8 +17,8 @@ $(eval $(call library,logger,$(LIBLOGGER_SOURCES),$(LIBLOGGER_LINK))) $(eval $(call nodejs_addon,logger,logger_js.cc filter_js.cc,logger js sigslot)) LIBLOG_METRICS_SOURCES := \ - kvp_logger_interface.cc kvp_logger_mongodb.cc easy_kvp_logger.cc logger_metrics_interface.cc \ - logger_metrics_mongo.cc logger_metrics_term.cc + kvp_logger_interface.cc easy_kvp_logger.cc logger_metrics_interface.cc \ + logger_metrics_term.cc LIBLOG_METRICS_LINK := \ mongoclient boost_filesystem boost_program_options types diff --git a/soa/logger/logger_metrics_interface.cc b/soa/logger/logger_metrics_interface.cc index dc105662a..285b580ea 100644 --- a/soa/logger/logger_metrics_interface.cc +++ b/soa/logger/logger_metrics_interface.cc @@ -1,5 +1,4 @@ #include "soa/logger/logger_metrics_interface.h" -#include "soa/logger/logger_metrics_mongo.h" #include "soa/logger/logger_metrics_void.h" #include "soa/logger/logger_metrics_term.h" #include "soa/jsoncpp/reader.h" @@ -53,10 +52,7 @@ ::setup(const string& configKey, const string& coll, string loggerType = config["type"].asString(); failSafe = config["failSafe"].asBool(); function fct = [&]{ - if(loggerType == "mongo"){ - logger = shared_ptr( - new LoggerMetricsMongo(config, coll, appName)); - }else if(loggerType == "term" || loggerType == "terminal"){ + if(loggerType == "term" || loggerType == "terminal"){ logger = shared_ptr( new LoggerMetricsTerm(config, coll, appName)); }else if(loggerType == "void"){ @@ -169,7 +165,7 @@ void ILoggerMetrics::logMetrics(const Json::Value& json){ failSafeHelper(fct); } -void ILoggerMetrics::failSafeHelper(std::function fct){ +void ILoggerMetrics::failSafeHelper(const std::function& fct){ if(failSafe){ try{ fct(); diff --git a/soa/logger/logger_metrics_interface.h b/soa/logger/logger_metrics_interface.h index 493e90f0f..016a41789 100644 --- a/soa/logger/logger_metrics_interface.h +++ b/soa/logger/logger_metrics_interface.h @@ -1,18 +1,26 @@ +/* logger_metrics_interface.h -*- C++ -*- + François-Michel L'Heureux, 21 May 2013 + Copyright (c) 2013 Datacratic. All rights reserved. +*/ + #pragma once #include #include #include -#include -#include "jml/arch/exception.h" -#include "soa/jsoncpp/json.h" #include -#include "boost/variant.hpp" #include +#include "boost/variant.hpp" +#include "jml/arch/exception.h" +#include "soa/jsoncpp/json.h" #include "soa/types/date.h" -namespace Datacratic{ +namespace Datacratic { + +/****************************************************************************/ +/* LOGGER METRICS */ +/****************************************************************************/ /** * KvpLogger are key-value-pair loggers @@ -22,107 +30,122 @@ namespace Datacratic{ * - Provides adaptor functions to avoid defining redundant functions in * implementations */ -class ILoggerMetrics{ - - private: - static bool failSafe; - const Date startDate; - ILoggerMetrics(){}; - - protected: - // ORDER OF VARIANT IMPORTANT! - typedef boost::variant Numeric; - typedef boost::variant NumOrStr; - - const static std::string METRICS; - const static std::string PROCESS; - const static std::string META; - - const std::string coll; - static std::string parentObjectId; - - ILoggerMetrics(const std::string& coll) : - startDate(Date::now()), coll(coll){}; - virtual void logInCategory(const std::string& category, - const std::vector& path, - const NumOrStr& val) = 0; - virtual void logInCategory(const std::string& category, - const Json::Value& j) = 0; - - void failSafeHelper(std::function); - virtual const std::string getProcessId() const = 0; - - public: - static std::shared_ptr setup( - const std::string& configKey, - const std::string& coll, - const std::string& appName); - /** - * Factory like getter for kvp - */ - static std::shared_ptr getSingleton(); - - void logMetrics(const Json::Value&); - void logProcess(const Json::Value& j){ - std::function fct = [&](){ - logInCategory(PROCESS, j); - }; - failSafeHelper(fct); - } - void logMeta(const Json::Value& j){ - std::function fct = [&](){ - logInCategory(META, j); - }; - failSafeHelper(fct); - } - - template - void logMetrics(const jsonifiable& j){ - std::function fct = [&](){ - Json::Value root = j.toJson(); - logMetrics(root); - }; - failSafeHelper(fct); +struct ILoggerMetrics { + // ORDER OF VARIANT IMPORTANT! + typedef boost::variant Numeric; + typedef boost::variant NumOrStr; + + ILoggerMetrics() = delete; + + static std::shared_ptr setup(const std::string & configKey, + const std::string & coll, + const std::string & appName); + static std::shared_ptr + setupFromJson(const Json::Value & config, + const std::string & coll, const std::string & appName); + + /** + * Factory like getter for kvp + */ + static std::shared_ptr getSingleton(); + + void logMetrics(const Json::Value &); + template + void logMetrics(const jsonifiable & j) + { + auto fct = [&] () { + Json::Value root = j.toJson(); + logMetrics(root); }; - template - void logProcess(const jsonifiable& j){ - std::function fct = [&](){ - Json::Value root = j.toJson(); - logProcess(root); - }; - failSafeHelper(fct); - }; - template - void logMeta(const jsonifiable& j){ - std::function fct = [&](){ - Json::Value root = j.toJson(); - logMeta(root); - }; - failSafeHelper(fct); + failSafeHelper(fct); + }; + void logMetrics(const std::vector & path, const Numeric & val) + { + auto fct = [&] () { + logInCategory(METRICS, path, val); }; + failSafeHelper(fct); + } - void logMetrics(const std::vector& path, const Numeric& val){ - std::function fct = [&](){ - logInCategory(METRICS, path, val); - }; - failSafeHelper(fct); - } - void logProcess(const std::vector& path, const NumOrStr& val){ - std::function fct = [&](){ - logInCategory(PROCESS, path, val); - }; - failSafeHelper(fct); - } - void logMeta(const std::vector& path, const NumOrStr& val){ - std::function fct = [&](){ - logInCategory(META, path, val); - }; - failSafeHelper(fct); - } - - void close(); - virtual ~ILoggerMetrics(){}; + void logProcess(const Json::Value & j) + { + auto fct = [&]() { + logInCategory(PROCESS, j); + }; + failSafeHelper(fct); + } + template + void logProcess(const jsonifiable & j) + { + auto fct = [&] () { + Json::Value root = j.toJson(); + logProcess(root); + }; + failSafeHelper(fct); + }; + void logProcess(const std::vector & path, const NumOrStr & val) + { + auto fct = [&] () { + logInCategory(PROCESS, path, val); + }; + failSafeHelper(fct); + } + void logMeta(const Json::Value & j) + { + auto fct = [&] () { + logInCategory(META, j); + }; + failSafeHelper(fct); + } + template + void logMeta(const jsonifiable & j) + { + auto fct = [&] () { + Json::Value root = j.toJson(); + logMeta(root); + }; + failSafeHelper(fct); + }; + void logMeta(const std::vector & path, const NumOrStr & val) + { + auto fct = [&] () { + logInCategory(META, path, val); + }; + failSafeHelper(fct); + } + + void close(); + virtual ~ILoggerMetrics(){}; + +protected: + const static std::string METRICS; + const static std::string PROCESS; + const static std::string META; + + ILoggerMetrics(const std::string & coll) + : coll(coll), startDate(Date::now()) + { + } + virtual void logInCategory(const std::string & category, + const std::vector & path, + const NumOrStr & val) = 0; + virtual void logInCategory(const std::string & category, + const Json::Value & j) = 0; + virtual std::string getProcessId() const = 0; + + void failSafeHelper(const std::function & fct); + + std::string coll; + +private: + static void setupLogger(const Json::Value & config, + const std::string & coll, + const std::string & appName); + static bool failSafe; + static std::string parentObjectId; + + const Date startDate; }; -}//namespace Datacratic +} // namespace Datacratic diff --git a/soa/logger/logger_metrics_mongo.cc b/soa/logger/logger_metrics_mongo.cc index a6cb507dd..adcd368cf 100644 --- a/soa/logger/logger_metrics_mongo.cc +++ b/soa/logger/logger_metrics_mongo.cc @@ -2,14 +2,23 @@ #include "mongo/bson/bson.h" #include "mongo/util/net/hostandport.h" #include "jml/utils/string_functions.h" +#include "soa/utils/mongo_init.h" namespace Datacratic{ using namespace std; using namespace mongo; +using namespace Datacratic; -LoggerMetricsMongo::LoggerMetricsMongo(Json::Value config, - const string& coll, const string& appName) : ILoggerMetrics(coll) + +/****************************************************************************/ +/* LOGGER METRICS MONGO */ +/****************************************************************************/ + +LoggerMetricsMongo:: +LoggerMetricsMongo(Json::Value config, const string & coll, + const string & appName) + : ILoggerMetrics(coll) { for(string s: {"hostAndPort", "database", "user", "pwd"}){ if(config[s].isNull()){ @@ -33,13 +42,30 @@ LoggerMetricsMongo::LoggerMetricsMongo(Json::Value config, conn = tmpConn; } db = config["database"].asString(); - string err; - if(!conn->auth(db, config["user"].asString(), - config["pwd"].asString(), err)) - { - throw ML::Exception( - "MongoDB connection failed with msg [%s]", err.c_str()); + + auto impl = [&] (string mechanism) { + BSONObj b = BSON("user" << config["user"].asString() + << "pwd" << config["pwd"].asString() + << "mechanism" << mechanism + << "db" << db); + try { + conn->auth(b); + } + catch (const UserException & _) { + return false; + } + return true; + }; + + if (!impl("SCRAM-SHA-1")) { + cerr << "Failed to authenticate with SCRAM-SHA-1, " + "trying with MONGODB-CR" << endl; + if (!impl("MONGODB-CR")) { + cerr << "Failed with MONGODB-CR as well" << endl; + throw ("Failed to auth"); + } } + BSONObj obj = BSON(GENOID); conn->insert(db + "." + coll, obj); objectId = obj["_id"].OID(); @@ -169,8 +195,12 @@ ::logInCategory(const std::string& category, true); } -const std::string LoggerMetricsMongo::getProcessId() const{ - return objectId.toString(); +std::string +LoggerMetricsMongo:: +getProcessId() + const +{ + return objectId.toString(); } diff --git a/soa/logger/logger_metrics_mongo.h b/soa/logger/logger_metrics_mongo.h index 9718d2107..77abea351 100644 --- a/soa/logger/logger_metrics_mongo.h +++ b/soa/logger/logger_metrics_mongo.h @@ -1,28 +1,38 @@ +/* logger_metrics_mongo.h -*- C++ -*- + François-Michel L'Heureux, 21 May 2013 + Copyright (c) 2013 Datacratic. All rights reserved. +*/ + #pragma once -#include "logger_metrics_interface.h" #include "mongo/client/dbclient.h" +#include "logger_metrics_interface.h" + + +namespace Datacratic { -namespace Datacratic{ -class LoggerMetricsMongo : public ILoggerMetrics{ +/****************************************************************************/ +/* LOGGER METRICS MONGO */ +/****************************************************************************/ + +struct LoggerMetricsMongo : public ILoggerMetrics { friend class ILoggerMetrics; - protected: - mongo::OID objectId; - std::string db; - std::shared_ptr conn; - - LoggerMetricsMongo(Json::Value config, - const std::string& coll, - const std::string& appName); - void logInCategory(const std::string&, - const Json::Value&); - void logInCategory(const std::string& category, - const std::vector& path, - const NumOrStr& val); - const std::string getProcessId() const; - - private: - bool logToTerm; +protected: + mongo::OID objectId; + std::string db; + std::shared_ptr conn; + + LoggerMetricsMongo(Json::Value config, const std::string & coll, + const std::string & appName); + void logInCategory(const std::string &, const Json::Value &); + void logInCategory(const std::string & category, + const std::vector & path, + const NumOrStr & val); + std::string getProcessId() const; + +private: + bool logToTerm; }; + }//namespace Datacratic diff --git a/soa/logger/logger_metrics_term.cc b/soa/logger/logger_metrics_term.cc index 0dd2ef3ad..80a365ead 100644 --- a/soa/logger/logger_metrics_term.cc +++ b/soa/logger/logger_metrics_term.cc @@ -1,13 +1,25 @@ -#include "logger_metrics_term.h" +/* logger_metrics_term.cc + François-Michel L'Heureux, 3 June 2013 + Copyright (c) 2013 Datacratic. All rights reserved. +*/ + #include #include -namespace Datacratic{ +#include "logger_metrics_term.h" using namespace std; +using namespace Datacratic; + -LoggerMetricsTerm::LoggerMetricsTerm(Json::Value config, - const string& coll, const string& appName) : ILoggerMetrics(coll) +/****************************************************************************/ +/* LOGGER METRICS TERM */ +/****************************************************************************/ + +LoggerMetricsTerm:: +LoggerMetricsTerm(Json::Value config, + const string & coll, const string & appName) + : ILoggerMetrics(coll) { stringstream ss; ss << getpid(); @@ -15,35 +27,39 @@ LoggerMetricsTerm::LoggerMetricsTerm(Json::Value config, cout << "Logger Metrics terminal: app " << appName << " under pid " << pid << endl; } -void LoggerMetricsTerm::logInCategory(const string& category, - const Json::Value& json) +void +LoggerMetricsTerm:: +logInCategory(const string & category, + const Json::Value & json) { cout << pid << "." << coll << "." << category << ": " << json.toStyledString() << endl; } -void LoggerMetricsTerm -::logInCategory(const std::string& category, - const std::vector& path, - const NumOrStr& val) +void +LoggerMetricsTerm:: +logInCategory(const std::string & category, + const std::vector & path, + const NumOrStr & val) { - if(path.size() == 0){ - throw new ML::Exception( - "You need to specify a path where to log the value"); + if (path.size() == 0) { + throw ML::Exception("You need to specify a path where to log" + " the value"); } stringstream ss; ss << val; stringstream newCat; newCat << pid << "." << coll << "." << category; - for(string part: path){ + for (const string & part: path) { newCat << "." << part; } cout << newCat.str() << ": " << ss.str() << endl; } -const std::string LoggerMetricsTerm::getProcessId() const{ +std::string +LoggerMetricsTerm:: +getProcessId() + const +{ return pid; } - - -}//namespace Datacratic diff --git a/soa/logger/logger_metrics_term.h b/soa/logger/logger_metrics_term.h index bd51f75b7..0cbe7e0ef 100644 --- a/soa/logger/logger_metrics_term.h +++ b/soa/logger/logger_metrics_term.h @@ -1,23 +1,34 @@ +/* logger_metrics_term.h -*- C++ -*- + François-Michel L'Heureux, 3 June 2013 + Copyright (c) 2013 Datacratic. All rights reserved. +*/ + #pragma once #include "logger_metrics_interface.h" -namespace Datacratic{ -class LoggerMetricsTerm : public ILoggerMetrics{ + +namespace Datacratic { + +/****************************************************************************/ +/* LOGGER METRICS TERM */ +/****************************************************************************/ + +struct LoggerMetricsTerm : public ILoggerMetrics { friend class ILoggerMetrics; - protected: - LoggerMetricsTerm(Json::Value config, - const std::string& coll, - const std::string& appName); - void logInCategory(const std::string&, - const Json::Value&); - void logInCategory(const std::string& category, - const std::vector& path, - const NumOrStr& val); - const std::string getProcessId() const; - - private: - std::string pid; +protected: + LoggerMetricsTerm(Json::Value config, + const std::string & coll, + const std::string & appName); + void logInCategory(const std::string &, const Json::Value &); + void logInCategory(const std::string & category, + const std::vector & path, + const NumOrStr & val); + std::string getProcessId() const; + +private: + std::string pid; }; -}//namespace Datacratic + +} // namespace Datacratic diff --git a/soa/logger/logger_metrics_void.h b/soa/logger/logger_metrics_void.h index 79469b093..d015d509a 100644 --- a/soa/logger/logger_metrics_void.h +++ b/soa/logger/logger_metrics_void.h @@ -1,20 +1,48 @@ +/* logger_metrics_void.h -*- C++ -*- + François-Michel L'Heureux, 21 May 2013 + Copyright (c) 2013 Datacratic. All rights reserved. +*/ + #pragma once #include "logger_metrics_interface.h" -namespace Datacratic{ -class LoggerMetricsVoid : public ILoggerMetrics{ + +namespace Datacratic { + +/****************************************************************************/ +/* LOGGER METRICS VOID */ +/****************************************************************************/ + +struct LoggerMetricsVoid : public ILoggerMetrics { friend class ILoggerMetrics; - protected: - LoggerMetricsVoid(Json::Value config, - const std::string& coll, - const std::string& appName) : ILoggerMetrics(coll){} - void logInCategory(const std::string&, - const Json::Value&){} - void logInCategory(const std::string& category, - const std::vector& path, - const NumOrStr& val){} - const std::string getProcessId() const{ return ""; } +protected: + LoggerMetricsVoid(Json::Value config, + const std::string & coll, + const std::string & appName) + : ILoggerMetrics(coll) + { + } + LoggerMetricsVoid(const std::string & coll) + : ILoggerMetrics(coll) + { + } + + void logInCategory(const std::string &, const Json::Value &) + { + } + void logInCategory(const std::string & category, + const std::vector & path, + const NumOrStr & val) + { + } + + std::string getProcessId() + const + { + return ""; + } }; -}//namespace Datacratic + +} // namespace Datacratic diff --git a/soa/logger/testing/logger_metrics_config.json b/soa/logger/testing/logger_metrics_config.json index 62bf11894..a4993817d 100644 --- a/soa/logger/testing/logger_metrics_config.json +++ b/soa/logger/testing/logger_metrics_config.json @@ -1,7 +1,7 @@ { "metricsLogger" : { "type" : "mongo", - "failSafe" : true, + "failSafe" : false, "hostAndPort" : "localhost:28356", "database" : "test", "user" : "testuser", diff --git a/soa/logger/testing/logger_metrics_test.cc b/soa/logger/testing/logger_metrics_test.cc index 2849047f1..4d0ed3593 100644 --- a/soa/logger/testing/logger_metrics_test.cc +++ b/soa/logger/testing/logger_metrics_test.cc @@ -26,9 +26,10 @@ using namespace ML; using namespace Datacratic; -using namespace std; +using namespace mongo; -BOOST_AUTO_TEST_CASE( test_logger_metrics ) { +BOOST_AUTO_TEST_CASE( test_logger_metrics ) +{ Mongo::MongoTemporaryServer mongo; setenv("CONFIG", "logger/testing/logger_metrics_config.json", 1); shared_ptr logger = @@ -81,4 +82,57 @@ BOOST_AUTO_TEST_CASE( test_logger_metrics ) { mongo::BSONObj p = cursor->next(); BOOST_CHECK_EQUAL(p["metrics"]["coco"]["sanchez"].Number(), 3); } + + Json::Value v; + v["coco"] = 123; + logger->logMetrics(v); + v.clear(); + v["expos"]["city"] = "baseball"; + v["expos"]["sport"] = "montreal"; + v["expos"]["players"][0] = "pedro"; + v["expos"]["players"][1] = "mario"; + v["expos"]["players"][2] = "octo"; + logger->logProcess(v); + + logger->logMeta({"octo"}, "sanchez"); + logger->close(); + + ML::sleep(2.0); + + string objectIdStr = getenv("METRICS_PARENT_ID"); + mongo::OID objectId(objectIdStr); + mongo::BSONObj where = BSON("_id" << objectId); + cursor = conn->query(database + ".metrics_test", where); + + BOOST_CHECK(cursor->more()); + { + mongo::BSONObj p = cursor->next(); + cerr << p.toString() << endl; + // conn->remove(database + ".metrics_test", p, 1); + BOOST_CHECK_EQUAL(p["process"]["appName"].String(), "test_app"); + BOOST_CHECK_EQUAL(p["metrics"]["coco"].Long(), 123); + BOOST_CHECK_EQUAL(p["meta"]["octo"].String(), "sanchez"); + BOOST_CHECK_EQUAL(p["process"]["expos"]["city"].String(), "baseball"); + BOOST_CHECK_EQUAL(p["process"]["expos"]["sport"].String(), "montreal"); + //BSONObj bsonPlayers = BSON("process.expos.players" << players); + auto players = p.getFieldDotted("process.expos.players").Array(); + BOOST_CHECK_EQUAL(players.size(), 3); + BOOST_CHECK_EQUAL(players[0].String(), "pedro"); + BOOST_CHECK_EQUAL(players[1].String(), "mario"); + BOOST_CHECK_EQUAL(players[2].String(), "octo"); + BOOST_CHECK(p["process"]["endDate"].toString() != "EOO"); + BOOST_CHECK(p["process"]["duration"].toString() != "EOO"); + } + + FILE * pipe = popen("echo -n $METRICS_PARENT_ID", "r"); + ExcAssert(pipe != nullptr); + char buffer[128]; + std::string result = ""; + while (!feof(pipe)) { + if (fgets(buffer, 128, pipe) != NULL) + result += buffer; + } + pclose(pipe); + + BOOST_CHECK_EQUAL(objectIdStr, result); } diff --git a/soa/logger/testing/logger_testing.mk b/soa/logger/testing/logger_testing.mk index e6c59c655..b8b372fcf 100644 --- a/soa/logger/testing/logger_testing.mk +++ b/soa/logger/testing/logger_testing.mk @@ -12,4 +12,4 @@ $(eval $(call test,rotating_file_logger_test,logger,manual boost)) $(eval $(call vowscoffee_test,logger_metrics_interface_js_test,iloggermetricscpp)) -$(eval $(call test,logger_metrics_test,log_metrics mongo_tmp_server utils,manual boost)) +#$(eval $(call test,logger_metrics_test,log_metrics mongo_tmp_server utils,manual boost)) diff --git a/soa/service/async_writer_source.cc b/soa/service/async_writer_source.cc index cbf8b5210..6d96bc853 100644 --- a/soa/service/async_writer_source.cc +++ b/soa/service/async_writer_source.cc @@ -25,7 +25,9 @@ AsyncWriterSource(const OnClosed & onClosed, const OnException & onException, size_t maxMessages, size_t readBufferSize) - : EpollLoop(onException), + : AsyncEventSource(), + epollFd_(-1), + numFds_(0), fd_(-1), closing_(false), readBufferSize_(readBufferSize), @@ -37,8 +39,13 @@ AsyncWriterSource(const OnClosed & onClosed, bytesReceived_(0), msgsSent_(0), onClosed_(onClosed), - onReceivedData_(onReceivedData) + onReceivedData_(onReceivedData), + onException_(onException) { + epollFd_ = ::epoll_create(666); + if (epollFd_ == -1) + throw ML::Exception(errno, "epoll_create"); + auto handleQueueEventCb = [&] (const ::epoll_event & event) { queue_.processOne(); }; @@ -51,6 +58,7 @@ AsyncWriterSource:: if (fd_ != -1) { closeFd(); } + closeEpollFd(); } void @@ -86,6 +94,16 @@ closeFd() handleClosing(false, false); } +void +AsyncWriterSource:: +closeEpollFd() +{ + if (epollFd_ != -1) { + ::close(epollFd_); + epollFd_ = -1; + } +} + bool AsyncWriterSource:: write(string data, const OnWriteResult & onWriteResult) @@ -174,6 +192,13 @@ handleWriteResult(int error, AsyncWrite && currentWrite) currentWrite.clear(); } +void +AsyncWriterSource:: +handleException() +{ + onException(current_exception()); +} + void AsyncWriterSource:: onClosed(bool fromPeer, const vector & msgs) @@ -192,6 +217,18 @@ onReceivedData(const char * buffer, size_t bufferSize) } } +void +AsyncWriterSource:: +onException(const exception_ptr & excPtr) +{ + if (onException_) { + onException(excPtr); + } + else { + rethrow_exception(excPtr); + } +} + void AsyncWriterSource:: requestClose() @@ -206,6 +243,38 @@ requestClose() } } +/* async event source */ +bool +AsyncWriterSource:: +processOne() +{ + struct epoll_event events[numFds_]; + + if (numFds_ > 0) { + try { + int res = epoll_wait(epollFd_, events, numFds_, 0); + if (res == -1) { + throw ML::Exception(errno, "epoll_wait"); + } + + for (int i = 0; i < res; i++) { + auto * fn = static_cast(events[i].data.ptr); + (*fn)(events[i]); + } + + for (auto & unreg: delayedUnregistrations_) { + auto cb = move(unreg.second); + unregisterFdCallback(unreg.first, false, cb); + } + } + catch (const std::exception & exc) { + handleException(); + } + } + + return false; +} + /* wakeup events */ void @@ -341,6 +410,99 @@ handleClosing(bool fromPeer, bool delayedUnregistration) } } +/* epoll operations */ + +void +AsyncWriterSource:: +registerFdCallback(int fd, const EpollCallback & cb) +{ + if (delayedUnregistrations_.count(fd) == 0) { + if (fdCallbacks_.find(fd) != fdCallbacks_.end()) { + throw ML::Exception("callback already registered for fd"); + } + } + else { + delayedUnregistrations_.erase(fd); + } + fdCallbacks_[fd] = cb; +} + +void +AsyncWriterSource:: +unregisterFdCallback(int fd, bool delayed, + const OnUnregistered & onUnregistered) +{ + if (fdCallbacks_.find(fd) == fdCallbacks_.end()) { + throw ML::Exception("callback not registered for fd"); + } + if (delayed) { + ExcAssert(delayedUnregistrations_.count(fd) == 0); + delayedUnregistrations_[fd] = onUnregistered; + } + else { + delayedUnregistrations_.erase(fd); + fdCallbacks_.erase(fd); + if (onUnregistered) { + onUnregistered(); + } + } +} + +void +AsyncWriterSource:: +performAddFd(int fd, bool readerFd, bool writerFd, bool modify, bool oneshot) +{ + if (epollFd_ == -1) + return; + + struct epoll_event event; + if (oneshot) { + event.events = EPOLLONESHOT; + } + else { + event.events = 0; + } + if (readerFd) { + event.events |= EPOLLIN; + } + if (writerFd) { + event.events |= EPOLLOUT; + } + + EpollCallback & cb = fdCallbacks_.at(fd); + event.data.ptr = &cb; + + int operation = modify ? EPOLL_CTL_MOD : EPOLL_CTL_ADD; + int res = epoll_ctl(epollFd_, operation, fd, &event); + if (res == -1) { + string message = (string("epoll_ctl:") + + " modify=" + to_string(modify) + + " fd=" + to_string(fd) + + " readerFd=" + to_string(readerFd) + + " writerFd=" + to_string(writerFd)); + throw ML::Exception(errno, message); + } + if (!modify) { + numFds_++; + } +} + +void +AsyncWriterSource:: +removeFd(int fd) +{ + if (epollFd_ == -1) + return; + + int res = epoll_ctl(epollFd_, EPOLL_CTL_DEL, fd, 0); + if (res == -1) + throw ML::Exception(errno, "epoll_ctl DEL " + to_string(fd)); + if (numFds_ == 0) { + throw ML::Exception("inconsistent number of fds registered"); + } + numFds_--; +} + std::vector AsyncWriterSource:: emptyMessageQueue() diff --git a/soa/service/async_writer_source.h b/soa/service/async_writer_source.h index 3f20b192b..dea6f9d9d 100644 --- a/soa/service/async_writer_source.h +++ b/soa/service/async_writer_source.h @@ -7,12 +7,20 @@ #pragma once +#include + #include +#include +#include #include #include +#include +#include + +#include "jml/utils/ring_buffer.h" -#include "soa/service/epoll_loop.h" -#include "soa/service/typed_message_channel.h" +#include "async_event_source.h" +#include "typed_message_channel.h" namespace Datacratic { @@ -47,9 +55,11 @@ struct AsyncWriteResult { /****************************************************************************/ /* A base class enabling the asynchronous and buffered writing of data to a - * file descriptor. */ + * file descriptor. This class currently implements two separate concerns (a + * read-write "Epoller" and a write queue) and might need to be split at some + * point. */ -struct AsyncWriterSource : public EpollLoop +struct AsyncWriterSource : public AsyncEventSource { /* type of callback used when the file descriptor has been closed */ typedef std::function OnReceivedData; + /* type of callback invoked whenever an uncaught exception occurs */ + typedef std::function OnException; + AsyncWriterSource(const OnClosed & onClosed, const OnReceivedData & onReceivedData, const OnException & onException, @@ -72,6 +85,11 @@ struct AsyncWriterSource : public EpollLoop size_t readBufferSize); virtual ~AsyncWriterSource(); + /* AsyncEventSource interface */ + virtual int selectFd() const + { return epollFd_; } + virtual bool processOne(); + /* enqueue "data" for writing, provided the file descriptor is open or * being opened, or throws */ bool write(std::string data, @@ -98,11 +116,14 @@ struct AsyncWriterSource : public EpollLoop * or due to a pipe reset. In the latter case, "msgs" also contains all * the unsent messages. */ virtual void onClosed(bool fromPeer, - const std::vector & msgs); + const std::vector & msgs); /* invoked when the data is available for reading */ virtual void onReceivedData(const char * data, size_t size); + /* invoked when an exception occurs during the handling of events */ + virtual void onException(const std::exception_ptr & excPtr); + /* number of bytes actually sent */ uint64_t bytesSent() const { return bytesSent_; } @@ -115,6 +136,8 @@ struct AsyncWriterSource : public EpollLoop { return msgsSent_; } protected: + typedef std::function OnUnregistered; + /* set the "main" file descriptor, for which epoll events are monitored * and the onWriteResult, onReceivedData and onClosed callbacks are * invoked automatically */ @@ -140,6 +163,41 @@ struct AsyncWriterSource : public EpollLoop operations */ virtual void closeFd(); + /* type of callback invoked whenever an epoll event is reported for a + * file descriptor */ + typedef std::function EpollCallback; + + /* register a file descriptor into the internal epoll queue for reading + and/or writing */ + void addFd(int fd, bool readerFd, bool writerFd) + { performAddFd(fd, readerFd, writerFd, false, false); } + + /* same as addFd, with the EPOLLONESHOT flag */ + void addFdOneShot(int fd, bool readerFd, bool writerFd) + { performAddFd(fd, readerFd, writerFd, false, true); } + + /* modify a file descriptor in the epoll queue */ + void modifyFd(int fd, bool readerFd, bool writerFd) + { performAddFd(fd, readerFd, writerFd, true, false); } + + /* same as modifyFd, with the EPOLLONESHOT flag */ + void modifyFdOneShot(int fd, bool readerFd, bool writerFd) + { performAddFd(fd, readerFd, writerFd, true, true); } + + /* remove a file descriptor from the internal epoll queue */ + void removeFd(int fd); + + /* associate a callback with a file descriptor for future epoll + operations */ + void registerFdCallback(int fd, const EpollCallback & cb); + + /* dissociate a callback and a file descriptor from the callback registry, + with the "delayed" parameter indicating whether the operation must + occur immediately or at the end of the epoll loop */ + void unregisterFdCallback(int fd, bool delayed, + const OnUnregistered & onUnregistered + = nullptr); + std::vector emptyMessageQueue(); private: @@ -169,6 +227,12 @@ struct AsyncWriterSource : public EpollLoop OnWriteResult onWriteResult; }; + void performAddFd(int fd, bool readerFd, bool writerFd, + bool modify, bool oneshot); + + /* epoll operations */ + void closeEpollFd(); + /* fd operations */ void flush(); @@ -177,10 +241,17 @@ struct AsyncWriterSource : public EpollLoop void handleWriteReady(); void handleWriteResult(int error, AsyncWrite && currentWrite); void handleClosing(bool fromPeer, bool delayedUnregistration); + void handleException(); /* wakeup operations */ void handleQueueNotification(); + int epollFd_; + size_t numFds_; + + std::map fdCallbacks_; + std::map delayedUnregistrations_; + int fd_; std::atomic closing_; size_t readBufferSize_; @@ -197,6 +268,7 @@ struct AsyncWriterSource : public EpollLoop OnClosed onClosed_; OnWriteResult onWriteResult_; OnReceivedData onReceivedData_; + OnException onException_; }; } diff --git a/soa/service/carbon_connector.cc b/soa/service/carbon_connector.cc index 5a426f005..39d75f72c 100644 --- a/soa/service/carbon_connector.cc +++ b/soa/service/carbon_connector.cc @@ -116,15 +116,15 @@ StatAggregator * createNewLevel() return new GaugeAggregator(GaugeAggregator::Level); } -StatAggregator * createNewOutcome(const std::vector& percentiles) +StatAggregator * createNewOutcome(std::vector percentiles) { - return new GaugeAggregator(GaugeAggregator::Outcome, percentiles); + return new GaugeAggregator(GaugeAggregator::Outcome, std::move(percentiles)); } void MultiAggregator:: record(const std::string & stat, - EventType type, + StatEventType type, float value, std::initializer_list extra) { @@ -180,9 +180,9 @@ recordLevel(const std::string & stat, float value) void MultiAggregator:: recordOutcome(const std::string & stat, float value, - const std::vector& percentiles) + std::vector percentiles) { - getAggregator(stat, createNewOutcome, percentiles).record(value); + getAggregator(stat, createNewOutcome, std::move(percentiles)).record(value); } diff --git a/soa/service/carbon_connector.h b/soa/service/carbon_connector.h index 47048a62a..240987abe 100644 --- a/soa/service/carbon_connector.h +++ b/soa/service/carbon_connector.h @@ -59,7 +59,7 @@ struct MultiAggregator { /** Record, generic version. */ void record(const std::string & stat, - EventType type = ET_COUNT, + StatEventType type = ET_COUNT, float value = 1.0, std::initializer_list extra = DefaultOutcomePercentiles); @@ -99,7 +99,7 @@ struct MultiAggregator { safe. */ void recordOutcome(const std::string & stat, float value, - const std::vector& percentiles = DefaultOutcomePercentiles); + std::vector percentiles = DefaultOutcomePercentiles); /** Dump synchronously (taking the lock). This should only be used in testing or debugging, not when connected to Carbon. diff --git a/soa/service/chunked_http_endpoint.cc b/soa/service/chunked_http_endpoint.cc index 7df06c476..8ca8d2813 100644 --- a/soa/service/chunked_http_endpoint.cc +++ b/soa/service/chunked_http_endpoint.cc @@ -92,7 +92,7 @@ status() const void ChunkedHttpHandler:: doEvent(const char * eventName, - EventType type, + StatEventType type, float value, const char * units) { diff --git a/soa/service/chunked_http_endpoint.h b/soa/service/chunked_http_endpoint.h index 0b5065328..60bcdb1d8 100644 --- a/soa/service/chunked_http_endpoint.h +++ b/soa/service/chunked_http_endpoint.h @@ -44,7 +44,7 @@ struct ChunkedHttpHandler virtual void handleError(const std::string & message); void doEvent(const char * eventName, - EventType type = ET_COUNT, + StatEventType type = ET_COUNT, float value = 1.0, const char * units = ""); @@ -79,14 +79,14 @@ struct ChunkedHttpEndpoint: public HttpEndpoint { 4. The units of the event (currently unused) */ typedef boost::function OnEvent; OnEvent onEvent; protected: void doEvent(const std::string & eventName, - EventType type = ET_COUNT, + StatEventType type = ET_COUNT, float value = 1.0, const char * units = "") { diff --git a/soa/service/connection_handler.cc b/soa/service/connection_handler.cc index 603cc67f6..3b4664e75 100644 --- a/soa/service/connection_handler.cc +++ b/soa/service/connection_handler.cc @@ -107,7 +107,7 @@ setTransport(TransportBase * transport) // << transport_ << endl; if (transport_) - throw Exception("can't switch transports from %8p to %8p", + throw Exception("can't switch transports from %08p to %08p", transport_, transport); transport_ = transport; } diff --git a/soa/service/endpoint.cc b/soa/service/endpoint.cc index 49f01fb14..807d2809c 100644 --- a/soa/service/endpoint.cc +++ b/soa/service/endpoint.cc @@ -117,7 +117,6 @@ spinup(int num_threads, bool synchronous) threadsActive_ = 0; - totalSleepTime.resize(num_threads, 1.0); resourceUsage.resize(num_threads); for (unsigned i = 0; i < num_threads; ++i) { @@ -540,6 +539,7 @@ EndpointBase:: doMinCtxSwitchPolling(int threadNum, int numThreads) { bool debug = false; + int epoch = 0; //debug = name() == "Backchannel"; //debug = threadNum == 7; @@ -602,6 +602,23 @@ doMinCtxSwitchPolling(int threadNum, int numThreads) << endl; } + // sync with the loop monitor request + int i = resourceEpoch; + if(i != epoch) { + // query the kernel for performance metrics + rusage now; + getrusage(RUSAGE_THREAD, &now); + + // if we're just started, assume we know nothing and don't update the usage + long s = now.ru_utime.tv_sec+now.ru_stime.tv_sec; + if(s > 1) { + std::lock_guard guard(usageLock); + resourceUsage[threadNum] = now; + } + + epoch = i; + } + // Are we in our timeslice? if (/* forceInSlice || */(fracms >= myStartUs && fracms < myEndUs)) { @@ -610,7 +627,6 @@ doMinCtxSwitchPolling(int threadNum, int numThreads) if (usToWait < 0 || usToWait > timesliceUs) usToWait = timesliceUs; - totalSleepTime[threadNum] += double(usToWait) / 1000000.0; int numHandled = handleEvents(usToWait, 4, handleEvent, beforeSleep, afterSleep); if (debug && false) @@ -641,7 +657,6 @@ doMinCtxSwitchPolling(int threadNum, int numThreads) cerr << "sleeping for " << usToSleep << " micros" << endl; double secToSleep = double(usToSleep) / 1000000.0; - totalSleepTime[threadNum] += secToSleep; ML::sleep(secToSleep); duty.notifyAfterSleep(); diff --git a/soa/service/endpoint.h b/soa/service/endpoint.h index 2ef5286e2..ebd8cf960 100644 --- a/soa/service/endpoint.h +++ b/soa/service/endpoint.h @@ -107,7 +107,6 @@ struct EndpointBase : public Epoller { /** Total number of seconds that this message loop has spent sleeping. Can be polled regularly to determine the duty cycle of the loop. */ - std::vector totalSleepSeconds() const { return totalSleepTime; } std::vector getResourceUsage() const { resourceEpoch++; std::vector result; diff --git a/soa/service/epoll_loop.cc b/soa/service/epoll_loop.cc index fdc0fad17..f53801250 100644 --- a/soa/service/epoll_loop.cc +++ b/soa/service/epoll_loop.cc @@ -68,12 +68,18 @@ loop(int maxEvents, int timeout) for (int i = 0; i < res; i++) { auto * fn = static_cast(events[i].data.ptr); + ExcAssert(fn != nullptr); (*fn)(events[i]); } - for (auto & unreg: delayedUnregistrations_) { - auto cb = move(unreg.second); - unregisterFdCallback(unreg.first, false, cb); + map delayedUnregistrations; + { + std::unique_lock guard(callbackLock_); + delayedUnregistrations = move(delayedUnregistrations_); + delayedUnregistrations_.clear(); + } + for (const auto & unreg: delayedUnregistrations) { + unregisterFdCallback(unreg.first, false, unreg.second); } } catch (const std::exception & exc) { @@ -160,6 +166,7 @@ void EpollLoop:: registerFdCallback(int fd, const EpollCallback & cb) { + std::unique_lock guard(callbackLock_); if (delayedUnregistrations_.count(fd) == 0) { if (fdCallbacks_.find(fd) != fdCallbacks_.end()) { throw ML::Exception("callback already registered for fd"); @@ -176,6 +183,7 @@ EpollLoop:: unregisterFdCallback(int fd, bool delayed, const OnUnregistered & onUnregistered) { + std::unique_lock guard(callbackLock_); if (fdCallbacks_.find(fd) == fdCallbacks_.end()) { throw ML::Exception("callback not registered for fd"); } diff --git a/soa/service/epoll_loop.h b/soa/service/epoll_loop.h index 9c8c54a0d..6d0183ea0 100644 --- a/soa/service/epoll_loop.h +++ b/soa/service/epoll_loop.h @@ -15,8 +15,9 @@ #include #include -#include #include +#include +#include #include "soa/service/async_event_source.h" @@ -46,7 +47,7 @@ struct EpollLoop : public AsyncEventSource typedef std::function EpollCallback; EpollLoop(const OnException & onException); - ~EpollLoop(); + virtual ~EpollLoop(); /* AsyncEventSource interface */ virtual int selectFd() const @@ -92,7 +93,7 @@ struct EpollLoop : public AsyncEventSource /* Remove a file descriptor from the internal epoll queue. If * "unregisterCallback" is specified, "unregisterFdCallback" will be - * specified on the given fd, in delayed mode. */ + * called on the given fd, in delayed mode. */ void removeFd(int fd, bool unregisterCallback = false); /* Associate a callback with a file descriptor for future epoll @@ -122,6 +123,7 @@ struct EpollLoop : public AsyncEventSource int epollFd_; size_t numFds_; + std::mutex callbackLock_; std::map fdCallbacks_; std::map delayedUnregistrations_; diff --git a/soa/service/epoller.h b/soa/service/epoller.h index 72364fbab..264ed5bbd 100644 --- a/soa/service/epoller.h +++ b/soa/service/epoller.h @@ -38,7 +38,7 @@ struct Epoller: public AsyncEventSource { { timeout_ = newTimeout; } - + /** Add the given fd to multiplex fd. It will repeatedly wake up the loop without being restarted. */ @@ -102,7 +102,7 @@ struct Epoller: public AsyncEventSource { virtual bool poll() const; virtual bool processOne(); - + private: /* Perform the fd addition and modification */ void performAddFd(int fd, void * data, bool oneShot, bool restart); diff --git a/soa/service/fs_utils.cc b/soa/service/fs_utils.cc index cfd7e60ef..22bcebae3 100644 --- a/soa/service/fs_utils.cc +++ b/soa/service/fs_utils.cc @@ -343,16 +343,4 @@ dirName(const std::string & filename) return dirname; } -/****************************************************************************/ -/* FILE COMMITER */ -/****************************************************************************/ - -FileCommiter:: -~FileCommiter() -{ - if (!commited_) { - tryEraseUriObject(fileUrl_); - } -} - } // namespace Datacratic diff --git a/soa/service/fs_utils.h b/soa/service/fs_utils.h index fe7f2e1a4..e092d348a 100644 --- a/soa/service/fs_utils.h +++ b/soa/service/fs_utils.h @@ -156,30 +156,4 @@ std::string baseName(const std::string & filename); std::string dirName(const std::string & filename); -/****************************************************************************/ -/* FILE COMMITER */ -/****************************************************************************/ - -/* The FileCommiter class is meant to ensure that a given file is in a - * consistent state and meant to exist. In practice, it gives a reasonable - * guarantee that exceptions or abandonned writes will not leave incomplete - * files lying around. Using RAII, we require the file to be "commited" at - * destruction time and we erase it otherwise. */ -struct FileCommiter { - FileCommiter(const std::string & fileUrl) - : fileUrl_(fileUrl), commited_(false) - { - } - ~FileCommiter(); - - void commit() - { - commited_ = true; - } - -private: - const std::string & fileUrl_; - bool commited_; -}; - } // namespace Datacratic diff --git a/soa/service/http_endpoint.cc b/soa/service/http_endpoint.cc index 931e1c90c..e09cd282f 100644 --- a/soa/service/http_endpoint.cc +++ b/soa/service/http_endpoint.cc @@ -86,7 +86,7 @@ handleData(const std::string & data) } if (readState != HEADER) { - throw Exception("invalid read state %d handling data '%s' for %p", + throw Exception("invalid read state %d handling data '%s' for %08xp", readState, data.c_str(), this); } diff --git a/soa/service/http_header.cc b/soa/service/http_header.cc index 3e694f8b7..cba8144d2 100644 --- a/soa/service/http_header.cc +++ b/soa/service/http_header.cc @@ -163,9 +163,6 @@ expectUrlEncodedString(ML::Parse_Context & context, } result += code; } - else if (c == '+') { - result += ' '; - } else { result += c; } diff --git a/soa/service/http_named_endpoint.cc b/soa/service/http_named_endpoint.cc index 53294a76e..c4d2218a8 100644 --- a/soa/service/http_named_endpoint.cc +++ b/soa/service/http_named_endpoint.cc @@ -61,7 +61,7 @@ bindTcpAddress(const std::string & address) return bindTcp(PortRange(port, last), hostPart); } - throw ML::Exception("invalid port %u", port); + throw ML::Exception("invalid port " + port); } return bindTcp(boost::lexical_cast(portPart), hostPart); diff --git a/soa/service/http_rest_proxy.cc b/soa/service/http_rest_proxy.cc index b5c8b8398..01df300a4 100644 --- a/soa/service/http_rest_proxy.cc +++ b/soa/service/http_rest_proxy.cc @@ -265,7 +265,8 @@ doneConnection(curlpp::Easy * conn) JsonRestProxy:: JsonRestProxy(const string & url) - : HttpRestProxy(url), maxRetries(10), maxBackoffTime(900) + : HttpRestProxy(url), protocolDate(0), + maxRetries(10), maxBackoffTime(900) { if (url.find("https://") == 0) { cerr << "warning: no validation will be performed on the SSL cert.\n"; @@ -290,6 +291,9 @@ performWithBackoff(const string & method, const string & resource, if (authToken.size() > 0) { headers.emplace_back(make_pair("Cookie", "token=\"" + authToken + "\"")); } + if (protocolDate > 0) { + headers.emplace_back(make_pair("X-Protocol-Date", to_string(protocolDate))); + } pid_t tid = gettid(); for (int retries = 0; diff --git a/soa/service/http_rest_proxy.h b/soa/service/http_rest_proxy.h index f9feaa547..ae33ac28e 100644 --- a/soa/service/http_rest_proxy.h +++ b/soa/service/http_rest_proxy.h @@ -304,6 +304,8 @@ struct JsonRestProxy : HttpRestProxy { /* authentication token */ std::string authToken; + int protocolDate; + /* number of exponential backoffs, -1 = unlimited */ int maxRetries; diff --git a/soa/service/logs.h b/soa/service/logs.h index 83b417d9e..cd820bc11 100644 --- a/soa/service/logs.h +++ b/soa/service/logs.h @@ -84,7 +84,7 @@ struct Logging struct ConsoleWriter : public Writer { ConsoleWriter(bool color = true) : - color(color && isatty(STDERR_FILENO)) { + color(color) { } void head(char const * timestamp, diff --git a/soa/service/loop_monitor.cc b/soa/service/loop_monitor.cc index 5212284ed..de8c79381 100644 --- a/soa/service/loop_monitor.cc +++ b/soa/service/loop_monitor.cc @@ -49,7 +49,6 @@ doLoops(uint64_t numTimeouts) LoadSample maxLoad; maxLoad.sequence = curLoad.sequence + 1; - for (auto& loop : loops) { double load = loop.second(updatePeriod * numTimeouts); if (load < 0.0 || load > 1.0) { @@ -74,37 +73,14 @@ LoopMonitor:: addMessageLoop(const string& name, const MessageLoop* loop) { // acts as a private member variable for sampleFn. - rusage lastSample; - auto lastTime = Date(); + double lastTimeSlept = 0.0; auto sampleFn = [=] (double elapsedTime) mutable { - auto sample = loop->getResourceUsage(); - - // get how much time elapsed since last time - auto now = Date::now(); - auto dt = now.secondsSince(lastTime); - - // first time? - if (lastTime == Date()) { - lastSample = sample; - } - - lastTime = now; - auto sec = double(sample.ru_utime.tv_sec - lastSample.ru_utime.tv_sec) - + double(sample.ru_stime.tv_sec - lastSample.ru_stime.tv_sec); - auto usec = double(sample.ru_utime.tv_usec - lastSample.ru_utime.tv_usec) - + double(sample.ru_stime.tv_usec - lastSample.ru_stime.tv_usec); - - auto load = sec + usec * 0.000001; - if (load >= dt) { - load = 1.0; - } else { - load /= dt; - } + double timeSlept = loop->totalSleepSeconds(); + double delta = std::min(timeSlept - lastTimeSlept, 1.0); + lastTimeSlept = timeSlept; - lastSample = sample; - //std::cerr << "THREAD LOAD :" << load << std::endl; - return load; + return 1.0 - (delta / elapsedTime); }; addCallback(name, sampleFn); diff --git a/soa/service/message_loop.cc b/soa/service/message_loop.cc index 4b0ec80ce..d17951bca 100644 --- a/soa/service/message_loop.cc +++ b/soa/service/message_loop.cc @@ -210,15 +210,6 @@ removeSourceSync(AsyncEventSource * source) return true; } -bool -MessageLoop:: -runInMessageLoopThread(std::function toRun) -{ - SourceEntry entry("", toRun, 0); - SourceAction newAction(SourceAction::RUN, move(entry)); - return sourceActions_.push_back(move(newAction)); -} - void MessageLoop:: wakeupMainThread() @@ -296,20 +287,10 @@ runWorkerThread() return; // Do any outstanding work now - int i = 0; - while (processOne()) { + while (processOne()) if (shutdown_) return; - if (i >= 50) { - getrusage(RUSAGE_THREAD, &resourceUsage); - i = 0; - } - i++; - } - - getrusage(RUSAGE_THREAD, &resourceUsage); - // At this point, we've done as much work as we can (there is no more // work to do). We will now sleep for the maximum allowable delay // time minus the time we spent working. This allows us to batch up @@ -391,9 +372,6 @@ handleSourceActions() else if (action.action_ == SourceAction::REMOVE) { processRemoveSource(action.entry_); } - else if (action.action_ == SourceAction::RUN) { - processRunAction(action.entry_); - } } } @@ -477,13 +455,6 @@ processRemoveSource(const SourceEntry & rmEntry) ML::futex_wake(entry.source->connectionState_); } -void -MessageLoop:: -processRunAction(const SourceEntry & entry) -{ - entry.run(); -} - bool MessageLoop:: poll() const diff --git a/soa/service/message_loop.h b/soa/service/message_loop.h index f43a882a1..cba3b830b 100644 --- a/soa/service/message_loop.h +++ b/soa/service/message_loop.h @@ -17,7 +17,6 @@ #include "async_event_source.h" #include "typed_message_channel.h" #include "logs.h" -#include "rusage.h" namespace Datacratic { @@ -124,12 +123,6 @@ struct MessageLoop : public Epoller { */ bool removeSourceSync(AsyncEventSource * source); - /** Run the given function in the main message loop thread. - WARNING: calling this function from the message loop thread will result - in a deadlock. - */ - bool runInMessageLoopThread(std::function toRun); - /** Re-check if anything needs to poll. */ void checkNeedsPoll(); @@ -137,7 +130,6 @@ struct MessageLoop : public Epoller { Can be polled regularly to determine the duty cycle of the loop. */ double totalSleepSeconds() const { return totalSleepTime_; } - rusage getResourceUsage() const { return resourceUsage; } void debug(bool debugOn); @@ -155,19 +147,12 @@ struct MessageLoop : public Epoller { SourceEntry(const std::string& name, std::shared_ptr source, int priority) - : name(name), source(std::move(source)), priority(priority) - {} - - SourceEntry(const std::string& name, - std::function run, - int priority) - : name(name), priority(priority), run(std::move(run)) + : name(name), source(source), priority(priority) {} std::string name; std::shared_ptr source; int priority; - std::function run; }; std::vector sources; @@ -176,7 +161,6 @@ struct MessageLoop : public Epoller { struct SourceAction { static constexpr int ADD = 0; static constexpr int REMOVE = 1; - static constexpr int RUN = 2; SourceAction() = default; @@ -205,7 +189,6 @@ struct MessageLoop : public Epoller { /** Number of secs that the message loop has spent sleeping. */ double totalSleepTime_; - rusage resourceUsage; /** Number of seconds of latency we're allowed to add in order to reduce the number of context switches. @@ -216,7 +199,6 @@ struct MessageLoop : public Epoller { void handleSourceActions(); void processAddSource(const SourceEntry & entry); void processRemoveSource(const SourceEntry & entry); - void processRunAction(const SourceEntry & entry); }; } // namespace Datacratic diff --git a/soa/service/redis.cc b/soa/service/redis.cc index cccdda841..c1be3488e 100644 --- a/soa/service/redis.cc +++ b/soa/service/redis.cc @@ -14,6 +14,7 @@ #include "jml/arch/atomic_ops.h" #include "jml/arch/backtrace.h" #include "jml/arch/futex.h" +#include "jml/arch/wakeup_fd.h" #include "jml/utils/vector_utils.h" @@ -416,7 +417,7 @@ size_t eventLoopsDestroyed = 0; struct AsyncConnection::EventLoop { - int wakeupfd[2]; + ML::Wakeup_Fd wakeupfd; volatile bool finished; AsyncConnection * connection; std::shared_ptr thread; @@ -424,18 +425,14 @@ struct AsyncConnection::EventLoop { volatile int disconnected; EventLoop(AsyncConnection * connection) - : finished(false), connection(connection), disconnected(1) + : wakeupfd(O_NONBLOCK) + , finished(false) + , connection(connection) + , disconnected(1) { ML::atomic_inc(eventLoopsCreated); - int res = pipe2(wakeupfd, O_NONBLOCK); - if (res == -1) - throw ML::Exception(errno, "pipe2"); - - //cerr << "connection on fd " << connection->context_->c.fd << endl; - - - fds[0].fd = wakeupfd[0]; + fds[0].fd = wakeupfd.fd(); fds[0].events = POLLIN; fds[1].fd = connection->context_->c.fd; fds[1].events = 0; @@ -443,13 +440,6 @@ struct AsyncConnection::EventLoop { registerMe(connection->context_); thread.reset(new boost::thread(boost::bind(&EventLoop::run, this))); - -#if 0 - char buf[1]; - res = read(wakeupfd[0], buf, 1); - if (res == -1) - throw ML::Exception(errno, "read"); -#endif } ~EventLoop() @@ -467,16 +457,11 @@ struct AsyncConnection::EventLoop { wakeup(); thread->join(); thread.reset(); - ::close(wakeupfd[0]); - ::close(wakeupfd[1]); } void wakeup() { - int res = write(wakeupfd[1], "x", 1); - if (res == -1) - throw ML::Exception("error waking up fd %d: %s", wakeupfd[1], - strerror(errno)); + wakeupfd.signal(); } void registerMe(redisAsyncContext * context) @@ -531,12 +516,18 @@ struct AsyncConnection::EventLoop { if (fds[0].revents & POLLIN) { //cerr << "got wakeup" << endl; - char buf[128]; - int res = read(fds[0].fd, buf, 128); - if (res == -1) - throw ML::Exception(errno, "read from wakeup pipe"); + wakeupfd.read(); //cerr << "woken up with " << res << " messages" << endl; } + if ((fds[1].revents & POLLHUP) + || (fds[1].revents & POLLERR)) { + /* For now, we simply disconnect when we receive a POLLHUP, but eventually + * we should try to reconnect or at least notify the user through a callback + */ + onDisconnect(REDIS_ERR_IO); + break; + } + if ((fds[1].revents & POLLOUT) && (fds[1].events & POLLOUT)) { //cerr << "got write on " << fds[1].fd << endl; diff --git a/soa/service/s3.cc b/soa/service/s3.cc index 750172a57..a4b11dcab 100644 --- a/soa/service/s3.cc +++ b/soa/service/s3.cc @@ -1390,7 +1390,7 @@ tryGetObjectInfoFull(const std::string & bucket, const std::string & object) auto listingResult = get(bucket, "/", Range::Full, "", {}, queryParams); if (listingResult.code_ != 200) { cerr << listingResult.bodyXmlStr() << endl; - throw ML::Exception("error getting object request: %ld", + throw ML::Exception("error getting object request: %d", listingResult.code_); } auto listingResultXml = listingResult.bodyXml(); @@ -1458,7 +1458,7 @@ eraseObject(const std::string & bucket, if (response.code_ != 204) { cerr << response.bodyXmlStr() << endl; - throw ML::Exception("error erasing object request: %ld", + throw ML::Exception("error erasing object request: %d", response.code_); } } diff --git a/soa/service/s3.h b/soa/service/s3.h index 67f0093ee..03f901b38 100644 --- a/soa/service/s3.h +++ b/soa/service/s3.h @@ -135,8 +135,7 @@ struct S3Api : public AwsApi { void adjust(size_t downloaded) { if (downloaded > size) { - throw ML::Exception("excessive adjustment size:" - " downloaded %zu size %lu", + throw ML::Exception("excessive adjustment size: downloaded %d size %d", downloaded, size); } offset += downloaded; diff --git a/soa/service/service.mk b/soa/service/service.mk index d1b77ce15..ff022b748 100644 --- a/soa/service/service.mk +++ b/soa/service/service.mk @@ -41,7 +41,6 @@ LIBSERVICES_SOURCES := \ passive_endpoint.cc \ chunked_http_endpoint.cc \ epoller.cc \ - epoll_loop.cc \ http_header.cc \ port_range_service.cc \ service_base.cc \ diff --git a/soa/service/service_base.cc b/soa/service/service_base.cc index 3660702f8..e42b0075c 100644 --- a/soa/service/service_base.cc +++ b/soa/service/service_base.cc @@ -79,7 +79,7 @@ void NullEventService:: onEvent(const std::string & name, const char * event, - EventType type, + StatEventType type, float value, std::initializer_list) { @@ -124,7 +124,7 @@ void CarbonEventService:: onEvent(const std::string & name, const char * event, - EventType type, + StatEventType type, float value, std::initializer_list extra) { @@ -683,7 +683,7 @@ EventRecorder(const std::string & eventPrefix, void EventRecorder:: -recordEventFmt(EventType type, +recordEventFmt(StatEventType type, float value, std::initializer_list extra, const char * fmt, ...) const diff --git a/soa/service/service_base.h b/soa/service/service_base.h index 174071038..493055846 100644 --- a/soa/service/service_base.h +++ b/soa/service/service_base.h @@ -48,7 +48,7 @@ struct EventService { virtual void onEvent(const std::string & name, const char * event, - EventType type, + StatEventType type, float value, std::initializer_list extra = DefaultOutcomePercentiles) = 0; @@ -72,7 +72,7 @@ struct NullEventService : public EventService { virtual void onEvent(const std::string & name, const char * event, - EventType type, + StatEventType type, float value, std::initializer_list extra = DefaultOutcomePercentiles); @@ -98,7 +98,7 @@ struct CarbonEventService : public EventService { virtual void onEvent(const std::string & name, const char * event, - EventType type, + StatEventType type, float value, std::initializer_list extra = std::initializer_list()); @@ -471,7 +471,7 @@ struct EventRecorder { units: the units of the event (eg, ms). Default is unitless. */ void recordEvent(const char * eventName, - EventType type = ET_COUNT, + StatEventType type = ET_COUNT, float value = 1.0, std::initializer_list extra = DefaultOutcomePercentiles) const { @@ -488,7 +488,7 @@ struct EventRecorder { es->onEvent(eventPrefix_, eventName, type, value, extra); } - void recordEventFmt(EventType type, + void recordEventFmt(StatEventType type, float value, std::initializer_list extra, const char * fmt, ...) const JML_FORMAT_STRING(5, 6); diff --git a/soa/service/stat_aggregator.cc b/soa/service/stat_aggregator.cc index d325b9085..ede1c5370 100644 --- a/soa/service/stat_aggregator.cc +++ b/soa/service/stat_aggregator.cc @@ -90,9 +90,9 @@ read(const std::string & prefix) /*****************************************************************************/ GaugeAggregator:: -GaugeAggregator(Verbosity verbosity, const std::vector& extra) +GaugeAggregator(Verbosity verbosity, std::vector extra) : verbosity(verbosity), values(new ML::distribution()) - , extra(extra) + , extra(std::move(extra)) { if (verbosity == Outcome) ExcCheck(this->extra.size() > 0, "Can not construct with empty percentiles"); diff --git a/soa/service/stat_aggregator.h b/soa/service/stat_aggregator.h index b6605969b..2a0f60b27 100644 --- a/soa/service/stat_aggregator.h +++ b/soa/service/stat_aggregator.h @@ -99,7 +99,7 @@ struct GaugeAggregator : public StatAggregator { }; GaugeAggregator(Verbosity verbosity = Outcome, - const std::vector& extra = DefaultOutcomePercentiles); + std::vector extra = DefaultOutcomePercentiles); virtual ~GaugeAggregator(); diff --git a/soa/service/stats_events.h b/soa/service/stats_events.h index b0fb9cce6..2b5aa9062 100644 --- a/soa/service/stats_events.h +++ b/soa/service/stats_events.h @@ -10,7 +10,7 @@ namespace Datacratic { -enum EventType { +enum StatEventType { ET_HIT, ///< Represents an extra count on a counter ET_COUNT, ///< Represents an extra value accumulated ET_STABLE_LEVEL, ///< Represents the current level of a stable something diff --git a/soa/service/tcp_client.cc b/soa/service/tcp_client.cc index ddec7d347..f43e6e8fc 100644 --- a/soa/service/tcp_client.cc +++ b/soa/service/tcp_client.cc @@ -166,7 +166,8 @@ connect(const OnConnectionResult & onConnectionResult) handleConnectionEventCb_ = [=] (const ::epoll_event & event) { this->handleConnectionEvent(socketFd, onConnectionResult); }; - addFdOneShot(socketFd, false, true, handleConnectionEventCb_); + registerFdCallback(socketFd, handleConnectionEventCb_); + addFdOneShot(socketFd, false, true); enableQueue(); state_ = TcpClientState::Connecting; // cerr << "connection in progress\n"; @@ -213,7 +214,8 @@ handleConnectionEvent(int socketFd, OnConnectionResult onConnectionResult) throw ML::Exception("unhandled error:" + to_string(result)); } - removeFd(socketFd, true); + removeFd(socketFd); + unregisterFdCallback(socketFd, true); if (connCode == Success) { errno = 0; setFd(socketFd); diff --git a/soa/service/testing/http_client_bench.cc b/soa/service/testing/http_client_bench.cc index 16b7b4edf..ee92d6568 100644 --- a/soa/service/testing/http_client_bench.cc +++ b/soa/service/testing/http_client_bench.cc @@ -147,12 +147,11 @@ int main(int argc, char *argv[]) { using namespace boost::program_options; - unsigned int concurrency(0); - unsigned int serverConcurrency(0); + size_t concurrency(0); int model(0); - unsigned int maxReqs(0); + size_t maxReqs(0); string method("GET"); - unsigned int payloadSize(0); + size_t payloadSize(0); string serveriface("127.0.0.1"); string clientiface(serveriface); @@ -163,8 +162,6 @@ int main(int argc, char *argv[]) "address:port to connect to (\"none\" for no client)") ("concurrency,c", value(&concurrency), "Number of concurrent requests") - ("server-concurrency", value(&serverConcurrency), - "Number of server worker threads (defaults to \"concurrency\")") ("method,M", value(&method), "Method to use (\"GET\"*, \"PUT\", \"POST\")") ("model,m", value(&model), @@ -201,9 +198,6 @@ int main(int argc, char *argv[]) if (concurrency == 0) { throw ML::Exception("'concurrency' must be specified"); } - if (serverConcurrency == 0) { - serverConcurrency = concurrency; - } if (payloadSize == 0) { throw ML::Exception("'payload-size' must be specified"); @@ -221,7 +215,7 @@ int main(int argc, char *argv[]) service.addResponse("GET", "/", 200, payload); service.addResponse("PUT", "/", 200, ""); service.addResponse("POST", "/", 200, ""); - service.start(serveriface, serverConcurrency); + service.start(serveriface, concurrency); } if (clientiface != "none") { @@ -271,7 +265,7 @@ int main(int argc, char *argv[]) } double qps = maxReqs / delta; double bps = double(maxReqs * payload.size()) / delta; - ::printf("%d\t%u\t%u\t%u\t%f\t%f\t%f\n", + ::printf("%d\t%lu\t%lu\t%lu\t%f\t%f\t%f\n", model, concurrency, maxReqs, payloadSize, delta, bps, qps); } else { diff --git a/soa/service/testing/http_client_test.cc b/soa/service/testing/http_client_test.cc index 698d16547..cae7bfad7 100644 --- a/soa/service/testing/http_client_test.cc +++ b/soa/service/testing/http_client_test.cc @@ -672,6 +672,7 @@ BOOST_AUTO_TEST_CASE( test_http_client_connection_closed ) auto proxies = make_shared(); HttpGetService service(proxies); + service.portToUse = 8080; service.addResponse("GET", "/", 200, "coucou"); service.start(); service.waitListening(); diff --git a/soa/service/testing/http_header_test.cc b/soa/service/testing/http_header_test.cc index 67b22306c..cf2f4ecdc 100644 --- a/soa/service/testing/http_header_test.cc +++ b/soa/service/testing/http_header_test.cc @@ -115,10 +115,3 @@ BOOST_AUTO_TEST_CASE(test_http_header_query_parser_g6) BOOST_CHECK_EQUAL(parser.queryParams[2].second, "="); BOOST_CHECK_EQUAL(parser.queryParams[3].second, ""); } - -BOOST_AUTO_TEST_CASE(test_http_header_space_as_plus) -{ - auto header = generateParser("/?arg1=1+2"); - - testQueryParam(header, "arg1", "1 2"); -} diff --git a/soa/service/testing/mongo_temporary_server.cc b/soa/service/testing/mongo_temporary_server.cc index bdfce4c17..2ac60a5fc 100644 --- a/soa/service/testing/mongo_temporary_server.cc +++ b/soa/service/testing/mongo_temporary_server.cc @@ -76,8 +76,7 @@ void MongoTemporaryServer::testConnection() { void MongoTemporaryServer::start() { namespace fs = boost::filesystem; // Check the unique path - if (uniquePath_ == "" || uniquePath_[0] == '/' || uniquePath_ == "." - || uniquePath_ == "..") { + if (uniquePath_.empty() || uniquePath_ == "." || uniquePath_ == "..") { throw ML::Exception("unacceptable unique path"); } @@ -85,15 +84,18 @@ void MongoTemporaryServer::start() { // First check that it doesn't exist struct stat stats; - int res = stat(uniquePath_.c_str(), &stats); - if (res != -1 || (errno != EEXIST && errno != ENOENT)) { - throw ML::Exception(errno, "unique path " + uniquePath_ - + " already exists"); + int res = ::stat(uniquePath_.c_str(), &stats); + if (res == -1) { + if (errno != EEXIST && errno != ENOENT) { + throw ML::Exception(errno, "unhandled exception"); + } + } else if (res == 0) { + throw ML::Exception("unique path " + uniquePath_ + " already exists"); } + cerr << "creating directory " << uniquePath_ << endl; - if(!fs::create_directory(fs::path(uniquePath_))) { - throw ML::Exception( - "could not create unique path %s",uniquePath_.c_str()); + if (!fs::create_directories(fs::path(uniquePath_))) { + throw ML::Exception("could not create unique path " + uniquePath_); } socketPath_ = uniquePath_ + "/mongo-socket"; diff --git a/soa/service/testing/py/mongo_temp_server_wrapping.cc b/soa/service/testing/py/mongo_temp_server_wrapping.cc index dbe60b571..9df40b71f 100644 --- a/soa/service/testing/py/mongo_temp_server_wrapping.cc +++ b/soa/service/testing/py/mongo_temp_server_wrapping.cc @@ -18,7 +18,7 @@ struct MongoTemporaryServerPtr { MongoTemporaryServerPtr(const std::string & uniquePath = "", const int portNum = 28356) : - mongoTmpServer(new MongoTemporaryServer(uniquePath, portNum)) + mongoTmpServer(new MongoTemporaryServer(uniquePath /*, portNum */)) { } diff --git a/soa/service/testing/redis_temporary_server.h b/soa/service/testing/redis_temporary_server.h index a916e5e5f..a50cf3dbb 100644 --- a/soa/service/testing/redis_temporary_server.h +++ b/soa/service/testing/redis_temporary_server.h @@ -31,8 +31,10 @@ struct RedisTemporaryServer : boost::noncopyable { using namespace std; if (uniquePath == "") { ML::Env_Option tmpDir("TMP", "./tmp"); + std::string dir = tmpDir; + if(dir[0] == '/') dir.insert(0, 1, '.'); uniquePath = ML::format("%s/redis-temporary-server-%d-%d", - tmpDir.get(), getpid(), index); + dir, getpid(), index); cerr << "starting redis temporary server under unique path " << uniquePath << endl; } @@ -51,8 +53,7 @@ struct RedisTemporaryServer : boost::noncopyable { using namespace std; // Check the unique path - if (uniquePath == "" || uniquePath[0] == '/' || uniquePath == "." - || uniquePath == "..") + if (uniquePath.empty() || uniquePath == "." || uniquePath == "..") throw ML::Exception("unacceptable unique path"); // 1. Create the directory diff --git a/soa/service/testing/runner_stress_test.cc b/soa/service/testing/runner_stress_test.cc index 8255426c1..197ac5975 100644 --- a/soa/service/testing/runner_stress_test.cc +++ b/soa/service/testing/runner_stress_test.cc @@ -106,10 +106,6 @@ BOOST_AUTO_TEST_CASE( test_stress_runner ) atomic nRunning(0); activeThreads = nThreads; - - auto onTerminate = [&] (const RunResult &) { - }; - auto runThread = [&] (int threadNum) { /* preparation */ HelperCommands commands; @@ -159,7 +155,7 @@ BOOST_AUTO_TEST_CASE( test_stress_runner ) auto & stdInSink = runner.getStdInSink(); runner.run({"build/x86_64/bin/runner_test_helper"}, - onTerminate, stdOutSink, stdErrSink); + nullptr, stdOutSink, stdErrSink); for (const string & command: commands) { while (!stdInSink.write(string(command))) { diff --git a/soa/service/testing/service_testing.mk b/soa/service/testing/service_testing.mk index 5071315bd..d66781ba9 100644 --- a/soa/service/testing/service_testing.mk +++ b/soa/service/testing/service_testing.mk @@ -1,4 +1,4 @@ -$(eval $(call library,mongo_tmp_server,mongo_temporary_server.cc, services)) +#$(eval $(call library,mongo_tmp_server,mongo_temporary_server.cc, services)) $(eval $(call test,epoll_test,services,boost)) $(eval $(call test,epoll_wait_test,services,boost manual)) @@ -37,7 +37,7 @@ $(eval $(call test,message_loop_test,services,boost)) $(eval $(call program,runner_test_helper,utils)) $(eval $(call test,runner_test,services,boost)) -$(eval $(call test,runner_stress_test,services,boost)) +$(eval $(call test,runner_stress_test,services,boost manual)) $(TESTS)/runner_test $(TESTS)/runner_stress_test: $(BIN)/runner_test_helper $(eval $(call test,sink_test,services,boost)) @@ -63,6 +63,5 @@ $(eval $(call test,sns_mock_test,cloud services,boost)) $(eval $(call test,zmq_message_loop_test,services,boost)) $(eval $(call test,event_handler_test,cloud services,boost manual)) -$(eval $(call test,mongo_basic_test,services boost_filesystem mongo_tmp_server,boost manual)) - -$(eval $(call include_sub_makes,py)) +#$(eval $(call test,mongo_basic_test,services boost_filesystem mongo_tmp_server,boost manual)) +#$(eval $(call include_sub_makes,py)) diff --git a/soa/service/testing/test_http_services.cc b/soa/service/testing/test_http_services.cc index d19923dbf..cc46b267a 100644 --- a/soa/service/testing/test_http_services.cc +++ b/soa/service/testing/test_http_services.cc @@ -10,7 +10,6 @@ HttpService(const shared_ptr & proxies) HttpEndpoint("http-test-service-ep"), portToUse(0), numReqs(0) { - HttpEndpoint::setPollingMode(EndpointBase::MIN_CPU_POLLING); } HttpService:: diff --git a/soa/types/basic_value_descriptions.h b/soa/types/basic_value_descriptions.h index 59e3986db..e5d4075ef 100644 --- a/soa/types/basic_value_descriptions.h +++ b/soa/types/basic_value_descriptions.h @@ -36,7 +36,7 @@ struct DefaultDescription val->val2 == 0 && val->val1 <= std::numeric_limits::max()) { context.writeInt(val->val1); } else { - context.writeStringUtf8(Utf8String(val->toString())); + context.writeString(val->toString()); } } @@ -51,7 +51,7 @@ struct StringIdDescription: public DefaultDescription { virtual void printJsonTyped(const Datacratic::Id * val, JsonPrintingContext & context) const { - context.writeStringUtf8(Utf8String(val->toString())); + context.writeString(val->toString()); } }; diff --git a/soa/types/date.cc b/soa/types/date.cc index c5e26b297..8b5c85ff7 100644 --- a/soa/types/date.cc +++ b/soa/types/date.cc @@ -1187,7 +1187,7 @@ toTm() const errno = 0; time_t t = toTimeT(); if (gmtime_r(&t, &result) == 0) - throw ML::Exception("error converting time: t = %lld (%s)", + throw ML::Exception("error converting time: t = %lld", (long long)t, strerror(errno)); return result; diff --git a/soa/types/id.cc b/soa/types/id.cc index 7667111ab..036cb5a16 100644 --- a/soa/types/id.cc +++ b/soa/types/id.cc @@ -4,7 +4,6 @@ */ -#include #include "id.h" #include "jml/arch/bit_range_ops.h" #include "jml/arch/format.h" @@ -136,7 +135,7 @@ parse(const char * value, size_t len, Type type) return; } - while ((type == UNKNOWN || type == UUID || type == UUID_CAPS) && len == 36) { + while ((type == UNKNOWN ||type == UUID) && len == 36) { // not really a while... // Try a uuid // AGID: --> 0828398c-5965-11e0-84c8-0026b937c8e1 @@ -152,34 +151,12 @@ parse(const char * value, size_t len, Type type) const char * p = value; bool failed = false; - int capsType = -1; // -1 = unknown, 0 = no, 1 = yes auto scanRange = [&] (int start, int len) -> unsigned long long { unsigned long long val = 0; for (unsigned i = start; i != start + len; ++i) { int c = p[i]; - { - // case handling - if (c >= 'a' && c <= 'f') { - if (capsType == -1) { - capsType = 0; - } - else if (capsType != 0) { - failed = true; - return val; - } - } - else if (c >= 'A' && c <= 'F') { - if (capsType == -1) { - capsType = 1; - } - else if (capsType != 1) { - failed = true; - return val; - } - } - } int v = hexToDec(c); if (v == -1) { failed = true; @@ -198,8 +175,10 @@ parse(const char * value, size_t len, Type type) f5 = scanRange(24, 12); if (failed) break; - r.type = capsType == 1 ? UUID_CAPS : UUID; + r.type = UUID; r.f1 = f1; r.f2 = f2; r.f3 = f3; r.f4 = f4; r.f5 = f5; + //r.val1 = ((uint64_t)f1 << 32) | ((uint64_t)f2 << 16) | f3; + //r.val2 = ((uint64_t)f4 << 48) | f5; finish(); return; } @@ -397,15 +376,10 @@ toString() const case NULLID: return "null"; case UUID: - return ML::format( - "%08lx-%04x-%04x-%04x-%012llx", - (unsigned)f1, (unsigned)f2, (unsigned)f3, (unsigned)f4, - (unsigned long long)f5); - case UUID_CAPS: - return ML::format( - "%08lX-%04X-%04X-%04X-%012llX", - (unsigned)f1, (unsigned)f2, (unsigned)f3, (unsigned)f4, - (unsigned long long)f5); + // AGID: --> 0828398c-5965-11e0-84c8-0026b937c8e1 + return ML::format("%08lx-%04x-%04x-%04x-%012llx", + (unsigned)f1, (unsigned)f2, (unsigned)f3, (unsigned)f4, + (unsigned long long)f5); case GOOG128: { // Google ID: --> CAESEAYra3NIxLT9C8twKrzqaA string result = "CAESE "; @@ -516,6 +490,10 @@ uint64_t Id:: complexHash() const { +#if ID_HASH_AS_STRING + std::string converted = toString(); + return CityHash64(converted.c_str(), converted.size()); +#else if (type == STR) return CityHash64(str, len); else if (type == COMPOUND2) { @@ -525,6 +503,7 @@ complexHash() const //else if (type == CUSTOM) // return controlFn(CF_HASH, data); else throw ML::Exception("unknown Id type"); +#endif } void @@ -579,7 +558,6 @@ serialize(ML::DB::Store_Writer & store) const case NONE: break; case NULLID: break; case UUID: - case UUID_CAPS: case GOOG128: case BIGDEC: store.save_binary(&val1, 8); @@ -638,7 +616,6 @@ reconstitute(ML::DB::Store_Reader & store) case NONE: break; case NULLID: break; case UUID: - case UUID_CAPS: case GOOG128: case BIGDEC: { store.load_binary(&r.val1, 8); diff --git a/soa/types/id.h b/soa/types/id.h index 3f98d9850..38d5a77c4 100644 --- a/soa/types/id.h +++ b/soa/types/id.h @@ -54,7 +54,6 @@ struct Id { BASE64_96 = 5, /// 16 character base64 string HEX128LC = 6, /// 32 character lowercase hex string INT64DEC = 7, /// obsolete type, do not use - UUID_CAPS = 8, /// uuid-ish string eg 0828398C-5965-11E0-84C8-0026B937C8E1 // other integer-encoded values go here @@ -205,8 +204,14 @@ struct Id { uint64_t hash() const { if (type == NONE || type == NULLID) return 0; +#if ID_HASH_AS_STRING + if (type == STR) + return CityHash64(str, len); + return complexHash(); +#else if (JML_UNLIKELY(type >= STR)) return complexHash(); return Hash128to64(std::make_pair(val1, val2)); +#endif } bool complexEqual(const Id & other) const; diff --git a/soa/types/json_parsing.h b/soa/types/json_parsing.h index 730719697..7c71a0b36 100644 --- a/soa/types/json_parsing.h +++ b/soa/types/json_parsing.h @@ -815,8 +815,15 @@ void parseJson(Id * output, Context & context) using namespace std; if (context.isString()) { - Utf8String value = context.expectStringUtf8(); - *output = Id(value.rawString()); + char buffer[4096]; + ssize_t realSize = context.expectStringAscii(buffer, sizeof(buffer)); + if (realSize > -1) { + *output = Id(buffer, realSize); + } + else { + std::string value = context.expectStringAscii(); + *output = Id(value); + } return; } diff --git a/soa/types/json_printing.cc b/soa/types/json_printing.cc index 5bc9a070b..c3344d4ce 100644 --- a/soa/types/json_printing.cc +++ b/soa/types/json_printing.cc @@ -8,7 +8,7 @@ #include "jml/utils/exc_assert.h" #include "json_printing.h" -#include "dtoa.h" + using namespace std; @@ -54,328 +54,4 @@ writeStringUtf8(const Utf8String & s) } -/*****************************************************************************/ -/* STREAM JSON PRINTING CONTEXT */ -/*****************************************************************************/ - -StreamJsonPrintingContext:: -StreamJsonPrintingContext(std::ostream & stream) - : stream(stream), writeUtf8(true) -{ -} - -void -StreamJsonPrintingContext:: -startObject() -{ - path.push_back(true /* isObject */); - stream << "{"; -} - -void -StreamJsonPrintingContext:: -startMember(const std::string & memberName) -{ - ExcAssert(path.back().isObject); - //path.back().memberName = memberName; - ++path.back().memberNum; - if (path.back().memberNum != 0) - stream << ","; - stream << '\"'; - ML::jsonEscape(memberName, stream); - stream << "\":"; -} - -void -StreamJsonPrintingContext:: -endObject() -{ - ExcAssert(path.back().isObject); - path.pop_back(); - stream << "}"; -} - -void -StreamJsonPrintingContext:: -startArray(int knownSize) -{ - path.push_back(false /* isObject */); - stream << "["; -} - -void -StreamJsonPrintingContext:: -newArrayElement() -{ - ExcAssert(!path.back().isObject); - ++path.back().memberNum; - if (path.back().memberNum != 0) - stream << ","; -} - -void -StreamJsonPrintingContext:: -endArray() -{ - ExcAssert(!path.back().isObject); - path.pop_back(); - stream << "]"; -} - -void -StreamJsonPrintingContext:: -skip() -{ - stream << "null"; -} - -void -StreamJsonPrintingContext:: -writeNull() -{ - stream << "null"; -} - -void -StreamJsonPrintingContext:: -writeInt(int i) -{ - stream << i; -} - -void -StreamJsonPrintingContext:: -writeUnsignedInt(unsigned int i) -{ - stream << i; -} - -void -StreamJsonPrintingContext:: -writeLong(long int i) -{ - stream << i; -} - -void -StreamJsonPrintingContext:: -writeUnsignedLong(unsigned long int i) -{ - stream << i; -} - -void -StreamJsonPrintingContext:: -writeLongLong(long long int i) -{ - stream << i; -} - -void -StreamJsonPrintingContext:: -writeUnsignedLongLong(unsigned long long int i) -{ - stream << i; -} - -void -StreamJsonPrintingContext:: -writeFloat(float f) -{ - if (std::isfinite(f)) - stream << Datacratic::dtoa(f); - else stream << "\"" << f << "\""; -} - -void -StreamJsonPrintingContext:: -writeDouble(double d) -{ - if (std::isfinite(d)) - stream << Datacratic::dtoa(d); - else stream << "\"" << d << "\""; -} - -void -StreamJsonPrintingContext:: -writeString(const std::string & s) -{ - stream << '\"'; - ML::jsonEscape(s, stream); - stream << '\"'; -} - -void -StreamJsonPrintingContext:: -writeStringUtf8(const Utf8String & s); - -void -StreamJsonPrintingContext:: -writeJson(const Json::Value & val) -{ - stream << val.toStringNoNewLine(); -} - -void -StreamJsonPrintingContext:: -writeBool(bool b) -{ - stream << (b ? "true": "false"); -} - - -/*****************************************************************************/ -/* STRUCTURED JSON PRINTING CONTEXT */ -/*****************************************************************************/ - -StructuredJsonPrintingContext:: -StructuredJsonPrintingContext() - : current(&output) -{ -} - -void -StructuredJsonPrintingContext:: -startObject() -{ - *current = Json::Value(Json::objectValue); - path.push_back(current); -} - -void -StructuredJsonPrintingContext:: -startMember(const std::string & memberName) -{ - current = &(*path.back())[memberName]; -} - -void -StructuredJsonPrintingContext:: -endObject() -{ - path.pop_back(); -} - -void -StructuredJsonPrintingContext:: -startArray(int knownSize) -{ - *current = Json::Value(Json::arrayValue); - path.push_back(current); -} - -void -StructuredJsonPrintingContext:: -newArrayElement() -{ - Json::Value & b = *path.back(); - current = &b[b.size()]; -} - -void -StructuredJsonPrintingContext:: -endArray() -{ - path.pop_back(); -} - -void -StructuredJsonPrintingContext:: -skip() -{ - *current = Json::Value(); -} - -void -StructuredJsonPrintingContext:: -writeNull() -{ - *current = Json::Value(); -} - -void -StructuredJsonPrintingContext:: -writeInt(int i) -{ - *current = i; -} - -void -StructuredJsonPrintingContext:: -writeUnsignedInt(unsigned int i) -{ - *current = i; -} - -void -StructuredJsonPrintingContext:: -writeLong(long int i) -{ - *current = i; -} - -void -StructuredJsonPrintingContext:: -writeUnsignedLong(unsigned long int i) -{ - *current = i; -} - -void -StructuredJsonPrintingContext:: -writeLongLong(long long int i) -{ - *current = i; -} - -void -StructuredJsonPrintingContext:: -writeUnsignedLongLong(unsigned long long int i) -{ - *current = i; -} - -void -StructuredJsonPrintingContext:: -writeFloat(float f) -{ - *current = f; -} - -void -StructuredJsonPrintingContext:: -writeDouble(double d) -{ - *current = d; -} - -void -StructuredJsonPrintingContext:: -writeString(const std::string & s) -{ - *current = s; -} - -void -StructuredJsonPrintingContext:: -writeStringUtf8(const Utf8String & s) -{ - *current = s; -} - -void -StructuredJsonPrintingContext:: -writeJson(const Json::Value & val) -{ - *current = val; -} - -void -StructuredJsonPrintingContext:: -writeBool(bool b) -{ - *current = b; -} - - } // namespace Datacratic diff --git a/soa/types/json_printing.h b/soa/types/json_printing.h index c7ee6e051..74fee9dca 100644 --- a/soa/types/json_printing.h +++ b/soa/types/json_printing.h @@ -63,7 +63,10 @@ struct JsonPrintingContext { struct StreamJsonPrintingContext : public JsonPrintingContext { - StreamJsonPrintingContext(std::ostream & stream); + StreamJsonPrintingContext(std::ostream & stream) + : stream(stream), writeUtf8(true) + { + } std::ostream & stream; bool writeUtf8; ///< If true, utf8 chars in binary. False: escaped ASCII @@ -81,45 +84,125 @@ struct StreamJsonPrintingContext std::vector path; - virtual void startObject(); + virtual void startObject() + { + path.push_back(true /* isObject */); + stream << "{"; + } - virtual void startMember(const std::string & memberName); + virtual void startMember(const std::string & memberName) + { + ExcAssert(path.back().isObject); + //path.back().memberName = memberName; + ++path.back().memberNum; + if (path.back().memberNum != 0) + stream << ","; + stream << '\"'; + ML::jsonEscape(memberName, stream); + stream << "\":"; + } - virtual void endObject(); + virtual void endObject() + { + ExcAssert(path.back().isObject); + path.pop_back(); + stream << "}"; + } - virtual void startArray(int knownSize = -1); + virtual void startArray(int knownSize = -1) + { + path.push_back(false /* isObject */); + stream << "["; + } - virtual void newArrayElement(); + virtual void newArrayElement() + { + ExcAssert(!path.back().isObject); + ++path.back().memberNum; + if (path.back().memberNum != 0) + stream << ","; + } - virtual void endArray(); + virtual void endArray() + { + ExcAssert(!path.back().isObject); + path.pop_back(); + stream << "]"; + } - virtual void skip(); + virtual void skip() + { + stream << "null"; + } - virtual void writeNull(); + virtual void writeNull() + { + stream << "null"; + } - virtual void writeInt(int i); + virtual void writeInt(int i) + { + stream << i; + } - virtual void writeUnsignedInt(unsigned int i); + virtual void writeUnsignedInt(unsigned int i) + { + stream << i; + } - virtual void writeLong(long int i); + virtual void writeLong(long int i) + { + stream << i; + } - virtual void writeUnsignedLong(unsigned long int i); + virtual void writeUnsignedLong(unsigned long int i) + { + stream << i; + } - virtual void writeLongLong(long long int i); + virtual void writeLongLong(long long int i) + { + stream << i; + } - virtual void writeUnsignedLongLong(unsigned long long int i); + virtual void writeUnsignedLongLong(unsigned long long int i) + { + stream << i; + } - virtual void writeFloat(float f); + virtual void writeFloat(float f) + { + if (std::isfinite(f)) + stream << f; + else stream << "\"" << f << "\""; + } - virtual void writeDouble(double d); + virtual void writeDouble(double d) + { + if (std::isfinite(d)) + stream << d; + else stream << "\"" << d << "\""; + } - virtual void writeString(const std::string & s); + virtual void writeString(const std::string & s) + { + stream << '\"'; + ML::jsonEscape(s, stream); + stream << '\"'; + } virtual void writeStringUtf8(const Utf8String & s); - virtual void writeJson(const Json::Value & val); + virtual void writeJson(const Json::Value & val) + { + stream << val.toStringNoNewLine(); + } + + virtual void writeBool(bool b) + { + stream << (b ? "true": "false"); + } - virtual void writeBool(bool b); }; @@ -135,49 +218,115 @@ struct StructuredJsonPrintingContext Json::Value output; Json::Value * current; - StructuredJsonPrintingContext(); + StructuredJsonPrintingContext() + : current(&output) + { + } std::vector path; - virtual void startObject(); + virtual void startObject() + { + *current = Json::Value(Json::objectValue); + path.push_back(current); + } - virtual void startMember(const std::string & memberName); + virtual void startMember(const std::string & memberName) + { + current = &(*path.back())[memberName]; + } - virtual void endObject(); + virtual void endObject() + { + path.pop_back(); + } - virtual void startArray(int knownSize = -1); + virtual void startArray(int knownSize = -1) + { + *current = Json::Value(Json::arrayValue); + path.push_back(current); + } - virtual void newArrayElement(); + virtual void newArrayElement() + { + Json::Value & b = *path.back(); + current = &b[b.size()]; + } - virtual void endArray(); + virtual void endArray() + { + path.pop_back(); + } - virtual void skip(); + virtual void skip() + { + *current = Json::Value(); + } - virtual void writeNull(); + virtual void writeNull() + { + *current = Json::Value(); + } - virtual void writeInt(int i); + virtual void writeInt(int i) + { + *current = i; + } - virtual void writeUnsignedInt(unsigned int i); + virtual void writeUnsignedInt(unsigned int i) + { + *current = i; + } - virtual void writeLong(long int i); + virtual void writeLong(long int i) + { + *current = i; + } - virtual void writeUnsignedLong(unsigned long int i); + virtual void writeUnsignedLong(unsigned long int i) + { + *current = i; + } - virtual void writeLongLong(long long int i); + virtual void writeLongLong(long long int i) + { + *current = i; + } - virtual void writeUnsignedLongLong(unsigned long long int i); + virtual void writeUnsignedLongLong(unsigned long long int i) + { + *current = i; + } - virtual void writeFloat(float f); + virtual void writeFloat(float f) + { + *current = f; + } - virtual void writeDouble(double d); + virtual void writeDouble(double d) + { + *current = d; + } - virtual void writeString(const std::string & s); + virtual void writeString(const std::string & s) + { + *current = s; + } - virtual void writeStringUtf8(const Utf8String & s); + virtual void writeStringUtf8(const Utf8String & s) + { + *current = s; + } - virtual void writeJson(const Json::Value & val); + virtual void writeJson(const Json::Value & val) + { + *current = val; + } - virtual void writeBool(bool b); + virtual void writeBool(bool b) + { + *current = b; + } }; } // namespace Datacratic diff --git a/soa/types/testing/id_test.cc b/soa/types/testing/id_test.cc index 35c92fb70..8fd8dde36 100644 --- a/soa/types/testing/id_test.cc +++ b/soa/types/testing/id_test.cc @@ -51,30 +51,11 @@ BOOST_AUTO_TEST_CASE( test_basic_id ) BOOST_AUTO_TEST_CASE( test_uuid_id ) { - // lower case string uuid = "0828398c-5965-11e0-84c8-0026b937c8e1"; Id id(uuid); BOOST_CHECK_EQUAL(id.type, Id::UUID); BOOST_CHECK_EQUAL(id.toString(), uuid); checkSerializeReconstitute(id); - - // upper case - string uuidCaps = "0828398C-5965-11E0-84C8-0026B937C8E1"; - Id idCaps(uuidCaps); - BOOST_CHECK_EQUAL(idCaps.type, Id::UUID_CAPS); - BOOST_CHECK_EQUAL(idCaps.toString(), uuidCaps); - checkSerializeReconstitute(idCaps); - BOOST_CHECK_NE(idCaps, id); - BOOST_CHECK_EQUAL(idCaps.hash(), id.hash()); - - // mixed case - string uuidMixed = "0828398C-5965-11e0-84c8-0026b937c8e1"; - Id idMixed(uuidMixed); - BOOST_CHECK_EQUAL(idMixed.type, Id::STR); - BOOST_CHECK_EQUAL(idMixed.toString(), uuidMixed); - checkSerializeReconstitute(idMixed); - BOOST_CHECK_NE(idMixed, id); - BOOST_CHECK_NE(idMixed, idCaps); } BOOST_AUTO_TEST_CASE( test_goog64_id ) @@ -348,3 +329,27 @@ BOOST_AUTO_TEST_CASE( test_compound_id ) { Id id(Id("hello"), Id("world")); } + +#if ID_HASH_AS_STRING +BOOST_AUTO_TEST_CASE( test_hash_as_string ) +{ + /* COMPOUND */ + { + Id typical(string("hello:big:world"), Id::STR); + Id id(Id("hello"), Id("big:world")); + BOOST_CHECK_EQUAL(typical.hash(), id.hash()); + } + + /* BIGDEC */ + { + Id typical(string("12345678"), Id::STR); + BOOST_CHECK_EQUAL(typical.type, Id::STR); + Id id(12345678); + BOOST_CHECK_EQUAL(typical.hash(), id.hash()); + + id = Id("12345678"); + BOOST_CHECK_EQUAL(id.type, Id::BIGDEC); + BOOST_CHECK_EQUAL(typical.hash(), id.hash()); + } +} +#endif diff --git a/soa/types/testing/string_test.cc b/soa/types/testing/string_test.cc index 2002c3a31..ae9a84701 100644 --- a/soa/types/testing/string_test.cc +++ b/soa/types/testing/string_test.cc @@ -9,7 +9,6 @@ #include #include #include "soa/jsoncpp/json.h" -#include "soa/types/dtoa.h" #include "jml/arch/format.h" using namespace std; @@ -118,21 +117,3 @@ BOOST_AUTO_TEST_CASE( test_u32_string ) BOOST_CHECK_EQUAL(plainAscii.extractAscii(), "Plain Ascii"); } - -BOOST_AUTO_TEST_CASE( test_basic_dtoa ) -{ - double value = 365.0; - BOOST_CHECK_EQUAL(dtoa(value) , "365"); - - value = 0.0; - BOOST_CHECK_EQUAL(dtoa(value) , "0"); - - value = 10.1; - BOOST_CHECK_EQUAL(dtoa(value) , "10.1"); - - value = -10.1; - BOOST_CHECK_EQUAL(dtoa(value) , "-10.1"); - - value = -1089000000000; - BOOST_CHECK_EQUAL(dtoa(value) , "-1.089e12"); -} diff --git a/soa/types/types.mk b/soa/types/types.mk index a898c28c7..374dc2833 100644 --- a/soa/types/types.mk +++ b/soa/types/types.mk @@ -11,8 +11,7 @@ LIBTYPES_SOURCES := \ id.cc \ url.cc \ periodic_utils.cc \ - csiphash.c \ - dtoa.c + csiphash.c LIBTYPES_LINK := \ boost_regex boost_date_time jsoncpp ACE db googleurl cityhash utils diff --git a/soa/utils/fixtures.h b/soa/utils/fixtures.h index b49e35af4..92d2b6931 100644 --- a/soa/utils/fixtures.h +++ b/soa/utils/fixtures.h @@ -54,15 +54,6 @@ struct TestFolderFixture _name_() : TestFolderFixture(#_name_) {} \ } -struct FileHolder -{ - std::string filename; - FileHolder(std::string filename) : filename(filename){} - ~FileHolder(){ - remove(filename.c_str()); - } -}; - } // namespace Datacratic diff --git a/soa/utils/mongo_init.h b/soa/utils/mongo_init.h new file mode 100644 index 000000000..98c9ab5bf --- /dev/null +++ b/soa/utils/mongo_init.h @@ -0,0 +1,33 @@ +/** + * mongo_init.h + * Mich, 2015-09-03 + * Copyright (c) 2015 Datacratic. All rights reserved. + **/ +#pragma once + +#include "mongo/bson/bson.h" +#include "mongo/util/net/hostandport.h" + + +using namespace mongo; + +namespace Datacratic { + bool _mongoInitialized; + +struct MongoAtInit { + + MongoAtInit() { + if (!_mongoInitialized) { + _mongoInitialized = true; + std::cerr << __FILE__ << ":" << __LINE__ << std::endl; + using mongo::client::initialize; + using mongo::client::Options; + auto status = initialize(); + if (!status.isOK()) { + throw ML::Exception("Mongo initialize failed"); + } + } + } +} atInit; + +} diff --git a/soa/valgrind.supp b/soa/valgrind.supp index 6d59a6c87..01059f4f0 100644 --- a/soa/valgrind.supp +++ b/soa/valgrind.supp @@ -48,6 +48,26 @@ ... } +{ + Logging::Category singletons + Memcheck:Leak + ... + fun:_ZN10Datacratic7Logging8CategoryC1EPKcS3_ + ... + fun:_dl_init + obj:/lib/x86_64-linux-gnu/ld-2.15.so +} + +{ + Logging::Category singletons + Memcheck:Leak + ... + fun:_ZN10Datacratic7Logging8CategoryC1EPKcb + ... + fun:_dl_init + obj:/lib/x86_64-linux-gnu/ld-2.15.so +} + { tcmalloc trips valgrind Memcheck:Param diff --git a/types/dtoa.c b/types/dtoa.c deleted file mode 100644 index ae3284f88..000000000 --- a/types/dtoa.c +++ /dev/null @@ -1,2787 +0,0 @@ -/**************************************************************** - * - * The author of this software is David M. Gay. - * - * Copyright (c) 1991, 2000, 2001 by Lucent Technologies. - * - * Permission to use, copy, modify, and distribute this software for any - * purpose without fee is hereby granted, provided that this entire notice - * is included in all copies of any software which is or includes a copy - * or modification of this software and in all copies of the supporting - * documentation for such software. - * - * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED - * WARRANTY. IN PARTICULAR, NEITHER THE AUTHOR NOR LUCENT MAKES ANY - * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE MERCHANTABILITY - * OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. - * - ***************************************************************/ - -/**************************************************************** - * This is dtoa.c by David M. Gay, downloaded from - * http://www.netlib.org/fp/dtoa.c on April 15, 2009 and modified for - * inclusion into the Python core by Mark E. T. Dickinson and Eric V. Smith. - * - * Please remember to check http://www.netlib.org/fp regularly (and especially - * before any Python release) for bugfixes and updates. - * - * The major modifications from Gay's original code are as follows: - * - * 0. The original code has been specialized to Python's needs by removing - * many of the #ifdef'd sections. In particular, code to support VAX and - * IBM floating-point formats, hex NaNs, hex floats, locale-aware - * treatment of the decimal point, and setting of the inexact flag have - * been removed. - * - * 1. We use PyMem_Malloc and PyMem_Free in place of malloc and free. - * - * 2. The public functions strtod, dtoa and freedtoa all now have - * a soa_ prefix. - * - * 3. Instead of assuming that PyMem_Malloc always succeeds, we thread - * PyMem_Malloc failures through the code. The functions - * - * Balloc, multadd, s2b, i2b, mult, pow5mult, lshift, diff, d2b - * - * of return type *Bigint all return NULL to indicate a malloc failure. - * Similarly, rv_alloc and nrv_alloc (return type char *) return NULL on - * failure. bigcomp now has return type int (it used to be void) and - * returns -1 on failure and 0 otherwise. soa_dtoa returns NULL - * on failure. soa_strtod indicates failure due to malloc failure - * by returning -1.0, setting errno=ENOMEM and *se to s00. - * - * 4. The static variable dtoa_result has been removed. Callers of - * soa_dtoa are expected to call soa_freedtoa to free - * the memory allocated by soa_dtoa. - * - * 5. The code has been reformatted to better fit with Python's - * C style guide (PEP 7). - * - * 6. A bug in the memory allocation has been fixed: to avoid FREEing memory - * that hasn't been MALLOC'ed, private_mem should only be used when k <= - * Kmax. - * - * 7. soa_strtod has been modified so that it doesn't accept strings with - * leading whitespace. - * - ***************************************************************/ - -/* Please send bug reports for the original dtoa.c code to David M. Gay (dmg - * at acm dot org, with " at " changed at "@" and " dot " changed to "."). - * Please report bugs for this modified version using the Python issue tracker - * (http://bugs.python.org). */ - -/* On a machine with IEEE extended-precision registers, it is - * necessary to specify double-precision (53-bit) rounding precision - * before invoking strtod or dtoa. If the machine uses (the equivalent - * of) Intel 80x87 arithmetic, the call - * _control87(PC_53, MCW_PC); - * does this with many compilers. Whether this or another call is - * appropriate depends on the compiler; for this to work, it may be - * necessary to #include "float.h" or another system-dependent header - * file. - */ - -/* strtod for IEEE-, VAX-, and IBM-arithmetic machines. - * - * This strtod returns a nearest machine number to the input decimal - * string (or sets errno to ERANGE). With IEEE arithmetic, ties are - * broken by the IEEE round-even rule. Otherwise ties are broken by - * biased rounding (add half and chop). - * - * Inspired loosely by William D. Clinger's paper "How to Read Floating - * Point Numbers Accurately" [Proc. ACM SIGPLAN '90, pp. 92-101]. - * - * Modifications: - * - * 1. We only require IEEE, IBM, or VAX double-precision - * arithmetic (not IEEE double-extended). - * 2. We get by with floating-point arithmetic in a case that - * Clinger missed -- when we're computing d * 10^n - * for a small integer d and the integer n is not too - * much larger than 22 (the maximum integer k for which - * we can represent 10^k exactly), we may be able to - * compute (d*10^k) * 10^(e-k) with just one roundoff. - * 3. Rather than a bit-at-a-time adjustment of the binary - * result in the hard case, we use floating-point - * arithmetic to determine the adjustment to within - * one bit; only in really hard cases do we need to - * compute a second residual. - * 4. Because of 3., we don't need a large table of powers of 10 - * for ten-to-e (just some small tables, e.g. of 10^k - * for 0 <= k <= 22). - */ - -#include -#include -#include -#include -#include -#include - -#define MALLOC malloc -#define FREE free -#define IEEE_8087 - -typedef uint32_t ULong; -typedef int32_t Long; -#define ULLong uint64_t - -//#define DEBUG - -#ifdef DEBUG -#define Bug(x) {fprintf(stderr, "%s\n", x); exit(1);} -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -typedef union { double d; ULong L[2]; } U; - -#ifdef IEEE_8087 -#define word0(x) (x)->L[1] -#define word1(x) (x)->L[0] -#else -#define word0(x) (x)->L[0] -#define word1(x) (x)->L[1] -#endif -#define dval(x) (x)->d - -#ifndef STRTOD_DIGLIM -#define STRTOD_DIGLIM 40 -#endif - -/* maximum permitted exponent value for strtod; exponents larger than - MAX_ABS_EXP in absolute value get truncated to +-MAX_ABS_EXP. MAX_ABS_EXP - should fit into an int. */ -#ifndef MAX_ABS_EXP -#define MAX_ABS_EXP 19999U -#endif - -/* The following definition of Storeinc is appropriate for MIPS processors. - * An alternative that might be better on some machines is - * #define Storeinc(a,b,c) (*a++ = b << 16 | c & 0xffff) - */ -#if defined(IEEE_8087) -#define Storeinc(a,b,c) (((unsigned short *)a)[1] = (unsigned short)b, \ - ((unsigned short *)a)[0] = (unsigned short)c, a++) -#else -#define Storeinc(a,b,c) (((unsigned short *)a)[0] = (unsigned short)b, \ - ((unsigned short *)a)[1] = (unsigned short)c, a++) -#endif - -/* #define P DBL_MANT_DIG */ -/* Ten_pmax = floor(P*log(2)/log(5)) */ -/* Bletch = (highest power of 2 < DBL_MAX_10_EXP) / 16 */ -/* Quick_max = floor((P-1)*log(FLT_RADIX)/log(10) - 1) */ -/* Int_max = floor(P*log(FLT_RADIX)/log(10) - 1) */ - -#define Exp_shift 20 -#define Exp_shift1 20 -#define Exp_msk1 0x100000 -#define Exp_msk11 0x100000 -#define Exp_mask 0x7ff00000 -#define P 53 -#define Nbits 53 -#define Bias 1023 -#define Emax 1023 -#define Emin (-1022) -#define Etiny (-1074) /* smallest denormal is 2**Etiny */ -#define Exp_1 0x3ff00000 -#define Exp_11 0x3ff00000 -#define Ebits 11 -#define Frac_mask 0xfffff -#define Frac_mask1 0xfffff -#define Ten_pmax 22 -#define Bletch 0x10 -#define Bndry_mask 0xfffff -#define Bndry_mask1 0xfffff -#define Sign_bit 0x80000000 -#define Log2P 1 -#define Tiny0 0 -#define Tiny1 1 -#define Quick_max 14 -#define Int_max 14 - -#ifndef Flt_Rounds -#ifdef FLT_ROUNDS -#define Flt_Rounds FLT_ROUNDS -#else -#define Flt_Rounds 1 -#endif -#endif /*Flt_Rounds*/ - -#define Rounding Flt_Rounds - -#define Big0 (Frac_mask1 | Exp_msk1*(DBL_MAX_EXP+Bias-1)) -#define Big1 0xffffffff - -/* struct BCinfo is used to pass information from soa_strtod to bigcomp */ - -typedef struct BCinfo BCinfo; -struct -BCinfo { - int e0, nd, nd0, scale; -}; - -#define FFFFFFFF 0xffffffffUL - -#define Kmax 7 - -/* struct Bigint is used to represent arbitrary-precision integers. These - integers are stored in sign-magnitude format, with the magnitude stored as - an array of base 2**32 digits. Bigints are always normalized: if x is a - Bigint then x->wds >= 1, and either x->wds == 1 or x[wds-1] is nonzero. - - The Bigint fields are as follows: - - - next is a header used by Balloc and Bfree to keep track of lists - of freed Bigints; it's also used for the linked list of - powers of 5 of the form 5**2**i used by pow5mult. - - k indicates which pool this Bigint was allocated from - - maxwds is the maximum number of words space was allocated for - (usually maxwds == 2**k) - - sign is 1 for negative Bigints, 0 for positive. The sign is unused - (ignored on inputs, set to 0 on outputs) in almost all operations - involving Bigints: a notable exception is the diff function, which - ignores signs on inputs but sets the sign of the output correctly. - - wds is the actual number of significant words - - x contains the vector of words (digits) for this Bigint, from least - significant (x[0]) to most significant (x[wds-1]). -*/ - -struct -Bigint { - struct Bigint *next; - int k, maxwds, sign, wds; - ULong x[1]; -}; - -typedef struct Bigint Bigint; - - /** Allocate a bigint */ - -static Bigint * -Balloc(int k) -{ - int x; - Bigint *rv; - unsigned int len; - - x = 1 << k; - len = (sizeof(Bigint) + (x-1)*sizeof(ULong) + sizeof(double) - 1) - /sizeof(double); - rv = (Bigint*)malloc(len*sizeof(double)); - if (rv == NULL) - return NULL; - rv->k = k; - rv->maxwds = x; - rv->sign = rv->wds = 0; - - return rv; -} - -/* Free a Bigint allocated with Balloc */ - -static void -Bfree(Bigint *v) -{ - free(v); -} - -#define Bcopy(x,y) memcpy((char *)&x->sign, (char *)&y->sign, \ - y->wds*sizeof(Long) + 2*sizeof(int)) - -/* Multiply a Bigint b by m and add a. Either modifies b in place and returns - a pointer to the modified b, or Bfrees b and returns a pointer to a copy. - On failure, return NULL. In this case, b will have been already freed. */ - -static Bigint * -multadd(Bigint *b, int m, int a) /* multiply by m and add a */ -{ - int i, wds; -#ifdef ULLong - ULong *x; - ULLong carry, y; -#else - ULong carry, *x, y; - ULong xi, z; -#endif - Bigint *b1; - - wds = b->wds; - x = b->x; - i = 0; - carry = a; - do { -#ifdef ULLong - y = *x * (ULLong)m + carry; - carry = y >> 32; - *x++ = (ULong)(y & FFFFFFFF); -#else - xi = *x; - y = (xi & 0xffff) * m + carry; - z = (xi >> 16) * m + (y >> 16); - carry = z >> 16; - *x++ = (z << 16) + (y & 0xffff); -#endif - } - while(++i < wds); - if (carry) { - if (wds >= b->maxwds) { - b1 = Balloc(b->k+1); - if (b1 == NULL){ - Bfree(b); - return NULL; - } - Bcopy(b1, b); - Bfree(b); - b = b1; - } - b->x[wds++] = (ULong)carry; - b->wds = wds; - } - return b; -} - -/* convert a string s containing nd decimal digits (possibly containing a - decimal separator at position nd0, which is ignored) to a Bigint. This - function carries on where the parsing code in soa_strtod leaves off: on - entry, y9 contains the result of converting the first 9 digits. Returns - NULL on failure. */ - -static Bigint * -s2b(const char *s, int nd0, int nd, ULong y9) -{ - Bigint *b; - int i, k; - Long x, y; - - x = (nd + 8) / 9; - for(k = 0, y = 1; x > y; y <<= 1, k++) ; - b = Balloc(k); - if (b == NULL) - return NULL; - b->x[0] = y9; - b->wds = 1; - - if (nd <= 9) - return b; - - s += 9; - for (i = 9; i < nd0; i++) { - b = multadd(b, 10, *s++ - '0'); - if (b == NULL) - return NULL; - } - s++; - for(; i < nd; i++) { - b = multadd(b, 10, *s++ - '0'); - if (b == NULL) - return NULL; - } - return b; -} - -/* count leading 0 bits in the 32-bit integer x. */ - -static int -hi0bits(ULong x) -{ - int k = 0; - - if (!(x & 0xffff0000)) { - k = 16; - x <<= 16; - } - if (!(x & 0xff000000)) { - k += 8; - x <<= 8; - } - if (!(x & 0xf0000000)) { - k += 4; - x <<= 4; - } - if (!(x & 0xc0000000)) { - k += 2; - x <<= 2; - } - if (!(x & 0x80000000)) { - k++; - if (!(x & 0x40000000)) - return 32; - } - return k; -} - -/* count trailing 0 bits in the 32-bit integer y, and shift y right by that - number of bits. */ - -static int -lo0bits(ULong *y) -{ - int k; - ULong x = *y; - - if (x & 7) { - if (x & 1) - return 0; - if (x & 2) { - *y = x >> 1; - return 1; - } - *y = x >> 2; - return 2; - } - k = 0; - if (!(x & 0xffff)) { - k = 16; - x >>= 16; - } - if (!(x & 0xff)) { - k += 8; - x >>= 8; - } - if (!(x & 0xf)) { - k += 4; - x >>= 4; - } - if (!(x & 0x3)) { - k += 2; - x >>= 2; - } - if (!(x & 1)) { - k++; - x >>= 1; - if (!x) - return 32; - } - *y = x; - return k; -} - -/* convert a small nonnegative integer to a Bigint */ - -static Bigint * -i2b(int i) -{ - Bigint *b; - - b = Balloc(1); - if (b == NULL) - return NULL; - b->x[0] = i; - b->wds = 1; - return b; -} - -/* multiply two Bigints. Returns a new Bigint, or NULL on failure. Ignores - the signs of a and b. */ - -static Bigint * -mult(Bigint *a, Bigint *b) -{ - Bigint *c; - int k, wa, wb, wc; - ULong *x, *xa, *xae, *xb, *xbe, *xc, *xc0; - ULong y; -#ifdef ULLong - ULLong carry, z; -#else - ULong carry, z; - ULong z2; -#endif - - if ((!a->x[0] && a->wds == 1) || (!b->x[0] && b->wds == 1)) { - c = Balloc(0); - if (c == NULL) - return NULL; - c->wds = 1; - c->x[0] = 0; - return c; - } - - if (a->wds < b->wds) { - c = a; - a = b; - b = c; - } - k = a->k; - wa = a->wds; - wb = b->wds; - wc = wa + wb; - if (wc > a->maxwds) - k++; - c = Balloc(k); - if (c == NULL) - return NULL; - for(x = c->x, xa = x + wc; x < xa; x++) - *x = 0; - xa = a->x; - xae = xa + wa; - xb = b->x; - xbe = xb + wb; - xc0 = c->x; -#ifdef ULLong - for(; xb < xbe; xc0++) { - if ((y = *xb++)) { - x = xa; - xc = xc0; - carry = 0; - do { - z = *x++ * (ULLong)y + *xc + carry; - carry = z >> 32; - *xc++ = (ULong)(z & FFFFFFFF); - } - while(x < xae); - *xc = (ULong)carry; - } - } -#else - for(; xb < xbe; xb++, xc0++) { - if (y = *xb & 0xffff) { - x = xa; - xc = xc0; - carry = 0; - do { - z = (*x & 0xffff) * y + (*xc & 0xffff) + carry; - carry = z >> 16; - z2 = (*x++ >> 16) * y + (*xc >> 16) + carry; - carry = z2 >> 16; - Storeinc(xc, z2, z); - } - while(x < xae); - *xc = carry; - } - if (y = *xb >> 16) { - x = xa; - xc = xc0; - carry = 0; - z2 = *xc; - do { - z = (*x & 0xffff) * y + (*xc >> 16) + carry; - carry = z >> 16; - Storeinc(xc, z, z2); - z2 = (*x++ >> 16) * y + (*xc & 0xffff) + carry; - carry = z2 >> 16; - } - while(x < xae); - *xc = z2; - } - } -#endif - for(xc0 = c->x, xc = xc0 + wc; wc > 0 && !*--xc; --wc) ; - c->wds = wc; - return c; -} - -#ifndef Py_USING_MEMORY_DEBUGGER - -/* p5s is a linked list of powers of 5 of the form 5**(2**i), i >= 2 */ - -static Bigint *p5s; - -/* multiply the Bigint b by 5**k. Returns a pointer to the result, or NULL on - failure; if the returned pointer is distinct from b then the original - Bigint b will have been Bfree'd. Ignores the sign of b. */ - -static Bigint * -pow5mult(Bigint *b, int k) -{ - Bigint *b1, *p5, *p51; - int i; - static int p05[3] = { 5, 25, 125 }; - - if ((i = k & 3)) { - b = multadd(b, p05[i-1], 0); - if (b == NULL) - return NULL; - } - - if (!(k >>= 2)) - return b; - p5 = p5s; - if (!p5) { - /* first time */ - p5 = i2b(625); - if (p5 == NULL) { - Bfree(b); - return NULL; - } - p5s = p5; - p5->next = 0; - } - for(;;) { - if (k & 1) { - b1 = mult(b, p5); - Bfree(b); - b = b1; - if (b == NULL) - return NULL; - } - if (!(k >>= 1)) - break; - p51 = p5->next; - if (!p51) { - p51 = mult(p5,p5); - if (p51 == NULL) { - Bfree(b); - return NULL; - } - p51->next = 0; - p5->next = p51; - } - p5 = p51; - } - return b; -} - -#else - -/* Version of pow5mult that doesn't cache powers of 5. Provided for - the benefit of memory debugging tools like Valgrind. */ - -static Bigint * -pow5mult(Bigint *b, int k) -{ - Bigint *b1, *p5, *p51; - int i; - static int p05[3] = { 5, 25, 125 }; - - if ((i = k & 3)) { - b = multadd(b, p05[i-1], 0); - if (b == NULL) - return NULL; - } - - if (!(k >>= 2)) - return b; - p5 = i2b(625); - if (p5 == NULL) { - Bfree(b); - return NULL; - } - - for(;;) { - if (k & 1) { - b1 = mult(b, p5); - Bfree(b); - b = b1; - if (b == NULL) { - Bfree(p5); - return NULL; - } - } - if (!(k >>= 1)) - break; - p51 = mult(p5, p5); - Bfree(p5); - p5 = p51; - if (p5 == NULL) { - Bfree(b); - return NULL; - } - } - Bfree(p5); - return b; -} - -#endif /* Py_USING_MEMORY_DEBUGGER */ - -/* shift a Bigint b left by k bits. Return a pointer to the shifted result, - or NULL on failure. If the returned pointer is distinct from b then the - original b will have been Bfree'd. Ignores the sign of b. */ - -static Bigint * -lshift(Bigint *b, int k) -{ - int i, k1, n, n1; - Bigint *b1; - ULong *x, *x1, *xe, z; - - if (!k || (!b->x[0] && b->wds == 1)) - return b; - - n = k >> 5; - k1 = b->k; - n1 = n + b->wds + 1; - for(i = b->maxwds; n1 > i; i <<= 1) - k1++; - b1 = Balloc(k1); - if (b1 == NULL) { - Bfree(b); - return NULL; - } - x1 = b1->x; - for(i = 0; i < n; i++) - *x1++ = 0; - x = b->x; - xe = x + b->wds; - if (k &= 0x1f) { - k1 = 32 - k; - z = 0; - do { - *x1++ = *x << k | z; - z = *x++ >> k1; - } - while(x < xe); - if ((*x1 = z)) - ++n1; - } - else do - *x1++ = *x++; - while(x < xe); - b1->wds = n1 - 1; - Bfree(b); - return b1; -} - -/* Do a three-way compare of a and b, returning -1 if a < b, 0 if a == b and - 1 if a > b. Ignores signs of a and b. */ - -static int -cmp(Bigint *a, Bigint *b) -{ - ULong *xa, *xa0, *xb, *xb0; - int i, j; - - i = a->wds; - j = b->wds; -#ifdef DEBUG - if (i > 1 && !a->x[i-1]) - Bug("cmp called with a->x[a->wds-1] == 0"); - if (j > 1 && !b->x[j-1]) - Bug("cmp called with b->x[b->wds-1] == 0"); -#endif - if (i -= j) - return i; - xa0 = a->x; - xa = xa0 + j; - xb0 = b->x; - xb = xb0 + j; - for(;;) { - if (*--xa != *--xb) - return *xa < *xb ? -1 : 1; - if (xa <= xa0) - break; - } - return 0; -} - -/* Take the difference of Bigints a and b, returning a new Bigint. Returns - NULL on failure. The signs of a and b are ignored, but the sign of the - result is set appropriately. */ - -static Bigint * -diff(Bigint *a, Bigint *b) -{ - Bigint *c; - int i, wa, wb; - ULong *xa, *xae, *xb, *xbe, *xc; -#ifdef ULLong - ULLong borrow, y; -#else - ULong borrow, y; - ULong z; -#endif - - i = cmp(a,b); - if (!i) { - c = Balloc(0); - if (c == NULL) - return NULL; - c->wds = 1; - c->x[0] = 0; - return c; - } - if (i < 0) { - c = a; - a = b; - b = c; - i = 1; - } - else - i = 0; - c = Balloc(a->k); - if (c == NULL) - return NULL; - c->sign = i; - wa = a->wds; - xa = a->x; - xae = xa + wa; - wb = b->wds; - xb = b->x; - xbe = xb + wb; - xc = c->x; - borrow = 0; -#ifdef ULLong - do { - y = (ULLong)*xa++ - *xb++ - borrow; - borrow = y >> 32 & (ULong)1; - *xc++ = (ULong)(y & FFFFFFFF); - } - while(xb < xbe); - while(xa < xae) { - y = *xa++ - borrow; - borrow = y >> 32 & (ULong)1; - *xc++ = (ULong)(y & FFFFFFFF); - } -#else - do { - y = (*xa & 0xffff) - (*xb & 0xffff) - borrow; - borrow = (y & 0x10000) >> 16; - z = (*xa++ >> 16) - (*xb++ >> 16) - borrow; - borrow = (z & 0x10000) >> 16; - Storeinc(xc, z, y); - } - while(xb < xbe); - while(xa < xae) { - y = (*xa & 0xffff) - borrow; - borrow = (y & 0x10000) >> 16; - z = (*xa++ >> 16) - borrow; - borrow = (z & 0x10000) >> 16; - Storeinc(xc, z, y); - } -#endif - while(!*--xc) - wa--; - c->wds = wa; - return c; -} - -/* Given a positive normal double x, return the difference between x and the - next double up. Doesn't give correct results for subnormals. */ - -static double -ulp(U *x) -{ - Long L; - U u; - - L = (word0(x) & Exp_mask) - (P-1)*Exp_msk1; - word0(&u) = L; - word1(&u) = 0; - return dval(&u); -} - -/* Convert a Bigint to a double plus an exponent */ - -static double -b2d(Bigint *a, int *e) -{ - ULong *xa, *xa0, w, y, z; - int k; - U d; - - xa0 = a->x; - xa = xa0 + a->wds; - y = *--xa; -#ifdef DEBUG - if (!y) Bug("zero y in b2d"); -#endif - k = hi0bits(y); - *e = 32 - k; - if (k < Ebits) { - word0(&d) = Exp_1 | y >> (Ebits - k); - w = xa > xa0 ? *--xa : 0; - word1(&d) = y << ((32-Ebits) + k) | w >> (Ebits - k); - goto ret_d; - } - z = xa > xa0 ? *--xa : 0; - if (k -= Ebits) { - word0(&d) = Exp_1 | y << k | z >> (32 - k); - y = xa > xa0 ? *--xa : 0; - word1(&d) = z << k | y >> (32 - k); - } - else { - word0(&d) = Exp_1 | y; - word1(&d) = z; - } - ret_d: - return dval(&d); -} - -/* Convert a scaled double to a Bigint plus an exponent. Similar to d2b, - except that it accepts the scale parameter used in soa_strtod (which - should be either 0 or 2*P), and the normalization for the return value is - different (see below). On input, d should be finite and nonnegative, and d - / 2**scale should be exactly representable as an IEEE 754 double. - - Returns a Bigint b and an integer e such that - - dval(d) / 2**scale = b * 2**e. - - Unlike d2b, b is not necessarily odd: b and e are normalized so - that either 2**(P-1) <= b < 2**P and e >= Etiny, or b < 2**P - and e == Etiny. This applies equally to an input of 0.0: in that - case the return values are b = 0 and e = Etiny. - - The above normalization ensures that for all possible inputs d, - 2**e gives ulp(d/2**scale). - - Returns NULL on failure. -*/ - -static Bigint * -sd2b(U *d, int scale, int *e) -{ - Bigint *b; - - b = Balloc(1); - if (b == NULL) - return NULL; - - /* First construct b and e assuming that scale == 0. */ - b->wds = 2; - b->x[0] = word1(d); - b->x[1] = word0(d) & Frac_mask; - *e = Etiny - 1 + (int)((word0(d) & Exp_mask) >> Exp_shift); - if (*e < Etiny) - *e = Etiny; - else - b->x[1] |= Exp_msk1; - - /* Now adjust for scale, provided that b != 0. */ - if (scale && (b->x[0] || b->x[1])) { - *e -= scale; - if (*e < Etiny) { - scale = Etiny - *e; - *e = Etiny; - /* We can't shift more than P-1 bits without shifting out a 1. */ - assert(0 < scale && scale <= P - 1); - if (scale >= 32) { - /* The bits shifted out should all be zero. */ - assert(b->x[0] == 0); - b->x[0] = b->x[1]; - b->x[1] = 0; - scale -= 32; - } - if (scale) { - /* The bits shifted out should all be zero. */ - assert(b->x[0] << (32 - scale) == 0); - b->x[0] = (b->x[0] >> scale) | (b->x[1] << (32 - scale)); - b->x[1] >>= scale; - } - } - } - /* Ensure b is normalized. */ - if (!b->x[1]) - b->wds = 1; - - return b; -} - -/* Convert a double to a Bigint plus an exponent. Return NULL on failure. - - Given a finite nonzero double d, return an odd Bigint b and exponent *e - such that fabs(d) = b * 2**e. On return, *bbits gives the number of - significant bits of b; that is, 2**(*bbits-1) <= b < 2**(*bbits). - - If d is zero, then b == 0, *e == -1010, *bbits = 0. - */ - -static Bigint * -d2b(U *d, int *e, int *bits) -{ - Bigint *b; - int de, k; - ULong *x, y, z; - int i; - - b = Balloc(1); - if (b == NULL) - return NULL; - x = b->x; - - z = word0(d) & Frac_mask; - word0(d) &= 0x7fffffff; /* clear sign bit, which we ignore */ - if ((de = (int)(word0(d) >> Exp_shift))) - z |= Exp_msk1; - if ((y = word1(d))) { - if ((k = lo0bits(&y))) { - x[0] = y | z << (32 - k); - z >>= k; - } - else - x[0] = y; - i = - b->wds = (x[1] = z) ? 2 : 1; - } - else { - k = lo0bits(&z); - x[0] = z; - i = - b->wds = 1; - k += 32; - } - if (de) { - *e = de - Bias - (P-1) + k; - *bits = P - k; - } - else { - *e = de - Bias - (P-1) + 1 + k; - *bits = 32*i - hi0bits(x[i-1]); - } - return b; -} - -/* Compute the ratio of two Bigints, as a double. The result may have an - error of up to 2.5 ulps. */ - -static double -ratio(Bigint *a, Bigint *b) -{ - U da, db; - int k, ka, kb; - - dval(&da) = b2d(a, &ka); - dval(&db) = b2d(b, &kb); - k = ka - kb + 32*(a->wds - b->wds); - if (k > 0) - word0(&da) += k*Exp_msk1; - else { - k = -k; - word0(&db) += k*Exp_msk1; - } - return dval(&da) / dval(&db); -} - -static const double -tens[] = { - 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, - 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, - 1e20, 1e21, 1e22 -}; - -static const double -bigtens[] = { 1e16, 1e32, 1e64, 1e128, 1e256 }; -static const double tinytens[] = { 1e-16, 1e-32, 1e-64, 1e-128, - 9007199254740992.*9007199254740992.e-256 - /* = 2^106 * 1e-256 */ -}; -/* The factor of 2^53 in tinytens[4] helps us avoid setting the underflow */ -/* flag unnecessarily. It leads to a song and dance at the end of strtod. */ -#define Scale_Bit 0x10 -#define n_bigtens 5 - -#define ULbits 32 -#define kshift 5 -#define kmask 31 - - -static int -dshift(Bigint *b, int p2) -{ - int rv = hi0bits(b->x[b->wds-1]) - 4; - if (p2 > 0) - rv -= p2; - return rv & kmask; -} - -/* special case of Bigint division. The quotient is always in the range 0 <= - quotient < 10, and on entry the divisor S is normalized so that its top 4 - bits (28--31) are zero and bit 27 is set. */ - -static int -quorem(Bigint *b, Bigint *S) -{ - int n; - ULong *bx, *bxe, q, *sx, *sxe; -#ifdef ULLong - ULLong borrow, carry, y, ys; -#else - ULong borrow, carry, y, ys; - ULong si, z, zs; -#endif - - n = S->wds; -#ifdef DEBUG - /*debug*/ if (b->wds > n) - /*debug*/ Bug("oversize b in quorem"); -#endif - if (b->wds < n) - return 0; - sx = S->x; - sxe = sx + --n; - bx = b->x; - bxe = bx + n; - q = *bxe / (*sxe + 1); /* ensure q <= true quotient */ -#ifdef DEBUG - /*debug*/ if (q > 9) - /*debug*/ Bug("oversized quotient in quorem"); -#endif - if (q) { - borrow = 0; - carry = 0; - do { -#ifdef ULLong - ys = *sx++ * (ULLong)q + carry; - carry = ys >> 32; - y = *bx - (ys & FFFFFFFF) - borrow; - borrow = y >> 32 & (ULong)1; - *bx++ = (ULong)(y & FFFFFFFF); -#else - si = *sx++; - ys = (si & 0xffff) * q + carry; - zs = (si >> 16) * q + (ys >> 16); - carry = zs >> 16; - y = (*bx & 0xffff) - (ys & 0xffff) - borrow; - borrow = (y & 0x10000) >> 16; - z = (*bx >> 16) - (zs & 0xffff) - borrow; - borrow = (z & 0x10000) >> 16; - Storeinc(bx, z, y); -#endif - } - while(sx <= sxe); - if (!*bxe) { - bx = b->x; - while(--bxe > bx && !*bxe) - --n; - b->wds = n; - } - } - if (cmp(b, S) >= 0) { - q++; - borrow = 0; - carry = 0; - bx = b->x; - sx = S->x; - do { -#ifdef ULLong - ys = *sx++ + carry; - carry = ys >> 32; - y = *bx - (ys & FFFFFFFF) - borrow; - borrow = y >> 32 & (ULong)1; - *bx++ = (ULong)(y & FFFFFFFF); -#else - si = *sx++; - ys = (si & 0xffff) + carry; - zs = (si >> 16) + (ys >> 16); - carry = zs >> 16; - y = (*bx & 0xffff) - (ys & 0xffff) - borrow; - borrow = (y & 0x10000) >> 16; - z = (*bx >> 16) - (zs & 0xffff) - borrow; - borrow = (z & 0x10000) >> 16; - Storeinc(bx, z, y); -#endif - } - while(sx <= sxe); - bx = b->x; - bxe = bx + n; - if (!*bxe) { - while(--bxe > bx && !*bxe) - --n; - b->wds = n; - } - } - return q; -} - -/* sulp(x) is a version of ulp(x) that takes bc.scale into account. - - Assuming that x is finite and nonnegative (positive zero is fine - here) and x / 2^bc.scale is exactly representable as a double, - sulp(x) is equivalent to 2^bc.scale * ulp(x / 2^bc.scale). */ - -static double -sulp(U *x, BCinfo *bc) -{ - U u; - - if (bc->scale && 2*P + 1 > (int)((word0(x) & Exp_mask) >> Exp_shift)) { - /* rv/2^bc->scale is subnormal */ - word0(&u) = (P+2)*Exp_msk1; - word1(&u) = 0; - return u.d; - } - else { - assert(word0(x) || word1(x)); /* x != 0.0 */ - return ulp(x); - } -} - -/* The bigcomp function handles some hard cases for strtod, for inputs - with more than STRTOD_DIGLIM digits. It's called once an initial - estimate for the double corresponding to the input string has - already been obtained by the code in soa_strtod. - - The bigcomp function is only called after soa_strtod has found a - double value rv such that either rv or rv + 1ulp represents the - correctly rounded value corresponding to the original string. It - determines which of these two values is the correct one by - computing the decimal digits of rv + 0.5ulp and comparing them with - the corresponding digits of s0. - - In the following, write dv for the absolute value of the number represented - by the input string. - - Inputs: - - s0 points to the first significant digit of the input string. - - rv is a (possibly scaled) estimate for the closest double value to the - value represented by the original input to soa_strtod. If - bc->scale is nonzero, then rv/2^(bc->scale) is the approximation to - the input value. - - bc is a struct containing information gathered during the parsing and - estimation steps of soa_strtod. Description of fields follows: - - bc->e0 gives the exponent of the input value, such that dv = (integer - given by the bd->nd digits of s0) * 10**e0 - - bc->nd gives the total number of significant digits of s0. It will - be at least 1. - - bc->nd0 gives the number of significant digits of s0 before the - decimal separator. If there's no decimal separator, bc->nd0 == - bc->nd. - - bc->scale is the value used to scale rv to avoid doing arithmetic with - subnormal values. It's either 0 or 2*P (=106). - - Outputs: - - On successful exit, rv/2^(bc->scale) is the closest double to dv. - - Returns 0 on success, -1 on failure (e.g., due to a failed malloc call). */ - -static int -bigcomp(U *rv, const char *s0, BCinfo *bc) -{ - Bigint *b, *d; - int b2, d2, dd, i, nd, nd0, odd, p2, p5; - - nd = bc->nd; - nd0 = bc->nd0; - p5 = nd + bc->e0; - b = sd2b(rv, bc->scale, &p2); - if (b == NULL) - return -1; - - /* record whether the lsb of rv/2^(bc->scale) is odd: in the exact halfway - case, this is used for round to even. */ - odd = b->x[0] & 1; - - /* left shift b by 1 bit and or a 1 into the least significant bit; - this gives us b * 2**p2 = rv/2^(bc->scale) + 0.5 ulp. */ - b = lshift(b, 1); - if (b == NULL) - return -1; - b->x[0] |= 1; - p2--; - - p2 -= p5; - d = i2b(1); - if (d == NULL) { - Bfree(b); - return -1; - } - /* Arrange for convenient computation of quotients: - * shift left if necessary so divisor has 4 leading 0 bits. - */ - if (p5 > 0) { - d = pow5mult(d, p5); - if (d == NULL) { - Bfree(b); - return -1; - } - } - else if (p5 < 0) { - b = pow5mult(b, -p5); - if (b == NULL) { - Bfree(d); - return -1; - } - } - if (p2 > 0) { - b2 = p2; - d2 = 0; - } - else { - b2 = 0; - d2 = -p2; - } - i = dshift(d, d2); - if ((b2 += i) > 0) { - b = lshift(b, b2); - if (b == NULL) { - Bfree(d); - return -1; - } - } - if ((d2 += i) > 0) { - d = lshift(d, d2); - if (d == NULL) { - Bfree(b); - return -1; - } - } - - /* Compare s0 with b/d: set dd to -1, 0, or 1 according as s0 < b/d, s0 == - * b/d, or s0 > b/d. Here the digits of s0 are thought of as representing - * a number in the range [0.1, 1). */ - if (cmp(b, d) >= 0) - /* b/d >= 1 */ - dd = -1; - else { - i = 0; - for(;;) { - b = multadd(b, 10, 0); - if (b == NULL) { - Bfree(d); - return -1; - } - dd = s0[i < nd0 ? i : i+1] - '0' - quorem(b, d); - i++; - - if (dd) - break; - if (!b->x[0] && b->wds == 1) { - /* b/d == 0 */ - dd = i < nd; - break; - } - if (!(i < nd)) { - /* b/d != 0, but digits of s0 exhausted */ - dd = -1; - break; - } - } - } - Bfree(b); - Bfree(d); - if (dd > 0 || (dd == 0 && odd)) - dval(rv) += sulp(rv, bc); - return 0; -} - -double -soa_strtod(const char *s00, char **se) -{ - int bb2, bb5, bbe, bd2, bd5, bs2, c, dsign, e, e1, error; - int esign, i, j, k, lz, nd, nd0, odd, sign; - const char *s, *s0, *s1; - double aadj, aadj1; - U aadj2, adj, rv, rv0; - ULong y, z, abs_exp; - Long L; - BCinfo bc; - Bigint *bb, *bb1, *bd, *bd0, *bs, *delta; - - dval(&rv) = 0.; - - /* Start parsing. */ - c = *(s = s00); - - /* Parse optional sign, if present. */ - sign = 0; - switch (c) { - case '-': - sign = 1; - /* no break */ - case '+': - c = *++s; - } - - /* Skip leading zeros: lz is true iff there were leading zeros. */ - s1 = s; - while (c == '0') - c = *++s; - lz = s != s1; - - /* Point s0 at the first nonzero digit (if any). nd0 will be the position - of the point relative to s0. nd will be the total number of digits - ignoring leading zeros. */ - s0 = s1 = s; - while ('0' <= c && c <= '9') - c = *++s; - nd0 = nd = s - s1; - - /* Parse decimal point and following digits. */ - if (c == '.') { - c = *++s; - if (!nd) { - s1 = s; - while (c == '0') - c = *++s; - lz = lz || s != s1; - nd0 -= s - s1; - s0 = s; - } - s1 = s; - while ('0' <= c && c <= '9') - c = *++s; - nd += s - s1; - } - - /* Now lz is true if and only if there were leading zero digits, and nd - gives the total number of digits ignoring leading zeros. A valid input - must have at least one digit. */ - if (!nd && !lz) { - if (se) - *se = (char *)s00; - goto parse_error; - } - - /* Parse exponent. */ - e = 0; - if (c == 'e' || c == 'E') { - s00 = s; - c = *++s; - - /* Exponent sign. */ - esign = 0; - switch (c) { - case '-': - esign = 1; - /* no break */ - case '+': - c = *++s; - } - - /* Skip zeros. lz is true iff there are leading zeros. */ - s1 = s; - while (c == '0') - c = *++s; - lz = s != s1; - - /* Get absolute value of the exponent. */ - s1 = s; - abs_exp = 0; - while ('0' <= c && c <= '9') { - abs_exp = 10*abs_exp + (c - '0'); - c = *++s; - } - - /* abs_exp will be correct modulo 2**32. But 10**9 < 2**32, so if - there are at most 9 significant exponent digits then overflow is - impossible. */ - if (s - s1 > 9 || abs_exp > MAX_ABS_EXP) - e = (int)MAX_ABS_EXP; - else - e = (int)abs_exp; - if (esign) - e = -e; - - /* A valid exponent must have at least one digit. */ - if (s == s1 && !lz) - s = s00; - } - - /* Adjust exponent to take into account position of the point. */ - e -= nd - nd0; - if (nd0 <= 0) - nd0 = nd; - - /* Finished parsing. Set se to indicate how far we parsed */ - if (se) - *se = (char *)s; - - /* If all digits were zero, exit with return value +-0.0. Otherwise, - strip trailing zeros: scan back until we hit a nonzero digit. */ - if (!nd) - goto ret; - for (i = nd; i > 0; ) { - --i; - if (s0[i < nd0 ? i : i+1] != '0') { - ++i; - break; - } - } - e += nd - i; - nd = i; - if (nd0 > nd) - nd0 = nd; - - /* Summary of parsing results. After parsing, and dealing with zero - * inputs, we have values s0, nd0, nd, e, sign, where: - * - * - s0 points to the first significant digit of the input string - * - * - nd is the total number of significant digits (here, and - * below, 'significant digits' means the set of digits of the - * significand of the input that remain after ignoring leading - * and trailing zeros). - * - * - nd0 indicates the position of the decimal point, if present; it - * satisfies 1 <= nd0 <= nd. The nd significant digits are in - * s0[0:nd0] and s0[nd0+1:nd+1] using the usual Python half-open slice - * notation. (If nd0 < nd, then s0[nd0] contains a '.' character; if - * nd0 == nd, then s0[nd0] could be any non-digit character.) - * - * - e is the adjusted exponent: the absolute value of the number - * represented by the original input string is n * 10**e, where - * n is the integer represented by the concatenation of - * s0[0:nd0] and s0[nd0+1:nd+1] - * - * - sign gives the sign of the input: 1 for negative, 0 for positive - * - * - the first and last significant digits are nonzero - */ - - /* put first DBL_DIG+1 digits into integer y and z. - * - * - y contains the value represented by the first min(9, nd) - * significant digits - * - * - if nd > 9, z contains the value represented by significant digits - * with indices in [9, min(16, nd)). So y * 10**(min(16, nd) - 9) + z - * gives the value represented by the first min(16, nd) sig. digits. - */ - - bc.e0 = e1 = e; - y = z = 0; - for (i = 0; i < nd; i++) { - if (i < 9) - y = 10*y + s0[i < nd0 ? i : i+1] - '0'; - else if (i < DBL_DIG+1) - z = 10*z + s0[i < nd0 ? i : i+1] - '0'; - else - break; - } - - k = nd < DBL_DIG + 1 ? nd : DBL_DIG + 1; - dval(&rv) = y; - if (k > 9) { - dval(&rv) = tens[k - 9] * dval(&rv) + z; - } - bd0 = 0; - if (nd <= DBL_DIG - && Flt_Rounds == 1 - ) { - if (!e) - goto ret; - if (e > 0) { - if (e <= Ten_pmax) { - dval(&rv) *= tens[e]; - goto ret; - } - i = DBL_DIG - nd; - if (e <= Ten_pmax + i) { - /* A fancier test would sometimes let us do - * this for larger i values. - */ - e -= i; - dval(&rv) *= tens[i]; - dval(&rv) *= tens[e]; - goto ret; - } - } - else if (e >= -Ten_pmax) { - dval(&rv) /= tens[-e]; - goto ret; - } - } - e1 += nd - k; - - bc.scale = 0; - - /* Get starting approximation = rv * 10**e1 */ - - if (e1 > 0) { - if ((i = e1 & 15)) - dval(&rv) *= tens[i]; - if (e1 &= ~15) { - if (e1 > DBL_MAX_10_EXP) - goto ovfl; - e1 >>= 4; - for(j = 0; e1 > 1; j++, e1 >>= 1) - if (e1 & 1) - dval(&rv) *= bigtens[j]; - /* The last multiplication could overflow. */ - word0(&rv) -= P*Exp_msk1; - dval(&rv) *= bigtens[j]; - if ((z = word0(&rv) & Exp_mask) - > Exp_msk1*(DBL_MAX_EXP+Bias-P)) - goto ovfl; - if (z > Exp_msk1*(DBL_MAX_EXP+Bias-1-P)) { - /* set to largest number */ - /* (Can't trust DBL_MAX) */ - word0(&rv) = Big0; - word1(&rv) = Big1; - } - else - word0(&rv) += P*Exp_msk1; - } - } - else if (e1 < 0) { - /* The input decimal value lies in [10**e1, 10**(e1+16)). - - If e1 <= -512, underflow immediately. - If e1 <= -256, set bc.scale to 2*P. - - So for input value < 1e-256, bc.scale is always set; - for input value >= 1e-240, bc.scale is never set. - For input values in [1e-256, 1e-240), bc.scale may or may - not be set. */ - - e1 = -e1; - if ((i = e1 & 15)) - dval(&rv) /= tens[i]; - if (e1 >>= 4) { - if (e1 >= 1 << n_bigtens) - goto undfl; - if (e1 & Scale_Bit) - bc.scale = 2*P; - for(j = 0; e1 > 0; j++, e1 >>= 1) - if (e1 & 1) - dval(&rv) *= tinytens[j]; - if (bc.scale && (j = 2*P + 1 - ((word0(&rv) & Exp_mask) - >> Exp_shift)) > 0) { - /* scaled rv is denormal; clear j low bits */ - if (j >= 32) { - word1(&rv) = 0; - if (j >= 53) - word0(&rv) = (P+2)*Exp_msk1; - else - word0(&rv) &= 0xffffffff << (j-32); - } - else - word1(&rv) &= 0xffffffff << j; - } - if (!dval(&rv)) - goto undfl; - } - } - - /* Now the hard part -- adjusting rv to the correct value.*/ - - /* Put digits into bd: true value = bd * 10^e */ - - bc.nd = nd; - bc.nd0 = nd0; /* Only needed if nd > STRTOD_DIGLIM, but done here */ - /* to silence an erroneous warning about bc.nd0 */ - /* possibly not being initialized. */ - if (nd > STRTOD_DIGLIM) { - /* ASSERT(STRTOD_DIGLIM >= 18); 18 == one more than the */ - /* minimum number of decimal digits to distinguish double values */ - /* in IEEE arithmetic. */ - - /* Truncate input to 18 significant digits, then discard any trailing - zeros on the result by updating nd, nd0, e and y suitably. (There's - no need to update z; it's not reused beyond this point.) */ - for (i = 18; i > 0; ) { - /* scan back until we hit a nonzero digit. significant digit 'i' - is s0[i] if i < nd0, s0[i+1] if i >= nd0. */ - --i; - if (s0[i < nd0 ? i : i+1] != '0') { - ++i; - break; - } - } - e += nd - i; - nd = i; - if (nd0 > nd) - nd0 = nd; - if (nd < 9) { /* must recompute y */ - y = 0; - for(i = 0; i < nd0; ++i) - y = 10*y + s0[i] - '0'; - for(; i < nd; ++i) - y = 10*y + s0[i+1] - '0'; - } - } - bd0 = s2b(s0, nd0, nd, y); - if (bd0 == NULL) - goto failed_malloc; - - /* Notation for the comments below. Write: - - - dv for the absolute value of the number represented by the original - decimal input string. - - - if we've truncated dv, write tdv for the truncated value. - Otherwise, set tdv == dv. - - - srv for the quantity rv/2^bc.scale; so srv is the current binary - approximation to tdv (and dv). It should be exactly representable - in an IEEE 754 double. - */ - - for(;;) { - - /* This is the main correction loop for soa_strtod. - - We've got a decimal value tdv, and a floating-point approximation - srv=rv/2^bc.scale to tdv. The aim is to determine whether srv is - close enough (i.e., within 0.5 ulps) to tdv, and to compute a new - approximation if not. - - To determine whether srv is close enough to tdv, compute integers - bd, bb and bs proportional to tdv, srv and 0.5 ulp(srv) - respectively, and then use integer arithmetic to determine whether - |tdv - srv| is less than, equal to, or greater than 0.5 ulp(srv). - */ - - bd = Balloc(bd0->k); - if (bd == NULL) { - Bfree(bd0); - goto failed_malloc; - } - Bcopy(bd, bd0); - bb = sd2b(&rv, bc.scale, &bbe); /* srv = bb * 2^bbe */ - if (bb == NULL) { - Bfree(bd); - Bfree(bd0); - goto failed_malloc; - } - /* Record whether lsb of bb is odd, in case we need this - for the round-to-even step later. */ - odd = bb->x[0] & 1; - - /* tdv = bd * 10**e; srv = bb * 2**bbe */ - bs = i2b(1); - if (bs == NULL) { - Bfree(bb); - Bfree(bd); - Bfree(bd0); - goto failed_malloc; - } - - if (e >= 0) { - bb2 = bb5 = 0; - bd2 = bd5 = e; - } - else { - bb2 = bb5 = -e; - bd2 = bd5 = 0; - } - if (bbe >= 0) - bb2 += bbe; - else - bd2 -= bbe; - bs2 = bb2; - bb2++; - bd2++; - - /* At this stage bd5 - bb5 == e == bd2 - bb2 + bbe, bb2 - bs2 == 1, - and bs == 1, so: - - tdv == bd * 10**e = bd * 2**(bbe - bb2 + bd2) * 5**(bd5 - bb5) - srv == bb * 2**bbe = bb * 2**(bbe - bb2 + bb2) - 0.5 ulp(srv) == 2**(bbe-1) = bs * 2**(bbe - bb2 + bs2) - - It follows that: - - M * tdv = bd * 2**bd2 * 5**bd5 - M * srv = bb * 2**bb2 * 5**bb5 - M * 0.5 ulp(srv) = bs * 2**bs2 * 5**bb5 - - for some constant M. (Actually, M == 2**(bb2 - bbe) * 5**bb5, but - this fact is not needed below.) - */ - - /* Remove factor of 2**i, where i = min(bb2, bd2, bs2). */ - i = bb2 < bd2 ? bb2 : bd2; - if (i > bs2) - i = bs2; - if (i > 0) { - bb2 -= i; - bd2 -= i; - bs2 -= i; - } - - /* Scale bb, bd, bs by the appropriate powers of 2 and 5. */ - if (bb5 > 0) { - bs = pow5mult(bs, bb5); - if (bs == NULL) { - Bfree(bb); - Bfree(bd); - Bfree(bd0); - goto failed_malloc; - } - bb1 = mult(bs, bb); - Bfree(bb); - bb = bb1; - if (bb == NULL) { - Bfree(bs); - Bfree(bd); - Bfree(bd0); - goto failed_malloc; - } - } - if (bb2 > 0) { - bb = lshift(bb, bb2); - if (bb == NULL) { - Bfree(bs); - Bfree(bd); - Bfree(bd0); - goto failed_malloc; - } - } - if (bd5 > 0) { - bd = pow5mult(bd, bd5); - if (bd == NULL) { - Bfree(bb); - Bfree(bs); - Bfree(bd0); - goto failed_malloc; - } - } - if (bd2 > 0) { - bd = lshift(bd, bd2); - if (bd == NULL) { - Bfree(bb); - Bfree(bs); - Bfree(bd0); - goto failed_malloc; - } - } - if (bs2 > 0) { - bs = lshift(bs, bs2); - if (bs == NULL) { - Bfree(bb); - Bfree(bd); - Bfree(bd0); - goto failed_malloc; - } - } - - /* Now bd, bb and bs are scaled versions of tdv, srv and 0.5 ulp(srv), - respectively. Compute the difference |tdv - srv|, and compare - with 0.5 ulp(srv). */ - - delta = diff(bb, bd); - if (delta == NULL) { - Bfree(bb); - Bfree(bs); - Bfree(bd); - Bfree(bd0); - goto failed_malloc; - } - dsign = delta->sign; - delta->sign = 0; - i = cmp(delta, bs); - if (bc.nd > nd && i <= 0) { - if (dsign) - break; /* Must use bigcomp(). */ - - /* Here rv overestimates the truncated decimal value by at most - 0.5 ulp(rv). Hence rv either overestimates the true decimal - value by <= 0.5 ulp(rv), or underestimates it by some small - amount (< 0.1 ulp(rv)); either way, rv is within 0.5 ulps of - the true decimal value, so it's possible to exit. - - Exception: if scaled rv is a normal exact power of 2, but not - DBL_MIN, then rv - 0.5 ulp(rv) takes us all the way down to the - next double, so the correctly rounded result is either rv - 0.5 - ulp(rv) or rv; in this case, use bigcomp to distinguish. */ - - if (!word1(&rv) && !(word0(&rv) & Bndry_mask)) { - /* rv can't be 0, since it's an overestimate for some - nonzero value. So rv is a normal power of 2. */ - j = (int)(word0(&rv) & Exp_mask) >> Exp_shift; - /* rv / 2^bc.scale = 2^(j - 1023 - bc.scale); use bigcomp if - rv / 2^bc.scale >= 2^-1021. */ - if (j - bc.scale >= 2) { - dval(&rv) -= 0.5 * sulp(&rv, &bc); - break; /* Use bigcomp. */ - } - } - - { - bc.nd = nd; - i = -1; /* Discarded digits make delta smaller. */ - } - } - - if (i < 0) { - /* Error is less than half an ulp -- check for - * special case of mantissa a power of two. - */ - if (dsign || word1(&rv) || word0(&rv) & Bndry_mask - || (word0(&rv) & Exp_mask) <= (2*P+1)*Exp_msk1 - ) { - break; - } - if (!delta->x[0] && delta->wds <= 1) { - /* exact result */ - break; - } - delta = lshift(delta,Log2P); - if (delta == NULL) { - Bfree(bb); - Bfree(bs); - Bfree(bd); - Bfree(bd0); - goto failed_malloc; - } - if (cmp(delta, bs) > 0) - goto drop_down; - break; - } - if (i == 0) { - /* exactly half-way between */ - if (dsign) { - if ((word0(&rv) & Bndry_mask1) == Bndry_mask1 - && word1(&rv) == ( - (bc.scale && - (y = word0(&rv) & Exp_mask) <= 2*P*Exp_msk1) ? - (0xffffffff & (0xffffffff << (2*P+1-(y>>Exp_shift)))) : - 0xffffffff)) { - /*boundary case -- increment exponent*/ - word0(&rv) = (word0(&rv) & Exp_mask) - + Exp_msk1 - ; - word1(&rv) = 0; - dsign = 0; - break; - } - } - else if (!(word0(&rv) & Bndry_mask) && !word1(&rv)) { - drop_down: - /* boundary case -- decrement exponent */ - if (bc.scale) { - L = word0(&rv) & Exp_mask; - if (L <= (2*P+1)*Exp_msk1) { - if (L > (P+2)*Exp_msk1) - /* round even ==> */ - /* accept rv */ - break; - /* rv = smallest denormal */ - if (bc.nd > nd) - break; - goto undfl; - } - } - L = (word0(&rv) & Exp_mask) - Exp_msk1; - word0(&rv) = L | Bndry_mask1; - word1(&rv) = 0xffffffff; - break; - } - if (!odd) - break; - if (dsign) - dval(&rv) += sulp(&rv, &bc); - else { - dval(&rv) -= sulp(&rv, &bc); - if (!dval(&rv)) { - if (bc.nd >nd) - break; - goto undfl; - } - } - dsign = 1 - dsign; - break; - } - if ((aadj = ratio(delta, bs)) <= 2.) { - if (dsign) - aadj = aadj1 = 1.; - else if (word1(&rv) || word0(&rv) & Bndry_mask) { - if (word1(&rv) == Tiny1 && !word0(&rv)) { - if (bc.nd >nd) - break; - goto undfl; - } - aadj = 1.; - aadj1 = -1.; - } - else { - /* special case -- power of FLT_RADIX to be */ - /* rounded down... */ - - if (aadj < 2./FLT_RADIX) - aadj = 1./FLT_RADIX; - else - aadj *= 0.5; - aadj1 = -aadj; - } - } - else { - aadj *= 0.5; - aadj1 = dsign ? aadj : -aadj; - if (Flt_Rounds == 0) - aadj1 += 0.5; - } - y = word0(&rv) & Exp_mask; - - /* Check for overflow */ - - if (y == Exp_msk1*(DBL_MAX_EXP+Bias-1)) { - dval(&rv0) = dval(&rv); - word0(&rv) -= P*Exp_msk1; - adj.d = aadj1 * ulp(&rv); - dval(&rv) += adj.d; - if ((word0(&rv) & Exp_mask) >= - Exp_msk1*(DBL_MAX_EXP+Bias-P)) { - if (word0(&rv0) == Big0 && word1(&rv0) == Big1) { - Bfree(bb); - Bfree(bd); - Bfree(bs); - Bfree(bd0); - Bfree(delta); - goto ovfl; - } - word0(&rv) = Big0; - word1(&rv) = Big1; - goto cont; - } - else - word0(&rv) += P*Exp_msk1; - } - else { - if (bc.scale && y <= 2*P*Exp_msk1) { - if (aadj <= 0x7fffffff) { - if ((z = (ULong)aadj) <= 0) - z = 1; - aadj = z; - aadj1 = dsign ? aadj : -aadj; - } - dval(&aadj2) = aadj1; - word0(&aadj2) += (2*P+1)*Exp_msk1 - y; - aadj1 = dval(&aadj2); - } - adj.d = aadj1 * ulp(&rv); - dval(&rv) += adj.d; - } - z = word0(&rv) & Exp_mask; - if (bc.nd == nd) { - if (!bc.scale) - if (y == z) { - /* Can we stop now? */ - L = (Long)aadj; - aadj -= L; - /* The tolerances below are conservative. */ - if (dsign || word1(&rv) || word0(&rv) & Bndry_mask) { - if (aadj < .4999999 || aadj > .5000001) - break; - } - else if (aadj < .4999999/FLT_RADIX) - break; - } - } - cont: - Bfree(bb); - Bfree(bd); - Bfree(bs); - Bfree(delta); - } - Bfree(bb); - Bfree(bd); - Bfree(bs); - Bfree(bd0); - Bfree(delta); - if (bc.nd > nd) { - error = bigcomp(&rv, s0, &bc); - if (error) - goto failed_malloc; - } - - if (bc.scale) { - word0(&rv0) = Exp_1 - 2*P*Exp_msk1; - word1(&rv0) = 0; - dval(&rv) *= dval(&rv0); - } - - ret: - return sign ? -dval(&rv) : dval(&rv); - - parse_error: - return 0.0; - - failed_malloc: - errno = ENOMEM; - return -1.0; - - undfl: - return sign ? -0.0 : 0.0; - - ovfl: - errno = ERANGE; - /* Can't trust HUGE_VAL */ - word0(&rv) = Exp_mask; - word1(&rv) = 0; - return sign ? -dval(&rv) : dval(&rv); - -} - -static char * -rv_alloc(int i) -{ - int j, k, *r; - - j = sizeof(ULong); - for(k = 0; - sizeof(Bigint) - sizeof(ULong) - sizeof(int) + j <= (unsigned)i; - j <<= 1) - k++; - r = (int*)Balloc(k); - if (r == NULL) - return NULL; - *r = k; - return (char *)(r+1); -} - -static char * -nrv_alloc(char *s, char **rve, int n) -{ - char *rv, *t; - - rv = rv_alloc(n); - if (rv == NULL) - return NULL; - t = rv; - while((*t = *s++)) t++; - if (rve) - *rve = t; - return rv; -} - -/* freedtoa(s) must be used to free values s returned by dtoa - * when MULTIPLE_THREADS is #defined. It should be used in all cases, - * but for consistency with earlier versions of dtoa, it is optional - * when MULTIPLE_THREADS is not defined. - */ - -void -soa_freedtoa(char *s) -{ - Bigint *b = (Bigint *)((int *)s - 1); - b->maxwds = 1 << (b->k = *(int*)b); - Bfree(b); -} - -/* dtoa for IEEE arithmetic (dmg): convert double to ASCII string. - * - * Inspired by "How to Print Floating-Point Numbers Accurately" by - * Guy L. Steele, Jr. and Jon L. White [Proc. ACM SIGPLAN '90, pp. 112-126]. - * - * Modifications: - * 1. Rather than iterating, we use a simple numeric overestimate - * to determine k = floor(log10(d)). We scale relevant - * quantities using O(log2(k)) rather than O(k) multiplications. - * 2. For some modes > 2 (corresponding to ecvt and fcvt), we don't - * try to generate digits strictly left to right. Instead, we - * compute with fewer bits and propagate the carry if necessary - * when rounding the final digit up. This is often faster. - * 3. Under the assumption that input will be rounded nearest, - * mode 0 renders 1e23 as 1e23 rather than 9.999999999999999e22. - * That is, we allow equality in stopping tests when the - * round-nearest rule will give the same floating-point value - * as would satisfaction of the stopping test with strict - * inequality. - * 4. We remove common factors of powers of 2 from relevant - * quantities. - * 5. When converting floating-point integers less than 1e16, - * we use floating-point arithmetic rather than resorting - * to multiple-precision integers. - * 6. When asked to produce fewer than 15 digits, we first try - * to get by with floating-point arithmetic; we resort to - * multiple-precision integer arithmetic only if we cannot - * guarantee that the floating-point calculation has given - * the correctly rounded result. For k requested digits and - * "uniformly" distributed input, the probability is - * something like 10^(k-15) that we must resort to the Long - * calculation. - */ - -/* Additional notes (METD): (1) returns NULL on failure. (2) to avoid memory - leakage, a successful call to soa_dtoa should always be matched by a - call to soa_freedtoa. */ - -char * -soa_dtoa(double dd, int mode, int ndigits, - int *decpt, int *sign, char **rve) -{ - /* Arguments ndigits, decpt, sign are similar to those - of ecvt and fcvt; trailing zeros are suppressed from - the returned string. If not null, *rve is set to point - to the end of the return value. If d is +-Infinity or NaN, - then *decpt is set to 9999. - - mode: - 0 ==> shortest string that yields d when read in - and rounded to nearest. - 1 ==> like 0, but with Steele & White stopping rule; - e.g. with IEEE P754 arithmetic , mode 0 gives - 1e23 whereas mode 1 gives 9.999999999999999e22. - 2 ==> max(1,ndigits) significant digits. This gives a - return value similar to that of ecvt, except - that trailing zeros are suppressed. - 3 ==> through ndigits past the decimal point. This - gives a return value similar to that from fcvt, - except that trailing zeros are suppressed, and - ndigits can be negative. - 4,5 ==> similar to 2 and 3, respectively, but (in - round-nearest mode) with the tests of mode 0 to - possibly return a shorter string that rounds to d. - With IEEE arithmetic and compilation with - -DHonor_FLT_ROUNDS, modes 4 and 5 behave the same - as modes 2 and 3 when FLT_ROUNDS != 1. - 6-9 ==> Debugging modes similar to mode - 4: don't try - fast floating-point estimate (if applicable). - - Values of mode other than 0-9 are treated as mode 0. - - Sufficient space is allocated to the return value - to hold the suppressed trailing zeros. - */ - - int bbits, b2, b5, be, dig, i, ieps, ilim, ilim0, ilim1, - j, j1, k, k0, k_check, leftright, m2, m5, s2, s5, - spec_case, try_quick; - Long L; - int denorm; - ULong x; - Bigint *b, *b1, *delta, *mlo, *mhi, *S; - U d2, eps, u; - double ds; - char *s, *s0; - - /* set pointers to NULL, to silence gcc compiler warnings and make - cleanup easier on error */ - mlo = mhi = S = 0; - s0 = 0; - - u.d = dd; - if (word0(&u) & Sign_bit) { - /* set sign for everything, including 0's and NaNs */ - *sign = 1; - word0(&u) &= ~Sign_bit; /* clear sign bit */ - } - else - *sign = 0; - - /* quick return for Infinities, NaNs and zeros */ - if ((word0(&u) & Exp_mask) == Exp_mask) - { - /* Infinity or NaN */ - *decpt = 9999; - if (!word1(&u) && !(word0(&u) & 0xfffff)) - return nrv_alloc("Infinity", rve, 8); - return nrv_alloc("NaN", rve, 3); - } - if (!dval(&u)) { - *decpt = 1; - return nrv_alloc("0", rve, 1); - } - - /* compute k = floor(log10(d)). The computation may leave k - one too large, but should never leave k too small. */ - b = d2b(&u, &be, &bbits); - if (b == NULL) - goto failed_malloc; - if ((i = (int)(word0(&u) >> Exp_shift1 & (Exp_mask>>Exp_shift1)))) { - dval(&d2) = dval(&u); - word0(&d2) &= Frac_mask1; - word0(&d2) |= Exp_11; - - /* log(x) ~=~ log(1.5) + (x-1.5)/1.5 - * log10(x) = log(x) / log(10) - * ~=~ log(1.5)/log(10) + (x-1.5)/(1.5*log(10)) - * log10(d) = (i-Bias)*log(2)/log(10) + log10(d2) - * - * This suggests computing an approximation k to log10(d) by - * - * k = (i - Bias)*0.301029995663981 - * + ( (d2-1.5)*0.289529654602168 + 0.176091259055681 ); - * - * We want k to be too large rather than too small. - * The error in the first-order Taylor series approximation - * is in our favor, so we just round up the constant enough - * to compensate for any error in the multiplication of - * (i - Bias) by 0.301029995663981; since |i - Bias| <= 1077, - * and 1077 * 0.30103 * 2^-52 ~=~ 7.2e-14, - * adding 1e-13 to the constant term more than suffices. - * Hence we adjust the constant term to 0.1760912590558. - * (We could get a more accurate k by invoking log10, - * but this is probably not worthwhile.) - */ - - i -= Bias; - denorm = 0; - } - else { - /* d is denormalized */ - - i = bbits + be + (Bias + (P-1) - 1); - x = i > 32 ? word0(&u) << (64 - i) | word1(&u) >> (i - 32) - : word1(&u) << (32 - i); - dval(&d2) = x; - word0(&d2) -= 31*Exp_msk1; /* adjust exponent */ - i -= (Bias + (P-1) - 1) + 1; - denorm = 1; - } - ds = (dval(&d2)-1.5)*0.289529654602168 + 0.1760912590558 + - i*0.301029995663981; - k = (int)ds; - if (ds < 0. && ds != k) - k--; /* want k = floor(ds) */ - k_check = 1; - if (k >= 0 && k <= Ten_pmax) { - if (dval(&u) < tens[k]) - k--; - k_check = 0; - } - j = bbits - i - 1; - if (j >= 0) { - b2 = 0; - s2 = j; - } - else { - b2 = -j; - s2 = 0; - } - if (k >= 0) { - b5 = 0; - s5 = k; - s2 += k; - } - else { - b2 -= k; - b5 = -k; - s5 = 0; - } - if (mode < 0 || mode > 9) - mode = 0; - - try_quick = 1; - - if (mode > 5) { - mode -= 4; - try_quick = 0; - } - leftright = 1; - ilim = ilim1 = -1; /* Values for cases 0 and 1; done here to */ - /* silence erroneous "gcc -Wall" warning. */ - switch(mode) { - case 0: - case 1: - i = 18; - ndigits = 0; - break; - case 2: - leftright = 0; - /* no break */ - case 4: - if (ndigits <= 0) - ndigits = 1; - ilim = ilim1 = i = ndigits; - break; - case 3: - leftright = 0; - /* no break */ - case 5: - i = ndigits + k + 1; - ilim = i; - ilim1 = i - 1; - if (i <= 0) - i = 1; - } - s0 = rv_alloc(i); - if (s0 == NULL) - goto failed_malloc; - s = s0; - - - if (ilim >= 0 && ilim <= Quick_max && try_quick) { - - /* Try to get by with floating-point arithmetic. */ - - i = 0; - dval(&d2) = dval(&u); - k0 = k; - ilim0 = ilim; - ieps = 2; /* conservative */ - if (k > 0) { - ds = tens[k&0xf]; - j = k >> 4; - if (j & Bletch) { - /* prevent overflows */ - j &= Bletch - 1; - dval(&u) /= bigtens[n_bigtens-1]; - ieps++; - } - for(; j; j >>= 1, i++) - if (j & 1) { - ieps++; - ds *= bigtens[i]; - } - dval(&u) /= ds; - } - else if ((j1 = -k)) { - dval(&u) *= tens[j1 & 0xf]; - for(j = j1 >> 4; j; j >>= 1, i++) - if (j & 1) { - ieps++; - dval(&u) *= bigtens[i]; - } - } - if (k_check && dval(&u) < 1. && ilim > 0) { - if (ilim1 <= 0) - goto fast_failed; - ilim = ilim1; - k--; - dval(&u) *= 10.; - ieps++; - } - dval(&eps) = ieps*dval(&u) + 7.; - word0(&eps) -= (P-1)*Exp_msk1; - if (ilim == 0) { - S = mhi = 0; - dval(&u) -= 5.; - if (dval(&u) > dval(&eps)) - goto one_digit; - if (dval(&u) < -dval(&eps)) - goto no_digits; - goto fast_failed; - } - if (leftright) { - /* Use Steele & White method of only - * generating digits needed. - */ - dval(&eps) = 0.5/tens[ilim-1] - dval(&eps); - for(i = 0;;) { - L = (Long)dval(&u); - dval(&u) -= L; - *s++ = '0' + (int)L; - if (dval(&u) < dval(&eps)) - goto ret1; - if (1. - dval(&u) < dval(&eps)) - goto bump_up; - if (++i >= ilim) - break; - dval(&eps) *= 10.; - dval(&u) *= 10.; - } - } - else { - /* Generate ilim digits, then fix them up. */ - dval(&eps) *= tens[ilim-1]; - for(i = 1;; i++, dval(&u) *= 10.) { - L = (Long)(dval(&u)); - if (!(dval(&u) -= L)) - ilim = i; - *s++ = '0' + (int)L; - if (i == ilim) { - if (dval(&u) > 0.5 + dval(&eps)) - goto bump_up; - else if (dval(&u) < 0.5 - dval(&eps)) { - while(*--s == '0'); - s++; - goto ret1; - } - break; - } - } - } - fast_failed: - s = s0; - dval(&u) = dval(&d2); - k = k0; - ilim = ilim0; - } - - /* Do we have a "small" integer? */ - - if (be >= 0 && k <= Int_max) { - /* Yes. */ - ds = tens[k]; - if (ndigits < 0 && ilim <= 0) { - S = mhi = 0; - if (ilim < 0 || dval(&u) <= 5*ds) - goto no_digits; - goto one_digit; - } - for(i = 1;; i++, dval(&u) *= 10.) { - L = (Long)(dval(&u) / ds); - dval(&u) -= L*ds; - *s++ = '0' + (int)L; - if (!dval(&u)) { - break; - } - if (i == ilim) { - dval(&u) += dval(&u); - if (dval(&u) > ds || (dval(&u) == ds && L & 1)) { - bump_up: - while(*--s == '9') - if (s == s0) { - k++; - *s = '0'; - break; - } - ++*s++; - } - break; - } - } - goto ret1; - } - - m2 = b2; - m5 = b5; - if (leftright) { - i = - denorm ? be + (Bias + (P-1) - 1 + 1) : - 1 + P - bbits; - b2 += i; - s2 += i; - mhi = i2b(1); - if (mhi == NULL) - goto failed_malloc; - } - if (m2 > 0 && s2 > 0) { - i = m2 < s2 ? m2 : s2; - b2 -= i; - m2 -= i; - s2 -= i; - } - if (b5 > 0) { - if (leftright) { - if (m5 > 0) { - mhi = pow5mult(mhi, m5); - if (mhi == NULL) - goto failed_malloc; - b1 = mult(mhi, b); - Bfree(b); - b = b1; - if (b == NULL) - goto failed_malloc; - } - if ((j = b5 - m5)) { - b = pow5mult(b, j); - if (b == NULL) - goto failed_malloc; - } - } - else { - b = pow5mult(b, b5); - if (b == NULL) - goto failed_malloc; - } - } - S = i2b(1); - if (S == NULL) - goto failed_malloc; - if (s5 > 0) { - S = pow5mult(S, s5); - if (S == NULL) - goto failed_malloc; - } - - /* Check for special case that d is a normalized power of 2. */ - - spec_case = 0; - if ((mode < 2 || leftright) - ) { - if (!word1(&u) && !(word0(&u) & Bndry_mask) - && word0(&u) & (Exp_mask & ~Exp_msk1) - ) { - /* The special case */ - b2 += Log2P; - s2 += Log2P; - spec_case = 1; - } - } - - /* Arrange for convenient computation of quotients: - * shift left if necessary so divisor has 4 leading 0 bits. - * - * Perhaps we should just compute leading 28 bits of S once - * and for all and pass them and a shift to quorem, so it - * can do shifts and ors to compute the numerator for q. - */ -#define iInc 28 - i = dshift(S, s2); - b2 += i; - m2 += i; - s2 += i; - if (b2 > 0) { - b = lshift(b, b2); - if (b == NULL) - goto failed_malloc; - } - if (s2 > 0) { - S = lshift(S, s2); - if (S == NULL) - goto failed_malloc; - } - if (k_check) { - if (cmp(b,S) < 0) { - k--; - b = multadd(b, 10, 0); /* we botched the k estimate */ - if (b == NULL) - goto failed_malloc; - if (leftright) { - mhi = multadd(mhi, 10, 0); - if (mhi == NULL) - goto failed_malloc; - } - ilim = ilim1; - } - } - if (ilim <= 0 && (mode == 3 || mode == 5)) { - if (ilim < 0) { - /* no digits, fcvt style */ - no_digits: - k = -1 - ndigits; - goto ret; - } - else { - S = multadd(S, 5, 0); - if (S == NULL) - goto failed_malloc; - if (cmp(b, S) <= 0) - goto no_digits; - } - one_digit: - *s++ = '1'; - k++; - goto ret; - } - if (leftright) { - if (m2 > 0) { - mhi = lshift(mhi, m2); - if (mhi == NULL) - goto failed_malloc; - } - - /* Compute mlo -- check for special case - * that d is a normalized power of 2. - */ - - mlo = mhi; - if (spec_case) { - mhi = Balloc(mhi->k); - if (mhi == NULL) - goto failed_malloc; - Bcopy(mhi, mlo); - mhi = lshift(mhi, Log2P); - if (mhi == NULL) - goto failed_malloc; - } - - for(i = 1;;i++) { - dig = quorem(b,S) + '0'; - /* Do we yet have the shortest decimal string - * that will round to d? - */ - j = cmp(b, mlo); - delta = diff(S, mhi); - if (delta == NULL) - goto failed_malloc; - j1 = delta->sign ? 1 : cmp(b, delta); - Bfree(delta); - if (j1 == 0 && mode != 1 && !(word1(&u) & 1) - ) { - if (dig == '9') - goto round_9_up; - if (j > 0) - dig++; - *s++ = dig; - goto ret; - } - if (j < 0 || (j == 0 && mode != 1 - && !(word1(&u) & 1) - )) { - if (!b->x[0] && b->wds <= 1) { - goto accept_dig; - } - if (j1 > 0) { - b = lshift(b, 1); - if (b == NULL) - goto failed_malloc; - j1 = cmp(b, S); - if ((j1 > 0 || (j1 == 0 && dig & 1)) - && dig++ == '9') - goto round_9_up; - } - accept_dig: - *s++ = dig; - goto ret; - } - if (j1 > 0) { - if (dig == '9') { /* possible if i == 1 */ - round_9_up: - *s++ = '9'; - goto roundoff; - } - *s++ = dig + 1; - goto ret; - } - *s++ = dig; - if (i == ilim) - break; - b = multadd(b, 10, 0); - if (b == NULL) - goto failed_malloc; - if (mlo == mhi) { - mlo = mhi = multadd(mhi, 10, 0); - if (mlo == NULL) - goto failed_malloc; - } - else { - mlo = multadd(mlo, 10, 0); - if (mlo == NULL) - goto failed_malloc; - mhi = multadd(mhi, 10, 0); - if (mhi == NULL) - goto failed_malloc; - } - } - } - else - for(i = 1;; i++) { - *s++ = dig = quorem(b,S) + '0'; - if (!b->x[0] && b->wds <= 1) { - goto ret; - } - if (i >= ilim) - break; - b = multadd(b, 10, 0); - if (b == NULL) - goto failed_malloc; - } - - /* Round off last digit */ - - b = lshift(b, 1); - if (b == NULL) - goto failed_malloc; - j = cmp(b, S); - if (j > 0 || (j == 0 && dig & 1)) { - roundoff: - while(*--s == '9') - if (s == s0) { - k++; - *s++ = '1'; - goto ret; - } - ++*s++; - } - else { - while(*--s == '0'); - s++; - } - ret: - Bfree(S); - if (mhi) { - if (mlo && mlo != mhi) - Bfree(mlo); - Bfree(mhi); - } - ret1: - Bfree(b); - *s = 0; - *decpt = k + 1; - if (rve) - *rve = s; - return s0; - failed_malloc: - if (S) - Bfree(S); - if (mlo && mlo != mhi) - Bfree(mlo); - if (mhi) - Bfree(mhi); - if (b) - Bfree(b); - if (s0) - soa_freedtoa(s0); - return NULL; -} -#ifdef __cplusplus -} -#endif diff --git a/types/dtoa.h b/types/dtoa.h deleted file mode 100644 index 8630c2a32..000000000 --- a/types/dtoa.h +++ /dev/null @@ -1,106 +0,0 @@ -/** dtoa.h - - Print a floating point number at the precision necessary to preserve its - binary representation, no more. - - Derived from dtoa.c. See copyright and authorship information in that file. -*/ - -#pragma once - -#include - -extern "C" { - - - /* Arguments ndigits, decpt, sign are similar to those - of ecvt and fcvt; trailing zeros are suppressed from - the returned string. If not null, *rve is set to point - to the end of the return value. If d is +-Infinity or NaN, - then *decpt is set to 9999. - - mode: - 0 ==> shortest string that yields d when read in - and rounded to nearest. - 1 ==> like 0, but with Steele & White stopping rule; - e.g. with IEEE P754 arithmetic , mode 0 gives - 1e23 whereas mode 1 gives 9.999999999999999e22. - 2 ==> max(1,ndigits) significant digits. This gives a - return value similar to that of ecvt, except - that trailing zeros are suppressed. - 3 ==> through ndigits past the decimal point. This - gives a return value similar to that from fcvt, - except that trailing zeros are suppressed, and - ndigits can be negative. - 4,5 ==> similar to 2 and 3, respectively, but (in - round-nearest mode) with the tests of mode 0 to - possibly return a shorter string that rounds to d. - With IEEE arithmetic and compilation with - -DHonor_FLT_ROUNDS, modes 4 and 5 behave the same - as modes 2 and 3 when FLT_ROUNDS != 1. - 6-9 ==> Debugging modes similar to mode - 4: don't try - fast floating-point estimate (if applicable). - - Values of mode other than 0-9 are treated as mode 0. - - Sufficient space is allocated to the return value - to hold the suppressed trailing zeros. - */ - -char * -soa_dtoa(double dd, int mode, int ndigits, - int *decpt, int *sign, char **rve); - -void -soa_freedtoa(char *s); - -double -soa_strtod(const char *s00, char **se); - -} // extern "C" - -namespace Datacratic { - -inline std::string dtoa(double floatVal) -{ - // Use dtoa to make sure we print a value that will be converted - // back to the same on input, without printing more digits than - // necessary. - int decpt; - int sign; - - char * result = soa_dtoa(floatVal, 1, -1 /* ndigits */, - &decpt, &sign, nullptr); - std::string toReturn(result); - soa_freedtoa(result); - - //cerr << "decpt = " << decpt << " sign = " << sign - // << " result = " << toReturn << " val = " << floatVal - // << endl; - if (decpt > 0 && decpt <= toReturn.size()) - toReturn.insert(decpt, "."); - else if (decpt == 9999) - ; - else if (decpt <= 0 && decpt > -6) { - toReturn.insert(0, "0."); - for (unsigned i = 0; i < -decpt; ++i) - toReturn.insert(2, "0"); - } - else { - if (toReturn.size() > 1) - toReturn.insert(1, "."); - toReturn += "e" + std::to_string(decpt - 1); - } - - if (sign) - toReturn.insert(0, "-"); - - if (toReturn.back() == '.') { - toReturn.erase(toReturn.size()-1, 1); - } - - return toReturn; -} - - -} // namespace Datacratic