From c965263fc6a98fc50d75504d186930e07024a533 Mon Sep 17 00:00:00 2001 From: WeijieSun Date: Tue, 26 Dec 2017 14:41:14 +0800 Subject: [PATCH 1/3] refactor: remove io-per-queue's io mode & fastrun's running mode --- include/dsn/tool-api/aio_provider.h | 4 +- include/dsn/tool-api/global_config.h | 33 +- include/dsn/tool-api/network.h | 4 +- include/dsn/tool-api/nfs.h | 2 +- include/dsn/tool-api/task.h | 5 +- include/dsn/tool-api/task_spec.h | 19 - include/dsn/tool-api/timer_service.h | 2 +- include/dsn/tool/fastrun.h | 54 -- include/dsn/tool/nfs/nfs_node_simple.h | 4 +- include/dsn/tool/node_scoper.h | 2 +- src/apps/echo/config.ini | 2 +- src/apps/nfs/nfs_node_impl.cpp | 2 +- src/apps/skv/config.ini | 1 - src/core/core/core_main.cpp | 2 - src/core/core/disk_engine.cpp | 4 +- src/core/core/disk_engine.h | 2 +- src/core/core/global_config.cpp | 40 -- src/core/core/lock_checker.cpp | 59 +- src/core/core/rpc_engine.cpp | 52 +- src/core/core/rpc_engine.h | 5 +- src/core/core/service_api_c.cpp | 4 +- src/core/core/service_engine.cpp | 242 ++------ src/core/core/service_engine.h | 28 +- src/core/core/task.cpp | 26 +- src/core/core/task_engine.cpp | 23 +- src/core/core/task_engine.h | 2 +- src/core/core/task_worker.cpp | 2 +- src/core/core/tool_api.cpp | 4 +- src/core/perf.tests/config-test.ini | 4 +- src/core/perf.tests/logger.cpp | 23 +- src/core/tests/CMakeLists.txt | 1 - .../tests/config-test-corrupt-message.ini | 3 +- src/core/tests/config-test-fastrun.ini | 80 --- src/core/tests/config-test-posix-aio.ini | 3 +- src/core/tests/config-test-sim.ini | 6 +- src/core/tests/config-test.ini | 8 +- src/core/tests/gtest.filter | 1 - src/core/tests/hpc_aio_provider.cpp | 154 ----- src/core/tests/hpc_aio_provider_for_test.h | 74 --- src/core/tests/hpc_io_looper.cpp | 186 ------- src/core/tests/hpc_tail_logger.cpp | 82 --- src/core/tests/main.cpp | 5 - src/core/tests/netprovider.cpp | 27 +- src/core/tests/nfs_test_file1 | 3 +- src/core/tests/nfs_test_file2 | 3 +- src/core/tools/common/asio_net_provider.cpp | 14 +- src/core/tools/common/asio_net_provider.h | 6 +- src/core/tools/common/empty_aio_provider.h | 2 +- .../common/native_aio_provider.linux.cpp | 8 +- .../tools/common/native_aio_provider.linux.h | 2 +- .../common/native_aio_provider.posix.cpp | 2 +- .../tools/common/native_aio_provider.posix.h | 2 +- .../tools/common/native_aio_provider.win.cpp | 2 +- src/core/tools/common/network.sim.cpp | 3 +- src/core/tools/common/network.sim.h | 2 +- src/core/tools/common/simple_task_queue.cpp | 11 +- src/core/tools/common/simple_task_queue.h | 2 +- src/core/tools/hpc/fastrun.cpp | 129 ----- src/core/tools/hpc/hpc_aio_provider.bsd.cpp | 257 --------- src/core/tools/hpc/hpc_aio_provider.h | 83 --- src/core/tools/hpc/hpc_aio_provider.linux.cpp | 246 -------- src/core/tools/hpc/hpc_aio_provider.win.cpp | 289 ---------- src/core/tools/hpc/hpc_env_provider.cpp | 54 -- src/core/tools/hpc/hpc_env_provider.h | 62 --- src/core/tools/hpc/hpc_logger.cpp | 389 ------------- src/core/tools/hpc/hpc_logger.h | 98 ---- .../tools/hpc/hpc_network_provider.bsd.cpp | 524 ------------------ src/core/tools/hpc/hpc_network_provider.h | 152 ----- .../tools/hpc/hpc_network_provider.linux.cpp | 518 ----------------- .../tools/hpc/hpc_network_provider.win.cpp | 506 ----------------- src/core/tools/hpc/hpc_tail_logger.cpp | 319 ----------- src/core/tools/hpc/hpc_tail_logger.h | 71 --- src/core/tools/hpc/hpc_task_queue.cpp | 66 +-- src/core/tools/hpc/hpc_task_queue.h | 32 +- src/core/tools/hpc/io_looper.bsd.cpp | 322 ----------- src/core/tools/hpc/io_looper.h | 154 ----- src/core/tools/hpc/io_looper.linux.cpp | 258 --------- src/core/tools/hpc/io_looper.win.cpp | 163 ------ src/core/tools/hpc/mix_all_io_looper.cpp | 247 --------- src/core/tools/hpc/mix_all_io_looper.h | 106 ---- src/core/tools/hpc/providers.hpc.cpp | 17 - src/core/tools/simulator/task_engine.sim.h | 2 +- .../replication/test/simple_kv/case-000.ini | 1 - .../replication/test/simple_kv/case-001.ini | 1 - .../replication/test/simple_kv/case-002.ini | 1 - .../replication/test/simple_kv/case-003.ini | 1 - .../replication/test/simple_kv/case-004.ini | 1 - .../replication/test/simple_kv/case-005.ini | 1 - .../replication/test/simple_kv/case-006.ini | 1 - .../replication/test/simple_kv/case-100.ini | 1 - .../replication/test/simple_kv/case-101.ini | 1 - .../replication/test/simple_kv/case-102.ini | 1 - .../replication/test/simple_kv/case-103.ini | 1 - .../replication/test/simple_kv/case-104.ini | 1 - .../replication/test/simple_kv/case-105.ini | 1 - .../replication/test/simple_kv/case-106.ini | 1 - .../replication/test/simple_kv/case-107.ini | 1 - .../replication/test/simple_kv/case-108.ini | 1 - .../replication/test/simple_kv/case-109.ini | 1 - .../replication/test/simple_kv/case-200.ini | 1 - .../replication/test/simple_kv/case-201.ini | 1 - .../replication/test/simple_kv/case-202-0.ini | 1 - .../replication/test/simple_kv/case-202-1.ini | 1 - .../replication/test/simple_kv/case-203-0.ini | 1 - .../replication/test/simple_kv/case-203-1.ini | 1 - .../replication/test/simple_kv/case-204.ini | 1 - .../replication/test/simple_kv/case-205.ini | 1 - .../replication/test/simple_kv/case-206.ini | 1 - .../replication/test/simple_kv/case-207.ini | 1 - .../replication/test/simple_kv/case-208.ini | 1 - .../replication/test/simple_kv/case-209.ini | 1 - .../replication/test/simple_kv/case-210.ini | 1 - .../replication/test/simple_kv/case-211.ini | 1 - .../replication/test/simple_kv/case-212.ini | 1 - .../replication/test/simple_kv/case-213.ini | 1 - .../replication/test/simple_kv/case-214.ini | 1 - .../replication/test/simple_kv/case-215.ini | 1 - .../replication/test/simple_kv/case-216.ini | 1 - .../replication/test/simple_kv/case-300-0.ini | 1 - .../replication/test/simple_kv/case-300-1.ini | 1 - .../replication/test/simple_kv/case-300-2.ini | 1 - .../replication/test/simple_kv/case-301.ini | 1 - .../replication/test/simple_kv/case-302.ini | 1 - .../replication/test/simple_kv/case-303.ini | 1 - .../replication/test/simple_kv/case-304.ini | 1 - .../replication/test/simple_kv/case-305.ini | 1 - .../replication/test/simple_kv/case-306.ini | 1 - .../replication/test/simple_kv/case-307.ini | 1 - .../replication/test/simple_kv/case-400.ini | 1 - .../replication/test/simple_kv/case-401.ini | 1 - .../replication/test/simple_kv/case-402.ini | 1 - .../replication/test/simple_kv/case-600.ini | 1 - .../replication/test/simple_kv/case-601.ini | 1 - .../replication/test/simple_kv/case-602.ini | 1 - .../replication/test/simple_kv/case-603.ini | 1 - .../replication/test/simple_kv/checker.cpp | 2 +- .../replication/test/simple_kv/config.ini | 1 - src/tests/dsn/config-test.ini | 3 +- 138 files changed, 168 insertions(+), 6303 deletions(-) delete mode 100644 include/dsn/tool/fastrun.h delete mode 100644 src/core/tests/config-test-fastrun.ini delete mode 100644 src/core/tests/hpc_aio_provider.cpp delete mode 100644 src/core/tests/hpc_aio_provider_for_test.h delete mode 100644 src/core/tests/hpc_io_looper.cpp delete mode 100644 src/core/tests/hpc_tail_logger.cpp delete mode 100644 src/core/tools/hpc/fastrun.cpp delete mode 100644 src/core/tools/hpc/hpc_aio_provider.bsd.cpp delete mode 100644 src/core/tools/hpc/hpc_aio_provider.h delete mode 100644 src/core/tools/hpc/hpc_aio_provider.linux.cpp delete mode 100644 src/core/tools/hpc/hpc_aio_provider.win.cpp delete mode 100644 src/core/tools/hpc/hpc_env_provider.cpp delete mode 100644 src/core/tools/hpc/hpc_env_provider.h delete mode 100644 src/core/tools/hpc/hpc_logger.cpp delete mode 100644 src/core/tools/hpc/hpc_logger.h delete mode 100644 src/core/tools/hpc/hpc_network_provider.bsd.cpp delete mode 100644 src/core/tools/hpc/hpc_network_provider.h delete mode 100644 src/core/tools/hpc/hpc_network_provider.linux.cpp delete mode 100644 src/core/tools/hpc/hpc_network_provider.win.cpp delete mode 100644 src/core/tools/hpc/hpc_tail_logger.cpp delete mode 100644 src/core/tools/hpc/hpc_tail_logger.h delete mode 100644 src/core/tools/hpc/io_looper.bsd.cpp delete mode 100644 src/core/tools/hpc/io_looper.h delete mode 100644 src/core/tools/hpc/io_looper.linux.cpp delete mode 100644 src/core/tools/hpc/io_looper.win.cpp delete mode 100644 src/core/tools/hpc/mix_all_io_looper.cpp delete mode 100644 src/core/tools/hpc/mix_all_io_looper.h diff --git a/include/dsn/tool-api/aio_provider.h b/include/dsn/tool-api/aio_provider.h index f280f17624..7863b01421 100644 --- a/include/dsn/tool-api/aio_provider.h +++ b/include/dsn/tool-api/aio_provider.h @@ -56,7 +56,7 @@ class task_queue; @{ */ // -// !!! all threads must be started with task::set_tls_dsn_context(null, provider->node()); +// !!! all threads must be started with task::set_tls_dsn_context(provider->node(), null); // class aio_provider { @@ -81,7 +81,7 @@ class aio_provider virtual void aio(aio_task *aio) = 0; virtual disk_aio *prepare_aio_context(aio_task *) = 0; - virtual void start(io_modifer &ctx) = 0; + virtual void start() = 0; protected: DSN_API void diff --git a/include/dsn/tool-api/global_config.h b/include/dsn/tool-api/global_config.h index dfd916976c..7d104b4b7a 100644 --- a/include/dsn/tool-api/global_config.h +++ b/include/dsn/tool-api/global_config.h @@ -103,7 +103,7 @@ struct service_app_spec int delay_seconds; bool run; int count; // index = 1,2,...,count - int ports_gap; // when count > 1 or service_spec.io_mode != IOE_PER_NODE + int ports_gap; // when count > 1 network_client_configs network_client_confs; network_server_configs network_server_confs; @@ -173,11 +173,7 @@ struct service_spec std::list rwlock_nr_aspects; std::list semaphore_aspects; - ioe_mode disk_io_mode; // whether disk is per node or per queue - ioe_mode rpc_io_mode; // whether rpc is per node or per queue - ioe_mode nfs_io_mode; // whether nfs is per node or per queue - ioe_mode timer_io_mode; // whether timer is per node or per queue - int io_worker_count; // for disk and rpc when per node + int io_worker_count; // for disk and rpc network_client_configs network_default_client_cfs; // default network configed by tools network_server_configs network_default_server_cfs; // default network configed by tools @@ -191,7 +187,6 @@ struct service_spec service_spec() {} DSN_API bool init(); DSN_API bool init_app_specs(); - DSN_API int get_ports_delta(int app_id, dsn::threadpool_code pool, int queue_index) const; }; CONFIG_BEGIN(service_spec) @@ -230,30 +225,6 @@ CONFIG_FLD_STRING_LIST(rwlock_nr_aspects, "non-recursive rwlock aspect providers, usually for tooling purpose") CONFIG_FLD_STRING_LIST(semaphore_aspects, "semaphore aspect providers, usually for tooling purpose") -CONFIG_FLD_ENUM(ioe_mode, - disk_io_mode, - IOE_PER_NODE, - IOE_INVALID, - false, - "how many disk engines? IOE_PER_NODE, or IOE_PER_QUEUE") -CONFIG_FLD_ENUM(ioe_mode, - rpc_io_mode, - IOE_PER_NODE, - IOE_INVALID, - false, - "how many rpc engines? IOE_PER_NODE, or IOE_PER_QUEUE") -CONFIG_FLD_ENUM(ioe_mode, - nfs_io_mode, - IOE_PER_NODE, - IOE_INVALID, - false, - "how many nfs engines? IOE_PER_NODE, or IOE_PER_QUEUE") -CONFIG_FLD_ENUM(ioe_mode, - timer_io_mode, - IOE_PER_NODE, - IOE_INVALID, - false, - "how many disk timer services? IOE_PER_NODE, or IOE_PER_QUEUE") CONFIG_FLD(int, uint64, io_worker_count, diff --git a/include/dsn/tool-api/network.h b/include/dsn/tool-api/network.h index 6021f2c243..a54c72d023 100644 --- a/include/dsn/tool-api/network.h +++ b/include/dsn/tool-api/network.h @@ -56,7 +56,7 @@ class task_queue; /*! network bound to a specific rpc_channel and port (see start) - !!! all threads must be started with task::set_tls_dsn_context(null, provider->node()); + !!! all threads must be started with task::set_tls_dsn_context(provider->node(), null); */ class network { @@ -85,7 +85,7 @@ class network // // when client_only is true, port is faked (equal to app id for tracing purpose) // - virtual error_code start(rpc_channel channel, int port, bool client_only, io_modifer &ctx) = 0; + virtual error_code start(rpc_channel channel, int port, bool client_only) = 0; // // the named address diff --git a/include/dsn/tool-api/nfs.h b/include/dsn/tool-api/nfs.h index 01bf987667..e844fb1749 100644 --- a/include/dsn/tool-api/nfs.h +++ b/include/dsn/tool-api/nfs.h @@ -87,7 +87,7 @@ class nfs_node virtual ~nfs_node() {} - virtual ::dsn::error_code start(io_modifer &ctx) = 0; + virtual ::dsn::error_code start() = 0; virtual error_code stop() = 0; diff --git a/include/dsn/tool-api/task.h b/include/dsn/tool-api/task.h index a3d665a394..a7f30b62b7 100644 --- a/include/dsn/tool-api/task.h +++ b/include/dsn/tool-api/task.h @@ -263,9 +263,8 @@ class task : public ref_counter, public extensible_object, public trans static timer_service *get_current_tsvc(); static void set_tls_dsn_context( - service_node *node, // cannot be null - task_worker *worker, // null for io or timer threads if they are not worker threads - task_queue *queue // owner queue if io_mode == IOE_PER_QUEUE + service_node *node, // cannot be null + task_worker *worker // null for io or timer threads if they are not worker threads ); protected: diff --git a/include/dsn/tool-api/task_spec.h b/include/dsn/tool-api/task_spec.h index 51240f7662..a6ea21e04c 100644 --- a/include/dsn/tool-api/task_spec.h +++ b/include/dsn/tool-api/task_spec.h @@ -94,18 +94,6 @@ ENUM_REG(TASK_STATE_FINISHED) ENUM_REG(TASK_STATE_CANCELLED) ENUM_END(task_state) -typedef enum ioe_mode { - IOE_PER_NODE, // each node has shared io engine (rpc/disk/nfs/timer) - IOE_PER_QUEUE, // each queue has shared io engine (rpc/disk/nfs/timer) - IOE_COUNT, - IOE_INVALID -} ioe_mode; - -ENUM_BEGIN(ioe_mode, IOE_INVALID) -ENUM_REG(IOE_PER_NODE) -ENUM_REG(IOE_PER_QUEUE) -ENUM_END(ioe_mode) - typedef enum grpc_mode_t { GRPC_TO_LEADER, // the rpc is sent to the leader (if exist) GRPC_TO_ALL, // the rpc is sent to all @@ -161,13 +149,6 @@ class message_ex; class admission_controller; typedef void (*task_rejection_handler)(task *, admission_controller *); -typedef struct __io_mode_modifier__ -{ - ioe_mode mode; // see ioe_mode for details - task_queue *queue; // when mode == IOE_PER_QUEUE - int port_shift_value; // port += port_shift_value -} io_modifer; - class task_spec : public extensible_object { public: diff --git a/include/dsn/tool-api/timer_service.h b/include/dsn/tool-api/timer_service.h index 6ca6c1f65f..9c7ac52394 100644 --- a/include/dsn/tool-api/timer_service.h +++ b/include/dsn/tool-api/timer_service.h @@ -61,7 +61,7 @@ class timer_service public: timer_service(service_node *node, timer_service *inner_provider) { _node = node; } - virtual void start(io_modifer &ctx) = 0; + virtual void start() = 0; // after milliseconds, the provider should call task->enqueue() virtual void add_timer(task *task) = 0; diff --git a/include/dsn/tool/fastrun.h b/include/dsn/tool/fastrun.h deleted file mode 100644 index ac601f5f40..0000000000 --- a/include/dsn/tool/fastrun.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * define the fast run tool for Zion, which aims at - * fast computation of the upper applications - * - * Revision history: - * Aug., 2015, @imzhenyu (Zhenyu Guo), first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#pragma once - -#include - -namespace dsn { -namespace tools { - -class fastrun : public tool_app -{ -public: - fastrun(const char *name) : tool_app(name) {} - - virtual void install(service_spec &s) override; - - virtual void run() override; -}; -} -} // end namespace dsn::tools diff --git a/include/dsn/tool/nfs/nfs_node_simple.h b/include/dsn/tool/nfs/nfs_node_simple.h index 07d82ea2d1..26c80b8c81 100644 --- a/include/dsn/tool/nfs/nfs_node_simple.h +++ b/include/dsn/tool/nfs/nfs_node_simple.h @@ -52,7 +52,7 @@ class nfs_node_simple : public nfs_node virtual void call(std::shared_ptr rci, aio_task *callback) override; - virtual ::dsn::error_code start(io_modifer &ctx) override; + virtual ::dsn::error_code start() override; virtual error_code stop() override; @@ -62,4 +62,4 @@ class nfs_node_simple : public nfs_node nfs_client_impl *_client; }; } -} \ No newline at end of file +} diff --git a/include/dsn/tool/node_scoper.h b/include/dsn/tool/node_scoper.h index 215f212fd8..47c0b10dba 100644 --- a/include/dsn/tool/node_scoper.h +++ b/include/dsn/tool/node_scoper.h @@ -46,7 +46,7 @@ class node_scoper node_scoper(service_node *node) { _old = tls_dsn; - task::set_tls_dsn_context(node, nullptr, nullptr); + task::set_tls_dsn_context(node, nullptr); } ~node_scoper() { tls_dsn = _old; } diff --git a/src/apps/echo/config.ini b/src/apps/echo/config.ini index c4598ee991..c0894b9374 100644 --- a/src/apps/echo/config.ini +++ b/src/apps/echo/config.ini @@ -203,7 +203,7 @@ timer_factory_name = timer_io_mode = ; use what tool to run this process, e.g., native or simulator -tool = fastrun +tool = nativerun ; use what toollets, e.g., tracer, profiler, fault_injector toollets = diff --git a/src/apps/nfs/nfs_node_impl.cpp b/src/apps/nfs/nfs_node_impl.cpp index 5549772480..c26cf0c1dd 100644 --- a/src/apps/nfs/nfs_node_impl.cpp +++ b/src/apps/nfs/nfs_node_impl.cpp @@ -59,7 +59,7 @@ void nfs_node_simple::call(std::shared_ptr rci, aio_task *c _client->begin_remote_copy(rci, callback); // copy file request entry } -error_code nfs_node_simple::start(io_modifer &ctx) +error_code nfs_node_simple::start() { _server = new nfs_service_impl(*_opts); _server->open_service(); diff --git a/src/apps/skv/config.ini b/src/apps/skv/config.ini index a7ddae809d..2e63f4a55f 100644 --- a/src/apps/skv/config.ini +++ b/src/apps/skv/config.ini @@ -44,7 +44,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun ;toollets = tracer ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/core/core/core_main.cpp b/src/core/core/core_main.cpp index 95fe2904f6..70803bd237 100644 --- a/src/core/core/core_main.cpp +++ b/src/core/core/core_main.cpp @@ -38,7 +38,6 @@ #include #include -#include #include #include #include @@ -65,7 +64,6 @@ void dsn_core_init() // register all possible tools and toollets dsn::tools::register_tool("nativerun"); - dsn::tools::register_tool("fastrun"); dsn::tools::register_tool("simulator"); dsn::tools::register_toollet("tracer"); dsn::tools::register_toollet("profiler"); diff --git a/src/core/core/disk_engine.cpp b/src/core/core/disk_engine.cpp index e563711a44..8d1d5d2a71 100644 --- a/src/core/core/disk_engine.cpp +++ b/src/core/core/disk_engine.cpp @@ -160,13 +160,13 @@ disk_engine::disk_engine(service_node *node) disk_engine::~disk_engine() {} -void disk_engine::start(aio_provider *provider, io_modifer &ctx) +void disk_engine::start(aio_provider *provider) { if (_is_running) return; _provider = provider; - _provider->start(ctx); + _provider->start(); _is_running = true; } diff --git a/src/core/core/disk_engine.h b/src/core/core/disk_engine.h index e36c352d93..76be4d2e74 100644 --- a/src/core/core/disk_engine.h +++ b/src/core/core/disk_engine.h @@ -82,7 +82,7 @@ class disk_engine disk_engine(service_node *node); ~disk_engine(); - void start(aio_provider *provider, io_modifer &ctx); + void start(aio_provider *provider); // asynchonous file read/write dsn_handle_t open(const char *file_name, int flag, int pmode); diff --git a/src/core/core/global_config.cpp b/src/core/core/global_config.cpp index e9d622ecfd..701a90ca88 100644 --- a/src/core/core/global_config.cpp +++ b/src/core/core/global_config.cpp @@ -351,23 +351,6 @@ bool service_spec::init_app_specs() // fix ports_gap when necessary int ports_gap = app.ports_gap; - switch (rpc_io_mode) { - case IOE_PER_NODE: - ports_gap *= 1; - break; - case IOE_PER_QUEUE: { - int number_of_ioes = 0; - for (auto &pl : app.pools) { - number_of_ioes += (this->threadpool_specs[pl].partitioned - ? this->threadpool_specs[pl].worker_count - : 1); - } - ports_gap *= number_of_ioes; - } break; - default: - dassert(false, "unsupport io mode"); - break; - } auto ports = app.ports; auto nsc = app.network_server_confs; @@ -400,27 +383,4 @@ bool service_spec::init_app_specs() return true; } - -int service_spec::get_ports_delta(int app_id, dsn::threadpool_code pool, int queue_index) const -{ - dassert(rpc_io_mode == IOE_PER_QUEUE, "only used for IOE_PER_QUEUE mode"); - - auto &aps = app_specs[app_id - 1]; - int number_of_ioes = 0; - for (auto &pl : aps.pools) { - if (pl != pool) { - number_of_ioes += - (this->threadpool_specs[pl].partitioned ? this->threadpool_specs[pl].worker_count - : 1); - } else { - number_of_ioes += (this->threadpool_specs[pl].partitioned ? (queue_index + 1) : 1); - break; - } - } - - dassert(number_of_ioes >= 1, "given pool not started"); - - return aps.ports_gap * (number_of_ioes - 1); -} - } // end namespace dsn diff --git a/src/core/core/lock_checker.cpp b/src/core/core/lock_checker.cpp index 3893b0b00c..85a2a1b6cf 100644 --- a/src/core/core/lock_checker.cpp +++ b/src/core/core/lock_checker.cpp @@ -66,48 +66,29 @@ void check_wait_task(task *waitee) { check_wait_safety(); - if ( - // in worker thread - task::get_current_worker() != nullptr + // not in worker thread + if (task::get_current_worker() == nullptr) + return; - // caller and callee share the same thread pool, - // or caller and io notification which notifies the callee share the same thread pool - && - (waitee->spec().type == TASK_TYPE_RPC_RESPONSE || - (waitee->spec().pool_code == task::get_current_worker()->pool_spec().pool_code) || - ( - // assuming wait is with the same thread when the io call is emitted - // in this case io notificaiton is received by the current thread - (waitee->spec().type == TASK_TYPE_AIO && - ::dsn::tools::spec().disk_io_mode == IOE_PER_QUEUE))) + // caller and callee don't share the same thread pool, + if (waitee->spec().type != TASK_TYPE_RPC_RESPONSE && + (waitee->spec().pool_code != task::get_current_worker()->pool_spec().pool_code)) + return; - // callee is not empty or we are using IOE_PER_QUEUE mode (io received by current thread) - && - (!waitee->is_empty() || ((waitee->spec().type == TASK_TYPE_AIO && - ::dsn::tools::spec().disk_io_mode == IOE_PER_QUEUE) || - (waitee->spec().type == TASK_TYPE_RPC_RESPONSE && - ::dsn::tools::spec().rpc_io_mode == IOE_PER_QUEUE))) + // callee is empty + if (waitee->is_empty()) + return; - // not much concurrency in the current pool - && - (task::get_current_worker()->pool_spec().partitioned || - task::get_current_worker()->pool_spec().worker_count == 1)) { - if ((::dsn::tools::spec().rpc_io_mode == IOE_PER_QUEUE || - ::dsn::tools::spec().disk_io_mode == IOE_PER_QUEUE) && - (task::get_current_worker()->pool_spec().partitioned || - task::get_current_worker()->pool_spec().worker_count == 1)) { - dwarn("cannot call task wait in worker thread '%s' that also serves as io thread " - "(disk/rpc_io_mode == IOE_PER_QUEUE) " - "when the thread pool is partitioned or the worker thread number is 1", - task::get_current_worker()->name().c_str()); - } else { - dwarn("task %s waits for another task %s sharing the same thread pool " - "- will lead to deadlocks easily (e.g., when worker_count = 1 or when the pool " - "is partitioned)", - task::get_current_task()->spec().code.to_string(), - waitee->spec().code.to_string()); - } - } + // there are enough concurrency + if (!task::get_current_worker()->pool_spec().partitioned && + task::get_current_worker()->pool_spec().worker_count > 1) + return; + + dwarn("task %s waits for another task %s sharing the same thread pool " + "- will lead to deadlocks easily (e.g., when worker_count = 1 or when the pool " + "is partitioned)", + task::get_current_task()->spec().code.to_string(), + waitee->spec().code.to_string()); } } } diff --git a/src/core/core/rpc_engine.cpp b/src/core/core/rpc_engine.cpp index cad235d1db..14dbc23f4a 100644 --- a/src/core/core/rpc_engine.cpp +++ b/src/core/core/rpc_engine.cpp @@ -429,8 +429,7 @@ rpc_engine::rpc_engine(service_node *node) : _node(node), _rpc_matcher(this) // network *rpc_engine::create_network(const network_server_config &netcs, bool client_only, - network_header_format client_hdr_format, - io_modifer &ctx) + network_header_format client_hdr_format) { const service_spec &spec = service_engine::fast_instance().spec(); network *net = utils::factory_store::create( @@ -443,7 +442,7 @@ network *rpc_engine::create_network(const network_server_config &netcs, } // start the net - error_code ret = net->start(netcs.channel, netcs.port + ctx.port_shift_value, client_only, ctx); + error_code ret = net->start(netcs.channel, netcs.port, client_only); if (ret == ERR_OK) { return net; } else { @@ -453,15 +452,12 @@ network *rpc_engine::create_network(const network_server_config &netcs, } } -error_code rpc_engine::start(const service_app_spec &aspec, io_modifer &ctx) +error_code rpc_engine::start(const service_app_spec &aspec) { if (_is_running) { return ERR_SERVICE_ALREADY_RUNNING; } - // local cache for shared networks with same provider and message format and port - std::map named_nets; // factory##fmt##port -> net - // start client networks _client_nets.resize(network_header_format::max_value() + 1); @@ -491,25 +487,16 @@ error_code rpc_engine::start(const service_app_spec &aspec, io_modifer &ctx) cs.factory_name = factory; cs.message_buffer_block_size = blk_size; - auto net = create_network(cs, true, client_hdr_format, ctx); + auto net = create_network(cs, true, client_hdr_format); if (!net) return ERR_NETWORK_INIT_FAILED; pnet[j] = net; - if (ctx.queue) { - ddebug("[%s.%s] network client started at port %u, channel = %s, fmt = %s ...", - node()->full_name(), - ctx.queue->get_name().c_str(), - (uint32_t)(cs.port + ctx.port_shift_value), - cs.channel.to_string(), - client_hdr_format.to_string()); - } else { - ddebug("[%s] network client started at port %u, channel = %s, fmt = %s ...", - node()->full_name(), - (uint32_t)(cs.port + ctx.port_shift_value), - cs.channel.to_string(), - client_hdr_format.to_string()); - } + ddebug("[%s] network client started at port %u, channel = %s, fmt = %s ...", + node()->full_name(), + (uint32_t)(cs.port), + cs.channel.to_string(), + client_hdr_format.to_string()); } } @@ -530,32 +517,23 @@ error_code rpc_engine::start(const service_app_spec &aspec, io_modifer &ctx) pnets = &it->second; } - auto net = create_network(sp.second, false, NET_HDR_DSN, ctx); + auto net = create_network(sp.second, false, NET_HDR_DSN); if (net == nullptr) { return ERR_NETWORK_INIT_FAILED; } (*pnets)[sp.second.channel] = net; - if (ctx.queue) { - dwarn("[%s.%s] network server started at port %u, channel = %s, ...", - node()->full_name(), - ctx.queue->get_name().c_str(), - (uint32_t)(port + ctx.port_shift_value), - sp.second.channel.to_string()); - } else { - dwarn("[%s] network server started at port %u, channel = %s, ...", - node()->full_name(), - (uint32_t)(port + ctx.port_shift_value), - sp.second.channel.to_string()); - } + dwarn("[%s] network server started at port %u, channel = %s, ...", + node()->full_name(), + (uint32_t)(port), + sp.second.channel.to_string()); } _uri_resolver_mgr.reset(new uri_resolver_manager()); _local_primary_address = _client_nets[NET_HDR_DSN][0]->address(); - _local_primary_address.set_port(aspec.ports.size() > 0 ? *aspec.ports.begin() - : aspec.id + ctx.port_shift_value); + _local_primary_address.set_port(aspec.ports.size() > 0 ? *aspec.ports.begin() : aspec.id); ddebug("=== service_node=[%s], primary_address=[%s] ===", _node->full_name(), diff --git a/src/core/core/rpc_engine.h b/src/core/core/rpc_engine.h index 11cfc26d99..ee1bfc8473 100644 --- a/src/core/core/rpc_engine.h +++ b/src/core/core/rpc_engine.h @@ -145,7 +145,7 @@ class rpc_engine // // management routines // - ::dsn::error_code start(const service_app_spec &spec, io_modifer &ctx); + ::dsn::error_code start(const service_app_spec &spec); void start_serving() { _is_serving = true; } // @@ -190,8 +190,7 @@ class rpc_engine private: network *create_network(const network_server_config &netcs, bool client_only, - network_header_format client_hdr_format, - io_modifer &ctx); + network_header_format client_hdr_format); private: service_node *_node; diff --git a/src/core/core/service_api_c.cpp b/src/core/core/service_api_c.cpp index 8de57937cb..daa6508f85 100644 --- a/src/core/core/service_api_c.cpp +++ b/src/core/core/service_api_c.cpp @@ -588,7 +588,7 @@ DSN_API bool dsn_mimic_app(const char *app_role, int index) for (auto &n : nodes) { if (n.second->spec().role_name == std::string(app_role) && n.second->spec().index == index) { - ::dsn::task::set_tls_dsn_context(n.second, nullptr, nullptr); + ::dsn::task::set_tls_dsn_context(n.second, nullptr); return true; } } @@ -682,7 +682,7 @@ bool run(const char *config_file, "dsn_runtime_init_time_ms = %" PRIu64 " (%s)", dsn_runtime_init_time_ms(), init_time_buf); dsn_core_init(); - ::dsn::task::set_tls_dsn_context(nullptr, nullptr, nullptr); + ::dsn::task::set_tls_dsn_context(nullptr, nullptr); dsn_all.engine_ready = false; dsn_all.config_completed = false; diff --git a/src/core/core/service_engine.cpp b/src/core/core/service_engine.cpp index 345d18cee6..659cf2e07e 100644 --- a/src/core/core/service_engine.cpp +++ b/src/core/core/service_engine.cpp @@ -62,158 +62,87 @@ bool service_node::rpc_register_handler(task_code code, const char *extra_name, const rpc_request_handler &h) { - for (auto &io : _ios) { - if (io.rpc) { - bool r = io.rpc->register_rpc_handler(code, extra_name, h); - if (!r) - return false; - } - } - return true; + return _node_io.rpc->register_rpc_handler(code, extra_name, h); } bool service_node::rpc_unregister_handler(dsn::task_code rpc_code) { - for (auto &io : _ios) { - if (io.rpc) { - bool r = io.rpc->unregister_rpc_handler(rpc_code); - if (!r) { - dassert(false, "unregister rpc failed(%s)", rpc_code.to_string()); - } - } - } - return true; + return _node_io.rpc->unregister_rpc_handler(rpc_code); } -error_code service_node::init_io_engine(io_engine &io, ioe_mode mode) +error_code service_node::init_io_engine() { auto &spec = service_engine::fast_instance().spec(); error_code err = ERR_OK; - io_modifer ctx; - ctx.queue = io.q; - ctx.port_shift_value = 0; - ctx.mode = mode; // init timer service - if (mode == spec.timer_io_mode) { - io.tsvc = factory_store::create( - service_engine::fast_instance().spec().timer_factory_name.c_str(), - PROVIDER_TYPE_MAIN, - this, - nullptr); - for (auto &s : service_engine::fast_instance().spec().timer_aspects) { - io.tsvc = factory_store::create( - s.c_str(), PROVIDER_TYPE_ASPECT, this, io.tsvc); - } - } else - io.tsvc = nullptr; + _node_io.tsvc = factory_store::create( + service_engine::fast_instance().spec().timer_factory_name.c_str(), + PROVIDER_TYPE_MAIN, + this, + nullptr); + for (auto &s : service_engine::fast_instance().spec().timer_aspects) { + _node_io.tsvc = factory_store::create( + s.c_str(), PROVIDER_TYPE_ASPECT, this, _node_io.tsvc); + } // init disk engine - if (mode == spec.disk_io_mode) { - io.disk = new disk_engine(this); - aio_provider *aio = factory_store::create( - spec.aio_factory_name.c_str(), ::dsn::PROVIDER_TYPE_MAIN, io.disk, nullptr); - for (auto it = spec.aio_aspects.begin(); it != spec.aio_aspects.end(); it++) { - aio = factory_store::create( - it->c_str(), PROVIDER_TYPE_ASPECT, io.disk, aio); - } - io.aio = aio; - } else - io.aio = nullptr; + _node_io.disk = new disk_engine(this); + aio_provider *aio = factory_store::create( + spec.aio_factory_name.c_str(), ::dsn::PROVIDER_TYPE_MAIN, _node_io.disk, nullptr); + for (auto it = spec.aio_aspects.begin(); it != spec.aio_aspects.end(); it++) { + aio = factory_store::create( + it->c_str(), PROVIDER_TYPE_ASPECT, _node_io.disk, aio); + } + _node_io.aio = aio; // init rpc engine - if (mode == spec.rpc_io_mode) { - if (ctx.mode == IOE_PER_QUEUE) { - // update ports if there are more than one rpc engines for one node - ctx.port_shift_value = - spec.get_ports_delta(_app_spec.id, io.pool->spec().pool_code, io.q->index()); - } - io.rpc = new rpc_engine(this); - } else - io.rpc = nullptr; + _node_io.rpc = new rpc_engine(this); // init nfs - io.nfs = nullptr; - if (mode == spec.nfs_io_mode) { - if (!spec.start_nfs) { - ddebug("nfs not started coz [core] start_nfs = false"); - } else if (spec.nfs_factory_name == "") { - dwarn("nfs not started coz no nfs_factory_name is specified," - " continue with no nfs"); - } else { - io.nfs = factory_store::create( - spec.nfs_factory_name.c_str(), PROVIDER_TYPE_MAIN, this); - } + _node_io.nfs = nullptr; + if (!spec.start_nfs) { + ddebug("nfs not started coz [core] start_nfs = false"); + } else if (spec.nfs_factory_name == "") { + dwarn("nfs not started coz no nfs_factory_name is specified," + " continue with no nfs"); + } else { + _node_io.nfs = factory_store::create( + spec.nfs_factory_name.c_str(), PROVIDER_TYPE_MAIN, this); } return err; } -error_code service_node::start_io_engine_in_main(const io_engine &io) +error_code service_node::start_io_engine_in_main() { auto &spec = service_engine::fast_instance().spec(); error_code err = ERR_OK; - io_modifer ctx; - ctx.queue = io.q; - ctx.port_shift_value = 0; // start timer service - if (io.tsvc) { - ctx.mode = spec.timer_io_mode; - io.tsvc->start(ctx); - } + _node_io.tsvc->start(); // start disk engine - if (io.disk) { - ctx.mode = spec.disk_io_mode; - io.disk->start(io.aio, ctx); - } + _node_io.disk->start(_node_io.aio); // start rpc engine - if (io.rpc) { - ctx.mode = spec.rpc_io_mode; - if (ctx.mode == IOE_PER_QUEUE) { - // update ports if there are more than one rpc engines for one node - ctx.port_shift_value = - spec.get_ports_delta(_app_spec.id, io.pool->spec().pool_code, io.q->index()); - } - err = io.rpc->start(_app_spec, ctx); - if (err != ERR_OK) - return err; - } - + err = _node_io.rpc->start(_app_spec); return err; } -error_code service_node::start_io_engine_in_node_start_task(const io_engine &io) +error_code service_node::start_io_engine_in_node_start_task() { - auto &spec = service_engine::fast_instance().spec(); error_code err = ERR_OK; - io_modifer ctx; - ctx.queue = io.q; - ctx.port_shift_value = 0; - - // start nfs delayed when the app is started - if (io.nfs) { - ctx.mode = spec.nfs_io_mode; - if (ctx.mode == IOE_PER_QUEUE) { - // update ports if there are more than one rpc engines for one node - ctx.port_shift_value = - spec.get_ports_delta(_app_spec.id, io.pool->spec().pool_code, io.q->index()); - } - - err = io.nfs->start(ctx); - if (err != ERR_OK) - return err; + if (_node_io.nfs) { + err = _node_io.nfs->start(); } - return err; } dsn::error_code service_node::start_app() { dassert(_entity.get(), "entity hasn't initialized"); - _entity->set_address(node_rpc()->primary_address()); + _entity->set_address(rpc()->primary_address()); std::vector args; utils::split_args(spec().arguments.c_str(), args); @@ -260,35 +189,12 @@ error_code service_node::start() _computation->create(_app_spec.pools); dassert(!_computation->is_started(), "task engine must not be started at this point"); - // init per node io engines - err = init_io_engine(_per_node_io, IOE_PER_NODE); + err = init_io_engine(); if (err != ERR_OK) return err; - _ios.push_back(_per_node_io); - - // init per queue io engines - for (auto &pl : _computation->pools()) { - if (pl == nullptr) - continue; - - for (auto &q : pl->queues()) { - io_engine io; - io.q = q; - io.pool = pl; - - err = init_io_engine(io, IOE_PER_QUEUE); - if (err != ERR_OK) - return err; - _per_queue_ios[q] = io; - - _ios.push_back(io); - } - } // start io engines (only timer, disk and rpc), others are started in app start task - for (auto &io : _ios) { - start_io_engine_in_main(io); - } + start_io_engine_in_main(); // start task engine _computation->start(); @@ -301,65 +207,11 @@ error_code service_node::start() } // start rpc serving - for (auto &io : _ios) { - if (io.rpc) - io.rpc->start_serving(); - } + _node_io.rpc->start_serving(); return err; } -void service_node::get_io(ioe_mode mode, task_queue *q, /*out*/ io_engine &io) const -{ - switch (mode) { - case IOE_PER_NODE: - io = _per_node_io; - break; - case IOE_PER_QUEUE: - if (q) { - auto it = _per_queue_ios.find(q); - dassert(it != _per_queue_ios.end(), "io engine must be created for the queue"); - io = it->second; - } else { - // nothing to do - } - break; - default: - dassert(false, "invalid io mode"); - } -} -rpc_engine *service_node::rpc(task_queue *q) const -{ - auto &spec = service_engine::fast_instance().spec(); - io_engine io; - get_io(spec.rpc_io_mode, q, io); - return io.rpc; -} - -disk_engine *service_node::disk(task_queue *q) const -{ - auto &spec = service_engine::fast_instance().spec(); - io_engine io; - get_io(spec.disk_io_mode, q, io); - return io.disk; -} - -nfs_node *service_node::nfs(task_queue *q) const -{ - auto &spec = service_engine::fast_instance().spec(); - io_engine io; - get_io(spec.nfs_io_mode, q, io); - return io.nfs; -} - -timer_service *service_node::tsvc(task_queue *q) const -{ - auto &spec = service_engine::fast_instance().spec(); - io_engine io; - get_io(spec.timer_io_mode, q, io); - return io.tsvc; -} - void service_node::get_runtime_info(const std::string &indent, const std::vector &args, /*out*/ std::stringstream &ss) @@ -454,20 +306,12 @@ void service_engine::register_system_rpc_handler(dsn::task_code code, { if (port == -1) { for (auto &n : _nodes_by_app_id) { - for (auto &io : n.second->ios()) { - if (io.rpc) { - io.rpc->register_rpc_handler(code, name, cb); - } - } + n.second->rpc_register_handler(code, name, cb); } } else { auto it = _nodes_by_app_port.find(port); if (it != _nodes_by_app_port.end()) { - for (auto &io : it->second->ios()) { - if (io.rpc) { - io.rpc->register_rpc_handler(code, name, cb); - } - } + it->second->rpc_register_handler(code, name, cb); } else { dwarn("cannot find service node with port %d", port); } diff --git a/src/core/core/service_engine.h b/src/core/core/service_engine.h index 3e05df1016..46b9cce2d3 100644 --- a/src/core/core/service_engine.h +++ b/src/core/core/service_engine.h @@ -70,9 +70,6 @@ class service_node disk_engine *disk; nfs_node *nfs; timer_service *tsvc; - - task_queue *q; - task_worker_pool *pool; aio_provider *aio; io_engine() { memset((void *)this, 0, sizeof(io_engine)); } @@ -81,24 +78,18 @@ class service_node public: explicit service_node(service_app_spec &app_spec); - rpc_engine *rpc(task_queue *q) const; - disk_engine *disk(task_queue *q) const; - nfs_node *nfs(task_queue *q) const; - timer_service *tsvc(task_queue *q) const; - - rpc_engine *node_rpc() const { return _per_node_io.rpc; } - disk_engine *node_disk() const { return _per_node_io.disk; } - nfs_node *node_nfs() const { return _per_node_io.nfs; } - timer_service *node_tsvc() const { return _per_node_io.tsvc; } + rpc_engine *rpc() const { return _node_io.rpc; } + disk_engine *disk() const { return _node_io.disk; } + nfs_node *nfs() const { return _node_io.nfs; } + timer_service *tsvc() const { return _node_io.tsvc; } task_engine *computation() const { return _computation; } - const std::list &ios() const { return _ios; } void get_runtime_info(const std::string &indent, const std::vector &args, /*out*/ std::stringstream &ss); void get_queue_info(/*out*/ std::stringstream &ss); - error_code start_io_engine_in_node_start_task(const io_engine &io); + error_code start_io_engine_in_node_start_task(); dsn::error_code start(); dsn::error_code start_app(); @@ -121,18 +112,15 @@ class service_node service_app_spec _app_spec; task_engine *_computation; - io_engine _per_node_io; - std::unordered_map _per_queue_ios; - std::list _ios; // all ios + io_engine _node_io; private: // the service entity is initialized after the engine // is initialized, so this should be call in start() void init_service_app(); - error_code init_io_engine(io_engine &io, ioe_mode mode); - error_code start_io_engine_in_main(const io_engine &io); - void get_io(ioe_mode mode, task_queue *q, /*out*/ io_engine &io) const; + error_code init_io_engine(); + error_code start_io_engine_in_main(); }; typedef std::map service_nodes_by_app_id; diff --git a/src/core/core/task.cpp b/src/core/core/task.cpp index 6d053274cc..8e138b234d 100644 --- a/src/core/core/task.cpp +++ b/src/core/core/task.cpp @@ -51,11 +51,8 @@ namespace dsn { __thread struct __tls_dsn__ tls_dsn; __thread uint16_t tls_dsn_lower32_task_id_mask = 0; -/*static*/ void task::set_tls_dsn_context( - service_node *node, // cannot be null - task_worker *worker, // null for io or timer threads if they are not worker threads - task_queue *queue // owner queue if io_mode == IOE_PER_QUEUE - ) +/*static*/ void task::set_tls_dsn_context(service_node *node, // cannot be null + task_worker *worker) { memset(static_cast(&tls_dsn), 0, sizeof(tls_dsn)); tls_dsn.magic = 0xdeadbeef; @@ -71,22 +68,15 @@ __thread uint16_t tls_dsn_lower32_task_id_mask = 0; node->full_name()); } - if (queue != nullptr) { - dassert(queue->pool()->node() == node, - "queue not belonging to the given node: %s vs %s", - queue->pool()->node()->full_name(), - node->full_name()); - } - tls_dsn.node = node; tls_dsn.worker = worker; tls_dsn.worker_index = worker ? worker->index() : -1; tls_dsn.current_task = nullptr; - tls_dsn.rpc = node->rpc(queue ? queue : (worker ? worker->queue() : nullptr)); - tls_dsn.disk = node->disk(queue ? queue : (worker ? worker->queue() : nullptr)); + tls_dsn.rpc = node->rpc(); + tls_dsn.disk = node->disk(); tls_dsn.env = service_engine::fast_instance().env(); - tls_dsn.nfs = node->nfs(queue ? queue : (worker ? worker->queue() : nullptr)); - tls_dsn.tsvc = node->tsvc(queue ? queue : (worker ? worker->queue() : nullptr)); + tls_dsn.nfs = node->nfs(); + tls_dsn.tsvc = node->tsvc(); } tls_dsn.node_pool_thread_ids = (node ? ((uint64_t)(uint8_t)node->id()) : 0) @@ -143,7 +133,7 @@ task::task(dsn::task_code code, int hash, service_node *node) } if (tls_dsn.magic != 0xdeadbeef) { - set_tls_dsn_context(nullptr, nullptr, nullptr); + set_tls_dsn_context(nullptr, nullptr); } _task_id = tls_dsn.node_pool_thread_ids + (++tls_dsn.last_lower32_task_id); @@ -582,8 +572,6 @@ aio_task::aio_task(dsn::task_code code, aio_handler &&cb, int hash, service_node set_error_code(ERR_IO_PENDING); auto disk = get_current_disk(); - if (!disk) - disk = node->node_disk(); _aio = disk->prepare_aio_context(this); } diff --git a/src/core/core/task_engine.cpp b/src/core/core/task_engine.cpp index 27614c5f31..dda478bf92 100644 --- a/src/core/core/task_engine.cpp +++ b/src/core/core/task_engine.cpp @@ -45,7 +45,7 @@ task_worker_pool::task_worker_pool(const threadpool_spec &opts, task_engine *own : _spec(opts), _owner(owner), _node(owner->node()) { _is_running = false; - _per_node_timer_svc = nullptr; + _timer_svc = nullptr; } void task_worker_pool::create() @@ -113,16 +113,7 @@ void task_worker_pool::start() _spec.partitioned ? "true" : "false"); // setup cached ptrs for fast timer service access - if (service_engine::fast_instance().spec().timer_io_mode == IOE_PER_QUEUE) { - for (size_t i = 0; i < _queues.size(); i++) { - auto svc = node()->tsvc(_queues[i]); - dassert(svc, "per queue timer service must be present"); - _per_queue_timer_svcs.push_back(svc); - } - } else { - _per_node_timer_svc = node()->tsvc(nullptr); - } - + _timer_svc = node()->tsvc(); _is_running = true; } @@ -131,15 +122,7 @@ void task_worker_pool::add_timer(task *t) dassert(t->delay_milliseconds() > 0, "task delayed should be dispatched to timer service first"); - if (_per_node_timer_svc) - _per_node_timer_svc->add_timer(t); - else { - unsigned int idx = - (_spec.partitioned - ? static_cast(t->hash()) % static_cast(_queues.size()) - : 0); - _per_queue_timer_svcs[idx]->add_timer(t); - } + _timer_svc->add_timer(t); } void task_worker_pool::enqueue(task *t) diff --git a/src/core/core/task_engine.h b/src/core/core/task_engine.h index ed652fefbd..f839cea02a 100644 --- a/src/core/core/task_engine.h +++ b/src/core/core/task_engine.h @@ -91,7 +91,7 @@ class task_worker_pool std::vector _controllers; // cached ptrs for fast access - timer_service *_per_node_timer_svc; + timer_service *_timer_svc; std::vector _per_queue_timer_svcs; bool _is_running; diff --git a/src/core/core/task_worker.cpp b/src/core/core/task_worker.cpp index 6253b2aae7..c015b8b8dc 100644 --- a/src/core/core/task_worker.cpp +++ b/src/core/core/task_worker.cpp @@ -266,7 +266,7 @@ void task_worker::run_internal() std::this_thread::sleep_for(std::chrono::milliseconds(1)); } - task::set_tls_dsn_context(pool()->node(), this, queue()); + task::set_tls_dsn_context(pool()->node(), this); _native_tid = ::dsn::utils::get_current_tid(); set_name(name().c_str()); diff --git a/src/core/core/tool_api.cpp b/src/core/core/tool_api.cpp index 97968d6a93..de85da98fe 100644 --- a/src/core/core/tool_api.cpp +++ b/src/core/core/tool_api.cpp @@ -57,9 +57,7 @@ class service_control_task : public task if (_start) { error_code err; - for (auto &io : _node->ios()) { - _node->start_io_engine_in_node_start_task(io); - } + err = _node->start_io_engine_in_node_start_task(); err = _node->start_app(); dassert(err == ERR_OK, "start app failed, err = %s", err.to_string()); diff --git a/src/core/perf.tests/config-test.ini b/src/core/perf.tests/config-test.ini index a486ff55a4..6104afef3e 100644 --- a/src/core/perf.tests/config-test.ini +++ b/src/core/perf.tests/config-test.ini @@ -32,9 +32,7 @@ pools = THREAD_POOL_DEFAULT, THREAD_POOL_TEST_SERVER [core] ;tool = simulator -;tool = nativerun -tool = fastrun - +tool = nativerun ;toollets = tracer, profiler, fault_injector pause_on_start = false cli_local = false diff --git a/src/core/perf.tests/logger.cpp b/src/core/perf.tests/logger.cpp index 2cd86f355d..991cb33301 100644 --- a/src/core/perf.tests/logger.cpp +++ b/src/core/perf.tests/logger.cpp @@ -44,8 +44,6 @@ #include "service_engine.h" #include "test_utils.h" -#include "../tools/hpc/hpc_logger.h" -#include "../tools/hpc/hpc_tail_logger.h" #include "../tools/common/simple_logger.h" using namespace ::dsn; @@ -70,7 +68,7 @@ void logger_test(int thread_count, int record_count) for (int i = 0; i < thread_count; ++i) { threads.push_back(new std::thread([&, i] { - task::set_tls_dsn_context(task::get_current_node2(), nullptr, nullptr); + task::set_tls_dsn_context(task::get_current_node2(), nullptr); for (int j = 0; j < record_count; j++) { logv(&logger, str, j, i); } @@ -108,22 +106,3 @@ TEST(core, simple_logger_test) logger_test(i, 100000); } } - -TEST(core, hpc_logger_test) -{ - std::cout << "thread_count\t\t record_count\t\t speed" << std::endl; - - auto threads_count = {1, 2, 5, 10}; - for (int i : threads_count) - logger_test(i, 100000); -} - -TEST(core, hpc_tail_logger_test) -{ - std::cout << "thread_count\t\t record_count\t\t speed" << std::endl; - - auto threads_count = {1, 2, 5, 10}; - for (int i : threads_count) { - logger_test(i, 10000); - } -} diff --git a/src/core/tests/CMakeLists.txt b/src/core/tests/CMakeLists.txt index 12ccf75e33..d77d4fed4c 100644 --- a/src/core/tests/CMakeLists.txt +++ b/src/core/tests/CMakeLists.txt @@ -29,7 +29,6 @@ set(MY_BINPLACES "${CMAKE_CURRENT_SOURCE_DIR}/config-bad-section.ini" "${CMAKE_CURRENT_SOURCE_DIR}/config-sample.ini" "${CMAKE_CURRENT_SOURCE_DIR}/config-test-corrupt-message.ini" "${CMAKE_CURRENT_SOURCE_DIR}/config-test.ini" - "${CMAKE_CURRENT_SOURCE_DIR}/config-test-fastrun.ini" "${CMAKE_CURRENT_SOURCE_DIR}/config-test-posix-aio.ini" "${CMAKE_CURRENT_SOURCE_DIR}/config-test-sim.ini" "${CMAKE_CURRENT_SOURCE_DIR}/config-unmatch-section.ini" diff --git a/src/core/tests/config-test-corrupt-message.ini b/src/core/tests/config-test-corrupt-message.ini index a5ee720c8c..37f6492c87 100644 --- a/src/core/tests/config-test-corrupt-message.ini +++ b/src/core/tests/config-test-corrupt-message.ini @@ -13,7 +13,7 @@ run = true ports = 20001 count = 1 delay_seconds = 1 -pools = THREAD_POOL_DEFAULT, THREAD_POOL_TEST_SERVER, THREAD_POOL_TEST_SERVER_2, THREAD_POOL_FOR_TEST_1, THREAD_POOL_FOR_TEST_2 +pools = THREAD_POOL_DEFAULT, THREAD_POOL_TEST_SERVER, THREAD_POOL_FOR_TEST_1, THREAD_POOL_FOR_TEST_2 [apps.server] type = test @@ -30,7 +30,6 @@ network.server.20103.RPC_CHANNEL_TCP = dsn::tools::asio_network_provider,65536 [core] ;tool = simulator tool = nativerun -;tool = fastrun toollets = fault_injector pause_on_start = false diff --git a/src/core/tests/config-test-fastrun.ini b/src/core/tests/config-test-fastrun.ini deleted file mode 100644 index 3304bd5ae7..0000000000 --- a/src/core/tests/config-test-fastrun.ini +++ /dev/null @@ -1,80 +0,0 @@ -[apps..default] -run = true -count = 1 -;network.client.RPC_CHANNEL_TCP = dsn::tools::sim_network_provider, 65536 -;network.client.RPC_CHANNEL_UDP = dsn::tools::sim_network_provider, 65536 -;network.server.0.RPC_CHANNEL_TCP = dsn::tools::sim_network_provider, 65536 - -[apps.client] -type = test -arguments = localhost 20101 -run = true -ports = -count = 1 -delay_seconds = 1 -pools = THREAD_POOL_DEFAULT, THREAD_POOL_TEST_SERVER, THREAD_POOL_TEST_SERVER_2, THREAD_POOL_FOR_TEST_1, THREAD_POOL_FOR_TEST_2 - -[core] -;tool = simulator -;tool = nativerun -tool = fastrun - -toollets = tracer, profiler -pause_on_start = false -cli_local = true -cli_remote = true - -logging_start_level = LOG_LEVEL_INFORMATION -logging_factory_name = dsn::tools::simple_logger - -disk_io_mode = IOE_PER_QUEUE -;rpc_io_mode = IOE_PER_QUEUE -nfs_io_mode = IOE_PER_QUEUE -timer_io_mode = IOE_PER_QUEUE - -io_worker_count = 1 - -[tools.simple_logger] -fast_flush = true -short_header = false -stderr_start_level = LOG_LEVEL_FATAL - -[tools.simulator] -random_seed = 0 - -[network] -; how many network threads for network library (used by asio) -io_service_worker_count = 2 - -[task..default] -is_trace = true -is_profile = true -allow_inline = false -rpc_call_channel = RPC_CHANNEL_TCP -rpc_message_header_format = dsn -rpc_timeout_milliseconds = 1000 - -[task.LPC_AIO_IMMEDIATE_CALLBACK] -is_trace = false -is_profile = false -allow_inline = false - -[task.LPC_RPC_TIMEOUT] -is_trace = false -is_profile = false - -; specification for each thread pool -[threadpool..default] -worker_count = 2 - -[threadpool.THREAD_POOL_DEFAULT] -partitioned = false -; max_input_queue_length = 1024 -worker_priority = THREAD_xPRIORITY_NORMAL - -[threadpool.THREAD_POOL_TEST_SERVER] -partitioned = false - -[uri-resolver.http://localhost:8080] -factory = partition_resolver_simple -arguments = 127.0.0.1:8080 diff --git a/src/core/tests/config-test-posix-aio.ini b/src/core/tests/config-test-posix-aio.ini index 72ed0543e4..5de0c7ca92 100644 --- a/src/core/tests/config-test-posix-aio.ini +++ b/src/core/tests/config-test-posix-aio.ini @@ -12,12 +12,11 @@ run = true ports = count = 1 delay_seconds = 1 -pools = THREAD_POOL_DEFAULT, THREAD_POOL_TEST_SERVER, THREAD_POOL_TEST_SERVER_2, THREAD_POOL_FOR_TEST_1, THREAD_POOL_FOR_TEST_2 +pools = THREAD_POOL_DEFAULT, THREAD_POOL_TEST_SERVER, THREAD_POOL_FOR_TEST_1, THREAD_POOL_FOR_TEST_2 [core] ;tool = simulator tool = nativerun -;tool = fastrun toollets = tracer, profiler pause_on_start = false diff --git a/src/core/tests/config-test-sim.ini b/src/core/tests/config-test-sim.ini index 5c973df135..da9a7d1dca 100644 --- a/src/core/tests/config-test-sim.ini +++ b/src/core/tests/config-test-sim.ini @@ -12,7 +12,7 @@ run = true ports = count = 1 delay_seconds = 1 -pools = THREAD_POOL_DEFAULT, THREAD_POOL_TEST_SERVER, THREAD_POOL_TEST_SERVER_2, THREAD_POOL_FOR_TEST_1, THREAD_POOL_FOR_TEST_2 +pools = THREAD_POOL_DEFAULT, THREAD_POOL_TEST_SERVER, THREAD_POOL_FOR_TEST_1, THREAD_POOL_FOR_TEST_2 [apps.server] type = test @@ -33,7 +33,6 @@ pools = THREAD_POOL_DEFAULT, THREAD_POOL_TEST_SERVER [core] tool = simulator ;tool = nativerun -;tool = fastrun toollets = tracer, profiler pause_on_start = false @@ -91,9 +90,6 @@ worker_priority = THREAD_xPRIORITY_NORMAL [threadpool.THREAD_POOL_TEST_SERVER] partitioned = false -[threadpool.THREAD_POOL_TEST_SERVER_2] -partitioned = false - [threadpool.THREAD_POOL_FOR_TEST_1] worker_count = 2 partitioned = false diff --git a/src/core/tests/config-test.ini b/src/core/tests/config-test.ini index 4552d7aa52..3ec2850348 100644 --- a/src/core/tests/config-test.ini +++ b/src/core/tests/config-test.ini @@ -13,7 +13,7 @@ run = true ports = 20001 count = 1 delay_seconds = 1 -pools = THREAD_POOL_DEFAULT, THREAD_POOL_TEST_SERVER, THREAD_POOL_TEST_SERVER_2, THREAD_POOL_FOR_TEST_1, THREAD_POOL_FOR_TEST_2 +pools = THREAD_POOL_DEFAULT, THREAD_POOL_TEST_SERVER, THREAD_POOL_FOR_TEST_1, THREAD_POOL_FOR_TEST_2 [apps.server] type = test @@ -46,7 +46,6 @@ pools = THREAD_POOL_DEFAULT, THREAD_POOL_TEST_SERVER [core] ;tool = simulator tool = nativerun -;tool = fastrun toollets = tracer, profiler pause_on_start = false @@ -106,11 +105,6 @@ worker_priority = THREAD_xPRIORITY_NORMAL partitioned = false admission_controller_factory_name = dsn::tools::admission_controller_for_test -[threadpool.THREAD_POOL_TEST_SERVER_2] -partitioned = false -queue_factory_name = dsn::tools::hpc_task_queue -worker_factory_name = dsn::task_worker - [threadpool.THREAD_POOL_FOR_TEST_1] worker_count = 2 worker_priority = THREAD_xPRIORITY_HIGHEST diff --git a/src/core/tests/gtest.filter b/src/core/tests/gtest.filter index e76adb78a3..81c4cb1514 100644 --- a/src/core/tests/gtest.filter +++ b/src/core/tests/gtest.filter @@ -1,4 +1,3 @@ config-test.ini -core.corrupt_message:core.aio*:core.operation_failed:tools_hpc.* config-test-sim.ini -core.corrupt_message:core.aio*:core.operation_failed:tools_hpc.* -config-test-fastrun.ini core.aio*:core.operation_failed:tools_hpc.* config-test-posix-aio.ini core.aio*:core.operation_failed diff --git a/src/core/tests/hpc_aio_provider.cpp b/src/core/tests/hpc_aio_provider.cpp deleted file mode 100644 index 5702ba959a..0000000000 --- a/src/core/tests/hpc_aio_provider.cpp +++ /dev/null @@ -1,154 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * Unit-test for hpc aio provider. - * - * Revision history: - * Nov., 2015, @xiaotz (Xiaotong Zhang), first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#include -#include -#include - -#include -#include -#include -#include -#include -#include "../core/disk_engine.h" -#include "test_utils.h" -#include "../tools/hpc/hpc_aio_provider.h" - -using namespace ::dsn; - -DEFINE_TASK_CODE_AIO(LPC_AIO_TEST, TASK_PRIORITY_COMMON, THREAD_POOL_TEST_SERVER); - -TEST(tools_hpc, aio) -{ - if (nullptr == task::get_current_disk()) - return; - - std::remove("test_hpc_aio.tmp"); // delete file - - // write to file - char buffer[11]; - int count = 10; - sprintf(buffer, "abcdefghij"); - dsn_handle_t file = dsn_file_open("test_hpc_aio.tmp", O_RDWR | O_CREAT, 0666); - dsn::task_tracker *tracker = new dsn::task_tracker(13); - - dsn::aio_task *callback = new dsn::aio_task(LPC_AIO_TEST, nullptr, 0); - callback->add_ref(); - - callback->set_tracker(tracker); - callback->aio()->buffer = (char *)buffer; - callback->aio()->buffer_size = count; - callback->aio()->engine = nullptr; - callback->aio()->file = file; - callback->aio()->file_offset = 0; - callback->aio()->type = ::dsn::AIO_Write; - ::dsn::task::get_current_disk()->write(callback); - - callback->wait(); - - dsn::error_code err = dsn_file_close(file); - EXPECT_TRUE(err == ERR_OK); - - callback->release_ref(); - delete tracker; - - // read from file - char buffer_read[11]; - buffer_read[10] = '\0'; - file = dsn_file_open("test_hpc_aio.tmp", O_RDWR | O_CREAT, 0666); - tracker = new dsn::task_tracker(13); - dsn::aio_task *callback_read = new dsn::aio_task(LPC_AIO_TEST, nullptr, 0); - callback_read->add_ref(); - - callback_read->set_tracker((dsn::task_tracker *)tracker); - callback_read->aio()->buffer = (char *)buffer_read; - callback_read->aio()->buffer_size = count; - callback_read->aio()->engine = nullptr; - callback_read->aio()->file = file; - callback_read->aio()->file_offset = 0; - callback_read->aio()->type = ::dsn::AIO_Read; - ::dsn::task::get_current_disk()->read(callback_read); - - callback_read->wait(); - - EXPECT_STREQ(buffer, buffer_read); - err = dsn_file_close(file); - EXPECT_TRUE(err == ERR_OK); - callback_read->release_ref(); - - delete tracker; - // invalid operation -} - -TEST(tools_hpc, aio_create) -{ - ::dsn::tools::hpc_aio_provider *p = new tools::hpc_aio_provider(nullptr, nullptr); - delete p; -} - -TEST(tools_hpc, aio_invalid_type) -{ - if (nullptr == task::get_current_disk()) - return; - - std::remove("test_hpc_aio2.tmp"); // delete file - - // invalid io - char buffer[10]; - int count = 10; - sprintf(buffer, "abcdefghi"); - dsn_handle_t file = dsn_file_open("test_hpc_aio2.tmp", O_RDWR | O_CREAT, 0666); - dsn::task_tracker *tracker = new dsn::task_tracker(13); - - dsn::aio_task *callback = new dsn::aio_task(LPC_AIO_TEST, nullptr, 0); - callback->add_ref(); - - callback->set_tracker(tracker); - callback->aio()->buffer = (char *)buffer; - callback->aio()->buffer_size = count; - callback->aio()->engine = nullptr; - callback->aio()->file = file; - callback->aio()->file_offset = 0; - callback->aio()->type = ::dsn::AIO_Invalid; - ::dsn::task::get_current_disk()->write(callback); - - callback->wait(); - - dsn::error_code err = dsn_file_close(file); - EXPECT_TRUE(err == ERR_OK); - - callback->release_ref(); - delete tracker; -} diff --git a/src/core/tests/hpc_aio_provider_for_test.h b/src/core/tests/hpc_aio_provider_for_test.h deleted file mode 100644 index cc75445234..0000000000 --- a/src/core/tests/hpc_aio_provider_for_test.h +++ /dev/null @@ -1,74 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * Unit-test for hpc aio provider. - * - * Revision history: - * Nov., 2015, @xiaotz (Xiaotong Zhang), first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#pragma once - -#include -#include -#include -#include -#include -#include "../core/disk_engine.h" -#include "test_utils.h" -#include "../tools/hpc/hpc_aio_provider.h" - -namespace dsn { -namespace test { -class hpc_aio_provider_for_test : public dsn::tools::hpc_aio_provider -{ -public: - hpc_aio_provider_for_test(disk_engine *disk, aio_provider *inner_provider) - : dsn::tools::hpc_aio_provider(disk, inner_provider), f(false) - { - } - - virtual void aio(aio_task *aio_tsk) - { - uint32_t b; - error_code err; - if (f) { - dsn::tools::hpc_aio_provider::aio(aio_tsk); - } else { - auto err = aio_internal(aio_tsk, f, &b); - complete_io(aio_tsk, err, b); - } - f = !f; - } - -private: - bool f; -}; -} -} diff --git a/src/core/tests/hpc_io_looper.cpp b/src/core/tests/hpc_io_looper.cpp deleted file mode 100644 index edf00543d8..0000000000 --- a/src/core/tests/hpc_io_looper.cpp +++ /dev/null @@ -1,186 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * Unit-test for io_looper. - * - * Revision history: - * Nov., 2015, @xiaotz (Xiaotong Zhang), first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "../core/service_engine.h" -#include "test_utils.h" -#include "../tools/hpc/io_looper.h" - -DEFINE_THREAD_POOL_CODE(THREAD_POOL_TEST_SERVER_2); -DEFINE_TASK_CODE_AIO(LPC_AIO_TEST_2, TASK_PRIORITY_COMMON, THREAD_POOL_TEST_SERVER_2); -DEFINE_TASK_CODE_AIO(LPC_AIO_TEST, TASK_PRIORITY_COMMON, THREAD_POOL_TEST_SERVER); - -#ifdef __linux__ - -#include -#include - -namespace dsn { -namespace test { -class io_looper_for_test : public dsn::tools::io_looper -{ -public: - io_looper_for_test() : io_looper() {} - virtual ~io_looper_for_test() {} -public: - virtual bool is_shared_timer_queue() override - { - if (io_looper::is_shared_timer_queue()) - return false; - else - return true; - } -}; -} -} - -TEST(tools_hpc, io_looper) -{ - dsn::test::io_looper_for_test *io_looper = new dsn::test::io_looper_for_test(); - io_looper->start(::dsn::task::get_current_node(), 1); - EXPECT_FALSE(io_looper->is_shared_timer_queue()); - - int event_fd = eventfd(0, EFD_NONBLOCK); - - io_context_t ctx; - memset(&ctx, 0, sizeof(ctx)); - auto ret = io_setup(128, &ctx); // 128 concurrent events - EXPECT_TRUE(ret == 0); - - utils::notify_event *evt = new utils::notify_event(); - dsn::tools::io_loop_callback callback = - [ctx, event_fd, evt](int native_error, uint32_t io_size, uintptr_t lolp_or_events) { - int64_t finished_aio = 0; - - if (read(event_fd, &finished_aio, sizeof(finished_aio)) != sizeof(finished_aio)) { - if (errno == EAGAIN || errno == EWOULDBLOCK) - return; - dassert(false, - "read number of aio completion from eventfd failed, err = %s", - strerror(errno)); - } - EXPECT_EQ(1, finished_aio); - - struct io_event events[1]; - int ret; - - struct timespec tms; - tms.tv_sec = 0; - tms.tv_nsec = 0; - - ret = io_getevents(ctx, 1, 1, events, &tms); - EXPECT_EQ(1, ret); - EXPECT_EQ(0, static_cast(events[0].res2)); - EXPECT_EQ(10, static_cast(events[0].res)); - evt->notify(); - }; - - auto err = - io_looper->bind_io_handle((dsn_handle_t)(intptr_t)event_fd, &callback, EPOLLIN | EPOLLET); - EXPECT_EQ(ERR_OK, err); - - std::remove("test_io_looper.tmp"); // delete file - std::ofstream myfile; - myfile.open("test_io_looper.tmp"); - myfile << "abcdefghi$"; - myfile.close(); - int fd = ::open("test_io_looper.tmp", O_RDWR, 0666); - char buffer[11]; - memset(buffer, 0, sizeof(buffer)); - - struct iocb *cb = new iocb; - memset(cb, 0, sizeof(iocb)); - io_prep_pread(cb, fd, buffer, 10, 0); - io_set_eventfd(cb, event_fd); - ret = io_submit(ctx, 1, &cb); - EXPECT_EQ(1, ret); - - evt->wait(); - EXPECT_STREQ("abcdefghi$", buffer); - - err = io_looper->unbind_io_handle((dsn_handle_t)(intptr_t)event_fd, &callback); - EXPECT_EQ(dsn::ERR_OK, err); - - delete io_looper; - delete cb; - delete evt; -} - -void timer_callback(void *param) -{ - int *a = (int *)param; - EXPECT_EQ(1, *a); - *a = 2; -} - -TEST(tools_hpc, io_looper_timer) -{ - if (dsn::service_engine::fast_instance().spec().semaphore_factory_name == - "dsn::tools::sim_semaphore_provider") - return; - - dsn::test::io_looper_for_test *io_looper = new dsn::test::io_looper_for_test(); - io_looper->start(::dsn::task::get_current_node(), 1); - - int *a = new int; - *a = 1; - dsn::raw_task *t = new dsn::raw_task(LPC_AIO_TEST, std::bind(&timer_callback, a)); - t->add_ref(); - - t->set_delay(1000); - t->add_ref(); - io_looper->add_timer(t); - t->wait(); - - EXPECT_EQ(2, *a); - - t->release_ref(); - - io_looper->stop(); - delete io_looper; -} - -#endif // ifdef __linux__ diff --git a/src/core/tests/hpc_tail_logger.cpp b/src/core/tests/hpc_tail_logger.cpp deleted file mode 100644 index 3f50bad07a..0000000000 --- a/src/core/tests/hpc_tail_logger.cpp +++ /dev/null @@ -1,82 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * Unit-test for hpc tail logger. - * - * Revision history: - * Nov., 2015, @xiaotz (Xiaotong Zhang), first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include "test_utils.h" -#include "../core/service_engine.h" -#include "../tools/hpc/hpc_tail_logger.h" - -TEST(tools_hpc, tail_logger) -{ - dwarn("this is a warning"); - dinfo("this is a log %d", 1); -} - -TEST(tools_hpc, tail_logger_cb) -{ - std::string output; - bool ret = dsn::command_manager::instance().run_command("tail-log", output); - if (!ret) - return; - - EXPECT_TRUE(strcmp(output.c_str(), "invalid arguments for tail-log command") == 0); - - std::ostringstream in; - in << "tail-log 12345 4 1 " << dsn::utils::get_current_tid(); - std::this_thread::sleep_for(std::chrono::seconds(3)); - dwarn("key:12345"); - std::this_thread::sleep_for(std::chrono::seconds(2)); - dwarn("key:12345"); - output.clear(); - dsn::command_manager::instance().run_command(in.str().c_str(), output); - EXPECT_TRUE(strstr(output.c_str(), "In total (1) log entries are found between") != nullptr); - dsn::command_manager::instance().run_command("tail-log-dump", output); - - ::dsn::logging_provider *logger = ::dsn::service_engine::fast_instance().logging(); - if (logger != nullptr) { - logger->flush(); - } -} diff --git a/src/core/tests/main.cpp b/src/core/tests/main.cpp index 232330e7fd..5e6fb95dca 100644 --- a/src/core/tests/main.cpp +++ b/src/core/tests/main.cpp @@ -37,8 +37,6 @@ #include "gtest/gtest.h" #include "test_utils.h" -#include "hpc_aio_provider_for_test.h" - extern void task_engine_module_init(); extern void command_manager_module_init(); @@ -49,9 +47,6 @@ GTEST_API_ int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); - dsn::tools::register_component_provider( - "dsn::test::hpc_aio_provider_for_test"); - // register all tools task_engine_module_init(); command_manager_module_init(); diff --git a/src/core/tests/netprovider.cpp b/src/core/tests/netprovider.cpp index 2bf8b0e0fd..fa0ffcb9c3 100644 --- a/src/core/tests/netprovider.cpp +++ b/src/core/tests/netprovider.cpp @@ -119,16 +119,13 @@ TEST(tools_common, asio_net_provider) asio_network_provider *asio_network = new asio_network_provider(task::get_current_rpc(), nullptr); - io_modifer modifier; - modifier.mode = IOE_PER_NODE; - modifier.queue = nullptr; error_code start_result; - start_result = asio_network->start(RPC_CHANNEL_TCP, TEST_PORT, true, modifier); + start_result = asio_network->start(RPC_CHANNEL_TCP, TEST_PORT, true); ASSERT_TRUE(start_result == ERR_OK); // the same asio network handle, start only client is ok - start_result = asio_network->start(RPC_CHANNEL_TCP, TEST_PORT, true, modifier); + start_result = asio_network->start(RPC_CHANNEL_TCP, TEST_PORT, true); ASSERT_TRUE(start_result == ERR_OK); rpc_address network_addr = asio_network->address(); @@ -136,14 +133,14 @@ TEST(tools_common, asio_net_provider) asio_network_provider *asio_network2 = new asio_network_provider(task::get_current_rpc(), nullptr); - start_result = asio_network2->start(RPC_CHANNEL_TCP, TEST_PORT, true, modifier); + start_result = asio_network2->start(RPC_CHANNEL_TCP, TEST_PORT, true); ASSERT_TRUE(start_result == ERR_OK); - start_result = asio_network2->start(RPC_CHANNEL_TCP, TEST_PORT, false, modifier); + start_result = asio_network2->start(RPC_CHANNEL_TCP, TEST_PORT, false); ASSERT_TRUE(start_result == ERR_OK); ddebug("result: %s", start_result.to_string()); - start_result = asio_network2->start(RPC_CHANNEL_TCP, TEST_PORT, false, modifier); + start_result = asio_network2->start(RPC_CHANNEL_TCP, TEST_PORT, false); ASSERT_TRUE(start_result == ERR_SERVICE_ALREADY_RUNNING); ddebug("result: %s", start_result.to_string()); @@ -168,15 +165,12 @@ TEST(tools_common, asio_udp_provider) RPC_TEST_NETPROVIDER, "rpc.test.netprovider", rpc_server_response)); auto client = new asio_udp_provider(task::get_current_rpc(), nullptr); - io_modifer modifier; - modifier.mode = IOE_PER_NODE; - modifier.queue = nullptr; error_code start_result; - start_result = client->start(RPC_CHANNEL_UDP, 0, true, modifier); + start_result = client->start(RPC_CHANNEL_UDP, 0, true); ASSERT_TRUE(start_result == ERR_OK); - start_result = client->start(RPC_CHANNEL_UDP, TEST_PORT, false, modifier); + start_result = client->start(RPC_CHANNEL_UDP, TEST_PORT, false); ASSERT_TRUE(start_result == ERR_OK); message_ex *msg = message_ex::create_request(RPC_TEST_NETPROVIDER, 0, 0); @@ -213,15 +207,12 @@ TEST(tools_common, sim_net_provider) RPC_TEST_NETPROVIDER, "rpc.test.netprovider", rpc_server_response)); sim_network_provider *sim_net = new sim_network_provider(task::get_current_rpc(), nullptr); - io_modifer modifer; - modifer.mode = IOE_PER_NODE; - modifer.queue = nullptr; error_code ans; - ans = sim_net->start(RPC_CHANNEL_TCP, TEST_PORT, false, modifer); + ans = sim_net->start(RPC_CHANNEL_TCP, TEST_PORT, false); ASSERT_TRUE(ans == ERR_OK); - ans = sim_net->start(RPC_CHANNEL_TCP, TEST_PORT, false, modifer); + ans = sim_net->start(RPC_CHANNEL_TCP, TEST_PORT, false); ASSERT_TRUE(ans == ERR_ADDRESS_ALREADY_USED); rpc_session_ptr client_session = diff --git a/src/core/tests/nfs_test_file1 b/src/core/tests/nfs_test_file1 index 7cf6e72477..7bf1621691 100644 --- a/src/core/tests/nfs_test_file1 +++ b/src/core/tests/nfs_test_file1 @@ -48,8 +48,7 @@ pools = THREAD_POOL_DEFAULT, THREAD_POOL_TEST_SERVER [core] ;tool = simulator -;tool = nativerun -tool = fastrun +tool = nativerun toollets = tracer, profiler, fault_injector pause_on_start = false diff --git a/src/core/tests/nfs_test_file2 b/src/core/tests/nfs_test_file2 index adab8dba23..468947af67 100644 --- a/src/core/tests/nfs_test_file2 +++ b/src/core/tests/nfs_test_file2 @@ -35,8 +35,7 @@ pools = THREAD_POOL_DEFAULT, THREAD_POOL_TEST_SERVER [core] ;tool = simulator -;tool = nativerun -tool = fastrun +tool = nativerun toollets = tracer, profiler, fault_injector pause_on_start = true diff --git a/src/core/tools/common/asio_net_provider.cpp b/src/core/tools/common/asio_net_provider.cpp index f723db23a3..1236aee5fa 100644 --- a/src/core/tools/common/asio_net_provider.cpp +++ b/src/core/tools/common/asio_net_provider.cpp @@ -45,8 +45,7 @@ asio_network_provider::asio_network_provider(rpc_engine *srv, network *inner_pro _acceptor = nullptr; } -error_code -asio_network_provider::start(rpc_channel channel, int port, bool client_only, io_modifer &ctx) +error_code asio_network_provider::start(rpc_channel channel, int port, bool client_only) { if (_acceptor != nullptr) return ERR_SERVICE_ALREADY_RUNNING; @@ -57,8 +56,8 @@ asio_network_provider::start(rpc_channel channel, int port, bool client_only, io 1, "thread number for io service (timer and boost network)"); for (int i = 0; i < io_service_worker_count; i++) { - _workers.push_back(std::shared_ptr(new std::thread([this, ctx, i]() { - task::set_tls_dsn_context(node(), nullptr, ctx.queue); + _workers.push_back(std::shared_ptr(new std::thread([this, i]() { + task::set_tls_dsn_context(node(), nullptr); const char *name = ::dsn::tools::get_service_node_name(node()); char buffer[128]; @@ -277,8 +276,7 @@ void asio_udp_provider::do_receive() }); } -error_code -asio_udp_provider::start(rpc_channel channel, int port, bool client_only, io_modifer &ctx) +error_code asio_udp_provider::start(rpc_channel channel, int port, bool client_only) { _is_client = client_only; int io_service_worker_count = @@ -341,8 +339,8 @@ asio_udp_provider::start(rpc_channel channel, int port, bool client_only, io_mod } for (int i = 0; i < io_service_worker_count; i++) { - _workers.push_back(std::shared_ptr(new std::thread([this, ctx, i]() { - task::set_tls_dsn_context(node(), nullptr, ctx.queue); + _workers.push_back(std::shared_ptr(new std::thread([this, i]() { + task::set_tls_dsn_context(node(), nullptr); const char *name = ::dsn::tools::get_service_node_name(node()); char buffer[128]; diff --git a/src/core/tools/common/asio_net_provider.h b/src/core/tools/common/asio_net_provider.h index 0f04dd0f5a..cf3cc333d2 100644 --- a/src/core/tools/common/asio_net_provider.h +++ b/src/core/tools/common/asio_net_provider.h @@ -46,8 +46,7 @@ class asio_network_provider : public connection_oriented_network public: asio_network_provider(rpc_engine *srv, network *inner_provider); - virtual error_code - start(rpc_channel channel, int port, bool client_only, io_modifer &ctx) override; + virtual error_code start(rpc_channel channel, int port, bool client_only) override; virtual ::dsn::rpc_address address() override { return _address; } virtual rpc_session_ptr create_client_session(::dsn::rpc_address server_addr) override; @@ -72,8 +71,7 @@ class asio_udp_provider : public network void send_message(message_ex *request) override; - virtual error_code - start(rpc_channel channel, int port, bool client_only, io_modifer &ctx) override; + virtual error_code start(rpc_channel channel, int port, bool client_only) override; virtual ::dsn::rpc_address address() override { return _address; } diff --git a/src/core/tools/common/empty_aio_provider.h b/src/core/tools/common/empty_aio_provider.h index 7a757b6e92..941df8efc8 100644 --- a/src/core/tools/common/empty_aio_provider.h +++ b/src/core/tools/common/empty_aio_provider.h @@ -52,7 +52,7 @@ class empty_aio_provider : public aio_provider virtual void aio(aio_task *aio) override; virtual disk_aio *prepare_aio_context(aio_task *tsk) override; - virtual void start(io_modifer &ctx) override {} + virtual void start() override {} }; } } diff --git a/src/core/tools/common/native_aio_provider.linux.cpp b/src/core/tools/common/native_aio_provider.linux.cpp index f6958a565e..6f13ab718d 100644 --- a/src/core/tools/common/native_aio_provider.linux.cpp +++ b/src/core/tools/common/native_aio_provider.linux.cpp @@ -58,10 +58,10 @@ native_linux_aio_provider::~native_linux_aio_provider() dassert(ret == 0, "io_destroy error, ret = %d", ret); } -void native_linux_aio_provider::start(io_modifer &ctx) +void native_linux_aio_provider::start() { - new std::thread([this, ctx]() { - task::set_tls_dsn_context(node(), nullptr, ctx.queue); + new std::thread([this]() { + task::set_tls_dsn_context(node(), nullptr); get_event(); }); } @@ -111,7 +111,7 @@ void native_linux_aio_provider::get_event() struct io_event events[1]; int ret; - task::set_tls_dsn_context(node(), nullptr, nullptr); + task::set_tls_dsn_context(node(), nullptr); const char *name = ::dsn::tools::get_service_node_name(node()); char buffer[128]; diff --git a/src/core/tools/common/native_aio_provider.linux.h b/src/core/tools/common/native_aio_provider.linux.h index b4c4c93666..f31aa7b08e 100644 --- a/src/core/tools/common/native_aio_provider.linux.h +++ b/src/core/tools/common/native_aio_provider.linux.h @@ -62,7 +62,7 @@ class native_linux_aio_provider : public aio_provider virtual void aio(aio_task *aio) override; virtual disk_aio *prepare_aio_context(aio_task *tsk) override; - virtual void start(io_modifer &ctx) override; + virtual void start() override; struct linux_disk_aio_context : public disk_aio { diff --git a/src/core/tools/common/native_aio_provider.posix.cpp b/src/core/tools/common/native_aio_provider.posix.cpp index 1f2a435d5e..dbd316a55c 100644 --- a/src/core/tools/common/native_aio_provider.posix.cpp +++ b/src/core/tools/common/native_aio_provider.posix.cpp @@ -107,7 +107,7 @@ void aio_completed(sigval sigval) auto ctx = (posix_disk_aio_context *)sigval.sival_ptr; if (dsn::tls_dsn.magic != 0xdeadbeef) - task::set_tls_dsn_context(ctx->tsk->node(), nullptr, nullptr); + task::set_tls_dsn_context(ctx->tsk->node(), nullptr); int err = aio_error(&ctx->cb); if (err != EINPROGRESS) { diff --git a/src/core/tools/common/native_aio_provider.posix.h b/src/core/tools/common/native_aio_provider.posix.h index 074d6d9997..1e44a1810e 100644 --- a/src/core/tools/common/native_aio_provider.posix.h +++ b/src/core/tools/common/native_aio_provider.posix.h @@ -57,7 +57,7 @@ class native_posix_aio_provider : public aio_provider virtual void aio(aio_task *aio) override; virtual disk_aio *prepare_aio_context(aio_task *tsk) override; - virtual void start(io_modifer &ctx) override {} + virtual void start() override {} protected: error_code aio_internal(aio_task *aio, bool async, /*out*/ uint32_t *pbytes = nullptr); diff --git a/src/core/tools/common/native_aio_provider.win.cpp b/src/core/tools/common/native_aio_provider.win.cpp index 04fa490290..c0729b3a0d 100644 --- a/src/core/tools/common/native_aio_provider.win.cpp +++ b/src/core/tools/common/native_aio_provider.win.cpp @@ -67,7 +67,7 @@ native_win_aio_provider::~native_win_aio_provider() void native_win_aio_provider::start(io_modifer &ctx) { _worker_thr = new std::thread([this, ctx]() { - task::set_tls_dsn_context(node(), nullptr, ctx.queue); + task::set_tls_dsn_context(node(), nullptr); const char *name = ::dsn::tools::get_service_node_name(node()); char buffer[128]; diff --git a/src/core/tools/common/network.sim.cpp b/src/core/tools/common/network.sim.cpp index acd40b012b..9b523d09b6 100644 --- a/src/core/tools/common/network.sim.cpp +++ b/src/core/tools/common/network.sim.cpp @@ -165,8 +165,7 @@ sim_network_provider::sim_network_provider(rpc_engine *rpc, network *inner_provi "max message delay (us)"); } -error_code -sim_network_provider::start(rpc_channel channel, int port, bool client_only, io_modifer &ctx) +error_code sim_network_provider::start(rpc_channel channel, int port, bool client_only) { dassert(channel == RPC_CHANNEL_TCP || channel == RPC_CHANNEL_UDP, "invalid given channel %s", diff --git a/src/core/tools/common/network.sim.h b/src/core/tools/common/network.sim.h index 71a713d092..2c4adbfd7e 100644 --- a/src/core/tools/common/network.sim.h +++ b/src/core/tools/common/network.sim.h @@ -83,7 +83,7 @@ class sim_network_provider : public connection_oriented_network sim_network_provider(rpc_engine *rpc, network *inner_provider); ~sim_network_provider(void) {} - virtual error_code start(rpc_channel channel, int port, bool client_only, io_modifer &ctx); + virtual error_code start(rpc_channel channel, int port, bool client_only); virtual ::dsn::rpc_address address() { return _address; } diff --git a/src/core/tools/common/simple_task_queue.cpp b/src/core/tools/common/simple_task_queue.cpp index 0faf377754..0aca67f9de 100644 --- a/src/core/tools/common/simple_task_queue.cpp +++ b/src/core/tools/common/simple_task_queue.cpp @@ -43,16 +43,13 @@ simple_timer_service::simple_timer_service(service_node *node, timer_service *in _worker = nullptr; } -void simple_timer_service::start(io_modifer &ctx) +void simple_timer_service::start() { - _worker = std::shared_ptr(new std::thread([this, ctx]() { - task::set_tls_dsn_context(node(), nullptr, ctx.queue); + _worker = std::shared_ptr(new std::thread([this]() { + task::set_tls_dsn_context(node(), nullptr); char buffer[128]; - sprintf(buffer, - "%s.%s.timer", - get_service_node_name(node()), - ctx.queue ? ctx.queue->get_name().c_str() : ""); + sprintf(buffer, "%s.timer", get_service_node_name(node())); task_worker::set_name(buffer); task_worker::set_priority(worker_priority_t::THREAD_xPRIORITY_ABOVE_NORMAL); diff --git a/src/core/tools/common/simple_task_queue.h b/src/core/tools/common/simple_task_queue.h index 2b59353ce1..e3851993af 100644 --- a/src/core/tools/common/simple_task_queue.h +++ b/src/core/tools/common/simple_task_queue.h @@ -62,7 +62,7 @@ class simple_timer_service : public timer_service // after milliseconds, the provider should call task->enqueue() virtual void add_timer(task *task) override; - virtual void start(io_modifer &ctx) override; + virtual void start() override; private: boost::asio::io_service _ios; diff --git a/src/core/tools/hpc/fastrun.cpp b/src/core/tools/hpc/fastrun.cpp deleted file mode 100644 index 33235c7368..0000000000 --- a/src/core/tools/hpc/fastrun.cpp +++ /dev/null @@ -1,129 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * What is this file about? - * - * Revision history: - * xxxx-xx-xx, author, first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#include -#include "mix_all_io_looper.h" - -namespace dsn { -namespace tools { -void fastrun::install(service_spec &spec) -{ - bool use_mixed_queue = false; - if (spec.disk_io_mode == IOE_PER_QUEUE || spec.rpc_io_mode == IOE_PER_QUEUE || - spec.nfs_io_mode == IOE_PER_QUEUE || spec.timer_io_mode == IOE_PER_QUEUE) - use_mixed_queue = true; - - if (spec.aio_factory_name == "") { - spec.aio_factory_name = ("dsn::tools::hpc_aio_provider"); - } - - if (spec.env_factory_name == "") - spec.env_factory_name = ("dsn::tools::hpc_env_provider"); - - if (spec.timer_factory_name == "") { - if (use_mixed_queue) - spec.timer_factory_name = "dsn::tools::io_looper_timer_service"; - else - spec.timer_factory_name = "dsn::tools::simple_timer_service"; - } - - { - network_client_config cs; - cs.factory_name = "dsn::tools::hpc_network_provider"; - cs.message_buffer_block_size = 1024 * 64; - spec.network_default_client_cfs[RPC_CHANNEL_TCP] = cs; - } - { - network_server_config cs2; - cs2.port = 0; - cs2.channel = RPC_CHANNEL_TCP; - cs2.factory_name = "dsn::tools::hpc_network_provider"; - cs2.message_buffer_block_size = 1024 * 64; - spec.network_default_server_cfs[cs2] = cs2; - } - - { - network_client_config cs; - cs.factory_name = "dsn::tools::asio_udp_provider"; - cs.message_buffer_block_size = 1024 * 64; - spec.network_default_client_cfs[RPC_CHANNEL_UDP] = cs; - } - { - - network_server_config cs2; - cs2.port = 0; - cs2.channel = RPC_CHANNEL_UDP; - cs2.factory_name = "dsn::tools::asio_udp_provider"; - cs2.message_buffer_block_size = 1024 * 64; - spec.network_default_server_cfs[cs2] = cs2; - } - - if (spec.perf_counter_factory_name == "") - spec.perf_counter_factory_name = "dsn::tools::simple_perf_counter"; - - if (spec.logging_factory_name == "") - spec.logging_factory_name = "dsn::tools::hpc_logger"; - - if (spec.lock_factory_name == "") - spec.lock_factory_name = ("dsn::tools::std_lock_provider"); - - if (spec.lock_nr_factory_name == "") - spec.lock_nr_factory_name = ("dsn::tools::std_lock_nr_provider"); - - if (spec.rwlock_nr_factory_name == "") - spec.rwlock_nr_factory_name = ("dsn::tools::std_rwlock_nr_provider"); - - if (spec.semaphore_factory_name == "") - spec.semaphore_factory_name = ("dsn::tools::std_semaphore_provider"); - - if (spec.nfs_factory_name == "") - spec.nfs_factory_name = "dsn::service::nfs_node_simple"; - - for (auto it = spec.threadpool_specs.begin(); it != spec.threadpool_specs.end(); ++it) { - threadpool_spec &tspec = *it; - - if (tspec.worker_factory_name == "") - tspec.worker_factory_name = - use_mixed_queue ? ("dsn::tools::io_looper_task_worker") : "dsn::task_worker"; - - if (tspec.queue_factory_name == "") - tspec.queue_factory_name = use_mixed_queue ? ("dsn::tools::io_looper_task_queue") - : "dsn::tools::hpc_concurrent_task_queue"; - } -} - -void fastrun::run() { tool_app::run(); } -} -} // end namespace dsn::tools diff --git a/src/core/tools/hpc/hpc_aio_provider.bsd.cpp b/src/core/tools/hpc/hpc_aio_provider.bsd.cpp deleted file mode 100644 index d0cdbde444..0000000000 --- a/src/core/tools/hpc/hpc_aio_provider.bsd.cpp +++ /dev/null @@ -1,257 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * AIO using kqueue on BSD - * - * Revision history: - * 2015-09-07, HX Lin(linmajia@live.com), first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#if defined(__APPLE__) || defined(__FreeBSD__) - -#include "hpc_aio_provider.h" -#include -#include -#include -#include -#include -#include "mix_all_io_looper.h" - -namespace dsn { -namespace tools { - -struct posix_disk_aio_context : public disk_aio -{ - struct aiocb cb; - aio_task *tsk; - hpc_aio_provider *this_; - utils::notify_event *evt; - error_code err; - uint32_t bytes; -}; - -#ifndef io_prep_pread -static inline void -io_prep_pread(struct aiocb *iocb, int fd, void *buf, size_t count, long long offset) -{ - memset(iocb, 0, sizeof(*iocb)); - iocb->aio_fildes = fd; - iocb->aio_lio_opcode = LIO_READ; - iocb->aio_reqprio = 0; - iocb->aio_buf = buf; - iocb->aio_nbytes = count; - iocb->aio_offset = offset; -} -#endif - -#ifndef io_prep_pwrite -static inline void -io_prep_pwrite(struct aiocb *iocb, int fd, void *buf, size_t count, long long offset) -{ - memset(iocb, 0, sizeof(*iocb)); - iocb->aio_fildes = fd; - iocb->aio_lio_opcode = LIO_WRITE; - iocb->aio_reqprio = 0; - iocb->aio_buf = buf; - iocb->aio_nbytes = count; - iocb->aio_offset = offset; -} -#endif - -hpc_aio_provider::hpc_aio_provider(disk_engine *disk, aio_provider *inner_provider) - : aio_provider(disk, inner_provider) -{ - _callback = [this](int native_error, uint32_t io_size, uintptr_t lolp_or_events) { - struct kevent *e; - struct aiocb *io; - int bytes; - int err; - - e = (struct kevent *)lolp_or_events; - if (e->filter != EVFILT_AIO) { - derror("Non-aio filter invokes the aio callback! e->filter = %hd", e->filter); - return; - } - - io = (struct aiocb *)(e->ident); - err = aio_error(io); - if (err == EINPROGRESS) { - return; - } - - bytes = aio_return(io); - if (bytes == -1) { - err = errno; - } - - complete_aio(io, bytes, err); - }; - - _looper = nullptr; -} - -void hpc_aio_provider::start(io_modifer &ctx) -{ - _looper = get_io_looper(node(), ctx.queue, ctx.mode); -} - -hpc_aio_provider::~hpc_aio_provider() {} - -dsn_handle_t hpc_aio_provider::open(const char *file_name, int oflag, int pmode) -{ - // No need to bind handle since EVFILT_AIO is registered when aio_* is called. - return (dsn_handle_t)(uintptr_t)::open(file_name, oflag, pmode); -} - -error_code hpc_aio_provider::close(dsn_handle_t fh) -{ - if (::close((int)(uintptr_t)(fh)) == 0) { - return ERR_OK; - } else { - derror("close file failed, err = %s", strerror(errno)); - return ERR_FILE_OPERATION_FAILED; - } -} - -error_code hpc_aio_provider::flush(dsn_handle_t fh) -{ - if (fh == DSN_INVALID_FILE_HANDLE || ::fsync((int)(uintptr_t)(fh)) == 0) { - return ERR_OK; - } else { - derror("flush file failed, err = %s", strerror(errno)); - return ERR_FILE_OPERATION_FAILED; - } -} - -disk_aio *hpc_aio_provider::prepare_aio_context(aio_task *tsk) -{ - auto r = new posix_disk_aio_context; - bzero((char *)&r->cb, sizeof(r->cb)); - r->tsk = tsk; - r->evt = nullptr; - return r; -} - -void hpc_aio_provider::aio(aio_task *aio_tsk) { aio_internal(aio_tsk, true); } - -error_code hpc_aio_provider::aio_internal(aio_task *aio_tsk, - bool async, - /*out*/ uint32_t *pbytes /*= nullptr*/) -{ - auto aio = (posix_disk_aio_context *)aio_tsk->aio(); - int r = 0; - - aio->this_ = this; - - if (!async) { - aio->evt = new utils::notify_event(); - aio->err = ERR_OK; - aio->bytes = 0; - } - - switch (aio->type) { - case AIO_Read: - io_prep_pread(&aio->cb, - static_cast((ssize_t)aio->file), - aio->buffer, - aio->buffer_size, - aio->file_offset); - break; - case AIO_Write: - io_prep_pwrite(&aio->cb, - static_cast((ssize_t)aio->file), - aio->buffer, - aio->buffer_size, - aio->file_offset); - break; - default: - dassert(false, "unknown aio type %u", static_cast(aio->type)); - break; - } - - // set up callback - aio->cb.aio_sigevent.sigev_notify = SIGEV_KEVENT; - aio->cb.aio_sigevent.sigev_notify_kqueue = (int)(uintptr_t)_looper->native_handle(); - aio->cb.aio_sigevent.sigev_notify_kevent_flags = EV_CLEAR; - aio->cb.aio_sigevent.sigev_value.sival_ptr = &_callback; - - r = (aio->type == AIO_Read) ? aio_read(&aio->cb) : aio_write(&aio->cb); - if (r != 0) { - derror("file op failed, err = %d (%s). On FreeBSD, you may need to load" - " aio kernel module by running 'sudo kldload aio'.", - errno, - strerror(errno)); - - if (async) { - complete_io(aio_tsk, ERR_FILE_OPERATION_FAILED, 0); - } else { - delete aio->evt; - aio->evt = nullptr; - } - return ERR_FILE_OPERATION_FAILED; - } else { - if (async) { - return ERR_IO_PENDING; - } else { - aio->evt->wait(); - delete aio->evt; - aio->evt = nullptr; - if (pbytes != nullptr) { - *pbytes = aio->bytes; - } - return aio->err; - } - } -} - -void hpc_aio_provider::complete_aio(struct aiocb *io, int bytes, int err) -{ - posix_disk_aio_context *ctx = CONTAINING_RECORD(io, posix_disk_aio_context, cb); - error_code ec; - - if (err != 0) { - derror("aio error, err = %s", strerror(err)); - ec = ERR_FILE_OPERATION_FAILED; - } else { - dassert(bytes >= 0, "bytes = %d.", bytes); - ec = bytes > 0 ? ERR_OK : ERR_HANDLE_EOF; - } - - if (!ctx->evt) { - aio_task *aio(ctx->tsk); - ctx->this_->complete_io(aio, ec, bytes); - } else { - ctx->err = ec; - ctx->bytes = bytes; - ctx->evt->notify(); - } -} -} -} // end namespace dsn::tools -#endif diff --git a/src/core/tools/hpc/hpc_aio_provider.h b/src/core/tools/hpc/hpc_aio_provider.h deleted file mode 100644 index 2da495cd76..0000000000 --- a/src/core/tools/hpc/hpc_aio_provider.h +++ /dev/null @@ -1,83 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * What is this file about? - * - * Revision history: - * xxxx-xx-xx, author, first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#pragma once - -#include -#include -#include "io_looper.h" - -#ifdef __linux__ -#include -#endif - -#if defined(__APPLE__) || defined(__FreeBSD__) -#include -#endif - -namespace dsn { -namespace tools { -class hpc_aio_provider : public aio_provider -{ -public: - hpc_aio_provider(disk_engine *disk, aio_provider *inner_provider); - virtual ~hpc_aio_provider(); - - virtual dsn_handle_t open(const char *file_name, int flag, int pmode) override; - virtual error_code close(dsn_handle_t fh) override; - virtual error_code flush(dsn_handle_t fh) override; - virtual void aio(aio_task *aio) override; - virtual disk_aio *prepare_aio_context(aio_task *tsk) override; - - virtual void start(io_modifer &ctx) override; - -protected: - error_code aio_internal(aio_task *aio, bool async, /*out*/ uint32_t *pbytes = nullptr); - -private: - io_looper *_looper; - io_loop_callback _callback; - -#ifdef __linux__ - void complete_aio(struct iocb *io, int bytes, int err); - - io_context_t _ctx; - int _event_fd; -#elif defined(__APPLE__) || defined(__FreeBSD__) - void complete_aio(struct aiocb *io, int bytes, int err); -#endif -}; -} -} diff --git a/src/core/tools/hpc/hpc_aio_provider.linux.cpp b/src/core/tools/hpc/hpc_aio_provider.linux.cpp deleted file mode 100644 index 847542fd03..0000000000 --- a/src/core/tools/hpc/hpc_aio_provider.linux.cpp +++ /dev/null @@ -1,246 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * What is this file about? - * - * Revision history: - * xxxx-xx-xx, author, first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#ifdef __linux__ -//# ifdef _WIN32 - -#include "hpc_aio_provider.h" -#include -#include -#include -#include -#include -#include -#include "mix_all_io_looper.h" - -namespace dsn { -namespace tools { - -struct linux_disk_aio_context : public disk_aio -{ - struct iocb cb; - aio_task *tsk; - hpc_aio_provider *this_; - utils::notify_event *evt; - error_code err; - uint32_t bytes; -}; - -hpc_aio_provider::hpc_aio_provider(disk_engine *disk, aio_provider *inner_provider) - : aio_provider(disk, inner_provider) -{ - _callback = [this](int native_error, uint32_t io_size, uintptr_t lolp_or_events) { - int64_t finished_aio = 0; - - if (read(_event_fd, &finished_aio, sizeof(finished_aio)) != sizeof(finished_aio)) { - if (errno == EAGAIN || errno == EWOULDBLOCK) - return; - - dassert(false, - "read number of aio completion from eventfd failed, err = %s", - strerror(errno)); - } - - struct io_event events[1]; - int ret; - - while (finished_aio-- > 0) { - struct timespec tms; - tms.tv_sec = 0; - tms.tv_nsec = 0; - - ret = io_getevents(_ctx, 1, 1, events, &tms); - dassert(ret == 1, - "aio must return 1 event as we already got " - "notification from eventfd"); - - struct iocb *io = events[0].obj; - complete_aio(io, static_cast(events[0].res), static_cast(events[0].res2)); - } - }; - - memset(&_ctx, 0, sizeof(_ctx)); - auto ret = io_setup(128, &_ctx); // 128 concurrent events - dassert(ret == 0, "io_setup error, err = %s", strerror(-ret)); - - _event_fd = eventfd(0, EFD_NONBLOCK); - _looper = nullptr; -} - -void hpc_aio_provider::start(io_modifer &ctx) -{ - _looper = get_io_looper(node(), ctx.queue, ctx.mode); - _looper->bind_io_handle((dsn_handle_t)(intptr_t)_event_fd, &_callback, EPOLLIN | EPOLLET); -} - -hpc_aio_provider::~hpc_aio_provider() -{ - auto ret = io_destroy(_ctx); - dassert(ret == 0, "io_destroy error, err = %s", strerror(-ret)); - - ::close(_event_fd); -} - -dsn_handle_t hpc_aio_provider::open(const char *file_name, int oflag, int pmode) -{ - return (dsn_handle_t)(uintptr_t)::open(file_name, oflag, pmode); -} - -error_code hpc_aio_provider::close(dsn_handle_t fh) -{ - if (fh == DSN_INVALID_FILE_HANDLE || ::close((int)(uintptr_t)(fh)) == 0) { - return ERR_OK; - } else { - derror("close file failed, err = %s", strerror(errno)); - return ERR_FILE_OPERATION_FAILED; - } -} - -error_code hpc_aio_provider::flush(dsn_handle_t fh) -{ - if (fh == DSN_INVALID_FILE_HANDLE || ::fsync((int)(uintptr_t)(fh)) == 0) { - return ERR_OK; - } else { - derror("flush file failed, err = %s", strerror(errno)); - return ERR_FILE_OPERATION_FAILED; - } -} - -disk_aio *hpc_aio_provider::prepare_aio_context(aio_task *tsk) -{ - auto r = new linux_disk_aio_context; - bzero((char *)&r->cb, sizeof(r->cb)); - r->tsk = tsk; - r->evt = nullptr; - return r; -} - -void hpc_aio_provider::aio(aio_task *aio_tsk) { aio_internal(aio_tsk, true); } - -error_code hpc_aio_provider::aio_internal(aio_task *aio_tsk, - bool async, - /*out*/ uint32_t *pbytes /*= nullptr*/) -{ - struct iocb *cbs[1]; - linux_disk_aio_context *aio; - int ret; - - aio = (linux_disk_aio_context *)aio_tsk->aio(); - - memset(&aio->cb, 0, sizeof(aio->cb)); - - aio->this_ = this; - - switch (aio->type) { - case AIO_Read: - io_prep_pread(&aio->cb, - static_cast((ssize_t)aio->file), - aio->buffer, - aio->buffer_size, - aio->file_offset); - break; - case AIO_Write: - io_prep_pwrite(&aio->cb, - static_cast((ssize_t)aio->file), - aio->buffer, - aio->buffer_size, - aio->file_offset); - break; - default: - derror("unknown aio type %u", static_cast(aio->type)); - } - - if (!async) { - aio->evt = new utils::notify_event(); - aio->err = ERR_OK; - aio->bytes = 0; - } - - cbs[0] = &aio->cb; - - io_set_eventfd(&aio->cb, _event_fd); - ret = io_submit(_ctx, 1, cbs); - - if (ret != 1) { - if (ret < 0) - derror("io_submit error, err = %s", strerror(-ret)); - else - derror("could not sumbit IOs, err = %s", strerror(-ret)); - - if (async) { - complete_io(aio_tsk, ERR_FILE_OPERATION_FAILED, 0); - } else { - delete aio->evt; - aio->evt = nullptr; - } - return ERR_FILE_OPERATION_FAILED; - } else { - if (async) { - return ERR_IO_PENDING; - } else { - aio->evt->wait(); - delete aio->evt; - aio->evt = nullptr; - if (pbytes != nullptr) { - *pbytes = aio->bytes; - } - return aio->err; - } - } -} - -void hpc_aio_provider::complete_aio(struct iocb *io, int bytes, int err) -{ - linux_disk_aio_context *aio = CONTAINING_RECORD(io, linux_disk_aio_context, cb); - error_code ec; - if (err != 0) { - derror("aio error, err = %s", strerror(err)); - ec = ERR_FILE_OPERATION_FAILED; - } else { - ec = bytes > 0 ? ERR_OK : ERR_HANDLE_EOF; - } - - if (!aio->evt) { - aio_task *aio_ptr(aio->tsk); - aio->this_->complete_io(aio_ptr, ec, bytes); - } else { - aio->err = ec; - aio->bytes = bytes; - aio->evt->notify(); - } -} -} -} // end namespace dsn::tools -#endif diff --git a/src/core/tools/hpc/hpc_aio_provider.win.cpp b/src/core/tools/hpc/hpc_aio_provider.win.cpp deleted file mode 100644 index 853d088fbf..0000000000 --- a/src/core/tools/hpc/hpc_aio_provider.win.cpp +++ /dev/null @@ -1,289 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * What is this file about? - * - * Revision history: - * xxxx-xx-xx, author, first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#ifdef _WIN32 - -#include "hpc_aio_provider.h" -#include -#include -#include -#include -#include -#include "mix_all_io_looper.h" - -namespace dsn { -namespace tools { - -struct windows_disk_aio_context : public disk_aio -{ - OVERLAPPED olp; - aio_task *tsk; - utils::notify_event *evt; - error_code err; - uint32_t bytes; -}; - -hpc_aio_provider::hpc_aio_provider(disk_engine *disk, aio_provider *inner_provider) - : aio_provider(disk, inner_provider) -{ - _looper = nullptr; - _callback = [this](int native_error, uint32_t io_size, uintptr_t lolp_or_events) { - windows_disk_aio_context *ctx = - CONTAINING_RECORD(lolp_or_events, windows_disk_aio_context, olp); - error_code err = - native_error == ERROR_SUCCESS - ? ERR_OK - : (native_error == ERROR_HANDLE_EOF ? ERR_HANDLE_EOF : ERR_FILE_OPERATION_FAILED); - if (!ctx->evt) { - aio_task *aio(ctx->tsk); - this->complete_io(aio, err, io_size); - } else { - ctx->err = err; - ctx->bytes = io_size; - ctx->evt->notify(); - } - }; -} - -void hpc_aio_provider::start(io_modifer &ctx) -{ - _looper = get_io_looper(node(), ctx.queue, ctx.mode); -} - -hpc_aio_provider::~hpc_aio_provider() {} - -dsn_handle_t hpc_aio_provider::open(const char *file_name, int oflag, int pmode) -{ - DWORD dwDesiredAccess = 0; - DWORD dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; - DWORD dwCreationDisposition = 0; - DWORD dwFlagsAndAttributes = FILE_FLAG_OVERLAPPED; - - SECURITY_ATTRIBUTES SecurityAttributes; - - SecurityAttributes.nLength = sizeof(SecurityAttributes); - SecurityAttributes.lpSecurityDescriptor = NULL; - - if (oflag & _O_NOINHERIT) { - SecurityAttributes.bInheritHandle = FALSE; - } else { - SecurityAttributes.bInheritHandle = TRUE; - } - - /* - * decode the access flags - */ - switch (oflag & (_O_RDONLY | _O_WRONLY | _O_RDWR)) { - - case _O_RDONLY: /* read access */ - dwDesiredAccess = GENERIC_READ; - break; - case _O_WRONLY: /* write access */ - /* giving it read access as well - * because in append (a, not a+), we need - * to read the BOM to determine the encoding - * (ie. ANSI, UTF8, UTF16) - */ - if ((oflag & _O_APPEND) && (oflag & (_O_WTEXT | _O_U16TEXT | _O_U8TEXT)) != 0) { - dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; - } else { - dwDesiredAccess = GENERIC_WRITE; - } - break; - case _O_RDWR: /* read and write access */ - dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; - break; - default: /* error, bad oflag */ - _doserrno = 0L; /* not an OS error */ - derror("Invalid open flag"); - } - - /* - * decode open/create method flags - */ - switch (oflag & (_O_CREAT | _O_EXCL | _O_TRUNC)) { - case 0: - case _O_EXCL: // ignore EXCL w/o CREAT - dwCreationDisposition = OPEN_EXISTING; - break; - - case _O_CREAT: - dwCreationDisposition = OPEN_ALWAYS; - break; - - case _O_CREAT | _O_EXCL: - case _O_CREAT | _O_TRUNC | _O_EXCL: - dwCreationDisposition = CREATE_NEW; - break; - - case _O_TRUNC: - case _O_TRUNC | _O_EXCL: // ignore EXCL w/o CREAT - dwCreationDisposition = TRUNCATE_EXISTING; - break; - - case _O_CREAT | _O_TRUNC: - dwCreationDisposition = CREATE_ALWAYS; - break; - - default: - // this can't happen ... all cases are covered - _doserrno = 0L; - derror("Invalid open flag"); - } - - /* - * try to open/create the file - */ - HANDLE fileHandle = ::CreateFileA(file_name, - dwDesiredAccess, - dwShareMode, - &SecurityAttributes, - dwCreationDisposition, - dwFlagsAndAttributes, - 0); - - if (fileHandle != INVALID_HANDLE_VALUE && fileHandle != nullptr) { - auto err = _looper->bind_io_handle((dsn_handle_t)fileHandle, &_callback); - if (err != ERR_OK) { - dassert(false, - "cannot associate file handle %s to io completion port, err = 0x%x", - file_name, - ::GetLastError()); - return 0; - } else { - return (dsn_handle_t)(fileHandle); - } - } else { - derror("cannot create file %s, err = 0x%x", file_name, ::GetLastError()); - return 0; - } -} - -error_code hpc_aio_provider::close(dsn_handle_t fh) -{ - if (::CloseHandle((HANDLE)(fh))) - return ERR_OK; - else { - derror("close file failed, err = 0x%x", ::GetLastError()); - return ERR_FILE_OPERATION_FAILED; - } -} - -error_code hpc_aio_provider::flush(dsn_handle_t fh) -{ - if (fh == DSN_INVALID_FILE_HANDLE || ::FlushFileBuffers((HANDLE)(fh))) { - return ERR_OK; - } else { - derror("close file failed, err = 0x%x", ::GetLastError()); - return ERR_FILE_OPERATION_FAILED; - } -} - -disk_aio *hpc_aio_provider::prepare_aio_context(aio_task *tsk) -{ - auto r = new windows_disk_aio_context; - ZeroMemory(&r->olp, sizeof(r->olp)); - r->tsk = tsk; - r->evt = nullptr; - return r; -} - -void hpc_aio_provider::aio(aio_task *aio_tsk) { aio_internal(aio_tsk, true); } - -error_code hpc_aio_provider::aio_internal(aio_task *aio_tsk, - bool async, - /*out*/ uint32_t *pbytes /*= nullptr*/) -{ - auto aio = (windows_disk_aio_context *)aio_tsk->aio(); - BOOL r = FALSE; - - aio->olp.Offset = (uint32_t)aio->file_offset; - aio->olp.OffsetHigh = (uint32_t)(aio->file_offset >> 32); - - if (!async) { - aio->evt = new utils::notify_event(); - aio->err = ERR_OK; - aio->bytes = 0; - } - - switch (aio->type) { - case AIO_Read: - r = ::ReadFile((HANDLE)aio->file, aio->buffer, aio->buffer_size, NULL, &aio->olp); - break; - case AIO_Write: - r = ::WriteFile((HANDLE)aio->file, aio->buffer, aio->buffer_size, NULL, &aio->olp); - break; - default: - dassert(false, "unknown aio type %u", static_cast(aio->type)); - break; - } - - if (!r) { - int native_error = ::GetLastError(); - - if (native_error != ERROR_IO_PENDING) { - derror("file operation failed, err = %u", native_error); - - error_code err = native_error == ERROR_SUCCESS - ? ERR_OK - : (native_error == ERROR_HANDLE_EOF ? ERR_HANDLE_EOF - : ERR_FILE_OPERATION_FAILED); - - if (async) { - complete_io(aio_tsk, err, 0); - } else { - delete aio->evt; - aio->evt = nullptr; - } - - return err; - } - } - - if (async) { - return ERR_IO_PENDING; - } else { - aio->evt->wait(); - delete aio->evt; - aio->evt = nullptr; - if (pbytes != nullptr) { - *pbytes = aio->bytes; - } - return aio->err; - } -} -} -} // end namespace dsn::tools -#endif diff --git a/src/core/tools/hpc/hpc_env_provider.cpp b/src/core/tools/hpc/hpc_env_provider.cpp deleted file mode 100644 index 38b29dc919..0000000000 --- a/src/core/tools/hpc/hpc_env_provider.cpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * What is this file about? - * - * Revision history: - * xxxx-xx-xx, author, first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#include "hpc_env_provider.h" - -namespace dsn { -namespace tools { -hpc_env_provider::hpc_env_provider(env_provider *inner_provider) : env_provider(inner_provider) -{ - _ns_start = utils::get_current_physical_time_ns(); -#if defined(_WIN32) - ::QueryPerformanceCounter((LARGE_INTEGER *)&_tick_start); - - uint64_t freq; - ::QueryPerformanceFrequency((LARGE_INTEGER *)&freq); - _tick_frequency_per_ns = (double)freq / 1000.0 / 1000.0 / 1000.0; -#else -// TODO: -#endif -} -} -} diff --git a/src/core/tools/hpc/hpc_env_provider.h b/src/core/tools/hpc/hpc_env_provider.h deleted file mode 100644 index 99000158e3..0000000000 --- a/src/core/tools/hpc/hpc_env_provider.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * What is this file about? - * - * Revision history: - * xxxx-xx-xx, author, first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#pragma once - -#include - -namespace dsn { -namespace tools { -class hpc_env_provider : public env_provider -{ -public: - hpc_env_provider(env_provider *inner_provider); - - virtual uint64_t now_ns() const - { -#if defined(_WIN32) - uint64_t now; - ::QueryPerformanceCounter((LARGE_INTEGER *)&now); - return _ns_start + (uint64_t)((double)(now - _tick_start) / _tick_frequency_per_ns); -#else - return utils::get_current_physical_time_ns(); -#endif - } - -private: - uint64_t _ns_start; -}; -} -} diff --git a/src/core/tools/hpc/hpc_logger.cpp b/src/core/tools/hpc/hpc_logger.cpp deleted file mode 100644 index f1ac450d53..0000000000 --- a/src/core/tools/hpc/hpc_logger.cpp +++ /dev/null @@ -1,389 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * What is this file about? - * - * Revision history: - * xxxx-xx-xx, author, first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -/************************************************************ -* hpc_logger (High-Performance Computing logger) -* -* Editor: Chang Lou (v-chlou@microsoft.com) -* -* -* The structure of the logger is like the following graph. -* -* For each thread: -* ------------------------- ------------------------- -* | new single log | -> | | -* ------------------------- | --------------------- | -* | | -* | --------------------- | -* | | -* | --------------------- | -* | | -* | --------------------- | -* ... -* | --------------------- | -* | | -* ------------------------- -* buffer (per thread) -* | -* | when the buffer is full, -* | push the buffer and buffer size into -_write_list, -* | malloc a new buffer for the thread to use -* V -* -========================================================================================================== -_write_list_lock -* -* ------------------------------------------------------------- -* {buf1, buf1_size} | {buf2, buf2_size} | {buf3, -buf3_size} | ... -* ------------------------------------------------------------- -* _write_list -* -* -========================================================================================================== -_write_list_lock -* | -* | when the _write_list is not -empty, -* | daemon thread is notified by -_write_list_cond -* V -* -* Daemon thread -* -* || -* ===========> log.x.txt -* -* Some other facts: -* 1. The log file size is restricted, when max size is achieved, a new log file will be -established. -* 2. When exiting, the logger flushes, in other words, print out the retained log info in buffers -of each thread and buffers in the buffer list. - -************************************************************/ - -#include "hpc_logger.h" -#include -#include -#include -#include -#include -#include -#include -#include - -#define MAX_FILE_SIZE 30 * 1024 * 1024 -namespace dsn { -namespace tools { -typedef struct __hpc_log_info__ -{ - uint32_t magic; - char *buffer; - char *next_write_ptr; -} hpc_log_tls_info; - -// log ptr for each thread -static __thread hpc_log_tls_info s_hpc_log_tls_info; - -// store log ptr for each thread -typedef ::dsn::utils::safe_singleton_store hpc_log_manager; - -// daemon thread -void hpc_logger::log_thread() -{ - std::vector saved_list; - - while (!_stop_thread) { - _write_list_lock.lock(); - _write_list_cond.wait(_write_list_lock, - [=] { return _stop_thread || _write_list.size() > 0; }); - saved_list = std::move(_write_list); - _write_list.clear(); - _is_writing = true; - _write_list_lock.unlock(); - - write_buffer_list(saved_list); - _is_writing = false; - } - - _write_list_lock.lock(); - saved_list = _write_list; - _write_list.clear(); - _is_writing = true; - _write_list_lock.unlock(); - - write_buffer_list(saved_list); - _is_writing = false; -} - -hpc_logger::hpc_logger(const char *log_dir) - : logging_provider(log_dir), _stop_thread(false), _is_writing(false) -{ - _log_dir = std::string(log_dir); - _per_thread_buffer_bytes = - (int)dsn_config_get_value_uint64("tools.hpc_logger", - "per_thread_buffer_bytes", - 64 * 1024, // 64 KB by default - "buffer size for per-thread logging"); - _max_number_of_log_files_on_disk = dsn_config_get_value_uint64( - "tools.hpc_logger", - "max_number_of_log_files_on_disk", - 20, - "max number of log files reserved on disk, older logs are auto deleted"); - - _start_index = 0; - _index = 1; - _current_log_file_bytes = 0; - - // check existing log files and decide start_index - std::vector sub_list; - if (!dsn::utils::filesystem::get_subfiles(_log_dir, sub_list, false)) { - dassert(false, "Fail to get subfiles in %s.", _log_dir.c_str()); - } - - for (auto &fpath : sub_list) { - auto &&name = dsn::utils::filesystem::get_file_name(fpath); - if (name.length() <= 5 || name.substr(0, 4) != "log.") - continue; - - int index; - if (1 != sscanf(name.c_str(), "log.%d.txt", &index) || index < 1) - continue; - - if (index > _index) - _index = index; - - if (_start_index == 0 || index < _start_index) - _start_index = index; - } - sub_list.clear(); - - if (_start_index == 0) - _start_index = _index; - else - _index++; - - _current_log = nullptr; - create_log_file(); - _log_thread = std::thread(&hpc_logger::log_thread, this); -} - -void hpc_logger::create_log_file() -{ - std::stringstream log; - log << _log_dir << "/log." << _index++ << ".txt"; - _current_log = new std::ofstream( - log.str().c_str(), std::ofstream::out | std::ofstream::app | std::ofstream::binary); - _current_log_file_bytes = 0; - - // TODO: move gc out of criticial path - while (_index - _start_index > _max_number_of_log_files_on_disk) { - std::stringstream str2; - str2 << "log." << _start_index++ << ".txt"; - auto dp = utils::filesystem::path_combine(_log_dir, str2.str()); - if (::remove(dp.c_str()) != 0) { - printf("Failed to remove garbage log file %s\n", dp.c_str()); - _start_index--; - break; - } - } -} - -hpc_logger::~hpc_logger(void) -{ - flush(); - - if (!_stop_thread) { - _stop_thread = true; - _write_list_cond.notify_one(); - _log_thread.join(); - } - - _current_log->close(); - delete _current_log; -} - -void hpc_logger::flush() -{ - std::vector threads; - hpc_log_manager::instance().get_all_keys(threads); - - for (auto &tid : threads) { - __hpc_log_info__ *log; - if (!hpc_log_manager::instance().get(tid, log)) - continue; - - _write_list_lock.lock(); - if (log->next_write_ptr != log->buffer) { - buffer_push(log->buffer, static_cast(log->next_write_ptr - log->buffer)); - log->buffer = (char *)malloc(_per_thread_buffer_bytes); - log->next_write_ptr = log->buffer; - } - _write_list_lock.unlock(); - } - - _write_list_cond.notify_one(); - - bool wait = true; - while (wait) { - _write_list_lock.lock(); - if (_write_list.size() == 0 && !_is_writing) - wait = false; - _write_list_lock.unlock(); - if (wait) - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } -} - -void hpc_logger::dsn_logv(const char *file, - const char *function, - const int line, - dsn_log_level_t log_level, - const char *fmt, - va_list args) -{ - if (s_hpc_log_tls_info.magic != 0xdeadbeef) { - s_hpc_log_tls_info.buffer = (char *)malloc(_per_thread_buffer_bytes); - s_hpc_log_tls_info.next_write_ptr = s_hpc_log_tls_info.buffer; - - hpc_log_manager::instance().put(::dsn::utils::get_current_tid(), &s_hpc_log_tls_info); - s_hpc_log_tls_info.magic = 0xdeadbeef; - } - - // get enough write space >= 1K - if (s_hpc_log_tls_info.next_write_ptr + 1024 > - s_hpc_log_tls_info.buffer + _per_thread_buffer_bytes) { - _write_list_lock.lock(); - buffer_push( - s_hpc_log_tls_info.buffer, - static_cast(s_hpc_log_tls_info.next_write_ptr - s_hpc_log_tls_info.buffer)); - s_hpc_log_tls_info.buffer = (char *)malloc(_per_thread_buffer_bytes); - s_hpc_log_tls_info.next_write_ptr = s_hpc_log_tls_info.buffer; - _write_list_lock.unlock(); - - _write_list_cond.notify_one(); - } - - char *ptr = s_hpc_log_tls_info.next_write_ptr; - char *ptr0 = ptr; // remember it - size_t capacity = - static_cast(s_hpc_log_tls_info.buffer + _per_thread_buffer_bytes - ptr); - - // print verbose log header - uint64_t ts = 0; - int tid = ::dsn::utils::get_current_tid(); - if (::dsn::tools::is_engine_ready()) - ts = dsn_now_ns(); - char str[24]; - ::dsn::utils::time_ms_to_string(ts / 1000000, str); - static const char s_level_char[] = "IDWEF"; - auto wn = snprintf_p( - ptr, capacity, "%c%s (%" PRIu64 " %04x) ", s_level_char[log_level], str, ts, tid); - ptr += wn; - capacity -= wn; - - auto t = task::get_current_task_id(); - if (t) { - if (nullptr != task::get_current_worker2()) { - wn = snprintf_p(ptr, - capacity, - "%6s.%7s%d.%016" PRIx64 ": ", - task::get_current_node_name(), - task::get_current_worker2()->pool_spec().name.c_str(), - task::get_current_worker2()->index(), - t); - } else { - wn = snprintf_p(ptr, - capacity, - "%6s.%7s.%05d.%016" PRIx64 ": ", - task::get_current_node_name(), - "io-thrd", - tid, - t); - } - } else { - wn = snprintf_p( - ptr, capacity, "%6s.%7s.%05d: ", task::get_current_node_name(), "io-thrd", tid); - } - - ptr += wn; - capacity -= wn; - - // print body - wn = std::vsnprintf(ptr, capacity - 1, fmt, args); - if (wn < 0) { - wn = snprintf_p(ptr, capacity - 1, "-- cannot printf due to that log entry has error ---"); - } else if (static_cast(wn) >= capacity) { - // log truncated - wn = capacity - 1; - } - - *(ptr + wn) = '\n'; - ptr += (wn + 1); - capacity -= (wn + 1); - - // set next write ptr - s_hpc_log_tls_info.next_write_ptr = ptr; - - // dump critical logs on screen - if (log_level >= LOG_LEVEL_WARNING) { - std::cout.write(ptr0, ptr - ptr0); - } -} - -void hpc_logger::buffer_push(char *buffer, int size) { _write_list.emplace_back(buffer, size); } - -void hpc_logger::write_buffer_list(std::vector &llist) -{ - for (auto &new_buffer_info : llist) { - if (_current_log_file_bytes + new_buffer_info.buffer_size >= MAX_FILE_SIZE) { - _current_log->close(); - delete _current_log; - _current_log = nullptr; - - create_log_file(); - } - - if (new_buffer_info.buffer_size > 0) { - _current_log->write(new_buffer_info.buffer, new_buffer_info.buffer_size); - _current_log_file_bytes += new_buffer_info.buffer_size; - } - } - _current_log->flush(); - llist.clear(); -} -} -} diff --git a/src/core/tools/hpc/hpc_logger.h b/src/core/tools/hpc/hpc_logger.h deleted file mode 100644 index f55a47886e..0000000000 --- a/src/core/tools/hpc/hpc_logger.h +++ /dev/null @@ -1,98 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * What is this file about? - * - * Revision history: - * xxxx-xx-xx, author, first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#pragma once - -#include -#include -#include -#include -#include - -namespace dsn { -namespace tools { -struct buffer_info -{ - char *buffer; - int buffer_size; - buffer_info(char *buffer, int buffer_size) : buffer(buffer), buffer_size(buffer_size) {} -}; - -class hpc_logger : public logging_provider -{ -public: - hpc_logger(const char *log_dir); - virtual ~hpc_logger(void); - - virtual void dsn_logv(const char *file, - const char *function, - const int line, - dsn_log_level_t log_level, - const char *fmt, - va_list args); - - virtual void flush(); - -private: - void log_thread(); - - void buffer_push(char *buffer, int size); - // print logs in log list - void write_buffer_list(std::vector &llist); - void create_log_file(); - -private: - bool _stop_thread; - std::thread _log_thread; - std::string _log_dir; - - // global buffer list - std::condition_variable_any _write_list_cond; - ::dsn::utils::ex_lock_nr_spin _write_list_lock; - std::vector _write_list; - volatile bool _is_writing; - - // log file and line count - int _start_index; - int _index; - int _per_thread_buffer_bytes; - int _current_log_file_bytes; - int _max_number_of_log_files_on_disk; - - // current write file - std::ofstream *_current_log; -}; -} -} diff --git a/src/core/tools/hpc/hpc_network_provider.bsd.cpp b/src/core/tools/hpc/hpc_network_provider.bsd.cpp deleted file mode 100644 index 7439ac3189..0000000000 --- a/src/core/tools/hpc/hpc_network_provider.bsd.cpp +++ /dev/null @@ -1,524 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * Network using kqueue on BSD. - * - * Revision history: - * 2015-09-06, HX Lin(linmajia@live.com), first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#if defined(__APPLE__) || defined(__FreeBSD__) - -#include "hpc_network_provider.h" -#include "mix_all_io_looper.h" -#include - -namespace dsn { -namespace tools { -static socket_t create_tcp_socket(sockaddr_in *addr) -{ - socket_t s = -1; - if ((s = socket(AF_INET, SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP)) == -1) { - dwarn("socket failed, err = %s", strerror(errno)); - return -1; - } - - int reuse = 1; - if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char *)&reuse, sizeof(int)) == -1) { - dwarn("setsockopt SO_REUSEADDR failed, err = %s", strerror(errno)); - } - - int nodelay = 1; - if (setsockopt(s, IPPROTO_TCP, TCP_NODELAY, (char *)&nodelay, sizeof(int)) != 0) { - dwarn("setsockopt TCP_NODELAY failed, err = %s", strerror(errno)); - } - - int buflen = 8 * 1024 * 1024; - if (setsockopt(s, SOL_SOCKET, SO_SNDBUF, (char *)&buflen, sizeof(buflen)) != 0) { - dwarn("setsockopt SO_SNDBUF failed, err = %s", strerror(errno)); - } - - buflen = 8 * 1024 * 1024; - if (setsockopt(s, SOL_SOCKET, SO_RCVBUF, (char *)&buflen, sizeof(buflen)) != 0) { - dwarn("setsockopt SO_RCVBUF failed, err = %s", strerror(errno)); - } - - int keepalive = 1; - if (setsockopt(s, SOL_SOCKET, SO_KEEPALIVE, (char *)&keepalive, sizeof(keepalive)) != 0) { - dwarn("setsockopt SO_KEEPALIVE failed, err = %s", strerror(errno)); - } - - if (addr != 0) { - if (bind(s, (struct sockaddr *)addr, sizeof(*addr)) != 0) { - derror("bind failed, err = %s", strerror(errno)); - ::close(s); - return -1; - } - } - - return s; -} - -hpc_network_provider::hpc_network_provider(rpc_engine *srv, network *inner_provider) - : connection_oriented_network(srv, inner_provider) -{ - _listen_fd = -1; - _looper = nullptr; - _max_buffer_block_count_per_send = 1; // TODO: after fixing we can increase it -} - -error_code -hpc_network_provider::start(rpc_channel channel, int port, bool client_only, io_modifer &ctx) -{ - if (_listen_fd != -1) - return ERR_SERVICE_ALREADY_RUNNING; - - _looper = get_io_looper(node(), ctx.queue, ctx.mode); - - dassert(channel == RPC_CHANNEL_TCP || channel == RPC_CHANNEL_UDP, - "invalid given channel %s", - channel.to_string()); - - _address.assign_ipv4(get_local_ipv4(), port); - - if (!client_only) { - struct sockaddr_in addr; - addr.sin_family = AF_INET; - addr.sin_addr.s_addr = INADDR_ANY; - addr.sin_port = htons(port); - - _listen_fd = create_tcp_socket(&addr); - if (_listen_fd == -1) { - dassert(false, "cannot create listen socket"); - } - - int forcereuse = 1; - if (setsockopt( - _listen_fd, SOL_SOCKET, SO_REUSEADDR, (char *)&forcereuse, sizeof(forcereuse)) != - 0) { - dwarn("setsockopt SO_REUSEDADDR failed, err = %s", strerror(errno)); - } - - if (listen(_listen_fd, SOMAXCONN) != 0) { - dwarn("listen failed, err = %s", strerror(errno)); - return ERR_NETWORK_START_FAILED; - } - - _accept_event.callback = [this](int err, uint32_t size, uintptr_t lpolp) { - this->do_accept(); - }; - - // bind for accept - _looper->bind_io_handle((dsn_handle_t)(intptr_t)_listen_fd, - &_accept_event.callback, - EVFILT_READ, - nullptr // network_provider is a global object - ); - } - - return ERR_OK; -} - -rpc_session_ptr hpc_network_provider::create_client_session(::dsn::rpc_address server_addr) -{ - struct sockaddr_in addr; - addr.sin_family = AF_INET; - addr.sin_addr.s_addr = INADDR_ANY; - addr.sin_port = 0; - - auto sock = create_tcp_socket(&addr); - dassert(sock != -1, "create client tcp socket failed!"); - message_parser_ptr parser(new_message_parser(_client_hdr_format)); - auto client = new hpc_rpc_session(sock, parser, *this, server_addr, true); - rpc_session_ptr c(client); - client->bind_looper(_looper, true); - return c; -} - -void hpc_network_provider::do_accept() -{ - while (true) { - struct sockaddr_in addr; - socklen_t addr_len = (socklen_t)sizeof(addr); - socket_t s = ::accept(_listen_fd, (struct sockaddr *)&addr, &addr_len); - if (s != -1) { - ::dsn::rpc_address client_addr(ntohl(addr.sin_addr.s_addr), ntohs(addr.sin_port)); - message_parser_ptr null_parser; - auto rs = new hpc_rpc_session(s, null_parser, *this, client_addr, false); - rpc_session_ptr s1(rs); - - rs->bind_looper(_looper); - this->on_server_session_accepted(s1); - } else { - if (errno != EAGAIN && errno != EWOULDBLOCK) { - derror("accept failed, err = %s", strerror(errno)); - } - break; - } - } -} - -void hpc_rpc_session::bind_looper(io_looper *looper, bool delay) -{ - _looper = looper; - if (!delay) { - // bind for send/recv - looper->bind_io_handle( - (dsn_handle_t)(intptr_t)_socket, &_ready_event, EVFILT_READ_WRITE, this); - } -} - -void hpc_rpc_session::do_read(int read_next) -{ - utils::auto_lock l(_send_lock); - - while (true) { - char *ptr = _reader.read_buffer_ptr(read_next); - int remaining = _reader.read_buffer_capacity(); - - int length = recv(_socket, ptr, remaining, 0); - int err = errno; - dinfo("(s = %d) call recv on %s, return %d, err = %s", - _socket, - _remote_addr.to_string(), - length, - strerror(err)); - - if (length > 0) { - _reader.mark_read(length); - - if (!_parser) { - read_next = prepare_parser(); - } - - if (_parser) { - message_ex *msg = _parser->get_message_on_receive(&_reader, read_next); - - while (msg != nullptr) { - this->on_read_completed(msg); - msg = _parser->get_message_on_receive(&_reader, read_next); - } - } - - if (read_next == -1) { - derror( - "(s = %d) recv failed on %s, parse failed", _socket, _remote_addr.to_string()); - on_failure(); - break; - } - } else { - if (err != EAGAIN && err != EWOULDBLOCK) { - derror("(s = %d) recv failed on %s, err = %s", - _socket, - _remote_addr.to_string(), - strerror(err)); - on_failure(); - } - break; - } - } -} - -void hpc_rpc_session::do_safe_write(uint64_t sig) -{ - utils::auto_lock l(_send_lock); - - if (0 == sig) { - if (_sending_signature) { - do_write(_sending_signature); - } else { - _send_lock.unlock(); // avoid recursion - on_send_completed(); // send next msg if there is. - _send_lock.lock(); - } - } else { - do_write(sig); - } -} - -void hpc_rpc_session::do_write(uint64_t sig) -{ - int flags; - - static_assert(sizeof(dsn_message_parser::send_buf) == sizeof(struct iovec), - "make sure they are compatible"); - - dbg_dassert(sig != 0, "cannot send empty msg"); - - // new msg - if (_sending_signature == 0) { - _sending_signature = sig; - _sending_buffer_start_index = 0; - } - - // continue old msg - else { - dassert(_sending_signature == sig, "only one sending msg is possible"); - } - - flags = -#ifdef __APPLE__ - SO_NOSIGPIPE -#else - MSG_NOSIGNAL -#endif - ; - - // prepare send buffer, make sure header is already in the buffer - while (true) { - int buffer_count = (int)_sending_buffers.size() - _sending_buffer_start_index; - struct msghdr hdr; - memset((void *)&hdr, 0, sizeof(hdr)); - hdr.msg_name = (void *)&_peer_addr; - hdr.msg_namelen = (socklen_t)sizeof(_peer_addr); - hdr.msg_iov = (struct iovec *)&_sending_buffers[_sending_buffer_start_index]; - hdr.msg_iovlen = (size_t)buffer_count; - - int sz = sendmsg(_socket, &hdr, flags); - int err = errno; - dinfo("(s = %d) call sendmsg on %s, return %d, err = %s", - _socket, - _remote_addr.to_string(), - sz, - strerror(err)); - - if (sz < 0) { - if (err != EAGAIN && err != EWOULDBLOCK) { - derror("(s = %d) sendmsg failed, err = %s", _socket, strerror(err)); - on_failure(true); - } else { - // wait for epoll_wait notification - } - return; - } else { - int len = (int)sz; - int buf_i = _sending_buffer_start_index; - while (len > 0) { - auto &buf = _sending_buffers[buf_i]; - if (len >= (int)buf.sz) { - buf_i++; - len -= (int)buf.sz; - } else { - buf.buf = (char *)buf.buf + len; - buf.sz -= len; - break; - } - } - _sending_buffer_start_index = buf_i; - - // message completed, continue next message - if (_sending_buffer_start_index == (int)_sending_buffers.size()) { - dassert(len == 0, "buffer must be sent completely"); - auto csig = _sending_signature; - _sending_signature = 0; - - _send_lock.unlock(); // avoid recursion - // try next msg recursively - on_send_completed(csig); - _send_lock.lock(); - return; - } - - else { - // try next while(true) loop to continue sending current msg - } - } - } -} - -void hpc_rpc_session::close() -{ - if (-1 != _socket) { - ::close(_socket); - dinfo("(s = %d) close socket %p", _socket, this); - _socket = -1; - } -} - -void hpc_rpc_session::on_send_recv_events_ready(uintptr_t lolp_or_events) -{ - struct kevent &e = *((struct kevent *)lolp_or_events); - // shutdown or send/recv error - if (((e.flags & EV_ERROR) != 0) || ((e.flags & EV_EOF) != 0)) { - dinfo("(s = %d) kevent failure on %s, events = 0x%x", - _socket, - _remote_addr.to_string(), - e.filter); - on_failure(); - return; - } - - // send - if (e.filter == EVFILT_WRITE) { - dinfo("(s = %d) kevent EVFILT_WRITE on %s, events = 0x%x", - _socket, - _remote_addr.to_string(), - e.filter); - - do_safe_write(0); - } - - // recv - if (e.filter == EVFILT_READ) { - dinfo("(s = %d) kevent EVFILT_READ on %s, events = 0x%x", - _socket, - _remote_addr.to_string(), - e.filter); - - start_read_next(); - } -} - -// client -hpc_rpc_session::hpc_rpc_session(socket_t sock, - message_parser_ptr &parser, - connection_oriented_network &net, - ::dsn::rpc_address remote_addr, - bool is_client) - : rpc_session(net, remote_addr, parser, is_client), _socket(sock) -{ - dassert(sock != -1, "invalid given socket handle"); - _sending_signature = 0; - _sending_buffer_start_index = 0; - _looper = nullptr; - - memset((void *)&_peer_addr, 0, sizeof(_peer_addr)); - _peer_addr.sin_family = AF_INET; - _peer_addr.sin_addr.s_addr = INADDR_ANY; - _peer_addr.sin_port = 0; - - if (is_client) { - _ready_event = [this](int err, uint32_t length, uintptr_t lolp_or_events) { - if (is_connecting()) - this->on_connect_events_ready(lolp_or_events); - else - this->on_send_recv_events_ready(lolp_or_events); - }; - } else { - socklen_t addr_len = (socklen_t)sizeof(_peer_addr); - if (getpeername(_socket, (struct sockaddr *)&_peer_addr, &addr_len) == -1) { - derror("(server) getpeername failed, err = %s", strerror(errno)); - } - - _ready_event = [this](int err, uint32_t length, uintptr_t lolp_or_events) { - struct kevent &e = *((struct kevent *)lolp_or_events); - dinfo("(s = %d) (server) epoll for send/recv to %s, events = %d", - _socket, - _remote_addr.to_string(), - e.filter); - this->on_send_recv_events_ready(lolp_or_events); - }; - } -} - -void hpc_rpc_session::on_connect_events_ready(uintptr_t lolp_or_events) -{ - dassert(is_connecting(), "session must be connecting at this time"); - - struct kevent &e = *((struct kevent *)lolp_or_events); - dinfo("(s = %d) epoll for connect to %s, events = 0x%x", - _socket, - _remote_addr.to_string(), - e.filter); - - if ((e.filter == EVFILT_WRITE) && ((e.flags & EV_ERROR) == 0) && ((e.flags & EV_EOF) == 0)) { - socklen_t addr_len = (socklen_t)sizeof(_peer_addr); - if (getpeername(_socket, (struct sockaddr *)&_peer_addr, &addr_len) == -1) { - derror("(s = %d) (client) getpeername failed, err = %s", _socket, strerror(errno)); - on_failure(); - return; - } - - dinfo("(s = %d) client session %s connected", _socket, _remote_addr.to_string()); - - set_connected(); - - if (_looper->bind_io_handle( - (dsn_handle_t)(intptr_t)_socket, &_ready_event, EVFILT_READ_WRITE) != ERR_OK) { - on_failure(); - return; - } - - // start first round send - do_safe_write(0); - } else { - int err = 0; - socklen_t err_len = (socklen_t)sizeof(err); - - if (getsockopt(_socket, SOL_SOCKET, SO_ERROR, (void *)&err, &err_len) < 0) { - derror("getsockopt for SO_ERROR failed, err = %s", strerror(errno)); - } - - derror("(s = %d) connect failed (in epoll), err = %s", _socket, strerror(err)); - on_failure(true); - } -} - -void hpc_rpc_session::on_failure(bool is_write) -{ - if (_socket != -1) { - _looper->unbind_io_handle((dsn_handle_t)(intptr_t)_socket, &_ready_event); - } - if (on_disconnected(is_write)) - close(); -} - -void hpc_rpc_session::connect() -{ - if (!set_connecting()) - return; - - dassert(_socket != -1, "invalid given socket handle"); - - struct sockaddr_in addr; - addr.sin_family = AF_INET; - addr.sin_addr.s_addr = htonl(_remote_addr.ip()); - addr.sin_port = htons(_remote_addr.port()); - - int rt = ::connect(_socket, (struct sockaddr *)&addr, (int)sizeof(addr)); - int err = errno; - dinfo("(s = %d) call connect to %s, return %d, err = %s", - _socket, - _remote_addr.to_string(), - rt, - strerror(err)); - - if (rt == -1 && err != EINPROGRESS) { - dwarn("(s = %d) connect failed, err = %s", _socket, strerror(err)); - on_failure(true); - return; - } - - // bind for connect - _looper->bind_io_handle((dsn_handle_t)(intptr_t)_socket, &_ready_event, EVFILT_WRITE, this); -} -} -} - -#endif diff --git a/src/core/tools/hpc/hpc_network_provider.h b/src/core/tools/hpc/hpc_network_provider.h deleted file mode 100644 index 216f324cc3..0000000000 --- a/src/core/tools/hpc/hpc_network_provider.h +++ /dev/null @@ -1,152 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * What is this file about? - * - * Revision history: - * xxxx-xx-xx, author, first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#pragma once - -#ifdef _WIN32 -#include -typedef SOCKET socket_t; -#else -#include -#include -#include -#include -typedef int socket_t; -#if defined(__FreeBSD__) -#include -#endif -#endif - -#include -#include "io_looper.h" - -namespace dsn { -namespace tools { - -class hpc_network_provider : public connection_oriented_network -{ -public: - hpc_network_provider(rpc_engine *srv, network *inner_provider); - - virtual error_code start(rpc_channel channel, int port, bool client_only, io_modifer &ctx); - virtual ::dsn::rpc_address address() { return _address; } - virtual rpc_session_ptr create_client_session(::dsn::rpc_address server_addr); - -private: - socket_t _listen_fd; - ::dsn::rpc_address _address; - io_looper *_looper; - -private: - void do_accept(); - -public: - struct ready_event - { -#ifdef _WIN32 - OVERLAPPED olp; -#endif - io_loop_callback callback; - }; - -private: - ready_event _accept_event; -#ifdef _WIN32 - socket_t _accept_sock; - char _accept_buffer[1024]; -#endif -}; - -class hpc_rpc_session : public rpc_session -{ -public: - virtual void connect() override; - virtual void close() override; - -public: - hpc_rpc_session(socket_t sock, - message_parser_ptr &parser, - connection_oriented_network &net, - ::dsn::rpc_address remote_addr, - bool is_client); - - virtual void send(uint64_t signature) override - { -#ifdef _WIN32 - do_write(signature); -#else - do_safe_write(signature); -#endif - } - - void bind_looper(io_looper *looper, bool delay = false); - virtual void do_read(int read_next) override; - -private: - void do_write(uint64_t signature); - void on_failure(bool is_write = false); - void on_read_completed(message_ex *msg) - { - if (!on_recv_message(msg, 0)) { - on_failure(false); - } - } - -protected: - socket_t _socket; - uint64_t _sending_signature; - int _sending_buffer_start_index; - -#ifdef _WIN32 - hpc_network_provider::ready_event _read_event; - hpc_network_provider::ready_event _write_event; - hpc_network_provider::ready_event _connect_event; -#else - io_loop_callback _ready_event; - struct sockaddr_in _peer_addr; - io_looper *_looper; - - // due to the bad design of EPOLLET, we need to - // use locks to avoid concurrent send/recv - ::dsn::utils::ex_lock_nr _send_lock; - ::dsn::utils::ex_lock_nr _recv_lock; - - void on_connect_events_ready(uintptr_t lolp_or_events); - void on_send_recv_events_ready(uintptr_t lolp_or_events); - void do_safe_write(uint64_t signature); -#endif -}; -} -} diff --git a/src/core/tools/hpc/hpc_network_provider.linux.cpp b/src/core/tools/hpc/hpc_network_provider.linux.cpp deleted file mode 100644 index 59ef836c77..0000000000 --- a/src/core/tools/hpc/hpc_network_provider.linux.cpp +++ /dev/null @@ -1,518 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * What is this file about? - * - * Revision history: - * xxxx-xx-xx, author, first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#ifdef __linux__ - -#include "hpc_network_provider.h" -#include "mix_all_io_looper.h" -#include - -namespace dsn { -namespace tools { -static socket_t create_tcp_socket(sockaddr_in *addr) -{ - socket_t s = -1; - if ((s = socket(AF_INET, SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP)) == -1) { - dwarn("socket failed, err = %s", strerror(errno)); - return -1; - } - - int reuse = 1; - if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char *)&reuse, sizeof(int)) == -1) { - dwarn("setsockopt SO_REUSEADDR failed, err = %s", strerror(errno)); - } - - int nodelay = 1; - if (setsockopt(s, IPPROTO_TCP, TCP_NODELAY, (char *)&nodelay, sizeof(int)) != 0) { - dwarn("setsockopt TCP_NODELAY failed, err = %s", strerror(errno)); - } - - int buflen = 8 * 1024 * 1024; - if (setsockopt(s, SOL_SOCKET, SO_SNDBUF, (char *)&buflen, sizeof(buflen)) != 0) { - dwarn("setsockopt SO_SNDBUF failed, err = %s", strerror(errno)); - } - - buflen = 8 * 1024 * 1024; - if (setsockopt(s, SOL_SOCKET, SO_RCVBUF, (char *)&buflen, sizeof(buflen)) != 0) { - dwarn("setsockopt SO_RCVBUF failed, err = %s", strerror(errno)); - } - - int keepalive = 1; - if (setsockopt(s, SOL_SOCKET, SO_KEEPALIVE, (char *)&keepalive, sizeof(keepalive)) != 0) { - dwarn("setsockopt SO_KEEPALIVE failed, err = %s", strerror(errno)); - } - - if (addr != 0) { - if (bind(s, (struct sockaddr *)addr, sizeof(*addr)) != 0) { - derror("bind failed, err = %s", strerror(errno)); - ::close(s); - return -1; - } - } - - return s; -} - -hpc_network_provider::hpc_network_provider(rpc_engine *srv, network *inner_provider) - : connection_oriented_network(srv, inner_provider) -{ - _listen_fd = -1; - _looper = nullptr; - _max_buffer_block_count_per_send = 128; -} - -error_code -hpc_network_provider::start(rpc_channel channel, int port, bool client_only, io_modifer &ctx) -{ - if (_listen_fd != -1) - return ERR_SERVICE_ALREADY_RUNNING; - - _looper = get_io_looper(node(), ctx.queue, ctx.mode); - - dassert(channel == RPC_CHANNEL_TCP || channel == RPC_CHANNEL_UDP, - "invalid given channel %s", - channel.to_string()); - - _address.assign_ipv4(get_local_ipv4(), port); - - if (!client_only) { - struct sockaddr_in addr; - addr.sin_family = AF_INET; - addr.sin_addr.s_addr = INADDR_ANY; - addr.sin_port = htons(port); - - _listen_fd = create_tcp_socket(&addr); - if (_listen_fd == -1) { - dassert(false, "cannot create listen socket"); - } - - int forcereuse = 1; - if (setsockopt( - _listen_fd, SOL_SOCKET, SO_REUSEADDR, (char *)&forcereuse, sizeof(forcereuse)) != - 0) { - dwarn("setsockopt SO_REUSEDADDR failed, err = %s", strerror(errno)); - } - - if (listen(_listen_fd, SOMAXCONN) != 0) { - dwarn("listen failed, err = %s", strerror(errno)); - return ERR_NETWORK_START_FAILED; - } - - _accept_event.callback = [this](int err, uint32_t size, uintptr_t lpolp) { - this->do_accept(); - }; - - // bind for accept - _looper->bind_io_handle((dsn_handle_t)(intptr_t)_listen_fd, - &_accept_event.callback, - EPOLLIN | EPOLLET, - nullptr // network_provider is a global object - ); - } - - return ERR_OK; -} - -rpc_session_ptr hpc_network_provider::create_client_session(::dsn::rpc_address server_addr) -{ - struct sockaddr_in addr; - addr.sin_family = AF_INET; - addr.sin_addr.s_addr = INADDR_ANY; - addr.sin_port = 0; - - auto sock = create_tcp_socket(&addr); - dassert(sock != -1, "create client tcp socket failed!"); - message_parser_ptr parser(new_message_parser(_client_hdr_format)); - auto client = new hpc_rpc_session(sock, parser, *this, server_addr, true); - rpc_session_ptr c(client); - client->bind_looper(_looper, true); - return c; -} - -void hpc_network_provider::do_accept() -{ - while (true) { - struct sockaddr_in addr; - socklen_t addr_len = (socklen_t)sizeof(addr); - socket_t s = ::accept(_listen_fd, (struct sockaddr *)&addr, &addr_len); - if (s != -1) { - ::dsn::rpc_address client_addr(ntohl(addr.sin_addr.s_addr), ntohs(addr.sin_port)); - message_parser_ptr null_parser; - auto rs = new hpc_rpc_session(s, null_parser, *this, client_addr, false); - rpc_session_ptr s1(rs); - - rs->bind_looper(_looper); - this->on_server_session_accepted(s1); - } else { - if (errno != EAGAIN && errno != EWOULDBLOCK) { - derror("accept failed, err = %s", strerror(errno)); - } - break; - } - } -} - -void hpc_rpc_session::bind_looper(io_looper *looper, bool delay) -{ - _looper = looper; - if (!delay) { - // bind for send/recv - looper->bind_io_handle((dsn_handle_t)(intptr_t)_socket, - &_ready_event, - EPOLLIN | EPOLLOUT | EPOLLRDHUP | EPOLLET, - this); - } -} - -void hpc_rpc_session::do_read(int read_next) -{ - utils::auto_lock l(_send_lock); - - while (true) { - char *ptr = _reader.read_buffer_ptr(read_next); - int remaining = _reader.read_buffer_capacity(); - - int length = recv(_socket, ptr, remaining, 0); - int err = errno; - dinfo("(s = %d) call recv on %s, return %d, err = %s", - _socket, - _remote_addr.to_string(), - length, - strerror(err)); - - if (length > 0) { - _reader.mark_read(length); - - if (!_parser) { - read_next = prepare_parser(); - } - - if (_parser) { - message_ex *msg = _parser->get_message_on_receive(&_reader, read_next); - - while (msg != nullptr) { - this->on_read_completed(msg); - msg = _parser->get_message_on_receive(&_reader, read_next); - } - } - - if (read_next == -1) { - derror( - "(s = %d) recv failed on %s, parse failed", _socket, _remote_addr.to_string()); - on_failure(); - break; - } - } else { - if (err != EAGAIN && err != EWOULDBLOCK) { - derror("(s = %d) recv failed on %s, err = %s", - _socket, - _remote_addr.to_string(), - strerror(err)); - on_failure(); - } - break; - } - } -} - -void hpc_rpc_session::do_safe_write(uint64_t sig) -{ - utils::auto_lock l(_send_lock); - - if (0 == sig) { - if (_sending_signature) { - do_write(_sending_signature); - } else { - _send_lock.unlock(); // avoid recursion - on_send_completed(); // send next msg if there is. - _send_lock.lock(); - } - } else { - do_write(sig); - } -} - -void hpc_rpc_session::do_write(uint64_t sig) -{ - static_assert(sizeof(message_parser::send_buf) == sizeof(struct iovec), - "make sure they are compatible"); - - dbg_dassert(sig != 0, "cannot send empty msg"); - - // new msg - if (_sending_signature == 0) { - _sending_signature = sig; - _sending_buffer_start_index = 0; - } - - // continue old msg - else { - dassert(_sending_signature == sig, "only one sending msg is possible"); - } - - // prepare send buffer, make sure header is already in the buffer - while (true) { - int buffer_count = (int)_sending_buffers.size() - _sending_buffer_start_index; - struct msghdr hdr; - memset((void *)&hdr, 0, sizeof(hdr)); - hdr.msg_name = (void *)&_peer_addr; - hdr.msg_namelen = (socklen_t)sizeof(_peer_addr); - hdr.msg_iov = (struct iovec *)&_sending_buffers[_sending_buffer_start_index]; - hdr.msg_iovlen = (size_t)buffer_count; - - int sz = sendmsg(_socket, &hdr, MSG_NOSIGNAL); - int err = errno; - dinfo("(s = %d) call sendmsg on %s, return %d, err = %s", - _socket, - _remote_addr.to_string(), - sz, - strerror(err)); - - if (sz < 0) { - if (err != EAGAIN && err != EWOULDBLOCK) { - derror("(s = %d) sendmsg failed, err = %s", _socket, strerror(err)); - on_failure(true); - } else { - // wait for epoll_wait notification - } - return; - } else { - int len = (int)sz; - int buf_i = _sending_buffer_start_index; - while (len > 0) { - auto &buf = _sending_buffers[buf_i]; - if (len >= (int)buf.sz) { - buf_i++; - len -= (int)buf.sz; - } else { - buf.buf = (char *)buf.buf + len; - buf.sz -= len; - break; - } - } - _sending_buffer_start_index = buf_i; - - // message completed, continue next message - if (_sending_buffer_start_index == (int)_sending_buffers.size()) { - dassert(len == 0, "buffer must be sent completely"); - - auto csig = _sending_signature; - _sending_signature = 0; - - _send_lock.unlock(); // avoid recursion - // try next msg recursively - on_send_completed(csig); - _send_lock.lock(); - return; - } - - else { - // try next while(true) loop to continue sending current msg - } - } - } -} - -void hpc_rpc_session::close() -{ - if (-1 != _socket) { - ::close(_socket); - dinfo("(s = %d) close socket %p", _socket, this); - _socket = -1; - } -} - -void hpc_rpc_session::on_send_recv_events_ready(uintptr_t lolp_or_events) -{ - uint32_t events = (uint32_t)lolp_or_events; - // shutdown or send/recv error - if ((events & EPOLLHUP) || (events & EPOLLRDHUP) || (events & EPOLLERR)) { - dinfo("(s = %d) epoll failure on %s, events = 0x%x", - _socket, - _remote_addr.to_string(), - events); - on_failure(); - return; - } - - // send - if (events & EPOLLOUT) { - dinfo("(s = %d) epoll EPOLLOUT on %s, events = 0x%x", - _socket, - _remote_addr.to_string(), - events); - - do_safe_write(0); - } - - // recv - if (events & EPOLLIN) { - dinfo("(s = %d) epoll EPOLLIN on %s, events = 0x%x", - _socket, - _remote_addr.to_string(), - events); - - start_read_next(); - } -} - -hpc_rpc_session::hpc_rpc_session(socket_t sock, - message_parser_ptr &parser, - connection_oriented_network &net, - ::dsn::rpc_address remote_addr, - bool is_client) - : rpc_session(net, remote_addr, parser, is_client), _socket(sock) -{ - dassert(sock != -1, "invalid given socket handle"); - _sending_signature = 0; - _sending_buffer_start_index = 0; - _looper = nullptr; - - memset((void *)&_peer_addr, 0, sizeof(_peer_addr)); - _peer_addr.sin_family = AF_INET; - _peer_addr.sin_addr.s_addr = INADDR_ANY; - _peer_addr.sin_port = 0; - - if (is_client) { - _ready_event = [this](int err, uint32_t length, uintptr_t lolp_or_events) { - if (is_connecting()) - this->on_connect_events_ready(lolp_or_events); - else - this->on_send_recv_events_ready(lolp_or_events); - }; - } else { - socklen_t addr_len = (socklen_t)sizeof(_peer_addr); - if (getpeername(_socket, (struct sockaddr *)&_peer_addr, &addr_len) == -1) { - derror("(server) getpeername failed, err = %s", strerror(errno)); - } - - _ready_event = [this](int err, uint32_t length, uintptr_t lolp_or_events) { - uint32_t events = (uint32_t)lolp_or_events; - dinfo("(s = %d) (server) epoll for send/recv to %s, events = 0x%x", - _socket, - _remote_addr.to_string(), - events); - this->on_send_recv_events_ready(events); - }; - } -} - -void hpc_rpc_session::on_connect_events_ready(uintptr_t lolp_or_events) -{ - dassert(is_connecting(), "session must be connecting at this time"); - uint32_t events = (uint32_t)lolp_or_events; - dinfo("(s = %d) epoll for connect to %s, events = 0x%x", - _socket, - _remote_addr.to_string(), - events); - - if ((events & EPOLLOUT) && !(events & EPOLLERR) && !(events & EPOLLHUP)) { - socklen_t addr_len = (socklen_t)sizeof(_peer_addr); - if (getpeername(_socket, (struct sockaddr *)&_peer_addr, &addr_len) == -1) { - derror("(s = %d) (client) getpeername failed, err = %s", _socket, strerror(errno)); - on_failure(); - return; - } - - dinfo("(s = %d) client session %s connected", _socket, _remote_addr.to_string()); - - set_connected(); - - struct epoll_event e; - e.data.ptr = &_ready_event; - e.events = EPOLLIN | EPOLLOUT | EPOLLRDHUP | EPOLLET; - - if (epoll_ctl((int)(intptr_t)_looper->native_handle(), EPOLL_CTL_MOD, _socket, &e) < 0) { - derror("(s = %d) (client) EPOLL_CTL_MOD failed, err = %s", _socket, strerror(errno)); - on_failure(); - return; - } - - // start first round send - do_safe_write(0); - } else { - int err = 0; - socklen_t err_len = (socklen_t)sizeof(err); - - if (getsockopt(_socket, SOL_SOCKET, SO_ERROR, (void *)&err, &err_len) < 0) { - derror("getsockopt for SO_ERROR failed, err = %s", strerror(errno)); - } - - derror("(s = %d) connect failed (in epoll), err = %s", _socket, strerror(err)); - on_failure(true); - } -} - -void hpc_rpc_session::on_failure(bool is_write) -{ - _looper->unbind_io_handle((dsn_handle_t)(intptr_t)_socket, &_ready_event); - if (on_disconnected(is_write)) - close(); -} - -void hpc_rpc_session::connect() -{ - if (!set_connecting()) - return; - - dassert(_socket != -1, "invalid given socket handle"); - - struct sockaddr_in addr; - addr.sin_family = AF_INET; - addr.sin_addr.s_addr = htonl(_remote_addr.ip()); - addr.sin_port = htons(_remote_addr.port()); - - int rt = ::connect(_socket, (struct sockaddr *)&addr, (int)sizeof(addr)); - int err = errno; - dinfo("(s = %d) call connect to %s, return %d, err = %s", - _socket, - _remote_addr.to_string(), - rt, - strerror(err)); - - if (rt == -1 && err != EINPROGRESS) { - dwarn("(s = %d) connect failed, err = %s", _socket, strerror(err)); - on_failure(true); - return; - } - - // bind for connect - _looper->bind_io_handle( - (dsn_handle_t)(intptr_t)_socket, &_ready_event, EPOLLOUT | EPOLLET, this); -} -} -} - -#endif diff --git a/src/core/tools/hpc/hpc_network_provider.win.cpp b/src/core/tools/hpc/hpc_network_provider.win.cpp deleted file mode 100644 index a23c792ca7..0000000000 --- a/src/core/tools/hpc/hpc_network_provider.win.cpp +++ /dev/null @@ -1,506 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * What is this file about? - * - * Revision history: - * xxxx-xx-xx, author, first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#ifdef _WIN32 - -#include "hpc_network_provider.h" -#include -#include "mix_all_io_looper.h" - -namespace dsn { -namespace tools { -static socket_t create_tcp_socket(sockaddr_in *addr) -{ - socket_t s = INVALID_SOCKET; - if ((s = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, 0, 0, WSA_FLAG_OVERLAPPED)) == - INVALID_SOCKET) { - dwarn("WSASocket failed, err = %d", ::GetLastError()); - return INVALID_SOCKET; - } - - int reuse = 1; - if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char *)&reuse, sizeof(int)) == -1) { - dwarn("setsockopt SO_REUSEADDR failed, err = %s", strerror(errno)); - } - - int nodelay = 1; - if (setsockopt(s, IPPROTO_TCP, TCP_NODELAY, (char *)&nodelay, sizeof(int)) != 0) { - dwarn("setsockopt TCP_NODELAY failed, err = %d", ::GetLastError()); - } - - int isopt = 1; - if (setsockopt(s, SOL_SOCKET, SO_DONTLINGER, (char *)&isopt, sizeof(int)) != 0) { - dwarn("setsockopt SO_DONTLINGER failed, err = %d", ::GetLastError()); - } - - int buflen = 8 * 1024 * 1024; - if (setsockopt(s, SOL_SOCKET, SO_SNDBUF, (char *)&buflen, sizeof(buflen)) != 0) { - dwarn("setsockopt SO_SNDBUF failed, err = %d", ::GetLastError()); - } - - buflen = 8 * 1024 * 1024; - if (setsockopt(s, SOL_SOCKET, SO_RCVBUF, (char *)&buflen, sizeof(buflen)) != 0) { - dwarn("setsockopt SO_RCVBUF failed, err = %d", ::GetLastError()); - } - - int keepalive = 1; - if (setsockopt(s, SOL_SOCKET, SO_KEEPALIVE, (char *)&keepalive, sizeof(keepalive)) != 0) { - dwarn("setsockopt SO_KEEPALIVE failed, err = %d", ::GetLastError()); - } - - if (addr != 0) { - if (bind(s, (struct sockaddr *)addr, sizeof(*addr)) != 0) { - derror("bind failed, err = %d", ::GetLastError()); - closesocket(s); - return INVALID_SOCKET; - } - } - - return s; -} - -static LPFN_ACCEPTEX s_lpfnAcceptEx = NULL; -static LPFN_CONNECTEX s_lpfnConnectEx = NULL; -static LPFN_GETACCEPTEXSOCKADDRS s_lpfnGetAcceptExSockaddrs = NULL; - -static void load_socket_functions() -{ - if (s_lpfnGetAcceptExSockaddrs != NULL) - return; - - socket_t s = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); - if (s == INVALID_SOCKET) { - dassert(false, "create Socket Failed, err = %d", ::GetLastError()); - } - - GUID GuidAcceptEx = WSAID_ACCEPTEX; - GUID GuidConnectEx = WSAID_CONNECTEX; - GUID GuidGetAcceptExSockaddrs = WSAID_GETACCEPTEXSOCKADDRS; - DWORD dwBytes; - - // Load the AcceptEx function into memory using WSAIoctl. - // The WSAIoctl function is an extension of the ioctlsocket() - // function that can use overlapped I/O. The function's 3rd - // through 6th parameters are input and output buffers where - // we pass the pointer to our AcceptEx function. This is used - // so that we can call the AcceptEx function directly, rather - // than refer to the Mswsock.lib library. - int rt = WSAIoctl(s, - SIO_GET_EXTENSION_FUNCTION_POINTER, - &GuidAcceptEx, - sizeof(GuidAcceptEx), - &s_lpfnAcceptEx, - sizeof(s_lpfnAcceptEx), - &dwBytes, - NULL, - NULL); - if (rt == SOCKET_ERROR) { - dwarn("WSAIoctl for AcceptEx failed, err = %d", ::WSAGetLastError()); - closesocket(s); - return; - } - - rt = WSAIoctl(s, - SIO_GET_EXTENSION_FUNCTION_POINTER, - &GuidConnectEx, - sizeof(GuidConnectEx), - &s_lpfnConnectEx, - sizeof(s_lpfnConnectEx), - &dwBytes, - NULL, - NULL); - if (rt == SOCKET_ERROR) { - dwarn("WSAIoctl for ConnectEx failed, err = %d", ::WSAGetLastError()); - closesocket(s); - return; - } - - rt = WSAIoctl(s, - SIO_GET_EXTENSION_FUNCTION_POINTER, - &GuidGetAcceptExSockaddrs, - sizeof(GuidGetAcceptExSockaddrs), - &s_lpfnGetAcceptExSockaddrs, - sizeof(s_lpfnGetAcceptExSockaddrs), - &dwBytes, - NULL, - NULL); - if (rt == SOCKET_ERROR) { - dwarn("WSAIoctl for GetAcceptExSockaddrs failed, err = %d", ::WSAGetLastError()); - closesocket(s); - return; - } - - closesocket(s); -} - -hpc_network_provider::hpc_network_provider(rpc_engine *srv, network *inner_provider) - : connection_oriented_network(srv, inner_provider) -{ - load_socket_functions(); - _listen_fd = INVALID_SOCKET; - _looper = nullptr; - _max_buffer_block_count_per_send = 64; -} - -error_code -hpc_network_provider::start(rpc_channel channel, int port, bool client_only, io_modifer &ctx) -{ - if (_listen_fd != INVALID_SOCKET) - return ERR_SERVICE_ALREADY_RUNNING; - - _looper = get_io_looper(node(), ctx.queue, ctx.mode); - - dassert(channel == RPC_CHANNEL_TCP || channel == RPC_CHANNEL_UDP, - "invalid given channel %s", - channel.to_string()); - - _address.assign_ipv4(get_local_ipv4(), port); - - if (!client_only) { - struct sockaddr_in addr; - addr.sin_family = AF_INET; - addr.sin_addr.s_addr = INADDR_ANY; - addr.sin_port = htons(port); - - _listen_fd = create_tcp_socket(&addr); - if (_listen_fd == INVALID_SOCKET) { - dassert(false, ""); - } - - int forcereuse = 1; - if (setsockopt( - _listen_fd, SOL_SOCKET, SO_REUSEADDR, (char *)&forcereuse, sizeof(forcereuse)) != - 0) { - dwarn("setsockopt SO_REUSEDADDR failed, err = %d", ::GetLastError()); - } - - _looper->bind_io_handle((dsn_handle_t)_listen_fd, &_accept_event.callback); - - if (listen(_listen_fd, SOMAXCONN) != 0) { - dwarn("listen failed, err = %d", ::GetLastError()); - return ERR_NETWORK_START_FAILED; - } - - do_accept(); - } - - return ERR_OK; -} - -rpc_session_ptr hpc_network_provider::create_client_session(::dsn::rpc_address server_addr) -{ - struct sockaddr_in addr; - addr.sin_family = AF_INET; - addr.sin_addr.s_addr = INADDR_ANY; - addr.sin_port = 0; - - auto sock = create_tcp_socket(&addr); - message_parser_ptr parser(new_message_parser(_client_hdr_format)); - auto client = new hpc_rpc_session(sock, parser, *this, server_addr, true); - rpc_session_ptr c(client); - client->bind_looper(_looper); - return c; -} - -void hpc_network_provider::do_accept() -{ - socket_t s = create_tcp_socket(nullptr); - dassert(s != INVALID_SOCKET, "cannot create socket for accept"); - - _accept_sock = s; - _accept_event.callback = [this](int err, uint32_t size, uintptr_t lpolp) { - // dinfo("accept completed, err = %d, size = %u", err, size); - dassert(&_accept_event.olp == (LPOVERLAPPED)lpolp, "must be this exact overlap"); - if (err == ERROR_SUCCESS) { - setsockopt(_accept_sock, - SOL_SOCKET, - SO_UPDATE_ACCEPT_CONTEXT, - (char *)&_listen_fd, - sizeof(_listen_fd)); - - struct sockaddr_in addr; - memset((void *)&addr, 0, sizeof(addr)); - - addr.sin_family = AF_INET; - addr.sin_addr.s_addr = INADDR_ANY; - addr.sin_port = 0; - - int addr_len = sizeof(addr); - if (getpeername(_accept_sock, (struct sockaddr *)&addr, &addr_len) == SOCKET_ERROR) { - dassert(false, "getpeername failed, err = %d", ::WSAGetLastError()); - } - - ::dsn::rpc_address client_addr(ntohl(addr.sin_addr.s_addr), ntohs(addr.sin_port)); - message_parser_ptr null_parser; - auto s = new hpc_rpc_session(_accept_sock, null_parser, *this, client_addr, false); - rpc_session_ptr s1(s); - s->bind_looper(_looper); - - this->on_server_session_accepted(s1); - - s->start_read_next(); - } else { - closesocket(_accept_sock); - } - - do_accept(); - }; - memset(&_accept_event.olp, 0, sizeof(_accept_event.olp)); - - DWORD bytes; - BOOL rt = s_lpfnAcceptEx(_listen_fd, - s, - _accept_buffer, - 0, - (sizeof(struct sockaddr_in) + 16), - (sizeof(struct sockaddr_in) + 16), - &bytes, - &_accept_event.olp); - - if (!rt && (WSAGetLastError() != ERROR_IO_PENDING)) { - dassert(false, "AcceptEx failed, err = %d", ::WSAGetLastError()); - closesocket(s); - } -} - -io_loop_callback s_ready_event = [](int err, uint32_t length, uintptr_t lolp) { - auto evt = CONTAINING_RECORD(lolp, hpc_network_provider::ready_event, olp); - evt->callback(err, length, lolp); -}; - -void hpc_rpc_session::bind_looper(io_looper *looper, bool delay) -{ - looper->bind_io_handle((dsn_handle_t)_socket, &s_ready_event); -} - -void hpc_rpc_session::do_read(int read_next) -{ - add_ref(); - _read_event.callback = [this](int err, uint32_t length, uintptr_t lolp) { - // dinfo("WSARecv completed, err = %d, size = %u", err, length); - dassert((LPOVERLAPPED)lolp == &_read_event.olp, "must be exact this overlapped"); - if (err != ERROR_SUCCESS) { - dwarn("WSARecv failed, err = %d", err); - on_failure(); - } else { - _reader.mark_read(length); - - int read_next = -1; - - if (!_parser) { - read_next = prepare_parser(); - } - - if (_parser) { - message_ex *msg = _parser->get_message_on_receive(&_reader, read_next); - - while (msg != nullptr) { - this->on_read_completed(msg); - msg = _parser->get_message_on_receive(&_reader, read_next); - } - } - - if (read_next == -1) { - derror( - "(s = %d) recv failed on %s, parse failed", _socket, _remote_addr.to_string()); - on_failure(); - } else { - start_read_next(read_next); - } - } - - release_ref(); - }; - memset(&_read_event.olp, 0, sizeof(_read_event.olp)); - - WSABUF buf[1]; - - void *ptr = _reader.read_buffer_ptr(read_next); - int remaining = _reader.read_buffer_capacity(); - buf[0].buf = (char *)ptr; - buf[0].len = remaining; - - DWORD bytes = 0; - DWORD flag = 0; - int rt = WSARecv(_socket, buf, 1, &bytes, &flag, &_read_event.olp, NULL); - - if (SOCKET_ERROR == rt && (WSAGetLastError() != ERROR_IO_PENDING)) { - dwarn("WSARecv failed, err = %d", ::WSAGetLastError()); - release_ref(); - on_failure(); - } - - // dinfo("WSARecv called, err = %d", rt); -} - -void hpc_rpc_session::do_write(uint64_t sig) -{ - add_ref(); - - _write_event.callback = [this](int err, uint32_t length, uintptr_t lolp) { - dassert((LPOVERLAPPED)lolp == &_write_event.olp, "must be exact this overlapped"); - if (err != ERROR_SUCCESS) { - dwarn("WSASend failed, err = %d", err); - on_failure(true); - } else { - int len = (int)length; - int buf_i = _sending_buffer_start_index; - while (len > 0) { - auto &buf = _sending_buffers[buf_i]; - if (len >= (int)buf.sz) { - buf_i++; - len -= (int)buf.sz; - } else { - buf.buf = (char *)buf.buf + len; - buf.sz -= (uint32_t)len; - break; - } - } - _sending_buffer_start_index = buf_i; - - // message completed, continue next message - if (_sending_buffer_start_index == (int)_sending_buffers.size()) { - dassert(len == 0, "buffer must be sent completely"); - auto sig = _sending_signature; - _sending_signature = 0; - on_send_completed(sig); - } else - do_write(_sending_signature); - } - - release_ref(); - }; - memset(&_write_event.olp, 0, sizeof(_write_event.olp)); - - // new msg - if (_sending_signature != sig) { - dassert(_sending_signature == 0, "only one sending msg is possible"); - _sending_signature = sig; - _sending_buffer_start_index = 0; - } - - // continue old msg - else { - dassert(_sending_signature == sig, "only one sending msg is possible"); - } - - int buffer_count = (int)_sending_buffers.size() - _sending_buffer_start_index; - static_assert(sizeof(message_parser::send_buf) == sizeof(WSABUF), - "make sure they are compatible"); - - DWORD bytes = 0; - int rt = WSASend(_socket, - (LPWSABUF)&_sending_buffers[_sending_buffer_start_index], - (DWORD)buffer_count, - &bytes, - 0, - &_write_event.olp, - NULL); - - if (SOCKET_ERROR == rt && (WSAGetLastError() != ERROR_IO_PENDING)) { - dwarn("WSASend failed, err = %d", ::WSAGetLastError()); - release_ref(); - on_failure(true); - } - - // dinfo("WSASend called, err = %d", rt); -} - -void hpc_rpc_session::close() -{ - if (0 != _socket) { - closesocket(_socket); - _socket = 0; - } -} - -hpc_rpc_session::hpc_rpc_session(socket_t sock, - message_parser_ptr &parser, - connection_oriented_network &net, - ::dsn::rpc_address remote_addr, - bool is_client) - : rpc_session(net, remote_addr, parser, is_client), _socket(sock) -{ - _sending_signature = 0; - _sending_buffer_start_index = 0; -} - -void hpc_rpc_session::on_failure(bool is_write) -{ - if (on_disconnected(is_write)) - close(); -} - -void hpc_rpc_session::connect() -{ - if (!set_connecting()) - return; - - _connect_event.callback = [this](int err, uint32_t io_size, uintptr_t lpolp) { - // dinfo("ConnectEx completed, err = %d, size = %u", err, io_size); - if (err != ERROR_SUCCESS) { - dwarn("ConnectEx failed, err = %d", err); - this->on_failure(true); - } else { - dinfo("client session %s connected", _remote_addr.to_string()); - - set_connected(); - on_send_completed(); - start_read_next(); - } - this->release_ref(); // added before ConnectEx - }; - memset(&_connect_event.olp, 0, sizeof(_connect_event.olp)); - - struct sockaddr_in addr; - addr.sin_family = AF_INET; - addr.sin_addr.s_addr = htonl(_remote_addr.ip()); - addr.sin_port = htons(_remote_addr.port()); - - this->add_ref(); // released in _connect_event.callback - BOOL rt = s_lpfnConnectEx( - _socket, (struct sockaddr *)&addr, (int)sizeof(addr), 0, 0, 0, &_connect_event.olp); - - if (!rt && (WSAGetLastError() != ERROR_IO_PENDING)) { - dwarn("ConnectEx failed, err = %d", ::WSAGetLastError()); - this->release_ref(); - - on_failure(true); - } -} -} -} - -#endif diff --git a/src/core/tools/hpc/hpc_tail_logger.cpp b/src/core/tools/hpc/hpc_tail_logger.cpp deleted file mode 100644 index 4fe58a3fc9..0000000000 --- a/src/core/tools/hpc/hpc_tail_logger.cpp +++ /dev/null @@ -1,319 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * What is this file about? - * - * Revision history: - * xxxx-xx-xx, author, first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#include "hpc_tail_logger.h" -#include -#include -#include -#include -#include -#include -#include - -namespace dsn { -namespace tools { -struct tail_log_hdr -{ - uint32_t log_break; // '\0' - uint32_t magic; - int32_t length; - uint64_t ts; - tail_log_hdr *prev; - - bool is_valid() { return magic == 0xdeadbeef; } -}; - -struct __tail_log_info__ -{ - uint32_t magic; - char *buffer; - char *next_write_ptr; - tail_log_hdr *last_hdr; -}; - -typedef ::dsn::utils::safe_singleton_store tail_log_manager; - -static __thread struct __tail_log_info__ s_tail_log_info; - -hpc_tail_logger::hpc_tail_logger(const char *log_dir) : logging_provider(log_dir) -{ - _log_dir = std::string(log_dir); - _per_thread_buffer_bytes = - (int)dsn_config_get_value_uint64("tools.hpc_tail_logger", - "per_thread_buffer_bytes", - 10 * 1024 * 1024, // 10 MB by default - "buffer size for per-thread logging"); - - static bool register_it = false; - if (register_it) { - return; - } - - register_it = true; - - // register command for tail logging - ::dsn::command_manager::instance().register_command( - {"tail-log"}, - "tail-log keyword back-seconds [back-start-seconds = 0] [tid1,tid2,...]", - "tail-log find logs with given keyword and within [now - back-seconds, now - " - "back-start-seconds]", - [this](const std::vector &args) { - if (args.size() < 2) - return std::string("invalid arguments for tail-log command"); - else { - std::unordered_set target_threads; - if (args.size() >= 4) { - std::list tids; - ::dsn::utils::split_args(args[3].c_str(), tids, ','); - for (auto &t : tids) { - target_threads.insert(atoi(t.c_str())); - } - } - - return this->search(args[0].c_str(), - atoi(args[1].c_str()), - args.size() >= 3 ? atoi(args[2].c_str()) : 0, - target_threads); - } - }); - - ::dsn::command_manager::instance().register_command( - {"tail-log-dump"}, - "tail-log-dump", - "tail-log-dump dump all tail logs to log files", - [this](const std::vector &args) { - hpc_tail_logs_dumpper(); - return std::string( - "logs are dumped to coredurmp dir started with hpc_tail_logs.xxx.log"); - }); -} - -hpc_tail_logger::~hpc_tail_logger(void) {} - -void hpc_tail_logger::flush() { hpc_tail_logs_dumpper(); } - -void hpc_tail_logger::hpc_tail_logs_dumpper() -{ - uint64_t nts = dsn_now_ns(); - std::stringstream log; - log << _log_dir << "/hpc_tail_logs." << nts << ".log"; - - std::ofstream olog(log.str().c_str()); - - std::vector threads; - tail_log_manager::instance().get_all_keys(threads); - - for (auto &tid : threads) { - __tail_log_info__ *log; - if (!tail_log_manager::instance().get(tid, log)) - continue; - - tail_log_hdr *hdr = log->last_hdr, *tmp = log->last_hdr; - while (tmp != nullptr && tmp != hdr) { - if (!tmp->is_valid()) - break; - - char *llog = (char *)(tmp)-tmp->length; - olog << llog << std::endl; - - // try previous log - tmp = tmp->prev; - }; - } - - olog.close(); -} - -std::string hpc_tail_logger::search(const char *keyword, - int back_seconds, - int back_start_seconds, - std::unordered_set &target_threads) -{ - uint64_t nts = dsn_now_ns(); - uint64_t start = - nts - static_cast(back_seconds) * 1000 * 1000 * 1000; // second to nanosecond - uint64_t end = - nts - - static_cast(back_start_seconds) * 1000 * 1000 * 1000; // second to nanosecond - - std::vector threads; - tail_log_manager::instance().get_all_keys(threads); - - std::stringstream ss; - int log_count = 0; - - for (auto &tid : threads) { - __tail_log_info__ *log; - if (!tail_log_manager::instance().get(tid, log)) - continue; - - // filter by tid - if (target_threads.size() > 0 && target_threads.find(tid) == target_threads.end()) - continue; - - tail_log_hdr *hdr = log->last_hdr, *tmp = log->last_hdr; - do { - if (!tmp->is_valid()) - break; - - // filter by time - if (tmp->ts < start) - break; - - if (tmp->ts > end) { - tmp = tmp->prev; - continue; - } - - // filter by keyword - char *llog = (char *)(tmp)-tmp->length; - if (strstr(llog, keyword)) { - ss << llog << std::endl; - log_count++; - } - - // try previous log - tmp = tmp->prev; - - } while (tmp != nullptr && tmp != hdr); - } - - char strb[24], stre[24]; - ::dsn::utils::time_ms_to_string(start / 1000000, strb); - ::dsn::utils::time_ms_to_string(end / 1000000, stre); - - ss << "------------------------------------------" << std::endl; - ss << "In total (" << log_count << ") log entries are found between [" << strb << ", " << stre - << "] " << std::endl; - - return ss.str(); -} - -void hpc_tail_logger::dsn_logv(const char *file, - const char *function, - const int line, - dsn_log_level_t log_level, - const char *fmt, - va_list args) -{ - // init log buffer if necessary - if (s_tail_log_info.magic != 0xdeadbeef) { - s_tail_log_info.buffer = (char *)malloc(_per_thread_buffer_bytes); - s_tail_log_info.next_write_ptr = s_tail_log_info.buffer; - s_tail_log_info.last_hdr = nullptr; - memset(s_tail_log_info.buffer, '\0', _per_thread_buffer_bytes); - - tail_log_manager::instance().put(::dsn::utils::get_current_tid(), &s_tail_log_info); - s_tail_log_info.magic = 0xdeadbeef; - } - - // get enough write space >= 1K - if (s_tail_log_info.next_write_ptr + 1024 > s_tail_log_info.buffer + _per_thread_buffer_bytes) { - s_tail_log_info.next_write_ptr = s_tail_log_info.buffer; - } - char *ptr = s_tail_log_info.next_write_ptr; - char *ptr0 = ptr; // remember it - size_t capacity = static_cast(s_tail_log_info.buffer + _per_thread_buffer_bytes - ptr); - - // print verbose log header - uint64_t ts = 0; - int tid = ::dsn::utils::get_current_tid(); - if (::dsn::tools::is_engine_ready()) - ts = dsn_now_ns(); - char str[24]; - ::dsn::utils::time_ms_to_string(ts / 1000000, str); - auto wn = sprintf(ptr, "%s (%" PRIu64 " %04x) ", str, ts, tid); - ptr += wn; - capacity -= wn; - - auto t = task::get_current_task_id(); - if (t) { - if (nullptr != task::get_current_worker2()) { - wn = sprintf(ptr, - "%6s.%7s%d.%016" PRIx64 ": ", - task::get_current_node_name(), - task::get_current_worker2()->pool_spec().name.c_str(), - task::get_current_worker2()->index(), - t); - } else { - wn = sprintf(ptr, - "%6s.%7s.%05d.%016" PRIx64 ": ", - task::get_current_node_name(), - "io-thrd", - tid, - t); - } - } else { - wn = sprintf(ptr, "%6s.%7s.%05d: ", task::get_current_node_name(), "io-thrd", tid); - } - - ptr += wn; - capacity -= wn; - - // print body - wn = std::vsnprintf(ptr, capacity, fmt, args); - if (wn < 0) { - wn = snprintf_p(ptr, capacity, "-- cannot printf due to that log entry has error ---"); - } else if (static_cast(wn) > capacity) { - // log truncated - wn = capacity; - } - - ptr += wn; - capacity -= wn; - - // set binary entry header on tail - tail_log_hdr *hdr = (tail_log_hdr *)ptr; - hdr->log_break = 0; - hdr->length = 0; - hdr->magic = 0xdeadbeef; - hdr->ts = ts; - hdr->length = static_cast(ptr - ptr0); - hdr->prev = s_tail_log_info.last_hdr; - s_tail_log_info.last_hdr = hdr; - - ptr += sizeof(tail_log_hdr); - capacity -= sizeof(tail_log_hdr); - - // set next write ptr - s_tail_log_info.next_write_ptr = ptr; - - // dump critical logs on screen - if (log_level >= LOG_LEVEL_WARNING) { - std::cout << ptr0 << std::endl; - } -} -} -} diff --git a/src/core/tools/hpc/hpc_tail_logger.h b/src/core/tools/hpc/hpc_tail_logger.h deleted file mode 100644 index 046e913623..0000000000 --- a/src/core/tools/hpc/hpc_tail_logger.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * What is this file about? - * - * Revision history: - * xxxx-xx-xx, author, first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#pragma once - -#include -#include - -namespace dsn { -namespace tools { - -class hpc_tail_logger : public logging_provider -{ -public: - hpc_tail_logger(const char *log_dir); - virtual ~hpc_tail_logger(void); - - virtual void dsn_logv(const char *file, - const char *function, - const int line, - dsn_log_level_t log_level, - const char *fmt, - va_list args); - - virtual void flush(); - -private: - std::string search(const char *keyword, - int back_seconds, - int back_start_seconds, - std::unordered_set &target_threads); - void hpc_tail_logs_dumpper(); - -private: - int _per_thread_buffer_bytes; - std::string _log_dir; -}; -} -} diff --git a/src/core/tools/hpc/hpc_task_queue.cpp b/src/core/tools/hpc/hpc_task_queue.cpp index 108a8bc747..d1aa6f13ca 100644 --- a/src/core/tools/hpc/hpc_task_queue.cpp +++ b/src/core/tools/hpc/hpc_task_queue.cpp @@ -38,71 +38,6 @@ namespace dsn { namespace tools { -hpc_task_queue::hpc_task_queue(task_worker_pool *pool, int index, task_queue *inner_provider) - : task_queue(pool, index, inner_provider) -{ -} - -void hpc_task_queue::enqueue(task *task) -{ - dassert(task->next == nullptr, "task is not alone"); - { - utils::auto_lock<::dsn::utils::ex_lock_nr_spin> l(_lock); - _tasks.add(task); - } - _cond.notify_one(); -} - -task *hpc_task_queue::dequeue(/*inout*/ int &batch_size) -{ - task *t; - - _lock.lock(); - _cond.wait(_lock, [=] { return !_tasks.is_empty(); }); - t = _tasks.pop_batch(batch_size); - _lock.unlock(); - - return t; -} - -hpc_task_priority_queue::hpc_task_priority_queue(task_worker_pool *pool, - int index, - task_queue *inner_provider) - : task_queue(pool, index, inner_provider) -{ -} - -void hpc_task_priority_queue::enqueue(task *task) -{ - dassert(task->next == nullptr, "task is not alone"); - auto idx = static_cast(task->spec().priority); - { - utils::auto_lock<::dsn::utils::ex_lock_nr_spin> l(_lock[idx]); - _tasks[idx].add(task); - } - - _sema.signal(); -} - -task *hpc_task_priority_queue::dequeue(/*inout*/ int &batch_size) -{ - task *t = nullptr; - - _sema.wait(); - - for (auto i = TASK_PRIORITY_COUNT - 1; i >= 0; --i) { - _lock[i].lock(); - t = _tasks[i].pop_one(); - _lock[i].unlock(); - - if (t) - break; - } - - batch_size = 1; - dassert(t != nullptr, "returned task cannot be null"); - return t; -} hpc_concurrent_task_queue::hpc_concurrent_task_queue(task_worker_pool *pool, int index, @@ -116,6 +51,7 @@ void hpc_concurrent_task_queue::enqueue(task *task) _queues[task->spec().priority].q.enqueue(task); _sema.signal(1); } + task *hpc_concurrent_task_queue::dequeue(int &batch_size) { batch_size = _sema.waitMany(batch_size); diff --git a/src/core/tools/hpc/hpc_task_queue.h b/src/core/tools/hpc/hpc_task_queue.h index 48dd1e770a..6989431672 100644 --- a/src/core/tools/hpc/hpc_task_queue.h +++ b/src/core/tools/hpc/hpc_task_queue.h @@ -35,41 +35,13 @@ #pragma once -#include -#include #include #include +#include + namespace dsn { namespace tools { -class hpc_task_queue : public task_queue -{ -public: - hpc_task_queue(task_worker_pool *pool, int index, task_queue *inner_provider); - - void enqueue(task *task) override; - task *dequeue(/*inout*/ int &batch_size) override; - -private: - utils::ex_lock_nr_spin _lock; - std::condition_variable_any _cond; - slist _tasks; -}; - -class hpc_task_priority_queue : public task_queue -{ -public: - hpc_task_priority_queue(task_worker_pool *pool, int index, task_queue *inner_provider); - - void enqueue(task *task) override; - task *dequeue(/*inout*/ int &batch_size) override; - -private: - utils::ex_lock_nr_spin _lock[TASK_PRIORITY_COUNT]; - slist _tasks[TASK_PRIORITY_COUNT]; - utils::semaphore _sema; -}; - class hpc_concurrent_task_queue : public task_queue { moodycamel::details::mpmc_sema::LightweightSemaphore _sema; diff --git a/src/core/tools/hpc/io_looper.bsd.cpp b/src/core/tools/hpc/io_looper.bsd.cpp deleted file mode 100644 index f3b83ccad7..0000000000 --- a/src/core/tools/hpc/io_looper.bsd.cpp +++ /dev/null @@ -1,322 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * io looper implementation for BSD. - * - * Revision history: - * 2015-09-01, HX Lin(linmajia@live.com), first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#if defined(__APPLE__) || defined(__FreeBSD__) - -#include "io_looper.h" - -#define IO_LOOPER_USER_NOTIFICATION_FD (-10) - -namespace dsn { -namespace tools { -io_looper::io_looper() -{ - _io_queue = -1; - _local_notification_fd = IO_LOOPER_USER_NOTIFICATION_FD; - _filters.insert(EVFILT_READ); - _filters.insert(EVFILT_WRITE); - // EVFILT_AIO is automatically registered. - //_filters.insert(EVFILT_AIO); - _filters.insert(EVFILT_READ_WRITE); - // Internal use - _filters.insert(EVFILT_USER); -} - -io_looper::~io_looper(void) { stop(); } - -error_code io_looper::bind_io_handle(dsn_handle_t handle, - io_loop_callback *cb, - unsigned int events, - ref_counter *ctx) -{ - int fd; - short filter; - int nr_filters; - struct kevent e[2]; - - if (cb == nullptr) { - derror("cb == nullptr"); - return ERR_INVALID_PARAMETERS; - } - - filter = (short)events; - if (_filters.find(filter) == _filters.end()) { - derror("The filter %hd is unsupported.", filter); - return ERR_INVALID_PARAMETERS; - } - - fd = (int)(intptr_t)(handle); - if (fd != IO_LOOPER_USER_NOTIFICATION_FD) { - if (fd < 0) { - derror("bind_io_handle: the fd %d is less than 0.", fd); - return ERR_INVALID_PARAMETERS; - } - - if (filter == EVFILT_USER) { - derror("EVFILT_USER is internally used."); - return ERR_INVALID_PARAMETERS; - } - } - - if (fd > 0) { - int flags = fcntl(fd, F_GETFL, 0); - dassert(flags != -1, "fcntl failed, err = %s, fd = %d", strerror(errno), fd); - - if (!(flags & O_NONBLOCK)) { - flags |= O_NONBLOCK; - flags = fcntl(fd, F_SETFL, flags); - dassert(flags != -1, "fcntl failed, err = %s, fd = %d", strerror(errno), fd); - } - } - - uintptr_t cb0 = (uintptr_t)cb; - dassert((cb0 & 0x1) == 0, "the least one bit must be zero for the callback address"); - - if (ctx) { - cb0 |= 0x1; // has ref_counter - - utils::auto_lock l(_io_sessions_lock); - auto pr = _io_sessions.insert(io_sessions::value_type(cb, ctx)); - dassert(pr.second, "the callback must not be registered before"); - } - - if (filter == EVFILT_READ_WRITE) { - e[0].filter = EVFILT_READ; - e[1].filter = EVFILT_WRITE; - nr_filters = 2; - } else { - e[0].filter = filter; - nr_filters = 1; - } - - for (int i = 0; i < nr_filters; i++) { - EV_SET(&e[i], fd, e[i].filter, (EV_ADD | EV_CLEAR), 0, 0, (void *)cb0); - } - - if (kevent(_io_queue, e, nr_filters, nullptr, 0, nullptr) == -1) { - derror("bind io handler to kqueue failed, err = %s, fd = %d", strerror(errno), fd); - unbind_io_handle(handle, cb); - return ERR_BIND_IOCP_FAILED; - } - - return ERR_OK; -} - -error_code io_looper::unbind_io_handle(dsn_handle_t handle, io_loop_callback *cb) -{ - int fd = (int)(intptr_t)handle; - int nr_filters; - struct kevent e[2]; - int cnt = 0; - bool succ = true; - - if (fd != IO_LOOPER_USER_NOTIFICATION_FD) { - if (fd < 0) { - derror("unbind_io_handle: the fd %d is less than 0.", fd); - return ERR_INVALID_PARAMETERS; - } - - e[0].filter = EVFILT_READ; - e[1].filter = EVFILT_WRITE; - nr_filters = 2; - } else { - e[0].filter = EVFILT_USER; - nr_filters = 1; - } - - for (int i = 0; i < nr_filters; i++) { - EV_SET(&e[i], fd, e[i].filter, EV_DELETE, 0, 0, nullptr); - - if (kevent(_io_queue, &e[i], 1, nullptr, 0, nullptr) == -1) { - if (errno != ENOENT) { - derror( - "unbind io handler to kqueue failed, err = %s, fd = %d", strerror(errno), fd); - succ = false; - } - } else { - cnt++; - } - } - - if ((cnt == 0) && succ) { - dwarn("fd = %d has not been bound yet.", fd); - } - - // in case the fd is already invalid - if (cb) { - utils::auto_lock l(_io_sessions_lock); - _io_sessions.erase(cb); - } - - return (succ ? ERR_OK : ERR_BIND_IOCP_FAILED); -} - -void io_looper::notify_local_execution() -{ - struct kevent e; - EV_SET(&e, - _local_notification_fd, - EVFILT_USER, - 0, - (NOTE_FFCOPY | NOTE_TRIGGER), - 0, - &_local_notification_callback); - - if (kevent(_io_queue, &e, 1, nullptr, 0, nullptr) == -1) { - dassert(false, "post local notification via eventfd failed, err = %s", strerror(errno)); - } - dinfo("notify local"); -} - -void io_looper::create_completion_queue() -{ - _io_queue = ::kqueue(); - dassert(_io_queue != -1, "Fail to create kqueue"); - - _local_notification_callback = - [this](int native_error, uint32_t io_size, uintptr_t lolp_or_events) { - this->handle_local_queues(); - }; - - bind_io_handle( - (dsn_handle_t)(intptr_t)_local_notification_fd, &_local_notification_callback, EVFILT_USER); -} - -void io_looper::close_completion_queue() -{ - if (_io_queue != -1) { - unbind_io_handle((dsn_handle_t)(intptr_t)_local_notification_fd, - &_local_notification_callback); - ::close(_io_queue); - _io_queue = -1; - } -} - -void io_looper::start(service_node *node, int worker_count) -{ - create_completion_queue(); - - for (int i = 0; i < worker_count; i++) { - std::thread *thr = new std::thread([this, node, i]() { - task::set_tls_dsn_context(node, nullptr, nullptr); - - const char *name = node ? ::dsn::tools::get_service_node_name(node) : "glb"; - char buffer[128]; - sprintf(buffer, "%s.io-loop.%d", name, i); - task_worker::set_name(buffer); - - this->loop_worker(); - }); - _workers.push_back(thr); - } -} - -void io_looper::stop() -{ - close_completion_queue(); - - if (_workers.size() > 0) { - for (auto thr : _workers) { - thr->join(); - delete thr; - } - _workers.clear(); - } -} - -void io_looper::loop_worker() -{ - struct timespec ts = {0, 1000000}; - - while (true) { - int nfds = kevent( - _io_queue, nullptr, 0, _events, IO_LOOPER_MAX_EVENT_COUNT, &ts); // 1ms for timers - if (nfds == 0) // timeout - { - handle_local_queues(); - continue; - } else if (-1 == nfds) { - if (errno == EINTR) { - continue; - } else { - derror("kevent loop exits, err = %s", strerror(errno)); - break; - } - } - - for (int i = 0; i < nfds; i++) { - dassert((_events[i].flags & EV_ERROR) == 0, "kevent should not be set EV_ERROR here."); - auto cb = (io_loop_callback *)_events[i].udata; - dinfo("kevent get events 0x%x, cb = %p", _events[i].filter, cb); - - uintptr_t cb0 = (uintptr_t)cb; - - // for those with ref_counter register entries - if (cb0 & 0x1) { - cb = (io_loop_callback *)(cb0 - 1); - - ref_counter *robj; - { - - utils::auto_lock l(_io_sessions_lock); - auto it = _io_sessions.find(cb); - if (it != _io_sessions.end()) { - robj = it->second; - // make sure callback is protected by ref counting - robj->add_ref(); - } else { - robj = nullptr; - } - } - - if (robj) { - (*cb)(0, 0, (uintptr_t)&_events[i]); - robj->release_ref(); - } else { - // context is gone (unregistered), let's skip - dwarn("kevent event 0x%x skipped as session is gone, cb = %p", - _events[i].filter, - cb); - } - } else { - (*cb)(0, 0, (uintptr_t)&_events[i]); - } - } - } -} -} -} - -#endif diff --git a/src/core/tools/hpc/io_looper.h b/src/core/tools/hpc/io_looper.h deleted file mode 100644 index 86f87730e0..0000000000 --- a/src/core/tools/hpc/io_looper.h +++ /dev/null @@ -1,154 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * define the base cross-platform asynchonous io looper interface - * - * Revision history: - * Aug., 2015, @imzhenyu (Zhenyu Guo), the first version - * xxxx-xx-xx, author, fix bug about xxx - */ -#pragma once - -#include -#include - -#ifndef _WIN32 - -#ifdef __linux__ -#include -#endif - -#if defined(__APPLE__) || defined(__FreeBSD__) -#include -#include -#include -#include -#ifndef EVFILT_NONE -#define EVFILT_NONE (-EVFILT_SYSCOUNT - 10) -#endif -#ifndef EVFILT_READ_WRITE -#define EVFILT_READ_WRITE (EVFILT_NONE - 1) -#endif -#endif - -#endif - -namespace dsn { -namespace tools { -// -// this structure is per io handle, and registered when bind_io_handle to completion queue -// the callback will be executed per io completion or ready -// -// windows (on completion): -// using lolp to differentiate multiple ops -// linux (on ready): -// using evens to differentiate differnt types of ops -// for the same type of ops, it seems Linux doesn't support op differentiation -// -// void handle_event(int native_error, uint32_t io_size, uintptr_t lolp_or_events) -typedef std::function io_loop_callback; - -// -// io looper on completion queue -// it is possible there are multiple loopers on the same io_queue -// -class io_looper -{ -public: - io_looper(); - virtual ~io_looper(void); - - void create_completion_queue(); - - void close_completion_queue(); - - error_code bind_io_handle(dsn_handle_t handle, - io_loop_callback *cb, - unsigned int events = 0, - ref_counter *ctx = nullptr); - - error_code unbind_io_handle(dsn_handle_t handle, io_loop_callback *cb = nullptr); - - void notify_local_execution(); - - void exit_loops(bool wait); - - dsn_handle_t native_handle() { return (dsn_handle_t)(uintptr_t)(_io_queue); } - - void loop_worker(); - - virtual void start(service_node *node, int worker_count); - - virtual void stop(); - - virtual void handle_local_queues() { exec_timer_tasks(false); } - - void add_timer(task *timer); // return next firing delay ms - -protected: - virtual bool is_shared_timer_queue() { return true; } - void exec_timer_tasks(bool local_exec); - -private: - std::vector _workers; -#ifdef _WIN32 - HANDLE _io_queue; -#else - int _io_queue; - -#define IO_LOOPER_MAX_EVENT_COUNT 128 -#ifdef __linux__ - struct epoll_event _events[IO_LOOPER_MAX_EVENT_COUNT]; -#elif defined(__APPLE__) || defined(__FreeBSD__) - struct kevent _events[IO_LOOPER_MAX_EVENT_COUNT]; - typedef std::unordered_set kqueue_filters; - kqueue_filters _filters; -#endif - - int _local_notification_fd; - io_loop_callback _local_notification_callback; - - // - // epoll notifications are not per-op, so we have to - // use a look-up layer to ensure the callback context - // is correctly referenced when the callback is executed. - // - typedef std::unordered_map io_sessions; - ::dsn::utils::ex_lock_nr_spin _io_sessions_lock; - io_sessions _io_sessions; -#endif - // timers - std::atomic _remote_timer_tasks_count; - ::dsn::utils::ex_lock_nr_spin _remote_timer_tasks_lock; - std::map> _remote_timer_tasks; // ts (ms) => task - std::map> _local_timer_tasks; -}; - -// --------------- inline implementation ------------------------- -} -} diff --git a/src/core/tools/hpc/io_looper.linux.cpp b/src/core/tools/hpc/io_looper.linux.cpp deleted file mode 100644 index 66c172730c..0000000000 --- a/src/core/tools/hpc/io_looper.linux.cpp +++ /dev/null @@ -1,258 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * What is this file about? - * - * Revision history: - * xxxx-xx-xx, author, first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#if defined(__linux__) - -#include "io_looper.h" -#include - -namespace dsn { -namespace tools { -io_looper::io_looper() : _remote_timer_tasks_count(0) -{ - _io_queue = 0; - _local_notification_fd = eventfd(0, EFD_NONBLOCK); -} - -io_looper::~io_looper(void) -{ - stop(); - close(_local_notification_fd); -} - -error_code io_looper::bind_io_handle(dsn_handle_t handle, - io_loop_callback *cb, - unsigned int events, - ref_counter *ctx) -{ - int fd = (int)(intptr_t)(handle); - - int flags = fcntl(fd, F_GETFL, 0); - dassert(flags != -1, "fcntl failed, err = %s, fd = %d", strerror(errno), fd); - - if (!(flags & O_NONBLOCK)) { - flags |= O_NONBLOCK; - flags = fcntl(fd, F_SETFL, flags); - dassert(flags != -1, "fcntl failed, err = %s, fd = %d", strerror(errno), fd); - } - - uintptr_t cb0 = (uintptr_t)cb; - dassert((cb0 & 0x1) == 0, "the least one bit must be zero for the callback address"); - - if (ctx) { - cb0 |= 0x1; // has ref_counter - - utils::auto_lock l(_io_sessions_lock); - auto pr = _io_sessions.insert(io_sessions::value_type(cb, ctx)); - dassert(pr.second, "the callback must not be registered before"); - } - - struct epoll_event e; - e.data.ptr = (void *)cb0; - e.events = events; - - if (epoll_ctl(_io_queue, EPOLL_CTL_ADD, fd, &e) < 0) { - derror("bind io handler to epoll_wait failed, err = %s, fd = %d", strerror(errno), fd); - - if (ctx) { - utils::auto_lock l(_io_sessions_lock); - auto r = _io_sessions.erase(cb); - dassert(r > 0, "the callback must be present"); - } - return ERR_BIND_IOCP_FAILED; - } else - return ERR_OK; -} - -error_code io_looper::unbind_io_handle(dsn_handle_t handle, io_loop_callback *cb) -{ - int fd = (int)(intptr_t)handle; - - if (epoll_ctl(_io_queue, EPOLL_CTL_DEL, fd, NULL) < 0) { - derror("unbind io handler to epoll_wait failed, err = %s, fd = %d", strerror(errno), fd); - - // in case the fd is already invalid - if (cb) { - utils::auto_lock l(_io_sessions_lock); - _io_sessions.erase(cb); - } - return ERR_BIND_IOCP_FAILED; - } else { - if (cb) { - utils::auto_lock l(_io_sessions_lock); - _io_sessions.erase(cb); - // without ctx, we are not sure whether cb exists in _io_sessions - // no assert here. - } - return ERR_OK; - } -} - -void io_looper::notify_local_execution() -{ - int64_t c = 1; - if (::write(_local_notification_fd, &c, sizeof(c)) < 0) { - dassert(false, "post local notification via eventfd failed, err = %s", strerror(errno)); - } - dinfo("notify local"); -} - -void io_looper::create_completion_queue() -{ - const int max_event_count = sizeof(_events) / sizeof(struct epoll_event); - - _io_queue = epoll_create(max_event_count); - - _local_notification_callback = - [this](int native_error, uint32_t io_size, uintptr_t lolp_or_events) { - int64_t notify_count = 0; - - if (read(_local_notification_fd, ¬ify_count, sizeof(notify_count)) != - sizeof(notify_count)) { - // possibly consumed already by others - // e.g., two contiguous write with two read, - // the second read will read nothing - return; - } - - this->handle_local_queues(); - }; - - bind_io_handle((dsn_handle_t)(intptr_t)_local_notification_fd, - &_local_notification_callback, - EPOLLIN | EPOLLET); -} - -void io_looper::close_completion_queue() -{ - if (_io_queue != 0) { - ::close(_io_queue); - _io_queue = 0; - } -} - -void io_looper::start(service_node *node, int worker_count) -{ - create_completion_queue(); - - for (int i = 0; i < worker_count; i++) { - std::thread *thr = new std::thread([this, node, i]() { - task::set_tls_dsn_context(node, nullptr, nullptr); - - const char *name = node ? ::dsn::tools::get_service_node_name(node) : "glb"; - char buffer[128]; - sprintf(buffer, "%s.io-loop.%d", name, i); - task_worker::set_name(buffer); - - this->loop_worker(); - }); - _workers.push_back(thr); - } -} - -void io_looper::stop() -{ - close_completion_queue(); - - if (_workers.size() > 0) { - for (auto thr : _workers) { - thr->join(); - delete thr; - } - _workers.clear(); - } -} - -void io_looper::loop_worker() -{ - const int max_event_count = sizeof(_events) / sizeof(struct epoll_event); - - while (true) { - int nfds = epoll_wait(_io_queue, _events, max_event_count, 1); // 1ms for timers - if (nfds == 0) // timeout - { - handle_local_queues(); - } else if (-1 == nfds) { - if (errno == EINTR) { - continue; - } else { - derror("epoll_wait loop exits, err = %s", strerror(errno)); - break; - } - } - - for (int i = 0; i < nfds; i++) { - auto cb = (io_loop_callback *)_events[i].data.ptr; - dinfo("epoll_wait get events 0x%x, cb = %p", _events[i].events, cb); - - uintptr_t cb0 = (uintptr_t)cb; - - // for those with ref_counter register entries - if (cb0 & 0x1) { - cb = (io_loop_callback *)(cb0 - 1); - - ref_counter *robj; - { - - utils::auto_lock l(_io_sessions_lock); - auto it = _io_sessions.find(cb); - if (it != _io_sessions.end()) { - robj = it->second; - // make sure callback is protected by ref counting - robj->add_ref(); - } else { - robj = nullptr; - } - } - - if (robj) { - (*cb)(0, 0, (uintptr_t)_events[i].events); - robj->release_ref(); - } else { - // context is gone (unregistered), let's skip - dwarn("epoll_wait event 0x%x skipped as session is gone, cb = %p", - _events[i].events, - cb); - } - } else { - (*cb)(0, 0, (uintptr_t)_events[i].events); - } - } - } -} -} -} - -#endif diff --git a/src/core/tools/hpc/io_looper.win.cpp b/src/core/tools/hpc/io_looper.win.cpp deleted file mode 100644 index 00110a9b9a..0000000000 --- a/src/core/tools/hpc/io_looper.win.cpp +++ /dev/null @@ -1,163 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * What is this file about? - * - * Revision history: - * xxxx-xx-xx, author, first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#ifdef _WIN32 - -#include "io_looper.h" - -#define NON_IO_TASK_NOTIFICATION_KEY 2 - -namespace dsn { -namespace tools { -io_looper::io_looper() { _io_queue = 0; } - -io_looper::~io_looper(void) { stop(); } - -error_code io_looper::bind_io_handle(dsn_handle_t handle, - io_loop_callback *cb, - unsigned int events, - ref_counter *ctx) -{ - events; // not used on windows - ctx; // not used on windows - if (NULL == ::CreateIoCompletionPort((HANDLE)handle, _io_queue, (ULONG_PTR)cb, 0)) { - derror("bind io handler to completion port failed, err = %d", ::GetLastError()); - return ERR_BIND_IOCP_FAILED; - } else - return ERR_OK; -} - -error_code io_looper::unbind_io_handle(dsn_handle_t handle, io_loop_callback *cb) -{ - // nothing to do - return ERR_OK; -} - -void io_looper::notify_local_execution() -{ - if (!::PostQueuedCompletionStatus(_io_queue, 0, NON_IO_TASK_NOTIFICATION_KEY, NULL)) { - dassert(false, "PostQueuedCompletionStatus failed, err = %d", ::GetLastError()); - } -} - -void io_looper::create_completion_queue() -{ - _io_queue = ::CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0); -} - -void io_looper::close_completion_queue() -{ - if (_io_queue != 0) { - ::CloseHandle(_io_queue); - _io_queue = 0; - } -} - -void io_looper::start(service_node *node, int worker_count) -{ - create_completion_queue(); - for (int i = 0; i < worker_count; i++) { - std::thread *thr = new std::thread([this, node, i]() { - task::set_tls_dsn_context(node, nullptr, nullptr); - - const char *name = node ? ::dsn::tools::get_service_node_name(node) : "glb"; - char buffer[128]; - sprintf(buffer, "%s.io-loop.%d", name, i); - task_worker::set_name(buffer); - - this->loop_worker(); - }); - _workers.push_back(thr); - } -} - -void io_looper::stop() -{ - close_completion_queue(); - - if (_workers.size() > 0) { - for (auto thr : _workers) { - thr->join(); - delete thr; - } - _workers.clear(); - } -} - -void io_looper::loop_worker() -{ - DWORD io_size; - uintptr_t completion_key; - LPOVERLAPPED lolp; - DWORD error; - - while (true) { - BOOL r = ::GetQueuedCompletionStatus( - _io_queue, &io_size, &completion_key, &lolp, 1); // 1ms timeout for timers - - // everything goes fine - if (r) { - error = ERROR_SUCCESS; - } - - // failed or timeout - else { - error = ::GetLastError(); - if (error == ERROR_ABANDONED_WAIT_0) { - derror("completion port loop exits"); - break; - } - - // only possible for timeout - if (NULL == lolp) { - handle_local_queues(); - continue; - } - - dinfo("io operation failed in iocp, err = 0x%x", error); - } - - if (NON_IO_TASK_NOTIFICATION_KEY == completion_key) { - handle_local_queues(); - } else { - io_loop_callback *cb = (io_loop_callback *)completion_key; - (*cb)((int)error, io_size, (uintptr_t)lolp); - } - } -} -} -} - -#endif diff --git a/src/core/tools/hpc/mix_all_io_looper.cpp b/src/core/tools/hpc/mix_all_io_looper.cpp deleted file mode 100644 index 193c245fb2..0000000000 --- a/src/core/tools/hpc/mix_all_io_looper.cpp +++ /dev/null @@ -1,247 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * What is this file about? - * - * Revision history: - * xxxx-xx-xx, author, first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#include "mix_all_io_looper.h" - -namespace dsn { -namespace tools { -class io_looper_holder : public utils::singleton -{ -public: - std::unordered_map per_node_loopers; - std::unordered_map per_queue_loopers; -}; - -io_looper *get_io_looper(service_node *node, task_queue *q, ioe_mode mode) -{ - switch (mode) { - case IOE_PER_NODE: { - dassert(node, "node is not given"); - auto it = io_looper_holder::instance().per_node_loopers.find(node); - if (it == io_looper_holder::instance().per_node_loopers.end()) { - auto looper = new io_looper(); - looper->start(node, spec().io_worker_count); - io_looper_holder::instance().per_node_loopers[node] = looper; - return looper; - } else - return it->second; - } - case IOE_PER_QUEUE: { - dassert(q, "task queue is not given"); - auto p = dynamic_cast(q); - dassert(p, "task queue must also be the io looper"); - return p; - } - default: - dassert(false, "invalid io loop type"); - return nullptr; - } -} - -//-------------------------------------------- -// -// when s_config.type == IOE_PER_QUEUE -// -io_looper_task_queue::io_looper_task_queue(task_worker_pool *pool, - int index, - task_queue *inner_provider) - : task_queue(pool, index, inner_provider) -{ - _remote_count = 0; -} - -io_looper_task_queue::~io_looper_task_queue() {} - -void io_looper_task_queue::start(service_node *node, int worker_count) -{ - create_completion_queue(); -} - -void io_looper_task_queue::stop() { io_looper::stop(); } - -void io_looper_task_queue::handle_local_queues() -{ - // execute timers in current thread - exec_timer_tasks(true); - - // execute local queue - task *t = _local_tasks.pop_all(), *next; - while (t) { - next = t->next; - t->exec_internal(); - t = next; - } - - // execute shared queue - { - utils::auto_lock<::dsn::utils::ex_lock_nr_spin> l(_lock); - t = _remote_tasks.pop_all(); - } - - while (t) { - next = t->next; - t->exec_internal(); - t = next; - } -} - -void io_looper::exec_timer_tasks(bool local_exec) -{ - // execute local timers - uint64_t nts = ::dsn::task::get_current_env()->now_ns() / 1000000; - while (_local_timer_tasks.size() > 0) { - auto it = _local_timer_tasks.begin(); - if (it->first <= nts) { - task *t = it->second.pop_all(), *next; - _local_timer_tasks.erase(it); - - while (t) { - next = t->next; - if (local_exec) - t->exec_internal(); - else { - t->enqueue(); - t->release_ref(); // added by first t->enqueue() - } - - t = next; - } - } else - break; - } - - // execute shared timers - while (_remote_timer_tasks_count.load(std::memory_order_relaxed) > 0) { - task *t = nullptr, *next; - { - utils::auto_lock<::dsn::utils::ex_lock_nr_spin> l(_remote_timer_tasks_lock); - if (_remote_timer_tasks.size() == 0) - break; - - auto it = _remote_timer_tasks.begin(); - if (it->first <= nts) { - t = it->second.pop_all(); - _remote_timer_tasks.erase(it); - } else - break; - } - - while (t) { - _remote_timer_tasks_count--; - - next = t->next; - - if (local_exec) - t->exec_internal(); - else { - t->enqueue(); - t->release_ref(); // added by first t->enqueue() - } - - t = next; - } - } -} - -void io_looper::add_timer(task *timer) -{ - uint64_t ts_ms = dsn_now_ms() + timer->delay_milliseconds(); - timer->set_delay(0); - - // put into locked queue when it is shared or from remote threads - if (is_shared_timer_queue()) { - { - utils::auto_lock<::dsn::utils::ex_lock_nr_spin> l(_remote_timer_tasks_lock); - auto pr = _remote_timer_tasks.insert( - std::map>::value_type(ts_ms, slist())); - pr.first->second.add(timer); - } - - _remote_timer_tasks_count++; - } - - // put into local queue - else { - auto pr = _local_timer_tasks.insert( - std::map>::value_type(ts_ms, slist())); - pr.first->second.add(timer); - } -} - -void io_looper_task_queue::enqueue(task *task) -{ - // put into locked queue when it is shared or from remote threads - if (is_shared() || task::get_current_worker() != this->owner_worker()) { - { - utils::auto_lock<::dsn::utils::ex_lock_nr_spin> l(_lock); - _remote_tasks.add(task); - } - - int old = _remote_count.fetch_add(1, std::memory_order_release); - if (old == 0) { - notify_local_execution(); - } - } - - // put into local queue - else { - _local_tasks.add(task); - } -} - -task *io_looper_task_queue::dequeue(/*inout*/ int &batch_size) -{ - dassert(false, "never execute here ..."); - batch_size = 0; - return nullptr; -} - -io_looper_task_worker::io_looper_task_worker(task_worker_pool *pool, - task_queue *q, - int index, - task_worker *inner_provider) - : task_worker(pool, q, index, inner_provider) -{ - io_looper_task_queue *looper = dynamic_cast(queue()); - looper->start(nullptr, 0); -} - -void io_looper_task_worker::loop() -{ - io_looper_task_queue *looper = dynamic_cast(queue()); - looper->loop_worker(); -} -} -} diff --git a/src/core/tools/hpc/mix_all_io_looper.h b/src/core/tools/hpc/mix_all_io_looper.h deleted file mode 100644 index 49f931a888..0000000000 --- a/src/core/tools/hpc/mix_all_io_looper.h +++ /dev/null @@ -1,106 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015 Microsoft Corporation - * - * -=- Robust Distributed System Nucleus (rDSN) -=- - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Description: - * io + computation looper interface, so everything uses the same thread - * - * Revision history: - * Aug., 2015, @imzhenyu (Zhenyu Guo), the first version - * xxxx-xx-xx, author, fix bug about xxx - */ - -#include "io_looper.h" - -namespace dsn { -namespace tools { -extern io_looper *get_io_looper(service_node *node, task_queue *q, ioe_mode mode); - -class io_looper_task_queue : public task_queue, public io_looper -{ -public: - io_looper_task_queue(task_worker_pool *pool, int index, task_queue *inner_provider); - virtual ~io_looper_task_queue(); - - virtual void start(service_node *node, int worker_count) override; - virtual void stop() override; - virtual void handle_local_queues() override; - - virtual void enqueue(task *task) override; - virtual task *dequeue(/*inout*/ int &batch_size) override; - -protected: - virtual bool is_shared_timer_queue() override - { - return is_shared() || task::get_current_worker() != owner_worker(); - } - -private: - std::atomic _remote_count; - - // tasks from remote threads - ::dsn::utils::ex_lock_nr_spin _lock; - slist _remote_tasks; - - // tasks from local thread - slist _local_tasks; -}; - -class io_looper_task_worker : public task_worker -{ -public: - io_looper_task_worker(task_worker_pool *pool, - task_queue *q, - int index, - task_worker *inner_provider); - virtual void loop(); -}; - -class io_looper_timer_service : public timer_service -{ -public: - io_looper_timer_service(service_node *node, timer_service *inner_provider) - : timer_service(node, inner_provider) - { - _looper = nullptr; - } - - virtual void start(io_modifer &ctx) - { - _looper = get_io_looper(node(), ctx.queue, ctx.mode); - dassert(_looper != nullptr, "correspondent looper is empty"); - } - - // after milliseconds, the provider should call task->enqueue() - virtual void add_timer(task *task) { _looper->add_timer(task); } - -private: - io_looper *_looper; -}; - -// ------------------ inline implementation -------------------- -} -} diff --git a/src/core/tools/hpc/providers.hpc.cpp b/src/core/tools/hpc/providers.hpc.cpp index 1e3561555c..f109f34045 100644 --- a/src/core/tools/hpc/providers.hpc.cpp +++ b/src/core/tools/hpc/providers.hpc.cpp @@ -35,29 +35,12 @@ #include #include "hpc_task_queue.h" -#include "hpc_tail_logger.h" -#include "hpc_logger.h" -#include "hpc_aio_provider.h" -#include "hpc_network_provider.h" -#include "hpc_env_provider.h" -#include "mix_all_io_looper.h" namespace dsn { namespace tools { void register_hpc_providers() { - register_component_provider("dsn::tools::hpc_tail_logger"); - register_component_provider("dsn::tools::hpc_logger"); - register_component_provider("dsn::tools::hpc_task_queue"); - register_component_provider("dsn::tools::hpc_task_priority_queue"); register_component_provider("dsn::tools::hpc_concurrent_task_queue"); - register_component_provider("dsn::tools::hpc_env_provider"); - - register_component_provider("dsn::tools::hpc_aio_provider"); - register_component_provider("dsn::tools::hpc_network_provider"); - register_component_provider("dsn::tools::io_looper_task_queue"); - register_component_provider("dsn::tools::io_looper_task_worker"); - register_component_provider("dsn::tools::io_looper_timer_service"); } } } diff --git a/src/core/tools/simulator/task_engine.sim.h b/src/core/tools/simulator/task_engine.sim.h index 25b5b8427e..bf9394f1d8 100644 --- a/src/core/tools/simulator/task_engine.sim.h +++ b/src/core/tools/simulator/task_engine.sim.h @@ -52,7 +52,7 @@ class sim_timer_service : public timer_service // after milliseconds, the provider should call task->enqueue() virtual void add_timer(task *task) override; - virtual void start(io_modifer &ctx) override {} + virtual void start() override {} }; class sim_task_queue : public task_queue diff --git a/src/dist/replication/test/simple_kv/case-000.ini b/src/dist/replication/test/simple_kv/case-000.ini index 7eb54d5b71..1f02483fe2 100644 --- a/src/dist/replication/test/simple_kv/case-000.ini +++ b/src/dist/replication/test/simple_kv/case-000.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-001.ini b/src/dist/replication/test/simple_kv/case-001.ini index f984522c80..0edac51ea6 100644 --- a/src/dist/replication/test/simple_kv/case-001.ini +++ b/src/dist/replication/test/simple_kv/case-001.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-002.ini b/src/dist/replication/test/simple_kv/case-002.ini index d792537ece..26fdcf3ac7 100644 --- a/src/dist/replication/test/simple_kv/case-002.ini +++ b/src/dist/replication/test/simple_kv/case-002.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-003.ini b/src/dist/replication/test/simple_kv/case-003.ini index 0643d3d62d..50cc09e54b 100644 --- a/src/dist/replication/test/simple_kv/case-003.ini +++ b/src/dist/replication/test/simple_kv/case-003.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-004.ini b/src/dist/replication/test/simple_kv/case-004.ini index 86b6139eac..2a5b1d3d77 100644 --- a/src/dist/replication/test/simple_kv/case-004.ini +++ b/src/dist/replication/test/simple_kv/case-004.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-005.ini b/src/dist/replication/test/simple_kv/case-005.ini index cd6abbef97..d115021ee3 100644 --- a/src/dist/replication/test/simple_kv/case-005.ini +++ b/src/dist/replication/test/simple_kv/case-005.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-006.ini b/src/dist/replication/test/simple_kv/case-006.ini index f984522c80..0edac51ea6 100644 --- a/src/dist/replication/test/simple_kv/case-006.ini +++ b/src/dist/replication/test/simple_kv/case-006.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-100.ini b/src/dist/replication/test/simple_kv/case-100.ini index 7eb54d5b71..1f02483fe2 100644 --- a/src/dist/replication/test/simple_kv/case-100.ini +++ b/src/dist/replication/test/simple_kv/case-100.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-101.ini b/src/dist/replication/test/simple_kv/case-101.ini index 7eb54d5b71..1f02483fe2 100644 --- a/src/dist/replication/test/simple_kv/case-101.ini +++ b/src/dist/replication/test/simple_kv/case-101.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-102.ini b/src/dist/replication/test/simple_kv/case-102.ini index 4fe0806311..5bafe318ec 100644 --- a/src/dist/replication/test/simple_kv/case-102.ini +++ b/src/dist/replication/test/simple_kv/case-102.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-103.ini b/src/dist/replication/test/simple_kv/case-103.ini index 7e1aca62d7..7459a02f8e 100644 --- a/src/dist/replication/test/simple_kv/case-103.ini +++ b/src/dist/replication/test/simple_kv/case-103.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-104.ini b/src/dist/replication/test/simple_kv/case-104.ini index 7eb54d5b71..1f02483fe2 100644 --- a/src/dist/replication/test/simple_kv/case-104.ini +++ b/src/dist/replication/test/simple_kv/case-104.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-105.ini b/src/dist/replication/test/simple_kv/case-105.ini index 7eb54d5b71..1f02483fe2 100644 --- a/src/dist/replication/test/simple_kv/case-105.ini +++ b/src/dist/replication/test/simple_kv/case-105.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-106.ini b/src/dist/replication/test/simple_kv/case-106.ini index 7eb54d5b71..1f02483fe2 100644 --- a/src/dist/replication/test/simple_kv/case-106.ini +++ b/src/dist/replication/test/simple_kv/case-106.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-107.ini b/src/dist/replication/test/simple_kv/case-107.ini index 7eb54d5b71..1f02483fe2 100644 --- a/src/dist/replication/test/simple_kv/case-107.ini +++ b/src/dist/replication/test/simple_kv/case-107.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-108.ini b/src/dist/replication/test/simple_kv/case-108.ini index 7eb54d5b71..1f02483fe2 100644 --- a/src/dist/replication/test/simple_kv/case-108.ini +++ b/src/dist/replication/test/simple_kv/case-108.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-109.ini b/src/dist/replication/test/simple_kv/case-109.ini index cae74b3b50..00d5325609 100644 --- a/src/dist/replication/test/simple_kv/case-109.ini +++ b/src/dist/replication/test/simple_kv/case-109.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-200.ini b/src/dist/replication/test/simple_kv/case-200.ini index 7eb54d5b71..1f02483fe2 100644 --- a/src/dist/replication/test/simple_kv/case-200.ini +++ b/src/dist/replication/test/simple_kv/case-200.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-201.ini b/src/dist/replication/test/simple_kv/case-201.ini index 7eb54d5b71..1f02483fe2 100644 --- a/src/dist/replication/test/simple_kv/case-201.ini +++ b/src/dist/replication/test/simple_kv/case-201.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-202-0.ini b/src/dist/replication/test/simple_kv/case-202-0.ini index 9d61ac7dd5..28be2b25c5 100644 --- a/src/dist/replication/test/simple_kv/case-202-0.ini +++ b/src/dist/replication/test/simple_kv/case-202-0.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-202-1.ini b/src/dist/replication/test/simple_kv/case-202-1.ini index 9d61ac7dd5..28be2b25c5 100644 --- a/src/dist/replication/test/simple_kv/case-202-1.ini +++ b/src/dist/replication/test/simple_kv/case-202-1.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-203-0.ini b/src/dist/replication/test/simple_kv/case-203-0.ini index 50fad55002..e123d13c3b 100644 --- a/src/dist/replication/test/simple_kv/case-203-0.ini +++ b/src/dist/replication/test/simple_kv/case-203-0.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-203-1.ini b/src/dist/replication/test/simple_kv/case-203-1.ini index 50fad55002..e123d13c3b 100644 --- a/src/dist/replication/test/simple_kv/case-203-1.ini +++ b/src/dist/replication/test/simple_kv/case-203-1.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-204.ini b/src/dist/replication/test/simple_kv/case-204.ini index 853e019b96..a4a4310c3e 100644 --- a/src/dist/replication/test/simple_kv/case-204.ini +++ b/src/dist/replication/test/simple_kv/case-204.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-205.ini b/src/dist/replication/test/simple_kv/case-205.ini index 853e019b96..a4a4310c3e 100644 --- a/src/dist/replication/test/simple_kv/case-205.ini +++ b/src/dist/replication/test/simple_kv/case-205.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-206.ini b/src/dist/replication/test/simple_kv/case-206.ini index 853e019b96..a4a4310c3e 100644 --- a/src/dist/replication/test/simple_kv/case-206.ini +++ b/src/dist/replication/test/simple_kv/case-206.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-207.ini b/src/dist/replication/test/simple_kv/case-207.ini index 853e019b96..a4a4310c3e 100644 --- a/src/dist/replication/test/simple_kv/case-207.ini +++ b/src/dist/replication/test/simple_kv/case-207.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-208.ini b/src/dist/replication/test/simple_kv/case-208.ini index 853e019b96..a4a4310c3e 100644 --- a/src/dist/replication/test/simple_kv/case-208.ini +++ b/src/dist/replication/test/simple_kv/case-208.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-209.ini b/src/dist/replication/test/simple_kv/case-209.ini index 853e019b96..a4a4310c3e 100644 --- a/src/dist/replication/test/simple_kv/case-209.ini +++ b/src/dist/replication/test/simple_kv/case-209.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-210.ini b/src/dist/replication/test/simple_kv/case-210.ini index 5c5a87feb1..22add149b7 100644 --- a/src/dist/replication/test/simple_kv/case-210.ini +++ b/src/dist/replication/test/simple_kv/case-210.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-211.ini b/src/dist/replication/test/simple_kv/case-211.ini index 5c5a87feb1..22add149b7 100644 --- a/src/dist/replication/test/simple_kv/case-211.ini +++ b/src/dist/replication/test/simple_kv/case-211.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-212.ini b/src/dist/replication/test/simple_kv/case-212.ini index 5c5a87feb1..22add149b7 100644 --- a/src/dist/replication/test/simple_kv/case-212.ini +++ b/src/dist/replication/test/simple_kv/case-212.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-213.ini b/src/dist/replication/test/simple_kv/case-213.ini index 6b2618b5db..554f28a7a7 100644 --- a/src/dist/replication/test/simple_kv/case-213.ini +++ b/src/dist/replication/test/simple_kv/case-213.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-214.ini b/src/dist/replication/test/simple_kv/case-214.ini index 853e019b96..a4a4310c3e 100644 --- a/src/dist/replication/test/simple_kv/case-214.ini +++ b/src/dist/replication/test/simple_kv/case-214.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-215.ini b/src/dist/replication/test/simple_kv/case-215.ini index 853e019b96..a4a4310c3e 100644 --- a/src/dist/replication/test/simple_kv/case-215.ini +++ b/src/dist/replication/test/simple_kv/case-215.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-216.ini b/src/dist/replication/test/simple_kv/case-216.ini index 853e019b96..a4a4310c3e 100644 --- a/src/dist/replication/test/simple_kv/case-216.ini +++ b/src/dist/replication/test/simple_kv/case-216.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-300-0.ini b/src/dist/replication/test/simple_kv/case-300-0.ini index 7eb54d5b71..1f02483fe2 100644 --- a/src/dist/replication/test/simple_kv/case-300-0.ini +++ b/src/dist/replication/test/simple_kv/case-300-0.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-300-1.ini b/src/dist/replication/test/simple_kv/case-300-1.ini index 7eb54d5b71..1f02483fe2 100644 --- a/src/dist/replication/test/simple_kv/case-300-1.ini +++ b/src/dist/replication/test/simple_kv/case-300-1.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-300-2.ini b/src/dist/replication/test/simple_kv/case-300-2.ini index 7eb54d5b71..1f02483fe2 100644 --- a/src/dist/replication/test/simple_kv/case-300-2.ini +++ b/src/dist/replication/test/simple_kv/case-300-2.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-301.ini b/src/dist/replication/test/simple_kv/case-301.ini index 7eb54d5b71..1f02483fe2 100644 --- a/src/dist/replication/test/simple_kv/case-301.ini +++ b/src/dist/replication/test/simple_kv/case-301.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-302.ini b/src/dist/replication/test/simple_kv/case-302.ini index 3b3d19b1a1..361bf821ed 100644 --- a/src/dist/replication/test/simple_kv/case-302.ini +++ b/src/dist/replication/test/simple_kv/case-302.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-303.ini b/src/dist/replication/test/simple_kv/case-303.ini index 619f08e736..6d34011852 100644 --- a/src/dist/replication/test/simple_kv/case-303.ini +++ b/src/dist/replication/test/simple_kv/case-303.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-304.ini b/src/dist/replication/test/simple_kv/case-304.ini index 7eb54d5b71..1f02483fe2 100644 --- a/src/dist/replication/test/simple_kv/case-304.ini +++ b/src/dist/replication/test/simple_kv/case-304.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-305.ini b/src/dist/replication/test/simple_kv/case-305.ini index 619f08e736..6d34011852 100644 --- a/src/dist/replication/test/simple_kv/case-305.ini +++ b/src/dist/replication/test/simple_kv/case-305.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-306.ini b/src/dist/replication/test/simple_kv/case-306.ini index 7eb54d5b71..1f02483fe2 100644 --- a/src/dist/replication/test/simple_kv/case-306.ini +++ b/src/dist/replication/test/simple_kv/case-306.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-307.ini b/src/dist/replication/test/simple_kv/case-307.ini index 7eb54d5b71..1f02483fe2 100644 --- a/src/dist/replication/test/simple_kv/case-307.ini +++ b/src/dist/replication/test/simple_kv/case-307.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-400.ini b/src/dist/replication/test/simple_kv/case-400.ini index 7eb54d5b71..1f02483fe2 100644 --- a/src/dist/replication/test/simple_kv/case-400.ini +++ b/src/dist/replication/test/simple_kv/case-400.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-401.ini b/src/dist/replication/test/simple_kv/case-401.ini index 7eb54d5b71..1f02483fe2 100644 --- a/src/dist/replication/test/simple_kv/case-401.ini +++ b/src/dist/replication/test/simple_kv/case-401.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-402.ini b/src/dist/replication/test/simple_kv/case-402.ini index 7eb54d5b71..1f02483fe2 100644 --- a/src/dist/replication/test/simple_kv/case-402.ini +++ b/src/dist/replication/test/simple_kv/case-402.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-600.ini b/src/dist/replication/test/simple_kv/case-600.ini index f984522c80..0edac51ea6 100644 --- a/src/dist/replication/test/simple_kv/case-600.ini +++ b/src/dist/replication/test/simple_kv/case-600.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-601.ini b/src/dist/replication/test/simple_kv/case-601.ini index 6601d8a7ae..63dc3f91b4 100644 --- a/src/dist/replication/test/simple_kv/case-601.ini +++ b/src/dist/replication/test/simple_kv/case-601.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = tracer,test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-602.ini b/src/dist/replication/test/simple_kv/case-602.ini index f984522c80..0edac51ea6 100644 --- a/src/dist/replication/test/simple_kv/case-602.ini +++ b/src/dist/replication/test/simple_kv/case-602.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/case-603.ini b/src/dist/replication/test/simple_kv/case-603.ini index a02cc8913f..9e012574ea 100644 --- a/src/dist/replication/test/simple_kv/case-603.ini +++ b/src/dist/replication/test/simple_kv/case-603.ini @@ -39,7 +39,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun toollets = test_injector ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/dist/replication/test/simple_kv/checker.cpp b/src/dist/replication/test/simple_kv/checker.cpp index f3c269030f..a08e0e718a 100644 --- a/src/dist/replication/test/simple_kv/checker.cpp +++ b/src/dist/replication/test/simple_kv/checker.cpp @@ -178,7 +178,7 @@ bool test_checker::init(const std::string &name, const std::vectorid(); std::string name = node.second->full_name(); - rpc_address paddr = node.second->rpc(nullptr)->primary_address(); + rpc_address paddr = node.second->rpc()->primary_address(); int port = paddr.port(); _node_to_address[name] = paddr; ddebug("=== node_to_address[%s]=%s", name.c_str(), paddr.to_string()); diff --git a/src/dist/replication/test/simple_kv/config.ini b/src/dist/replication/test/simple_kv/config.ini index 5322f61969..ccde4f922e 100644 --- a/src/dist/replication/test/simple_kv/config.ini +++ b/src/dist/replication/test/simple_kv/config.ini @@ -56,7 +56,6 @@ start_nfs = true tool = simulator ;tool = nativerun -;tool = fastrun ;toollets = tracer ;toollets = fault_injector ;toollets = tracer, fault_injector diff --git a/src/tests/dsn/config-test.ini b/src/tests/dsn/config-test.ini index 94ef4bb4b8..c486598b4c 100644 --- a/src/tests/dsn/config-test.ini +++ b/src/tests/dsn/config-test.ini @@ -48,8 +48,7 @@ pools = THREAD_POOL_DEFAULT, THREAD_POOL_FD [core] ;tool = simulator -;tool = nativerun -tool = fastrun +tool = nativerun ;toollets = tracer, profiler ;fault_injector From 8667ab1491c026d6a225fd7ab1c647f590afb838 Mon Sep 17 00:00:00 2001 From: WeijieSun Date: Mon, 11 Jun 2018 14:17:23 +0800 Subject: [PATCH 2/3] core: assign a timer thread for each worker thread --- src/core/core/task_engine.cpp | 48 ++++++++++++++++++++++------------- src/core/core/task_engine.h | 2 -- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/src/core/core/task_engine.cpp b/src/core/core/task_engine.cpp index dda478bf92..48b3794848 100644 --- a/src/core/core/task_engine.cpp +++ b/src/core/core/task_engine.cpp @@ -42,10 +42,8 @@ using namespace dsn::utils; namespace dsn { task_worker_pool::task_worker_pool(const threadpool_spec &opts, task_engine *owner) - : _spec(opts), _owner(owner), _node(owner->node()) + : _spec(opts), _owner(owner), _node(owner->node()), _is_running(false) { - _is_running = false; - _timer_svc = nullptr; } void task_worker_pool::create() @@ -80,6 +78,19 @@ void task_worker_pool::create() } } + for (int i = 0; i < qCount; ++i) { + auto tsvc = factory_store::create( + service_engine::fast_instance().spec().timer_factory_name.c_str(), + PROVIDER_TYPE_MAIN, + _node, + nullptr); + for (auto &s : service_engine::fast_instance().spec().timer_aspects) { + tsvc = + factory_store::create(s.c_str(), PROVIDER_TYPE_ASPECT, _node, tsvc); + } + _per_queue_timer_svcs.push_back(tsvc); + } + for (int i = 0; i < _spec.worker_count; i++) { auto q = _queues[qCount == 1 ? 0 : i]; task_worker *worker = factory_store::create( @@ -100,6 +111,8 @@ void task_worker_pool::start() if (_is_running) return; + for (auto &tsvc : _per_queue_timer_svcs) + tsvc->start(); for (auto &wk : _workers) wk->start(); @@ -112,8 +125,6 @@ void task_worker_pool::start() _spec.worker_share_core ? "true" : "false", _spec.partitioned ? "true" : "false"); - // setup cached ptrs for fast timer service access - _timer_svc = node()->tsvc(); _is_running = true; } @@ -122,7 +133,11 @@ void task_worker_pool::add_timer(task *t) dassert(t->delay_milliseconds() > 0, "task delayed should be dispatched to timer service first"); - _timer_svc->add_timer(t); + unsigned int idx = (_spec.partitioned + ? static_cast(t->hash()) % + static_cast(_per_queue_timer_svcs.size()) + : 0); + _per_queue_timer_svcs[idx]->add_timer(t); } void task_worker_pool::enqueue(task *t) @@ -132,18 +147,15 @@ void task_worker_pool::enqueue(task *t) dassert(t->delay_milliseconds() == 0, "task delayed should be dispatched to timer service first"); - if (_is_running) { - unsigned int idx = - (_spec.partitioned - ? static_cast(t->hash()) % static_cast(_queues.size()) - : 0); - return _queues[idx]->enqueue_internal(t); - } else { - dassert(false, - "worker pool %s must be started before enqueue task %s", - spec().name.c_str(), - t->spec().name.c_str()); - } + dassert(_is_running, + "worker pool %s must be started before enqueue task %s", + spec().name.c_str(), + t->spec().name.c_str()); + unsigned int idx = + (_spec.partitioned + ? static_cast(t->hash()) % static_cast(_queues.size()) + : 0); + return _queues[idx]->enqueue_internal(t); } bool task_worker_pool::shared_same_worker_with_current_task(task *tsk) const diff --git a/src/core/core/task_engine.h b/src/core/core/task_engine.h index f839cea02a..2fbc238706 100644 --- a/src/core/core/task_engine.h +++ b/src/core/core/task_engine.h @@ -90,8 +90,6 @@ class task_worker_pool std::vector _queues; std::vector _controllers; - // cached ptrs for fast access - timer_service *_timer_svc; std::vector _per_queue_timer_svcs; bool _is_running; From 2ebbc1cbdcba8bcca816f00631b14e91a9ff0d77 Mon Sep 17 00:00:00 2001 From: WeijieSun Date: Mon, 11 Jun 2018 14:52:56 +0800 Subject: [PATCH 3/3] core: remove timer service from service node, hide it into task worker pool as a private module --- include/dsn/tool-api/task.h | 8 -------- src/core/core/service_engine.cpp | 14 -------------- src/core/core/service_engine.h | 2 -- src/core/core/task.cpp | 1 - 4 files changed, 25 deletions(-) diff --git a/include/dsn/tool-api/task.h b/include/dsn/tool-api/task.h index a7f30b62b7..445817d7f1 100644 --- a/include/dsn/tool-api/task.h +++ b/include/dsn/tool-api/task.h @@ -85,7 +85,6 @@ struct __tls_dsn__ disk_engine *disk; env_provider *env; nfs_node *nfs; - timer_service *tsvc; int last_worker_queue_size; uint64_t node_pool_thread_ids; // 8,8,16 bits @@ -260,7 +259,6 @@ class task : public ref_counter, public extensible_object, public trans static disk_engine *get_current_disk(); static env_provider *get_current_env(); static nfs_node *get_current_nfs(); - static timer_service *get_current_tsvc(); static void set_tls_dsn_context( service_node *node, // cannot be null @@ -715,10 +713,4 @@ __inline /*static*/ nfs_node *task::get_current_nfs() return tls_dsn.nfs; } -__inline /*static*/ timer_service *task::get_current_tsvc() -{ - check_tls_dsn(); - return tls_dsn.tsvc; -} - } // end namespace diff --git a/src/core/core/service_engine.cpp b/src/core/core/service_engine.cpp index 659cf2e07e..cdd0ca3f70 100644 --- a/src/core/core/service_engine.cpp +++ b/src/core/core/service_engine.cpp @@ -75,17 +75,6 @@ error_code service_node::init_io_engine() auto &spec = service_engine::fast_instance().spec(); error_code err = ERR_OK; - // init timer service - _node_io.tsvc = factory_store::create( - service_engine::fast_instance().spec().timer_factory_name.c_str(), - PROVIDER_TYPE_MAIN, - this, - nullptr); - for (auto &s : service_engine::fast_instance().spec().timer_aspects) { - _node_io.tsvc = factory_store::create( - s.c_str(), PROVIDER_TYPE_ASPECT, this, _node_io.tsvc); - } - // init disk engine _node_io.disk = new disk_engine(this); aio_provider *aio = factory_store::create( @@ -119,9 +108,6 @@ error_code service_node::start_io_engine_in_main() auto &spec = service_engine::fast_instance().spec(); error_code err = ERR_OK; - // start timer service - _node_io.tsvc->start(); - // start disk engine _node_io.disk->start(_node_io.aio); diff --git a/src/core/core/service_engine.h b/src/core/core/service_engine.h index 46b9cce2d3..008ad3690c 100644 --- a/src/core/core/service_engine.h +++ b/src/core/core/service_engine.h @@ -69,7 +69,6 @@ class service_node rpc_engine *rpc; disk_engine *disk; nfs_node *nfs; - timer_service *tsvc; aio_provider *aio; io_engine() { memset((void *)this, 0, sizeof(io_engine)); } @@ -81,7 +80,6 @@ class service_node rpc_engine *rpc() const { return _node_io.rpc; } disk_engine *disk() const { return _node_io.disk; } nfs_node *nfs() const { return _node_io.nfs; } - timer_service *tsvc() const { return _node_io.tsvc; } task_engine *computation() const { return _computation; } void get_runtime_info(const std::string &indent, diff --git a/src/core/core/task.cpp b/src/core/core/task.cpp index 8e138b234d..2ce6521b6e 100644 --- a/src/core/core/task.cpp +++ b/src/core/core/task.cpp @@ -76,7 +76,6 @@ __thread uint16_t tls_dsn_lower32_task_id_mask = 0; tls_dsn.disk = node->disk(); tls_dsn.env = service_engine::fast_instance().env(); tls_dsn.nfs = node->nfs(); - tls_dsn.tsvc = node->tsvc(); } tls_dsn.node_pool_thread_ids = (node ? ((uint64_t)(uint8_t)node->id()) : 0)