From 83e249acdbaf330f613c297addb1c949ba750b4c Mon Sep 17 00:00:00 2001 From: Haomai Wang Date: Tue, 6 Mar 2018 11:28:54 +0800 Subject: [PATCH] msg/async: avoid put message within write_lock message deconstruct under busy env isn't a very short period, because cpu may stuck into tcmalloc library to do gc. from a rough bench(3k iops), reduce send_message latency from 54us to 37.5us. in real env(higher iops), it should be much more better. Signed-off-by: Haomai Wang --- src/msg/async/AsyncConnection.cc | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/msg/async/AsyncConnection.cc b/src/msg/async/AsyncConnection.cc index 8927201587a37..b919702c2a61b 100644 --- a/src/msg/async/AsyncConnection.cc +++ b/src/msg/async/AsyncConnection.cc @@ -2299,15 +2299,21 @@ void AsyncConnection::handle_ack(uint64_t seq) { ldout(async_msgr->cct, 15) << __func__ << " got ack seq " << seq << dendl; // trim sent list - std::lock_guard l(write_lock); - while (!sent.empty() && sent.front()->get_seq() <= seq) { + static const int max_pending = 128; + int i = 0; + Message *pending[max_pending]; + write_lock.lock(); + while (!sent.empty() && sent.front()->get_seq() <= seq && i < max_pending) { Message* m = sent.front(); sent.pop_front(); + pending[i++] = m; ldout(async_msgr->cct, 10) << __func__ << " got ack seq " << seq << " >= " << m->get_seq() << " on " << m << " " << *m << dendl; - m->put(); } + write_lock.unlock(); + for (int k = 0; k < i; k++) + pending[k]->put(); } void AsyncConnection::DelayedDelivery::do_request(uint64_t id)