From 4cdc8f9588b0c2d0b62dc91a3a1e884deebcf534 Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Tue, 21 Mar 2017 14:03:23 +0800 Subject: [PATCH] tests: ceph_test_rados_api_watch_notify: test timeout using rados_watch3() objecter resends the watch request upon tcp reconnecting, and OSD resets the watch timeout when handling the resent watch request, so, with a small "ms tcp read timeout", the timeout test always fails. in this change, we use rados_watch3() which supports the "timeout" option, so we can pass a timeout smaller than "ms tcp read timeout", and hence the test can pass. Fixes: http://tracker.ceph.com/issues/19312 Signed-off-by: Kefu Chai --- src/test/librados/watch_notify.cc | 182 ++++++++++++++++-------------- 1 file changed, 97 insertions(+), 85 deletions(-) diff --git a/src/test/librados/watch_notify.cc b/src/test/librados/watch_notify.cc index 87bb579215656..953df293f9cd2 100644 --- a/src/test/librados/watch_notify.cc +++ b/src/test/librados/watch_notify.cc @@ -319,91 +319,6 @@ TEST_F(LibRadosWatchNotify, AioWatchDelete) { rados_aio_release(comp); } -TEST_F(LibRadosWatchNotify, Watch2Timeout) { - notify_io = ioctx; - notify_oid = "foo"; - notify_cookies.clear(); - notify_err = 0; - char buf[128]; - memset(buf, 0xcc, sizeof(buf)); - ASSERT_EQ(0, rados_write(ioctx, notify_oid, buf, sizeof(buf), 0)); - uint64_t handle; - time_t start = time(0); - ASSERT_EQ(0, - rados_watch2(ioctx, notify_oid, &handle, - watch_notify2_test_cb, - watch_notify2_test_errcb, this)); - int age = rados_watch_check(ioctx, handle); - time_t age_bound = time(0) + 1 - start; - ASSERT_LT(age, age_bound * 1000); - ASSERT_GT(age, 0); - rados_conf_set(cluster, "objecter_inject_no_watch_ping", "true"); - int left = 900; - std::cout << "waiting up to " << left << " for osd to time us out ..." - << std::endl; - while (notify_err == 0 && --left) { - sleep(1); - } - ASSERT_TRUE(left > 0); - rados_conf_set(cluster, "objecter_inject_no_watch_ping", "false"); - ASSERT_EQ(-ENOTCONN, notify_err); - ASSERT_EQ(-ENOTCONN, rados_watch_check(ioctx, handle)); - - // a subsequent notify should not reach us - char *reply_buf = 0; - size_t reply_buf_len; - ASSERT_EQ(0, rados_notify2(ioctx, notify_oid, - "notify", 6, 300000, - &reply_buf, &reply_buf_len)); - { - bufferlist reply; - reply.append(reply_buf, reply_buf_len); - std::map, bufferlist> reply_map; - std::set > missed_map; - bufferlist::iterator reply_p = reply.begin(); - ::decode(reply_map, reply_p); - ::decode(missed_map, reply_p); - ASSERT_EQ(0u, reply_map.size()); - ASSERT_EQ(0u, missed_map.size()); - } - ASSERT_EQ(0u, notify_cookies.size()); - ASSERT_EQ(-ENOTCONN, rados_watch_check(ioctx, handle)); - rados_buffer_free(reply_buf); - - // re-watch - rados_unwatch2(ioctx, handle); - handle = 0; - ASSERT_EQ(0, - rados_watch2(ioctx, notify_oid, &handle, - watch_notify2_test_cb, - watch_notify2_test_errcb, this)); - ASSERT_GT(rados_watch_check(ioctx, handle), 0); - - // and now a notify will work. - ASSERT_EQ(0, rados_notify2(ioctx, notify_oid, - "notify", 6, 300000, - &reply_buf, &reply_buf_len)); - { - bufferlist reply; - reply.append(reply_buf, reply_buf_len); - std::map, bufferlist> reply_map; - std::set > missed_map; - bufferlist::iterator reply_p = reply.begin(); - ::decode(reply_map, reply_p); - ::decode(missed_map, reply_p); - ASSERT_EQ(1u, reply_map.size()); - ASSERT_EQ(0u, missed_map.size()); - ASSERT_EQ(1u, notify_cookies.count(handle)); - ASSERT_EQ(5u, reply_map.begin()->second.length()); - ASSERT_EQ(0, strncmp("reply", reply_map.begin()->second.c_str(), 5)); - } - ASSERT_EQ(1u, notify_cookies.size()); - ASSERT_GT(rados_watch_check(ioctx, handle), 0); - - rados_buffer_free(reply_buf); - rados_unwatch2(ioctx, handle); -} - // -- TEST_F(LibRadosWatchNotify, WatchNotify2) { @@ -849,6 +764,103 @@ TEST_P(LibRadosWatchNotifyPP, WatchNotify3) { ioctx.unwatch2(handle); } +TEST_F(LibRadosWatchNotify, Watch3Timeout) { + notify_io = ioctx; + notify_oid = "foo"; + notify_cookies.clear(); + notify_err = 0; + char buf[128]; + memset(buf, 0xcc, sizeof(buf)); + ASSERT_EQ(0, rados_write(ioctx, notify_oid, buf, sizeof(buf), 0)); + uint64_t handle; + time_t start = time(0); + const uint32_t timeout = 4; + { + // make sure i timeout before the messenger reconnects to the OSD, + // it will resend a watch request on behalf of the client, and the + // timer of timeout on OSD side will be reset by the new request. + char conf[128]; + ASSERT_EQ(0, rados_conf_get(cluster, + "ms_tcp_read_timeout", + conf, sizeof(conf))); + auto tcp_read_timeout = std::stoll(conf); + ASSERT_LT(timeout, tcp_read_timeout); + } + ASSERT_EQ(0, + rados_watch3(ioctx, notify_oid, &handle, + watch_notify2_test_cb, watch_notify2_test_errcb, + timeout, this)); + int age = rados_watch_check(ioctx, handle); + time_t age_bound = time(0) + 1 - start; + ASSERT_LT(age, age_bound * 1000); + ASSERT_GT(age, 0); + rados_conf_set(cluster, "objecter_inject_no_watch_ping", "true"); + int left = 2 * timeout; + std::cout << "waiting up to " << left << " for osd to time us out ..." + << std::endl; + while (notify_err == 0 && --left) { + sleep(1); + } + ASSERT_GT(left, 0); + rados_conf_set(cluster, "objecter_inject_no_watch_ping", "false"); + ASSERT_EQ(-ENOTCONN, notify_err); + ASSERT_EQ(-ENOTCONN, rados_watch_check(ioctx, handle)); + + // a subsequent notify should not reach us + char *reply_buf = nullptr; + size_t reply_buf_len; + ASSERT_EQ(0, rados_notify2(ioctx, notify_oid, + "notify", 6, 300000, + &reply_buf, &reply_buf_len)); + { + bufferlist reply; + reply.append(reply_buf, reply_buf_len); + std::map, bufferlist> reply_map; + std::set > missed_map; + bufferlist::iterator reply_p = reply.begin(); + ::decode(reply_map, reply_p); + ::decode(missed_map, reply_p); + ASSERT_EQ(0u, reply_map.size()); + ASSERT_EQ(0u, missed_map.size()); + } + ASSERT_EQ(0u, notify_cookies.size()); + ASSERT_EQ(-ENOTCONN, rados_watch_check(ioctx, handle)); + rados_buffer_free(reply_buf); + + // re-watch + rados_unwatch2(ioctx, handle); + handle = 0; + ASSERT_EQ(0, + rados_watch2(ioctx, notify_oid, &handle, + watch_notify2_test_cb, + watch_notify2_test_errcb, this)); + ASSERT_GT(rados_watch_check(ioctx, handle), 0); + + // and now a notify will work. + ASSERT_EQ(0, rados_notify2(ioctx, notify_oid, + "notify", 6, 300000, + &reply_buf, &reply_buf_len)); + { + bufferlist reply; + reply.append(reply_buf, reply_buf_len); + std::map, bufferlist> reply_map; + std::set > missed_map; + bufferlist::iterator reply_p = reply.begin(); + ::decode(reply_map, reply_p); + ::decode(missed_map, reply_p); + ASSERT_EQ(1u, reply_map.size()); + ASSERT_EQ(0u, missed_map.size()); + ASSERT_EQ(1u, notify_cookies.count(handle)); + ASSERT_EQ(5u, reply_map.begin()->second.length()); + ASSERT_EQ(0, strncmp("reply", reply_map.begin()->second.c_str(), 5)); + } + ASSERT_EQ(1u, notify_cookies.size()); + ASSERT_GT(rados_watch_check(ioctx, handle), 0); + + rados_buffer_free(reply_buf); + rados_unwatch2(ioctx, handle); +} + TEST_F(LibRadosWatchNotify, AioWatchDelete2) { notify_io = ioctx; notify_oid = "foo";