msg/pipe: add an option to disable immediate failure on ECONNREFUSED

New option was added (ms immediate fail on connection refused) with "true"
as default setting, that controls whether the new failure behavior is active.

Signed-off-by: Piotr Dałek <git@predictor.org.pl>

src/common/config_opts.h

@@ -188,6 +188,7 @@ OPTION(ms_rwthread_stack_bytes, OPT_U64, 1024 << 10)
 OPTION(ms_tcp_read_timeout, OPT_U64, 900)
 OPTION(ms_pq_max_tokens_per_priority, OPT_U64, 16777216)
 OPTION(ms_pq_min_cost, OPT_U64, 65536)
+OPTION(ms_immediate_fail_on_connection_refused, OPT_BOOL, true) // mark osd down immediately after receiving ECONNREFUSED during (re)connection
 OPTION(ms_inject_socket_failures, OPT_U64, 0)
 OPTION(ms_inject_delay_type, OPT_STR, "")          // "osd mds mon client" allowed
 OPTION(ms_inject_delay_msg_type, OPT_STR, "")      // the type of message to delay, as returned by Message::get_type_name(). This is an additional restriction on the general type filter ms_inject_delay_type.

src/msg/simple/Pipe.cc

@@ -919,7 +919,9 @@ int Pipe::connect()
 	     << ", " << cpp_strerror(stored_errno) << dendl;
     if (stored_errno == ECONNREFUSED) {
       ldout(msgr->cct, 10) << "connection refused!" << dendl;
-      msgr->dispatch_queue.queue_refused(connection_state.get());
+      if (conf->ms_immediate_fail_on_connection_refused) {
+        msgr->dispatch_queue.queue_refused(connection_state.get());
+      }
     }
     goto fail;
   }