rgw: move all curl related global init to one place

since libcurl needs to setup the various ssl libraries based on backends, moving
all the global init to one place. A TODO is to use std::call_once on the
global_init and similarly the init functions of libopenssl. A further TODO is to
implement locking primitives for gnutls

Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>

src/rgw/CMakeLists.txt

@@ -60,6 +60,7 @@ set(rgw_a_srcs
   rgw_frontend.cc
   rgw_gc.cc
   rgw_http_client.cc
+  rgw_http_client_curl.cc
   rgw_json_enc.cc
   rgw_keystone.cc
   rgw_ldap.cc
@@ -171,11 +172,6 @@ if (WITH_RADOSGW_BEAST_FRONTEND)
     rgw_asio_frontend.cc)
 endif (WITH_RADOSGW_BEAST_FRONTEND)
 
-if (WITH_RADOSGW_OPENSSL_LOCK_COMPAT)
-  list(APPEND radosgw_srcs
-    rgw_http_client_ssl.cc)
-endif (WITH_RADOSGW_OPENSSL_LOCK_COMPAT)
-
 add_library(radosgw_a STATIC ${radosgw_srcs}
   $<TARGET_OBJECTS:civetweb_common_objs>)
 target_link_libraries(radosgw_a rgw_a ${SSL_LIBRARIES})

src/rgw/rgw_http_client_curl.cc

@@ -0,0 +1,103 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+extern "C" {
+#include <curl/curl.h>
+}
+
+#include "rgw_http_client_curl.h"
+#include <mutex>
+#include <vector>
+
+#include "rgw_common.h"
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rgw
+
+#ifdef WITH_RADOSGW_OPENSSL_LOCK_COMPAT
+#include <openssl/crypto.h>
+#endif
+
+namespace rgw {
+namespace curl {
+
+#ifdef HAVE_CURL_MULTI_WAIT
+static void check_curl()
+{
+}
+#else
+static void check_curl()
+{
+  derr << "WARNING: libcurl doesn't support curl_multi_wait()" << dendl;
+  derr << "WARNING: cross zone / region transfer performance may be affected" << dendl;
+}
+#endif
+
+void init_ssl(){
+#ifdef WITH_RADOSGW_OPENSSL_LOCK_COMPAT
+  rgw::curl::openssl::init_ssl();
+#endif
+}
+
+std::once_flag curl_init_flag;
+
+void setup_curl(){
+  check_curl();
+  init_ssl();
+  std::call_once(curl_init_flag, curl_global_init, CURL_GLOBAL_ALL);
+}
+
+void cleanup_curl(){
+  curl_global_cleanup();
+}
+
+
+namespace openssl {
+
+class RGWSSLSetup
+{
+  std::vector <std::mutex> locks;
+public:
+  RGWSSLSetup(int n) : locks (n){}
+
+  void set_lock(int id){
+    try {
+      locks.at(id).lock();
+    } catch (std::out_of_range& e) {
+      dout(0) << __func__ << "failed to set locks" << dendl;
+    }
+  }
+
+  void clear_lock(int id){
+    try {
+      locks.at(id).unlock();
+    } catch (std::out_of_range& e) {
+      dout(0) << __func__ << "failed to unlock" << dendl;
+    }
+  }
+};
+
+
+#ifdef WITH_RADOSGW_OPENSSL_LOCK_COMPAT
+void rgw_ssl_locking_callback(int mode, int id, const char *file, int line)
+{
+  static RGWSSLSetup locks(CRYPTO_num_locks());
+  if (mode & CRYPTO_LOCK)
+    locks.set_lock(id);
+  else
+    locks.clear_lock(id);
+}
+
+unsigned long rgw_ssl_thread_id_callback(){
+  return (unsigned long)pthread_self();
+}
+
+void init_ssl(){
+  CRYPTO_set_id_callback((unsigned long (*) ()) rgw_ssl_thread_id_callback);
+  CRYPTO_set_locking_callback(rgw_ssl_locking_callback);
+}
+#else
+void init_ssl(){}
+#endif // WITH_RADOSGW_OPENSSL_LOCK_COMPAT
+
+}
+}
+}

src/rgw/rgw_http_client_ssl.h → src/rgw/rgw_http_client_curl.h

@@ -11,11 +11,21 @@
  * Foundation.  See file COPYING.
  *
  */
+#ifndef RGW_HTTP_CLIENT_CURL_H
+#define RGW_HTTP_CLIENT_CURL_H
 
-#ifndef RGW_HTTP_CLIENT_SSL_H
-#define RGW_HTTP_CLIENT_SSL_H
+namespace rgw {
+namespace curl {
 
-void rgw_ssl_locking_callback(int mode, int n, const char *file, int line);
-unsigned long rgw_ssl_thread_id_callback();
+void init_ssl();
+void setup_curl();
+void cleanup_curl();
+
+namespace openssl {
+void init_ssl();
+}
+
+}
+}
 
 #endif

src/rgw/rgw_main.cc

@@ -1,10 +1,5 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
-
-extern "C" {
-#include <curl/curl.h>
-}
-
 #include "common/ceph_argparse.h"
 #include "global/global_init.h"
 #include "global/signal_handler.h"
@@ -40,14 +35,11 @@ extern "C" {
 #include "rgw_request.h"
 #include "rgw_process.h"
 #include "rgw_frontend.h"
+#include "rgw_http_client_curl.h"
 #if defined(WITH_RADOSGW_BEAST_FRONTEND)
 #include "rgw_asio_frontend.h"
 #endif /* WITH_RADOSGW_BEAST_FRONTEND */
 
-#if defined(WITH_RADOSGW_OPENSSL_LOCK_COMPAT)
-#include "rgw_http_client_ssl.h"
-#include <openssl/crypto.h>
-#endif /* WITH_RADOSGW_OPENSSL_LOCK_COMPAT */
 
 #ifdef HAVE_SYS_PRCTL_H
 #include <sys/prctl.h>
@@ -119,17 +111,6 @@ static void godown_alarm(int signum)
   _exit(0);
 }
 
-#ifdef HAVE_CURL_MULTI_WAIT
-static void check_curl()
-{
-}
-#else
-static void check_curl()
-{
-  derr << "WARNING: libcurl doesn't support curl_multi_wait()" << dendl;
-  derr << "WARNING: cross zone / region transfer performance may be affected" << dendl;
-}
-#endif
 
 class C_InitTimeout : public Context {
 public:
@@ -275,8 +256,6 @@ int main(int argc, const char **argv)
     g_conf->set_val_or_die("rgw_zonegroup", g_conf->rgw_region.c_str());
   }
 
-  check_curl();
-
   if (g_conf->daemonize) {
     global_init_daemonize(g_ceph_context);
   }
@@ -299,13 +278,8 @@ int main(int argc, const char **argv)
   }
 
   rgw_init_resolver();
+  rgw::curl::setup_curl();
 
-#if defined(WITH_RADOSGW_OPENSSL_LOCK_COMPAT)
-  CRYPTO_set_id_callback((unsigned long (*) ()) rgw_ssl_thread_id_callback);
-  CRYPTO_set_locking_callback(rgw_ssl_locking_callback);
-#endif
-  curl_global_init(CURL_GLOBAL_ALL);
-
 #if defined(WITH_RADOSGW_FCGI_FRONTEND)
   FCGX_Init();
 #endif
@@ -580,7 +554,7 @@ int main(int argc, const char **argv)
 
   rgw_tools_cleanup();
   rgw_shutdown_resolver();
-  curl_global_cleanup();
+  rgw::curl::cleanup_curl();
 
   rgw_perf_stop(g_ceph_context);