Handle multiple listening addreses w/ optional ssl "correctly" with c…

…ivetweb.

For civetweb: accept a range of port numbers joined with '+'.
Port numbers may include an ipaddress: prefix and 's' suffix.
Additionally, use "mg_get_local_addr" to correctly deduce host port per
incoming connection.

civetweb can accept connections on multiple ports, some of which
might have SSL turned on and some not.  Both s3 and swift have various
authorization protocols in which the port number matters.  In the generic
radosgw frontend process, each frontend only has one port number, but
we should want to have both ssl and non-ssl connections managed within
one rgw frontend, because the thread pool is also per front-end, and
that *is* a scarce resource.

So, this patch enables the use of multiple ports with a single civetweb
frontend.  To indicate https: append an 's' to portno.  To use multiple
ports, use +.  So 80+443s indicates use of the usual default http ports.
The parsed port is not stored in the frontend structure,

So instead, this patch adds logic to use the results of
mg_get_local_addr() on a per-connection basis insetad of the generic
front-end port number.  This will affect "v4" s3 authorization, and also
affect swift pre-signed URLs.

mg_get_local_addr() is a new customization to civetweb; that submodule
was updated (in a temporary repository) by the previous commit to this.

Signed-off-by: Marcus Watts <mwatts@redhat.com>

src/rgw/rgw_civetweb.cc

@@ -29,13 +29,23 @@ size_t RGWCivetWeb::write_data(const char *buf, const size_t len)
   return len;
 }
 
-RGWCivetWeb::RGWCivetWeb(mg_connection* const conn, const int port)
+RGWCivetWeb::RGWCivetWeb(mg_connection* const conn)
   : conn(conn),
-    port(port),
     explicit_keepalive(false),
     explicit_conn_close(false),
     txbuf(*this)
 {
+    sockaddr *lsa = mg_get_local_addr(conn);
+    switch(lsa->sa_family) {
+    case AF_INET:
+	port = ntohs(((struct sockaddr_in*)lsa)->sin_port);
+	break;
+    case AF_INET6:
+	port = ntohs(((struct sockaddr_in6*)lsa)->sin6_port);
+	break;
+    default:
+	port = -1;
+    }
 }
 
 size_t RGWCivetWeb::read_data(char *buf, size_t len)
@@ -110,14 +120,12 @@ void RGWCivetWeb::init_env(CephContext *cct)
     env.set("REMOTE_USER", info->remote_user);
   }
 
+  if (port <= 0)
+    lderr(cct) << "init_env: bug: invalid port number" << dendl;
   char port_buf[16];
   snprintf(port_buf, sizeof(port_buf), "%d", port);
   env.set("SERVER_PORT", port_buf);
-
   if (info->is_ssl) {
-    if (port == 0) {
-      strcpy(port_buf,"443");
-    }
     env.set("SERVER_PORT_SECURE", port_buf);
   }
 }

src/rgw/rgw_civetweb.h

@@ -52,7 +52,7 @@ class RGWCivetWeb : public rgw::io::RestfulClient,
     return env;
   }
 
-  RGWCivetWeb(mg_connection *_conn, int _port);
+  RGWCivetWeb(mg_connection *_conn);
 };
 
 #endif

src/rgw/rgw_civetweb_frontend.cc

@@ -22,7 +22,7 @@ int RGWCivetWebFrontend::process(struct mg_connection*  const conn)
   /* Hold a read lock over access to env.store for reconfiguration. */
   RWLock::RLocker lock(env.mutex);
 
-  RGWCivetWeb cw_client(conn, env.port);
+  RGWCivetWeb cw_client(conn);
   auto real_client_io = rgw::io::add_reordering(
                           rgw::io::add_buffering(
                             rgw::io::add_chunking(
@@ -45,14 +45,17 @@ int RGWCivetWebFrontend::process(struct mg_connection*  const conn)
 int RGWCivetWebFrontend::run()
 {
   auto& conf_map = conf->get_config_map();
+  string port_str;
 
   set_conf_default(conf_map, "num_threads",
                    std::to_string(g_conf->rgw_thread_pool_size));
   set_conf_default(conf_map, "decode_url", "no");
   set_conf_default(conf_map, "enable_keep_alive", "yes");
-  conf_map["listening_ports"] = conf->get_val("port", "80");
   set_conf_default(conf_map, "validate_http_method", "no");
   set_conf_default(conf_map, "canonicalize_url_path", "no");
+  conf->get_val("port", "80", &port_str);
+  std::replace(port_str.begin(), port_str.end(), '+', ',');
+  conf_map["listening_ports"] = port_str;
 
   /* Set run_as_user. This will cause civetweb to invoke setuid() and setgid()
    * based on pw_uid and pw_gid obtained from pw_name. */

src/rgw/rgw_main.cc

@@ -187,7 +187,6 @@ static void reloader_handler(int signum)
   sighup_handler(signum);
 }
 
-
 /*
  * start up the RADOS connection and then handle HTTP messages as they come in
  */
@@ -470,12 +469,10 @@ int main(int argc, const char **argv)
 
       fe = new RGWFCGXFrontend(fcgi_pe, config);
     } else if (framework == "civetweb" || framework == "mongoose") {
-      int port;
-      config->get_val("port", 80, &port);
       std::string uri_prefix;
       config->get_val("prefix", "", &uri_prefix);
 
-      RGWProcessEnv env = { store, &rest, olog, port, uri_prefix };
+      RGWProcessEnv env = { store, &rest, olog, 0, uri_prefix };
 
       fe = new RGWCivetWebFrontend(env, config);
     } else if (framework == "loadgen") {

src/rgw/rgw_rest_s3.cc

@@ -3798,10 +3798,10 @@ int RGW_Auth_S3::authorize_v4(RGWRados *store, struct req_state *s, bool force_b
     }
     string token_value = string(t);
     if (force_boto2_compat && using_qs && (token == "host")) {
-      if (!port.empty() && port != "80" && port != "0") {
-        token_value = token_value + ":" + port;
-      } else if (!secure_port.empty() && secure_port != "443") {
+      if (!secure_port.empty() && secure_port != "443") {
         token_value = token_value + ":" + secure_port;
+      } else if (!port.empty() && port != "80" && port != "0") {
+        token_value = token_value + ":" + port;
       }
     }
     canonical_hdrs_map[token] = rgw_trim_whitespace(token_value);