Merge pull request #16161 from smithfarm/wip-19614-kraken

kraken: multisite: rest api fails to decode large period on 'period commit'

Reviewed-by: Casey Bodley <cbodley@redhat.com>

src/common/config_opts.h

@@ -1361,6 +1361,7 @@ OPTION(rgw_max_chunk_size, OPT_INT, 4 * 1024 * 1024)
 OPTION(rgw_put_obj_min_window_size, OPT_INT, 16 * 1024 * 1024)
 OPTION(rgw_put_obj_max_window_size, OPT_INT, 64 * 1024 * 1024)
 OPTION(rgw_max_put_size, OPT_U64, 5ULL*1024*1024*1024)
+OPTION(rgw_max_put_param_size, OPT_U64, 1 * 1024 * 1024) // max input size for PUT requests accepting json/xml params
 
 /**
  * override max bucket index shards in zone configuration (if not zero)

src/rgw/rgw_admin.cc

@@ -3067,6 +3067,9 @@ int main(int argc, char **argv)
 	  cerr << "realm.rename failed: " << cpp_strerror(-ret) << std::endl;
 	  return -ret;
 	}
+        cout << "Realm name updated. Note that this change only applies to "
+            "the current cluster, so this command must be run separately "
+            "on each of the realm's other clusters." << std::endl;
       }
       break;
     case OPT_REALM_SET:

src/rgw/rgw_op.cc

@@ -435,6 +435,12 @@ int rgw_build_bucket_policies(RGWRados* store, struct req_state* s)
     if (!r) {
       if (!zonegroup.endpoints.empty()) {
 	s->zonegroup_endpoint = zonegroup.endpoints.front();
+      } else {
+        // use zonegroup's master zone endpoints
+        auto z = zonegroup.zones.find(zonegroup.master_zone);
+        if (z != zonegroup.zones.end() && !z->second.endpoints.empty()) {
+          s->zonegroup_endpoint = z->second.endpoints.front();
+        }
       }
       s->zonegroup_name = zonegroup.get_name();
     }

src/rgw/rgw_op.h

@@ -1065,7 +1065,7 @@ class RGWGetACLs : public RGWOp {
 
 class RGWPutACLs : public RGWOp {
 protected:
-  size_t len;
+  int len;
   char *data;
   ACLOwner owner;
 
@@ -1108,7 +1108,7 @@ class RGWGetLC : public RGWOp {
 
 class RGWPutLC : public RGWOp {
 protected:
-  size_t len;
+  int len;
   char *data;
   string cookie;
 
@@ -1524,7 +1524,7 @@ class RGWGetHealthCheck : public RGWOp {
 class RGWDeleteMultiObj : public RGWOp {
 protected:
   int max_to_delete;
-  size_t len;
+  int len;
   char *data;
   rgw_bucket bucket;
   bool quiet;

src/rgw/rgw_rados.h

@@ -3186,7 +3186,7 @@ class RGWRados
   }
 
   bool need_to_log_metadata() {
-    return get_zone().log_meta;
+    return is_meta_master() && get_zone().log_meta;
   }
 
   librados::Rados* get_rados_handle();

src/rgw/rgw_rest.cc

@@ -776,7 +776,7 @@ void abort_early(struct req_state *s, RGWOp *op, int err_no,
       if (!s->redirect.empty()) {
         dest_uri = s->redirect;
       } else if (!s->zonegroup_endpoint.empty()) {
-        string dest_uri = s->zonegroup_endpoint;
+        dest_uri = s->zonegroup_endpoint;
         /*
          * reqest_uri is always start with slash, so we need to remove
          * the unnecessary slash at the end of dest_uri.
@@ -1254,55 +1254,19 @@ int RGWPostObj_ObjStore::verify_params()
 
 int RGWPutACLs_ObjStore::get_params()
 {
-  size_t cl = 0;
-  if (s->length)
-    cl = atoll(s->length);
-  if (cl) {
-    data = (char *)malloc(cl + 1);
-    if (!data) {
-       op_ret = -ENOMEM;
-       return op_ret;
-    }
-    const auto read_len = recv_body(s, data, cl);
-    if (read_len < 0) {
-      return read_len;
-    } else {
-      len = read_len;
-    }
-    data[len] = '\0';
-  } else {
-    len = 0;
-  }
-
+  const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+  op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false);
   return op_ret;
 }
 
 int RGWPutLC_ObjStore::get_params()
 {
-  size_t cl = 0;
-  if (s->length)
-    cl = atoll(s->length);
-  if (cl) {
-    data = (char *)malloc(cl + 1);
-    if (!data) {
-       op_ret = -ENOMEM;
-       return op_ret;
-    }
-    const auto read_len = recv_body(s, data, cl);
-    if (read_len < 0) {
-      return read_len;
-    } else {
-      len = read_len;
-    }
-    data[len] = '\0';
-  } else {
-    len = 0;
-  }
-
+  const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+  op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false);
   return op_ret;
 }
 
-static int read_all_chunked_input(req_state *s, char **pdata, int *plen, int max_read)
+static int read_all_chunked_input(req_state *s, char **pdata, int *plen, const uint64_t max_read)
 {
 #define READ_CHUNK 4096
 #define MAX_READ_CHUNK (128 * 1024)
@@ -1326,7 +1290,7 @@ static int read_all_chunked_input(req_state *s, char **pdata, int *plen, int max
       if (need_to_read < MAX_READ_CHUNK)
 	need_to_read *= 2;
 
-      if (total > max_read) {
+      if ((unsigned)total > max_read) {
 	free(data);
 	return -ERANGE;
       }
@@ -1352,14 +1316,17 @@ static int read_all_chunked_input(req_state *s, char **pdata, int *plen, int max
 }
 
 int rgw_rest_read_all_input(struct req_state *s, char **pdata, int *plen,
-			    int max_len)
+			    const uint64_t max_len, const bool allow_chunked)
 {
   size_t cl = 0;
   int len = 0;
   char *data = NULL;
 
   if (s->length)
     cl = atoll(s->length);
+  else if (!allow_chunked)
+    return -ERR_LENGTH_REQUIRED;
+
   if (cl) {
     if (cl > (size_t)max_len) {
       return -ERANGE;
@@ -1374,7 +1341,7 @@ int rgw_rest_read_all_input(struct req_state *s, char **pdata, int *plen,
       return len;
     }
     data[len] = '\0';
-  } else if (!s->length) {
+  } else if (allow_chunked && !s->length) {
     const char *encoding = s->info.env->get("HTTP_TRANSFER_ENCODING");
     if (!encoding || strcmp(encoding, "chunked") != 0)
       return -ERR_LENGTH_REQUIRED;
@@ -1462,28 +1429,8 @@ int RGWDeleteMultiObj_ObjStore::get_params()
   // everything is probably fine, set the bucket
   bucket = s->bucket;
 
-  size_t cl = 0;
-
-  if (s->length)
-    cl = atoll(s->length);
-  if (cl) {
-    data = (char *)malloc(cl + 1);
-    if (!data) {
-      op_ret = -ENOMEM;
-      return op_ret;
-    }
-    const auto read_len = recv_body(s, data, cl);
-    if (read_len < 0) {
-      op_ret = read_len;
-      return op_ret;
-    } else {
-      len = read_len;
-    }
-    data[len] = '\0';
-  } else {
-    return -EINVAL;
-  }
-
+  const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+  op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false);
   return op_ret;
 }
 

src/rgw/rgw_rest.h

@@ -29,11 +29,11 @@ extern void rgw_flush_formatter(struct req_state *s,
 				ceph::Formatter *formatter);
 
 extern int rgw_rest_read_all_input(struct req_state *s, char **data, int *plen,
-				   int max_len);
+				   uint64_t max_len, bool allow_chunked=true);
 
 template <class T>
 int rgw_rest_get_json_input(CephContext *cct, req_state *s, T& out,
-			    int max_len, bool *empty)
+			    uint64_t max_len, bool *empty)
 {
   int rv, data_len;
   char *data;
@@ -72,7 +72,7 @@ int rgw_rest_get_json_input(CephContext *cct, req_state *s, T& out,
 }
 
 template <class T>
-int rgw_rest_get_json_input_keep_data(CephContext *cct, req_state *s, T& out, int max_len, char **pdata, int *len)
+int rgw_rest_get_json_input_keep_data(CephContext *cct, req_state *s, T& out, uint64_t max_len, char **pdata, int *len)
 {
   int rv, data_len;
   char *data;

src/rgw/rgw_rest_realm.cc

@@ -84,9 +84,9 @@ void RGWOp_Period_Post::execute()
   period.init(cct, store, false);
 
   // decode the period from input
-#define PERIOD_MAX_LEN 4096
+  const auto max_size = cct->_conf->rgw_max_put_param_size;
   bool empty;
-  http_ret = rgw_rest_get_json_input(cct, s, period, PERIOD_MAX_LEN, &empty);
+  http_ret = rgw_rest_get_json_input(cct, s, period, max_size, &empty);
   if (http_ret < 0) {
     lderr(cct) << "failed to decode period" << dendl;
     return;

src/rgw/rgw_rest_s3.cc

@@ -738,12 +738,10 @@ class RGWSetBucketVersioningParser : public RGWXMLParser
 
 int RGWSetBucketVersioning_ObjStore_S3::get_params()
 {
-#define GET_BUCKET_VERSIONING_BUF_MAX (128 * 1024)
-
   char *data = nullptr;
   int len = 0;
   int r =
-    rgw_rest_read_all_input(s, &data, &len, GET_BUCKET_VERSIONING_BUF_MAX);
+    rgw_rest_read_all_input(s, &data, &len, s->cct->_conf->rgw_max_put_param_size, false);
   if (r < 0) {
     return r;
   }
@@ -791,11 +789,11 @@ void RGWSetBucketVersioning_ObjStore_S3::send_response()
 
 int RGWSetBucketWebsite_ObjStore_S3::get_params()
 {
-  static constexpr uint32_t GET_BUCKET_WEBSITE_BUF_MAX = (128 * 1024);
-
   char *data = nullptr;
   int len = 0;
-  int r = rgw_rest_read_all_input(s, &data, &len, GET_BUCKET_WEBSITE_BUF_MAX);
+  const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+  int r = rgw_rest_read_all_input(s, &data, &len, max_size, false);
+
   if (r < 0) {
     return r;
   }
@@ -964,8 +962,10 @@ int RGWCreateBucket_ObjStore_S3::get_params()
 
   int len = 0;
   char *data = nullptr;
-#define CREATE_BUCKET_MAX_REQ_LEN (512 * 1024) /* this is way more than enough */
-  op_ret = rgw_rest_read_all_input(s, &data, &len, CREATE_BUCKET_MAX_REQ_LEN);
+
+  const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+  op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false);
+
   if ((op_ret < 0) && (op_ret != -ERR_LENGTH_REQUIRED))
     return op_ret;
 
@@ -2433,53 +2433,38 @@ void RGWGetCORS_ObjStore_S3::send_response()
 int RGWPutCORS_ObjStore_S3::get_params()
 {
   int r;
-  char *data = NULL;
+  char *data = nullptr;
   int len = 0;
-  size_t cl = 0;
   RGWCORSXMLParser_S3 parser(s->cct);
   RGWCORSConfiguration_S3 *cors_config;
 
-  if (s->length)
-    cl = atoll(s->length);
-  if (cl) {
-    data = (char *)malloc(cl + 1);
-    if (!data) {
-       r = -ENOMEM;
-       goto done_err;
-    }
-    len = recv_body(s, data, cl);
-    if (len < 0) {
-      r = len;
-      goto done_err;
-    }
-    data[len] = '\0';
-  } else {
-    len = 0;
+  const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+  r = rgw_rest_read_all_input(s, &data, &len, max_size, false);
+  if (r < 0) {
+    return r;
   }
 
+  auto data_deleter = std::unique_ptr<char, decltype(free)*>{data, free};
+
   if (s->aws4_auth_needs_complete) {
-    int ret_auth = do_aws4_auth_completion();
-    if (ret_auth < 0) {
-      r = ret_auth;
-      goto done_err;
+    r = do_aws4_auth_completion();
+    if (r < 0) {
+      return r;
     }
   }
 
   if (!parser.init()) {
-    r = -EINVAL;
-    goto done_err;
+    return -EINVAL;
   }
 
   if (!data || !parser.parse(data, len, 1)) {
-    r = -EINVAL;
-    goto done_err;
+    return -EINVAL;
   }
   cors_config =
     static_cast<RGWCORSConfiguration_S3 *>(parser.find_first(
 					     "CORSConfiguration"));
   if (!cors_config) {
-    r = -EINVAL;
-    goto done_err;
+    return -EINVAL;
   }
 
   if (s->cct->_conf->subsys.should_gather(ceph_subsys_rgw, 15)) {
@@ -2490,11 +2475,7 @@ int RGWPutCORS_ObjStore_S3::get_params()
 
   cors_config->encode(cors_bl);
 
-  free(data);
   return 0;
-done_err:
-  free(data);
-  return r;
 }
 
 void RGWPutCORS_ObjStore_S3::send_response()
@@ -2588,10 +2569,11 @@ class RGWSetRequestPaymentParser : public RGWXMLParser
 
 int RGWSetRequestPayment_ObjStore_S3::get_params()
 {
-#define GET_REQUEST_PAYMENT_BUF_MAX (128 * 1024)
   char *data;
   int len = 0;
-  int r = rgw_rest_read_all_input(s, &data, &len, GET_REQUEST_PAYMENT_BUF_MAX);
+  const auto max_size = s->cct->_conf->rgw_max_put_param_size;
+  int r = rgw_rest_read_all_input(s, &data, &len, max_size, false);
+
   if (r < 0) {
     return r;
   }