Skip to content

Commit

Permalink
rgw: make a configurable size for requests with xml params
Browse files Browse the repository at this point in the history 
We currently read all user input for a few apis accepting xml, avoid
doing this and error out early, most s3 apis limit to about 1000 xml
entries, make this configurable via a new config param,
`rgw_max_put_param_size` defaulting to 1MB. Also modified
`rgw_rest_read_all_input` with an additional param to not allow for
chunked uploads, though we error out in a better way by responding with
content length required rather than S3's 501 if chunked is set.

This also adds the same behavior in RGWPutCORS reusing
`rgw_rest_read_all_input`, and using a data_deleter to manage the char*

Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>

rgw: rgw_rest_s3: make PUT CORS accept only a configured max input

Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
  • Loading branch information
@theanalyst
theanalyst committed Mar 24, 2017
1 parent 72097c7 commit 35375b7
Show file tree
Hide file tree
Showing 5 changed files with 44 additions and 114 deletions.
1 change: 1 addition & 0 deletions src/common/config_opts.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1402,6 +1402,7 @@ OPTION(rgw_max_chunk_size, OPT_INT, 4 * 1024 * 1024)
OPTION(rgw_put_obj_min_window_size, OPT_INT, 16 * 1024 * 1024)
OPTION(rgw_put_obj_max_window_size, OPT_INT, 64 * 1024 * 1024)
OPTION(rgw_max_put_size, OPT_U64, 5ULL*1024*1024*1024)
OPTION(rgw_max_put_param_size, OPT_U64, 1 * 1024 * 1024) // max input size for PUT requests accepting json/xml params

/**
* override max bucket index shards in zone configuration (if not zero)
Expand Down
6 changes: 3 additions & 3 deletions src/rgw/rgw_op.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1065,7 +1065,7 @@ class RGWGetACLs : public RGWOp {

class RGWPutACLs : public RGWOp {
protected:
size_t len;
int len;
char *data;
ACLOwner owner;

Expand Down Expand Up @@ -1108,7 +1108,7 @@ class RGWGetLC : public RGWOp {

class RGWPutLC : public RGWOp {
protected:
size_t len;
int len;
char *data;
string cookie;

Expand Down Expand Up @@ -1524,7 +1524,7 @@ class RGWGetHealthCheck : public RGWOp {
class RGWDeleteMultiObj : public RGWOp {
protected:
int max_to_delete;
size_t len;
int len;
char *data;
rgw_bucket bucket;
bool quiet;
Expand Down
79 changes: 13 additions & 66 deletions src/rgw/rgw_rest.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1253,55 +1253,19 @@ int RGWPostObj_ObjStore::verify_params()

int RGWPutACLs_ObjStore::get_params()
{
size_t cl = 0;
if (s->length)
cl = atoll(s->length);
if (cl) {
data = (char *)malloc(cl + 1);
if (!data) {
op_ret = -ENOMEM;
return op_ret;
}
const auto read_len = recv_body(s, data, cl);
if (read_len < 0) {
return read_len;
} else {
len = read_len;
}
data[len] = '\0';
} else {
len = 0;
}

const auto max_size = s->cct->_conf->rgw_max_put_param_size;
op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false);
return op_ret;
}

int RGWPutLC_ObjStore::get_params()
{
size_t cl = 0;
if (s->length)
cl = atoll(s->length);
if (cl) {
data = (char *)malloc(cl + 1);
if (!data) {
op_ret = -ENOMEM;
return op_ret;
}
const auto read_len = recv_body(s, data, cl);
if (read_len < 0) {
return read_len;
} else {
len = read_len;
}
data[len] = '\0';
} else {
len = 0;
}

const auto max_size = s->cct->_conf->rgw_max_put_param_size;
op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false);
return op_ret;
}

static int read_all_chunked_input(req_state *s, char **pdata, int *plen, int max_read)
static int read_all_chunked_input(req_state *s, char **pdata, int *plen, const uint64_t max_read)
{
#define READ_CHUNK 4096
#define MAX_READ_CHUNK (128 * 1024)
Expand All @@ -1325,7 +1289,7 @@ static int read_all_chunked_input(req_state *s, char **pdata, int *plen, int max
if (need_to_read < MAX_READ_CHUNK)
need_to_read *= 2;

if (total > max_read) {
if ((unsigned)total > max_read) {
free(data);
return -ERANGE;
}
Expand All @@ -1351,14 +1315,17 @@ static int read_all_chunked_input(req_state *s, char **pdata, int *plen, int max
}

int rgw_rest_read_all_input(struct req_state *s, char **pdata, int *plen,
int max_len)
const uint64_t max_len, const bool allow_chunked)
{
size_t cl = 0;
int len = 0;
char *data = NULL;

if (s->length)
cl = atoll(s->length);
else if (!allow_chunked)
return -ERR_LENGTH_REQUIRED;

if (cl) {
if (cl > (size_t)max_len) {
return -ERANGE;
Expand All @@ -1373,7 +1340,7 @@ int rgw_rest_read_all_input(struct req_state *s, char **pdata, int *plen,
return len;
}
data[len] = '\0';
} else if (!s->length) {
} else if (allow_chunked && !s->length) {
const char *encoding = s->info.env->get("HTTP_TRANSFER_ENCODING");
if (!encoding || strcmp(encoding, "chunked") != 0)
return -ERR_LENGTH_REQUIRED;
Expand Down Expand Up @@ -1461,28 +1428,8 @@ int RGWDeleteMultiObj_ObjStore::get_params()
// everything is probably fine, set the bucket
bucket = s->bucket;

size_t cl = 0;

if (s->length)
cl = atoll(s->length);
if (cl) {
data = (char *)malloc(cl + 1);
if (!data) {
op_ret = -ENOMEM;
return op_ret;
}
const auto read_len = recv_body(s, data, cl);
if (read_len < 0) {
op_ret = read_len;
return op_ret;
} else {
len = read_len;
}
data[len] = '\0';
} else {
return -EINVAL;
}

const auto max_size = s->cct->_conf->rgw_max_put_param_size;
op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false);
return op_ret;
}

Expand Down
6 changes: 3 additions & 3 deletions src/rgw/rgw_rest.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@ extern void rgw_flush_formatter(struct req_state *s,
ceph::Formatter *formatter);

extern int rgw_rest_read_all_input(struct req_state *s, char **data, int *plen,
int max_len);
uint64_t max_len, bool allow_chunked=true);

template <class T>
int rgw_rest_get_json_input(CephContext *cct, req_state *s, T& out,
int max_len, bool *empty)
uint64_t max_len, bool *empty)
{
int rv, data_len;
char *data;
Expand Down Expand Up @@ -72,7 +72,7 @@ int rgw_rest_get_json_input(CephContext *cct, req_state *s, T& out,
}

template <class T>
int rgw_rest_get_json_input_keep_data(CephContext *cct, req_state *s, T& out, int max_len, char **pdata, int *len)
int rgw_rest_get_json_input_keep_data(CephContext *cct, req_state *s, T& out, uint64_t max_len, char **pdata, int *len)
{
int rv, data_len;
char *data;
Expand Down
66 changes: 24 additions & 42 deletions src/rgw/rgw_rest_s3.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -733,12 +733,10 @@ class RGWSetBucketVersioningParser : public RGWXMLParser

int RGWSetBucketVersioning_ObjStore_S3::get_params()
{
#define GET_BUCKET_VERSIONING_BUF_MAX (128 * 1024)

char *data = nullptr;
int len = 0;
int r =
rgw_rest_read_all_input(s, &data, &len, GET_BUCKET_VERSIONING_BUF_MAX);
rgw_rest_read_all_input(s, &data, &len, s->cct->_conf->rgw_max_put_param_size, false);
if (r < 0) {
return r;
}
Expand Down Expand Up @@ -786,11 +784,11 @@ void RGWSetBucketVersioning_ObjStore_S3::send_response()

int RGWSetBucketWebsite_ObjStore_S3::get_params()
{
static constexpr uint32_t GET_BUCKET_WEBSITE_BUF_MAX = (128 * 1024);

char *data = nullptr;
int len = 0;
int r = rgw_rest_read_all_input(s, &data, &len, GET_BUCKET_WEBSITE_BUF_MAX);
const auto max_size = s->cct->_conf->rgw_max_put_param_size;
int r = rgw_rest_read_all_input(s, &data, &len, max_size, false);

if (r < 0) {
return r;
}
Expand Down Expand Up @@ -959,8 +957,10 @@ int RGWCreateBucket_ObjStore_S3::get_params()

int len = 0;
char *data = nullptr;
#define CREATE_BUCKET_MAX_REQ_LEN (512 * 1024) /* this is way more than enough */
op_ret = rgw_rest_read_all_input(s, &data, &len, CREATE_BUCKET_MAX_REQ_LEN);

const auto max_size = s->cct->_conf->rgw_max_put_param_size;
op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false);

if ((op_ret < 0) && (op_ret != -ERR_LENGTH_REQUIRED))
return op_ret;

Expand Down Expand Up @@ -2425,53 +2425,38 @@ void RGWGetCORS_ObjStore_S3::send_response()
int RGWPutCORS_ObjStore_S3::get_params()
{
int r;
char *data = NULL;
char *data = nullptr;
int len = 0;
size_t cl = 0;
RGWCORSXMLParser_S3 parser(s->cct);
RGWCORSConfiguration_S3 *cors_config;

if (s->length)
cl = atoll(s->length);
if (cl) {
data = (char *)malloc(cl + 1);
if (!data) {
r = -ENOMEM;
goto done_err;
}
len = recv_body(s, data, cl);
if (len < 0) {
r = len;
goto done_err;
}
data[len] = '\0';
} else {
len = 0;
const auto max_size = s->cct->_conf->rgw_max_put_param_size;
r = rgw_rest_read_all_input(s, &data, &len, max_size, false);
if (r < 0) {
return r;
}

auto data_deleter = std::unique_ptr<char, decltype(free)*>{data, free};

if (s->aws4_auth_needs_complete) {
int ret_auth = do_aws4_auth_completion();
if (ret_auth < 0) {
r = ret_auth;
goto done_err;
r = do_aws4_auth_completion();
if (r < 0) {
return r;
}
}

if (!parser.init()) {
r = -EINVAL;
goto done_err;
return -EINVAL;
}

if (!data || !parser.parse(data, len, 1)) {
r = -EINVAL;
goto done_err;
return -EINVAL;
}
cors_config =
static_cast<RGWCORSConfiguration_S3 *>(parser.find_first(
"CORSConfiguration"));
if (!cors_config) {
r = -EINVAL;
goto done_err;
return -EINVAL;
}

if (s->cct->_conf->subsys.should_gather(ceph_subsys_rgw, 15)) {
Expand All @@ -2482,11 +2467,7 @@ int RGWPutCORS_ObjStore_S3::get_params()

cors_config->encode(cors_bl);

free(data);
return 0;
done_err:
free(data);
return r;
}

void RGWPutCORS_ObjStore_S3::send_response()
Expand Down Expand Up @@ -2580,10 +2561,11 @@ class RGWSetRequestPaymentParser : public RGWXMLParser

int RGWSetRequestPayment_ObjStore_S3::get_params()
{
#define GET_REQUEST_PAYMENT_BUF_MAX (128 * 1024)
char *data;
int len = 0;
int r = rgw_rest_read_all_input(s, &data, &len, GET_REQUEST_PAYMENT_BUF_MAX);
const auto max_size = s->cct->_conf->rgw_max_put_param_size;
int r = rgw_rest_read_all_input(s, &data, &len, max_size, false);

if (r < 0) {
return r;
}
Expand Down

0 comments on commit 35375b7

Please sign in to comment.