Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

luminous: rgw: handle S3 version 2 pre-signed urls with meta-data #25901

Merged
merged 2 commits into from Feb 25, 2019
Merged

luminous: rgw: handle S3 version 2 pre-signed urls with meta-data #25901

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dd7d957
rgw: distinguish parse_v4_credentials
mattbenjamin Oct 19, 2018
966c6dc
rgw: handle S3 version 2 pre-signed urls with meta-data
mattbenjamin Oct 19, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
34 changes: 25 additions & 9 deletions src/rgw/rgw_auth_s3.cc
View file
Open in desktop
Expand Up @@ -116,6 +116,7 @@ void rgw_create_s3_canonical_header(
const char* const content_type,
const char* const date,
const std::map<std::string, std::string>& meta_map,
const std::map<std::string, std::string>& qs_map,
const char* const request_uri,
const std::map<std::string, std::string>& sub_resources,
std::string& dest_str)
Expand Down Expand Up @@ -143,6 +144,7 @@ void rgw_create_s3_canonical_header(
dest.append("\n");

dest.append(get_canon_amz_hdr(meta_map));
dest.append(get_canon_amz_hdr(qs_map));
dest.append(get_canon_resource(request_uri, sub_resources));

dest_str = dest;
Expand All @@ -152,6 +154,17 @@ static inline bool is_base64_for_content_md5(unsigned char c) {
return (isalnum(c) || isspace(c) || (c == '+') || (c == '/') || (c == '='));
}

static inline void get_v2_qs_map(const req_info& info,
std::map<std::string, std::string>& qs_map) {
const auto& params = const_cast<RGWHTTPArgs&>(info.args).get_params();
for (const auto& elt : params) {
std::string k = boost::algorithm::to_lower_copy(elt.first);
if (k.find("x-amz-meta-") == /* offset */ 0) {
add_amz_meta_header(qs_map, k, elt.second);
}
}
}

/*
* get the header authentication information required to
* compute a request's signature
Expand All @@ -175,7 +188,10 @@ bool rgw_create_s3_canonical_header(const req_info& info,
const char *content_type = info.env->get("CONTENT_TYPE");

std::string date;
std::map<std::string, std::string> qs_map;

if (qsr) {
get_v2_qs_map(info, qs_map); // handle qs metadata
date = info.args.get("Expires");
} else {
const char *str = info.env->get("HTTP_X_AMZ_DATE");
Expand Down Expand Up @@ -214,8 +230,8 @@ bool rgw_create_s3_canonical_header(const req_info& info,
}

rgw_create_s3_canonical_header(info.method, content_md5, content_type,
date.c_str(), meta_map, request_uri.c_str(),
sub_resources, dest);
date.c_str(), meta_map, qs_map,
request_uri.c_str(), sub_resources, dest);
return true;
}

Expand Down Expand Up @@ -412,13 +428,13 @@ static inline int parse_v4_auth_header(const req_info& info, /* in
return 0;
}

int parse_credentials(const req_info& info, /* in */
boost::string_view& access_key_id, /* out */
boost::string_view& credential_scope, /* out */
boost::string_view& signedheaders, /* out */
boost::string_view& signature, /* out */
boost::string_view& date, /* out */
bool& using_qs) /* out */
int parse_v4_credentials(const req_info& info, /* in */
boost::string_view& access_key_id, /* out */
boost::string_view& credential_scope, /* out */
boost::string_view& signedheaders, /* out */
boost::string_view& signature, /* out */
boost::string_view& date, /* out */
bool& using_qs) /* out */
{
const char* const http_auth = info.env->get("HTTP_AUTHORIZATION");
using_qs = http_auth == nullptr || http_auth[0] == '\0';
Expand Down
15 changes: 8 additions & 7 deletions src/rgw/rgw_auth_s3.h
View file
Open in desktop
Expand Up @@ -350,6 +350,7 @@ void rgw_create_s3_canonical_header(
const char *content_type,
const char *date,
const std::map<std::string, std::string>& meta_map,
const std::map<std::string, std::string>& qs_map,
const char *request_uri,
const std::map<std::string, std::string>& sub_resources,
std::string& dest_str);
Expand Down Expand Up @@ -381,13 +382,13 @@ static constexpr char AWS4_UNSIGNED_PAYLOAD_HASH[] = "UNSIGNED-PAYLOAD";
static constexpr char AWS4_STREAMING_PAYLOAD_HASH[] = \
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD";

int parse_credentials(const req_info& info, /* in */
boost::string_view& access_key_id, /* out */
boost::string_view& credential_scope, /* out */
boost::string_view& signedheaders, /* out */
boost::string_view& signature, /* out */
boost::string_view& date, /* out */
bool& using_qs); /* out */
int parse_v4_credentials(const req_info& info, /* in */
boost::string_view& access_key_id, /* out */
boost::string_view& credential_scope, /* out */
boost::string_view& signedheaders, /* out */
boost::string_view& signature, /* out */
boost::string_view& date, /* out */
bool& using_qs); /* out */

static inline std::string get_v4_canonical_uri(const req_info& info) {
/* The code should normalize according to RFC 3986 but S3 does NOT do path
Expand Down
3 changes: 0 additions & 3 deletions src/rgw/rgw_common.cc
View file
Open in desktop
Expand Up @@ -6,8 +6,6 @@
#include <algorithm>
#include <string>
#include <boost/tokenizer.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/utility/string_view.hpp>

#include "json_spirit/json_spirit.h"
#include "common/ceph_json.h"
Expand Down Expand Up @@ -404,7 +402,6 @@ struct str_len meta_prefixes[] = { STR_LEN_ENTRY("HTTP_X_AMZ"),
STR_LEN_ENTRY("HTTP_X_ACCOUNT"),
{NULL, 0} };


void req_info::init_meta_info(bool *found_bad_meta)
{
x_meta_map.clear();
Expand Down
20 changes: 20 additions & 0 deletions src/rgw/rgw_common.h
View file
Open in desktop
Expand Up @@ -18,6 +18,7 @@

#include <array>

#include <boost/algorithm/string.hpp>
#include <boost/utility/string_view.hpp>

#include "common/ceph_crypto.h"
Expand Down Expand Up @@ -2207,6 +2208,25 @@ static inline uint64_t rgw_rounded_objsize_kb(uint64_t bytes)
return ((bytes + 4095) & ~4095) / 1024;
}

/* implement combining step, S3 header canonicalization; k is a
* valid header and in lc form */
static inline void add_amz_meta_header(
std::map<std::string, std::string>& x_meta_map,
const std::string& k,
const std::string& v)
{
auto it = x_meta_map.find(k);
if (it != x_meta_map.end()) {
std::string old = it->second;
boost::algorithm::trim_right(old);
old.append(",");
old.append(v);
x_meta_map[k] = old;
} else {
x_meta_map[k] = v;
}
} /* add_amz_meta_header */

extern string rgw_string_unquote(const string& s);
extern void parse_csv_string(const string& ival, vector<string>& ovals);
extern int parse_key_value(string& in_str, string& key, string& val);
Expand Down
1 change: 1 addition & 0 deletions src/rgw/rgw_loadgen.cc
View file
Open in desktop
Expand Up @@ -29,6 +29,7 @@ int RGWLoadGenRequestEnv::sign(RGWAccessKey& access_key)
content_type.c_str(),
date_str.c_str(),
meta_map,
map<string, string>{},
uri.c_str(),
sub_resources,
canonical_header);
Expand Down
2 changes: 1 addition & 1 deletion src/rgw/rgw_rest_client.cc
View file
Open in desktop
Expand Up @@ -123,7 +123,7 @@ int RGWRESTSimpleRequest::execute(RGWAccessKey& key, const char *method, const c
map<string, string> meta_map;
map<string, string> sub_resources;
rgw_create_s3_canonical_header(method, NULL, NULL, date_str.c_str(),
meta_map, new_url.c_str(), sub_resources,
meta_map, meta_map, new_url.c_str(), sub_resources,
canonical_header);

string digest;
Expand Down
33 changes: 26 additions & 7 deletions src/rgw/rgw_rest_s3.cc
View file
Open in desktop
Expand Up @@ -1243,6 +1243,19 @@ void RGWDeleteBucket_ObjStore_S3::send_response()
}
}

static inline void map_qs_metadata(struct req_state* s)
{
/* merge S3 valid user metadata from the query-string into
* x_meta_map, which maps them to attributes */
const auto& params = const_cast<RGWHTTPArgs&>(s->info.args).get_params();
for (const auto& elt : params) {
std::string k = boost::algorithm::to_lower_copy(elt.first);
if (k.find("x-amz-meta-") == /* offset */ 0) {
add_amz_meta_header(s->info.x_meta_map, k, elt.second);
}
}
}

int RGWPutObj_ObjStore_S3::get_params()
{
if (!s->length)
Expand All @@ -1253,6 +1266,8 @@ int RGWPutObj_ObjStore_S3::get_params()
size_t pos;
int ret;

map_qs_metadata(s);

RGWAccessControlPolicy_S3 s3policy(s->cct);
ret = create_s3_policy(s, store, s3policy, s->owner);
if (ret < 0)
Expand Down Expand Up @@ -1547,6 +1562,8 @@ int RGWPostObj_ObjStore_S3::get_params()
return op_ret;
}

map_qs_metadata(s);

ldout(s->cct, 20) << "adding bucket to policy env: " << s->bucket.name
<< dendl;
env.add_var("bucket", s->bucket.name);
Expand Down Expand Up @@ -2539,6 +2556,8 @@ int RGWCompleteMultipart_ObjStore_S3::get_params()
return ret;
}

map_qs_metadata(s);

return do_aws4_auth_completion();
}

Expand Down Expand Up @@ -3736,13 +3755,13 @@ AWSGeneralAbstractor::get_auth_data_v4(const req_state* const s,
boost::string_view credential_scope;
boost::string_view client_signature;

int ret = rgw::auth::s3::parse_credentials(s->info,
access_key_id,
credential_scope,
signed_hdrs,
client_signature,
date,
using_qs);
int ret = rgw::auth::s3::parse_v4_credentials(s->info,
access_key_id,
credential_scope,
signed_hdrs,
client_signature,
date,
using_qs);
if (ret < 0) {
throw ret;
}
Expand Down