Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

rgw: aws sync module fix #3

Merged
merged 4 commits into from
Nov 22, 2017
Merged

rgw: aws sync module fix #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
45234f8
rgw: decrease the log level in RGWCoroutinesStack::unwind
Leeshine Nov 16, 2017
69a883a
rgw: implement S3RESTConn to send request to S3
Leeshine Nov 21, 2017
d1948de
rgw: add populate_zonegroup() and populate_uid() in RGWRESTConn
Leeshine Nov 21, 2017
e58a791
rgw: modify testcase in src/test/rgw
Leeshine Nov 21, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/rgw/rgw_coroutine.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,7 @@ int RGWCoroutinesStack::unwind(int retcode)
rgw_spawned_stacks *src_spawned = &(*pos)->spawned;

if (pos == ops.begin()) {
ldout(cct, 0) << "stack " << (void *)this << " end" << dendl;
ldout(cct, 15) << "stack " << (void *)this << " end" << dendl;
spawned.inherit(src_spawned);
ops.clear();
pos = ops.end();
Expand Down
17 changes: 5 additions & 12 deletions src/rgw/rgw_rest_conn.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,17 +81,10 @@ string RGWRESTConn::get_url()
return endpoint;
}

static void populate_params(param_vec_t& params, const rgw_user *uid, const string& zonegroup)
void RGWRESTConn::populate_params(param_vec_t& params, const rgw_user *uid, const string& zonegroup)
{
if (uid) {
string uid_str = uid->to_str();
if (!uid->empty()) {
params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "uid", uid_str));
}
}
if (!zonegroup.empty()) {
params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "zonegroup", zonegroup));
}
populate_uid(params, uid);
populate_zonegroup(params, zonegroup);
}

int RGWRESTConn::forward(const rgw_user& uid, req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl)
Expand Down Expand Up @@ -371,7 +364,7 @@ RGWRESTReadResource::RGWRESTReadResource(RGWRESTConn *_conn,

void RGWRESTReadResource::init_common(param_vec_t *extra_headers)
{
populate_params(params, nullptr, conn->get_self_zonegroup());
conn->populate_params(params, nullptr, conn->get_self_zonegroup());

if (extra_headers) {
headers.insert(extra_headers->begin(), extra_headers->end());
Expand Down Expand Up @@ -431,7 +424,7 @@ RGWRESTSendResource::RGWRESTSendResource(RGWRESTConn *_conn,

void RGWRESTSendResource::init_common(param_vec_t *extra_headers)
{
populate_params(params, nullptr, conn->get_self_zonegroup());
conn->populate_params(params, nullptr, conn->get_self_zonegroup());

if (extra_headers) {
headers.insert(extra_headers->begin(), extra_headers->end());
Expand Down
33 changes: 33 additions & 0 deletions src/rgw/rgw_rest_conn.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ class RGWRESTConn
}
size_t get_endpoint_count() const { return endpoints.size(); }

virtual void populate_params(param_vec_t& params, const rgw_user *uid, const string& zonegroup);
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Leeshine populate_params() is too generic, let's in addition have populate_zonegroup() and use that one instead.


/* sync request */
int forward(const rgw_user& uid, req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl);

Expand Down Expand Up @@ -147,6 +149,37 @@ class RGWRESTConn
int get_json_resource(const string& resource, param_vec_t *params, T& t);
template <class T>
int get_json_resource(const string& resource, const rgw_http_param_pair *pp, T& t);

private:
void populate_zonegroup(param_vec_t& params, const string& zonegroup) {
if (!zonegroup.empty()) {
params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "zonegroup", zonegroup));
}
}
void populate_uid(param_vec_t& params, const rgw_user *uid) {
if (uid) {
string uid_str = uid->to_str();
if (!uid->empty()){
params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "uid", uid_str));
}
}
}
};

class S3RESTConn : public RGWRESTConn {

public:

S3RESTConn(CephContext *_cct, RGWRados *store, const string& _remote_id, const list<string>& endpoints) :
RGWRESTConn(_cct, store, _remote_id, endpoints) {}

S3RESTConn(CephContext *_cct, RGWRados *store, const string& _remote_id, const list<string>& endpoints, RGWAccessKey _cred):
RGWRESTConn(_cct, store, _remote_id, endpoints, _cred) {}

void populate_params(param_vec_t& params, const rgw_user *uid, const string& zonegroup) override {
// do not populate any params in S3 REST Connection.
return;
}
};


Expand Down
2 changes: 1 addition & 1 deletion src/rgw/rgw_sync_module_aws.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -914,7 +914,7 @@ class RGWAWSDataSyncModule: public RGWDataSyncModule {

void init(RGWDataSyncEnv *sync_env, uint64_t instance_id) {
instance.id = string("s3:") + instance.conf.s3_endpoint;
instance.conn.reset(new RGWRESTConn(cct,
instance.conn.reset(new S3RESTConn(cct,
sync_env->store,
instance.id,
{ instance.conf.s3_endpoint },
Expand Down
6 changes: 3 additions & 3 deletions src/test/rgw/test_rgw_compression.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

#include "rgw/rgw_compression.h"

class ut_get_sink : public RGWGetDataCB {
class ut_get_sink : public RGWGetObj_Filter {
bufferlist sink;
public:
ut_get_sink() {}
Expand All @@ -30,7 +30,7 @@ class ut_get_sink : public RGWGetDataCB {
}
};

class ut_get_sink_size : public RGWGetDataCB {
class ut_get_sink_size : public RGWGetObj_Filter {
size_t max_size = 0;
public:
ut_get_sink_size() {}
Expand Down Expand Up @@ -71,7 +71,7 @@ class ut_put_sink: public RGWPutObjDataProcessor
};


struct MockGetDataCB : public RGWGetDataCB {
struct MockGetDataCB : public RGWGetObj_Filter {
int handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) override {
return 0;
}
Expand Down
2 changes: 1 addition & 1 deletion src/test/rgw/test_rgw_crypto.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ using namespace std;
std::unique_ptr<BlockCrypt> AES_256_CBC_create(CephContext* cct, const uint8_t* key, size_t len);


class ut_get_sink : public RGWGetDataCB {
class ut_get_sink : public RGWGetObj_Filter {
std::stringstream sink;
public:
ut_get_sink() {}
Expand Down