Merge pull request #3 from Leeshine/wip-rgw-cloud-sync

rgw: aws sync module fix Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
yehudasa · Nov 22, 2017 · df516b3 · df516b3
2 parents bfb64bb + e58a791
commit df516b3
Show file tree

Hide file tree

Showing 6 changed files with 44 additions and 18 deletions.
diff --git a/src/rgw/rgw_coroutine.cc b/src/rgw/rgw_coroutine.cc
@@ -332,7 +332,7 @@ int RGWCoroutinesStack::unwind(int retcode)
   rgw_spawned_stacks *src_spawned = &(*pos)->spawned;
 
   if (pos == ops.begin()) {
-    ldout(cct, 0) << "stack " << (void *)this << " end" << dendl;
+    ldout(cct, 15) << "stack " << (void *)this << " end" << dendl;
     spawned.inherit(src_spawned);
     ops.clear();
     pos = ops.end();

diff --git a/src/rgw/rgw_rest_conn.cc b/src/rgw/rgw_rest_conn.cc
@@ -81,17 +81,10 @@ string RGWRESTConn::get_url()
   return endpoint;
 }
 
-static void populate_params(param_vec_t& params, const rgw_user *uid, const string& zonegroup)
+void RGWRESTConn::populate_params(param_vec_t& params, const rgw_user *uid, const string& zonegroup)
 {
-  if (uid) {
-    string uid_str = uid->to_str();
-    if (!uid->empty()) {
-      params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "uid", uid_str));
-    }
-  }
-  if (!zonegroup.empty()) {
-    params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "zonegroup", zonegroup));
-  }
+  populate_uid(params, uid);
+  populate_zonegroup(params, zonegroup);
 }
 
 int RGWRESTConn::forward(const rgw_user& uid, req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl)
@@ -371,7 +364,7 @@ RGWRESTReadResource::RGWRESTReadResource(RGWRESTConn *_conn,
 
 void RGWRESTReadResource::init_common(param_vec_t *extra_headers)
 {
-  populate_params(params, nullptr, conn->get_self_zonegroup());
+  conn->populate_params(params, nullptr, conn->get_self_zonegroup());
 
   if (extra_headers) {
     headers.insert(extra_headers->begin(), extra_headers->end());
@@ -431,7 +424,7 @@ RGWRESTSendResource::RGWRESTSendResource(RGWRESTConn *_conn,
 
 void RGWRESTSendResource::init_common(param_vec_t *extra_headers)
 {
-  populate_params(params, nullptr, conn->get_self_zonegroup());
+  conn->populate_params(params, nullptr, conn->get_self_zonegroup());
 
   if (extra_headers) {
     headers.insert(extra_headers->begin(), extra_headers->end());

diff --git a/src/rgw/rgw_rest_conn.h b/src/rgw/rgw_rest_conn.h
@@ -90,6 +90,8 @@ class RGWRESTConn
   }
   size_t get_endpoint_count() const { return endpoints.size(); }
 
+  virtual void populate_params(param_vec_t& params, const rgw_user *uid, const string& zonegroup);
+
   /* sync request */
   int forward(const rgw_user& uid, req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl);
 
@@ -147,6 +149,37 @@ class RGWRESTConn
   int get_json_resource(const string& resource, param_vec_t *params, T& t);
   template <class T>
   int get_json_resource(const string& resource, const rgw_http_param_pair *pp, T& t);
+
+private:
+  void populate_zonegroup(param_vec_t& params, const string& zonegroup) {
+    if (!zonegroup.empty()) {
+      params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "zonegroup", zonegroup));
+    }
+  }
+  void populate_uid(param_vec_t& params, const rgw_user *uid) {
+    if (uid) {
+      string uid_str = uid->to_str();
+      if (!uid->empty()){
+        params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "uid", uid_str));
+      }
+    }
+  }
+};
+
+class S3RESTConn : public RGWRESTConn {
+
+public:
+
+  S3RESTConn(CephContext *_cct, RGWRados *store, const string& _remote_id, const list<string>& endpoints) :
+    RGWRESTConn(_cct, store, _remote_id, endpoints) {}
+
+  S3RESTConn(CephContext *_cct, RGWRados *store, const string& _remote_id, const list<string>& endpoints, RGWAccessKey _cred):
+    RGWRESTConn(_cct, store, _remote_id, endpoints, _cred) {}
+
+  void populate_params(param_vec_t& params, const rgw_user *uid, const string& zonegroup) override {
+    // do not populate any params in S3 REST Connection.
+    return;
+  }
 };
 
 

diff --git a/src/rgw/rgw_sync_module_aws.cc b/src/rgw/rgw_sync_module_aws.cc
@@ -914,7 +914,7 @@ class RGWAWSDataSyncModule: public RGWDataSyncModule {
 
   void init(RGWDataSyncEnv *sync_env, uint64_t instance_id) {
     instance.id = string("s3:") + instance.conf.s3_endpoint;
-    instance.conn.reset(new RGWRESTConn(cct,
+    instance.conn.reset(new S3RESTConn(cct,
                                     sync_env->store,
                                     instance.id,
                                     { instance.conf.s3_endpoint },

diff --git a/src/test/rgw/test_rgw_compression.cc b/src/test/rgw/test_rgw_compression.cc
@@ -4,7 +4,7 @@
 
 #include "rgw/rgw_compression.h"
 
-class ut_get_sink : public RGWGetDataCB {
+class ut_get_sink : public RGWGetObj_Filter {
   bufferlist sink;
 public:
   ut_get_sink() {}
@@ -30,7 +30,7 @@ class ut_get_sink : public RGWGetDataCB {
   }
 };
 
-class ut_get_sink_size : public RGWGetDataCB {
+class ut_get_sink_size : public RGWGetObj_Filter {
   size_t max_size = 0;
 public:
   ut_get_sink_size() {}
@@ -71,7 +71,7 @@ class ut_put_sink: public RGWPutObjDataProcessor
 };
 
 
-struct MockGetDataCB : public RGWGetDataCB {
+struct MockGetDataCB : public RGWGetObj_Filter {
   int handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) override {
     return 0;
   }

diff --git a/src/test/rgw/test_rgw_crypto.cc b/src/test/rgw/test_rgw_crypto.cc
@@ -27,7 +27,7 @@ using namespace std;
 std::unique_ptr<BlockCrypt> AES_256_CBC_create(CephContext* cct, const uint8_t* key, size_t len);
 
 
-class ut_get_sink : public RGWGetDataCB {
+class ut_get_sink : public RGWGetObj_Filter {
   std::stringstream sink;
 public:
   ut_get_sink() {}