rgw: initial multi-pool support for atomic upload

Fixes: http://tracker.ceph.com/issues/22565
Signed-off-by: Jeegn Chen <jeegnchen@gmail.com>

src/rgw/rgw_admin.cc

@@ -269,8 +269,16 @@ void usage()
   cout << "   --index-pool=<pool>       placement target index pool\n";
   cout << "   --data-pool=<pool>        placement target data pool\n";
   cout << "   --data-extra-pool=<pool>  placement target data extra (non-ec) pool\n";
+  cout << "   --current-tail-pool=<pool>\n";
+  cout << "                             placement target tail pool\n";
+  cout << "   --data-tail-pools=<pool>[,...]\n";
+  cout << "                             placement candidate tail pools\n";
+  cout << "   --new-tail-pool=<pool>\n";
+  cout << "                             pool to be added to existing placement candidate tail pool list\n";
   cout << "   --placement-index-type=<type>\n";
   cout << "                             placement target index type (normal, indexless, or #id)\n";
+  cout << "   --placement-data-layout-type=<type>\n";
+  cout << "                             placement target data layout type (normal, splitted, or #id)\n";
   cout << "   --compression=<type>      placement target compression type (plugin name or empty/none)\n";
   cout << "   --tier-type=<type>        zone tier type\n";
   cout << "   --tier-config=<k>=<v>[,...]\n";
@@ -2501,8 +2509,13 @@ int main(int argc, const char **argv)
   boost::optional<string> index_pool;
   boost::optional<string> data_pool;
   boost::optional<string> data_extra_pool;
+  boost::optional<string> current_tail_pool;
+  boost::optional<string> new_tail_pool;
+  boost::optional< list<string> > tail_pools;
   RGWBucketIndexType placement_index_type = RGWBIType_Normal;
+  RGWBucketDataLayoutType placement_data_layout_type = RGWDLType_SinglePool;
   bool index_type_specified = false;
+  bool data_layout_type_specified = false;
 
   boost::optional<std::string> compression_type;
 
@@ -2832,6 +2845,27 @@ int main(int argc, const char **argv)
       perm_policy_doc = val;
     } else if (ceph_argparse_witharg(args, i, &val, "--path-prefix", (char*)NULL)) {
       path_prefix = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--placement-data-layout-type", (char*)NULL)) {
+      if (val == "single-pool") {
+        placement_data_layout_type= RGWDLType_SinglePool;
+      } else if (val == "split-pool") {
+        placement_data_layout_type = RGWDLType_SplitPool;
+      } else {
+        placement_data_layout_type = (RGWBucketDataLayoutType)strict_strtol(val.c_str(), 10, &err);
+        if (!err.empty()) {
+          cerr << "ERROR: failed to parse data layout type: " << err << std::endl;
+          return EINVAL;
+        }
+      }
+      data_layout_type_specified = true;
+    } else if (ceph_argparse_witharg(args, i, &val, "--current-tail-pool", (char*)NULL)) {
+      current_tail_pool = val;
+    } else if (ceph_argparse_witharg(args, i, &val, "--data-tail-pools", (char*)NULL)) {
+      list<string> tmp_tail_pools;
+      get_str_list(val, tmp_tail_pools);
+			tail_pools = tmp_tail_pools;
+    } else if (ceph_argparse_witharg(args, i, &val, "--new-tail-pool", (char*)NULL)) {
+      new_tail_pool = val;
     } else if (strncmp(*i, "-", 1) == 0) {
       cerr << "ERROR: invalid flag " << *i << std::endl;
       return EINVAL;
@@ -4323,6 +4357,36 @@ int main(int argc, const char **argv)
           if (compression_type) {
             info.compression_type = *compression_type;
           }
+          if (data_layout_type_specified && current_tail_pool && tail_pools) {
+            if (placement_data_layout_type != RGWDLType_SplitPool) {
+              cerr << "ERROR: --placement-data-layout-type should be splitted to make "
+                "--current-tail-pool and --data-tail-pools valid." << std::endl;
+              return EINVAL;
+            }
+            if (! tail_pools ||
+                std::find(tail_pools->begin(),
+                          tail_pools->end(),
+                          *current_tail_pool) == tail_pools->end()) {
+              cerr << "ERROR: the value specified by --current-tail-pool should be "
+                   << "in the list specified by --data-tail-pools valid"
+                   << std::endl;
+              return EINVAL;
+            }
+            info.data_layout_type = placement_data_layout_type;
+            info.current_tail_pool = *current_tail_pool;
+            info.data_tail_pools.clear();
+            for (auto &p : *tail_pools) {
+              info.data_tail_pools.push_back(rgw_pool(p));
+            }
+          } else if ((data_layout_type_specified &&
+              placement_data_layout_type==RGWDLType_SplitPool) ||
+              current_tail_pool ||
+              tail_pools) {
+            cerr << "ERROR: --placement-data-layout-type, --current-tail-pool or "
+                 << "--data-tail-pools is missing."
+                 << std::endl;
+            return EINVAL;
+					}
 
           ret = check_pool_support_omap(info.get_data_extra_pool());
           if (ret < 0) {
@@ -4353,6 +4417,31 @@ int main(int argc, const char **argv)
           if (compression_type) {
             info.compression_type = *compression_type;
           }
+          if (data_layout_type_specified) {
+            info.data_layout_type = placement_data_layout_type;
+          }
+          if (current_tail_pool) {
+            info.current_tail_pool = *current_tail_pool;
+          }
+          if (tail_pools) {
+            for (auto &p : *tail_pools) {
+              info.data_tail_pools.push_back(rgw_pool(p));
+            }
+          }
+          if (new_tail_pool) {
+            info.data_tail_pools.push_back(rgw_pool(*new_tail_pool));
+          }
+          if (info.data_layout_type == RGWDLType_SplitPool) {
+            if (info.current_tail_pool.empty() || info.data_tail_pools.empty() ||
+                std::find(info.data_tail_pools.begin(),
+                        info.data_tail_pools.end(),
+                        info.current_tail_pool) == info.data_tail_pools.end()) {
+              cerr << "ERROR: --current-tail-pool or --data-tail-pools " 
+                   << "is improper for splited data layout"
+                   << std::endl;
+              return EINVAL;
+            }
+          }
 
           ret = check_pool_support_omap(info.get_data_extra_pool());
           if (ret < 0) {

src/rgw/rgw_common.h

@@ -1197,6 +1197,24 @@ inline ostream& operator<<(ostream& out, const RGWBucketIndexType &index_type)
   }
 }
 
+enum RGWBucketDataLayoutType {
+  RGWDLType_SinglePool = 0,
+  RGWDLType_SplitPool = 1,
+};
+
+inline ostream& operator<<(ostream& out, const RGWBucketDataLayoutType &data_layout_type) 
+{
+  switch (data_layout_type) {
+    case RGWDLType_SinglePool:
+      return out << "SinglePool";
+    case RGWDLType_SplitPool:
+      return out << "SplitPool";
+    default:
+      return out << "Unknown";
+  }
+}
+
+
 struct RGWBucketInfo
 {
   enum BIShardsHashType {
@@ -2111,6 +2129,39 @@ struct rgw_cache_entry_info {
   rgw_cache_entry_info() : gen(0) {}
 };
 
+struct rgw_data_placement_volatile_config {
+  RGWBucketDataLayoutType data_layout_type;
+  rgw_pool tail_data_pool;
+
+  rgw_data_placement_volatile_config(): data_layout_type(RGWDLType_SinglePool) {}
+  rgw_data_placement_volatile_config(const rgw_data_placement_volatile_config&) = default;
+  rgw_data_placement_volatile_config(rgw_data_placement_volatile_config&&) = default;
+
+  rgw_data_placement_volatile_config(RGWBucketDataLayoutType data_layout_type,
+                                           const rgw_pool& tail_data_pool)
+     : data_layout_type(data_layout_type), tail_data_pool(tail_data_pool) {
+  }
+
+  rgw_data_placement_volatile_config&
+  operator=(const rgw_data_placement_volatile_config&) = default;
+
+  RGWBucketDataLayoutType get_data_layout_type() const {
+    return data_layout_type;
+  }
+
+  const rgw_pool& get_tail_data_pool() const {
+    return tail_data_pool;
+  }
+
+  bool empty() const {
+    return (data_layout_type == RGWDLType_SinglePool);
+  }
+
+  void dump(Formatter *f) const;
+  void decode_json(JSONObj *obj);
+};
+
+
 inline ostream& operator<<(ostream& out, const rgw_obj &o) {
   return out << o.bucket.name << ":" << o.get_oid();
 }

src/rgw/rgw_dencoder.cc

@@ -173,9 +173,12 @@ void RGWObjManifest::get_implicit_location(uint64_t cur_part_id, uint64_t cur_st
     oid = *override_prefix;
   }
 
+  location->set_head(false);
+  location->set_data_placement_volatile_config(data_placement_vc);
   if (!cur_part_id) {
     if (ofs < max_head_size) {
       location->set_placement_rule(head_placement_rule);
+      location->set_head(true);
       *location = obj;
       return;
     } else {

src/rgw/rgw_json_enc.cc

@@ -107,6 +107,7 @@ void RGWObjManifest::dump(Formatter *f) const
   ::encode_json("rules", rules, f);
   ::encode_json("tail_instance", tail_instance, f);
   ::encode_json("tail_placement", tail_placement, f);
+  ::encode_json("data_placement_vc", data_placement_vc, f);
 }
 
 void rgw_log_entry::dump(Formatter *f) const
@@ -583,6 +584,19 @@ void rgw_data_placement_target::decode_json(JSONObj *obj) {
   JSONDecoder::decode_json("index_pool", index_pool, obj);
 }
 
+void rgw_data_placement_volatile_config::dump(Formatter *f) const
+{
+  encode_json("data_layout_type", (uint32_t)data_layout_type, f);
+  encode_json("tail_data_pool", tail_data_pool, f);
+}
+
+void rgw_data_placement_volatile_config::decode_json(JSONObj *obj) {
+  uint32_t it(0);
+  JSONDecoder::decode_json("data_layout_type", it, obj);
+  data_layout_type = (RGWBucketDataLayoutType)it;
+  JSONDecoder::decode_json("tail_data_pool", tail_data_pool, obj);
+}
+
 void rgw_bucket::dump(Formatter *f) const
 {
   encode_json("name", name, f);
@@ -930,8 +944,11 @@ void RGWZonePlacementInfo::dump(Formatter *f) const
   encode_json("index_pool", index_pool, f);
   encode_json("data_pool", data_pool, f);
   encode_json("data_extra_pool", data_extra_pool, f);
+  encode_json("current_tail_pool", current_tail_pool, f);
+  encode_json("data_tail_pools", data_tail_pools, f);
   encode_json("index_type", (uint32_t)index_type, f);
   encode_json("compression", compression_type, f);
+  encode_json("data_layout_type", (uint32_t)data_layout_type, f);
 }
 
 void RGWZonePlacementInfo::decode_json(JSONObj *obj)

src/rgw/rgw_op.cc

@@ -3038,7 +3038,12 @@ int RGWPutObjProcessor_Multipart::prepare(RGWRados *store, string *oid_rand)
 
   manifest.set_multipart_part_rule(store->ctx()->_conf->rgw_obj_stripe_size, num);
 
-  int r = manifest_gen.create_begin(store->ctx(), &manifest, s->bucket_info.placement_rule, bucket, target_obj);
+  int r = manifest_gen.create_begin(store->ctx(),
+                                    &manifest,
+                                    s->bucket_info.placement_rule,
+                                    data_placement_vc,
+                                    bucket,
+                                    target_obj);
   if (r < 0) {
     return r;
   }
@@ -3141,6 +3146,9 @@ RGWPutObjProcessor *RGWPutObj::select_processor(RGWObjectCtx& obj_ctx, bool *is_
     processor = new RGWPutObjProcessor_Atomic(obj_ctx, s->bucket_info, s->bucket, s->object.name, part_size, s->req_id, s->bucket_info.versioning_enabled());
     (static_cast<RGWPutObjProcessor_Atomic *>(processor))->set_olh_epoch(olh_epoch);
     (static_cast<RGWPutObjProcessor_Atomic *>(processor))->set_version_id(version_id);
+    rgw_data_placement_volatile_config dpvc;
+    store->get_zone_params().get_data_placement_volatile_config(s->bucket_info.placement_rule, &dpvc);
+    (static_cast<RGWPutObjProcessor_Atomic *>(processor))->set_data_placement_volatile_config(dpvc);
   } else {
     processor = new RGWPutObjProcessor_Multipart(obj_ctx, s->bucket_info, part_size, s);
   }