Merge branch 'wip-sharded-scan' of git://github.com/jcsp/ceph into gr…

…eg-fs-testing

#7034

Reviewed-by: Yan, Zheng <zyan@redhat.com>

Conflicts:
	src/tools/cephfs/DataScan.cc

src/CMakeLists.txt

@@ -1244,6 +1244,7 @@ add_custom_target(cephfs_testing DEPENDS
     vstart
     rados
     cephfs
+    cls_cephfs
     ceph-fuse
     ceph-dencoder
     cephfs-journal-tool

src/include/rados/librados.h

@@ -994,6 +994,8 @@ CEPH_RADOS_API int rados_object_list(rados_ioctx_t io,
     const rados_object_list_cursor start,
     const rados_object_list_cursor finish,
     const size_t result_size,
+    const char *filter_buf,
+    const size_t filter_buf_len,
     rados_object_list_item *results,
     rados_object_list_cursor *next);
 

src/include/rados/librados.hpp

@@ -818,6 +818,7 @@ namespace librados
     bool object_list_is_end(const ObjectCursor &oc);
     int object_list(const ObjectCursor &start, const ObjectCursor &finish,
                     const size_t result_count,
+                    const bufferlist &filter,
                     std::vector<ObjectItem> *result,
                     ObjectCursor *next);
     void object_list_slice(

src/librados/librados.cc

@@ -3678,6 +3678,8 @@ extern "C" int rados_object_list(rados_ioctx_t io,
     const rados_object_list_cursor start,
     const rados_object_list_cursor finish,
     const size_t result_item_count,
+    const char *filter_buf,
+    const size_t filter_buf_len,
     rados_object_list_item *result_items,
     rados_object_list_cursor *next)
 {
@@ -3691,13 +3693,19 @@ extern "C" int rados_object_list(rados_ioctx_t io,
   std::list<librados::ListObjectImpl> result;
   hobject_t next_hash;
 
+  bufferlist filter_bl;
+  if (filter_buf != nullptr) {
+    filter_bl.append(filter_buf, filter_buf_len);
+  }
+
   C_SaferCond cond;
   ctx->objecter->enumerate_objects(
       ctx->poolid,
       ctx->oloc.nspace,
       *((hobject_t*)start),
       *((hobject_t*)finish),
       result_item_count,
+      filter_bl,
       &result,
       &next_hash,
       &cond);
@@ -5223,6 +5231,7 @@ bool librados::IoCtx::object_list_is_end(const ObjectCursor &oc)
 int librados::IoCtx::object_list(const ObjectCursor &start,
                 const ObjectCursor &finish,
                 const size_t result_item_count,
+                const bufferlist &filter,
                 std::vector<ObjectItem> *result,
                 ObjectCursor *next)
 {
@@ -5239,6 +5248,7 @@ int librados::IoCtx::object_list(const ObjectCursor &start,
       *((hobject_t*)start.c_cursor),
       *((hobject_t*)finish.c_cursor),
       result_item_count,
+      filter,
       &obj_result,
       &next_hash,
       &cond);

src/osdc/Objecter.cc

@@ -4861,6 +4861,7 @@ void Objecter::enumerate_objects(
     const hobject_t &start,
     const hobject_t &end,
     const uint32_t max,
+    const bufferlist &filter_bl,
     std::list<librados::ListObjectImpl> *result, 
     hobject_t *next,
     Context *on_finish)
@@ -4909,11 +4910,8 @@ void Objecter::enumerate_objects(
   C_EnumerateReply *on_ack = new C_EnumerateReply(
       this, next, result, end, pool_id, on_finish);
 
-  // Construct pgls operation
-  bufferlist filter; // FIXME pass in?
-
   ObjectOperation op;
-  op.pg_nls(max, filter, start, 0);
+  op.pg_nls(max, filter_bl, start, 0);
 
   // Issue.  See you later in _enumerate_reply
   object_locator_t oloc(pool_id, ns);

src/osdc/Objecter.h

@@ -187,7 +187,7 @@ struct ObjectOperation {
     osd_op.op.pgls.start_epoch = start_epoch;
     ::encode(cookie, osd_op.indata);
   }
-  void add_pgls_filter(int op, uint64_t count, bufferlist& filter,
+  void add_pgls_filter(int op, uint64_t count, const bufferlist& filter,
 		       collection_list_handle_t cookie, epoch_t start_epoch) {
     OSDOp& osd_op = add_op(op);
     osd_op.op.pgls.count = count;
@@ -219,7 +219,7 @@ struct ObjectOperation {
     flags |= CEPH_OSD_FLAG_PGOP;
   }
 
-  void pg_nls(uint64_t count, bufferlist& filter,
+  void pg_nls(uint64_t count, const bufferlist& filter,
 	      collection_list_handle_t cookie, epoch_t start_epoch) {
     if (filter.length() == 0)
       add_pgls(CEPH_OSD_OP_PGNLS, count, cookie, start_epoch);
@@ -2579,6 +2579,7 @@ class Objecter : public md_config_obs_t, public Dispatcher {
     const hobject_t &start,
     const hobject_t &end,
     const uint32_t max,
+    const bufferlist &filter_bl,
     std::list<librados::ListObjectImpl> *result, 
     hobject_t *next,
     Context *on_finish);

src/test/librados/list.cc

@@ -693,7 +693,7 @@ TEST_F(LibRadosList, EnumerateObjects) {
     memset(results, 0, sizeof(rados_object_list_item) * 12);
     int r = rados_object_list(ioctx,
             c, rados_object_list_end(ioctx),
-            12, results, &c);
+            12, NULL, 0, results, &c);
     ASSERT_GE(r, 0);
     for (int i = 0; i < r; ++i) {
       std::string oid(results[i].oid, results[i].oid_length);
@@ -761,7 +761,7 @@ TEST_F(LibRadosList, EnumerateObjectsSplit) {
         memset(results, 0, sizeof(rados_object_list_item) * 12);
         int r = rados_object_list(ioctx,
                 c, shard_end,
-                12, results, &c);
+                12, NULL, 0, results, &c);
         ASSERT_GE(r, 0);
         for (int i = 0; i < r; ++i) {
           std::string oid(results[i].oid, results[i].oid_length);
@@ -806,7 +806,7 @@ TEST_F(LibRadosListPP, EnumerateObjectsPP) {
   while(!ioctx.object_list_is_end(c))
   {
     std::vector<ObjectItem> result;
-    int r = ioctx.object_list(c, end, 12, &result, &c);
+    int r = ioctx.object_list(c, end, 12, {}, &result, &c);
     ASSERT_GE(r, 0);
     ASSERT_EQ(r, (int)result.size());
     for (int i = 0; i < r; ++i) {
@@ -861,7 +861,7 @@ TEST_F(LibRadosListPP, EnumerateObjectsSplitPP) {
       while(c < shard_end)
       {
         std::vector<ObjectItem> result;
-        int r = ioctx.object_list(c, shard_end, 12, &result, &c);
+        int r = ioctx.object_list(c, shard_end, 12, {}, &result, &c);
         ASSERT_GE(r, 0);
 
         for (const auto & i : result) {
@@ -906,5 +906,53 @@ TEST_F(LibRadosListNP, ListObjectsError) {
   rados_shutdown(cluster);
 }
 
+TEST_F(LibRadosListPP, EnumerateObjectsFilterPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist obj_content;
+  obj_content.append(buf, sizeof(buf));
+
+  std::string target_str = "content";
+
+  // Write xattr bare, no ::encod'ing
+  bufferlist target_val;
+  target_val.append(target_str);
+  bufferlist nontarget_val;
+  nontarget_val.append("rhubarb");
+
+  ASSERT_EQ(0, ioctx.write("has_xattr", obj_content, obj_content.length(), 0));
+  ASSERT_EQ(0, ioctx.write("has_wrong_xattr", obj_content, obj_content.length(), 0));
+  ASSERT_EQ(0, ioctx.write("no_xattr", obj_content, obj_content.length(), 0));
+
+  ASSERT_EQ(0, ioctx.setxattr("has_xattr", "theattr", target_val));
+  ASSERT_EQ(0, ioctx.setxattr("has_wrong_xattr", "theattr", nontarget_val));
+
+  bufferlist filter_bl;
+  std::string filter_name = "plain";
+  ::encode(filter_name, filter_bl);
+  ::encode("_theattr", filter_bl);
+  ::encode(target_str, filter_bl);
+
+  ObjectCursor c = ioctx.object_list_begin();
+  ObjectCursor end = ioctx.object_list_end();
+  bool foundit = false;
+  while(!ioctx.object_list_is_end(c))
+  {
+    std::vector<ObjectItem> result;
+    int r = ioctx.object_list(c, end, 12, filter_bl, &result, &c);
+    ASSERT_GE(r, 0);
+    ASSERT_EQ(r, result.size());
+    for (int i = 0; i < r; ++i) {
+      auto oid = result[i].oid;
+      // We should only see the object that matches the filter
+      ASSERT_EQ(oid, "has_xattr");
+      // We should only see it once
+      ASSERT_FALSE(foundit);
+      foundit = true;
+    }
+  }
+  ASSERT_TRUE(foundit);
+}
+
 #pragma GCC diagnostic pop
 #pragma GCC diagnostic warning "-Wpragmas"