rgw: implement GetObjectAttributes

.gitmodules

@@ -81,3 +81,6 @@
 [submodule "src/boost_redis"]
 	path = src/boost_redis
 	url = https://github.com/boostorg/redis.git
+[submodule "src/BLAKE3"]
+	path = src/BLAKE3
+	url = git@github.com:BLAKE3-team/BLAKE3.git

src/CMakeLists.txt

@@ -203,6 +203,10 @@ if(HAVE_INTEL)
   else()
     set(object_format "elf64")
   endif()
+
+  set(SAVE_CMAKE_ASM_FLAGS ${CMAKE_ASM_FLAGS})
+  set(SAVE_CMAKE_ASM_COMPILER ${CMAKE_ASM_COMPILER})
+
   set(CMAKE_ASM_FLAGS "-f ${object_format}")
   set(CMAKE_ASM_COMPILER ${PROJECT_SOURCE_DIR}/src/nasm-wrapper)
   if(NOT WIN32)

src/common/ceph_crypto.h

@@ -20,6 +20,7 @@
 #include <openssl/evp.h>
 #include <openssl/ossl_typ.h>
 #include <openssl/hmac.h>
+#include "BLAKE3/c/blake3.h"
 
 #include "include/ceph_assert.h"
 
@@ -50,6 +51,26 @@ namespace TOPNSPC::crypto {
 	{}
   };
 
+  class Blake3 {
+    private:
+      blake3_hasher h;
+
+    public:
+      static constexpr uint16_t digest_size = BLAKE3_OUT_LEN /* 32 bytes */;
+
+      Blake3() { Restart(); }
+
+      void Restart() { blake3_hasher_init(&h); }
+
+      void Update(const unsigned char *data, uint64_t len) {
+	blake3_hasher_update(&h, data, len);
+      }
+
+      void Final(unsigned char* digest) {
+	blake3_hasher_finalize(&h, digest, digest_size);
+      }
+  }; /* Blake3 */
+
   namespace ssl {
     class OpenSSLDigest {
       private:
@@ -91,7 +112,6 @@ namespace TOPNSPC::crypto {
         SHA512 () : OpenSSLDigest(EVP_sha512()) { }
     };
 
-
 # if OPENSSL_VERSION_NUMBER < 0x10100000L
   class HMAC {
   private:

src/rgw/CMakeLists.txt

@@ -30,6 +30,33 @@ endfunction()
 
 find_package(ICU 52.0 COMPONENTS uc REQUIRED)
 
+set(blake3_srcs
+  ${CMAKE_SOURCE_DIR}/src/BLAKE3/c/blake3.c
+  ${CMAKE_SOURCE_DIR}/src/BLAKE3/c/blake3_dispatch.c
+  ${CMAKE_SOURCE_DIR}/src/BLAKE3/c/blake3_portable.c)
+
+if (HAVE_INTEL)
+  # restore compiler detected assembler, following use gas syntax
+  set(CMAKE_ASM_FLAGS ${SAVE_CMAKE_ASM_FLAGS})
+  set(CMAKE_ASM_COMPILER ${SAVE_CMAKE_ASM_COMPILER})
+  if(WIN32)
+  list(APPEND blake3_srcs
+    ${CMAKE_SOURCE_DIR}/src/BLAKE3/c/blake3_sse2_x86-64_windows_gnu.S
+    ${CMAKE_SOURCE_DIR}/src/BLAKE3/c/blake3_sse41_x86-64_windows_gnu.S
+    ${CMAKE_SOURCE_DIR}/src/BLAKE3/c/blake3_avx2_x86-64_windows_gnu.S
+    ${CMAKE_SOURCE_DIR}/src/BLAKE3/c/blake3_avx512_x86-64_windows_gnu.S)
+  else()
+  list(APPEND blake3_srcs
+    ${CMAKE_SOURCE_DIR}/src/BLAKE3/c/blake3.c
+    ${CMAKE_SOURCE_DIR}/src/BLAKE3/c/blake3_dispatch.c
+    ${CMAKE_SOURCE_DIR}/src/BLAKE3/c/blake3_portable.c
+    ${CMAKE_SOURCE_DIR}/src/BLAKE3/c/blake3_sse2_x86-64_unix.S
+    ${CMAKE_SOURCE_DIR}/src/BLAKE3/c/blake3_sse41_x86-64_unix.S
+    ${CMAKE_SOURCE_DIR}/src/BLAKE3/c/blake3_avx2_x86-64_unix.S
+    ${CMAKE_SOURCE_DIR}/src/BLAKE3/c/blake3_avx512_x86-64_unix.S)
+  endif()
+endif()
+
 set(librgw_common_srcs
   services/svc_finisher.cc
   services/svc_bi_rados.cc
@@ -57,6 +84,7 @@ set(librgw_common_srcs
   services/svc_user_rados.cc
   services/svc_zone.cc
   services/svc_zone_utils.cc
+  ${blake3_srcs}
   rgw_account.cc
   rgw_acl.cc
   rgw_acl_s3.cc
@@ -70,6 +98,7 @@ set(librgw_common_srcs
   rgw_bucket.cc
   rgw_bucket_layout.cc
   rgw_cache.cc
+  rgw_cksum_pipe.cc
   rgw_common.cc
   rgw_compression.cc
   rgw_cors.cc

src/rgw/driver/d4n/rgw_sal_d4n.cc

@@ -914,6 +914,7 @@ int D4NFilterWriter::process(bufferlist&& data, uint64_t offset)
 int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag,
                        ceph::real_time *mtime, ceph::real_time set_mtime,
                        std::map<std::string, bufferlist>& attrs,
+		       const std::optional<rgw::cksum::Cksum>& cksum,
                        ceph::real_time delete_at,
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
@@ -933,7 +934,7 @@ int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag,
     ldpp_dout(save_dpp, 10) << "D4NFilterWriter::" << __func__ << "(): ObjectDirectory set method failed." << dendl;
 
   /* Retrieve complete set of attrs */
-  int ret = next->complete(accounted_size, etag, mtime, set_mtime, attrs,
+  int ret = next->complete(accounted_size, etag, mtime, set_mtime, attrs, cksum,
 			delete_at, if_match, if_nomatch, user_data, zones_trace,
 			canceled, rctx, flags);
   obj->get_obj_attrs(rctx.y, save_dpp, NULL);

src/rgw/driver/d4n/rgw_sal_d4n.h

@@ -235,14 +235,15 @@ class D4NFilterWriter : public FilterWriter {
     virtual int prepare(optional_yield y);
     virtual int process(bufferlist&& data, uint64_t offset) override;
     virtual int complete(size_t accounted_size, const std::string& etag,
-                       ceph::real_time *mtime, ceph::real_time set_mtime,
-                       std::map<std::string, bufferlist>& attrs,
-                       ceph::real_time delete_at,
-                       const char *if_match, const char *if_nomatch,
-                       const std::string *user_data,
-                       rgw_zone_set *zones_trace, bool *canceled,
-                       const req_context& rctx,
-                       uint32_t flags) override;
+			 ceph::real_time *mtime, ceph::real_time set_mtime,
+			 std::map<std::string, bufferlist>& attrs,
+			 const std::optional<rgw::cksum::Cksum>& cksum,
+			 ceph::real_time delete_at,
+			 const char *if_match, const char *if_nomatch,
+			 const std::string *user_data,
+			 rgw_zone_set *zones_trace, bool *canceled,
+			 const req_context& rctx,
+			 uint32_t flags) override;
    bool is_atomic() { return atomic; };
    const DoutPrefixProvider* dpp() { return save_dpp; } 
 };

src/rgw/driver/daos/rgw_sal_daos.cc

@@ -1598,6 +1598,7 @@ int DaosMultipartUpload::init(const DoutPrefixProvider* dpp, optional_yield y,
 
   multipart_upload_info upload_info;
   upload_info.dest_placement = dest_placement;
+  upload_info.cksum_type = cksum_type;
 
   ent.encode(bl);
   encode(attrs, bl);
@@ -1970,6 +1971,7 @@ int DaosMultipartUpload::get_info(const DoutPrefixProvider* dpp,
 
   // Now decode the placement rule
   decode(upload_info, iter);
+  cksum_type = upload_info.cksum_type;
   placement = upload_info.dest_placement;
   *rule = &placement;
 

src/rgw/driver/daos/rgw_sal_daos.h

@@ -742,6 +742,7 @@ class DaosAtomicWriter : public StoreWriter {
   virtual int complete(size_t accounted_size, const std::string& etag,
                        ceph::real_time* mtime, ceph::real_time set_mtime,
                        std::map<std::string, bufferlist>& attrs,
+		       const std::optional<rgw::cksum::Cksum>& cksum,
                        ceph::real_time delete_at, const char* if_match,
                        const char* if_nomatch, const std::string* user_data,
                        rgw_zone_set* zones_trace, bool* canceled,
@@ -788,6 +789,7 @@ class DaosMultipartWriter : public StoreWriter {
   virtual int complete(size_t accounted_size, const std::string& etag,
                        ceph::real_time* mtime, ceph::real_time set_mtime,
                        std::map<std::string, bufferlist>& attrs,
+		       const std::optional<rgw::cksum::Cksum>& cksum,
                        ceph::real_time delete_at, const char* if_match,
                        const char* if_nomatch, const std::string* user_data,
                        rgw_zone_set* zones_trace, bool* canceled,
@@ -810,6 +812,10 @@ class DaosMultipartPart : public StoreMultipartPart {
   virtual const std::string& get_etag() { return info.etag; }
   virtual ceph::real_time& get_mtime() { return info.modified; }
 
+  virtual const std::optional<rgw::cksum::Cksum>& get_cksum() {
+    return info.cksum;
+  }
+
   friend class DaosMultipartUpload;
 };
 

src/rgw/driver/motr/rgw_sal_motr.h

@@ -791,6 +791,7 @@ class MotrAtomicWriter : public StoreWriter {
   virtual int complete(size_t accounted_size, const std::string& etag,
                        ceph::real_time *mtime, ceph::real_time set_mtime,
                        std::map<std::string, bufferlist>& attrs,
+		       const std::optional<rgw::cksum::Cksum>& cksum,
                        ceph::real_time delete_at,
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
@@ -839,6 +840,7 @@ class MotrMultipartWriter : public StoreWriter {
   virtual int complete(size_t accounted_size, const std::string& etag,
                        ceph::real_time *mtime, ceph::real_time set_mtime,
                        std::map<std::string, bufferlist>& attrs,
+		       const std::optional<rgw::cksum::Cksum>& cksum,
                        ceph::real_time delete_at,
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
@@ -888,6 +890,9 @@ class MotrMultipartPart : public StoreMultipartPart {
   virtual uint64_t get_size() { return info.accounted_size; }
   virtual const std::string& get_etag() { return info.etag; }
   virtual ceph::real_time& get_mtime() { return info.modified; }
+  virtual const std::optional<rgw::cksum::Cksum>& get_cksum() {
+    return info.cksum;
+  }
 
   RGWObjManifest& get_manifest() { return info.manifest; }
 

src/rgw/driver/posix/rgw_sal_posix.cc

@@ -14,6 +14,7 @@
  */
 
 #include "rgw_sal_posix.h"
+#include <asm-generic/errno.h>
 #include <dirent.h>
 #include <sys/stat.h>
 #include <sys/xattr.h>
@@ -1543,6 +1544,14 @@ int POSIXObject::copy_object(const ACLOwner& owner,
   }
 }
 
+int POSIXObject::list_parts(const DoutPrefixProvider* dpp, CephContext* cct,
+			    int max_parts, int marker, int* next_marker,
+			    bool* truncated, list_parts_each_t each_func,
+			    optional_yield y)
+{
+  return -EOPNOTSUPP;
+}
+
 int POSIXObject::get_obj_state(const DoutPrefixProvider* dpp, RGWObjState **pstate, optional_yield y, bool follow_olh)
 {
   int ret = stat(dpp);
@@ -2522,6 +2531,7 @@ int POSIXMultipartUpload::init(const DoutPrefixProvider *dpp, optional_yield y,
 
   meta_obj = get_meta_obj();
 
+  mp_obj.upload_info.cksum_type = cksum_type;
   mp_obj.upload_info.dest_placement = dest_placement;
 
   bufferlist bl;
@@ -2809,9 +2819,12 @@ int POSIXMultipartWriter::process(bufferlist&& data, uint64_t offset)
   return obj->write(offset, data, dpp, null_yield);
 }
 
-int POSIXMultipartWriter::complete(size_t accounted_size, const std::string& etag,
+int POSIXMultipartWriter::complete(
+		       size_t accounted_size,
+		       const std::string& etag,
                        ceph::real_time *mtime, ceph::real_time set_mtime,
                        std::map<std::string, bufferlist>& attrs,
+		       const std::optional<rgw::cksum::Cksum>& cksum,
                        ceph::real_time delete_at,
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,
@@ -2841,6 +2854,7 @@ int POSIXMultipartWriter::complete(size_t accounted_size, const std::string& eta
 
   info.num = part_num;
   info.etag = etag;
+  info.cksum = cksum;
   info.mtime = set_mtime;
 
   bufferlist bl;
@@ -2880,6 +2894,7 @@ int POSIXAtomicWriter::process(bufferlist&& data, uint64_t offset)
 int POSIXAtomicWriter::complete(size_t accounted_size, const std::string& etag,
                        ceph::real_time *mtime, ceph::real_time set_mtime,
                        std::map<std::string, bufferlist>& attrs,
+		       const std::optional<rgw::cksum::Cksum>& cksum,
                        ceph::real_time delete_at,
                        const char *if_match, const char *if_nomatch,
                        const std::string *user_data,

src/rgw/driver/posix/rgw_sal_posix.h

@@ -339,6 +339,13 @@ class POSIXObject : public StoreObject {
                const DoutPrefixProvider* dpp, optional_yield y) override;
   virtual RGWAccessControlPolicy& get_acl(void) override { return acls; }
   virtual int set_acl(const RGWAccessControlPolicy& acl) override { acls = acl; return 0; }
+
+    /** If multipart, enumerate (a range [marker..marker+[min(max_parts, parts_count-1)] of) parts of the object */
+    virtual int list_parts(const DoutPrefixProvider* dpp, CephContext* cct,
+			   int max_parts, int marker, int* next_marker,
+			   bool* truncated, list_parts_each_t each_func,
+			   optional_yield y) override;
+
   virtual int get_obj_state(const DoutPrefixProvider* dpp, RGWObjState **state, optional_yield y, bool follow_olh = true) override;
   virtual int set_obj_attrs(const DoutPrefixProvider* dpp, Attrs* setattrs,
 			    Attrs* delattrs, optional_yield y) override;
@@ -479,19 +486,24 @@ struct POSIXUploadPartInfo {
   uint32_t num{0};
   std::string etag;
   ceph::real_time mtime;
+  std::optional<rgw::cksum::Cksum> cksum;
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(1, 1, bl);
+    ENCODE_START(2, 1, bl);
     encode(num, bl);
     encode(etag, bl);
     encode(mtime, bl);
+    encode(cksum, bl);
     ENCODE_FINISH(bl);
   }
   void decode(bufferlist::const_iterator& bl) {
-    DECODE_START(1, bl);
+    DECODE_START_LEGACY_COMPAT_LEN(2, 1, 1, bl);
     decode(num, bl);
     decode(etag, bl);
     decode(mtime, bl);
+    if (struct_v > 1) {
+      decode(cksum, bl);
+    }
     DECODE_FINISH(bl);
   }
 };
@@ -514,8 +526,12 @@ class POSIXMultipartPart : public StoreMultipartPart {
   virtual uint64_t get_size() { return shadow->get_obj_size(); }
   virtual const std::string& get_etag() { return info.etag; }
   virtual ceph::real_time& get_mtime() { return info.mtime; }
+  virtual const std::optional<rgw::cksum::Cksum>& get_cksum() {
+    return info.cksum;
+  }
 
-  int load(const DoutPrefixProvider* dpp, optional_yield y, POSIXDriver* driver, rgw_obj_key& key);
+  int load(const DoutPrefixProvider* dpp, optional_yield y, POSIXDriver* driver,
+	   rgw_obj_key& key);
 
   friend class POSIXMultipartUpload;
 };
@@ -605,6 +621,7 @@ class POSIXAtomicWriter : public StoreWriter {
   virtual int complete(size_t accounted_size, const std::string& etag,
                        ceph::real_time *mtime, ceph::real_time set_mtime,
 		       std::map<std::string, bufferlist>& attrs,
+		       const std::optional<rgw::cksum::Cksum>& cksum,
 		       ceph::real_time delete_at,
 		       const char *if_match, const char *if_nomatch,
 		       const std::string *user_data,
@@ -645,6 +662,7 @@ class POSIXMultipartWriter : public StoreWriter {
   virtual int complete(size_t accounted_size, const std::string& etag,
                        ceph::real_time *mtime, ceph::real_time set_mtime,
 		       std::map<std::string, bufferlist>& attrs,
+		       const std::optional<rgw::cksum::Cksum>& cksum,
 		       ceph::real_time delete_at,
 		       const char *if_match, const char *if_nomatch,
 		       const std::string *user_data,