Merge pull request #10167: jewel: rgw: aws4: add STREAMING-AWS4-HMAC-…

…SHA256-PAYLOAD support

Reviewed-by: Loic Dachary <ldachary@redhat.com>

src/rgw/rgw_auth_s3.cc

@@ -277,7 +277,11 @@ void rgw_create_s3_v4_canonical_request(struct req_state *s, const string& canon
     if (s->aws4_auth_needs_complete) {
       request_payload_hash = STREAM_IO(s)->grab_aws4_sha256_hash();
     } else {
-      rgw_hash_s3_string_sha256(request_payload.c_str(), request_payload.size(), request_payload_hash);
+      if (s->aws4_auth_streaming_mode) {
+        request_payload_hash = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
+      } else {
+        rgw_hash_s3_string_sha256(request_payload.c_str(), request_payload.size(), request_payload_hash);
+      }
     }
   }
 
@@ -392,13 +396,16 @@ int rgw_calculate_s3_v4_aws_signature(struct req_state *s,
 
   /* aws4_request */
 
-  char signing_k[CEPH_CRYPTO_HMACSHA256_DIGESTSIZE];
+  char *signing_k = s->aws4_auth->signing_k;
+
   calc_hmac_sha256(service_k, CEPH_CRYPTO_HMACSHA256_DIGESTSIZE, "aws4_request", 12, signing_k);
 
   buf_to_hex((unsigned char *) signing_k, CEPH_CRYPTO_HMACSHA256_DIGESTSIZE, aux);
 
   ldout(s->cct, 10) << "signing_k     = " << string(aux) << dendl;
 
+  s->aws4_auth->signing_key = aux;
+
   /* new signature */
 
   char signature_k[CEPH_CRYPTO_HMACSHA256_DIGESTSIZE];

src/rgw/rgw_common.cc

@@ -195,6 +195,7 @@ req_state::req_state(CephContext* _cct, RGWEnv* e, RGWUserInfo* u)
   object_acl = NULL;
   expect_cont = false;
   aws4_auth_needs_complete = false;
+  aws4_auth_streaming_mode = false;
 
   header_ended = false;
   obj_size = 0;
@@ -213,16 +214,13 @@ req_state::req_state(CephContext* _cct, RGWEnv* e, RGWUserInfo* u)
   http_auth = NULL;
   local_source = false;
 
-  aws4_auth = NULL;
-
   obj_ctx = NULL;
 }
 
 req_state::~req_state() {
   delete formatter;
   delete bucket_acl;
   delete object_acl;
-  delete aws4_auth;
 }
 
 struct str_len {

src/rgw/rgw_common.h

@@ -1188,6 +1188,10 @@ struct rgw_aws4_auth {
   string signature;
   string new_signature;
   string payload_hash;
+  string seed_signature;
+  string signing_key;
+  char signing_k[CEPH_CRYPTO_HMACSHA256_DIGESTSIZE];
+  bufferlist bl;
 };
 
 struct req_init_state {
@@ -1260,7 +1264,8 @@ struct req_state {
 
   /* aws4 auth support */
   bool aws4_auth_needs_complete;
-  rgw_aws4_auth *aws4_auth;
+  bool aws4_auth_streaming_mode;
+  unique_ptr<rgw_aws4_auth> aws4_auth;
 
   string canned_acl;
   bool has_acl_header;

src/rgw/rgw_op.cc

@@ -2437,15 +2437,22 @@ void RGWPutObj::execute()
   }
 
   do {
-    bufferlist data;
-    len = get_data(data);
+    bufferlist data_in;
+    len = get_data(data_in);
     if (len < 0) {
       op_ret = len;
       goto done;
     }
     if (!len)
       break;
 
+    bufferlist &data = data_in;
+    if (s->aws4_auth_streaming_mode) {
+      /* use unwrapped data */
+      data = s->aws4_auth->bl;
+      len = data.length();
+    }
+
     /* do we need this operation to be synchronous? if we're dealing with an object with immutable
      * head, e.g., multipart object we need to make sure we're the first one writing to this object
      */
@@ -2497,7 +2504,9 @@ void RGWPutObj::execute()
     ofs += len;
   } while (len > 0);
 
-  if (!chunked_upload && ofs != s->content_length) {
+  if (!chunked_upload &&
+      ofs != s->content_length &&
+      !s->aws4_auth_streaming_mode) {
     op_ret = -ERR_REQUEST_TIMEOUT;
     goto done;
   }
@@ -2543,6 +2552,7 @@ void RGWPutObj::execute()
   hash.Final(m);
 
   buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5);
+
   etag = calc_md5;
 
   if (supplied_md5_b64 && strcmp(calc_md5, supplied_md5)) {

src/rgw/rgw_rest.cc

@@ -1005,6 +1005,49 @@ int RGWPutObj_ObjStore::get_params()
   return 0;
 }
 
+int RGWPutObj_ObjStore::get_padding_last_aws4_chunk_encoded(bufferlist &bl, uint64_t chunk_size) {
+
+  const int chunk_str_min_len = 1 + 17 + 64 + 2; /* len('0') = 1 */
+
+  char *chunk_str = bl.c_str();
+  int budget = bl.length();
+
+  unsigned int chunk_data_size;
+  unsigned int chunk_offset = 0;
+
+  while (1) {
+
+    /* check available metadata */
+    if (budget < chunk_str_min_len) {
+      return -ERR_SIGNATURE_NO_MATCH;
+    }
+
+    chunk_offset = 0;
+
+    /* grab chunk size */
+    while ((*(chunk_str+chunk_offset) != ';') && (chunk_offset < chunk_str_min_len))
+      chunk_offset++;
+    string str = string(chunk_str, chunk_offset);
+    stringstream ss;
+    ss << std::hex << str;
+    ss >> chunk_data_size;
+
+    /* next chunk */
+    chunk_offset += 17 + 64 + 2 + chunk_data_size;
+
+    /* last chunk? */
+    budget -= chunk_offset;
+    if (budget < 0) {
+      budget *= -1;
+      break;
+    }
+
+    chunk_str += chunk_offset;
+  }
+
+  return budget;
+}
+
 int RGWPutObj_ObjStore::get_data(bufferlist& bl)
 {
   size_t cl;
@@ -1030,6 +1073,30 @@ int RGWPutObj_ObjStore::get_data(bufferlist& bl)
 
     len = read_len;
     bl.append(bp, 0, len);
+
+    /* read last aws4 chunk padding */
+    if (s->aws4_auth_streaming_mode && len == (int)chunk_size) {
+      int ret_auth = get_padding_last_aws4_chunk_encoded(bl, chunk_size);
+      if (ret_auth < 0) {
+        return ret_auth;
+      }
+      int len_padding = ret_auth;
+      if (len_padding) {
+        int read_len;
+        bufferptr bp_extra(len_padding);
+        int r = STREAM_IO(s)->read(bp_extra.c_str(), len_padding, &read_len,
+                                   s->aws4_auth_needs_complete);
+        if (r < 0) {
+          return r;
+        }
+        if (read_len != len_padding) {
+          return -ERR_SIGNATURE_NO_MATCH;
+        }
+        bl.append(bp_extra.c_str(), len_padding);
+        bl.rebuild();
+      }
+    }
+
   }
 
   if ((uint64_t)ofs + len > s->cct->_conf->rgw_max_put_size) {

src/rgw/rgw_rest.h

@@ -217,6 +217,8 @@ class RGWPutObj_ObjStore : public RGWPutObj
   virtual int verify_params();
   virtual int get_params();
   virtual int get_data(bufferlist& bl);
+
+  int get_padding_last_aws4_chunk_encoded(bufferlist &bl, uint64_t chunk_size);
 };
 
 class RGWPostObj_ObjStore : public RGWPostObj