rgw: aws4: add STREAMING-AWS4-HMAC-SHA256-PAYLOAD support

When authenticating requests using the Authorization header in AWS4, you have
the option of uploading the payload in chunks. You can send data in fixed size
or variable size chunks.

This patch enables streaming mode and signed headers support with chunked
uploads.

Fixes: http://tracker.ceph.com/issues/16311

Signed-off-by: Javier M. Mellid <jmunhoz@igalia.com>
(cherry picked from commit 5de5876)

src/rgw/rgw_auth_s3.cc

@@ -277,7 +277,11 @@ void rgw_create_s3_v4_canonical_request(struct req_state *s, const string& canon
     if (s->aws4_auth_needs_complete) {
       request_payload_hash = STREAM_IO(s)->grab_aws4_sha256_hash();
     } else {
-      rgw_hash_s3_string_sha256(request_payload.c_str(), request_payload.size(), request_payload_hash);
+      if (s->aws4_auth_streaming_mode) {
+        request_payload_hash = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
+      } else {
+        rgw_hash_s3_string_sha256(request_payload.c_str(), request_payload.size(), request_payload_hash);
+      }
     }
   }
 
@@ -392,13 +396,16 @@ int rgw_calculate_s3_v4_aws_signature(struct req_state *s,
 
   /* aws4_request */
 
-  char signing_k[CEPH_CRYPTO_HMACSHA256_DIGESTSIZE];
+  char *signing_k = s->aws4_auth->signing_k;
+
   calc_hmac_sha256(service_k, CEPH_CRYPTO_HMACSHA256_DIGESTSIZE, "aws4_request", 12, signing_k);
 
   buf_to_hex((unsigned char *) signing_k, CEPH_CRYPTO_HMACSHA256_DIGESTSIZE, aux);
 
   ldout(s->cct, 10) << "signing_k     = " << string(aux) << dendl;
 
+  s->aws4_auth->signing_key = aux;
+
   /* new signature */
 
   char signature_k[CEPH_CRYPTO_HMACSHA256_DIGESTSIZE];

src/rgw/rgw_common.cc

@@ -195,6 +195,7 @@ req_state::req_state(CephContext* _cct, RGWEnv* e, RGWUserInfo* u)
   object_acl = NULL;
   expect_cont = false;
   aws4_auth_needs_complete = false;
+  aws4_auth_streaming_mode = false;
 
   header_ended = false;
   obj_size = 0;
@@ -222,7 +223,6 @@ req_state::~req_state() {
   delete formatter;
   delete bucket_acl;
   delete object_acl;
-  delete aws4_auth;
 }
 
 struct str_len {

src/rgw/rgw_common.h

@@ -1188,6 +1188,10 @@ struct rgw_aws4_auth {
   string signature;
   string new_signature;
   string payload_hash;
+  string seed_signature;
+  string signing_key;
+  char signing_k[CEPH_CRYPTO_HMACSHA256_DIGESTSIZE];
+  bufferlist bl;
 };
 
 struct req_init_state {
@@ -1260,7 +1264,8 @@ struct req_state {
 
   /* aws4 auth support */
   bool aws4_auth_needs_complete;
-  rgw_aws4_auth *aws4_auth;
+  bool aws4_auth_streaming_mode;
+  unique_ptr<rgw_aws4_auth> aws4_auth;
 
   string canned_acl;
   bool has_acl_header;

src/rgw/rgw_op.cc

@@ -2422,15 +2422,24 @@ void RGWPutObj::execute()
   }
 
   do {
-    bufferlist data;
-    len = get_data(data);
+    bufferlist data_in;
+    len = get_data(data_in);
     if (len < 0) {
       op_ret = len;
       goto done;
     }
     if (!len)
       break;
 
+    bufferlist data;
+    if (s->aws4_auth_streaming_mode) {
+      /* use unwrapped data */
+      data = s->aws4_auth->bl;
+      len = data.length();
+    } else {
+      data = data_in;
+    }
+
     /* do we need this operation to be synchronous? if we're dealing with an object with immutable
      * head, e.g., multipart object we need to make sure we're the first one writing to this object
      */
@@ -2482,7 +2491,9 @@ void RGWPutObj::execute()
     ofs += len;
   } while (len > 0);
 
-  if (!chunked_upload && ofs != s->content_length) {
+  if (!chunked_upload &&
+      ofs != s->content_length &&
+      !s->aws4_auth_streaming_mode) {
     op_ret = -ERR_REQUEST_TIMEOUT;
     goto done;
   }
@@ -2528,6 +2539,7 @@ void RGWPutObj::execute()
   hash.Final(m);
 
   buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5);
+
   etag = calc_md5;
 
   if (supplied_md5_b64 && strcmp(calc_md5, supplied_md5)) {

src/rgw/rgw_rest.cc

@@ -997,6 +997,49 @@ int RGWPutObj_ObjStore::get_params()
   return 0;
 }
 
+int RGWPutObj_ObjStore::get_padding_last_aws4_chunk_encoded(bufferlist &bl, uint64_t chunk_size) {
+
+  const int chunk_str_min_len = 1 + 17 + 64 + 2; /* len('0') = 1 */
+
+  char *chunk_str = bl.c_str();
+  int budget = bl.length();
+
+  unsigned int chunk_data_size;
+  unsigned int chunk_offset = 0;
+
+  while (1) {
+
+    /* check available metadata */
+    if (budget < chunk_str_min_len) {
+      return -ERR_SIGNATURE_NO_MATCH;
+    }
+
+    chunk_offset = 0;
+
+    /* grab chunk size */
+    while ((*(chunk_str+chunk_offset) != ';') && (chunk_offset < chunk_str_min_len))
+      chunk_offset++;
+    string str = string(chunk_str, chunk_offset);
+    stringstream ss;
+    ss << std::hex << str;
+    ss >> chunk_data_size;
+
+    /* next chunk */
+    chunk_offset += 17 + 64 + 2 + chunk_data_size;
+
+    /* last chunk? */
+    budget -= chunk_offset;
+    if (budget < 0) {
+      budget *= -1;
+      break;
+    }
+
+    chunk_str += chunk_offset;
+  }
+
+  return budget;
+}
+
 int RGWPutObj_ObjStore::get_data(bufferlist& bl)
 {
   size_t cl;
@@ -1022,6 +1065,30 @@ int RGWPutObj_ObjStore::get_data(bufferlist& bl)
 
     len = read_len;
     bl.append(bp, 0, len);
+
+    /* read last aws4 chunk padding */
+    if (s->aws4_auth_streaming_mode && len == (int)chunk_size) {
+      int ret_auth = get_padding_last_aws4_chunk_encoded(bl, chunk_size);
+      if (ret_auth < 0) {
+        return ret_auth;
+      }
+      int len_padding = ret_auth;
+      if (len_padding) {
+        int read_len;
+        bufferptr bp_extra(len_padding);
+        int r = STREAM_IO(s)->read(bp_extra.c_str(), len_padding, &read_len,
+                                   s->aws4_auth_needs_complete);
+        if (r < 0) {
+          return r;
+        }
+        if (read_len != len_padding) {
+          return -ERR_SIGNATURE_NO_MATCH;
+        }
+        bl.append(bp_extra.c_str(), len_padding);
+        bl.rebuild();
+      }
+    }
+
   }
 
   if ((uint64_t)ofs + len > s->cct->_conf->rgw_max_put_size) {

src/rgw/rgw_rest.h

@@ -217,6 +217,8 @@ class RGWPutObj_ObjStore : public RGWPutObj
   virtual int verify_params();
   virtual int get_params();
   virtual int get_data(bufferlist& bl);
+
+  int get_padding_last_aws4_chunk_encoded(bufferlist &bl, uint64_t chunk_size);
 };
 
 class RGWPostObj_ObjStore : public RGWPostObj