rgw: error out if frontend did not send all data

Fixes: #11851
The civetweb mg_write() doesn't return error when it can't flush all data
to the user, it just sends the total number of bytes written. Modified the
client io to return total number of bytes and return an error if didn't
send anything.

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
(cherry picked from commit daa679c)

src/rgw/rgw_civetweb.cc

@@ -13,13 +13,18 @@ int RGWMongoose::write_data(const char *buf, int len)
 {
   if (!header_done) {
     header_data.append(buf, len);
-    return 0;
+    return len;
   }
   if (!sent_header) {
     data.append(buf, len);
-    return 0;
+    return len;
+  }
+  int r = mg_write(conn, buf, len);
+  if (r == 0) {
+    /* didn't send anything, error out */
+    return -EIO;
   }
-  return mg_write(conn, buf, len);
+  return r;
 }
 
 RGWMongoose::RGWMongoose(mg_connection *_conn, int _port) : conn(_conn), port(_port), header_done(false), sent_header(false), has_content_length(false),

src/rgw/rgw_client_io.cc

@@ -56,7 +56,12 @@ int RGWClientIO::write(const char *buf, int len)
     return ret;
 
   if (account)
-    bytes_sent += len;
+    bytes_sent += ret;
+
+  if (ret < len) {
+    /* sent less than tried to send, error out */
+    return -EIO;
+  }
 
   return 0;
 }