rgw: fix memory fragmentation problem reading data from client.

mg_read returns 0 on EOF.  In some versions of civetweb, mg_read can
return "short" reads.  The logic in the rest of ceph depends on a read
always filling its buffer when possible.  So loop here and fill the
buffer.  Looping to fill the buffer means we must also track when
mg_read returns EOF, because after that further calls to mg_read will return -1.

Fixes: https://tracker.ceph.com/issues/23207

Signed-off-by: Marcus Watts <mwatts@redhat.com>
(cherry picked from commit d9a150b)

src/rgw/rgw_civetweb.cc

@@ -33,6 +33,7 @@ RGWCivetWeb::RGWCivetWeb(mg_connection* const conn)
   : conn(conn),
     explicit_keepalive(false),
     explicit_conn_close(false),
+    got_eof_on_read(false),
     txbuf(*this)
 {
     sockaddr *lsa = mg_get_local_addr(conn);
@@ -50,11 +51,21 @@ RGWCivetWeb::RGWCivetWeb(mg_connection* const conn)
 
 size_t RGWCivetWeb::read_data(char *buf, size_t len)
 {
-  const int ret = mg_read(conn, buf, len);
-  if (ret < 0) {
-    throw rgw::io::Exception(EIO, std::system_category());
+  int c, ret;
+  if (got_eof_on_read) {
+    return 0;
+  }
+  for (c = 0; c < len; c += ret) {
+    ret = mg_read(conn, buf, len);
+    if (ret < 0) {
+      throw rgw::io::Exception(EIO, std::system_category());
+    }
+    if (!ret) {
+      got_eof_on_read = true;
+      break;
+    }
   }
-  return ret;
+  return c;
 }
 
 void RGWCivetWeb::flush()

src/rgw/rgw_civetweb.h

@@ -19,6 +19,7 @@ class RGWCivetWeb : public rgw::io::RestfulClient,
 
   bool explicit_keepalive;
   bool explicit_conn_close;
+  bool got_eof_on_read;
 
   rgw::io::StaticOutputBufferer<> txbuf;