osd: bail out if transaction size overflows

with a large MOSDMap message, the transaction size could be greater than
UINT_MAX. so fail early with error messages.

Fixes: http://tracker.ceph.com/issues/16982
Signed-off-by: Kefu Chai <kchai@redhat.com>

src/osd/OSD.cc

@@ -6704,10 +6704,16 @@ void OSD::handle_osd_map(MOSDMap *m)
   }
 
   ObjectStore::Transaction t;
+  uint64_t txn_size = 0;
 
   // store new maps: queue for disk and put in the osdmap cache
   epoch_t start = MAX(superblock.newest_map + 1, first);
   for (epoch_t e = start; e <= last; e++) {
+    if (txn_size > t.get_num_bytes()) {
+      derr << __func << " transaction size overflowed" << dendl;
+      assert(txn_size > t.get_num_bytes());
+    }
+    txn_size = t.get_num_bytes();
     map<epoch_t,bufferlist>::iterator p;
     p = m->maps.find(e);
     if (p != m->maps.end()) {