Merge pull request #2618 from STEllAR-GROUP/parcelport_libfabric3

Some fixes to libfabric that didn't get caught before the merge
STEllAR-GROUP · May 10, 2017 · 1cc259c · 1cc259c
2 parents 53dbad1 + 2bf30de
commit 1cc259c
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 10 deletions.
diff --git a/plugins/parcelport/libfabric/header.hpp b/plugins/parcelport/libfabric/header.hpp
@@ -148,8 +148,11 @@ namespace libfabric
                 else {
                     // the message isn't piggybacked and neither is the chunk data
                     // so we must add rma-get information for the message
-                    detail::message_chunk *mc =
-                        reinterpret_cast<detail::message_chunk*>(&data_[chunkbytes]);
+                    detail::message_chunk *mc = reinterpret_cast<detail::message_chunk*>
+                        (&data_[message_info_offset()]);
+                    LOG_DEBUG_MSG("Setting chunk free message size to "
+                        << decnumber(buffer.size_)
+                        << "offset " << decnumber(message_info_offset()));
                     mc->message_rma =
                         serialization::create_pointer_chunk(nullptr, buffer.size_, 0);
                 }
@@ -326,7 +329,11 @@ namespace libfabric
             // if the data is not piggybacked then look at the final chunk
             chunktype *chunks = reinterpret_cast<chunktype *>(chunk_ptr());
             if (!chunks) {
-                throw std::runtime_error("Don't call message_size without chunk data");
+                detail::message_chunk *mc = message_chunk_ptr();
+                LOG_DEBUG_MSG("chunk free message size is "
+                    << decnumber(mc->message_rma.size_)
+                    << "offset was " << decnumber(message_info_offset()));
+                return mc->message_rma.size_;
             }
             return chunks[message_header.num_chunks-1].size_;
         }
@@ -345,9 +352,12 @@ namespace libfabric
                 detail::message_chunk *mc = message_chunk_ptr();
                 chunks = &mc->message_rma;
             }
+            else {
+                chunks = &chunks[message_header.num_chunks-1];
+            }
             // the last chunk will be our RMA message chunk
-            chunks[message_header.num_chunks-1].rkey_       = key;
-            chunks[message_header.num_chunks-1].data_.cpos_ = addr;
+            chunks->rkey_       = key;
+            chunks->data_.cpos_ = addr;
         }
 
         std::uint32_t num_chunks()

diff --git a/plugins/parcelport/libfabric/libfabric_controller.hpp b/plugins/parcelport/libfabric/libfabric_controller.hpp
@@ -230,7 +230,7 @@ namespace libfabric
                 locality new_locality;
                 std::copy(binary_t(encoded_data),
                           binary_t(encoded_data + encoded_length),
-                          (char*)(new_locality.fabric_data()));
+                          (new_locality.fabric_data_writable()));
 
                 // insert locality into address vector
                 LOG_DEBUG_MSG("Calling insert_address for " << decnumber(i)
@@ -720,11 +720,11 @@ namespace libfabric
                 int err_sz = fi_cq_readerr(txcq_, &e ,0);
                 // flags might not be set correctly
                 if (e.flags == (FI_MSG | FI_SEND)) {
-                    LOG_ERROR_MSG("txcq Error for FI_SEND with len " << e.len
+                    LOG_ERROR_MSG("txcq Error for FI_SEND with len " << hexlength(e.len)
                         << "context " << hexpointer(e.op_context));
                 }
                 if (e.flags & FI_RMA) {
-                    LOG_ERROR_MSG("txcq Error for FI_RMA with len " << e.len
+                    LOG_ERROR_MSG("txcq Error for FI_RMA with len " << hexlength(e.len)
                         << "context " << hexpointer(e.op_context));
                 }
                 rma_base *base = reinterpret_cast<rma_base*>(e.op_context);
@@ -788,7 +788,8 @@ namespace libfabric
             else if (ret == -FI_EAVAIL) {
                 struct fi_cq_err_entry e = {};
                 int err_sz = fi_cq_readerr(rxcq_, &e ,0);
-                LOG_ERROR_MSG("rxcq Error with flags " << e.flags << " len " << e.len);
+                LOG_ERROR_MSG("rxcq Error with flags " << hexlength(e.flags)
+                    << "len " << hexlength(e.len));
             }
             else {
                 LOG_ERROR_MSG("unknown error in completion rxcq read");

diff --git a/plugins/parcelport/libfabric/locality.hpp b/plugins/parcelport/libfabric/locality.hpp
@@ -144,6 +144,9 @@ struct locality {
 
     const void *fabric_data() const { return data_.data(); }
 
+    char *fabric_data_writable() { return reinterpret_cast<char*>(data_.data()); }
+
+
     bool valid() { return true; }
 
 private:

diff --git a/plugins/parcelport/libfabric/rma_receiver.cpp b/plugins/parcelport/libfabric/rma_receiver.cpp
@@ -537,7 +537,8 @@ namespace libfabric
     // --------------------------------------------------------------------
     void rma_receiver::handle_error(struct fi_cq_err_entry err)
     {
-        LOG_ERROR_MSG("rma_receiver handling an error " << hexpointer(this));
+        LOG_ERROR_MSG("rma_receiver handling an error (unimplemented)"
+            << hexpointer(this));
     }
 
 }}}}