Skip to content

Commit

Permalink
several performance oriented updates
Browse files Browse the repository at this point in the history
  • Loading branch information
@ct-clmsn
ct-clmsn committed Mar 25, 2024
1 parent b94200f commit 766d572
Show file tree
Hide file tree
Showing 6 changed files with 55 additions and 46 deletions.
8 changes: 4 additions & 4 deletions libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,10 @@ namespace hpx { namespace util {

struct openshmem_seginfo_t
{
std::uint8_t* addr;
std::uint8_t* size;
std::uint8_t* beg_addr;
std::uint8_t* end_addr;
unsigned int * rcv;
unsigned int * xmt;
};

struct HPX_CORE_EXPORT openshmem_environment
Expand Down Expand Up @@ -100,8 +102,6 @@ namespace hpx { namespace util {
static hpx::mutex* segment_mutex;
static openshmem_seginfo_t* segments;
static std::uint8_t* shmem_buffer;
static unsigned int rcv;
static unsigned int xmt;
};
}} // namespace hpx::util

Expand Down
23 changes: 15 additions & 8 deletions libs/core/openshmem_base/src/openshmem_environment.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,6 @@ namespace hpx::util {
hpx::mutex* openshmem_environment::segment_mutex = nullptr;
openshmem_seginfo_t* openshmem_environment::segments = nullptr;
std::uint8_t* hpx::util::openshmem_environment::shmem_buffer = nullptr;
unsigned int openshmem_environment::rcv = 0;
unsigned int openshmem_environment::xmt = 0;

///////////////////////////////////////////////////////////////////////////
int openshmem_environment::init([[maybe_unused]] int* argc,
Expand Down Expand Up @@ -146,19 +144,28 @@ namespace hpx::util {
//
//segments.resize(hpx::threads::hardware_concurrency() * size());
//
openshmem_environment::segments = new openshmem_seginfo_t[size()];
openshmem_environment::segment_mutex = new hpx::mutex[size()];
const std::size_t page_count = size();
const std::size_t signal_count = page_count*2;
openshmem_environment::segments = new openshmem_seginfo_t[page_count];
openshmem_environment::segment_mutex = new hpx::mutex[page_count];

const std::size_t byte_count = ((sizeof(std::uint8_t)*OPENSHMEM_PER_RANK_PAGESIZE*page_count)+(sizeof(unsigned int)*signal_count)) / sizeof(std::uint8_t);

hpx::util::openshmem_environment::shmem_buffer =
static_cast<std::uint8_t*>(shmem_calloc(
OPENSHMEM_PER_RANK_PAGESIZE, sizeof(std::uint8_t)));
byte_count, sizeof(std::uint8_t)));

for (int i = 0; i < size(); ++i)
std::size_t beg_signal = (OPENSHMEM_PER_RANK_PAGESIZE*page_count);
for (std::size_t i = 0; i < page_count; ++i)
{
segments[i].addr = hpx::util::openshmem_environment::shmem_buffer +
segments[i].beg_addr = hpx::util::openshmem_environment::shmem_buffer +
(i * OPENSHMEM_PER_RANK_PAGESIZE);
segments[i].size = static_cast<std::uint8_t*>(segments[i].addr) +
segments[i].end_addr = static_cast<std::uint8_t*>(segments[i].beg_addr) +
OPENSHMEM_PER_RANK_PAGESIZE;
segments[i].rcv = reinterpret_cast<unsigned int*>(hpx::util::openshmem_environment::shmem_buffer +
beg_signal + signal_count);
segments[i].xmt = reinterpret_cast<unsigned int*>(hpx::util::openshmem_environment::shmem_buffer +
beg_signal + signal_count + 1);
}

shmem_barrier_all();
Expand Down
5 changes: 3 additions & 2 deletions libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,13 +143,14 @@ namespace hpx::parcelset::policies::openshmem {
{
header h = rcv_header_;
rcv_header_.reset();
auto self_ = hpx::util::openshmem_environment::rank();

while (rcv_header_.data() == 0)
{
bo(&hpx::util::openshmem_environment::rcv);
bo(hpx::util::openshmem_environment::segments[self_].rcv);
}

hpx::util::openshmem_environment::rcv = 0;
(*(hpx::util::openshmem_environment::segments[self_].rcv)) = 0;
return h;
}

Expand Down
28 changes: 14 additions & 14 deletions libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,14 +105,14 @@ namespace hpx::parcelset::policies::openshmem {
{
hpx::util::openshmem_environment::scoped_lock l;
hpx::util::openshmem_environment::wait_until(
1, &hpx::util::openshmem_environment::rcv);
hpx::util::openshmem_environment::rcv = 0;
1, hpx::util::openshmem_environment::segments[self_].rcv);
(*(hpx::util::openshmem_environment::segments[self_].rcv)) = 0;

hpx::util::openshmem_environment::get(
reinterpret_cast<std::uint8_t*>(
buffer_.transmission_chunks_.data()),
self_,
hpx::util::openshmem_environment::segments[self_].addr,
hpx::util::openshmem_environment::segments[self_].beg_addr,
static_cast<int>(buffer_.transmission_chunks_.size() *
sizeof(buffer_type::transmission_chunk_type)));

Expand Down Expand Up @@ -142,14 +142,14 @@ namespace hpx::parcelset::policies::openshmem {
{
auto self_ = hpx::util::openshmem_environment::rank();
hpx::util::openshmem_environment::scoped_lock l;
hpx::util::openshmem_environment::wait_until(
1, &hpx::util::openshmem_environment::rcv);
hpx::util::openshmem_environment::rcv = 0;
hpx::util::openshmem_environment::wait_until(
1, hpx::util::openshmem_environment::segments[self_].rcv);
(*(hpx::util::openshmem_environment::segments[self_].rcv)) = 0;

hpx::util::openshmem_environment::get(
reinterpret_cast<std::uint8_t*>(buffer_.data_.data()),
self_,
hpx::util::openshmem_environment::segments[self_].addr,
hpx::util::openshmem_environment::segments[self_].beg_addr,
buffer_.data_.size());

request_ptr_ = true;
Expand Down Expand Up @@ -180,16 +180,16 @@ namespace hpx::parcelset::policies::openshmem {
hpx::util::openshmem_environment::scoped_lock l;

hpx::util::openshmem_environment::wait_until(
1, &hpx::util::openshmem_environment::rcv);
hpx::util::openshmem_environment::rcv = 0;
1, hpx::util::openshmem_environment::segments[self_].rcv);
(*(hpx::util::openshmem_environment::segments[self_].rcv)) = 0;

hpx::util::openshmem_environment::get(
reinterpret_cast<std::uint8_t*>(c.data()), self_,
hpx::util::openshmem_environment::segments[self_].addr,
hpx::util::openshmem_environment::segments[self_].beg_addr,
c.size());

hpx::util::openshmem_environment::put_signal(nullptr, src_,
nullptr, 0, &hpx::util::openshmem_environment::xmt);
nullptr, 0, hpx::util::openshmem_environment::segments[self_].xmt);

request_ptr_ = true;
}
Expand All @@ -215,12 +215,12 @@ namespace hpx::parcelset::policies::openshmem {
hpx::util::openshmem_environment::scoped_lock l;

hpx::util::openshmem_environment::wait_until(
1, &hpx::util::openshmem_environment::rcv);
hpx::util::openshmem_environment::rcv = 0;
1, hpx::util::openshmem_environment::segments[self_].rcv);
(*(hpx::util::openshmem_environment::segments[self_].rcv)) = 0;

hpx::util::openshmem_environment::get(
reinterpret_cast<std::uint8_t*>(&tag_), self_,
hpx::util::openshmem_environment::segments[self_].addr,
hpx::util::openshmem_environment::segments[self_].beg_addr,
sizeof(int));

request_ptr_ = true;
Expand Down
2 changes: 1 addition & 1 deletion libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ namespace hpx::parcelset::policies::openshmem {
std::memcpy(&next_free,
hpx::util::openshmem_environment::segments
[hpx::util::openshmem_environment::rank()]
.addr,
.beg_addr,
sizeof(int));

return next_free;
Expand Down
35 changes: 18 additions & 17 deletions libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,9 +166,8 @@ namespace hpx::parcelset::policies::openshmem {
//
hpx::util::openshmem_environment::put_signal(
reinterpret_cast<std::uint8_t*>(chunks), dst_,
static_cast<std::uint8_t*>(
hpx::util::openshmem_environment::segments[dst_].addr),
sizeof_chunks, &hpx::util::openshmem_environment::rcv);
static_cast<std::uint8_t*>(hpx::util::openshmem_environment::segments[dst_].beg_addr),
sizeof_chunks, reinterpret_cast<unsigned int*>(hpx::util::openshmem_environment::segments[dst_].rcv));
}

state_ = sent_header;
Expand All @@ -187,16 +186,18 @@ namespace hpx::parcelset::policies::openshmem {
chunks = buffer_.transmission_chunks_;
if (!chunks.empty())
{
const std::size_t rank_ =
hpx::util::openshmem_environment::rank();
hpx::util::openshmem_environment::scoped_lock l;
hpx::util::openshmem_environment::put_signal(
reinterpret_cast<std::uint8_t*>(chunks.data()), dst_,
static_cast<std::uint8_t*>(
hpx::util::openshmem_environment::segments
[hpx::util::openshmem_environment::rank()]
.addr),
static_cast<std::uint8_t*>(hpx::util::openshmem_environment::segments
[rank_].beg_addr),
static_cast<int>(chunks.size() *
sizeof(parcel_buffer_type::transmission_chunk_type)),
&hpx::util::openshmem_environment::rcv);
reinterpret_cast<unsigned int*>(hpx::util::openshmem_environment::segments
[rank_].rcv)
);
}

state_ = sent_transmission_chunks;
Expand All @@ -216,10 +217,10 @@ namespace hpx::parcelset::policies::openshmem {
hpx::util::openshmem_environment::scoped_lock l;
hpx::util::openshmem_environment::put_signal(
reinterpret_cast<std::uint8_t*>(buffer_.data_.data()), dst_,
static_cast<std::uint8_t*>(
hpx::util::openshmem_environment::segments[dst_].addr),
static_cast<std::uint8_t*>(hpx::util::openshmem_environment::segments[dst_].beg_addr),
buffer_.data_.size(),
&hpx::util::openshmem_environment::rcv);
reinterpret_cast<unsigned int*>(hpx::util::openshmem_environment::segments[dst_].rcv)
);
}
state_ = sent_data;

Expand All @@ -245,15 +246,15 @@ namespace hpx::parcelset::policies::openshmem {
hpx::util::openshmem_environment::put_signal(
reinterpret_cast<const std::uint8_t*>(c.data_.cpos_),
dst_,
static_cast<std::uint8_t*>(
hpx::util::openshmem_environment::segments[dst_]
.addr),
static_cast<std::uint8_t*>(hpx::util::openshmem_environment::segments[dst_]
.beg_addr),
static_cast<int>(c.size_),
&hpx::util::openshmem_environment::rcv);
reinterpret_cast<unsigned int*>(hpx::util::openshmem_environment::segments[dst_].rcv)
);

hpx::util::openshmem_environment::wait_until(
1, &hpx::util::openshmem_environment::xmt);
hpx::util::openshmem_environment::xmt = 0;
1, hpx::util::openshmem_environment::segments[dst_].xmt);
(*(hpx::util::openshmem_environment::segments[dst_].xmt)) = 0;
}

++chunks_idx_;
Expand Down

0 comments on commit 766d572

Please sign in to comment.