From 6921665ecba7156b3f6ebf70b93def9e09226244 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Fri, 12 Nov 2021 09:32:56 +0000 Subject: [PATCH 1/4] track message type in execution graph --- include/faabric/util/exec_graph.h | 2 ++ src/scheduler/MpiWorld.cpp | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/include/faabric/util/exec_graph.h b/include/faabric/util/exec_graph.h index bc000a3ad..f233f1ab8 100644 --- a/include/faabric/util/exec_graph.h +++ b/include/faabric/util/exec_graph.h @@ -16,4 +16,6 @@ void incrementCounter(faabric::Message& msg, const int valueToIncrement = 1); static inline std::string const mpiMsgCountPrefix = "mpi-msgcount-torank-"; + +static inline std::string const mpiMsgTypeCountPrefix = "mpi-msgtype-torank"; } diff --git a/src/scheduler/MpiWorld.cpp b/src/scheduler/MpiWorld.cpp index e6a11498d..35f0fc39e 100644 --- a/src/scheduler/MpiWorld.cpp +++ b/src/scheduler/MpiWorld.cpp @@ -592,6 +592,15 @@ void MpiWorld::send(int sendRank, *thisRankMsg, faabric::util::exec_graph::mpiMsgCountPrefix + std::to_string(recvRank)); + + // Work out the message type breakdown + faabric::util::exec_graph::incrementCounter( + *thisRankMsg, + fmt::format("{}-{}-{}", + faabric::util::exec_graph::mpiMsgTypeCountPrefix, + std::to_string(messageType), + std::to_string(recvRank)) + ); } } From 945de7512a78912d8f68f3d78047401a61dd5b46 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Fri, 12 Nov 2021 10:04:58 +0000 Subject: [PATCH 2/4] adding test for tracing by message type --- src/scheduler/MpiWorld.cpp | 3 +- tests/test/scheduler/test_mpi_exec_graph.cpp | 70 +++++++++++++++++++- 2 files changed, 68 insertions(+), 5 deletions(-) diff --git a/src/scheduler/MpiWorld.cpp b/src/scheduler/MpiWorld.cpp index 35f0fc39e..058463af4 100644 --- a/src/scheduler/MpiWorld.cpp +++ b/src/scheduler/MpiWorld.cpp @@ -599,8 +599,7 @@ void MpiWorld::send(int sendRank, fmt::format("{}-{}-{}", faabric::util::exec_graph::mpiMsgTypeCountPrefix, std::to_string(messageType), - std::to_string(recvRank)) - ); + std::to_string(recvRank))); } } diff --git a/tests/test/scheduler/test_mpi_exec_graph.cpp b/tests/test/scheduler/test_mpi_exec_graph.cpp index 03138ea51..352bfa71b 100644 --- a/tests/test/scheduler/test_mpi_exec_graph.cpp +++ b/tests/test/scheduler/test_mpi_exec_graph.cpp @@ -36,8 +36,6 @@ TEST_CASE_METHOD(MpiTestFixture, rankA1, rankA2, BYTES(buffer), MPI_INT, messageData.size(), &status); } - REQUIRE(msg.intexecgraphdetails_size() == 1); - REQUIRE(msg.execgraphdetails_size() == 0); REQUIRE(msg.intexecgraphdetails().count(expectedKey) == 1); REQUIRE(msg.intexecgraphdetails().at(expectedKey) == numToSend); } @@ -126,8 +124,74 @@ TEST_CASE_METHOD(MpiBaseTestFixture, std::string expectedKey = faabric::util::exec_graph::mpiMsgCountPrefix + std::to_string(rank); REQUIRE(otherMsg.mpirank() == otherRank); - REQUIRE(otherMsg.intexecgraphdetails_size() == 1); REQUIRE(otherMsg.intexecgraphdetails().count(expectedKey) == 1); REQUIRE(otherMsg.intexecgraphdetails().at(expectedKey) == 1); } + +TEST_CASE_METHOD(MpiTestFixture, + "Test tracing the number of MPI messages by type", + "[util][exec-graph]") +{ + msg.set_recordexecgraph(true); + + // Send one message + int rankA1 = 0; + int rankA2 = 1; + MPI_Status status{}; + + std::vector messageData = { 0, 1, 2 }; + auto buffer = new int[messageData.size()]; + + std::string expectedKey; + int msgCount; + + SECTION("Normal send") + { + expectedKey = + fmt::format("{}-{}-{}", + faabric::util::exec_graph::mpiMsgTypeCountPrefix, + faabric::MPIMessage::NORMAL, + std::to_string(rankA2)); + msgCount = 1; + + world.send(rankA1, + rankA2, + BYTES(messageData.data()), + MPI_INT, + messageData.size()); + world.recv( + rankA1, rankA2, BYTES(buffer), MPI_INT, messageData.size(), &status); + } + + SECTION("Reduce") + { + std::vector data(2, 0); + + expectedKey = + fmt::format("{}-{}-{}", + faabric::util::exec_graph::mpiMsgTypeCountPrefix, + faabric::MPIMessage::REDUCE, + std::to_string(rankA2)); + msgCount = worldSize - 1; + + // Reduce expects to receive a message from all other ranks + for (int r = 0; r < worldSize; r++) { + if (r != rankA2) { + world.reduce( + r, rankA2, BYTES(&data[0]), nullptr, MPI_INT, 1, MPI_SUM); + } + } + + world.reduce(rankA2, + rankA2, + BYTES(&data[1]), + BYTES(&data[1]), + MPI_INT, + 1, + MPI_SUM); + } + + REQUIRE(msg.intexecgraphdetails().count(expectedKey) == 1); + REQUIRE(msg.intexecgraphdetails().at(expectedKey) == msgCount); +} } From 58dddfb4a1f5fa1a233e777bb413bea031cbd716 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Fri, 12 Nov 2021 11:49:23 +0000 Subject: [PATCH 3/4] switching to define-based constants for prefixes and keep them in an MPI header --- include/faabric/scheduler/MpiWorld.h | 7 +++++ include/faabric/util/exec_graph.h | 5 +--- src/scheduler/MpiWorld.cpp | 6 ++-- tests/test/scheduler/test_mpi_exec_graph.cpp | 31 ++++++++++---------- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/include/faabric/scheduler/MpiWorld.h b/include/faabric/scheduler/MpiWorld.h index 4d9c9bf34..a80b20b66 100644 --- a/include/faabric/scheduler/MpiWorld.h +++ b/include/faabric/scheduler/MpiWorld.h @@ -12,6 +12,13 @@ #include #include +// Constants for profiling MPI parameters like number of messages sent or +// message breakdown by type in the execution graph. Remember to increase the +// counter if you add another one +#define NUM_MPI_EXEC_GRAPH_DETAILS 2 +#define MPI_MSG_COUNT_PREFIX "mpi-msgcount-torank-" +#define MPI_MSGTYPE_COUNT_PREFIX "mpi-msgtype-torank" + namespace faabric::scheduler { typedef faabric::util::Queue> InMemoryMpiQueue; diff --git a/include/faabric/util/exec_graph.h b/include/faabric/util/exec_graph.h index f233f1ab8..6e9863267 100644 --- a/include/faabric/util/exec_graph.h +++ b/include/faabric/util/exec_graph.h @@ -7,6 +7,7 @@ #include namespace faabric::util::exec_graph { + void addDetail(faabric::Message& msg, const std::string& key, const std::string& value); @@ -14,8 +15,4 @@ void addDetail(faabric::Message& msg, void incrementCounter(faabric::Message& msg, const std::string& key, const int valueToIncrement = 1); - -static inline std::string const mpiMsgCountPrefix = "mpi-msgcount-torank-"; - -static inline std::string const mpiMsgTypeCountPrefix = "mpi-msgtype-torank"; } diff --git a/src/scheduler/MpiWorld.cpp b/src/scheduler/MpiWorld.cpp index 058463af4..3618dcc34 100644 --- a/src/scheduler/MpiWorld.cpp +++ b/src/scheduler/MpiWorld.cpp @@ -589,15 +589,13 @@ void MpiWorld::send(int sendRank, // If the message is set and recording on, track we have sent this message if (thisRankMsg != nullptr && thisRankMsg->recordexecgraph()) { faabric::util::exec_graph::incrementCounter( - *thisRankMsg, - faabric::util::exec_graph::mpiMsgCountPrefix + - std::to_string(recvRank)); + *thisRankMsg, MPI_MSG_COUNT_PREFIX + std::to_string(recvRank)); // Work out the message type breakdown faabric::util::exec_graph::incrementCounter( *thisRankMsg, fmt::format("{}-{}-{}", - faabric::util::exec_graph::mpiMsgTypeCountPrefix, + MPI_MSGTYPE_COUNT_PREFIX, std::to_string(messageType), std::to_string(recvRank))); } diff --git a/tests/test/scheduler/test_mpi_exec_graph.cpp b/tests/test/scheduler/test_mpi_exec_graph.cpp index 352bfa71b..1fc178d58 100644 --- a/tests/test/scheduler/test_mpi_exec_graph.cpp +++ b/tests/test/scheduler/test_mpi_exec_graph.cpp @@ -23,8 +23,7 @@ TEST_CASE_METHOD(MpiTestFixture, auto buffer = new int[messageData.size()]; int numToSend = 10; - std::string expectedKey = - faabric::util::exec_graph::mpiMsgCountPrefix + std::to_string(rankA2); + std::string expectedKey = MPI_MSG_COUNT_PREFIX + std::to_string(rankA2); for (int i = 0; i < numToSend; i++) { world.send(rankA1, @@ -36,6 +35,8 @@ TEST_CASE_METHOD(MpiTestFixture, rankA1, rankA2, BYTES(buffer), MPI_INT, messageData.size(), &status); } + REQUIRE(msg.intexecgraphdetails_size() == NUM_MPI_EXEC_GRAPH_DETAILS); + REQUIRE(msg.execgraphdetails_size() == 0); REQUIRE(msg.intexecgraphdetails().count(expectedKey) == 1); REQUIRE(msg.intexecgraphdetails().at(expectedKey) == numToSend); } @@ -56,8 +57,7 @@ TEST_CASE_METHOD(MpiTestFixture, auto buffer = new int[messageData.size()]; int numToSend = 10; - std::string expectedKey = - faabric::util::exec_graph::mpiMsgCountPrefix + std::to_string(rankA2); + std::string expectedKey = MPI_MSG_COUNT_PREFIX + std::to_string(rankA2); for (int i = 0; i < numToSend; i++) { world.send(rankA1, @@ -121,8 +121,7 @@ TEST_CASE_METHOD(MpiBaseTestFixture, otherWorldThread.join(); } - std::string expectedKey = - faabric::util::exec_graph::mpiMsgCountPrefix + std::to_string(rank); + std::string expectedKey = MPI_MSG_COUNT_PREFIX + std::to_string(rank); REQUIRE(otherMsg.mpirank() == otherRank); REQUIRE(otherMsg.intexecgraphdetails().count(expectedKey) == 1); REQUIRE(otherMsg.intexecgraphdetails().at(expectedKey) == 1); @@ -147,11 +146,10 @@ TEST_CASE_METHOD(MpiTestFixture, SECTION("Normal send") { - expectedKey = - fmt::format("{}-{}-{}", - faabric::util::exec_graph::mpiMsgTypeCountPrefix, - faabric::MPIMessage::NORMAL, - std::to_string(rankA2)); + expectedKey = fmt::format("{}-{}-{}", + MPI_MSGTYPE_COUNT_PREFIX, + faabric::MPIMessage::NORMAL, + std::to_string(rankA2)); msgCount = 1; world.send(rankA1, @@ -167,11 +165,10 @@ TEST_CASE_METHOD(MpiTestFixture, { std::vector data(2, 0); - expectedKey = - fmt::format("{}-{}-{}", - faabric::util::exec_graph::mpiMsgTypeCountPrefix, - faabric::MPIMessage::REDUCE, - std::to_string(rankA2)); + expectedKey = fmt::format("{}-{}-{}", + MPI_MSGTYPE_COUNT_PREFIX, + faabric::MPIMessage::REDUCE, + std::to_string(rankA2)); msgCount = worldSize - 1; // Reduce expects to receive a message from all other ranks @@ -191,6 +188,8 @@ TEST_CASE_METHOD(MpiTestFixture, MPI_SUM); } + REQUIRE(msg.intexecgraphdetails_size() == NUM_MPI_EXEC_GRAPH_DETAILS); + REQUIRE(msg.execgraphdetails_size() == 0); REQUIRE(msg.intexecgraphdetails().count(expectedKey) == 1); REQUIRE(msg.intexecgraphdetails().at(expectedKey) == msgCount); } From b34d53cec5c16450883dced8e59bb6dc6f7842a3 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Fri, 12 Nov 2021 15:08:26 +0000 Subject: [PATCH 4/4] consistent trailing - in constants --- include/faabric/scheduler/MpiWorld.h | 2 +- src/scheduler/MpiWorld.cpp | 3 ++- tests/test/scheduler/test_mpi_exec_graph.cpp | 9 ++++++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/include/faabric/scheduler/MpiWorld.h b/include/faabric/scheduler/MpiWorld.h index a80b20b66..10fdbee5d 100644 --- a/include/faabric/scheduler/MpiWorld.h +++ b/include/faabric/scheduler/MpiWorld.h @@ -16,7 +16,7 @@ // message breakdown by type in the execution graph. Remember to increase the // counter if you add another one #define NUM_MPI_EXEC_GRAPH_DETAILS 2 -#define MPI_MSG_COUNT_PREFIX "mpi-msgcount-torank-" +#define MPI_MSG_COUNT_PREFIX "mpi-msgcount-torank" #define MPI_MSGTYPE_COUNT_PREFIX "mpi-msgtype-torank" namespace faabric::scheduler { diff --git a/src/scheduler/MpiWorld.cpp b/src/scheduler/MpiWorld.cpp index 3618dcc34..b017a00e4 100644 --- a/src/scheduler/MpiWorld.cpp +++ b/src/scheduler/MpiWorld.cpp @@ -589,7 +589,8 @@ void MpiWorld::send(int sendRank, // If the message is set and recording on, track we have sent this message if (thisRankMsg != nullptr && thisRankMsg->recordexecgraph()) { faabric::util::exec_graph::incrementCounter( - *thisRankMsg, MPI_MSG_COUNT_PREFIX + std::to_string(recvRank)); + *thisRankMsg, + fmt::format("{}-{}", MPI_MSG_COUNT_PREFIX, std::to_string(recvRank))); // Work out the message type breakdown faabric::util::exec_graph::incrementCounter( diff --git a/tests/test/scheduler/test_mpi_exec_graph.cpp b/tests/test/scheduler/test_mpi_exec_graph.cpp index 1fc178d58..513df0e71 100644 --- a/tests/test/scheduler/test_mpi_exec_graph.cpp +++ b/tests/test/scheduler/test_mpi_exec_graph.cpp @@ -23,7 +23,8 @@ TEST_CASE_METHOD(MpiTestFixture, auto buffer = new int[messageData.size()]; int numToSend = 10; - std::string expectedKey = MPI_MSG_COUNT_PREFIX + std::to_string(rankA2); + std::string expectedKey = + fmt::format("{}-{}", MPI_MSG_COUNT_PREFIX, std::to_string(rankA2)); for (int i = 0; i < numToSend; i++) { world.send(rankA1, @@ -57,7 +58,8 @@ TEST_CASE_METHOD(MpiTestFixture, auto buffer = new int[messageData.size()]; int numToSend = 10; - std::string expectedKey = MPI_MSG_COUNT_PREFIX + std::to_string(rankA2); + std::string expectedKey = + fmt::format("{}-{}", MPI_MSG_COUNT_PREFIX, std::to_string(rankA2)); for (int i = 0; i < numToSend; i++) { world.send(rankA1, @@ -121,7 +123,8 @@ TEST_CASE_METHOD(MpiBaseTestFixture, otherWorldThread.join(); } - std::string expectedKey = MPI_MSG_COUNT_PREFIX + std::to_string(rank); + std::string expectedKey = + fmt::format("{}-{}", MPI_MSG_COUNT_PREFIX, std::to_string(rank)); REQUIRE(otherMsg.mpirank() == otherRank); REQUIRE(otherMsg.intexecgraphdetails().count(expectedKey) == 1); REQUIRE(otherMsg.intexecgraphdetails().at(expectedKey) == 1);