From 1e7e2f4e0dcc58fac8fddc0fe8d0df171a27e834 Mon Sep 17 00:00:00 2001 From: Fred Hornsey Date: Tue, 29 Jul 2025 14:00:00 -0500 Subject: [PATCH 01/18] Priority in Microgrid Controller Selection Fixes https://github.com/OpenDDS/DeveloperResources/issues/12 --- tactical-microgrid-standard/CMakeLists.txt | 4 +- .../common/ControllerSelector.cpp | 30 ++- .../common/ControllerSelector.h | 66 +++++- .../common/TimerHandler.h | 7 + .../controller/Controller.cpp | 2 +- .../controller/Controller.h | 7 +- .../power_devices/PowerDevice.h | 5 + .../tests/mc-sel/CMakeLists.txt | 8 +- .../tests/mc-sel/basic-dev.cpp | 35 ++++ .../tests/mc-sel/basic-mc.cpp | 46 +---- .../tests/mc-sel/common.h | 39 ++++ .../tests/mc-sel/mc-sel-test.cpp | 194 ------------------ .../tests/mc-sel/run_test.pl | 67 ++++++ 13 files changed, 261 insertions(+), 249 deletions(-) create mode 100644 tactical-microgrid-standard/tests/mc-sel/basic-dev.cpp create mode 100644 tactical-microgrid-standard/tests/mc-sel/common.h delete mode 100644 tactical-microgrid-standard/tests/mc-sel/mc-sel-test.cpp create mode 100755 tactical-microgrid-standard/tests/mc-sel/run_test.pl diff --git a/tactical-microgrid-standard/CMakeLists.txt b/tactical-microgrid-standard/CMakeLists.txt index 81f90ca..90fe4c1 100644 --- a/tactical-microgrid-standard/CMakeLists.txt +++ b/tactical-microgrid-standard/CMakeLists.txt @@ -87,4 +87,6 @@ add_executable(Distribution target_include_directories(Distribution PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(Distribution PRIVATE PowerSim_Idl) -add_subdirectory(tests) +if(BUILD_TESTING) + add_subdirectory(tests) +endif() diff --git a/tactical-microgrid-standard/common/ControllerSelector.cpp b/tactical-microgrid-standard/common/ControllerSelector.cpp index 8510364..45a6fe3 100644 --- a/tactical-microgrid-standard/common/ControllerSelector.cpp +++ b/tactical-microgrid-standard/common/ControllerSelector.cpp @@ -1,7 +1,8 @@ #include "ControllerSelector.h" ControllerSelector::ControllerSelector(const tms::Identity& device_id) - : device_id_(device_id) + : ControllerCallbacks(lock_) + , device_id_(device_id) { } @@ -42,6 +43,11 @@ void ControllerSelector::got_device_info(const tms::DeviceInfo& di) if (di.role() == tms::DeviceRole::ROLE_MICROGRID_CONTROLLER && !all_controllers_.count(di.deviceId())) { all_controllers_[di.deviceId()] = TimePoint::min(); + all_controllers_by_priority_.insert(Priority(di)); + ACE_DEBUG((LM_INFO, "Got %C\n", di.deviceId().c_str())); + for (auto it = all_controllers_by_priority_.begin(); it != all_controllers_by_priority_.end(); ++it) { + ACE_DEBUG((LM_INFO, "%d %C\n", it->priority, it->id.c_str())); + } } } @@ -65,8 +71,7 @@ void ControllerSelector::timer_fired(Timer& timer) } if (now - it->second < heartbeat_deadline) { - selected_ = mc_id; - send_controller_state(); + select(mc_id, std::chrono::duration_cast(now - it->second)); } } @@ -76,6 +81,9 @@ void ControllerSelector::timer_fired(Timer& timer) const auto& timer_id = timer.id; ACE_DEBUG((LM_INFO, "(%P|%t) INFO: ControllerSelector::timed_event(MissedHeartbeat): " "\"%C\". Timer id: %d\n", selected_.c_str(), timer_id)); + if (missed_heartbeat_callback_) { + missed_heartbeat_callback_(selected_); + } schedule_once(LostController{}, lost_active_controller_delay); // Start a No MC timer if the device has missed heartbeats from all MCs @@ -98,15 +106,19 @@ void ControllerSelector::timer_fired(Timer&) Guard g(lock_); ACE_DEBUG((LM_INFO, "(%P|%t) INFO: ControllerSelector::timed_event(LostController): " "\"%C\"\n", selected_.c_str())); + if (lost_controller_callback_) { + lost_controller_callback_(selected_); + } selected_.clear(); // Select a new controller if possible. If there are no recent controllers // and we don't hear from another, the NoControllers timer will fire. const TimePoint now = Clock::now(); - for (auto it = all_controllers_.begin(); it != all_controllers_.end(); ++it) { - const auto last_hb = now - it->second; + for (auto it = all_controllers_by_priority_.begin(); it != all_controllers_by_priority_.end(); ++it) { + auto mc_info = all_controllers_.find(it->id); + const auto last_hb = now - mc_info->second; if (last_hb < heartbeat_deadline) { - select(it->first, std::chrono::duration_cast(last_hb)); + select(mc_info->first, std::chrono::duration_cast(last_hb)); break; } } @@ -116,6 +128,9 @@ void ControllerSelector::timer_fired(Timer&) { Guard g(lock_); ACE_DEBUG((LM_INFO, "(%P|%t) INFO: ControllerSelector::timed_event(NoControllers)\n")); + if (no_controllers_callback_) { + no_controllers_callback_(); + } // TODO: CONFIG_ON_COMMS_LOSS } @@ -123,6 +138,9 @@ void ControllerSelector::select(const tms::Identity& id, Sec last_hb) { ACE_DEBUG((LM_INFO, "(%P|%t) INFO: ControllerSelector::select: \"%C\"\n", id.c_str())); selected_ = id; + if (new_controller_callback_) { + new_controller_callback_(selected_); + } send_controller_state(); schedule_once(MissedHeartbeat{}, heartbeat_deadline - last_hb); } diff --git a/tactical-microgrid-standard/common/ControllerSelector.h b/tactical-microgrid-standard/common/ControllerSelector.h index 8eca3d9..35ea0b1 100644 --- a/tactical-microgrid-standard/common/ControllerSelector.h +++ b/tactical-microgrid-standard/common/ControllerSelector.h @@ -7,6 +7,8 @@ #include #include +#include +#include struct NewController { tms::Identity id; @@ -25,7 +27,46 @@ struct NoControllers { static const char* name() { return "NoControllers"; } }; -class PowerDevice; +class OpenDDS_TMS_Export ControllerCallbacks { +public: + using IdCallback = std::function; + using Callback = std::function; + + explicit ControllerCallbacks(Mutex& lock) : cb_lock_(lock) {} + + void set_new_controller_callback(IdCallback cb) + { + Guard g(cb_lock_); + new_controller_callback_ = cb; + } + + void set_missed_heartbeat_callback(IdCallback cb) + { + Guard g(cb_lock_); + missed_heartbeat_callback_ = cb; + } + + void set_lost_controller_callback(IdCallback cb) + { + Guard g(cb_lock_); + lost_controller_callback_ = cb; + } + + void set_no_controllers_callback(Callback cb) + { + Guard g(cb_lock_); + no_controllers_callback_ = cb; + } + +protected: + IdCallback new_controller_callback_; + IdCallback missed_heartbeat_callback_; + IdCallback lost_controller_callback_; + Callback no_controllers_callback_; + +private: + Mutex& cb_lock_; +}; /** * Logic for determining what controller should be used, based on TMS A.7.5. @@ -55,6 +96,7 @@ class PowerDevice; * S: If there's a selectable controller with a recent heartbeat */ class OpenDDS_TMS_Export ControllerSelector : + public ControllerCallbacks, public TimerHandler { public: explicit ControllerSelector(const tms::Identity& device_id); @@ -75,12 +117,6 @@ class OpenDDS_TMS_Export ControllerSelector : return selected_ == id; } - ACE_Reactor* get_reactor() const - { - Guard g(lock_); - return reactor_; - } - void set_ActiveMicrogridControllerState_writer(tms::ActiveMicrogridControllerStateDataWriter_var amcs_dw) { amcs_dw_ = amcs_dw; @@ -107,6 +143,22 @@ class OpenDDS_TMS_Export ControllerSelector : tms::Identity selected_; std::map all_controllers_; + struct Priority { + uint16_t priority; + tms::Identity id; + + Priority(const tms::DeviceInfo& di) + : priority(di.controlService()->mc()->priorityRanking()) + , id(di.deviceId()) + { + } + + bool operator<(const Priority& lhs) const + { + return std::tie(priority, id) < std::tie(lhs.priority, lhs.id); + } + }; + std::set all_controllers_by_priority_; // Device ID to which this controller selector belong. tms::Identity device_id_; diff --git a/tactical-microgrid-standard/common/TimerHandler.h b/tactical-microgrid-standard/common/TimerHandler.h index a1dd6f5..2bfced4 100644 --- a/tactical-microgrid-standard/common/TimerHandler.h +++ b/tactical-microgrid-standard/common/TimerHandler.h @@ -146,6 +146,12 @@ class TimerHandler : public ACE_Event_Handler, protected TimerHolder } } + ACE_Reactor* get_reactor() const + { + Guard g(lock_); + return reactor_; + } + template typename Timer::Ptr get_timer(const std::string& name = "") { @@ -244,6 +250,7 @@ class TimerHandler : public ACE_Event_Handler, protected TimerHolder { Guard g(lock_); auto key = *reinterpret_cast(arg); + ACE_DEBUG((LM_DEBUG, "(%P|%t) TimerHandler::handle_timeout: %C\n", key.c_str())); if (active_timers_.count(key) == 0) { ACE_ERROR((LM_WARNING, "(%P|%t) WARNING: TimerHandler::handle_timeout: timer key %C does NOT exist\n", key.c_str())); } diff --git a/tactical-microgrid-standard/controller/Controller.cpp b/tactical-microgrid-standard/controller/Controller.cpp index bc07b4c..b13de75 100644 --- a/tactical-microgrid-standard/controller/Controller.cpp +++ b/tactical-microgrid-standard/controller/Controller.cpp @@ -100,7 +100,7 @@ tms::DeviceInfo Controller::populate_device_info() const tms::MicrogridControllerInfo mc_info; { mc_info.features().push_back(tms::MicrogridControllerFeature::MCF_GENERAL); - mc_info.priorityRanking(0); + mc_info.priorityRanking(priority_); } csi.mc() = mc_info; } diff --git a/tactical-microgrid-standard/controller/Controller.h b/tactical-microgrid-standard/controller/Controller.h index eeef828..f7f0c76 100644 --- a/tactical-microgrid-standard/controller/Controller.h +++ b/tactical-microgrid-standard/controller/Controller.h @@ -6,7 +6,11 @@ class Controller : public Handshaking { public: - explicit Controller(const tms::Identity& id) : Handshaking(id) {} + explicit Controller(const tms::Identity& id, uint16_t priority= 0) + : Handshaking(id) + , priority_(priority) + { + } DDS::ReturnCode_t init(DDS::DomainId_t domain_id, int argc = 0, char* argv[] = nullptr); int run(); @@ -28,6 +32,7 @@ class Controller : public Handshaking { private: mutable std::mutex mut_; PowerDevices power_devices_; + uint16_t priority_; DDS::DomainId_t tms_domain_id_ = OpenDDS::DOMAIN_UNKNOWN;; }; diff --git a/tactical-microgrid-standard/power_devices/PowerDevice.h b/tactical-microgrid-standard/power_devices/PowerDevice.h index 2c6a90b..02094f8 100644 --- a/tactical-microgrid-standard/power_devices/PowerDevice.h +++ b/tactical-microgrid-standard/power_devices/PowerDevice.h @@ -73,6 +73,11 @@ class PowerSim_Idl_Export PowerDevice : public Handshaking { return essl_; } + ControllerCallbacks& controller_callbacks() + { + return controller_selector_; + } + protected: virtual int run_i() { diff --git a/tactical-microgrid-standard/tests/mc-sel/CMakeLists.txt b/tactical-microgrid-standard/tests/mc-sel/CMakeLists.txt index 543fa6e..1b72765 100644 --- a/tactical-microgrid-standard/tests/mc-sel/CMakeLists.txt +++ b/tactical-microgrid-standard/tests/mc-sel/CMakeLists.txt @@ -1,15 +1,15 @@ cmake_minimum_required(VERSION 3.27) -project(opendds_tms_mc_sel_test CXX) +project(opendds_tms_mc_sel CXX) enable_testing() find_package(OpenDDS REQUIRED) include(opendds_testing) -add_executable(mc-sel-test mc-sel-test.cpp) -target_link_libraries(mc-sel-test PRIVATE PowerSim_Idl) +add_executable(basic-dev basic-dev.cpp) +target_link_libraries(basic-dev PRIVATE PowerSim_Idl) add_executable(basic-mc ${CMAKE_SOURCE_DIR}/controller/Controller.cpp basic-mc.cpp) target_link_libraries(basic-mc PRIVATE Commands_Idl) -opendds_add_test(COMMAND ./mc-sel-test) +opendds_add_test(ARGS remove_dcs_after=0 test_verbose=1) diff --git a/tactical-microgrid-standard/tests/mc-sel/basic-dev.cpp b/tactical-microgrid-standard/tests/mc-sel/basic-dev.cpp new file mode 100644 index 0000000..dfa88fb --- /dev/null +++ b/tactical-microgrid-standard/tests/mc-sel/basic-dev.cpp @@ -0,0 +1,35 @@ +#include "common.h" + +#include + +#include + +int main(int argc, char* argv[]) +{ + const std::string name = "dev"; + PowerDevice pd(name); + if (pd.init(domain, argc, argv) != DDS::RETCODE_OK) { + return 1; + } + + DistributedConditionSet_rch dcs = + OpenDDS::DCPS::make_rch(); + + ControllerCallbacks& cbs = pd.controller_callbacks(); + cbs.set_new_controller_callback([dcs, name](const tms::Identity& id) { + printf("new_controller\n"); + dcs->post(name, "new controller " + id); + }); + cbs.set_missed_heartbeat_callback([dcs, name](const tms::Identity& id) { + dcs->post(name, "missed controller " + id); + }); + cbs.set_lost_controller_callback([dcs, name](const tms::Identity& id) { + dcs->post(name, "lost controller " + id); + }); + cbs.set_no_controllers_callback([dcs, name]() { + dcs->post(name, "no controllers"); + }); + + Exiter exiter(pd); + return pd.run(); +} diff --git a/tactical-microgrid-standard/tests/mc-sel/basic-mc.cpp b/tactical-microgrid-standard/tests/mc-sel/basic-mc.cpp index 2010106..f1d2b7f 100644 --- a/tactical-microgrid-standard/tests/mc-sel/basic-mc.cpp +++ b/tactical-microgrid-standard/tests/mc-sel/basic-mc.cpp @@ -1,46 +1,22 @@ -#include "controller/Controller.h" +#include "common.h" -struct Timeout { - static const char* name() { return "Timeout"; } -}; - -class Test : public TimerHandler { -public: - Test() - { - reactor_->register_handler(SIGINT, this); - schedule(Timeout{}, Sec(0), Sec(30)); - } - -private: - void timer_fired(Timer& t) - { - ACE_DEBUG((LM_INFO, "(%P|%t) Timeout\n")); - t.exit_after = true; - } - - void any_timer_fired(AnyTimer timer) - { - std::visit([&](auto&& value) { this->timer_fired(*value); }, timer); - } - - int handle_signal(int, siginfo_t*, ucontext_t*) - { - ACE_DEBUG((LM_INFO, "(%P|%t) SIGINT\n")); - return end_event_loop(); - } -}; +#include int main(int argc, char* argv[]) { - const int domain = std::stoi(argv[1]); - const std::string device_id = argv[2]; + if (argc < 4) { + ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: requires device id, priority, and time duration arguments\n")); + return 1; + } + const std::string device_id = argv[1]; + const uint16_t priority = std::stoi(argv[2]); + const Sec exit_after(std::stoi(argv[3])); - Controller mc(device_id); + Controller mc(device_id, priority); if (mc.init(domain, argc, argv) != DDS::RETCODE_OK) { return 1; } - Test test; + Exiter exiter(mc, exit_after); return mc.run(); } diff --git a/tactical-microgrid-standard/tests/mc-sel/common.h b/tactical-microgrid-standard/tests/mc-sel/common.h new file mode 100644 index 0000000..acae97f --- /dev/null +++ b/tactical-microgrid-standard/tests/mc-sel/common.h @@ -0,0 +1,39 @@ +#include +#include + +const int domain = 1; + +struct Timeout { + static const char* name() { return "Timeout"; } +}; + +class Exiter : public TimerHandler { +public: + Exiter(Handshaking& handshaking, Sec exit_after = Sec(0)) + : TimerHandler(handshaking.get_reactor()) + { + reactor_->register_handler(SIGINT, this); + if (exit_after > Sec(0)) { + schedule_once(Timeout{}, exit_after); + } + } + +private: + void timer_fired(Timer& t) + { + ACE_DEBUG((LM_INFO, "(%P|%t) Timeout\n")); + exit(0); + } + + void any_timer_fired(AnyTimer timer) + { + std::visit([&](auto&& value) { this->timer_fired(*value); }, timer); + } + + int handle_signal(int, siginfo_t*, ucontext_t*) + { + ACE_DEBUG((LM_INFO, "(%P|%t) SIGINT\n")); + exit(1); + return -1; + } +}; diff --git a/tactical-microgrid-standard/tests/mc-sel/mc-sel-test.cpp b/tactical-microgrid-standard/tests/mc-sel/mc-sel-test.cpp deleted file mode 100644 index 6913ecf..0000000 --- a/tactical-microgrid-standard/tests/mc-sel/mc-sel-test.cpp +++ /dev/null @@ -1,194 +0,0 @@ -#include "power_devices/PowerDevice.h" - -#include - -#include -#include -#include -#include -#include - -namespace fs = std::filesystem; - -const int domain = 1; -const std::string device_id = "dev"; - -struct McStart { - std::vector ids; - Sec run_for; - static const char* name() { return "McStart"; }; -}; -struct McSelected { - std::string id; - bool exit_after = false; - static const char* name() { return "McSelected"; } -}; -struct McStop { - std::vector pids; - static const char* name() { return "McStop"; } -}; -const std::string mc1 = "mc1"; -const std::string mc2 = "mc2"; -const std::string mc3 = "mc3"; -class Test : public TimerHandler { -public: - Test(PowerDevice& pd, const std::string& mc_path) - : pd_(pd) - , mc_path_(mc_path) - { - proc_man_.open(10, reactor_); - reactor_->register_handler(SIGINT, this); - - /* - * Time (s): 01234567890123456789012345678901234567890123456789012345678901234567890 - * Checks: 1 2 3 4 5 - * selected: [~~mc1~~~~~~~~~~][~mc2~~~~~~~~~~~~~~] [-m1--~~~~~~~~~~~] - * mc1: <--------> <-------------> - * mc2: <-----------------------> - * mc3: <-----------------------> - */ - schedule_once(mc1, McStart{{mc1}, Sec(10)}, Sec(1)); - schedule_once("mc2 & mc3", McStart{{mc2, mc3}, Sec(25)}, Sec(5)); - schedule_once("1st check", McSelected{mc1}, Sec(10)); - schedule_once("2nd check", McSelected{mc1}, Sec(15)); - schedule_once("3rd check", McSelected{mc2}, Sec(25)); - schedule_once("4th check", McSelected{""}, Sec(55)); - schedule_once("mc1 again", McStart{{mc1}, Sec(15)}, Sec(60)); - schedule_once("5th check", McSelected{mc1, /* exit_after = */ true}, Sec(70)); - } - - ~Test() - { - stop_all(); - } - - int exit_status() - { - return failed_; - } - -private: - void stop_mc(pid_t pid) - { - ACE_DEBUG((LM_INFO, "(%P|%t) sigint %d\n", pid)); - if (pid != ACE_INVALID_PID) { - proc_man_.terminate(pid, SIGINT); - } - } - - void stop_all() - { - for (auto& pair: procs_) { - if (!pair.second.status) { - stop_mc(pair.second.pid); - } - } - } - - void timer_fired(Timer& t) - { - std::vector pids; - for (const auto& id : t.arg.ids) { - ACE_Process_Options proc_opts; - proc_opts.command_line("%s %d %s", mc_path_.c_str(), domain, id.c_str()); - Proc proc{proc_opts.command_line_buf()}; - proc.pid = proc_man_.spawn(proc_opts, this); - if (proc.ran()) { - ACE_ERROR((LM_ERROR, "(%P|%t) McStart: \"%C\" failed: %p\n", proc.cmd.c_str())); - failed_ = true; - t.exit_after = true; - return; - } - ACE_DEBUG((LM_INFO, "(%P|%t) McStart: \"%C\" pid: %d\n", proc.cmd.c_str(), proc.pid)); - procs_[proc.pid] = proc; - pids.push_back(proc.pid); - } - schedule_once(t.name, McStop{pids}, t.arg.run_for); - } - - void timer_fired(Timer& t) - { - const auto sel = pd_.selected(); - ACE_DEBUG((LM_INFO, "(%P|%t) McSelected: expect: \"%C\" selected: \"%C\"\n", t.arg.id.c_str(), sel.c_str())); - if (sel != t.arg.id) { - ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: McSelected: failed\n")); - failed_ = true; - t.exit_after = true; - } else { - t.exit_after = t.arg.exit_after; - } - } - - void timer_fired(Timer& t) - { - for (const pid_t pid : t.arg.pids) { - stop_mc(pid); - } - } - - void any_timer_fired(AnyTimer timer) - { - std::visit([&](auto&& value) { this->timer_fired(*value); }, timer); - } - - int handle_exit(ACE_Process* p) - { - Proc& proc = procs_.find(p->getpid())->second; - proc.status = p->exit_code(); - if (*proc.status) { - ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: \"%C\" exited with status %d\n", - proc.cmd.c_str(), *proc.status)); - failed_ = true; - } else { - ACE_DEBUG((LM_INFO, "(%P|%t) \"%C\" exited\n", proc.cmd.c_str())); - } - return 0; - } - - int handle_signal(int, siginfo_t*, ucontext_t*) - { - ACE_DEBUG((LM_INFO, "(%P|%t) SIGINT\n")); - stop_all(); - return end_event_loop(); - } - - struct Proc { - std::string cmd; - pid_t pid = ACE_INVALID_PID; - std::optional status; - - bool ran() const - { - return pid == ACE_INVALID_PID; - } - - bool failed() const - { - return !ran() || status.value_or(1); - } - }; - - bool failed_ = false; - ACE_Process_Manager proc_man_; - std::map procs_; - PowerDevice& pd_; - const std::string mc_path_; -}; - -int main(int argc, char* argv[]) -{ - const auto our_path = fs::path(argv[0]); - const auto mc_path = our_path.parent_path() / ("basic-mc" + our_path.extension().string()); - - PowerDevice pd(device_id); - if (pd.init(domain, argc, argv) != DDS::RETCODE_OK) { - return 1; - } - - Test test(pd, mc_path.string()); - if (ACE_Reactor::instance()->run_reactor_event_loop() != 0) { - return 1; - } - - return test.exit_status(); -} diff --git a/tactical-microgrid-standard/tests/mc-sel/run_test.pl b/tactical-microgrid-standard/tests/mc-sel/run_test.pl new file mode 100755 index 0000000..5a42a97 --- /dev/null +++ b/tactical-microgrid-standard/tests/mc-sel/run_test.pl @@ -0,0 +1,67 @@ +eval '(exit $?0)' && eval 'exec perl -S $0 ${1+"$@"}' + & eval 'exec perl -S $0 $argv:q' + if 0; + +use strict; +use warnings; + +use PerlDDS::Run_Test; + +my $test = new PerlDDS::TestFramework(); + + +sub start_mc { + my $name = shift(); + my $priority = shift(); + my $time = shift(); + + $test->process($name, 'basic-mc', "$name $priority $time"); + $test->start_process($name); + + return $name, time() + $time; +} + +$test->process('dev', 'basic-dev'); +$test->start_process('dev'); + +my $extra = 3; +my $lost_controller = 9 + $extra; # 3s for missed, 6s for lost + +sub expect_new_mc { + my $name = shift(); + my $max = shift(); + + return $test->wait_for('dev', "new controller $name", max_wait => $max); +} + +sub expect_stop_mc { + my $name = shift(); + my $ends_at = shift(); + + $test->stop_process($ends_at - time() + $extra, $name); +} + +sub expect_lost_mc { + my $name = shift(); + my $max = shift(); + + return $test->wait_for('dev', "lost controller $name", max_wait => $max); +} + +my ($mc1, $mc1_ends_at) = start_mc('mc1', 0, 20); +expect_new_mc($mc1, 10); + +sleep(3); +my ($mc2, $mc2_ends_at) = start_mc('mc2', 10, 30); +my ($mc3, $mc3_ends_at) = start_mc('mc3', 0, 30); +my ($mc4, $mc4_ends_at) = start_mc('mc4', 10, 30); + +expect_stop_mc($mc1, $mc1_ends_at); +expect_lost_mc($mc1, $lost_controller); +expect_new_mc($mc3, 5); +expect_stop_mc($mc2, $mc2_ends_at); +expect_stop_mc($mc3, $mc3_ends_at); +expect_stop_mc($mc4, $mc4_ends_at); + +$test->kill_process(2, 'dev'); +exit $test->finish(2); From 126953eecadac1bafaa253b2a4bd06a6c4ee4648 Mon Sep 17 00:00:00 2001 From: Fred Hornsey Date: Fri, 1 Aug 2025 13:58:53 -0500 Subject: [PATCH 02/18] Add Testing to GHA and Tweaks --- .github/workflows/TMS.yml | 38 +++++++++++++++++-- .../common/ControllerSelector.cpp | 8 +--- .../common/ControllerSelector.h | 20 +++++++--- .../tests/mc-sel/common.h | 2 +- 4 files changed, 51 insertions(+), 17 deletions(-) diff --git a/.github/workflows/TMS.yml b/.github/workflows/TMS.yml index 6eca17b..dea584a 100644 --- a/.github/workflows/TMS.yml +++ b/.github/workflows/TMS.yml @@ -226,7 +226,7 @@ jobs: cd OpenDDS . setenv.sh cd ../tactical-microgrid-standard - cmake -B build_static + cmake -B build_static -D BUILD_TESTING=TRUE cmake --build build_static - name: 'Build TMS application with shared libs (Linux /macOS)' if: runner.os == 'Linux' || runner.os == 'macOS' @@ -235,7 +235,7 @@ jobs: cd OpenDDS . setenv.sh cd ../tactical-microgrid-standard - cmake -DBUILD_SHARED_LIBS=yes -B build_shared + cmake -DBUILD_SHARED_LIBS=yes -B build_shared -D BUILD_TESTING=TRUE cmake --build build_shared - name: 'Build TMS application with static libs (Windows)' if: runner.os == 'Windows' @@ -244,7 +244,7 @@ jobs: cd OpenDDS call setenv.cmd cd ..\tactical-microgrid-standard - cmake -B build_static + cmake -B build_static -D BUILD_TESTING=TRUE cmake --build build_static - name: 'Build TMS application with shared libs (Windows)' if: runner.os == 'Windows' @@ -253,5 +253,35 @@ jobs: cd OpenDDS call setenv.cmd cd ..\tactical-microgrid-standard - cmake -DBUILD_SHARED_LIBS=yes -B build_shared + cmake -DBUILD_SHARED_LIBS=yes -B build_shared -D BUILD_TESTING=TRUE cmake --build build_shared + - name: 'Run tests with static libs (Linux / macOS)' + if: runner.os == 'Linux' || runner.os == 'macOS' + run: |- + cd OpenDDS + . setenv.sh + cd ../tactical-microgrid-standard/build_static + ctest --verbose --output-on-failure + - name: 'Run tests with shared libs (Linux /macOS)' + if: runner.os == 'Linux' || runner.os == 'macOS' + run: |- + cd OpenDDS + . setenv.sh + cd ../tactical-microgrid-standard/build_shared + ctest --verbose --output-on-failure + - name: 'Run tests with static libs (Windows)' + if: runner.os == 'Windows' + shell: cmd + run: |- + cd OpenDDS + call setenv.cmd + cd ..\tactical-microgrid-standard\build_static + ctest --verbose --output-on-failure -C Debug + - name: 'Run tests with shared libs (Windows)' + if: runner.os == 'Windows' + shell: cmd + run: |- + cd OpenDDS + call setenv.cmd + cd ..\tactical-microgrid-standard\build_shared + ctest --verbose --output-on-failure -C Debug diff --git a/tactical-microgrid-standard/common/ControllerSelector.cpp b/tactical-microgrid-standard/common/ControllerSelector.cpp index 38e5adc..a433a49 100644 --- a/tactical-microgrid-standard/common/ControllerSelector.cpp +++ b/tactical-microgrid-standard/common/ControllerSelector.cpp @@ -43,11 +43,7 @@ void ControllerSelector::got_device_info(const tms::DeviceInfo& di) if (di.role() == tms::DeviceRole::ROLE_MICROGRID_CONTROLLER && !all_controllers_.count(di.deviceId())) { all_controllers_[di.deviceId()] = TimePoint::min(); - all_controllers_by_priority_.insert(Priority(di)); - ACE_DEBUG((LM_INFO, "Got %C\n", di.deviceId().c_str())); - for (auto it = all_controllers_by_priority_.begin(); it != all_controllers_by_priority_.end(); ++it) { - ACE_DEBUG((LM_INFO, "%d %C\n", it->priority, it->id.c_str())); - } + prioritized_controllers_.insert(PrioritizedController(di)); } } @@ -130,7 +126,7 @@ bool ControllerSelector::select_controller() const TimePoint now = Clock::now(); // Select an available controller with smallest identity alphabetically - for (auto it = all_controllers_by_priority_.begin(); it != all_controllers_by_priority_.end(); ++it) { + for (auto it = prioritized_controllers_.begin(); it != prioritized_controllers_.end(); ++it) { auto mc_info = all_controllers_.find(it->id); const auto last_hb = now - mc_info->second; // TMS spec doesn't specify this. But it should make sure the controller is still available diff --git a/tactical-microgrid-standard/common/ControllerSelector.h b/tactical-microgrid-standard/common/ControllerSelector.h index 7aa86c0..d83f22c 100644 --- a/tactical-microgrid-standard/common/ControllerSelector.h +++ b/tactical-microgrid-standard/common/ControllerSelector.h @@ -144,22 +144,30 @@ class OpenDDS_TMS_Export ControllerSelector : tms::Identity selected_; std::map all_controllers_; - struct Priority { - uint16_t priority; + + struct PrioritizedController { + uint16_t priority = 0; tms::Identity id; - Priority(const tms::DeviceInfo& di) - : priority(di.controlService()->mc()->priorityRanking()) + explicit PrioritizedController(const tms::DeviceInfo& di) + : priority(0) , id(di.deviceId()) { + const auto& cs = di.controlService(); + if (cs) { + const auto& mc = cs->mc(); + if (mc) { + priority = mc->priorityRanking(); + } + } } - bool operator<(const Priority& lhs) const + bool operator<(const PrioritizedController& lhs) const { return std::tie(priority, id) < std::tie(lhs.priority, lhs.id); } }; - std::set all_controllers_by_priority_; + std::set prioritized_controllers_; // Device ID to which this controller selector belong. tms::Identity device_id_; diff --git a/tactical-microgrid-standard/tests/mc-sel/common.h b/tactical-microgrid-standard/tests/mc-sel/common.h index acae97f..c3f10be 100644 --- a/tactical-microgrid-standard/tests/mc-sel/common.h +++ b/tactical-microgrid-standard/tests/mc-sel/common.h @@ -33,7 +33,7 @@ class Exiter : public TimerHandler { int handle_signal(int, siginfo_t*, ucontext_t*) { ACE_DEBUG((LM_INFO, "(%P|%t) SIGINT\n")); - exit(1); + exit(0); return -1; } }; From ecdcad12c33ae7e20d8baa41c83dda6b49c2660e Mon Sep 17 00:00:00 2001 From: Fred Hornsey Date: Mon, 4 Aug 2025 13:17:55 -0500 Subject: [PATCH 03/18] Respond to Review and Use SIGINT --- .../common/ControllerSelector.cpp | 5 +++-- .../common/ControllerSelector.h | 12 ++++++------ tactical-microgrid-standard/controller/Controller.h | 2 +- tactical-microgrid-standard/tests/mc-sel/run_test.pl | 11 +++++++++++ 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/tactical-microgrid-standard/common/ControllerSelector.cpp b/tactical-microgrid-standard/common/ControllerSelector.cpp index a433a49..d4c02ed 100644 --- a/tactical-microgrid-standard/common/ControllerSelector.cpp +++ b/tactical-microgrid-standard/common/ControllerSelector.cpp @@ -1,7 +1,8 @@ #include "ControllerSelector.h" -ControllerSelector::ControllerSelector(const tms::Identity& device_id) - : ControllerCallbacks(lock_) +ControllerSelector::ControllerSelector(const tms::Identity& device_id, ACE_Reactor* reactor) + : TimerHandler(reactor) + , ControllerCallbacks(lock_) , device_id_(device_id) { } diff --git a/tactical-microgrid-standard/common/ControllerSelector.h b/tactical-microgrid-standard/common/ControllerSelector.h index d83f22c..577c0f6 100644 --- a/tactical-microgrid-standard/common/ControllerSelector.h +++ b/tactical-microgrid-standard/common/ControllerSelector.h @@ -95,11 +95,12 @@ class OpenDDS_TMS_Export ControllerCallbacks { * A: If heartbeat is from selected * S: If there's a selectable controller with a recent heartbeat */ -class OpenDDS_TMS_Export ControllerSelector : - public ControllerCallbacks, - public TimerHandler { +class OpenDDS_TMS_Export ControllerSelector + : public TimerHandler + , public ControllerCallbacks +{ public: - explicit ControllerSelector(const tms::Identity& device_id); + explicit ControllerSelector(const tms::Identity& device_id, ACE_Reactor* reactor = nullptr); ~ControllerSelector(); void got_heartbeat(const tms::Heartbeat& hb); @@ -150,8 +151,7 @@ class OpenDDS_TMS_Export ControllerSelector : tms::Identity id; explicit PrioritizedController(const tms::DeviceInfo& di) - : priority(0) - , id(di.deviceId()) + : id(di.deviceId()) { const auto& cs = di.controlService(); if (cs) { diff --git a/tactical-microgrid-standard/controller/Controller.h b/tactical-microgrid-standard/controller/Controller.h index 311ffcc..6bb9a37 100644 --- a/tactical-microgrid-standard/controller/Controller.h +++ b/tactical-microgrid-standard/controller/Controller.h @@ -6,7 +6,7 @@ class Controller : public Handshaking { public: - explicit Controller(const tms::Identity& id, uint16_t priority= 0) + explicit Controller(const tms::Identity& id, uint16_t priority = 0) : Handshaking(id) , priority_(priority) { diff --git a/tactical-microgrid-standard/tests/mc-sel/run_test.pl b/tactical-microgrid-standard/tests/mc-sel/run_test.pl index 5a42a97..85217d3 100755 --- a/tactical-microgrid-standard/tests/mc-sel/run_test.pl +++ b/tactical-microgrid-standard/tests/mc-sel/run_test.pl @@ -34,10 +34,21 @@ sub expect_new_mc { return $test->wait_for('dev', "new controller $name", max_wait => $max); } +sub wait_until { + my $when = shift(); + + my $til = $when - time(); + if ($til > 0) { + sleep($til); + } +} + sub expect_stop_mc { my $name = shift(); my $ends_at = shift(); + wait_until($ends_at); + kill('INT', PerlDDS::TestFramework::_getpid($test->{processes}->{process}->{$name}->{process})); $test->stop_process($ends_at - time() + $extra, $name); } From 12e2e0482630ef6b82e45ccc670097c2085332e0 Mon Sep 17 00:00:00 2001 From: Fred Hornsey Date: Tue, 5 Aug 2025 12:06:11 -0500 Subject: [PATCH 04/18] Remove Testing for macOS macOS seems to have issues with IPv6 Multicast. --- .github/workflows/TMS.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/TMS.yml b/.github/workflows/TMS.yml index dea584a..28399d7 100644 --- a/.github/workflows/TMS.yml +++ b/.github/workflows/TMS.yml @@ -255,15 +255,15 @@ jobs: cd ..\tactical-microgrid-standard cmake -DBUILD_SHARED_LIBS=yes -B build_shared -D BUILD_TESTING=TRUE cmake --build build_shared - - name: 'Run tests with static libs (Linux / macOS)' - if: runner.os == 'Linux' || runner.os == 'macOS' + - name: 'Run tests with static libs (Linux)' + if: runner.os == 'Linux' run: |- cd OpenDDS . setenv.sh cd ../tactical-microgrid-standard/build_static ctest --verbose --output-on-failure - - name: 'Run tests with shared libs (Linux /macOS)' - if: runner.os == 'Linux' || runner.os == 'macOS' + - name: 'Run tests with shared libs (Linux)' + if: runner.os == 'Linux' run: |- cd OpenDDS . setenv.sh From ac8a098c6aabf0def9ba39be9e5c3968ea32e86f Mon Sep 17 00:00:00 2001 From: Fred Hornsey Date: Wed, 6 Aug 2025 13:30:15 -0500 Subject: [PATCH 05/18] Workaround Escaping in opnedds_add_test https://github.com/OpenDDS/OpenDDS/pull/5071 --- tactical-microgrid-standard/tests/mc-sel/run_test.pl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tactical-microgrid-standard/tests/mc-sel/run_test.pl b/tactical-microgrid-standard/tests/mc-sel/run_test.pl index 85217d3..8a928e7 100755 --- a/tactical-microgrid-standard/tests/mc-sel/run_test.pl +++ b/tactical-microgrid-standard/tests/mc-sel/run_test.pl @@ -5,6 +5,10 @@ use strict; use warnings; +# Workaround https://github.com/OpenDDS/OpenDDS/pull/5071 +use Env (ACE_ROOT); +use lib "$ACE_ROOT/bin"; + use PerlDDS::Run_Test; my $test = new PerlDDS::TestFramework(); From 7fca4000bedbc0c8c0807f22a8c98693d1816417 Mon Sep 17 00:00:00 2001 From: Fred Hornsey Date: Wed, 6 Aug 2025 13:45:15 -0500 Subject: [PATCH 06/18] Use qw --- tactical-microgrid-standard/tests/mc-sel/run_test.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tactical-microgrid-standard/tests/mc-sel/run_test.pl b/tactical-microgrid-standard/tests/mc-sel/run_test.pl index 8a928e7..3f27f77 100755 --- a/tactical-microgrid-standard/tests/mc-sel/run_test.pl +++ b/tactical-microgrid-standard/tests/mc-sel/run_test.pl @@ -6,7 +6,7 @@ use warnings; # Workaround https://github.com/OpenDDS/OpenDDS/pull/5071 -use Env (ACE_ROOT); +use Env qw(ACE_ROOT); use lib "$ACE_ROOT/bin"; use PerlDDS::Run_Test; From f77f7889bf3d966e0e0c471df51de7f4f74822c1 Mon Sep 17 00:00:00 2001 From: Fred Hornsey Date: Tue, 12 Aug 2025 14:12:02 -0500 Subject: [PATCH 07/18] Move Windows Stack Change to TMS_Common This allows all programs to use it. --- tactical-microgrid-standard/CMakeLists.txt | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/tactical-microgrid-standard/CMakeLists.txt b/tactical-microgrid-standard/CMakeLists.txt index f5838ea..7258274 100644 --- a/tactical-microgrid-standard/CMakeLists.txt +++ b/tactical-microgrid-standard/CMakeLists.txt @@ -24,6 +24,12 @@ opendds_target_sources(TMS_Common target_link_libraries(TMS_Common PUBLIC OpenDDS::Rtps_Udp) target_compile_features(TMS_Common PUBLIC cxx_std_17) opendds_bigobj(TMS_Common) +# Generated code with complete type objects enabled for the TMS IDL file +# causes stack overflow on Windows. +# Increase the stack size to 2MB (default is 1MB). +if (MSVC) + target_link_options(TMS_Common PUBLIC "/STACK:2097152") +endif() add_library(Commands_Idl) opendds_target_sources(Commands_Idl @@ -87,17 +93,6 @@ add_executable(Distribution target_include_directories(Distribution PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(Distribution PRIVATE PowerSim_Idl) -# Generated code with complete type objects enabled for the TMS IDL file -# causes stack overflow on Windows. -# Increase the stack size to 2MB (default is 1MB). -if (MSVC) - target_link_options(Controller PRIVATE "/STACK:2097152") - target_link_options(CLI PRIVATE "/STACK:2097152") - target_link_options(Source PRIVATE "/STACK:2097152") - target_link_options(Load PRIVATE "/STACK:2097152") - target_link_options(Distribution PRIVATE "/STACK:2097152") -endif() - if(BUILD_TESTING) add_subdirectory(tests) endif() From 9abe3afadbfb5771706149207dd6990f435d7070 Mon Sep 17 00:00:00 2001 From: Fred Hornsey Date: Fri, 15 Aug 2025 14:38:45 -0500 Subject: [PATCH 08/18] Don't do SIGINT on Windows and try trace on Linux --- .github/workflows/TMS.yml | 15 ++++++++++++++- .../tests/mc-sel/run_test.pl | 8 +++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.github/workflows/TMS.yml b/.github/workflows/TMS.yml index 28399d7..be870d2 100644 --- a/.github/workflows/TMS.yml +++ b/.github/workflows/TMS.yml @@ -58,11 +58,22 @@ jobs: fetch-depth: 1 submodules: true + # Setup stacktrace + - name: install gdb + run: | + sudo apt-get update + sudo apt-get install gdb + echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope + - name: change core file pattern + run: | + sudo sysctl -w kernel.core_pattern=core.%e.%p + echo Core file pattern set to: + cat /proc/sys/kernel/core_pattern + # Get 3rd-party Dependencies - name: 'Install openssl, xerces (Linux)' if: runner.os == 'Linux' run: |- - sudo apt-get update sudo apt-get -y install libssl-dev libxerces-c-dev - name: 'Install xerces (macOS)' if: runner.os == 'macOS' @@ -258,6 +269,7 @@ jobs: - name: 'Run tests with static libs (Linux)' if: runner.os == 'Linux' run: |- + ulimit -c unlimited cd OpenDDS . setenv.sh cd ../tactical-microgrid-standard/build_static @@ -265,6 +277,7 @@ jobs: - name: 'Run tests with shared libs (Linux)' if: runner.os == 'Linux' run: |- + ulimit -c unlimited cd OpenDDS . setenv.sh cd ../tactical-microgrid-standard/build_shared diff --git a/tactical-microgrid-standard/tests/mc-sel/run_test.pl b/tactical-microgrid-standard/tests/mc-sel/run_test.pl index 3f27f77..f598ab0 100755 --- a/tactical-microgrid-standard/tests/mc-sel/run_test.pl +++ b/tactical-microgrid-standard/tests/mc-sel/run_test.pl @@ -12,7 +12,7 @@ use PerlDDS::Run_Test; my $test = new PerlDDS::TestFramework(); - +$test->ignore_error("failed send: host unreachable"); sub start_mc { my $name = shift(); @@ -33,7 +33,7 @@ sub start_mc { sub expect_new_mc { my $name = shift(); - my $max = shift(); + my $max = shift() + $extra; return $test->wait_for('dev', "new controller $name", max_wait => $max); } @@ -52,7 +52,9 @@ sub expect_stop_mc { my $ends_at = shift(); wait_until($ends_at); - kill('INT', PerlDDS::TestFramework::_getpid($test->{processes}->{process}->{$name}->{process})); + if ($^O ne 'MSWin32') { + kill('INT', PerlDDS::TestFramework::_getpid($test->{processes}->{process}->{$name}->{process})); + } $test->stop_process($ends_at - time() + $extra, $name); } From dec436c02dc509c48086f502d4dbe5e352b89211 Mon Sep 17 00:00:00 2001 From: Fred Hornsey Date: Fri, 15 Aug 2025 15:17:16 -0500 Subject: [PATCH 09/18] Add Missing OS if --- .github/workflows/TMS.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/TMS.yml b/.github/workflows/TMS.yml index be870d2..ab48b27 100644 --- a/.github/workflows/TMS.yml +++ b/.github/workflows/TMS.yml @@ -60,11 +60,13 @@ jobs: # Setup stacktrace - name: install gdb + if: runner.os == 'Linux' run: | sudo apt-get update sudo apt-get install gdb echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope - name: change core file pattern + if: runner.os == 'Linux' run: | sudo sysctl -w kernel.core_pattern=core.%e.%p echo Core file pattern set to: From d3e3ea05c4d861a1c3902f8c198b4eaaaa78d037 Mon Sep 17 00:00:00 2001 From: Fred Hornsey Date: Fri, 15 Aug 2025 20:27:14 -0500 Subject: [PATCH 10/18] Try Kill(..., 0) on Windows --- .../tests/mc-sel/run_test.pl | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tactical-microgrid-standard/tests/mc-sel/run_test.pl b/tactical-microgrid-standard/tests/mc-sel/run_test.pl index f598ab0..d24afbd 100755 --- a/tactical-microgrid-standard/tests/mc-sel/run_test.pl +++ b/tactical-microgrid-standard/tests/mc-sel/run_test.pl @@ -5,6 +5,11 @@ use strict; use warnings; +if ($^O eq 'MSWin32') { + eval('use Win32::Process;'); + die($@) if ($@); +} + # Workaround https://github.com/OpenDDS/OpenDDS/pull/5071 use Env qw(ACE_ROOT); use lib "$ACE_ROOT/bin"; @@ -52,8 +57,12 @@ sub expect_stop_mc { my $ends_at = shift(); wait_until($ends_at); - if ($^O ne 'MSWin32') { - kill('INT', PerlDDS::TestFramework::_getpid($test->{processes}->{process}->{$name}->{process})); + my $proc = $test->{processes}->{process}->{$name}->{process}->{PROCESS}; + if ($^O eq 'MSWin32') { + Win32::Process::Kill($proc, 0); + } + else { + kill('INT', $proc); } $test->stop_process($ends_at - time() + $extra, $name); } From 787324c7279e5e518f0a5c2c4813b57b7f2d9c9f Mon Sep 17 00:00:00 2001 From: Fred Hornsey Date: Fri, 22 Aug 2025 13:32:24 -0500 Subject: [PATCH 11/18] Explicit Shutdown in Test --- .../common/Handshaking.cpp | 26 +++++++++++++++---- .../common/Handshaking.h | 5 ++++ .../controller/Controller.h | 1 - .../power_devices/PowerDevice.h | 5 ++++ .../tests/mc-sel/basic-mc.cpp | 7 +++-- .../tests/mc-sel/common.h | 15 +++++++++-- .../tests/mc-sel/run_test.pl | 2 +- 7 files changed, 48 insertions(+), 13 deletions(-) diff --git a/tactical-microgrid-standard/common/Handshaking.cpp b/tactical-microgrid-standard/common/Handshaking.cpp index 205968b..cfb8461 100644 --- a/tactical-microgrid-standard/common/Handshaking.cpp +++ b/tactical-microgrid-standard/common/Handshaking.cpp @@ -14,11 +14,27 @@ Handshaking::~Handshaking() { - if (participant_) { - participant_->delete_contained_entities(); - } - if (dpf_) { - dpf_->delete_participant(participant_); + delete_all_entities(); +} + +void Handshaking::delete_all_entities() +{ + delete_extra_entities(); + delete_entities(participant_); +} + +void Handshaking::delete_extra_entities() +{ +} + +void Handshaking::delete_entities(DDS::DomainParticipant_var& part) +{ + if (part) { + part->delete_contained_entities(); + if (dpf_) { + dpf_->delete_participant(part); + } + part = nullptr; } } diff --git a/tactical-microgrid-standard/common/Handshaking.h b/tactical-microgrid-standard/common/Handshaking.h index bf4c1f1..992e4e1 100644 --- a/tactical-microgrid-standard/common/Handshaking.h +++ b/tactical-microgrid-standard/common/Handshaking.h @@ -61,7 +61,12 @@ class OpenDDS_TMS_Export Handshaking : public TimerHandler { return device_id_; } + void delete_all_entities(); + protected: + void delete_entities(DDS::DomainParticipant_var& part); + virtual void delete_extra_entities(); + virtual tms::DeviceInfo get_device_info() const { tms::DeviceInfo device_info; diff --git a/tactical-microgrid-standard/controller/Controller.h b/tactical-microgrid-standard/controller/Controller.h index 6bb9a37..7ad25e7 100644 --- a/tactical-microgrid-standard/controller/Controller.h +++ b/tactical-microgrid-standard/controller/Controller.h @@ -31,7 +31,6 @@ class Controller : public Handshaking { void heartbeat_cb(const tms::Heartbeat& hb, const DDS::SampleInfo& si); tms::DeviceInfo populate_device_info() const; -private: mutable std::mutex mut_; PowerDevices power_devices_; uint16_t priority_; diff --git a/tactical-microgrid-standard/power_devices/PowerDevice.h b/tactical-microgrid-standard/power_devices/PowerDevice.h index 02094f8..18f4fdb 100644 --- a/tactical-microgrid-standard/power_devices/PowerDevice.h +++ b/tactical-microgrid-standard/power_devices/PowerDevice.h @@ -92,6 +92,11 @@ class PowerSim_Idl_Export PowerDevice : public Handshaking { return ret; } + void delete_extra_entities() + { + delete_entities(sim_participant_); + } + // Concrete power device should override this function depending on their role. virtual tms::DeviceInfo populate_device_info() const; diff --git a/tactical-microgrid-standard/tests/mc-sel/basic-mc.cpp b/tactical-microgrid-standard/tests/mc-sel/basic-mc.cpp index f1d2b7f..e2214c0 100644 --- a/tactical-microgrid-standard/tests/mc-sel/basic-mc.cpp +++ b/tactical-microgrid-standard/tests/mc-sel/basic-mc.cpp @@ -4,19 +4,18 @@ int main(int argc, char* argv[]) { - if (argc < 4) { - ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: requires device id, priority, and time duration arguments\n")); + if (argc < 3) { + ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: requires device id and priority time duration arguments\n")); return 1; } const std::string device_id = argv[1]; const uint16_t priority = std::stoi(argv[2]); - const Sec exit_after(std::stoi(argv[3])); Controller mc(device_id, priority); if (mc.init(domain, argc, argv) != DDS::RETCODE_OK) { return 1; } - Exiter exiter(mc, exit_after); + Exiter exiter(mc); return mc.run(); } diff --git a/tactical-microgrid-standard/tests/mc-sel/common.h b/tactical-microgrid-standard/tests/mc-sel/common.h index c3f10be..aa0c52a 100644 --- a/tactical-microgrid-standard/tests/mc-sel/common.h +++ b/tactical-microgrid-standard/tests/mc-sel/common.h @@ -4,6 +4,7 @@ const int domain = 1; struct Timeout { + bool exit_after = true; static const char* name() { return "Timeout"; } }; @@ -11,6 +12,7 @@ class Exiter : public TimerHandler { public: Exiter(Handshaking& handshaking, Sec exit_after = Sec(0)) : TimerHandler(handshaking.get_reactor()) + , handshaking_(handshaking) { reactor_->register_handler(SIGINT, this); if (exit_after > Sec(0)) { @@ -19,10 +21,18 @@ class Exiter : public TimerHandler { } private: + Handshaking& handshaking_; + + void shutdown() + { + handshaking_.delete_all_entities(); + TheServiceParticipant->shutdown(); + } + void timer_fired(Timer& t) { ACE_DEBUG((LM_INFO, "(%P|%t) Timeout\n")); - exit(0); + shutdown(); } void any_timer_fired(AnyTimer timer) @@ -33,7 +43,8 @@ class Exiter : public TimerHandler { int handle_signal(int, siginfo_t*, ucontext_t*) { ACE_DEBUG((LM_INFO, "(%P|%t) SIGINT\n")); - exit(0); + shutdown(); + reactor_->end_reactor_event_loop(); return -1; } }; diff --git a/tactical-microgrid-standard/tests/mc-sel/run_test.pl b/tactical-microgrid-standard/tests/mc-sel/run_test.pl index d24afbd..878a83b 100755 --- a/tactical-microgrid-standard/tests/mc-sel/run_test.pl +++ b/tactical-microgrid-standard/tests/mc-sel/run_test.pl @@ -24,7 +24,7 @@ sub start_mc { my $priority = shift(); my $time = shift(); - $test->process($name, 'basic-mc', "$name $priority $time"); + $test->process($name, 'basic-mc', "$name $priority"); $test->start_process($name); return $name, time() + $time; From ae0446fc6ae4de57d904f8efca76b670faa706d7 Mon Sep 17 00:00:00 2001 From: Fred Hornsey Date: Fri, 29 Aug 2025 23:08:19 -0500 Subject: [PATCH 12/18] More Fixes For Occasional Failures - Logging for heartbeats. - Give more time for mc3 to be selected. - Don't base mc kill timeout on when it was supposed to stop. - Use double-based duration to avoid situations when time duration math results are rounded to zero. --- .../common/ControllerSelector.cpp | 14 ++++++++++++++ .../common/TimerHandler.h | 16 ++++++++++++++-- .../tests/mc-sel/run_test.pl | 4 ++-- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/tactical-microgrid-standard/common/ControllerSelector.cpp b/tactical-microgrid-standard/common/ControllerSelector.cpp index d4c02ed..a4e8b34 100644 --- a/tactical-microgrid-standard/common/ControllerSelector.cpp +++ b/tactical-microgrid-standard/common/ControllerSelector.cpp @@ -1,5 +1,7 @@ #include "ControllerSelector.h" +#include + ControllerSelector::ControllerSelector(const tms::Identity& device_id, ACE_Reactor* reactor) : TimerHandler(reactor) , ControllerCallbacks(lock_) @@ -16,6 +18,9 @@ void ControllerSelector::got_heartbeat(const tms::Heartbeat& hb) Guard g(lock_); auto it = all_controllers_.find(hb.deviceId()); if (it != all_controllers_.end()) { + ACE_DEBUG((LM_DEBUG, "(%P|%t) ControllerSelector::got_heartbeat: from mc \"%C\"\n", + hb.deviceId().c_str())); + it->second = Clock::now(); // Update last heartbeat cancel(); @@ -32,6 +37,9 @@ void ControllerSelector::got_heartbeat(const tms::Heartbeat& hb) schedule_once(MissedHeartbeat{}, heartbeat_deadline); } } + } else { + ACE_DEBUG((LM_DEBUG, "(%P|%t) ControllerSelector::got_heartbeat: from unknown \"%C\"\n", + hb.deviceId().c_str())); } } @@ -129,12 +137,18 @@ bool ControllerSelector::select_controller() // Select an available controller with smallest identity alphabetically for (auto it = prioritized_controllers_.begin(); it != prioritized_controllers_.end(); ++it) { auto mc_info = all_controllers_.find(it->id); + const auto& id = mc_info->first; const auto last_hb = now - mc_info->second; // TMS spec doesn't specify this. But it should make sure the controller is still available // i.e., last heartbeat received within 3 seconds. if (last_hb < heartbeat_deadline) { select(mc_info->first, std::chrono::duration_cast(last_hb)); break; + } else { + std::ostringstream oss; + oss << std::chrono::duration_cast(last_hb - heartbeat_deadline).count(); + ACE_DEBUG((LM_DEBUG, "(%P|%t) ControllerSelector::select_controller: \"%C\" missed heatbeat by %Cs\n", + id.c_str(), oss.str().c_str())); } } return false; diff --git a/tactical-microgrid-standard/common/TimerHandler.h b/tactical-microgrid-standard/common/TimerHandler.h index 2bfced4..608db71 100644 --- a/tactical-microgrid-standard/common/TimerHandler.h +++ b/tactical-microgrid-standard/common/TimerHandler.h @@ -16,7 +16,7 @@ #include #include -using Sec = std::chrono::seconds; +using Sec = std::chrono::duration; using Clock = std::chrono::system_clock; using TimePoint = std::chrono::time_point; using TimerId = long; @@ -25,6 +25,18 @@ constexpr TimerId null_timer_id = 0; using Mutex = std::recursive_mutex; using Guard = std::lock_guard; +// Workaround https://github.com/DOCGroup/ACE_TAO/pull/2462 +template +inline ACE_Time_Value to_time_value(const std::chrono::duration& duration) +{ + using namespace std::chrono; + + const seconds s{duration_cast(duration)}; + const microseconds usec{duration_cast(duration - s)}; + return ACE_Time_Value(ACE_Utils::truncate_cast(s.count()), + ACE_Utils::truncate_cast(usec.count())); +} + // EventType must implement: static const char* name(); template struct Timer { @@ -172,7 +184,7 @@ class TimerHandler : public ACE_Event_Handler, protected TimerHolder Guard g(lock_); assert_inactive(timer); const TimerId id = reactor_->schedule_timer( - this, &timer->key, ACE_Time_Value(timer->delay), ACE_Time_Value(timer->period)); + this, &timer->key, to_time_value(timer->delay), to_time_value(timer->period)); timer->activate(id); active_timers_[timer->key] = timer; } diff --git a/tactical-microgrid-standard/tests/mc-sel/run_test.pl b/tactical-microgrid-standard/tests/mc-sel/run_test.pl index 878a83b..4300d92 100755 --- a/tactical-microgrid-standard/tests/mc-sel/run_test.pl +++ b/tactical-microgrid-standard/tests/mc-sel/run_test.pl @@ -64,7 +64,7 @@ sub expect_stop_mc { else { kill('INT', $proc); } - $test->stop_process($ends_at - time() + $extra, $name); + $test->stop_process(6, $name); } sub expect_lost_mc { @@ -84,7 +84,7 @@ sub expect_lost_mc { expect_stop_mc($mc1, $mc1_ends_at); expect_lost_mc($mc1, $lost_controller); -expect_new_mc($mc3, 5); +expect_new_mc($mc3, 10); expect_stop_mc($mc2, $mc2_ends_at); expect_stop_mc($mc3, $mc3_ends_at); expect_stop_mc($mc4, $mc4_ends_at); From 991abb20db6fcb39e55d4e1207ecacbfb6121a4f Mon Sep 17 00:00:00 2001 From: Fred Hornsey Date: Tue, 2 Sep 2025 13:15:23 -0500 Subject: [PATCH 13/18] Try Removing IPv6 and Log Ignore --- .github/workflows/TMS.yml | 2 +- tactical-microgrid-standard/tests/mc-sel/run_test.pl | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/TMS.yml b/.github/workflows/TMS.yml index ab48b27..e089445 100644 --- a/.github/workflows/TMS.yml +++ b/.github/workflows/TMS.yml @@ -137,7 +137,7 @@ jobs: echo "TAO_ROOT=$GITHUB_WORKSPACE\\ACE_TAO\\TAO" >> $GITHUB_ENV echo "DDS_ROOT=$GITHUB_WORKSPACE\\OpenDDS" >> $GITHUB_ENV echo "MPC_ROOT=$GITHUB_WORKSPACE\\MPC" >> $GITHUB_ENV - CONFIG_OPTIONS+=" --no-tests --std=c++17 --ipv6 --security --xerces3=\"$VCPKG_INSTALLED_DIR\\x64-windows\" --openssl=\"$VCPKG_INSTALLED_DIR\\x64-windows\"" + CONFIG_OPTIONS+=" --no-tests --std=c++17 --security --xerces3=\"$VCPKG_INSTALLED_DIR\\x64-windows\" --openssl=\"$VCPKG_INSTALLED_DIR\\x64-windows\"" echo "CONFIG_OPTIONS=$CONFIG_OPTIONS" >> $GITHUB_ENV export COMPILER_VERSION=$("C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" -property catalog_productDisplayVersion) echo "COMPILER_VERSION=$COMPILER_VERSION" >> $GITHUB_ENV diff --git a/tactical-microgrid-standard/tests/mc-sel/run_test.pl b/tactical-microgrid-standard/tests/mc-sel/run_test.pl index 4300d92..35b4e5f 100755 --- a/tactical-microgrid-standard/tests/mc-sel/run_test.pl +++ b/tactical-microgrid-standard/tests/mc-sel/run_test.pl @@ -17,7 +17,6 @@ use PerlDDS::Run_Test; my $test = new PerlDDS::TestFramework(); -$test->ignore_error("failed send: host unreachable"); sub start_mc { my $name = shift(); From d0694b847e0fea08676763df05c584e0320cd069 Mon Sep 17 00:00:00 2001 From: Fred Hornsey Date: Wed, 3 Sep 2025 14:53:40 -0500 Subject: [PATCH 14/18] Put --ipv6 Back, Try TARGET_RUNTIME_DLL_DIRS --- .github/workflows/TMS.yml | 2 +- tactical-microgrid-standard/tests/mc-sel/CMakeLists.txt | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/TMS.yml b/.github/workflows/TMS.yml index e089445..ab48b27 100644 --- a/.github/workflows/TMS.yml +++ b/.github/workflows/TMS.yml @@ -137,7 +137,7 @@ jobs: echo "TAO_ROOT=$GITHUB_WORKSPACE\\ACE_TAO\\TAO" >> $GITHUB_ENV echo "DDS_ROOT=$GITHUB_WORKSPACE\\OpenDDS" >> $GITHUB_ENV echo "MPC_ROOT=$GITHUB_WORKSPACE\\MPC" >> $GITHUB_ENV - CONFIG_OPTIONS+=" --no-tests --std=c++17 --security --xerces3=\"$VCPKG_INSTALLED_DIR\\x64-windows\" --openssl=\"$VCPKG_INSTALLED_DIR\\x64-windows\"" + CONFIG_OPTIONS+=" --no-tests --std=c++17 --ipv6 --security --xerces3=\"$VCPKG_INSTALLED_DIR\\x64-windows\" --openssl=\"$VCPKG_INSTALLED_DIR\\x64-windows\"" echo "CONFIG_OPTIONS=$CONFIG_OPTIONS" >> $GITHUB_ENV export COMPILER_VERSION=$("C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" -property catalog_productDisplayVersion) echo "COMPILER_VERSION=$COMPILER_VERSION" >> $GITHUB_ENV diff --git a/tactical-microgrid-standard/tests/mc-sel/CMakeLists.txt b/tactical-microgrid-standard/tests/mc-sel/CMakeLists.txt index b2f0274..c7b2925 100644 --- a/tactical-microgrid-standard/tests/mc-sel/CMakeLists.txt +++ b/tactical-microgrid-standard/tests/mc-sel/CMakeLists.txt @@ -15,4 +15,7 @@ add_executable(basic-mc basic-mc.cpp) target_link_libraries(basic-mc PRIVATE Commands_Idl) -opendds_add_test(ARGS remove_dcs_after=0 test_verbose=1) +opendds_add_test( + EXTRA_LIB_DIRS "$" + ARGS remove_dcs_after=0 test_verbose=1 +) From 0f536596eee308bff057b9fea848d41ad68e2f3e Mon Sep 17 00:00:00 2001 From: Fred Hornsey Date: Wed, 3 Sep 2025 14:54:21 -0500 Subject: [PATCH 15/18] Try mc-sel Test on macOS Without IPv6 --- .github/workflows/TMS.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/TMS.yml b/.github/workflows/TMS.yml index ab48b27..c611a55 100644 --- a/.github/workflows/TMS.yml +++ b/.github/workflows/TMS.yml @@ -109,13 +109,15 @@ jobs: echo "DDS_ROOT=$GITHUB_WORKSPACE/OpenDDS" >> $GITHUB_ENV echo "MPC_ROOT=$GITHUB_WORKSPACE/MPC" >> $GITHUB_ENV echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/ACE_TAO/ACE/lib:$GITHUB_WORKSPACE/OpenDDS/lib" >> $GITHUB_ENV - CONFIG_OPTIONS+=" --no-tests --std=c++17 --ipv6 --security" + CONFIG_OPTIONS+=" --no-tests --std=c++17 --security" if [ '${{ runner.os }}' == 'macOS' ]; then if [ '${{ matrix.runner }}' == 'macos-13' ]; then CONFIG_OPTIONS+=" --xerces3=$(brew --prefix xerces-c) --openssl=/usr/local/opt/openssl@1.1" else CONFIG_OPTIONS+=" --xerces3=$(brew --prefix xerces-c) --openssl=$(brew --prefix openssl)" fi + else + CONFIG_OPTIONS+=" --ipv6" fi echo "CONFIG_OPTIONS=$CONFIG_OPTIONS" >> $GITHUB_ENV export COMPILER_VERSION=$(c++ --version 2>&1 | head -n 1) @@ -268,16 +270,16 @@ jobs: cd ..\tactical-microgrid-standard cmake -DBUILD_SHARED_LIBS=yes -B build_shared -D BUILD_TESTING=TRUE cmake --build build_shared - - name: 'Run tests with static libs (Linux)' - if: runner.os == 'Linux' + - name: 'Run tests with static libs (Linux/macOS)' + if: runner.os == 'Linux' || runner.os == 'macOS' run: |- ulimit -c unlimited cd OpenDDS . setenv.sh cd ../tactical-microgrid-standard/build_static ctest --verbose --output-on-failure - - name: 'Run tests with shared libs (Linux)' - if: runner.os == 'Linux' + - name: 'Run tests with shared libs (Linux/macOS)' + if: runner.os == 'Linux' || runner.os == 'macOS' run: |- ulimit -c unlimited cd OpenDDS From c1fe616449c2e7832714797b68cf244e9cefe47f Mon Sep 17 00:00:00 2001 From: Fred Hornsey Date: Wed, 3 Sep 2025 16:31:36 -0500 Subject: [PATCH 16/18] Revert "Try mc-sel Test on macOS Without IPv6" This reverts commit 0f536596eee308bff057b9fea848d41ad68e2f3e. The mc-sel Test can work in GHA, but the controllers seem to have a high chance of crashing when the run_test.pl sends a SIGINT. --- .github/workflows/TMS.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/TMS.yml b/.github/workflows/TMS.yml index c611a55..ab48b27 100644 --- a/.github/workflows/TMS.yml +++ b/.github/workflows/TMS.yml @@ -109,15 +109,13 @@ jobs: echo "DDS_ROOT=$GITHUB_WORKSPACE/OpenDDS" >> $GITHUB_ENV echo "MPC_ROOT=$GITHUB_WORKSPACE/MPC" >> $GITHUB_ENV echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/ACE_TAO/ACE/lib:$GITHUB_WORKSPACE/OpenDDS/lib" >> $GITHUB_ENV - CONFIG_OPTIONS+=" --no-tests --std=c++17 --security" + CONFIG_OPTIONS+=" --no-tests --std=c++17 --ipv6 --security" if [ '${{ runner.os }}' == 'macOS' ]; then if [ '${{ matrix.runner }}' == 'macos-13' ]; then CONFIG_OPTIONS+=" --xerces3=$(brew --prefix xerces-c) --openssl=/usr/local/opt/openssl@1.1" else CONFIG_OPTIONS+=" --xerces3=$(brew --prefix xerces-c) --openssl=$(brew --prefix openssl)" fi - else - CONFIG_OPTIONS+=" --ipv6" fi echo "CONFIG_OPTIONS=$CONFIG_OPTIONS" >> $GITHUB_ENV export COMPILER_VERSION=$(c++ --version 2>&1 | head -n 1) @@ -270,16 +268,16 @@ jobs: cd ..\tactical-microgrid-standard cmake -DBUILD_SHARED_LIBS=yes -B build_shared -D BUILD_TESTING=TRUE cmake --build build_shared - - name: 'Run tests with static libs (Linux/macOS)' - if: runner.os == 'Linux' || runner.os == 'macOS' + - name: 'Run tests with static libs (Linux)' + if: runner.os == 'Linux' run: |- ulimit -c unlimited cd OpenDDS . setenv.sh cd ../tactical-microgrid-standard/build_static ctest --verbose --output-on-failure - - name: 'Run tests with shared libs (Linux/macOS)' - if: runner.os == 'Linux' || runner.os == 'macOS' + - name: 'Run tests with shared libs (Linux)' + if: runner.os == 'Linux' run: |- ulimit -c unlimited cd OpenDDS From 25de2931fea5b31a884a3d1a5c9639bc2747b405 Mon Sep 17 00:00:00 2001 From: Fred Hornsey Date: Sat, 6 Sep 2025 00:24:01 -0500 Subject: [PATCH 17/18] Configurable Debug Messages --- tactical-microgrid-standard/README.md | 18 ++- .../common/Configurable.h | 111 ++++++++++++++++++ .../common/ControllerSelector.cpp | 44 +++++-- .../common/ControllerSelector.h | 7 ++ .../common/TimerHandler.h | 2 +- .../controller/Controller.cpp | 54 ++++++--- .../controller/Controller.h | 15 ++- .../power_devices/PowerDevice.cpp | 7 +- .../tests/mc-sel/run_test.pl | 16 ++- 9 files changed, 231 insertions(+), 43 deletions(-) create mode 100644 tactical-microgrid-standard/common/Configurable.h diff --git a/tactical-microgrid-standard/README.md b/tactical-microgrid-standard/README.md index e14177a..1b64694 100644 --- a/tactical-microgrid-standard/README.md +++ b/tactical-microgrid-standard/README.md @@ -32,7 +32,7 @@ cmake --build - `cli/`: Command-line interface implementation - `cli_idl/`: Interface definition files for CLI commands - `common/`: Shared libraries and utilities - - TMS data model definitions (mil-std-3071_data_model.idl) + - TMS data model definitions (`mil-std-3071_data_model.idl`) - TMS Handshaking function - TMS microgrid controller selection - TMS QoS profiles @@ -46,13 +46,27 @@ cmake --build ## Testing -Tests can be run using CTest after building the project: +Tests can be run using CTest after building the project with `-D BUILD_TESTING=TRUE`: ```bash cd ctest ``` +## Configuration + +These programs support the following OpenDDS configuration properties. There +are multiple ways to set these, see +[OpenDDS runtime configuration](https://opendds.readthedocs.io/en/master/devguide/run_time_configuration.html) +for more info. + +- `TMS_SELECTOR_DEBUG=` + - Enables debug logging of the microgrid controller selection process of power devices. + - Command line option example: `-OpenDDS-tms-selector-debug true` +- `TMS_CONTROLLER_DEBUG=` + - Enables debug logging of what devices the controller has learned about. + - Command line option example: `-OpenDDS-tms-controller-debug true` + ## References - [Tactical Microgrid Standard (MIL-STD-3071)](https://quicksearch.dla.mil/qsDocDetails.aspx?ident_number=285095) diff --git a/tactical-microgrid-standard/common/Configurable.h b/tactical-microgrid-standard/common/Configurable.h new file mode 100644 index 0000000..76a2e21 --- /dev/null +++ b/tactical-microgrid-standard/common/Configurable.h @@ -0,0 +1,111 @@ +#ifndef TMS_COMMON_CONFIGURABLE +#define TMS_COMMON_CONFIGURABLE + +#include +#include +#include + +#include + +class Configurable { +public: + using Mutex = std::recursive_mutex; + using Guard = std::lock_guard; + + Configurable(const std::string& prefix) + : config_prefix_(prefix) + { + } + + virtual ~Configurable() + { + Guard g(config_lock_); + + if (config_reader_) { + TheServiceParticipant->config_topic()->disconnect(config_reader_); + } + } + + void setup_config() + { + Guard g(config_lock_); + + if (!config_reader_) { + config_listener_ = OpenDDS::DCPS::make_rch(this, config_prefix_ + "_"); + config_reader_ = OpenDDS::DCPS::make_rch( + TheServiceParticipant->config_store()->datareader_qos(), config_listener_); + TheServiceParticipant->config_topic()->connect(config_reader_); + } + } + + const std::string& config_prefix() const + { + return config_prefix_; + } + + static bool convert_bool(const OpenDDS::DCPS::ConfigPair& pair, bool& value) + { + DDS::Boolean x = 0; + if (pair.value() == "true") { + value = true; + return true; + } else if (pair.value() == "false") { + value = false; + return true; + } else if (OpenDDS::DCPS::convertToInteger(pair.value(), x)) { + value = x; + return true; + } else { + ACE_ERROR((LM_WARNING, "(%P|%t) WARNING: Configurable::convert_bool: failed to parse boolean for %C=%C\n", + pair.key().c_str(), pair.value().c_str())); + return false; + } + } + + virtual bool got_config(const std::string& name, const OpenDDS::DCPS::ConfigPair& pair) = 0; + +private: + class ConfigListener : public OpenDDS::DCPS::ConfigListener { + public: + explicit ConfigListener(Configurable* configurable, const std::string& prefix) + : OpenDDS::DCPS::ConfigListener(TheServiceParticipant->job_queue()) + , configurable_(*configurable) + , prefix_(prefix) + { + } + + void on_data_available(InternalDataReader_rch reader) override + { + OpenDDS::DCPS::ConfigReader::SampleSequence samples; + OpenDDS::DCPS::InternalSampleInfoSequence infos; + reader->read(samples, infos, DDS::LENGTH_UNLIMITED, + DDS::NOT_READ_SAMPLE_STATE, DDS::ANY_VIEW_STATE, DDS::ANY_INSTANCE_STATE); + for (size_t idx = 0; idx != samples.size(); ++idx) { + const auto& info = infos[idx]; + if (info.valid_data) { + const auto& pair = samples[idx]; + // Match key to prefix and pass rest of key as a short name + if (pair.key().substr(0, prefix_.length()) == prefix_) { + const auto name = pair.key().substr(prefix_.length()); + if (!configurable_.got_config(name, pair)) { + ACE_ERROR((LM_WARNING, "(%P|%t) WARNING: Configurable::ConfigListener::on_data_available: " + "%C is not a valid property for %C\n", + name.c_str(), configurable_.config_prefix().c_str())); + } + } + } + } + } + + private: + Configurable& configurable_; + const std::string prefix_; + }; + + mutable Mutex config_lock_; + const std::string config_prefix_; + OpenDDS::DCPS::RcHandle config_reader_; + OpenDDS::DCPS::RcHandle config_listener_; +}; + +#endif diff --git a/tactical-microgrid-standard/common/ControllerSelector.cpp b/tactical-microgrid-standard/common/ControllerSelector.cpp index a4e8b34..4b362a9 100644 --- a/tactical-microgrid-standard/common/ControllerSelector.cpp +++ b/tactical-microgrid-standard/common/ControllerSelector.cpp @@ -5,6 +5,7 @@ ControllerSelector::ControllerSelector(const tms::Identity& device_id, ACE_Reactor* reactor) : TimerHandler(reactor) , ControllerCallbacks(lock_) + , Configurable("TMS_SELECTOR") , device_id_(device_id) { } @@ -13,13 +14,33 @@ ControllerSelector::~ControllerSelector() { } +bool ControllerSelector::got_config(const std::string& name, const OpenDDS::DCPS::ConfigPair& pair) +{ + if (name == "DEBUG") { + bool tmp; + if (convert_bool(pair, tmp)) { + set_debug(tmp); + } + return true; + } + return false; +} + +void ControllerSelector::set_debug(bool value) +{ + Guard g(lock_); + debug_ = value; +} + void ControllerSelector::got_heartbeat(const tms::Heartbeat& hb) { Guard g(lock_); auto it = all_controllers_.find(hb.deviceId()); if (it != all_controllers_.end()) { - ACE_DEBUG((LM_DEBUG, "(%P|%t) ControllerSelector::got_heartbeat: from mc \"%C\"\n", - hb.deviceId().c_str())); + if (debug_) { + ACE_DEBUG((LM_DEBUG, "(%P|%t) ControllerSelector::got_heartbeat: from mc \"%C\"\n", + hb.deviceId().c_str())); + } it->second = Clock::now(); // Update last heartbeat cancel(); @@ -37,7 +58,7 @@ void ControllerSelector::got_heartbeat(const tms::Heartbeat& hb) schedule_once(MissedHeartbeat{}, heartbeat_deadline); } } - } else { + } else if (debug_) { ACE_DEBUG((LM_DEBUG, "(%P|%t) ControllerSelector::got_heartbeat: from unknown \"%C\"\n", hb.deviceId().c_str())); } @@ -60,8 +81,11 @@ void ControllerSelector::timer_fired(Timer& timer) { Guard g(lock_); const auto& mc_id = timer.arg.id; - ACE_DEBUG((LM_INFO, "(%P|%t) INFO: ControllerSelector::timed_event(NewController): " - "triggered by \"%C\" heartbeat\n", mc_id.c_str())); + + if (debug_) { + ACE_DEBUG((LM_DEBUG, "(%P|%t) ControllerSelector::timed_event(NewController): " + "triggered by \"%C\" heartbeat\n", mc_id.c_str())); + } if (all_controllers_.find(mc_id) == all_controllers_.end()) { ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: ControllerSelector::timed_event(NewController): Controller \"%C\" not found!\n", @@ -81,8 +105,10 @@ void ControllerSelector::timer_fired(Timer& timer) { Guard g(lock_); const auto& timer_id = timer.id; - ACE_DEBUG((LM_INFO, "(%P|%t) INFO: ControllerSelector::timed_event(MissedHeartbeat): " - "\"%C\". Timer id: %d\n", selected_.c_str(), timer_id)); + if (debug_) { + ACE_DEBUG((LM_DEBUG, "(%P|%t) ControllerSelector::timed_event(MissedHeartbeat): " + "\"%C\". Timer id: %d\n", selected_.c_str(), timer_id)); + } if (missed_heartbeat_callback_) { missed_heartbeat_callback_(selected_); } @@ -121,7 +147,7 @@ void ControllerSelector::timer_fired(Timer&) void ControllerSelector::timer_fired(Timer&) { Guard g(lock_); - ACE_DEBUG((LM_INFO, "(%P|%t) INFO: ControllerSelector::timed_event(NoControllers)\n")); + ACE_ERROR((LM_WARNING, "(%P|%t) WARNING: ControllerSelector::timed_event(NoControllers)\n")); if (no_controllers_callback_) { no_controllers_callback_(); } @@ -144,7 +170,7 @@ bool ControllerSelector::select_controller() if (last_hb < heartbeat_deadline) { select(mc_info->first, std::chrono::duration_cast(last_hb)); break; - } else { + } else if (debug_) { std::ostringstream oss; oss << std::chrono::duration_cast(last_hb - heartbeat_deadline).count(); ACE_DEBUG((LM_DEBUG, "(%P|%t) ControllerSelector::select_controller: \"%C\" missed heatbeat by %Cs\n", diff --git a/tactical-microgrid-standard/common/ControllerSelector.h b/tactical-microgrid-standard/common/ControllerSelector.h index 577c0f6..6e3287b 100644 --- a/tactical-microgrid-standard/common/ControllerSelector.h +++ b/tactical-microgrid-standard/common/ControllerSelector.h @@ -4,6 +4,7 @@ #include "TimerHandler.h" #include +#include #include #include @@ -98,11 +99,15 @@ class OpenDDS_TMS_Export ControllerCallbacks { class OpenDDS_TMS_Export ControllerSelector : public TimerHandler , public ControllerCallbacks + , public Configurable { public: explicit ControllerSelector(const tms::Identity& device_id, ACE_Reactor* reactor = nullptr); ~ControllerSelector(); + bool got_config(const std::string& name, const OpenDDS::DCPS::ConfigPair& pair); + void set_debug(bool value); + void got_heartbeat(const tms::Heartbeat& hb); void got_device_info(const tms::DeviceInfo& di); @@ -138,6 +143,8 @@ class OpenDDS_TMS_Export ControllerSelector std::visit([&](auto&& value) { this->timer_fired(*value); }, timer); } + bool debug_ = false; + void select(const tms::Identity& id, Sec last_hb = Sec(0)); bool select_controller(); diff --git a/tactical-microgrid-standard/common/TimerHandler.h b/tactical-microgrid-standard/common/TimerHandler.h index 608db71..128e725 100644 --- a/tactical-microgrid-standard/common/TimerHandler.h +++ b/tactical-microgrid-standard/common/TimerHandler.h @@ -262,7 +262,7 @@ class TimerHandler : public ACE_Event_Handler, protected TimerHolder { Guard g(lock_); auto key = *reinterpret_cast(arg); - ACE_DEBUG((LM_DEBUG, "(%P|%t) TimerHandler::handle_timeout: %C\n", key.c_str())); + // ACE_DEBUG((LM_DEBUG, "(%P|%t) TimerHandler::handle_timeout: %C\n", key.c_str())); if (active_timers_.count(key) == 0) { ACE_ERROR((LM_WARNING, "(%P|%t) WARNING: TimerHandler::handle_timeout: timer key %C does NOT exist\n", key.c_str())); } diff --git a/tactical-microgrid-standard/controller/Controller.cpp b/tactical-microgrid-standard/controller/Controller.cpp index ae2bfa1..9f71a1d 100644 --- a/tactical-microgrid-standard/controller/Controller.cpp +++ b/tactical-microgrid-standard/controller/Controller.cpp @@ -1,7 +1,8 @@ #include "Controller.h" #include "ActiveMicrogridControllerStateDataReaderListenerImpl.h" -#include "common/Utils.h" -#include "common/QosHelper.h" + +#include +#include #include @@ -12,6 +13,8 @@ DDS::ReturnCode_t Controller::init(DDS::DomainId_t domain_id, int argc, char* ar return rc; } + setup_config(); + rc = create_subscribers( [&](const auto& di, const auto& si) { device_info_cb(di, si); }, [&](const auto& hb, const auto& si) { heartbeat_cb(hb, si); }); @@ -29,7 +32,7 @@ DDS::ReturnCode_t Controller::init(DDS::DomainId_t domain_id, int argc, char* ar // Subscribe to the tms::ActiveMicrogridControllerState topic tms::ActiveMicrogridControllerStateTypeSupport_var amcs_ts = new tms::ActiveMicrogridControllerStateTypeSupportImpl; if (DDS::RETCODE_OK != amcs_ts->register_type(dp, "")) { - ACE_ERROR((LM_ERROR, "(%P|%t) CLIClient::init: register_type ActiveMicrogridControllerState failed\n")); + ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: Controller::init: register_type ActiveMicrogridControllerState failed\n")); return DDS::RETCODE_ERROR; } @@ -40,7 +43,7 @@ DDS::ReturnCode_t Controller::init(DDS::DomainId_t domain_id, int argc, char* ar nullptr, ::OpenDDS::DCPS::DEFAULT_STATUS_MASK); if (!amcs_topic) { - ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: CLIClient::init: create_topic \"%C\" failed\n", + ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: Controller::init: create_topic \"%C\" failed\n", tms::topic::TOPIC_ACTIVE_MICROGRID_CONTROLLER_STATE.c_str())); return DDS::RETCODE_ERROR; } @@ -50,7 +53,7 @@ DDS::ReturnCode_t Controller::init(DDS::DomainId_t domain_id, int argc, char* ar nullptr, ::OpenDDS::DCPS::DEFAULT_STATUS_MASK); if (!tms_sub) { - ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: CLIClient::init: create_subscriber with TMS QoS failed\n")); + ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: Controller::init: create_subscriber with TMS QoS failed\n")); return DDS::RETCODE_ERROR; } @@ -61,7 +64,7 @@ DDS::ReturnCode_t Controller::init(DDS::DomainId_t domain_id, int argc, char* ar amcs_listener, ::OpenDDS::DCPS::DEFAULT_STATUS_MASK); if (!amcs_dr_base) { - ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: CLIClient::init: create_datareader for topic \"%C\" failed\n", + ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: Controller::init: create_datareader for topic \"%C\" failed\n", tms::topic::TOPIC_ACTIVE_MICROGRID_CONTROLLER_STATE.c_str())); return DDS::RETCODE_ERROR; } @@ -84,6 +87,24 @@ void Controller::terminate() reactor_->end_reactor_event_loop(); } +bool Controller::got_config(const std::string& name, const OpenDDS::DCPS::ConfigPair& pair) +{ + if (name == "DEBUG") { + bool tmp; + if (convert_bool(pair, tmp)) { + set_debug(tmp); + } + return true; + } + return false; +} + +void Controller::set_debug(bool value) +{ + std::lock_guard guard(mut_); + debug_ = value; +} + tms::Identity Controller::id() const { return device_id_; @@ -110,12 +131,14 @@ void Controller::device_info_cb(const tms::DeviceInfo& di, const DDS::SampleInfo return; } - ACE_DEBUG((LM_INFO, "(%P|%t) INFO: Controller::device_info_cb: device: \"%C\"\n", di.deviceId().c_str())); + std::lock_guard guard(mut_); + if (debug_) { + ACE_DEBUG((LM_DEBUG, "(%P|%t) Controller::device_info_cb: device: \"%C\"\n", di.deviceId().c_str())); + } // Ignore other control devices, such as microgrid controllers. // Store all power devices, including those that select a different MC as its active MC. if (di.role() != tms::DeviceRole::ROLE_MICROGRID_CONTROLLER) { - std::lock_guard guard(mut_); power_devices_.insert(std::make_pair(di.deviceId(), cli::PowerDeviceInfo(di, tms::EnergyStartStopLevel::ESSL_OPERATIONAL, std::optional()))); @@ -128,16 +151,11 @@ void Controller::heartbeat_cb(const tms::Heartbeat& hb, const DDS::SampleInfo& s return; } - const tms::Identity& id = hb.deviceId(); - const uint32_t seqnum = hb.sequenceNumber(); - - if (OpenDDS::DCPS::DCPS_debug_level >= 8) { - std::lock_guard guard(mut_); - if (power_devices_.count(id) > 0) { - ACE_DEBUG((LM_INFO, "(%P|%t) INFO: Controller::heartbeat_cb: known device: \"%C\", seqnum: %u\n", id.c_str(), seqnum)); - } else { - ACE_DEBUG((LM_INFO, "(%P|%t) INFO: Controller::heartbeat_cb: unknown device: \"%C\", seqnum: %u\n", id.c_str(), seqnum)); - } + std::lock_guard guard(mut_); + if (debug_) { + const tms::Identity& id = hb.deviceId(); + ACE_DEBUG((LM_DEBUG, "(%P|%t) Controller::heartbeat_cb: %C device: \"%C\", seqnum: %u\n", + power_devices_.count(id) > 0 ? "known" : "other", id.c_str(), hb.sequenceNumber())); } } diff --git a/tactical-microgrid-standard/controller/Controller.h b/tactical-microgrid-standard/controller/Controller.h index 7ad25e7..dac7c76 100644 --- a/tactical-microgrid-standard/controller/Controller.h +++ b/tactical-microgrid-standard/controller/Controller.h @@ -2,13 +2,16 @@ #define CONTROLLER_CONTROLLER_H #include "Common.h" -#include "common/Handshaking.h" -class Controller : public Handshaking { +#include +#include + +class Controller : public Handshaking, Configurable { public: explicit Controller(const tms::Identity& id, uint16_t priority = 0) - : Handshaking(id) - , priority_(priority) + : Handshaking(id) + , Configurable("TMS_CONTROLLER") + , priority_(priority) { } @@ -19,6 +22,9 @@ class Controller : public Handshaking { void update_essl(const tms::Identity& pd_id, tms::EnergyStartStopLevel to_level); void terminate(); + bool got_config(const std::string& name, const OpenDDS::DCPS::ConfigPair& pair); + void set_debug(bool value); + DDS::DomainId_t tms_domain_id() const { return tms_domain_id_; @@ -32,6 +38,7 @@ class Controller : public Handshaking { tms::DeviceInfo populate_device_info() const; mutable std::mutex mut_; + bool debug_ = false; PowerDevices power_devices_; uint16_t priority_; diff --git a/tactical-microgrid-standard/power_devices/PowerDevice.cpp b/tactical-microgrid-standard/power_devices/PowerDevice.cpp index 1ea9f49..3b3e2f6 100644 --- a/tactical-microgrid-standard/power_devices/PowerDevice.cpp +++ b/tactical-microgrid-standard/power_devices/PowerDevice.cpp @@ -13,6 +13,8 @@ DDS::ReturnCode_t PowerDevice::init(DDS::DomainId_t domain, int argc, char* argv return rc; } + controller_selector_.setup_config(); + rc = create_subscribers( [&](const auto& di, const auto& si) { got_device_info(di, si); }, [&](const auto& hb, const auto& si) { got_heartbeat(hb, si); }); @@ -223,9 +225,6 @@ void PowerDevice::got_heartbeat(const tms::Heartbeat& hb, const DDS::SampleInfo& return; } - if (OpenDDS::DCPS::DCPS_debug_level >= 8) { - ACE_DEBUG((LM_INFO, "(%P|%t) INFO: Handshaking::power_device_got_heartbeat: from %C\n", hb.deviceId().c_str())); - } controller_selector_.got_heartbeat(hb); } @@ -234,7 +233,7 @@ void PowerDevice::got_device_info(const tms::DeviceInfo& di, const DDS::SampleIn if (!si.valid_data || di.deviceId() == device_id_) { return; } - ACE_DEBUG((LM_INFO, "(%P|%t) INFO: Handshaking::power_device_got_device_info: from %C\n", di.deviceId().c_str())); + controller_selector_.got_device_info(di); } diff --git a/tactical-microgrid-standard/tests/mc-sel/run_test.pl b/tactical-microgrid-standard/tests/mc-sel/run_test.pl index 35b4e5f..c5e11aa 100755 --- a/tactical-microgrid-standard/tests/mc-sel/run_test.pl +++ b/tactical-microgrid-standard/tests/mc-sel/run_test.pl @@ -23,15 +23,12 @@ sub start_mc { my $priority = shift(); my $time = shift(); - $test->process($name, 'basic-mc', "$name $priority"); + $test->process($name, 'basic-mc', "$name $priority -OpenDDS-tms-controller-debug true"); $test->start_process($name); return $name, time() + $time; } -$test->process('dev', 'basic-dev'); -$test->start_process('dev'); - my $extra = 3; my $lost_controller = 9 + $extra; # 3s for missed, 6s for lost @@ -73,20 +70,29 @@ sub expect_lost_mc { return $test->wait_for('dev', "lost controller $name", max_wait => $max); } +# Start device we will be using to see what controllers are selected +$test->process('dev', 'basic-dev', '-OpenDDS-tms-selector-debug true'); +$test->start_process('dev'); + +# Start controller mc1 and expect selection by device. my ($mc1, $mc1_ends_at) = start_mc('mc1', 0, 20); expect_new_mc($mc1, 10); sleep(3); + +# Start a few controllers with different priorities. my ($mc2, $mc2_ends_at) = start_mc('mc2', 10, 30); my ($mc3, $mc3_ends_at) = start_mc('mc3', 0, 30); my ($mc4, $mc4_ends_at) = start_mc('mc4', 10, 30); +# Stop mc1 then the device should pick up mc3 because it has the lowest +# priority value. expect_stop_mc($mc1, $mc1_ends_at); expect_lost_mc($mc1, $lost_controller); expect_new_mc($mc3, 10); + expect_stop_mc($mc2, $mc2_ends_at); expect_stop_mc($mc3, $mc3_ends_at); expect_stop_mc($mc4, $mc4_ends_at); - $test->kill_process(2, 'dev'); exit $test->finish(2); From 96ba18b695c4538c3348e98bd075c1a935fafe51 Mon Sep 17 00:00:00 2001 From: Fred Hornsey Date: Mon, 8 Sep 2025 14:49:37 -0500 Subject: [PATCH 18/18] Remove Commented Line --- tactical-microgrid-standard/common/TimerHandler.h | 1 - 1 file changed, 1 deletion(-) diff --git a/tactical-microgrid-standard/common/TimerHandler.h b/tactical-microgrid-standard/common/TimerHandler.h index 128e725..88a507f 100644 --- a/tactical-microgrid-standard/common/TimerHandler.h +++ b/tactical-microgrid-standard/common/TimerHandler.h @@ -262,7 +262,6 @@ class TimerHandler : public ACE_Event_Handler, protected TimerHolder { Guard g(lock_); auto key = *reinterpret_cast(arg); - // ACE_DEBUG((LM_DEBUG, "(%P|%t) TimerHandler::handle_timeout: %C\n", key.c_str())); if (active_timers_.count(key) == 0) { ACE_ERROR((LM_WARNING, "(%P|%t) WARNING: TimerHandler::handle_timeout: timer key %C does NOT exist\n", key.c_str())); }