diff --git a/.github/workflows/TMS.yml b/.github/workflows/TMS.yml index 6eca17b..ab48b27 100644 --- a/.github/workflows/TMS.yml +++ b/.github/workflows/TMS.yml @@ -58,11 +58,24 @@ jobs: fetch-depth: 1 submodules: true + # Setup stacktrace + - name: install gdb + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install gdb + echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope + - name: change core file pattern + if: runner.os == 'Linux' + run: | + sudo sysctl -w kernel.core_pattern=core.%e.%p + echo Core file pattern set to: + cat /proc/sys/kernel/core_pattern + # Get 3rd-party Dependencies - name: 'Install openssl, xerces (Linux)' if: runner.os == 'Linux' run: |- - sudo apt-get update sudo apt-get -y install libssl-dev libxerces-c-dev - name: 'Install xerces (macOS)' if: runner.os == 'macOS' @@ -226,7 +239,7 @@ jobs: cd OpenDDS . setenv.sh cd ../tactical-microgrid-standard - cmake -B build_static + cmake -B build_static -D BUILD_TESTING=TRUE cmake --build build_static - name: 'Build TMS application with shared libs (Linux /macOS)' if: runner.os == 'Linux' || runner.os == 'macOS' @@ -235,7 +248,7 @@ jobs: cd OpenDDS . setenv.sh cd ../tactical-microgrid-standard - cmake -DBUILD_SHARED_LIBS=yes -B build_shared + cmake -DBUILD_SHARED_LIBS=yes -B build_shared -D BUILD_TESTING=TRUE cmake --build build_shared - name: 'Build TMS application with static libs (Windows)' if: runner.os == 'Windows' @@ -244,7 +257,7 @@ jobs: cd OpenDDS call setenv.cmd cd ..\tactical-microgrid-standard - cmake -B build_static + cmake -B build_static -D BUILD_TESTING=TRUE cmake --build build_static - name: 'Build TMS application with shared libs (Windows)' if: runner.os == 'Windows' @@ -253,5 +266,37 @@ jobs: cd OpenDDS call setenv.cmd cd ..\tactical-microgrid-standard - cmake -DBUILD_SHARED_LIBS=yes -B build_shared + cmake -DBUILD_SHARED_LIBS=yes -B build_shared -D BUILD_TESTING=TRUE cmake --build build_shared + - name: 'Run tests with static libs (Linux)' + if: runner.os == 'Linux' + run: |- + ulimit -c unlimited + cd OpenDDS + . setenv.sh + cd ../tactical-microgrid-standard/build_static + ctest --verbose --output-on-failure + - name: 'Run tests with shared libs (Linux)' + if: runner.os == 'Linux' + run: |- + ulimit -c unlimited + cd OpenDDS + . setenv.sh + cd ../tactical-microgrid-standard/build_shared + ctest --verbose --output-on-failure + - name: 'Run tests with static libs (Windows)' + if: runner.os == 'Windows' + shell: cmd + run: |- + cd OpenDDS + call setenv.cmd + cd ..\tactical-microgrid-standard\build_static + ctest --verbose --output-on-failure -C Debug + - name: 'Run tests with shared libs (Windows)' + if: runner.os == 'Windows' + shell: cmd + run: |- + cd OpenDDS + call setenv.cmd + cd ..\tactical-microgrid-standard\build_shared + ctest --verbose --output-on-failure -C Debug diff --git a/tactical-microgrid-standard/CMakeLists.txt b/tactical-microgrid-standard/CMakeLists.txt index 6c8dfba..7258274 100644 --- a/tactical-microgrid-standard/CMakeLists.txt +++ b/tactical-microgrid-standard/CMakeLists.txt @@ -24,6 +24,12 @@ opendds_target_sources(TMS_Common target_link_libraries(TMS_Common PUBLIC OpenDDS::Rtps_Udp) target_compile_features(TMS_Common PUBLIC cxx_std_17) opendds_bigobj(TMS_Common) +# Generated code with complete type objects enabled for the TMS IDL file +# causes stack overflow on Windows. +# Increase the stack size to 2MB (default is 1MB). +if (MSVC) + target_link_options(TMS_Common PUBLIC "/STACK:2097152") +endif() add_library(Commands_Idl) opendds_target_sources(Commands_Idl @@ -87,16 +93,6 @@ add_executable(Distribution target_include_directories(Distribution PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(Distribution PRIVATE PowerSim_Idl) -# Generated code with complete type objects enabled for the TMS IDL file -# causes stack overflow on Windows. -# Increase the stack size to 2MB (default is 1MB). -if (MSVC) - target_link_options(Controller PRIVATE "/STACK:2097152") - target_link_options(CLI PRIVATE "/STACK:2097152") - target_link_options(Source PRIVATE "/STACK:2097152") - target_link_options(Load PRIVATE "/STACK:2097152") - target_link_options(Distribution PRIVATE "/STACK:2097152") +if(BUILD_TESTING) + add_subdirectory(tests) endif() - - -add_subdirectory(tests) diff --git a/tactical-microgrid-standard/README.md b/tactical-microgrid-standard/README.md index e14177a..1b64694 100644 --- a/tactical-microgrid-standard/README.md +++ b/tactical-microgrid-standard/README.md @@ -32,7 +32,7 @@ cmake --build - `cli/`: Command-line interface implementation - `cli_idl/`: Interface definition files for CLI commands - `common/`: Shared libraries and utilities - - TMS data model definitions (mil-std-3071_data_model.idl) + - TMS data model definitions (`mil-std-3071_data_model.idl`) - TMS Handshaking function - TMS microgrid controller selection - TMS QoS profiles @@ -46,13 +46,27 @@ cmake --build ## Testing -Tests can be run using CTest after building the project: +Tests can be run using CTest after building the project with `-D BUILD_TESTING=TRUE`: ```bash cd ctest ``` +## Configuration + +These programs support the following OpenDDS configuration properties. There +are multiple ways to set these, see +[OpenDDS runtime configuration](https://opendds.readthedocs.io/en/master/devguide/run_time_configuration.html) +for more info. + +- `TMS_SELECTOR_DEBUG=` + - Enables debug logging of the microgrid controller selection process of power devices. + - Command line option example: `-OpenDDS-tms-selector-debug true` +- `TMS_CONTROLLER_DEBUG=` + - Enables debug logging of what devices the controller has learned about. + - Command line option example: `-OpenDDS-tms-controller-debug true` + ## References - [Tactical Microgrid Standard (MIL-STD-3071)](https://quicksearch.dla.mil/qsDocDetails.aspx?ident_number=285095) diff --git a/tactical-microgrid-standard/common/Configurable.h b/tactical-microgrid-standard/common/Configurable.h new file mode 100644 index 0000000..76a2e21 --- /dev/null +++ b/tactical-microgrid-standard/common/Configurable.h @@ -0,0 +1,111 @@ +#ifndef TMS_COMMON_CONFIGURABLE +#define TMS_COMMON_CONFIGURABLE + +#include +#include +#include + +#include + +class Configurable { +public: + using Mutex = std::recursive_mutex; + using Guard = std::lock_guard; + + Configurable(const std::string& prefix) + : config_prefix_(prefix) + { + } + + virtual ~Configurable() + { + Guard g(config_lock_); + + if (config_reader_) { + TheServiceParticipant->config_topic()->disconnect(config_reader_); + } + } + + void setup_config() + { + Guard g(config_lock_); + + if (!config_reader_) { + config_listener_ = OpenDDS::DCPS::make_rch(this, config_prefix_ + "_"); + config_reader_ = OpenDDS::DCPS::make_rch( + TheServiceParticipant->config_store()->datareader_qos(), config_listener_); + TheServiceParticipant->config_topic()->connect(config_reader_); + } + } + + const std::string& config_prefix() const + { + return config_prefix_; + } + + static bool convert_bool(const OpenDDS::DCPS::ConfigPair& pair, bool& value) + { + DDS::Boolean x = 0; + if (pair.value() == "true") { + value = true; + return true; + } else if (pair.value() == "false") { + value = false; + return true; + } else if (OpenDDS::DCPS::convertToInteger(pair.value(), x)) { + value = x; + return true; + } else { + ACE_ERROR((LM_WARNING, "(%P|%t) WARNING: Configurable::convert_bool: failed to parse boolean for %C=%C\n", + pair.key().c_str(), pair.value().c_str())); + return false; + } + } + + virtual bool got_config(const std::string& name, const OpenDDS::DCPS::ConfigPair& pair) = 0; + +private: + class ConfigListener : public OpenDDS::DCPS::ConfigListener { + public: + explicit ConfigListener(Configurable* configurable, const std::string& prefix) + : OpenDDS::DCPS::ConfigListener(TheServiceParticipant->job_queue()) + , configurable_(*configurable) + , prefix_(prefix) + { + } + + void on_data_available(InternalDataReader_rch reader) override + { + OpenDDS::DCPS::ConfigReader::SampleSequence samples; + OpenDDS::DCPS::InternalSampleInfoSequence infos; + reader->read(samples, infos, DDS::LENGTH_UNLIMITED, + DDS::NOT_READ_SAMPLE_STATE, DDS::ANY_VIEW_STATE, DDS::ANY_INSTANCE_STATE); + for (size_t idx = 0; idx != samples.size(); ++idx) { + const auto& info = infos[idx]; + if (info.valid_data) { + const auto& pair = samples[idx]; + // Match key to prefix and pass rest of key as a short name + if (pair.key().substr(0, prefix_.length()) == prefix_) { + const auto name = pair.key().substr(prefix_.length()); + if (!configurable_.got_config(name, pair)) { + ACE_ERROR((LM_WARNING, "(%P|%t) WARNING: Configurable::ConfigListener::on_data_available: " + "%C is not a valid property for %C\n", + name.c_str(), configurable_.config_prefix().c_str())); + } + } + } + } + } + + private: + Configurable& configurable_; + const std::string prefix_; + }; + + mutable Mutex config_lock_; + const std::string config_prefix_; + OpenDDS::DCPS::RcHandle config_reader_; + OpenDDS::DCPS::RcHandle config_listener_; +}; + +#endif diff --git a/tactical-microgrid-standard/common/ControllerSelector.cpp b/tactical-microgrid-standard/common/ControllerSelector.cpp index 83208b4..4b362a9 100644 --- a/tactical-microgrid-standard/common/ControllerSelector.cpp +++ b/tactical-microgrid-standard/common/ControllerSelector.cpp @@ -1,7 +1,12 @@ #include "ControllerSelector.h" -ControllerSelector::ControllerSelector(const tms::Identity& device_id) - : device_id_(device_id) +#include + +ControllerSelector::ControllerSelector(const tms::Identity& device_id, ACE_Reactor* reactor) + : TimerHandler(reactor) + , ControllerCallbacks(lock_) + , Configurable("TMS_SELECTOR") + , device_id_(device_id) { } @@ -9,11 +14,34 @@ ControllerSelector::~ControllerSelector() { } +bool ControllerSelector::got_config(const std::string& name, const OpenDDS::DCPS::ConfigPair& pair) +{ + if (name == "DEBUG") { + bool tmp; + if (convert_bool(pair, tmp)) { + set_debug(tmp); + } + return true; + } + return false; +} + +void ControllerSelector::set_debug(bool value) +{ + Guard g(lock_); + debug_ = value; +} + void ControllerSelector::got_heartbeat(const tms::Heartbeat& hb) { Guard g(lock_); auto it = all_controllers_.find(hb.deviceId()); if (it != all_controllers_.end()) { + if (debug_) { + ACE_DEBUG((LM_DEBUG, "(%P|%t) ControllerSelector::got_heartbeat: from mc \"%C\"\n", + hb.deviceId().c_str())); + } + it->second = Clock::now(); // Update last heartbeat cancel(); @@ -30,6 +58,9 @@ void ControllerSelector::got_heartbeat(const tms::Heartbeat& hb) schedule_once(MissedHeartbeat{}, heartbeat_deadline); } } + } else if (debug_) { + ACE_DEBUG((LM_DEBUG, "(%P|%t) ControllerSelector::got_heartbeat: from unknown \"%C\"\n", + hb.deviceId().c_str())); } } @@ -42,6 +73,7 @@ void ControllerSelector::got_device_info(const tms::DeviceInfo& di) if (di.role() == tms::DeviceRole::ROLE_MICROGRID_CONTROLLER && !all_controllers_.count(di.deviceId())) { all_controllers_[di.deviceId()] = TimePoint::min(); + prioritized_controllers_.insert(PrioritizedController(di)); } } @@ -49,8 +81,11 @@ void ControllerSelector::timer_fired(Timer& timer) { Guard g(lock_); const auto& mc_id = timer.arg.id; - ACE_DEBUG((LM_INFO, "(%P|%t) INFO: ControllerSelector::timed_event(NewController): " - "triggered by \"%C\" heartbeat\n", mc_id.c_str())); + + if (debug_) { + ACE_DEBUG((LM_DEBUG, "(%P|%t) ControllerSelector::timed_event(NewController): " + "triggered by \"%C\" heartbeat\n", mc_id.c_str())); + } if (all_controllers_.find(mc_id) == all_controllers_.end()) { ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: ControllerSelector::timed_event(NewController): Controller \"%C\" not found!\n", @@ -70,8 +105,13 @@ void ControllerSelector::timer_fired(Timer& timer) { Guard g(lock_); const auto& timer_id = timer.id; - ACE_DEBUG((LM_INFO, "(%P|%t) INFO: ControllerSelector::timed_event(MissedHeartbeat): " - "\"%C\". Timer id: %d\n", selected_.c_str(), timer_id)); + if (debug_) { + ACE_DEBUG((LM_DEBUG, "(%P|%t) ControllerSelector::timed_event(MissedHeartbeat): " + "\"%C\". Timer id: %d\n", selected_.c_str(), timer_id)); + } + if (missed_heartbeat_callback_) { + missed_heartbeat_callback_(selected_); + } schedule_once(LostController{}, lost_active_controller_delay); // Start a No MC timer if the device has missed heartbeats from all MCs @@ -94,6 +134,9 @@ void ControllerSelector::timer_fired(Timer&) Guard g(lock_); ACE_DEBUG((LM_INFO, "(%P|%t) INFO: ControllerSelector::timed_event(LostController): " "\"%C\"\n", selected_.c_str())); + if (lost_controller_callback_) { + lost_controller_callback_(selected_); + } selected_.clear(); // Select a new controller if possible. If there are no recent controllers @@ -104,7 +147,10 @@ void ControllerSelector::timer_fired(Timer&) void ControllerSelector::timer_fired(Timer&) { Guard g(lock_); - ACE_DEBUG((LM_INFO, "(%P|%t) INFO: ControllerSelector::timed_event(NoControllers)\n")); + ACE_ERROR((LM_WARNING, "(%P|%t) WARNING: ControllerSelector::timed_event(NoControllers)\n")); + if (no_controllers_callback_) { + no_controllers_callback_(); + } // TODO: CONFIG_ON_COMMS_LOSS } @@ -115,13 +161,20 @@ bool ControllerSelector::select_controller() const TimePoint now = Clock::now(); // Select an available controller with smallest identity alphabetically - for (auto it = all_controllers_.begin(); it != all_controllers_.end(); ++it) { - const auto last_hb = now - it->second; + for (auto it = prioritized_controllers_.begin(); it != prioritized_controllers_.end(); ++it) { + auto mc_info = all_controllers_.find(it->id); + const auto& id = mc_info->first; + const auto last_hb = now - mc_info->second; // TMS spec doesn't specify this. But it should make sure the controller is still available // i.e., last heartbeat received within 3 seconds. if (last_hb < heartbeat_deadline) { - select(it->first, std::chrono::duration_cast(last_hb)); - return true; + select(mc_info->first, std::chrono::duration_cast(last_hb)); + break; + } else if (debug_) { + std::ostringstream oss; + oss << std::chrono::duration_cast(last_hb - heartbeat_deadline).count(); + ACE_DEBUG((LM_DEBUG, "(%P|%t) ControllerSelector::select_controller: \"%C\" missed heatbeat by %Cs\n", + id.c_str(), oss.str().c_str())); } } return false; @@ -131,6 +184,9 @@ void ControllerSelector::select(const tms::Identity& id, Sec last_hb) { ACE_DEBUG((LM_INFO, "(%P|%t) INFO: ControllerSelector::select: \"%C\"\n", id.c_str())); selected_ = id; + if (new_controller_callback_) { + new_controller_callback_(selected_); + } send_controller_state(); schedule_once(MissedHeartbeat{}, heartbeat_deadline - last_hb); } diff --git a/tactical-microgrid-standard/common/ControllerSelector.h b/tactical-microgrid-standard/common/ControllerSelector.h index f000099..6e3287b 100644 --- a/tactical-microgrid-standard/common/ControllerSelector.h +++ b/tactical-microgrid-standard/common/ControllerSelector.h @@ -4,9 +4,12 @@ #include "TimerHandler.h" #include +#include #include #include +#include +#include struct NewController { tms::Identity id; @@ -25,7 +28,46 @@ struct NoControllers { static const char* name() { return "NoControllers"; } }; -class PowerDevice; +class OpenDDS_TMS_Export ControllerCallbacks { +public: + using IdCallback = std::function; + using Callback = std::function; + + explicit ControllerCallbacks(Mutex& lock) : cb_lock_(lock) {} + + void set_new_controller_callback(IdCallback cb) + { + Guard g(cb_lock_); + new_controller_callback_ = cb; + } + + void set_missed_heartbeat_callback(IdCallback cb) + { + Guard g(cb_lock_); + missed_heartbeat_callback_ = cb; + } + + void set_lost_controller_callback(IdCallback cb) + { + Guard g(cb_lock_); + lost_controller_callback_ = cb; + } + + void set_no_controllers_callback(Callback cb) + { + Guard g(cb_lock_); + no_controllers_callback_ = cb; + } + +protected: + IdCallback new_controller_callback_; + IdCallback missed_heartbeat_callback_; + IdCallback lost_controller_callback_; + Callback no_controllers_callback_; + +private: + Mutex& cb_lock_; +}; /** * Logic for determining what controller should be used, based on TMS A.7.5. @@ -54,12 +96,18 @@ class PowerDevice; * A: If heartbeat is from selected * S: If there's a selectable controller with a recent heartbeat */ -class OpenDDS_TMS_Export ControllerSelector : - public TimerHandler { +class OpenDDS_TMS_Export ControllerSelector + : public TimerHandler + , public ControllerCallbacks + , public Configurable +{ public: - explicit ControllerSelector(const tms::Identity& device_id); + explicit ControllerSelector(const tms::Identity& device_id, ACE_Reactor* reactor = nullptr); ~ControllerSelector(); + bool got_config(const std::string& name, const OpenDDS::DCPS::ConfigPair& pair); + void set_debug(bool value); + void got_heartbeat(const tms::Heartbeat& hb); void got_device_info(const tms::DeviceInfo& di); @@ -75,12 +123,6 @@ class OpenDDS_TMS_Export ControllerSelector : return selected_ == id; } - ACE_Reactor* get_reactor() const - { - Guard g(lock_); - return reactor_; - } - void set_ActiveMicrogridControllerState_writer(tms::ActiveMicrogridControllerStateDataWriter_var amcs_dw) { amcs_dw_ = amcs_dw; @@ -101,6 +143,8 @@ class OpenDDS_TMS_Export ControllerSelector : std::visit([&](auto&& value) { this->timer_fired(*value); }, timer); } + bool debug_ = false; + void select(const tms::Identity& id, Sec last_hb = Sec(0)); bool select_controller(); @@ -109,6 +153,29 @@ class OpenDDS_TMS_Export ControllerSelector : tms::Identity selected_; std::map all_controllers_; + struct PrioritizedController { + uint16_t priority = 0; + tms::Identity id; + + explicit PrioritizedController(const tms::DeviceInfo& di) + : id(di.deviceId()) + { + const auto& cs = di.controlService(); + if (cs) { + const auto& mc = cs->mc(); + if (mc) { + priority = mc->priorityRanking(); + } + } + } + + bool operator<(const PrioritizedController& lhs) const + { + return std::tie(priority, id) < std::tie(lhs.priority, lhs.id); + } + }; + std::set prioritized_controllers_; + // Device ID to which this controller selector belong. tms::Identity device_id_; diff --git a/tactical-microgrid-standard/common/Handshaking.cpp b/tactical-microgrid-standard/common/Handshaking.cpp index 205968b..cfb8461 100644 --- a/tactical-microgrid-standard/common/Handshaking.cpp +++ b/tactical-microgrid-standard/common/Handshaking.cpp @@ -14,11 +14,27 @@ Handshaking::~Handshaking() { - if (participant_) { - participant_->delete_contained_entities(); - } - if (dpf_) { - dpf_->delete_participant(participant_); + delete_all_entities(); +} + +void Handshaking::delete_all_entities() +{ + delete_extra_entities(); + delete_entities(participant_); +} + +void Handshaking::delete_extra_entities() +{ +} + +void Handshaking::delete_entities(DDS::DomainParticipant_var& part) +{ + if (part) { + part->delete_contained_entities(); + if (dpf_) { + dpf_->delete_participant(part); + } + part = nullptr; } } diff --git a/tactical-microgrid-standard/common/Handshaking.h b/tactical-microgrid-standard/common/Handshaking.h index bf4c1f1..992e4e1 100644 --- a/tactical-microgrid-standard/common/Handshaking.h +++ b/tactical-microgrid-standard/common/Handshaking.h @@ -61,7 +61,12 @@ class OpenDDS_TMS_Export Handshaking : public TimerHandler { return device_id_; } + void delete_all_entities(); + protected: + void delete_entities(DDS::DomainParticipant_var& part); + virtual void delete_extra_entities(); + virtual tms::DeviceInfo get_device_info() const { tms::DeviceInfo device_info; diff --git a/tactical-microgrid-standard/common/TimerHandler.h b/tactical-microgrid-standard/common/TimerHandler.h index a1dd6f5..88a507f 100644 --- a/tactical-microgrid-standard/common/TimerHandler.h +++ b/tactical-microgrid-standard/common/TimerHandler.h @@ -16,7 +16,7 @@ #include #include -using Sec = std::chrono::seconds; +using Sec = std::chrono::duration; using Clock = std::chrono::system_clock; using TimePoint = std::chrono::time_point; using TimerId = long; @@ -25,6 +25,18 @@ constexpr TimerId null_timer_id = 0; using Mutex = std::recursive_mutex; using Guard = std::lock_guard; +// Workaround https://github.com/DOCGroup/ACE_TAO/pull/2462 +template +inline ACE_Time_Value to_time_value(const std::chrono::duration& duration) +{ + using namespace std::chrono; + + const seconds s{duration_cast(duration)}; + const microseconds usec{duration_cast(duration - s)}; + return ACE_Time_Value(ACE_Utils::truncate_cast(s.count()), + ACE_Utils::truncate_cast(usec.count())); +} + // EventType must implement: static const char* name(); template struct Timer { @@ -146,6 +158,12 @@ class TimerHandler : public ACE_Event_Handler, protected TimerHolder } } + ACE_Reactor* get_reactor() const + { + Guard g(lock_); + return reactor_; + } + template typename Timer::Ptr get_timer(const std::string& name = "") { @@ -166,7 +184,7 @@ class TimerHandler : public ACE_Event_Handler, protected TimerHolder Guard g(lock_); assert_inactive(timer); const TimerId id = reactor_->schedule_timer( - this, &timer->key, ACE_Time_Value(timer->delay), ACE_Time_Value(timer->period)); + this, &timer->key, to_time_value(timer->delay), to_time_value(timer->period)); timer->activate(id); active_timers_[timer->key] = timer; } diff --git a/tactical-microgrid-standard/controller/Controller.cpp b/tactical-microgrid-standard/controller/Controller.cpp index 955d329..9f71a1d 100644 --- a/tactical-microgrid-standard/controller/Controller.cpp +++ b/tactical-microgrid-standard/controller/Controller.cpp @@ -1,7 +1,8 @@ #include "Controller.h" #include "ActiveMicrogridControllerStateDataReaderListenerImpl.h" -#include "common/Utils.h" -#include "common/QosHelper.h" + +#include +#include #include @@ -12,6 +13,8 @@ DDS::ReturnCode_t Controller::init(DDS::DomainId_t domain_id, int argc, char* ar return rc; } + setup_config(); + rc = create_subscribers( [&](const auto& di, const auto& si) { device_info_cb(di, si); }, [&](const auto& hb, const auto& si) { heartbeat_cb(hb, si); }); @@ -29,7 +32,7 @@ DDS::ReturnCode_t Controller::init(DDS::DomainId_t domain_id, int argc, char* ar // Subscribe to the tms::ActiveMicrogridControllerState topic tms::ActiveMicrogridControllerStateTypeSupport_var amcs_ts = new tms::ActiveMicrogridControllerStateTypeSupportImpl; if (DDS::RETCODE_OK != amcs_ts->register_type(dp, "")) { - ACE_ERROR((LM_ERROR, "(%P|%t) CLIClient::init: register_type ActiveMicrogridControllerState failed\n")); + ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: Controller::init: register_type ActiveMicrogridControllerState failed\n")); return DDS::RETCODE_ERROR; } @@ -40,7 +43,7 @@ DDS::ReturnCode_t Controller::init(DDS::DomainId_t domain_id, int argc, char* ar nullptr, ::OpenDDS::DCPS::DEFAULT_STATUS_MASK); if (!amcs_topic) { - ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: CLIClient::init: create_topic \"%C\" failed\n", + ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: Controller::init: create_topic \"%C\" failed\n", tms::topic::TOPIC_ACTIVE_MICROGRID_CONTROLLER_STATE.c_str())); return DDS::RETCODE_ERROR; } @@ -50,7 +53,7 @@ DDS::ReturnCode_t Controller::init(DDS::DomainId_t domain_id, int argc, char* ar nullptr, ::OpenDDS::DCPS::DEFAULT_STATUS_MASK); if (!tms_sub) { - ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: CLIClient::init: create_subscriber with TMS QoS failed\n")); + ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: Controller::init: create_subscriber with TMS QoS failed\n")); return DDS::RETCODE_ERROR; } @@ -61,7 +64,7 @@ DDS::ReturnCode_t Controller::init(DDS::DomainId_t domain_id, int argc, char* ar amcs_listener, ::OpenDDS::DCPS::DEFAULT_STATUS_MASK); if (!amcs_dr_base) { - ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: CLIClient::init: create_datareader for topic \"%C\" failed\n", + ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: Controller::init: create_datareader for topic \"%C\" failed\n", tms::topic::TOPIC_ACTIVE_MICROGRID_CONTROLLER_STATE.c_str())); return DDS::RETCODE_ERROR; } @@ -84,6 +87,24 @@ void Controller::terminate() reactor_->end_reactor_event_loop(); } +bool Controller::got_config(const std::string& name, const OpenDDS::DCPS::ConfigPair& pair) +{ + if (name == "DEBUG") { + bool tmp; + if (convert_bool(pair, tmp)) { + set_debug(tmp); + } + return true; + } + return false; +} + +void Controller::set_debug(bool value) +{ + std::lock_guard guard(mut_); + debug_ = value; +} + tms::Identity Controller::id() const { return device_id_; @@ -110,12 +131,14 @@ void Controller::device_info_cb(const tms::DeviceInfo& di, const DDS::SampleInfo return; } - ACE_DEBUG((LM_INFO, "(%P|%t) INFO: Controller::device_info_cb: device: \"%C\"\n", di.deviceId().c_str())); + std::lock_guard guard(mut_); + if (debug_) { + ACE_DEBUG((LM_DEBUG, "(%P|%t) Controller::device_info_cb: device: \"%C\"\n", di.deviceId().c_str())); + } // Ignore other control devices, such as microgrid controllers. // Store all power devices, including those that select a different MC as its active MC. if (di.role() != tms::DeviceRole::ROLE_MICROGRID_CONTROLLER) { - std::lock_guard guard(mut_); power_devices_.insert(std::make_pair(di.deviceId(), cli::PowerDeviceInfo(di, tms::EnergyStartStopLevel::ESSL_OPERATIONAL, std::optional()))); @@ -128,16 +151,11 @@ void Controller::heartbeat_cb(const tms::Heartbeat& hb, const DDS::SampleInfo& s return; } - const tms::Identity& id = hb.deviceId(); - const uint32_t seqnum = hb.sequenceNumber(); - - if (OpenDDS::DCPS::DCPS_debug_level >= 8) { - std::lock_guard guard(mut_); - if (power_devices_.count(id) > 0) { - ACE_DEBUG((LM_INFO, "(%P|%t) INFO: Controller::heartbeat_cb: known device: \"%C\", seqnum: %u\n", id.c_str(), seqnum)); - } else { - ACE_DEBUG((LM_INFO, "(%P|%t) INFO: Controller::heartbeat_cb: unknown device: \"%C\", seqnum: %u\n", id.c_str(), seqnum)); - } + std::lock_guard guard(mut_); + if (debug_) { + const tms::Identity& id = hb.deviceId(); + ACE_DEBUG((LM_DEBUG, "(%P|%t) Controller::heartbeat_cb: %C device: \"%C\", seqnum: %u\n", + power_devices_.count(id) > 0 ? "known" : "other", id.c_str(), hb.sequenceNumber())); } } @@ -163,7 +181,7 @@ tms::DeviceInfo Controller::populate_device_info() const tms::MicrogridControllerInfo mc_info; { mc_info.features().push_back(tms::MicrogridControllerFeature::MCF_GENERAL); - mc_info.priorityRanking(0); + mc_info.priorityRanking(priority_); } csi.mc() = mc_info; } diff --git a/tactical-microgrid-standard/controller/Controller.h b/tactical-microgrid-standard/controller/Controller.h index f2c5bdb..dac7c76 100644 --- a/tactical-microgrid-standard/controller/Controller.h +++ b/tactical-microgrid-standard/controller/Controller.h @@ -2,11 +2,18 @@ #define CONTROLLER_CONTROLLER_H #include "Common.h" -#include "common/Handshaking.h" -class Controller : public Handshaking { +#include +#include + +class Controller : public Handshaking, Configurable { public: - explicit Controller(const tms::Identity& id) : Handshaking(id) {} + explicit Controller(const tms::Identity& id, uint16_t priority = 0) + : Handshaking(id) + , Configurable("TMS_CONTROLLER") + , priority_(priority) + { + } DDS::ReturnCode_t init(DDS::DomainId_t domain_id, int argc = 0, char* argv[] = nullptr); int run(); @@ -15,6 +22,9 @@ class Controller : public Handshaking { void update_essl(const tms::Identity& pd_id, tms::EnergyStartStopLevel to_level); void terminate(); + bool got_config(const std::string& name, const OpenDDS::DCPS::ConfigPair& pair); + void set_debug(bool value); + DDS::DomainId_t tms_domain_id() const { return tms_domain_id_; @@ -27,9 +37,10 @@ class Controller : public Handshaking { void heartbeat_cb(const tms::Heartbeat& hb, const DDS::SampleInfo& si); tms::DeviceInfo populate_device_info() const; -private: mutable std::mutex mut_; + bool debug_ = false; PowerDevices power_devices_; + uint16_t priority_; DDS::DomainId_t tms_domain_id_ = OpenDDS::DOMAIN_UNKNOWN;; }; diff --git a/tactical-microgrid-standard/power_devices/PowerDevice.cpp b/tactical-microgrid-standard/power_devices/PowerDevice.cpp index 1ea9f49..3b3e2f6 100644 --- a/tactical-microgrid-standard/power_devices/PowerDevice.cpp +++ b/tactical-microgrid-standard/power_devices/PowerDevice.cpp @@ -13,6 +13,8 @@ DDS::ReturnCode_t PowerDevice::init(DDS::DomainId_t domain, int argc, char* argv return rc; } + controller_selector_.setup_config(); + rc = create_subscribers( [&](const auto& di, const auto& si) { got_device_info(di, si); }, [&](const auto& hb, const auto& si) { got_heartbeat(hb, si); }); @@ -223,9 +225,6 @@ void PowerDevice::got_heartbeat(const tms::Heartbeat& hb, const DDS::SampleInfo& return; } - if (OpenDDS::DCPS::DCPS_debug_level >= 8) { - ACE_DEBUG((LM_INFO, "(%P|%t) INFO: Handshaking::power_device_got_heartbeat: from %C\n", hb.deviceId().c_str())); - } controller_selector_.got_heartbeat(hb); } @@ -234,7 +233,7 @@ void PowerDevice::got_device_info(const tms::DeviceInfo& di, const DDS::SampleIn if (!si.valid_data || di.deviceId() == device_id_) { return; } - ACE_DEBUG((LM_INFO, "(%P|%t) INFO: Handshaking::power_device_got_device_info: from %C\n", di.deviceId().c_str())); + controller_selector_.got_device_info(di); } diff --git a/tactical-microgrid-standard/power_devices/PowerDevice.h b/tactical-microgrid-standard/power_devices/PowerDevice.h index 2c6a90b..18f4fdb 100644 --- a/tactical-microgrid-standard/power_devices/PowerDevice.h +++ b/tactical-microgrid-standard/power_devices/PowerDevice.h @@ -73,6 +73,11 @@ class PowerSim_Idl_Export PowerDevice : public Handshaking { return essl_; } + ControllerCallbacks& controller_callbacks() + { + return controller_selector_; + } + protected: virtual int run_i() { @@ -87,6 +92,11 @@ class PowerSim_Idl_Export PowerDevice : public Handshaking { return ret; } + void delete_extra_entities() + { + delete_entities(sim_participant_); + } + // Concrete power device should override this function depending on their role. virtual tms::DeviceInfo populate_device_info() const; diff --git a/tactical-microgrid-standard/tests/mc-sel/CMakeLists.txt b/tactical-microgrid-standard/tests/mc-sel/CMakeLists.txt index 4eb9448..c7b2925 100644 --- a/tactical-microgrid-standard/tests/mc-sel/CMakeLists.txt +++ b/tactical-microgrid-standard/tests/mc-sel/CMakeLists.txt @@ -1,13 +1,13 @@ cmake_minimum_required(VERSION 3.27) -project(opendds_tms_mc_sel_test CXX) +project(opendds_tms_mc_sel CXX) enable_testing() find_package(OpenDDS REQUIRED) include(opendds_testing) -add_executable(mc-sel-test mc-sel-test.cpp) -target_link_libraries(mc-sel-test PRIVATE PowerSim_Idl) +add_executable(basic-dev basic-dev.cpp) +target_link_libraries(basic-dev PRIVATE PowerSim_Idl) add_executable(basic-mc ${CMAKE_SOURCE_DIR}/controller/Controller.cpp @@ -15,4 +15,7 @@ add_executable(basic-mc basic-mc.cpp) target_link_libraries(basic-mc PRIVATE Commands_Idl) -opendds_add_test(COMMAND ./mc-sel-test) +opendds_add_test( + EXTRA_LIB_DIRS "$" + ARGS remove_dcs_after=0 test_verbose=1 +) diff --git a/tactical-microgrid-standard/tests/mc-sel/basic-dev.cpp b/tactical-microgrid-standard/tests/mc-sel/basic-dev.cpp new file mode 100644 index 0000000..dfa88fb --- /dev/null +++ b/tactical-microgrid-standard/tests/mc-sel/basic-dev.cpp @@ -0,0 +1,35 @@ +#include "common.h" + +#include + +#include + +int main(int argc, char* argv[]) +{ + const std::string name = "dev"; + PowerDevice pd(name); + if (pd.init(domain, argc, argv) != DDS::RETCODE_OK) { + return 1; + } + + DistributedConditionSet_rch dcs = + OpenDDS::DCPS::make_rch(); + + ControllerCallbacks& cbs = pd.controller_callbacks(); + cbs.set_new_controller_callback([dcs, name](const tms::Identity& id) { + printf("new_controller\n"); + dcs->post(name, "new controller " + id); + }); + cbs.set_missed_heartbeat_callback([dcs, name](const tms::Identity& id) { + dcs->post(name, "missed controller " + id); + }); + cbs.set_lost_controller_callback([dcs, name](const tms::Identity& id) { + dcs->post(name, "lost controller " + id); + }); + cbs.set_no_controllers_callback([dcs, name]() { + dcs->post(name, "no controllers"); + }); + + Exiter exiter(pd); + return pd.run(); +} diff --git a/tactical-microgrid-standard/tests/mc-sel/basic-mc.cpp b/tactical-microgrid-standard/tests/mc-sel/basic-mc.cpp index 2010106..e2214c0 100644 --- a/tactical-microgrid-standard/tests/mc-sel/basic-mc.cpp +++ b/tactical-microgrid-standard/tests/mc-sel/basic-mc.cpp @@ -1,46 +1,21 @@ -#include "controller/Controller.h" +#include "common.h" -struct Timeout { - static const char* name() { return "Timeout"; } -}; - -class Test : public TimerHandler { -public: - Test() - { - reactor_->register_handler(SIGINT, this); - schedule(Timeout{}, Sec(0), Sec(30)); - } - -private: - void timer_fired(Timer& t) - { - ACE_DEBUG((LM_INFO, "(%P|%t) Timeout\n")); - t.exit_after = true; - } - - void any_timer_fired(AnyTimer timer) - { - std::visit([&](auto&& value) { this->timer_fired(*value); }, timer); - } - - int handle_signal(int, siginfo_t*, ucontext_t*) - { - ACE_DEBUG((LM_INFO, "(%P|%t) SIGINT\n")); - return end_event_loop(); - } -}; +#include int main(int argc, char* argv[]) { - const int domain = std::stoi(argv[1]); - const std::string device_id = argv[2]; + if (argc < 3) { + ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: requires device id and priority time duration arguments\n")); + return 1; + } + const std::string device_id = argv[1]; + const uint16_t priority = std::stoi(argv[2]); - Controller mc(device_id); + Controller mc(device_id, priority); if (mc.init(domain, argc, argv) != DDS::RETCODE_OK) { return 1; } - Test test; + Exiter exiter(mc); return mc.run(); } diff --git a/tactical-microgrid-standard/tests/mc-sel/common.h b/tactical-microgrid-standard/tests/mc-sel/common.h new file mode 100644 index 0000000..aa0c52a --- /dev/null +++ b/tactical-microgrid-standard/tests/mc-sel/common.h @@ -0,0 +1,50 @@ +#include +#include + +const int domain = 1; + +struct Timeout { + bool exit_after = true; + static const char* name() { return "Timeout"; } +}; + +class Exiter : public TimerHandler { +public: + Exiter(Handshaking& handshaking, Sec exit_after = Sec(0)) + : TimerHandler(handshaking.get_reactor()) + , handshaking_(handshaking) + { + reactor_->register_handler(SIGINT, this); + if (exit_after > Sec(0)) { + schedule_once(Timeout{}, exit_after); + } + } + +private: + Handshaking& handshaking_; + + void shutdown() + { + handshaking_.delete_all_entities(); + TheServiceParticipant->shutdown(); + } + + void timer_fired(Timer& t) + { + ACE_DEBUG((LM_INFO, "(%P|%t) Timeout\n")); + shutdown(); + } + + void any_timer_fired(AnyTimer timer) + { + std::visit([&](auto&& value) { this->timer_fired(*value); }, timer); + } + + int handle_signal(int, siginfo_t*, ucontext_t*) + { + ACE_DEBUG((LM_INFO, "(%P|%t) SIGINT\n")); + shutdown(); + reactor_->end_reactor_event_loop(); + return -1; + } +}; diff --git a/tactical-microgrid-standard/tests/mc-sel/mc-sel-test.cpp b/tactical-microgrid-standard/tests/mc-sel/mc-sel-test.cpp deleted file mode 100644 index 6913ecf..0000000 --- a/tactical-microgrid-standard/tests/mc-sel/mc-sel-test.cpp +++ /dev/null @@ -1,194 +0,0 @@ -#include "power_devices/PowerDevice.h" - -#include - -#include -#include -#include -#include -#include - -namespace fs = std::filesystem; - -const int domain = 1; -const std::string device_id = "dev"; - -struct McStart { - std::vector ids; - Sec run_for; - static const char* name() { return "McStart"; }; -}; -struct McSelected { - std::string id; - bool exit_after = false; - static const char* name() { return "McSelected"; } -}; -struct McStop { - std::vector pids; - static const char* name() { return "McStop"; } -}; -const std::string mc1 = "mc1"; -const std::string mc2 = "mc2"; -const std::string mc3 = "mc3"; -class Test : public TimerHandler { -public: - Test(PowerDevice& pd, const std::string& mc_path) - : pd_(pd) - , mc_path_(mc_path) - { - proc_man_.open(10, reactor_); - reactor_->register_handler(SIGINT, this); - - /* - * Time (s): 01234567890123456789012345678901234567890123456789012345678901234567890 - * Checks: 1 2 3 4 5 - * selected: [~~mc1~~~~~~~~~~][~mc2~~~~~~~~~~~~~~] [-m1--~~~~~~~~~~~] - * mc1: <--------> <-------------> - * mc2: <-----------------------> - * mc3: <-----------------------> - */ - schedule_once(mc1, McStart{{mc1}, Sec(10)}, Sec(1)); - schedule_once("mc2 & mc3", McStart{{mc2, mc3}, Sec(25)}, Sec(5)); - schedule_once("1st check", McSelected{mc1}, Sec(10)); - schedule_once("2nd check", McSelected{mc1}, Sec(15)); - schedule_once("3rd check", McSelected{mc2}, Sec(25)); - schedule_once("4th check", McSelected{""}, Sec(55)); - schedule_once("mc1 again", McStart{{mc1}, Sec(15)}, Sec(60)); - schedule_once("5th check", McSelected{mc1, /* exit_after = */ true}, Sec(70)); - } - - ~Test() - { - stop_all(); - } - - int exit_status() - { - return failed_; - } - -private: - void stop_mc(pid_t pid) - { - ACE_DEBUG((LM_INFO, "(%P|%t) sigint %d\n", pid)); - if (pid != ACE_INVALID_PID) { - proc_man_.terminate(pid, SIGINT); - } - } - - void stop_all() - { - for (auto& pair: procs_) { - if (!pair.second.status) { - stop_mc(pair.second.pid); - } - } - } - - void timer_fired(Timer& t) - { - std::vector pids; - for (const auto& id : t.arg.ids) { - ACE_Process_Options proc_opts; - proc_opts.command_line("%s %d %s", mc_path_.c_str(), domain, id.c_str()); - Proc proc{proc_opts.command_line_buf()}; - proc.pid = proc_man_.spawn(proc_opts, this); - if (proc.ran()) { - ACE_ERROR((LM_ERROR, "(%P|%t) McStart: \"%C\" failed: %p\n", proc.cmd.c_str())); - failed_ = true; - t.exit_after = true; - return; - } - ACE_DEBUG((LM_INFO, "(%P|%t) McStart: \"%C\" pid: %d\n", proc.cmd.c_str(), proc.pid)); - procs_[proc.pid] = proc; - pids.push_back(proc.pid); - } - schedule_once(t.name, McStop{pids}, t.arg.run_for); - } - - void timer_fired(Timer& t) - { - const auto sel = pd_.selected(); - ACE_DEBUG((LM_INFO, "(%P|%t) McSelected: expect: \"%C\" selected: \"%C\"\n", t.arg.id.c_str(), sel.c_str())); - if (sel != t.arg.id) { - ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: McSelected: failed\n")); - failed_ = true; - t.exit_after = true; - } else { - t.exit_after = t.arg.exit_after; - } - } - - void timer_fired(Timer& t) - { - for (const pid_t pid : t.arg.pids) { - stop_mc(pid); - } - } - - void any_timer_fired(AnyTimer timer) - { - std::visit([&](auto&& value) { this->timer_fired(*value); }, timer); - } - - int handle_exit(ACE_Process* p) - { - Proc& proc = procs_.find(p->getpid())->second; - proc.status = p->exit_code(); - if (*proc.status) { - ACE_ERROR((LM_ERROR, "(%P|%t) ERROR: \"%C\" exited with status %d\n", - proc.cmd.c_str(), *proc.status)); - failed_ = true; - } else { - ACE_DEBUG((LM_INFO, "(%P|%t) \"%C\" exited\n", proc.cmd.c_str())); - } - return 0; - } - - int handle_signal(int, siginfo_t*, ucontext_t*) - { - ACE_DEBUG((LM_INFO, "(%P|%t) SIGINT\n")); - stop_all(); - return end_event_loop(); - } - - struct Proc { - std::string cmd; - pid_t pid = ACE_INVALID_PID; - std::optional status; - - bool ran() const - { - return pid == ACE_INVALID_PID; - } - - bool failed() const - { - return !ran() || status.value_or(1); - } - }; - - bool failed_ = false; - ACE_Process_Manager proc_man_; - std::map procs_; - PowerDevice& pd_; - const std::string mc_path_; -}; - -int main(int argc, char* argv[]) -{ - const auto our_path = fs::path(argv[0]); - const auto mc_path = our_path.parent_path() / ("basic-mc" + our_path.extension().string()); - - PowerDevice pd(device_id); - if (pd.init(domain, argc, argv) != DDS::RETCODE_OK) { - return 1; - } - - Test test(pd, mc_path.string()); - if (ACE_Reactor::instance()->run_reactor_event_loop() != 0) { - return 1; - } - - return test.exit_status(); -} diff --git a/tactical-microgrid-standard/tests/mc-sel/run_test.pl b/tactical-microgrid-standard/tests/mc-sel/run_test.pl new file mode 100755 index 0000000..c5e11aa --- /dev/null +++ b/tactical-microgrid-standard/tests/mc-sel/run_test.pl @@ -0,0 +1,98 @@ +eval '(exit $?0)' && eval 'exec perl -S $0 ${1+"$@"}' + & eval 'exec perl -S $0 $argv:q' + if 0; + +use strict; +use warnings; + +if ($^O eq 'MSWin32') { + eval('use Win32::Process;'); + die($@) if ($@); +} + +# Workaround https://github.com/OpenDDS/OpenDDS/pull/5071 +use Env qw(ACE_ROOT); +use lib "$ACE_ROOT/bin"; + +use PerlDDS::Run_Test; + +my $test = new PerlDDS::TestFramework(); + +sub start_mc { + my $name = shift(); + my $priority = shift(); + my $time = shift(); + + $test->process($name, 'basic-mc', "$name $priority -OpenDDS-tms-controller-debug true"); + $test->start_process($name); + + return $name, time() + $time; +} + +my $extra = 3; +my $lost_controller = 9 + $extra; # 3s for missed, 6s for lost + +sub expect_new_mc { + my $name = shift(); + my $max = shift() + $extra; + + return $test->wait_for('dev', "new controller $name", max_wait => $max); +} + +sub wait_until { + my $when = shift(); + + my $til = $when - time(); + if ($til > 0) { + sleep($til); + } +} + +sub expect_stop_mc { + my $name = shift(); + my $ends_at = shift(); + + wait_until($ends_at); + my $proc = $test->{processes}->{process}->{$name}->{process}->{PROCESS}; + if ($^O eq 'MSWin32') { + Win32::Process::Kill($proc, 0); + } + else { + kill('INT', $proc); + } + $test->stop_process(6, $name); +} + +sub expect_lost_mc { + my $name = shift(); + my $max = shift(); + + return $test->wait_for('dev', "lost controller $name", max_wait => $max); +} + +# Start device we will be using to see what controllers are selected +$test->process('dev', 'basic-dev', '-OpenDDS-tms-selector-debug true'); +$test->start_process('dev'); + +# Start controller mc1 and expect selection by device. +my ($mc1, $mc1_ends_at) = start_mc('mc1', 0, 20); +expect_new_mc($mc1, 10); + +sleep(3); + +# Start a few controllers with different priorities. +my ($mc2, $mc2_ends_at) = start_mc('mc2', 10, 30); +my ($mc3, $mc3_ends_at) = start_mc('mc3', 0, 30); +my ($mc4, $mc4_ends_at) = start_mc('mc4', 10, 30); + +# Stop mc1 then the device should pick up mc3 because it has the lowest +# priority value. +expect_stop_mc($mc1, $mc1_ends_at); +expect_lost_mc($mc1, $lost_controller); +expect_new_mc($mc3, 10); + +expect_stop_mc($mc2, $mc2_ends_at); +expect_stop_mc($mc3, $mc3_ends_at); +expect_stop_mc($mc4, $mc4_ends_at); +$test->kill_process(2, 'dev'); +exit $test->finish(2);