Feature: restart unresponsive inverter

we found that the inverter sometimes stops responding to commands,
especially to the "start producing" command. we now count the number of
consecutive timeouts when trying to send a new limit or power state
commands. after two timeouts were recorded, every additional timeout
will send a restart command to the inverter.

as a last resort, if the counter keeps climbing, the DTU is restarted.

notice that this only targets unresponsive inverters which are
reachable. unreachable inverters are not restarted and do not cause a
DTU reboot. this is important for solar-driven inverters, which are
unreachable during the night. the DPL will not calculate a new limit and
hence the updateInverter() method will do nothing while the target
inverter is unreachable.

publish the timeout counter to MQTT for monitoring purposes.

include/PowerLimiter.h

@@ -41,6 +41,7 @@ class PowerLimiterClass {
     };
 
     void init(Scheduler& scheduler);
+    uint8_t getInverterUpdateTimeouts() const { return _inverterUpdateTimeouts; }
     uint8_t getPowerLimiterState();
     int32_t getLastRequestedPowerLimit() { return _lastRequestedPowerLimit; }
 
@@ -77,6 +78,7 @@ class PowerLimiterClass {
     uint32_t _nextCalculateCheck = 5000; // time in millis for next NTP check to calulate restart
     bool _fullSolarPassThroughEnabled = false;
     bool _verboseLogging = true;
+    uint8_t _inverterUpdateTimeouts = 0;
 
     frozen::string const& getStatusText(Status status);
     void announceStatus(Status status);

src/MqttHandlePowerLimiter.cpp

@@ -76,6 +76,8 @@ void MqttHandlePowerLimiterClass::loop()
     auto val = static_cast<unsigned>(PowerLimiter.getMode());
     MqttSettings.publish("powerlimiter/status/mode", String(val));
 
+    MqttSettings.publish("powerlimiter/status/inverter_update_timeouts", String(PowerLimiter.getInverterUpdateTimeouts()));
+
     // no thresholds are relevant for setups without a battery
     if (config.PowerLimiter.IsInverterSolarPowered) { return; }
 

src/PowerLimiter.cpp

@@ -3,6 +3,7 @@
  * Copyright (C) 2022 Thomas Basler and others
  */
 
+#include "Utils.h"
 #include "Battery.h"
 #include "PowerMeter.h"
 #include "PowerLimiter.h"
@@ -537,15 +538,39 @@ bool PowerLimiterClass::updateInverter()
 
     if (nullptr == _inverter) { return reset(); }
 
+    // do not reset _inverterUpdateTimeouts below if no state change requested
+    if (!_oTargetPowerState.has_value() && !_oTargetPowerLimitWatts.has_value()) {
+        return reset();
+    }
+
     if (!_oUpdateStartMillis.has_value()) {
         _oUpdateStartMillis = millis();
     }
 
     if ((millis() - *_oUpdateStartMillis) > 30 * 1000) {
-        MessageOutput.printf("[DPL::updateInverter] timeout, "
+        ++_inverterUpdateTimeouts;
+        MessageOutput.printf("[DPL::updateInverter] timeout (%d in succession), "
                 "state transition pending: %s, limit pending: %s\r\n",
+                _inverterUpdateTimeouts,
                 (_oTargetPowerState.has_value()?"yes":"no"),
                 (_oTargetPowerLimitWatts.has_value()?"yes":"no"));
+
+        // NOTE that this is not always 5 minutes, since this counts timeouts,
+        // not absolute time. after any timeout, an update cycle ends. a new
+        // timeout can only happen after starting a new update cycle, which in
+        // turn is only started if the DPL did calculate a new limit, which in
+        // turn does not happen while the inverter is unreachable, no matter
+        // how long (a whole night) that might be.
+        if (_inverterUpdateTimeouts >= 10) {
+            MessageOutput.println("[DPL::loop] issuing inverter restart command after update timed out repeatedly");
+            _inverter->sendRestartControlRequest();
+        }
+
+        if (_inverterUpdateTimeouts >= 20) {
+            MessageOutput.println("[DPL::loop] restarting system since inverter is unresponsive");
+            Utils::restartDtu();
+        }
+
         return reset();
     }
 
@@ -648,6 +673,8 @@ bool PowerLimiterClass::updateInverter()
     // enable power production only after setting the desired limit
     if (switchPowerState(true)) { return true; }
 
+    _inverterUpdateTimeouts = 0;
+
     return reset();
 }