Merge remote-tracking branch 'tbnobody/OpenDTU/master'

.gitignore

@@ -4,4 +4,5 @@
 .vscode/launch.json
 .vscode/ipch
 .vscode/settings.json
-platformio-device-monitor*.log
+platformio-device-monitor*.log
+platformio_override.ini

README.md

@@ -59,9 +59,15 @@ Sends text raw data as difined in VE.Direct spec.
 * Hoymiles HM-1000
 * Hoymiles HM-1200
 * Hoymiles HM-1500
-* TSUN TSOL-M350 (Maybe depending on firmware on the inverter)
-* TSUN TSOL-M800 (Maybe depending on firmware on the inverter)
-* TSUN TSOL-M1600 (Maybe depending on firmware on the inverter)
+* TSUN TSOL-M350 (Maybe depending on firmware/serial number on the inverter)
+* TSUN TSOL-M800 (Maybe depending on firmware/serial number on the inverter)
+* TSUN TSOL-M1600 (Maybe depending on firmware/serial number on the inverter)
+
+**TSUN compatibility remark:**
+Compatibility with OpenDTU seems to be related to serial numbers. Current findings indicate that TSUN inverters with a serial number starting with "11" are supported, whereby inverters with a serial number starting with "10" are not.
+Firmware version seems to play not a significant role and cannot be read from the stickers. For completeness, the following firmware version have been reported to work with OpenDTU:
+* v1.0.10 TSOL-M800 (DE)
+* v1.0.12 TSOL-M1600
 
 ## Features for end users
 * Read live data from inverter
@@ -80,6 +86,7 @@ Sends text raw data as difined in VE.Direct spec.
 * Ve.Direct interface (via web-interface, REST-api, or MQTT)
 * Ethernet support
 * Prometheus API endpoint (/api/prometheus/metrics)
+* English and german web interface
 
 ## Features for developers
 * The microcontroller part
@@ -132,7 +139,7 @@ Use a power suppy with 5 V and 1 A. The USB cable connected to your PC/Notebook
 
 ### Change pin assignment
 Its possible to change all the pins of the NRF24L01+ module.
-This can be achieved by editing the 'platformio.ini' file and add/change one or more of the following lines to the 'build_flags' parameter:
+This can be achieved by copying one of the [env:....] sections from 'platformio.ini' to 'platformio_override.ini' and editing the 'platformio_override.ini' file and add/change one or more of the following lines to the 'build_flags' parameter:
 ```
 -DHOYMILES_PIN_MISO=19
 -DHOYMILES_PIN_MOSI=23
@@ -143,15 +150,16 @@ This can be achieved by editing the 'platformio.ini' file and add/change one or
 -DVICTRON_PIN_TX=21
 -DVICTRON_PIN_RX=22 
 ```
+It is recommended to make all changes only in the  'platformio_override.ini', this is your personal copy.
 
 ## Flashing and starting up
 ### with Visual Studio Code
 * Install [Visual Studio Code](https://code.visualstudio.com/download) (from now named "vscode")
 * In Visual Studio Code, install the [PlatformIO Extension](https://marketplace.visualstudio.com/items?itemName=platformio.platformio-ide)
 * Install git and enable git in vscode - [git download](https://git-scm.com/downloads/) - [Instructions](https://www.jcchouinard.com/install-git-in-vscode/)
 * Clone this repository (you really have to clone it, don't just download the ZIP file. During the build process the git hash gets embedded into the firmware. If you download the ZIP file a build error will occur): Inside vscode open the command palette by pressing `CTRL` + `SHIFT` + `P`. Enter `git clone`, add the repository-URL `https://github.com/tbnobody/OpenDTU`. Next you have to choose (or create) a target directory.
-* In vscode, choose File --> Open Folder and select the previously downloaded source code. (You have to select the folder which contains the "platformio.ini" file)
-* Adjust the COM port in the file "platformio.ini" for your USB-serial-converter. It occurs twice:
+* In vscode, choose File --> Open Folder and select the previously downloaded source code. (You have to select the folder which contains the "platformio.ini" and "platformio_override.ini" file)
+* Adjust the COM port in the file "platformio_override.ini" for your USB-to-serial-converter. It occurs twice:
     * upload_port
     * monitor_port
 * Select the arrow button in the blue bottom status bar (PlatformIO: Upload) to compile and upload the firmware. During the compilation, all required libraries are downloaded automatically.
@@ -162,7 +170,7 @@ This can be achieved by editing the 'platformio.ini' file and add/change one or
 ### on the commandline with PlatformIO Core
 * Install [PlatformIO Core](https://platformio.org/install/cli)
 * Clone this repository (you really have to clone it, don't just download the ZIP file. During the build process the git hash gets embedded into the firmware. If you download the ZIP file a build error will occur)
-* Adjust the COM port in the file "platformio.ini". It occurs twice:
+* Adjust the COM port in the file "platformio_override.ini". It occurs twice:
     * upload_port
     * monitor_port
 * build: `platformio run -e generic`

docs/MQTT_Topics.md

@@ -67,7 +67,7 @@ cmd topics are used to set values. Status topics are updated from values set in
 | [serial]/status/limit_absolute            | R     | Current applied production limit of the inverter     | Watt (W)                   |
 | [serial]/cmd/limit_persistent_relative    | W     | Set the inverter limit as a percentage of total production capability. The  value will survive the night without power. The updated value will show up in the web GUI and limit_relative topic immediatly. | %                          |
 | [serial]/cmd/limit_persistent_absolute    | W     | Set the inverter limit as a absolute value. The  value will survive the night without power. The updated value will set immediatly within the inverter but show up in the web GUI and limit_relative topic after around 4 minutes. If you are using a already known inverter (known Hardware ID), the updated value will show up within a few seconds. | Watt (W)                   |
-| [serial]/cmd/limit_nonpersistent_relative | W     | Set the inverter limit as a percentage of total production capability. The  value will reset to the last persistent value at night without power. The updated value will show up in the web GUI and limit_relative topic immediatly. | %                          |
-| [serial]/cmd/limit_nonpersistent_absolute | W     | Set the inverter limit as a absolute value. The  value will reset to the last persistent value at night without power. The updated value will set immediatly within the inverter but show up in the web GUI and limit_relative topic after around 4 minutes. If you are using a already known inverter (known Hardware ID), the updated value will show up within a few seconds. | Watt (W)                   |
+| [serial]/cmd/limit_nonpersistent_relative | W     | Set the inverter limit as a percentage of total production capability. The  value will reset to the last persistent value at night without power. The updated value will show up in the web GUI and limit_relative topic immediatly. The value must be published non-retained, otherwise it will be ignored! | %                          |
+| [serial]/cmd/limit_nonpersistent_absolute | W     | Set the inverter limit as a absolute value. The  value will reset to the last persistent value at night without power. The updated value will set immediatly within the inverter but show up in the web GUI and limit_relative topic after around 4 minutes. If you are using a already known inverter (known Hardware ID), the updated value will show up within a few seconds. The value must be published non-retained, otherwise it will be ignored! | Watt (W)                   |
 | [serial]/cmd/power                        | W      | Turn the inverter on (1) or off (0)                 | 0 or 1                     |
 | [serial]/cmd/restart                      | W      | Restarts the inverters (also resets YieldDay)       | 1                          |

include/MessageOutput.h

@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#pragma once
+
+#include <AsyncWebSocket.h>
+#include <HardwareSerial.h>
+#include <Stream.h>
+
+#define BUFFER_SIZE 500
+
+class MessageOutputClass : public Print {
+public:
+    MessageOutputClass();
+    void loop();
+    size_t write(uint8_t c);
+    void register_ws_output(AsyncWebSocket* output);
+
+private:
+    AsyncWebSocket* _ws = NULL;
+    char _buffer[BUFFER_SIZE];
+    uint16_t _buff_pos = 0;
+    uint32_t _lastSend = 0;
+    bool _forceSend = false;
+
+    SemaphoreHandle_t _lock;
+};
+
+extern MessageOutputClass MessageOutput;

include/MqttHandleDtu.h

@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#pragma once
+
+#include <cstdint>
+
+class MqttHandleDtuClass {
+public:
+    void init();
+    void loop();
+
+private:
+    uint32_t _lastPublish;
+};
+
+extern MqttHandleDtuClass MqttHandleDtu;

include/MqttHassPublishing.h → include/MqttHandleHass.h

@@ -1,11 +1,8 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 #pragma once
 
-#include "Configuration.h"
-#include <Arduino.h>
 #include <ArduinoJson.h>
 #include <Hoymiles.h>
-#include <memory>
 
 // mqtt discovery device classes
 enum {
@@ -51,14 +48,15 @@ const byteAssign_fieldDeviceClass_t deviceFieldAssignment[] = {
 };
 #define DEVICE_CLS_ASSIGN_LIST_LEN (sizeof(deviceFieldAssignment) / sizeof(byteAssign_fieldDeviceClass_t))
 
-class MqttHassPublishingClass {
+class MqttHandleHassClass {
 public:
     void init();
     void loop();
     void publishConfig();
     void forceUpdate();
 
 private:
+    void publish(const String& subtopic, const String& payload);
     void publishField(std::shared_ptr<InverterAbstract> inv, uint8_t channel, byteAssign_fieldDeviceClass_t fieldType, bool clear = false);
     void publishInverterButton(std::shared_ptr<InverterAbstract> inv, const char* caption, const char* icon, const char* category, const char* deviceClass, const char* subTopic, const char* payload);
     void publishInverterNumber(std::shared_ptr<InverterAbstract> inv, const char* caption, const char* icon, const char* category, const char* commandTopic, const char* stateTopic, const char* unitOfMeasure, int16_t min = 1, int16_t max = 100);
@@ -69,4 +67,4 @@ class MqttHassPublishingClass {
     bool _updateForced = false;
 };
 
-extern MqttHassPublishingClass MqttHassPublishing;
+extern MqttHandleHassClass MqttHandleHass;

include/MqttPublishing.h → include/MqttHandleInverter.h

@@ -2,11 +2,10 @@
 #pragma once
 
 #include "Configuration.h"
-#include <Arduino.h>
 #include <Hoymiles.h>
-#include <memory>
+#include <espMqttClient.h>
 
-class MqttPublishingClass {
+class MqttHandleInverterClass {
 public:
     void init();
     void loop();
@@ -15,6 +14,7 @@ class MqttPublishingClass {
 
 private:
     void publishField(std::shared_ptr<InverterAbstract> inv, uint8_t channel, uint8_t fieldId);
+    void onMqttMessage(const espMqttClientTypes::MessageProperties& properties, const char* topic, const uint8_t* payload, size_t len, size_t index, size_t total);
 
     uint32_t _lastPublishStats[INV_MAX_COUNT];
     uint32_t _lastPublish;
@@ -37,4 +37,4 @@ class MqttPublishingClass {
     };
 };
 
-extern MqttPublishingClass MqttPublishing;
+extern MqttHandleInverterClass MqttHandleInverter;

include/MqttVedirectPublishing.h → include/MqttHandleVedirect.h

@@ -14,7 +14,7 @@
 #define VICTRON_PIN_TX 21
 #endif
 
-class MqttVedirectPublishingClass {
+class MqttHandleVedirectClass {
 public:
     void init();
     void loop();
@@ -23,4 +23,4 @@ class MqttVedirectPublishingClass {
     uint32_t _lastPublish;
 };
 
-extern MqttVedirectPublishingClass MqttVedirectPublishing;
+extern MqttHandleVedirectClass MqttHandleVedirect;

include/MqttSettings.h

@@ -2,10 +2,9 @@
 #pragma once
 
 #include "NetworkSettings.h"
-#include <Arduino.h>
+#include <MqttSubscribeParser.h>
 #include <Ticker.h>
 #include <espMqttClient.h>
-#include <memory>
 
 class MqttSettingsClass {
 public:
@@ -14,7 +13,10 @@ class MqttSettingsClass {
     void performReconnect();
     bool getConnected();
     void publish(const String& subtopic, const String& payload);
-    void publishHass(const String& subtopic, const String& payload);
+    void publishGeneric(const String& topic, const String& payload, bool retain, uint8_t qos = 0);
+
+    void subscribe(const String& topic, uint8_t qos, const espMqttClientTypes::OnMessageCallback& cb);
+    void unsubscribe(const String& topic);
 
     String getPrefix();
 
@@ -34,6 +36,7 @@ class MqttSettingsClass {
     String clientId;
     String willTopic;
     Ticker mqttReconnectTimer;
+    MqttSubscribeParser _mqttSubscribeParser;
 };
 
 extern MqttSettingsClass MqttSettings;

include/NetworkSettings.h

@@ -3,7 +3,6 @@
 
 #include <DNSServer.h>
 #include <WiFi.h>
-#include <memory>
 #include <vector>
 
 enum class network_mode {

include/NtpSettings.h

@@ -1,8 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 #pragma once
 
-#include <Arduino.h>
-
 class NtpSettingsClass {
 public:
     NtpSettingsClass();

include/WebApi.h

@@ -17,6 +17,7 @@
 #include "WebApi_security.h"
 #include "WebApi_sysstatus.h"
 #include "WebApi_webapp.h"
+#include "WebApi_ws_console.h"
 #include "WebApi_ws_live.h"
 #include "WebApi_ws_vedirect_live.h"
 #include "WebApi_vedirect.h"
@@ -51,6 +52,7 @@ class WebApiClass {
     WebApiSecurityClass _webApiSecurity;
     WebApiSysstatusClass _webApiSysstatus;
     WebApiWebappClass _webApiWebapp;
+    WebApiWsConsoleClass _webApiWsConsole;
     WebApiWsLiveClass _webApiWsLive;
     WebApiWsVedirectLiveClass _webApiWsVedirectLive;
     WebApiVedirectClass _webApiVedirect;

include/WebApi_errors.h

@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#pragma once
+
+enum WebApiError {
+    GenericBase = 1000,
+    GenericSuccess,
+    GenericNoValueFound,
+    GenericDataTooLarge,
+    GenericParseError,
+    GenericValueMissing,
+
+    DtuBase = 2000,
+    DtuSerialZero,
+    DtuPollZero,
+    DtuInvalidPowerLevel,
+
+    ConfigBase = 3000,
+    ConfigNotDeleted,
+    ConfigSuccess,
+
+    InverterBase = 4000,
+    InverterSerialZero,
+    InverterNameLength,
+    InverterCount,
+    InverterAdded,
+    InverterInvalidId,
+    InverterInvalidMaxChannel,
+    InverterChanged,
+    InverterDeleted,
+
+    LimitBase = 5000,
+    LimitSerialZero,
+    LimitInvalidLimit,
+    LimitInvalidType,
+    LimitInvalidInverter,
+
+    MaintenanceBase = 6000,
+    MaintenanceRebootTriggered,
+    MaintenanceRebootCancled,
+
+    MqttBase = 7000,
+    MqttHostnameLength,
+    MqttUsernameLength,
+    MqttPasswordLength,
+    MqttTopicLength,
+    MqttTopicCharacter,
+    MqttTopicTrailingSlash,
+    MqttPort,
+    MqttCertificateLength,
+    MqttLwtTopicLength,
+    MqttLwtTopicCharacter,
+    MqttLwtOnlineLength,
+    MqttLwtOfflineLength,
+    MqttPublishInterval,
+    MqttHassTopicLength,
+    MqttHassTopicCharacter,
+
+    NetworkBase = 8000,
+    NetworkIpInvalid,
+    NetworkNetmaskInvalid,
+    NetworkGatewayInvalid,
+    NetworkDns1Invalid,
+    NetworkDns2Invalid,
+
+    NtpBase = 9000,
+    NtpServerLength,
+    NtpTimezoneLength,
+    NtpTimezoneDescriptionLength,
+    NtpYearInvalid,
+    NtpMonthInvalid,
+    NtpDayInvalid,
+    NtpHourInvalid,
+    NtpMinuteInvalid,
+    NtpSecondInvalid,
+    NtpTimeUpdated,
+
+    SecurityBase = 10000,
+    SecurityPasswordLength,
+    SecurityAuthSuccess,
+
+    PowerBase = 11000,
+    PowerSerialZero,
+    PowerInvalidInverter,
+};

include/WebApi_prometheus.h

@@ -1,7 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
 #pragma once
 
-#include "Hoymiles.h"
 #include <ESPAsyncWebServer.h>
+#include <Hoymiles.h>
 
 class WebApiPrometheusClass {
 public:

include/WebApi_ws_console.h

@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#pragma once
+
+#include <ESPAsyncWebServer.h>
+
+class WebApiWsConsoleClass {
+public:
+    WebApiWsConsoleClass();
+    void init(AsyncWebServer* server);
+    void loop();
+
+private:
+    void onWebsocketEvent(AsyncWebSocket* server, AsyncWebSocketClient* client, AwsEventType type, void* arg, uint8_t* data, size_t len);
+
+    AsyncWebServer* _server;
+    AsyncWebSocket _ws;
+
+    uint32_t _lastWsCleanup = 0;
+};