Merge pull request #3250 from TD-er/bugfix/controller_http_advanced

[HTTP Advanced] Do not percent-encode body (#1306)

.travis.yml

@@ -45,8 +45,8 @@ env:
   - ENV=minimal_IRext_ESP8266_1M
   - ENV=minimal_IRext_ESP8266_4M1M
   - ENV=minimal_IRext_ESP8266_4M2M
-  - ENV=minimal_core_242_ESP8266_1M_OTA
-  - ENV=minimal_core_242_ESP8285_1M_OTA
+  #- ENV=minimal_core_242_ESP8266_1M_OTA
+  #- ENV=minimal_core_242_ESP8285_1M_OTA
   - ENV=minimal_core_274_ESP8266_1M_OTA_Domoticz
   - ENV=minimal_core_274_ESP8266_1M_OTA_FHEM_HA
   - ENV=minimal_core_274_ESP8285_1M_OTA_Domoticz
@@ -80,7 +80,7 @@ env:
   - ENV=test_ESP8266_4M1M_VCC
   #- ENV=test_ESP8266_4M1M_VCC_MDNS_SD
   - ENV=test_alt_wifi_ESP8266_4M1M_VCC
-  - ENV=test_beta_ESP8266_16M_LittleFS
+  #- ENV=test_beta_ESP8266_16M_LittleFS
   - ENV=test_beta_ESP8266_4M1M
 
 script:

docs/source/Controller/C011.rst

@@ -37,3 +37,61 @@ Change log
 
 Description
 -----------
+
+Many systems provide a simple HTTP API where data can be changed using a HTTP request with a properly constructed querystring.
+
+This Controller allows you to send HTTP GET, POST and PUT Requests to the given webserver.
+
+For every device you have to choose to which Controllers it is sending its data. When configuring the Controller you can access this data and use them to generate your querystring.
+
+You can use this placeholders in http header and in the http body:
+
+* ``%systime%`` will be replaced with the local system time
+* ``%vcc%`` will be replaced with the power supply voltage (if enable in this build)
+* ``%ip%`` will be replaced with the local ip adress
+* ``%sysload%`` will be replaced with the system load
+* ``%uptime%`` will be replaced with the system uptime
+* ``%CR%`` will be replaced with "\r"
+* ``%LF%`` will be replaced with "\n" (newline)
+* ``%sysname%`` will be replaced with the system name
+* ``%tskname%`` will be replaced with the name of the device which is sending data to this controller
+* ``%id%`` will be replaced with IDX value.
+* ``%vname1%`` will be replaced with Valuename 1
+* ``%vname2%`` will be replaced with Valuename 2
+* ``%vname3%`` will be replaced with Valuename 3
+* ``%vname4%`` will be replaced with Valuename 4
+* ``%val1%`` will be replaced with the value 1 of the device which is sending data to this controller
+* ``%val2%`` will be replaced with the value 2 of the device which is sending data to this controller
+
+You can also write things like this:
+
+``%1%_some_text_and_placeholder_%/1%`` everything between ``%1%`` and ``%/1%`` will only print when there is a value ``1``
+
+
+Examples
+--------
+
+InfluxDB HTTP Api
+^^^^^^^^^^^^^^^^^
+
+* HTTP Method: ``POST``
+* HTTP URI: ``write?db=testdb``
+* HTTP Header: ``Content-Type: application/x-www-form-urlencoded``
+* HTTP Body: 
+
+.. code-block:: none
+
+  %1%%vname1%,Standort=%tskname%
+  Wert=%val1%%/1%%2%%LF%%vname2%,Standort=%taskname%
+  Wert=%val2%%/2%%3%%LF%%vname2%,Standort=%taskname%
+  Wert=%val3%%/3%%4%%LF%%vname2%,Standort=%taskname%
+  Wert=%val4%%/4%
+
+See also `InfluxDB API description <https://docs.influxdata.com/influxdb/v1.5/guides/writing_data/#writing-multiple-points>`_
+
+Nettemp HTTP Api
+^^^^^^^^^^^^^^^^
+
+untested but should work with something like this:
+
+``/receiver.php?device=ip&type=%1%%vname1%%/1%%2%;%vname2%%/2%%3%;%vname3%%/3%%4%;%vname4%%/4%&value=%1%%val1%%/1%%2%;%val2%%/2%%3%;%val3%%/3%%4%;%val4%%/4%``

docs/source/Controller/_Controller.rst

@@ -67,6 +67,11 @@ before WiFi connection is made or during lost connection.
   Especially useful for controllers which cannot send samples in a burst. This makes the receiving time stamp useless to detect what samples were taken around the same time.
   The sample set counter value can help matching received samples to a single set.
 
+.. note::
+  Be careful when setting the timeout too high.
+  For almost all controllers, sending data is a blocking call, so it may halt execution of other code on the node.
+  With timouts longer than 2 seconds, the ESP may reboot as the software watchdog may step in.
+
 
 
 Sample ThingSpeak configuration

docs/source/Controller/_controller_substitutions.repl

@@ -111,9 +111,9 @@
 .. |C010_maintainer| replace:: `.`
 .. |C010_compileinfo| replace:: `.`
 
-.. |C011_name| replace:: :cyan:`Generic HTTP`
+.. |C011_name| replace:: :cyan:`Generic HTTP Advanced`
 .. |C011_type| replace:: :cyan:`Controller`
-.. |C011_typename| replace:: :cyan:`Controller - Generic HTTP`
+.. |C011_typename| replace:: :cyan:`Controller - Generic HTTP Advanced`
 .. |C011_status| replace:: :yellow:`TESTING`
 .. |C011_github| replace:: C011.ino
 .. _C011_github: https://github.com/letscontrolit/ESPEasy/blob/mega/src/_C011.ino

lib/RN2483-Arduino-Library/src/rn2xx3.cpp

@@ -160,6 +160,11 @@ rn2xx3_handler::RN_state rn2xx3::async_loop()
   return _rn2xx3_handler.get_state();
 }
 
+uint8_t rn2xx3::get_busy_count() const
+{
+  return _rn2xx3_handler.get_busy_count();
+}
+
 rn2xx3_handler::RN_state rn2xx3::wait_command_finished(unsigned long timeout)
 {
   return _rn2xx3_handler.wait_command_finished(timeout);

lib/RN2483-Arduino-Library/src/rn2xx3.h

@@ -206,6 +206,8 @@ class rn2xx3 {
    */
   rn2xx3_handler::RN_state async_loop();
 
+  uint8_t get_busy_count() const;
+
   rn2xx3_handler::RN_state wait_command_finished(unsigned long timeout = 10000);
 
   rn2xx3_handler::RN_state wait_command_accepted(unsigned long timeout = 10000);

lib/RN2483-Arduino-Library/src/rn2xx3_handler.cpp

@@ -678,6 +678,10 @@ rn2xx3_handler::RN_state rn2xx3_handler::get_state() const {
   return _state;
 }
 
+uint8_t rn2xx3_handler::get_busy_count() const {
+  return _busy_count;
+}
+
 String rn2xx3_handler::sysver() {
   String ver = sendRawCommand(F("sys get ver"));
 
@@ -1014,7 +1018,7 @@ void rn2xx3_handler::handle_reply_received() {
       }
       else
       {
-        delay(1000);
+        delay(100);
       }
       break;
     }

lib/RN2483-Arduino-Library/src/rn2xx3_handler.h

@@ -181,6 +181,8 @@ class rn2xx3_handler {
 
   RN_state      get_state() const;
 
+  uint8_t       get_busy_count() const;
+
   String        sysver();
 
   // delay from last moment of sending to receive RX1 and RX2 window

platformio_core_defs.ini

@@ -143,6 +143,9 @@ platform_packages         =
 [core_esp32_1_12_2]
 platform                  = espressif32@1.12.4
 
+[core_esp32_2_0_0]
+platform                  = espressif32@2.0.0
+
 
 [core_esp32_stage]
 platform                  = https://github.com/platformio/platform-espressif32.git#feature/stage

platformio_esp32_envs.ini

@@ -8,7 +8,7 @@
 
 [esp32_common]
 extends                   = common
-platform                  = ${core_esp32_1_12_2.platform}
+platform                  = ${core_esp32_2_0_0.platform}
 lib_ignore                = ESP8266WiFi, ESP8266Ping, ESP8266WebServer, ESP8266HTTPUpdateServer, ESP8266mDNS, IRremoteESP8266, ESPEasy_ESP8266Ping, ESP32_ping, HeatpumpIR
 lib_deps                  = https://github.com/TD-er/ESPEasySerial.git#v2.0.3, Adafruit ILI9341,  Adafruit GFX Library, LOLIN_EPD, Adafruit BusIO
 board_build.f_flash       = 80000000L

requirements.txt

@@ -18,7 +18,7 @@ MarkupSafe==1.1.1
 marshmallow==3.7.1
 packaging==20.4
 pathtools==0.1.2
-platformio>=4.3.4
+platformio>=5.0.1
 port-for==0.3.1
 pycparser==2.20
 pyelftools==0.26

src/Controller.ino

@@ -153,6 +153,7 @@ bool MQTTConnect(controllerIndex_t controller_idx)
   }
   updateMQTTclient_connected();
 //  mqtt = WiFiClient(); // workaround see: https://github.com/esp8266/Arduino/issues/4497#issuecomment-373023864
+  yield();
   mqtt.setTimeout(ControllerSettings.ClientTimeout);
   MQTTclient.setClient(mqtt);
 
@@ -198,7 +199,7 @@ bool MQTTConnect(controllerIndex_t controller_idx)
 
 
   byte controller_number = Settings.Protocol[controller_idx];
-  count_connection_results(MQTTresult, F("MQTT : Broker "), controller_number, ControllerSettings);
+  count_connection_results(MQTTresult, F("MQTT : Broker "), controller_number);
 
   if (!MQTTresult) {
     MQTTclient.disconnect();
@@ -272,46 +273,55 @@ bool MQTTCheck(controllerIndex_t controller_idx)
 
   if (Protocol[ProtocolIndex].usesMQTT)
   {
-    MakeControllerSettings(ControllerSettings);
-    if (!AllocatedControllerSettings()) {
-      addLog(LOG_LEVEL_ERROR, F("MQTT : Cannot check, out of RAM"));
-      return false;
-    }
+    bool mqtt_sendLWT = false;
+    String LWTTopic, LWTMessageConnect;
+    bool willRetain = false;
+    {
+      MakeControllerSettings(ControllerSettings);
+      if (!AllocatedControllerSettings()) {
+        addLog(LOG_LEVEL_ERROR, F("MQTT : Cannot check, out of RAM"));
+        return false;
+      }
+
+      LoadControllerSettings(controller_idx, ControllerSettings);
 
-    LoadControllerSettings(controller_idx, ControllerSettings);
+      // FIXME TD-er: Is this still needed?
+      /*
+      #ifdef USES_ESPEASY_NOW
+      if (!MQTTclient.connected()) {
+        if (ControllerSettings.enableESPEasyNowFallback()) {
+          return true;
+        }
+      }
+      #endif
+      */
 
-    // FIXME TD-er: Is this still needed?
-    /*
-    #ifdef USES_ESPEASY_NOW
-    if (!MQTTclient.connected()) {
-      if (ControllerSettings.enableESPEasyNowFallback()) {
+      if (!ControllerSettings.isSet()) {
         return true;
       }
+
+      if (ControllerSettings.mqtt_sendLWT()) {
+        mqtt_sendLWT = true;
+        LWTTopic          = getLWT_topic(ControllerSettings);
+        LWTMessageConnect = getLWT_messageConnect(ControllerSettings);
+        willRetain        = ControllerSettings.mqtt_willRetain();
+      }
     }
-    #endif
-    */
-
-    if (ControllerSettings.isSet()) {
-      if (MQTTclient_should_reconnect || !MQTTclient.connected())
-      {
-        if (MQTTclient_should_reconnect) {
-          addLog(LOG_LEVEL_ERROR, F("MQTT : Intentional reconnect"));
-        }
-        return MQTTConnect(controller_idx);
+    if (MQTTclient_should_reconnect || !MQTTclient.connected())
+    {
+      if (MQTTclient_should_reconnect) {
+        addLog(LOG_LEVEL_ERROR, F("MQTT : Intentional reconnect"));
       }
+      return MQTTConnect(controller_idx);
+    }
 
-      if (MQTTclient_must_send_LWT_connected) {
-        if (ControllerSettings.mqtt_sendLWT()) {
-          String LWTTopic          = getLWT_topic(ControllerSettings);
-          String LWTMessageConnect = getLWT_messageConnect(ControllerSettings);
-          bool   willRetain        = ControllerSettings.mqtt_willRetain();
-
-          if (MQTTclient.publish(LWTTopic.c_str(), LWTMessageConnect.c_str(), willRetain)) {
-            MQTTclient_must_send_LWT_connected = false;
-          }
-        } else {
+    if (MQTTclient_must_send_LWT_connected) {
+      if (mqtt_sendLWT) {
+        if (MQTTclient.publish(LWTTopic.c_str(), LWTMessageConnect.c_str(), willRetain)) {
           MQTTclient_must_send_LWT_connected = false;
         }
+      } else {
+        MQTTclient_must_send_LWT_connected = false;
       }
     }
   }