Merge remote-tracking branch 'origin/release/v3.x' into develop

CHANGELOG.md

@@ -1,3 +1,31 @@
+## 3.0.0-rc.2
+
+### FEATURES
+
+- [Cellular] Battery presence detection when charging is disabled [#2272](https://github.com/particle-iot/device-os/pull/2272)
+- Increase the maximum DTLS packet size and payload of the cloud primitives [#2260](https://github.com/particle-iot/device-os/pull/2260)
+
+### ENHANCEMENTS
+
+- [Cellular] Update ICCID/IMSI to APN map with a new Kore ICCID prefix [#2276](https://github.com/particle-iot/device-os/pull/2276)
+- Disable some of the elliptic-curves not in use to save flash space [#2273](https://github.com/particle-iot/device-os/pull/2273)
+- [Cellular] Update LTE signal strength/quality parameters (RSRP/RSRQ) mapping to percentages [#2285](https://github.com/particle-iot/device-os/pull/2285)
+- [Cellular] System power management improvements [#2272](https://github.com/particle-iot/device-os/pull/2272)
+- [B5 SoM / Quectel] Improve warm and cold boot behavior [#2300](https://github.com/particle-iot/device-os/pull/2300)
+
+### BUGFIXES
+
+- [Gen 3] Use `PIN_INVALID` when initializing SPI peripheral to avoid overriding the pin mode of the default CS pin on reinitialization [#2275](https://github.com/particle-iot/device-os/pull/2275)
+- [Argon / Tracker] Make sure that ESP32 NCP power state is correctly initialized on boot [#2279](https://github.com/particle-iot/device-os/pull/2279)
+- [Electron] Increase `AT+COPS` timeout to 5 minutes [#2281](https://github.com/particle-iot/device-os/pull/2281)
+- [Electron] Fix Sleep 2.0 APIs taking up to 10 minutes to power-off the cellular modem while it's attempting network registration [#2284](https://github.com/particle-iot/device-os/pull/2284)
+- [B5 SoM / Tracker] Fix warm boot sometimes requiring modem reset [#2289](https://github.com/particle-iot/device-os/pull/2289)
+- [Gen 3] Fix micros/millis/unixtime becoming non-monotonic when RTC overflow event occurs [2a4fcb82b](https://github.com/particle-iot/device-os/commit/2a4fcb82b0968300b8a0227f665ffe94203f9f38)
+
+### INTERNAL
+
+- Startup SLO automated tests [#2277](https://github.com/particle-iot/device-os/pull/2277) [#2274](https://github.com/particle-iot/device-os/pull/2274)
+
 ## 3.0.0-rc.1
 
 ### FEATURES

build/release.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -o errexit -o pipefail -o noclobber -o nounset
 
-VERSION="3.0.0-rc.1"
+VERSION="3.0.0-rc.2"
 
 function display_help ()
 {

build/version.mk

@@ -1,7 +1,7 @@
-VERSION_STRING = 3.0.0-rc.1
+VERSION_STRING = 3.0.0-rc.2
 
 # PRODUCT_FIRMWARE_VERSION reported by default
 # FIXME: Unclear if this is used, PRODUCT_FIRMWARE_VERSION defaults to 65535 every release
-VERSION = 3003
+VERSION = 3004
 
 CFLAGS += -DSYSTEM_VERSION_STRING=$(VERSION_STRING)

communication/inc/protocol.h

@@ -185,6 +185,11 @@ class Protocol
 	 */
 	uint16_t system_version;
 
+	/**
+	 * Maximum size of an outgoing CoAP message.
+	 */
+	size_t max_transmit_message_size;
+
 	void set_protocol_flags(uint32_t flags)
 	{
 		protocol_flags = flags;
@@ -348,7 +353,8 @@ class Protocol
 			initialized(false),
 			max_binary_size(0), // Unlimited
 			ota_chunk_size(DEFAULT_OTA_CHUNK_SIZE),
-			system_version(0) // Unknown
+			system_version(0), // Unknown
+			max_transmit_message_size(0) // Limited by compile-time maximum
 	{
 	}
 
@@ -399,6 +405,16 @@ class Protocol
 		ota_chunk_size = size;
 	}
 
+	void set_max_transmit_message_size(size_t size)
+	{
+		max_transmit_message_size = size;
+	}
+
+	size_t get_max_transmit_message_size() const
+	{
+		return max_transmit_message_size;
+	}
+
 	void set_handlers(CommunicationsHandlers& handlers)
 	{
 		copy_and_init(&this->handlers, sizeof(this->handlers), &handlers, handlers.size);

communication/inc/protocol_defs.h

@@ -150,7 +150,8 @@ enum Enum
     COMPRESSED_OTA = 3, ///< Enable support for compressed/combined OTA updates.
     SYSTEM_MODULE_VERSION = 4, ///< Module version of the system firmware.
     MAX_BINARY_SIZE = 5, ///< Maximum size of a firmware binary.
-    OTA_CHUNK_SIZE = 6 ///< Size of an OTA update chunk.
+    OTA_CHUNK_SIZE = 6, ///< Size of an OTA update chunk.
+    MAX_TRANSMIT_MESSAGE_SIZE = 7 ///< Maximum size of of outgoing CoAP message.
 };
 
 }

communication/src/messages.cpp

@@ -316,7 +316,7 @@ size_t Messages::separate_response_with_payload(unsigned char *buf, uint16_t mes
 }
 
 size_t Messages::event(uint8_t buf[], uint16_t message_id, const char *event_name,
-             const char *data, int ttl, EventType::Enum event_type, bool confirmable)
+             const char *data, size_t data_size, int ttl, EventType::Enum event_type, bool confirmable)
 {
   uint8_t *p = buf;
   *p++ = confirmable ? 0x40 : 0x50; // non-confirmable /confirmable, no token
@@ -337,13 +337,12 @@ size_t Messages::event(uint8_t buf[], uint16_t message_id, const char *event_nam
     *p++ = ttl & 0xff;
   }
 
-  if (NULL != data)
+  if (NULL != data && data_size > 0)
   {
-    name_data_len = strnlen(data, MAX_EVENT_DATA_LENGTH);
-
     *p++ = 0xff;
-    memcpy(p, data, name_data_len);
-    p += name_data_len;
+
+    memcpy(p, data, data_size);
+    p += data_size;
   }
 
   return p - buf;

communication/src/messages.h

@@ -80,7 +80,7 @@ class Messages
 			unsigned payload_len, bool confirmable);
 
 	static size_t event(uint8_t buf[], uint16_t message_id, const char *event_name,
-	             const char *data, int ttl, EventType::Enum event_type, bool confirmable);
+	             const char *data, size_t data_size, int ttl, EventType::Enum event_type, bool confirmable);
 
 
     static inline size_t empty_ack(unsigned char *buf,

communication/src/publisher.cpp

@@ -19,6 +19,61 @@
 
 #include "protocol.h"
 
-void particle::protocol::Publisher::add_ack_handler(message_id_t msg_id, CompletionHandler handler) {
+namespace particle {
+
+namespace protocol {
+
+void Publisher::add_ack_handler(message_id_t msg_id, CompletionHandler handler) {
     protocol->add_ack_handler(msg_id, std::move(handler), SEND_EVENT_ACK_TIMEOUT);
 }
+
+ProtocolError Publisher::send_event(MessageChannel& channel, const char* event_name,
+            const char* data, int ttl, EventType::Enum event_type, int flags,
+            system_tick_t time, CompletionHandler handler) {
+    bool is_system_event = is_system(event_name);
+    bool rate_limited = is_rate_limited(is_system_event, time);
+    if (rate_limited) {
+        g_rateLimitedEventsCounter++;
+        return BANDWIDTH_EXCEEDED;
+    }
+
+    Message message;
+    channel.create(message);
+    bool confirmable = channel.is_unreliable();
+    if (flags & EventType::NO_ACK) {
+        confirmable = false;
+    } else if (flags & EventType::WITH_ACK) {
+        confirmable = true;
+    }
+
+    // Adjust the maximum event payload size if MAX_TRANSMIT_MESSAGE_SIZE property
+	// indicates that we cannot transmit over a certain limit.
+    const auto max_transmit_message_size = protocol->get_max_transmit_message_size();
+    size_t data_size = 0;
+    if (data) {
+        size_t max_size = MAX_EVENT_DATA_LENGTH;
+        if (max_transmit_message_size && max_transmit_message_size < MAX_EVENT_MESSAGE_SIZE) {
+            const auto overhead = MAX_EVENT_MESSAGE_SIZE - MAX_EVENT_NAME_LENGTH - MAX_EVENT_DATA_LENGTH;
+            const size_t event_name_length = strnlen(event_name, MAX_EVENT_NAME_LENGTH);
+            max_size = std::min(max_size, max_transmit_message_size - overhead - event_name_length);
+        }
+        data_size = strnlen(data, max_size);
+    }
+    size_t msglen = Messages::event(message.buf(), 0, event_name, data, data_size, ttl,
+            event_type, confirmable);
+    message.set_length(msglen);
+    const ProtocolError result = channel.send(message);
+    if (result == NO_ERROR) {
+        // Register completion handler only if acknowledgement was requested explicitly
+        if ((flags & EventType::WITH_ACK) && message.has_id()) {
+            add_ack_handler(message.get_id(), std::move(handler));
+        } else {
+            handler.setResult();
+        }
+    }
+    return result;
+}
+
+} // protocol
+
+} // particle

communication/src/publisher.h

@@ -89,37 +89,7 @@ class Publisher
 
 	ProtocolError send_event(MessageChannel& channel, const char* event_name,
 			const char* data, int ttl, EventType::Enum event_type, int flags,
-			system_tick_t time, CompletionHandler handler)
-	{
-		bool is_system_event = is_system(event_name);
-		bool rate_limited = is_rate_limited(is_system_event, time);
-		if (rate_limited) {
-			g_rateLimitedEventsCounter++;
-			return BANDWIDTH_EXCEEDED;
-		}
-
-		Message message;
-		channel.create(message);
-		bool confirmable = channel.is_unreliable();
-		if (flags & EventType::NO_ACK) {
-			confirmable = false;
-		} else if (flags & EventType::WITH_ACK) {
-			confirmable = true;
-		}
-		size_t msglen = Messages::event(message.buf(), 0, event_name, data, ttl,
-				event_type, confirmable);
-		message.set_length(msglen);
-		const ProtocolError result = channel.send(message);
-		if (result == NO_ERROR) {
-			// Register completion handler only if acknowledgement was requested explicitly
-			if ((flags & EventType::WITH_ACK) && message.has_id()) {
-			    add_ack_handler(message.get_id(), std::move(handler));
-			} else {
-			    handler.setResult();
-			}
-		}
-		return result;
-	}
+			system_tick_t time, CompletionHandler handler);
 
 private:
 	Protocol* protocol;

communication/src/spark_protocol_functions.cpp

@@ -225,6 +225,10 @@ int spark_protocol_set_connection_property(ProtocolFacade* protocol, unsigned pr
         protocol->set_ota_chunk_size(value);
         return 0;
     }
+    case particle::protocol::Connection::MAX_TRANSMIT_MESSAGE_SIZE: {
+        protocol->set_max_transmit_message_size(value);
+        return 0;
+    }
     default:
         return particle::protocol::ProtocolError::NOT_IMPLEMENTED;
     }

communication/src/variables.cpp

@@ -133,6 +133,16 @@ ProtocolError Variables::encode_response(Message& message, token_t token, const
     if (value_size > MAX_VARIABLE_VALUE_LENGTH) {
         value_size = MAX_VARIABLE_VALUE_LENGTH; // Truncate the value data
     }
+    // Adjust the maximum variable payload size if MAX_TRANSMIT_MESSAGE_SIZE property
+	// indicates that we cannot transmit over a certain limit.
+    const size_t max_transmit_message_size = protocol_->get_max_transmit_message_size();
+    if (max_transmit_message_size > 0 && max_transmit_message_size < MAX_VARIABLE_VALUE_MESSAGE_SIZE) {
+        auto max_variable_value_length = max_transmit_message_size -
+                (MAX_VARIABLE_VALUE_MESSAGE_SIZE - MAX_VARIABLE_VALUE_LENGTH);
+        if (value_size > max_variable_value_length) {
+            value_size = max_variable_value_length;
+        }
+    }
     size_t msg_size = 0;
     switch (value_type) {
     case SparkReturnType::BOOLEAN: {

hal/inc/hal_platform.h

@@ -442,4 +442,8 @@
 #define HAL_PLATFORM_WIFI_SCAN_ONLY (0)
 #endif // HAL_PLATFORM_WIFI_SCAN_ONLY
 
+#ifndef HAL_PLATFORM_BROKEN_MTU
+#define HAL_PLATFORM_BROKEN_MTU (0)
+#endif // HAL_PLATFORM_BROKEN_MTU
+
 #endif /* HAL_PLATFORM_H */

hal/network/lwip/cellular/pppncpnetif.cpp

@@ -322,6 +322,11 @@ void PppNcpNetif::pppEventHandler(uint64_t ev) {
         LOG(TRACE, "Negotiated MTU: %u", mtu);
         // Reset
         connectStart_ = 0;
+        const auto ncpMtu = celMan_->ncpClient()->getMtu();
+        if (ncpMtu > 0 && (unsigned)ncpMtu < mtu) {
+            LOG(TRACE, "Updating MTU to: %u", ncpMtu);
+            client_.getIf()->mtu = ncpMtu;
+        }
     } else if (ev == particle::net::ppp::Client::EVENT_CONNECTING) {
         if (connectStart_ == 0) {
             connectStart_ = HAL_Timer_Get_Milli_Seconds();

hal/network/lwip/ppp_client.cpp

@@ -278,7 +278,15 @@ void Client::setAuth(const char* user, const char* password) {
 
   {
     LwipTcpIpCoreLock lk;
-    ppp_set_auth(pcb_, PPPAUTHTYPE_ANY, user_.get(), pass_.get());
+    auto authtype = PPPAUTHTYPE_CHAP | PPPAUTHTYPE_PAP;
+    // FIXME: CHAP authentication is broken on Quectel modems when
+    // resuming into an ongoing PPP session. We are not getting a CHAP
+    // challenge request and just sit in AUTH stage indefinitely.
+    // As a workaround enable only PAP
+    if (PLATFORM_NCP_MANUFACTURER(platform_primary_ncp_identifier()) == PLATFORM_NCP_MANUFACTURER_QUECTEL) {
+      authtype = PPPAUTHTYPE_PAP;
+    }
+    ppp_set_auth(pcb_, authtype, user_.get(), pass_.get());
   }
 }
 

hal/network/ncp/cellular/cellular_ncp_client.h

@@ -163,6 +163,7 @@ class CellularNcpClient: public NcpClient {
     virtual int setRegistrationTimeout(unsigned timeout) = 0;
     virtual int getTxDelayInDataChannel() = 0;
     virtual int enterDataMode() = 0;
+    virtual int getMtu() = 0;
 };
 
 inline CellularNcpClientConfig::CellularNcpClientConfig() :