From 5fe67ab3cf015b72807f847564062614c5664697 Mon Sep 17 00:00:00 2001 From: Scott Brust Date: Wed, 11 Oct 2023 17:09:48 -0700 Subject: [PATCH] Implement reachability test message. Prioritize networks on tester round trip time --- system/src/system_cloud_connection_posix.cpp | 2 +- system/src/system_connection_manager.cpp | 157 ++++++++++++------- system/src/system_connection_manager.h | 7 +- system/src/system_network_manager.cpp | 2 +- 4 files changed, 112 insertions(+), 56 deletions(-) diff --git a/system/src/system_cloud_connection_posix.cpp b/system/src/system_cloud_connection_posix.cpp index 74565ec5e1..6b705e15dd 100644 --- a/system/src/system_cloud_connection_posix.cpp +++ b/system/src/system_cloud_connection_posix.cpp @@ -213,7 +213,7 @@ int system_cloud_connect(int protocol, const ServerAddress* address, sockaddr* s if_index_to_name(cloudInterface, ifr.ifr_name); auto sockOptRet = sock_setsockopt(s, SOL_SOCKET, SO_BINDTODEVICE, &ifr, sizeof(ifr)); - LOG(TRACE, "Bound socket result %d from internal index %lu to lwip interface %s", sockOptRet, cloudInterface, ifr.ifr_name); + LOG(TRACE, "%d Bound cloud socket to lwip interface %s", sockOptRet, ifr.ifr_name); } /* FIXME: timeout for TCP */ diff --git a/system/src/system_connection_manager.cpp b/system/src/system_connection_manager.cpp index e6cc16b8e0..fd334409b5 100644 --- a/system/src/system_connection_manager.cpp +++ b/system/src/system_connection_manager.cpp @@ -28,13 +28,11 @@ LOG_SOURCE_CATEGORY("system.cm") #include "system_connection_manager.h" #include "system_cloud.h" #include "spark_wiring_network.h" -#include "spark_wiring_udp.h" #include "spark_wiring_vector.h" +#include "endian_util.h" // TODO: not use workarounds -#include "spark_wiring_ethernet.h" -#include "spark_wiring_cellular.h" -#include "spark_wiring_wifi.h" +#include "spark_wiring_udp.h" namespace particle { namespace system { @@ -53,9 +51,12 @@ void ConnectionManager::setPreferredNetwork(network_handle_t network, bool prefe // auto r = spark_set_connection_property(SPARK_CLOUD_BIND_NETWORK_INTERFACE, preferredNetwork, nullptr, nullptr); //LOG(INFO, "%d preferredNetwork %lu setPreferredNetwork %lu", r, preferredNetwork, network); - network_interface_t defaultInterface = NETWORK_INTERFACE_ALL; - LOG(INFO, "setPreferredNetwork network: %lu preferredNetwork_: %lu", network, preferredNetwork_); - preferredNetwork_ = preferred ? network : defaultInterface; + //LOG(INFO, "setPreferredNetwork network: %lu preferredNetwork_: %lu", network, preferredNetwork_); + if (preferred) { + preferredNetwork_ = network; + } else if (network == preferredNetwork_) { + preferredNetwork_ = NETWORK_INTERFACE_ALL; + } } network_handle_t ConnectionManager::getPreferredNetwork() { @@ -66,7 +67,7 @@ network_handle_t ConnectionManager::getPreferredNetwork() { // LOG(INFO, "%d getPreferredNetwork %lu", r, network); // return network; - LOG(INFO, "getPreferredNetwork %lu", preferredNetwork_); + //LOG(INFO, "getPreferredNetwork %lu", preferredNetwork_); return preferredNetwork_; } @@ -77,7 +78,7 @@ network_handle_t ConnectionManager::getCloudConnectionNetwork() { socklen_t len = sizeof(socketNetIfIndex); sock_handle_t cloudSocket = system_cloud_get_socket_handle(); int r = sock_getsockopt(cloudSocket, SOL_SOCKET, SO_BINDTODEVICE, &socketNetIfIndex, &len); - LOG(INFO, "getCloudConnectionNetwork %d : %lu", r, socketNetIfIndex); + LOG(TRACE, "getCloudConnectionNetwork %d : %lu", r, socketNetIfIndex); } return socketNetIfIndex; @@ -91,7 +92,7 @@ network_handle_t ConnectionManager::selectCloudConnectionNetwork() { size_t n = sizeof(boundNetwork); // TODO: error handling auto r = spark_get_connection_property(SPARK_CLOUD_BIND_NETWORK_INTERFACE, &boundNetwork, &n, nullptr); - //LOG(INFO, "%d selectCloudConnectionNetwork %lu", r, boundNetwork); + LOG(TRACE, "%d selectCloudConnectionNetwork %lu", r, boundNetwork); if (boundNetwork != NETWORK_INTERFACE_ALL) { LOG(TRACE, "%d using bound network: %lu", r, boundNetwork); @@ -106,24 +107,13 @@ network_handle_t ConnectionManager::selectCloudConnectionNetwork() { // 3: If no preferred network, use the 'best' network based on criteria // 3.1: Network is ready: ie configured + connected (see ipv4 routable hook) - // 3.2: Network has best criteria based on network tester stats - // TODO: More correct prioritization than just recreating the eth > wifi > cell order - // IE use netif tester - if (spark::Ethernet.ready()) { - return NETWORK_INTERFACE_ETHERNET; - } - -#if HAL_PLATFORM_WIFI - if (spark::WiFi.ready()) { - return NETWORK_INTERFACE_WIFI_STA; + // 3.2: Network has best criteria based on network tester stats (vector should be sorted in "best" order) + for (auto& i: *NetIfTester::instance()->getDiagnostics()) { + if (spark::Network.from(i.interface).ready()) { + LOG(TRACE, "using best tested network: %lu", i.interface); + return i.interface; + } } -#endif - -#if HAL_PLATFORM_CELLULAR - if (spark::Cellular.ready()) { - return NETWORK_INTERFACE_CELLULAR; - } -#endif // TODO: Determine a specific interface to bind to, even in the default case. // ie: How should we handle selecting a cloud connection when no interfaces are up/ready? @@ -131,16 +121,14 @@ network_handle_t ConnectionManager::selectCloudConnectionNetwork() { return bestNetwork; } - - NetIfTester::NetIfTester() { network_interface_t interfaceList[] = { NETWORK_INTERFACE_ETHERNET, -#if HAL_PLATFORM_CELLULAR - NETWORK_INTERFACE_CELLULAR, -#endif #if HAL_PLATFORM_WIFI - NETWORK_INTERFACE_WIFI_STA + NETWORK_INTERFACE_WIFI_STA, +#endif +#if HAL_PLATFORM_CELLULAR + NETWORK_INTERFACE_CELLULAR #endif }; @@ -157,53 +145,116 @@ NetIfTester* NetIfTester::instance() { } void NetIfTester::testInterfaces() { + LOG(INFO, "Connecting to %s#%d ", DEVICE_SERVICE_HOSTNAME, DEVICE_SERVICE_PORT); // GOAL: To maintain a list of which network interface is "best" at any given time for (auto& i: ifDiagnostics_) { testInterface(&i); } + // sort list by packet latency + std::sort(ifDiagnostics_.begin(), ifDiagnostics_.end(), [](const NetIfDiagnostics& dg1, const NetIfDiagnostics& dg2) { + return (dg1.avgPacketRoundTripTime < dg2.avgPacketRoundTripTime); // In ascending order, ie fastest to slowest + }); + + for (auto& i: ifDiagnostics_) { + LOG(TRACE, "If %lu latency %lu", i.interface, i.avgPacketRoundTripTime); + } +} + + // TODO: Make size dynamic / realistic for typical coap payload lengths + const unsigned REACHABILITY_TEST = 252; + const unsigned REACHABILITY_PAYLOAD_SIZE = 32; + + typedef struct { + uint8_t type; + uint8_t version[2]; + uint16_t epoch; + uint8_t sequence_number[6]; + uint16_t length; + uint8_t fragment[REACHABILITY_PAYLOAD_SIZE]; + } __attribute__((__packed__)) DTLSPlaintext_t; + + +static const char* netifToName(uint8_t interfaceNumber) { + switch(interfaceNumber) { + case NETWORK_INTERFACE_ETHERNET: + return "Ethernet"; + case NETWORK_INTERFACE_CELLULAR: + return "Cellular"; + case NETWORK_INTERFACE_WIFI_STA: + return "WiFi "; + default: + return ""; + } } -int NetIfTester::testInterface(NetIfDiagnostics* diagnostics) { +int NetIfTester::testInterface(NetIfDiagnostics* diagnostics) { auto network = spark::Network.from(diagnostics->interface); if (!network) { - return SYSTEM_ERROR_NONE; + return SYSTEM_ERROR_BAD_DATA; } - // TODO: Make this more CoAP like test message - uint8_t udpTxBuffer[128] = {'H', 'e', 'l', 'l', 'o', 0}; + DTLSPlaintext_t reachabilityMessage = { + REACHABILITY_TEST, // DTLS Message Type + {0xfe, 0xfd}, // DTLS type 1.2 + 0x8005, // TODO: Differentiate interfaces by epoch field + {}, // TODO: Sequence number + REACHABILITY_PAYLOAD_SIZE, + {} + }; + memset(reachabilityMessage.fragment, 0xFF, sizeof(reachabilityMessage.fragment)); + + reachabilityMessage.epoch = nativeToBigEndian(reachabilityMessage.epoch); + reachabilityMessage.length = nativeToBigEndian(reachabilityMessage.length); + + uint8_t udpTxBuffer[128] = {}; + size_t udpTxMessageSize = sizeof(reachabilityMessage); + memcpy(udpTxBuffer, &reachabilityMessage, udpTxMessageSize); + uint8_t udpRxBuffer[128] = {}; - uint8_t beginResult = 0; int sendResult, receiveResult = -1; IPAddress echoServer; diagnostics->dnsResolutionAttempts++; - echoServer = network.resolve(UDP_ECHO_SERVER_HOSTNAME); + echoServer = network.resolve(DEVICE_SERVICE_HOSTNAME); if (!echoServer) { + LOG(WARN, "%s failed to resolve DNS", netifToName(diagnostics->interface)); + diagnostics->avgPacketRoundTripTime = 0; diagnostics->dnsResolutionFailures++; - LOG(WARN, "IF #%d failed to resolve DNS for %s : %s", diagnostics->interface, UDP_ECHO_SERVER_HOSTNAME, echoServer.toString().c_str()); - return SYSTEM_ERROR_PROTOCOL; + return SYSTEM_ERROR_NETWORK; } + // TODO: Dont use UDP wiring instance, use sockets directly and use LWIP poll() UDP udpInstance = UDP(); - // TODO: What error conditions are the on UDP socket bind (ie the begin call here)? - beginResult = udpInstance.begin(NetIfTester::UDP_ECHO_PORT, diagnostics->interface); - - LOG(INFO, "Testing IF #%d with %s : %s", diagnostics->interface, UDP_ECHO_SERVER_HOSTNAME, echoServer.toString().c_str()); + udpInstance.begin(NetIfTester::DEVICE_SERVICE_PORT, diagnostics->interface); // TODO: Error conditions on sock_sendto - // TODO: start packet rount trip timer - sendResult = udpInstance.sendPacket(udpTxBuffer, strlen((char*)udpTxBuffer), echoServer, UDP_ECHO_PORT); + sendResult = udpInstance.sendPacket(udpTxBuffer, udpTxMessageSize, echoServer, DEVICE_SERVICE_PORT); + auto startMillis = HAL_Timer_Get_Milli_Seconds(); if (sendResult > 0) { - diagnostics->txBytes += strlen((char*)udpTxBuffer); - } + diagnostics->txBytes += udpTxMessageSize; + } - receiveResult = udpInstance.receivePacket(udpRxBuffer, strlen((char*)udpTxBuffer), 5000); + bool timedOut = true; + receiveResult = udpInstance.receivePacket(udpRxBuffer, udpTxMessageSize, 5000); if (receiveResult > 0) { - diagnostics->rxBytes += strlen((char*)udpRxBuffer); - } + diagnostics->rxBytes += udpTxMessageSize; + timedOut = false; + } + + // TODO: Better metrics for latency rather than single packet timing + // TODO: Validate recevied data, ie sequence number, packet size + contents + auto endMillis = HAL_Timer_Get_Milli_Seconds(); + diagnostics->packetCount++; + diagnostics->avgPacketRoundTripTime = endMillis - startMillis; + + LOG(TRACE, "%s bytes tx: %d rx: %d roundtrip time: %d %s", + netifToName(diagnostics->interface), + diagnostics->txBytes, + diagnostics->rxBytes, + diagnostics->avgPacketRoundTripTime, + timedOut ? "timed out" : ""); - LOG(INFO, "UDP begin %d send: %d receive: %d message: %s", beginResult, sendResult, receiveResult, (char*)udpRxBuffer); udpInstance.stop(); return SYSTEM_ERROR_NONE; diff --git a/system/src/system_connection_manager.h b/system/src/system_connection_manager.h index 7424f51f7a..2c636e0b79 100644 --- a/system/src/system_connection_manager.h +++ b/system/src/system_connection_manager.h @@ -49,11 +49,12 @@ class ConnectionManager { struct NetIfDiagnostics { - network_interface_t interface; // TODO: Change to NetworkClass& ? + network_interface_t interface; uint32_t dnsResolutionAttempts; uint32_t dnsResolutionFailures; uint32_t socketConnAttempts; uint32_t socketConnFailures; + uint32_t packetCount; uint32_t txBytes; uint32_t rxBytes; uint32_t rxTimeouts; @@ -75,6 +76,10 @@ class NetIfTester { const uint16_t UDP_ECHO_PORT = 40000; const char * UDP_ECHO_SERVER_HOSTNAME = "publish-receiver-udp.particle.io"; + + // TODO: Query at runtime + const uint16_t DEVICE_SERVICE_PORT = 5684; + const char * DEVICE_SERVICE_HOSTNAME = "a10aced202194944a0429fc.v5.udp-mesh.staging.particle.io"; Vector ifDiagnostics_; diff --git a/system/src/system_network_manager.cpp b/system/src/system_network_manager.cpp index c9689bc610..1aad086841 100644 --- a/system/src/system_network_manager.cpp +++ b/system/src/system_network_manager.cpp @@ -618,7 +618,7 @@ void NetworkManager::handleIfLink(if_t iface, const struct if_event* ev) { /* Interface link state changed to DOWN */ if (state_ == State::IP_CONFIGURED || state_ == State::IFACE_LINK_UP) { if (countIfacesWithFlags(IFF_UP | IFF_LOWER_UP) == 0) { - transition(State::IFACE_DOWN); + transition(State::IFACE_UP); } else { refreshIpState(); }