[coap-secure] introduce maximum connection attempt limit for CoAP agent

This commit introduces a new feature in `SecureTransport` and
`CoapSecure` that allows us to specify the maximum number of allowed
connection attempts before the socket is automatically closed and the
CoAP agent is stopped. This can be used to enhance security by
preventing attacks and excessive retries. This commit also adds a
callback mechanism to notify when the limit is reached and the CoAP
agent is stopped.

src/core/coap/coap_secure.cpp

@@ -55,13 +55,18 @@ CoapSecure::CoapSecure(Instance &aInstance, bool aLayerTwoSecurity)
 {
 }
 
-Error CoapSecure::Start(uint16_t aPort)
+Error CoapSecure::Start(uint16_t aPort) { return Start(aPort, /* aMaxAttempts */ 0, nullptr, nullptr); }
+
+Error CoapSecure::Start(uint16_t aPort, uint16_t aMaxAttempts, AutoStopCallback aCallback, void *aContext)
 {
-    Error error = kErrorNone;
+    Error error = kErrorAlready;
+
+    SuccessOrExit(mDtls.SetMaxConnectionAttempts(aMaxAttempts, HandleDtlsAutoClose, this));
+    mAutoStopCallback.Set(aCallback, aContext);
 
     mConnectedCallback.Clear();
 
-    SuccessOrExit(error = mDtls.Open(&CoapSecure::HandleDtlsReceive, &CoapSecure::HandleDtlsConnected, this));
+    SuccessOrExit(error = mDtls.Open(HandleDtlsReceive, HandleDtlsConnected, this));
     SuccessOrExit(error = mDtls.Bind(aPort));
 
 exit:
@@ -70,12 +75,13 @@ Error CoapSecure::Start(uint16_t aPort)
 
 Error CoapSecure::Start(MeshCoP::SecureTransport::TransportCallback aCallback, void *aContext)
 {
-    Error error = kErrorNone;
+    Error error = kErrorAlready;
 
     mConnectedCallback.Clear();
-
-    SuccessOrExit(error = mDtls.Open(&CoapSecure::HandleDtlsReceive, &CoapSecure::HandleDtlsConnected, this));
-    SuccessOrExit(error = mDtls.Bind(aCallback, aContext));
+    SuccessOrExit(mDtls.SetMaxConnectionAttempts(0, nullptr, nullptr));
+    SuccessOrExit(mDtls.Open(HandleDtlsReceive, HandleDtlsConnected, this));
+    SuccessOrExit(mDtls.Bind(aCallback, aContext));
+    error = kErrorNone;
 
 exit:
     return error;
@@ -172,6 +178,17 @@ void CoapSecure::HandleDtlsConnected(void *aContext, bool aConnected)
 
 void CoapSecure::HandleDtlsConnected(bool aConnected) { mConnectedCallback.InvokeIfSet(aConnected); }
 
+void CoapSecure::HandleDtlsAutoClose(void *aContext)
+{
+    return static_cast<CoapSecure *>(aContext)->HandleDtlsAutoClose();
+}
+
+void CoapSecure::HandleDtlsAutoClose(void)
+{
+    Stop();
+    mAutoStopCallback.InvokeIfSet();
+}
+
 void CoapSecure::HandleDtlsReceive(void *aContext, uint8_t *aBuf, uint16_t aLength)
 {
     return static_cast<CoapSecure *>(aContext)->HandleDtlsReceive(aBuf, aLength);

src/core/coap/coap_secure.hpp

@@ -61,6 +61,15 @@ class CoapSecure : public CoapBase
      */
     typedef void (*ConnectedCallback)(bool aConnected, void *aContext);
 
+    /**
+     * Callback to notify when the agent is automatically stopped due to reaching the maximum number of connection
+     * attempts.
+     *
+     * @param[in] aContext    A pointer to arbitrary context information.
+     *
+     */
+    typedef void (*AutoStopCallback)(void *aContext);
+
     /**
      * Initializes the object.
      *
@@ -81,6 +90,21 @@ class CoapSecure : public CoapBase
      */
     Error Start(uint16_t aPort);
 
+    /**
+     * Starts the secure CoAP agent and sets the maximum number of allowed connection attempts before stopping the
+     * agent automatically.
+     *
+     * @param[in] aPort           The local UDP port to bind to.
+     * @param[in] aMaxAttempts    Maximum number of allowed connection request attempts. Zero indicates no limit.
+     * @param[in] aCallback       Callback to notify if max number of attempts has reached and agent is stopped.
+     * @param[in] aContext        A pointer to arbitrary context to use with `AutoStopCallback`.
+     *
+     * @retval kErrorNone        Successfully started the CoAP agent.
+     * @retval kErrorAlready     Already started.
+     *
+     */
+    Error Start(uint16_t aPort, uint16_t aMaxAttempts, AutoStopCallback aCallback, void *aContext);
+
     /**
      * Starts the secure CoAP agent, but do not use socket to transmit/receive messages.
      *
@@ -391,6 +415,9 @@ class CoapSecure : public CoapBase
     static void HandleDtlsConnected(void *aContext, bool aConnected);
     void        HandleDtlsConnected(bool aConnected);
 
+    static void HandleDtlsAutoClose(void *aContext);
+    void        HandleDtlsAutoClose(void);
+
     static void HandleDtlsReceive(void *aContext, uint8_t *aBuf, uint16_t aLength);
     void        HandleDtlsReceive(uint8_t *aBuf, uint16_t aLength);
 
@@ -399,6 +426,7 @@ class CoapSecure : public CoapBase
 
     MeshCoP::SecureTransport    mDtls;
     Callback<ConnectedCallback> mConnectedCallback;
+    Callback<AutoStopCallback>  mAutoStopCallback;
     ot::MessageQueue            mTransmitQueue;
     TaskletContext              mTransmitTask;
 };

src/core/meshcop/secure_transport.cpp

@@ -83,6 +83,8 @@ SecureTransport::SecureTransport(Instance &aInstance, bool aLayerTwoSecurity, bo
     , mTimerSet(false)
     , mLayerTwoSecurity(aLayerTwoSecurity)
     , mDatagramTransport(aDatagramTransport)
+    , mMaxConnectionAttempts(0)
+    , mRemainingConnectionAttempts(0)
     , mReceiveMessage(nullptr)
     , mSocket(aInstance)
     , mMessageSubType(Message::kSubTypeNone)
@@ -159,18 +161,39 @@ Error SecureTransport::Open(ReceiveHandler aReceiveHandler, ConnectedHandler aCo
 
     mConnectedCallback.Set(aConnectedHandler, aContext);
     mReceiveCallback.Set(aReceiveHandler, aContext);
+
+    mRemainingConnectionAttempts = mMaxConnectionAttempts;
+
     SetState(kStateOpen);
 
 exit:
     return error;
 }
 
+Error SecureTransport::SetMaxConnectionAttempts(uint16_t aMaxAttempts, AutoCloseCallback aCallback, void *aContext)
+{
+    Error error = kErrorNone;
+
+    VerifyOrExit(IsStateClosed(), error = kErrorInvalidState);
+
+    mMaxConnectionAttempts = aMaxAttempts;
+    mAutoCloseCallback.Set(aCallback, aContext);
+
+exit:
+    return error;
+}
+
 Error SecureTransport::Connect(const Ip6::SockAddr &aSockAddr)
 {
     Error error;
 
     VerifyOrExit(IsStateOpen(), error = kErrorInvalidState);
 
+    if (mRemainingConnectionAttempts > 0)
+    {
+        mRemainingConnectionAttempts--;
+    }
+
     mMessageInfo.SetPeerAddr(aSockAddr.GetAddress());
     mMessageInfo.SetPeerPort(aSockAddr.mPort);
 
@@ -191,6 +214,11 @@ void SecureTransport::HandleReceive(Message &aMessage, const Ip6::MessageInfo &a
 
     if (IsStateOpen())
     {
+        if (mRemainingConnectionAttempts > 0)
+        {
+            mRemainingConnectionAttempts--;
+        }
+
         IgnoreError(mSocket.Connect(Ip6::SockAddr(aMessageInfo.GetPeerAddr(), aMessageInfo.GetPeerPort())));
 
         mMessageInfo.SetPeerAddr(aMessageInfo.GetPeerAddr());
@@ -388,8 +416,16 @@ Error SecureTransport::Setup(bool aClient)
 exit:
     if (IsStateInitializing() && (rval != 0))
     {
-        SetState(kStateOpen);
-        FreeMbedtls();
+        if ((mMaxConnectionAttempts > 0) && (mRemainingConnectionAttempts == 0))
+        {
+            Close();
+            mAutoCloseCallback.InvokeIfSet();
+        }
+        else
+        {
+            SetState(kStateOpen);
+            FreeMbedtls();
+        }
     }
 
     return Crypto::MbedTls::MapError(rval);

src/core/meshcop/secure_transport.hpp

@@ -128,6 +128,15 @@ class SecureTransport : public InstanceLocator
      */
     typedef Error (*TransportCallback)(void *aContext, ot::Message &aMessage, const Ip6::MessageInfo &aMessageInfo);
 
+    /**
+     * Callback to notify when the socket is automatically closed due to reaching the maximum number of connection
+     * attempts (set from `SetMaxConnectionAttempts()`).
+     *
+     * @param[in] aContext    A pointer to arbitrary context information.
+     *
+     */
+    typedef void (*AutoCloseCallback)(void *aContext);
+
     /**
      * Opens the socket.
      *
@@ -141,6 +150,24 @@ class SecureTransport : public InstanceLocator
      */
     Error Open(ReceiveHandler aReceiveHandler, ConnectedHandler aConnectedHandler, void *aContext);
 
+    /**
+     * Sets the maximum number of allowed connection requests before socket is automatically closed.
+     *
+     * This method can be called when socket is closed. Otherwise `kErrorInvalidSatet` is returned.
+     *
+     * If @p aMaxAttempts is zero, no limit is applied and connections are allowed until the socket is closed. This is
+     * the default behavior if `SetMaxConnectionAttempts()` is not called.
+     *
+     * @param[in] aMaxAttempts    Maximum number of allowed connection attempts.
+     * @param[in] aCallback       Callback to notify when max number of attempts has reached and socket is closed.
+     * @param[in] aContext        A pointer to arbitrary context to use with `AutoCloseCallback`.
+     *
+     * @retval kErrorNone          Successfully set the maximum allowed connection attempts and callback.
+     * @retval kErrorInvalidState  Socket is not closed.
+     *
+     */
+    Error SetMaxConnectionAttempts(uint16_t aMaxAttempts, AutoCloseCallback aCallback, void *aContext);
+
     /**
      * Binds this DTLS to a UDP port.
      *
@@ -616,6 +643,10 @@ class SecureTransport : public InstanceLocator
     bool mLayerTwoSecurity : 1;
     bool mDatagramTransport : 1;
 
+    uint16_t                    mMaxConnectionAttempts;
+    uint16_t                    mRemainingConnectionAttempts;
+    Callback<AutoCloseCallback> mAutoCloseCallback;
+
     Message *mReceiveMessage;
 
     Callback<ConnectedHandler> mConnectedCallback;