diff --git a/examples/echoserver/echoserver.c b/examples/echoserver/echoserver.c
index 1b0aba8d27..338780bc9f 100644
--- a/examples/echoserver/echoserver.c
+++ b/examples/echoserver/echoserver.c
@@ -92,6 +92,11 @@ typedef struct {
 } thread_ctx_t;
 
 
+#ifndef DEFAULT_HIGHWATER_MARK
+    #define DEFAULT_HIGHWATER_MARK 0
+#endif
+
+
 #ifdef __GNUC__
     #define WS_NORETURN __attribute__((noreturn))
 #else
@@ -587,11 +592,29 @@ static int wsUserAuth(uint8_t authType,
 }
 
 
+static int wsHighwater(uint8_t side, void* ctx)
+{
+    if (ctx) {
+        WOLFSSH* ssh = (WOLFSSH*)ctx;
+        uint32_t highwaterMark = wolfSSH_GetHighwater(ssh);
+
+        printf("HIGHWATER ALERT: (%u) %s\n", highwaterMark,
+                (side == WOLFSSH_HWSIDE_RECEIVE) ? "receive" : "transmit");
+        highwaterMark *= 2;
+        printf("  Doubling the highwater mark to %u.\n", highwaterMark);
+        wolfSSH_SetHighwater(ssh, highwaterMark);
+    }
+
+    return 0;
+}
+
+
 int main(void)
 {
     WOLFSSH_CTX* ctx = NULL;
     PwMapList pwMapList;
     SOCKET_T listenFd = 0;
+    uint32_t defaultHighwater = DEFAULT_HIGHWATER_MARK;
 
     #ifdef DEBUG_WOLFSSH
         wolfSSH_Debugging_ON();
@@ -610,6 +633,8 @@ int main(void)
 
     memset(&pwMapList, 0, sizeof(pwMapList));
     wolfSSH_SetUserAuth(ctx, wsUserAuth);
+    if (defaultHighwater > 0)
+        wolfSSH_SetHighwaterCb(ctx, defaultHighwater, wsHighwater);
 
     {
         uint8_t buf[SCRATCH_BUFFER_SIZE];
@@ -663,6 +688,9 @@ int main(void)
             exit(EXIT_FAILURE);
         }
         wolfSSH_SetUserAuthCtx(ssh, &pwMapList);
+        /* Use the session object for its own highwater callback ctx */
+        if (defaultHighwater > 0)
+            wolfSSH_SetHighwaterCtx(ssh, (void*)ssh);
 
         if (listen(listenFd, 5) != 0)
             err_sys("tcp listen failed");
diff --git a/src/internal.c b/src/internal.c
index 896ada113f..8bb47cf2d1 100644
--- a/src/internal.c
+++ b/src/internal.c
@@ -146,6 +146,139 @@ const char* GetErrorString(int err)
 }
 
 
+WOLFSSH_CTX* CtxInit(WOLFSSH_CTX* ctx, void* heap)
+{
+    WLOG(WS_LOG_DEBUG, "Entering CtxInit()");
+
+    if (ctx == NULL)
+        return ctx;
+
+    WMEMSET(ctx, 0, sizeof(WOLFSSH_CTX));
+
+    if (heap)
+        ctx->heap = heap;
+
+#ifndef WOLFSSH_USER_IO
+    ctx->ioRecvCb = wsEmbedRecv;
+    ctx->ioSendCb = wsEmbedSend;
+#endif /* WOLFSSH_USER_IO */
+    ctx->countHighwater = DEFAULT_COUNT_HIGHWATER;
+
+    return ctx;
+}
+
+
+void CtxResourceFree(WOLFSSH_CTX* ctx)
+{
+    WLOG(WS_LOG_DEBUG, "Entering CtxResourceFree()");
+
+    if (ctx->privateKey) {
+        ForceZero(ctx->privateKey, ctx->privateKeySz);
+        WFREE(ctx->privateKey, ctx->heap, DYNTYPE_KEY);
+    }
+    WFREE(ctx->cert, ctx->heap, DYNTYPE_CERT);
+    WFREE(ctx->caCert, ctx->heap, DYNTYPE_CA);
+}
+
+
+WOLFSSH* SshInit(WOLFSSH* ssh, WOLFSSH_CTX* ctx)
+{
+    HandshakeInfo* handshake = NULL;
+    RNG*           rng = NULL;
+    void*          heap;
+
+    WLOG(WS_LOG_DEBUG, "Entering SshInit()");
+
+    if (ssh == NULL || ctx == NULL)
+        return ssh;
+    heap = ctx->heap;
+
+    handshake = (HandshakeInfo*)WMALLOC(sizeof(HandshakeInfo),
+                                        heap, DYNTYPE_HS);
+    rng = (RNG*)WMALLOC(sizeof(RNG), heap, DYNTYPE_RNG);
+
+    if (handshake == NULL || rng == NULL || wc_InitRng(rng) != 0) {
+
+        WLOG(WS_LOG_DEBUG, "SshInit: Cannot allocate memory.\n");
+        WFREE(handshake, heap, DYNTYPE_HS);
+        WFREE(rng, heap, DYNTYPE_RNG);
+        wolfSSH_free(ssh);
+        return NULL;
+    }
+
+    WMEMSET(ssh, 0, sizeof(WOLFSSH));  /* default init to zeros */
+    WMEMSET(handshake, 0, sizeof(HandshakeInfo));
+
+    ssh->ctx         = ctx;
+    ssh->error       = WS_SUCCESS;
+    ssh->rfd         = -1;         /* set to invalid */
+    ssh->wfd         = -1;         /* set to invalid */
+    ssh->ioReadCtx   = &ssh->rfd;  /* prevent invalid access if not correctly */
+    ssh->ioWriteCtx  = &ssh->wfd;  /* set */
+    ssh->countHighwater = ctx->countHighwater;
+    ssh->acceptState = ACCEPT_BEGIN;
+    ssh->clientState = CLIENT_BEGIN;
+    ssh->nextChannel = DEFAULT_NEXT_CHANNEL;
+    ssh->blockSz     = MIN_BLOCK_SZ;
+    ssh->encryptId   = ID_NONE;
+    ssh->macId       = ID_NONE;
+    ssh->peerBlockSz = MIN_BLOCK_SZ;
+    ssh->rng         = rng;
+    ssh->kSz         = sizeof(ssh->k);
+    ssh->handshake   = handshake;
+    handshake->kexId = ID_NONE;
+    handshake->pubKeyId  = ID_NONE;
+    handshake->encryptId = ID_NONE;
+    handshake->macId = ID_NONE;
+    handshake->blockSz = MIN_BLOCK_SZ;
+
+    if (BufferInit(&ssh->inputBuffer, 0, ctx->heap) != WS_SUCCESS ||
+        BufferInit(&ssh->outputBuffer, 0, ctx->heap) != WS_SUCCESS ||
+        wc_InitSha(&ssh->handshake->hash) != 0) {
+
+        wolfSSH_free(ssh);
+        ssh = NULL;
+    }
+
+    return ssh;
+}
+
+
+void SshResourceFree(WOLFSSH* ssh, void* heap)
+{
+    /* when ssh holds resources, free here */
+    (void)heap;
+
+    WLOG(WS_LOG_DEBUG, "Entering sshResourceFree()");
+
+    ShrinkBuffer(&ssh->inputBuffer, 1);
+    ShrinkBuffer(&ssh->outputBuffer, 1);
+    ForceZero(ssh->k, ssh->kSz);
+    if (ssh->handshake) {
+        ForceZero(ssh->handshake, sizeof(HandshakeInfo));
+        WFREE(ssh->handshake, heap, DYNTYPE_HS);
+    }
+    ForceZero(&ssh->clientKeys, sizeof(Keys));
+    ForceZero(&ssh->serverKeys, sizeof(Keys));
+    if (ssh->rng) {
+        wc_FreeRng(ssh->rng);
+        WFREE(ssh->rng, heap, DYNTYPE_RNG);
+    }
+    if (ssh->userName) {
+        WFREE(ssh->userName, heap, DYNTYPE_STRING);
+    }
+    if (ssh->channelList) {
+        WOLFSSH_CHANNEL* cur = ssh->channelList;
+        WOLFSSH_CHANNEL* next;
+        while (cur) {
+            next = cur->next;
+            ChannelDelete(cur, heap);
+            cur = next;
+        }
+    }
+}
+
+
 typedef struct {
     uint8_t id;
     const char* name;
@@ -954,9 +1087,13 @@ static int DoKexInit(WOLFSSH* ssh, uint8_t* buf, uint32_t len, uint32_t* idx)
         }
         else {
             ssh->handshake->encryptId = algoId;
-            ssh->handshake->blockSz = ssh->ivClientSz = ssh->ivServerSz
-                = BlockSzForId(algoId);
-            ssh->encKeyClientSz = ssh->encKeyServerSz = KeySzForId(algoId);
+            ssh->handshake->blockSz =
+                ssh->handshake->clientKeys.ivSz =
+                ssh->handshake->serverKeys.ivSz =
+                BlockSzForId(algoId);
+            ssh->handshake->clientKeys.encKeySz =
+                ssh->handshake->serverKeys.encKeySz =
+                KeySzForId(algoId);
         }
     }
 
@@ -987,7 +1124,9 @@ static int DoKexInit(WOLFSSH* ssh, uint8_t* buf, uint32_t len, uint32_t* idx)
             else {
                 ssh->handshake->macId = algoId;
                 ssh->handshake->macSz = MacSzForId(algoId);
-                ssh->macKeyClientSz = ssh->macKeyServerSz = KeySzForId(algoId);
+                ssh->handshake->clientKeys.macKeySz =
+                    ssh->handshake->serverKeys.macKeySz =
+                    KeySzForId(algoId);
             }
         }
     }
@@ -1176,6 +1315,7 @@ static int DoNewKeys(WOLFSSH* ssh, uint8_t* buf, uint32_t len, uint32_t* idx)
         ssh->peerMacId = ssh->handshake->macId;
         ssh->peerBlockSz = ssh->handshake->blockSz;
         ssh->peerMacSz = ssh->handshake->macSz;
+        WMEMCPY(&ssh->clientKeys, &ssh->handshake->clientKeys, sizeof(Keys));
 
         switch (ssh->peerEncryptId) {
             case ID_NONE:
@@ -1185,8 +1325,9 @@ static int DoNewKeys(WOLFSSH* ssh, uint8_t* buf, uint32_t len, uint32_t* idx)
             case ID_AES128_CBC:
                 WLOG(WS_LOG_DEBUG, "DNK: peer using cipher aes128-cbc");
                 ret = wc_AesSetKey(&ssh->decryptCipher.aes,
-                                   ssh->encKeyClient, ssh->encKeyClientSz,
-                                   ssh->ivClient, AES_DECRYPTION);
+                                   ssh->clientKeys.encKey,
+                                   ssh->clientKeys.encKeySz,
+                                   ssh->clientKeys.iv, AES_DECRYPTION);
                 break;
 
             default:
@@ -1289,31 +1430,37 @@ int GenerateKey(uint8_t hashId, uint8_t keyId,
 
 static int GenerateKeys(WOLFSSH* ssh)
 {
+    Keys* cK;
+    Keys* sK;
+
     if (ssh == NULL)
         return WS_BAD_ARGUMENT;
 
+    cK = &ssh->handshake->clientKeys;
+    sK = &ssh->handshake->serverKeys;
+
     GenerateKey(0, 'A',
-                ssh->ivClient, ssh->ivClientSz,
+                cK->iv, cK->ivSz,
                 ssh->k, ssh->kSz, ssh->h, ssh->hSz,
                 ssh->sessionId, ssh->sessionIdSz);
     GenerateKey(0, 'B',
-                ssh->ivServer, ssh->ivServerSz,
+                sK->iv, sK->ivSz,
                 ssh->k, ssh->kSz, ssh->h, ssh->hSz,
                 ssh->sessionId, ssh->sessionIdSz);
     GenerateKey(0, 'C',
-                ssh->encKeyClient, ssh->encKeyClientSz,
+                cK->encKey, cK->encKeySz,
                 ssh->k, ssh->kSz, ssh->h, ssh->hSz,
                 ssh->sessionId, ssh->sessionIdSz);
     GenerateKey(0, 'D',
-                ssh->encKeyServer, ssh->encKeyServerSz,
+                sK->encKey, sK->encKeySz,
                 ssh->k, ssh->kSz, ssh->h, ssh->hSz,
                 ssh->sessionId, ssh->sessionIdSz);
     GenerateKey(0, 'E',
-                ssh->macKeyClient, ssh->macKeyClientSz,
+                cK->macKey, cK->macKeySz,
                 ssh->k, ssh->kSz, ssh->h, ssh->hSz,
                 ssh->sessionId, ssh->sessionIdSz);
     GenerateKey(0, 'F',
-                ssh->macKeyServer, ssh->macKeyServerSz,
+                sK->macKey, sK->macKeySz,
                 ssh->k, ssh->kSz, ssh->h, ssh->hSz,
                 ssh->sessionId, ssh->sessionIdSz);
 
@@ -1325,17 +1472,17 @@ static int GenerateKeys(WOLFSSH* ssh)
     printf("Session ID:\n");
     DumpOctetString(ssh->sessionId, ssh->sessionIdSz);
     printf("A:\n");
-    DumpOctetString(ssh->ivClient, ssh->ivClientSz);
+    DumpOctetString(cK->iv, cK->ivSz);
     printf("B:\n");
-    DumpOctetString(ssh->ivServer, ssh->ivServerSz);
+    DumpOctetString(sK->iv, sK->ivSz);
     printf("C:\n");
-    DumpOctetString(ssh->encKeyClient, ssh->encKeyClientSz);
+    DumpOctetString(cK->encKey, cK->encKeySz);
     printf("D:\n");
-    DumpOctetString(ssh->encKeyServer, ssh->encKeyServerSz);
+    DumpOctetString(sK->encKey, sK->encKeySz);
     printf("E:\n");
-    DumpOctetString(ssh->macKeyClient, ssh->macKeyClientSz);
+    DumpOctetString(cK->macKey, cK->macKeySz);
     printf("F:\n");
-    DumpOctetString(ssh->macKeyServer, ssh->macKeyServerSz);
+    DumpOctetString(sK->macKey, sK->macKeySz);
     printf("\n");
 #endif /* SHOW_SECRETS */
 
@@ -2258,6 +2405,11 @@ static INLINE int Encrypt(WOLFSSH* ssh, uint8_t* cipher, const uint8_t* input,
     }
 
     ssh->txCount += sz;
+    if (ssh->countHighwater && ssh->txCount > ssh->countHighwater) {
+        WLOG(WS_LOG_DEBUG, "Transmit over high water mark");
+        if (ssh->ctx->highwaterCb)
+            ssh->ctx->highwaterCb(WOLFSSH_HWSIDE_TRANSMIT, ssh->highwaterCtx);
+    }
 
     return ret;
 }
@@ -2290,6 +2442,11 @@ static INLINE int Decrypt(WOLFSSH* ssh, uint8_t* plain, const uint8_t* input,
     }
 
     ssh->rxCount += sz;
+    if (ssh->countHighwater && ssh->rxCount > ssh->countHighwater) {
+        WLOG(WS_LOG_DEBUG, "Receive over high water mark");
+        if (ssh->ctx->highwaterCb)
+            ssh->ctx->highwaterCb(WOLFSSH_HWSIDE_RECEIVE, ssh->highwaterCtx);
+    }
 
     return ret;
 }
@@ -2316,7 +2473,8 @@ static INLINE int CreateMac(WOLFSSH* ssh, const uint8_t* in, uint32_t inSz,
                 uint8_t digest[SHA_DIGEST_SIZE];
 
                 wc_HmacSetKey(&hmac, SHA,
-                              ssh->macKeyServer, ssh->macKeyServerSz);
+                              ssh->serverKeys.macKey,
+                              ssh->serverKeys.macKeySz);
                 wc_HmacUpdate(&hmac, flatSeq, sizeof(flatSeq));
                 wc_HmacUpdate(&hmac, in, inSz);
                 wc_HmacFinal(&hmac, digest);
@@ -2329,7 +2487,8 @@ static INLINE int CreateMac(WOLFSSH* ssh, const uint8_t* in, uint32_t inSz,
                 Hmac hmac;
 
                 wc_HmacSetKey(&hmac, SHA,
-                              ssh->macKeyServer, ssh->macKeyServerSz);
+                              ssh->serverKeys.macKey,
+                              ssh->serverKeys.macKeySz);
                 wc_HmacUpdate(&hmac, flatSeq, sizeof(flatSeq));
                 wc_HmacUpdate(&hmac, in, inSz);
                 wc_HmacFinal(&hmac, mac);
@@ -2341,7 +2500,8 @@ static INLINE int CreateMac(WOLFSSH* ssh, const uint8_t* in, uint32_t inSz,
                 Hmac hmac;
 
                 wc_HmacSetKey(&hmac, SHA256,
-                              ssh->macKeyServer, ssh->macKeyServerSz);
+                              ssh->serverKeys.macKey,
+                              ssh->serverKeys.macKeySz);
                 wc_HmacUpdate(&hmac, flatSeq, sizeof(flatSeq));
                 wc_HmacUpdate(&hmac, in, inSz);
                 wc_HmacFinal(&hmac, mac);
@@ -2370,7 +2530,7 @@ static INLINE int VerifyMac(WOLFSSH* ssh, const uint8_t* in, uint32_t inSz,
     WLOG(WS_LOG_DEBUG, "VerifyMac %s", IdToName(ssh->peerMacId));
     WLOG(WS_LOG_DEBUG, "VM: inSz = %u", inSz);
     WLOG(WS_LOG_DEBUG, "VM: seq = %u", ssh->peerSeq);
-    WLOG(WS_LOG_DEBUG, "VM: keyLen = %u", ssh->macKeyClientSz);
+    WLOG(WS_LOG_DEBUG, "VM: keyLen = %u", ssh->clientKeys.macKeySz);
 
     switch (ssh->peerMacId) {
         case ID_NONE:
@@ -2378,7 +2538,8 @@ static INLINE int VerifyMac(WOLFSSH* ssh, const uint8_t* in, uint32_t inSz,
 
         case ID_HMAC_SHA1:
         case ID_HMAC_SHA1_96:
-            wc_HmacSetKey(&hmac, SHA, ssh->macKeyClient, ssh->macKeyClientSz);
+            wc_HmacSetKey(&hmac, SHA,
+                          ssh->clientKeys.macKey, ssh->clientKeys.macKeySz);
             wc_HmacUpdate(&hmac, flatSeq, sizeof(flatSeq));
             wc_HmacUpdate(&hmac, in, inSz);
             wc_HmacFinal(&hmac, checkMac);
@@ -2388,7 +2549,7 @@ static INLINE int VerifyMac(WOLFSSH* ssh, const uint8_t* in, uint32_t inSz,
 
         case ID_HMAC_SHA2_256:
             wc_HmacSetKey(&hmac, SHA256,
-                          ssh->macKeyClient, ssh->macKeyClientSz);
+                          ssh->clientKeys.macKey, ssh->clientKeys.macKeySz);
             wc_HmacUpdate(&hmac, flatSeq, sizeof(flatSeq));
             wc_HmacUpdate(&hmac, in, inSz);
             wc_HmacFinal(&hmac, checkMac);
@@ -2996,6 +3157,7 @@ int SendNewKeys(WOLFSSH* ssh)
         ssh->encryptId = ssh->handshake->encryptId;
         ssh->macSz = ssh->handshake->macSz;
         ssh->macId = ssh->handshake->macId;
+        WMEMCPY(&ssh->serverKeys, &ssh->handshake->serverKeys, sizeof(Keys));
 
         switch (ssh->encryptId) {
             case ID_NONE:
@@ -3005,8 +3167,9 @@ int SendNewKeys(WOLFSSH* ssh)
             case ID_AES128_CBC:
                 WLOG(WS_LOG_DEBUG, "SNK: using cipher aes128-cbc");
                 ret = wc_AesSetKey(&ssh->encryptCipher.aes,
-                                  ssh->encKeyServer, ssh->encKeyServerSz,
-                                  ssh->ivServer, AES_ENCRYPTION);
+                                  ssh->serverKeys.encKey,
+                                  ssh->serverKeys.encKeySz,
+                                  ssh->serverKeys.iv, AES_ENCRYPTION);
                 break;
 
             default:
diff --git a/src/ssh.c b/src/ssh.c
index 6771f5522b..cbfff9eb1d 100644
--- a/src/ssh.c
+++ b/src/ssh.c
@@ -64,27 +64,6 @@ int wolfSSH_Cleanup(void)
 }
 
 
-static WOLFSSH_CTX* CtxInit(WOLFSSH_CTX* ctx, void* heap)
-{
-    WLOG(WS_LOG_DEBUG, "Entering CtxInit()");
-
-    if (ctx == NULL)
-        return ctx;
-
-    WMEMSET(ctx, 0, sizeof(WOLFSSH_CTX));
-
-    if (heap)
-        ctx->heap = heap;
-
-#ifndef WOLFSSH_USER_IO
-    ctx->ioRecvCb = wsEmbedRecv;
-    ctx->ioSendCb = wsEmbedSend;
-#endif /* WOLFSSH_USER_IO */
-
-    return ctx;
-}
-
-
 WOLFSSH_CTX* wolfSSH_CTX_new(uint8_t side, void* heap)
 {
     WOLFSSH_CTX* ctx;
@@ -105,19 +84,6 @@ WOLFSSH_CTX* wolfSSH_CTX_new(uint8_t side, void* heap)
 }
 
 
-static void CtxResourceFree(WOLFSSH_CTX* ctx)
-{
-    WLOG(WS_LOG_DEBUG, "Entering CtxResourceFree()");
-
-    if (ctx->privateKey) {
-        ForceZero(ctx->privateKey, ctx->privateKeySz);
-        WFREE(ctx->privateKey, ctx->heap, DYNTYPE_KEY);
-    }
-    WFREE(ctx->cert, ctx->heap, DYNTYPE_CERT);
-    WFREE(ctx->caCert, ctx->heap, DYNTYPE_CA);
-}
-
-
 void wolfSSH_CTX_free(WOLFSSH_CTX* ctx)
 {
     WLOG(WS_LOG_DEBUG, "Entering wolfSSH_CTX_free()");
@@ -129,66 +95,6 @@ void wolfSSH_CTX_free(WOLFSSH_CTX* ctx)
 }
 
 
-static WOLFSSH* SshInit(WOLFSSH* ssh, WOLFSSH_CTX* ctx)
-{
-    HandshakeInfo* handshake;
-    RNG*           rng;
-
-    WLOG(WS_LOG_DEBUG, "Entering SshInit()");
-
-    if (ssh == NULL)
-        return ssh;
-
-    handshake = (HandshakeInfo*)WMALLOC(sizeof(HandshakeInfo),
-                                        ctx->heap, DYNTYPE_HS);
-    if (handshake == NULL) {
-        wolfSSH_free(ssh);
-        return NULL;
-    }
-
-    rng = (RNG*)WMALLOC(sizeof(RNG), ctx->heap, DYNTYPE_RNG);
-    if (rng == NULL || wc_InitRng(rng) != 0) {
-        wolfSSH_free(ssh);
-        return NULL;
-    }
-
-    WMEMSET(ssh, 0, sizeof(WOLFSSH));  /* default init to zeros */
-    WMEMSET(handshake, 0, sizeof(HandshakeInfo));
-
-    ssh->ctx         = ctx;
-    ssh->error       = WS_SUCCESS;
-    ssh->rfd         = -1;         /* set to invalid */
-    ssh->wfd         = -1;         /* set to invalid */
-    ssh->ioReadCtx   = &ssh->rfd;  /* prevent invalid access if not correctly */
-    ssh->ioWriteCtx  = &ssh->wfd;  /* set */
-    ssh->acceptState = ACCEPT_BEGIN;
-    ssh->clientState = CLIENT_BEGIN;
-    ssh->nextChannel = DEFAULT_NEXT_CHANNEL;
-    ssh->blockSz     = MIN_BLOCK_SZ;
-    ssh->encryptId   = ID_NONE;
-    ssh->macId       = ID_NONE;
-    ssh->peerBlockSz = MIN_BLOCK_SZ;
-    ssh->rng         = rng;
-    ssh->kSz         = sizeof(ssh->k);
-    ssh->handshake   = handshake;
-    handshake->kexId = ID_NONE;
-    handshake->pubKeyId  = ID_NONE;
-    handshake->encryptId = ID_NONE;
-    handshake->macId = ID_NONE;
-    handshake->blockSz = MIN_BLOCK_SZ;
-
-    if (BufferInit(&ssh->inputBuffer, 0, ctx->heap) != WS_SUCCESS ||
-        BufferInit(&ssh->outputBuffer, 0, ctx->heap) != WS_SUCCESS ||
-        wc_InitSha(&ssh->handshake->hash) != 0) {
-
-        wolfSSH_free(ssh);
-        ssh = NULL;
-    }
-
-    return ssh;
-}
-
-
 WOLFSSH* wolfSSH_new(WOLFSSH_CTX* ctx)
 {
     WOLFSSH* ssh;
@@ -214,46 +120,12 @@ WOLFSSH* wolfSSH_new(WOLFSSH_CTX* ctx)
 }
 
 
-static void SshResourceFree(WOLFSSH* ssh, void* heap)
-{
-    /* when ssh holds resources, free here */
-    (void)heap;
-
-    WLOG(WS_LOG_DEBUG, "Entering sshResourceFree()");
-
-    ShrinkBuffer(&ssh->inputBuffer, 1);
-    ShrinkBuffer(&ssh->outputBuffer, 1);
-    ForceZero(ssh->k, ssh->kSz);
-    if (ssh->handshake) {
-        WMEMSET(ssh->handshake, 0, sizeof(HandshakeInfo));
-        WFREE(ssh->handshake, heap, DYNTYPE_HS);
-    }
-    if (ssh->rng) {
-        wc_FreeRng(ssh->rng);
-        WFREE(ssh->rng, heap, DYNTYPE_RNG);
-    }
-    if (ssh->userName) {
-        WFREE(ssh->userName, heap, DYNTYPE_STRING);
-    }
-    if (ssh->channelList) {
-        WOLFSSH_CHANNEL* cur = ssh->channelList;
-        WOLFSSH_CHANNEL* next;
-        while (cur) {
-            next = cur->next;
-            ChannelDelete(cur, heap);
-            cur = next;
-        }
-    }
-}
-
-
 void wolfSSH_free(WOLFSSH* ssh)
 {
-    void* heap = ssh->ctx ? ssh->ctx->heap : NULL;
-
     WLOG(WS_LOG_DEBUG, "Entering wolfSSH_free()");
 
     if (ssh) {
+        void* heap = ssh->ctx ? ssh->ctx->heap : NULL;
         SshResourceFree(ssh, heap);
         WFREE(ssh, heap, DYNTYPE_SSH);
     }
@@ -288,6 +160,63 @@ int wolfSSH_get_fd(const WOLFSSH* ssh)
 }
 
 
+int wolfSSH_SetHighwater(WOLFSSH* ssh, uint32_t highwater)
+{
+    WLOG(WS_LOG_DEBUG, "Entering wolfSSH_SetHighwater()");
+
+    if (ssh) {
+        ssh->countHighwater = highwater;
+
+        return WS_SUCCESS;
+    }
+
+    return WS_BAD_ARGUMENT;
+}
+
+
+uint32_t wolfSSH_GetHighwater(WOLFSSH* ssh)
+{
+    WLOG(WS_LOG_DEBUG, "Entering wolfSSH_GetHighwater()");
+
+    if (ssh)
+        return ssh->countHighwater;
+
+    return 0;
+}
+
+
+void wolfSSH_SetHighwaterCb(WOLFSSH_CTX* ctx, uint32_t highwater,
+                            WS_CallbackHighwater cb)
+{
+    WLOG(WS_LOG_DEBUG, "Entering wolfSSH_SetHighwaterCb()");
+
+    if (ctx) {
+        ctx->countHighwater = highwater;
+        ctx->highwaterCb = cb;
+    }
+}
+
+
+void wolfSSH_SetHighwaterCtx(WOLFSSH* ssh, void* ctx)
+{
+    WLOG(WS_LOG_DEBUG, "Entering wolfSSH_SetHighwaterCtx()");
+
+    if (ssh)
+        ssh->highwaterCtx = ctx;
+}
+
+
+void* wolfSSH_GetHighwaterCtx(WOLFSSH* ssh)
+{
+    WLOG(WS_LOG_DEBUG, "Entering wolfSSH_GetHighwaterCtx()");
+
+    if (ssh)
+        return ssh->highwaterCtx;
+
+    return NULL;
+}
+
+
 int wolfSSH_get_error(const WOLFSSH* ssh)
 {
     WLOG(WS_LOG_DEBUG, "Entering wolfSSH_get_error()");
diff --git a/wolfssh/internal.h b/wolfssh/internal.h
index 15e69015c9..9cd5dc89de 100644
--- a/wolfssh/internal.h
+++ b/wolfssh/internal.h
@@ -101,6 +101,7 @@ enum {
 #define MSG_ID_SZ        1
 #define SHA1_96_SZ       12
 #define UINT32_SZ        4
+#define DEFAULT_COUNT_HIGHWATER (1024 * 1024 * 1024)
 #define DEFAULT_WINDOW_SZ     (1024 * 1024)
 #define DEFAULT_MAX_PACKET_SZ (16 * 1024)
 #define DEFAULT_NEXT_CHANNEL  13013
@@ -138,6 +139,7 @@ struct WOLFSSH_CTX {
     WS_CallbackIORecv   ioRecvCb;    /* I/O Receive Callback */
     WS_CallbackIOSend   ioSendCb;    /* I/O Send Callback */
     WS_CallbackUserAuth userAuthCb;  /* User Authentication Callback */
+    WS_CallbackHighwater highwaterCb; /* Data Highwater Mark Callback */
 
     uint8_t*            cert;        /* Owned by CTX */
     uint32_t            certSz;
@@ -145,6 +147,7 @@ struct WOLFSSH_CTX {
     uint32_t            caCertSz;
     uint8_t*            privateKey;  /* Owned by CTX */
     uint32_t            privateKeySz;
+    uint32_t            countHighwater;
 };
 
 
@@ -153,6 +156,16 @@ typedef struct Ciphers {
 } Ciphers;
 
 
+typedef struct Keys {
+    uint8_t        iv[AES_BLOCK_SIZE];
+    uint8_t        ivSz;
+    uint8_t        encKey[AES_BLOCK_SIZE];
+    uint8_t        encKeySz;
+    uint8_t        macKey[MAX_HMAC_SZ];
+    uint8_t        macKeySz;
+} Keys;
+
+
 typedef struct HandshakeInfo {
     uint8_t        kexId;
     uint8_t        pubKeyId;
@@ -163,6 +176,8 @@ typedef struct HandshakeInfo {
     uint8_t        blockSz;
     uint8_t        macSz;
 
+    Keys           clientKeys;
+    Keys           serverKeys;
     Sha            hash;
     uint8_t        e[257]; /* May have a leading zero, for unsigned. */
     uint32_t       eSz;
@@ -182,6 +197,7 @@ struct WOLFSSH {
     uint32_t       txCount;
     uint32_t       rxCount;
     uint32_t       countHighwater;
+    void*          highwaterCtx;
     uint32_t       curSz;
     uint32_t       seq;
     uint32_t       peerSeq;
@@ -222,19 +238,8 @@ struct WOLFSSH {
     uint8_t        sessionId[SHA_DIGEST_SIZE];
     uint32_t       sessionIdSz;
 
-    uint8_t        ivClient[AES_BLOCK_SIZE];
-    uint8_t        ivClientSz;
-    uint8_t        ivServer[AES_BLOCK_SIZE];
-    uint8_t        ivServerSz;
-    uint8_t        encKeyClient[AES_BLOCK_SIZE];
-    uint8_t        encKeyClientSz;
-    uint8_t        encKeyServer[AES_BLOCK_SIZE];
-    uint8_t        encKeyServerSz;
-    uint8_t        macKeyClient[MAX_HMAC_SZ];
-    uint8_t        macKeyClientSz;
-    uint8_t        macKeyServer[MAX_HMAC_SZ];
-    uint8_t        macKeyServerSz;
-
+    Keys           clientKeys;
+    Keys           serverKeys;
     HandshakeInfo* handshake;
 
     void*          userAuthCtx;
@@ -260,6 +265,11 @@ struct WOLFSSH_CHANNEL {
 };
 
 
+WOLFSSL_LOCAL WOLFSSH_CTX* CtxInit(WOLFSSH_CTX*, void*);
+WOLFSSL_LOCAL void CtxResourceFree(WOLFSSH_CTX*);
+WOLFSSH_LOCAL WOLFSSH* SshInit(WOLFSSH*, WOLFSSH_CTX*);
+WOLFSSL_LOCAL void SshResourceFree(WOLFSSH*, void*);
+
 WOLFSSH_LOCAL WOLFSSH_CHANNEL* ChannelNew(WOLFSSH*, uint8_t, uint32_t,
                                           uint32_t, uint32_t);
 WOLFSSH_LOCAL void ChannelDelete(WOLFSSH_CHANNEL*, void*);
diff --git a/wolfssh/ssh.h b/wolfssh/ssh.h
index 5b56bc1bfe..7394a01ab2 100644
--- a/wolfssh/ssh.h
+++ b/wolfssh/ssh.h
@@ -59,6 +59,17 @@ WOLFSSH_API void     wolfSSH_free(WOLFSSH*);
 WOLFSSH_API int  wolfSSH_set_fd(WOLFSSH*, int);
 WOLFSSH_API int  wolfSSH_get_fd(const WOLFSSH*);
 
+/* data high water mark functions */
+WOLFSSH_API int wolfSSH_SetHighwater(WOLFSSH*, uint32_t);
+WOLFSSH_API uint32_t wolfSSH_GetHighwater(WOLFSSH*);
+
+typedef int (*WS_CallbackHighwater)(uint8_t, void*);
+WOLFSSH_API void wolfSSH_SetHighwaterCb(WOLFSSH_CTX*, uint32_t,
+                                        WS_CallbackHighwater);
+WOLFSSH_API void wolfSSH_SetHighwaterCtx(WOLFSSH*, void*);
+WOLFSSH_API void* wolfSSH_GetHighwaterCtx(WOLFSSH*);
+
+
 WOLFSSH_API int wolfSSH_get_error(const WOLFSSH*);
 WOLFSSH_API const char* wolfSSH_get_error_name(const WOLFSSH*);
 
@@ -131,6 +142,13 @@ WOLFSSH_API int wolfSSH_KDF(uint8_t, uint8_t, uint8_t*, uint32_t,
                 const uint8_t*, uint32_t, const uint8_t*, uint32_t,
                 const uint8_t*, uint32_t);
 
+
+enum WS_HighwaterSide {
+    WOLFSSH_HWSIDE_TRANSMIT,
+    WOLFSSH_HWSIDE_RECEIVE
+};
+
+
 enum WS_EndpointTypes {
     WOLFSSH_ENDPOINT_SERVER,
     WOLFSSH_ENDPOINT_CLIENT