Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
276 changes: 228 additions & 48 deletions src/mbedtls.c
Original file line number Diff line number Diff line change
Expand Up @@ -109,45 +109,108 @@ _libssh2_mbedtls_cipher_init(_libssh2_cipher_ctx *h,
unsigned char *secret,
int encrypt)
{
const mbedtls_cipher_info_t *cipher_info;
int ret, op;
struct _libssh2_mbedtls_cipher_ctx *cctx;
int ret;

if(!h)
return -1;

op = encrypt ? MBEDTLS_ENCRYPT : MBEDTLS_DECRYPT;

cipher_info = mbedtls_cipher_info_from_type(algo);
if(!cipher_info)
/* Allocate unified context structure */
cctx = (struct _libssh2_mbedtls_cipher_ctx *)
mbedtls_calloc(1, sizeof(struct _libssh2_mbedtls_cipher_ctx));
if(!cctx)
return -1;

mbedtls_cipher_init(h);
ret = mbedtls_cipher_setup(h, cipher_info);

/* libssh2 computes and adds SSH packet padding itself, so for CBC
* tell mbedTLS to expect no padding on the cipher layer. Only call
* set_padding_mode for CBC ciphers since other modes (CTR, stream)
* are not applicable and will cause an error. */
if(!ret) {
if(algo == MBEDTLS_CIPHER_AES_128_CBC ||
algo == MBEDTLS_CIPHER_AES_192_CBC ||
algo == MBEDTLS_CIPHER_AES_256_CBC ||
algo == MBEDTLS_CIPHER_DES_EDE3_CBC) {
ret = mbedtls_cipher_set_padding_mode(h, MBEDTLS_PADDING_NONE);
/* Store algorithm type and encryption flag */
cctx->algo = algo;
cctx->encrypt = encrypt;

#if LIBSSH2_AES_GCM
/* Check if this is a GCM algorithm */
if(algo == MBEDTLS_CIPHER_AES_128_GCM ||
algo == MBEDTLS_CIPHER_AES_256_GCM) {
unsigned int keybits;

/* Store the initial IV (will be incremented per packet) */
memcpy(cctx->ctx.gcm.iv, iv, 12);

/* Initialize GCM context */
mbedtls_gcm_init(&cctx->ctx.gcm.gcm_ctx);

/* Set key length based on algorithm */
if(algo == MBEDTLS_CIPHER_AES_128_GCM)
keybits = 128;
else /* MBEDTLS_CIPHER_AES_256_GCM */
keybits = 256;

/* Setup GCM with AES cipher and key */
ret = mbedtls_gcm_setkey(&cctx->ctx.gcm.gcm_ctx,
MBEDTLS_CIPHER_ID_AES,
secret,
keybits);

if(ret) {
mbedtls_gcm_free(&cctx->ctx.gcm.gcm_ctx);
mbedtls_free(cctx);
return -1;
}

/* Store the context pointer */
*h = cctx;
return 0;
}
Comment on lines +157 to 161
Copy link

Copilot AI May 1, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

_libssh2_mbedtls_cipher_init() stores cctx via *(void **)h = cctx;, but in the mbedTLS backend _libssh2_cipher_ctx is defined as mbedtls_cipher_context_t (a struct), not a pointer type. This is undefined behavior (strict aliasing/alignment) and makes the _libssh2_cipher_ctx ABI inconsistent. Fix by redefining _libssh2_cipher_ctx for mbedTLS to be a pointer type (and updating init/crypt/dtor accordingly) or by embedding the unified context directly in _libssh2_cipher_ctx.

Copilot uses AI. Check for mistakes.
#endif

if(!ret)
ret = mbedtls_cipher_setkey(h,
secret,
(int)mbedtls_cipher_info_get_key_bitlen(cipher_info),
op);
/* Non-GCM algorithms: use standard cipher API */
{
const mbedtls_cipher_info_t *cipher_info;
int op;

if(!ret)
ret = mbedtls_cipher_set_iv(h, iv,
mbedtls_cipher_info_get_iv_size(cipher_info));
op = encrypt ? MBEDTLS_ENCRYPT : MBEDTLS_DECRYPT;

return ret == 0 ? 0 : -1;
cipher_info = mbedtls_cipher_info_from_type(algo);
if(!cipher_info) {
mbedtls_free(cctx);
return -1;
}

mbedtls_cipher_init(&cctx->ctx.cipher_ctx);
ret = mbedtls_cipher_setup(&cctx->ctx.cipher_ctx, cipher_info);

/* libssh2 computes and adds SSH packet padding itself, so for CBC
* tell mbedTLS to expect no padding on the cipher layer. Only call
* set_padding_mode for CBC ciphers since other modes (CTR, stream)
* are not applicable and will cause an error. */
if(!ret) {
if(algo == MBEDTLS_CIPHER_AES_128_CBC ||
algo == MBEDTLS_CIPHER_AES_192_CBC ||
algo == MBEDTLS_CIPHER_AES_256_CBC ||
algo == MBEDTLS_CIPHER_DES_EDE3_CBC) {
ret = mbedtls_cipher_set_padding_mode(&cctx->ctx.cipher_ctx,
MBEDTLS_PADDING_NONE);
}
}

if(!ret)
ret = mbedtls_cipher_setkey(&cctx->ctx.cipher_ctx,
secret,
(int)mbedtls_cipher_info_get_key_bitlen(cipher_info),
op);

if(!ret)
ret = mbedtls_cipher_set_iv(&cctx->ctx.cipher_ctx, iv,
mbedtls_cipher_info_get_iv_size(cipher_info));

if(ret) {
mbedtls_cipher_free(&cctx->ctx.cipher_ctx);
mbedtls_free(cctx);
return -1;
}

/* Store the context pointer */
*h = cctx;
return 0;
}
}

int
Expand All @@ -157,43 +220,160 @@ _libssh2_mbedtls_cipher_crypt(_libssh2_cipher_ctx *ctx,
unsigned char *block,
size_t blocksize, int firstlast)
{
struct _libssh2_mbedtls_cipher_ctx *cctx;
int ret;
unsigned char *output;
size_t osize, olen, finish_olen;

(void)encrypt;
cctx = *ctx;
if(!cctx)
return -1;

(void)algo;
(void)firstlast;
(void)encrypt;

osize = blocksize + mbedtls_cipher_get_block_size(ctx);
#if LIBSSH2_AES_GCM
/* Check if this is a GCM algorithm based on stored algo type */
if(cctx->algo == MBEDTLS_CIPHER_AES_128_GCM ||
cctx->algo == MBEDTLS_CIPHER_AES_256_GCM) {
const int authlen = 16; /* GCM authentication tag length */
const int aadlen = IS_FIRST(firstlast) ? 4 : 0;
const int authenticationtag = IS_LAST(firstlast) ? authlen : 0;
const int cryptlen = (int)blocksize - aadlen - authenticationtag;
unsigned char tag[16];
unsigned char *buf = NULL;

/* Allocate temporary buffer for output */
if(cryptlen > 0) {
buf = (unsigned char *)mbedtls_calloc(cryptlen, 1);
if(!buf)
return -1;
}
Comment on lines +245 to +249
Copy link

Copilot AI May 1, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The temporary buf in the GCM path can hold plaintext during decrypt, but it’s freed with mbedtls_free() (and on some error paths too) without being wiped. Use _libssh2_mbedtls_safe_free(buf, cryptlen) (or equivalent explicit zeroization) for all frees of this buffer to avoid leaving plaintext in heap memory.

Copilot uses AI. Check for mistakes.

output = (unsigned char *)mbedtls_calloc(osize, sizeof(char));
if(output) {
ret = mbedtls_cipher_reset(ctx);
/* First block: start GCM operation */
if(IS_FIRST(firstlast)) {
ret = mbedtls_gcm_starts(&cctx->ctx.gcm.gcm_ctx,
cctx->encrypt ? MBEDTLS_GCM_ENCRYPT :
MBEDTLS_GCM_DECRYPT,
cctx->ctx.gcm.iv, 12);
if(ret) {
_libssh2_mbedtls_safe_free(buf, cryptlen);
return -1;
}

/* Process AAD (Additional Authenticated Data) */
if(aadlen) {
ret = mbedtls_gcm_update_ad(&cctx->ctx.gcm.gcm_ctx,
block, aadlen);
if(ret) {
_libssh2_mbedtls_safe_free(buf, cryptlen);
return -1;
}
}
}

if(!ret)
ret = mbedtls_cipher_update(ctx, block, blocksize, output, &olen);
/* Process payload */
if(cryptlen > 0) {
size_t olen = 0;
ret = mbedtls_gcm_update(&cctx->ctx.gcm.gcm_ctx,
block + aadlen, cryptlen,
buf, cryptlen, &olen);
if(ret) {
_libssh2_mbedtls_safe_free(buf, cryptlen);
return -1;
}
memcpy(block + aadlen, buf, olen);
_libssh2_mbedtls_safe_free(buf, cryptlen);
}

if(!ret)
ret = mbedtls_cipher_finish(ctx, output + olen, &finish_olen);
/* Last block: finalize and handle authentication tag */
if(IS_LAST(firstlast)) {
size_t olen_finish = 0;
unsigned char finish_buf[16];
int i;

if(!ret) {
olen += finish_olen;
memcpy(block, output, olen);
ret = mbedtls_gcm_finish(&cctx->ctx.gcm.gcm_ctx,
finish_buf, sizeof(finish_buf),
&olen_finish, tag, authlen);
if(ret)
return -1;

if(cctx->encrypt) {
memcpy(block + blocksize - authlen, tag, authlen);
}
else {
if(_libssh2_timingsafe_bcmp(tag, block + blocksize - authlen,
authlen) != 0)
return MBEDTLS_ERR_GCM_AUTH_FAILED;
}

/* Increment IV for next packet */
for(i = 11; i >= 4; i--) {
if(++cctx->ctx.gcm.iv[i])
break;
}
}

_libssh2_mbedtls_safe_free(output, osize);
return 0;
}
else
ret = -1;
#endif

return ret == 0 ? 0 : -1;
/* Non-GCM algorithms: use standard cipher API */
{
unsigned char *output;
size_t osize, olen, finish_olen;

(void)firstlast;

osize = blocksize +
mbedtls_cipher_get_block_size(&cctx->ctx.cipher_ctx);

output = (unsigned char *)mbedtls_calloc(osize, sizeof(char));
if(output) {
ret = mbedtls_cipher_reset(&cctx->ctx.cipher_ctx);

if(!ret)
ret = mbedtls_cipher_update(&cctx->ctx.cipher_ctx, block,
blocksize, output, &olen);

if(!ret)
ret = mbedtls_cipher_finish(&cctx->ctx.cipher_ctx,
output + olen, &finish_olen);

if(!ret) {
olen += finish_olen;
memcpy(block, output, olen);
}

_libssh2_mbedtls_safe_free(output, osize);
}
else
ret = -1;

return ret == 0 ? 0 : -1;
}
}

void
_libssh2_mbedtls_cipher_dtor(_libssh2_cipher_ctx *ctx)
{
mbedtls_cipher_free(ctx);
struct _libssh2_mbedtls_cipher_ctx *cctx = *ctx;

if(!cctx)
return;

#if LIBSSH2_AES_GCM
if(cctx->algo == MBEDTLS_CIPHER_AES_128_GCM ||
cctx->algo == MBEDTLS_CIPHER_AES_256_GCM) {
mbedtls_gcm_free(&cctx->ctx.gcm.gcm_ctx);
mbedtls_free(cctx);
*ctx = NULL;
return;
}
#endif

/* Regular cipher context */
mbedtls_cipher_free(&cctx->ctx.cipher_ctx);
mbedtls_free(cctx);
*ctx = NULL;
}

int
Expand Down
30 changes: 26 additions & 4 deletions src/mbedtls.h
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
#include <mbedtls/rsa.h>
#include <mbedtls/bignum.h>
#include <mbedtls/cipher.h>
#include <mbedtls/gcm.h>
#ifdef MBEDTLS_ECDH_C
# include <mbedtls/ecdh.h>
#endif
Expand All @@ -67,7 +68,7 @@

#define LIBSSH2_AES_CBC 1
#define LIBSSH2_AES_CTR 1
#define LIBSSH2_AES_GCM 0
#define LIBSSH2_AES_GCM 1
#ifdef MBEDTLS_CIPHER_BLOWFISH_CBC
# define LIBSSH2_BLOWFISH 1
#else
Expand Down Expand Up @@ -337,7 +338,26 @@ typedef enum {
* mbedTLS backend: Cipher Context structure
*/

#define _libssh2_cipher_ctx mbedtls_cipher_context_t
/*
* Unified cipher context structure.
* Handles both regular ciphers and AEAD ciphers (GCM)
* using a union to optimize memory usage.
*/
struct _libssh2_mbedtls_cipher_ctx {
mbedtls_cipher_type_t algo;
int encrypt;
union {
mbedtls_cipher_context_t cipher_ctx;
#if LIBSSH2_AES_GCM
struct {
mbedtls_gcm_context gcm_ctx;
unsigned char iv[12];
} gcm;
#endif
} ctx;
};

#define _libssh2_cipher_ctx struct _libssh2_mbedtls_cipher_ctx *

#define _libssh2_cipher_type(algo) mbedtls_cipher_type_t algo

Expand All @@ -354,15 +374,17 @@ typedef enum {
#define _libssh2_cipher_arcfour MBEDTLS_CIPHER_ARC4_128
#endif
#define _libssh2_cipher_3des MBEDTLS_CIPHER_DES_EDE3_CBC
#define _libssh2_cipher_aes128gcm MBEDTLS_CIPHER_AES_128_GCM
#define _libssh2_cipher_aes256gcm MBEDTLS_CIPHER_AES_256_GCM
#define _libssh2_cipher_chacha20 MBEDTLS_CIPHER_CHACHA20_POLY1305

/*******************************************************************/
/*
* mbedTLS backend: Cipher functions
*/

#define _libssh2_cipher_init(ctx, type, iv, secret, encrypt) \
_libssh2_mbedtls_cipher_init(ctx, type, iv, secret, encrypt)
#define _libssh2_cipher_init(h, type, iv, secret, encrypt) \
_libssh2_mbedtls_cipher_init(h, type, iv, secret, encrypt)
Comment on lines +386 to +387
Copy link

Copilot AI May 1, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The PR description says AES-GCM is enabled for the mbedTLS backend, but in the current codebase LIBSSH2_AES_GCM is still defined as 0 in src/mbedtls.h and there are no _libssh2_cipher_aes{128,256}gcm mappings for mbedTLS. As a result, all the new GCM code in mbedtls.c is compiled out and GCM won’t actually be negotiable/usable. Please flip LIBSSH2_AES_GCM to 1, add the required mbedtls/gcm.h include, and add the cipher type mappings (consistent with how openssl.h exposes _libssh2_cipher_aes128gcm/_libssh2_cipher_aes256gcm).

Copilot uses AI. Check for mistakes.
#define _libssh2_cipher_crypt(ctx, type, encrypt, block, blocklen, fl) \
_libssh2_mbedtls_cipher_crypt(ctx, type, encrypt, block, blocklen, fl)
#define _libssh2_cipher_dtor(ctx) \
Expand Down
Loading