Add support for AES-GCM crypto protocols (#797)

Add support for aes256-gcm@openssh.com and aes128-gcm@openssh.com
ciphers, which are the OpenSSH implementations of AES-GCM cryptography.
It is similar to RFC5647 but has changes to the MAC protocol
negotiation.  These are implemented for recent versions of OpenSSL only.

The ciphers work differently than most previous ones in two big areas:
the cipher includes its own integrated MAC, and the packet length field
in the SSH frame is left unencrypted.  The code changes necessary are
gated by flags in the LIBSSH2_CRYPT_METHOD configuration structure.

These differences mean that both the first and last parts of a block
require special handling during encryption. The first part is where the
packet length field is, which must be kept out of the encryption path
but in the authenticated part (as AAD).  The last part is where the
Authentication Tag is found, which is calculated and appended during
encryption or removed and validated on decryption. As encryption/
decryption is performed on each packet in a loop, one block at a time,
flags indicating when the first and last blocks are being processed are
passed down to the encryption layers.

The strict block-by-block encryption that occurs with other protocols is
inappropriate for AES-GCM, since the packet length shifts the first
encrypted byte 4 bytes into the block. Additionally, the final part of
the block must contain the AES-GCM's Authentication Tag, so it must be
presented to the lower encryption layer whole. These requirements mean
added code to consolidate blocks as they are passed down.

When AES-GCM is negotiated as the cipher, its built-in MAC is
automatically used as the SSH MAC so further MAC negotiation is not
necessary.  The SSH negotiation is skipped when _libssh2_mac_override()
indicates that such a cipher is in use.  The virtual MAC configuration
block mac_method_hmac_aesgcm is then used as the MAC placeholder.

This work was sponsored by Anders Borum.

Integration-patches-by: Viktor Szakats

* fix checksrc errors
* fix openssl.c warning
* fix transport.c warnings
* switch to `LIBSSH2_MIN/MAX()` from `MIN()`/`MAX()`
* fix indent
* fix libgcrypt unused warning
* fix mbedtls unused warning
* fix wincng unused warning
* fix old openssl unused variable warnings
* delete blank lines
* updates to help merging with the ETM patch

docs/HACKING-CRYPTO

@@ -323,7 +323,8 @@ int _libssh2_cipher_crypt(_libssh2_cipher_ctx *ctx,
                           _libssh2_cipher_type(algo),
                           int encrypt,
                           unsigned char *block,
-                          size_t blocksize);
+                          size_t blocksize,
+                          int firstlast);
 Encrypt or decrypt in-place data at (block, blocksize) using the given
 context and/or algorithm.
 Return 0 if OK, else -1.

src/crypt.c

@@ -54,7 +54,7 @@
  */
 static int
 crypt_none_crypt(LIBSSH2_SESSION * session, unsigned char *buf,
-                 void **abstract)
+                 void **abstract, int firstlast)
 {
     /* Do nothing to the data! */
     return 0;
@@ -106,12 +106,12 @@ crypt_init(LIBSSH2_SESSION * session,
 
 static int
 crypt_encrypt(LIBSSH2_SESSION * session, unsigned char *block,
-              size_t blocksize, void **abstract)
+              size_t blocksize, void **abstract, int firstlast)
 {
     struct crypt_ctx *cctx = *(struct crypt_ctx **) abstract;
     (void) session;
     return _libssh2_cipher_crypt(&cctx->h, cctx->algo, cctx->encrypt, block,
-                                 blocksize);
+                                 blocksize, firstlast);
 }
 
 static int
@@ -126,6 +126,34 @@ crypt_dtor(LIBSSH2_SESSION * session, void **abstract)
     return 0;
 }
 
+#if LIBSSH2_AES_GCM
+static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_aes256_gcm = {
+    "aes256-gcm@openssh.com",
+    "",
+    16,                         /* blocksize */
+    12,                         /* initial value length */
+    32,                         /* secret length -- 32*8 == 256bit */
+    LIBSSH2_CRYPT_FLAG_INTEGRATED_MAC | LIBSSH2_CRYPT_FLAG_PKTLEN_AAD,
+    &crypt_init,
+    &crypt_encrypt,
+    &crypt_dtor,
+    _libssh2_cipher_aes256gcm
+};
+
+static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_aes128_gcm = {
+    "aes128-gcm@openssh.com",
+    "",
+    16,                         /* blocksize */
+    12,                         /* initial value length */
+    16,                         /* secret length -- 16*8 == 128bit */
+    LIBSSH2_CRYPT_FLAG_INTEGRATED_MAC | LIBSSH2_CRYPT_FLAG_PKTLEN_AAD,
+    &crypt_init,
+    &crypt_encrypt,
+    &crypt_dtor,
+    _libssh2_cipher_aes128gcm
+};
+#endif
+
 #if LIBSSH2_AES_CTR
 static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_aes128_ctr = {
     "aes128-ctr",
@@ -269,7 +297,8 @@ crypt_init_arcfour128(LIBSSH2_SESSION * session,
         size_t discard = 1536;
         for(; discard; discard -= 8)
             _libssh2_cipher_crypt(&cctx->h, cctx->algo, cctx->encrypt, block,
-                                  method->blocksize);
+                                  method->blocksize, MIDDLE_BLOCK);
+                               /* Not all middle, but here it doesn't matter */
     }
 
     return rc;
@@ -322,6 +351,10 @@ static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_3des_cbc = {
 /* These are the crypt methods that are available to be negotiated. Methods
    towards the start are chosen in preference to ones further down the list. */
 static const LIBSSH2_CRYPT_METHOD *_libssh2_crypt_methods[] = {
+#if LIBSSH2_AES_GCM
+    &libssh2_crypt_method_aes256_gcm,
+    &libssh2_crypt_method_aes128_gcm,
+#endif /* LIBSSH2_AES_GCM */
 #if LIBSSH2_AES_CTR
     &libssh2_crypt_method_aes256_ctr,
     &libssh2_crypt_method_aes192_ctr,

src/crypto.h

@@ -332,7 +332,8 @@ int _libssh2_cipher_init(_libssh2_cipher_ctx * h,
 
 int _libssh2_cipher_crypt(_libssh2_cipher_ctx * ctx,
                           _libssh2_cipher_type(algo),
-                          int encrypt, unsigned char *block, size_t blocksize);
+                          int encrypt, unsigned char *block, size_t blocksize,
+                          int firstlast);
 
 int _libssh2_pub_priv_keyfile(LIBSSH2_SESSION *session,
                               unsigned char **method,

src/kex.c

@@ -3582,9 +3582,18 @@ static int kex_agree_mac(LIBSSH2_SESSION * session,
                          size_t mac_len)
 {
     const LIBSSH2_MAC_METHOD **macp = _libssh2_mac_methods();
+    const LIBSSH2_MAC_METHOD *override;
     unsigned char *s;
     (void)session;
 
+    override = _libssh2_mac_override(endpoint->crypt);
+    if(override) {
+        /* This crypto method has its own hmac method built-in, so a separate
+         * negotiation (and use) of a separate hmac method is unnecessary */
+        endpoint->mac = override;
+        return 0;
+    }
+
     if(endpoint->mac_prefs) {
         s = (unsigned char *) endpoint->mac_prefs;
 
@@ -3747,6 +3756,8 @@ static int kex_agree_methods(LIBSSH2_SESSION * session, unsigned char *data,
         return -1;
     }
 
+    /* This must happen after kex_agree_crypt since some MACs depend on the
+       negotiated crypto method */
     if(kex_agree_mac(session, &session->local, mac_cs, mac_cs_len) ||
         kex_agree_mac(session, &session->remote, mac_sc, mac_sc_len)) {
         return -1;

src/libgcrypt.c

@@ -607,11 +607,13 @@ _libssh2_cipher_init(_libssh2_cipher_ctx * h,
 int
 _libssh2_cipher_crypt(_libssh2_cipher_ctx * ctx,
                       _libssh2_cipher_type(algo),
-                      int encrypt, unsigned char *block, size_t blklen)
+                      int encrypt, unsigned char *block, size_t blklen,
+                      int firstlast)
 {
     int ret;
 
     (void)algo;
+    (void)firstlast;
 
     if(encrypt) {
         ret = gcry_cipher_encrypt(*ctx, block, blklen, block, blklen);

src/libgcrypt.h

@@ -51,6 +51,7 @@
 
 #define LIBSSH2_AES 1
 #define LIBSSH2_AES_CTR 1
+#define LIBSSH2_AES_GCM 0
 #define LIBSSH2_BLOWFISH 1
 #define LIBSSH2_RC4 1
 #define LIBSSH2_CAST 1

src/libssh2_priv.h

@@ -948,12 +948,36 @@ struct _LIBSSH2_CRYPT_METHOD
                  int *free_iv, unsigned char *secret, int *free_secret,
                  int encrypt, void **abstract);
     int (*crypt) (LIBSSH2_SESSION * session, unsigned char *block,
-                  size_t blocksize, void **abstract);
+                  size_t blocksize, void **abstract, int firstlast);
     int (*dtor) (LIBSSH2_SESSION * session, void **abstract);
 
       _libssh2_cipher_type(algo);
 };
 
+/* Bit flags for _LIBSSH2_CRYPT_METHOD */
+
+/* Crypto method has integrated message authentication */
+#define LIBSSH2_CRYPT_FLAG_INTEGRATED_MAC            1
+/* Crypto method does not encrypt the packet length */
+#define LIBSSH2_CRYPT_FLAG_PKTLEN_AAD                2
+
+/* Convenience macros for accessing crypt flags */
+/* Local crypto flags */
+#define CRYPT_FLAG_L(session, flag) ((session)->local.crypt && \
+    ((session)->local.crypt->flags & LIBSSH2_CRYPT_FLAG_##flag))
+/* Remote crypto flags */
+#define CRYPT_FLAG_R(session, flag) ((session)->remote.crypt && \
+    ((session)->remote.crypt->flags & LIBSSH2_CRYPT_FLAG_##flag))
+
+/* Values for firstlast */
+#define FIRST_BLOCK 1
+#define MIDDLE_BLOCK 0
+#define LAST_BLOCK 2
+
+/* Convenience macros for accessing firstlast */
+#define IS_FIRST(firstlast) (firstlast & FIRST_BLOCK)
+#define IS_LAST(firstlast) (firstlast & LAST_BLOCK)
+
 struct _LIBSSH2_COMP_METHOD
 {
     const char *name;

src/mac.c

@@ -423,3 +423,32 @@ _libssh2_mac_methods(void)
 {
     return mac_methods;
 }
+
+#if LIBSSH2_AES_GCM
+/* Stub for aes256-gcm@openssh.com crypto type, which has an integrated
+   HMAC method. This must not be added to mac_methods[] since it cannot be
+   negotiated separately. */
+static const LIBSSH2_MAC_METHOD mac_method_hmac_aesgcm = {
+    "INTEGRATED-AES-GCM",  /* made up name for display only */
+    16,
+    16,
+    NULL,
+    NULL,
+    NULL,
+};
+#endif /* LIBSSH2_AES_GCM */
+
+/* See if the negotiated crypto method has its own authentication scheme that
+ * obviates the need for a separate negotiated hmac method */
+const LIBSSH2_MAC_METHOD *
+_libssh2_mac_override(const LIBSSH2_CRYPT_METHOD *crypt)
+{
+#if LIBSSH2_AES_GCM
+    if(!strcmp(crypt->name, "aes256-gcm@openssh.com") ||
+       !strcmp(crypt->name, "aes128-gcm@openssh.com"))
+        return &mac_method_hmac_aesgcm;
+#else
+    (void) crypt;
+#endif /* LIBSSH2_AES_GCM */
+    return NULL;
+}

src/mac.h

@@ -62,5 +62,7 @@ struct _LIBSSH2_MAC_METHOD
 typedef struct _LIBSSH2_MAC_METHOD LIBSSH2_MAC_METHOD;
 
 const LIBSSH2_MAC_METHOD **_libssh2_mac_methods(void);
+const LIBSSH2_MAC_METHOD *_libssh2_mac_override(
+        const LIBSSH2_CRYPT_METHOD *crypt);
 
 #endif /* __LIBSSH2_MAC_H */

src/mbedtls.c

@@ -139,14 +139,15 @@ _libssh2_mbedtls_cipher_crypt(_libssh2_cipher_ctx *ctx,
                               _libssh2_cipher_type(algo),
                               int encrypt,
                               unsigned char *block,
-                              size_t blocklen)
+                              size_t blocklen, int firstlast)
 {
     int ret;
     unsigned char *output;
     size_t osize, olen, finish_olen;
 
     (void)encrypt;
     (void)algo;
+    (void)firstlast;
 
     osize = blocklen + mbedtls_cipher_get_block_size(ctx);
 

src/mbedtls.h

@@ -67,6 +67,7 @@
 
 #define LIBSSH2_AES             1
 #define LIBSSH2_AES_CTR         1
+#define LIBSSH2_AES_GCM         0
 #ifdef MBEDTLS_CIPHER_BLOWFISH_CBC
 # define LIBSSH2_BLOWFISH       1
 #else
@@ -390,8 +391,8 @@ typedef enum {
 
 #define _libssh2_cipher_init(ctx, type, iv, secret, encrypt) \
     _libssh2_mbedtls_cipher_init(ctx, type, iv, secret, encrypt)
-#define _libssh2_cipher_crypt(ctx, type, encrypt, block, blocklen) \
-    _libssh2_mbedtls_cipher_crypt(ctx, type, encrypt, block, blocklen)
+#define _libssh2_cipher_crypt(ctx, type, encrypt, block, blocklen, fl) \
+    _libssh2_mbedtls_cipher_crypt(ctx, type, encrypt, block, blocklen, fl)
 #define _libssh2_cipher_dtor(ctx) \
     _libssh2_mbedtls_cipher_dtor(ctx)
 
@@ -472,7 +473,7 @@ _libssh2_mbedtls_cipher_crypt(_libssh2_cipher_ctx *ctx,
                               _libssh2_cipher_type(type),
                               int encrypt,
                               unsigned char *block,
-                              size_t blocklen);
+                              size_t blocklen, int firstlast);
 void
 _libssh2_mbedtls_cipher_dtor(_libssh2_cipher_ctx *ctx);