blowfish: build improvements

- include `blowfish.c` into `bcrypt_pbkdf.c`, instead of
  compiling it as a distinct object.

- make low-level blowfish functions static. This prevents this symbols
  to pollute the public namespace of libssh2. It also allows the
  compiler to inline these functions.

- integrate `blf.h` header into `bcrypt_pbkdf.c` as well.

- use `_DEBUG_BLOWFISH` instead of `#if 0`.

- fix `_DEBUG_BLOWFISH` compiler warnings and other nits.

- `#undef` `inline` before redefining it in `libssh2_priv.h`.
  (copied from `blowfish.c`)

- delete unused `inline` redefinitions from `blowfish.c`.

- disable unused low-level blowfish functions.

- formatting, header order.

Closes libssh2#938

Makefile.inc

@@ -2,8 +2,8 @@ CSOURCES = channel.c comp.c crypt.c hostkey.c kex.c mac.c misc.c \
   packet.c publickey.c scp.c session.c sftp.c userauth.c transport.c \
   userauth_kbd_packet.c \
   version.c knownhost.c agent.c $(CRYPTO_CSOURCES) pem.c keepalive.c global.c \
-  blowfish.c bcrypt_pbkdf.c agent_win.c os400qc3.c
+  bcrypt_pbkdf.c agent_win.c os400qc3.c
 
 HHEADERS = libssh2_priv.h libssh2_setup.h $(CRYPTO_HHEADERS) transport.h \
   channel.h comp.h mac.h misc.h packet.h userauth.h session.h sftp.h crypto.h \
-  blf.h agent.h userauth_kbd_packet.h os400qc3.h
+  agent.h userauth_kbd_packet.h os400qc3.h

src/CMakeLists.txt

@@ -108,9 +108,7 @@ set(SOURCES
   ${CRYPTO_SOURCES}
   agent.c
   agent_win.c
-  blf.h
   bcrypt_pbkdf.c
-  blowfish.c
   channel.c
   channel.h
   comp.c

src/bcrypt_pbkdf.c

@@ -25,7 +25,7 @@
 #include <sys/param.h>
 #endif
 
-#include "blf.h"
+#include "blowfish.c"
 
 #define MINIMUM(a,b) (((a) < (b)) ? (a) : (b))
 

src/blowfish.c

@@ -1,6 +1,7 @@
 /* $OpenBSD: blowfish.c,v 1.18 2004/11/02 17:23:26 hshoexer Exp $ */
 /*
- * Blowfish block cipher for OpenBSD
+ * Blowfish for OpenBSD - a fast block cipher designed by Bruce Schneier
+ *
  * Copyright 1997 Niels Provos <provos@physnet.uni-hamburg.de>
  * All rights reserved.
  *
@@ -36,40 +37,79 @@
  * Bruce Schneier.
  */
 
-
 #if !defined(HAVE_BCRYPT_PBKDF) && (!defined(HAVE_BLOWFISH_INITSTATE) || \
                                     !defined(HAVE_BLOWFISH_EXPAND0STATE) || \
                                     !defined(HAVE_BLF_ENC))
 
-#if 0
-#include <stdio.h>              /* used for debugging */
+#ifdef _DEBUG_BLOWFISH
+#include <stdio.h>
 #include <string.h>
+#include <inttypes.h>
 #endif
 
-#include <sys/types.h>
+/* Schneier specifies a maximum key length of 56 bytes.
+ * This ensures that every key bit affects every cipher
+ * bit.  However, the subkeys can hold up to 72 bytes.
+ * Warning: For normal blowfish encryption only 56 bytes
+ * of the key affect all cipherbits.
+ */
+
+#define BLF_N   16                      /* Number of Subkeys */
+#define BLF_MAXKEYLEN ((BLF_N-2)*4)     /* 448 bits */
+#define BLF_MAXUTILIZED ((BLF_N + 2)*4)   /* 576 bits */
+
+/* Blowfish context */
+typedef struct BlowfishContext {
+        uint32_t S[4][256];     /* S-Boxes */
+        uint32_t P[BLF_N + 2];  /* Subkeys */
+} blf_ctx;
+
+/* Raw access to customized Blowfish
+ *      blf_key is just:
+ *      Blowfish_initstate( state )
+ *      Blowfish_expand0state( state, key, keylen )
+ */
+
+static void Blowfish_encipher(blf_ctx *, uint32_t *, uint32_t *);
+#ifdef _DEBUG_BLOWFISH
+static void Blowfish_decipher(blf_ctx *, uint32_t *, uint32_t *);
+#endif
+static void Blowfish_initstate(blf_ctx *);
+static void Blowfish_expand0state(blf_ctx *, const uint8_t *, uint16_t);
+static void Blowfish_expandstate
+(blf_ctx *, const uint8_t *, uint16_t, const uint8_t *, uint16_t);
 
-#include "libssh2.h"
-#include "blf.h"
+/* Standard Blowfish */
 
-#undef inline
-#ifdef __GNUC__
-#define inline __inline__
-#elif defined(_MSC_VER)
-#define inline __inline
-#else
-#define inline
+#ifdef _DEBUG_BLOWFISH
+static void blf_key(blf_ctx *, const uint8_t *, uint16_t);
+#endif
+static void blf_enc(blf_ctx *, uint32_t *, uint16_t);
+#ifdef _DEBUG_BLOWFISH
+static void blf_dec(blf_ctx *, uint32_t *, uint16_t);
 #endif
 
+#if 0
+static void blf_ecb_encrypt(blf_ctx *, uint8_t *, uint32_t);
+static void blf_ecb_decrypt(blf_ctx *, uint8_t *, uint32_t);
+
+static void blf_cbc_encrypt(blf_ctx *, uint8_t *, uint8_t *, uint32_t);
+static void blf_cbc_decrypt(blf_ctx *, uint8_t *, uint8_t *, uint32_t);
+#endif
+
+/* Converts uint8_t to uint32_t */
+static uint32_t Blowfish_stream2word(const uint8_t *, uint16_t, uint16_t *);
+
 /* Function for Feistel Networks */
 
-#define F(s, x) ((((s)[        (((x)>>24)&0xFF)]        \
-                   + (s)[0x100 + (((x)>>16)&0xFF)])     \
-                  ^ (s)[0x200 + (((x)>> 8)&0xFF)])      \
+#define F(s, x) ((((s)[        (((x)>>24)&0xFF)]      \
+                 + (s)[0x100 + (((x)>>16)&0xFF)])     \
+                 ^ (s)[0x200 + (((x)>> 8)&0xFF)])     \
                  + (s)[0x300 + ( (x)     &0xFF)])
 
 #define BLFRND(s,p,i,j,n) (i ^= F(s,j) ^ (p)[n])
 
-void
+static void
 Blowfish_encipher(blf_ctx *c, uint32_t *xl, uint32_t *xr)
 {
     uint32_t Xl;
@@ -94,7 +134,8 @@ Blowfish_encipher(blf_ctx *c, uint32_t *xl, uint32_t *xr)
     *xr = Xl;
 }
 
-void
+#ifdef _DEBUG_BLOWFISH
+static void
 Blowfish_decipher(blf_ctx *c, uint32_t *xl, uint32_t *xr)
 {
     uint32_t Xl;
@@ -118,8 +159,9 @@ Blowfish_decipher(blf_ctx *c, uint32_t *xl, uint32_t *xr)
     *xl = Xr ^ p[0];
     *xr = Xl;
 }
+#endif
 
-void
+static void
 Blowfish_initstate(blf_ctx *c)
 {
     /* P-box and S-box tables initialized with digits of Pi */
@@ -398,7 +440,7 @@ Blowfish_initstate(blf_ctx *c)
     *c = initstate;
 }
 
-uint32_t
+static uint32_t
 Blowfish_stream2word(const uint8_t *data, uint16_t databytes,
                      uint16_t *current)
 {
@@ -419,7 +461,7 @@ Blowfish_stream2word(const uint8_t *data, uint16_t databytes,
     return temp;
 }
 
-void
+static void
 Blowfish_expand0state(blf_ctx *c, const uint8_t *key, uint16_t keybytes)
 {
     int i;
@@ -456,8 +498,7 @@ Blowfish_expand0state(blf_ctx *c, const uint8_t *key, uint16_t keybytes)
     }
 }
 
-
-void
+static void
 Blowfish_expandstate(blf_ctx *c, const uint8_t *data, uint16_t databytes,
                      const uint8_t *key, uint16_t keybytes)
 {
@@ -500,7 +541,8 @@ Blowfish_expandstate(blf_ctx *c, const uint8_t *data, uint16_t databytes,
 
 }
 
-void
+#ifdef _DEBUG_BLOWFISH
+static void
 blf_key(blf_ctx *c, const uint8_t *k, uint16_t len)
 {
     /* Initialize S-boxes and subkeys with Pi */
@@ -509,8 +551,9 @@ blf_key(blf_ctx *c, const uint8_t *k, uint16_t len)
     /* Transform S-boxes and subkeys with key */
     Blowfish_expand0state(c, k, len);
 }
+#endif
 
-void
+static void
 blf_enc(blf_ctx *c, uint32_t *data, uint16_t blocks)
 {
     uint32_t *d;
@@ -523,7 +566,8 @@ blf_enc(blf_ctx *c, uint32_t *data, uint16_t blocks)
     }
 }
 
-void
+#ifdef _DEBUG_BLOWFISH
+static void
 blf_dec(blf_ctx *c, uint32_t *data, uint16_t blocks)
 {
     uint32_t *d;
@@ -535,8 +579,10 @@ blf_dec(blf_ctx *c, uint32_t *data, uint16_t blocks)
         d += 2;
     }
 }
+#endif
 
-void
+#if 0
+static void
 blf_ecb_encrypt(blf_ctx *c, uint8_t *data, uint32_t len)
 {
     uint32_t l, r;
@@ -558,7 +604,7 @@ blf_ecb_encrypt(blf_ctx *c, uint8_t *data, uint32_t len)
     }
 }
 
-void
+static void
 blf_ecb_decrypt(blf_ctx *c, uint8_t *data, uint32_t len)
 {
     uint32_t l, r;
@@ -580,7 +626,7 @@ blf_ecb_decrypt(blf_ctx *c, uint8_t *data, uint32_t len)
     }
 }
 
-void
+static void
 blf_cbc_encrypt(blf_ctx *c, uint8_t *iv, uint8_t *data, uint32_t len)
 {
     uint32_t l, r;
@@ -605,7 +651,7 @@ blf_cbc_encrypt(blf_ctx *c, uint8_t *iv, uint8_t *data, uint32_t len)
     }
 }
 
-void
+static void
 blf_cbc_decrypt(blf_ctx *c, uint8_t *iva, uint8_t *data, uint32_t len)
 {
     uint32_t l, r;
@@ -645,20 +691,20 @@ blf_cbc_decrypt(blf_ctx *c, uint8_t *iva, uint8_t *data, uint32_t len)
     for(j = 0; j < 8; j++)
         data[j] ^= iva[j];
 }
+#endif
 
-#if 0
-void
+#ifdef _DEBUG_BLOWFISH
+static void
 report(uint32_t data[], uint16_t len)
 {
-    uint16_t i;
+    int i;
     for(i = 0; i < len; i += 2)
-        printf("Block %0hd: %08lx %08lx.\n",
-               i / 2, data[i], data[i + 1]);
+        printf("Block %d: 0x%08lx 0x%08lx.\n",
+               i / 2, (unsigned long)data[i], (unsigned long)data[i + 1]);
 }
-void
+int
 main(void)
 {
-
     blf_ctx c;
     char    key[] = "AAAAA";
     char    key2[] = "abcdefghijklmnopqrstuvwxyz";
@@ -681,12 +727,15 @@ main(void)
     report(data, 10);
 
     /* Second test */
-    blf_key(&c, (uint8_t *) key2, strlen(key2));
+    blf_key(&c, (uint8_t *) key2, (uint16_t)strlen(key2));
     blf_enc(&c, data2, 1);
     printf("\nShould read as: 0x324ed0fe 0xf413a203.\n");
     report(data2, 2);
     blf_dec(&c, data2, 1);
+    printf("\nShould read as: 0x424c4f57 0x46495348.\n");
     report(data2, 2);
+
+    return 0;
 }
 #endif
 

src/libssh2_priv.h

@@ -120,8 +120,10 @@ int _libssh2_snprintf(char *cp, size_t cp_max_len, const char *fmt, ...);
 
 /* "inline" keyword is valid only with C++ engine! */
 #ifdef __GNUC__
+#undef inline
 #define inline __inline__
 #elif defined(_MSC_VER)
+#undef inline
 #define inline __inline
 #endif