musl: improve crypt() size hack

Instead of silently downgrading any non-MD5 crypt() request to DES,
cleanly fail with return NULL and errno = ENOSYS. This allows callers
to notice the missing support instead of the unwanted silent fallback
to DES.

Also add a menuconfig toolchain option to optionally disable the crypt
size hack completely. This can be probably made dependant on SMALL_FLASH
or a similar feature indicator in a future commit.

Ref: openwrt#1331
Signed-off-by: Jo-Philipp Wich <jo@mein.io>

toolchain/Config.in

@@ -262,6 +262,7 @@ choice
 endchoice
 
 source "toolchain/uClibc/Config.in"
+source "toolchain/musl/Config.in"
 
 comment "Debuggers"
 	depends on TOOLCHAINOPTS

toolchain/musl/Config.in

@@ -0,0 +1,12 @@
+# Password crypt stubbing
+
+config MUSL_DISABLE_CRYPT_SIZE_HACK
+	bool "Include crypt() support for SHA256, SHA512 and Blowfish ciphers"
+	depends on TOOLCHAINOPTS && USE_MUSL && !EXTERNAL_TOOLCHAIN
+	default n
+	help
+	  Enable this option to re-include crypt() support for the SHA256, SHA512 and
+	  Blowfish ciphers. Without this option, attempting to hash a string with a salt
+	  requesting one of these ciphers will cause the crypt() function to call stub
+	  implementations which will always fail with errno ENOSYS. Including the ciphers
+	  will increase the library size by about 14KB after LZMA compression.

toolchain/musl/common.mk

@@ -29,6 +29,7 @@ include $(INCLUDE_DIR)/host-build.mk
 include $(INCLUDE_DIR)/hardening.mk
 
 TARGET_CFLAGS:= $(filter-out -O%,$(TARGET_CFLAGS))
+TARGET_CFLAGS+= $(if $(CONFIG_MUSL_DISABLE_CRYPT_SIZE_HACK),,-DCRYPT_SIZE_HACK)
 
 MUSL_CONFIGURE:= \
 	$(TARGET_CONFIGURE_OPTS) \

toolchain/musl/patches/901-crypt_size_hack.patch

@@ -1,59 +1,74 @@
---- a/src/crypt/crypt_r.c
-+++ b/src/crypt/crypt_r.c
-@@ -19,12 +19,6 @@ char *__crypt_r(const char *key, const c
- 	if (salt[0] == '$' && salt[1] && salt[2]) {
- 		if (salt[1] == '1' && salt[2] == '$')
- 			return __crypt_md5(key, salt, output);
--		if (salt[1] == '2' && salt[3] == '$')
--			return __crypt_blowfish(key, salt, output);
--		if (salt[1] == '5' && salt[2] == '$')
--			return __crypt_sha256(key, salt, output);
--		if (salt[1] == '6' && salt[2] == '$')
--			return __crypt_sha512(key, salt, output);
- 	}
- 	return __crypt_des(key, salt, output);
- }
 --- a/src/crypt/crypt_sha512.c
 +++ b/src/crypt/crypt_sha512.c
-@@ -12,6 +12,7 @@
- #include <stdio.h>
+@@ -13,6 +13,17 @@
  #include <string.h>
  #include <stdint.h>
-+#if 0
 
++#ifdef CRYPT_SIZE_HACK
++#include <errno.h>
++
++char *__crypt_sha512(const char *key, const char *setting, char *output)
++{
++	errno = ENOSYS;
++	return NULL;
++}
++
++#else
++
  /* public domain sha512 implementation based on fips180-3 */
  /* >=2^64 bits messages are not supported (about 2000 peta bytes) */
-@@ -369,3 +370,4 @@ char *__crypt_sha512(const char *key, co
+
+@@ -369,3 +380,4 @@ char *__crypt_sha512(const char *key, co
  		return "*";
  	return p;
  }
 +#endif
 --- a/src/crypt/crypt_blowfish.c
 +++ b/src/crypt/crypt_blowfish.c
-@@ -50,6 +50,7 @@
+@@ -50,6 +50,17 @@
  #include <string.h>
  #include <stdint.h>
 
-+#if 0
++#ifdef CRYPT_SIZE_HACK
++#include <errno.h>
++
++char *__crypt_blowfish(const char *key, const char *setting, char *output)
++{
++	errno = ENOSYS;
++	return NULL;
++}
++
++#else
++
  typedef uint32_t BF_word;
  typedef int32_t BF_word_signed;
 
-@@ -796,3 +797,4 @@ char *__crypt_blowfish(const char *key,
+@@ -796,3 +807,4 @@ char *__crypt_blowfish(const char *key,
 
  	return "*";
  }
 +#endif
 --- a/src/crypt/crypt_sha256.c
 +++ b/src/crypt/crypt_sha256.c
-@@ -13,6 +13,7 @@
+@@ -13,6 +13,17 @@
  #include <string.h>
  #include <stdint.h>
 
-+#if 0
++#ifdef CRYPT_SIZE_HACK
++#include <errno.h>
++
++char *__crypt_sha256(const char *key, const char *setting, char *output)
++{
++	errno = ENOSYS;
++	return NULL;
++}
++
++#else
++
  /* public domain sha256 implementation based on fips180-3 */
 
  struct sha256 {
-@@ -320,3 +321,4 @@ char *__crypt_sha256(const char *key, co
+@@ -320,3 +331,4 @@ char *__crypt_sha256(const char *key, co
  		return "*";
  	return p;
  }