Make the IS_*() macros work correctly for 8-bit ASCII characters

Fixes openssl#5778, openssl#5840

The various IS_*() macros did not work correctly for 8-bit ASCII
characters with the high bit set, because the CVT(a) preprocessor
macro and'ed the given ASCII value with 0x7F, effectively folding
the high value range 128-255 over the low value range 0-127.
As a consequence, some of the IS_*() erroneously returned TRUE.

This commit fixes the issue by mapping CVT(a) to 127 for all values
a >= 127. This works because the lookup tables CONF_type_default
and CONF_type_win32 have all bits cleared at entry 127, whence
IS_*(a) returns FALSE for all values outside the range 0-127.

The IS_*() macros were also changed to return TRUE or FALSE (1 or 0)
instead of a nonzero or zero value.

Note that with this change, the macro IS_*(c,a) evaluates the 'a'
argument twice, unless CHARSET_EBCDIC is defined. This prohibits the
use of arguments with side effects like IS_*(c, *p++).

crypto/conf/conf_def.h

@@ -28,21 +28,26 @@
 #define KEYTYPES(c)       ((const unsigned short *)((c)->meth_data))
 
 #ifndef CHARSET_EBCDIC
-# define CVT(a) ((a) & 0x7F)
+# define CVT(a) ((unsigned char)(a) <= 127 ? (a) : 127)
 #else
-# define CVT(a) os_toascci[(a) & 0x7F]
+# define CVT(a) os_toascii[(unsigned char)(a)]
 #endif
 
-#define IS_COMMENT(c,a)     (KEYTYPES(c)[CVT(a)] & CONF_COMMENT)
-#define IS_FCOMMENT(c,a)    (KEYTYPES(c)[CVT(a)] & CONF_FCOMMENT)
-#define IS_EOF(c,a)         (KEYTYPES(c)[CVT(a)] & CONF_EOF)
-#define IS_ESC(c,a)         (KEYTYPES(c)[CVT(a)] & CONF_ESC)
-#define IS_NUMBER(c,a)      (KEYTYPES(c)[CVT(a)] & CONF_NUMBER)
-#define IS_WS(c,a)          (KEYTYPES(c)[CVT(a)] & CONF_WS)
-#define IS_ALNUM(c,a)       (KEYTYPES(c)[CVT(a)] & CONF_ALNUM)
-#define IS_ALNUM_PUNCT(c,a) (KEYTYPES(c)[CVT(a)] & CONF_ALNUM_PUNCT)
-#define IS_QUOTE(c,a)       (KEYTYPES(c)[CVT(a)] & CONF_QUOTE)
-#define IS_DQUOTE(c,a)      (KEYTYPES(c)[CVT(a)] & CONF_DQUOTE)
+/*
+ * Attention: because the macro argument 'a' is evaluated twice in CVT(a),
+ * it is not allowed pass 'a' arguments with side effects to IS_*(c,a)
+ * like for example IS_*(c, *p++).
+ */
+#define IS_COMMENT(c,a)     ((KEYTYPES(c)[CVT(a)] & CONF_COMMENT) ? 1 : 0)
+#define IS_FCOMMENT(c,a)    ((KEYTYPES(c)[CVT(a)] & CONF_FCOMMENT) ? 1 : 0)
+#define IS_EOF(c,a)         ((KEYTYPES(c)[CVT(a)] & CONF_EOF) ? 1 : 0)
+#define IS_ESC(c,a)         ((KEYTYPES(c)[CVT(a)] & CONF_ESC) ? 1 : 0)
+#define IS_NUMBER(c,a)      ((KEYTYPES(c)[CVT(a)] & CONF_NUMBER) ? 1 : 0)
+#define IS_WS(c,a)          ((KEYTYPES(c)[CVT(a)] & CONF_WS) ? 1 : 0)
+#define IS_ALNUM(c,a)       ((KEYTYPES(c)[CVT(a)] & CONF_ALNUM) ? 1 : 0)
+#define IS_ALNUM_PUNCT(c,a) ((KEYTYPES(c)[CVT(a)] & CONF_ALNUM_PUNCT) ? 1 : 0)
+#define IS_QUOTE(c,a)       ((KEYTYPES(c)[CVT(a)] & CONF_QUOTE) ? 1 : 0)
+#define IS_DQUOTE(c,a)      ((KEYTYPES(c)[CVT(a)] & CONF_DQUOTE) ? 1 : 0)
 
 static const unsigned short CONF_type_default[128] = {
     0x0008, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,

crypto/conf/keysets.pl

@@ -88,21 +88,26 @@
 #define KEYTYPES(c)       ((const unsigned short *)((c)->meth_data))
 
 #ifndef CHARSET_EBCDIC
-# define CVT(a) ((a) & 0x7F)
+# define CVT(a) ((unsigned char)(a) <= 127 ? (a) : 127)
 #else
-# define CVT(a) os_toascci[(a) & 0x7F]
+# define CVT(a) os_toascii[(unsigned char)(a)]
 #endif
 
-#define IS_COMMENT(c,a)     (KEYTYPES(c)[CVT(a)] & CONF_COMMENT)
-#define IS_FCOMMENT(c,a)    (KEYTYPES(c)[CVT(a)] & CONF_FCOMMENT)
-#define IS_EOF(c,a)         (KEYTYPES(c)[CVT(a)] & CONF_EOF)
-#define IS_ESC(c,a)         (KEYTYPES(c)[CVT(a)] & CONF_ESC)
-#define IS_NUMBER(c,a)      (KEYTYPES(c)[CVT(a)] & CONF_NUMBER)
-#define IS_WS(c,a)          (KEYTYPES(c)[CVT(a)] & CONF_WS)
-#define IS_ALNUM(c,a)       (KEYTYPES(c)[CVT(a)] & CONF_ALNUM)
-#define IS_ALNUM_PUNCT(c,a) (KEYTYPES(c)[CVT(a)] & CONF_ALNUM_PUNCT)
-#define IS_QUOTE(c,a)       (KEYTYPES(c)[CVT(a)] & CONF_QUOTE)
-#define IS_DQUOTE(c,a)      (KEYTYPES(c)[CVT(a)] & CONF_DQUOTE)
+/*
+ * Attention: because the macro argument 'a' is evaluated twice in CVT(a),
+ * it is not allowed pass 'a' arguments with side effects to IS_*(c,a)
+ * like for example IS_*(c, *p++).
+ */
+#define IS_COMMENT(c,a)     ((KEYTYPES(c)[CVT(a)] & CONF_COMMENT) ? 1 : 0)
+#define IS_FCOMMENT(c,a)    ((KEYTYPES(c)[CVT(a)] & CONF_FCOMMENT) ? 1 : 0)
+#define IS_EOF(c,a)         ((KEYTYPES(c)[CVT(a)] & CONF_EOF) ? 1 : 0)
+#define IS_ESC(c,a)         ((KEYTYPES(c)[CVT(a)] & CONF_ESC) ? 1 : 0)
+#define IS_NUMBER(c,a)      ((KEYTYPES(c)[CVT(a)] & CONF_NUMBER) ? 1 : 0)
+#define IS_WS(c,a)          ((KEYTYPES(c)[CVT(a)] & CONF_WS) ? 1 : 0)
+#define IS_ALNUM(c,a)       ((KEYTYPES(c)[CVT(a)] & CONF_ALNUM) ? 1 : 0)
+#define IS_ALNUM_PUNCT(c,a) ((KEYTYPES(c)[CVT(a)] & CONF_ALNUM_PUNCT) ? 1 : 0)
+#define IS_QUOTE(c,a)       ((KEYTYPES(c)[CVT(a)] & CONF_QUOTE) ? 1 : 0)
+#define IS_DQUOTE(c,a)      ((KEYTYPES(c)[CVT(a)] & CONF_DQUOTE) ? 1 : 0)
 
 EOF