correct handling of double-byte in C and C++ compilers

open-watcom · Dec 15, 2023 · 934654b · 934654b
1 parent d2d37a6
commit 934654b
Show file tree

Hide file tree

Showing 14 changed files with 58 additions and 39 deletions.
diff --git a/bld/cc/c/cmodel.c b/bld/cc/c/cmodel.c
@@ -335,7 +335,6 @@ void InitModInfo( void )
     CompFlags.emit_library_names                = true;
     CompFlags.emit_dependencies                 = true;
     CompFlags.emit_targimp_symbols              = true;
-    CompFlags.use_unicode                       = true;
     CompFlags.no_debug_type_names               = false;
     CompFlags.auto_agg_inits                    = false;
     CompFlags.no_check_inits                    = false;

diff --git a/bld/cc/c/coptions.c b/bld/cc/c/coptions.c
@@ -215,29 +215,27 @@ static void SetCharacterEncoding( void )
         LoadUnicodeTable( unicode_CP );
         break;
     case ENC_ZK0U:
-        CompFlags.use_unicode = false;
-        SetDBChar( 0 );                     /* set double-byte char type */
         CompFlags.jis_to_unicode = true;
-        break;
+        /* fall through */
     case ENC_ZK:
     case ENC_ZK0:
-        CompFlags.use_unicode = false;
+        CompFlags.use_double_byte = true;
         SetDBChar( 0 );                     /* set double-byte char type */
         break;
     case ENC_ZK1:
-        CompFlags.use_unicode = false;
+        CompFlags.use_double_byte = true;
         SetDBChar( 1 );                     /* set double-byte char type */
         break;
     case ENC_ZK2:
-        CompFlags.use_unicode = false;
+        CompFlags.use_double_byte = true;
         SetDBChar( 2 );                     /* set double-byte char type */
         break;
     case ENC_ZK3:
-        CompFlags.use_unicode = false;
+        CompFlags.use_double_byte = true;
         SetDBChar( 3 );                     /* set double-byte char type */
         break;
     case ENC_ZKL:
-        CompFlags.use_unicode = false;
+        CompFlags.use_double_byte = true;
         SetDBChar( -1 );                    /* set double-byte char type to default */
         break;
     }
@@ -791,7 +789,7 @@ static void MacroDefs( void )
     if( CompFlags.zc_switch_used ) {
         DefSwitchMacro( "ZC" );
     }
-    if( !CompFlags.use_unicode ) {
+    if( CompFlags.use_double_byte ) {
         DefSwitchMacro( "ZK" );
     }
 #if _INTEL_CPU

diff --git a/bld/cc/c/cscan.c b/bld/cc/c/cscan.c
@@ -1416,6 +1416,19 @@ int ESCChar( int c, escinp_fn ifn, msg_codes *perr_msg, escout_fn ofn )
     return( n );
 }
 
+int EncodeWchar( int c )
+/**********************/
+{
+    if( CompFlags.use_double_byte ) {
+        if( CompFlags.jis_to_unicode ) {
+            c = JIS2Unicode( c );
+        }
+    } else {
+        c = UniCode[c];
+    }
+    return( c );
+}
+
 static TOKEN doScanCharConst( DATA_TYPE char_type )
 /**************************************************
  * TokenLen is alway lower then BUF_SIZE that
@@ -1508,11 +1521,7 @@ static TOKEN doScanCharConst( DATA_TYPE char_type )
                     Buffer[TokenLen++] = CurrChar;
                     NextChar();
                 } else if( char_type == TYP_WCHAR ) {
-                    if( CompFlags.use_unicode ) {
-                        c = UniCode[c];
-                    } else if( CompFlags.jis_to_unicode ) {
-                        c = JIS2Unicode( c );
-                    }
+                    c = EncodeWchar( c );
                     ++i;
                     value = (value << 8) + ((c & 0xFF00) >> 8);
                     c &= 0x00FF;

diff --git a/bld/cc/c/cstring.c b/bld/cc/c/cstring.c
@@ -172,11 +172,7 @@ static target_size RemoveEscapes( char *buf, const char *inbuf, target_size ilen
                     c = read_inp();
                 }
             } else if( CompFlags.wide_char_string ) {
-                if( CompFlags.use_unicode ) {
-                    c = UniCode[c];
-                } else if( CompFlags.jis_to_unicode ) {
-                    c = JIS2Unicode( c );
-                }
+                c = EncodeWchar( c );
                 WRITE_BYTE( c );
                 c = c >> 8;
 #if _CPU == 370

diff --git a/bld/cc/h/ctypes.h b/bld/cc/h/ctypes.h
@@ -616,7 +616,7 @@ typedef struct comp_flags {
     boolbit make_enums_an_int                   : 1;    /* force all enums to be int */
     boolbit original_enum_setting               : 1;    /* reset value if pragma used*/
     boolbit zc_switch_used                      : 1;    /* -zc switch specified   */
-    boolbit use_unicode                         : 1;    /* use unicode for L"abc" */
+    boolbit use_double_byte                     : 1;    /* use double-byte encoding for L"abc" */
     boolbit op_switch_used                      : 1;    /* -op force floats to mem */
     boolbit no_debug_type_names                 : 1;    /* -d2~ switch specified  */
 

diff --git a/bld/cc/h/scan.h b/bld/cc/h/scan.h
@@ -2,7 +2,7 @@
 *
 *                            Open Watcom Project
 *
-* Copyright (c) 2002-2020 The Open Watcom Contributors. All Rights Reserved.
+* Copyright (c) 2002-2023 The Open Watcom Contributors. All Rights Reserved.
 *    Portions Copyright (c) 1983-2002 Sybase, Inc. All Rights Reserved.
 *
 *  ========================================================================
@@ -74,4 +74,6 @@ typedef enum charset_flags {
 #define HEX_BASE        'a'
 #endif
 
-extern charset_flags CharSet[LCHR_MAX];
+extern charset_flags    CharSet[LCHR_MAX];
+
+extern int              EncodeWchar( int c );
diff --git a/bld/plusplus/c/cmdlnany.c b/bld/plusplus/c/cmdlnany.c
@@ -627,22 +627,25 @@ static void analyseAnyTargetOptions( OPT_STORAGE *data )
     }
     switch( data->char_set ) {
     case OPT_ENUM_char_set_zku:
-        CompFlags.use_unicode = true;
         loadUnicodeTable( data->zku_value );
         break;
     case OPT_ENUM_char_set_zk0u:
         CompFlags.jis_to_unicode = true;
         /* fall through */
     case OPT_ENUM_char_set_zk0:
+        CompFlags.use_double_byte = true;
         SetDBChar( 0 );
         break;
     case OPT_ENUM_char_set_zk1:
+        CompFlags.use_double_byte = true;
         SetDBChar( 1 );
         break;
     case OPT_ENUM_char_set_zk2:
+        CompFlags.use_double_byte = true;
         SetDBChar( 2 );
         break;
     case OPT_ENUM_char_set_zkl:
+        CompFlags.use_double_byte = true;
         SetDBChar( -1 );
         break;
     }

diff --git a/bld/plusplus/c/cmdlnrsc.c b/bld/plusplus/c/cmdlnrsc.c
@@ -225,7 +225,7 @@ static void macroDefs( void )
     if( CompFlags.emit_names ) {
         DefSwitchMacro( "EN" );
     }
-    if( CompFlags.use_unicode ) {
+    if( CompFlags.use_double_byte ) {
         DefSwitchMacro( "ZK" );
     }
     if( CompFlags.bm_switch_used ) {

diff --git a/bld/plusplus/c/cmdlnx86.c b/bld/plusplus/c/cmdlnx86.c
@@ -928,7 +928,7 @@ static void macroDefs( void )
     if( CompFlags.zc_switch_used ) {
         DefSwitchMacro( "ZC" );
     }
-    if( CompFlags.use_unicode ) {
+    if( CompFlags.use_double_byte ) {
         DefSwitchMacro( "ZK" );
     }
     if( CompFlags.sg_switch_used ) {

diff --git a/bld/plusplus/c/cscan.c b/bld/plusplus/c/cscan.c
@@ -354,6 +354,19 @@ static int doESCChar( int c, bool expanding, type_id char_type )
     return( n );
 }
 
+int EncodeWchar( int c )
+/**********************/
+{
+    if( CompFlags.use_double_byte ) {
+        if( CompFlags.jis_to_unicode ) {
+            c = JIS2Unicode( c );
+        }
+    } else {
+        c = UniCode[c];
+    }
+    return( c );
+}
+
 static TOKEN doScanCharConst( type_id char_type, bool expanding )
 {
     int c;
@@ -402,11 +415,7 @@ static TOKEN doScanCharConst( type_id char_type, bool expanding )
                     c &= 0x00FF;
                     flag.double_byte_char = true;
                 } else if( char_type == TYP_WCHAR ) {
-                    if( CompFlags.use_unicode ) {
-                        c = UniCode[c];
-                    } else if( CompFlags.jis_to_unicode ) {
-                        c = JIS2Unicode( c );
-                    }
+                    c = EncodeWchar( c );
                     ++i;
                     value = (value << 8) + ((c & 0xFF00) >> 8);
                     c &= 0x00FF;

diff --git a/bld/plusplus/c/stringl.c b/bld/plusplus/c/stringl.c
@@ -2,7 +2,7 @@
 *
 *                            Open Watcom Project
 *
-* Copyright (c) 2002-2020 The Open Watcom Contributors. All Rights Reserved.
+* Copyright (c) 2002-2023 The Open Watcom Contributors. All Rights Reserved.
 *    Portions Copyright (c) 1983-2002 Sybase, Inc. All Rights Reserved.
 *
 *  ========================================================================
@@ -217,11 +217,7 @@ static size_t compressLiteral( char *tgt, const char *s, size_t len, bool wide )
                 }
                 --len;
             } else if( wide ) {
-                if( CompFlags.use_unicode ) {
-                    chr = UniCode[chr];
-                } else if( CompFlags.jis_to_unicode ) {
-                    chr = JIS2Unicode( chr );
-                }
+                chr = EncodeWchar( chr );
                 STORE_WCHAR( tgt, chr, new_len );
             } else {
 //              _ASCIIOUT( chr );

diff --git a/bld/plusplus/h/compflag.h b/bld/plusplus/h/compflag.h
@@ -126,7 +126,7 @@ typedef struct comp_flags {
     boolbit     no_debug_type_names             : 1;
     boolbit     emit_names                      : 1;
     boolbit     warnings_cause_bad_exit         : 1;
-    boolbit     use_unicode                     : 1;
+    boolbit     use_double_byte                 : 1;
     boolbit     unique_functions                : 1;
     boolbit     jis_to_unicode                  : 1;
     boolbit     emit_dependencies               : 1;

diff --git a/bld/plusplus/h/scan.h b/bld/plusplus/h/scan.h
@@ -2,7 +2,7 @@
 *
 *                            Open Watcom Project
 *
-* Copyright (c) 2002-2020 The Open Watcom Contributors. All Rights Reserved.
+* Copyright (c) 2002-2023 The Open Watcom Contributors. All Rights Reserved.
 *    Portions Copyright (c) 1983-2002 Sybase, Inc. All Rights Reserved.
 *
 *  ========================================================================
@@ -84,5 +84,6 @@ typedef enum charset_flags {
 extern charset_flags        CharSet[LCHR_MAX];      // character characterizations
 extern unsigned             JIS2Unicode( unsigned );
 extern const unsigned char  TokValue[];
+extern int                  EncodeWchar( int c );
 
 #endif // __SCAN_H__
diff --git a/docs/doc/cg/cpopts.gml b/docs/doc/cg/cpopts.gml
@@ -5589,6 +5589,12 @@ execution time.
 .np
 This group of options deals with compile-time aspects of character
 sets used in the source code.
+.np
+By default, the compiler uses output Unicode encoding and source code page 437
+(US-ASCII) to output wide characters. 
+.if &e'&$SWzku ne 0 .do begin
+This setting is equivalent to using the -zku=437 option.
+.do end
 .*
 :OPTLIST.
 .*