Accept field names which contain Japanese characters

cobj/codegen.c

@@ -233,11 +233,28 @@ static void get_java_identifier_helper(struct cb_field *f, char *buf) {
 }
 
 static void strcpy_identifier_cobol_to_java(char *buf, const char *identifier) {
-  for (; *identifier != '\0'; ++identifier, ++buf) {
-    if (*identifier == '-') {
-      *buf = '_';
-    } else {
-      *buf = *identifier;
+  int all_ascii = 1;
+  unsigned char *p = (unsigned char *)identifier;
+  for (; *p; ++p) {
+    unsigned char c = *p;
+    if (c < 0x0A || 0x80 <= c) {
+      all_ascii = 0;
+      break;
+    }
+  }
+
+  if (all_ascii) {
+    for (; *identifier; ++identifier, ++buf) {
+      if (*identifier == '-') {
+        *buf = '_';
+      } else {
+        *buf = *identifier;
+      }
+    }
+  } else {
+    for (; *identifier; ++identifier) {
+      sprintf(buf, "%02x", (unsigned char)*identifier);
+      buf += 2;
     }
   }
   *buf = '\0';
@@ -432,11 +449,17 @@ static void joutput_indent(const char *str) {
   }
 }
 
+enum cb_string_category {
+  CB_STRING_CATEGORY_ALL_ASCII,
+  CB_STRING_CATEGORY_ALL_SJIS,
+  CB_STRING_CATEGORY_CONTAINS_NON_SJIS,
+};
+
 struct string_literal_cache {
   unsigned char *string_value;
   int size;
   int param_wrap_string_flag;
-  int printable;
+  enum cb_string_category category;
   char *var_name;
   struct string_literal_cache *next;
 };
@@ -461,26 +484,29 @@ static void free_string_literal_list() {
   }
 }
 
-static int is_string_printable(const unsigned char *s, int size) {
+static enum cb_string_category get_string_category(const unsigned char *s,
+                                                   int size) {
   int i;
+  enum cb_string_category category = CB_STRING_CATEGORY_ALL_ASCII;
   for (i = 0; i < size;) {
     int c = s[i];
     if (0x20 <= c && c <= 0x7e) {
       i += 1;
     } else if ((0x81 <= c && c <= 0x9f) || (0xe0 <= c && c <= 0xef)) {
       i += 2;
+      category = CB_STRING_CATEGORY_ALL_SJIS;
     } else {
-      return 0;
+      return CB_STRING_CATEGORY_CONTAINS_NON_SJIS;
     }
   }
-  return 1;
+  return category;
 }
 
 static void joutput_string_write(const unsigned char *s, int size,
-                                 int printable) {
+                                 enum cb_string_category category) {
   int i;
 
-  if (printable) {
+  if (category == CB_STRING_CATEGORY_ALL_ASCII) {
     if (param_wrap_string_flag) {
       joutput("new CobolDataStorage(");
     } else {
@@ -529,7 +555,7 @@ static void joutput_string(const unsigned char *s, int size) {
 
   new_literal_cache->size = size;
   new_literal_cache->param_wrap_string_flag = param_wrap_string_flag;
-  new_literal_cache->printable = is_string_printable(s, size);
+  new_literal_cache->category = get_string_category(s, size);
 
   new_literal_cache->string_value = malloc(size);
   memcpy(new_literal_cache->string_value, s, size);
@@ -580,7 +606,7 @@ static void joutput_all_string_literals() {
     joutput_prefix();
     joutput("public static final %s %s = ", data_type, l->var_name);
     param_wrap_string_flag = l->param_wrap_string_flag;
-    joutput_string_write(l->string_value, l->size, l->printable);
+    joutput_string_write(l->string_value, l->size, l->category);
     joutput(";\n");
     l = l->next;
   }
@@ -5728,8 +5754,8 @@ static void joutput_label_variable_name(char *s, int key,
                                         struct cb_label *section) {
   joutput(CB_PREFIX_LABEL);
   if (s) {
-    const char *c;
     if (section && section->name) {
+      const char *c;
       for (c = (const char *)section->name; *c; ++c) {
         if (*c == ' ') {
           joutput("_");
@@ -5741,15 +5767,26 @@ static void joutput_label_variable_name(char *s, int key,
       }
       joutput("__");
     }
-    for (c = s; *c; ++c) {
-      if (*c == ' ') {
-        joutput("_");
-      } else if (*c == '-') {
-        joutput("_");
+    char buf[COB_SMALL_BUFF];
+    strcpy_identifier_cobol_to_java(buf, s);
+    char *p = buf;
+    while (*p) {
+      if (*p < 0x80) {
+        if (*p == '-') {
+          *p = '_';
+        } else if (*p == ' ') {
+          *p = '_';
+        }
+        p++;
       } else {
-        joutput("%c", *c);
+        if (*(p + 1) == '\0') {
+          break;
+        } else {
+          p = p + 2;
+        }
       }
     }
+    joutput("%s", buf);
   } else {
     joutput("anonymous__%d", key);
   }

cobj/tree.c

@@ -2714,16 +2714,22 @@ cb_tree cb_build_intrinsic(cb_tree name, cb_tree args, cb_tree refmod) {
 
 char *cb_get_hexword(char *name) {
   unsigned char *p;
-  int non_ascii = 0;
+  int non_sjis = 0;
   char *rt = NULL;
 
-  for (p = (unsigned char *)name; *p; p++) {
-    if (0x80 & *p) {
-      non_ascii = 1;
+  for (p = (unsigned char *)name; *p;) {
+    unsigned char c = *p;
+    if (c < 0x80) {
+      p++;
+    } else if ((0x81 <= c && c <= 0x9F) || (0xE0 <= c && c <= 0xEF)) {
+      p += 2;
+    } else {
+      non_sjis = 1;
       break;
     }
   }
-  if (!non_ascii) {
+
+  if (!non_sjis) {
     rt = strdup(name);
   } else {
     rt = cobc_malloc(strlen(name) * 2 + 7);

libcobj/app/src/main/java/jp/osscons/opensourcecobol/libcobj/common/CobolUtil.java

@@ -296,7 +296,9 @@ public static void runtimeError(String s) {
     if (sourceFile != null) {
       System.err.print(String.format("%s:%d: ", sourceFile, sourceLine));
     }
-    System.err.println("libcobj: " + s);
+    byte[] messageBytes = ("libcobj: " + s).getBytes(AbstractCobolField.charSetSJIS);
+    System.err.write(messageBytes, 0, messageBytes.length);
+    System.err.println();
     System.err.flush();
   }
 

libcobj/app/src/main/java/jp/osscons/opensourcecobol/libcobj/data/AbstractCobolField.java

@@ -40,7 +40,7 @@ public abstract class AbstractCobolField {
 
   static int lastsize = 0;
   static CobolDataStorage lastdata = null;
-  static Charset CharSetSJIS = Charset.forName("SHIFT-JIS");
+  public static Charset charSetSJIS = Charset.forName("SHIFT-JIS");
 
   static final int[] cobExp10 = {
     1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000
@@ -586,7 +586,7 @@ protected void moveFromAll(AbstractCobolField src) {
   public void moveFrom(String s) {
     // The maximum number of digits of int type in decimal is 10
 
-    byte[] bytes = s.getBytes(CharSetSJIS);
+    byte[] bytes = s.getBytes(charSetSJIS);
 
     CobolDataStorage storage = new CobolDataStorage(bytes.length);
     storage.memcpy(bytes);
@@ -682,8 +682,8 @@ public void moveFrom(double number) {
   public void checkNumeric(byte[] s) throws CobolStopRunException {
     if (!this.isNumeric()) {
       byte[] buff = this.getDataStorage().getByteArrayRef(0, this.getSize());
-      String name = new String(s, CharSetSJIS);
-      String content = new String(buff, CharSetSJIS);
+      String name = new String(s, charSetSJIS);
+      String content = new String(buff, charSetSJIS);
       CobolUtil.runtimeError("'" + name + "' not numeric: '" + content + "'");
       CobolStopRunException.stopRunAndThrow(1);
     }

tests/i18n_sjis.src/program-id.at

@@ -19,7 +19,7 @@ AT_CLEANUP
 
 
 AT_SETUP([PROGRAM-ID NATIONAL C89 warning])
-
+AT_CHECK([${SKIP_TEST}])
 AT_DATA([test.conf], [
 include "default.conf"
 c89-identifier-length-check: yes

tests/i18n_sjis.src/user-defined-word.at

@@ -178,7 +178,6 @@ AT_CHECK([java prog], [0], [OK])
 AT_CLEANUP
 
 AT_SETUP([Nihongo field name in numeric test msg.])
-AT_CHECK([${SKIP_TEST}])
 
 AT_DATA([prog.cob], [
        IDENTIFICATION    DIVISION.
@@ -270,7 +269,6 @@ AT_CHECK([java prog], [1], [],
 AT_CLEANUP
 
 AT_SETUP([Nihongo field name in length of ref_mod test msg.])
-AT_CHECK([${SKIP_TEST}])
 
 AT_DATA([prog.cob], [
        IDENTIFICATION    DIVISION.
@@ -293,7 +291,6 @@ AT_CHECK([java prog], [1], [],
 AT_CLEANUP
 
 AT_SETUP([Nihongo field name in offset of ref_mod test msg.])
-AT_CHECK([${SKIP_TEST}])
 
 AT_DATA([prog.cob], [
        IDENTIFICATION    DIVISION.
@@ -391,7 +388,6 @@ AT_CHECK([java prog], [1], [],
 AT_CLEANUP
 
 AT_SETUP([Nihongo field name in undefined error msg.])
-AT_CHECK([${SKIP_TEST}])
 
 AT_DATA([prog.cob], [
        IDENTIFICATION   DIVISION.
@@ -431,7 +427,6 @@ prog.cob:19: Error: 'NO
 AT_CLEANUP
 
 AT_SETUP([Nihongo field name in ambiguous error msg.])
-AT_CHECK([${SKIP_TEST}])
 
 AT_DATA([prog.cob], [
        IDENTIFICATION   DIVISION.
@@ -468,7 +463,6 @@ prog.cob:12: Error: 'B
 AT_CLEANUP
 
 AT_SETUP([Nihongo label name in ambiguous error msg.])
-AT_CHECK([${SKIP_TEST}])
 
 AT_DATA([prog.cob], [
        IDENTIFICATION   DIVISION.