Unicode support for the Windows runtime (#1200)

* Add support code

* Explicitly reference ANSI Windows APIs

* Adapt Sys.is_directory

* Adapt ocamlrun

* Add Changes entry

* Add testsuite

* Adapt Unix.open_process{_in,_out,_full,}, Unix.create_process{_env,}

* Adapt headernt.c

* Adapt Pervasives.open_{in,out}, Filename.temp_file, etc.

* Adapt Sys.file_exists

* Adapt Sys.remove

* Adapt Sys.chdir

* Adapt Sys.getcwd

* Adapt Sys.getenv

* Adapt Sys.command

* Adapt Sys.readdir

* Adapt CPLUGINS

* Remove use of FormatMessageA, CreateFileA

* Adapt Unix.mkdir

* Adapt Unix.openfile

* Adapt Unix.readlink

* Adapt Unix.rename

* Adapt Unix.{LargeFile,}.{l,}stat

* Adapt Unix.system

* Adapt Unix.{open,read}dir

* Adapt Unix.link

* Adapt Unix.symlink

* Adapt Unix.getcwd

* Adapt Unix.rmdir

* Adapt Unix.utimes

* Adapt Unix.unlink

* Adapt Unix.chdir

* Adapt Unix.chmod

* Adapt Unix.{execv,execve,execvp,execvpe}

* Compile with -DUNICODE -D_UNICODE under Windows

* Add configure-time switch, Config.windows_unicode

* Adapt Unix.putenv

* Re-implement Unix.environment using GetEnvironmentStrings()

* Use Unicode-aware flexdll

* Adapt Unix.environment

* AppVeyor: bootstrap flexdll

* Adapt tests/embedded/cmmain.c

* Adapt tests/lib-dynlink-csharp/entry.c

* Remove exec tests

* Fixup

* Pass -municode to MinGW compiler

* Try to fix tests/embedded

* Adapt Sys.rename

* Correct Changes entry

* Makefile.several: use $(O) and $(NATIVECODE_ONLY)

* Display => skipped correctly for tests/win-unicode

* Add missing casts to execv* calls

It's not clear why these aren't necessary for with char, but they are
necessary with wchar_t on GCC (but not MSVC).

* Missing header in systhreads (Win32 only)

* Revert "Pass -municode to MinGW compiler"

This reverts commit a4ce7fb.

* Revert "Try to fix tests/embedded"

This reverts commit 5197d89.

* Revert "Remove exec tests"

This reverts commit 306ccef.

* Don't pass $(LDFLAGS) when build ocamlc.opt

It's already included via CC anyway, and it causes Unicode problems for
Winodws (because the linker options need to be prefixed "-link" to go via
flexlink).

* Use wmain on Windows for ocamlrun

* Build Unicode applications on Windows

* Use wmain in headernt.c

* Minor correction to win-unicode Makefile

* Switch submodule to FlexDLL 0.36

* Build ocamlyacc as an ANSI application

* Revert "Fixup"

This reverts commit 500bd6b.

* Fix casts for execvp/execve

* Remove tabs from test code

* Fix Changes entry

* shell32.lib is no longer necessary

* Free allocated string

* Changes: signal breaking change

* Disable exec_tests

* Protect with CAML_INTERNALS

.gitignore

@@ -351,6 +351,8 @@ _ocamltest
 /testsuite/tests/warnings/w55.opt.opt_result
 /testsuite/tests/warnings/w58.opt.opt_result
 
+/testsuite/tests/win-unicode/symlink_tests.precheck
+
 /testsuite/tools/expect_test
 
 /tools/ocamldep

Changes

@@ -544,6 +544,10 @@ Working version
 
 ### Runtime system:
 
+* MPR#3771, GPR#153, GPR#1200: Unicode support for the Windows runtime.
+  (ygrek, Clement Franchini, Nicolas Ojeda Bar, review by Alain Frisch, David
+  Allsopp)
+
 - GPR#1070, GPR#1295: enable gcc typechecking for caml_alloc_sprintf,
   caml_gc_message. Make caml_gc_message a variadic function. Fix many
   caml_gc_message format strings.

Makefile

@@ -70,12 +70,6 @@ else
 OCAML_NATDYNLINKOPTS = -ccopt "$(NATDYNLINKOPTS)"
 endif
 
-ifeq "$(strip $(LDFLAGS))" ""
-OCAML_LDFLAGS=
-else
-OCAML_LDFLAGS = -ccopt "$(LDFLAGS)"
-endif
-
 YACCFLAGS=-v --strict
 CAMLLEX=$(CAMLRUN) boot/ocamllex
 CAMLDEP=$(CAMLRUN) tools/ocamldep
@@ -336,7 +330,8 @@ utils/config.ml: utils/config.mlp config/Makefile
 	    -e 's|%%LIBUNWIND_AVAILABLE%%|$(LIBUNWIND_AVAILABLE)|' \
 	    -e 's|%%LIBUNWIND_LINK_FLAGS%%|$(LIBUNWIND_LINK_FLAGS)|' \
 	    -e 's|%%MKDLL%%|$(subst \,\\,$(MKDLL))|' \
-	    -e 's|%%MKEXE%%|$(subst \,\\,$(MKEXE))|' \
+	    -e 's|%%MKEXE%%|$(subst ",\\",$(subst \,\\,$(MKEXE)))|' \
+	    -e 's|%%FLEXLINK_LDFLAGS%%|$(subst ",\\",$(subst \,\\,$(if $(LDFLAGS), -link "$(LDFLAGS)")))|' \
 	    -e 's|%%MKMAINDLL%%|$(subst \,\\,$(MKMAINDLL))|' \
 	    -e 's|%%MODEL%%|$(MODEL)|' \
 	    -e 's|%%NATIVECCLIBS%%|$(NATIVECCLIBS)|' \
@@ -349,6 +344,7 @@ utils/config.ml: utils/config.mlp config/Makefile
 	    -e 's|%%PROFINFO_WIDTH%%|$(PROFINFO_WIDTH)|' \
 	    -e 's|%%RANLIBCMD%%|$(RANLIBCMD)|' \
 	    -e 's|%%SAFE_STRING%%|$(SAFE_STRING)|' \
+	    -e 's|%%WINDOWS_UNICODE%%|$(WINDOWS_UNICODE)|' \
 	    -e 's|%%SYSTEM%%|$(SYSTEM)|' \
 	    -e 's|%%SYSTHREAD_SUPPORT%%|$(SYSTHREAD_SUPPORT)|' \
 	    -e 's|%%TARGET%%|$(TARGET)|' \
@@ -873,8 +869,7 @@ partialclean::
 
 ocamlc.opt: compilerlibs/ocamlcommon.cmxa compilerlibs/ocamlbytecomp.cmxa \
             $(BYTESTART:.cmo=.cmx)
-	$(CAMLOPT) $(LINKFLAGS) $(OCAML_LDFLAGS) -o $@ \
-	  $^ -cclib "$(BYTECCLIBS)"
+	$(CAMLOPT) $(LINKFLAGS) -o $@ $^ -cclib "$(BYTECCLIBS)"
 
 partialclean::
 	rm -f ocamlc.opt

appveyor_build.sh

@@ -32,9 +32,9 @@ PREFIX="C:/Program Files/OCaml"
 wmic cpu get name
 
 if [[ $1 = "msvc32-only" ]] ; then
-  cd $APPVEYOR_BUILD_FOLDER/flexdll-0.35
-  make MSVC_DETECT=0 CHAINS=msvc MSVC_FLAGS="-nologo -MD -D_CRT_NO_DEPRECATE -GS- -WX" support
-  cp flexdll*_msvc.obj "$PREFIX/bin/flexdll"
+#  cd $APPVEYOR_BUILD_FOLDER/flexdll-0.35
+#  make MSVC_DETECT=0 CHAINS=msvc MSVC_FLAGS="-nologo -MD -D_CRT_NO_DEPRECATE -GS- -WX" support
+#  cp flexdll*_msvc.obj "$PREFIX/bin/flexdll"
 
   cd $APPVEYOR_BUILD_FOLDER/../build-msvc32
   cp config/m-nt.h byterun/caml/m.h
@@ -46,6 +46,8 @@ if [[ $1 = "msvc32-only" ]] ; then
   echo "Edit config/Makefile to set PREFIX=$PREFIX"
   sed -e "s|PREFIX=.*|PREFIX=$PREFIX|" -e "/^ *CFLAGS *=/s/\r\?$/ -WX\0/" config/Makefile.msvc > config/Makefile
 
+  git submodule update --init flexdll
+  run "make flexdll" make flexdll
   run "make world" make world
   run "make runtimeopt" make runtimeopt
   run "make -C otherlibs/systhreads libthreadsnat.lib" make -C otherlibs/systhreads libthreadsnat.lib
@@ -63,11 +65,11 @@ git submodule update --init flexdll
 
 cd $APPVEYOR_BUILD_FOLDER
 
-tar -xzf flexdll.tar.gz
-cd flexdll-0.35
-make MSVC_DETECT=0 CHAINS=msvc64 support
-cp flexdll*_msvc64.obj "$PREFIX/bin/flexdll"
-cd ..
+# tar -xzf flexdll.tar.gz
+# cd flexdll-0.35
+# make MSVC_DETECT=0 CHAINS=msvc64 support
+# cp flexdll*_msvc64.obj "$PREFIX/bin/flexdll"
+# cd ..
 
 cp config/m-nt.h byterun/caml/m.h
 cp config/s-nt.h byterun/caml/s.h
@@ -76,6 +78,8 @@ echo "Edit config/Makefile to set PREFIX=$PREFIX"
 sed -e "s|PREFIX=.*|PREFIX=$PREFIX|" -e "/^ *CFLAGS *=/s/\r\?$/ -WX\0/" config/Makefile.msvc64 > config/Makefile
 #run "Content of config/Makefile" cat config/Makefile
 
+git submodule update --init flexdll
+run "make flexdll" make flexdll
 run "make world" make world
 run "make bootstrap" make bootstrap
 run "make opt" make opt

asmrun/natdynlink.c

@@ -69,11 +69,11 @@ CAMLprim value caml_natdynlink_open(value filename, value global)
   CAMLlocal3 (res, handle, header);
   void *sym;
   void *dlhandle;
-  char *p;
+  charnat *p;
 
   /* TODO: dlclose in case of error... */
 
-  p = caml_stat_strdup(String_val(filename));
+  p = caml_stat_strdup_to_utf16(String_val(filename));
   caml_enter_blocking_section();
   dlhandle = caml_dlopen(p, 1, Int_val(global));
   caml_leave_blocking_section();
@@ -151,11 +151,11 @@ CAMLprim value caml_natdynlink_run_toplevel(value filename, value symbol)
   CAMLparam2 (filename, symbol);
   CAMLlocal3 (res, v, handle_v);
   void *handle;
-  char *p;
+  charnat *p;
 
   /* TODO: dlclose in case of error... */
 
-  p = caml_stat_strdup(String_val(filename));
+  p = caml_stat_strdup_to_utf16(String_val(filename));
   caml_enter_blocking_section();
   handle = caml_dlopen(p, 1, 1);
   caml_leave_blocking_section();

asmrun/startup.c

@@ -103,9 +103,9 @@ extern void caml_install_invalid_parameter_handler();
 
 #endif
 
-value caml_startup_common(char **argv, int pooling)
+value caml_startup_common(charnat **argv, int pooling)
 {
-  char * exe_name, * proc_self_exe;
+  charnat * exe_name, * proc_self_exe;
   char tos;
 
   /* Determine options */
@@ -142,7 +142,7 @@ value caml_startup_common(char **argv, int pooling)
   caml_init_backtrace();
   caml_debugger_init (); /* force debugger.o stub to be linked */
   exe_name = argv[0];
-  if (exe_name == NULL) exe_name = "";
+  if (exe_name == NULL) exe_name = _T("");
   proc_self_exe = caml_executable_name();
   if (proc_self_exe != NULL)
     exe_name = proc_self_exe;
@@ -156,29 +156,29 @@ value caml_startup_common(char **argv, int pooling)
   return caml_start_program();
 }
 
-value caml_startup_exn(char **argv)
+value caml_startup_exn(charnat **argv)
 {
   return caml_startup_common(argv, /* pooling */ 0);
 }
 
-void caml_startup(char **argv)
+void caml_startup(charnat **argv)
 {
   value res = caml_startup_exn(argv);
   if (Is_exception_result(res))
     caml_fatal_uncaught_exception(Extract_exception(res));
 }
 
-void caml_main(char **argv)
+void caml_main(charnat **argv)
 {
   caml_startup(argv);
 }
 
-value caml_startup_pooled_exn(char **argv)
+value caml_startup_pooled_exn(charnat **argv)
 {
   return caml_startup_common(argv, /* pooling */ 1);
 }
 
-void caml_startup_pooled(char **argv)
+void caml_startup_pooled(charnat **argv)
 {
   value res = caml_startup_pooled_exn(argv);
   if (Is_exception_result(res))

byterun/Makefile

@@ -65,7 +65,7 @@ endif
 DBGO=d.$(O)
 
 ifeq "$(UNIX_OR_WIN32)" "win32"
-LIBS = $(call SYSLIB,ws2_32) $(EXTRALIBS)
+LIBS = $(call SYSLIB,ws2_32) $(call SYSLIB,shell32) $(EXTRALIBS)
 ifdef BOOTSTRAPPING_FLEXLINK
 MAKE_OCAMLRUN=$(MKEXE_BOOT)
 else

byterun/backtrace_prim.c

@@ -46,7 +46,7 @@
 /* The table of debug information fragments */
 struct ext_table caml_debug_info;
 
-CAMLexport char * caml_cds_file = NULL;
+CAMLexport charnat * caml_cds_file = NULL;
 
 /* Location of fields in the Instruct.debug_event record */
 enum {
@@ -329,7 +329,7 @@ static void read_main_debug_info(struct debug_info *di)
 {
   CAMLparam0();
   CAMLlocal3(events, evl, l);
-  char *exec_name;
+  charnat *exec_name;
   int fd, num_events, orig, i;
   struct channel *chan;
   struct exec_trailer trail;

byterun/caml/backtrace.h

@@ -109,7 +109,7 @@ CAMLprim value caml_record_backtrace(value vflag);
 #ifndef NATIVE_CODE
 
 /* Path to the file containing debug information, if any, or NULL. */
-CAMLextern char * caml_cds_file;
+CAMLextern charnat * caml_cds_file;
 
 /* Primitive called _only_ by runtime to record unwinded frames to
  * backtrace.  A similar primitive exists for native code, but with a

byterun/caml/callback.h

@@ -47,11 +47,11 @@ CAMLextern value * caml_named_value (char const * name);
 typedef void (*caml_named_action) (value*, char *);
 CAMLextern void caml_iterate_named_values(caml_named_action f);
 
-CAMLextern void caml_main (char ** argv);
-CAMLextern void caml_startup (char ** argv);
-CAMLextern value caml_startup_exn (char ** argv);
-CAMLextern void caml_startup_pooled (char ** argv);
-CAMLextern value caml_startup_pooled_exn (char ** argv);
+CAMLextern void caml_main (charnat ** argv);
+CAMLextern void caml_startup (charnat ** argv);
+CAMLextern value caml_startup_exn (charnat ** argv);
+CAMLextern void caml_startup_pooled (charnat ** argv);
+CAMLextern value caml_startup_pooled_exn (charnat ** argv);
 CAMLextern void caml_shutdown (void);
 
 CAMLextern int caml_callback_depth;

byterun/caml/config.h

@@ -37,6 +37,14 @@
 #define ARCH_SIZET_PRINTF_FORMAT "z"
 #endif
 
+/* Types for Windows wide strings */
+
+#ifdef _WIN32
+#define ARCH_CHARNATSTR_PRINTF_FORMAT "S"
+#else
+#define ARCH_CHARNATSTR_PRINTF_FORMAT "s"
+#endif
+
 /* Types for 32-bit integers, 64-bit integers, and
    native integers (as wide as a pointer type) */
 

byterun/caml/dynlink.h

@@ -27,8 +27,8 @@
    (all three 0-separated in char arrays).
    Abort the runtime system on error.
    Calling this frees caml_shared_libs_path (not touching its contents). */
-extern void caml_build_primitive_table(char * lib_path,
-                                       char * libs,
+extern void caml_build_primitive_table(charnat * lib_path,
+                                       charnat * libs,
                                        char * req_prims);
 
 /* The search path for shared libraries */

byterun/caml/memory.h

@@ -156,6 +156,9 @@ typedef char* caml_stat_string;
    the request fails, and so requires the runtime lock to be held.
 */
 CAMLextern caml_stat_string caml_stat_strdup(const char *s);
+#ifdef _WIN32
+CAMLextern wchar_t* caml_stat_wcsdup(const wchar_t *s);
+#endif
 
 /* [caml_stat_strdup_noexc] is a variant of [caml_stat_strdup] that returns NULL
    in case the request fails, and doesn't require the runtime lock.
@@ -168,6 +171,9 @@ CAMLextern caml_stat_string caml_stat_strdup_noexc(const char *s);
    request fails, and so requires the runtime lock to be held.
 */
 CAMLextern caml_stat_string caml_stat_strconcat(int n, ...);
+#ifdef _WIN32
+CAMLextern wchar_t* caml_stat_wcsconcat(int n, ...);
+#endif
 
 
 /* void caml_shrink_heap (char *);        Only used in compact.c */