Resolve Windows path problems

tools/mkbuildoptglobals.py

@@ -187,10 +187,14 @@
 import glob
 import os
 import platform
+import traceback
 import sys
 import textwrap
 import time
 
+import locale
+import codecs
+
 # Need to work on signature line used for match to avoid conflicts with
 # existing embedded documentation methods.
 build_opt_signature = "/*@create-file:build.opt@"
@@ -201,6 +205,7 @@
 err_print_flag = False
 msg_print_buf = ""
 debug_enabled = False
+default_locale = (None, None)
 
 # Issues trying to address through buffered printing
 # 1. Arduino IDE 2.0 RC5 does not show stderr text in color. Text printed does
@@ -254,6 +259,7 @@ def handle_error(err_no):
     # on err_no != 0, commit print buffer to stderr and sys exist with err_no
     global msg_print_buf
     global err_print_flag
+    global default_locale
     if len(msg_print_buf):
         if err_no or err_print_flag:
             fd = sys.stderr
@@ -295,16 +301,16 @@ def copy_create_build_file(source_fqfn, build_target_fqfn):
                 pass
     return True     # file changed
 
-
 def add_include_line(build_opt_fqfn, include_fqfn):
+    global default_locale
     if not os.path.exists(include_fqfn):
         # If file is missing, we need an place holder
-        with open(include_fqfn, 'w', encoding="utf-8"):
+        with open(include_fqfn, 'w', encoding=default_locale[1]):
             pass
-        print("add_include_line: Created " + include_fqfn)
-    with open(build_opt_fqfn, 'a', encoding="utf-8") as build_opt:
-        build_opt.write('-include "' + include_fqfn.replace('\\', '\\\\') + '"\n')
+        print_msg("add_include_line: Created " + include_fqfn)
 
+    with open(build_opt_fqfn, 'a', encoding=default_locale[1]) as build_opt:
+        build_opt.write('-include "' + include_fqfn.replace('\\', '\\\\') + '"\n')
 
 def extract_create_build_opt_file(globals_h_fqfn, file_name, build_opt_fqfn):
     """
@@ -313,8 +319,9 @@ def extract_create_build_opt_file(globals_h_fqfn, file_name, build_opt_fqfn):
     copy of Sketch.ino.globals.h.
     """
     global build_opt_signature
+    global default_locale
 
-    build_opt = open(build_opt_fqfn, 'w', encoding="utf-8")
+    build_opt = open(build_opt_fqfn, 'w', encoding=default_locale[1])
     if not os.path.exists(globals_h_fqfn) or (0 == os.path.getsize(globals_h_fqfn)):
         build_opt.close()
         return False
@@ -609,8 +616,19 @@ def main():
     global build_opt_signature
     global docs_url
     global debug_enabled
+    global default_locale
     num_include_lines = 1
 
+    # Given that GCC will handle lines from an @file as if they were on
+    # the command line. I assume that the contents of @file need to be encoded
+    # to match that of the shell running GCC runs. I am not 100% sure this API
+    # gives me that, but it appears to work.
+    #
+    # However, elsewhere when dealing with source code we continue to use 'utf-8',
+    # ref. https://gcc.gnu.org/onlinedocs/gcc-4.1.2/cpp/Character-sets.html
+    default_locale = locale.getdefaultlocale()
+    print_msg(f'default locale: {default_locale}')
+
     args = parse_args()
     debug_enabled = args.debug
     runtime_ide_path = os.path.normpath(args.runtime_ide_path)
@@ -655,6 +673,10 @@ def main():
     print_dbg(f"first_time:             {first_time}")
     print_dbg(f"use_aggressive_caching_workaround: {use_aggressive_caching_workaround}")
 
+    if not os.path.exists(build_path_core):
+        os.makedirs(build_path_core)
+        print_msg("Clean build, created dir " + build_path_core)
+
     if first_time or \
     not use_aggressive_caching_workaround or \
     not os.path.exists(commonhfile_fqfn):
@@ -667,10 +689,6 @@ def main():
         touch(commonhfile_fqfn)
         print_err(f"Neutralized future timestamp on build file: {commonhfile_fqfn}")
 
-    if not os.path.exists(build_path_core):
-        os.makedirs(build_path_core)
-        print_msg("Clean build, created dir " + build_path_core)
-
     if os.path.exists(source_globals_h_fqfn):
         print_msg("Using global include from " + source_globals_h_fqfn)
 
@@ -750,4 +768,10 @@ def main():
     handle_error(0)   # commit print buffer
 
 if __name__ == '__main__':
-    sys.exit(main())
+    rc = 1
+    try:
+        rc = main()
+    except:
+        print_err(traceback.format_exc())
+        handle_error(0)
+    sys.exit(rc)

tools/sizes.py

@@ -72,8 +72,10 @@ def get_segment_sizes(elf, path, mmu):
         (".bss", "BSS"),
     )
 
+    import locale
+    shell_encoding = locale.getdefaultlocale()[1]
     cmd = [os.path.join(path, "xtensa-lx106-elf-size"), "-A", elf]
-    with subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True) as proc:
+    with subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True, encoding=shell_encoding) as proc:
         lines = proc.stdout.readlines()
         for line in lines:
             words = line.split()