minor: handle unicode in cmdline consts; clarify const doc.; deprecat…

…e custom preprocessor

README.md

@@ -403,6 +403,7 @@ end
 Keep in mind that in some cases, Shrinko8 will play it safe and avoid a computation whose result is questionable or has a high potential to change between pico8 versions. If this prevents a `--[[const]]` variable from being assigned a constant, Shrinko8 will warn about this:
 
 ```lua
+-- here, abs overflows (due to receiving -0x8000), and shrinko8 chooses not to rely on the overflow behaviour
 --[[const]] x = abs(0x7fff+1)-1
 ?x
 

pico_constfold.py

@@ -1,5 +1,5 @@
 from utils import *
-from pico_defs import fixnum_is_negative, k_fixnum_mask
+from pico_defs import fixnum_is_negative, k_fixnum_mask, to_p8str
 from pico_tokenize import Token, TokenType, ConstToken, k_skip_children, tokenize
 from pico_parse import Node, NodeType, VarKind, is_global_or_builtin_local, is_vararg_expr
 from pico_output import format_fixnum
@@ -585,8 +585,11 @@ def visit_assign(node):
 
     root.traverse_nodes(pre=skip_special, post=update_node_constval)
 
-def parse_constant(value):
-    tokens, errors = tokenize(Source(None, value))
+def parse_constant(value, as_str=False):
+    if as_str:
+        return LuaString(to_p8str(value))
+
+    tokens, errors = tokenize(Source(None, to_p8str(value)))
     if not errors:
         token = None
         if len(tokens) == 1:

pico_preprocess.py

@@ -279,8 +279,7 @@ def handle(m, path, code, i, start_i, out_i, outparts, outmappings):
                 outparts.append(value)
                 out_i += len(value)
 
-        # (do not keep empty lines, unlike PicoPreprocessor)
-        return end_i + 1, end_i + 1, out_i
+        return end_i, end_i, out_i
 
     def handle_inline(m, path, code, i, start_i, out_i, outparts, outmappings):
         if not m.active:

shrinko8.py

@@ -5,7 +5,7 @@
 from pico_cart import Cart, CartFormat, read_cart, write_cart, get_bbs_cart_url, merge_cart
 from pico_export import read_cart_export, read_pod_file, ListOp
 from pico_tokenize import k_hint_split_re
-from pico_constfold import parse_constant, LuaString
+from pico_constfold import parse_constant
 import argparse
 
 k_version = 'v1.2.0b'
@@ -130,9 +130,9 @@ def ParsableCountHandler(prefix, name, size, limit):
 pgroup.add_argument("--global-builtins-only", action="store_true", help="assume all builtins are global, corresponds to pico8's -global_api option")
 pgroup.add_argument("--local-builtin", type=SplitBySeps, action=extend_arg, help="treat identifier(s) as a local builtin (probably no use outside of testing)")
 pgroup.add_argument("--ignore-hints", action="store_true", help="ignore shrinko8 hint comments")
-pgroup.add_argument("--custom-preprocessor", action="store_true", help="enable a custom preprocessor (#define X 123, #ifdef X, #[X], #[X[[print('X enabled')]]])")
 pgroup.add_argument("--output-sections", type=SplitBySeps, action=extend_arg, help="only output the specified p8 sections {%s,...}" % ",".join(k_def_sections))
 pgroup.add_argument("--export-name", help="name to use for the export (by default, taken from output name)")
+pgroup.add_argument("--custom-preprocessor", action="store_true", help=argparse.SUPPRESS) # might remove this one day
 
 def default_format(input, for_output=False):
     ext = path_extension(input)[1:].lower()
@@ -243,11 +243,11 @@ def main_inner(raw_args):
 
     if args.const or args.str_const:
         if args.const:
-            args.const = {name: parse_constant(val) or throw(f"cannot parse <{val}>. If it's meant to be a string, try using --str-const instead") 
+            args.const = {name: parse_constant(val) or throw(f"cannot parse const <{val}>. If it's meant to be a string, try using --str-const instead") 
                           for name, val in args.const}
         if args.str_const:
             args.const = args.const or {}
-            args.const.update({name: LuaString(val) for name, val in args.str_const})
+            args.const.update({name: parse_constant(val, as_str=True) for name, val in args.str_const})
 
     args.preproc_cb, args.postproc_cb, args.sublang_cb = None, None, None
     if args.script: