ls-colors -- colorize file arguments in subproc mode command lines

news/appimage.rst

@@ -1,3 +1,23 @@
 **Added:**
 
 * Added building process of standalone rootless AppImage for xonsh.
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* <news item>
+
+**Security:**
+
+* <news item>

news/ls-color.rst

@@ -0,0 +1,24 @@
+**Added:**
+
+* added event on_lscolors_changed which fires when an item in $LS_COLORS changed.
+
+**Changed:**
+
+* modified subprocess mode to colorize files per LS_COLORS, when they appear as arguments in the command line.
+  Yet another approximation of ls -c file coloring behavior.
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* Modified base_shell._TeeStdBuf to feed bytes not str to console window under VS Code.
+
+**Security:**
+
+* <news item>

news/unit_tests1.rst

@@ -0,0 +1,24 @@
+**Added:**
+
+* <news item>
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* Unit test failures in test_integrations under ubuntu 19.10 with Python 3.8.0
+* .gitignore entries for venv under project root (as for autovox) and for VS Code.
+
+**Security:**
+
+* <news item>

scripts/xonsh

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3 -u
+#!/usr/bin/env -S python3 -u
 
 from xonsh.main import main
 main()

scripts/xonsh-cat

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3 -u
+#!/usr/bin/env -S python3 -u
 
 from xonsh.xoreutils.cat import cat_main as main
 main()

tests/test_environ.py

@@ -299,3 +299,36 @@ def test_lscolors_target():
     lsc = LsColors.fromstring("ln=target")
     assert lsc["ln"] == ("TARGET",)
     assert lsc.detype() == "ln=target"
+
+
+@pytest.mark.parametrize( "key_in,old_in,new_in,test", 
+    [('rs', ('NO_COLOR',), ('BLUE',), 'existing key, change value')
+    ,('rs', ('NO_COLOR',), ('NO_COLOR',), 'existing key, no change in value')
+    ,('tw', None, ('NO_COLOR',), 'create new key')
+    ,('pi', ('BACKGROUND_BLACK', 'YELLOW'), None, 'delete existing key')
+    ],
+)
+
+def test_lscolors_events(key_in, old_in, new_in, test, xonsh_builtins):
+    lsc = LsColors.fromstring('rs=0:di=01;34:pi=40;33')
+    # corresponding colors: [('NO_COLOR',), ('BOLD_CYAN',), ('BOLD_CYAN',), ('BACKGROUND_BLACK', 'YELLOW')]
+
+    event_fired = False
+    @xonsh_builtins.events.on_lscolors_change
+    def handler(key,oldvalue,newvalue,**kwargs):
+        nonlocal old_in, new_in, key_in, event_fired
+        assert key==key_in and oldvalue==old_in and newvalue==new_in, "Old and new event values match"
+        event_fired = True
+
+    xonsh_builtins.__xonsh__.env["LS_COLORS"] = lsc
+
+    if new_in is None:
+        old = lsc.pop(key_in, 'argle')
+    else:
+        lsc[key_in] = new_in
+
+    if old_in == new_in:
+        assert not event_fired, "No event if value doesn't change"
+    else:
+        assert event_fired
+

tests/test_ptk_highlight.py

@@ -22,7 +22,10 @@
 from xonsh.platform import ON_WINDOWS
 from xonsh.built_ins import load_builtins, unload_builtins
 from xonsh.execer import Execer
-from xonsh.pyghooks import XonshLexer
+from xonsh.pyghooks import XonshLexer, Color, XonshStyle, on_pre_cmdloop
+from xonsh.environ import LsColors
+from xonsh.events import events
+from tools import DummyShell
 
 
 @pytest.fixture(autouse=True)
@@ -141,16 +144,24 @@ def test_nested():
 
 
 @skip_if_on_windows
-def test_path(tmpdir):
+def test_path(tmpdir, xonsh_builtins):
+
+    xonsh_builtins.__xonsh__.shell = DummyShell()       # because load_command_cache zaps it. 
+    xonsh_builtins.__xonsh__.shell.shell_type = 'prompt_toolkit2'
+    xonsh_builtins.__xonsh__.shell.shell.styler = XonshStyle()  # default style
+    lsc = LsColors( LsColors.default_settings)
+    xonsh_builtins.__xonsh__.env["LS_COLORS"] = lsc 
+    on_pre_cmdloop()      # to add lscolors to style
+
     test_dir = str(tmpdir.mkdir("xonsh-test-highlight-path"))
     check_token(
-        "cd {}".format(test_dir), [(Name.Builtin, "cd"), (Name.Constant, test_dir)]
+        "cd {}".format(test_dir), [(Name.Builtin, "cd"), (Color.BOLD_BLUE, test_dir)]
     )
     check_token(
         "cd {}-xxx".format(test_dir),
         [(Name.Builtin, "cd"), (Text, "{}-xxx".format(test_dir))],
     )
-    check_token("cd X={}".format(test_dir), [(Name.Constant, test_dir)])
+    check_token("cd X={}".format(test_dir), [(Color.BOLD_BLUE, test_dir)])
 
     with builtins.__xonsh__.env.swap(AUTO_CD=True):
         check_token(test_dir, [(Name.Constant, test_dir)])
@@ -194,3 +205,4 @@ def test_macro():
             (String, "export var=42; echo $var"),
         ],
     )
+

tests/test_pyghooks.py

@@ -1,8 +1,15 @@
 """Tests pygments hooks."""
 import pytest
+import os
+import builtins
+import stat
 
-from xonsh.pyghooks import Color, color_name_to_pygments_code, code_by_name
+from tempfile import TemporaryDirectory
 
+from xonsh.pyghooks import XonshStyle, Color, color_name_to_pygments_code, code_by_name, color_by_name, on_pre_cmdloop, color_file, file_color_tokens
+
+from xonsh.environ import LsColors
+from tools import skip_if_on_windows, skip_if_on_unix
 
 DEFAULT_STYLES = {
     # Reset
@@ -91,3 +98,121 @@ def test_code_by_name(name, exp):
     styles = DEFAULT_STYLES.copy()
     obs = code_by_name(name, styles)
     assert obs == exp
+
+
+@pytest.mark.parametrize(
+    "in_tuple, exp_ct, exp_ansi_colors"
+    , [
+        (("NO_COLOR",), Color.NO_COLOR, "noinherit")
+        ,(("GREEN",), Color.GREEN, "ansigreen")
+        ,(("BOLD_RED",), Color.BOLD_RED, "bold ansired")
+        ,(("BACKGROUND_BLACK", "BOLD_GREEN"), Color.BACKGROUND_BLACK__BOLD_GREEN, "bg:ansiblack bold ansigreen")
+        ,
+    ]
+
+)
+def test_color_token_by_name( in_tuple, exp_ct, exp_ansi_colors, xonsh_builtins):
+    from xonsh.pyghooks import XonshStyle, color_token_by_name
+
+    xonsh_builtins.__xonsh__.shell.shell_type = 'prompt_toolkit2'
+    styler = XonshStyle() # default style
+    xonsh_builtins.__xonsh__.shell.shell.styler = styler
+    # can't really instantiate XonshStyle separate from a shell??
+
+    ct = color_token_by_name( in_tuple)
+    ansi_colors = styler.styles[ct] # if keyerror, ct was not cached
+    assert ct == exp_ct, "returned color token is right"
+    assert ansi_colors == exp_ansi_colors, "color token mapped to correct color string"
+
+#todo test color changes when ls_colors is updated.  Easy case: new color already in cache; hard case, new color is never-before-seen
+
+_cf = {
+    'rs': 'regular',
+    'di': 'simple_dir',
+ 'ln': 'simple_link',
+ 'mh': None,
+ 'pi': 'pipe',
+ 'so': None,
+ 'do': None,
+ 'bd': '/dev/sda',
+ 'cd': '/dev/tty',
+ 'or': 'orphan_link',
+ 'mi': None,
+ 'su': 'set_uid',
+ 'sg': 'set_gid',
+ 'ca': None,
+ 'tw': 'sticky_ow_dir',
+ 'ow': 'other_writable_dir',
+ 'st': 'sticky_dir',
+ 'ex': 'executable',
+ '*.emf': 'foo.emf',
+ '*.zip': 'foo.zip',
+ '*.ogg': 'foo.ogg'
+
+}
+def colorizable_files(xonsh_builtins):
+    """populate temp dir with sample files and iniialize (hopefully consistent) LS_COLORS"""
+
+    xonsh_builtins.__xonsh__.shell.shell_type = 'prompt_toolkit2'
+    xonsh_builtins.__xonsh__.shell.shell.styler = XonshStyle()  # default style
+    lsc = LsColors( LsColors.default_settings)
+    xonsh_builtins.__xonsh__.env["LS_COLORS"] = lsc 
+    on_pre_cmdloop()      # to add lscolors to style
+
+    with TemporaryDirectory() as tempdir:
+        for k,v in _cf.items():
+
+            if v is None:
+                continue
+            if v.startswith('/'):
+                file_path = v
+            else:
+                file_path = tempdir  + '/' + v
+            try:
+                mode = os.lstat(file_path)
+            except FileNotFoundError:
+                if file_path.endswith('_dir'):
+                    os.mkdir(file_path)
+                else:
+                    open(file_path, 'a').close()
+                if k in ('di', 'rg'):
+                    pass
+                elif k == 'ex':
+                    os.chmod(file_path, stat.S_IXUSR)
+                elif k == 'ln':
+                    os.rename(file_path, file_path+'_target')
+                    os.symlink(file_path+'_target', file_path)
+                elif k == 'or':
+                    os.rename(file_path, file_path+'_target')
+                    os.symlink(file_path+'_target', file_path)
+                    os.remove(file_path+'_target')
+                elif k == 'pi':
+                    os.remove(file_path)
+                    os.mkfifo(file_path)
+                elif k == 'su':
+                    os.chmod(file_path, stat.S_ISUID)
+                elif k == 'sg':
+                    os.chmod(file_path, stat.S_ISGID)
+                elif k == 'st':
+                    os.chmod(file_path, stat.S_ISVTX | stat.S_IRUSR | stat.S_IWUSR) # TempDir requires o:r
+                elif k == 'tw':
+                    os.chmod(file_path, stat.S_ISVTX | stat.S_IWOTH | stat.S_IRUSR | stat.S_IWUSR)
+                elif k == 'ow':
+                    os.chmod(file_path, stat.S_IWOTH | stat.S_IRUSR | stat.S_IWUSR)
+                else:
+                    pass # cauterize those elseless ifs!
+
+            yield k, file_path, file_color_tokens[k]
+
+    pass # tempdir get cleaned up here.
+
+@skip_if_on_windows
+def test_colorize_file(xonsh_builtins):
+#    # someday, should parameterize this test, so you get all the failures in one run.
+    for (key, file_path, exp_tok) in colorizable_files(xonsh_builtins):
+        mode = (os.lstat( file_path)).st_mode
+        color_token, color_key = color_file(file_path, mode)
+
+        assert color_key == key, "File classified as expected kind"
+        assert color_token == exp_tok, "Color token is as expected"
+

xonsh/base_shell.py

@@ -71,8 +71,8 @@ def __init__(
         self.errors = env.get("XONSH_ENCODING_ERRORS") if errors is None else errors
         self.prestd = prestd
         self.poststd = poststd
-        self._std_is_binary = not hasattr(stdbuf, "encoding")
-
+        self._std_is_binary = (not hasattr(stdbuf, "encoding")) or hasattr(stdbuf, "_redirect_to") # VS Code terminal window - has encoding attr but won't accept str                
+        
     def fileno(self):
         """Returns the file descriptor of the std buffer."""
         return self.stdbuf.fileno()
@@ -95,6 +95,8 @@ def readinto(self, b):
 
     def write(self, b):
         """Write bytes into both buffers."""
+        if type(b) is str:
+            raise TypeError( "who dares call write() with str?")
         std_b = b
         if self.prestd:
             std_b = self.prestd + b

xonsh/environ.py

@@ -123,6 +123,20 @@
 )
 
 
+events.doc(
+    "on_lscolors_change",
+    """
+on_lscolors_change(key: str, oldvalue: Any, newvalue: Any) -> None
+
+Fires after a value in LS_COLORS changes, when a new key is added (oldvalue is None) 
+or when an existing key is deleted (newvalue is None).  
+LS_COLORS values must be (ANSI color) strings, None is unambiguous.
+Does not fire when the whole environment variable changes (see on_envvar_change).
+Does not fire for each value when LS_COLORS is first instantiated.
+Normal usage is to arm the event handler, then read (not modify) all existing values.
+""",
+)
+
 @lazyobject
 def HELP_TEMPLATE():
     return (
@@ -330,11 +344,16 @@ def __getitem__(self, key):
 
     def __setitem__(self, key, value):
         self._detyped = None
+        old_value = self._d.get(key, None)
         self._d[key] = value
+        if old_value != value:
+            events.on_lscolors_change.fire(key=key, oldvalue=old_value, newvalue=value)
 
     def __delitem__(self, key):
         self._detyped = None
+        old_value = self._d.get(key, None)
         del self._d[key]
+        events.on_lscolors_change.fire(key=key, oldvalue=old_value, newvalue=None)
 
     def __len__(self):
         return len(self._d)