Merge pull request #1928 from timbrel/streaming-line-history

core/commands/line_history.py

@@ -11,7 +11,7 @@
 from ..fns import filter_, pairwise
 from ..git_command import GitCommand
 from ..parse_diff import SplittedDiff
-from ..runtime import enqueue_on_worker
+from ..runtime import run_on_new_thread
 from ..utils import flash
 from ..view import clamp, replace_view_content
 from ...common import util
@@ -263,10 +263,10 @@ def render():
                 ]
                 + [commit]
             )
-            output = self.git(*cmd)
-            replace_view_content(view, output)
+            for line in self.git_streaming(*cmd):
+                replace_view_content(view, line, sublime.Region(view.size()))
 
-        enqueue_on_worker(render)
+        run_on_new_thread(render)
 
 
 class gs_line_history_open_commit(TextCommand, GitCommand):

core/commands/log_graph.py

@@ -459,25 +459,6 @@ def wait_for_first_item(it):
     return chain(head, iterable)
 
 
-def log_git_command(fn):
-    # type: (Callable[..., Iterator[T]]) -> Callable[..., Iterator[T]]
-    def decorated(self, *args, **kwargs):
-        start_time = time.perf_counter()
-        stderr = ''
-        saved_exception = None
-        try:
-            yield from fn(self, *args, **kwargs)
-        except GitSavvyError as e:
-            stderr = e.stderr
-            saved_exception = e
-        finally:
-            end_time = time.perf_counter()
-            util.debug.log_git(args, self.repo_path, None, "<SNIP>", stderr, end_time - start_time)
-            if saved_exception:
-                raise saved_exception from None
-    return decorated
-
-
 class Done(Exception):
     pass
 
@@ -519,19 +500,6 @@ class GraphLine(NamedTuple):
     parents: str
 
 
-def try_kill_proc(proc):
-    if proc:
-        try:
-            utils.kill_proc(proc)
-        except ProcessLookupError:
-            pass
-        proc.got_killed = True
-
-
-def proc_has_been_killed(proc):
-    return getattr(proc, "got_killed", False)
-
-
 def selection_is_before_region(view, region):
     # type: (sublime.View, sublime.Region) -> bool
     try:
@@ -834,7 +802,7 @@ def remember_proc(proc):
         def ensure_not_aborted(fn):
             def decorated(*args, **kwargs):
                 if should_abort():
-                    try_kill_proc(current_proc)
+                    utils.try_kill_proc(current_proc)
                 else:
                     return fn(*args, **kwargs)
             return decorated
@@ -918,7 +886,7 @@ def process_graph(lines):
                             yield "...\n"
                             yield line
                     else:
-                        try_kill_proc(current_proc)
+                        utils.try_kill_proc(current_proc)
                         yield "..."
                         break
 
@@ -1083,7 +1051,7 @@ def reader():
                 try:
                     lines = run_or_timeout(lambda: wait_for_first_item(graph), timeout=1.0)
                 except TimeoutError:
-                    try_kill_proc(current_proc)
+                    utils.try_kill_proc(current_proc)
                     settings.set('git_savvy.log_graph_view.decoration', None)
                     enqueue_on_worker(
                         self.view.run_command,
@@ -1251,43 +1219,10 @@ def apply_token(view, token, offset):
 
         run_on_new_thread(reader)
 
-    @log_git_command
-    def git_stdout(self, *args, show_panel_on_error=True, throw_on_error=True, got_proc=None, **kwargs):
-        # type: (...) -> Iterator[str]
-        # Note: Can't use `self.lax_decode` because it internally uses
-        # `self.get_encoding_candidates()` which blocks the main thread as it
-        # needs to access the settings!
-        decode = lax_decoder(self.get_encoding_candidates())
-        proc = self.git(*args, just_the_proc=True, **kwargs)
-        if got_proc:
-            got_proc(proc)
-        received_some_stdout = False
-        with proc:
-            for line in iter(proc.stdout.readline, b''):
-                yield decode(line)
-                if not received_some_stdout:
-                    received_some_stdout = True
-
-            stderr = ''.join(map(decode, proc.stderr.readlines()))
-
-        if throw_on_error and not proc.returncode == 0 and not proc_has_been_killed(proc):
-            stdout = "<STDOUT SNIPPED>\n" if received_some_stdout else ""
-            raise GitSavvyError(
-                "$ {}\n\n{}".format(
-                    util.debug.pretty_git_command(args),
-                    ''.join([stdout, stderr])
-                ),
-                cmd=proc.args,
-                stdout=stdout,
-                stderr=stderr,
-                show_panel=show_panel_on_error,
-                window=self.view.window(),
-            )
-
     def read_graph(self, got_proc=None):
         # type: (Callable[[subprocess.Popen], None]) -> Iterator[str]
         args = self.build_git_command()
-        yield from self.git_stdout(*args, got_proc=got_proc)
+        yield from self.git_streaming(*args, got_proc=got_proc)
 
     def build_git_command(self):
         settings = self.view.settings()
@@ -1365,19 +1300,6 @@ def build_git_command(self):
         return args
 
 
-def lax_decoder(encodings):
-    # type: (Sequence[str]) -> Callable[[bytes], str]
-    def decode(bytes):
-        # type: (bytes) -> str
-        for encoding in encodings:
-            try:
-                return bytes.decode(encoding)
-            except UnicodeDecodeError:
-                pass
-        return bytes.decode('utf8', errors='replace')
-    return decode
-
-
 def prelude(view):
     # type: (sublime.View) -> str
     settings = view.settings()

core/git_command.py

@@ -26,7 +26,7 @@
 from GitSavvy.core import store
 from GitSavvy.core.fns import consume, filter_, pairwise
 from GitSavvy.core.runtime import auto_timeout, enqueue_on_worker, run_as_future
-from GitSavvy.core.utils import kill_proc, paths_upwards, resolve_path
+from GitSavvy.core.utils import try_kill_proc, paths_upwards, proc_has_been_killed, resolve_path
 
 
 from typing import (
@@ -149,7 +149,7 @@ def on_line(line: bytes,
             except IndexError:
                 time.sleep(next(delay) / 1000)
                 if timeout_manager.has_timed_out():
-                    kill_proc(proc)
+                    try_kill_proc(proc)
                     raise TimeoutError("timed out after {} seconds".format(timeout))
 
     # Check and raise exceptions if any
@@ -192,6 +192,26 @@ def _group_bytes_to_lines(bytewise: Iterator[bytes]) -> Iterator[bytes]:
         yield line
 
 
+def log_git_runtime(fn):
+    # type: (Callable[..., Iterator[T]]) -> Callable[..., Iterator[T]]
+    """A specialized log decorator for `git_streaming`."""
+    def decorated(self, *args, **kwargs):
+        start_time = time.perf_counter()
+        stderr = ''
+        saved_exception = None
+        try:
+            yield from fn(self, *args, **kwargs)
+        except GitSavvyError as e:
+            stderr = e.stderr
+            saved_exception = e
+        finally:
+            end_time = time.perf_counter()
+            util.debug.log_git(args, self.repo_path, None, "<SNIP>", stderr, end_time - start_time)
+            if saved_exception:
+                raise saved_exception from None
+    return decorated
+
+
 STARTUPINFO = None
 if sys.platform == "win32":
     STARTUPINFO = subprocess.STARTUPINFO()
@@ -476,6 +496,36 @@ def git_throwing_silently(self, *args, **kwargs):
             **kwargs
         )
 
+    @log_git_runtime
+    def git_streaming(self, *args, show_panel_on_error=True, throw_on_error=True, got_proc=None, **kwargs):
+        # type: (...) -> Iterator[str]
+        decode = partial(self.lax_decode_, self.get_encoding_candidates())
+        proc = self.git(*args, just_the_proc=True, **kwargs)
+        if got_proc:
+            got_proc(proc)
+        received_some_stdout = False
+        with proc:
+            for line in iter(proc.stdout.readline, b''):
+                yield decode(line)
+                if not received_some_stdout:
+                    received_some_stdout = True
+
+            stderr = ''.join(map(decode, proc.stderr.readlines()))
+
+        if throw_on_error and not proc.returncode == 0 and not proc_has_been_killed(proc):
+            stdout = "<STDOUT SNIPPED>\n" if received_some_stdout else ""
+            raise GitSavvyError(
+                "$ {}\n\n{}".format(
+                    util.debug.pretty_git_command(args),
+                    ''.join([stdout, stderr])
+                ),
+                cmd=proc.args,
+                stdout=stdout,
+                stderr=stderr,
+                show_panel=show_panel_on_error,
+                window=self.some_window(),
+            )
+
     def get_encoding_candidates(self):
         # type: () -> Sequence[str]
         return [
@@ -498,10 +548,16 @@ def ensure_decoded(self, input):
 
     def lax_decode(self, input):
         # type: (bytes) -> str
-        try:
-            return self.strict_decode(input)
-        except UnicodeDecodeError:
-            return input.decode("utf-8", "replace")
+        return self.lax_decode_(self.get_encoding_candidates(), input)
+
+    def lax_decode_(self, encodings, input):
+        # type: (Sequence[str], bytes) -> str
+        for encoding in encodings:
+            try:
+                return input.decode(encoding)
+            except UnicodeDecodeError:
+                pass
+        return input.decode('utf8', errors='replace')
 
     def try_decode(self, input, encodings):
         # type: (bytes, Sequence[str]) -> Tuple[str, str]

core/utils.py

@@ -375,6 +375,19 @@ def kill_proc(proc):
         proc.terminate()
 
 
+def try_kill_proc(proc):
+    if proc:
+        try:
+            kill_proc(proc)
+        except ProcessLookupError:
+            pass
+        proc.got_killed = True
+
+
+def proc_has_been_killed(proc):
+    return getattr(proc, "got_killed", False)
+
+
 # `realpath` also supports `bytes` and we don't, hence the indirection
 def _resolve_path(path):
     # type: (str) -> str