Merge pull request #1528 from mheinzler/multi-files

WIP: Support linter messages from multiple files

lint/backend.py

@@ -58,13 +58,13 @@ def get_lint_tasks(linters, view, view_has_changed):
         tasks = []
         for region in regions:
             code = view.substr(region)
-            offset = view.rowcol(region.begin())
+            offsets = view.rowcol(region.begin()) + (region.begin(),)
 
             # Due to a limitation in python 3.3, we cannot 'name' a thread when
             # using the ThreadPoolExecutor. (This feature has been introduced
             # in python 3.6.) So, we do this manually.
             task_name = make_good_task_name(linter, view)
-            task = partial(execute_lint_task, linter, code, offset, view_has_changed)
+            task = partial(execute_lint_task, linter, code, offsets, view_has_changed)
             executor = partial(modify_thread_name, task_name, task)
             tasks.append(executor)
 
@@ -106,10 +106,10 @@ def reduced_concurrency():
 
 
 @reduced_concurrency()
-def execute_lint_task(linter, code, offset, view_has_changed):
+def execute_lint_task(linter, code, offsets, view_has_changed):
     try:
         errors = linter.lint(code, view_has_changed) or []
-        finalize_errors(linter, errors, offset)
+        finalize_errors(linter, errors, offsets)
 
         return errors
     except linter_module.TransientError:
@@ -121,37 +121,60 @@ def execute_lint_task(linter, code, offset, view_has_changed):
         return []  # Empty list here to clear old errors
 
 
-def finalize_errors(linter, errors, offset):
+def error_json_serializer(o):
+    """Return a JSON serializable representation of error properties."""
+    if isinstance(o, sublime.Region):
+        return (o.a, o.b)
+
+    return o
+
+
+def finalize_errors(linter, errors, offsets):
     linter_name = linter.name
     view = linter.view
-    line_offset, col_offset = offset
+    line_offset, col_offset, pt_offset = offsets
 
     for error in errors:
+        # see if this error belongs to the main file
+        belongs_to_main_file = True
+        if 'filename' in error:
+            if (os.path.normcase(error['filename']) != os.path.normcase(view.file_name() or '') and
+                    error['filename'] != "<untitled {}>".format(view.buffer_id())):
+                belongs_to_main_file = False
+
         line, start, end = error['line'], error['start'], error['end']
-        if line == 0:
-            start += col_offset
-            end += col_offset
+        if belongs_to_main_file:  # offsets are for the main file only
+            if line == 0:
+                start += col_offset
+                end += col_offset
+
+            line += line_offset
 
-        line += line_offset
+        try:
+            region = error['region']
+        except KeyError:
+            line_start = view.text_point(line, 0)
+            region = sublime.Region(line_start + start, line_start + end)
+            if len(region) == 0:
+                region.b = region.b + 1
+
+        else:
+            if belongs_to_main_file:  # offsets are for the main file only
+                region = sublime.Region(region.a + pt_offset, region.b + pt_offset)
 
         error.update({
             'line': line,
             'start': start,
             'end': end,
-            'linter': linter_name
+            'linter': linter_name,
+            'region': region
         })
 
         uid = hashlib.sha256(
-            json.dumps(error, sort_keys=True).encode('utf-8')).hexdigest()
-
-        line_start = view.text_point(line, 0)
-        region = sublime.Region(line_start + start, line_start + end)
-        if len(region) == 0:
-            region.b = region.b + 1
+            json.dumps(error, sort_keys=True, default=error_json_serializer).encode('utf-8')).hexdigest()
 
         error.update({
             'uid': uid,
-            'region': region,
             'priority': style.get_value('priority', error, 0)
         })
 

lint/linter.py

@@ -152,6 +152,12 @@ def max_lines(self):
     # def text_point(self, row, col) => Point
     # def rowcol(self, point) => (row, col)
 
+    @staticmethod
+    def from_file(filename):
+        """Return a VirtualView with the contents of file."""
+        with open(filename, 'r', encoding='utf8') as f:
+            return VirtualView(f.read())
+
 
 class ViewSettings:
     """
@@ -612,6 +618,8 @@ def __init__(self, view, settings):
         if self.defaults is not None:
             self.defaults = self.defaults.copy()
 
+        self.temp_filename = None
+
     @property
     def filename(self):
         """Return the view's file path or '' if unsaved."""
@@ -989,7 +997,7 @@ def filter_errors(self, errors):
         return [
             error
             for error in errors
-            if not any(
+            if error is not None and not any(
                 pattern.search(': '.join([error['error_type'], error['code'], error['msg']]))
                 for pattern in filters
             )
@@ -1116,6 +1124,23 @@ def process_match(self, m, vv):
         error_type = m.error_type or self.get_error_type(m.error, m.warning)
         code = m.code or m.error or m.warning or ''
 
+        # determine a filename for this match
+        filename = self.normalize_filename(m.filename)
+
+        if filename:
+            # this is a match for a different file so we need its contents for
+            # the below checks
+            try:
+                vv = VirtualView.from_file(filename)
+            except OSError as err:
+                # warn about the error and drop this match
+                logger.warning('Exception: {}'.format(str(err)))
+                self.notify_failure()
+                return None
+        else:  # main file
+            # use the filename of the current view
+            filename = self.view.file_name() or "<untitled {}>".format(self.view.buffer_id())
+
         col = m.col
         line = m.line
 
@@ -1137,10 +1162,19 @@ def process_match(self, m, vv):
             col = max(min(col, (end - start) - 1), 0)
 
         line, start, end = self.reposition_match(line, col, m, vv)
+
+        # find the region to highlight for this error
+        line_start, _ = vv.full_line(line)
+        region = sublime.Region(line_start + start, line_start + end)
+        if len(region) == 0:
+            region.b += 1
+
         return {
+            "filename": filename,
             "line": line,
             "start": start,
             "end": end,
+            "region": region,
             "error_type": error_type,
             "code": code,
             "msg": m.message.strip(),
@@ -1154,6 +1188,33 @@ def get_error_type(self, error, warning):
         else:
             return self.default_type
 
+    def normalize_filename(self, filename):
+        """Return an absolute filename if it is not the main file."""
+        if filename and not self.is_stdin_filename(filename):
+            # ensure that the filename is absolute by basing relative paths on
+            # the working directory
+            cwd = self.get_working_dir(self.settings) or os.path.realpath('.')
+            filename = os.path.normpath(os.path.join(cwd, filename))
+
+            # only return a filename if it is a different file
+            normed_filename = os.path.normcase(filename)
+            if normed_filename == os.path.normcase(self.filename):
+                return None
+
+            # when the command was run on a temporary file we also need to
+            # compare this filename with that temporary filename
+            if self.temp_filename and normed_filename == os.path.normcase(self.temp_filename):
+                return None
+
+            return filename
+
+        # must be the main file
+        return None
+
+    @staticmethod
+    def is_stdin_filename(filename):
+        return filename in ["stdin", "<stdin>", "-"]
+
     def maybe_fix_tab_width(self, line, col, vv):
         # Adjust column numbers to match the linter's tabs if necessary
         if self.tab_width > 1:
@@ -1274,6 +1335,9 @@ def tmpfile(self, cmd, code, suffix=None):
             suffix = self.get_tempfile_suffix()
 
         with make_temp_file(suffix, code) as file:
+            # store this filename to assign its errors to the main file later
+            self.temp_filename = file.name
+
             ctx = get_view_context(self.view)
             ctx['file_on_disk'] = self.filename
             ctx['temp_file'] = file.name

sublime_linter.py

@@ -176,6 +176,7 @@ def on_pre_close(self, view):
             persist.view_linters.pop(bid, None)
 
             guard_check_linters_for_view.pop(bid, None)
+            affected_filenames_per_bid.pop(bid, None)
             buffer_syntaxes.pop(bid, None)
             queue.cleanup(bid)
 
@@ -289,7 +290,7 @@ def lint(view, view_has_changed, lock, reason=None):
     with remember_runtime(
         "Linting '{}' took {{:.2f}}s".format(util.canonical_filename(view))
     ):
-        sink = partial(update_buffer_errors, bid, view_has_changed)
+        sink = partial(group_by_filename_and_update, bid, view_has_changed)
         backend.lint_view(linters, view, view_has_changed, sink)
 
     events.broadcast(events.LINT_END, {'buffer_id': bid})
@@ -308,6 +309,63 @@ def kill_active_popen_calls(bid):
         proc.friendly_terminated = True
 
 
+affected_filenames_per_bid = defaultdict(lambda: defaultdict(set))
+
+
+def group_by_filename_and_update(bid, view_has_changed, linter, errors):
+    """Group lint errors by filename and update them."""
+    if view_has_changed():  # abort early
+        return
+
+    window = linter.view.window()
+
+    # group all errors by filenames to update them separately
+    grouped = defaultdict(list)
+    for error in errors:
+        grouped[error.get('filename')].append(error)
+
+    # The contract for a simple linter is that it reports `[errors]` or an
+    # empty list `[]` if the buffer is clean. For linters that report errors
+    # for multiple files we collect information about which files are actually
+    # reported by a given `bid` so that we can clean the results. Basically,
+    # we must fake a `[]` response for every filename that is no longer
+    # reported.
+
+    current_filenames = set(grouped.keys())  # `set` for the immutable version
+    previous_filenames = affected_filenames_per_bid[bid][linter.name]
+    clean_files = previous_filenames - current_filenames
+
+    for filename in clean_files:
+        grouped[filename]  # For the side-effect of creating a new empty `list`
+
+    did_update_main_view = False
+    for filename, errors in grouped.items():
+        if not filename:  # backwards compatibility
+            update_buffer_errors(bid, view_has_changed, linter, errors)
+        else:
+            # search for an open view for this file to get a bid
+            view = window.find_open_file(filename)
+            if view:
+                this_bid = view.buffer_id()
+
+                # ignore errors of other files if their view is dirty
+                if this_bid != bid and view.is_dirty() and errors:
+                    continue
+
+                update_buffer_errors(this_bid, view_has_changed, linter, errors)
+
+                if this_bid == bid:
+                    did_update_main_view = True
+
+    # For the main view we MUST *always* report an outcome. This is not for
+    # cleanup but functions as a signal that we're done. Merely for the status
+    # bar view.
+    if not did_update_main_view:
+        update_buffer_errors(bid, view_has_changed, linter, [])
+
+    affected_filenames_per_bid[bid][linter.name] = current_filenames
+
+
 def update_buffer_errors(bid, view_has_changed, linter, errors):
     """Persist lint error changes and broadcast."""
     if view_has_changed():  # abort early

tests/test_filter_results.py

@@ -46,7 +46,7 @@ def tearDown(self):
 
 VIEW_UNCHANGED = lambda: False  # noqa: E731
 execute_lint_task = partial(
-    backend.execute_lint_task, offset=(0, 0), view_has_changed=VIEW_UNCHANGED
+    backend.execute_lint_task, offsets=(0, 0, 0), view_has_changed=VIEW_UNCHANGED
 )