Bug 1192290 - Factor up splitlines

dxr/app.py

@@ -28,7 +28,7 @@
 from dxr.plugins import plugins_named
 from dxr.query import Query, filter_menu_items
 from dxr.utils import (non_negative_int, decode_es_datetime, DXR_BLUEPRINT,
-                       format_number, append_update, append_by_line, build_offset_map)
+                       format_number, append_update, append_by_line, build_offset_map, split_content_lines)
 from dxr.vcs import file_contents_at_rev
 
 # Look in the 'dxr' package for static files, etc.:
@@ -552,7 +552,7 @@ def rev(tree, revision, path):
         # We do some wrapping to mimic the JSON returned by an ES lines query.
         return _browse_file(tree,
                             path,
-                            [{'content': line} for line in contents.splitlines(True)],
+                            [{'content': line} for line in split_content_lines(contents)],
                             {},
                             config,
                             not is_text,

dxr/build.py

@@ -29,7 +29,7 @@
 from dxr.lines import es_lines, finished_tags
 from dxr.mime import decode_data, is_binary_image
 from dxr.utils import (open_log, deep_update, append_update,
-                       append_update_by_line, append_by_line, bucket)
+                       append_update_by_line, append_by_line, bucket, split_content_lines)
 from dxr.vcs import VcsCache
 
 
@@ -450,7 +450,7 @@ def index_file(tree, tree_indexers, path, es, index):
     # Index by line if the contents are text and the path is not a symlink.
     index_by_line = is_text and not is_link
     if index_by_line:
-        lines = contents.splitlines(True)
+        lines = split_content_lines(contents)
         num_lines = len(lines)
         needles_by_line = [{} for _ in xrange(num_lines)]
         annotations_by_line = [[] for _ in xrange(num_lines)]

dxr/indexers.py

@@ -9,7 +9,7 @@
 
 from funcy import group_by, decorator, imapcat
 
-from dxr.utils import build_offset_map
+from dxr.utils import build_offset_map, split_content_lines
 
 
 STRING_PROPERTY = {
@@ -329,7 +329,7 @@ def _line_offsets(self):
             if not self.contains_text():
                 raise ValueError("Can't get line offsets for a file that isn't"
                                  " text.")
-            lines = self.contents.splitlines(True) if self.contents is not None else []
+            lines = split_content_lines(self.contents) if self.contents is not None else []
             self._line_offset_list = build_offset_map(lines)
         return self._line_offset_list
 

dxr/plugins/core.py

@@ -20,7 +20,7 @@
 from dxr.plugins import direct_search
 from dxr.trigrammer import (regex_grammar, NGRAM_LENGTH, es_regex_filter,
                             NoTrigrams, PythonRegexVisitor)
-from dxr.utils import glob_to_regex
+from dxr.utils import glob_to_regex, split_content_lines
 
 __all__ = ['mappings', 'analyzers', 'TextFilter', 'PathFilter', 'FilenameFilter',
            'ExtFilter', 'RegexpFilter', 'IdFilter', 'RefFilter']
@@ -477,7 +477,7 @@ def needles(self):
 
     def needles_by_line(self):
         """Fill out line number and content for every line."""
-        for number, text in enumerate(self.contents.splitlines(True), 1):
+        for number, text in enumerate(split_content_lines(self.contents), 1):
             yield [('number', number),
                    ('content', text)]
 

dxr/plugins/python/utils.py

@@ -2,7 +2,7 @@
 import os
 import re
 from contextlib import contextmanager
-
+from dxr.utils import split_content_lines
 
 # The actual check that Python uses seems to be done in C, but this
 # regex was taken from lib2to3.pgen.tokenize.
@@ -21,7 +21,7 @@ def ast_parse(contents):
         u''.join(
             # The encoding declaration is only meaningful in the top two lines.
             u'\n' if i < 2 and encoding_re.match(line) else line
-            for i, line in enumerate(contents.splitlines(True))
+            for i, line in enumerate(split_content_lines(contents))
         )
     )
 

dxr/plugins/xpidl/indexers.py

@@ -3,6 +3,7 @@
 
 import dxr.indexers
 from dxr.indexers import iterable_per_line, with_start_and_end, split_into_lines
+from dxr.utils import split_content_lines
 from dxr.plugins.xpidl.filters import PLUGIN_NAME
 from dxr.plugins.xpidl.visitor import IdlVisitor
 
@@ -24,7 +25,7 @@ def idl(self):
         # Don't try again if we already excepted.
         if not self._idl and not self._had_idl_exception:
             try:
-                self._idl = IdlVisitor(self.parser, self.contents, self.contents.splitlines(True),
+                self._idl = IdlVisitor(self.parser, self.contents, split_content_lines(self.contents),
                                        self.path, self.absolute_path(),
                                        self.plugin_config.include_folders,
                                        self.plugin_config.header_path, self.tree)

dxr/utils.py

@@ -285,3 +285,7 @@ def is_in(needle, haystack):
 def without_ending(ending, string):
     """If ``string`` ends with ``ending``, strip it off."""
     return string[:-len(ending)] if string.endswith(ending) else string
+
+def split_content_lines(string):
+    """Line breaks are included in the resulting list"""
+    return string.splitlines(True)