Merge branch 'development' into refactor/st3plus

plugin.py

@@ -4,31 +4,31 @@
 import sublime
 
 if int(sublime.version()) < 3143:
-	print("CTags requires Sublime Text 3143+")
+    print("CTags requires Sublime Text 3143+")
 
 else:
-	import sys
+    import sys
 
-	# Clear module cache to force reloading all modules of this package.
-	prefix = __package__ + "."  # don't clear the base package
-	for module_name in [
-	    module_name
-	    for module_name in sys.modules
-	    if module_name.startswith(prefix) and module_name != __name__
-	]:
-	    del sys.modules[module_name]
-	del prefix
-	del sys
+    # Clear module cache to force reloading all modules of this package.
+    prefix = __package__ + "."  # don't clear the base package
+    for module_name in [
+        module_name
+        for module_name in sys.modules
+        if module_name.startswith(prefix) and module_name != __name__
+    ]:
+        del sys.modules[module_name]
+    del prefix
+    del sys
 
-	# Publish Commands and EventListeners
-	from .plugins.cmds import (
-		CTagsAutoComplete,
-		JumpPrev,
-		NavigateToDefinition,
-		RebuildTags,
-		SearchForDefinition,
-		ShowSymbols,
-		TestCtags,
-	)
+    # Publish Commands and EventListeners
+    from .plugins.cmds import (
+        CTagsAutoComplete,
+        JumpPrev,
+        NavigateToDefinition,
+        RebuildTags,
+        SearchForDefinition,
+        ShowSymbols,
+        TestCtags,
+    )
 
-	from .plugins.edit import apply_edit
+    from .plugins.edit import apply_edit

plugins/cmds.py

@@ -325,7 +325,7 @@ def follow_tag_path(view, tag_path, pattern):
 
     # find the ex_command pattern
     pattern_region = find_source(
-        view, '^' + escape_regex(pattern) + '$', start_at, flags=0)
+        view, r'^' + escape_regex(pattern), start_at, flags=0)
 
     if setting('debug'):  # leave a visual trail for easy debugging
         regions = regions + ([pattern_region] if pattern_region else [])
@@ -419,7 +419,7 @@ def compile_lists(sorter):
 # File collection helper functions
 
 
-def get_rel_path_to_source(path, tag_file, multiple=True):
+def get_rel_path_to_source(path, tag_file):
     """
     Get relative path from tag_file to source file.
 
@@ -429,14 +429,11 @@ def get_rel_path_to_source(path, tag_file, multiple=True):
 
     :returns: list containing relative path from tag_file to source file
     """
-    if multiple:
-        return []
-
     tag_dir = os.path.dirname(tag_file)  # get tag directory
-    common_prefix = os.path.commonprefix([tag_dir, path])
+    common_prefix = os.path.commonprefix((tag_dir, path))
     relative_path = os.path.relpath(path, common_prefix)
 
-    return [relative_path]
+    return relative_path
 
 
 def get_current_file_suffix(path):
@@ -746,20 +743,28 @@ def run(self, view, args, tags_file):
         if not tags_file:
             return
 
-        multi = args.get('type') == 'multi'
-        lang = args.get('type') == 'lang'
-
-        if view.file_name():
-            files = get_rel_path_to_source(
-                view.file_name(), tags_file, multi)
+        symbol_type = args.get('type')
+        multi = symbol_type == 'multi'
+        lang = symbol_type == 'lang'
 
         if lang:
+            # filter and cache by file suffix
             suffix = get_current_file_suffix(view.file_name())
             key = suffix
+        elif multi:
+            # request all symbols of given tags file
+            key = "__all__"
+            files = []
         else:
-            key = ','.join(files)
+            # request symbols of current view's file
+            key = view.file_name()
+            if not key:
+                return
+            key = get_rel_path_to_source(key, tags_file)
+            key = key.replace('\\', '/')
+
+            files = [key]
 
-        tags_file = tags_file + '_sorted_by_file'
         base_path = get_common_ancestor_folder(
             view.file_name(), view.window().folders())
 
@@ -895,66 +900,51 @@ def tags_built(tag_file):
 
             tags_built(result)
 
-        GetAllCTagsList.ctags_list = []  # clear the cached ctags list
+        if tag_file in ctags_completions:
+            del ctags_completions[tag_file]  # clear the cached ctags list
 
 # Autocomplete commands
 
 
-class GetAllCTagsList():
-    """
-    Cache all the ctags list.
-    """
-    ctags_list = []
-
-    def __init__(self, list):
-        self.ctags_list = list
+ctags_completions = {}
 
 
 class CTagsAutoComplete(sublime_plugin.EventListener):
 
     def on_query_completions(self, view, prefix, locations):
-        if setting('autocomplete'):
-            prefix = prefix.strip().lower()
-            tags_path = view.window().folders()[0] + '/' + setting('tag_file')
+        if not setting('autocomplete'):
+            return None
 
-            sub_results = [v.extract_completions(prefix)
-                           for v in sublime.active_window().views()]
-            sub_results = [(item, item) for sublist in sub_results
-                           for item in sublist]  # flatten
+        prefix = prefix.lower()
 
-            if GetAllCTagsList.ctags_list:
-                results = [sublist for sublist in GetAllCTagsList.ctags_list
-                           if sublist[0].lower().startswith(prefix)]
-                results = sorted(set(results).union(set(sub_results)))
+        tags_path = find_tags_relative_to(
+            view.file_name(), setting('tag_file'))
 
-                return results
-            else:
-                tags = []
+        if not tags_path:
+            return None
 
-                # check if a project is open and the tags file exists
-                if not (view.window().folders() and os.path.exists(tags_path)):
-                    return tags
+        if not os.path.exists(tags_path):
+            return None
 
-                if sublime.platform() == "windows":
-                    prefix = ""
-                else:
-                    prefix = "\\"
+        if os.path.getsize(tags_path) > 100 * 1024 * 1024:
+            return None
+
+        if tags_path not in ctags_completions:
+            tags = set()
 
-                f = os.popen(
-                    "awk \"{ print " + prefix + "$1 }\" \"" + tags_path + "\"")
+            with open(tags_path, "r", encoding="utf-8") as fobj:
+                for line in fobj:
+                    line = line.strip()
+                    if not line or line.startswith("!_TAG"):
+                        continue
+                    cols = line.split("\t", 1)
+                    tags.add(cols[0])
 
-                for i in f.readlines():
-                    tags.append([i.strip()])
+            ctags_completions[tags_path] = tags
 
-                tags = [(item, item) for sublist in tags
-                        for item in sublist]  # flatten
-                tags = sorted(set(tags))  # make unique
-                GetAllCTagsList.ctags_list = tags
-                results = [sublist for sublist in GetAllCTagsList.ctags_list
-                           if sublist[0].lower().startswith(prefix)]
-                results = sorted(set(results).union(set(sub_results)))
+        return [tag for tag in ctags_completions[tags_path]
+                if tag.lower().startswith(prefix)]
 
-                return results
 
 # Test CTags commands
 

plugins/ctags.py

@@ -338,7 +338,7 @@ def resort_ctags(tag_file):
             If not exists, create an empty array and store in the
                 dictionary with the file name as key
             Save the line to this list
-        Create a new ``[tagfile]_sorted_by_file`` file
+        Create a new ``tagfile`` file
         For each key in the sorted dictionary
             For each line in the list indicated by the key
                 Split the line on tab character
@@ -350,19 +350,39 @@ def resort_ctags(tag_file):
 
     :returns: None
     """
-    keys = {}
+    meta = []
+    symbols = []
+    tmp_file = tag_file + '.tmp'
 
     with codecs.open(tag_file, encoding='utf-8', errors='replace') as file_:
         for line in file_:
-            keys.setdefault(line.split('\t')[FILENAME], []).append(line)
+            if line.startswith('!_TAG'):
+                meta.append(line)
+                continue
+
+            # read all valid symbol tags, which contain at least 
+            # symbol name and containing file and build a list of tuples
+            split = line.split('\t')
+            if len(split) > FILENAME:
+                symbols.append((split[FILENAME], split))
+
+    # sort inplace to save some RAM with large .tags files
+    meta.sort()
+    symbols.sort()
 
-    with codecs.open(tag_file+'_sorted_by_file', 'w', encoding='utf-8',
+    with codecs.open(tmp_file, 'w', encoding='utf-8',
                      errors='replace') as file_:
-        for k in sorted(keys):
-            for line in keys[k]:
-                split = line.split('\t')
-                split[FILENAME] = split[FILENAME].lstrip('.\\')
-                file_.write('\t'.join(split))
+
+        # write sourted metadata
+        file_.writelines(meta)
+
+        # followed by sorted list of symbols
+        for _, split in symbols:
+            split[FILENAME] = split[FILENAME].lstrip('.\\')
+            file_.write('\t'.join(split))
+
+    os.remove(tag_file)
+    os.rename(tmp_file, tag_file)
 
 #
 # Models
@@ -390,16 +410,26 @@ def __init__(self, line, column=0):
         self.column = column
 
     def __lt__(self, other):
-        return self.line.split('\t')[self.column] < other
+        try:
+            return self.key < other
+        except IndexError:
+            return False
 
     def __gt__(self, other):
-        return self.line.split('\t')[self.column] > other
+        try:
+            return self.key > other
+        except IndexError:
+            return False
 
     def __getitem__(self, index):
-        return self.line.split('\t')[index]
+        return self.line.split('\t', self.column + 1)[index]
 
     def __len__(self):
-        return len(self.line.split('\t'))
+        return self.line.count('\t') + 1
+
+    @property
+    def key(self):
+        return self[self.column]
 
 class TagFile(object):
     """
@@ -440,6 +470,8 @@ def __getitem__(self, index):
             result = self.mapped.readline()  # get a complete line
 
         result = result.strip()
+        if not result:
+            raise IndexError("Invalid tag at index %d." % index)
 
         return Tag(result, self.column)
 
@@ -473,7 +505,7 @@ def open(self):
         """
         Open file.
         """
-        self.file_o = codecs.open(self.path, 'r+b', encoding='ascii')
+        self.file_o = codecs.open(self.path, 'r+b', encoding='utf-8')
         self.mapped = mmap.mmap(self.file_o.fileno(), 0,
                                 access=mmap.ACCESS_READ)
 
@@ -497,20 +529,21 @@ def search(self, exact_match=True, *tags):
         if not tags:
             while self.mapped.tell() < self.mapped.size():
                 result = Tag(self.mapped.readline().strip(), self.column)
-                yield(result)
+                if result.line:
+                    yield result
             return
 
         for key in tags:
             left_index = bisect.bisect_left(self, key)
             if exact_match:
                 result = self[left_index]
                 while result.line and result[result.column] == key:
-                    yield(result)
+                    yield result
                     result = Tag(self.mapped.readline().strip(), self.column)
             else:
                 result = self[left_index]
                 while result.line and result[result.column].startswith(key):
-                    yield(result)
+                    yield result
                     result = Tag(self.mapped.readline().strip(), self.column)
 
     def search_by_suffix(self, suffix):
@@ -526,10 +559,9 @@ def search_by_suffix(self, suffix):
         :returns: matching tags
         """
         for line in self.file_o:
-            if line.split('\t')[self.column].endswith(suffix):
-                yield Tag(line)
-            else:
-                continue
+            tag = Tag(line, self.column)
+            if tag.key.endswith(suffix):
+                yield tag
 
     def tag_class(self):
         """

plugins/tests/test_ctagsplugin.py

@@ -202,24 +202,22 @@ def test_get_rel_path_to_source__source_file_in_sibling_directory(self):
         temp = '/c/users/temporary_file'
         tag_file = '/c/users/tags'
 
-        result = cmds.get_rel_path_to_source(
-            temp, tag_file, multiple=False)
+        result = cmds.get_rel_path_to_source(temp, tag_file)
 
         relative_path = 'temporary_file'
 
-        self.assertEqual([relative_path], result)
+        self.assertEqual(relative_path, result)
 
     def test_get_rel_path_to_source__source_file_in_child_directory(self):
         temp = '/c/users/folder/temporary_file'
         tag_file = '/c/users/tags'
 
-        result = cmds.get_rel_path_to_source(
-            temp, tag_file, multiple=False)
+        result = cmds.get_rel_path_to_source(temp, tag_file)
 
         # handle [windows, unix] paths
         relative_paths = ['folder\\temporary_file', 'folder/temporary_file']
 
-        self.assertIn(result[0], relative_paths)
+        self.assertIn(result, relative_paths)
 
 if __name__ == '__main__':
     unittest.main()