This repository has been archived by the owner on Jan 5, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 137
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Implemented new,
libclang-based, back-end for syntax highlighting s…
…ervice.
This replaces previous `ctags`-based approach which was falling short in some cases and especially with modern C++
codebases. `libclang` compared to `ctags` is a full-featured C++ parser and, when configured properly, is able to
provide us with much more complete and correct results.
Not only that this new back-end is used for syntax highlighting, but it will be certainly reused to implement a whole
other set of features such as refactoring tools, fix-its, indexers, class browsers and alike.
Integration was somewhat more complex and has been changed, so right now syntax highlighting:
* Is triggered on (almost) each text change (highlighting as you type).
* Previously, it was only done after saving changes done on a file.
* Triggering syntax highlighting after each and every character would be too excesive and
for bigger and more complex files probably not even feasible. Hence, a simple heuristic method
has been implemented which tries to identify situations when it is not absolutely necessary to
trigger highlighting (i.e. still typing a word or inserting whitespaces in a way that it doesn't
affect syntax highlighting). Although not perfect, this method certainly minimizes the unnecessary
traffic.
* Requires additional configuration in a form of project-specific compiler flags to get best results:
* i.e. include paths, -Wall, -Werror and whatever else is used
* Configuration is done via `g:project_compiler_args` found in `.yavide_proj` in project root directory.
* Necessary system includes such as `/usr/bin/../lib64/clang/3.8.0/include` are detected automatically and
are not required to be part of the configuration.
* Requires an update of colorscheme as additional highlight groups have been introduced.
* `yaflandia` colorscheme has been accordingly updated
* Should one want to use some other colorscheme, a similar update will be needed in order to make a full
use of syntax highlighting service.
Quirks:
* Scrolling will become really really slow if `cursorline` is set. This is unfortunately very old, very known and
very annoying Vim issue. One can circumvent it by turning it off (`set nocursorline`) for bigger files or turning
it off permanently if one does not require such a feature.- Loading branch information
1 parent
f36050d
commit 9d44c4b
Showing
14 changed files
with
577 additions
and
286 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| import sys | ||
| import logging | ||
| import subprocess | ||
| import clang.cindex | ||
| from services.syntax_highlighter.token_identifier import TokenIdentifier | ||
|
|
||
| def get_system_includes(): | ||
| output = subprocess.Popen(["g++", "-v", "-E", "-x", "c++", "-"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() | ||
| pattern = ["#include <...> search starts here:", "End of search list."] | ||
| output = str(output) | ||
| return output[output.find(pattern[0]) + len(pattern[0]) : output.find(pattern[1])].replace(' ', '-I').split('\\n') | ||
|
|
||
| class ClangTokenizer(): | ||
| def __init__(self): | ||
| self.filename = '' | ||
| self.token_list = [] | ||
| self.index = clang.cindex.Index.create() | ||
| self.default_args = ['-x', 'c++', '-std=c++14'] + get_system_includes() | ||
|
|
||
| def run(self, filename, compiler_args, project_root_directory): | ||
| self.filename = filename | ||
| self.token_list = [] | ||
| logging.info('Filename = {0}'.format(self.filename)) | ||
| logging.info('Default args = {0}'.format(self.default_args)) | ||
| logging.info('User-provided compiler args = {0}'.format(compiler_args)) | ||
| logging.info('Compiler working-directory = {0}'.format(project_root_directory)) | ||
| translation_unit = self.index.parse( | ||
| path = self.filename, | ||
| args = self.default_args + compiler_args + ['-working-directory=' + project_root_directory], | ||
| options = clang.cindex.TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD | ||
| ) | ||
|
|
||
| diag = translation_unit.diagnostics | ||
| for d in diag: | ||
| logging.info('Parsing error: ' + str(d)) | ||
|
|
||
| logging.info('Translation unit: '.format(translation_unit.spelling)) | ||
| self.__visit_all_nodes(translation_unit.cursor) | ||
|
|
||
| def get_token_list(self): | ||
| return self.token_list | ||
|
|
||
| def get_token_id(self, token): | ||
| if token.referenced: | ||
| return ClangTokenizer.to_token_id(token.referenced.kind) | ||
| return ClangTokenizer.to_token_id(token.kind) | ||
|
|
||
| def get_token_name(self, token): | ||
| if (token.referenced): | ||
| return token.referenced.spelling | ||
| else: | ||
| return token.spelling | ||
|
|
||
| def get_token_line(self, token): | ||
| return token.location.line | ||
|
|
||
| def get_token_column(self, token): | ||
| return token.location.column | ||
|
|
||
| def dump_token_list(self): | ||
| for idx, token in enumerate(self.token_list): | ||
| logging.debug( | ||
| '%-12s' % ('[' + str(token.location.line) + ', ' + str(token.location.column) + ']') + | ||
| '%-40s ' % str(token.spelling) + | ||
| '%-40s ' % str(token.kind) + | ||
| ('%-40s ' % str(token.referenced.spelling) if (token.kind.is_reference()) else '') + | ||
| ('%-40s ' % str(token.referenced.kind) if (token.kind.is_reference()) else '')) | ||
|
|
||
| def __visit_all_nodes(self, node): | ||
| for n in node.get_children(): | ||
| if n.location.file and n.location.file.name == self.filename: | ||
| self.token_list.append(n) | ||
| self.__visit_all_nodes(n) | ||
|
|
||
| @staticmethod | ||
| def to_token_id(kind): | ||
| if (kind == clang.cindex.CursorKind.NAMESPACE): | ||
| return TokenIdentifier.getNamespaceId() | ||
| if (kind in [clang.cindex.CursorKind.CLASS_DECL, clang.cindex.CursorKind.CLASS_TEMPLATE, clang.cindex.CursorKind.CLASS_TEMPLATE_PARTIAL_SPECIALIZATION]): | ||
| return TokenIdentifier.getClassId() | ||
| if (kind == clang.cindex.CursorKind.STRUCT_DECL): | ||
| return TokenIdentifier.getStructId() | ||
| if (kind == clang.cindex.CursorKind.ENUM_DECL): | ||
| return TokenIdentifier.getEnumId() | ||
| if (kind == clang.cindex.CursorKind.ENUM_CONSTANT_DECL): | ||
| return TokenIdentifier.getEnumValueId() | ||
| if (kind == clang.cindex.CursorKind.UNION_DECL): | ||
| return TokenIdentifier.getUnionId() | ||
| if (kind == clang.cindex.CursorKind.FIELD_DECL): | ||
| return TokenIdentifier.getFieldId() | ||
| if (kind == clang.cindex.CursorKind.VAR_DECL): | ||
| return TokenIdentifier.getLocalVariableId() | ||
| if (kind in [clang.cindex.CursorKind.FUNCTION_DECL, clang.cindex.CursorKind.FUNCTION_TEMPLATE]): | ||
| return TokenIdentifier.getFunctionId() | ||
| if (kind in [clang.cindex.CursorKind.CXX_METHOD, clang.cindex.CursorKind.CONSTRUCTOR, clang.cindex.CursorKind.DESTRUCTOR]): | ||
| return TokenIdentifier.getMethodId() | ||
| if (kind == clang.cindex.CursorKind.PARM_DECL): | ||
| return TokenIdentifier.getFunctionParameterId() | ||
| if (kind == clang.cindex.CursorKind.TEMPLATE_TYPE_PARAMETER): | ||
| return TokenIdentifier.getTemplateTypeParameterId() | ||
| if (kind == clang.cindex.CursorKind.TEMPLATE_NON_TYPE_PARAMETER): | ||
| return TokenIdentifier.getTemplateNonTypeParameterId() | ||
| if (kind == clang.cindex.CursorKind.TEMPLATE_TEMPLATE_PARAMETER): | ||
| return TokenIdentifier.getTemplateTemplateParameterId() | ||
| if (kind == clang.cindex.CursorKind.MACRO_DEFINITION): | ||
| return TokenIdentifier.getMacroDefinitionId() | ||
| if (kind == clang.cindex.CursorKind.MACRO_INSTANTIATION): | ||
| return TokenIdentifier.getMacroInstantiationId() | ||
| if (kind in [clang.cindex.CursorKind.TYPEDEF_DECL, clang.cindex.CursorKind.TYPE_ALIAS_DECL]): | ||
| return TokenIdentifier.getTypedefId() | ||
| if (kind == clang.cindex.CursorKind.NAMESPACE_ALIAS): | ||
| return TokenIdentifier.getNamespaceAliasId() | ||
| if (kind == clang.cindex.CursorKind.USING_DIRECTIVE): | ||
| return TokenIdentifier.getUsingDirectiveId() | ||
| if (kind == clang.cindex.CursorKind.USING_DECLARATION): | ||
| return TokenIdentifier.getUsingDeclarationId() | ||
| return TokenIdentifier.getUnsupportedId() | ||
|
|
Oops, something went wrong.