From 5dbffa3a48cc2ef7b1f273fe8dcc9a53fcf9d8f5 Mon Sep 17 00:00:00 2001 From: Akshay Ganeshen Date: Tue, 19 Sep 2017 17:35:14 -0400 Subject: [PATCH] Cleaned up and refactored. Split apart large modules. Modified the logging system to hook into a formatter, instead of trying to modify records with a filter. This makes it a lot less fragile. Split the server logic into multiple smaller modules. Split the sublime-utility logic into multiple smaller modules. Added a dedicated schema module to define the json structures. --- .coveragerc | 20 + lib/process/__init__.py | 11 + lib/process/filehandles.py | 99 +++ lib/{ => process}/process.py | 123 +--- lib/schema/__init__.py | 12 + lib/{jsonmodels.py => schema/completions.py} | 255 +------- lib/schema/request.py | 246 +++++++ lib/subl/__init__.py | 6 + lib/subl/constants.py | 48 ++ lib/subl/dummy.py | 64 ++ lib/subl/settings.py | 274 ++++++++ lib/{st.py => subl/view.py} | 435 ++----------- lib/util/__init__.py | 6 + lib/util/format.py | 76 +++ lib/{ => util}/fs.py | 15 +- lib/util/hmac.py | 76 +++ lib/{logutils.py => util/log.py} | 328 +++++----- lib/{strutils.py => util/str.py} | 108 +--- lib/ycmd.py | 642 ------------------- lib/ycmd/__init__.py | 6 + lib/ycmd/constants.py | 96 +++ lib/ycmd/server.py | 377 +++++++++++ lib/ycmd/settings.py | 102 +++ lib/ycmd/start.py | 198 ++++++ runtests.py | 25 +- syplugin.py | 128 ++-- tests/test_fs.py | 25 +- tests/test_logutils.py | 99 ++- tests/test_process.py | 31 +- tests/utils.py | 5 +- 30 files changed, 2161 insertions(+), 1775 deletions(-) create mode 100755 .coveragerc create mode 100644 lib/process/__init__.py create mode 100644 lib/process/filehandles.py rename lib/{ => process}/process.py (58%) create mode 100644 lib/schema/__init__.py rename lib/{jsonmodels.py => schema/completions.py} (55%) create mode 100644 lib/schema/request.py create mode 100644 lib/subl/__init__.py create mode 100644 lib/subl/constants.py create mode 100644 lib/subl/dummy.py create mode 100644 lib/subl/settings.py rename lib/{st.py => subl/view.py} (54%) create mode 100644 lib/util/__init__.py create mode 100644 lib/util/format.py rename lib/{ => util}/fs.py (97%) create mode 100644 lib/util/hmac.py rename lib/{logutils.py => util/log.py} (54%) rename lib/{strutils.py => util/str.py} (54%) delete mode 100644 lib/ycmd.py create mode 100644 lib/ycmd/__init__.py create mode 100644 lib/ycmd/constants.py create mode 100644 lib/ycmd/server.py create mode 100644 lib/ycmd/settings.py create mode 100644 lib/ycmd/start.py diff --git a/.coveragerc b/.coveragerc new file mode 100755 index 0000000..4b01423 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,20 @@ +[run] +branch = True +source = + lib + +[paths] +source = + lib/ + +[report] +exclude_lines = + pragma: no cover + def __repr__ + raise NotImplementedError + +omit = + */__init__.py + */tests/* + +ignore_errors = True diff --git a/lib/process/__init__.py b/lib/process/__init__.py new file mode 100644 index 0000000..7310155 --- /dev/null +++ b/lib/process/__init__.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 + +''' +lib/process +sublime-ycmd process management module. +''' + +from lib.process.process import Process # noqa +from lib.process.filehandles import FileHandles # noqa + +__all__ = ['process', 'filehandles'] diff --git a/lib/process/filehandles.py b/lib/process/filehandles.py new file mode 100644 index 0000000..9095c43 --- /dev/null +++ b/lib/process/filehandles.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 + +''' +lib/process/process.py +Process file handle wrapper class. + +Provides a way to customize the stdin, stdout, and stderr file handles of a +process before launching it. By default, all handles will be closed, which +means it would be impossible to write data to it or read output from it. +''' + +import io +import logging +import subprocess + +logger = logging.getLogger('sublime-ycmd.' + __name__) + + +class FileHandles(object): + ''' + Container class for process file handles (stdin, stdout, stderr). + Provides an option to set up PIPEs when starting processes, and then allows + reading/writing to those pipes with helper methods. + ''' + + PIPE = subprocess.PIPE + DEVNULL = subprocess.DEVNULL + STDOUT = subprocess.STDOUT + + def __init__(self): + self._stdin = None + self._stdout = None + self._stderr = None + + def configure_filehandles(self, stdin=None, stdout=None, stderr=None): + ''' + Sets the file handle behaviour for the launched process. Each handle + can be assigned one of the subprocess handle constants: + stdin = None, PIPE + stdout = None, PIPE, DEVNULL + stderr = None, PIPE, DEVNULL, STDOUT + These value gets forwarded directly to Popen. + ''' + self.stdin = stdin + self.stdout = stdout + self.stderr = stderr + + @staticmethod + def valid_filehandle(handle): + ''' Validates that a handle is `None`, or a file-like object. ''' + if handle is None or isinstance(handle, io.IOBase): + return True + + # explicitly allow the special flags as well + return handle in [ + FileHandles.PIPE, + FileHandles.DEVNULL, + FileHandles.STDOUT, + ] + + @property + def stdin(self): + ''' Returns the configured stdin handle. ''' + return self._stdin + + @stdin.setter + def stdin(self, stdin): + ''' Sets the stdin file handle. ''' + if not self.valid_filehandle(stdin): + raise ValueError('stdin handle must be a file instance') + if stdin == self.STDOUT: + # what? + raise ValueError('stdin handle cannot be redirected to stdout') + self._stdin = stdin + + @property + def stdout(self): + ''' Returns the configured stdout handle. ''' + return self._stdout + + @stdout.setter + def stdout(self, stdout): + if not self.valid_filehandle(stdout): + raise ValueError('stdout handle must be a file instance') + # do not allow STDOUT + if stdout == self.STDOUT: + raise ValueError('stdout handle cannot be redirected to stdout') + self._stdout = stdout + + @property + def stderr(self): + ''' Returns the configured stderr handle. ''' + return self._stderr + + @stderr.setter + def stderr(self, stderr): + if not self.valid_filehandle(stderr): + raise ValueError('stderr handle must be a file instance') + self._stderr = stderr diff --git a/lib/process.py b/lib/process/process.py similarity index 58% rename from lib/process.py rename to lib/process/process.py index d1b2eec..fd084d9 100644 --- a/lib/process.py +++ b/lib/process/process.py @@ -1,106 +1,27 @@ #!/usr/bin/env python3 ''' -lib/process.py -Contains utilities for working with processes. This is used to start and manage -the ycmd server process. +lib/process/process.py +Process wrapper class. + +Provides utilities for managing processes. This is useful for starting the ycmd +server process, checking if it's still alive, and shutting it down. ''' -import io import logging -import os import subprocess -from lib.fs import ( +from lib.process.filehandles import FileHandles +from lib.util.fs import ( + is_directory, + is_file, resolve_binary_path, ) logger = logging.getLogger('sublime-ycmd.' + __name__) -class SYfileHandles(object): - ''' - Container class for process file handles (stdin, stdout, stderr). - Provides an option to set up PIPEs when starting processes, and then allows - reading/writing to those pipes with helper methods. - ''' - - PIPE = subprocess.PIPE - DEVNULL = subprocess.DEVNULL - STDOUT = subprocess.STDOUT - - def __init__(self): - self._stdin = None - self._stdout = None - self._stderr = None - - def configure_filehandles(self, stdin=None, stdout=None, stderr=None): - ''' - Sets the file handle behaviour for the launched process. Each handle - can be assigned one of the subprocess handle constants: - stdin = None, PIPE - stdout = None, PIPE, DEVNULL - stderr = None, PIPE, DEVNULL, STDOUT - These value gets forwarded directly to Popen. - ''' - self.stdin = stdin - self.stdout = stdout - self.stderr = stderr - - @staticmethod - def valid_filehandle(handle): - ''' Validates that a handle is None, or a file-like object. ''' - if handle is None or isinstance(handle, io.IOBase): - return True - # explicitly allow the special flags as well - if handle in [SYfileHandles.PIPE, - SYfileHandles.DEVNULL, - SYfileHandles.STDOUT]: - return True - return False - - @property - def stdin(self): - ''' Returns the configured stdin handle. ''' - return self._stdin - - @stdin.setter - def stdin(self, stdin): - ''' Sets the stdin file handle. ''' - if not self.valid_filehandle(stdin): - raise ValueError('stdin handle must be a file instance') - if stdin == self.STDOUT: - # what? - raise ValueError('stdin handle cannot be redirected to stdout') - self._stdin = stdin - - @property - def stdout(self): - ''' Returns the configured stdout handle. ''' - return self._stdout - - @stdout.setter - def stdout(self, stdout): - if not self.valid_filehandle(stdout): - raise ValueError('stdout handle must be a file instance') - # do not allow STDOUT - if stdout == self.STDOUT: - raise ValueError('stdout handle cannot be redirected to stdout') - self._stdout = stdout - - @property - def stderr(self): - ''' Returns the configured stderr handle. ''' - return self._stderr - - @stderr.setter - def stderr(self, stderr): - if not self.valid_filehandle(stderr): - raise ValueError('stderr handle must be a file instance') - self._stderr = stderr - - -class SYprocess(object): +class Process(object): ''' Process class This class represents a managed process. @@ -111,7 +32,7 @@ def __init__(self): self._args = None self._env = None self._cwd = None - self._filehandles = SYfileHandles() + self._filehandles = FileHandles() self._handle = None @@ -125,7 +46,7 @@ def binary(self, binary): ''' Sets the process binary. ''' assert isinstance(binary, str), 'binary must be a string: %r' % binary binary = resolve_binary_path(binary) - assert os.path.isfile(binary), 'binary path invalid: %r' % binary + assert is_file(binary), 'binary path invalid: %r' % binary logger.debug('setting binary to: %s', binary) self._binary = binary @@ -144,6 +65,8 @@ def args(self, args): logger.warning('process already started... no point setting args') assert hasattr(args, '__iter__'), 'args must be iterable: %r' % args + # collect the arguments, so we don't exhaust the iterator + args = list(args) if self._args is not None: logger.warning('overwriting existing process args: %r', self._args) @@ -153,7 +76,7 @@ def args(self, args): @property def env(self): - ''' Returns the process env. Initializes it if it is None. ''' + ''' Returns the process env. Initializes it if it is `None`. ''' if self._env is None: self._env = {} return self._env @@ -184,7 +107,7 @@ def cwd(self, cwd): logger.warning('process already started... no point setting cwd') assert isinstance(cwd, str), 'cwd must be a string: %r' % cwd - if not os.path.isdir(cwd): + if not is_directory(cwd): logger.warning('invalid working directory: %s', cwd) logger.debug('setting cwd to: %s', cwd) @@ -235,15 +158,15 @@ def communicate(self, inpt=None, timeout=None): ''' Sends data via stdin and reads data from stdout, stderr. This will likely block if the process is still alive. - When input is None, stdin is immediately closed. - When timeout is None, this waits indefinitely for the process to + When `inpt` is `None`, stdin is immediately closed. + When `timeout` is `None`, this waits indefinitely for the process to terminate. Otherwise, it is interpreted as the number of seconds to - wait for, until raising a TimeoutExpired exception. + wait for until raising a `TimeoutExpired` exception. ''' if not self.alive(): logger.debug('process not alive, unlikely to block') - assert self._handle is not None + assert self._handle is not None, '[internal] process handle is null' return self._handle.communicate(inpt, timeout) def wait(self, maxwait=10): @@ -252,14 +175,14 @@ def wait(self, maxwait=10): logger.debug('process not alive, nothing to wait for') return - assert self._handle is not None + assert self._handle is not None, '[internal] process handle is null' self._handle.wait(maxwait) def kill(self): - ''' Kills the associated process, by sending a signal. ''' + ''' Kills the associated process by sending a signal. ''' if not self.alive(): logger.debug('process is already dead, not sending signal') return - assert self._handle is not None + assert self._handle is not None, '[internal] process handle is null' self._handle.kill() diff --git a/lib/schema/__init__.py b/lib/schema/__init__.py new file mode 100644 index 0000000..2509667 --- /dev/null +++ b/lib/schema/__init__.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 + +''' +lib/schema +sublime-ycmd json schema definitions. +''' + +from lib.schema.request import RequestParameters # noqa +from lib.schema.completions import ( # noqa + Completions, + CompletionOption, +) diff --git a/lib/jsonmodels.py b/lib/schema/completions.py similarity index 55% rename from lib/jsonmodels.py rename to lib/schema/completions.py index f24ae32..c035d35 100644 --- a/lib/jsonmodels.py +++ b/lib/schema/completions.py @@ -1,235 +1,24 @@ #!/usr/bin/env python3 ''' -lib/jsonmodels.py -Wrappers around the JSON messages passed between this plugin and ycmd. Provides -concrete classes with attributes to make it easier to work with the arbitrary -keys in the ycmd API. +lib/schema/completions.py +Schema definition for responses from completion requests. ''' import logging -from lib.strutils import ( - parse_json, -) +from lib.schema.request import RequestParameters +from lib.util.format import json_parse logger = logging.getLogger('sublime-ycmd.' + __name__) -class SYrequestParameters(object): +class Completions(object): ''' - Wrapper around JSON parameters used in ycmd requests. - All requests need to have certain parameters for the ycmd server to even - consider handling. These parameters are given default values if they are - not filled in by the time it gets serialized to JSON. Setting parameters - will also automatically validate that they are the right type. - - TODO : - Certain handlers use additional parameters, e.g. event notifications - require the event type as part of the request body. These parameters can - also get checked when specifying the target handler. - ''' - - def __init__(self, - file_path=None, file_contents=None, file_types=None, - line_num=None, column_num=None): - self.clear() - self.file_path = file_path - self.file_contents = file_contents - self.file_types = file_types - self.line_num = line_num - self.column_num = column_num - - def clear(self): - ''' Deletes all stored parameters. ''' - self._file_path = None - self._file_contents = None - self._file_types = None - self._line_num = None - self._column_num = None - self._extra_params = {} - - def to_json(self): - file_path = self.file_path - file_contents = self.file_contents - file_types = self.file_types - line_num = self.line_num - column_num = self.column_num - extra_params = self._extra_params - - # validate - if not file_path: - raise ValueError('no file path specified') - if not isinstance(file_path, str): - raise TypeError('file path must be a str: %r' % (file_path)) - - if not file_contents: - file_contents = '' - if not isinstance(file_contents, str): - raise TypeError( - 'file contents must be a str: %r' % (file_contents) - ) - - if file_types is None: - file_types = [] - if not isinstance(file_types, (tuple, list)): - raise TypeError('file types must be a list: %r' % (file_types)) - - if line_num is None: - line_num = 1 - if not isinstance(line_num, int): - raise TypeError('line num must be an int: %r' % (line_num)) - - if column_num is None: - column_num = 1 - if not isinstance(column_num, int): - raise TypeError('column num must be an int: %r' % (column_num)) - - if extra_params is None: - extra_params = {} - if not isinstance(extra_params, dict): - raise TypeError( - 'extra parameters must be a dict: %r' % (extra_params) - ) - - json_params = { - 'filepath': file_path, - 'file_data': { - file_path: { - 'filetypes': file_types, - 'contents': file_contents, - }, - }, - - 'line_num': line_num, - 'column_num': column_num, - } - json_params.update(extra_params) - - return json_params - - @property - def handler(self): - if not self._handler: - logger.warning('no handler set') - return '' - return self._handler - - @handler.setter - def handler(self, handler): - if not isinstance(handler, str): - raise TypeError - self._handler = handler - - @property - def file_path(self): - if not self._file_path: - logger.warning('no file path set') - return '' - return self._file_path - - @file_path.setter - def file_path(self, file_path): - if not isinstance(file_path, str): - raise TypeError - self._file_path = file_path - - @property - def file_contents(self): - if not self._file_contents: - logger.warning('no file contents set') - return '' - return self._file_contents - - @file_contents.setter - def file_contents(self, file_contents): - if not isinstance(file_contents, str): - raise TypeError - self._file_contents = file_contents - - @property - def file_types(self): - if not self._file_types: - logger.warning('no file types set') - return [] - return self._file_types - - @file_types.setter - def file_types(self, file_types): - if isinstance(file_types, str): - file_types = [file_types] - if not isinstance(file_types, (tuple, list)): - raise TypeError - # create a shallow copy - self._file_types = list(file_types) - - @property - def line_num(self): - if not self._line_num: - logger.warning('no line number set') - return 1 - return self._line_num - - @line_num.setter - def line_num(self, line_num): - if not isinstance(line_num, int): - raise TypeError - if line_num <= 0: - raise ValueError - self._line_num = line_num - - @property - def column_num(self): - if not self._column_num: - logger.warning('no column number set') - return 1 - return self._column_num - - @column_num.setter - def column_num(self, column_num): - if not isinstance(column_num, int): - raise TypeError - if column_num <= 0: - raise ValueError - self._column_num = column_num - - def __getitem__(self, key): - ''' Retrieves `key` from the extra parameters. ''' - if self._extra_params is None: - self._extra_params = {} - return self._extra_params[key] - - def get(self, key, default=None): - ''' - Retrieves `key` from the extra parameters. Returns `default` if unset. - ''' - if self._extra_params is None: - self._extra_params = {} - return self._extra_params.get(key, default) - - def __setitem__(self, key, value): - ''' - Sets `key` in the extra parameters. These parameters have higher - priority than the file-based parameters, and may overwrite them if the - same key is used. - ''' - if self._extra_params is None: - self._extra_params = {} - self._extra_params[key] = value - - def __delitem__(self, key): - ''' Clears the `key` extra parameter. ''' - if self._extra_params is None: - return - del self._extra_params[key] - - -class SYcompletions(object): - ''' - Wrapper around the JSON response received from ycmd completions. + Wrapper around the json response received from ycmd completions. Contains top-level metadata like where the completion was requested, and what prefix was matched by ycmd. This class also acts as a collection for - individual `SYcompletionOption` instances, which act as the possible + individual `CompletionOption` instances, which act as the possible choices for finishing the current identifier. This class behaves like a list. The completion options are ordered by ycmd, @@ -270,9 +59,9 @@ def __repr__(self): ) -class SYcompletionOption(object): +class CompletionOption(object): ''' - Wrapper around individual JSON entries received from ycmd completions. + Wrapper around individual json entries received from ycmd completions. All completion options have metadata indicating what kind of symbol it is, and how they can be displayed. This base class is used to define the common attributes available in all completion options. Subclasses further include @@ -349,7 +138,7 @@ def __repr__(self): } if self._file_types: repr_params['file_types'] = self._file_types - return '' % (repr_params) + return '' % (repr_params) def _has_file_type(self, file_type): if self._file_types is None: @@ -364,7 +153,7 @@ def _has_file_type(self, file_type): def _parse_completion_option(node, file_types=None): ''' Parses a single item in the completions list at `node` into an - `SYcompletionOption` instance. + `CompletionOption` instance. If `file_types` is provided, it should be a list of strings indicating the file types of the original source code. This will be used to post-process and normalize the ycmd descriptions depending on the syntax. @@ -380,33 +169,33 @@ def _parse_completion_option(node, file_types=None): extra_data = node.get('extra_data', None) detailed_info = node.get('detailed_info', None) - return SYcompletionOption( + return CompletionOption( menu_info=menu_info, insertion_text=insertion_text, extra_data=extra_data, detailed_info=detailed_info, file_types=file_types, ) -def parse_completion_options(json, request_parameters=None): +def parse_completions(json, request_parameters=None): ''' - Parses a `json` response from ycmd into an `SYcompletions` instance. - This expects a certain format in the input JSON, or it won't be able to + Parses a `json` response from ycmd into an `Completions` instance. + This expects a certain format in the input json, or it won't be able to properly build the completion options. If `request_parameters` is provided, it should be an instance of - `SYrequestParameters`. It may be used to post-process the completion - options depending on the syntax of the file. For example, this will attempt - to normalize differences in the way ycmd displays functions. + `RequestParameters`. It may be used to post-process the completion options + depending on the syntax of the file. For example, this will attempt to + normalize differences in the way ycmd displays functions. ''' assert isinstance(json, (str, bytes, dict)), \ 'json must be a dict: %r' % (json) assert request_parameters is None or \ - isinstance(request_parameters, SYrequestParameters), \ - 'request parameters must be SYrequestParameters: %r' % \ + isinstance(request_parameters, RequestParameters), \ + 'request parameters must be RequestParameters: %r' % \ (request_parameters) if isinstance(json, (str, bytes)): logger.debug('parsing json string into a dict') - json = parse_json(json) + json = json_parse(json) if 'errors' not in json or not isinstance(json['errors'], list): raise TypeError('json is missing "errors" list') @@ -440,7 +229,7 @@ def _is_completions(completions): # just assume it's an int start_column = json_start_column - return SYcompletions( + return Completions( completion_options=completion_options, start_column=start_column, ) diff --git a/lib/schema/request.py b/lib/schema/request.py new file mode 100644 index 0000000..a080712 --- /dev/null +++ b/lib/schema/request.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 + +''' +lib/schema/request.py +Schema definition for request parameters. + +The request parameter class wraps the json parameters required in most ycmd +handlers. These parameters are required for the ycmd server to even consider +handling. Without them, an error response is sent during the validation stage. + +Some handlers will end up ignoring these required parameters, which is slightly +annoying. In that case, this class is able to fill in default values if they +are not filled in by the time it gets serialized to json. Setting parameters +will also automatically validate that they are the correct type. + +TODO : +Certain handlers use additional parameters, e.g. event notifications require +the event type as part of the request body. These parameters can also get +checked when specifying the target handler. +''' + +import logging + +logger = logging.getLogger('sublime-ycmd.' + __name__) + + +class RequestParameters(object): + ''' + Wrapper around json parameters used in ycmd requests. Supports arbitrary + extra parameters using a `dict`-like interface. + ''' + + def __init__(self, file_path=None, file_contents=None, file_types=None, + line_num=None, column_num=None): + self.file_path = file_path + self.file_contents = file_contents + self.file_types = file_types + self.line_num = line_num + self.column_num = column_num + self._extra_params = {} + + def reset(self): + ''' Deletes all stored parameters. ''' + self._file_path = None + self._file_contents = None + self._file_types = None + self._line_num = None + self._column_num = None + self._extra_params = {} + + def to_json(self): + file_path = self.file_path + file_contents = self.file_contents + file_types = self.file_types + line_num = self.line_num + column_num = self.column_num + extra_params = self._extra_params + + # validate + if not file_path: + raise ValueError('no file path specified') + if not isinstance(file_path, str): + raise TypeError('file path must be a str: %r' % (file_path)) + + if not file_contents: + file_contents = '' + if not isinstance(file_contents, str): + raise TypeError( + 'file contents must be a str: %r' % (file_contents) + ) + + if file_types is None: + file_types = [] + if not isinstance(file_types, (tuple, list)): + raise TypeError('file types must be a list: %r' % (file_types)) + + if line_num is None: + line_num = 1 + if not isinstance(line_num, int): + raise TypeError('line num must be an int: %r' % (line_num)) + + if column_num is None: + column_num = 1 + if not isinstance(column_num, int): + raise TypeError('column num must be an int: %r' % (column_num)) + + if extra_params is None: + extra_params = {} + if not isinstance(extra_params, dict): + raise TypeError( + 'extra parameters must be a dict: %r' % (extra_params) + ) + + json_params = { + 'filepath': file_path, + 'file_data': { + file_path: { + 'filetypes': file_types, + 'contents': file_contents, + }, + }, + + 'line_num': line_num, + 'column_num': column_num, + } + json_params.update(extra_params) + + return json_params + + @property + def handler(self): + if not self._handler: + logger.warning('no handler set') + return '' + return self._handler + + @handler.setter + def handler(self, handler): + if not isinstance(handler, str): + raise TypeError + self._handler = handler + + @property + def file_path(self): + if not self._file_path: + logger.warning('no file path set') + return '' + return self._file_path + + @file_path.setter + def file_path(self, file_path): + if not isinstance(file_path, str): + raise TypeError + self._file_path = file_path + + @property + def file_contents(self): + if not self._file_contents: + logger.warning('no file contents set') + return '' + return self._file_contents + + @file_contents.setter + def file_contents(self, file_contents): + if not isinstance(file_contents, str): + raise TypeError + self._file_contents = file_contents + + @property + def file_types(self): + if not self._file_types: + logger.warning('no file types set') + return [] + return self._file_types + + @file_types.setter + def file_types(self, file_types): + if isinstance(file_types, str): + file_types = [file_types] + if not isinstance(file_types, (tuple, list)): + raise TypeError + # create a shallow copy + self._file_types = list(file_types) + + @property + def line_num(self): + if not self._line_num: + logger.warning('no line number set') + return 1 + return self._line_num + + @line_num.setter + def line_num(self, line_num): + if not isinstance(line_num, int): + raise TypeError + if line_num <= 0: + raise ValueError + self._line_num = line_num + + @property + def column_num(self): + if not self._column_num: + logger.warning('no column number set') + return 1 + return self._column_num + + @column_num.setter + def column_num(self, column_num): + if not isinstance(column_num, int): + raise TypeError + if column_num <= 0: + raise ValueError + self._column_num = column_num + + def __getitem__(self, key): + ''' Retrieves `key` from the extra parameters. ''' + if self._extra_params is None: + self._extra_params = {} + return self._extra_params[key] + + def get(self, key, default=None): + ''' + Retrieves `key` from the extra parameters. Returns `default` if unset. + ''' + if self._extra_params is None: + self._extra_params = {} + return self._extra_params.get(key, default) + + def __setitem__(self, key, value): + ''' + Sets `key` in the extra parameters. These parameters have higher + priority than the file-based parameters, and may overwrite them if the + same key is used. + ''' + if self._extra_params is None: + self._extra_params = {} + self._extra_params[key] = value + + def __delitem__(self, key): + ''' Clears the `key` extra parameter. ''' + if self._extra_params is None: + return + del self._extra_params[key] + + def __iter__(self): + ''' Dictionary-compatible iterator. ''' + base_items = [ + ('file_path', self._file_path), + ('file_contents', self._file_contents), + ('file_types', self._file_types), + ('line_num', self._line_num), + ('column_num', self._column_num), + ] + if not self._extra_params: + return iter(base_items) + + extra_items = self._extra_params.items() + + all_items = list(base_items) + list(extra_items) + return iter(all_items) + + def __str__(self): + return str(dict(self)) + + def __repr__(self): + return '%s(%r)' % ('RequestParameters', dict(self)) diff --git a/lib/subl/__init__.py b/lib/subl/__init__.py new file mode 100644 index 0000000..7e33bc4 --- /dev/null +++ b/lib/subl/__init__.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 + +''' +lib/subl +sublime-ycmd sublime utility module. +''' diff --git a/lib/subl/constants.py b/lib/subl/constants.py new file mode 100644 index 0000000..1451be0 --- /dev/null +++ b/lib/subl/constants.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +''' +lib/subl/constants.py + +Constants for use in sublime apis, including plugin-specific settings. +''' + +''' +Settings file name. + +This name is passed to sublime when loading the settings. +''' +SUBLIME_SETTINGS_FILENAME = 'sublime-ycmd.sublime-settings' + +''' +Settings keys. + +The "watched" keys are used for detecting changes to the settings file(s). + +The "server" keys are used for configuring ycmd servers. Changes to these +settings should trigger a restart on all running ycmd servers. +''' +SUBLIME_SETTINGS_WATCHED_KEYS = [ + 'ycmd_root_directory', + 'ycmd_default_settings_path', + 'ycmd_python_binary_path', + 'ycmd_language_whitelist', + 'ycmd_language_blacklist', +] +SUBLIME_SETTINGS_YCMD_SERVER_KEYS = [ + 'ycmd_root_directory', + 'ycmd_default_settings_path', + 'ycmd_python_binary_path', +] + +''' +Sane defaults for settings. This will be part of `SUBLIME_SETTINGS_FILENAME` +already, so it's mainly here for reference. + +The scope mapping is used to map syntax scopes to ycmd file types. They don't +line up exactly, so this scope mapping defines the required transformations. +For example, the syntax defines 'c++', but ycmd expects 'cpp'. +''' +SUBLIME_DEFAULT_LANGUAGE_SCOPE_MAPPING = { + 'c++': 'cpp', + 'js': 'javascript', +} diff --git a/lib/subl/dummy.py b/lib/subl/dummy.py new file mode 100644 index 0000000..3ec5e37 --- /dev/null +++ b/lib/subl/dummy.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 + +''' +lib/subl/dummy.py +Dummy sublime module polyfills. + +Generates a dummy implementation of the module to allow the rest of the library +to still load (i.e. no import errors, and no name errors). This isn't meant to +emulate the sublime module, however. The dummy implementations do nothing. + +In modules that require sublime, use the following to fall back on this module: +``` +try: + import sublime + import sublime_plugin +except ImportError: + from lib.subl.dummy import sublime + from lib.subl.dummy import sublime_plugin +``` +''' + +import collections +import logging + +logger = logging.getLogger('sublime-ycmd.' + __name__) + + +class SublimeDummyBase(object): + pass + + +class SublimeDummySettings(dict): + def clear_on_change(self, key): + pass + + def add_on_change(self, key): + pass + + +def sublime_dummy_load_settings(filename): + logger.debug('supplying dummy data for settings file: %s', filename) + return SublimeDummySettings() + + +SublimeDummy = collections.namedtuple('SublimeDummy', [ + 'Settings', + 'View', + 'load_settings', +]) +SublimePluginDummy = collections.namedtuple('SublimePluginDummy', [ + 'EventListener', + 'TextCommand', +]) + + +sublime = SublimeDummy( + SublimeDummyBase, + SublimeDummyBase, + sublime_dummy_load_settings, +) +sublime_plugin = SublimePluginDummy( + SublimeDummyBase, + SublimeDummyBase, +) diff --git a/lib/subl/settings.py b/lib/subl/settings.py new file mode 100644 index 0000000..21c4ec0 --- /dev/null +++ b/lib/subl/settings.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 + +''' +lib/subl/settings.py +Plugin settings class. + +Wraps the settings `dict` and exposes them as attributes on the class. Default +settings will be calculated for missing/blank settings if possible. + +This is also meant to abstract the setting key names, so there aren't as many +hard-coded strings in the main plugin logic. +''' + +import logging + +try: + import sublime +except ImportError: + from lib.subl.dummy import sublime + +from lib.subl.constants import ( + SUBLIME_SETTINGS_FILENAME, + SUBLIME_SETTINGS_WATCHED_KEYS, + SUBLIME_SETTINGS_YCMD_SERVER_KEYS, +) +from lib.util.fs import ( + resolve_binary_path, + default_python_binary_path, +) +from lib.ycmd.settings import get_default_settings_path + +logger = logging.getLogger('sublime-ycmd.' + __name__) + + +class Settings(object): + ''' + Wrapper class that exposes the loaded settings as attributes. + ''' + + def __init__(self, settings=None): + self._ycmd_root_directory = None + self._ycmd_default_settings_path = None + self._ycmd_python_binary_path = None + self._ycmd_language_whitelist = None + self._ycmd_language_blacklist = None + + if settings is not None: + self.parse(settings) + + def parse(self, settings): + ''' + Assigns the contents of `settings` to the internal instance variables. + The settings may be provided as a `dict` or as a `sublime.Settings` + instance. + TODO : note the setting keys, note that all variables are reset + ''' + assert isinstance(settings, (sublime.Settings, dict)) + + # this logic relies on both instance types having a `get` method + self._ycmd_root_directory = settings.get('ycmd_root_directory', None) + self._ycmd_default_settings_path = \ + settings.get('ycmd_default_settings_path', None) + self._ycmd_python_binary_path = \ + settings.get('ycmd_python_binary_path', None) + self._ycmd_language_whitelist = \ + settings.get('ycmd_language_whitelist', None) + self._ycmd_language_blacklist = \ + settings.get('ycmd_language_blacklist', None) + + self._normalize() + + def _normalize(self): + ''' + Calculates and updates any values that haven't been set after parsing + settings provided to the `parse` method. + This will calculate things like the default settings path based on the + ycmd root directory, or the python binary based on the system PATH. + ''' + if not self._ycmd_default_settings_path: + ycmd_root_directory = self._ycmd_root_directory + if ycmd_root_directory: + ycmd_default_settings_path = \ + get_default_settings_path(ycmd_root_directory) + logger.debug( + 'calculated default settings path from ' + 'ycmd root directory: %s', ycmd_default_settings_path + ) + self._ycmd_default_settings_path = ycmd_default_settings_path + + if not self._ycmd_python_binary_path: + self._ycmd_python_binary_path = default_python_binary_path() + ycmd_python_binary_path = self._ycmd_python_binary_path + + resolved_python_binary_path = \ + resolve_binary_path(ycmd_python_binary_path) + if resolved_python_binary_path: + if resolved_python_binary_path != ycmd_python_binary_path: + logger.debug( + 'calculated %s binary path: %s', + ycmd_python_binary_path, resolved_python_binary_path, + ) + self._ycmd_python_binary_path = resolved_python_binary_path + else: + logger.error( + 'failed to locate %s binary, ' + 'might not be able to start ycmd servers', + ycmd_python_binary_path + ) + + if self._ycmd_language_whitelist is None: + logger.debug('using empty whitelist - enable for all scopes') + self._ycmd_language_whitelist = [] + + if self._ycmd_language_blacklist is None: + logger.debug('using empty blacklist - disable for no scopes') + self._ycmd_language_blacklist = [] + + @property + def ycmd_root_directory(self): + ''' + Returns the path to the ycmd root directory. + If set, this will be a string. If unset, this will be `None`. + ''' + return self._ycmd_root_directory + + @property + def ycmd_default_settings_path(self): + ''' + Returns the path to the ycmd default settings file. + If set, this will be a string. If unset, it is calculated based on the + ycmd root directory. If that fails, this will be `None`. + ''' + return self._ycmd_default_settings_path + + @property + def ycmd_python_binary_path(self): + ''' + Returns the path to the python executable used to start ycmd. + If set, this will be a string. If unset, it is calculated based on the + PATH environment variable. If that fails, this will be `None`. + ''' + return self._ycmd_python_binary_path + + @property + def ycmd_language_whitelist(self): + ''' + Returns the language/scope whitelist to perform completions on. + This will be a list of strings, but may be empty. + ''' + return self._ycmd_language_whitelist + + @property + def ycmd_language_blacklist(self): + ''' + Returns the language/scope blacklist to prevent completions on. + This will be a list of strings, but may be empty. + ''' + return self._ycmd_language_blacklist + + def as_dict(self): + logger.warning('deprecated: call dict() on Settings directly') + return { + 'ycmd_root_directory': self.ycmd_root_directory, + 'ycmd_default_settings_path': self.ycmd_default_settings_path, + 'ycmd_python_binary_path': self.ycmd_python_binary_path, + 'ycmd_language_whitelist': self.ycmd_language_whitelist, + 'ycmd_language_blacklist': self.ycmd_language_blacklist, + } + + def __eq__(self, other): + ''' + Returns true if the settings instance `other` has the same ycmd server + configuration as this instance. + Other settings, like language whitelist/blacklist, are not compared. + If `other` is not an instance of `Settings`, this returns false. + ''' + if not isinstance(other, Settings): + return False + + for ycmd_server_settings_key in SUBLIME_SETTINGS_YCMD_SERVER_KEYS: + self_setting = getattr(self, ycmd_server_settings_key) + other_setting = getattr(other, ycmd_server_settings_key) + + if self_setting != other_setting: + return False + + return True + + def __bool__(self): + return not not self._ycmd_root_directory + + def __hash__(self): + return hash(( + self.ycmd_root_directory, + self.ycmd_default_settings_path, + self.ycmd_python_binary_path, + )) + + def __iter__(self): + ''' Dictionary-compatible iterator. ''' + return iter([ + ('ycmd_root_directory', self.ycmd_root_directory), + ('ycmd_default_settings_path', self.ycmd_default_settings_path), + ('ycmd_python_binary_path', self.ycmd_python_binary_path), + ('ycmd_language_whitelist', self.ycmd_language_whitelist), + ('ycmd_language_blacklist', self.ycmd_language_blacklist), + ]) + + def __str__(self): + return str(dict(self)) + + def __repr__(self): + return '%s(%r)' % ('Settings', dict(self)) + + +def load_settings(filename=SUBLIME_SETTINGS_FILENAME): + ''' + Fetches the resolved settings file `filename` from sublime, and parses + it into a `Settings` instance. The file name should be the base name of + the file (i.e. not the absolute/relative path to it). + ''' + logger.debug('loading settings from: %s', filename) + logger.critical('load_settings reference: %r', sublime.load_settings) + try: + logger.critical('source: %s', sublime._source) + finally: + pass + settings = sublime.load_settings(filename) + + logger.debug('parsing/extracting settings') + return Settings(settings=settings) + + +def bind_on_change_settings(callback, + filename=SUBLIME_SETTINGS_FILENAME, + setting_keys=SUBLIME_SETTINGS_WATCHED_KEYS): + ''' + Binds `callback` to the on-change-settings event. The settings are loaded + from `filename`, which should be the base name of the file (i.e. not the + path to it). When loading, the settings are parsed into a `Settings` + instance, and this instance is supplied as an argument to the callback. + The keys in `setting_keys` are used to bind an event listener. Changes to + settings that use these keys will trigger a reload. + When called, this will automatically load the settings for the first time, + and immediately invoke `callback` with the initial settings. + ''' + logger.debug('loading settings from: %s', filename) + settings = sublime.load_settings(filename) + + def generate_on_change_settings(key, + callback=callback, + settings=settings): + def on_change_settings(): + logger.debug('settings changed, name: %s', key) + extracted_settings = Settings(settings=settings) + callback(extracted_settings) + return on_change_settings + + logger.debug('binding on-change handlers for keys: %s', setting_keys) + for setting_key in setting_keys: + settings.clear_on_change(setting_key) + settings.add_on_change( + setting_key, generate_on_change_settings(setting_key) + ) + + logger.debug('loading initial settings') + initial_settings = Settings(settings=settings) + + logger.debug( + 'triggering callback with initial settings: %s', initial_settings + ) + callback(initial_settings) + + return initial_settings diff --git a/lib/st.py b/lib/subl/view.py similarity index 54% rename from lib/st.py rename to lib/subl/view.py index b20c168..1b56d98 100644 --- a/lib/st.py +++ b/lib/subl/view.py @@ -1,260 +1,38 @@ #!/usr/bin/env python3 ''' -lib/st.py -High-level helpers for working with the sublime module. +lib/subl/settings.py +Plugin settings class. + +Wraps the settings `dict` and exposes them as attributes on the class. Default +settings will be calculated for missing/blank settings if possible. + +This is also meant to abstract the setting key names, so there aren't as many +hard-coded strings in the main plugin logic. ''' import logging -from lib.fs import ( +try: + import sublime +except ImportError: + from lib.subl.dummy import sublime + +from lib.schema.request import RequestParameters +from lib.subl.constants import SUBLIME_DEFAULT_LANGUAGE_SCOPE_MAPPING +from lib.util.fs import ( + get_common_ancestor, get_directory_name, resolve_abspath, - get_common_ancestor, - resolve_binary_path, - default_python_binary_path, -) -from lib.jsonmodels import ( - SYrequestParameters, -) -from lib.ycmd import ( - get_ycmd_default_settings_path, ) -try: - import sublime # noqa - import sublime_plugin # noqa - _HAS_LOADED_ST = True -except ImportError: - import collections - _HAS_LOADED_ST = False - - class SublimeDummyBase(object): - pass - SublimeDummy = collections.namedtuple('SublimeDummy', { - 'Settings', - 'View', - }) - SublimePluginDummy = collections.namedtuple('SublimePluginDummy', [ - 'EventListener', - 'TextCommand', - ]) - - sublime = SublimeDummy( - SublimeDummyBase, - SublimeDummyBase, - ) - sublime_plugin = SublimePluginDummy( - SublimeDummyBase, - SublimeDummyBase, - ) -finally: - assert isinstance(_HAS_LOADED_ST, bool) +# for type annotations only: +from lib.ycmd.server import Server # noqa: F401 logger = logging.getLogger('sublime-ycmd.' + __name__) -# EXPORT -SublimeEventListener = sublime_plugin.EventListener -SublimeTextCommand = sublime_plugin.TextCommand - -DEFAULT_SUBLIME_SETTINGS_FILENAME = 'sublime-ycmd.sublime-settings' -DEFAULT_SUBLIME_SETTINGS_KEYS = [ - 'ycmd_root_directory', - 'ycmd_default_settings_path', - 'ycmd_python_binary_path', - 'ycmd_language_whitelist', - 'ycmd_language_blacklist', -] -YCMD_SERVER_SETTINGS_KEYS = [ - 'ycmd_root_directory', - 'ycmd_default_settings_path', - 'ycmd_python_binary_path', -] - -# Mapping from sublime scope names to the corresponding syntax name. This is -# required for ycmd, as it does not do the mapping itself, and will instead -# complain when it is not understood. -# TODO : Move this language mapping into the settings file, so the user can -# configure it if required. -YCMD_SCOPE_MAPPING = { - 'js': 'javascript', - 'c++': 'cpp', -} - - -class SYsettings(object): - ''' - Wrapper class that exposes the loaded settings as attributes. - This is meant to abstract the setting key names, so there aren't as many - hard-coded strings in the main plugin logic. - ''' - - def __init__(self, settings=None): - self._ycmd_root_directory = None - self._ycmd_default_settings_path = None - self._ycmd_python_binary_path = None - self._ycmd_language_whitelist = None - self._ycmd_language_blacklist = None - - if settings is not None: - self.parse(settings) - - def parse(self, settings): - ''' - Assigns the contents of `settings` to the internal instance variables. - The settings may be provided as a `dict` or as a `sublime.Settings` - instance. - TODO : note the setting keys, note that all variables are reset - ''' - assert isinstance(settings, (sublime.Settings, dict)) - - # this logic relies on both instance types having a `get` method - self._ycmd_root_directory = settings.get('ycmd_root_directory', None) - self._ycmd_default_settings_path = \ - settings.get('ycmd_default_settings_path', None) - self._ycmd_python_binary_path = \ - settings.get('ycmd_python_binary_path', None) - self._ycmd_language_whitelist = \ - settings.get('ycmd_language_whitelist', None) - self._ycmd_language_blacklist = \ - settings.get('ycmd_language_blacklist', None) - - self._normalize() - - def _normalize(self): - ''' - Calculates and updates any values that haven't been set after parsing - settings provided to the `parse` method. - This will calculate things like the default settings path based on the - ycmd root directory, or the python binary based on the system PATH. - ''' - if not self._ycmd_default_settings_path: - ycmd_root_directory = self._ycmd_root_directory - if ycmd_root_directory: - ycmd_default_settings_path = \ - get_ycmd_default_settings_path(ycmd_root_directory) - logger.debug( - 'calculated default settings path from ' - 'ycmd root directory: %s', ycmd_default_settings_path - ) - self._ycmd_default_settings_path = ycmd_default_settings_path - - if not self._ycmd_python_binary_path: - self._ycmd_python_binary_path = default_python_binary_path() - ycmd_python_binary_path = self._ycmd_python_binary_path - - resolved_python_binary_path = \ - resolve_binary_path(ycmd_python_binary_path) - if resolved_python_binary_path: - if resolved_python_binary_path != ycmd_python_binary_path: - logger.debug( - 'calculated %s binary path: %s', - ycmd_python_binary_path, resolved_python_binary_path, - ) - self._ycmd_python_binary_path = resolved_python_binary_path - else: - logger.error( - 'failed to locate %s binary, ' - 'might not be able to start ycmd servers', - ycmd_python_binary_path - ) - - if self._ycmd_language_whitelist is None: - logger.debug('using empty whitelist - enable for all scopes') - self._ycmd_language_whitelist = [] - - if self._ycmd_language_blacklist is None: - logger.debug('using empty blacklist - disable for no scopes') - self._ycmd_language_blacklist = [] - - @property - def ycmd_root_directory(self): - ''' - Returns the path to the ycmd root directory. - If set, this will be a string. If unset, this will be `None`. - ''' - return self._ycmd_root_directory - - @property - def ycmd_default_settings_path(self): - ''' - Returns the path to the ycmd default settings file. - If set, this will be a string. If unset, it is calculated based on the - ycmd root directory. If that fails, this will be `None`. - ''' - return self._ycmd_default_settings_path - - @property - def ycmd_python_binary_path(self): - ''' - Returns the path to the python executable used to start ycmd. - If set, this will be a string. If unset, it is calculated based on the - PATH environment variable. If that fails, this will be `None`. - ''' - return self._ycmd_python_binary_path - - @property - def ycmd_language_whitelist(self): - ''' - Returns the language/scope whitelist to perform completions on. - This will be a list of strings, but may be empty. - ''' - return self._ycmd_language_whitelist - - @property - def ycmd_language_blacklist(self): - ''' - Returns the language/scope blacklist to prevent completions on. - This will be a list of strings, but may be empty. - ''' - return self._ycmd_language_blacklist - - def as_dict(self): - return { - 'ycmd_root_directory': self.ycmd_root_directory, - 'ycmd_default_settings_path': self.ycmd_default_settings_path, - 'ycmd_python_binary_path': self.ycmd_python_binary_path, - 'ycmd_language_whitelist': self.ycmd_language_whitelist, - 'ycmd_language_blacklist': self.ycmd_language_blacklist, - } - - def __eq__(self, other): - ''' - Returns true if the settings instance `other` has the same ycmd server - configuration as this instance. - Other settings, like language whitelist/blacklist, are not compared. - If `other` is not an instance of `SYsettings`, this returns false. - ''' - if not isinstance(other, SYsettings): - return False - - for ycmd_server_settings_key in YCMD_SERVER_SETTINGS_KEYS: - self_setting = getattr(self, ycmd_server_settings_key) - other_setting = getattr(other, ycmd_server_settings_key) - if self_setting != other_setting: - return False - - return True - - def __str__(self): - return str(self.as_dict()) - - def __repr__(self): - return '%s(%s)' % ('SYsettings', str(self.as_dict())) - - def __bool__(self): - return not not self._ycmd_root_directory - - def __hash__(self): - return hash(( - self.ycmd_root_directory, - self.ycmd_default_settings_path, - self.ycmd_python_binary_path, - )) - - -class SYview(object): +class View(object): ''' Wrapper class that provides extra functionality over `sublime.View`. This allows tracking state for each view independently. @@ -315,10 +93,11 @@ def dirty(self): def to_file_params(self): ''' Generates and returns the keyword arguments required for making a call - to a handler method on `SYserver`. These parameters include information + to a handler method on `Server`. These parameters include information about the file, and also the contents, and file type(s), if available. The result can be unpacked into a call to one of the server methods. ''' + logger.warning('deprecated: use View to RequestParameters instead') if not self._view: logger.error('no view handle has been set') return None @@ -378,8 +157,8 @@ def to_file_params(self): def generate_request_parameters(self): ''' - Generates and returns file-related `SYrequestParameters` for use in the - `SYserver` handlers. + Generates and returns file-related `RequestParameters` for use in the + `Server` handlers. These parameters include information about the file like the name, contents, and file type(s). Additional parameters may still be added to the result before passing it off to the server. @@ -435,7 +214,7 @@ def generate_request_parameters(self): line_num = file_row + 1 column_num = file_col + 1 - return SYrequestParameters( + return RequestParameters( file_path=file_path, file_contents=file_contents, file_types=file_types, @@ -476,7 +255,7 @@ def __eq__(self, other): if isinstance(other, sublime.View): other_id = other.id() - elif isinstance(other, SYview): + elif isinstance(other, View): other_id = other._view.id() else: raise TypeError('view must be a View: %r' % (other)) @@ -516,25 +295,14 @@ def __contains__(self, key): self._cache = {} return key in self._cache - ''' - # pass-through to `sublime.View` attributes: - def __getattr__(self, name): - if not self._view: - logger.error( - 'no view handle has been set, cannot get attribute: %s', name, - ) - raise AttributeError - return getattr(self._view, name) - ''' - # pass-through to `sublime.View` methods: + def id(self): if not self._view: logger.error('no view handle has been set') return None return self._view.id() - ''' def size(self): if not self._view: logger.error('no view handle has been set') @@ -552,79 +320,23 @@ def scope_name(self, point): logger.error('no view handle has been set') return None return self._view.scope_name(point) - ''' -def load_known_settings(filename=DEFAULT_SUBLIME_SETTINGS_FILENAME): - ''' - Fetches the resolved settings file `filename` from sublime, and parses - it into a `SYsettings` instance. The file name should be the base name of - the file (i.e. not the absolute/relative path to it). +def get_view_id(view): ''' - if not _HAS_LOADED_ST: - logger.debug('debug mode, returning empty values') - return SYsettings() + Returns the id of a given `view`. - logger.debug('loading settings from: %s', filename) - settings = sublime.load_settings(filename) + If the view is already a number, it is returned as-is. It is assumed to be + the view id already. - logger.debug('parsing/extracting settings') - return SYsettings(settings=settings) - - -def bind_on_change_settings(callback, - filename=DEFAULT_SUBLIME_SETTINGS_FILENAME, - setting_keys=DEFAULT_SUBLIME_SETTINGS_KEYS): - ''' - Binds `callback` to the on-change-settings event. The settings are loaded - from `filename`, which should be the base name of the file (i.e. not the - path to it). When loading, the settings are parsed into a `SYsettings` - instance, and this instance is supplied as an argument to the callback. - The keys in `setting_keys` are used to bind an event listener. Changes to - settings that use these keys will trigger a reload. - When called, this will automatically load the settings for the first time, - and immediately invoke `callback` with the initial settings. + If the view is a `sublime.View` or `View`, the `view.id()` method is called + to get the id. ''' - if not _HAS_LOADED_ST: - logger.debug('debug mode, will only trigger initial load event') - initial_settings = SYsettings() - else: - logger.debug('loading settings from: %s', filename) - settings = sublime.load_settings(filename) - - def generate_on_change_settings(key, - callback=callback, - settings=settings): - def on_change_settings(): - logger.debug('settings changed, name: %s', key) - extracted_settings = SYsettings(settings=settings) - callback(extracted_settings) - return on_change_settings - - logger.debug('binding on-change handlers for keys: %s', setting_keys) - for setting_key in setting_keys: - settings.clear_on_change(setting_key) - settings.add_on_change( - setting_key, generate_on_change_settings(setting_key) - ) - - logger.debug('loading initial settings') - initial_settings = SYsettings(settings=settings) - - logger.debug( - 'triggering callback with initial settings: %s', initial_settings - ) - callback(initial_settings) - - return initial_settings - - -def get_view_id(view): if isinstance(view, int): # already a view ID, so return it as-is return view - assert isinstance(view, (sublime.View, SYview)), \ + assert isinstance(view, (sublime.View, View)), \ 'view must be a View: %r' % (view) # duck type, both support the method: @@ -718,10 +430,10 @@ def get_path_for_view(view): Finally, if the path cannot be determined by either of the two strategies above, then this will just return `None`. ''' - assert isinstance(view, (sublime.View, SYview)), \ + assert isinstance(view, (sublime.View, View)), \ 'view must be a View: %r' % (view) - if isinstance(view, SYview): + if isinstance(view, View): cached_path = view.path if cached_path is not None: return cached_path @@ -788,12 +500,12 @@ def get_file_types(view, scope_position=0): will appear in the result. If no 'source' scopes are found, an empty list is returned. ''' - assert isinstance(view, (sublime.View, SYview)), \ + assert isinstance(view, (sublime.View, View)), \ 'view must be a View: %r' % (view) assert isinstance(scope_position, int), \ 'scope position must be an int: %r' % (scope_position) - if isinstance(view, SYview): + if isinstance(view, View): if scope_position == 0: cached_file_types = view.file_types if cached_file_types is not None: @@ -822,8 +534,10 @@ def get_file_types(view, scope_position=0): lambda s: s[len(SOURCE_PREFIX):], source_scope_names )) + # TODO : Use `Settings` to get the scope mapping dynamically. source_types = list(map( - lambda s: YCMD_SCOPE_MAPPING.get(s, s), source_names + lambda s: SUBLIME_DEFAULT_LANGUAGE_SCOPE_MAPPING.get(s, s), + source_names )) logger.debug( 'extracted source scope names: %s -> %s -> %s', @@ -831,70 +545,3 @@ def get_file_types(view, scope_position=0): ) return source_types - -# DEPRECATED: - - -def sublime_defer(callback, *args, **kwargs): - ''' - Calls the supplied `callback` asynchronously, when running under sublime. - In debug mode, the `callback` is executed immediately. - ''' - def bound_callback(): - return callback(*args, **kwargs) - - if _HAS_LOADED_ST: - sublime.set_timeout(bound_callback, 0) - else: - bound_callback() - - -def is_sublime_type(value, expected): - ''' - Checks to see if a provided value is of the expected type. This wraps - classes in the sublime module to allow noops when in debug mode. - Returns True if it is as expected, and False otherwise. This also logs a - warning when returning False. - ''' - if not _HAS_LOADED_ST: - # return True always - this is for debugging - return True - - def try_get_module(cls): - ''' Returns the class `__module__` attribute, if defined, or None. ''' - try: - return cls.__module__ - except AttributeError: - return None - - def try_get_name(cls): - ''' Returns the class `__name__` attribute, if defined, or None. ''' - try: - return cls.__name__ - except AttributeError: - return None - - expected_from_module = try_get_module(expected) - if expected_from_module not in ['sublime', 'sublime_plugin']: - logger.warning( - 'invalid sublime type, expected "sublime" or "sublime_plugin", ' - 'got: %r', expected_from_module, - ) - - if isinstance(value, expected): - return True - - expected_classname = try_get_name(expected) - - if expected_from_module is not None and expected_classname is not None: - expected_description = \ - '%s.%s' % (expected_from_module, expected_classname) - elif expected_classname is not None: - expected_description = expected_classname - else: - expected_description = '?' - - logger.warning( - 'value is not an instance of %s: %r', expected_description, value - ) - return False diff --git a/lib/util/__init__.py b/lib/util/__init__.py new file mode 100644 index 0000000..d3977df --- /dev/null +++ b/lib/util/__init__.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 + +''' +lib/util +sublime-ycmd utility function module. +''' diff --git a/lib/util/format.py b/lib/util/format.py new file mode 100644 index 0000000..47fc4db --- /dev/null +++ b/lib/util/format.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 + +''' +lib/util/format.py +Data formatting functions. Includes base64 encode/decode functions as well as +json parse/serialize functions. +''' + +import base64 +import json +import logging + +from lib.util.str import ( + bytes_to_str, + str_to_bytes, +) + +logger = logging.getLogger('sublime-ycmd.' + __name__) + + +def base64_encode(data): + ''' + Encodes the given `data` in base-64. The result will either be a `str`, + or `bytes`, depending on the input type (same as input type). + ''' + assert isinstance(data, (str, bytes)), \ + 'data must be str or bytes: %r' % (data) + + is_str = isinstance(data, str) + if is_str: + data = str_to_bytes(data) + + encoded = base64.b64encode(data) + + if is_str: + encoded = bytes_to_str(encoded) + + return encoded + + +def base64_decode(data): + ''' + Decodes the given `data` from base-64. The result will either be a `str`, + or `bytes`, depending on the input type (same as input type). + ''' + assert isinstance(data, (str, bytes)), \ + 'data must be str or bytes: %r' % (data) + + is_str = isinstance(data, str) + if is_str: + data = str_to_bytes(data) + + decoded = base64.b64decode(data) + + if is_str: + decoded = bytes_to_str(decoded) + + return decoded + + +def json_serialize(data): + ''' Serializes `data` from a `dict` to a json `str`. ''' + assert isinstance(data, dict), 'data must be a dict: %r' % (data) + serialized = json.dumps(data) + return serialized + + +def json_parse(data): + ''' Parses `data` from a json `str` a `dict`. ''' + assert isinstance(data, (str, bytes)), \ + 'data must be str or bytes: %r' % (data) + if isinstance(data, bytes): + data = bytes_to_str(data) + + parsed = json.loads(data) + return parsed diff --git a/lib/fs.py b/lib/util/fs.py similarity index 97% rename from lib/fs.py rename to lib/util/fs.py index c6a7a50..dcd14a0 100644 --- a/lib/fs.py +++ b/lib/util/fs.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' -lib/fs.py +lib/util/fs.py File-system utilities. ''' @@ -91,7 +91,7 @@ def save_json_file(fp, data, encoding='utf-8'): Serializes and writes out `data` to `fp`. The data should be provided as a `dict`, and will be serialized to a JSON string. The `fp` parameter should support a `write` method. - If `encoding` is `b''`, the serialized JSON will be written as `bytes`. + The `encoding` parameter is used when encoding the serialized data. ''' json_str = json.dumps(data) json_bytes = json_str.encode(encoding=encoding) @@ -336,17 +336,6 @@ def _commonpath_polyfill(paths): common_path_components[0] += os.sep return os.path.join(*common_path_components) - ''' - commonprefix = os.path.commonprefix(paths) - - if not commonprefix: - raise ValueError('Paths don\'t have a common ancestor') - - head, tail = os.path.split(commonprefix) - # always drop `tail`, that ensures we end up with a valid prefix path - return head - ''' - def get_common_ancestor(paths, default=None): common_path = default diff --git a/lib/util/hmac.py b/lib/util/hmac.py new file mode 100644 index 0000000..5436f28 --- /dev/null +++ b/lib/util/hmac.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 + +''' +lib/util/hmac.py +Contains HMAC utility functions. The ycmd server expects an HMAC header with +all requests to verify the client's identity. The ycmd server also includes an +HMAC header in all responses to allow the client to verify the responses. +''' + +import hashlib +import hmac +import logging +import os + +from lib.util.str import ( + bytes_to_str, + str_to_bytes, +) +from lib.util.format import ( + base64_encode, +) + +logger = logging.getLogger('sublime-ycmd.' + __name__) + + +def new_hmac_secret(num_bytes=32): + ''' Generates and returns an HMAC secret in binary encoding. ''' + hmac_secret_binary = os.urandom(num_bytes) + return hmac_secret_binary + + +def _calculate_hmac(hmac_secret, data, digestmod=hashlib.sha256): + assert isinstance(hmac_secret, (str, bytes)), \ + 'hmac secret must be str or bytes: %r' % (hmac_secret) + assert isinstance(data, (str, bytes)), \ + 'data must be str or bytes: %r' % (data) + + hmac_secret = str_to_bytes(hmac_secret) + data = str_to_bytes(data) + + hmac_instance = hmac.new(hmac_secret, msg=data, digestmod=digestmod) + hmac_digest_bytes = hmac_instance.digest() + assert isinstance(hmac_digest_bytes, bytes), \ + '[internal] hmac digest should be bytes: %r' % (hmac_digest_bytes) + + return hmac_digest_bytes + + +def calculate_hmac(hmac_secret, *content, digestmod=hashlib.sha256): + ''' + Calculates the HMAC for the given `content` using the `hmac_secret`. + This is calculated by first generating the HMAC for each item in + `content` separately, then concatenating them, and finally running another + HMAC on the concatenated intermediate result. Finally, the result of that + is base-64 encoded, so it is suitable for use in headers. + ''' + assert isinstance(hmac_secret, (str, bytes)), \ + 'hmac secret must be str or bytes: %r' % (hmac_secret) + hmac_secret = str_to_bytes(hmac_secret) + + content_hmac_digests = map( + lambda data: _calculate_hmac( + hmac_secret, data=data, digestmod=digestmod, + ), content, + ) + + concatenated_hmac_digests = b''.join(content_hmac_digests) + + hmac_digest_binary = _calculate_hmac( + hmac_secret, data=concatenated_hmac_digests, digestmod=digestmod, + ) + + hmac_digest_bytes = base64_encode(hmac_digest_binary) + hmac_digest_str = bytes_to_str(hmac_digest_bytes) + + return hmac_digest_str diff --git a/lib/logutils.py b/lib/util/log.py similarity index 54% rename from lib/logutils.py rename to lib/util/log.py index 3cfce4f..75f7150 100644 --- a/lib/logutils.py +++ b/lib/util/log.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 ''' -lib/logutils.py +lib/util/log.py Contains additional logic to help improve logging output. ''' +import collections import functools import logging import os @@ -86,46 +87,44 @@ def get_default_messagefmt(): (LEVELNAME_MAXLEN, FILENAME_MAXLEN, LINENO_MAXLEN, FUNCNAME_MAXLEN) -def get_extended_messagefmt(): +def get_default_format_props(): ''' - Returns a record format string for use in logging configuration. This - version includes some non-standard record items that are meant to be - generated by log filters in this module. Remember to add these filters to - the logger so that the extra record items are calculated. + Returns a map from record attribute names to desired max smart-truncation + length. This `dict` is suitable for use in `SmartTruncateFormatter`. ''' - return \ - '[%%(asctime)s] %%(xlevelname)%ds %%(xfilename)%ds:%%(lineno)-%dd ' \ - '%%(xfuncname)-%ss %%(message)s' % \ - (LEVELNAME_MAXLEN, FILENAME_MAXLEN, LINENO_MAXLEN, FUNCNAME_MAXLEN) + return { + 'levelname': LEVELNAME_MAXLEN, + 'filename': FILENAME_MAXLEN, + # doesn't make sense to truncate ints: + # 'lineno': LINENO_MAXLEN, + 'funcName': FUNCNAME_MAXLEN, + } -def register_extension_filters(loginstance=None): +def get_smart_truncate_formatter(fmt=None, datefmt=None, props=None): ''' - Registers the filters that generated extended record entries. This includes - all entries used in get_extended_messagefmt. - If no instance is provided, this targets the root logger instance. + Generates and returns a `SmartTruncateFormatter` with defaults. + If any parameter is omitted, a default one is calculated using the + `get_default_` helpers above. ''' - if loginstance is None: - root_logger = logging.getLogger() - - if not root_logger.hasHandlers(): - raise Exception('Root logger has no handlers, cannot bind to it') + if fmt is None: + fmt = get_default_messagefmt() + if datefmt is None: + datefmt = get_default_datefmt() + if props is None: + props = get_default_format_props() - loginstance = root_logger.handlers[0] + assert isinstance(fmt, str), 'fmt must be a str: %r' % (fmt) + assert isinstance(datefmt, str), 'datefmt must be a str: %r' % (datefmt) + # no need to validate `props`, that's done in the constructor - assert isinstance(loginstance, (logging.Logger, logging.Handler)), \ - 'loginstance must be a Logger or Handler: %r' % loginstance + formatter = SmartTruncateFormatter( + fmt=fmt, + datefmt=datefmt, + props=props, + ) - xlevelname_filter = SYlogPropertyShortener().set_length(LEVELNAME_MAXLEN) \ - .set_property('levelname').set_result('xlevelname') - xfilename_filter = SYlogPropertyShortener().set_length(FILENAME_MAXLEN) \ - .set_property('filename').set_result('xfilename') - xfuncname_filter = SYlogPropertyShortener().set_length(FUNCNAME_MAXLEN) \ - .set_property('funcName').set_result('xfuncname') - - loginstance.addFilter(xlevelname_filter) - loginstance.addFilter(xfilename_filter) - loginstance.addFilter(xfuncname_filter) + return formatter # Used to strip common prefixes from script paths (namely, absolute paths) @@ -140,7 +139,7 @@ def strip_common_path_prefix(basepath, relativeto=_SY_LIB_DIR): ''' assert isinstance(basepath, str), \ 'basepath must be a string: %r' % basepath - # Unfortunately, os.path.commonpath is only available in python 3.5+ + # Unfortunately, `os.path.commonpath` is only available in python 3.5+ # To be compatible with 3.3, need to use commonprefix and then process it common_path_chars = os.path.commonprefix([basepath, relativeto]) common_path_components = os.path.split(common_path_chars) @@ -246,8 +245,8 @@ def get_reconstructed(): def apply_truncate_until_satisfied(truncatefn): ''' Applies the specified truncation function to the current working - components until the length requirement is satisfied. Returns True if - successful, and False if it was not enough. Either way, the current + components until the length requirement is satisfied. Returns `True` if + successful, and `False` if it was not enough. Either way, the current component state is updated after the application. ''' for i, component in enumerate(current_components): @@ -287,139 +286,156 @@ def apply_truncate_until_satisfied(truncatefn): return '' -class SYlogPropertyShortener(logging.Filter): +class SmartTruncateFormatter(logging.Formatter): ''' - Logging filter that shortens property values to a desired target length. - To use this class, the source property and target length must be set. If - they are not set, this filter does not modify the record in any way. + Logging formatter that shortens property values to the target length + indicated by the format-code length. ''' - def __init__(self): - super(SYlogPropertyShortener, self).__init__() - self._source_property = None - self._target_length = None - self._result_property = None - - def set_property(self, source_property): - ''' - Sets the property that this filter targets. When filtering, this - property is looked up in the log record and processed. - ''' - assert source_property is None or isinstance(source_property, str), \ - 'source_property must be a str: %r' % source_property - self._source_property = source_property - return self - - def set_length(self, target_length): - ''' - Sets the target length for the shortened property. When filtering, the - source property will be shortened to be this length (at most). - ''' - assert target_length is None or isinstance(target_length, int), \ - 'target_length must be an int: %r' % target_length - if target_length <= 0: - raise ValueError('Target length must be a positive integer') - self._target_length = target_length - return self - - def set_result(self, result_property): - ''' - Sets the property to store the shortened result into. When filtering, - the shortened property will be placed in the log record under this - name. If not set, the filter will update the source property in-place. - ''' - assert result_property is None or isinstance(result_property, str), \ - 'result_property must be a str: %r' % result_property - self._result_property = result_property - return self - - def filter(self, record): - ''' - Log filter callback. - Called by the parent logger when a record is to be logged. Always - returns True, to indicate that the filter does not reject the message. - ''' - if self._source_property is None or self._target_length is None: - # Not configured, can't do anything - return True - - source_property = self._source_property - target_length = self._target_length - result_property = self._result_property \ - if self._result_property is not None else source_property - - assert isinstance(source_property, str), \ - '[internal] source_property is not a str: %r' % source_property - assert isinstance(target_length, int) and target_length > 0, \ - '[internal] target_length is not an int: %r' % target_length - assert isinstance(result_property, str), \ - '[internal] result_property is not a str: %r' % result_property - - # Always initialize the result property - this ensures that the record - # can populate the message format even if there is a failure - if not hasattr(record, result_property): - setattr(record, result_property, '?') - - if not hasattr(record, source_property): - # Property unavailable, can't do anything - return True - - source_value = getattr(record, source_property) - try: - truncated_value = smart_truncate(source_value, target_length) - except Exception: - # Failed to truncate, can't do anything - return True - - setattr(record, result_property, truncated_value) - return True - - def __str__(self): - source_property = self._source_property - target_length = self._target_length - result_property = self._result_property \ - if self._result_property is not None else source_property - - return 'shorten("%s", %d) -> "%s"' % ( - source_property, target_length, result_property + def __init__(self, fmt=None, datefmt=None, style='%', props=None): + super(SmartTruncateFormatter, self).__init__( + fmt=fmt, datefmt=datefmt, style=style, ) + if props and not isinstance(props, dict): + logger.error('invalid property map: %r', props) + # fall through and use it anyway + self._props = props if props is not None else {} + self._debug = False + + def format(self, record): + def format_record_entry(name, length): + if not isinstance(name, str): + if self._debug: + logger.error('invalid formattable field: %r', name) + return + if not isinstance(length, int): + if self._debug: + logger.error('invalid format field width: %r', length) + return + + if not hasattr(record, name): + # name not in it, so nothing to format + return + + value = getattr(record, name) + if not isinstance(value, str): + # not a str, don't format it, it's hard to handle it right... + if self._debug: + logger.warning( + 'cannot format value for %s: %r', name, value + ) + return + + truncated = smart_truncate(value, length) + if not truncated: + if self._debug: + logger.warning( + 'failed to truncate value, size: %s, %d', + value, length, + ) + return + + setattr(record, name, truncated) + + for k, v in self._props.items(): + format_record_entry(k, v) + + return super(SmartTruncateFormatter, self).format(record) + + def __getitem__(self, key): + if not hasattr(self._props, '__getitem__'): + logger.error('invalid props, cannot get item: %r', self._props) + if not self._props: + raise KeyError(key,) + + return self._props[key] + + def __setitem__(self, key, value): + if not hasattr(self._props, '__setitem__'): + logger.error('invalid props, cannot set item: %r', self._props) + if not isinstance(value, int): + logger.error('invalid length, should be int: %r', value) + + self._props[key] = value + + def __contains__(self, key): + if not hasattr(self._props, '__contains__'): + logger.error('invalid props, : %r', self._props) + + def __iter__(self): + ''' Dictionary-compatible iterator. ''' + for key in self._props: + value = self._props[key] + + yield (key, value) def __repr__(self): - source_property = self._source_property - target_length = self._target_length - result_property = self._result_property \ - if self._result_property is not None else source_property + return '%s(%r)' % ('SmartTruncateFormatter', dict(self)) - return '' % ( - source_property, target_length, result_property, - ) +FormatField = collections.namedtuple('FormatField', [ + 'name', + 'zero', + 'minus', + 'space', + 'plus', + 'width', + 'point', + 'type', +]) -class SYaddPrefixLogAdapter(logging.LoggerAdapter): - ''' - Trivial log adapter wrapper class. This class helps the built-in python - logger with calculating the caller frame. It must exist in a separate - module for the caller lookup logic to grab the correct frame. Otherwise, - the log adapter will tend to grab the caller frame of the logging module - itself, which is probably not the desired behaviour... - ''' - def __init__(self, logger, extra=None): - super(SYaddPrefixLogAdapter, self).__init__(logger, extra or {}) +def parse_fields(fmt, style='%', default=None): + ''' + Parses and yields template fields in a format string. - def process(self, msg, kwargs): - p = self.prefix() + For %-style formatting, these look like: '%(foo)15s', where '15' refers to + the field width for 'foo'. - return '%s%s' % (p, msg), kwargs + NOTE : This is a best-effort implementation. It might not work 100%. + ''' + if style != '%': + raise NotImplementedError( + 'unimplemented: field width for non %%-style formats' % () + ) - def prefix(self): + def _get_parser(): ''' - Override method to generate the prefix for log messages. This method - should return a string, which will appear as a prefix in front of any - log messages passed to the underlying logger. - The default implementation returns an empty string, which will end up - returning the original message as-is. - Methods have access to the logger and extra parameters as - `self.logger`, and `self.extra`, respectively. + Compiles and returns a regex parser for parsing fields. + + The regex matches something in the form: '%(foo) 15s' + % - prefix + (name) - named field (optional) + 0 - for numbers, left pad with 0, override space (optional) + - - left-pad, override 0 (optional) + space - whitespace before before positive numbers (optional) + + - for numbers, always include +/- sign (optional) + number - field width + + NOTE : Attribute names must match regex parameter names. ''' - return '' + + field_pattern = ''.join(( + r'%', + r'(?:\((?P\w+)\))?', + r'(?P0)?', + r'(?P-)?', + r'(?P\s)?', + r'(?P\+)?', + r'(?P\d+)?', + r'(?P(?:\.\d+))?', + r'(?P[hlL]*[srxXeEfFdiou])', + )) + + return re.compile(field_pattern) + + try: + field_parser = _get_parser() + except re.error as e: + logger.error('field width parser is invalid: %r', e) + return None + + def _match_to_field(match): + return FormatField(**match.groupdict()) + + return map(_match_to_field, field_parser.finditer(fmt)) diff --git a/lib/strutils.py b/lib/util/str.py similarity index 54% rename from lib/strutils.py rename to lib/util/str.py index e3192c7..6a704cb 100644 --- a/lib/strutils.py +++ b/lib/util/str.py @@ -1,22 +1,23 @@ #!/usr/bin/env python3 ''' -lib/strutils.py +lib/util/str.py Contains string utility functions. This includes conversions between `str` and `bytes`, and hash calculations. ''' -import base64 -import hashlib -import hmac -import json import logging -import os logger = logging.getLogger('sublime-ycmd.' + __name__) def str_to_bytes(data): + ''' + Converts `data` to `bytes`. + + If data is a `str`, it is encoded into `bytes`. + If data is `bytes`, it is returned as-is. + ''' assert isinstance(data, (str, bytes)), \ 'data must be str or bytes: %r' % (data) if isinstance(data, bytes): @@ -26,6 +27,12 @@ def str_to_bytes(data): def bytes_to_str(data): + ''' + Converts `data` to `str`. + + If data is a `bytes`, it is decoded into `str`. + If data is `str`, it is returned as-is. + ''' assert isinstance(data, (str, bytes)), \ 'data must be str or bytes: %r' % (data) if isinstance(data, str): @@ -34,95 +41,6 @@ def bytes_to_str(data): return data.decode() -def base64_encode(data): - ''' - Encodes the given `data` in base-64. The result will either be a `str`, - or `bytes`, depending on the input type (same as input type). - ''' - assert isinstance(data, (str, bytes)), \ - 'data must be str or bytes: %r' % (data) - - is_str = isinstance(data, str) - if is_str: - data = str_to_bytes(data) - - encoded = base64.b64encode(data) - - if is_str: - encoded = bytes_to_str(encoded) - - return encoded - - -def new_hmac_secret(num_bytes=32): - ''' Generates and returns an HMAC secret in binary encoding. ''' - hmac_secret_binary = os.urandom(num_bytes) - return hmac_secret_binary - - -def _calculate_hmac(hmac_secret, data, digestmod=hashlib.sha256): - assert isinstance(hmac_secret, (str, bytes)), \ - 'hmac secret must be str or bytes: %r' % (hmac_secret) - assert isinstance(data, (str, bytes)), \ - 'data must be str or bytes: %r' % (data) - - hmac_secret = str_to_bytes(hmac_secret) - data = str_to_bytes(data) - - hmac_instance = hmac.new(hmac_secret, msg=data, digestmod=digestmod) - hmac_digest_bytes = hmac_instance.digest() - assert isinstance(hmac_digest_bytes, bytes), \ - '[internal] hmac digest should be bytes: %r' % (hmac_digest_bytes) - - return hmac_digest_bytes - - -def calculate_hmac(hmac_secret, *content, digestmod=hashlib.sha256): - ''' - Calculates the HMAC for the given `content` using the `hmac_secret`. - This is calculated by first generating the HMAC for each item in - `content` separately, then concatenating them, and finally running another - HMAC on the concatenated intermediate result. Finally, the result of that - is base-64 encoded, so it is suitable for use in headers. - ''' - assert isinstance(hmac_secret, (str, bytes)), \ - 'hmac secret must be str or bytes: %r' % (hmac_secret) - hmac_secret = str_to_bytes(hmac_secret) - - content_hmac_digests = map( - lambda data: _calculate_hmac( - hmac_secret, data=data, digestmod=digestmod, - ), content, - ) - - concatenated_hmac_digests = b''.join(content_hmac_digests) - - hmac_digest_binary = _calculate_hmac( - hmac_secret, data=concatenated_hmac_digests, digestmod=digestmod, - ) - - hmac_digest_bytes = base64_encode(hmac_digest_binary) - hmac_digest_str = bytes_to_str(hmac_digest_bytes) - - return hmac_digest_str - - -def format_json(data): - assert isinstance(data, dict), 'data must be a dict: %r' % (data) - serialized = json.dumps(data) - return serialized - - -def parse_json(data): - assert isinstance(data, (str, bytes)), \ - 'data must be str or bytes: %r' % (data) - if isinstance(data, bytes): - data = bytes_to_str(data) - - parsed = json.loads(data) - return parsed - - def truncate(data, max_sz=16): ''' Truncates the input `data` in a somewhat-intelligent way. The purpose of diff --git a/lib/ycmd.py b/lib/ycmd.py deleted file mode 100644 index 37f9236..0000000 --- a/lib/ycmd.py +++ /dev/null @@ -1,642 +0,0 @@ -#!/usr/bin/env python3 - -''' -lib/ycmd.py -High-level helpers for managing the youcompleteme daemon. -''' - -import http -import logging -import os -import socket -import tempfile - -# for type annotations only: -import io # noqa: F401 - -from lib.fs import ( - is_directory, - is_file, - get_base_name, - load_json_file, - save_json_file, - default_python_binary_path, -) -from lib.jsonmodels import ( - SYrequestParameters, - parse_completion_options, -) -from lib.process import ( - SYprocess, - SYfileHandles, -) -from lib.strutils import ( - truncate, - new_hmac_secret, - calculate_hmac, - format_json, - parse_json, - base64_encode, - bytes_to_str, - str_to_bytes, -) - -logger = logging.getLogger('sublime-ycmd.' + __name__) -# special logger instance for use in the server class -# this logger uses a filter to add server information to all log statements -server_logger = logging.getLogger('sublime-ycmd.' + __name__ + '.server') - -# most constants were taken from the ycmd example client: -# https://github.com/Valloric/ycmd/blob/master/examples/example_client.py - -# the missing ones were taken from the ycmd handler logic: -# https://github.com/Valloric/ycmd/blob/master/ycmd/handlers.py - -YCMD_HMAC_HEADER = 'X-Ycm-Hmac' -YCMD_HMAC_SECRET_LENGTH = 16 -# YCMD_SERVER_IDLE_SUICIDE_SECONDS = 30 # 30 secs -YCMD_SERVER_IDLE_SUICIDE_SECONDS = 5 * 60 # 5 mins -# YCMD_SERVER_IDLE_SUICIDE_SECONDS = 3 * 60 * 60 # 3 hrs -YCMD_MAX_SERVER_WAIT_TIME_SECONDS = 5 - -YCMD_HANDLER_GET_COMPLETIONS = '/completions' -YCMD_HANDLER_RUN_COMPLETER_COMMAND = '/run_completer_command' -YCMD_HANDLER_EVENT_NOTIFICATION = '/event_notification' -YCMD_HANDLER_DEFINED_SUBCOMMANDS = '/defined_subcommands' -YCMD_HANDLER_DETAILED_DIAGNOSTIC = '/detailed_diagnostic' -YCMD_HANDLER_LOAD_EXTRA_CONF = '/load_extra_conf_file' -YCMD_HANDLER_IGNORE_EXTRA_CONF = '/ignore_extra_conf_file' -YCMD_HANDLER_DEBUG_INFO = '/debug_info' -YCMD_HANDLER_SHUTDOWN = '/shutdown' - -YCMD_COMMAND_GET_TYPE = 'GetType' -YCMD_COMMAND_GET_PARENT = 'GetParent' -YCMD_COMMAND_GO_TO_DECLARATION = 'GoToDeclaration' -YCMD_COMMAND_GO_TO_DEFINTION = 'GoToDefinition' -YCMD_COMMAND_GO_TO = 'GoTo' -YCMD_COMMAND_GO_TO_IMPRECISE = 'GoToImprecise' -YCMD_COMMAND_CLEAR_COMPILATION_FLAG_CACHE = 'ClearCompilationFlagCache' - -YCMD_EVENT_FILE_READY_TO_PARSE = 'FileReadyToParse' -YCMD_EVENT_BUFFER_UNLOAD = 'BufferUnload' -YCMD_EVENT_BUFFER_VISIT = 'BufferVisit' -YCMD_EVENT_INSERT_LEAVE = 'InsertLeave' -YCMD_EVENT_CURRENT_IDENTIFIER_FINISHED = 'CurrentIdentifierFinished' - - -class SYserver(object): - ''' - Self-contained ycmd server object. Creates and maintains a persistent - connection to a ycmd server process. Provides a simple-ish way to send - API requests to the backend, including control functions like stopping and - pinging the server. - - TODO : Run all this stuff off-thread. - TODO : Unit tests. - ''' - - def __init__(self, process_handle=None, - hostname=None, port=None, hmac=None, label=None): - - self._process_handle = process_handle - - self._hostname = hostname - self._port = port - self._hmac = hmac - self._label = label - - self._reset_logger() - - def stop(self, hard=False): - if not self.alive(): - return - - if hard: - self._process_handle.kill() - else: - self._send_request(YCMD_HANDLER_SHUTDOWN, method='POST') - - def alive(self): - if not self._process_handle: - self._logger.debug('no process handle, ycmd server must be dead') - return False - - assert isinstance(self._process_handle, SYprocess), \ - '[internal] process handle is not SYprocess: %r' % \ - (self._process_handle) - # TODO : Also use the '/healthy' handler to check if it's alive. - return self._process_handle.alive() - - def communicate(self, inpt=None, timeout=None): - if not self._process_handle: - self._logger.debug('no process handle, cannot get process output') - return None, None - - assert isinstance(self._process_handle, SYprocess), \ - '[internal] process handle is not SYprocess: %r' % \ - (self._process_handle) - return self._process_handle.communicate(inpt=inpt, timeout=timeout) - - def _generate_hmac_header(self, method, path, body=None): - if body is None: - body = b'' - assert isinstance(body, bytes), 'body must be bytes: %r' % (body) - - content_hmac = calculate_hmac( - self._hmac, method, path, body, - ) - - return { - YCMD_HMAC_HEADER: content_hmac, - } - - def _send_request(self, handler, request_params=None, method=None): - ''' - Sends a request to the associated ycmd server and returns the response. - The `handler` should be one of the ycmd handler constants. - If `request_params` are supplied, it should be an instance of - `SYrequestParameters`. Most handlers require these parameters, and ycmd - will reject any requests that are missing them. - If `method` is provided, it should be an HTTP verb (e.g. 'GET', - 'POST'). If omitted, it is set to 'GET' when no parameters are given, - and 'POST' otherwise. - ''' - assert request_params is None or \ - isinstance(request_params, SYrequestParameters), \ - '[internal] request parameters is not SYrequestParameters: %r' % \ - (request_params) - assert method is None or isinstance(method, str), \ - '[internal] method is not a str: %r' % (method) - - has_params = request_params is not None - - if has_params: - logger.debug('generating json body from parameters') - json_params = request_params.to_json() - body = format_json(json_params) - else: - json_params = None - body = None - - if isinstance(body, str): - body = str_to_bytes(body) - - if not method: - method = 'GET' if not has_params else 'POST' - - hmac_headers = self._generate_hmac_header(method, handler, body) - content_type_headers = \ - {'Content-Type': 'application/json'} if has_params else None - - headers = {} - if hmac_headers: - headers.update(hmac_headers) - if content_type_headers: - headers.update(content_type_headers) - - self._logger.debug( - 'about to send a request with ' - 'method, handler, params, headers: %s, %s, %s, %s', - method, handler, truncate(json_params), truncate(headers), - ) - - response_status = None - response_reason = None - response_headers = None - response_data = None - try: - connection = http.client.HTTPConnection( - host=self.hostname, port=self.port, - ) - connection.request( - method=method, - url=handler, - body=body, - headers=headers, - ) - - response = connection.getresponse() - - response_status = response.status - response_reason = response.reason - - # TODO : Move http response logic somewhere else. - response_headers = response.getheaders() - response_content_type = response.getheader('Content-Type') - response_content_length = response.getheader('Content-Length', 0) - self._logger.critical('[TODO] verify hmac for response') - - try: - response_content_length = int(response_content_length) - except ValueError: - pass - - if response_content_length > 0: - response_content = response.read() - # TODO : EAFP, always try parsing as JSON. - if response_content_type == 'application/json': - response_data = parse_json(response_content) - else: - response_data = response_content - except http.client.HTTPException as e: - self._logger.error('error during ycmd request: %s', e) - except ConnectionError as e: - self._logger.error( - 'connection error, ycmd server may be dead: %s', e, - ) - - self._logger.critical( - '[REMOVEME] parsed status, reason, headers, data: %s, %s, %s, %s', - response_status, response_reason, response_headers, response_data, - ) - - return response_data - - def get_completer_commands(self, request_params): - return self._send_request( - YCMD_HANDLER_DEFINED_SUBCOMMANDS, - request_params=request_params, - method='POST', - ) - - def get_debug_info(self, request_params): - return self._send_request( - YCMD_HANDLER_DEBUG_INFO, - request_params=request_params, - method='POST', - ) - - def get_code_completions(self, request_params): - assert isinstance(request_params, SYrequestParameters), \ - 'request parameters must be SYrequestParameters: %r' % \ - (request_params) - - completion_data = self._send_request( - YCMD_HANDLER_GET_COMPLETIONS, - request_params=request_params, - method='POST', - ) - self._logger.debug( - 'received completion results: %s', truncate(completion_data), - ) - - completions = parse_completion_options(completion_data, request_params) - self._logger.debug('parsed completions: %r', completions) - - return completions - - def _notify_event(self, event_name, request_params, method='POST'): - assert isinstance(request_params, SYrequestParameters), \ - 'request parameters must be SYrequestParameters: %r' % \ - (request_params) - - self._logger.debug( - 'sending event notification for event: %s', event_name, - ) - - request_params['event_name'] = event_name - return self._send_request( - YCMD_HANDLER_EVENT_NOTIFICATION, - request_params=request_params, - method=method, - ) - - def notify_file_ready_to_parse(self, request_params): - return self._notify_event( - YCMD_EVENT_FILE_READY_TO_PARSE, - request_params=request_params, - ) - - def notify_buffer_enter(self, request_params): - return self._notify_event( - YCMD_EVENT_BUFFER_VISIT, - request_params=request_params, - ) - - def notify_buffer_leave(self, request_params): - return self._notify_event( - YCMD_EVENT_BUFFER_UNLOAD, - request_params=request_params, - ) - - def notify_leave_insert_mode(self, request_params): - return self._notify_event( - YCMD_EVENT_INSERT_LEAVE, - request_params=request_params, - ) - - def notify_current_identifier_finished(self, request_params): - return self._notify_event( - YCMD_EVENT_CURRENT_IDENTIFIER_FINISHED, - request_params=request_params, - ) - - @property - def hostname(self): - if not self._hostname: - self._logger.warning('server hostname is not set') - return self._hostname - - @hostname.setter - def hostname(self, hostname): - if not isinstance(hostname, str): - self._logger.warning('hostname is not a str: %r', hostname) - self._hostname = hostname - self._reset_logger() - - @property - def port(self): - if not self._port: - self._logger.warning('server port is not set') - return self._port - - @port.setter - def port(self, port): - if not isinstance(port, int): - self._logger.warning('port is not an int: %r', port) - self._port = port - self._reset_logger() - - @property - def hmac(self): - self._logger.error('returning server hmac secret... ' - 'nobody else should need it...') - return self._hmac - - @hmac.setter - def hmac(self, hmac): - if not isinstance(hmac, str): - self._logger.warning('server hmac secret is not a str: %r', hmac) - self._hmac = hmac - - @property - def label(self): - return self._label - - @label.setter - def label(self, label): - if not isinstance(label, str): - self._logger.warning('server label is not a str: %r', label) - self._label = label - - def _reset_logger(self): - self._logger = SYserverLoggerAdapter(server_logger, { - 'hostname': self._hostname or '?', - 'port': self._port or '?', - }) - - def pretty_str(self): - label_desc = ' "%s"' % (self._label) if self._label else '' - server_desc = 'ycmd server%s' % (label_desc) - - if not self._hmac: - return '%s - null' % (server_desc) - - if self._hostname is None or self._port is None: - return '%s - unknown' % (server_desc) - - return '%s - %s:%d' % (server_desc, self._hostname, self._port) - - def __str__(self): - return '%s:%s' % (self._hostname or '', self._port or '') - - -def get_ycmd_default_settings_path(ycmd_root_directory): - if not is_directory(ycmd_root_directory): - logger.warning('invalid ycmd root directory: %s', ycmd_root_directory) - # but whatever, fall through and provide the expected path anyway - - return os.path.join(ycmd_root_directory, 'ycmd', 'default_settings.json') - - -def generate_settings_data(ycmd_settings_path, hmac_secret): - ''' - Generates and returns a settings `dict` containing the options for - starting a ycmd server. This settings object should be written to a JSON - file and supplied as a command-line argument to the ycmd module. - The `hmac_secret` argument should be the binary-encoded HMAC secret. It - will be base64-encoded before adding it to the settings object. - ''' - assert isinstance(ycmd_settings_path, str), \ - 'ycmd settings path must be a str: %r' % (ycmd_settings_path) - if not is_file(ycmd_settings_path): - logger.warning( - 'ycmd settings path appears to be invalid: %r', ycmd_settings_path - ) - - ycmd_settings = load_json_file(ycmd_settings_path) - logger.debug('loaded ycmd settings: %s', ycmd_settings) - - assert isinstance(ycmd_settings, dict), \ - 'ycmd settings should be valid json: %r' % (ycmd_settings) - - # WHITELIST - # Enable for everything. This plugin will decide when to send requests. - if 'filetype_whitelist' not in ycmd_settings: - logger.warning( - 'ycmd settings template is missing the ' - 'filetype_whitelist placeholder' - ) - ycmd_settings['filetype_whitelist'] = {} - ycmd_settings['filetype_whitelist']['*'] = 1 - - # BLACKLIST - # Disable for nothing. This plugin will decide what to ignore. - if 'filetype_blacklist' not in ycmd_settings: - logger.warning( - 'ycmd settings template is missing the ' - 'filetype_blacklist placeholder' - ) - ycmd_settings['filetype_blacklist'] = {} - - # HMAC - # Pass in the hmac parameter. It needs to be base-64 encoded first. - if 'hmac_secret' not in ycmd_settings: - logger.warning( - 'ycmd settings template is missing the hmac_secret placeholder' - ) - - if not isinstance(hmac_secret, bytes): - logger.warning( - 'hmac secret was not passed in as binary, it might be incorrect' - ) - else: - logger.debug('converting hmac secret to base64') - hmac_secret_binary = hmac_secret - hmac_secret_encoded = base64_encode(hmac_secret_binary) - hmac_secret_str = bytes_to_str(hmac_secret_encoded) - hmac_secret = hmac_secret_str - - ycmd_settings['hmac_secret'] = hmac_secret - - # MISC - # Settings to ensure that the ycmd server is enabled whenever possible. - ycmd_settings['min_num_of_chars_for_completion'] = 0 - ycmd_settings['min_num_identifier_candidate_chars'] = 0 - ycmd_settings['collect_identifiers_from_comments_and_strings'] = 1 - ycmd_settings['complete_in_comments'] = 1 - ycmd_settings['complete_in_strings'] = 1 - - return ycmd_settings - - -def start_ycmd_server(ycmd_root_directory, - ycmd_settings_path=None, - ycmd_python_binary_path=None, - working_directory=None): - assert isinstance(ycmd_root_directory, str), \ - 'ycmd root directory must be a str: %r' % (ycmd_root_directory) - - if ycmd_settings_path is None: - ycmd_settings_path = \ - get_ycmd_default_settings_path(ycmd_root_directory) - assert isinstance(ycmd_settings_path, str), \ - 'ycmd settings path must be a str: %r' % (ycmd_settings_path) - - if ycmd_python_binary_path is None: - ycmd_python_binary_path = default_python_binary_path() - assert isinstance(ycmd_python_binary_path, str), \ - 'ycmd python binary path must be a str: %r' % (ycmd_python_binary_path) - - if working_directory is None: - working_directory = os.getcwd() - assert isinstance(working_directory, str), \ - 'working directory must be a str: %r' % (working_directory) - - ycmd_module_directory = os.path.join(ycmd_root_directory, 'ycmd') - - logger.debug( - 'preparing to start ycmd server with ' - 'ycmd path, default settings path, python binary path, ' - 'working directory: %s, %s, %s, %s', - ycmd_root_directory, ycmd_settings_path, - ycmd_python_binary_path, working_directory, - ) - - ycmd_hmac_secret = new_hmac_secret(num_bytes=YCMD_HMAC_SECRET_LENGTH) - ycmd_settings_data = \ - generate_settings_data(ycmd_settings_path, ycmd_hmac_secret) - - # no point using `with` for this, since we also use `delete=True` - temp_file_object = tempfile.NamedTemporaryFile( - prefix='ycmd_settings_', suffix='.json', delete=False, - ) - temp_file_name = temp_file_object.name - temp_file_handle = temp_file_object.file # type: io.TextIOWrapper - - save_json_file(temp_file_handle, ycmd_settings_data) - - temp_file_handle.flush() - temp_file_object.close() - - logger.critical('[REMOVEME] generating temporary files for log output') - - # toggle-able log file naming scheme - # use whichever to test/debug - _GENERATE_UNIQUE_TMPLOGS = False - if _GENERATE_UNIQUE_TMPLOGS: - stdout_file_object = tempfile.NamedTemporaryFile( - prefix='ycmd_stdout_', suffix='.log', delete=False, - ) - stderr_file_object = tempfile.NamedTemporaryFile( - prefix='ycmd_stderr_', suffix='.log', delete=False, - ) - stdout_file_name = stdout_file_object.name - stderr_file_name = stderr_file_object.name - stdout_file_object.close() - stderr_file_object.close() - else: - tempdir = tempfile.gettempdir() - alnum_working_directory = \ - ''.join(c if c.isalnum() else '_' for c in working_directory) - stdout_file_name = os.path.join( - tempdir, 'ycmd_stdout_%s.log' % (alnum_working_directory) - ) - stderr_file_name = os.path.join( - tempdir, 'ycmd_stderr_%s.log' % (alnum_working_directory) - ) - - logger.critical( - '[REMOVEME] keeping log files for stdout, stderr: %s, %s', - stdout_file_name, stderr_file_name, - ) - - ycmd_process_handle = SYprocess() - ycmd_server_hostname = '127.0.0.1' - ycmd_server_port = _get_unused_port(ycmd_server_hostname) - ycmd_server_label = get_base_name(working_directory) - - ycmd_process_handle.binary = ycmd_python_binary_path - ycmd_process_handle.args.extend([ - ycmd_module_directory, - '--host=%s' % (ycmd_server_hostname), - '--port=%s' % (ycmd_server_port), - '--idle_suicide_seconds=%s' % (YCMD_SERVER_IDLE_SUICIDE_SECONDS), - '--check_interval_seconds=%s' % (YCMD_MAX_SERVER_WAIT_TIME_SECONDS), - '--options_file=%s' % (temp_file_name), - # XXX : REMOVE ME - testing only - '--log=debug', - '--stdout=%s' % (stdout_file_name), - '--stderr=%s' % (stderr_file_name), - '--keep_logfiles', - ]) - ycmd_process_handle.cwd = working_directory - - # don't start it up just yet... set up the return value while we can - ycmd_server = SYserver( - process_handle=ycmd_process_handle, - hostname=ycmd_server_hostname, - port=ycmd_server_port, - hmac=ycmd_hmac_secret, - label=ycmd_server_label, - ) - - ycmd_process_handle.filehandles.stdout = SYfileHandles.PIPE - ycmd_process_handle.filehandles.stderr = SYfileHandles.PIPE - - try: - ycmd_process_handle.start() - except ValueError as e: - logger.critical('failed to launch ycmd server, argument error: %s', e) - except OSError as e: - logger.warning('failed to launch ycmd server, system error: %s', e) - finally: - pass - # TODO : Add this into the SYserver logic. - # It should check that the file is deleted after exit. - ''' - if is_file(temp_file_name): - # was not removed by startup, so we should clean up after it... - os.remove(temp_file_name) - ''' - - if not ycmd_process_handle.alive(): - stdout, stderr = ycmd_process_handle.communicate(timeout=0) - logger.error('failed to launch ycmd server, error output: %s', stderr) - - return ycmd_server - - -def _get_unused_port(interface='127.0.0.1'): - ''' Finds an available port for the ycmd server process to listen on. ''' - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.bind((interface, 0)) - - port = s.getsockname()[1] - logger.debug('found unused port: %d', port) - - s.close() - return port - - -class SYserverLoggerAdapter(logging.LoggerAdapter): - def __init__(self, logger, extra=None): - super(SYserverLoggerAdapter, self).__init__(logger, extra or {}) - - def process(self, msg, kwargs): - server_id = '(%s:%s)' % ( - self.extra.get('hostname', '?'), - self.extra.get('port', '?'), - ) - - return '%-16s %s' % (server_id, msg), kwargs diff --git a/lib/ycmd/__init__.py b/lib/ycmd/__init__.py new file mode 100644 index 0000000..e3bbead --- /dev/null +++ b/lib/ycmd/__init__.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 + +''' +lib/ycmd +sublime-ycmd ycmd server module. +''' diff --git a/lib/ycmd/constants.py b/lib/ycmd/constants.py new file mode 100644 index 0000000..8c40ffa --- /dev/null +++ b/lib/ycmd/constants.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +''' +lib/ycmd/constants.py +Constants for use in ycmd server configuration, including routes/handlers. + +All constants have the prefix `YCMD_`. The user-configurable constants have a +default value with the prefix `YCMD_DEFAULT_`. + +Most constants were taken from the ycmd example client: +https://github.com/Valloric/ycmd/blob/master/examples/example_client.py + +The missing ones were taken from the ycmd handler logic: +https://github.com/Valloric/ycmd/blob/master/ycmd/handlers.py +''' + +''' +HMAC header information. + +Not user configurable (likely won't change). +''' +YCMD_HMAC_HEADER = 'X-Ycm-Hmac' +YCMD_HMAC_SECRET_LENGTH = 16 + +''' +Server will automatically shut down when idle for `int` seconds. This can be +set fairly low if idle resource usage is an issue. The plugin is able to detect +when a server has shut down, and will create a new one if so. + +Use `0` to disable the behaviour entirely. This is definitely not recommended. +If this plugin fails to shut it down, it will continue to run indefinitely... + +User configurable. +''' +# YCMD_DEFAULT_SERVER_IDLE_SUICIDE_SECONDS = 30 # 30 secs +YCMD_DEFAULT_SERVER_IDLE_SUICIDE_SECONDS = 5 * 60 # 5 mins +# YCMD_DEFAULT_SERVER_IDLE_SUICIDE_SECONDS = 3 * 60 * 60 # 3 hrs + +''' +Server will wait at most `int` seconds for a response from a semantic +completion subserver (e.g. tern, jedi). If the subserver takes too long, ycmd +can at least return the list of identifiers. The delayed response will get +cached, and may be used in the next completion request. + +User configurable. +''' +YCMD_DEFAULT_MAX_SERVER_WAIT_TIME_SECONDS = 5 + +''' +Server handlers/routes. The server has an HTTP+JSON api, so these correspond to +the top-level functions available to clients (i.e. this plugin). + +Not user configurable (likely won't change). +''' +YCMD_HANDLER_GET_COMPLETIONS = '/completions' +YCMD_HANDLER_RUN_COMPLETER_COMMAND = '/run_completer_command' +YCMD_HANDLER_EVENT_NOTIFICATION = '/event_notification' +YCMD_HANDLER_DEFINED_SUBCOMMANDS = '/defined_subcommands' +YCMD_HANDLER_DETAILED_DIAGNOSTIC = '/detailed_diagnostic' +YCMD_HANDLER_LOAD_EXTRA_CONF = '/load_extra_conf_file' +YCMD_HANDLER_IGNORE_EXTRA_CONF = '/ignore_extra_conf_file' +YCMD_HANDLER_DEBUG_INFO = '/debug_info' +YCMD_HANDLER_READY = '/ready' +YCMD_HANDLER_HEALTHY = '/healthy' +YCMD_HANDLER_SHUTDOWN = '/shutdown' + +''' +Server request parameters, specific to `YCMD_HANDLER_RUN_COMPLETER_COMMAND`. + +Not all commands are available for all file types. To get the list of commands +for a given file type, use `YCMD_HANDLER_DEFINED_SUBCOMMANDS`, along with the +intended file type in the request body. + +Not user configurable (but should be, maybe). +''' +YCMD_COMMAND_GET_TYPE = 'GetType' +YCMD_COMMAND_GET_PARENT = 'GetParent' +YCMD_COMMAND_GO_TO_DECLARATION = 'GoToDeclaration' +YCMD_COMMAND_GO_TO_DEFINTION = 'GoToDefinition' +YCMD_COMMAND_GO_TO = 'GoTo' +YCMD_COMMAND_GO_TO_IMPRECISE = 'GoToImprecise' +YCMD_COMMAND_CLEAR_COMPILATION_FLAG_CACHE = 'ClearCompilationFlagCache' + +''' +Server request parameters, specific to `YCMD_HANDLER_EVENT_NOTIFICATION`. + +The only real required one is `YCMD_EVENT_FILE_READY_TO_PARSE`. The other ones +are optional, but help ycmd cache the data efficiently. + +Not user configurable (but should be, maybe). +''' +YCMD_EVENT_FILE_READY_TO_PARSE = 'FileReadyToParse' +YCMD_EVENT_BUFFER_UNLOAD = 'BufferUnload' +YCMD_EVENT_BUFFER_VISIT = 'BufferVisit' +YCMD_EVENT_INSERT_LEAVE = 'InsertLeave' +YCMD_EVENT_CURRENT_IDENTIFIER_FINISHED = 'CurrentIdentifierFinished' diff --git a/lib/ycmd/server.py b/lib/ycmd/server.py new file mode 100644 index 0000000..9b9851c --- /dev/null +++ b/lib/ycmd/server.py @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 + +''' +lib/ycmd/server.py +Server abstraction layer. + +Defines a server class that represents a connection to a ycmd server process. +Information about the actual server process is available via the properties. + +The ycmd server handlers are exposed as methods on this class. To send a +request to the server process, call the method, and it will package up the +parameters and send the request. These calls will block, and return the result +of the request (or raise an exception for unexpected errors). +''' + +import http +import logging + +from lib.process import Process +from lib.schema.completions import parse_completions +from lib.schema.request import RequestParameters +from lib.util.format import ( + json_serialize, + json_parse, +) +from lib.util.hmac import calculate_hmac +from lib.util.str import ( + str_to_bytes, + truncate, +) +from lib.ycmd.constants import ( + YCMD_HMAC_HEADER, + YCMD_HANDLER_SHUTDOWN, + YCMD_HANDLER_DEFINED_SUBCOMMANDS, + YCMD_HANDLER_DEBUG_INFO, + YCMD_HANDLER_GET_COMPLETIONS, + YCMD_HANDLER_EVENT_NOTIFICATION, + YCMD_EVENT_FILE_READY_TO_PARSE, + YCMD_EVENT_BUFFER_VISIT, + YCMD_EVENT_BUFFER_UNLOAD, + YCMD_EVENT_INSERT_LEAVE, + YCMD_EVENT_CURRENT_IDENTIFIER_FINISHED, +) + +logger = logging.getLogger('sublime-ycmd.' + __name__) +# special logger instance for use in the server class +# this logger uses a filter to add server information to all log statements +server_logger = logging.getLogger('sublime-ycmd.' + __name__ + '.server') + + +class Server(object): + ''' + Self-contained ycmd server object. Creates and maintains a persistent + connection to a ycmd server process. Provides a simple-ish way to send + API requests to the backend, including control functions like stopping and + pinging the server. + + TODO : Run all this stuff off-thread. + TODO : Unit tests. + ''' + + def __init__(self, process_handle=None, + hostname=None, port=None, hmac=None, label=None): + self._process_handle = process_handle + + self._hostname = hostname + self._port = port + self._hmac = hmac + self._label = label + + self._reset_logger() + + def stop(self, hard=False): + if not self.alive(): + return + + if hard: + self._process_handle.kill() + else: + self._send_request(YCMD_HANDLER_SHUTDOWN, method='POST') + + def alive(self): + if not self._process_handle: + self._logger.debug('no process handle, ycmd server must be dead') + return False + + assert isinstance(self._process_handle, Process), \ + '[internal] process handle is not Process: %r' % \ + (self._process_handle) + # TODO : Also use the '/healthy' handler to check if it's alive. + return self._process_handle.alive() + + def communicate(self, inpt=None, timeout=None): + if not self._process_handle: + self._logger.debug('no process handle, cannot get process output') + return None, None + + assert isinstance(self._process_handle, Process), \ + '[internal] process handle is not Process: %r' % \ + (self._process_handle) + return self._process_handle.communicate(inpt=inpt, timeout=timeout) + + def _generate_hmac_header(self, method, path, body=None): + if body is None: + body = b'' + assert isinstance(body, bytes), 'body must be bytes: %r' % (body) + + content_hmac = calculate_hmac( + self._hmac, method, path, body, + ) + + return { + YCMD_HMAC_HEADER: content_hmac, + } + + def _send_request(self, handler, request_params=None, method=None): + ''' + Sends a request to the associated ycmd server and returns the response. + The `handler` should be one of the ycmd handler constants. + If `request_params` are supplied, it should be an instance of + `RequestParameters`. Most handlers require these parameters, and ycmd + will reject any requests that are missing them. + If `method` is provided, it should be an HTTP verb (e.g. 'GET', + 'POST'). If omitted, it is set to 'GET' when no parameters are given, + and 'POST' otherwise. + ''' + assert request_params is None or \ + isinstance(request_params, RequestParameters), \ + '[internal] request parameters is not RequestParameters: %r' % \ + (request_params) + assert method is None or isinstance(method, str), \ + '[internal] method is not a str: %r' % (method) + + has_params = request_params is not None + + if has_params: + logger.debug('generating json body from parameters') + json_params = request_params.to_json() + body = json_serialize(json_params) + else: + json_params = None + body = None + + if isinstance(body, str): + body = str_to_bytes(body) + + if not method: + method = 'GET' if not has_params else 'POST' + + hmac_headers = self._generate_hmac_header(method, handler, body) + content_type_headers = \ + {'Content-Type': 'application/json'} if has_params else None + + headers = {} + if hmac_headers: + headers.update(hmac_headers) + if content_type_headers: + headers.update(content_type_headers) + + self._logger.debug( + 'about to send a request with ' + 'method, handler, params, headers: %s, %s, %s, %s', + method, handler, truncate(json_params), truncate(headers), + ) + + response_status = None + response_reason = None + response_headers = None + response_data = None + try: + connection = http.client.HTTPConnection( + host=self.hostname, port=self.port, + ) + connection.request( + method=method, + url=handler, + body=body, + headers=headers, + ) + + response = connection.getresponse() + + response_status = response.status + response_reason = response.reason + + # TODO : Move http response logic somewhere else. + response_headers = response.getheaders() + response_content_type = response.getheader('Content-Type') + response_content_length = response.getheader('Content-Length', 0) + self._logger.critical('[TODO] verify hmac for response') + + try: + response_content_length = int(response_content_length) + except ValueError: + pass + + if response_content_length > 0: + response_content = response.read() + # TODO : EAFP, always try parsing as JSON. + if response_content_type == 'application/json': + response_data = json_parse(response_content) + else: + response_data = response_content + except http.client.HTTPException as e: + self._logger.error('error during ycmd request: %s', e) + except ConnectionError as e: + self._logger.error( + 'connection error, ycmd server may be dead: %s', e, + ) + + self._logger.critical( + '[REMOVEME] parsed status, reason, headers, data: %s, %s, %s, %s', + response_status, response_reason, response_headers, response_data, + ) + + return response_data + + def get_completer_commands(self, request_params): + return self._send_request( + YCMD_HANDLER_DEFINED_SUBCOMMANDS, + request_params=request_params, + method='POST', + ) + + def get_debug_info(self, request_params): + return self._send_request( + YCMD_HANDLER_DEBUG_INFO, + request_params=request_params, + method='POST', + ) + + def get_code_completions(self, request_params): + assert isinstance(request_params, RequestParameters), \ + 'request parameters must be RequestParameters: %r' % \ + (request_params) + + completion_data = self._send_request( + YCMD_HANDLER_GET_COMPLETIONS, + request_params=request_params, + method='POST', + ) + self._logger.debug( + 'received completion results: %s', truncate(completion_data), + ) + + completions = parse_completions(completion_data, request_params) + self._logger.debug('parsed completions: %r', completions) + + return completions + + def _notify_event(self, event_name, request_params, method='POST'): + assert isinstance(request_params, RequestParameters), \ + 'request parameters must be RequestParameters: %r' % \ + (request_params) + + self._logger.debug( + 'sending event notification for event: %s', event_name, + ) + + request_params['event_name'] = event_name + return self._send_request( + YCMD_HANDLER_EVENT_NOTIFICATION, + request_params=request_params, + method=method, + ) + + def notify_file_ready_to_parse(self, request_params): + return self._notify_event( + YCMD_EVENT_FILE_READY_TO_PARSE, + request_params=request_params, + ) + + def notify_buffer_enter(self, request_params): + return self._notify_event( + YCMD_EVENT_BUFFER_VISIT, + request_params=request_params, + ) + + def notify_buffer_leave(self, request_params): + return self._notify_event( + YCMD_EVENT_BUFFER_UNLOAD, + request_params=request_params, + ) + + def notify_leave_insert_mode(self, request_params): + return self._notify_event( + YCMD_EVENT_INSERT_LEAVE, + request_params=request_params, + ) + + def notify_current_identifier_finished(self, request_params): + return self._notify_event( + YCMD_EVENT_CURRENT_IDENTIFIER_FINISHED, + request_params=request_params, + ) + + @property + def hostname(self): + if not self._hostname: + self._logger.warning('server hostname is not set') + return self._hostname + + @hostname.setter + def hostname(self, hostname): + if not isinstance(hostname, str): + self._logger.warning('hostname is not a str: %r', hostname) + self._hostname = hostname + self._reset_logger() + + @property + def port(self): + if not self._port: + self._logger.warning('server port is not set') + return self._port + + @port.setter + def port(self, port): + if not isinstance(port, int): + self._logger.warning('port is not an int: %r', port) + self._port = port + self._reset_logger() + + @property + def hmac(self): + self._logger.error('returning server hmac secret... ' + 'nobody else should need it...') + return self._hmac + + @hmac.setter + def hmac(self, hmac): + if not isinstance(hmac, str): + self._logger.warning('server hmac secret is not a str: %r', hmac) + self._hmac = hmac + + @property + def label(self): + return self._label + + @label.setter + def label(self, label): + if not isinstance(label, str): + self._logger.warning('server label is not a str: %r', label) + self._label = label + + def _reset_logger(self): + self._logger = ServerLoggerAdapter(server_logger, { + 'hostname': self._hostname or '?', + 'port': self._port or '?', + }) + + def pretty_str(self): + label_desc = ' "%s"' % (self._label) if self._label else '' + server_desc = 'ycmd server%s' % (label_desc) + + if not self._hmac: + return '%s - null' % (server_desc) + + if self._hostname is None or self._port is None: + return '%s - unknown' % (server_desc) + + return '%s - %s:%d' % (server_desc, self._hostname, self._port) + + def __str__(self): + return '%s:%s' % (self._hostname or '', self._port or '') + + +class ServerLoggerAdapter(logging.LoggerAdapter): + def __init__(self, logger, extra=None): + super(ServerLoggerAdapter, self).__init__(logger, extra or {}) + + def process(self, msg, kwargs): + server_id = '(%s:%s)' % ( + self.extra.get('hostname', '?'), + self.extra.get('port', '?'), + ) + + return '%-16s %s' % (server_id, msg), kwargs diff --git a/lib/ycmd/settings.py b/lib/ycmd/settings.py new file mode 100644 index 0000000..3d48387 --- /dev/null +++ b/lib/ycmd/settings.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 + +''' +lib/ycmd/settings.py +Utility functions for ycmd settings. +''' + +import logging +import os + +from lib.util.format import base64_encode +from lib.util.fs import ( + is_directory, + is_file, + load_json_file, +) +from lib.util.str import bytes_to_str + +logger = logging.getLogger('sublime-ycmd.' + __name__) + + +def get_default_settings_path(ycmd_root_directory): + ''' + Generates the path to the default settings json file from the ycmd module. + The `ycmd_root_directory` should refer to the path to the repository. + ''' + if not is_directory(ycmd_root_directory): + logger.warning('invalid ycmd root directory: %s', ycmd_root_directory) + # but whatever, fall through and provide the expected path anyway + + return os.path.join(ycmd_root_directory, 'ycmd', 'default_settings.json') + + +def generate_settings_data(ycmd_settings_path, hmac_secret): + ''' + Generates and returns a settings `dict` containing the options for + starting a ycmd server. This settings object should be written to a json + file and supplied as a command-line argument to the ycmd module. + The `hmac_secret` argument should be the binary-encoded HMAC secret. It + will be base64-encoded before adding it to the settings object. + ''' + assert isinstance(ycmd_settings_path, str), \ + 'ycmd settings path must be a str: %r' % (ycmd_settings_path) + if not is_file(ycmd_settings_path): + logger.warning( + 'ycmd settings path appears to be invalid: %r', ycmd_settings_path + ) + + ycmd_settings = load_json_file(ycmd_settings_path) + logger.debug('loaded ycmd settings: %s', ycmd_settings) + + assert isinstance(ycmd_settings, dict), \ + 'ycmd settings should be valid json: %r' % (ycmd_settings) + + # WHITELIST + # Enable for everything. This plugin will decide when to send requests. + if 'filetype_whitelist' not in ycmd_settings: + logger.warning( + 'ycmd settings template is missing the ' + 'filetype_whitelist placeholder' + ) + ycmd_settings['filetype_whitelist'] = {} + ycmd_settings['filetype_whitelist']['*'] = 1 + + # BLACKLIST + # Disable for nothing. This plugin will decide what to ignore. + if 'filetype_blacklist' not in ycmd_settings: + logger.warning( + 'ycmd settings template is missing the ' + 'filetype_blacklist placeholder' + ) + ycmd_settings['filetype_blacklist'] = {} + + # HMAC + # Pass in the hmac parameter. It needs to be base-64 encoded first. + if 'hmac_secret' not in ycmd_settings: + logger.warning( + 'ycmd settings template is missing the hmac_secret placeholder' + ) + + if not isinstance(hmac_secret, bytes): + logger.warning( + 'hmac secret was not passed in as binary, it might be incorrect' + ) + else: + logger.debug('converting hmac secret to base64') + hmac_secret_binary = hmac_secret + hmac_secret_encoded = base64_encode(hmac_secret_binary) + hmac_secret_str = bytes_to_str(hmac_secret_encoded) + hmac_secret = hmac_secret_str + + ycmd_settings['hmac_secret'] = hmac_secret + + # MISC + # Settings to ensure that the ycmd server is enabled whenever possible. + ycmd_settings['min_num_of_chars_for_completion'] = 0 + ycmd_settings['min_num_identifier_candidate_chars'] = 0 + ycmd_settings['collect_identifiers_from_comments_and_strings'] = 1 + ycmd_settings['complete_in_comments'] = 1 + ycmd_settings['complete_in_strings'] = 1 + + return ycmd_settings diff --git a/lib/ycmd/start.py b/lib/ycmd/start.py new file mode 100644 index 0000000..3b49dd4 --- /dev/null +++ b/lib/ycmd/start.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 + +''' +lib/ycmd/start.py +Server bootstrap logic. Starts up a ycmd server process and returns a handle. +''' + +import logging +import os +import socket +import tempfile + +from lib.process import ( + FileHandles, + Process, +) +from lib.util.fs import ( + default_python_binary_path, + get_base_name, + save_json_file, +) +from lib.util.hmac import new_hmac_secret +from lib.ycmd.constants import ( + YCMD_HMAC_SECRET_LENGTH, + YCMD_DEFAULT_SERVER_IDLE_SUICIDE_SECONDS, + YCMD_DEFAULT_MAX_SERVER_WAIT_TIME_SECONDS, +) +from lib.ycmd.server import Server +from lib.ycmd.settings import ( + get_default_settings_path, + generate_settings_data, +) + +logger = logging.getLogger('sublime-ycmd.' + __name__) + + +def start_ycmd_server(ycmd_root_directory, + ycmd_settings_path=None, working_directory=None, + ycmd_python_binary_path=None, + ycmd_server_idle_suicide_seconds=None, + ycmd_max_server_wait_time_seconds=None): + assert isinstance(ycmd_root_directory, str), \ + 'ycmd root directory must be a str: %r' % (ycmd_root_directory) + + if ycmd_settings_path is None: + ycmd_settings_path = \ + get_default_settings_path(ycmd_root_directory) + assert isinstance(ycmd_settings_path, str), \ + 'ycmd settings path must be a str: %r' % (ycmd_settings_path) + + if ycmd_python_binary_path is None: + ycmd_python_binary_path = default_python_binary_path() + assert isinstance(ycmd_python_binary_path, str), \ + 'ycmd python binary path must be a str: %r' % (ycmd_python_binary_path) + + if ycmd_server_idle_suicide_seconds is None: + ycmd_server_idle_suicide_seconds = \ + YCMD_DEFAULT_SERVER_IDLE_SUICIDE_SECONDS + assert isinstance(ycmd_server_idle_suicide_seconds, int), \ + 'ycmd server idle suicide seconds must be an int: %r' % \ + (ycmd_server_idle_suicide_seconds) + + if ycmd_max_server_wait_time_seconds is None: + ycmd_max_server_wait_time_seconds = \ + YCMD_DEFAULT_MAX_SERVER_WAIT_TIME_SECONDS + assert isinstance(ycmd_max_server_wait_time_seconds, int), \ + 'ycmd max server wait time seconds must be an int: %r' % \ + (ycmd_max_server_wait_time_seconds) + + if working_directory is None: + working_directory = os.getcwd() + assert isinstance(working_directory, str), \ + 'working directory must be a str: %r' % (working_directory) + + ycmd_module_directory = os.path.join(ycmd_root_directory, 'ycmd') + + logger.debug( + 'preparing to start ycmd server with ' + 'ycmd path, default settings path, python binary path, ' + 'working directory: %s, %s, %s, %s', + ycmd_root_directory, ycmd_settings_path, + ycmd_python_binary_path, working_directory, + ) + + ycmd_hmac_secret = new_hmac_secret(num_bytes=YCMD_HMAC_SECRET_LENGTH) + ycmd_settings_data = \ + generate_settings_data(ycmd_settings_path, ycmd_hmac_secret) + + # no point using `with` for this, since we also use `delete=True` + temp_file_object = tempfile.NamedTemporaryFile( + prefix='ycmd_settings_', suffix='.json', delete=False, + ) + temp_file_name = temp_file_object.name + temp_file_handle = temp_file_object.file # type: io.TextIOWrapper + + save_json_file(temp_file_handle, ycmd_settings_data) + + temp_file_handle.flush() + temp_file_object.close() + + logger.critical('[REMOVEME] generating temporary files for log output') + + # toggle-able log file naming scheme + # use whichever to test/debug + _GENERATE_UNIQUE_TMPLOGS = False + if _GENERATE_UNIQUE_TMPLOGS: + stdout_file_object = tempfile.NamedTemporaryFile( + prefix='ycmd_stdout_', suffix='.log', delete=False, + ) + stderr_file_object = tempfile.NamedTemporaryFile( + prefix='ycmd_stderr_', suffix='.log', delete=False, + ) + stdout_file_name = stdout_file_object.name + stderr_file_name = stderr_file_object.name + stdout_file_object.close() + stderr_file_object.close() + else: + tempdir = tempfile.gettempdir() + alnum_working_directory = \ + ''.join(c if c.isalnum() else '_' for c in working_directory) + stdout_file_name = os.path.join( + tempdir, 'ycmd_stdout_%s.log' % (alnum_working_directory) + ) + stderr_file_name = os.path.join( + tempdir, 'ycmd_stderr_%s.log' % (alnum_working_directory) + ) + + logger.critical( + '[REMOVEME] keeping log files for stdout, stderr: %s, %s', + stdout_file_name, stderr_file_name, + ) + + ycmd_process_handle = Process() + ycmd_server_hostname = '127.0.0.1' + ycmd_server_port = _get_unused_port(ycmd_server_hostname) + ycmd_server_label = get_base_name(working_directory) + + ycmd_process_handle.binary = ycmd_python_binary_path + ycmd_process_handle.args.extend([ + ycmd_module_directory, + '--host=%s' % (ycmd_server_hostname), + '--port=%s' % (ycmd_server_port), + '--idle_suicide_seconds=%s' % (ycmd_server_idle_suicide_seconds), + '--check_interval_seconds=%s' % (ycmd_max_server_wait_time_seconds), + '--options_file=%s' % (temp_file_name), + # XXX : REMOVE ME - testing only + '--log=debug', + '--stdout=%s' % (stdout_file_name), + '--stderr=%s' % (stderr_file_name), + '--keep_logfiles', + ]) + ycmd_process_handle.cwd = working_directory + + # don't start it up just yet... set up the return value while we can + ycmd_server = Server( + process_handle=ycmd_process_handle, + hostname=ycmd_server_hostname, + port=ycmd_server_port, + hmac=ycmd_hmac_secret, + label=ycmd_server_label, + ) + + ycmd_process_handle.filehandles.stdout = FileHandles.PIPE + ycmd_process_handle.filehandles.stderr = FileHandles.PIPE + + try: + ycmd_process_handle.start() + except ValueError as e: + logger.critical('failed to launch ycmd server, argument error: %s', e) + except OSError as e: + logger.warning('failed to launch ycmd server, system error: %s', e) + finally: + pass + # TODO : Add this into the Server logic. + # It should check that the file is deleted after exit. + ''' + if is_file(temp_file_name): + # was not removed by startup, so we should clean up after it... + os.remove(temp_file_name) + ''' + + if not ycmd_process_handle.alive(): + stdout, stderr = ycmd_process_handle.communicate(timeout=0) + logger.error('failed to launch ycmd server, error output: %s', stderr) + + return ycmd_server + + +def _get_unused_port(interface='127.0.0.1'): + ''' Finds an available port for the ycmd server process to listen on. ''' + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind((interface, 0)) + + port = s.getsockname()[1] + logger.debug('found unused port: %d', port) + + s.close() + return port diff --git a/runtests.py b/runtests.py index 3b48fc6..e751d7a 100755 --- a/runtests.py +++ b/runtests.py @@ -15,8 +15,8 @@ sys.path.append(os.path.dirname(__file__)) # noqa: E402 -import cli.args -import lib.logutils +from cli.args import base_cli_argparser +from lib.util.log import get_smart_truncate_formatter logger = logging.getLogger('sublime-ycmd.' + __name__) @@ -51,13 +51,9 @@ def configure_logging(log_level=None, output_stream=None): logger_stream = logging.StreamHandler(stream=output_stream) logger_stream.setLevel(log_level) - logger_formatter = logging.Formatter( - fmt=lib.logutils.get_extended_messagefmt(), - datefmt=lib.logutils.get_default_datefmt(), - ) + logger_formatter = get_smart_truncate_formatter() logger_stream.setFormatter(logger_formatter) logger_instance.addHandler(logger_stream) - lib.logutils.register_extension_filters(logger_stream) def get_test_suite_items(test_suite): @@ -89,7 +85,7 @@ def get_cli_argparser(): Generates and returns an argparse.ArgumentParser instance for use with parsing test-related options. ''' - parser = cli.args.base_cli_argparser( + parser = base_cli_argparser( description='sublime-ycmd unit test runner', ) @@ -106,7 +102,7 @@ def get_cli_argparser(): return parser -class SYtestRunnerLogStream(io.TextIOBase): +class TestRunnerLogStream(io.TextIOBase): ''' File stream wrapper class for use with TestRunner. Instances of this class will accept messages written by a test runner @@ -115,7 +111,7 @@ class SYtestRunnerLogStream(io.TextIOBase): ''' def __init__(self): - super(SYtestRunnerLogStream, self).__init__() + super(TestRunnerLogStream, self).__init__() self._buffer = '' def consume_line(self): @@ -164,7 +160,7 @@ def close(self): for buffered_line in self._buffer.splitlines(): self.testrunner_log(buffered_line) - super(SYtestRunnerLogStream, self).close() + super(TestRunnerLogStream, self).close() def main(): @@ -186,9 +182,10 @@ def main(): get_test_suite_items(test_suite)) logger.debug('about to run tests') - test_runner_logstream = SYtestRunnerLogStream() - test_runner = unittest.TextTestRunner(stream=test_runner_logstream, - verbosity=2) + test_runner_logstream = TestRunnerLogStream() + test_runner = unittest.TextTestRunner( + stream=test_runner_logstream, verbosity=2, + ) test_runner.run(test_suite) diff --git a/syplugin.py b/syplugin.py index 6bb3475..5b6b5c1 100755 --- a/syplugin.py +++ b/syplugin.py @@ -13,45 +13,37 @@ sys.path.append(os.path.dirname(__file__)) # noqa: E402 from cli.args import base_cli_argparser -from lib.jsonmodels import ( - SYcompletions, - SYcompletionOption, +from lib.util.log import ( + get_smart_truncate_formatter, ) -from lib.logutils import ( - get_default_datefmt, - get_extended_messagefmt, - register_extension_filters, +from lib.schema import ( + Completions, + CompletionOption, ) -from lib.st import ( - SYsettings, - SYview, +from lib.subl.settings import ( + Settings, bind_on_change_settings, +) +from lib.subl.view import ( + View, get_view_id, get_path_for_view, ) -from lib.ycmd import ( - SYserver, +from lib.ycmd.server import Server +from lib.ycmd.start import ( start_ycmd_server, ) logger = logging.getLogger('sublime-ycmd.' + __name__) -# TODO : refactor import wrapper logic try: import sublime import sublime_plugin _HAS_LOADED_ST = True except ImportError: - import collections + from lib.subl.dummy import sublime + from lib.subl.dummy import sublime_plugin _HAS_LOADED_ST = False - - class EventListenerDummy(object): - pass - - sublime = {} - SublimePluginDummy = \ - collections.namedtuple('SublimePluginDummy', ['EventListener']) - sublime_plugin = SublimePluginDummy(EventListenerDummy) finally: assert isinstance(_HAS_LOADED_ST, bool) @@ -94,13 +86,9 @@ def configure_logging(log_level=None, output_stream=None): # Don't log after here! Extension filters not set up yet logger_stream = logging.StreamHandler(stream=output_stream) logger_stream.setLevel(log_level) - logger_formatter = logging.Formatter( - fmt=get_extended_messagefmt(), - datefmt=get_default_datefmt(), - ) + logger_formatter = get_smart_truncate_formatter() logger_stream.setFormatter(logger_formatter) logger_instance.addHandler(logger_stream) - register_extension_filters(logger_stream) # Safe to log again logger.debug('successfully configured logging') @@ -124,7 +112,7 @@ def reset(self, ycmd_python_binary_path=None): if self._servers: for server in self._servers: - if not isinstance(server, SYserver): + if not isinstance(server, Server): logger.error( 'invalid server handle, clearing it: %r', server ) @@ -161,17 +149,17 @@ def reset(self, def get_servers(self): ''' - Returns a shallow-copy of the list of managed `SYserver` instances. + Returns a shallow-copy of the list of managed `Server` instances. ''' return self._servers[:] - def get_server_for_view(self, view): # type (sublime.View) -> SYserver + def get_server_for_view(self, view): # type (sublime.View) -> Server ''' - Returns a `SYserver` instance that has a suitable working directory for + Returns a `Server` instance that has a suitable working directory for use with the supplied `view`. If one does not exist, it will be created. ''' - if not isinstance(view, (sublime.View, SYview)): + if not isinstance(view, (sublime.View, View)): raise TypeError('view must be a View: %r' % (view)) view_id = get_view_id(view) @@ -217,11 +205,11 @@ def get_server_for_view(self, view): # type (sublime.View) -> SYserver self._view_id_to_server[view_id] = server self._working_directory_to_server[view_working_dir] = server - return server # type: SYserver + return server # type: Server def _unregister_server(self, server): - assert isinstance(server, SYserver), \ - '[internal] server must be SYserver: %r' % (server) + assert isinstance(server, Server), \ + '[internal] server must be Server: %r' % (server) if server not in self._servers: logger.error( 'server was never registered in server manager: %s', @@ -260,7 +248,7 @@ def __contains__(self, view): return view_id in self._view_id_to_server def __getitem__(self, view): - return self.get_server_for_view(view) # type: SYserver + return self.get_server_for_view(view) # type: Server def __len__(self): return len(self._servers) @@ -269,14 +257,14 @@ def __len__(self): class SublimeYcmdViewManager(object): ''' Singleton helper class. Manages wrappers around sublime view instances. - The wrapper class `SYview` is used around `sublime.View` to cache certain + The wrapper class `View` is used around `sublime.View` to cache certain calculations, and to store view-specific variables/state. Although this abstraction isn't strictly necessary, it can save expensive operations like file path calculation and ycmd event notification. ''' def __init__(self): - # maps view IDs to `SYview` instances + # maps view IDs to `View` instances self._views = {} self.reset() @@ -293,18 +281,18 @@ def reset(self): def get_wrapped_view(self, view): ''' - Returns an instance of `SYview` corresponding to `view`. If one does + Returns an instance of `View` corresponding to `view`. If one does not exist, it will be created, if possible. If the view is provided as an ID (int), then the lookup is performed as normal, but a `KeyError` will be raised if it does not exist. If the view is an instance of `sublime.View`, then the lookup is again performed as usual, but will be created if it does not exist. - Finally, if the view is an instance of `SYview`, it is returned as-is. + Finally, if the view is an instance of `View`, it is returned as-is. ''' - assert isinstance(view, (int, sublime.View, SYview)), \ + assert isinstance(view, (int, sublime.View, View)), \ 'view must be a View: %r' % (view) - if isinstance(view, SYview): + if isinstance(view, View): return view view_id = get_view_id(view) @@ -320,12 +308,12 @@ def get_wrapped_view(self, view): raise KeyError('view id is not registered: %r' % (view_id)) logger.debug('view is not registered, creating a wrapper for it') - wrapped_view = SYview(view) + wrapped_view = View(view) self._views[view_id] = wrapped_view assert view_id in self._views, \ '[internal] view id was not registered properly: %r' % (view_id) - return self._views[view_id] # type: SYview + return self._views[view_id] # type: View def has_notified_ready_to_parse(self, view, server): ''' @@ -347,7 +335,7 @@ def has_notified_ready_to_parse(self, view, server): logger.debug('view has not been sent to any server: %s', view) return False - # accept servers either as strings or as `SYserver` instances: + # accept servers either as strings or as `Server` instances: supplied_server_key = str(server) notified_server_key = view['last_notified_server'] @@ -400,7 +388,7 @@ def set_notified_ready_to_parse(self, view, server, has_notified=True): # initialize, but leave it empty view['notified_servers'] = {} - # accept servers either as strings or as `SYserver` instances: + # accept servers either as strings or as `Server` instances: supplied_server_key = str(server) notified_server_key = view['last_notified_server'] @@ -421,7 +409,7 @@ def set_notified_ready_to_parse(self, view, server, has_notified=True): del notified_servers[supplied_server_key] def _register_view(self, view): - assert isinstance(view, (sublime.View, SYview)), \ + assert isinstance(view, (sublime.View, View)), \ 'view must be a View: %r' % (view) view_id = get_view_id(view) @@ -433,8 +421,8 @@ def _register_view(self, view): logger.warning('view has already been registered, id: %s', view_id) if isinstance(view, sublime.View): - view = SYview(view) - elif not isinstance(view, SYview): + view = View(view) + elif not isinstance(view, View): logger.error('unknown view type: %r', view) raise TypeError('view must be a View: %r' % (view)) @@ -458,7 +446,7 @@ def _unregister_view(self, view): def get_views(self): ''' - Returns a shallow-copy of the map of managed `SYview` instances. + Returns a shallow-copy of the map of managed `View` instances. ''' return self._views.copy() @@ -501,7 +489,7 @@ def reset(self): def configure(self, settings): ''' Receives a `settings` object and reconfigures the state from it. - The settings should be an instance of `SYsettings`. See `lib.st` for + The settings should be an instance of `Settings`. See `lib.st` for helpers to generate these settings. If there are changes to the ycmd server settings, then the state will automatically stop all currently running servers. They will be @@ -527,8 +515,8 @@ def configure(self, settings): of the constants from the `logging` module: 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'. ''' - assert isinstance(settings, SYsettings), \ - 'settings must be SYsettings: %r' % (settings) + assert isinstance(settings, Settings), \ + 'settings must be Settings: %r' % (settings) requires_restart = self._requires_ycmd_restart(settings) if requires_restart: @@ -557,8 +545,8 @@ def activate_view(self, view): logger.debug('no plugin state, ignoring activate event') return False - server = self._server_manager[view] # type: SYserver - view = self._view_manager[view] # type: SYview + server = self._server_manager[view] # type: Server + view = self._view_manager[view] # type: View if not server: logger.debug('no server for view, ignoring activate event') return False @@ -602,8 +590,8 @@ def deactivate_view(self, view): logger.debug('no plugin state, ignoring deactivate event') return False - server = self._server_manager[view] # type: SYserver - view = self._view_manager[view] # type: SYview + server = self._server_manager[view] # type: Server + view = self._view_manager[view] # type: View if not server: logger.debug('no server for view, ignoring deactivate event') return False @@ -656,8 +644,8 @@ def completions_for_view(self, view): logger.debug('no plugin state, cannot provide completions') return None - server = self._server_manager[view] # type: SYserver - view = self._view_manager[view] # type: SYview + server = self._server_manager[view] # type: Server + view = self._view_manager[view] # type: View if not server: logger.debug('no server for view, cannot request completions') return None @@ -687,12 +675,12 @@ def completions_for_view(self, view): logger.debug('got completions for view: %s', completions) # TODO : Gracefully handle this by returning None - assert isinstance(completions, SYcompletions), \ - '[TODO] completions must be SYcompletions: %r' % (completions) + assert isinstance(completions, Completions), \ + '[TODO] completions must be Completions: %r' % (completions) def _st_completion_tuple(completion): - assert isinstance(completion, SYcompletionOption), \ - '[internal] completion is not SYcompletionOption: %r' % \ + assert isinstance(completion, CompletionOption), \ + '[internal] completion is not CompletionOption: %r' % \ (completion) # TODO : Calculate trigger properly st_trigger = completion.text() @@ -722,7 +710,7 @@ def __contains__(self, view): def __getitem__(self, view): ''' Wrapper around server manager. ''' - return self._server_manager[view] # type: SYserver + return self._server_manager[view] # type: Server def __len__(self): ''' Wrapper around server manager. ''' @@ -738,14 +726,14 @@ def _requires_ycmd_restart(self, settings): ycmd servers. This basically just compares the settings to the internal copy of the settings, and returns true if any ycmd parameters differ. ''' - assert isinstance(settings, SYsettings), \ - '[internal] settings must be SYsettings: %r' % (settings) + assert isinstance(settings, Settings), \ + '[internal] settings must be Settings: %r' % (settings) if not self._settings: # no settings - always trigger restart return True - assert isinstance(self._settings, SYsettings), \ - '[internal] settings must be SYsettings: %r' % (self._settings) + assert isinstance(self._settings, Settings), \ + '[internal] settings must be Settings: %r' % (self._settings) return self._settings != settings @@ -849,10 +837,10 @@ def views(self): def lookup_view(self, view): ''' - Returns a wrapped `SYview` for the given `sublime.View` in the view + Returns a wrapped `View` for the given `sublime.View` in the view manager, if one exists. Does NOT create one if it doesn't exist, just returns None. - If `view` is already a `SYview`, it is returned as-is. + If `view` is already a `View`, it is returned as-is. ''' view_manager = self._view_manager if view in view_manager: diff --git a/tests/test_fs.py b/tests/test_fs.py index d372da1..e47422f 100644 --- a/tests/test_fs.py +++ b/tests/test_fs.py @@ -8,8 +8,11 @@ import os import unittest -import lib.fs -import tests.utils +from lib.util.fs import get_common_ancestor +from tests.utils import ( + applytestfunction, + logtest, +) logger = logging.getLogger('sublime-ycmd.' + __name__) @@ -22,20 +25,20 @@ FS_ROOT = '' -class SYTgetCommonAncestor(unittest.TestCase): +class TestGetCommonAncestor(unittest.TestCase): ''' Unit tests for calculating the common ancestor between paths. The common ancestor should represent the path to a common file node between all the input paths. ''' - @tests.utils.logtest('get common ancestor : empty') + @logtest('get common ancestor : empty') def test_gca_empty(self): ''' Ensures that `None` is returned for an empty path list. ''' - result = lib.fs.get_common_ancestor([]) + result = get_common_ancestor([]) self.assertEqual(None, result, 'common ancestor of empty paths should be None') - @tests.utils.logtest('get common ancestor : single') + @logtest('get common ancestor : single') def test_gca_single(self): ''' Ensures that a single path always results in that same path. ''' single_paths = [ @@ -54,14 +57,14 @@ def test_gca_single(self): ] def test_gca_single_one(path): - result = lib.fs.get_common_ancestor([path]) + result = get_common_ancestor([path]) self.assertEqual(path, result) - tests.utils.applytestfunction( + applytestfunction( self, test_gca_single_one, single_path_args, ) - @tests.utils.logtest('get common ancestor : mostly similar') + @logtest('get common ancestor : mostly similar') def test_gca_mostly_similar(self): ''' Ensures that the common path can be found for many paths, with a long @@ -88,9 +91,9 @@ def test_gca_mostly_similar(self): ] def test_gca_mostly_similar_one(*paths, expected=''): - result = lib.fs.get_common_ancestor(paths) + result = get_common_ancestor(paths) self.assertEqual(expected, result) - tests.utils.applytestfunction( + applytestfunction( self, test_gca_mostly_similar_one, mostly_similar_path_args, ) diff --git a/tests/test_logutils.py b/tests/test_logutils.py index bd2c39e..5a158da 100644 --- a/tests/test_logutils.py +++ b/tests/test_logutils.py @@ -7,49 +7,86 @@ import logging import unittest -import lib.logutils -import tests.utils +from lib.util.log import ( + SmartTruncateFormatter, + FormatField, + parse_fields, +) +from tests.utils import ( + applytestfunction, + logtest, +) logger = logging.getLogger('sublime-ycmd.' + __name__) -def get_dummy_log_record(): +def make_log_record(msg='', **kwargs): + log_record = logging.LogRecord( + name=__name__, pathname=__file__, lineno=1, + level=logging.DEBUG, msg=msg, args=kwargs, + exc_info=None, func=None, sinfo=None, + ) + + for k, v in kwargs.items(): + setattr(log_record, k, v) + + return log_record + + +def make_format_field(name=None, zero=None, minus=None, space=None, + plus=None, width=None, point=None, type=None): + return FormatField( + name=name, zero=zero, minus=minus, space=space, + plus=plus, width=width, point=point, type=type, + ) + + +class TestFieldIterator(unittest.TestCase): ''' - Generates and returns a dummy log record for use in tests. These dummy - records contain a couple of extra parameters to play around with. + Unit tests for the log-format field iterator. This iterator should extract + information about each field in a format string. ''' - dummy_log_record = \ - logging.LogRecord(name=__name__, level=logging.DEBUG, - pathname=__file__, lineno=1, - msg='dummy log record! x = %(x)s, y = %(y)s', - args={'x': 'hello', 'y': 'world'}, - exc_info=None, func=None, sinfo=None) - setattr(dummy_log_record, 'x', 'hello') - setattr(dummy_log_record, 'y', 'world') - return dummy_log_record + @logtest('log-format field iterator : simple percent') + def test_fi_simple_percent(self): + ''' Ensures that single-item `%`-format fields are parsed. ''' + + single_fields = [ + ('%(foo)15s', make_format_field(name='foo', width='15', type='s')), + ('% 5ld', make_format_field(space=' ', width='5', type='ld')), + ('%-2s', make_format_field(minus='-', width='2', type='s')), + ] + single_field_args = [ + (f, {}) for f in single_fields + ] + + def test_lffi_single_percent(field, expected): + result = next(parse_fields(field)) + self.assertEqual(expected, result) + applytestfunction( + self, test_lffi_single_percent, single_field_args, + ) -class SYTlogPropertyShortener(unittest.TestCase): + +class TestTruncateFormatter(unittest.TestCase): ''' - Unit tests for the log-property shortening filter. This filter will shorten - a LogRecord property to a target length. + Unit tests for the log smart-truncate formatter. This formatter should try + to truncate fields that are longer than the target field width. ''' - @tests.utils.logtest('log-property shortener : in-place shorten property') - def test_lps_valid_property(self): - ''' Ensures that valid properties/target-lengths get shortened. ''' - x_shortener = lib.logutils.SYlogPropertyShortener() - x_shortener.set_property('x').set_length(1) - - dummy = get_dummy_log_record() - initial_x = getattr(dummy, 'x') + @logtest('log smart-truncate formatter : simple fields') + def test_tf_simple_percent_fields(self): + ''' Ensures that simple %-style field widths are handled. ''' - result = x_shortener.filter(dummy) - self.assertTrue(result, 'utility filter should always return True') + format_string = '%(short)4s %(long)-8s' + formatter = SmartTruncateFormatter(fmt=format_string, props={ + 'short': 4, + 'long': 8, + }) - result_x = getattr(dummy, 'x') - expected_x = initial_x[0] + record = make_log_record(short='hello', long='world') + formatted = formatter.format(record) - self.assertEqual(expected_x, result_x, - 'filter did not shorten property to 1 letter') + expected = ' hll world ' + self.assertEqual(expected, formatted) diff --git a/tests/test_process.py b/tests/test_process.py index 1fbbf24..890825f 100644 --- a/tests/test_process.py +++ b/tests/test_process.py @@ -8,8 +8,11 @@ import logging import unittest -import lib.process -import tests.utils +from lib.process import ( + FileHandles, + Process, +) +from tests.utils import logtest logger = logging.getLogger('sublime-ycmd.' + __name__) @@ -19,8 +22,8 @@ def make_output_stream(process): Creates an in-memory output stream and binds the given process' stdout and stderr to it. ''' - assert isinstance(process, lib.process.SYprocess), \ - '[internal] process is not a SYprocess instance: %r' % process + assert isinstance(process, Process), \ + '[internal] process is not a Process instance: %r' % process memstream = io.StringIO() @@ -35,14 +38,14 @@ def set_process_output_pipes(process): Sets the stdout and stderr handles for the process to be a PIPE. This allows reading stdout and stderr from the process. ''' - assert isinstance(process, lib.process.SYprocess), \ - '[internal] process is not a SYprocess instance: %r' % process + assert isinstance(process, Process), \ + '[internal] process is not a Process instance: %r' % process assert not process.alive(), \ '[internal] process is running already, cannot redirect outputs' - process.filehandles.stdout = lib.process.SYfileHandles.PIPE - process.filehandles.stderr = lib.process.SYfileHandles.PIPE + process.filehandles.stdout = FileHandles.PIPE + process.filehandles.stderr = FileHandles.PIPE def poll_process_output(process): @@ -51,8 +54,8 @@ def poll_process_output(process): This blocks until the process has either terminated, or has closed the output file descriptors. ''' - assert isinstance(process, lib.process.SYprocess), \ - '[internal] process is not a SYprocess instance: %r' % process + assert isinstance(process, Process), \ + '[internal] process is not a Process instance: %r' % process if process.alive(): logger.debug('process is still alive, this will likely block') @@ -61,16 +64,16 @@ def poll_process_output(process): return process.communicate(None, 3) -class SYTprocess(unittest.TestCase): +class TestProcess(unittest.TestCase): ''' Unit tests for the process class. This class should allow configuration and management of a generic process. ''' - @tests.utils.logtest('process : echo') - def test_pr_echo(self): + @logtest('process : echo') + def test_process_echo(self): ''' Ensures that the process can launch a simple echo command. ''' - echo_process = lib.process.SYprocess() + echo_process = Process() set_process_output_pipes(echo_process) echo_process.binary = 'echo' diff --git a/tests/utils.py b/tests/utils.py index 047b62c..42c0df3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -54,8 +54,9 @@ def applytestfunction(testinstance, testfunction, testcases): try: testfunction(*testcaseargs, **testcasekwargs) except Exception as exc: - exc_info = str(exc).splitlines()[0] - logger.debug('test case #%d resulted in an error: %s', + exc_lines = str(exc).splitlines() + exc_info = exc_lines[0] if exc_lines else exc + logger.debug('test case #%d resulted in an error: %r', testcounter, exc_info) raise else: