From e43ec5130aef57c623028601eb59629af9bc81a2 Mon Sep 17 00:00:00 2001 From: Milad Rastian Date: Sun, 13 Aug 2017 22:03:44 +0200 Subject: [PATCH] first stab on elixir_sense intgration start working on #97 --- .gitignore | 2 +- .travis.yml | 2 +- after/ftplugin/elixir.vim | 2 +- after/plugin/alchemist.vim | 21 +- alchemist_client | 35 +- autoload/elixircomplete.vim | 72 ++- elixir_sense.py | 367 +++++++++++ elixir_sense/.gitignore | 17 + elixir_sense/README.md | 125 ++++ elixir_sense/config/config.exs | 30 + .../lib/alchemist/helpers/complete.ex | 427 ++++++++++++ .../lib/alchemist/helpers/module_info.ex | 131 ++++ elixir_sense/lib/elixir_sense.ex | 300 +++++++++ elixir_sense/lib/elixir_sense/core/ast.ex | 156 +++++ .../lib/elixir_sense/core/introspection.ex | 599 +++++++++++++++++ .../lib/elixir_sense/core/metadata.ex | 82 +++ .../lib/elixir_sense/core/metadata_builder.ex | 364 +++++++++++ elixir_sense/lib/elixir_sense/core/parser.ex | 103 +++ elixir_sense/lib/elixir_sense/core/source.ex | 186 ++++++ elixir_sense/lib/elixir_sense/core/state.ex | 251 ++++++++ .../lib/elixir_sense/providers/definition.ex | 77 +++ .../lib/elixir_sense/providers/docs.ex | 28 + .../lib/elixir_sense/providers/eval.ex | 94 +++ .../lib/elixir_sense/providers/expand.ex | 47 ++ .../lib/elixir_sense/providers/signature.ex | 38 ++ .../lib/elixir_sense/providers/suggestion.ex | 150 +++++ elixir_sense/lib/elixir_sense/server.ex | 38 ++ .../lib/elixir_sense/server/context_loader.ex | 95 +++ .../elixir_sense/server/request_handler.ex | 53 ++ .../lib/elixir_sense/server/tcp_server.ex | 145 +++++ elixir_sense/mix.exs | 52 ++ elixir_sense/mix.lock | 14 + elixir_sense/run.exs | 28 + elixir_sense/run_test.exs | 6 + elixir_sense/t.exs | 33 + .../alchemist/helpers/module_info_test.exs | 42 ++ .../test/elixir_sense/all_modules_test.exs | 16 + .../test/elixir_sense/core/ast_test.exs | 32 + .../elixir_sense/core/introspection_test.exs | 106 +++ .../core/metadata_builder_test.exs | 608 ++++++++++++++++++ .../test/elixir_sense/core/metadata_test.exs | 91 +++ .../test/elixir_sense/core/parser_test.exs | 119 ++++ .../test/elixir_sense/core/source_test.exs | 379 +++++++++++ .../test/elixir_sense/definition_test.exs | 171 +++++ elixir_sense/test/elixir_sense/docs_test.exs | 297 +++++++++ elixir_sense/test/elixir_sense/eval_test.exs | 145 +++++ .../providers/suggestion_test.exs | 65 ++ .../test/elixir_sense/signature_test.exs | 146 +++++ .../test/elixir_sense/suggestions_test.exs | 230 +++++++ elixir_sense/test/elixir_sense_test.exs | 5 + elixir_sense/test/server_test.exs | 198 ++++++ elixir_sense/test/test_helper.exs | 1 + erl_terms.py | 301 +++++++++ t/fixtures/alchemist_server/valid.log | 2 +- 54 files changed, 7058 insertions(+), 66 deletions(-) create mode 100644 elixir_sense.py create mode 100644 elixir_sense/.gitignore create mode 100644 elixir_sense/README.md create mode 100644 elixir_sense/config/config.exs create mode 100644 elixir_sense/lib/alchemist/helpers/complete.ex create mode 100644 elixir_sense/lib/alchemist/helpers/module_info.ex create mode 100644 elixir_sense/lib/elixir_sense.ex create mode 100644 elixir_sense/lib/elixir_sense/core/ast.ex create mode 100644 elixir_sense/lib/elixir_sense/core/introspection.ex create mode 100644 elixir_sense/lib/elixir_sense/core/metadata.ex create mode 100644 elixir_sense/lib/elixir_sense/core/metadata_builder.ex create mode 100644 elixir_sense/lib/elixir_sense/core/parser.ex create mode 100644 elixir_sense/lib/elixir_sense/core/source.ex create mode 100644 elixir_sense/lib/elixir_sense/core/state.ex create mode 100644 elixir_sense/lib/elixir_sense/providers/definition.ex create mode 100644 elixir_sense/lib/elixir_sense/providers/docs.ex create mode 100644 elixir_sense/lib/elixir_sense/providers/eval.ex create mode 100644 elixir_sense/lib/elixir_sense/providers/expand.ex create mode 100644 elixir_sense/lib/elixir_sense/providers/signature.ex create mode 100644 elixir_sense/lib/elixir_sense/providers/suggestion.ex create mode 100644 elixir_sense/lib/elixir_sense/server.ex create mode 100644 elixir_sense/lib/elixir_sense/server/context_loader.ex create mode 100644 elixir_sense/lib/elixir_sense/server/request_handler.ex create mode 100644 elixir_sense/lib/elixir_sense/server/tcp_server.ex create mode 100644 elixir_sense/mix.exs create mode 100644 elixir_sense/mix.lock create mode 100644 elixir_sense/run.exs create mode 100644 elixir_sense/run_test.exs create mode 100644 elixir_sense/t.exs create mode 100644 elixir_sense/test/alchemist/helpers/module_info_test.exs create mode 100644 elixir_sense/test/elixir_sense/all_modules_test.exs create mode 100644 elixir_sense/test/elixir_sense/core/ast_test.exs create mode 100644 elixir_sense/test/elixir_sense/core/introspection_test.exs create mode 100644 elixir_sense/test/elixir_sense/core/metadata_builder_test.exs create mode 100644 elixir_sense/test/elixir_sense/core/metadata_test.exs create mode 100644 elixir_sense/test/elixir_sense/core/parser_test.exs create mode 100644 elixir_sense/test/elixir_sense/core/source_test.exs create mode 100644 elixir_sense/test/elixir_sense/definition_test.exs create mode 100644 elixir_sense/test/elixir_sense/docs_test.exs create mode 100644 elixir_sense/test/elixir_sense/eval_test.exs create mode 100644 elixir_sense/test/elixir_sense/providers/suggestion_test.exs create mode 100644 elixir_sense/test/elixir_sense/signature_test.exs create mode 100644 elixir_sense/test/elixir_sense/suggestions_test.exs create mode 100644 elixir_sense/test/elixir_sense_test.exs create mode 100644 elixir_sense/test/server_test.exs create mode 100644 elixir_sense/test/test_helper.exs create mode 100644 erl_terms.py diff --git a/.gitignore b/.gitignore index 461355a..0b9d272 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,4 @@ erl_crash.dump *.ez *.sw? -*.py* +*.py? diff --git a/.travis.yml b/.travis.yml index 2842568..a55a6fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ sudo: false script: - - python ./alchemist.py + - python ./elixir_sense.py notifications: irc: "irc.freenode.org#vim-elixir" diff --git a/after/ftplugin/elixir.vim b/after/ftplugin/elixir.vim index 7da75cc..12f7ce5 100644 --- a/after/ftplugin/elixir.vim +++ b/after/ftplugin/elixir.vim @@ -17,7 +17,7 @@ if !exists('g:alchemist#omnifunc') endif if exists('&omnifunc') && g:alchemist#omnifunc - setl omnifunc=elixircomplete#Complete + setl omnifunc=elixircomplete#auto_complete endif runtime! ftplugin/man.vim diff --git a/after/plugin/alchemist.vim b/after/plugin/alchemist.vim index 1ef218f..861fcae 100644 --- a/after/plugin/alchemist.vim +++ b/after/plugin/alchemist.vim @@ -8,18 +8,19 @@ if !exists('g:alchemist#alchemist_client') let g:alchemist#alchemist_client = expand(":p:h:h") . '/../alchemist_client' endif -function! alchemist#alchemist_client(req) - let req = a:req . "\n" +function! alchemist#alchemist_client(req, lnum, cnum, lines) + echom "alchemist_client" + let req = a:req let cmd = g:alchemist#alchemist_client - if !alchemist#ansi_enabled() - let cmd = cmd . ' --colors=false ' - endif - if exists('g:alchemist#elixir_erlang_src') - let cmd = cmd . ' -s ' . g:alchemist#elixir_erlang_src - endif + "if exists('g:alchemist#elixir_erlang_src') + " let cmd = cmd . ' -s ' . g:alchemist#elixir_erlang_src + "endif let cmd = cmd . ' -d "' . expand('%:p:h') . '"' - - return system(cmd, req) + let cmd = cmd . ' --line=' . a:lnum + let cmd = cmd . ' --column=' . a:cnum + let cmd = cmd . ' --request=' . a:req + echom cmd + return system(cmd, join(a:lines, "\n")) endfunction function! alchemist#get_doc(word) diff --git a/alchemist_client b/alchemist_client index ce7ca9a..63bcbb7 100755 --- a/alchemist_client +++ b/alchemist_client @@ -1,7 +1,7 @@ #!/usr/bin/env python from __future__ import print_function import os, sys, getopt -from alchemist import AlchemistClient +from elixir_sense import ElixirSenseClient debug = False @@ -22,14 +22,16 @@ def alchemist_help(): """ def main(argv): - cmd_type = "" - cmd = "" + request = "" + line = 0 + column = 0 cwd = "" alchemist_script = "" ansi = True + elixir_erlang_source_dir = "" source = "" try: - opts, args = getopt.getopt(argv,"hc:t:d:a:s:",["type=","command-type=", "directory=", "alchemist-server=", "colors=", "source="]) + opts, args = getopt.getopt(argv,"hr:l:c:d:",["request=","line=", "column=", "directory=", "alchemist-server=", "colors=", "source_dir="]) except getopt.GetoptError: print(alchemist_help()) sys.exit(2) @@ -37,36 +39,35 @@ def main(argv): if opt == '-h': print(alchemist_help()) sys.exit() - elif opt in ("-c", "--command"): - cmd = arg - elif opt in ("-t", "--command-type"): - cmd_type = arg + elif opt in ("-r", "--request"): + request = arg + elif opt in ("-l", "--line"): + line = arg + elif opt in ("-c", "--column"): + column = arg elif opt in ("-d", "--directory"): cwd = arg elif opt in ("-a", "--alchemist-server"): alchemist_script = arg elif opt in ("-s", "--source"): - source = arg + elixir_erlang_source_dir = arg elif opt in ("--colors"): if arg == "false": ansi = False if os.path.exists(cwd.strip()) == False: raise Exception("working directory [%s] doesn't exist" % cwd) cwd = os.path.abspath(cwd) - if cmd == "": - cmd = read_stdin() + source = sys.stdin.read() if alchemist_script == "": - alchemist_script = "%s/alchemist-server/run.exs" % where_am_i() - if cmd_type == "": - cmd_type = cmd.split(" ")[0] + alchemist_script = "%s/elixir_sense/run.exs" % where_am_i() - if cmd == "" or cmd_type == "" or alchemist_script == "" or cwd == "": + if "" in [request, alchemist_script, request, line, column]: print("Invalid command, alchemist_script or working directory") print(alchemist_help()) sys.exit(2) - a = AlchemistClient(debug=debug, cwd=cwd, ansi=ansi, alchemist_script=alchemist_script, source=source) - print(a.process_command(cmd), end="") + s = ElixirSenseClient(debug=debug, cwd=cwd, ansi=ansi, elixir_sense_script=alchemist_script) + print(s.process_command(request, source, line ,column), end="") def where_am_i(): diff --git a/autoload/elixircomplete.vim b/autoload/elixircomplete.vim index 0a52ddf..359139c 100644 --- a/autoload/elixircomplete.vim +++ b/autoload/elixircomplete.vim @@ -1,18 +1,8 @@ -" This code partially is based on helpex.vim project -" Authors: -" * sanmiguel -" * Milad - if !exists('g:alchemist#alchemist_client') let g:alchemist#alchemist_client = expand(":p:h:h") . '/../alchemist_client' endif -let s:elixir_namespace= '\<[A-Z][[:alnum:]]\+\(\.[A-Z][[:alnum:].]\+\)*.*$' -let s:erlang_module= ':\<' -let s:elixir_fun_w_arity = '.*/[0-9]$' -let s:elixir_module = '[A-Z][[:alnum:]_]\+\([A_Z][[:alnum:]_]+\)*' - function! elixircomplete#ExDocComplete(ArgLead, CmdLine, CursorPos, ...) let suggestions = elixircomplete#Complete(0, a:ArgLead) if type(suggestions) != type([]) @@ -25,23 +15,27 @@ function! s:strip_dot(input_string) return substitute(a:input_string, '^\s*\(.\{-}\)\.*$', '\1', '') endfunction -function! elixircomplete#Complete(findstart, base_or_suggestions) +function! elixircomplete#auto_complete(findstart, base_or_suggestions) + let cnum = col('.') if a:findstart - return s:FindStart() - else - return s:build_completions(a:base_or_suggestions) + return s:find_start() + end + let suggestions = elixircomplete#get_suggestions(a:base_or_suggestions) + if len(suggestions) == 0 + return -1 endif + return suggestions endfunction " Omni findstart phase. -function! s:FindStart() +function! s:find_start() " return int 0 < n <= col('.') " " if the column left of us is whitespace, or [(){}[]] " no word let col = col('.') " get the column to the left of us - if strpart(getline(line('.')), col-2, 1) =~ '[{}() ]' + if strpart(getline(line('.')), col-2, 1) =~ '[{}() ]' return col - 1 endif " TODO This is a pretty dirty way to go about this @@ -55,28 +49,36 @@ function! s:FindStart() endfunction -function! s:build_completions(base_or_suggestions) - if type(a:base_or_suggestions) == type([]) - let suggestions = a:base_or_suggestions - else - let suggestions = elixircomplete#get_suggestions(a:base_or_suggestions) - endif - - if len(suggestions) == 0 - return -1 - endif - return suggestions -endfunction - -function! elixircomplete#get_suggestions(hint) - let req = alchemist#alchemist_format("COMPX", a:hint, "Elixir", [], []) - let result = alchemist#alchemist_client(req) - let suggestions = filter(split(result, '\n'), 'v:val != "END-OF-COMPX"') +function! elixircomplete#get_suggestions(base_or_suggestions) + let req = 'suggestions' + let lnum = line('.') + let cnum = col('.') + len(a:base_or_suggestions) + let blines = getline(1, lnum -1) + let cline = getline('.') + let before_c = strpart(getline('.'), 0, col('.')) + let after_c = strpart(getline('.'), col('.'), len(getline('.'))) + let cline = before_c . a:base_or_suggestions . after_c + let alines = getline(lnum +1 , '$') + let lines = blines + [cline] + alines + let result = alchemist#alchemist_client(req, lnum, cnum, lines) + let suggestions = split(result, '\n') let parsed_suggestion = [] for sugg in suggestions - let details = matchlist(sugg, 'kind:\(.*\), word:\(.*\), abbr:\(.*\)$') + let details = matchlist(sugg, 'kind:\(.*\), word:\(.*\), abbr:\(.*\), menu:\(.*\)$') if len(details) > 0 - let a = {'kind': details[1], 'word': details[2], 'abbr': details[3]} + if details[1] == 'f' + let word = details[2] + let sug_parts = split(a:base_or_suggestions, '\.') + if len(sug_parts) == 1 && a:base_or_suggestions[len(a:base_or_suggestions) -1] != '.' + let word_parts = split(word, '\.') + let word_size = len(word_parts) - 1 + let word = word_parts[word_size] + endif + let a = {'kind': details[1], 'word': word, 'abbr': details[3], 'menu': details[4], 'dup': 1} + elseif details[1] == 'm' || details[1] == 'p' || details[1] == 'e' || details[1] == 's' + let word = details[2] + let a = {'kind': details[1], 'word': word, 'menu': details[4], 'abbr': details[3]} + endif call add(parsed_suggestion, a) endif endfor diff --git a/elixir_sense.py b/elixir_sense.py new file mode 100644 index 0000000..d7ecb4c --- /dev/null +++ b/elixir_sense.py @@ -0,0 +1,367 @@ +from __future__ import print_function +import os +import tempfile +import re +import pprint +import subprocess, shlex +import select, socket +import time +import syslog +import struct +import erl_terms + +class ElixirSenseClient: + + + def __init__(self, **kw): + self._debug = kw.get('debug', False) + self._cwd = kw.get('cwd', '') + self.__create_tmp_dir() + self._cwd = self.get_project_base_dir() + self._ansi = kw.get('ansi', True) + self._alchemist_script = kw.get('elixir_sense_script', None) + self._source = kw.get('source', None) + self.re_elixir_fun_with_arity = re.compile(r'(?P.*)/[0-9]+$') + self.re_elixir_module_and_fun = re.compile(r'^(?P[A-Z][A-Za-z0-9\._]+)\.(?P[a-z_?!]+)') + self.re_erlang_module = re.compile(r'^\:(?P.*)') + self.re_elixir_module = re.compile(r'^(?P[A-Z][A-Za-z0-9\._]+)') + self.re_x_base = re.compile(r'^.*{\s*"(?P.*)"\s*') + self.re_elixir_src = re.compile(r'.*(/lib/elixir/lib.*)') + self.re_erlang_src = re.compile(r'.*otp.*(/lib/.*\.erl)') + + + def __create_tmp_dir(self): + dir_tmp = self._get_tmp_dir() + if os.path.exists(dir_tmp) == False: + os.makedirs(self._get_tmp_dir()) + def process_command(self, request, source, line, column): + self._log('column: %s' % column) + self._log('line: %s' % line) + self._log('source: %s' % source) + py_struct = { + 'request_id': 1, + 'auth_token': None, + 'request': request, + 'payload': { + 'buffer': source, + 'line': int(line), + 'column': int(column) + } + } + + req_erl_struct = erl_terms.encode(py_struct) + server_log = self._get_running_server_log() + if server_log == None: + server_log = self._create_server_log() + self._run_alchemist_server(server_log) + + connection = self._extract_connection_settings(server_log) + if connection == None: + self._run_alchemist_server(server_log) + connection = self._extract_connection_settings(server_log) + + sock = self._connect(connection) + if sock == None: + self._run_alchemist_server(server_log) + connection = self._extract_connection_settings(server_log) + if connection == None: + self._log("Couldn't find the connection settings from server_log: %s" % (server_log)) + return None + sock = self._connect(connection) + + resp_erl_struct = self._send_command(sock, req_erl_struct) + #f = open("/tmp/erl_bin.txt", "wb") + #f.write(resp_erl_struct) + rep_py_struct = erl_terms.decode(resp_erl_struct) + self._log('ElixirSense: %s' % rep_py_struct) + if request == "suggestions": + return self.to_vim_suggestions(rep_py_struct['payload']) + + def to_vim_suggestions(self, suggestions): + """ + >>> alchemist = ElixirSenseClient() + >>> alchemist.to_vim_suggestions([{'type': 'hint', 'value': 'Enum.ma'}, {'origin': 'Enum', 'arity': 2, 'name': 'map', 'args': 'enumerable,fun', 'type': 'function', 'spec': '@spec map(t, (element -> any)) :: list', 'summary': 'Returns a list where each item is the result of invoking`fun` on each corresponding item of `enumerable`.'}]) + 'kind:f, word:Enum.map, abbr:map(enumerable,fun), menu: Enum\\n' + """ + result = '' + prefix_module = '' + for s in suggestions: + if s['type'] == 'hint': + if s['value'][-1] == '.': + prefix_module = s['value'] + continue + if s['type'] == 'module': + if ('%s.' % s['name']) == prefix_module: + continue + mtype = s['subtype'] or s['type'] + result = "%skind:%s, word:%s%s, abbr:%s, menu: %s\n" % (result, 'm', prefix_module, s['name'], s['name'], mtype) + if s['type'] == 'function': + args = '%s(%s)' % (s['name'], s['args']) + result = "%skind:%s, word:%s.%s, abbr:%s, menu: %s\n" % (result, 'f', s['origin'], s['name'], args, s['origin']) + + return result + + def _log(self, text): + f = open("/tmp/log.log", "a") + f.write("%s\n" % text) + f.close() + + if self._debug == False: + return + syslog.openlog("alchemist_client") + syslog.syslog(syslog.LOG_ALERT, text) + + def _get_path_unique_name(self, path): + """ + >>> alchemist = ElixirSenseClient() + >>> alchemist._get_path_unique_name("/Users/milad/dev/ex_guard/") + 'zSUserszSmiladzSdevzSex_guard' + """ + return os.path.abspath(path).replace("/", "zS") + + def _create_server_log(self): + dir_tmp = self._get_tmp_dir() + log_tmp = "%s/%s" % (dir_tmp, self._cwd.replace("/", "zS")) + if os.path.exists(dir_tmp) == False: + os.makedirs(dir_tmp) + + if os.path.exists(log_tmp) == False: + return log_tmp + + return None + + def _get_running_server_log(self): + dir_tmp = self._get_tmp_dir() + log_tmp = "%s/%s" % (dir_tmp, self._get_path_unique_name(self._cwd)) + self._log("Load server settings from: %s" % (log_tmp)) + if os.path.exists(dir_tmp) == False: + return None + + if os.path.exists(log_tmp) == True: + return log_tmp + + return None + + def _run_alchemist_server(self, server_log): + """ + execute alchemist server and wait until it has printed a line + into STDOUT + """ + alchemist_script = self._alchemist_script + if os.path.exists(alchemist_script) == False: + raise Exception("alchemist script does not exist in (%s)" % alchemist_script) + alchemist_script = "elixir %s unix 0 dev" % alchemist_script + self._log(alchemist_script) + arg = shlex.split(alchemist_script) + log_file = open(server_log, "w") + subprocess.Popen(arg, stdout=log_file, stderr=log_file, stdin=log_file, cwd=self._cwd) + for t in range(0, 50): + time.sleep(0.1) + r = open(server_log).readlines() + if len(r) > 0: + break + + def _connect(self, host_port): + if host_port == None: return None + (host, port) = host_port + if isinstance(port, str): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + host_port = port + else: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + try: + sock.connect(host_port) + except socket.error as e: + self._log("Can not establish connection to %s, error: %s" % (host_port, e)) + return None + + return sock + + def _send_command(self, sock, cmd): + packer = struct.Struct('!I') + packed_data = packer.pack(len(cmd)) + sock.sendall(packed_data + cmd) + try: + return self._sock_readlines(sock) + except socket.error as e: + if e.errno == 35: + raise Exception("reached 10 sec timeout, error:Resource temporarily unavailable") + else: + raise e + + self._log("response for %s: %s" % (cmd.split(" ")[0], result.replace('\n', '\\n'))) + return '' + + def _find_elixir_erlang_src(self, filename): + if self._is_readable(filename): + return filename + if self.re_elixir_src.match(filename): + elixir_src_file = "%s/elixir/%s" % (self._source, self.re_elixir_src.match(filename).group(1)) + if self._is_readable(elixir_src_file): + return os.path.realpath(elixir_src_file) + elif self.re_erlang_src.match(filename): + erlang_src_file = "%s/otp/%s" % (self._source, self.re_erlang_src.match(filename).group(1)) + if self._is_readable(erlang_src_file): + return os.path.realpath(erlang_src_file) + return filename + + def _find_module_line(self, filename, module): + return self._find_pattern_in_file( + filename, + ["defmodule %s" % module, "-module(%s)." % module[1:]]) + + def _find_function_line(self, filename, function): + return self._find_pattern_in_file( + filename, + ["def %s" % function, "defp %s" % function, "-spec %s" % function]) + + def _find_pattern_in_file(self, filename, patterns): + if not os.path.isfile(filename) or not os.access(filename, os.R_OK): + return 1 + lines = open(filename, "r").readlines() + for line_num, line_str in enumerate(lines): + matched_p = list(filter(lambda p: p in line_str, patterns)) + if len(matched_p) > 0: + return line_num + 1 + return 1 + + def _is_readable(self, filename): + if os.path.isfile(filename) and os.access(filename, os.R_OK): + return True + return False + + def _sock_readlines(self, sock, recv_buffer=4096, timeout=10): + sock.setblocking(0) + packet_size = -1 + + select.select([sock], [], [], timeout) + data = sock.recv(recv_buffer) + (packet_size, ) = struct.unpack('!I', data[:4]) + buf = data[4:] + while len(buf) < packet_size: + select.select([sock], [], [], timeout) + data = sock.recv(recv_buffer) + buf = buf + data + return buf + + def _extract_connection_settings(self, server_log): + """ + >>> alchemist = ElixirSenseClient() + >>> server_log = "t/fixtures/alchemist_server/valid.log" + >>> print(alchemist._extract_connection_settings(server_log)) + ('localhost', '/tmp/elixir-sense-1502654288590225000.sock') + + >>> server_log = "t/fixtures/alchemist_server/invalid.log" + >>> print(alchemist._extract_connection_settings(server_log)) + None + """ + for line in open(server_log, "r").readlines(): + self._log(line) + match = re.search(r'ok\:(?P\w+):(?P.*\.sock)', line) + if match: + (host, port) = match.groups() + try : + return (host, int(port)) + except : + return (host, port) + return None + + def _get_tmp_dir(self): + """ + >>> alchemist = ElixirSenseClient() + >>> os.environ['TMPDIR'] = '/tmp/foo01/' + >>> alchemist._get_tmp_dir() + '/tmp/foo01/alchemist_server' + >>> del os.environ['TMPDIR'] + >>> os.environ['TEMP'] = '/tmp/foo02/' + >>> alchemist._get_tmp_dir() + '/tmp/foo02/alchemist_server' + >>> del os.environ['TEMP'] + >>> os.environ['TMP'] = '/tmp/foo03/' + >>> alchemist._get_tmp_dir() + '/tmp/foo03/alchemist_server' + >>> del os.environ['TMP'] + >>> alchemist._get_tmp_dir() == tempfile.tempdir #TODO: revert + False + """ + for var in ['TMPDIR', 'TEMP', 'TMP']: + if var in os.environ: + return os.path.abspath("%s/alchemist_server" % os.environ[var]) + if tempfile.tempdir != None: + return os.path.abspath("%s/alchemist_server" % tempfile.tempdir) + + return "%s/alchemist_server" % "/tmp" + + pass + + def get_project_base_dir(self, running_servers_logs=None): + """ + >>> #prepare the test env + >>> tmp_dir = tempfile.mkdtemp() + >>> p01_dir = os.path.join(tmp_dir, "p01") + >>> os.mkdir(p01_dir) + >>> p01_lib_dir = os.path.join(p01_dir, "lib") + >>> os.mkdir(p01_lib_dir) + >>> p01_log = p01_dir.replace("/", "zS") + + >>> #detect that base dir is already running + >>> alchemist = ElixirSenseClient(cwd=p01_dir) + >>> alchemist.get_project_base_dir([p01_log]) == p01_dir + True + >>> #since server is running on base dir, if lib dir is given, should return base dir + >>> alchemist = ElixirSenseClient(cwd=p01_lib_dir) + >>> alchemist.get_project_base_dir([p01_log]) == p01_dir + True + >>> #if given dir is out of base dir, should return the exact dir + >>> alchemist = ElixirSenseClient(cwd=tmp_dir) + >>> alchemist.get_project_base_dir([p01_log]) == tmp_dir + True + >>> #since there is no running server, lib dir is detected as base dir + >>> alchemist = ElixirSenseClient(cwd=p01_lib_dir) + >>> alchemist.get_project_base_dir([]) == p01_lib_dir + True + >>> #prepare mix test + >>> open(os.path.join(p01_dir, "mix.exs"), 'a').close() + >>> #should find mix.exs recursively and return base dir + >>> alchemist = ElixirSenseClient(cwd=p01_lib_dir) + >>> alchemist.get_project_base_dir([]) == p01_dir + True + >>> #find directory of parent when running inside a nested project + >>> apps = os.path.join(p01_dir, "apps") + >>> os.mkdir(apps) + >>> nested = os.path.join(apps, "nested_project") + >>> os.mkdir(nested) + >>> nested_lib = os.path.join(nested, "lib") + >>> os.mkdir(nested_lib) + >>> open(os.path.join(nested, "mix.exs"), 'a').close() + >>> alchemist = ElixirSenseClient(cwd=nested_lib) + >>> alchemist.get_project_base_dir([]) == p01_dir + True + """ + + if running_servers_logs == None: + running_servers_logs = os.listdir(self._get_tmp_dir()) + paths = self._cwd.split(os.sep) + mix_dir = [] + for i in range(len(paths)): + project_dir = os.sep.join(paths[:len(paths)-i]) + if not project_dir: + continue + log_tmp = "%s" % project_dir.replace("/", "zS") + if log_tmp in running_servers_logs: + self._log("project_dir(matched): "+str(project_dir)) + return project_dir + + if os.path.exists(os.path.join(project_dir, "mix.exs")): + mix_dir.append(project_dir) + + self._log("mix_dir: "+str(mix_dir)) + if len(mix_dir): + return mix_dir.pop() + + return self._cwd + +if __name__ == "__main__": + import doctest + doctest.testmod() diff --git a/elixir_sense/.gitignore b/elixir_sense/.gitignore new file mode 100644 index 0000000..6e1db0f --- /dev/null +++ b/elixir_sense/.gitignore @@ -0,0 +1,17 @@ +# The directory Mix will write compiled artifacts to. +/_build + +# If you run "mix test --cover", coverage assets end up here. +/cover + +# The directory Mix downloads your dependencies sources to. +/deps + +# Where 3rd-party dependencies like ExDoc output generated docs. +/doc + +# If the VM crashes, it generates a dump, let's ignore it too. +erl_crash.dump + +# Also ignore archive artifacts (built via "mix archive.build"). +*.ez diff --git a/elixir_sense/README.md b/elixir_sense/README.md new file mode 100644 index 0000000..e8db68a --- /dev/null +++ b/elixir_sense/README.md @@ -0,0 +1,125 @@ +# ElixirSense + +An API/Server for Elixir projects that provides context-aware information for code completion, documentation, go/jump to definition, signature info and more. + +## Usage + +``` +git clone https://github.com/msaraiva/elixir_sense.git +cd elixir_sense +elixir run.exs socket_type port env +``` + +Where: +- `socket_type` - Can be either `unix` (Unix domain socket) or `tcpip`. +- `port` - Specifies which port number to use. Setting port to `0` will make the underlying OS assign an available port number. +- `env` - The environment. Possible values are `dev` or `test` + +Example (using Unix domain socket): + +``` +$ elixir run.exs unix 0 dev +ok:localhost:/tmp/elixir-sense-some_generated_unique_id.sock +``` + +Example (using TCP/IP): + +``` +$ elixir run.exs tcpip 0 dev +ok:localhost:56789:AUTH_TOKEN +``` + +> Note: AUTH_TOKEN is an authentication token generated by the server. All requests sent over tcp/ip must contain this token. + +## Connecting to the server + +The TCP server sends/receives data using a simple binary protocol. All messages are serialized into Erlang's [External Term Format](http://erlang.org/doc/apps/erts/erl_ext_dist.html). Clients that want to communicate with the server must serialize/deserialize data into/from this format. + +### Example using :gen_tcp + +```elixir +{:ok, socket} = :gen_tcp.connect({:local, '/tmp/elixir-sense-some_generated_unique_id.sock'}, 0, [:binary, active: false, packet: 4]) + +ElixirSense.Server.ContextLoader.set_context("dev", "PATH_TO_YOUR_PROJECT") + +code = """ +defmodule MyModule do + alias List, as: MyList + MyList.flatten(par0, +end +""" + +request = %{ + "request_id" => 1, + "auth_token" => nil, + "request" => "signature", + "payload" => %{ + "buffer" => code, + "line" => 3, + "column" => 23 + } +} + +data = :erlang.term_to_binary(request) +:ok = :gen_tcp.send(socket, data) +{:ok, response} = :gen_tcp.recv(socket, 0) +:erlang.binary_to_term(response) + +``` + +The output: + +```elixir +%{request_id: 1, + payload: %{ + active_param: 1, + pipe_before: false, + signatures: [ + %{documentation: "Flattens the given `list` of nested lists.", + name: "flatten", + params: ["list"], + spec: "@spec flatten(deep_list) :: list when deep_list: [any | deep_list]" + }, + %{documentation: "Flattens the given `list` of nested lists.\nThe list `tail` will be added at the end of\nthe flattened list.", + name: "flatten", + params: ["list", "tail"], + spec: "@spec flatten(deep_list, [elem]) :: [elem] when deep_list: [elem | deep_list], elem: var" + } + ]}, + error: nil +} +``` + +### Example using `elixir-sense-client.js` + +```javascript +let client = new ElixirSenseClient('localhost', '/tmp/elixir-sense-some_generated_unique_id.sock', null, "dev", PATH_TO_YOUR_PROJECT) + +code = ` + defmodule MyModule do + alias List, as: MyList + MyList.flatten(par0, + end +`; + +client.send("signature", { buffer: code, line: 4, column: 25 }, (result) => { + console.log(result); +}); +``` + +## Testing + +``` +$ mix deps.get +$ mix test +``` + +For coverage: + +``` +mix coveralls +``` + +## WIP + +Credits, License, ... diff --git a/elixir_sense/config/config.exs b/elixir_sense/config/config.exs new file mode 100644 index 0000000..92a851b --- /dev/null +++ b/elixir_sense/config/config.exs @@ -0,0 +1,30 @@ +# This file is responsible for configuring your application +# and its dependencies with the aid of the Mix.Config module. +use Mix.Config + +# This configuration is loaded before any dependency and is restricted +# to this project. If another project depends on this project, this +# file won't be loaded nor affect the parent project. For this reason, +# if you want to provide default values for your application for +# 3rd-party users, it should be done in your "mix.exs" file. + +# You can configure for your application as: +# +# config :elixir_sense, key: :value +# +# And access this configuration in your application as: +# +# Application.get_env(:elixir_sense, :key) +# +# Or configure a 3rd-party app: +# +# config :logger, level: :info +# + +# It is also possible to import configuration files, relative to this +# directory. For example, you can emulate configuration per environment +# by uncommenting the line below and defining dev.exs, test.exs and such. +# Configuration from the imported file will override the ones defined +# here (which is why it is important to import them last). +# +# import_config "#{Mix.env}.exs" diff --git a/elixir_sense/lib/alchemist/helpers/complete.ex b/elixir_sense/lib/alchemist/helpers/complete.ex new file mode 100644 index 0000000..e7f5279 --- /dev/null +++ b/elixir_sense/lib/alchemist/helpers/complete.ex @@ -0,0 +1,427 @@ +defmodule Alchemist.Helpers.Complete do + + @builtin_functions [{:__info__, 1}, {:module_info, 0}, {:module_info, 1}] + + alias Alchemist.Helpers.ModuleInfo + alias ElixirSense.Core.Introspection + + @moduledoc false + + # This Alchemist.Completer holds a codebase copy of the + # IEx.Autocomplete because for the use of context specific + # aliases. + # + # With the release of Elixir v1.1 the IEx.Autocomplete will + # look for aliases in a certain environment variable + # `Application.get_env(:iex, :autocomplete_server)` and until + # then we'll use our own autocomplete codebase. + + def run(exp) do + code = case is_bitstring(exp) do + true -> exp |> String.to_charlist + _ -> exp + end + + {status, result, list} = expand(code |> Enum.reverse) + + case {status, result, list} do + {:no, _, _} -> '' + {:yes, [], _} -> List.insert_at(list, 0, %{type: :hint, value: "#{exp}"}) + {:yes, _, []} -> run(code ++ result) + {:yes, _, _} -> List.insert_at(run(code ++ result), 1, Enum.at(list, 0)) + # + end + end + + def run(hint, modules) do + context_module = modules |> Enum.at(0) + + exported? = fn mod, f, a -> + !({f, a} in @builtin_functions) and (function_exported?(mod, f, a) or macro_exported?(mod, f, a)) + end + accept_function = fn + (mod, mod, _, _, _) -> true + (_ , _ , _, _, :undefined) -> false + (_ , mod, f, a, _) -> exported?.(mod, f, a) + end + + for module <- modules, module != Elixir do + funs = ModuleInfo.get_functions(module, hint) + funs_info = Introspection.module_functions_info(module) + + for {f, a} <- funs, + {func_kind, fun_args, desc, spec} = Map.get(funs_info, {f, a}, {:undefined, "", "", ""}), + accept_function.(context_module, module, f, a, func_kind) + do + kind = case {context_module, module, func_kind} do + {m, m, :defmacro} -> "public_macro" + {_, _, :defmacro} -> "macro" + {m, m, :def} -> "public_function" + {_, _, :def} -> "function" + {m, m, :undefined} -> if ({f, a} in @builtin_functions) or exported?.(module, f, a), do: "public_function", else: "private_function" + _ -> "unknown" + end + + func_name = Atom.to_string(f) + mod_name = module |> Introspection.module_to_string + %{type: kind, name: func_name, arity: a, args: fun_args, origin: mod_name, summary: desc, spec: spec} + end + end |> List.flatten + end + + def expand('') do + expand_import("") + end + + def expand([h|t] = expr) do + cond do + h === ?. and t != [] -> + expand_dot(reduce(t)) + h === ?: -> + expand_erlang_modules() + identifier?(h) -> + expand_expr(reduce(expr)) + (h == ?/) and t != [] and identifier?(hd(t)) -> + expand_expr(reduce(t)) + h in '([{' -> + expand('') + true -> + no() + end + end + + defp identifier?(h) do + (h in ?a..?z) or (h in ?A..?Z) or (h in ?0..?9) or h in [?_, ??, ?!] + end + + defp expand_dot(expr) do + case Code.string_to_quoted expr do + {:ok, atom} when is_atom(atom) -> + expand_call(atom, "") + {:ok, {:__aliases__, _, list}} -> + expand_elixir_modules(list, "") + _ -> + no() + end + end + + defp expand_expr(expr) do + case Code.string_to_quoted expr do + {:ok, atom} when is_atom(atom) -> + expand_erlang_modules(Atom.to_string(atom)) + {:ok, {atom, _, nil}} when is_atom(atom) -> + expand_import(Atom.to_string(atom)) + {:ok, {:__aliases__, _, [root]}} -> + expand_elixir_modules([], Atom.to_string(root)) + {:ok, {:__aliases__, _, [h|_] = list}} when is_atom(h) -> + hint = Atom.to_string(List.last(list)) + list = Enum.take(list, length(list) - 1) + expand_elixir_modules(list, hint) + {:ok, {{:., _, [mod, fun]}, _, []}} when is_atom(fun) -> + expand_call(mod, Atom.to_string(fun)) + _ -> + no() + end + end + + defp reduce(expr) do + Enum.reverse Enum.reduce ' ([{', expr, fn token, acc -> + hd(:string.tokens(acc, [token])) + end + end + + defp yes(hint, entries) do + {:yes, String.to_charlist(hint), entries} + end + + defp no do + {:no, '', []} + end + + ## Formatting + + defp format_expansion([], _) do + no() + end + + defp format_expansion([uniq], hint) do + case to_hint(uniq, hint) do + "" -> yes("", to_uniq_entries(uniq)) + hint -> yes(hint, to_uniq_entries(uniq)) + end + end + + defp format_expansion([first|_] = entries, hint) do + binary = Enum.map(entries, &(&1.name)) + length = byte_size(hint) + prefix = :binary.longest_common_prefix(binary) + if prefix in [0, length] do + yes("", Enum.flat_map(entries, &to_entries/1)) + else + yes(:binary.part(first.name, prefix, length - prefix), []) + end + end + + ## Expand calls + + # :atom.fun + defp expand_call(mod, hint) when is_atom(mod) do + expand_require(mod, hint) + end + + # Elixir.fun + defp expand_call({:__aliases__, _, list}, hint) do + list + |> expand_alias() + |> normalize_module() + |> expand_require(hint) + end + + defp expand_call(_, _) do + no() + end + + defp expand_require(mod, hint) do + format_expansion match_module_funs(mod, hint), hint + end + + defp expand_import(hint) do + funs = + match_module_funs(Kernel, hint) ++ + match_module_funs(Kernel.SpecialForms, hint) + format_expansion funs, hint + end + + ## Erlang modules + + defp expand_erlang_modules(hint \\ "") do + format_expansion match_erlang_modules(hint), hint + end + + defp match_erlang_modules(hint) do + for mod <- match_modules(hint, true) do + %{kind: :module, name: mod, type: :erlang, desc: ""} + end + end + + ## Elixir modules + + defp expand_elixir_modules([], hint) do + expand_elixir_modules(Elixir, hint, match_aliases(hint)) + end + + defp expand_elixir_modules(list, hint) do + list + |> expand_alias() + |> normalize_module() + |> expand_elixir_modules(hint, []) + end + + defp expand_elixir_modules(mod, hint, aliases) do + aliases + |> Kernel.++(match_elixir_modules(mod, hint)) + |> Kernel.++(match_module_funs(mod, hint)) + |> format_expansion(hint) + end + + defp expand_alias([name | rest] = list) do + module = Module.concat(Elixir, name) + Enum.find_value env_aliases(), list, fn {alias, mod} -> + if alias === module do + case Atom.to_string(mod) do + "Elixir." <> mod -> + Module.concat [mod|rest] + _ -> + mod + end + end + end + end + + defp env_aliases do + :"alchemist.el" + |> Application.get_env(:aliases) + |> format_aliases + end + + defp format_aliases(nil), do: [] + defp format_aliases(list), do: list + + defp match_aliases(hint) do + for {alias, _mod} <- env_aliases(), + [name] = Module.split(alias), + starts_with?(name, hint) do + %{kind: :module, type: :alias, name: name, desc: ""} + end + end + + defp match_elixir_modules(module, hint) do + name = Atom.to_string(module) + depth = length(String.split(name, ".")) + 1 + base = name <> "." <> hint + + for mod <- match_modules(base, module === Elixir), + parts = String.split(mod, "."), + depth <= length(parts) do + mod_as_atom = mod |> String.to_atom + desc = Introspection.get_module_docs_summary(mod_as_atom) + subtype = Introspection.get_module_subtype(mod_as_atom) + %{kind: :module, type: :elixir, name: Enum.at(parts, depth - 1), + desc: desc, subtype: subtype} + end + |> Enum.uniq_by(fn %{name: name} -> name end) + end + + ## Helpers + + defp normalize_module(mod) do + if is_list(mod) do + Module.concat(mod) + else + mod + end + end + + defp match_modules(hint, root) do + root + |> get_modules() + |> :lists.usort() + |> Enum.drop_while(& not starts_with?(&1, hint)) + |> Enum.take_while(& starts_with?(&1, hint)) + end + + defp get_modules(true) do + ["Elixir.Elixir"] ++ get_modules(false) + end + + defp get_modules(false) do + modules = Enum.map(:code.all_loaded(), &Atom.to_string(elem(&1, 0))) + case :code.get_mode() do + :interactive -> modules ++ get_modules_from_applications() + _otherwise -> modules + end + end + + defp get_modules_from_applications do + for [app] <- loaded_applications(), + {:ok, modules} = :application.get_key(app, :modules), + module <- modules do + Atom.to_string(module) + end + end + + defp loaded_applications do + # If we invoke :application.loaded_applications/0, + # it can error if we don't call safe_fixtable before. + # Since in both cases we are reaching over the + # application controller internals, we choose to match + # for performance. + :ets.match(:ac_tab, {{:loaded, :"$1"}, :_}) + end + + defp match_module_funs(mod, hint) do + case ensure_loaded(mod) do + {:module, _} -> + falist = get_module_funs(mod) + + list = Enum.reduce falist, [], fn {f, a, func_kind, doc, spec}, acc -> + case :lists.keyfind(f, 1, acc) do + {f, aa, func_kind, docs, specs} -> + :lists.keyreplace(f, 1, acc, {f, [a|aa], func_kind, [doc|docs], [spec|specs]}) + false -> [{f, [a], func_kind, [doc], [spec]}|acc] + end + end + + for {fun, arities, func_kind, docs, specs} <- list, + name = Atom.to_string(fun), + starts_with?(name, hint) do + %{kind: :function, name: name, arities: arities, module: mod, + func_kind: func_kind, docs: docs, specs: specs} + end |> :lists.sort() + + _otherwise -> [] + end + end + + defp get_module_funs(mod) do + if function_exported?(mod, :__info__, 1) do + funs = if docs = Code.get_docs(mod, :docs) do + specs = Introspection.get_module_specs(mod) + for {{f, a}, _line, func_kind, _sign, doc} = func_doc <- docs, doc != false do + spec = Map.get(specs, {f, a}, "") + {f, a, func_kind, func_doc, spec} + end + else + macros = :macros + |> mod.__info__() + |> Enum.map(fn {f, a} -> {f, a, :macro, nil, nil} end) + functions = :functions + |> mod.__info__() + |> Enum.map(fn {f, a} -> {f, a, :function, nil, nil} end) + macros ++ functions + end + funs ++ (@builtin_functions |> Enum.map(fn {f, a} -> {f, a, :function, nil, nil} end)) + else + for {f, a} <- mod.module_info(:exports) do + case f |> Atom.to_string do + "MACRO-" <> name -> {String.to_atom(name), a, :macro, nil, nil} + _name -> {f, a, :function, nil, nil} + end + end + end + end + + defp ensure_loaded(Elixir), do: {:error, :nofile} + defp ensure_loaded(mod), do: Code.ensure_compiled(mod) + + defp starts_with?(_string, ""), do: true + defp starts_with?(string, hint), do: String.starts_with?(string, hint) + + ## Ad-hoc conversions + + defp to_entries(%{kind: :module, name: name, desc: desc, subtype: subtype}) when subtype != nil do + [%{type: :module, name: name, subtype: subtype, summary: desc}] + end + + defp to_entries(%{kind: :module, name: name, desc: desc}) do + [%{type: :module, name: name, subtype: nil, summary: desc}] + end + + defp to_entries(%{kind: :function, name: name, arities: arities, module: mod, func_kind: func_kind, docs: docs, specs: specs}) do + docs_specs = docs |> Enum.zip(specs) + arities_docs_specs = arities |> Enum.zip(docs_specs) + + for {a, {doc, spec}} <- arities_docs_specs do + {fun_args, desc} = Introspection.extract_fun_args_and_desc(doc) + kind = case func_kind do + :defmacro -> "macro" + _ -> "function" + end + mod_name = mod + |> Introspection.module_to_string + %{type: kind, name: name, arity: a, args: fun_args, origin: mod_name, summary: desc, spec: spec} + end + end + + defp to_uniq_entries(%{kind: :module} = mod) do + to_entries(mod) + end + + defp to_uniq_entries(%{kind: :function} = fun) do + to_entries(fun) + end + + defp to_hint(%{kind: :module, name: name}, hint) do + format_hint(name, hint) <> "." + end + + defp to_hint(%{kind: :function, name: name}, hint) do + format_hint(name, hint) + end + + defp format_hint(name, hint) do + hint_size = byte_size(hint) + :binary.part(name, hint_size, byte_size(name) - hint_size) + end + +end diff --git a/elixir_sense/lib/alchemist/helpers/module_info.ex b/elixir_sense/lib/alchemist/helpers/module_info.ex new file mode 100644 index 0000000..7de310f --- /dev/null +++ b/elixir_sense/lib/alchemist/helpers/module_info.ex @@ -0,0 +1,131 @@ +defmodule Alchemist.Helpers.ModuleInfo do + + @moduledoc false + + def moduledoc?(module) do + case Code.get_docs module, :moduledoc do + {_, doc} -> is_binary doc + _ -> false + end + end + + def docs?(module, function) do + docs = Code.get_docs module, :docs + do_docs?(docs, function) + end + + def expand_alias([name | rest] = list, aliases) do + module = Module.concat(Elixir, name) + aliases + |> Enum.find_value(list, fn {alias, mod} -> + if alias === module do + case Atom.to_string(mod) do + "Elixir." <> mod -> + Module.concat [mod|rest] + _ -> + mod + end + end + end) + |> normalize_module + end + + def get_functions(module, hint) do + hint = to_string hint + {module, _} = Code.eval_string(module) + functions = get_module_funs(module) + + list = Enum.reduce functions, [], fn({f, a}, acc) -> + case :lists.keyfind(f, 1, acc) do + {f, aa} -> :lists.keyreplace(f, 1, acc, {f, [a|aa]}) + false -> [{f, [a]}|acc] + end + end + list + |> do_get_functions(hint) + |> :lists.sort() + end + + def has_function?(module, function) do + List.keymember? get_module_funs(module), function, 0 + end + + defp do_get_functions(list, "") do + all_functions(list) + end + + defp do_get_functions(list, hint) do + all_functions(list, hint) + end + + defp get_module_funs(module) do + case Code.ensure_loaded(module) do + {:module, _} -> + (:functions + |> module.module_info() + |> filter_module_funs) + ++ module.__info__(:macros) + _otherwise -> + [] + end + end + + defp filter_module_funs(list) do + for fun = {f, _a} <- list, !(f |> Atom.to_string |> String.starts_with?(["MACRO-", "-"])) do + fun + end + end + + defp all_functions(list) do + for {fun, arities} <- list do + for arity <- arities do + {fun, arity} + end + end |> List.flatten + end + + defp all_functions(list, hint) do + for {fun, arities} <- list, name = Atom.to_string(fun), String.starts_with?(name, hint) do + for arity <- arities do + {fun, arity} + end + end |> List.flatten + end + + def all_applications_modules do + for [app] <- loaded_applications(), + {:ok, modules} = :application.get_key(app, :modules), + module <- modules do + module + end + end + + defp do_docs?([head|tail], function) do + {{func, _}, _, _, _, doc} = head + if func == function and is_binary(doc) do + true + else + do_docs?(tail, function) + end + end + defp do_docs?([], _function), do: false + defp do_docs?(nil, _function), do: false + + defp loaded_applications do + # If we invoke :application.loaded_applications/0, + # it can error if we don't call safe_fixtable before. + # Since in both cases we are reaching over the + # application controller internals, we choose to match + # for performance. + :ets.match(:ac_tab, {{:loaded, :"$1"}, :_}) + end + + defp normalize_module(mod) do + if is_list(mod) do + Module.concat(mod) + else + mod + end + end + +end diff --git a/elixir_sense/lib/elixir_sense.ex b/elixir_sense/lib/elixir_sense.ex new file mode 100644 index 0000000..7a732db --- /dev/null +++ b/elixir_sense/lib/elixir_sense.ex @@ -0,0 +1,300 @@ +defmodule ElixirSense do + + @moduledoc """ + ElxirSense is a Elixir library that implements useful features for any editor/tool that needs + to introspect context-aware information about Elixir source code. + + This module provides the basic functionality for context-aware code completion, docs, signature info and more. + """ + + alias ElixirSense.Core.State + alias ElixirSense.Core.Metadata + alias ElixirSense.Core.Parser + alias ElixirSense.Core.Introspection + alias ElixirSense.Core.Source + alias ElixirSense.Providers.Docs + alias ElixirSense.Providers.Definition + alias ElixirSense.Providers.Suggestion + alias ElixirSense.Providers.Signature + alias ElixirSense.Providers.Expand + alias ElixirSense.Providers.Eval + + @doc ~S""" + Returns all documentation related a module or function, including types and callback information. + + ## Example + + iex> code = ~S''' + ...> defmodule MyModule do + ...> alias Enum, as: MyEnum + ...> MyEnum.to_list(1..3) + ...> end + ...> ''' + iex> %{docs: %{types: types, docs: docs}} = ElixirSense.docs(code, 3, 11) + iex> docs |> String.split("\n") |> Enum.at(6) + "Converts `enumerable` to a list." + iex> types |> String.split("\n") |> Enum.at(0) + " `@type t :: Enumerable.t" + """ + @spec docs(String.t, pos_integer, pos_integer) :: %{subject: String.t, actual_subject: String.t, docs: Introspection.docs} + def docs(code, line, column) do + subject = Source.subject(code, line, column) + metadata = Parser.parse_string(code, true, true, line) + %State.Env{ + imports: imports, + aliases: aliases, + module: module + } = Metadata.get_env(metadata, line) + + {actual_subject, docs} = Docs.all(subject, imports, aliases, module) + %{subject: subject, actual_subject: actual_subject, docs: docs} + end + + @doc ~S""" + Returns the location (file and line) where a module, function or macro was defined. + + ## Example + + iex> code = ~S''' + ...> defmodule MyModule do + ...> alias Enum, as: MyEnum + ...> MyEnum.to_list(1..3) + ...> end + ...> ''' + iex> {path, line} = ElixirSense.definition(code, 3, 11) + iex> "#{Path.basename(path)}:#{to_string(line)}" + "enum.ex:2523" + """ + @spec definition(String.t, pos_integer, pos_integer) :: Definition.location + def definition(code, line, column) do + subject = Source.subject(code, line, column) + buffer_file_metadata = Parser.parse_string(code, true, true, line) + %State.Env{ + imports: imports, + aliases: aliases, + module: module + } = Metadata.get_env(buffer_file_metadata, line) + + Definition.find(subject, imports, aliases, module) + end + + @doc ~S""" + Returns a sorted list of all available modules + + ## Example + + iex> ElixirSense.all_modules() |> Enum.take(4) + [":application", ":application_controller", ":application_master", ":application_starter"] + + iex> ElixirSense.all_modules() |> Enum.take(-4) + ["Version.Parser", "Version.Parser.DSL", "Version.Requirement", "WithClauseError"] + + """ + def all_modules do + Introspection.all_modules() + |> Enum.map(&Atom.to_string(&1)) + |> Enum.map(fn x -> if String.downcase(x) == x do ":" <> x else x end end) + |> Enum.map(&String.replace_prefix(&1, "Elixir.", "")) + |> Enum.sort() + end + + @doc """ + Finds suggestions by a given hint. + + Returned suggestions: + + * Modules, functions, variables, function params and module attributes available in the current scope. + * Callbacks defined in behaviours (works also when @behaviour is injected by use directives) + * Lists the accepted "returns" specs when inside a callback implementation + + Additional information: + + * Type of the module (Module, Struct, Protocol, Implementation or Exception) + * Documentation summary for each module or function + * Function and callback specs + * Origin: where the function was originally defined (for aliased, imported modules or callbacks) + * Smart snippets for functions + + ## Example + + iex> code = ~S''' + ...> defmodule MyModule do + ...> alias List, as: MyList + ...> MyList.fi + ...> end + ...> ''' + iex> ElixirSense.suggestions(code, 3, 12) + [%{type: :hint, value: "MyList.first"}, + %{type: "function", name: "first", arity: 1, origin: "List", + spec: "@spec first([elem]) :: nil | elem when elem: var", + summary: "Returns the first element in `list` or `nil` if `list` is empty.", + args: "list"}] + """ + @spec suggestions(String.t, non_neg_integer, non_neg_integer) :: [Suggestion.suggestion] + def suggestions(buffer, line, column) do + hint = Source.prefix(buffer, line, column) + buffer_file_metadata = Parser.parse_string(buffer, true, true, line) + %State.Env{ + imports: imports, + aliases: aliases, + vars: vars, + attributes: attributes, + behaviours: behaviours, + module: module, + scope: scope + } = Metadata.get_env(buffer_file_metadata, line) + + Suggestion.find(hint, [module|imports], aliases, vars, attributes, behaviours, scope) + end + + @doc """ + Returns the signature info from the function when inside a function call. + + ## Example + + iex> code = ~S''' + ...> defmodule MyModule do + ...> alias List, as: MyList + ...> MyList.flatten(par0, + ...> end + ...> ''' + iex> ElixirSense.signature(code, 3, 23) + %{active_param: 1, + pipe_before: false, + signatures: [ + %{name: "flatten", + params: ["list"], + documentation: "Flattens the given `list` of nested lists.", + spec: "@spec flatten(deep_list) :: list when deep_list: [any | deep_list]"}, + %{name: "flatten", + params: ["list", "tail"], + documentation: "Flattens the given `list` of nested lists.\\nThe list `tail` will be added at the end of\\nthe flattened list.", + spec: "@spec flatten(deep_list, [elem]) :: [elem] when deep_list: [elem | deep_list], elem: var"} + ] + } + """ + @spec signature(String.t, pos_integer, pos_integer) :: Signature.signature_info + def signature(code, line, column) do + prefix = Source.text_before(code, line, column) + buffer_file_metadata = Parser.parse_string(code, true, true, line) + %State.Env{ + imports: imports, + aliases: aliases, + module: module, + } = Metadata.get_env(buffer_file_metadata, line) + + Signature.find(prefix, imports, aliases, module, buffer_file_metadata) + end + + @doc """ + Returns a map containing the results of all different code expansion methods + available. + + Available axpansion methods: + + * `expand_once` - Calls `Macro.expand_once/2` + * `expand` - Calls `Macro.expand/2` + * `expand_all` - Recursively calls `Macro.expand/2` + * `expand_partial` - The same as `expand_all`, but does not expand `:def, :defp, :defmodule, :@, :defmacro, + :defmacrop, :defoverridable, :__ENV__, :__CALLER__, :raise, :if, :unless, :in` + + > **Notice**: In order to expand the selected code properly, ElixirSense parses/expands the source file and tries to introspect context information + like requires, aliases, import, etc. However the environment during the real compilation process may still be diffent from the one we + try to simulate, therefore, in some cases, the expansion might not work as expected or, in some cases, not even be possible. + + ## Example + + Given the following code: + + ``` + unless ok do + IO.puts to_string(:error) + else + IO.puts to_string(:ok) + end + + ``` + + A full expansion will generate the following results based on each method: + + ### expand_once + + ``` + if(ok) do + IO.puts(to_string(:ok)) + else + IO.puts(to_string(:error)) + end + ``` + + ### expand + + ``` + case(ok) do + x when x in [false, nil] -> + IO.puts(to_string(:error)) + _ -> + IO.puts(to_string(:ok)) + end + ``` + + ### expand_partial + + ``` + unless(ok) do + IO.puts(String.Chars.to_string(:error)) + else + IO.puts(String.Chars.to_string(:ok)) + end + ``` + + ### expand_all + + ``` + case(ok) do + x when :erlang.or(:erlang.=:=(x, nil), :erlang.=:=(x, false)) -> + IO.puts(String.Chars.to_string(:error)) + _ -> + IO.puts(String.Chars.to_string(:ok)) + end + ``` + + """ + @spec expand_full(String.t, String.t, pos_integer) :: Expand.expanded_code_map + def expand_full(buffer, code, line) do + buffer_file_metadata = Parser.parse_string(buffer, true, true, line) + %State.Env{ + requires: requires, + imports: imports, + module: module + } = Metadata.get_env(buffer_file_metadata, line) + + Expand.expand_full(code, requires, imports, module) + end + + @doc """ + Converts a string to its quoted form. + """ + @spec quote(String.t) :: String.t + def quote(code) do + Eval.quote(code) + end + + @doc ~S""" + Evaluate a pattern matching expression and format its results, including + the list of bindings, if any. + + ## Example + + iex> code = ''' + ...> {_, %{status: status, msg: message}, [arg1|_]} = {:error, %{status: 404, msg: "Not found"}, [1,2,3]} + ...> ''' + iex> ElixirSense.match(code) + "# Bindings\n\nstatus = 404\n\nmessage = \"Not found\"\n\narg1 = 1" + """ + @spec match(String.t) :: Eval.bindings + def match(code) do + Eval.match_and_format(code) + end + +end diff --git a/elixir_sense/lib/elixir_sense/core/ast.ex b/elixir_sense/lib/elixir_sense/core/ast.ex new file mode 100644 index 0000000..0c97b32 --- /dev/null +++ b/elixir_sense/lib/elixir_sense/core/ast.ex @@ -0,0 +1,156 @@ +defmodule ElixirSense.Core.Ast do + @moduledoc """ + Abstract Syntax Tree support + """ + + alias ElixirSense.Core.Introspection + + @empty_env_info %{requires: [], imports: [], behaviours: []} + + @partials [:def, :defp, :defmodule, :@, :defmacro, :defmacrop, :defoverridable, + :__ENV__, :__CALLER__, :raise, :if, :unless, :in] + + @max_expand_count 30_000 + + def extract_use_info(use_ast, module) do + env = Map.put(__ENV__, :module, module) + {expanded_ast, _requires} = Macro.prewalk(use_ast, {env, 1}, &do_expand/2) + {_ast, env_info} = Macro.prewalk(expanded_ast, @empty_env_info, &pre_walk_expanded/2) + env_info + catch + {:expand_error, _} -> + IO.puts(:stderr, "Info: ignoring recursive macro") + @empty_env_info + end + + def expand_partial(ast, env) do + {expanded_ast, _} = Macro.prewalk(ast, {env, 1}, &do_expand_partial/2) + expanded_ast + + rescue + _e -> ast + catch + e -> e + end + + def expand_all(ast, env) do + {expanded_ast, _} = Macro.prewalk(ast, {env, 1}, &do_expand_all/2) + expanded_ast + rescue + _e -> ast + catch + e -> e + end + + def set_module_for_env(env, module) do + Map.put(env, :module, module) + end + + def add_requires_to_env(env, modules) do + add_directive_modules_to_env(env, :require, modules) + end + + def add_imports_to_env(env, modules) do + add_directive_modules_to_env(env, :import, modules) + end + + defp add_directive_modules_to_env(env, directive, modules) do + directive_string = modules + |> Enum.map(&"#{directive} #{Introspection.module_to_string(&1)}") + |> Enum.join("; ") + {new_env, _} = Code.eval_string("#{directive_string}; __ENV__", [], env) + new_env + end + + defp do_expand_all(ast, acc) do + do_expand(ast, acc) + end + + defp do_expand_partial({name, _, _} = ast, acc) when name in @partials do + {ast, acc} + end + defp do_expand_partial(ast, acc) do + do_expand(ast, acc) + end + + defp do_expand({:require, _, _} = ast, {env, count}) do + modules = extract_directive_modules(:require, ast) + new_env = add_requires_to_env(env, modules) + {ast, {new_env, count}} + end + + defp do_expand(ast, acc) do + do_expand_with_fixes(ast, acc) + end + + # Fix inexpansible `use ExUnit.Case` + defp do_expand_with_fixes({:use, _, [{:__aliases__, _, [:ExUnit, :Case]}|_]}, acc) do + ast = quote do + import ExUnit.Callbacks + import ExUnit.Assertions + import ExUnit.Case + import ExUnit.DocTest + end + {ast, acc} + end + + defp do_expand_with_fixes(ast, {env, count}) do + if count > @max_expand_count do + throw {:expand_error, "Cannot expand recursive macro"} + end + expanded_ast = Macro.expand(ast, env) + {expanded_ast, {env, count + 1}} + end + + defp pre_walk_expanded({:__block__, _, _} = ast, acc) do + {ast, acc} + end + defp pre_walk_expanded({:require, _, _} = ast, acc) do + modules = extract_directive_modules(:require, ast) + {ast, %{acc | requires: (acc.requires ++ modules)}} + end + defp pre_walk_expanded({:import, _, _} = ast, acc) do + modules = extract_directive_modules(:import, ast) + {ast, %{acc | imports: (acc.imports ++ modules)}} + end + defp pre_walk_expanded({:@, _, [{:behaviour, _, [behaviour]}]} = ast, acc) do + {ast, %{acc | behaviours: [behaviour|acc.behaviours]}} + end + defp pre_walk_expanded({{:., _, [Module, :put_attribute]}, _, [_module, :behaviour, behaviour | _]} = ast, acc) do + {ast, %{acc | behaviours: [behaviour|acc.behaviours]}} + end + defp pre_walk_expanded({_name, _meta, _args}, acc) do + {nil, acc} + end + defp pre_walk_expanded(ast, acc) do + {ast, acc} + end + + defp extract_directive_modules(directive, ast) do + case ast do + # v1.2 notation + {^directive, _, [{{:., _, [{:__aliases__, _, prefix_atoms}, :{}]}, _, aliases}]} -> + aliases |> Enum.map(fn {:__aliases__, _, mods} -> + Module.concat(prefix_atoms ++ mods) + end) + # with options + {^directive, _, [{_, _, module_atoms = [mod|_]}, _opts]} when is_atom(mod) -> + [module_atoms |> Module.concat] + # with options + {^directive, _, [module, _opts]} when is_atom(module) -> + [module] + # with options + {^directive, _, [{:__aliases__, _, module_parts}, _opts]} -> + [module_parts |> Module.concat] + # without options + {^directive, _, [{:__aliases__, _, module_parts}]} -> + [module_parts |> Module.concat] + # without options + {^directive, _, [{:__aliases__, [alias: false, counter: _], module_parts}]} -> + [module_parts |> Module.concat] + # without options + {^directive, _, [module]} -> + [module] + end + end +end diff --git a/elixir_sense/lib/elixir_sense/core/introspection.ex b/elixir_sense/lib/elixir_sense/core/introspection.ex new file mode 100644 index 0000000..ae4af0e --- /dev/null +++ b/elixir_sense/lib/elixir_sense/core/introspection.ex @@ -0,0 +1,599 @@ +defmodule ElixirSense.Core.Introspection do + @moduledoc """ + A collection of functions to introspect/format docs, specs, types and callbacks. + + Based on: + https://github.com/elixir-lang/elixir/blob/c983b3db6936ce869f2668b9465a50007ffb9896/lib/iex/lib/iex/introspection.ex + https://github.com/elixir-lang/ex_doc/blob/82463a56053b29a406fd271e9e2e2f05e87d6248/lib/ex_doc/retriever.ex + """ + + alias Kernel.Typespec + alias Alchemist.Helpers.ModuleInfo + + @type mod_fun :: {mod :: module | nil, fun :: atom | nil} + @type markdown :: String.t + @type mod_docs :: %{docs: markdown, types: markdown, callbacks: markdown} + @type fun_docs :: %{docs: markdown, types: markdown} + @type docs :: mod_docs | fun_docs + + @wrapped_behaviours %{ + :gen_server => GenServer, + :gen_event => GenEvent + } + + def all_modules do + ModuleInfo.all_applications_modules() + end + + @spec get_all_docs(mod_fun) :: docs + def get_all_docs({mod, nil}) do + %{docs: get_docs_md(mod), types: get_types_md(mod), callbacks: get_callbacks_md(mod)} + end + + def get_all_docs({mod, fun}) do + %{docs: get_func_docs_md(mod, fun), types: get_types_md(mod)} + end + + def get_signatures(mod, fun, code_docs \\ nil) do + docs = code_docs || Code.get_docs(mod, :docs) || [] + for {{f, arity}, _, _, args, text} <- docs, f == fun do + fun_args = Enum.map(args, &format_doc_arg(&1)) + fun_str = Atom.to_string(fun) + doc = extract_summary_from_docs(text) + spec = get_spec(mod, fun, arity) + %{name: fun_str, params: fun_args, documentation: doc, spec: spec} + end + end + + def get_func_docs_md(mod, fun) do + docs = + case Code.get_docs(mod, :docs) do + nil -> nil + docs -> + for {{f, arity}, _, _, args, text} <- docs, f == fun do + fun_args_text = args + |> Enum.map_join(", ", &format_doc_arg(&1)) + |> String.replace("\\\\", "\\\\\\\\") + mod_str = module_to_string(mod) + fun_str = Atom.to_string(fun) + "> #{mod_str}.#{fun_str}(#{fun_args_text})\n\n#{get_spec_text(mod, fun, arity)}#{text}" + end + end + + case docs do + [_|_] -> Enum.join(docs, "\n\n____\n\n") + _ -> "No documentation available" + end + end + + def get_docs_md(mod) when is_atom(mod) do + mod_str = module_to_string(mod) + case Code.get_docs(mod, :moduledoc) do + {_line, doc} when is_binary(doc) -> + "> #{mod_str}\n\n" <> doc + _ -> + "No documentation available" + end + end + + def get_types_md(mod) when is_atom(mod) do + for %{type: type, doc: doc} <- get_types_with_docs(mod) do + """ + `#{type}` + + #{doc} + """ + end |> Enum.join("\n\n____\n\n") + end + + def get_callbacks_md(mod) when is_atom(mod) do + for %{callback: callback, signature: signature, doc: doc} <- get_callbacks_with_docs(mod) do + """ + > #{signature} + + ### Specs + + `#{callback}` + + #{doc} + """ + end + |> Enum.join("\n\n____\n\n") + end + + def get_callbacks_with_docs(mod) when is_atom(mod) do + mod = + @wrapped_behaviours + |> Map.get(mod, mod) + + case get_callbacks_and_docs(mod) do + {callbacks, []} -> + Enum.map(callbacks, fn {{name, arity}, [spec | _]} -> + spec_ast = Typespec.spec_to_ast(name, spec) + signature = get_typespec_signature(spec_ast, arity) + definition = format_spec_ast(spec_ast) + %{name: name, arity: arity, callback: "@callback #{definition}", signature: signature, doc: nil} + end) + {callbacks, docs} -> + Enum.map docs, fn + {{fun, arity}, _, :macrocallback, doc} -> + fun + |> get_callback_with_doc(:macrocallback, doc, {:"MACRO-#{fun}", arity + 1}, callbacks) + |> Map.put(:arity, arity) + {{fun, arity}, _, kind, doc} -> + get_callback_with_doc(fun, kind, doc, {fun, arity}, callbacks) + end + end + end + + def get_types_with_docs(module) when is_atom(module) do + module + |> get_types() + |> Enum.map(fn {_, {t, _, _args}} = type -> + %{type: format_type(type), doc: get_type_doc(module, t)} + end) + end + + defp get_types(module) when is_atom(module) do + case Typespec.beam_types(module) do + nil -> [] + types -> types + end + end + + def extract_summary_from_docs(doc) when doc in [nil, "", false], do: "" + def extract_summary_from_docs(doc) do + doc + |> String.split("\n\n") + |> Enum.at(0) + end + + defp format_type({:opaque, type}) do + {:::, _, [ast, _]} = Typespec.type_to_ast(type) + "@opaque #{format_spec_ast(ast)}" + end + + defp format_type({kind, type}) do + ast = Typespec.type_to_ast(type) + "@#{kind} #{format_spec_ast(ast)}" + end + + def format_spec_ast_single_line(spec_ast) do + spec_ast + |> Macro.prewalk(&drop_macro_env/1) + |> spec_ast_to_string() + end + + def format_spec_ast(spec_ast) do + parts = + spec_ast + |> Macro.prewalk(&drop_macro_env/1) + |> extract_spec_ast_parts + + name_str = Macro.to_string(parts.name) + + when_str = + case parts[:when_part] do + nil -> "" + ast -> + {:when, [], [:fake_lhs, ast]} + |> Macro.to_string + |> String.replace_prefix(":fake_lhs", "") + end + + returns_str = + parts.returns + |> Enum.map(&Macro.to_string(&1)) + |> Enum.join(" |\n ") + + formated_spec = + case length(parts.returns) do + 1 -> "#{name_str} :: #{returns_str}#{when_str}\n" + _ -> "#{name_str} ::\n #{returns_str}#{when_str}\n" + end + + formated_spec |> String.replace("()", "") + end + + def define_callback?(mod, fun, arity) do + mod + |> Kernel.Typespec.beam_callbacks() + |> Enum.any?(fn {{f, a}, _} -> {f, a} == {fun, arity} end) + end + + def get_returns_from_callback(module, func, arity) do + parts = + @wrapped_behaviours + |> Map.get(module, module) + |> get_callback_ast(func, arity) + |> Macro.prewalk(&drop_macro_env/1) + |> extract_spec_ast_parts + + for return <- parts.returns do + ast = return |> strip_return_types() + return = + case parts[:when_part] do + nil -> return + _ -> {:when, [], [return, parts.when_part]} + end + + spec = return |> spec_ast_to_string() + stripped = ast |> spec_ast_to_string() + snippet = ast |> return_to_snippet() + %{description: stripped, spec: spec, snippet: snippet} + end + end + + defp extract_spec_ast_parts({:when, _, [{:::, _, [name_part, return_part]}, when_part]}) do + %{name: name_part, returns: extract_return_part(return_part, []), when_part: when_part} + end + + defp extract_spec_ast_parts({:::, _, [name_part, return_part]}) do + %{name: name_part, returns: extract_return_part(return_part, [])} + end + + defp extract_return_part({:|, _, [lhs, rhs]}, returns) do + [lhs|extract_return_part(rhs, returns)] + end + + defp extract_return_part(ast, returns) do + [ast|returns] + end + + defp get_type_doc(module, type) do + case Code.get_docs(module, :type_docs) do + nil -> "" + docs -> + {{_, _}, _, _, description} = Enum.find(docs, fn({{name, _}, _, _, _}) -> + type == name + end) + description || "" + end + end + + defp get_callback_with_doc(name, kind, doc, key, callbacks) do + {_, [spec | _]} = List.keyfind(callbacks, key, 0) + {_f, arity} = key + + spec_ast = name + |> Typespec.spec_to_ast(spec) + |> Macro.prewalk(&drop_macro_env/1) + signature = get_typespec_signature(spec_ast, arity) + definition = format_spec_ast(spec_ast) + + %{name: name, arity: arity, callback: "@#{kind} #{definition}", signature: signature, doc: doc} + end + + defp get_callbacks_and_docs(mod) do + callbacks = Typespec.beam_callbacks(mod) + docs = + @wrapped_behaviours + |> Map.get(mod, mod) + |> Code.get_docs(:callback_docs) + + {callbacks || [], docs || []} + end + + defp drop_macro_env({name, meta, [{:::, _, [{:env, _, _}, _ | _]} | args]}), do: {name, meta, args} + defp drop_macro_env(other), do: other + + defp get_typespec_signature({:when, _, [{:::, _, [{name, meta, args}, _]}, _]}, arity) do + Macro.to_string {name, meta, strip_types(args, arity)} + end + + defp get_typespec_signature({:::, _, [{name, meta, args}, _]}, arity) do + Macro.to_string {name, meta, strip_types(args, arity)} + end + + defp get_typespec_signature({name, meta, args}, arity) do + Macro.to_string {name, meta, strip_types(args, arity)} + end + + defp strip_types(args, arity) do + args + |> Enum.take(-arity) + |> Enum.with_index() + |> Enum.map(fn + {{:::, _, [left, _]}, i} -> to_var(left, i) + {{:|, _, _}, i} -> to_var({}, i) + {left, i} -> to_var(left, i) + end) + end + + defp strip_return_types(returns) when is_list(returns) do + returns |> Enum.map(&strip_return_types/1) + end + defp strip_return_types({:::, _, [left, _]}) do + left + end + defp strip_return_types({:|, meta, args}) do + {:|, meta, strip_return_types(args)} + end + defp strip_return_types({:{}, meta, args}) do + {:{}, meta, strip_return_types(args)} + end + defp strip_return_types(value) do + value + end + + defp return_to_snippet(ast) do + {ast, _} = Macro.prewalk(ast, 1, &term_to_snippet/2) + ast |> Macro.to_string + end + defp term_to_snippet({name, _, nil} = ast, index) when is_atom(name) do + next_snippet(ast, index) + end + defp term_to_snippet({:__aliases__, _, _} = ast, index) do + next_snippet(ast, index) + end + defp term_to_snippet({{:., _, _}, _, _} = ast, index) do + next_snippet(ast, index) + end + defp term_to_snippet({:|, _, _} = ast, index) do + next_snippet(ast, index) + end + defp term_to_snippet(ast, index) do + {ast, index} + end + defp next_snippet(ast, index) do + {"${#{index}:#{spec_ast_to_string(ast)}}$", index + 1} + end + + def param_to_var({{:=, _, [_lhs, {name, _, _} = rhs]}, arg_index}) when is_atom(name) do + rhs + |> to_var(arg_index + 1) + |> Macro.to_string + end + + def param_to_var({{:=, _, [{name, _, _} = lhs, _rhs]}, arg_index}) when is_atom(name) do + lhs + |> to_var(arg_index + 1) + |> Macro.to_string + end + + def param_to_var({{:\\, _, _} = ast, _}) do + ast + |> Macro.to_string + end + + def param_to_var({ast, arg_index}) do + ast + |> to_var(arg_index + 1) + |> Macro.to_string + end + + defp to_var({:{}, _, _}, _), + do: {:tuple, [], nil} + defp to_var({_, _}, _), + do: {:tuple, [], nil} + defp to_var({name, meta, _}, _) when is_atom(name), + do: {name, meta, nil} + defp to_var({:<<>>, _, _}, _), + do: {:binary, [], nil} + defp to_var({:%{}, _, _}, _), + do: {:map, [], nil} + defp to_var(integer, _) when is_integer(integer), + do: {:integer, [], nil} + defp to_var(float, _) when is_float(float), + do: {:float, [], nil} + defp to_var(list, _) when is_list(list), + do: {:list, [], nil} + defp to_var(atom, _) when is_atom(atom), + do: {:atom, [], nil} + defp to_var(_, i), + do: {:"arg#{i}", [], nil} + + def get_module_docs_summary(module) do + case Code.get_docs module, :moduledoc do + {_, doc} -> extract_summary_from_docs(doc) + _ -> "" + end + end + + def get_module_subtype(module) do + has_func = fn f, a -> Code.ensure_loaded?(module) && Kernel.function_exported?(module, f, a) end + cond do + has_func.(:__protocol__, 1) -> :protocol + has_func.(:__impl__, 1) -> :implementation + has_func.(:__struct__, 0) -> + if Map.get(module.__struct__, :__exception__) do + :exception + else + :struct + end + true -> nil + end + end + + def extract_fun_args_and_desc({{_fun, _}, _line, _kind, args, doc}) do + formatted_args = + args + |> Enum.map_join(",", &format_doc_arg(&1)) + |> String.replace(~r/\s+/, " ") + desc = extract_summary_from_docs(doc) + {formatted_args, desc} + end + + def extract_fun_args_and_desc(nil) do + {"", ""} + end + + def get_module_specs(module) do + case beam_specs(module) do + nil -> %{} + specs -> + for {_kind, {{f, a}, _spec}} = spec <- specs, into: %{} do + {{f, a}, spec_to_string(spec)} + end + end + end + + def get_spec(module, function, arity) when is_atom(module) and is_atom(function) and is_integer(arity) do + module + |> get_module_specs + |> Map.get({function, arity}, "") + end + + def get_spec_text(mod, fun, arity) do + case get_spec(mod, fun, arity) do + "" -> "" + spec -> + "### Specs\n\n`#{spec}`\n\n" + end + end + + def module_to_string(module) do + case module |> Atom.to_string do + "Elixir." <> name -> name + name -> ":#{name}" + end + end + + def split_mod_fun_call(call) do + case Code.string_to_quoted(call) do + {:error, _} -> + {nil, nil} + {:ok, quoted} when is_atom(quoted) -> + {quoted, nil} + {:ok, quoted} -> + split_mod_quoted_fun_call(quoted) + end + end + + def split_mod_quoted_fun_call(quoted) do + case Macro.decompose_call(quoted) do + {{:__aliases__, _, mod_parts}, fun, _args} -> + {Module.concat(mod_parts), fun} + {:__aliases__, mod_parts} -> + {Module.concat(mod_parts), nil} + {mod, func, []} when is_atom(mod) and is_atom(func) -> + {mod, func} + {func, []} when is_atom(func) -> + {nil, func} + _ -> {nil, nil} + end + end + + def module_functions_info(module) do + docs = Code.get_docs(module, :docs) || [] + specs = get_module_specs(module) + for {{f, a}, _line, func_kind, _sign, doc} = func_doc <- docs, doc != false, into: %{} do + spec = Map.get(specs, {f, a}, "") + {fun_args, desc} = extract_fun_args_and_desc(func_doc) + {{f, a}, {func_kind, fun_args, desc, spec}} + end + end + + def get_callback_ast(module, callback, arity) do + {{name, _}, [spec | _]} = module + |> Kernel.Typespec.beam_callbacks() + |> Enum.find(fn {{f, a}, _} -> {f, a} == {callback, arity} end) + + Kernel.Typespec.spec_to_ast(name, spec) + end + + defp format_doc_arg({:\\, _, [left, right]}) do + format_doc_arg(left) <> " \\\\ " <> Macro.to_string(right) + end + + defp format_doc_arg({var, _, _}) do + Atom.to_string(var) + end + + defp spec_ast_to_string(ast) do + ast |> Macro.to_string |> String.replace("()", "") + end + + defp spec_to_string({kind, {{name, _arity}, specs}}) do + spec = hd(specs) + binary = Macro.to_string Typespec.spec_to_ast(name, spec) + "@#{kind} #{binary}" |> String.replace("()", "") + end + + defp beam_specs(module) do + beam_specs_tag(Typespec.beam_specs(module), :spec) + end + + defp beam_specs_tag(nil, _), do: nil + defp beam_specs_tag(specs, tag) do + Enum.map(specs, &{tag, &1}) + end + + def actual_mod_fun(mod_fun, imports, aliases, current_module) do + with {nil, nil} <- find_kernel_function(mod_fun), + {nil, nil} <- find_imported_function(mod_fun, imports), + {nil, nil} <- find_aliased_function(mod_fun, aliases), + {nil, nil} <- find_function_in_module(mod_fun), + {nil, nil} <- find_function_in_current_module(mod_fun, current_module) + do + mod_fun + else + new_mod_fun -> new_mod_fun + end + end + + defp find_kernel_function({nil, fun}) do + cond do + ModuleInfo.docs?(Kernel, fun) -> + {Kernel, fun} + ModuleInfo.docs?(Kernel.SpecialForms, fun) -> + {Kernel.SpecialForms, fun} + true -> {nil, nil} + end + end + + defp find_kernel_function({_mod, _fun}) do + {nil, nil} + end + + defp find_imported_function({nil, fun}, imports) do + case imports |> Enum.find(&ModuleInfo.has_function?(&1, fun)) do + nil -> {nil, nil} + mod -> {mod, fun} + end + end + + defp find_imported_function({_mod, _fun}, _imports) do + {nil, nil} + end + + defp find_aliased_function({nil, _fun}, _aliases) do + {nil, nil} + end + + defp find_aliased_function({mod, fun}, aliases) do + if elixir_module?(mod) do + module = + mod + |> Module.split + |> ModuleInfo.expand_alias(aliases) + {module, fun} + else + {nil, nil} + end + end + + defp find_function_in_module({mod, fun}) do + if elixir_module?(mod) && ModuleInfo.has_function?(mod, fun) do + {mod, fun} + else + {nil, nil} + end + end + + defp find_function_in_current_module({nil, fun}, current_module) do + {current_module, fun} + end + + defp find_function_in_current_module(_, _) do + {nil, nil} + end + + defp elixir_module?(module) when is_atom(module) do + module == Module.concat(Elixir, module) + end + defp elixir_module?(_) do + false + end + +end diff --git a/elixir_sense/lib/elixir_sense/core/metadata.ex b/elixir_sense/lib/elixir_sense/core/metadata.ex new file mode 100644 index 0000000..852affb --- /dev/null +++ b/elixir_sense/lib/elixir_sense/core/metadata.ex @@ -0,0 +1,82 @@ +defmodule ElixirSense.Core.Metadata do + @moduledoc """ + Core Metadata + """ + + alias ElixirSense.Core.State + alias ElixirSense.Core.Introspection + + defstruct source: nil, + mods_funs_to_lines: %{}, + lines_to_env: %{}, + error: nil + + def get_env(%__MODULE__{} = metadata, line_number) do + case Map.get(metadata.lines_to_env, line_number) do + nil -> %State.Env{} + ctx -> ctx + end + end + + def get_function_line(%__MODULE__{} = metadata, module, function) do + case Map.get(metadata.mods_funs_to_lines, {module, function, nil}) do + nil -> get_function_line_using_docs(module, function) + %{lines: lines} -> List.last(lines) + end + end + + def get_function_info(%__MODULE__{} = metadata, module, function) do + case Map.get(metadata.mods_funs_to_lines, {module, function, nil}) do + nil -> %{lines: [], params: []} + info -> info + end + end + + def get_function_params(%__MODULE__{} = metadata, module, function) do + params = + metadata + |> get_function_info(module, function) + |> Map.get(:params) + |> Enum.reverse + + Enum.map(params, fn param -> + param + |> Macro.to_string() + |> String.slice(1..-2) + end) + end + + def get_function_signatures(%__MODULE__{} = metadata, module, function, code_docs \\ nil) do + docs = code_docs || Code.get_docs(module, :docs) || [] + + params_list = + metadata + |> get_function_info(module, function) + |> Map.get(:params) + |> Enum.reverse + + Enum.map(params_list, fn params -> + arity = length(params) + {doc, spec} = + Enum.find_value(docs, {"", ""}, fn {{f, a}, _, _, _, text} -> + f == function && + a == arity && + {Introspection.extract_summary_from_docs(text), Introspection.get_spec(module, function, arity)} + end) + %{name: Atom.to_string(function), + params: params |> Enum.with_index() |> Enum.map(&Introspection.param_to_var/1), + documentation: doc, + spec: spec + } + end) + end + + defp get_function_line_using_docs(module, function) do + docs = Code.get_docs(module, :docs) + + for {{func, _arity}, line, _kind, _, _} <- docs, func == function do + line + end |> Enum.at(0) + end + +end diff --git a/elixir_sense/lib/elixir_sense/core/metadata_builder.ex b/elixir_sense/lib/elixir_sense/core/metadata_builder.ex new file mode 100644 index 0000000..9a3e48f --- /dev/null +++ b/elixir_sense/lib/elixir_sense/core/metadata_builder.ex @@ -0,0 +1,364 @@ +defmodule ElixirSense.Core.MetadataBuilder do + + @moduledoc """ + This module is responsible for building/retrieving environment information from an AST. + """ + + import ElixirSense.Core.State + alias ElixirSense.Core.Ast + alias ElixirSense.Core.State + + @scope_keywords [:for, :try, :fn] + @block_keywords [:do, :else, :rescue, :catch, :after] + @defs [:def, :defp, :defmacro, :defmacrop] + + @doc """ + Traverses the AST building/retrieving the environment information. + It returns a `ElixirSense.Core.State` struct containing the information. + """ + def build(ast) do + {_ast, state} = Macro.traverse(ast, %State{}, &pre/2, &post/2) + state + end + + defp pre_module(ast, state, line, module) do + state + |> new_namespace(module) + |> add_current_module_to_index(line) + |> create_alias_for_current_module + |> new_attributes_scope + |> new_behaviours_scope + |> new_alias_scope + |> new_import_scope + |> new_require_scope + |> new_vars_scope + |> result(ast) + end + + defp post_module(ast, state, module) do + state + |> remove_module_from_namespace(module) + |> remove_attributes_scope + |> remove_behaviours_scope + |> remove_alias_scope + |> remove_import_scope + |> remove_require_scope + |> remove_vars_scope + |> result(ast) + end + + defp pre_func(ast, state, line, name, params) do + state + |> new_named_func(name, length(params || [])) + |> add_current_env_to_line(line) + |> add_func_to_index(name, params || [], line) + |> new_alias_scope + |> new_import_scope + |> new_require_scope + |> new_func_vars_scope + |> add_vars(find_vars(params)) + |> result(ast) + end + + defp post_func(ast, state) do + state + |> remove_alias_scope + |> remove_import_scope + |> remove_require_scope + |> remove_func_vars_scope + |> remove_last_scope_from_scopes + |> result(ast) + end + + defp pre_scope_keyword(ast, state, line) do + state + |> add_current_env_to_line(line) + |> new_vars_scope + |> result(ast) + end + + defp post_scope_keyword(ast, state) do + state + |> remove_vars_scope + |> result(ast) + end + + defp pre_block_keyword(ast, state) do + state + |> new_alias_scope + |> new_import_scope + |> new_require_scope + |> new_vars_scope + |> result(ast) + end + + defp post_block_keyword(ast, state) do + state + |> remove_alias_scope + |> remove_import_scope + |> remove_require_scope + |> remove_vars_scope + |> result(ast) + end + + defp pre_clause(ast, state, lhs) do + state + |> new_alias_scope + |> new_import_scope + |> new_require_scope + |> new_vars_scope + |> add_vars(find_vars(lhs)) + |> result(ast) + end + + defp post_clause(ast, state) do + state + |> remove_alias_scope + |> remove_import_scope + |> remove_require_scope + |> remove_vars_scope + |> result(ast) + end + + defp pre_alias(ast, state, line, aliases_tuples) when is_list(aliases_tuples) do + state + |> add_current_env_to_line(line) + |> add_aliases(aliases_tuples) + |> result(ast) + end + + defp pre_alias(ast, state, line, alias_tuple) do + state + |> add_current_env_to_line(line) + |> add_alias(alias_tuple) + |> result(ast) + end + + defp pre_import(ast, state, line, modules) when is_list(modules) do + state + |> add_current_env_to_line(line) + |> add_imports(modules) + |> result(ast) + end + + defp pre_import(ast, state, line, module) do + state + |> add_current_env_to_line(line) + |> add_import(module) + |> result(ast) + end + + defp pre_require(ast, state, line, modules) when is_list(modules) do + state + |> add_current_env_to_line(line) + |> add_requires(modules) + |> result(ast) + end + + defp pre_require(ast, state, line, module) do + state + |> add_current_env_to_line(line) + |> add_require(module) + |> result(ast) + end + + defp pre_module_attribute(ast, state, line, name) do + state + |> add_current_env_to_line(line) + |> add_attribute(name) + |> result(ast) + end + + defp pre_behaviour(ast, state, line, module) do + state + |> add_current_env_to_line(line) + |> add_behaviour(module) + |> result(ast) + end + + defp pre({:defmodule, [line: line], [{:__aliases__, _, module}, _]} = ast, state) do + pre_module(ast, state, line, module) + end + + defp pre({def_name, meta, [{:when, _, [head|_]}, body]}, state) when def_name in @defs do + pre({def_name, meta, [head, body]}, state) + end + + defp pre({def_name, [line: line], [{name, _, params}, _body]} = ast, state) when def_name in @defs and is_atom(name) do + pre_func(ast, state, line, name, params) + end + + defp pre({def_name, _, _} = ast, state) when def_name in @defs do + {ast, state} + end + + defp pre({:@, [line: line], [{:behaviour, _, [{:__aliases__, _, module_atoms}]}]} = ast, state) do + module = module_atoms |> Module.concat + pre_behaviour(ast, state, line, module) + end + + defp pre({:@, [line: line], [{:behaviour, _, [erlang_module]}]} = ast, state) do + pre_behaviour(ast, state, line, erlang_module) + end + + defp pre({:@, [line: line], [{name, _, _}]} = ast, state) do + pre_module_attribute(ast, state, line, name) + end + + # import with v1.2 notation + defp pre({:import, [line: line], [{{:., _, [{:__aliases__, _, prefix_atoms}, :{}]}, _, imports}]} = ast, state) do + imports_modules = imports |> Enum.map(fn {:__aliases__, _, mods} -> + Module.concat(prefix_atoms ++ mods) + end) + pre_import(ast, state, line, imports_modules) + end + + # import without options + defp pre({:import, meta, [module_info]}, state) do + pre({:import, meta, [module_info, []]}, state) + end + + # import with options + defp pre({:import, [line: line], [{_, _, module_atoms = [mod|_]}, _opts]} = ast, state) when is_atom(mod) do + module = module_atoms |> Module.concat + pre_import(ast, state, line, module) + end + + # require with v1.2 notation + defp pre({:require, [line: line], [{{:., _, [{:__aliases__, _, prefix_atoms}, :{}]}, _, requires}]} = ast, state) do + requires_modules = requires |> Enum.map(fn {:__aliases__, _, mods} -> + Module.concat(prefix_atoms ++ mods) + end) + pre_require(ast, state, line, requires_modules) + end + + # require without options + defp pre({:require, meta, [module_info]}, state) do + pre({:require, meta, [module_info, []]}, state) + end + + # require with options + defp pre({:require, [line: line], [{_, _, module_atoms = [mod|_]}, _opts]} = ast, state) when is_atom(mod) do + module = module_atoms |> Module.concat + pre_require(ast, state, line, module) + end + + # alias with v1.2 notation + defp pre({:alias, [line: line], [{{:., _, [{:__aliases__, _, prefix_atoms}, :{}]}, _, aliases}]} = ast, state) do + aliases_tuples = aliases |> Enum.map(fn {:__aliases__, _, mods} -> + {Module.concat(mods), Module.concat(prefix_atoms ++ mods)} + end) + pre_alias(ast, state, line, aliases_tuples) + end + + # alias without options + defp pre({:alias, [line: line], [{:__aliases__, _, module_atoms = [mod|_]}]} = ast, state) when is_atom(mod) do + alias_tuple = {Module.concat([List.last(module_atoms)]), Module.concat(module_atoms)} + pre_alias(ast, state, line, alias_tuple) + end + + # alias with `as` option + defp pre({:alias, [line: line], [{_, _, module_atoms = [mod|_]}, [as: {:__aliases__, _, alias_atoms = [al|_]}]]} = ast, state) when is_atom(mod) and is_atom(al) do + alias_tuple = {Module.concat(alias_atoms), Module.concat(module_atoms)} + pre_alias(ast, state, line, alias_tuple) + end + + defp pre({atom, [line: line], _} = ast, state) when atom in @scope_keywords do + pre_scope_keyword(ast, state, line) + end + + defp pre({atom, _block} = ast, state) when atom in @block_keywords do + pre_block_keyword(ast, state) + end + + defp pre({:->, [line: _line], [lhs, _rhs]} = ast, state) do + pre_clause(ast, state, lhs) + end + + defp pre({:=, _meta, [lhs, _rhs]} = ast, state) do + state + |> add_vars(find_vars(lhs)) + |> result(ast) + end + + defp pre({:<-, _meta, [lhs, _rhs]} = ast, state) do + state + |> add_vars(find_vars(lhs)) + |> result(ast) + end + + # Kernel: defmacro use(module, opts \\ []) + defp pre({:use, [line: _], [{param, _, nil}|_]} = ast, state) when is_atom(param) do + state + |> result(ast) + end + + defp pre({:use, [line: line], _} = ast, state) do + %{requires: requires, imports: imports, behaviours: behaviours} = Ast.extract_use_info(ast, get_current_module(state)) + + state + |> add_current_env_to_line(line) + |> add_requires(requires) + |> add_imports(imports) + |> add_behaviours(behaviours) + |> result(ast) + end + + # Any other tuple with a line + defp pre({_, [line: line], _} = ast, state) do + state + |> add_current_env_to_line(line) + |> result(ast) + end + + # No line defined + defp pre(ast, state) do + {ast, state} + end + + defp post({:defmodule, _, [{:__aliases__, _, module}, _]} = ast, state) do + post_module(ast, state, module) + end + + defp post({def_name, [line: _line], [{name, _, _params}, _]} = ast, state) when def_name in @defs and is_atom(name) do + post_func(ast, state) + end + + defp post({def_name, _, _} = ast, state) when def_name in @defs do + {ast, state} + end + + defp post({atom, _, _} = ast, state) when atom in @scope_keywords do + post_scope_keyword(ast, state) + end + + defp post({atom, _block} = ast, state) when atom in @block_keywords do + post_block_keyword(ast, state) + end + + defp post({:->, [line: _line], [_lhs, _rhs]} = ast, state) do + post_clause(ast, state) + end + + defp post(ast, state) do + {ast, state} + end + + defp result(state, ast) do + {ast, state} + end + + defp find_vars(ast) do + {_ast, vars} = Macro.prewalk(ast, [], &match_var/2) + vars |> Enum.uniq_by(&(&1)) + end + + defp match_var({var, [line: _], context} = ast, vars) when is_atom(var) and context in [nil, Elixir] do + {ast, [var|vars]} + end + + defp match_var(ast, vars) do + {ast, vars} + end + +end diff --git a/elixir_sense/lib/elixir_sense/core/parser.ex b/elixir_sense/lib/elixir_sense/core/parser.ex new file mode 100644 index 0000000..a7f1169 --- /dev/null +++ b/elixir_sense/lib/elixir_sense/core/parser.ex @@ -0,0 +1,103 @@ +defmodule ElixirSense.Core.Parser do + @moduledoc """ + Core Parser + """ + + alias ElixirSense.Core.MetadataBuilder + alias ElixirSense.Core.Metadata + + def parse_file(file, try_to_fix_parse_error, try_to_fix_line_not_found, cursor_line_number) do + case File.read(file) do + {:ok, source} -> + parse_string(source, try_to_fix_parse_error, try_to_fix_line_not_found, cursor_line_number) + error -> error + end + end + + def parse_string(source, try_to_fix_parse_error, try_to_fix_line_not_found, cursor_line_number) do + case string_to_ast(source, try_to_fix_parse_error, cursor_line_number) do + {:ok, ast} -> + acc = MetadataBuilder.build(ast) + if Map.has_key?(acc.lines_to_env, cursor_line_number) or !try_to_fix_line_not_found do + %Metadata{ + source: source, + mods_funs_to_lines: acc.mods_funs_to_lines, + lines_to_env: acc.lines_to_env + } + else + # IO.puts :stderr, "LINE NOT FOUND" + source + |> fix_line_not_found(cursor_line_number) + |> parse_string(false, false, cursor_line_number) + end + {:error, error} -> + # IO.puts :stderr, "CAN'T FIX IT" + # IO.inspect :stderr, error, [] + %Metadata{ + source: source, + error: error + } + end + end + + defp string_to_ast(source, try_to_fix_parse_error, cursor_line_number) do + case Code.string_to_quoted(source) do + {:ok, ast} -> + {:ok, ast} + error -> + # IO.puts :stderr, "PARSE ERROR" + # IO.inspect :stderr, error, [] + if try_to_fix_parse_error do + source + |> fix_parse_error(cursor_line_number, error) + |> string_to_ast(false, cursor_line_number) + else + error + end + end + end + + defp fix_parse_error(source, _cursor_line_number, {:error, {line, {"\"" <> <<_::bytes-size(1)>> <> "\" is missing terminator" <> _, _}, _}}) when is_integer(line) do + source + |> replace_line_with_marker(line) + end + + defp fix_parse_error(source, _cursor_line_number, {:error, {_line, {_error_type, text}, _token}}) do + [_, line] = Regex.run(~r/line\s(\d+)/, text) + line = line |> String.to_integer + source + |> replace_line_with_marker(line) + end + + defp fix_parse_error(source, cursor_line_number, {:error, {line, "syntax" <> _, "'end'"}}) when is_integer(line) do + source + |> replace_line_with_marker(cursor_line_number) + end + + defp fix_parse_error(source, _cursor_line_number, {:error, {line, "syntax" <> _, _token}}) when is_integer(line) do + source + |> replace_line_with_marker(line) + end + + defp fix_parse_error(_, nil, error) do + error + end + + defp fix_parse_error(source, cursor_line_number, _error) do + source + |> replace_line_with_marker(cursor_line_number) + end + + defp fix_line_not_found(source, line_number) do + source |> replace_line_with_marker(line_number) + end + + defp replace_line_with_marker(source, line) do + # IO.puts :stderr, "REPLACING LINE: #{line}" + source + |> String.split(["\n", "\r\n"]) + |> List.replace_at(line - 1, "(__atom_elixir_marker_#{line}__())") + |> Enum.join("\n") + end + +end diff --git a/elixir_sense/lib/elixir_sense/core/source.ex b/elixir_sense/lib/elixir_sense/core/source.ex new file mode 100644 index 0000000..8ff7f0c --- /dev/null +++ b/elixir_sense/lib/elixir_sense/core/source.ex @@ -0,0 +1,186 @@ +defmodule ElixirSense.Core.Source do + @moduledoc """ + Source parsing + """ + + @empty_graphemes [" ", "\n", "\r\n"] + @stop_graphemes ~w/{ } ( ) [ ] < > + - * & ^ , ; ~ % = " ' \\ \/ $ ! ?`#/ ++ @empty_graphemes + + def prefix(code, line, col) do + line = code |> String.split("\n") |> Enum.at(line - 1) + line_str = line |> String.slice(0, col - 1) + case Regex.run(~r/[\w0-9\._!\?\:@]+$/, line_str) do + nil -> "" + [prefix] -> prefix + end + end + + def text_before(code, line, col) do + pos = find_position(code, line, col, {0, 1, 1}) + {text, _rest} = String.split_at(code, pos) + text + end + + def subject(code, line, col) do + case walk_text(code, &find_subject/5, %{line: line, col: col, pos_found: false, candidate: []}) do + %{candidate: []} -> + nil + %{candidate: candidate} -> + candidate |> Enum.reverse |> Enum.join + end + end + + defp find_subject(grapheme, rest, line, col, %{pos_found: false, line: line, col: col} = acc) do + find_subject(grapheme, rest, line, col, %{acc | pos_found: true}) + end + defp find_subject("." = grapheme, rest, _line, _col, %{pos_found: false} = acc) do + {rest, %{acc | candidate: [grapheme|acc.candidate]}} + end + defp find_subject(".", _rest, _line, _col, %{pos_found: true} = acc) do + {"", acc} + end + defp find_subject(grapheme, rest, _line, _col, %{candidate: [_|_]} = acc) when grapheme in ["!", "?"] do + {rest, %{acc | candidate: [grapheme|acc.candidate]}} + end + defp find_subject(grapheme, rest, _line, _col, %{candidate: ["."|_]} = acc) when grapheme in @stop_graphemes do + {rest, acc} + end + defp find_subject(grapheme, rest, _line, _col, %{pos_found: false} = acc) when grapheme in @stop_graphemes do + {rest, %{acc | candidate: []}} + end + defp find_subject(grapheme, _rest, _line, _col, %{pos_found: true} = acc) when grapheme in @stop_graphemes do + {"", acc} + end + defp find_subject(grapheme, rest, _line, _col, acc) do + {rest, %{acc | candidate: [grapheme|acc.candidate]}} + end + + defp walk_text(text, func, acc) do + do_walk_text(text, func, 1, 1, acc) + end + + defp do_walk_text(text, func, line, col, acc) do + case String.next_grapheme(text) do + nil -> + acc + {grapheme, rest} -> + {new_rest, new_acc} = func.(grapheme, rest, line, col, acc) + {new_line, new_col} = + if grapheme in ["\n", "\r\n"] do + {line + 1, 1} + else + {line, col + 1} + end + + do_walk_text(new_rest, func, new_line, new_col, new_acc) + end + end + + defp find_position(_text, line, col, {pos, line, col}) do + pos + end + + defp find_position(text, line, col, {pos, current_line, current_col}) do + case String.next_grapheme(text) do + {grapheme, rest} -> + {new_pos, new_line, new_col} = + if grapheme in ["\n", "\r\n"] do + {pos + 1, current_line + 1, 1} + else + {pos + 1, current_line, current_col + 1} + end + find_position(rest, line, col, {new_pos, new_line, new_col}) + nil -> + pos + end + end + + def which_func(prefix) do + tokens = + case prefix |> String.to_charlist |> :elixir_tokenizer.tokenize(1, []) do + {:ok, _, _, tokens} -> + tokens |> Enum.reverse + {:error, {_line, _error_prefix, _token}, _rest, sofar} -> + # DEBUG + # IO.puts :stderr, :elixir_utils.characters_to_binary(error_prefix) + # IO.inspect(:stderr, {:sofar, sofar}, []) + # IO.inspect(:stderr, {:rest, rest}, []) + sofar + end + pattern = %{npar: 0, count: 0, count2: 0, candidate: [], pos: nil, pipe_before: false} + result = scan(tokens, pattern) + %{candidate: candidate, npar: npar, pipe_before: pipe_before, pos: pos} = result + + %{ + candidate: normalize_candidate(candidate), + npar: normalize_npar(npar, pipe_before), + pipe_before: pipe_before, + pos: pos + } + end + + defp normalize_candidate(candidate) do + case candidate do + [] -> :none + [func] -> {nil, func} + [mod, func] -> {mod, func} + list -> + [func|mods] = Enum.reverse(list) + {Module.concat(Enum.reverse(mods)), func} + end + end + + defp normalize_npar(npar, true), do: npar + 1 + defp normalize_npar(npar, _pipe_before), do: npar + + defp scan([{:",", _}|_], %{count: 1} = state), do: state + defp scan([{:",", _}|tokens], %{count: 0, count2: 0} = state) do + scan(tokens, %{state | npar: state.npar + 1, candidate: []}) + end + defp scan([{:"(", _}|_], %{count: 1} = state), do: state + defp scan([{:"(", _}|tokens], state) do + scan(tokens, %{state | count: state.count + 1, candidate: []}) + end + defp scan([{:")", _}|tokens], state) do + scan(tokens, %{state | count: state.count - 1, candidate: []}) + end + defp scan([{token, _}|tokens], %{count2: 0} = state) when token in [:"[", :"{"] do + scan(tokens, %{state | npar: 0, count2: 0}) + end + defp scan([{token, _}|tokens], state) when token in [:"[", :"{"] do + scan(tokens, %{state | count2: state.count2 + 1}) + end + defp scan([{token, _}|tokens], state) when token in [:"]", :"}"]do + scan(tokens, %{state | count2: state.count2 - 1}) + end + defp scan([{:paren_identifier, pos, value}|tokens], %{count: 1} = state) do + scan(tokens, %{state | candidate: [value|state.candidate], pos: update_pos(pos, state.pos)}) + end + defp scan([{:aliases, pos, [value]}|tokens], %{count: 1} = state) do + updated_pos = update_pos(pos, state.pos) + scan(tokens, %{state | candidate: [Module.concat([value])|state.candidate], pos: updated_pos}) + end + defp scan([{:atom, pos, value}|tokens], %{count: 1} = state) do + scan(tokens, %{state | candidate: [value|state.candidate], pos: update_pos(pos, state.pos)}) + end + defp scan([{:fn, _}|tokens], %{count: 1} = state) do + scan(tokens, %{state | npar: 0, count: 0}) + end + defp scan([{:., _}|tokens], state), do: scan(tokens, state) + defp scan([{:arrow_op, _, :|>}|_], %{count: 1} = state), do: pipe_before(state) + defp scan([_|_], %{count: 1} = state), do: state + defp scan([_token|tokens], state), do: scan(tokens, state) + defp scan([], state), do: state + + defp update_pos({line, init_col, end_col}, nil) do + {{line, init_col}, {line, end_col}} + end + defp update_pos({new_init_line, new_init_col, _}, {{_, _}, {end_line, end_col}}) do + {{new_init_line, new_init_col}, {end_line, end_col}} + end + + defp pipe_before(state) do + %{state | pipe_before: true} + end + +end diff --git a/elixir_sense/lib/elixir_sense/core/state.ex b/elixir_sense/lib/elixir_sense/core/state.ex new file mode 100644 index 0000000..02382ef --- /dev/null +++ b/elixir_sense/lib/elixir_sense/core/state.ex @@ -0,0 +1,251 @@ +defmodule ElixirSense.Core.State do + @moduledoc """ + Core State + """ + + defstruct [ + namespace: [:Elixir], + scopes: [:Elixir], + imports: [[]], + requires: [[]], + aliases: [[]], + attributes: [[]], + scope_attributes: [[]], + behaviours: [[]], + scope_behaviours: [[]], + vars: [[]], + scope_vars: [[]], + mods_funs_to_lines: %{}, + lines_to_env: %{} + ] + + defmodule Env do + @moduledoc false + defstruct imports: [], requires: [], aliases: [], module: nil, vars: [], attributes: [], behaviours: [], scope: nil + end + + def get_current_env(state) do + current_module = get_current_module(state) + current_imports = state.imports |> :lists.reverse |> List.flatten + current_requires = state.requires |> :lists.reverse |> List.flatten + current_aliases = state.aliases |> :lists.reverse |> List.flatten + current_vars = state.scope_vars |> :lists.reverse |> List.flatten + current_attributes = state.scope_attributes |> :lists.reverse |> List.flatten + current_behaviours = hd(state.behaviours) + current_scope = hd(state.scopes) + + %Env{ + imports: current_imports, + requires: current_requires, + aliases: current_aliases, + module: current_module, + vars: current_vars, + attributes: current_attributes, + behaviours: current_behaviours, + scope: current_scope + } + end + + def get_current_module(state) do + state.namespace |> :lists.reverse |> Module.concat + end + + def add_current_env_to_line(state, line) do + env = get_current_env(state) + %{state | lines_to_env: Map.put(state.lines_to_env, line, env)} + end + + def get_current_scope_name(state) do + scope = case hd(state.scopes) do + {fun, _} -> fun + mod -> mod + end + scope |> Atom.to_string() + end + + def add_mod_fun_to_line(state, {module, fun, arity}, line, params) do + current_info = Map.get(state.mods_funs_to_lines, {module, fun, arity}, %{}) + current_params = current_info |> Map.get(:params, []) + current_lines = current_info |> Map.get(:lines, []) + new_params = [params|current_params] + new_lines = [line|current_lines] + + mods_funs_to_lines = Map.put(state.mods_funs_to_lines, {module, fun, arity}, %{lines: new_lines, params: new_params}) + %{state | mods_funs_to_lines: mods_funs_to_lines} + end + + def new_namespace(state, module) do + module_reversed = :lists.reverse(module) + namespace = module_reversed ++ state.namespace + scopes = module_reversed ++ state.scopes + %{state | namespace: namespace, scopes: scopes} + end + + def remove_module_from_namespace(state, module) do + outer_mods = Enum.drop(state.namespace, length(module)) + outer_scopes = Enum.drop(state.scopes, length(module)) + %{state | namespace: outer_mods, scopes: outer_scopes} + end + + def new_named_func(state, name, arity) do + %{state | scopes: [{name, arity}|state.scopes]} + end + + def remove_last_scope_from_scopes(state) do + %{state | scopes: tl(state.scopes)} + end + + def add_current_module_to_index(state, line) do + current_module = state.namespace |> :lists.reverse |> Module.concat + add_mod_fun_to_line(state, {current_module, nil, nil}, line, nil) + end + + def add_func_to_index(state, func, params, line) do + current_module = state.namespace |> :lists.reverse |> Module.concat + state + |> add_mod_fun_to_line({current_module, func, length(params)}, line, params) + |> add_mod_fun_to_line({current_module, func, nil}, line, params) + end + + def new_alias_scope(state) do + %{state | aliases: [[]|state.aliases]} + end + + def create_alias_for_current_module(state) do + if length(state.namespace) > 2 do + current_module = state.namespace |> :lists.reverse |> Module.concat + alias_tuple = {Module.concat([hd(state.namespace)]), current_module} + state |> add_alias(alias_tuple) + else + state + end + end + + def remove_alias_scope(state) do + %{state | aliases: tl(state.aliases)} + end + + def new_vars_scope(state) do + %{state | vars: [[]|state.vars], scope_vars: [[]|state.scope_vars]} + end + + def new_func_vars_scope(state) do + %{state | vars: [[]|state.vars], scope_vars: [[]]} + end + + def new_attributes_scope(state) do + %{state | attributes: [[]|state.attributes], scope_attributes: [[]]} + end + + def new_behaviours_scope(state) do + %{state | behaviours: [[]|state.behaviours], scope_behaviours: [[]]} + end + + def remove_vars_scope(state) do + %{state | vars: tl(state.vars), scope_vars: tl(state.scope_vars)} + end + + def remove_func_vars_scope(state) do + vars = tl(state.vars) + %{state | vars: vars, scope_vars: vars} + end + + def remove_attributes_scope(state) do + attributes = tl(state.attributes) + %{state | attributes: attributes, scope_attributes: attributes} + end + + def remove_behaviours_scope(state) do + behaviours = tl(state.behaviours) + %{state | behaviours: behaviours, scope_behaviours: behaviours} + end + + def add_alias(state, alias_tuple) do + [aliases_from_scope|inherited_aliases] = state.aliases + %{state | aliases: [[alias_tuple|aliases_from_scope]|inherited_aliases]} + end + + def add_aliases(state, aliases_tuples) do + Enum.reduce(aliases_tuples, state, fn(tuple, state) -> add_alias(state, tuple) end) + end + + def new_import_scope(state) do + %{state | imports: [[]|state.imports]} + end + + def new_require_scope(state) do + %{state | requires: [[]|state.requires]} + end + + def remove_import_scope(state) do + %{state | imports: tl(state.imports)} + end + + def remove_require_scope(state) do + %{state | requires: tl(state.requires)} + end + + def add_import(state, module) do + [imports_from_scope|inherited_imports] = state.imports + %{state | imports: [[module|imports_from_scope]|inherited_imports]} + end + + def add_imports(state, modules) do + Enum.reduce(modules, state, fn(mod, state) -> add_import(state, mod) end) + end + + def add_require(state, module) do + [requires_from_scope|inherited_requires] = state.requires + %{state | requires: [[module|requires_from_scope]|inherited_requires]} + end + + def add_requires(state, modules) do + Enum.reduce(modules, state, fn(mod, state) -> add_require(state, mod) end) + end + + def add_var(state, var) do + scope = get_current_scope_name(state) + [vars_from_scope|other_vars] = state.vars + + vars_from_scope = + if var in vars_from_scope do + vars_from_scope + else + case Atom.to_string(var) do + "_" <> _ -> vars_from_scope + ^scope -> vars_from_scope + _ -> [var|vars_from_scope] + end + end + + %{state | vars: [vars_from_scope|other_vars], scope_vars: [vars_from_scope|tl(state.scope_vars)]} + end + + def add_attribute(state, attribute) do + [attributes_from_scope|other_attributes] = state.attributes + + attributes_from_scope = + if attribute in attributes_from_scope do + attributes_from_scope + else + [attribute|attributes_from_scope] + end + attributes = [attributes_from_scope|other_attributes] + scope_attributes = [attributes_from_scope|tl(state.scope_attributes)] + %{state | attributes: attributes, scope_attributes: scope_attributes} + end + + def add_behaviour(state, module) do + [behaviours_from_scope|other_behaviours] = state.behaviours + %{state | behaviours: [[module|behaviours_from_scope]|other_behaviours]} + end + + def add_behaviours(state, modules) do + Enum.reduce(modules, state, fn(mod, state) -> add_behaviour(state, mod) end) + end + + def add_vars(state, vars) do + vars |> Enum.reduce(state, fn(var, state) -> add_var(state, var) end) + end + +end diff --git a/elixir_sense/lib/elixir_sense/providers/definition.ex b/elixir_sense/lib/elixir_sense/providers/definition.ex new file mode 100644 index 0000000..dc34a75 --- /dev/null +++ b/elixir_sense/lib/elixir_sense/providers/definition.ex @@ -0,0 +1,77 @@ +defmodule ElixirSense.Providers.Definition do + + @moduledoc """ + Provides a function to find out where symbols are defined. + + Currently finds definition of modules, functions and macros. + """ + + alias ElixirSense.Core.Metadata + alias ElixirSense.Core.Parser + alias ElixirSense.Core.Introspection + + @type file :: String.t + @type line :: pos_integer + @type location :: {file, line | nil} + + @doc """ + Finds out where a module, function or macro was defined. + """ + @spec find(String.t, [module], [{module, module}], module) :: location + def find(subject, imports, aliases, module) do + subject + |> Introspection.split_mod_fun_call + |> Introspection.actual_mod_fun(imports, aliases, module) + |> find_source() + end + + defp find_source({mod, fun}) do + mod + |> find_mod_file() + |> find_fun_line(fun) + end + + defp find_mod_file(module) do + file = if Code.ensure_loaded? module do + case module.module_info(:compile)[:source] do + nil -> nil + source -> List.to_string(source) + end + end + file = if file && File.exists?(file) do + file + else + erl_file = module |> :code.which |> to_string |> String.replace(~r/(.+)\/ebin\/([^\s]+)\.beam$/, "\\1/src/\\2.erl") + if File.exists?(erl_file) do + erl_file + end + end + {module, file} + end + + defp find_fun_line({_, file}, _fun) when file in ["non_existing", nil, ""] do + {"non_existing", nil} + end + + defp find_fun_line({mod, file}, fun) do + line = if String.ends_with?(file, ".erl") do + find_fun_line_in_erl_file(file, fun) + else + file_metadata = Parser.parse_file(file, false, false, nil) + Metadata.get_function_line(file_metadata, mod, fun) + end + {file, line} + end + + defp find_fun_line_in_erl_file(file, fun) do + fun_name = Atom.to_string(fun) + index = + file + |> File.read! + |> String.split(["\n", "\r\n"]) + |> Enum.find_index(&String.match?(&1, ~r/^#{fun_name}\b/)) + + (index || 0) + 1 + end + +end diff --git a/elixir_sense/lib/elixir_sense/providers/docs.ex b/elixir_sense/lib/elixir_sense/providers/docs.ex new file mode 100644 index 0000000..7e32df7 --- /dev/null +++ b/elixir_sense/lib/elixir_sense/providers/docs.ex @@ -0,0 +1,28 @@ +defmodule ElixirSense.Providers.Docs do + @moduledoc """ + Doc Provider + """ + alias ElixirSense.Core.Introspection + + @spec all(String.t, [module], [{module, module}], module) :: {actual_mod_fun :: String.t, docs :: Introspection.docs} + def all(subject, imports, aliases, module) do + mod_fun = + subject + |> Introspection.split_mod_fun_call + |> Introspection.actual_mod_fun(imports, aliases, module) + {mod_fun_to_string(mod_fun), Introspection.get_all_docs(mod_fun)} + end + + defp mod_fun_to_string({nil, fun}) do + Atom.to_string(fun) + end + + defp mod_fun_to_string({mod, nil}) do + Introspection.module_to_string(mod) + end + + defp mod_fun_to_string({mod, fun}) do + Introspection.module_to_string(mod) <> "." <> Atom.to_string(fun) + end + +end diff --git a/elixir_sense/lib/elixir_sense/providers/eval.ex b/elixir_sense/lib/elixir_sense/providers/eval.ex new file mode 100644 index 0000000..6aa1ef2 --- /dev/null +++ b/elixir_sense/lib/elixir_sense/providers/eval.ex @@ -0,0 +1,94 @@ +defmodule ElixirSense.Providers.Eval do + + @moduledoc """ + Provider responsible for evaluating Elixr expressions. + """ + + alias ElixirSense.Core.Introspection + + @type binding :: {name :: String.t, value :: String.t} + @type bindings :: [binding] | :no_match | {:error, message :: String.t} + + @doc """ + Converts a string to its quoted form. + """ + def quote(code) do + code + |> Code.string_to_quoted + |> Tuple.to_list + |> List.last + |> inspect + end + + @doc """ + Evaluate a pattern matching expression and returns its bindings, if any. + """ + @spec match(String.t) :: bindings + def match(code) do + try do + {:=, _, [pattern|_]} = code |> Code.string_to_quoted! + vars = extract_vars(pattern) + + bindings = + code + |> Code.eval_string + |> Tuple.to_list + |> List.last + + Enum.map(vars, fn var -> + {var, Keyword.get(bindings, var)} + end) + rescue + MatchError -> + :no_match + e -> + %{__struct__: type, description: description, line: line} = e + {:error, "# #{Introspection.module_to_string(type)} on line #{line}:\n# ↳ #{description}"} + end + end + + @doc """ + Evaluate a pattern matching expression using `ElixirSense.Providers.Eval.match/1` + and format the results. + """ + @spec match_and_format(String.t) :: bindings + def match_and_format(code) do + case match(code) do + :no_match -> + "# No match" + {:error, message} -> + message + bindings -> + bindings_to_string(bindings) + end + end + + defp bindings_to_string(bindings) do + header = + if Enum.empty?(bindings) do + "# No bindings" + else + "# Bindings" + end + + body = + Enum.map_join(bindings, "\n\n", fn {var, val} -> + "#{var} = #{inspect(val)}" + end) + header <> "\n\n" <> body + end + + defp extract_vars(ast) do + {_ast, acc} = Macro.postwalk(ast, [], &extract_var/2) + acc |> Enum.reverse + end + + defp extract_var(ast = {var_name, [line: _], nil}, acc) when is_atom(var_name) and var_name != :_ do + {ast, [var_name|acc]} + end + + defp extract_var(ast, acc) do + {ast, acc} + end + +end diff --git a/elixir_sense/lib/elixir_sense/providers/expand.ex b/elixir_sense/lib/elixir_sense/providers/expand.ex new file mode 100644 index 0000000..4f26a79 --- /dev/null +++ b/elixir_sense/lib/elixir_sense/providers/expand.ex @@ -0,0 +1,47 @@ +defmodule ElixirSense.Providers.Expand do + + @moduledoc """ + Provider responsible for code expansion features. + """ + + alias ElixirSense.Core.Ast + + @type expanded_code_map :: %{ + expand_once: String.t, + expand: String.t, + expand_partial: String.t, + expand_all: String.t, + } + + @doc """ + Returns a map containing the results of all different code expansion methods + available (expand_once, expand, expand_partial and expand_all). + """ + @spec expand_full(String.t, [module], [module], module) :: expanded_code_map + def expand_full(code, requires, imports, module) do + env = + __ENV__ + |> Ast.add_requires_to_env(requires) + |> Ast.add_imports_to_env(imports) + |> Ast.set_module_for_env(module) + + try do + {_, expr} = code |> Code.string_to_quoted + %{ + expand_once: expr |> Macro.expand_once(env) |> Macro.to_string, + expand: expr |> Macro.expand(env) |> Macro.to_string, + expand_partial: expr |> Ast.expand_partial(env) |> Macro.to_string, + expand_all: expr |> Ast.expand_all(env) |> Macro.to_string, + } + rescue + e -> + message = inspect(e) + %{ + expand_once: message, + expand: message, + expand_partial: message, + expand_all: message, + } + end + end +end diff --git a/elixir_sense/lib/elixir_sense/providers/signature.ex b/elixir_sense/lib/elixir_sense/providers/signature.ex new file mode 100644 index 0000000..e046348 --- /dev/null +++ b/elixir_sense/lib/elixir_sense/providers/signature.ex @@ -0,0 +1,38 @@ +defmodule ElixirSense.Providers.Signature do + + @moduledoc """ + Provider responsible for introspection information about function signatures. + """ + + alias ElixirSense.Core.Introspection + alias ElixirSense.Core.Source + alias ElixirSense.Core.Metadata + + @type signature :: %{name: String.t, params: [String.t]} + @type signature_info :: %{active_param: pos_integer, signatures: [signature]} | :none + + @doc """ + Returns the signature info from the function defined in the prefix, if any. + """ + @spec find(String.t, [module], [{module, module}], module, map) :: signature_info + def find(prefix, imports, aliases, module, metadata) do + case Source.which_func(prefix) do + %{candidate: {mod, fun}, npar: npar, pipe_before: pipe_before} -> + {mod, fun} = Introspection.actual_mod_fun({mod, fun}, imports, aliases, module) + signatures = find_signatures({mod, fun}, metadata) + %{active_param: npar, pipe_before: pipe_before, signatures: signatures} + _ -> + :none + end + end + + defp find_signatures({mod, fun}, metadata) do + docs = Code.get_docs(mod, :docs) + signatures = case Metadata.get_function_signatures(metadata, mod, fun, docs) do + [] -> Introspection.get_signatures(mod, fun, docs) + signatures -> signatures + end + signatures |> Enum.uniq_by(fn sig -> sig.params end) + end + +end diff --git a/elixir_sense/lib/elixir_sense/providers/suggestion.ex b/elixir_sense/lib/elixir_sense/providers/suggestion.ex new file mode 100644 index 0000000..486d2d6 --- /dev/null +++ b/elixir_sense/lib/elixir_sense/providers/suggestion.ex @@ -0,0 +1,150 @@ +defmodule ElixirSense.Providers.Suggestion do + + @moduledoc """ + Provider responsible for finding suggestions for auto-completing + """ + + alias Alchemist.Helpers.Complete + alias ElixirSense.Core.Introspection + + @type fun_arity :: {atom, non_neg_integer} + @type scope :: module | fun_arity + + @type attribute :: %{ + type: :attribute, + name: String.t + } + + @type variable :: %{ + type: :var, + name: String.t + } + + @type return :: %{ + type: :return, + description: String.t, + spec: String.t, + snippet: String.t, + } + + @type callback :: %{ + type: :callback, + name: String.t, + arity: non_neg_integer, + args: String.t, + origin: String.t, + summary: String.t, + spec: String.t + } + + @type func :: %{ + type: :function, + name: String.t, + arity: non_neg_integer, + args: String.t, + origin: String.t, + summary: String.t, + spec: String.t + } + + @type mod :: %{ + type: :module, + name: String.t, + subtype: String.t, + summary: String.t + } + + @type hint :: %{ + type: :hint, + value: String.t + } + + @type suggestion :: attribute + | variable + | return + | callback + | func + | mod + | hint + + @doc """ + Finds all suggestions for a hint based on context information. + """ + @spec find(String.t, [module], [{module, module}], [String.t], [String.t], [module], scope) :: [suggestion] + def find(hint, imports, aliases, vars, attributes, behaviours, scope) do + %{hint: hint_suggestion, suggestions: mods_and_funcs} = find_hint_mods_funcs(hint, imports, aliases) + + callbacks_or_returns = + case scope do + {_f, _a} -> find_returns(behaviours, hint, scope) + _mod -> find_callbacks(behaviours, hint) + end + + [hint_suggestion] + |> Kernel.++(callbacks_or_returns) + |> Kernel.++(find_attributes(attributes, hint)) + |> Kernel.++(find_vars(vars, hint)) + |> Kernel.++(mods_and_funcs) + |> Enum.uniq_by(&(&1)) + end + + @spec find_hint_mods_funcs(String.t, [module], [{module, module}]) :: %{hint: hint, suggestions: [mod | func]} + defp find_hint_mods_funcs(hint, imports, aliases) do + Application.put_env(:"alchemist.el", :aliases, aliases) + + list1 = Complete.run(hint, imports) + list2 = Complete.run(hint) + + {hint_suggestion, suggestions} = + case List.first(list2) do + %{type: :hint} = sug -> + {sug, list1 ++ List.delete_at(list2, 0)} + _ -> + {%{type: :hint, value: "#{hint}"}, list1 ++ list2} + end + + %{hint: hint_suggestion, suggestions: suggestions} + end + + @spec find_vars([String.t], String.t) :: [variable] + defp find_vars(vars, hint) do + for var <- vars, hint == "" or String.starts_with?("#{var}", hint) do + %{type: :variable, name: var} + end |> Enum.sort + end + + @spec find_attributes([String.t], String.t) :: [attribute] + defp find_attributes(attributes, hint) do + for attribute <- attributes, hint in ["", "@"] or String.starts_with?("@#{attribute}", hint) do + %{type: :attribute, name: "@#{attribute}"} + end |> Enum.sort + end + + @spec find_returns([module], String.t, scope) :: [return] + defp find_returns(behaviours, "", {fun, arity}) do + for mod <- behaviours, Introspection.define_callback?(mod, fun, arity) do + for return <- Introspection.get_returns_from_callback(mod, fun, arity) do + %{type: :return, description: return.description, spec: return.spec, snippet: return.snippet} + end + end |> List.flatten + end + defp find_returns(_behaviours, _hint, _module) do + [] + end + + @spec find_callbacks([module], String.t) :: [callback] + defp find_callbacks(behaviours, hint) do + behaviours |> Enum.flat_map(fn mod -> + mod_name = mod |> Introspection.module_to_string + for %{name: name, arity: arity, callback: spec, signature: signature, doc: doc} <- Introspection.get_callbacks_with_docs(mod), + hint == "" or String.starts_with?("#{name}", hint) + do + desc = Introspection.extract_summary_from_docs(doc) + [_, args_str] = Regex.run(~r/.\((.*)\)/, signature) + args = args_str |> String.replace(~r/\s/, "") + %{type: :callback, name: name, arity: arity, args: args, origin: mod_name, summary: desc, spec: spec} + end + end) |> Enum.sort + end + +end diff --git a/elixir_sense/lib/elixir_sense/server.ex b/elixir_sense/lib/elixir_sense/server.ex new file mode 100644 index 0000000..13c65bc --- /dev/null +++ b/elixir_sense/lib/elixir_sense/server.ex @@ -0,0 +1,38 @@ +defmodule ElixirSense.Server do + @moduledoc """ + Server entry point and coordinator + """ + + alias ElixirSense.Server.TCPServer + + def start(args) do + [socket_type, port, env] = validate_args(args) + IO.puts(:stderr, "Initializing ElixirSense server for environment \"#{env}\" (Elixir version #{System.version})") + IO.puts(:stderr, "Working directory is \"#{Path.expand(".")}\"") + TCPServer.start([socket_type: socket_type, port: port, env: env]) + loop() + end + + defp validate_args(["unix", _port, env] = args) do + {version, _} = :otp_release |> :erlang.system_info() |> :string.to_integer() + if version < 19 do + IO.puts(:stderr, "Warning: Erlang version < 19. Cannot use Unix domain sockets. Using tcp/ip instead.") + ["tcpip", "0", env] + else + args + end + end + defp validate_args(args) do + args + end + + defp loop do + case IO.gets("") do + :eof -> + IO.puts(:stderr, "Stopping ElixirSense server") + _ -> + loop() + end + end + +end diff --git a/elixir_sense/lib/elixir_sense/server/context_loader.ex b/elixir_sense/lib/elixir_sense/server/context_loader.ex new file mode 100644 index 0000000..f98f48d --- /dev/null +++ b/elixir_sense/lib/elixir_sense/server/context_loader.ex @@ -0,0 +1,95 @@ +defmodule ElixirSense.Server.ContextLoader do + @moduledoc """ + Server Context Loader + """ + use GenServer + + @minimal_reload_time 2000 + + def start_link(env) do + GenServer.start_link(__MODULE__, env, [name: __MODULE__]) + end + + def init(env) do + {:ok, {all_loaded(), [], [], env, Path.expand("."), 0}} + end + + def set_context(env, cwd) do + GenServer.call(__MODULE__, {:set_context, {env, cwd}}) + end + + def get_state do + GenServer.call(__MODULE__, :get_state) + end + + def reload do + GenServer.call(__MODULE__, :reload) + end + + def handle_call(:reload, _from, {loaded, paths, apps, env, cwd, last_load_time}) do + time = :erlang.system_time(:milli_seconds) + reload = time - last_load_time > @minimal_reload_time + + {new_paths, new_apps} = + if reload do + purge_modules(loaded) + purge_paths(paths) + purge_apps(apps) + {load_paths(env, cwd), load_apps(env, cwd)} + else + {paths, apps} + end + + {:reply, :ok, {loaded, new_paths, new_apps, env, cwd, time}} + end + + def handle_call({:set_context, {env, cwd}}, _from, {loaded, paths, apps, _env, _cwd, last_load_time}) do + {:reply, {env, cwd}, {loaded, paths, apps, env, cwd, last_load_time}} + end + + def handle_call(:get_state, _from, state) do + {:reply, state, state} + end + + defp preload_modules(modules) do + modules |> Enum.each(fn mod -> + {:module, _} = Code.ensure_loaded(mod) + end) + end + + defp all_loaded do + preload_modules([Inspect, :base64, :crypto]) + for {m, _} <- :code.all_loaded, do: m + end + + defp load_paths(env, cwd) do + for path <- Path.wildcard(Path.join(cwd, "_build/#{env}/lib/*/ebin")) do + Code.prepend_path(path) + path + end + end + + defp load_apps(env, cwd) do + for path <- Path.wildcard(Path.join(cwd, "_build/#{env}/lib/*/ebin/*.app")) do + app = path |> Path.basename() |> Path.rootname() |> String.to_atom + Application.load(app) + app + end + end + + defp purge_modules(loaded) do + for m <- (all_loaded() -- loaded) do + :code.delete(m) + :code.purge(m) + end + end + + defp purge_paths(paths) do + for p <- paths, do: Code.delete_path(p) + end + + defp purge_apps(apps) do + for a <- apps, do: Application.unload(a) + end + +end diff --git a/elixir_sense/lib/elixir_sense/server/request_handler.ex b/elixir_sense/lib/elixir_sense/server/request_handler.ex new file mode 100644 index 0000000..e41d3ee --- /dev/null +++ b/elixir_sense/lib/elixir_sense/server/request_handler.ex @@ -0,0 +1,53 @@ +defmodule ElixirSense.Server.RequestHandler do + + @moduledoc """ + Handles all requests received by the TCP Server and maps those requests to ElixirSense API calls. + """ + + alias ElixirSense.Server.ContextLoader + + def handle_request("signature", %{"buffer" => buffer, "line" => line, "column" => column}) do + ElixirSense.signature(buffer, line, column) + end + + def handle_request("docs", %{"buffer" => buffer, "line" => line, "column" => column}) do + ElixirSense.docs(buffer, line, column) + end + + def handle_request("definition", %{"buffer" => buffer, "line" => line, "column" => column}) do + case ElixirSense.definition(buffer, line, column) do + {"non_existing", nil} -> "non_existing:0" + {file, nil} -> "#{file}:0" + {file, line} -> "#{file}:#{line}" + end + end + + def handle_request("suggestions", %{"buffer" => buffer, "line" => line, "column" => column}) do + ElixirSense.suggestions(buffer, line, column) + end + + def handle_request("expand_full", %{"buffer" => buffer, "selected_code" => selected_code, "line" => line}) do + ElixirSense.expand_full(buffer, selected_code, line) + end + + def handle_request("quote", %{"code" => code}) do + ElixirSense.quote(code) + end + + def handle_request("match", %{"code" => code}) do + ElixirSense.match(code) + end + + def handle_request("all_modules", %{}) do + ElixirSense.all_modules() + end + + def handle_request("set_context", %{"env" => env, "cwd" => cwd}) do + env |> ContextLoader.set_context(cwd) |> Tuple.to_list() + end + + def handle_request(request, payload) do + IO.puts :stderr, "Cannot handle request \"#{request}\". Payload: #{inspect(payload)}" + end + +end diff --git a/elixir_sense/lib/elixir_sense/server/tcp_server.ex b/elixir_sense/lib/elixir_sense/server/tcp_server.ex new file mode 100644 index 0000000..ac627e3 --- /dev/null +++ b/elixir_sense/lib/elixir_sense/server/tcp_server.ex @@ -0,0 +1,145 @@ +defmodule ElixirSense.Server.TCPServer do + @moduledoc """ + TCP Server connection endpoint + """ + use Bitwise + + alias ElixirSense.Server.{RequestHandler, ContextLoader} + + @connection_handler_supervisor ElixirSense.Server.TCPServer.ConnectionHandlerSupervisor + @default_listen_options [:binary, active: false, reuseaddr: true, packet: 4] + + def start([socket_type: socket_type, port: port, env: env]) do + import Supervisor.Spec + + children = [ + worker(Task, [__MODULE__, :listen, [socket_type, "localhost", port]]), + supervisor(Task.Supervisor, [[name: @connection_handler_supervisor]]), + worker(ContextLoader, [env]) + ] + + opts = [strategy: :one_for_one, name: __MODULE__] + Supervisor.start_link(children, opts) + end + + def listen(socket_type, host, port) do + {port_or_file, opts} = listen_options(socket_type, port) + {:ok, socket} = :gen_tcp.listen(port_or_file, opts) + {:ok, port_or_file} = :inet.port(socket) + auth_token = create_auth_token(socket_type) + + socket_type + |> format_output(host, port_or_file, auth_token) + |> IO.puts + + accept(socket, auth_token) + end + + defp create_auth_token("tcpip") do + :base64.encode(:crypto.strong_rand_bytes(64)) + end + defp create_auth_token("unix") do + nil + end + + defp format_output("tcpip", host, port, auth_token) do + "ok:#{host}:#{port}:#{auth_token}" + end + + defp format_output("unix", host, file, _auth_token) do + "ok:#{host}:#{file}" + end + + defp listen_options("tcpip", port) do + {String.to_integer(port), @default_listen_options ++ [ip: {127, 0, 0, 1}]} + end + + defp listen_options("unix", _port) do + {0, @default_listen_options ++ [ifaddr: {:local, socket_file()}]} + end + + defp accept(socket, auth_token) do + {:ok, client_socket} = :gen_tcp.accept(socket) + {:ok, pid} = start_connection_handler(client_socket, auth_token) + :ok = :gen_tcp.controlling_process(client_socket, pid) + + accept(socket, auth_token) + end + + defp start_connection_handler(client_socket, auth_token) do + Task.Supervisor.start_child(@connection_handler_supervisor, fn -> + connection_handler(client_socket, auth_token) + end) + end + + defp connection_handler(socket, auth_token) do + case :gen_tcp.recv(socket, 0) do + {:error, :closed} -> + IO.puts :stderr, "Client socket is closed" + {:ok, data} -> + data + |> process_request(auth_token) + |> send_response(socket) + connection_handler(socket, auth_token) + end + end + + defp process_request(data, auth_token) do + try do + data + |> :erlang.binary_to_term() + |> dispatch_request(auth_token) + |> :erlang.term_to_binary() + rescue + e -> + IO.puts(:stderr, "Server Error: \n" <> Exception.message(e) <> "\n" <> Exception.format_stacktrace(System.stacktrace)) + :erlang.term_to_binary(%{request_id: nil, payload: nil, error: Exception.message(e)}) + catch + e -> + error = "Uncaught value #{inspect(e)}" + IO.puts(:stderr, "Server Error: #{error}\n" <> Exception.format_stacktrace(System.stacktrace)) + :erlang.term_to_binary(%{request_id: nil, payload: nil, error: error}) + end + end + + defp dispatch_request(%{ + "request_id" => request_id, + "auth_token" => req_token, + "request" => request, + "payload" => payload}, auth_token) do + if secure_compare(auth_token, req_token) do + ContextLoader.reload() + payload = RequestHandler.handle_request(request, payload) + %{request_id: request_id, payload: payload, error: nil} + else + %{request_id: request_id, payload: nil, error: "unauthorized"} + end + end + + defp send_response(data, socket) do + :gen_tcp.send(socket, data) + end + + defp socket_file do + sock_id = :erlang.system_time() + String.to_charlist("/tmp/elixir-sense-#{sock_id}.sock") + end + + # Adapted from https://github.com/plackemacher/secure_compare/blob/master/lib/secure_compare.ex + defp secure_compare(nil, nil), do: true + defp secure_compare(a, b) when is_nil(a) or is_nil(b), do: false + defp secure_compare(a, b) when byte_size(a) != byte_size(b), do: false + defp secure_compare(a, b) when is_binary(a) and is_binary(b) do + a_list = String.to_charlist(a) + b_list = String.to_charlist(b) + secure_compare(a_list, b_list) + end + defp secure_compare(a, b) when is_list(a) and is_list(b) do + res = a + |> Enum.zip(b) + |> Enum.reduce(0, fn({a_byte, b_byte}, acc) -> + acc ||| bxor(a_byte, b_byte) + end) + res == 0 + end +end diff --git a/elixir_sense/mix.exs b/elixir_sense/mix.exs new file mode 100644 index 0000000..5c2b975 --- /dev/null +++ b/elixir_sense/mix.exs @@ -0,0 +1,52 @@ +defmodule ElixirSense.Mixfile do + @moduledoc false + use Mix.Project + + def project do + [app: :elixir_sense, + version: "0.2.0", + elixir: "~> 1.5", + build_embedded: Mix.env == :prod, + start_permanent: Mix.env == :prod, + test_coverage: [tool: ExCoveralls], + preferred_cli_env: ["coveralls": :test, "coveralls.detail": :test, "coveralls.html": :test], + dialyzer: [ + flags: ["-Wunmatched_returns", "-Werror_handling", "-Wrace_conditions", "-Wunderspecs", "-Wno_match"] + ], + deps: deps(), + docs: docs(), + description: description(), + package: package(), + ] + end + + def application do + [applications: [:logger]] + end + + defp deps do + [{:excoveralls, "~> 0.6", only: :test}, + {:dialyxir, "~> 0.4", only: [:dev]}, + {:credo, "~> 0.8.4", only: [:dev]}, + {:ex_doc, "~> 0.14", only: [:dev]}] + end + + defp docs do + [main: "ElixirSense"] + end + + defp description do + """ + An API/Server for Elixir projects that provides context-aware information + for code completion, documentation, go/jump to definition, signature info + and more. + """ + end + + defp package do + [maintainers: ["Marlus Saraiva (@msaraiva)"], + licenses: ["Apache 2.0"], + links: %{"GitHub" => "https://github.com/msaraiva/elixir_sense"}] + end + +end diff --git a/elixir_sense/mix.lock b/elixir_sense/mix.lock new file mode 100644 index 0000000..1354392 --- /dev/null +++ b/elixir_sense/mix.lock @@ -0,0 +1,14 @@ +%{"bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [], [], "hexpm"}, + "certifi": {:hex, :certifi, "0.7.0", "861a57f3808f7eb0c2d1802afeaae0fa5de813b0df0979153cbafcd853ababaf", [:rebar3], []}, + "credo": {:hex, :credo, "0.8.4", "4e50acac058cf6292d6066e5b0d03da5e1483702e1ccde39abba385c9f03ead4", [], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}], "hexpm"}, + "dialyxir": {:hex, :dialyxir, "0.4.3", "a4daeebd0107de10d3bbae2ccb6b8905e69544db1ed5fe9148ad27cd4cb2c0cd", [:mix], []}, + "earmark": {:hex, :earmark, "1.1.0", "8c2bf85d725050a92042bc1edf362621004d43ca6241c756f39612084e95487f", [:mix], []}, + "ex_doc": {:hex, :ex_doc, "0.14.5", "c0433c8117e948404d93ca69411dd575ec6be39b47802e81ca8d91017a0cf83c", [:mix], [{:earmark, "~> 1.0", [hex: :earmark, optional: false]}]}, + "excoveralls": {:hex, :excoveralls, "0.6.1", "9e946b6db84dba592f47632157ecd135a46384b98a430fd16007dc910c70348b", [:mix], [{:exjsx, "~> 3.0", [hex: :exjsx, optional: false]}, {:hackney, ">= 0.12.0", [hex: :hackney, optional: false]}]}, + "exjsx": {:hex, :exjsx, "3.2.1", "1bc5bf1e4fd249104178f0885030bcd75a4526f4d2a1e976f4b428d347614f0f", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, optional: false]}]}, + "hackney": {:hex, :hackney, "1.6.5", "8c025ee397ac94a184b0743c73b33b96465e85f90a02e210e86df6cbafaa5065", [:rebar3], [{:certifi, "0.7.0", [hex: :certifi, optional: false]}, {:idna, "1.2.0", [hex: :idna, optional: false]}, {:metrics, "1.0.1", [hex: :metrics, optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, optional: false]}]}, + "idna": {:hex, :idna, "1.2.0", "ac62ee99da068f43c50dc69acf700e03a62a348360126260e87f2b54eced86b2", [:rebar3], []}, + "jsx": {:hex, :jsx, "2.8.1", "1453b4eb3615acb3e2cd0a105d27e6761e2ed2e501ac0b390f5bbec497669846", [:mix, :rebar3], []}, + "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], []}, + "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], []}, + "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.1", "28a4d65b7f59893bc2c7de786dec1e1555bd742d336043fe644ae956c3497fbe", [:make, :rebar], []}} diff --git a/elixir_sense/run.exs b/elixir_sense/run.exs new file mode 100644 index 0000000..40dfde7 --- /dev/null +++ b/elixir_sense/run.exs @@ -0,0 +1,28 @@ +requires = [ + "elixir_sense/core/introspection.ex", + "elixir_sense/core/ast.ex", + "elixir_sense/core/state.ex", + "elixir_sense/core/metadata_builder.ex", + "elixir_sense/core/metadata.ex", + "elixir_sense/core/parser.ex", + "elixir_sense/core/source.ex", + "alchemist/helpers/module_info.ex", + "alchemist/helpers/complete.ex", + "elixir_sense/providers/definition.ex", + "elixir_sense/providers/docs.ex", + "elixir_sense/providers/suggestion.ex", + "elixir_sense/providers/signature.ex", + "elixir_sense/providers/expand.ex", + "elixir_sense/providers/eval.ex", + "elixir_sense/server/request_handler.ex", + "elixir_sense/server/context_loader.ex", + "elixir_sense/server/tcp_server.ex", + "elixir_sense.ex", + "elixir_sense/server.ex" +] + +requires |> Enum.each(fn file -> + Code.require_file("lib/#{file}", __DIR__) +end) + +ElixirSense.Server.start(System.argv) diff --git a/elixir_sense/run_test.exs b/elixir_sense/run_test.exs new file mode 100644 index 0000000..c2c53ea --- /dev/null +++ b/elixir_sense/run_test.exs @@ -0,0 +1,6 @@ +Code.require_file "run.exs", __DIR__ +ExUnit.start() + +for path <- Path.wildcard(Path.join(__DIR__, "/test/**/*.exs")) do + Code.require_file path +end diff --git a/elixir_sense/t.exs b/elixir_sense/t.exs new file mode 100644 index 0000000..0547410 --- /dev/null +++ b/elixir_sense/t.exs @@ -0,0 +1,33 @@ +socket = '/tmp/elixir-sense-1500874896440962000.sock' +{:ok, socket} = :gen_tcp.connect({:local, socket}, 0, [:binary, active: false, packet: 4]) + +code = """ +defmodule MyModule do + alias List, as: MyList + List.flatten + Interface.UserService +end +""" + +scode = """ +defmodule MyModule do + import List +end +""" + +request = %{ + "request_id" => 3, + "auth_token" => nil, + "request" => "definition", + "payload" => %{ + "buffer" => code, + "line" => 3, + "column" => 6 + } +} + +data = :erlang.term_to_binary(request) +:ok = :gen_tcp.send(socket, data) +{:ok, response} = :gen_tcp.recv(socket, 0) +:erlang.binary_to_term(response) +|> IO.inspect diff --git a/elixir_sense/test/alchemist/helpers/module_info_test.exs b/elixir_sense/test/alchemist/helpers/module_info_test.exs new file mode 100644 index 0000000..eda1397 --- /dev/null +++ b/elixir_sense/test/alchemist/helpers/module_info_test.exs @@ -0,0 +1,42 @@ +defmodule Alchemist.Helpers.ModuleTest do + + use ExUnit.Case + + alias Alchemist.Helpers.ModuleInfo + + test "moduledoc? returns true" do + assert ModuleInfo.moduledoc?(List) == true + end + + test "moduledoc? returns false" do + assert ModuleInfo.moduledoc?(List.Chars.Atom) == false + end + + test "docs? returns true" do + assert ModuleInfo.docs?(List, :flatten) == true + assert ModuleInfo.docs?(Kernel, :def) == true + end + + test "docs? returns false" do + assert ModuleInfo.docs?(List, :dance) == false + assert ModuleInfo.docs?(nil, :dance) == false + end + + test "expand_alias return expanded module alias" do + aliases = [{MyList, List}, {MyGenServer, :gen_server}] + + assert ModuleInfo.expand_alias([MyList], aliases) == List + assert ModuleInfo.expand_alias([MyGenServer], aliases) == :gen_server + assert ModuleInfo.expand_alias([MyList], aliases) == List + end + + test "has_function? return true" do + assert ModuleInfo.has_function?(List, :flatten) == true + assert ModuleInfo.has_function?(List, :to_string) == true + end + + test "has_function? return false" do + assert ModuleInfo.has_function?(List, :split) == false + assert ModuleInfo.has_function?(List, :map) == false + end +end diff --git a/elixir_sense/test/elixir_sense/all_modules_test.exs b/elixir_sense/test/elixir_sense/all_modules_test.exs new file mode 100644 index 0000000..3d15acd --- /dev/null +++ b/elixir_sense/test/elixir_sense/all_modules_test.exs @@ -0,0 +1,16 @@ +defmodule ElixirSense.Providers.ModulesTest do + + use ExUnit.Case + alias ElixirSense.Providers.Definition + + doctest Definition + + test "test all modules available modules are listed" do + modules = ElixirSense.all_modules() + assert "ElixirSense" in modules + assert not "ElixirSense.Providers" in modules + assert "ElixirSense.Providers.Definition" in modules + assert ":kernel" in modules + end + +end diff --git a/elixir_sense/test/elixir_sense/core/ast_test.exs b/elixir_sense/test/elixir_sense/core/ast_test.exs new file mode 100644 index 0000000..6857160 --- /dev/null +++ b/elixir_sense/test/elixir_sense/core/ast_test.exs @@ -0,0 +1,32 @@ +defmodule ElixirSense.Core.AstTest do + + use ExUnit.Case + alias ElixirSense.Core.Ast + + defmodule ExpandRecursive do + defmacro my_macro do + quote do + my_macro = "Hi" + end + end + end + + test "expand_partial cannot expand recursive macros" do + import ExpandRecursive + result = + quote do + my_macro() + end |> Ast.expand_partial(__ENV__) + assert result == {:expand_error, "Cannot expand recursive macro"} + end + + test "expand_all cannot expand recursive macros" do + import ExpandRecursive + result = + quote do + my_macro() + end |> Ast.expand_all(__ENV__) + assert result == {:expand_error, "Cannot expand recursive macro"} + end + +end diff --git a/elixir_sense/test/elixir_sense/core/introspection_test.exs b/elixir_sense/test/elixir_sense/core/introspection_test.exs new file mode 100644 index 0000000..4e4516a --- /dev/null +++ b/elixir_sense/test/elixir_sense/core/introspection_test.exs @@ -0,0 +1,106 @@ +defmodule ElixirSense.Core.IntrospectionTest do + + use ExUnit.Case + + import ElixirSense.Core.Introspection + + test "get_callbacks_with_docs for erlang behaviours" do + assert get_callbacks_with_docs(:supervisor) == [%{ + name: :init, + arity: 1, + callback: """ + @callback init(args :: term) :: + {:ok, {supFlags :: sup_flags, [childSpec :: child_spec]}} | + :ignore + """, + signature: "init(args)", + doc: nil + }] + end + + test "get_callbacks_with_docs for Elixir behaviours with no docs defined" do + assert get_callbacks_with_docs(Exception) == [ + %{name: :exception, arity: 1, callback: "@callback exception(term) :: t\n", signature: "exception(term)", doc: nil}, + %{name: :message, arity: 1, callback: "@callback message(t) :: String.t\n", signature: "message(t)", doc: nil} + ] + end + + test "get_callbacks_with_docs for Elixir behaviours with docs defined" do + info = get_callbacks_with_docs(GenServer) |> Enum.at(0) + + assert info.name == :code_change + assert info.arity == 3 + assert info.callback == """ + @callback code_change(old_vsn, state :: term, extra :: term) :: + {:ok, new_state :: term} | + {:error, reason :: term} when old_vsn: term | {:down, term} + """ + assert info.doc =~ "Invoked to change the state of the `GenServer`" + assert info.signature == "code_change(old_vsn, state, extra)" + end + + test "format_spec_ast with one return option does not aplit the returns" do + type_ast = get_type_ast(GenServer, :debug) + + assert format_spec_ast(type_ast) == """ + debug :: [:trace | :log | :statistics | {:log_to_file, Path.t}] + """ + end + + test "format_spec_ast with more than one return option aplits the returns" do + type_ast = get_type_ast(GenServer, :on_start) + + assert format_spec_ast(type_ast) == """ + on_start :: + {:ok, pid} | + :ignore | + {:error, {:already_started, pid} | term} + """ + end + + test "format_spec_ast for callback" do + ast = get_callback_ast(GenServer, :code_change, 3) + assert format_spec_ast(ast) == """ + code_change(old_vsn, state :: term, extra :: term) :: + {:ok, new_state :: term} | + {:error, reason :: term} when old_vsn: term | {:down, term} + """ + end + + test "get_returns_from_callback" do + returns = get_returns_from_callback(GenServer, :code_change, 3) + assert returns == [ + %{description: "{:ok, new_state}", snippet: "{:ok, \"${1:new_state}$\"}", spec: "{:ok, new_state :: term} when old_vsn: term | {:down, term}"}, + %{description: "{:error, reason}", snippet: "{:error, \"${1:reason}$\"}", spec: "{:error, reason :: term} when old_vsn: term | {:down, term}"} + ] + end + + test "get_returns_from_callback (all types in 'when')" do + returns = get_returns_from_callback(:gen_server, :handle_call, 3) + assert returns == [ + %{description: "{:reply, reply, new_state}", snippet: "{:reply, \"${1:reply}$\", \"${2:new_state}$\"}", spec: "{:reply, reply, new_state} when reply: term, new_state: term, reason: term"}, + %{description: "{:reply, reply, new_state, timeout | :hibernate}", snippet: "{:reply, \"${1:reply}$\", \"${2:new_state}$\", \"${3:timeout | :hibernate}$\"}", spec: "{:reply, reply, new_state, timeout | :hibernate} when reply: term, new_state: term, reason: term"}, + %{description: "{:noreply, new_state}", snippet: "{:noreply, \"${1:new_state}$\"}", spec: "{:noreply, new_state} when reply: term, new_state: term, reason: term"}, + %{description: "{:noreply, new_state, timeout | :hibernate}", snippet: "{:noreply, \"${1:new_state}$\", \"${2:timeout | :hibernate}$\"}", spec: "{:noreply, new_state, timeout | :hibernate} when reply: term, new_state: term, reason: term"}, + %{description: "{:stop, reason, reply, new_state}", snippet: "{:stop, \"${1:reason}$\", \"${2:reply}$\", \"${3:new_state}$\"}", spec: "{:stop, reason, reply, new_state} when reply: term, new_state: term, reason: term"}, + %{description: "{:stop, reason, new_state}", snippet: "{:stop, \"${1:reason}$\", \"${2:new_state}$\"}", spec: "{:stop, reason, new_state} when reply: term, new_state: term, reason: term"} + ] + end + + test "get_returns_from_callback (erlang specs)" do + returns = get_returns_from_callback(:gen_fsm, :handle_event, 3) + assert returns == [ + %{description: "{:next_state, nextStateName, newStateData}", snippet: "{:next_state, \"${1:nextStateName}$\", \"${2:newStateData}$\"}", spec: "{:next_state, nextStateName :: atom, newStateData :: term}"}, + %{description: "{:next_state, nextStateName, newStateData, timeout | :hibernate}", snippet: "{:next_state, \"${1:nextStateName}$\", \"${2:newStateData}$\", \"${3:timeout | :hibernate}$\"}", spec: "{:next_state, nextStateName :: atom, newStateData :: term, timeout | :hibernate}"}, + %{description: "{:stop, reason, newStateData}", snippet: "{:stop, \"${1:reason}$\", \"${2:newStateData}$\"}", spec: "{:stop, reason :: term, newStateData :: term}"} + ] + end + + defp get_type_ast(module, type) do + {_kind, type} = + Kernel.Typespec.beam_types(module) + |> Enum.find(fn {_, {name, _, _}} -> name == type end) + Kernel.Typespec.type_to_ast(type) + end + +end diff --git a/elixir_sense/test/elixir_sense/core/metadata_builder_test.exs b/elixir_sense/test/elixir_sense/core/metadata_builder_test.exs new file mode 100644 index 0000000..a34f910 --- /dev/null +++ b/elixir_sense/test/elixir_sense/core/metadata_builder_test.exs @@ -0,0 +1,608 @@ +defmodule ElixirSense.Core.MetadataBuilderTest do + + use ExUnit.Case + + alias ElixirSense.Core.MetadataBuilder + + test "build metadata from kernel.ex" do + assert get_subject_definition_line(Kernel, :defmodule, nil) =~ "defmacro defmodule(alias, do: block) do" + end + + test "build metadata from kernel/special_forms.ex" do + assert get_subject_definition_line(Kernel.SpecialForms, :alias, nil) =~ "defmacro alias(module, opts)" + end + + test "module attributes" do + state = """ + defmodule MyModule do + @myattribute 1 + IO.puts @myattribute + defmodule InnerModule do + @inner_attr module_var + IO.puts @inner_attr + end + IO.puts "" + end + """ + |> string_to_state + + assert get_line_attributes(state, 3) == [:myattribute] + assert get_line_attributes(state, 6) == [:inner_attr] + assert get_line_attributes(state, 8) == [:myattribute] + end + + test "vars defined inside a function without params" do + state = """ + defmodule MyModule do + var_out1 = 1 + def func do + var_in1 = 1 + var_in2 = 1 + IO.puts "" + end + var_out2 = 1 + end + """ + |> string_to_state + + vars = state |> get_line_vars(6) + assert vars == [:var_in1, :var_in2] + end + + test "vars defined inside a function with params" do + + state = """ + defmodule MyModule do + var_out1 = 1 + def func(%{key1: par1, key2: [par2|[par3, _]]}, par4) do + var_in1 = 1 + var_in2 = 1 + IO.puts "" + end + var_out2 = 1 + end + """ + |> string_to_state + + vars = state |> get_line_vars(6) + assert vars == [:par1, :par2, :par3, :par4, :var_in1, :var_in2] + end + + test "vars defined inside a module" do + + state = + """ + defmodule MyModule do + var_out1 = 1 + def func do + var_in = 1 + end + var_out2 = 1 + IO.puts "" + end + """ + |> string_to_state + + vars = state |> get_line_vars(7) + assert vars == [:var_out1, :var_out2] + end + + test "vars defined in a `for` comprehension" do + + state = + """ + defmodule MyModule do + var_out1 = 1 + IO.puts "" + for var_on <- [1,2], var_on != 2 do + var_in = 1 + IO.puts "" + end + var_out2 = 1 + IO.puts "" + end + """ + |> string_to_state + + assert get_line_vars(state, 3) == [:var_out1] + assert get_line_vars(state, 6) == [:var_in, :var_on, :var_out1] + assert get_line_vars(state, 9) == [:var_out1, :var_out2] + end + + test "vars defined in a `if/else` statement" do + + state = + """ + defmodule MyModule do + var_out1 = 1 + if var_on = true do + var_in_if = 1 + IO.puts "" + else + var_in_else = 1 + IO.puts "" + end + var_out2 = 1 + IO.puts "" + end + """ + |> string_to_state + + assert get_line_vars(state, 5) == [:var_in_if, :var_on, :var_out1] + assert get_line_vars(state, 8) == [:var_in_else, :var_on, :var_out1] + # This assert fails: + # assert get_line_vars(state, 11) == [:var_in_else, :var_in_if, :var_on, :var_out1, :var_out2] + end + + test "vars defined inside a `fn`" do + + state = + """ + defmodule MyModule do + var_out1 = 1 + fn var_on -> + var_in = 1 + IO.puts "" + end + var_out2 = 1 + IO.puts "" + end + """ + |> string_to_state + + assert get_line_vars(state, 5) == [:var_in, :var_on, :var_out1] + assert get_line_vars(state, 8) == [:var_out1, :var_out2] + end + + test "vars defined inside a `case`" do + + state = + """ + defmodule MyModule do + var_out1 = 1 + case var_out1 do + {var_on1} -> + var_in1 = 1 + IO.puts "" + {var_on2} -> + var_in2 = 2 + IO.puts "" + end + var_out2 = 1 + IO.puts "" + end + """ + |> string_to_state + + assert get_line_vars(state, 6) == [:var_in1, :var_on1, :var_out1] + assert get_line_vars(state, 9) == [:var_in2, :var_on2, :var_out1] + # This assert fails + # assert get_line_vars(state, 12) == [:var_in1, :var_in2, :var_out1, :var_out2] + end + + test "vars defined inside a `cond`" do + + state = + """ + defmodule MyModule do + var_out1 = 1 + cond do + 1 == 1 -> + var_in = 1 + IO.puts "" + end + var_out2 = 1 + IO.puts "" + end + """ + |> string_to_state + + assert get_line_vars(state, 6) == [:var_in, :var_out1] + # This assert fails: + # assert get_line_vars(state, 9) == [:var_in, :var_out1, :var_out2] + end + + test "a variable should only be added once to the vars list" do + + state = + """ + defmodule MyModule do + var = 1 + var = 2 + IO.puts "" + end + """ + |> string_to_state + + assert get_line_vars(state, 4) == [:var] + end + + test "functions of arity 0 should not be in the vars list" do + + state = + """ + defmodule MyModule do + myself = self + mynode = node() + IO.puts "" + end + """ + |> string_to_state + + assert get_line_vars(state, 3) == [:mynode, :myself] + end + + test "inherited vars" do + + state = + """ + top_level_var = 1 + IO.puts "" + defmodule OuterModule do + outer_module_var = 1 + IO.puts "" + defmodule InnerModule do + inner_module_var = 1 + IO.puts "" + def func do + func_var = 1 + IO.puts "" + end + IO.puts "" + end + IO.puts "" + end + IO.puts "" + """ + |> string_to_state + + assert get_line_vars(state, 2) == [:top_level_var] + assert get_line_vars(state, 5) == [:outer_module_var, :top_level_var] + assert get_line_vars(state, 8) == [:inner_module_var, :outer_module_var, :top_level_var] + assert get_line_vars(state, 11) == [:func_var] + assert get_line_vars(state, 13) == [:inner_module_var, :outer_module_var, :top_level_var] + assert get_line_vars(state, 15) == [:outer_module_var, :top_level_var] + assert get_line_vars(state, 17) == [:top_level_var] + end + + test "aliases" do + + state = + """ + defmodule OuterModule do + alias List, as: MyList + IO.puts "" + defmodule InnerModule do + alias Enum, as: MyEnum + IO.puts "" + def func do + alias String, as: MyString + IO.puts "" + if true do + alias Macro, as: MyMacro + IO.puts "" + end + IO.puts "" + end + IO.puts "" + end + alias Code, as: MyCode + IO.puts "" + end + """ + |> string_to_state + + assert get_line_aliases(state, 3) == [{MyList, List}] + assert get_line_aliases(state, 6) == [{InnerModule, OuterModule.InnerModule}, {MyList, List}, {MyEnum, Enum}] + assert get_line_aliases(state, 9) == [{InnerModule, OuterModule.InnerModule}, {MyList, List}, {MyEnum, Enum}, {MyString, String}] + assert get_line_aliases(state, 12) == [{InnerModule, OuterModule.InnerModule}, {MyList, List}, {MyEnum, Enum}, {MyString, String}, {MyMacro, Macro}] + assert get_line_aliases(state, 14) == [{InnerModule, OuterModule.InnerModule}, {MyList, List}, {MyEnum, Enum}, {MyString, String}] + assert get_line_aliases(state, 16) == [{InnerModule, OuterModule.InnerModule}, {MyList, List}, {MyEnum, Enum}] + assert get_line_aliases(state, 19) == [{MyCode, Code}, {InnerModule, OuterModule.InnerModule}, {MyList, List}] + end + + test "aliases with `fn`" do + + state = + """ + defmodule MyModule do + alias Enum, as: MyEnum + IO.puts "" + fn var_on -> + alias List, as: MyList + IO.puts "" + end + IO.puts "" + end + """ + |> string_to_state + + assert get_line_aliases(state, 3) == [{MyEnum, Enum}] + assert get_line_aliases(state, 6) == [{MyEnum, Enum}, {MyList, List}] + assert get_line_aliases(state, 8) == [{MyEnum, Enum}] + end + + test "aliases defined with v1.2 notation" do + + state = + """ + defmodule MyModule do + alias Foo.{User, Email} + IO.puts "" + end + """ + |> string_to_state + + assert get_line_aliases(state, 3) == [{Email, Foo.Email}, {User, Foo.User}] + end + + test "aliases without options" do + + state = + """ + defmodule MyModule do + alias Foo.User + IO.puts "" + end + """ + |> string_to_state + + assert get_line_aliases(state, 3) == [{User, Foo.User}] + end + + test "imports defined with v1.2 notation" do + + state = + """ + defmodule MyModule do + import Foo.Bar.{User, Email} + IO.puts "" + end + """ + |> string_to_state + + assert get_line_imports(state, 3) == [Foo.Bar.Email, Foo.Bar.User] + end + + test "imports" do + + state = + """ + defmodule OuterModule do + import List + IO.puts "" + defmodule InnerModule do + import Enum + IO.puts "" + def func do + import String + IO.puts "" + if true do + import Macro + IO.puts "" + end + IO.puts "" + end + IO.puts "" + end + import Code + IO.puts "" + end + """ + |> string_to_state + + assert get_line_imports(state, 3) == [List] + assert get_line_imports(state, 6) == [List, Enum] + assert get_line_imports(state, 9) == [List, Enum, String] + assert get_line_imports(state, 12) == [List, Enum, String, Macro] + assert get_line_imports(state, 14) == [List, Enum, String] + assert get_line_imports(state, 16) == [List, Enum] + assert get_line_imports(state, 19) == [Code, List] + end + + test "requires" do + + state = + """ + defmodule MyModule do + require Mod + IO.puts "" + end + """ + |> string_to_state + + assert get_line_requires(state, 3) == [Mod] + end + + test "requires with 1.2 notation" do + + state = + """ + defmodule MyModule do + require Mod.{Mo1, Mod2} + IO.puts "" + end + """ + |> string_to_state + + assert get_line_requires(state, 3) == [Mod.Mod2, Mod.Mo1] + end + + test "current module" do + + state = + """ + IO.puts "" + defmodule OuterModule do + IO.puts "" + defmodule InnerModule do + def func do + if true do + IO.puts "" + end + end + end + IO.puts "" + end + """ + |> string_to_state + + assert get_line_module(state, 1) == Elixir + assert get_line_module(state, 3) == OuterModule + assert get_line_module(state, 7) == OuterModule.InnerModule + assert get_line_module(state, 11) == OuterModule + end + + test "behaviours" do + + state = + """ + IO.puts "" + defmodule OuterModule do + use Application + @behaviour SomeModule.SomeBehaviour + IO.puts "" + defmodule InnerModuleWithUse do + use GenServer + IO.puts "" + end + defmodule InnerModuleWithBh do + @behaviour SomeOtherBehaviour + IO.puts "" + end + defmodule InnerModuleWithoutBh do + IO.puts "" + end + IO.puts "" + end + """ + |> string_to_state + + assert get_line_behaviours(state, 1) == [] + assert get_line_behaviours(state, 5) == [Application, SomeModule.SomeBehaviour] + assert get_line_behaviours(state, 8) == [GenServer] + assert get_line_behaviours(state, 12) == [SomeOtherBehaviour] + assert get_line_behaviours(state, 15) == [] + assert get_line_behaviours(state, 17) == [Application, SomeModule.SomeBehaviour] + end + + test "behaviour from erlang module" do + + state = + """ + defmodule OuterModule do + @behaviour :gen_server + IO.puts "" + end + """ + |> string_to_state + + assert get_line_behaviours(state, 3) == [:gen_server] + end + + test "current scope" do + + state = + """ + defmodule MyModule do + def func do + IO.puts "" + end + IO.puts "" + def func_with_when(par) when is_list(par) do + IO.puts "" + end + IO.puts "" + defmacro macro1(ast) do + IO.puts "" + end + IO.puts "" + defmacro import(module, opts) + IO.puts "" + end + """ + |> string_to_state + + assert get_scope_name(state, 3) == {:func, 0} + assert get_scope_name(state, 5) == :MyModule + assert get_scope_name(state, 7) == {:func_with_when, 1} + assert get_scope_name(state, 9) == :MyModule + assert get_scope_name(state, 11) == {:macro1, 1} + assert get_scope_name(state, 13) == :MyModule + assert get_scope_name(state, 15) == :MyModule + end + + defp string_to_state(string) do + string + |> Code.string_to_quoted + |> (fn {:ok, ast} -> ast end).() + |> MetadataBuilder.build + end + + defp get_scope_name(state, line) do + case state.lines_to_env[line] do + nil -> nil + env -> env.scope + end + end + + defp get_line_vars(state, line) do + case state.lines_to_env[line] do + nil -> [] + env -> env.vars + end |> Enum.sort + end + + defp get_line_aliases(state, line) do + case state.lines_to_env[line] do + nil -> [] + env -> env.aliases + end + end + + defp get_line_imports(state, line) do + case state.lines_to_env[line] do + nil -> [] + env -> env.imports + end + end + + defp get_line_requires(state, line) do + case state.lines_to_env[line] do + nil -> [] + env -> env.requires + end + end + + defp get_line_attributes(state, line) do + case state.lines_to_env[line] do + nil -> [] + env -> env.attributes + end |> Enum.sort + end + + defp get_line_behaviours(state, line) do + case state.lines_to_env[line] do + nil -> [] + env -> env.behaviours + end |> Enum.sort + end + + defp get_line_module(state, line) do + (env = state.lines_to_env[line]) && env.module + end + + defp get_subject_definition_line(module, func, arity) do + file = module.module_info(:compile)[:source] + acc = + File.read!(file) + |> Code.string_to_quoted + |> MetadataBuilder.build + + %{lines: lines} = Map.get(acc.mods_funs_to_lines, {module, func, arity}) + line_number = List.last(lines) + + File.read!(file) |> String.split("\n") |> Enum.at(line_number-1) + end + +end diff --git a/elixir_sense/test/elixir_sense/core/metadata_test.exs b/elixir_sense/test/elixir_sense/core/metadata_test.exs new file mode 100644 index 0000000..c6bdf21 --- /dev/null +++ b/elixir_sense/test/elixir_sense/core/metadata_test.exs @@ -0,0 +1,91 @@ +defmodule ElixirSense.Core.MetadataTest do + + use ExUnit.Case + + alias ElixirSense.Core.Parser + alias ElixirSense.Core.Metadata + + test "get_function_params" do + code = + """ + defmodule MyModule do + defp func(1) do + IO.puts "" + end + + defp func(par1) do + IO.puts par1 + end + + defp func(par1, {a, _b} = par2) do + IO.puts par1 <> a <> par2 + end + + defp func([head|_], par2) do + IO.puts head <> par2 + end + end + """ + + params = + Parser.parse_string(code, true, true, 0) + |> Metadata.get_function_params(MyModule, :func) + + assert params == [ + "1", + "par1", + "par1, {a, _b} = par2", + "[head | _], par2" + ] + end + + test "get_function_signatures" do + code = + """ + defmodule MyModule do + defp func(par) do + IO.inspect par + end + + defp func([] = my_list) do + IO.inspect my_list + end + + defp func(par1 = {a, _}, {_b, _c} = par2) do + IO.inspect {a, par2} + end + + defp func([head|_], par2) do + IO.inspect head <> par2 + end + + defp func(par1, [head|_]) do + IO.inspect {par1, head} + end + + defp func("a_string", par2) do + IO.inspect par2 + end + + defp func({_, _, _}, optional \\\\ true) do + IO.inspect optional + end + end + """ + + signatures = + Parser.parse_string(code, true, true, 0) + |> Metadata.get_function_signatures(MyModule, :func) + + assert signatures == [ + %{name: "func", params: ["par"], documentation: "", spec: ""}, + %{name: "func", params: ["my_list"], documentation: "", spec: ""}, + %{name: "func", params: ["par1", "par2"], documentation: "", spec: ""}, + %{name: "func", params: ["list", "par2"], documentation: "", spec: ""}, + %{name: "func", params: ["par1", "list"], documentation: "", spec: ""}, + %{name: "func", params: ["arg1", "par2"], documentation: "", spec: ""}, + %{name: "func", params: ["tuple", "optional \\\\ true"], documentation: "", spec: ""} + ] + end + +end diff --git a/elixir_sense/test/elixir_sense/core/parser_test.exs b/elixir_sense/test/elixir_sense/core/parser_test.exs new file mode 100644 index 0000000..52683f8 --- /dev/null +++ b/elixir_sense/test/elixir_sense/core/parser_test.exs @@ -0,0 +1,119 @@ +defmodule ElixirSense.Core.ParserTest do + use ExUnit.Case + + import ElixirSense.Core.Parser + alias ElixirSense.Core.{Metadata, State.Env} + + test "parse_string creates a Metadata struct" do + source = """ + defmodule MyModule do + import List + + end + """ + assert %Metadata{ + error: nil, + mods_funs_to_lines: %{{MyModule, nil, nil} => %{lines: [1]}}, + lines_to_env: %{ + 2 => %Env{imports: []}, + 3 => %Env{imports: [List]} + }, + source: "defmodule MyModule" <> _ + } = parse_string(source, true, true, 3) + end + + test "parse_string with syntax error" do + source = """ + defmodule MyModule do + import List + Enum + + end + """ + assert %Metadata{ + error: nil, + lines_to_env: %{ + 2 => %Env{imports: []}, + 3 => %Env{imports: [List]} + } + } = parse_string(source, true, true, 3) + end + + test "parse_string with syntax error (missing param)" do + source = """ + defmodule MyModule do + import List + IO.puts(:stderr, ) + end + """ + assert %Metadata{ + error: nil, + lines_to_env: %{ + 2 => %Env{imports: []}, + 3 => %Env{imports: [List]} + } + } = parse_string(source, true, true, 3) + end + + test "parse_string with missing terminator \")\"" do + source = """ + defmodule MyModule do + import List + func( + end + """ + assert %Metadata{ + error: nil, + lines_to_env: %{ + 2 => %Env{imports: []}, + 3 => %Env{imports: [List]} + } + } = parse_string(source, true, true, 3) + end + + test "parse_string with missing terminator \"]\"" do + source = """ + defmodule MyModule do + import List + list = [ + end + """ + assert %Metadata{ + error: nil, + lines_to_env: %{ + 2 => %Env{imports: []}, + 3 => %Env{imports: [List]} + } + } = parse_string(source, true, true, 3) + end + + test "parse_string with missing terminator \"}\"" do + source = """ + defmodule MyModule do + import List + tuple = { + end + """ + assert %Metadata{ + error: nil, + lines_to_env: %{ + 2 => %Env{imports: []}, + 3 => %Env{imports: [List]} + } + } = parse_string(source, true, true, 3) + end + + test "parse_string with missing terminator \"end\"" do + source = """ + defmodule MyModule do + + """ + assert parse_string(source, true, true, 2) == + %ElixirSense.Core.Metadata{ + error: {3,"missing terminator: end (for \"do\" starting at line 1)", ""}, + lines_to_env: %{}, + mods_funs_to_lines: %{}, + source: "defmodule MyModule do\n\n" + } + end + +end diff --git a/elixir_sense/test/elixir_sense/core/source_test.exs b/elixir_sense/test/elixir_sense/core/source_test.exs new file mode 100644 index 0000000..28f0040 --- /dev/null +++ b/elixir_sense/test/elixir_sense/core/source_test.exs @@ -0,0 +1,379 @@ +defmodule ElixirSense.Core.SourceTest do + use ExUnit.Case + + import ElixirSense.Core.Source + + describe "which_func/1" do + + test "functions without namespace" do + assert which_func("var = func(") == %{ + candidate: {nil, :func}, + npar: 0, + pipe_before: false, + pos: {{1, 7}, {1, 11}} + } + assert which_func("var = func(param1, ") == %{ + candidate: {nil, :func}, + npar: 1, + pipe_before: false, + pos: {{1, 7}, {1, 11}} + } + end + + test "functions with namespace" do + assert which_func("var = Mod.func(param1, par") == %{ + candidate: {Mod, :func}, + npar: 1, + pipe_before: false, + pos: {{1, 7}, {1, 15}} + } + assert which_func("var = Mod.SubMod.func(param1, param2, par") == %{ + candidate: {Mod.SubMod, :func}, + npar: 2, + pipe_before: false, + pos: {{1, 7}, {1, 22}} + } + end + + test "nested functions calls" do + assert which_func("var = outer_func(Mod.SubMod.func(param1,") == %{ + candidate: {Mod.SubMod, :func}, + npar: 1, + pipe_before: false, + pos: {{1, 18}, {1, 33}} + } + assert which_func("var = outer_func(Mod.SubMod.func(param1, [inner_func(") == %{ + candidate: {nil, :inner_func}, + npar: 0, + pipe_before: false, + pos: {{1, 43}, {1, 53}} + } + assert which_func("var = outer_func(func(param1, inner_func, ") == %{ + candidate: {nil, :func}, + npar: 2, + pipe_before: false, + pos: {{1, 18}, {1, 22}} + } + assert which_func("var = outer_func(func(param1, inner_func(), ") == %{ + candidate: {nil, :func}, + npar: 2, + pipe_before: false, + pos: {{1, 18}, {1, 22}} + } + assert which_func("var = func(param1, func2(fun(p3), 4, 5), func3(p1, p2), ") == %{ + candidate: {nil, :func}, + npar: 3, + pipe_before: false, + pos: {{1, 7}, {1, 11}} + } + end + + test "function call with multiple lines" do + assert which_func(""" + var = Mod.func(param1, + param2, + + """) == %{candidate: {Mod, :func}, npar: 2, pipe_before: false, pos: {{1, 7}, {1, 15}}} + end + + test "after double quotes" do + assert which_func("var = func(param1, \"not_a_func(, ") == %{ + candidate: {nil, :func}, + npar: 1, + pipe_before: false, + pos: {{1, 7}, {1, 11}} + } + assert which_func("var = func(\"a_string_(param1\", ") == %{ + candidate: {nil, :func}, + npar: 1, + pipe_before: false, + pos: {{1, 7}, {1, 11}} + } + end + + test "with operators" do + assert which_func("var = Mod.func1(param) + func2(param1, ") == %{ + candidate: {nil, :func2}, + npar: 1, + pipe_before: false, + pos: {{1, 26}, {1, 31}} + } + end + + test "erlang functions" do + assert which_func("var = :global.whereis_name( ") == %{ + candidate: {:global, :whereis_name}, + npar: 0, + pipe_before: false, + pos: {{1, 7}, {1, 27}} + } + end + + test "with fn" do + assert which_func("fn(a, ") == %{candidate: :none, npar: 0, pipe_before: false, pos: nil} + end + + test "with another fn before" do + assert which_func("var = Enum.sort_by(list, fn(i) -> i*i end, fn(a, ") == %{ + candidate: {Enum, :sort_by}, + npar: 2, + pipe_before: false, + pos: {{1, 7}, {1, 19}} + } + end + + test "inside fn body" do + assert which_func("var = Enum.map([1,2], fn(i) -> i*") == %{ + candidate: {Enum, :map}, + npar: 1, + pipe_before: false, + pos: {{1, 7}, {1, 15}} + } + end + + test "inside a list" do + assert which_func("var = Enum.map([1,2,3") == %{ + candidate: {Enum, :map}, + npar: 0, + pipe_before: false, + pos: {{1, 7}, {1, 15}} + } + end + + test "inside a list after comma" do + assert which_func("var = Enum.map([1,") == %{ + candidate: {Enum, :map}, + npar: 0, + pipe_before: false, + pos: {{1, 7}, {1, 15}} + } + end + + test "inside an list without items" do + assert which_func("var = Enum.map([") == %{ + candidate: {Enum, :map}, + npar: 0, + pipe_before: false, + pos: {{1, 7}, {1, 15}} + } + end + + test "inside a list with a list before" do + assert which_func("var = Enum.map([1,2], [1, ") == %{ + candidate: {Enum, :map}, + npar: 1, + pipe_before: false, + pos: {{1, 7}, {1, 15}} + } + end + + test "inside a tuple" do + assert which_func("var = Enum.map({1,2,3") == %{ + candidate: {Enum, :map}, + npar: 0, + pipe_before: false, + pos: {{1, 7}, {1, 15}} + } + end + + test "inside a tuple with another tuple before" do + assert which_func("var = Enum.map({1,2}, {1, ") == %{ + candidate: {Enum, :map}, + npar: 1, + pipe_before: false, + pos: {{1, 7}, {1, 15}} + } + end + + test "inside a tuple inside a list" do + assert which_func("var = Enum.map({1,2}, [{1, ") == %{ + candidate: {Enum, :map}, + npar: 1, + pipe_before: false, + pos: {{1, 7}, {1, 15}} + } + end + + test "inside a tuple after comma" do + assert which_func("var = Enum.map([{1,") == %{ + candidate: {Enum, :map}, + npar: 0, + pipe_before: false, + pos: {{1, 7}, {1, 15}} + } + end + + test "inside a list inside a tuple inside a list" do + assert which_func("var = Enum.map([{1,[a, ") == %{ + candidate: {Enum, :map}, + npar: 0, + pipe_before: false, + pos: {{1, 7}, {1, 15}} + } + end + + test "fails when code has parsing errors before the cursor" do + assert which_func("} = Enum.map(list, ") == %{candidate: :none, npar: 0, pipe_before: false, pos: nil} + end + + end + + describe "text_before/3" do + + test "functions without namespace" do + code = """ + defmodule MyMod do + def my_func(par1, ) + end + """ + text = """ + defmodule MyMod do + def my_func(par1, + """ |> String.trim() + + assert text_before(code, 2, 20) == text + end + + end + describe "subject" do + + test "functions without namespace" do + code = """ + defmodule MyMod do + my_func(par1, ) + end + """ + + assert subject(code, 2, 5) == "my_func" + end + + test "functions with namespace" do + code = """ + defmodule MyMod do + Mod.func(par1, ) + end + """ + + assert subject(code, 2, 8) == "Mod.func" + end + + test "functions ending with !" do + code = """ + defmodule MyMod do + Mod.func! + end + """ + + assert subject(code, 2, 8) == "Mod.func!" + end + + test "functions ending with ?" do + code = """ + defmodule MyMod do + func?(par1, ) + end + """ + + assert subject(code, 2, 8) == "func?" + end + + test "erlang modules" do + code = """ + :lists.concat([1,2]) + """ + + assert subject(code, 1, 5) == ":lists" + assert subject(code, 1, 5) == ":lists" + end + + test "functions from erlang modules" do + code = """ + :lists.concat([1,2]) + """ + + assert subject(code, 1, 12) == ":lists.concat" + end + + test "capture operator" do + code = """ + Emum.map(list, &func/1) + """ + + assert subject(code, 1, 21) == "func" + end + + test "functions with `!` operator before" do + code = """ + if !match({_,_}, var) do + """ + + assert subject(code, 1, 8) == "match" + end + + test "module and function in different lines" do + code = """ + Mod. + func + """ + + assert subject(code, 2, 7) == "Mod.func" + end + + test "elixir module" do + code = """ + defmodule MyMod do + ModA.ModB.func + end + """ + + assert subject(code, 2, 4) == "ModA" + assert subject(code, 2, 9) == "ModA.ModB" + assert subject(code, 2, 14) == "ModA.ModB.func" + end + + test "anonymous functions call" do + code = """ + my_func.(1,2) + """ + + assert subject(code, 1, 4) == "my_func" + end + + test "no empty/stop grapheme after subject" do + code = "Mod.my_func" + + assert subject(code, 1, 2) == "Mod" + assert subject(code, 1, 6) == "Mod.my_func" + end + + test "find closest on the edges" do + code = """ + defmodule MyMod do + Mod.my_func(par1, par2) + end + """ + + assert subject(code, 2, 2) == nil + assert subject(code, 2, 3) == "Mod" + assert subject(code, 2, 5) == "Mod" + assert subject(code, 2, 6) == "Mod" + assert subject(code, 2, 7) == "Mod.my_func" + assert subject(code, 2, 14) == "Mod.my_func" + assert subject(code, 2, 15) == "par1" + assert subject(code, 2, 19) == "par1" + assert subject(code, 2, 20) == nil + assert subject(code, 2, 21) == "par2" + end + + test "module from struct" do + code = """ + defmodule MyMod do + Mod.my_func(%MyMod{a: 1}) + end + """ + + assert subject(code, 2, 17) == "MyMod" + end + + end +end diff --git a/elixir_sense/test/elixir_sense/definition_test.exs b/elixir_sense/test/elixir_sense/definition_test.exs new file mode 100644 index 0000000..3aad3f2 --- /dev/null +++ b/elixir_sense/test/elixir_sense/definition_test.exs @@ -0,0 +1,171 @@ +defmodule ElixirSense.Providers.DefinitionTest do + + use ExUnit.Case + alias ElixirSense.Providers.Definition + + doctest Definition + + test "find definition of functions from Kernel" do + buffer = """ + defmodule MyModule do + + end + """ + {file, line} = ElixirSense.definition(buffer, 1, 2) + assert file =~ "lib/elixir/lib/kernel.ex" + assert read_line(file, line) =~ "defmacro defmodule" + end + + test "find definition of functions from Kernel.SpecialForms" do + buffer = """ + defmodule MyModule do + import List + end + """ + {file, line} = ElixirSense.definition(buffer, 2, 4) + assert file =~ "lib/elixir/lib/kernel/special_forms.ex" + assert read_line(file, line) =~ "defmacro import" + end + + test "find definition of functions from imports" do + buffer = """ + defmodule MyModule do + import Mix.Generator + create_file( + end + """ + {file, line} = ElixirSense.definition(buffer, 3, 4) + assert file =~ "lib/mix/lib/mix/generator.ex" + assert read_line(file, line) =~ "def create_file" + end + + test "find definition of functions from aliased modules" do + buffer = """ + defmodule MyModule do + alias List, as: MyList + MyList.flatten([[1],[3]]) + end + """ + {file, line} = ElixirSense.definition(buffer, 3, 11) + assert file =~ "lib/elixir/lib/list.ex" + assert read_line(file, line) =~ "def flatten" + end + + test "find definition of modules" do + buffer = """ + defmodule MyModule do + alias List, as: MyList + String.to_atom("erlang") + end + """ + {file, line} = ElixirSense.definition(buffer, 3, 4) + assert file =~ "lib/elixir/lib/string.ex" + assert read_line(file, line) =~ "defmodule String" + end + + test "find definition of erlang modules" do + buffer = """ + defmodule MyModule do + def dup(x) do + :lists.duplicate(2, x) + end + end + """ + {file, line} = ElixirSense.definition(buffer, 3, 7) + assert file =~ "/src/lists.erl" + assert line == 1 + end + + test "find definition of remote erlang functions" do + buffer = """ + defmodule MyModule do + def dup(x) do + :lists.duplicate(2, x) + end + end + """ + {file, line} = ElixirSense.definition(buffer, 3, 15) + assert file =~ "/src/lists.erl" + assert read_line(file, line) =~ "duplicate(N, X)" + end + + test "non existing modules" do + buffer = """ + defmodule MyModule do + SilverBulletModule.run + end + """ + assert ElixirSense.definition(buffer, 2, 24) == {"non_existing", nil} + end + + test "cannot find map field calls" do + buffer = """ + defmodule MyModule do + env = __ENV__ + IO.puts(env.file) + end + """ + assert ElixirSense.definition(buffer, 3, 16) == {"non_existing", nil} + end + + test "cannot find vars" do + buffer = """ + defmodule MyModule do + var = 1 + end + """ + assert ElixirSense.definition(buffer, 2, 4) == {"non_existing", nil} + end + + test "cannot find map fields" do + buffer = """ + defmodule MyModule do + var = %{count: 1} + end + """ + assert ElixirSense.definition(buffer, 2, 12) == {"non_existing", nil} + end + + test "preloaded modules" do + buffer = """ + defmodule MyModule do + :erlang.node + end + """ + assert ElixirSense.definition(buffer, 2, 5) == {"non_existing", nil} + end + + # Call this when running `mix test`, but not when running `elixir run_test.exs` + if Process.whereis(Elixir.Mix.Supervisor) do + test "erlang modules from deps" do + buffer = """ + defmodule MyModule do + :hackney + end + """ + {file, line} = ElixirSense.definition(buffer, 2, 5) + assert file =~ "deps/hackney/src/hackney.erl" + assert line == 1 + end + end + + test "find the related module when searching for built-in functions" do + buffer = """ + defmodule MyModule do + List.module_info() + end + """ + {file, line} = ElixirSense.definition(buffer, 2, 10) + assert file =~ "lib/elixir/lib/list.ex" + assert line == nil + end + + defp read_line(file, line) do + file + |> File.read! + |> String.split(["\n", "\r\n"]) + |> Enum.at(line-1) + |> String.trim + end + +end diff --git a/elixir_sense/test/elixir_sense/docs_test.exs b/elixir_sense/test/elixir_sense/docs_test.exs new file mode 100644 index 0000000..1e14ebd --- /dev/null +++ b/elixir_sense/test/elixir_sense/docs_test.exs @@ -0,0 +1,297 @@ +defmodule ElixirSense.DocsTest do + + use ExUnit.Case + + describe "docs" do + + test "retrieve documentation" do + buffer = """ + defmodule MyModule do + + end + """ + + %{ + subject: subject, + actual_subject: actual_subject, + docs: %{docs: docs} + } = ElixirSense.docs(buffer, 1, 2) + + assert subject == "defmodule" + assert actual_subject == "Kernel.defmodule" + assert docs =~ """ + Defines a module given by name with the given contents. + """ + end + + test "retrieve function documentation" do + buffer = """ + defmodule MyModule do + def func(list) do + List.flatten(list) + end + end + """ + + %{ + subject: subject, + actual_subject: actual_subject, + docs: %{docs: docs} + } = ElixirSense.docs(buffer, 3, 12) + + assert subject == "List.flatten" + assert actual_subject == "List.flatten" + assert docs == """ + > List.flatten(list) + + ### Specs + + `@spec flatten(deep_list) :: list when deep_list: [any | deep_list]` + + Flattens the given `list` of nested lists. + + ## Examples + + iex> List.flatten([1, [[2], 3]]) + [1, 2, 3] + + + + ____ + + > List.flatten(list, tail) + + ### Specs + + `@spec flatten(deep_list, [elem]) :: [elem] when deep_list: [elem | deep_list], elem: var` + + Flattens the given `list` of nested lists. + The list `tail` will be added at the end of + the flattened list. + + ## Examples + + iex> List.flatten([1, [[2], 3]], [4, 5]) + [1, 2, 3, 4, 5] + + """ + end + + test "retrieve function documentation from aliased modules" do + buffer = """ + defmodule MyModule do + alias List, as: MyList + MyList.flatten + end + """ + + %{ + subject: subject, + actual_subject: actual_subject, + docs: %{docs: docs} + } = ElixirSense.docs(buffer, 3, 12) + + assert subject == "MyList.flatten" + assert actual_subject == "List.flatten" + assert docs == """ + > List.flatten(list) + + ### Specs + + `@spec flatten(deep_list) :: list when deep_list: [any | deep_list]` + + Flattens the given `list` of nested lists. + + ## Examples + + iex> List.flatten([1, [[2], 3]]) + [1, 2, 3] + + + + ____ + + > List.flatten(list, tail) + + ### Specs + + `@spec flatten(deep_list, [elem]) :: [elem] when deep_list: [elem | deep_list], elem: var` + + Flattens the given `list` of nested lists. + The list `tail` will be added at the end of + the flattened list. + + ## Examples + + iex> List.flatten([1, [[2], 3]], [4, 5]) + [1, 2, 3, 4, 5] + + """ + end + + test "retrive function documentation from imported modules" do + buffer = """ + defmodule MyModule do + import Mix.Generator + create_file( + end + """ + + %{ + subject: subject, + actual_subject: actual_subject, + docs: %{docs: docs} + } = ElixirSense.docs(buffer, 3, 5) + + assert subject == "create_file" + assert actual_subject == "Mix.Generator.create_file" + assert docs =~ """ + > Mix.Generator.create_file(path, contents, opts \\\\\\\\ []) + + ### Specs + + `@spec create_file(Path.t, iodata, Keyword.t) :: any` + + Creates a file with the given contents. + If the file already exists, asks for user confirmation. + + ## Options + + * `:force` - forces installation without a shell prompt. + """ + end + + test "request for defmacro" do + buffer = """ + defmodule MyModule do + defmacro my_macro do + end + end + """ + + %{subject: subject, docs: %{docs: docs}} = ElixirSense.docs(buffer, 2, 5) + + assert subject == "defmacro" + assert docs =~ """ + > Kernel.defmacro(call, expr \\\\\\\\ nil) + + Defines a macro with the given name and body. + + ## Examples + + defmodule MyLogic do + defmacro unless(expr, opts) do + quote do + if !unquote(expr), unquote(opts) + end + end + end + + require MyLogic + MyLogic.unless false do + IO.puts \"It works\" + end + + """ + end + + test "retrieve documentation from modules" do + buffer = """ + defmodule MyModule do + use GenServer + end + """ + + %{ + subject: subject, + actual_subject: actual_subject, + docs: %{docs: docs} + } = ElixirSense.docs(buffer, 2, 8) + + assert subject == "GenServer" + assert actual_subject == "GenServer" + assert docs =~ """ + > GenServer + + A behaviour module for implementing the server of a client-server relation. + + A GenServer is a process like any other Elixir process and it can be used + to keep state, execute code asynchronously and so on. The advantage of using + a generic server process (GenServer) implemented using this module is that it + will have a standard set of interface functions and include functionality for + tracing and error reporting. It will also fit into a supervision tree. + """ + end + + test "retrieve type information from modules" do + buffer = """ + defmodule MyModule do + use GenServer + end + """ + + %{subject: subject, docs: %{types: docs}} = ElixirSense.docs(buffer, 2, 8) + + assert subject == "GenServer" + assert docs =~ """ + `@type on_start :: + {:ok, pid} | + :ignore | + {:error, {:already_started, pid} | term} + ` + + Return values of `start*` functions + + + ____ + + `@type name :: + atom | + {:global, term} | + {:via, module, term} + ` + + The GenServer name + """ + end + + test "retrieve callback information from modules" do + buffer = """ + defmodule MyModule do + use Application + end + """ + + %{subject: subject, docs: %{callbacks: docs}} = ElixirSense.docs(buffer, 2, 8) + + assert subject == "Application" + assert docs =~ """ + > start(start_type, start_args) + + ### Specs + + `@callback start(start_type, start_args :: term) :: + {:ok, pid} | + {:ok, pid, state} | + {:error, reason :: term} + ` + + Called when an application is started. + """ + end + + test "no docs" do + buffer = """ + defmodule MyModule do + raise ArgumentError, "Error" + end + """ + + %{subject: subject, docs: %{docs: docs}} = ElixirSense.docs(buffer, 2, 11) + + assert subject == "ArgumentError" + assert docs == "No documentation available" + end + + end +end diff --git a/elixir_sense/test/elixir_sense/eval_test.exs b/elixir_sense/test/elixir_sense/eval_test.exs new file mode 100644 index 0000000..49f10aa --- /dev/null +++ b/elixir_sense/test/elixir_sense/eval_test.exs @@ -0,0 +1,145 @@ +defmodule ElixirSense.Evaltest do + + use ExUnit.Case + + describe "match" do + + test "with bindings" do + code = "{name, _, [par1, par2]} = {:func, [line: 1], [{:par1, [line: 1], nil}, {:par2, [line: 1], nil}]}" + assert ElixirSense.match(code) =~ """ + # Bindings + + name = :func + + par1 = {:par1, [line: 1], nil} + + par2 = {:par2, [line: 1], nil} + """ |> String.trim + end + + test "without bindings" do + code = "{_, _, [_, _]} = {:func, [line: 1], [{:par1, [line: 1], nil}, {:par2, [line: 1], nil}]}" + assert ElixirSense.match(code) == """ + # No bindings + + """ + end + + test "with token missing error" do + code = "{ = {:func, [line: 1], [{:par1, [line: 1], nil}, {:par2, [line: 1], nil}]}" + assert ElixirSense.match(code) =~ """ + # TokenMissingError on line 1: + # ↳ missing terminator: } (for "{" starting at line 1) + """ |> String.trim + end + + test "EVAL request match with match error" do + code = "{var} = {:func, [line: 1], [{:par1, [line: 1], nil}, {:par2, [line: 1], nil}]}" + assert ElixirSense.match(code) == "# No match" + end + + end + + describe "expand full" do + + test "without errors" do + buffer = """ + defmodule MyModule do + + end + """ + code = "use Application" + result = ElixirSense.expand_full(buffer, code, 2) + + assert result.expand_once =~ """ + ( + require(Application) + Application.__using__([]) + ) + """ |> String.trim + + assert result.expand =~ """ + ( + require(Application) + Application.__using__([]) + ) + """ |> String.trim + + assert result.expand_partial =~ """ + ( + require(Application) + ( + @behaviour(Application) + @doc(false) + def(stop(_state)) do + :ok + end + defoverridable(stop: 1) + ) + ) + """ |> String.trim + + if (Version.match?(System.version, ">=1.4.0")) do + assert result.expand_all =~ """ + ( + require(Application) + ( + Module.put_attribute(MyModule, :behaviour, Application, nil, nil) + Module.put_attribute(MyModule, :doc, {0, false}, [{MyModule, :__MODULE__, 0, [file: "lib/elixir_sense/providers/expand.ex", line: 0]}], nil) + """ |> String.trim + else + assert result.expand_all =~ """ + ( + require(Application) + ( + Module.put_attribute(MyModule, :behaviour, Application) + Module.put_attribute(MyModule, :doc, {0, false}, [{MyModule, :__MODULE__, 0, [file: "lib/elixir_sense/providers/expand.ex", line: 0]}]) + """ |> String.trim + end + end + + test "with errors" do + buffer = """ + defmodule MyModule do + + end + """ + code = "{" + result = ElixirSense.expand_full(buffer, code, 2) + + assert result.expand_once =~ """ + {1, "missing terminator: } (for \\"{\\" starting at line 1)", ""} + """ |> String.trim + + assert result.expand =~ """ + {1, "missing terminator: } (for \\"{\\" starting at line 1)", ""} + """ |> String.trim + + assert result.expand_partial =~ """ + {1, "missing terminator: } (for \\"{\\" starting at line 1)", ""} + """ |> String.trim + + assert result.expand_all =~ """ + {1, "missing terminator: } (for \\"{\\" starting at line 1)", ""} + """ |> String.trim + end + + end + + describe "quote" do + + test "without error" do + code = "func(par1, par2)" + assert ElixirSense.quote(code) =~ "{:func, [line: 1], [{:par1, [line: 1], nil}, {:par2, [line: 1], nil}]}" + end + + test "with error" do + code = "func(par1, par2" + assert ElixirSense.quote(code) =~ """ + {1, "missing terminator: ) (for \\"(\\" starting at line 1)", \""} + """ |> String.trim + end + + end + +end diff --git a/elixir_sense/test/elixir_sense/providers/suggestion_test.exs b/elixir_sense/test/elixir_sense/providers/suggestion_test.exs new file mode 100644 index 0000000..68d6eff --- /dev/null +++ b/elixir_sense/test/elixir_sense/providers/suggestion_test.exs @@ -0,0 +1,65 @@ +defmodule ElixirSense.Providers.SuggestionTest do + + use ExUnit.Case + alias ElixirSense.Providers.Suggestion + + doctest Suggestion + + defmodule MyModule do + def say_hi, do: true + end + + test "find definition of functions from Kernel" do + assert [ + %{type: :hint, value: "List."}, + %{name: "List", subtype: nil, summary: "" <> _, type: :module}, + %{name: "Chars", subtype: :protocol, summary: "The List.Chars protocol" <> _, type: :module}, + %{args: "", arity: 1, name: "__info__", origin: "List", spec: nil, summary: "", type: "function"}, + %{args: "list", arity: 1, name: "first", origin: "List", spec: "@spec first([elem]) :: nil | elem when elem: var", summary: "Returns the first " <> _, type: "function"}, + %{args: "list", arity: 1, name: "last", origin: "List", spec: "@spec last([elem]) :: nil | elem when elem: var", summary: "Returns the last element " <> _, type: "function"}, + %{args: "charlist", arity: 1, name: "to_atom", origin: "List", spec: "@spec to_atom(charlist) :: atom", summary: "Converts a charlist to an atom.", type: "function"}, + %{args: "charlist", arity: 1, name: "to_existing_atom", origin: "List", spec: "@spec to_existing_atom(charlist) :: atom", summary: "Converts a charlist" <> _, type: "function"}, + %{args: "charlist", arity: 1, name: "to_float", origin: "List", spec: "@spec to_float(charlist) :: float", summary: "Returns the float " <> _, type: "function"}, + %{args: "list", arity: 1, name: "to_string", origin: "List", spec: "@spec to_string(:unicode.charlist) :: String.t", summary: "Converts a list " <> _, type: "function"}, + %{args: "list", arity: 1, name: "to_tuple", origin: "List", spec: "@spec to_tuple(list) :: tuple", summary: "Converts a list to a tuple.", type: "function"}, + %{args: "list", arity: 1, name: "wrap", origin: "List", spec: "@spec wrap(list | any) :: list", summary: "Wraps the " <> _, type: "function"}, + %{args: "list_of_lists", arity: 1, name: "zip", origin: "List", spec: "@spec zip([list]) :: [tuple]", summary: "Zips corresponding " <> _, type: "function"}, + %{args: "", arity: 1, name: "module_info", origin: "List", spec: nil, summary: "", type: "function"}, + %{args: "", arity: 0, name: "module_info", origin: "List", spec: nil, summary: "", type: "function"}, + %{args: "list,item", arity: 2, name: "delete", origin: "List", spec: "@spec delete(list, any) :: list", summary: "Deletes the given " <> _, type: "function"} + | _] = Suggestion.find("List", [], [], [], [], [], SomeModule) + end + + test "return completion candidates for 'Str'" do + assert Suggestion.find("Str", [], [], [], [], [], SomeModule) == [ + %{type: :hint, value: "Str"}, + %{name: "Stream", subtype: :struct, summary: "Module for creating and composing streams.", type: :module}, + %{name: "String", subtype: nil, summary: "A String in Elixir is a UTF-8 encoded binary.", type: :module}, + %{name: "StringIO", subtype: nil, summary: "Controls an IO device process that wraps a string.", type: :module} + ] + end + + test "return completion candidates for 'List.del'" do + assert [ + %{type: :hint, value: "List.delete"}, + %{args: "list,item", arity: 2, name: "delete", origin: "List", spec: "@spec delete(list, any) :: list", summary: "Deletes the given" <> _, type: "function"}, + %{args: "list,index", arity: 2, name: "delete_at", origin: "List", spec: "@spec delete_at(list, integer) :: list", summary: "Produces a new list by " <> _, type: "function"} + ] = Suggestion.find("List.del", [], [], [], [], [], SomeModule) + end + + test "return completion candidates for module with alias" do + assert [ + %{type: :hint, value: "MyList.delete"}, + %{args: "list,item", arity: 2, name: "delete", origin: "List", spec: "@spec delete(list, any) :: list", summary: "Deletes the given " <> _, type: "function"}, + %{args: "list,index", arity: 2, name: "delete_at", origin: "List", spec: "@spec delete_at(list, integer) :: list", summary: "Produces a new list " <> _, type: "function"} + ] = Suggestion.find("MyList.del", [], [{MyList, List}], [], [], [], SomeModule) + end + + test "return completion candidates for functions from import" do + assert Suggestion.find("say", [MyModule], [], [], [], [], SomeModule) == [ + %{type: :hint, value: "say"}, + %{args: "", arity: 0, name: "say_hi", origin: "ElixirSense.Providers.SuggestionTest.MyModule", spec: "", summary: "", type: "public_function"} + ] + end + +end diff --git a/elixir_sense/test/elixir_sense/signature_test.exs b/elixir_sense/test/elixir_sense/signature_test.exs new file mode 100644 index 0000000..689a18f --- /dev/null +++ b/elixir_sense/test/elixir_sense/signature_test.exs @@ -0,0 +1,146 @@ +defmodule ElixirSense.SignatureTest do + + use ExUnit.Case + alias ElixirSense.Providers.Signature + + doctest Signature + + describe "signature" do + + test "find signatures from aliased modules" do + code = """ + defmodule MyModule do + alias List, as: MyList + MyList.flatten(par1, + end + """ + assert ElixirSense.signature(code, 3, 23) == %{ + active_param: 1, + pipe_before: false, + signatures: [ + %{ + name: "flatten", + params: ["list"], + documentation: "Flattens the given `list` of nested lists.", + spec: "@spec flatten(deep_list) :: list when deep_list: [any | deep_list]" + }, + %{ + name: "flatten", + params: ["list", "tail"], + documentation: "Flattens the given `list` of nested lists.\nThe list `tail` will be added at the end of\nthe flattened list.", + spec: "@spec flatten(deep_list, [elem]) :: [elem] when deep_list: [elem | deep_list], elem: var" + } + ] + } + end + + test "finds signatures from Kernel functions" do + code = """ + defmodule MyModule do + apply(par1, + end + """ + assert ElixirSense.signature(code, 2, 14) == %{ + active_param: 1, + pipe_before: false, + signatures: [ + %{ + name: "apply", + params: ["fun", "args"], + documentation: "Invokes the given `fun` with the list of arguments `args`.", + spec: "@spec apply((... -> any), [any]) :: any" + }, + %{ + name: "apply", + params: ["module", "fun", "args"], + documentation: "Invokes the given `fun` from `module` with the list of arguments `args`.", + spec: "@spec apply(module, atom, [any]) :: any" + } + ] + } + end + + test "finds signatures from local functions" do + code = """ + defmodule MyModule do + + def run do + sum(a, + end + + defp sum(a, b) do + a + b + end + + defp sum({a, b}) do + a + b + end + end + """ + assert ElixirSense.signature(code, 4, 12) == %{ + active_param: 1, + pipe_before: false, + signatures: [ + %{ + name: "sum", + params: ["a", "b"], + documentation: "", + spec: "" + }, + %{ + name: "sum", + params: ["tuple"], + documentation: "", + spec: "" + } + ] + } + end + + test "returns :none when it cannot identify a function call" do + code = """ + defmodule MyModule do + fn(a, + end + """ + assert ElixirSense.signature(code, 2, 8) == :none + end + + test "return empty signature list when no signature is found" do + code = """ + defmodule MyModule do + a_func( + end + """ + assert ElixirSense.signature(code, 2, 10) == %{active_param: 0, signatures: [], pipe_before: false} + end + + test "after |>" do + code = """ + defmodule MyModule do + {1, 2} |> IO.inspect( + end + """ + assert ElixirSense.signature(code, 2, 24) == %{ + active_param: 1, + pipe_before: true, + signatures: [ + %{ + name: "inspect", + params: ["item", "opts \\\\ []"], + documentation: "Inspects and writes the given `item` to the device.", + spec: "@spec inspect(item, Keyword.t) :: item when item: var" + }, + %{ + name: "inspect", + params: ["device", "item", "opts"], + documentation: "Inspects `item` according to the given options using the IO `device`.", + spec: "@spec inspect(device, item, Keyword.t) :: item when item: var" + } + ] + } + end + + end + +end diff --git a/elixir_sense/test/elixir_sense/suggestions_test.exs b/elixir_sense/test/elixir_sense/suggestions_test.exs new file mode 100644 index 0000000..390a360 --- /dev/null +++ b/elixir_sense/test/elixir_sense/suggestions_test.exs @@ -0,0 +1,230 @@ +defmodule ElixirSense.SuggestionsTest do + + use ExUnit.Case + + test "empty hint" do + buffer = """ + defmodule MyModule do + + end + """ + + list = ElixirSense.suggestions(buffer, 2, 7) + + assert Enum.find(list, fn s -> match?(%{name: "import", arity: 2}, s) end) == %{ + args: "module,opts", arity: 2, name: "import", + origin: "Kernel.SpecialForms", spec: "", + summary: "Imports functions and macros from other modules.", + type: "macro" + } + assert Enum.find(list, fn s -> match?(%{name: "quote", arity: 2}, s) end) == %{ + arity: 2, origin: "Kernel.SpecialForms", + spec: "", type: "macro", args: "opts,block", + name: "quote", + summary: "Gets the representation of any expression." + } + assert Enum.find(list, fn s -> match?(%{name: "require", arity: 2}, s) end) == %{ + arity: 2, origin: "Kernel.SpecialForms", + spec: "", type: "macro", args: "module,opts", + name: "require", + summary: "Requires a given module to be compiled and loaded." + } + + end + + test "without empty hint" do + + buffer = """ + defmodule MyModule do + is_b + end + """ + + list = ElixirSense.suggestions(buffer, 2, 11) + + assert list == [ + %{type: :hint, value: "is_b"}, + %{args: "term", arity: 1, name: "is_binary", origin: "Kernel", + spec: "@spec is_binary(term) :: boolean", + summary: "Returns `true` if `term` is a binary; otherwise returns `false`.", + type: "function"}, + %{args: "term", arity: 1, name: "is_bitstring", origin: "Kernel", + spec: "@spec is_bitstring(term) :: boolean", + summary: "Returns `true` if `term` is a bitstring (including a binary); otherwise returns `false`.", + type: "function"}, + %{args: "term", arity: 1, name: "is_boolean", origin: "Kernel", + spec: "@spec is_boolean(term) :: boolean", + summary: "Returns `true` if `term` is either the atom `true` or the atom `false` (i.e.,\na boolean); otherwise returns `false`.", + type: "function"} + ] + end + + test "with an alias" do + buffer = """ + defmodule MyModule do + alias List, as: MyList + MyList.flat + end + """ + + list = ElixirSense.suggestions(buffer, 3, 14) + + assert list == [ + %{type: :hint, value: "MyList.flatten"}, + %{args: "list,tail", arity: 2, name: "flatten", origin: "List", + spec: "@spec flatten(deep_list, [elem]) :: [elem] when deep_list: [elem | deep_list], elem: var", + summary: "Flattens the given `list` of nested lists.\nThe list `tail` will be added at the end of\nthe flattened list.", + type: "function"}, + %{args: "list", arity: 1, name: "flatten", origin: "List", + spec: "@spec flatten(deep_list) :: list when deep_list: [any | deep_list]", + summary: "Flattens the given `list` of nested lists.", + type: "function"} + ] + end + + test "with a module hint" do + buffer = """ + defmodule MyModule do + Str + end + """ + + list = ElixirSense.suggestions(buffer, 2, 6) + + assert list == [ + %{type: :hint, value: "Str"}, + %{name: "Stream", subtype: :struct, + summary: "Module for creating and composing streams.", + type: :module}, + %{name: "String", subtype: nil, + summary: "A String in Elixir is a UTF-8 encoded binary.", + type: :module}, + %{name: "StringIO", subtype: nil, + summary: "Controls an IO device process that wraps a string.", + type: :module} + ] + end + + test "lists callbacks" do + buffer = """ + defmodule MyServer do + use GenServer + + end + """ + + list = + ElixirSense.suggestions(buffer, 3, 7) + |> Enum.filter(fn s -> s.type == :callback && s.name == :code_change end) + + assert list == [%{ + args: "old_vsn,state,extra", arity: 3, name: :code_change, + origin: "GenServer", + spec: "@callback code_change(old_vsn, state :: term, extra :: term) ::\n {:ok, new_state :: term} |\n {:error, reason :: term} when old_vsn: term | {:down, term}\n", + summary: "Invoked to change the state of the `GenServer` when a different version of a\nmodule is loaded (hot code swapping) and the state's term structure should be\nchanged.", + type: :callback + }] + end + + test "lists returns" do + buffer = """ + defmodule MyServer do + use GenServer + + def handle_call(request, from, state) do + + end + + end + """ + + list = + ElixirSense.suggestions(buffer, 5, 5) + |> Enum.filter(fn s -> s.type == :return end) + + assert list == [ + %{description: "{:reply, reply, new_state}", + snippet: "{:reply, \"${1:reply}$\", \"${2:new_state}$\"}", + spec: "{:reply, reply, new_state} when reply: term, new_state: term, reason: term", + type: :return}, + %{description: "{:reply, reply, new_state, timeout | :hibernate}", + snippet: "{:reply, \"${1:reply}$\", \"${2:new_state}$\", \"${3:timeout | :hibernate}$\"}", + spec: "{:reply, reply, new_state, timeout | :hibernate} when reply: term, new_state: term, reason: term", + type: :return}, + %{description: "{:noreply, new_state}", + snippet: "{:noreply, \"${1:new_state}$\"}", + spec: "{:noreply, new_state} when reply: term, new_state: term, reason: term", + type: :return}, + %{description: "{:noreply, new_state, timeout | :hibernate}", + snippet: "{:noreply, \"${1:new_state}$\", \"${2:timeout | :hibernate}$\"}", + spec: "{:noreply, new_state, timeout | :hibernate} when reply: term, new_state: term, reason: term", + type: :return}, + %{description: "{:stop, reason, reply, new_state}", + snippet: "{:stop, \"${1:reason}$\", \"${2:reply}$\", \"${3:new_state}$\"}", + spec: "{:stop, reason, reply, new_state} when reply: term, new_state: term, reason: term", + type: :return}, + %{description: "{:stop, reason, new_state}", + snippet: "{:stop, \"${1:reason}$\", \"${2:new_state}$\"}", + spec: "{:stop, reason, new_state} when reply: term, new_state: term, reason: term", + type: :return} + ] + end + + test "lists params and vars" do + buffer = """ + defmodule MyServer do + use GenServer + + def handle_call(request, from, state) do + var1 = true + + end + + end + """ + + list = + ElixirSense.suggestions(buffer, 6, 5) + |> Enum.filter(fn s -> s.type == :variable end) + + assert list == [ + %{name: :from, type: :variable}, + %{name: :request, type: :variable}, + %{name: :state, type: :variable}, + %{name: :var1, type: :variable} + ] + end + + test "lists attributes" do + buffer = """ + defmodule MyModule do + @my_attribute1 true + @my_attribute2 false + @ + end + """ + + list = + ElixirSense.suggestions(buffer, 4, 4) + |> Enum.filter(fn s -> s.type == :attribute end) + + assert list == [ + %{name: "@my_attribute1", type: :attribute}, + %{name: "@my_attribute2", type: :attribute} + ] + end + + test "Elixir module" do + buffer = """ + defmodule MyModule do + El + end + """ + + list = ElixirSense.suggestions(buffer, 2, 5) + + assert Enum.at(list,0) == %{type: :hint, value: "Elixir"} + assert Enum.at(list,1) == %{type: :module, name: "Elixir", subtype: nil, summary: ""} + end + +end diff --git a/elixir_sense/test/elixir_sense_test.exs b/elixir_sense/test/elixir_sense_test.exs new file mode 100644 index 0000000..90a804a --- /dev/null +++ b/elixir_sense/test/elixir_sense_test.exs @@ -0,0 +1,5 @@ +defmodule ElixirSenseTest do + use ExUnit.Case + + doctest ElixirSense +end diff --git a/elixir_sense/test/server_test.exs b/elixir_sense/test/server_test.exs new file mode 100644 index 0000000..43f55c5 --- /dev/null +++ b/elixir_sense/test/server_test.exs @@ -0,0 +1,198 @@ +defmodule TCPHelper do + + def send_request(socket, request) do + data = :erlang.term_to_binary(request) + send_and_recv(socket, data) + |> :erlang.binary_to_term + |> Map.get(:payload) + end + + def send_and_recv(socket, data) do + :ok = :gen_tcp.send(socket, data) + {:ok, response} = :gen_tcp.recv(socket, 0, 1000) + response + end + +end + +defmodule ElixirSense.ServerTest do + use ExUnit.Case + + alias ElixirSense.Server.ContextLoader + import ExUnit.CaptureIO + import TCPHelper + + setup_all do + ["ok", "localhost", port, auth_token] = capture_io(fn -> + ElixirSense.Server.start(["tcpip", "0", "dev"]) + end) |> String.split(":") + port = port |> String.trim |> String.to_integer + auth_token = auth_token|> String.trim + {:ok, socket} = :gen_tcp.connect('localhost', port, [:binary, active: false, packet: 4]) + + {:ok, socket: socket, auth_token: auth_token} + end + + test "definition request", %{socket: socket, auth_token: auth_token} do + request = %{ + "request_id" => 1, + "auth_token" => auth_token, + "request" => "definition", + "payload" => %{ + "buffer" => "Enum.to_list", + "line" => 1, + "column" => 6 + } + } + assert send_request(socket, request) =~ "enum.ex:2523" + end + + test "signature request", %{socket: socket, auth_token: auth_token} do + request = %{ + "request_id" => 1, + "auth_token" => auth_token, + "request" => "signature", + "payload" => %{ + "buffer" => "List.flatten(par, ", + "line" => 1, + "column" => 18 + } + } + assert send_request(socket, request).active_param == 1 + end + + test "quote request", %{socket: socket, auth_token: auth_token} do + request = %{ + "request_id" => 1, + "auth_token" => auth_token, + "request" => "quote", + "payload" => %{ + "code" => "var = 1", + } + } + assert send_request(socket, request) == "{:=, [line: 1], [{:var, [line: 1], nil}, 1]}" + end + + test "match request", %{socket: socket, auth_token: auth_token} do + request = %{ + "request_id" => 1, + "auth_token" => auth_token, + "request" => "match", + "payload" => %{ + "code" => "{var1, var2} = {1, 2}", + } + } + assert send_request(socket, request) == "# Bindings\n\nvar1 = 1\n\nvar2 = 2" + end + + test "expand request", %{socket: socket, auth_token: auth_token} do + request = %{ + "request_id" => 1, + "auth_token" => auth_token, + "request" => "expand_full", + "payload" => %{ + "buffer" => "", + "selected_code" => "unless true, do: false", + "line" => 1 + } + } + assert send_request(socket, request).expand_once == "if(true) do\n nil\nelse\n false\nend" + end + + test "docs request", %{socket: socket, auth_token: auth_token} do + request = %{ + "request_id" => 1, + "auth_token" => auth_token, + "request" => "docs", + "payload" => %{ + "buffer" => "Enum.to_list", + "line" => 1, + "column" => 6 + } + } + assert send_request(socket, request).docs.docs =~ "> Enum.to_list" + end + + test "suggestions request", %{socket: socket, auth_token: auth_token} do + request = %{ + "request_id" => 1, + "auth_token" => auth_token, + "request" => "suggestions", + "payload" => %{ + "buffer" => "List.", + "line" => 1, + "column" => 6 + } + } + assert send_request(socket, request) |> Enum.at(0) == %{type: :hint, value: "List."} + end + + test "set_context request", %{socket: socket, auth_token: auth_token} do + {_, _, _, env, cwd, _} = ContextLoader.get_state() + + assert env == "dev" + + request = %{ + "request_id" => 1, + "auth_token" => auth_token, + "request" => "set_context", + "payload" => %{ + "env" => "test", + "cwd" => cwd + } + } + send_request(socket, request) + + {_, _, _, env, _, _} = ContextLoader.get_state() + assert env == "test" + end + + test "unauthorized request", %{socket: socket} do + request = %{ + "request_id" => 1, + "auth_token" => "not the right token", + "request" => "match", + "payload" => %{ + "code" => "{var1, var2} = {1, 2}", + } + } + data = :erlang.term_to_binary(request) + response = send_and_recv(socket, data) |> :erlang.binary_to_term + + assert response.payload == nil + assert response.error == "unauthorized" + end + +end + +defmodule ElixirSense.ServerUnixSocketTest do + use ExUnit.Case + + import ExUnit.CaptureIO + import TCPHelper + + setup_all do + ["ok", "localhost", file] = capture_io(fn -> + ElixirSense.Server.start(["unix", "0", "dev"]) + end) |> String.split(":") + file = file |> String.trim |> String.to_charlist + {:ok, socket} = :gen_tcp.connect({:local, file}, 0, [:binary, active: false, packet: 4]) + + {:ok, socket: socket} + end + + test "suggestions request", %{socket: socket} do + request = %{ + "request_id" => 1, + "auth_token" => nil, + "request" => "suggestions", + "payload" => %{ + "buffer" => "List.", + "line" => 1, + "column" => 6 + } + } + assert send_request(socket, request) |> Enum.at(0) == %{type: :hint, value: "List."} + end + +end diff --git a/elixir_sense/test/test_helper.exs b/elixir_sense/test/test_helper.exs new file mode 100644 index 0000000..869559e --- /dev/null +++ b/elixir_sense/test/test_helper.exs @@ -0,0 +1 @@ +ExUnit.start() diff --git a/erl_terms.py b/erl_terms.py new file mode 100644 index 0000000..16620fe --- /dev/null +++ b/erl_terms.py @@ -0,0 +1,301 @@ +import struct +#def encode(py_struct): + +FORMAT_VERSION = '\x83' #struct.pack("b", 131) + +NEW_FLOAT_EXT = 70 # [Float64:IEEE float] +BIT_BINARY_EXT = 77 # [UInt32:Len, UInt8:Bits, Len:Data] +SMALL_INTEGER_EXT = struct.pack("b", 97) # [UInt8:Int] +INTEGER_EXT = struct.pack("b", 98) # [Int32:Int] +FLOAT_EXT = 99 # [31:Float String] Float in string format (formatted "%.20e", sscanf "%lf"). Superseded by NEW_FLOAT_EXT +ATOM_EXT = struct.pack("b", 100) # 100 [UInt16:Len, Len:AtomName] max Len is 255 +REFERENCE_EXT = 101 # 101 [atom:Node, UInt32:ID, UInt8:Creation] +PORT_EXT = 102 # [atom:Node, UInt32:ID, UInt8:Creation] +PID_EXT = 103 # [atom:Node, UInt32:ID, UInt32:Serial, UInt8:Creation] +SMALL_TUPLE_EXT = 104 # [UInt8:Arity, N:Elements] +LARGE_TUPLE_EXT = 105 # [UInt32:Arity, N:Elements] +NIL_EXT = struct.pack("b", 106) # empty list +STRING_EXT = 107 # [UInt32:Len, Len:Characters] +LIST_EXT = struct.pack("b", 108) # [UInt32:Len, Elements, Tail] +BINARY_EXT = struct.pack("b", 109) # [UInt32:Len, Len:Data] +SMALL_BIG_EXT = 110 # [UInt8:n, UInt8:Sign, n:nums] +LARGE_BIG_EXT = 111 # [UInt32:n, UInt8:Sign, n:nums] +NEW_FUN_EXT = 112 # [UInt32:Size, UInt8:Arity, 16*Uint6-MD5:Uniq, UInt32:Index, UInt32:NumFree, atom:Module, int:OldIndex, int:OldUniq, pid:Pid, NunFree*ext:FreeVars] +EXPORT_EXT = 113 # [atom:Module, atom:Function, smallint:Arity] +NEW_REFERENCE_EXT = 114 # [UInt16:Len, atom:Node, UInt8:Creation, Len*UInt32:ID] +SMALL_ATOM_EXT = 115 # [UInt8:Len, Len:AtomName] +MAP_EXT = struct.pack("b", 116) +FUN_EXT = 117 # [UInt4:NumFree, pid:Pid, atom:Module, int:Index, int:Uniq, NumFree*ext:FreeVars] +COMPRESSED = 80 # [UInt4:UncompressedSize, N:ZlibCompressedData] + +def decode(binary): + """ + >>> decode('\\x83' + SMALL_INTEGER_EXT + '\x01') + 1 + >>> decode('\\x83\\x74\\x00\\x00\\x00\\x01\\x64\\x00\\x05\\x65\\x72\\x72\\x6F\\x72\\x64\\x00\\x03\\x6E\\x69\\x6c') + {'error': None} + >>> decode(encode(-256)) + -256 + >>> decode(encode(False)) + False + >>> decode(encode(True)) + True + >>> decode(encode(None)) + >>> decode(encode("Hello")) + 'Hello' + >>> decode(encode([])) + [] + >>> decode(encode([1])) + [1] + >>> decode(encode(['a'])) + ['a'] + >>> decode(encode({'error': None, 'payload': {'active_param': 1, 'pipe_before': False}, 'signatures': [{'docs': 'docs', 'name': 'name', 'params': ['list']}, {'docs': 'snd doc', 'params': ['list']}], 'request_id': 1 })) + {'error': None, 'signatures': [{'docs': 'docs', 'params': ['list'], 'name': 'name'}, {'docs': 'snd doc', 'params': ['list']}], 'payload': {'active_param': 1, 'pipe_before': False}, 'request_id': 1} + """ + if binary[0] != FORMAT_VERSION: + raise NotImplementedError("Unable to serialize version %s" % binary[0]) + binary = binary[1:] + + (obj_size, fn) = __decode_func(binary) + return fn(binary[0: obj_size]) + +def __decode_func(binary): + if binary[0] == SMALL_INTEGER_EXT: + return (2, __decode_int) + elif binary[0] == INTEGER_EXT: + return (5, __decode_int) + elif binary[0] == BINARY_EXT: + (size, ) = struct.unpack(">L", binary[1:5]) + return (1 + 4 + size, __decode_string) + elif binary[0] == ATOM_EXT: + (size, ) = struct.unpack(">H", binary[1:3]) + return (1 + 2 + size, __decode_atom) + elif binary[0] == NIL_EXT: + return (1, __decode_list) + elif binary[0] == LIST_EXT: + (list_size, ) = struct.unpack(">L", binary[1:5]) + tmp_binary = binary[5:] + byte_size = 0 + for i in xrange(list_size): + (obj_size, fn) = __decode_func(tmp_binary) + byte_size = byte_size + obj_size + tmp_binary = tmp_binary[obj_size:] + return (1 + 4 + byte_size + 1, __decode_list) + elif binary[0] == MAP_EXT: + (map_size, ) = struct.unpack(">L", binary[1:5]) + tmp_binary = binary[5:] + byte_size = 0 + for i in xrange(map_size): + (obj_size, fn) = __decode_func(tmp_binary) + byte_size = byte_size + obj_size + tmp_binary = tmp_binary[obj_size:] + + + (obj_size, fn) = __decode_func(tmp_binary) + byte_size = byte_size + obj_size + tmp_binary = tmp_binary[obj_size:] + return (1 + 4 + byte_size , __decode_map) + else: + raise NotImplementedError("Unable to unserialize %r" % binary[0]) + +def __decode_map(binary): + """ + >>> __decode_map(__encode_map({'foo': 1})) + {'foo': 1} + >>> __decode_map(__encode_map({'foo': 'bar'})) + {'foo': 'bar'} + >>> __decode_map(__encode_map({'foo': {'bar': 4938}})) + {'foo': {'bar': 4938}} + >>> __decode_map(__encode_map({'error': None, 'payload': {'active_param': 1, 'pipe_before': False}, 'signatures': [{'docs': 'docs', 'name': 'name', 'params': ['list']}, {'docs': 'snd doc', 'params': ['list']}], 'request_id': 1 })) + {'error': None, 'signatures': [{'docs': 'docs', 'params': ['list'], 'name': 'name'}, {'docs': 'snd doc', 'params': ['list']}], 'payload': {'active_param': 1, 'pipe_before': False}, 'request_id': 1} + """ + (size,) = struct.unpack(">L", binary[1:5]) + result = {} + binary = binary[5:] + for i in xrange(size): + + (key_obj_size, key_fn) = __decode_func(binary) + key = key_fn(binary[0: key_obj_size]) + binary = binary[key_obj_size:] + + (value_obj_size, value_fn) = __decode_func(binary) + value = value_fn(binary[0: value_obj_size]) + + binary = binary[value_obj_size:] + + result.update({key: value}) + + return result + + +def __decode_list(binary): + """ + >>> __decode_list(__encode_list([])) + [] + >>> __decode_list(__encode_list(['a'])) + ['a'] + >>> __decode_list(__encode_list([1])) + [1] + >>> __decode_list(__encode_list([1, 'a'])) + [1, 'a'] + >>> __decode_list(__encode_list([True, None, 1, 'a'])) + [True, None, 1, 'a'] + """ + if binary == NIL_EXT: return [] + (size, ) = struct.unpack(">L", binary[1:5]) + result = [] + binary = binary[5:] + for i in xrange(size): + (obj_size, fn) = __decode_func(binary) + result.append(fn(binary[0: obj_size])) + binary = binary[obj_size:] + + return result + +def __decode_string(binary): + """ + >>> __decode_string(__encode_string("h")) + 'h' + """ + return binary[5:] + +def __decode_atom(binary): + """ + >>> __decode_atom(__encode_atom("nil")) + >>> __decode_atom(__encode_atom("true")) + True + >>> __decode_atom(__encode_atom("false")) + False + >>> __decode_atom(__encode_atom("my_key")) + 'my_key' + """ + atom = binary[3:] + if atom == 'true': + return True + elif atom == 'false': + return False + elif atom == 'nil': + return None + return atom + +def __decode_int(binary): + """ + >>> __decode_int(__encode_int(1)) + 1 + >>> __decode_int(__encode_int(256)) + 256 + """ + if binary[0] == 'a' : + (num,) = struct.unpack("B", binary[1]) + return num + (num,) = struct.unpack(">l", binary[1:]) + return num + +def encode(struct): + """ + >>> encode(False) + '\\x83d\\x00\\x05false' + >>> encode([]) + '\\x83j' + """ + return FORMAT_VERSION + __encoder_func(struct)(struct) + +def __encode_list(obj): + """ + >>> __encode_list([]) + 'j' + >>> __encode_list(['a']) + 'l\\x00\\x00\\x00\\x01m\\x00\\x00\\x00\\x01aj' + >>> __encode_list([1]) + 'l\\x00\\x00\\x00\\x01a\\x01j' + """ + if len(obj) == 0: + return NIL_EXT + b = struct.pack(">L", len(obj)) + for i in obj: + b = '%s%s' %(b, __encoder_func(i)(i)) + return LIST_EXT + b + NIL_EXT + +def __encode_map(obj): + """ + >>> __encode_map({'foo': 1}) + 't\\x00\\x00\\x00\\x01m\\x00\\x00\\x00\\x03fooa\\x01' + >>> __encode_map({'foo': 'bar'}) + 't\\x00\\x00\\x00\\x01m\\x00\\x00\\x00\\x03foom\\x00\\x00\\x00\\x03bar' + >>> __encode_map({'foo': {'bar': 4938}}) + 't\\x00\\x00\\x00\\x01m\\x00\\x00\\x00\\x03foot\\x00\\x00\\x00\\x01m\\x00\\x00\\x00\\x03barb\\x00\\x00\\x13J' + """ + b = struct.pack(">L", len(obj)) + for k,v in obj.iteritems(): + b = '%s%s%s' % (b, __encoder_func(k)(k), __encoder_func(v)(v)) + return MAP_EXT + b + +def __encoder_func(obj): + if isinstance(obj, str): + return __encode_string + elif isinstance(obj, bool): + return __encode_boolean + elif isinstance(obj, int): + return __encode_int + elif isinstance(obj, dict): + return __encode_map + elif isinstance(obj, list): + return __encode_list + elif obj is None: + return __encode_none + else: + raise NotImplementedError("Unable to serialize %r" % obj) + +def __encode_string(obj): + """ + >>> __encode_string("h") + 'm\\x00\\x00\\x00\\x01h' + >>> __encode_string("hello world!") + 'm\\x00\\x00\\x00\\x0chello world!' + """ + return BINARY_EXT + struct.pack(">L", len(obj)) + obj + +def __encode_none(obj): + """ + >>> __encode_none(None) + 'd\\x00\\x03nil' + """ + return __encode_atom("nil") + +def __encode_boolean(obj): + """ + >>> __encode_boolean(True) + 'd\\x00\\x04true' + >>> __encode_boolean(False) + 'd\\x00\\x05false' + """ + if obj == True: + return __encode_atom("true") + elif obj == False: + return __encode_atom("false") + else: + raise "Maybe later" + +def __encode_atom(obj): + return ATOM_EXT + struct.pack(">H", len(obj)) + (b"%s" % obj) + +def __encode_int(obj): + """ + >>> __encode_int(1) + 'a\\x01' + >>> __encode_int(256) + 'b\\x00\\x00\\x01\\x00' + """ + if 0 <= obj <= 255: + return SMALL_INTEGER_EXT + struct.pack("B", obj) + elif -2147483648 <= obj <= 2147483647: + return INTEGER_EXT + struct.pack(">l", obj) + else: + raise "Maybe later" +if __name__ == "__main__": + import doctest + doctest.testmod() + #f = open('/tmp/erl_bin.txt', 'rb') + #data = f.read() + #print(len(data)) + #print(decode(data)) diff --git a/t/fixtures/alchemist_server/valid.log b/t/fixtures/alchemist_server/valid.log index 195f3b9..f56e0b2 100644 --- a/t/fixtures/alchemist_server/valid.log +++ b/t/fixtures/alchemist_server/valid.log @@ -1 +1 @@ -ok|localhost:2433 +ok:localhost:/tmp/elixir-sense-1502654288590225000.sock