Permalink
Browse files

Change repository layout, include dependencies

  • Loading branch information...
1 parent 1375a75 commit 120fabea46233615fc98606cefa8741cd47026fd @xolox committed May 25, 2011
View
4 INSTALL.md
@@ -1,3 +1,3 @@
-**Note to GitHub users:** This git repository doesn't include all of the auto-load scripts required by the plug-in. The easiest way to get the plug-in including its dependencies is to download [the latest ZIP archive](http://peterodding.com/code/vim/downloads/lua-inspect).
+If you're looking for the simplest way to get the plug-in up and running, download [the latest ZIP archive](http://peterodding.com/code/vim/downloads/lua-inspect.zip) from [Vim Online](http://www.vim.org/scripts/script.php?script_id=3169), unzip that in `~/.vim/` (on UNIX) or `%USERPROFILE%\vimfiles` (on Windows) and you're good to go.
-If you're using [Pathogen](http://www.vim.org/scripts/script.php?script_id=2332) and want to keep the plug-in up to date using git, you can use the [luainspect.vim mirror repository](https://github.com/vim-scripts/luainspect.vim) by [vim-scripts.org](http://vim-scripts.org/) which is synchronized with the latest release on [Vim Online](http://www.vim.org/scripts/script.php?script_id=3169) multiple times a day.
+If you're using git and/or [Pathogen](http://www.vim.org/scripts/script.php?script_id=2332), [Vundle](https://github.com/gmarik/vundle) or a similar plug-in manager and want to keep the plug-in up to date using git, you can use the GitHub repository directly, it should just work.
View
2 README.md
@@ -20,7 +20,7 @@ The Vim plug-in `luainspect.vim` uses the [LuaInspect](http://lua-users.org/wiki
## Installation
-Unzip the most recent [ZIP archive](http://peterodding.com/code/vim/downloads/lua-inspect) file inside your Vim profile directory (usually this is `~/.vim` on UNIX and `%USERPROFILE%\vimfiles` on Windows), restart Vim and execute the command `:helptags ~/.vim/doc` (use `:helptags ~\vimfiles\doc` instead on Windows). Now try it out: Edit a Lua file and within a few seconds semantic highlighting should be enabled automatically!
+Unzip the most recent [ZIP archive](http://peterodding.com/code/vim/downloads/lua-inspect.zip) file inside your Vim profile directory (usually this is `~/.vim` on UNIX and `%USERPROFILE%\vimfiles` on Windows), restart Vim and execute the command `:helptags ~/.vim/doc` (use `:helptags ~\vimfiles\doc` instead on Windows). Now try it out: Edit a Lua file and within a few seconds semantic highlighting should be enabled automatically!
Note that on Windows a command prompt window pops up whenever LuaInspect is run as an external process. If this bothers you then you can install my [shell.vim](http://peterodding.com/code/vim/shell/) plug-in which includes a [DLL](http://en.wikipedia.org/wiki/Dynamic-link_library) that works around this issue. Once you've installed both plug-ins it should work out of the box!
View
40 autoload.vim → autoload/xolox/luainspect.vim
@@ -1,19 +1,19 @@
" Vim script.
" Author: Peter Odding <peter@peterodding.com>
-" Last Change: October 9, 2010
+" Last Change: May 25, 2011
" URL: http://peterodding.com/code/vim/lua-inspect/
" License: MIT
let s:script = expand('<sfile>:p:~')
-function! luainspect#auto_enable() " {{{1
+function! xolox#luainspect#auto_enable() " {{{1
if !&diff && !exists('b:luainspect_disabled')
" Disable easytags.vim because it doesn't play nice with luainspect.vim!
let b:easytags_nohl = 1
" Define buffer local mappings for rename / goto definition features.
- inoremap <buffer> <silent> <F2> <C-o>:call luainspect#make_request('rename')<CR>
- nnoremap <buffer> <silent> <F2> :call luainspect#make_request('rename')<CR>
- nnoremap <buffer> <silent> gd :call luainspect#make_request('goto')<CR>
+ inoremap <buffer> <silent> <F2> <C-o>:call xolox#luainspect#make_request('rename')<CR>
+ nnoremap <buffer> <silent> <F2> :call xolox#luainspect#make_request('rename')<CR>
+ nnoremap <buffer> <silent> gd :call xolox#luainspect#make_request('goto')<CR>
" Enable balloon evaluation / dynamic tool tips.
setlocal ballooneval balloonexpr=LuaInspectToolTip()
" Install automatic commands to update the highlighting.
@@ -23,7 +23,7 @@ function! luainspect#auto_enable() " {{{1
endif
endfunction
-function! luainspect#highlight_cmd(disable) " {{{1
+function! xolox#luainspect#highlight_cmd(disable) " {{{1
if a:disable
call s:clear_previous_matches()
unlet! b:luainspect_input
@@ -32,12 +32,12 @@ function! luainspect#highlight_cmd(disable) " {{{1
let b:luainspect_disabled = 1
else
unlet! b:luainspect_disabled
- call luainspect#make_request('highlight')
+ call xolox#luainspect#make_request('highlight')
endif
endfunction
-function! luainspect#make_request(action) " {{{1
- let starttime = xolox#timer#start()
+function! xolox#luainspect#make_request(action) " {{{1
+ let starttime = xolox#misc#timer#start()
let bufnr = a:action != 'tooltip' ? bufnr('%') : v:beval_bufnr
let bufname = bufname(bufnr)
if bufname != ''
@@ -72,41 +72,41 @@ function! luainspect#make_request(action) " {{{1
let error_cmd = 'syntax match luaInspectSyntaxError /\%%>%il\%%<%il.*/ containedin=ALLBUT,lua*Comment*'
execute printf(error_cmd, linenum - 1, (linenum2 ? linenum2 : line('$')) + 1)
endif
- call xolox#timer#stop("%s: Found a syntax error in %s in %s.", s:script, friendlyname, starttime)
+ call xolox#misc#timer#stop("%s: Found a syntax error in %s in %s.", s:script, friendlyname, starttime)
" But always let the user know that a syntax error exists.
- call xolox#warning("Syntax error around line %i in %s: %s", linenum, friendlyname, b:luainspect_syntax_error)
+ call xolox#misc#msg#warn("Syntax error around line %i in %s: %s", linenum, friendlyname, b:luainspect_syntax_error)
return
endif
unlet! b:luainspect_syntax_error
if response == 'highlight'
call s:define_default_styles()
call s:clear_previous_matches()
call s:highlight_variables()
- call xolox#timer#stop("%s: Highlighted variables in %s in %s.", s:script, friendlyname, starttime)
+ call xolox#misc#timer#stop("%s: Highlighted variables in %s in %s.", s:script, friendlyname, starttime)
elseif response == 'goto'
if len(b:luainspect_output) < 3
- call xolox#warning("No variable under cursor!")
+ call xolox#misc#msg#warn("No variable under cursor!")
else
let linenum = b:luainspect_output[1] + 0
let colnum = b:luainspect_output[2] + 1
call setpos('.', [0, linenum, colnum, 0])
- call xolox#timer#stop("%s: Jumped to definition in %s in %s.", s:script, friendlyname, starttime)
+ call xolox#misc#timer#stop("%s: Jumped to definition in %s in %s.", s:script, friendlyname, starttime)
if &verbose == 0
" Clear previous "No variable under cursor!" message to avoid confusion.
- call xolox#message("")
+ call xolox#misc#msg#info("")
endif
endif
elseif response == 'tooltip'
if len(b:luainspect_output) > 1
- call xolox#timer#stop("%s: Rendered tool tip for %s in %s.", s:script, friendlyname, starttime)
+ call xolox#misc#timer#stop("%s: Rendered tool tip for %s in %s.", s:script, friendlyname, starttime)
return join(b:luainspect_output[1:-1], "\n")
endif
elseif response == 'rename'
if len(b:luainspect_output) > 1
- call xolox#timer#stop("%s: Prepared for rename in %s in %s.", s:script, friendlyname, starttime)
+ call xolox#misc#timer#stop("%s: Prepared for rename in %s in %s.", s:script, friendlyname, starttime)
call s:rename_variable()
else
- call xolox#warning("No variable under cursor!")
+ call xolox#misc#msg#warn("No variable under cursor!")
endif
endif
endif
@@ -278,15 +278,15 @@ function! s:rename_variable() " {{{1
let num_renamed += 1
endfor
let msg = "Renamed %i occurrences of %s to %s"
- call xolox#message(msg, num_renamed, oldname, newname)
+ call xolox#misc#msg#info(msg, num_renamed, oldname, newname)
endif
endfunction
function! s:check_output(line, pattern) " {{{1
if match(a:line, a:pattern) >= 0
return 1
else
- call xolox#warning("Invalid output from luainspect4vim.lua: '%s'", strtrans(a:line))
+ call xolox#misc#msg#warn("Invalid output from luainspect4vim.lua: '%s'", strtrans(a:line))
return 0
endif
endfunction
View
177 doc/luainspect.txt
@@ -0,0 +1,177 @@
+*luainspect.txt* Semantic highlighting for Lua in Vim
+
+The Vim plug-in 'luainspect.vim' uses the LuaInspect [1] tool to
+(automatically) perform semantic highlighting of variables in Lua source code.
+It was inspired by lua2-mode [2] (for Emacs [3]) and the SciTE [4] plug-in
+included with LuaInspect. In addition to the semantic highlighting the
+following features are currently supported:
+
+ - Press '<F2>' with the text cursor on a variable and the plug-in will prompt
+ you to rename the variable.
+
+ - Press 'gd' (in normal mode) with the text cursor on a variable and you'll
+ jump to its declaration / first occurrence.
+
+ - When you hover over a variable with the mouse cursor in graphical Vim,
+ information about the variable is displayed in a tooltip.
+
+ - If the text cursor is on a variable while the highlighting is refreshed then
+ all occurrences of the variable will be marked in the style of Vim's
+ cursorline option (see |'cursorline'|).
+
+ - When luainspect reports a wrong argument count for a function call the text
+ will be highlighted with a green underline. When you hover over the
+ highlighted text a tooltip shows the associated warning message.
+
+ - When LuaInspect reports warnings about unused variables, wrong argument
+ counts, etc. they are shown in a location list window (see |location-list|).
+
+ - When a syntax error is found (during highlighting or using the rename
+ functionality) the lines where the error is reported will be marked like a
+ spelling error.
+
+ Screenshot of semantic highlighting, see reference [5]
+
+==============================================================================
+ *luainspect-installation*
+Installation ~
+
+Unzip the most recent ZIP archive [6] file inside your Vim profile directory
+(usually this is '~/.vim' on UNIX and '%USERPROFILE%\vimfiles' on Windows),
+restart Vim and execute the command ':helptags ~/.vim/doc' (use ':helptags
+~\vimfiles\doc' instead on Windows). Now try it out: Edit a Lua file and
+within a few seconds semantic highlighting should be enabled automatically!
+
+Note that on Windows a command prompt window pops up whenever LuaInspect is
+run as an external process. If this bothers you then you can install my
+shell.vim [7] plug-in which includes a DLL [8] that works around this issue.
+Once you've installed both plug-ins it should work out of the box!
+
+==============================================================================
+ *luainspect-usage*
+Usage ~
+
+When you open any Lua file the semantic highlighting should be enabled
+automatically within a few seconds, so you don't have to configure anything if
+you're happy with the defaults.
+
+------------------------------------------------------------------------------
+ *:luainspect-command*
+The ':LuaInspect' command ~
+
+You don't need to use this command unless you've disabled automatic
+highlighting using 'g:lua_inspect_events'. When you execute this command the
+plug-in runs the LuaInspect tool and then highlights all variables in the
+current buffer using one of the following highlighting groups:
+
+ - luaInspectGlobalDefined
+
+ - luaInspectGlobalUndefined
+
+ - luaInspectLocalUnused
+
+ - luaInspectLocalMutated
+
+ - luaInspectUpValue
+
+ - luaInspectParam
+
+ - luaInspectLocal
+
+ - luaInspectFieldDefined
+
+ - luaInspectFieldUndefined
+
+ - luaInspectSelectedVariable
+
+ - luaInspectWrongArgCount
+
+ - luaInspectSyntaxError
+
+If you don't like one or more of the default styles the Vim documentation
+describes how to change them (see |:hi-default|). If you want to disable the
+semantic highlighting in a specific Vim buffer execute ':LuaInspect!' in that
+buffer. When you want to reenable the highlighting execute ':LuaInspect'
+again, but now without the bang (see |:command-bang|).
+
+------------------------------------------------------------------------------
+ *g:loaded_luainspect-option*
+The 'g:loaded_luainspect' option ~
+
+This variable isn't really an option but if you want to avoid loading the
+'luainspect.vim' plug-in you can set this variable to any value in your |vimrc|
+script:
+>
+ :let g:loaded_luainspect = 1
+
+------------------------------------------------------------------------------
+ *luainspect-g:lua_inspect_warnings-option*
+The 'g:lua_inspect_warnings' option ~
+
+When LuaInspect reports warnings about unused variables, wrong argument
+counts, etc. they are automatically shown in a location list window (see
+|location-list|). If you don't like this add the following to your |vimrc| script:
+>
+ :let g:lua_inspect_warnings = 0
+
+------------------------------------------------------------------------------
+ *luainspect-g:lua_inspect_events-option*
+The 'g:lua_inspect_events' option ~
+
+By default semantic highlighting is automatically enabled after a short
+timeout and when you save a buffer. If you want to disable automatic
+highlighting altogether add the following to your |vimrc| script:
+>
+ :let g:lua_inspect_events = ''
+
+You can also add events, for example if you also want to run ':LuaInspect' the
+moment you edit a Lua file then try this:
+>
+ :let g:lua_inspect_events = 'CursorHold,CursorHoldI,BufReadPost,BufWritePost'
+
+Note that this only works when the plug-in is loaded (or reloaded) after
+setting the 'g:lua_inspect_events' option.
+
+------------------------------------------------------------------------------
+ *luainspect-g:lua_inspect_internal-option*
+The 'g:lua_inspect_internal' option ~
+
+The plug-in uses the Lua interface for Vim when available so that it doesn't
+have to run LuaInspect as an external program (which can slow things down). If
+you insist on running LuaInspect as an external program you can set this
+variable to false (0) in your |vimrc| script:
+>
+ :let g:lua_inspect_internal = 0
+
+==============================================================================
+ *luainspect-contact*
+Contact ~
+
+If you have questions, bug reports, suggestions, etc. the author can be
+contacted at peter@peterodding.com. The latest version is available at
+http://peterodding.com/code/vim/lua-inspect/ and http://github.com/xolox/vim-lua-inspect.
+If you like this plug-in please vote for it on Vim Online [9].
+
+==============================================================================
+ *luainspect-license*
+License ~
+
+This software is licensed under the MIT license [10]. Copyright 2010 Peter
+Odding <peter@peterodding.com>.
+
+==============================================================================
+ *luainspect-references*
+References ~
+
+[1] http://lua-users.org/wiki/LuaInspect
+[2] http://www.enyo.de/fw/software/lua-emacs/lua2-mode.html
+[3] http://www.gnu.org/software/emacs/
+[4] http://www.scintilla.org/SciTE.html
+[5] http://peterodding.com/code/vim/luainspect/screenshot.png
+[6] http://peterodding.com/code/vim/downloads/lua-inspect.zip
+[7] http://peterodding.com/code/vim/shell/
+[8] http://en.wikipedia.org/wiki/Dynamic-link_library
+[9] http://www.vim.org/scripts/script.php?script_id=3169
+[10] http://en.wikipedia.org/wiki/MIT_License
+
+vim: ft=help
View
747 misc/luainspect/gg.lua
@@ -0,0 +1,747 @@
+----------------------------------------------------------------------
+-- Metalua.
+--
+-- Summary: parser generator. Collection of higher order functors,
+-- which allow to build and combine parsers. Relies on a lexer
+-- that supports the same API as the one exposed in mll.lua.
+--
+----------------------------------------------------------------------
+--
+-- Copyright (c) 2006-2008, Fabien Fleutot <metalua@gmail.com>.
+--
+-- This software is released under the MIT Licence, see licence.txt
+-- for details.
+--
+----------------------------------------------------------------------
+
+--------------------------------------------------------------------------------
+--
+-- Exported API:
+--
+-- Parser generators:
+-- * [gg.sequence()]
+-- * [gg.multisequence()]
+-- * [gg.expr()]
+-- * [gg.list()]
+-- * [gg.onkeyword()]
+-- * [gg.optkeyword()]
+--
+-- Other functions:
+-- * [gg.parse_error()]
+-- * [gg.make_parser()]
+-- * [gg.is_parser()]
+--
+--------------------------------------------------------------------------------
+
+module("gg", package.seeall)
+
+-------------------------------------------------------------------------------
+-- parser metatable, which maps __call to method parse, and adds some
+-- error tracing boilerplate.
+-------------------------------------------------------------------------------
+local parser_metatable = { }
+function parser_metatable.__call (parser, lx, ...)
+ --printf ("Call parser %q of type %q", parser.name or "?", parser.kind)
+ if mlc.metabugs then
+ return parser:parse (lx, ...)
+ --local x = parser:parse (lx, ...)
+ --printf ("Result of parser %q: %s",
+ -- parser.name or "?",
+ -- _G.table.tostring(x, "nohash", 80))
+ --return x
+ else
+ local li = lx:lineinfo_right() or { "?", "?", "?", "?" }
+ local status, ast = pcall (parser.parse, parser, lx, ...)
+ if status then return ast else
+ error (string.format ("%s\n - (l.%s, c.%s, k.%s) in parser %s",
+ ast:strmatch "gg.lua:%d+: (.*)" or ast,
+ li[1], li[2], li[3], parser.name or parser.kind))
+ end
+ end
+end
+
+-------------------------------------------------------------------------------
+-- Turn a table into a parser, mainly by setting the metatable.
+-------------------------------------------------------------------------------
+function make_parser(kind, p)
+ p.kind = kind
+ if not p.transformers then p.transformers = { } end
+ function p.transformers:add (x)
+ table.insert (self, x)
+ end
+ setmetatable (p, parser_metatable)
+ return p
+end
+
+-------------------------------------------------------------------------------
+-- Return true iff [x] is a parser.
+-- If it's a gg-generated parser, return the name of its kind.
+-------------------------------------------------------------------------------
+function is_parser (x)
+ return type(x)=="function" or getmetatable(x)==parser_metatable and x.kind
+end
+
+-------------------------------------------------------------------------------
+-- Parse a sequence, without applying builder nor transformers
+-------------------------------------------------------------------------------
+local function raw_parse_sequence (lx, p)
+ local r = { }
+ for i=1, #p do
+ e=p[i]
+ if type(e) == "string" then
+ if not lx:is_keyword (lx:next(), e) then
+ parse_error (lx, "Keyword '%s' expected", e) end
+ elseif is_parser (e) then
+ table.insert (r, e (lx))
+ else
+ gg.parse_error (lx,"Sequence `%s': element #%i is not a string "..
+ "nor a parser: %s",
+ p.name, i, table.tostring(e))
+ end
+ end
+ return r
+end
+
+-------------------------------------------------------------------------------
+-- Parse a multisequence, without applying multisequence transformers.
+-- The sequences are completely parsed.
+-------------------------------------------------------------------------------
+local function raw_parse_multisequence (lx, sequence_table, default)
+ local seq_parser = sequence_table[lx:is_keyword(lx:peek())]
+ if seq_parser then return seq_parser (lx)
+ elseif default then return default (lx)
+ else return false end
+end
+
+-------------------------------------------------------------------------------
+-- Applies all transformers listed in parser on ast.
+-------------------------------------------------------------------------------
+local function transform (ast, parser, fli, lli)
+ if parser.transformers then
+ for _, t in ipairs (parser.transformers) do ast = t(ast) or ast end
+ end
+ if type(ast) == 'table'then
+ local ali = ast.lineinfo
+ if not ali or ali.first~=fli or ali.last~=lli then
+ ast.lineinfo = { first = fli, last = lli }
+ end
+ end
+ return ast
+end
+
+-------------------------------------------------------------------------------
+-- Generate a tracable parsing error (not implemented yet)
+-------------------------------------------------------------------------------
+function parse_error(lx, fmt, ...)
+ local li = lx:lineinfo_left() or {-1,-1,-1, "<unknown file>"}
+ local msg = string.format("line %i, char %i: "..fmt, li[1], li[2], ...)
+ local src = lx.src
+ if li[3]>0 and src then
+ local i, j = li[3], li[3]
+ while src:sub(i,i) ~= '\n' and i>=0 do i=i-1 end
+ while src:sub(j,j) ~= '\n' and j<=#src do j=j+1 end
+ local srcline = src:sub (i+1, j-1)
+ local idx = string.rep (" ", li[2]).."^"
+ msg = string.format("%s\n>>> %s\n>>> %s", msg, srcline, idx)
+ end
+ error(msg)
+end
+
+-------------------------------------------------------------------------------
+--
+-- Sequence parser generator
+--
+-------------------------------------------------------------------------------
+-- Input fields:
+--
+-- * [builder]: how to build an AST out of sequence parts. let [x] be the list
+-- of subparser results (keywords are simply omitted). [builder] can be:
+-- - [nil], in which case the result of parsing is simply [x]
+-- - a string, which is then put as a tag on [x]
+-- - a function, which takes [x] as a parameter and returns an AST.
+--
+-- * [name]: the name of the parser. Used for debug messages
+--
+-- * [transformers]: a list of AST->AST functions, applied in order on ASTs
+-- returned by the parser.
+--
+-- * Table-part entries corresponds to keywords (strings) and subparsers
+-- (function and callable objects).
+--
+-- After creation, the following fields are added:
+-- * [parse] the parsing function lexer->AST
+-- * [kind] == "sequence"
+-- * [name] is set, if it wasn't in the input.
+--
+-------------------------------------------------------------------------------
+function sequence (p)
+ make_parser ("sequence", p)
+
+ -------------------------------------------------------------------
+ -- Parsing method
+ -------------------------------------------------------------------
+ function p:parse (lx)
+ -- Raw parsing:
+ local fli = lx:lineinfo_right()
+ local seq = raw_parse_sequence (lx, self)
+ local lli = lx:lineinfo_left()
+
+ -- Builder application:
+ local builder, tb = self.builder, type (self.builder)
+ if tb == "string" then seq.tag = builder
+ elseif tb == "function" or builder and builder.__call then seq = builder(seq)
+ elseif builder == nil then -- nothing
+ else error ("Invalid builder of type "..tb.." in sequence") end
+ seq = transform (seq, self, fli, lli)
+ assert (not seq or seq.lineinfo)
+ return seq
+ end
+
+ -------------------------------------------------------------------
+ -- Construction
+ -------------------------------------------------------------------
+ -- Try to build a proper name
+ if not p.name and type(p[1])=="string" then
+ p.name = p[1].." ..."
+ if type(p[#p])=="string" then p.name = p.name .. " " .. p[#p] end
+ else
+ p.name = "<anonymous>"
+ end
+
+ return p
+end --</sequence>
+
+
+-------------------------------------------------------------------------------
+--
+-- Multiple, keyword-driven, sequence parser generator
+--
+-------------------------------------------------------------------------------
+-- in [p], useful fields are:
+--
+-- * [transformers]: as usual
+--
+-- * [name]: as usual
+--
+-- * Table-part entries must be sequence parsers, or tables which can
+-- be turned into a sequence parser by [gg.sequence]. These
+-- sequences must start with a keyword, and this initial keyword
+-- must be different for each sequence. The table-part entries will
+-- be removed after [gg.multisequence] returns.
+--
+-- * [default]: the parser to run if the next keyword in the lexer is
+-- none of the registered initial keywords. If there's no default
+-- parser and no suitable initial keyword, the multisequence parser
+-- simply returns [false].
+--
+-- After creation, the following fields are added:
+--
+-- * [parse] the parsing function lexer->AST
+--
+-- * [sequences] the table of sequences, indexed by initial keywords.
+--
+-- * [add] method takes a sequence parser or a config table for
+-- [gg.sequence], and adds/replaces the corresponding sequence
+-- parser. If the keyword was already used, the former sequence is
+-- removed and a warning is issued.
+--
+-- * [get] method returns a sequence by its initial keyword
+--
+-- * [kind] == "multisequence"
+--
+-------------------------------------------------------------------------------
+function multisequence (p)
+ make_parser ("multisequence", p)
+
+ -------------------------------------------------------------------
+ -- Add a sequence (might be just a config table for [gg.sequence])
+ -------------------------------------------------------------------
+ function p:add (s)
+ -- compile if necessary:
+ local keyword = s[1]
+ if not is_parser(s) then sequence(s) end
+ if is_parser(s) ~= 'sequence' or type(keyword) ~= "string" then
+ if self.default then -- two defaults
+ error ("In a multisequence parser, all but one sequences "..
+ "must start with a keyword")
+ else self.default = s end -- first default
+ elseif self.sequences[keyword] then -- duplicate keyword
+ eprintf (" *** Warning: keyword %q overloaded in multisequence ***", keyword)
+ self.sequences[keyword] = s
+ else -- newly caught keyword
+ self.sequences[keyword] = s
+ end
+ end -- </multisequence.add>
+
+ -------------------------------------------------------------------
+ -- Get the sequence starting with this keyword. [kw :: string]
+ -------------------------------------------------------------------
+ function p:get (kw) return self.sequences [kw] end
+
+ -------------------------------------------------------------------
+ -- Remove the sequence starting with keyword [kw :: string]
+ -------------------------------------------------------------------
+ function p:del (kw)
+ if not self.sequences[kw] then
+ eprintf("*** Warning: trying to delete sequence starting "..
+ "with %q from a multisequence having no such "..
+ "entry ***", kw) end
+ local removed = self.sequences[kw]
+ self.sequences[kw] = nil
+ return removed
+ end
+
+ -------------------------------------------------------------------
+ -- Parsing method
+ -------------------------------------------------------------------
+ function p:parse (lx)
+ local fli = lx:lineinfo_right()
+ local x = raw_parse_multisequence (lx, self.sequences, self.default)
+ local lli = lx:lineinfo_left()
+ return transform (x, self, fli, lli)
+ end
+
+ -------------------------------------------------------------------
+ -- Construction
+ -------------------------------------------------------------------
+ -- Register the sequences passed to the constructor. They're going
+ -- from the array part of the parser to the hash part of field
+ -- [sequences]
+ p.sequences = { }
+ for i=1, #p do p:add (p[i]); p[i] = nil end
+
+ -- FIXME: why is this commented out?
+ --if p.default and not is_parser(p.default) then sequence(p.default) end
+ return p
+end --</multisequence>
+
+
+-------------------------------------------------------------------------------
+--
+-- Expression parser generator
+--
+-------------------------------------------------------------------------------
+--
+-- Expression configuration relies on three tables: [prefix], [infix]
+-- and [suffix]. Moreover, the primary parser can be replaced by a
+-- table: in this case the [primary] table will be passed to
+-- [gg.multisequence] to create a parser.
+--
+-- Each of these tables is a modified multisequence parser: the
+-- differences with respect to regular multisequence config tables are:
+--
+-- * the builder takes specific parameters:
+-- - for [prefix], it takes the result of the prefix sequence parser,
+-- and the prefixed expression
+-- - for [infix], it takes the left-hand-side expression, the results
+-- of the infix sequence parser, and the right-hand-side expression.
+-- - for [suffix], it takes the suffixed expression, and theresult
+-- of the suffix sequence parser.
+--
+-- * the default field is a list, with parameters:
+-- - [parser] the raw parsing function
+-- - [transformers], as usual
+-- - [prec], the operator's precedence
+-- - [assoc] for [infix] table, the operator's associativity, which
+-- can be "left", "right" or "flat" (default to left)
+--
+-- In [p], useful fields are:
+-- * [transformers]: as usual
+-- * [name]: as usual
+-- * [primary]: the atomic expression parser, or a multisequence config
+-- table (mandatory)
+-- * [prefix]: prefix operators config table, see above.
+-- * [infix]: infix operators config table, see above.
+-- * [suffix]: suffix operators config table, see above.
+--
+-- After creation, these fields are added:
+-- * [kind] == "expr"
+-- * [parse] as usual
+-- * each table is turned into a multisequence, and therefore has an
+-- [add] method
+--
+-------------------------------------------------------------------------------
+function expr (p)
+ make_parser ("expr", p)
+
+ -------------------------------------------------------------------
+ -- parser method.
+ -- In addition to the lexer, it takes an optional precedence:
+ -- it won't read expressions whose precedence is lower or equal
+ -- to [prec].
+ -------------------------------------------------------------------
+ function p:parse (lx, prec)
+ prec = prec or 0
+
+ ------------------------------------------------------
+ -- Extract the right parser and the corresponding
+ -- options table, for (pre|in|suff)fix operators.
+ -- Options include prec, assoc, transformers.
+ ------------------------------------------------------
+ local function get_parser_info (tab)
+ local p2 = tab:get (lx:is_keyword (lx:peek()))
+ if p2 then -- keyword-based sequence found
+ local function parser(lx) return raw_parse_sequence(lx, p2) end
+ return parser, p2
+ else -- Got to use the default parser
+ local d = tab.default
+ if d then return d.parse or d.parser, d
+ else return false, false end
+ end
+ end
+
+ ------------------------------------------------------
+ -- Look for a prefix sequence. Multiple prefixes are
+ -- handled through the recursive [p.parse] call.
+ -- Notice the double-transform: one for the primary
+ -- expr, and one for the one with the prefix op.
+ ------------------------------------------------------
+ local function handle_prefix ()
+ local fli = lx:lineinfo_right()
+ local p2_func, p2 = get_parser_info (self.prefix)
+ local op = p2_func and p2_func (lx)
+ if op then -- Keyword-based sequence found
+ local ili = lx:lineinfo_right() -- Intermediate LineInfo
+ local e = p2.builder (op, self:parse (lx, p2.prec))
+ local lli = lx:lineinfo_left()
+ return transform (transform (e, p2, ili, lli), self, fli, lli)
+ else -- No prefix found, get a primary expression
+ local e = self.primary(lx)
+ local lli = lx:lineinfo_left()
+ return transform (e, self, fli, lli)
+ end
+ end --</expr.parse.handle_prefix>
+
+ ------------------------------------------------------
+ -- Look for an infix sequence+right-hand-side operand.
+ -- Return the whole binary expression result,
+ -- or false if no operator was found.
+ ------------------------------------------------------
+ local function handle_infix (e)
+ local p2_func, p2 = get_parser_info (self.infix)
+ if not p2 then return false end
+
+ -----------------------------------------
+ -- Handle flattening operators: gather all operands
+ -- of the series in [list]; when a different operator
+ -- is found, stop, build from [list], [transform] and
+ -- return.
+ -----------------------------------------
+ if (not p2.prec or p2.prec>prec) and p2.assoc=="flat" then
+ local fli = lx:lineinfo_right()
+ local pflat, list = p2, { e }
+ repeat
+ local op = p2_func(lx)
+ if not op then break end
+ table.insert (list, self:parse (lx, p2.prec))
+ local _ -- We only care about checking that p2==pflat
+ _, p2 = get_parser_info (self.infix)
+ until p2 ~= pflat
+ local e2 = pflat.builder (list)
+ local lli = lx:lineinfo_left()
+ return transform (transform (e2, pflat, fli, lli), self, fli, lli)
+
+ -----------------------------------------
+ -- Handle regular infix operators: [e] the LHS is known,
+ -- just gather the operator and [e2] the RHS.
+ -- Result goes in [e3].
+ -----------------------------------------
+ elseif p2.prec and p2.prec>prec or
+ p2.prec==prec and p2.assoc=="right" then
+ local fli = e.lineinfo.first -- lx:lineinfo_right()
+ local op = p2_func(lx)
+ if not op then return false end
+ local e2 = self:parse (lx, p2.prec)
+ local e3 = p2.builder (e, op, e2)
+ local lli = lx:lineinfo_left()
+ return transform (transform (e3, p2, fli, lli), self, fli, lli)
+
+ -----------------------------------------
+ -- Check for non-associative operators, and complain if applicable.
+ -----------------------------------------
+ elseif p2.assoc=="none" and p2.prec==prec then
+ parser_error (lx, "non-associative operator!")
+
+ -----------------------------------------
+ -- No infix operator suitable at that precedence
+ -----------------------------------------
+ else return false end
+
+ end --</expr.parse.handle_infix>
+
+ ------------------------------------------------------
+ -- Look for a suffix sequence.
+ -- Return the result of suffix operator on [e],
+ -- or false if no operator was found.
+ ------------------------------------------------------
+ local function handle_suffix (e)
+ -- FIXME bad fli, must take e.lineinfo.first
+ local p2_func, p2 = get_parser_info (self.suffix)
+ if not p2 then return false end
+ if not p2.prec or p2.prec>=prec then
+ --local fli = lx:lineinfo_right()
+ local fli = e.lineinfo.first
+ local op = p2_func(lx)
+ if not op then return false end
+ local lli = lx:lineinfo_left()
+ e = p2.builder (e, op)
+ e = transform (transform (e, p2, fli, lli), self, fli, lli)
+ return e
+ end
+ return false
+ end --</expr.parse.handle_suffix>
+
+ ------------------------------------------------------
+ -- Parser body: read suffix and (infix+operand)
+ -- extensions as long as we're able to fetch more at
+ -- this precedence level.
+ ------------------------------------------------------
+ local e = handle_prefix()
+ repeat
+ local x = handle_suffix (e); e = x or e
+ local y = handle_infix (e); e = y or e
+ until not (x or y)
+
+ -- No transform: it already happened in operators handling
+ return e
+ end --</expr.parse>
+
+ -------------------------------------------------------------------
+ -- Construction
+ -------------------------------------------------------------------
+ if not p.primary then p.primary=p[1]; p[1]=nil end
+ for _, t in ipairs{ "primary", "prefix", "infix", "suffix" } do
+ if not p[t] then p[t] = { } end
+ if not is_parser(p[t]) then multisequence(p[t]) end
+ end
+ function p:add(...) return self.primary:add(...) end
+ return p
+end --</expr>
+
+
+-------------------------------------------------------------------------------
+--
+-- List parser generator
+--
+-------------------------------------------------------------------------------
+-- In [p], the following fields can be provided in input:
+--
+-- * [builder]: takes list of subparser results, returns AST
+-- * [transformers]: as usual
+-- * [name]: as usual
+--
+-- * [terminators]: list of strings representing the keywords which
+-- might mark the end of the list. When non-empty, the list is
+-- allowed to be empty. A string is treated as a single-element
+-- table, whose element is that string, e.g. ["do"] is the same as
+-- [{"do"}].
+--
+-- * [separators]: list of strings representing the keywords which can
+-- separate elements of the list. When non-empty, one of these
+-- keyword has to be found between each element. Lack of a separator
+-- indicates the end of the list. A string is treated as a
+-- single-element table, whose element is that string, e.g. ["do"]
+-- is the same as [{"do"}]. If [terminators] is empty/nil, then
+-- [separators] has to be non-empty.
+--
+-- After creation, the following fields are added:
+-- * [parse] the parsing function lexer->AST
+-- * [kind] == "list"
+--
+-------------------------------------------------------------------------------
+function list (p)
+ make_parser ("list", p)
+
+ -------------------------------------------------------------------
+ -- Parsing method
+ -------------------------------------------------------------------
+ function p:parse (lx)
+
+ ------------------------------------------------------
+ -- Used to quickly check whether there's a terminator
+ -- or a separator immediately ahead
+ ------------------------------------------------------
+ local function peek_is_in (keywords)
+ return keywords and lx:is_keyword(lx:peek(), unpack(keywords)) end
+
+ local x = { }
+ local fli = lx:lineinfo_right()
+
+ -- if there's a terminator to start with, don't bother trying
+ if not peek_is_in (self.terminators) then
+ repeat table.insert (x, self.primary (lx)) -- read one element
+ until
+ -- First reason to stop: There's a separator list specified,
+ -- and next token isn't one. Otherwise, consume it with [lx:next()]
+ self.separators and not(peek_is_in (self.separators) and lx:next()) or
+ -- Other reason to stop: terminator token ahead
+ peek_is_in (self.terminators) or
+ -- Last reason: end of file reached
+ lx:peek().tag=="Eof"
+ end
+
+ local lli = lx:lineinfo_left()
+
+ -- Apply the builder. It can be a string, or a callable value,
+ -- or simply nothing.
+ local b = self.builder
+ if b then
+ if type(b)=="string" then x.tag = b -- b is a string, use it as a tag
+ elseif type(b)=="function" then x=b(x)
+ else
+ local bmt = getmetatable(b)
+ if bmt and bmt.__call then x=b(x) end
+ end
+ end
+ return transform (x, self, fli, lli)
+ end --</list.parse>
+
+ -------------------------------------------------------------------
+ -- Construction
+ -------------------------------------------------------------------
+ if not p.primary then p.primary = p[1]; p[1] = nil end
+ if type(p.terminators) == "string" then p.terminators = { p.terminators }
+ elseif p.terminators and #p.terminators == 0 then p.terminators = nil end
+ if type(p.separators) == "string" then p.separators = { p.separators }
+ elseif p.separators and #p.separators == 0 then p.separators = nil end
+
+ return p
+end --</list>
+
+
+-------------------------------------------------------------------------------
+--
+-- Keyword-conditionned parser generator
+--
+-------------------------------------------------------------------------------
+--
+-- Only apply a parser if a given keyword is found. The result of
+-- [gg.onkeyword] parser is the result of the subparser (modulo
+-- [transformers] applications).
+--
+-- lineinfo: the keyword is *not* included in the boundaries of the
+-- resulting lineinfo. A review of all usages of gg.onkeyword() in the
+-- implementation of metalua has shown that it was the appropriate choice
+-- in every case.
+--
+-- Input fields:
+--
+-- * [name]: as usual
+--
+-- * [transformers]: as usual
+--
+-- * [peek]: if non-nil, the conditionning keyword is left in the lexeme
+-- stream instead of being consumed.
+--
+-- * [primary]: the subparser.
+--
+-- * [keywords]: list of strings representing triggering keywords.
+--
+-- * Table-part entries can contain strings, and/or exactly one parser.
+-- Strings are put in [keywords], and the parser is put in [primary].
+--
+-- After the call, the following fields will be set:
+--
+-- * [parse] the parsing method
+-- * [kind] == "onkeyword"
+-- * [primary]
+-- * [keywords]
+--
+-------------------------------------------------------------------------------
+function onkeyword (p)
+ make_parser ("onkeyword", p)
+
+ -------------------------------------------------------------------
+ -- Parsing method
+ -------------------------------------------------------------------
+ function p:parse(lx)
+ if lx:is_keyword (lx:peek(), unpack(self.keywords)) then
+ --local fli = lx:lineinfo_right()
+ if not self.peek then lx:next() end
+ local content = self.primary (lx)
+ --local lli = lx:lineinfo_left()
+ local fli, lli = content.lineinfo.first, content.lineinfo.last
+ return transform (content, p, fli, lli)
+ else return false end
+ end
+
+ -------------------------------------------------------------------
+ -- Construction
+ -------------------------------------------------------------------
+ if not p.keywords then p.keywords = { } end
+ for _, x in ipairs(p) do
+ if type(x)=="string" then table.insert (p.keywords, x)
+ else assert (not p.primary and is_parser (x)); p.primary = x end
+ end
+ if not next (p.keywords) then
+ eprintf("Warning, no keyword to trigger gg.onkeyword") end
+ assert (p.primary, 'no primary parser in gg.onkeyword')
+ return p
+end --</onkeyword>
+
+
+-------------------------------------------------------------------------------
+--
+-- Optional keyword consummer pseudo-parser generator
+--
+-------------------------------------------------------------------------------
+--
+-- This doesn't return a real parser, just a function. That function parses
+-- one of the keywords passed as parameters, and returns it. It returns
+-- [false] if no matching keyword is found.
+--
+-- Notice that tokens returned by lexer already carry lineinfo, therefore
+-- there's no need to add them, as done usually through transform() calls.
+-------------------------------------------------------------------------------
+function optkeyword (...)
+ local args = {...}
+ if type (args[1]) == "table" then
+ assert (#args == 1)
+ args = args[1]
+ end
+ for _, v in ipairs(args) do assert (type(v)=="string") end
+ return function (lx)
+ local x = lx:is_keyword (lx:peek(), unpack (args))
+ if x then lx:next(); return x
+ else return false end
+ end
+end
+
+
+-------------------------------------------------------------------------------
+--
+-- Run a parser with a special lexer
+--
+-------------------------------------------------------------------------------
+--
+-- This doesn't return a real parser, just a function.
+-- First argument is the lexer class to be used with the parser,
+-- 2nd is the parser itself.
+-- The resulting parser returns whatever the argument parser does.
+--
+-------------------------------------------------------------------------------
+function with_lexer(new_lexer, parser)
+
+ -------------------------------------------------------------------
+ -- Most gg functions take their parameters in a table, so it's
+ -- better to silently accept when with_lexer{ } is called with
+ -- its arguments in a list:
+ -------------------------------------------------------------------
+ if not parser and #new_lexer==2 and type(new_lexer[1])=='table' then
+ return with_lexer(unpack(new_lexer))
+ end
+
+ -------------------------------------------------------------------
+ -- Save the current lexer, switch it for the new one, run the parser,
+ -- restore the previous lexer, even if the parser caused an error.
+ -------------------------------------------------------------------
+ return function (lx)
+ local old_lexer = getmetatable(lx)
+ lx:sync()
+ setmetatable(lx, new_lexer)
+ local status, result = pcall(parser, lx)
+ lx:sync()
+ setmetatable(lx, old_lexer)
+ if status then return result else error(result) end
+ end
+end
View
512 misc/luainspect/lexer.lua
@@ -0,0 +1,512 @@
+----------------------------------------------------------------------
+-- Metalua: $Id: mll.lua,v 1.3 2006/11/15 09:07:50 fab13n Exp $
+--
+-- Summary: generic Lua-style lexer definition. You need this plus
+-- some keyword additions to create the complete Lua lexer,
+-- as is done in mlp_lexer.lua.
+--
+-- TODO:
+--
+-- * Make it easy to define new flavors of strings. Replacing the
+-- lexer.patterns.long_string regexp by an extensible list, with
+-- customizable token tag, would probably be enough. Maybe add:
+-- + an index of capture for the regexp, that would specify
+-- which capture holds the content of the string-like token
+-- + a token tag
+-- + or a string->string transformer function.
+--
+-- * There are some _G.table to prevent a namespace clash which has
+-- now disappered. remove them.
+----------------------------------------------------------------------
+--
+-- Copyright (c) 2006, Fabien Fleutot <metalua@gmail.com>.
+--
+-- This software is released under the MIT Licence, see licence.txt
+-- for details.
+--
+----------------------------------------------------------------------
+
+module ("lexer", package.seeall)
+
+require 'metalua.runtime'
+
+
+lexer = { alpha={ }, sym={ } }
+lexer.__index=lexer
+
+local debugf = function() end
+--local debugf=printf
+
+----------------------------------------------------------------------
+-- Patterns used by [lexer:extract] to decompose the raw string into
+-- correctly tagged tokens.
+----------------------------------------------------------------------
+lexer.patterns = {
+ spaces = "^[ \r\n\t]*()",
+ short_comment = "^%-%-([^\n]*)()\n",
+ final_short_comment = "^%-%-([^\n]*)()$",
+ long_comment = "^%-%-%[(=*)%[\n?(.-)%]%1%]()",
+ long_string = "^%[(=*)%[\n?(.-)%]%1%]()",
+ number_mantissa = { "^%d+%.?%d*()", "^%d*%.%d+()" },
+ number_exponant = "^[eE][%+%-]?%d+()",
+ number_hex = "^0[xX]%x+()",
+ word = "^([%a_][%w_]*)()"
+}
+
+----------------------------------------------------------------------
+-- unescape a whole string, applying [unesc_digits] and
+-- [unesc_letter] as many times as required.
+----------------------------------------------------------------------
+local function unescape_string (s)
+
+ -- Turn the digits of an escape sequence into the corresponding
+ -- character, e.g. [unesc_digits("123") == string.char(123)].
+ local function unesc_digits (backslashes, digits)
+ if #backslashes%2==0 then
+ -- Even number of backslashes, they escape each other, not the digits.
+ -- Return them so that unesc_letter() can treaat them
+ return backslashes..digits
+ else
+ -- Remove the odd backslash, which escapes the number sequence.
+ -- The rest will be returned and parsed by unesc_letter()
+ backslashes = backslashes :sub (1,-2)
+ end
+ local k, j, i = digits:reverse():byte(1, 3)
+ local z = _G.string.byte "0"
+ local code = (k or z) + 10*(j or z) + 100*(i or z) - 111*z
+ if code > 255 then
+ error ("Illegal escape sequence '\\"..digits..
+ "' in string: ASCII codes must be in [0..255]")
+ end
+ return backslashes .. string.char (code)
+ end
+
+ -- Take a letter [x], and returns the character represented by the
+ -- sequence ['\\'..x], e.g. [unesc_letter "n" == "\n"].
+ local function unesc_letter(x)
+ local t = {
+ a = "\a", b = "\b", f = "\f",
+ n = "\n", r = "\r", t = "\t", v = "\v",
+ ["\\"] = "\\", ["'"] = "'", ['"'] = '"', ["\n"] = "\n" }
+ return t[x] or error([[Unknown escape sequence '\]]..x..[[']])
+ end
+
+ return s
+ :gsub ("(\\+)([0-9][0-9]?[0-9]?)", unesc_digits)
+ :gsub ("\\(%D)",unesc_letter)
+end
+
+lexer.extractors = {
+ "skip_whitespaces_and_comments",
+ "extract_short_string", "extract_word", "extract_number",
+ "extract_long_string", "extract_symbol" }
+
+lexer.token_metatable = {
+-- __tostring = function(a)
+-- return string.format ("`%s{'%s'}",a.tag, a[1])
+-- end
+}
+
+lexer.lineinfo_metatable = { }
+
+----------------------------------------------------------------------
+-- Really extract next token fron the raw string
+-- (and update the index).
+-- loc: offset of the position just after spaces and comments
+-- previous_i: offset in src before extraction began
+----------------------------------------------------------------------
+function lexer:extract ()
+ local previous_i = self.i
+ local loc = self.i
+ local eof, token
+
+ -- Put line info, comments and metatable around the tag and content
+ -- provided by extractors, thus returning a complete lexer token.
+ -- first_line: line # at the beginning of token
+ -- first_column_offset: char # of the last '\n' before beginning of token
+ -- i: scans from beginning of prefix spaces/comments to end of token.
+ local function build_token (tag, content)
+ assert (tag and content)
+ local i, first_line, first_column_offset, previous_line_length =
+ previous_i, self.line, self.column_offset, nil
+
+ -- update self.line and first_line. i := indexes of '\n' chars
+ while true do
+ i = self.src:match ("\n()", i, true)
+ --PATCHED:LuaInspect: above line was not counting line numbers
+ -- correctly when first character of file was a \n.
+ if not i or i>self.i then break end -- no more '\n' until end of token
+ previous_line_length = i - self.column_offset
+ if loc and i <= loc then -- '\n' before beginning of token
+ first_column_offset = i
+ first_line = first_line+1
+ end
+ self.line = self.line+1
+ self.column_offset = i
+ end
+
+ -- lineinfo entries: [1]=line, [2]=column, [3]=char, [4]=filename
+ local fli = { first_line, loc-first_column_offset, loc, self.src_name }
+ local lli = { self.line, self.i-self.column_offset-1, self.i-1, self.src_name }
+ --Pluto barfes when the metatable is set:(
+ setmetatable(fli, lexer.lineinfo_metatable)
+ setmetatable(lli, lexer.lineinfo_metatable)
+ local a = { tag = tag, lineinfo = { first=fli, last=lli }, content }
+ if lli[2]==-1 then lli[1], lli[2] = lli[1]-1, previous_line_length-1 end
+ if #self.attached_comments > 0 then
+ a.lineinfo.comments = self.attached_comments
+ fli.comments = self.attached_comments
+ if self.lineinfo_last then
+ self.lineinfo_last.comments = self.attached_comments
+ end
+ end
+ self.attached_comments = { }
+ return setmetatable (a, self.token_metatable)
+ end --</function build_token>
+
+ for ext_idx, extractor in ipairs(self.extractors) do
+ -- printf("method = %s", method)
+ local tag, content = self [extractor] (self)
+ -- [loc] is placed just after the leading whitespaces and comments;
+ -- for this to work, the whitespace extractor *must be* at index 1.
+ if ext_idx==1 then loc = self.i end
+
+ if tag then
+ --printf("`%s{ %q }\t%i", tag, content, loc);
+ return build_token (tag, content)
+ end
+ end
+
+ error "None of the lexer extractors returned anything!"
+end
+
+----------------------------------------------------------------------
+-- skip whites and comments
+-- FIXME: doesn't take into account:
+-- - unterminated long comments
+-- - short comments at last line without a final \n
+----------------------------------------------------------------------
+function lexer:skip_whitespaces_and_comments()
+ local table_insert = _G.table.insert
+ repeat -- loop as long as a space or comment chunk is found
+ local _, j
+ local again = false
+ local last_comment_content = nil
+ -- skip spaces
+ self.i = self.src:match (self.patterns.spaces, self.i)
+ -- skip a long comment if any
+ _, last_comment_content, j =
+ self.src :match (self.patterns.long_comment, self.i)
+ if j then
+ table_insert(self.attached_comments,
+ {last_comment_content, self.i, j, "long"})
+ self.i=j; again=true
+ end
+ -- skip a short comment if any
+ last_comment_content, j = self.src:match (self.patterns.short_comment, self.i)
+ if j then
+ table_insert(self.attached_comments,
+ {last_comment_content, self.i, j, "short"})
+ self.i=j; again=true
+ end
+ if self.i>#self.src then return "Eof", "eof" end
+ until not again
+
+ if self.src:match (self.patterns.final_short_comment, self.i) then
+ return "Eof", "eof" end
+ --assert (not self.src:match(self.patterns.short_comment, self.i))
+ --assert (not self.src:match(self.patterns.long_comment, self.i))
+ -- --assert (not self.src:match(self.patterns.spaces, self.i))
+ return
+end
+
+----------------------------------------------------------------------
+-- extract a '...' or "..." short string
+----------------------------------------------------------------------
+function lexer:extract_short_string()
+ -- [k] is the first unread char, [self.i] points to [k] in [self.src]
+ local j, k = self.i, self.src :sub (self.i,self.i)
+ if k~="'" and k~='"' then return end
+ local i = self.i + 1
+ local j = i
+ while true do
+ -- k = opening char: either simple-quote or double-quote
+ -- i = index of beginning-of-string
+ -- x = next "interesting" character
+ -- j = position after interesting char
+ -- y = char just after x
+ local x, y
+ x, j, y = self.src :match ("([\\\r\n"..k.."])()(.?)", j)
+ if x == '\\' then j=j+1 -- don't parse escaped char
+ elseif x == k then break -- unescaped end of string
+ else -- eof or '\r' or '\n' reached before end of string
+ assert (not x or x=="\r" or x=="\n")
+ error "Unterminated string"
+ end
+ end
+ self.i = j
+
+ return "String", unescape_string (self.src:sub (i,j-2))
+end
+
+----------------------------------------------------------------------
+--
+----------------------------------------------------------------------
+function lexer:extract_word()
+ -- Id / keyword
+ local word, j = self.src:match (self.patterns.word, self.i)
+ if word then
+ self.i = j
+ if self.alpha [word] then return "Keyword", word
+ else return "Id", word end
+ end
+end
+
+----------------------------------------------------------------------
+--
+----------------------------------------------------------------------
+function lexer:extract_number()
+ -- Number
+ local j = self.src:match(self.patterns.number_hex, self.i)
+ if not j then
+ j = self.src:match (self.patterns.number_mantissa[1], self.i) or
+ self.src:match (self.patterns.number_mantissa[2], self.i)
+ if j then
+ j = self.src:match (self.patterns.number_exponant, j) or j;
+ end
+ end
+ if not j then return end
+ -- Number found, interpret with tonumber() and return it
+ local n = tonumber (self.src:sub (self.i, j-1))
+ self.i = j
+ return "Number", n
+end
+
+----------------------------------------------------------------------
+--
+----------------------------------------------------------------------
+function lexer:extract_long_string()
+ -- Long string
+ local _, content, j = self.src:match (self.patterns.long_string, self.i)
+ if j then self.i = j; return "String", content end
+end
+
+----------------------------------------------------------------------
+--
+----------------------------------------------------------------------
+function lexer:extract_symbol()
+ -- compound symbol
+ local k = self.src:sub (self.i,self.i)
+ local symk = self.sym [k]
+ if not symk then
+ self.i = self.i + 1
+ return "Keyword", k
+ end
+ for _, sym in pairs (symk) do
+ if sym == self.src:sub (self.i, self.i + #sym - 1) then
+ self.i = self.i + #sym;
+ return "Keyword", sym
+ end
+ end
+ -- single char symbol
+ self.i = self.i+1
+ return "Keyword", k
+end
+
+----------------------------------------------------------------------
+-- Add a keyword to the list of keywords recognized by the lexer.
+----------------------------------------------------------------------
+function lexer:add (w, ...)
+ assert(not ..., "lexer:add() takes only one arg, although possibly a table")
+ if type (w) == "table" then
+ for _, x in ipairs (w) do self:add (x) end
+ else
+ if w:match (self.patterns.word .. "$") then self.alpha [w] = true
+ elseif w:match "^%p%p+$" then
+ local k = w:sub(1,1)
+ local list = self.sym [k]
+ if not list then list = { }; self.sym [k] = list end
+ _G.table.insert (list, w)
+ elseif w:match "^%p$" then return
+ else error "Invalid keyword" end
+ end
+end
+
+----------------------------------------------------------------------
+-- Return the [n]th next token, without consumming it.
+-- [n] defaults to 1. If it goes pass the end of the stream, an EOF
+-- token is returned.
+----------------------------------------------------------------------
+function lexer:peek (n)
+ if not n then n=1 end
+ if n > #self.peeked then
+ for i = #self.peeked+1, n do
+ self.peeked [i] = self:extract()
+ end
+ end
+ return self.peeked [n]
+end
+
+----------------------------------------------------------------------
+-- Return the [n]th next token, removing it as well as the 0..n-1
+-- previous tokens. [n] defaults to 1. If it goes pass the end of the
+-- stream, an EOF token is returned.
+----------------------------------------------------------------------
+function lexer:next (n)
+ n = n or 1
+ self:peek (n)
+ local a
+ for i=1,n do
+ a = _G.table.remove (self.peeked, 1)
+ if a then
+ --debugf ("lexer:next() ==> %s %s",
+ -- table.tostring(a), tostring(a))
+ end
+ self.lastline = a.lineinfo.last[1]
+ end
+ self.lineinfo_last = a.lineinfo.last
+ return a or eof_token
+end
+
+----------------------------------------------------------------------
+-- Returns an object which saves the stream's current state.
+----------------------------------------------------------------------
+-- FIXME there are more fields than that to save
+function lexer:save () return { self.i; _G.table.cat(self.peeked) } end
+
+----------------------------------------------------------------------
+-- Restore the stream's state, as saved by method [save].
+----------------------------------------------------------------------
+-- FIXME there are more fields than that to restore
+function lexer:restore (s) self.i=s[1]; self.peeked=s[2] end
+
+----------------------------------------------------------------------
+-- Resynchronize: cancel any token in self.peeked, by emptying the
+-- list and resetting the indexes
+----------------------------------------------------------------------
+function lexer:sync()
+ local p1 = self.peeked[1]
+ if p1 then
+ li = p1.lineinfo.first
+ self.line, self.i = li[1], li[3]
+ self.column_offset = self.i - li[2]
+ self.peeked = { }
+ self.attached_comments = p1.lineinfo.first.comments or { }
+ end
+end
+
+----------------------------------------------------------------------
+-- Take the source and offset of an old lexer.
+----------------------------------------------------------------------
+function lexer:takeover(old)
+ self:sync()
+ self.line, self.column_offset, self.i, self.src, self.attached_comments =
+ old.line, old.column_offset, old.i, old.src, old.attached_comments
+ return self
+end
+
+-- function lexer:lineinfo()
+-- if self.peeked[1] then return self.peeked[1].lineinfo.first
+-- else return { self.line, self.i-self.column_offset, self.i } end
+-- end
+
+
+----------------------------------------------------------------------
+-- Return the current position in the sources. This position is between
+-- two tokens, and can be within a space / comment area, and therefore
+-- have a non-null width. :lineinfo_left() returns the beginning of the
+-- separation area, :lineinfo_right() returns the end of that area.
+--
+-- ____ last consummed token ____ first unconsummed token
+-- / /
+-- XXXXX <spaces and comments> YYYYY
+-- \____ \____
+-- :lineinfo_left() :lineinfo_right()
+----------------------------------------------------------------------
+function lexer:lineinfo_right()
+ return self:peek(1).lineinfo.first
+end
+
+function lexer:lineinfo_left()
+ return self.lineinfo_last
+end
+
+----------------------------------------------------------------------
+-- Create a new lexstream.
+----------------------------------------------------------------------
+function lexer:newstream (src_or_stream, name)
+ name = name or "?"
+ if type(src_or_stream)=='table' then -- it's a stream
+ return setmetatable ({ }, self) :takeover (src_or_stream)
+ elseif type(src_or_stream)=='string' then -- it's a source string
+ local src = src_or_stream
+ local stream = {
+ src_name = name; -- Name of the file
+ src = src; -- The source, as a single string
+ peeked = { }; -- Already peeked, but not discarded yet, tokens
+ i = 1; -- Character offset in src
+ line = 1; -- Current line number
+ column_offset = 0; -- distance from beginning of file to last '\n'
+ attached_comments = { },-- comments accumulator
+ lineinfo_last = { 1, 1, 1, name }
+ }
+ setmetatable (stream, self)
+
+ -- skip initial sharp-bang for unix scripts
+ -- FIXME: redundant with mlp.chunk()
+ if src and src :match "^#" then stream.i = src :find "\n" + 1 end
+ return stream
+ else
+ assert(false, ":newstream() takes a source string or a stream, not a "..
+ type(src_or_stream))
+ end
+end
+
+----------------------------------------------------------------------
+-- if there's no ... args, return the token a (whose truth value is
+-- true) if it's a `Keyword{ }, or nil. If there are ... args, they
+-- have to be strings. if the token a is a keyword, and it's content
+-- is one of the ... args, then returns it (it's truth value is
+-- true). If no a keyword or not in ..., return nil.
+----------------------------------------------------------------------
+function lexer:is_keyword (a, ...)
+ if not a or a.tag ~= "Keyword" then return false end
+ local words = {...}
+ if #words == 0 then return a[1] end
+ for _, w in ipairs (words) do
+ if w == a[1] then return w end
+ end
+ return false
+end
+
+----------------------------------------------------------------------
+-- Cause an error if the next token isn't a keyword whose content
+-- is listed among ... args (which have to be strings).
+----------------------------------------------------------------------
+function lexer:check (...)
+ local words = {...}
+ local a = self:next()
+ local function err ()
+ error ("Got " .. tostring (a) ..
+ ", expected one of these keywords : '" ..
+ _G.table.concat (words,"', '") .. "'") end
+
+ if not a or a.tag ~= "Keyword" then err () end
+ if #words == 0 then return a[1] end
+ for _, w in ipairs (words) do
+ if w == a[1] then return w end
+ end
+ err ()
+end
+
+----------------------------------------------------------------------
+--
+----------------------------------------------------------------------
+function lexer:clone()
+ local clone = {
+ alpha = table.deep_copy(self.alpha),
+ sym = table.deep_copy(self.sym) }
+ setmetatable(clone, self)
+ clone.__index = clone
+ return clone
+end
View
26 misc/luainspect/luainspect-copyright.txt
@@ -0,0 +1,26 @@
+LuaInspect License
+
+Copyright (C) 2010 David Manura
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+===============================================================================
+
+Uses Metalua libraries (see metalua/LICENSE).
+Uses jquery (see COPYRIGHT-jquery).
View
325 misc/luainspect/luainspect-readme.txt
@@ -0,0 +1,325 @@
+LuaInspect - LuaInspect is a tool that does Lua code analysis.
+It includes an extensive plugin for the SciTE [1] text editor,
+there is also a plugin for the VIM editor [2], and it includes
+an export to DHTML as well.
+
+== Project Page ==
+
+For further details, see http://lua-users.org/wiki/LuaInspect .
+
+== Status ==
+
+WARNING: This code is not yet stable. It is usable but you
+may need to sometimes fix things yourself. Many additional
+features could be added too.
+
+== Features ==
+
+ * analysis:
+ * identifies global (red) and local variables (blue), including locals that are
+ function arguments (dark blue) and upvalues (light blue)
+ * identifies unused local variables: e.g. `do local x=1 end` (white-on-blue)
+ * identifies local variables masking other locals (same name): e.g. `local x=1; local x=2`
+ (strikethrough and squiggle line)
+ * identifies local variables that have non-constant binding (`local x = 1; x = 2`) (italic)
+ * identifies unknown global variables (white-on-red) and table fields (red), inferred by
+ static and dynamic evaluation.
+ * infers values of variables (e.g. `local sum = math.pi + 2` is 5.14.
+ and defined-ness of members of imported modules
+ (`local mt = require "math"; math.sqrtt(2) -- undefined`)
+ * infers signatures of functions (including local, global, and module functions)
+ * checks number of function arguments against signatures (SciTE only)
+ * cross-references variables (locals and module fields) with their definitions and uses
+ (pink highlight), identifies range of lines/scope where the local is defined
+ (currently SciTE only), and supports jump-to-definition and jump-to-uses (SciTE only)
+ * identifies all keywords in selected block (underline)
+ * evaluate special comments (prefixed by '!') to inject semantic information into analysis
+ (similar to luaanalyze / lint).
+ * basic type inferences (e.g. number + number -> number)
+ * infer function return values (e.g. `function f(x) if x then return 1,2,3 else return 1,3,'z' end end`
+ returns 1, number, unknown).
+ * detect dead-code (e.g. `do return end dead()`) (SciTE only) (diagonal hatching)
+ * refactoring:
+ * command to rename all occurrences of selected variable (SciTE only)
+ * browsing:
+ * inspect members of selected table.
+ * select statement or comment containing current cursor selection (SciTE only)
+ * display real-time annotations of all local variables, like an Excel/Mathcad worksheet
+ (experimental feature via ANNOTATE_ALL_LOCALS) (currently SciTE only)
+ * auto-complete typing support (SciTE only) (experimental)
+ * interfaces: SciTE plugin, VIM plugin, and HTML output.
+
+== Files in this directory ==
+
+metalualib/* - Copy of Metalua libraries under here
+luainspectlib/* - LuaInspect libraries under here
+htmllib/* - HTML resources under here
+
+Note: the metalualib contains this version of metalua:
+ http://github.com/fab13n/metalua/tree/fcee97b8d0091ceb471902ee457dbccaab98234e
+with a few bug fixes (search for "PATCHED:LuaInspect" in the source)
+
+== Command-line Usage (HTML output) ==
+
+Example:
+
+ $ lua luainspectlib/luainspect/command.lua examples.lua > test-output/examples.html
+
+== Installation in SciTE ==
+
+Install SciTE version. Version 2.12 and 2.20 work (older versions might not work).
+
+First install http://lua-users.org/wiki/SciteExtMan .
+
+Add this to your SciTE Lua startup script (but change LUAINSPECT_PATH):
+
+=============================
+local LUAINSPECT_PATH = "c:/luainspect"
+package.path = package.path .. ";" .. LUAINSPECT_PATH .. "/metalualib/?.lua"
+package.path = package.path .. ";" .. LUAINSPECT_PATH .. "/luainspectlib/?.lua"
+require "luainspect.scite" : install()
+=============================
+
+Dependencies:
+ Tested with SciTE version 2.12 (older versions might not work).
+ Requires http://lua-users.org/wiki/SciteExtMan .
+ ctagsdx.lua from the full SciteExtMan is optional (allows "goto mark" command
+ to return to previous location following a "go to definition" or "show all variable uses").
+
+If you want to customize styles, add the contents of the
+`light_styles` or `dark_styles` variable in the scite.lua file to your
+SciTEGlobal.properties, SciTEUser.properties, or SciTE.properties file.
+
+== Configuring SciTE options ==
+
+The following LuaInspect options can be configured in one of your
+SciTE property files:
+
+ luainspect.update.always (0 or 1, default 1)
+ luainspect.delay.count (integer >= 1, default 5)
+ luainspect.annotate.all.locals (0 or 1, default 0)
+ luainspect.incremental.compilation (0 or 1, default 1)
+ luainspect.performance.tests (0 or 1, default 0)
+ luainspect.autocomplete.vars (0 or 1, default 0)
+ luainspect.autocomplete.syntax (0 or 1, default 0)
+ luainspect.path.append (string, default '')
+ luainspect.cpath.append (string, default '')
+ style.script_lua.scheme (string, '' or 'dark', default '')
+
+For details, see scite.lua.
+
+== Installation on VIM ==
+
+See [2] for VIM editor support.
+
+== Preliminary support for luaanalyze style comments ==
+
+To make all variables in scope match name 'ast$' be recognized by LuaInspect as a
+table with field 'tag' of type string, add this to your code:
+
+ --! context.apply_value('ast$', {tag=''})
+
+The LuaInspect code itself uses this:
+
+ --! require 'luainspect.typecheck' (context)
+
+== Design Notes ==
+
+The font styles are intended to make the more dangerous
+or questionable code stand out more.
+
+== LICENSE ==
+
+See LICENSE file.
+
+== Credits ==
+
+David Manura, original author.
+Steve Donovan for discussions on design, SciTE and ExtMan.
+Fabien Fleutot for Metalua and discussions.
+SciTE suggestions/fixes by Tymur Gubayev.
+Peter Odding for VIM editor support [2]
+
+== Bugs ==
+
+Please report bugs via github <http://github.com/davidm/lua-inspect/issues>
+or just "dee em dot el you ae at em ae tee ayche two dot ow ar gee", or
+if you prefer neither, append to the wiki page
+<http://lua-users.org/wiki/LuaInspect>.
+
+== References ==
+
+[1] http://www.scintilla.org/SciTE.html
+[2] http://peterodding.com/code/vim/lua-inspect/ - VIM editor support
+
+== Changes ==
+
+20100818
+ [!] HTML: fix missing chars at end-of-file
+ [!] Metalua: fix lexer line number count off-by-one error
+ [!] SciTE: fix Unicode/UTF-8 encoding breaking formatting
+ [!] core: fix performance problem with tinsertlist function
+ [!] core/performance: cleanup invalidated_code function
+
+20100817
+ [!] core: fix keyword token recognition problems
+ [!] core: skip inspection on require loops
+ [+] core: infer function return values (temporarily disabled)
+ [+] core: detect dead-code (temporarily disabled)
+ [*] core: internal refactoring (ast.valueknown)
+
+20100816
+ core: make reporting optional
+ metalua: patches to metalua lineinfo
+ (was corrupting HTML output and SciTE highlighting)
+
+20100814
+ core: add basic type inferences (e.g. number+number -> number)
+
+20100813
+ core: inspect required modules too
+ (e.g. enables use of imported function signatures)
+ core/SciTE: add list all warnings command (SciTE: Ctrl+Alt+W lists, and F4 iterates them)
+
+20100811
+ SciTE: autocomplete functions arguments when cursor after '('
+ core: fix signatures for os/debug libraries
+ core/SciTE: display function argument list or helpinfo for variables
+ SciTE: Ctrl+Alt+I changed to Ctrl+Alt+B to avoid conflict with
+ SciTE 2.20 incremental search
+
+20100810
+ SciTE: improved "inspect variable" command, supports browsing nested tables.
+ SciTE: split luainspect.autocomplete property into two properties
+ SciTE: add autocomplete function
+ SciTE: autocomplete table fields.
+
+20100809
+ core/SciTE: add function argument count check
+ core/SciTE: jump to definition now supports functions in different files.
+ core/SciTE/HTML: improvements to displaying masking/masked lexicals.
+ core/SciTE: add command to just to previous statement
+ core/SciTE: preliminary variable autocomplete support
+ (luainspect.autocomplete currently disabled by default)
+ SciTE: add missing style.script_lua.local_param_mutate style.
+
+20100807
+ SciTE: Add luainspect.path.append/luainspect.cpath.append properties
+ to append to package.path/cpath
+ SciTE: Add custom searcher function to locate modules in same path as current buffer.
+ SciTE: Added "force reinspect" command to force full reinspection of code.
+ Note: this will also attempt to unload any modules loaded by previous inspection.
+ SciTE: Improve luainspect.update.delay to delay inspection for given tick count
+ following user typing. Also displays blue '+' marker when inspection has been delayed.
+
+20100806
+ SciTE: jump to uses, not jumps to exact position, not just line number
+ SciTE: mark lines of invalidated code upon introducing code errors and display
+ error message below invalidated code (not on exact line of error)
+ SciTE: add styling delay option to improve performance (luainspect.update.delay)
+ SciTE: preliminary auto-complete typing support (luainspect.autocomplete)
+ (experimental and currently off by default)
+
+20100805
+ core: Major internal refactoring to simplify incremental compilation
+ (lineinfo managed in tokenlist). Breaks API.
+ core/SciTE/HTML: identifies local variables that mask other locals (same name):
+ e.g. local x=1; local x=2 (strikethrough)
+ core: added version number variable APIVERSION to luainspect.init.
+ HTML: highlight keywords in selected block
+ SciTE: the incremental compilation feature is now on by default.
+
+20100803
+ core:Evaluate special comments (prefixed by '!') to inject semantic information into analysis
+ (similar to luaanalyze).
+ core: Further work on incremental compilation feature.
+
+20100802
+ core: improve field value inferences
+ SciTE: improve dark style clarity
+ SciTE: make margin markers for variable scope and block mutually exclusive
+
+20100731
+ SciTE: allow styles in properties to be specified by name and more flexibly overridden.
+ SciTE: add optional dark style
+ SciTE/HTML: support mutate upvalues, cleanup styles
+ SciTE: improve keyword highlighting (always highlight containing block)
+
+20100730
+ core: fix scoping of `for` statements (in globals.lua)
+ core/SciTE: highlight keywords and show all keywords in selected statement.
+
+20100729
+ SciTE: options can now be set with SciTE properties.
+ SciTE: refactor: select statement
+ core/SciTE: more work on incremental compilation (luainspect.incremental.compilation)
+
+20100728
+ core/SciTE: add command to select statement or comment containing current cursor selection.
+ core/SciTE: experimental incremental compilation option (ALLOW_INCREMENTAL_COMPILATION)
+ core/SciTE: add special styling (background color) for tab whitespace
+
+20100727
+ SciTE: Fix limited styling range may skip styling (broke in 20100726)
+
+20100726
+ SciTE: apply default styles in script if not specified in properties file.
+ SciTE: initial implementation of folding (but currently disabled due to SciTE problems)
+ SciTE: improve OnStyle only over provided byte range
+ Note: you may now remove LuaInspect styles from your properties file.
+
+20100725
+ SciTE: fix memory overflow when code contains buffer.notes.
+
+20100724
+ SciTE: list all uses of selected variable (currently locals only)
+ SciTE: display errors about mismatched blocks or parens at both top and bottom of problem
+ SciTE: support shebang line
+
+20100723
+ core/SciTE/HTML: Initial support for table fields
+ core/SciTE: initial dynamic value determination
+ core: fix recursive local scoping (`Localrec) in globals.lua
+ SciTE: Mark all range of selected variable's scope in margin
+ SciTE: New command to rename all occurrences of selected variable
+ SciTE: Significant performance gain utilizing loadstring in addition
+ to metalua libraries
+ SciTE: Mark upvalues (lighter blue)
+ SciTE: Fix handling multiple buffers.
+ SciTE: display variable info on double click
+ SciTE: display real-time annotations of all local variables, like a Mathcad worksheet
+ (experimental feature via ANNOTATE_ALL_LOCALS)
+ SciTE: jump (goto) definition of selected variable (currently locals only)
+ ctagsdx.lua from the full SciteExtMan is optional (allows "goto mark" command
+ to return to previous location following a "go to definition").
+ SciTE: add command to inspect table contents.
+ Note: SciTE*.properties and luainspect.css have been updated; please update when upgrading
+
+20100720
+ core: support for detecting unused locals (white on blue)
+ SciTE: display callinfo help on top-level standard library globals
+ SciTE: display local parameters distinctly (dark blue)
+ SciTE: display compiler errors as annotations
+ SciTE: partial workaround for conflict with other lexers
+ SciTE: option to recompile only when cursor line number changes to improve performance
+ and reduce error reporting (set UPDATE_ALWAYS to true in scite.lua to enable this)
+ SciTE: workaround for Metalua libraries sometimes not returning line number in error report
+ Note: SciTE*.properties and luainspect.css have been updated; please update when upgrading
+
+20100719
+ core: Fixed "repeat" statement scope handling (globals.lua)
+ SciTE: Improve performance (not recompile when code not changing)
+ SciTE: Add "!" marker near compiler error.
+ SciTE: Add hotspots on local variables
+
+20100717-2
+ SciTE: highlight all instances of selected identifier
+ Now requires http://lua-users.org/wiki/SciteExtMan
+
+20100717
+ added initial SciTE text editor plugin
+
+20100622
+ initial version with HTML output
+
+--David Manura
View
982 misc/luainspect/luainspect/ast.lua
@@ -0,0 +1,982 @@
+-- luainspect.ast - Lua Abstract Syntax Tree (AST) and token list operations.
+--
+-- Two main structures are maintained. A Metalua-style AST represents the
+-- nested syntactic structure obtained from the parse.
+-- A separate linear ordered list of tokens represents the syntactic structure
+-- from the lexing, including line information (character positions only not row/columns),
+-- comments, and keywords, which is originally built from the lineinfo attributes
+-- injected by Metalua into the AST (IMPROVE: it probably would be simpler
+-- to obtain this from the lexer directly rather then inferring it from the parsing).
+-- During AST manipulations, the lineinfo maintained in the AST is ignored
+-- because it was found more difficult to maintain and not in the optimal format.
+--
+-- The contained code deals with
+-- - Building the AST from source.
+-- - Building the tokenlist from the AST lineinfo.
+-- - Querying the AST+tokenlist.
+-- - Modifying the AST+tokenlist (including incremental parsing source -> AST)
+-- - Annotating the AST with navigational info (e.g. parent links) to assist queries.
+-- - Dumping the AST+tokenlist for debugging.
+--
+-- (c) 2010 David Manura, MIT License.
+
+
+--! require 'luainspect.typecheck' (context)
+
+-- boilerplate/utility
+-- LUA_PATH="?.lua;/path/to/metalua/src/compiler/?.lua;/path/to/metalua/src/lib/?.lua"
+-- import modules -- order is important
+require "lexer"
+require "gg"
+require "mlp_lexer"
+require "mlp_misc"
+require "mlp_table"
+require "mlp_meta"
+require "mlp_expr"
+require "mlp_stat"
+--require "mlp_ext"
+_G.mlc = {} -- make gg happy
+-- Metalua:IMPROVE: make above imports simpler
+
+local M = {}
+
+
+-- CATEGORY: debug
+local function DEBUG(...)
+ if LUAINSPECT_DEBUG then
+ print('DEBUG:', ...)
+ end
+end
+
+
+-- Convert character position to row,column position in string src.
+-- Add values are 1-indexed.
+function M.pos_to_linecol(pos, src)
+ local linenum = 1
+ local lasteolpos = 0
+ for eolpos in src:gmatch"()\n" do
+ if eolpos > pos then break end
+ linenum = linenum + 1
+ lasteolpos = eolpos
+ end
+ local colnum = pos - lasteolpos
+ return linenum, colnum
+end
+
+-- Remove any sheband ("#!") line from Lua source string.
+-- CATEGORY: Lua parsing
+function M.remove_shebang(src)
+ local shebang = src:match("^#![^\r\n]*")
+ return shebang and (" "):rep(#shebang) .. src:sub(#shebang+1) or src
+end
+
+
+-- Custom version of loadstring that parses out line number info
+-- CATEGORY: Lua parsing
+function M.loadstring(src)
+ local f, err = loadstring(src, "")
+ if f then
+ return f
+ else
+ err = err:gsub('^%[string ""%]:', "")
+ local linenum = assert(err:match("(%d+):"))
+ local colnum = 0
+ local linenum2 = err:match("^%d+: '[^']+' expected %(to close '[^']+' at line (%d+)")
+ return nil, err, linenum, colnum, linenum2
+ end
+end
+
+
+-- helper for ast_from_string. Raises on error.
+-- FIX? filename currently ignored in Metalua
+-- CATEGORY: Lua parsing
+local function ast_from_string_helper(src, filename)
+ filename = filename or '(string)'
+ local lx = mlp.lexer:newstream (src, filename)
+ local ast = mlp.chunk(lx)
+ return ast
+end
+
+
+-- Count number of lines in text.
+-- Warning: the decision of whether to count a trailing new-line in a file
+-- or an empty file as a line is a little subjective. This function currently
+-- defines the line count as 1 plus the number of new line characters.
+-- CATEGORY: utility/string
+local function linecount(text)
+ local n = 1
+ for _ in text:gmatch'\n' do
+ n = n + 1
+ end
+ return n
+end
+
+
+-- Converts Lua source string to Lua AST (via mlp/gg).
+-- CATEGORY: Lua parsing
+function M.ast_from_string(src, filename)
+ local ok, ast = pcall(ast_from_string_helper, src, filename)
+ if not ok then
+ local err = ast
+ err = err:match('[^\n]*')
+ err = err:gsub("^.-:%s*line", "line")
+ -- mlp.chunk prepending this is undesirable. error(msg,0) would be better in gg.lua. Reported.
+ -- TODO-Metalua: remove when fixed in Metalua.
+ local linenum, colnum = err:match("line (%d+), char (%d+)")
+ if not linenum then
+ -- Metalua libraries may return "...gg.lua:56: .../mlp_misc.lua:179: End-of-file expected"
+ -- without the normal line/char numbers given things like "if x then end end". Should be
+ -- fixed probably with gg.parse_error in _chunk in mlp_misc.lua.
+ -- TODO-Metalua: remove when fixed in Metalua.
+ linenum = linecount(src)
+ colnum = 1
+ end
+ local linenum2 = nil
+ return nil, err, linenum, colnum, linenum2
+ else
+ return ast
+ end
+end
+
+
+-- Simple comment parser. Returns Metalua-style comment.
+-- CATEGORY: Lua lexing
+local function quick_parse_comment(src)
+ local s = src:match"^%-%-([^\n]*)()\n$"
+ if s then return {s, 1, #src, 'short'} end
+ local _, s = src:match(lexer.lexer.patterns.long_comment .. '\r?\n?$')
+ if s then return {s, 1, #src, 'long'} end
+ return nil
+end
+--FIX:check new-line correctness
+--note: currently requiring \n at end of single line comment to avoid
+-- incremental compilation with `--x\nf()` and removing \n from still
+-- recognizing as comment `--x`.
+-- currently allowing \r\n at end of long comment since Metalua includes
+-- it in lineinfo of long comment (FIX:Metalua?)
+
+
+-- Gets length of longest prefix string in both provided strings.
+-- Returns max n such that text1:sub(1,n) == text2:sub(1,n) and n <= max(#text1,#text2)
+-- CATEGORY: string utility
+local function longest_prefix(text1, text2)
+ local nmin = 0
+ local nmax = math.min(#text1, #text2)
+ while nmax > nmin do
+ local nmid = math.ceil((nmin+nmax)/2)
+ if text1:sub(1,nmid) == text2:sub(1,nmid) then
+ nmin = nmid
+ else
+ nmax = nmid-1
+ end
+ end
+ return nmin
+end
+
+
+-- Gets length of longest postfix string in both provided strings.
+-- Returns max n such that text1:sub(-n) == text2:sub(-n) and n <= max(#text1,#text2)
+-- CATEGORY: string utility
+local function longest_postfix(text1, text2)
+ local nmin = 0
+ local nmax = math.min(#text1, #text2)
+ while nmax > nmin do
+ local nmid = math.ceil((nmin+nmax)/2)
+ if text1:sub(-nmid) == text2:sub(-nmid) then --[*]
+ nmin = nmid
+ else
+ nmax = nmid-1
+ end
+ end
+ return nmin
+end -- differs from longest_prefix only on line [*]
+
+
+
+-- Determines AST node that must be re-evaluated upon changing code string from
+-- `src` to `bsrc`, given previous top_ast/tokenlist/src.
+-- Note: decorates top_ast as side-effect.
+-- If preserve is true, then does not expand AST match even if replacement is invalid.
+-- CATEGORY: AST/tokenlist manipulation
+function M.invalidated_code(top_ast, tokenlist, src, bsrc, preserve)
+ -- Converts posiiton range in src to position range in bsrc.
+ local function range_transform(src_fpos, src_lpos)
+ local src_nlpos = #src - src_lpos
+ local bsrc_fpos = src_fpos
+ local bsrc_lpos = #bsrc - src_nlpos
+ return bsrc_fpos, bsrc_lpos
+ end
+
+ if src == bsrc then return end -- up-to-date
+
+ -- Find range of positions in src that differences correspond to.
+ -- Note: for zero byte range, src_pos2 = src_pos1 - 1.
+ local npre = longest_prefix(src, bsrc)
+ local npost = math.min(#src-npre, longest_postfix(src, bsrc))
+ -- note: min avoids overlap ambiguity
+ local src_fpos, src_lpos = 1 + npre, #src - npost
+
+ -- Find smallest AST node containing src range above. May also
+ -- be contained in (smaller) comment or whitespace.
+ local match_ast, match_comment, iswhitespace =
+ M.smallest_ast_containing_range(top_ast, tokenlist, src, src_fpos, src_lpos)
+ DEBUG('invalidate-smallest:', match_ast and (match_ast.tag or 'notag'), match_comment, iswhitespace)
+
+ -- Determine which (ast, comment, or whitespace) to match, and get its pos range in src and bsrc.
+ local srcm_fpos, srcm_lpos, bsrcm_fpos, bsrcm_lpos, mast, mtype
+ if iswhitespace then
+ mast, mtype = nil, 'whitespace'
+ srcm_fpos, srcm_lpos = src_fpos, src_lpos
+ elseif match_comment then
+ mast, mtype = match_comment, 'comment'
+ srcm_fpos, srcm_lpos = match_comment.fpos, match_comment.lpos
+ else
+ mast, mtype = match_ast, 'ast'
+ srcm_fpos, srcm_lpos = M.ast_pos_range(match_ast, tokenlist)
+ end
+ bsrcm_fpos, bsrcm_lpos = range_transform(srcm_fpos, srcm_lpos)
+
+ -- Never expand match if preserve specified.
+ if preserve then
+ return srcm_fpos, srcm_lpos, bsrcm_fpos, bsrcm_lpos, mast, mtype
+ end
+
+ -- Determine if replacement could break parent nodes.
+ local isreplacesafe
+ if mtype == 'whitespace' then
+ if bsrc:sub(bsrcm_fpos, bsrcm_lpos):match'^%s*$' then -- replaced with whitespace
+ if bsrc:sub(bsrcm_fpos-1, bsrcm_lpos+1):match'%s' then -- not eliminating whitespace
+ isreplacesafe = true
+ end
+ end
+ elseif mtype == 'comment' then
+ local m2src = bsrc:sub(bsrcm_fpos, bsrcm_lpos)
+ DEBUG('invalidate-comment[' .. m2src .. ']')
+ if quick_parse_comment(m2src) then -- replaced with comment
+ isreplacesafe = true
+ end
+ end
+ if isreplacesafe then -- return on safe replacement
+ return srcm_fpos, srcm_lpos, bsrcm_fpos, bsrcm_lpos, mast, mtype
+ end
+
+ -- Find smallest containing statement block that will compile.
+ while 1 do
+ match_ast = M.get_containing_statementblock(match_ast, top_ast)
+ local srcm_fpos, srcm_lpos = M.ast_pos_range(match_ast, tokenlist)
+ local bsrcm_fpos, bsrcm_lpos = range_transform(srcm_fpos, srcm_lpos)
+ local msrc = bsrc:sub(bsrcm_fpos, bsrcm_lpos)
+ DEBUG('invalidate-statblock:', match_ast and match_ast.tag, '[' .. msrc .. ']')
+ if loadstring(msrc) then -- compiled
+ return srcm_fpos, srcm_lpos, bsrcm_fpos, bsrcm_lpos, match_ast, 'statblock'
+ end
+ M.ensure_parents_marked(top_ast)
+ match_ast = match_ast.parent
+ if not match_ast then
+ return nil, nil, nil, nil, top_ast, 'full' -- entire AST invalidated
+ end
+ end
+end
+
+
+-- Walks AST `ast` in arbitrary order, visiting each node `n`, executing `fdown(n)` (if specified)
+-- when doing down and `fup(n)` (if specified) when going if.
+-- CATEGORY: AST walk
+function M.walk(ast, fdown, fup)
+ assert(type(ast) == 'table')
+ if fdown then fdown(ast) end
+ for _,bast in ipairs(ast) do
+ if type(bast) == 'table' then
+ M.walk(bast, fdown, fup)
+ end
+ end
+ if fup then fup(ast) end
+end
+
+
+-- Replaces contents of table t1 with contents of table t2.
+-- Does not change metatable (if any).
+-- This function is useful for swapping one AST node with another
+-- while preserving any references to the node.
+-- CATEGORY: table utility
+function M.switchtable(t1, t2)
+ for k in pairs(t1) do t1[k] = nil end
+ for k in pairs(t2) do t1[k] = t2[k] end
+end
+
+
+-- Inserts all elements in list bt at index i in list t.
+-- CATEGORY: table utility
+local function tinsertlist(t, i, bt)
+ local oldtlen, delta = #t, i - 1
+ for ti = #t + 1, #t + #bt do t[ti] = false end -- preallocate (avoid holes)
+ for ti = oldtlen, i, -1 do t[ti + #bt] = t[ti] end -- shift
+ for bi = 1, #bt do t[bi + delta] = bt[bi] end -- fill
+end
+--[[TESTSUITE:
+local function _tinsertlist(t, i, bt)
+ for bi=#bt,1,-1 do table.insert(t, i, bt[bi]) end
+end -- equivalent but MUCH less efficient for large tables
+local function _tinsertlist(t, i, bt)
+ for bi=1,#bt do table.insert(t, i+bi-1, bt[bi]) end
+end -- equivalent but MUCH less efficient for large tables
+local t = {}; tinsertlist(t, 1, {}); assert(table.concat(t)=='')
+local t = {}; tinsertlist(t, 1, {2,3}); assert(table.concat(t)=='23')
+local t = {4}; tinsertlist(t, 1, {2,3}); assert(table.concat(t)=='234')
+local t = {2}; tinsertlist(t, 2, {3,4}); assert(table.concat(t)=='234')
+local t = {4,5}; tinsertlist(t, 1, {2,3}); assert(table.concat(t)=='2345')
+local t = {2,5}; tinsertlist(t, 2, {3,4}); assert(table.concat(t)=='2345')
+local t = {2,3}; tinsertlist(t, 3, {4,5}); assert(table.concat(t)=='2345')
+print 'DONE'
+--]]
+
+
+
+-- Gets list of keyword positions related to node ast in source src
+-- note: ast must be visible, i.e. have lineinfo (e.g. unlike `Id "self" definition).
+-- Note: includes operators.
+-- Note: Assumes ast Metalua-style lineinfo is valid.
+-- CATEGORY: tokenlist build
+function M.get_keywords(ast, src)
+ local list = {}
+ if not ast.lineinfo then return list end
+ -- examine space between each pair of children i and j.
+ -- special cases: 0 is before first child and #ast+1 is after last child
+ local i = 0
+ while i <= #ast do
+ -- j is node following i that has lineinfo
+ local j = i+1; while j < #ast+1 and not ast[j].lineinfo do j=j+1 end
+
+ -- Get position range [fpos,lpos] between subsequent children.
+ local fpos
+ if i == 0 then -- before first child
+ fpos = ast.lineinfo.first[3]
+ else
+ local last = ast[i].lineinfo.last; local c = last.comments
+ fpos = (c and #c > 0 and c[#c][3] or last[3]) + 1
+ end
+ local lpos
+ if j == #ast+1 then -- after last child
+ lpos = ast.lineinfo.last[3]
+ else
+ local first = ast[j].lineinfo.first; local c = first.comments
+ --DEBUG('first', ast.tag, first[3], src:sub(first[3], first[3]+3))
+ lpos = (c and #c > 0 and c[1][2] or first[3]) - 1
+ end
+
+ -- Find keyword in range.
+ local spos = fpos
+ repeat
+ local mfpos, tok, mlppos = src:match("^%s*()(%a+)()", spos)
+ if not mfpos then
+ mfpos, tok, mlppos = src:match("^%s*()(%p+)()", spos)
+ end
+ if mfpos then
+ local mlpos = mlppos-1
+ if mlpos > lpos then mlpos = lpos end
+ --DEBUG('look', ast.tag, #ast,i,j,'*', mfpos, tok, mlppos, fpos, lpos, src:sub(fpos, fpos+5))
+ if mlpos >= mfpos then
+ list[#list+1] = mfpos
+ list[#list+1] = mlpos
+ end
+ end
+ spos = mlppos
+ until not spos or spos > lpos
+ -- note: finds single keyword. in `local function` returns only `local`
+ --DEBUG(i,j ,'test[' .. src:sub(fpos, lpos) .. ']')