lib/metalua/base.lua lib/metalua/string2.lua lib/metalua/table2.lua @@ -1,3 +1,6 @@ +2009-03-13 + upgraded to metalua 0.5 from git + 2008-09-20 upgraded to metalua libraries to git version c995760d4ba67c4250ac6029c30ce9491e34ccc2 , which incorporates parsing CHANGES @@ -1,730 +1,747 @@ ----------------------------------------------------------------------- --- Metalua. --- --- Summary: parser generator. Collection of higher order functors, --- which allow to build and combine parsers. Relies on a lexer --- that supports the same API as the one exposed in mll.lua. --- ----------------------------------------------------------------------- --- --- Copyright (c) 2006-2008, Fabien Fleutot <metalua@gmail.com>. --- --- This software is released under the MIT Licence, see licence.txt --- for details. --- ----------------------------------------------------------------------- - --------------------------------------------------------------------------------- --- --- Exported API: --- --- Parser generators: --- * [gg.sequence()] --- * [gg.multisequence()] --- * [gg.expr()] --- * [gg.list()] --- * [gg.onkeyword()] --- * [gg.optkeyword()] --- --- Other functions: --- * [gg.parse_error()] --- * [gg.make_parser()] --- * [gg.is_parser()] --- --------------------------------------------------------------------------------- - -module("gg", package.seeall) - -------------------------------------------------------------------------------- --- parser metatable, which maps __call to method parse, and adds some --- error tracing boilerplate. -------------------------------------------------------------------------------- -local parser_metatable = { } -function parser_metatable.__call (parser, lx, ...) - --printf ("Call parser %q of type %q", parser.name or "?", parser.kind) - if mlc.metabugs then - return parser:parse (lx, ...) - --local x = parser:parse (lx, ...) - --printf ("Result of parser %q: %s", - -- parser.name or "?", - -- _G.table.tostring(x, "nohash", 80)) - --return x - else - local li = lx:lineinfo_right() or { "?", "?", "?", "?" } - local status, ast = pcall (parser.parse, parser, lx, ...) - if status then return ast else - error (string.format ("%s\n - (l.%s, c.%s, k.%s) in parser %s", - ast:strmatch "gg.lua:%d+: (.*)" or ast, - li[1], li[2], li[3], parser.name or parser.kind)) - end - end -end - -------------------------------------------------------------------------------- --- Turn a table into a parser, mainly by setting the metatable. -------------------------------------------------------------------------------- -function make_parser(kind, p) - p.kind = kind - if not p.transformers then p.transformers = { } end - function p.transformers:add (x) - table.insert (self, x) - end - setmetatable (p, parser_metatable) - return p -end - -------------------------------------------------------------------------------- --- Return true iff [x] is a parser. --- If it's a gg-generated parser, reutrn the name of its kind. -------------------------------------------------------------------------------- -function is_parser (x) - return type(x)=="function" or getmetatable(x)==parser_metatable and x.kind -end - -------------------------------------------------------------------------------- --- Parse a sequence, without applying builder nor transformers -------------------------------------------------------------------------------- -local function raw_parse_sequence (lx, p) - local r = { } - for i=1, #p do - e=p[i] - if type(e) == "string" then - if not lx:is_keyword (lx:next(), e) then - parse_error (lx, "Keyword '%s' expected", e) end - elseif is_parser (e) then - table.insert (r, e (lx)) - else - gg.parse_error (lx,"Sequence `%s': element #%i is not a string ".. - "nor a parser: %s", - p.name, i, table.tostring(e)) - end - end - return r -end - -------------------------------------------------------------------------------- --- Parse a multisequence, without applying multisequence transformers. --- The sequences are completely parsed. -------------------------------------------------------------------------------- -local function raw_parse_multisequence (lx, sequence_table, default) - local seq_parser = sequence_table[lx:is_keyword(lx:peek())] - if seq_parser then return seq_parser (lx) - elseif default then return default (lx) - else return false end -end - -------------------------------------------------------------------------------- --- Applies all transformers listed in parser on ast. -------------------------------------------------------------------------------- -local function transform (ast, parser, fli, lli) - if parser.transformers then - for _, t in ipairs (parser.transformers) do ast = t(ast) or ast end - end - if type(ast) == 'table'then - local ali = ast.lineinfo - if not ali or ali.first~=fli or ali.last~=lli then - ast.lineinfo = { first = fli, last = lli } - end - end - return ast -end - -------------------------------------------------------------------------------- --- Generate a tracable parsing error (not implemented yet) -------------------------------------------------------------------------------- -function parse_error(lx, fmt, ...) - local li = lx:lineinfo_left() or {-1,-1,-1, "<unknown file>"} - local msg = string.format("line %i, char %i: "..fmt, li[1], li[2], ...) - local src = lx.src - if li[3]>0 and src then - local i, j = li[3], li[3] - while src:sub(i,i) ~= '\n' and i>=0 do i=i-1 end - while src:sub(j,j) ~= '\n' and j<=#src do j=j+1 end - local srcline = src:sub (i+1, j-1) - local idx = string.rep (" ", li[2]).."^" - msg = string.format("%s\n>>> %s\n>>> %s", msg, srcline, idx) - end - error(msg) -end - -------------------------------------------------------------------------------- --- --- Sequence parser generator --- -------------------------------------------------------------------------------- --- Input fields: --- --- * [builder]: how to build an AST out of sequence parts. let [x] be the list --- of subparser results (keywords are simply omitted). [builder] can be: --- - [nil], in which case the result of parsing is simply [x] --- - a string, which is then put as a tag on [x] --- - a function, which takes [x] as a parameter and returns an AST. --- --- * [name]: the name of the parser. Used for debug messages --- --- * [transformers]: a list of AST->AST functions, applied in order on ASTs --- returned by the parser. --- --- * Table-part entries corresponds to keywords (strings) and subparsers --- (function and callable objects). --- --- After creation, the following fields are added: --- * [parse] the parsing function lexer->AST --- * [kind] == "sequence" --- * [name] is set, if it wasn't in the input. --- -------------------------------------------------------------------------------- -function sequence (p) - make_parser ("sequence", p) - - ------------------------------------------------------------------- - -- Parsing method - ------------------------------------------------------------------- - function p:parse (lx) - -- Raw parsing: - local fli = lx:lineinfo_right() - local seq = raw_parse_sequence (lx, self) - local lli = lx:lineinfo_left() - - -- Builder application: - local builder, tb = self.builder, type (self.builder) - if tb == "string" then seq.tag = builder - elseif tb == "function" or builder and builder.__call then seq = builder(seq) - elseif builder == nil then -- nothing - else error("Invalid builder of type "..tb.." in sequence") end - seq = transform (seq, self, fli, lli) - return seq - end - - ------------------------------------------------------------------- - -- Construction - ------------------------------------------------------------------- - -- Try to build a proper name - if not p.name and type(p[1])=="string" then - p.name = p[1].." ..." - if type(p[#p])=="string" then p.name = p.name .. " " .. p[#p] end - else - p.name = "<anonymous>" - end - - return p -end --</sequence> - - -------------------------------------------------------------------------------- --- --- Multiple, keyword-driven, sequence parser generator --- -------------------------------------------------------------------------------- --- in [p], useful fields are: --- --- * [transformers]: as usual --- --- * [name]: as usual --- --- * Table-part entries must be sequence parsers, or tables which can --- be turned into a sequence parser by [gg.sequence]. These --- sequences must start with a keyword, and this initial keyword --- must be different for each sequence. The table-part entries will --- be removed after [gg.multisequence] returns. --- --- * [default]: the parser to run if the next keyword in the lexer is --- none of the registered initial keywords. If there's no default --- parser and no suitable initial keyword, the multisequence parser --- simply returns [false]. --- --- After creation, the following fields are added: --- --- * [parse] the parsing function lexer->AST --- --- * [sequences] the table of sequences, indexed by initial keywords. --- --- * [add] method takes a sequence parser or a config table for --- [gg.sequence], and adds/replaces the corresponding sequence --- parser. If the keyword was already used, the former sequence is --- removed and a warning is issued. --- --- * [get] method returns a sequence by its initial keyword --- --- * [kind] == "multisequence" --- -------------------------------------------------------------------------------- -function multisequence (p) - make_parser ("multisequence", p) - - ------------------------------------------------------------------- - -- Add a sequence (might be just a config table for [gg.sequence]) - ------------------------------------------------------------------- - function p:add (s) - -- compile if necessary: - if not is_parser(s) then sequence(s) end - if type(s[1]) ~= "string" then - error "Invalid sequence for multiseq" - elseif self.sequences[s[1]] then - printf (" *** Warning: keyword %q overloaded in multisequence ***", s[1]) - end - self.sequences[s[1]] = s - end -- </multisequence.add> - - ------------------------------------------------------------------- - -- Get the sequence starting with this keyword. [kw :: string] - ------------------------------------------------------------------- - function p:get (kw) return self.sequences [kw] end - - ------------------------------------------------------------------- - -- Remove the sequence starting with keyword [kw :: string] - ------------------------------------------------------------------- - function p:del (kw) - if not self.sequences[kw] then - printf("*** Warning: trying to delete sequence starting ".. - "with %q from a multisequence having no such ".. - "entry ***", kw) end - local removed = self.sequences[kw] - self.sequences[kw] = nil - return removed - end - - ------------------------------------------------------------------- - -- Parsing method - ------------------------------------------------------------------- - function p:parse (lx) - local fli = lx:lineinfo_right() - local x = raw_parse_multisequence (lx, self.sequences, self.default) - local lli = lx:lineinfo_left() - return transform (x, self, fli, lli) - end - - ------------------------------------------------------------------- - -- Construction - ------------------------------------------------------------------- - -- Register the sequences passed to the constructor. They're going - -- from the array part of the parser to the hash part of field - -- [sequences] - p.sequences = { } - for i=1, #p do p:add (p[i]); p[i] = nil end - - -- FIXME: why is this commented out? - --if p.default and not is_parser(p.default) then sequence(p.default) end - return p -end --</multisequence> - - -------------------------------------------------------------------------------- --- --- Expression parser generator --- -------------------------------------------------------------------------------- --- --- Expression configuration relies on three tables: [prefix], [infix] --- and [suffix]. Moreover, the primary parser can be replaced by a --- table: in this case the [primary] table will be passed to --- [gg.multisequence] to create a parser. --- --- Each of these tables is a modified multisequence parser: the --- differences with respect to regular multisequence config tables are: --- --- * the builder takes specific parameters: --- - for [prefix], it takes the result of the prefix sequence parser, --- and the prefixed expression --- - for [infix], it takes the left-hand-side expression, the results --- of the infix sequence parser, and the right-hand-side expression. --- - for [suffix], it takes the suffixed expression, and theresult --- of the suffix sequence parser. --- --- * the default field is a list, with parameters: --- - [parser] the raw parsing function --- - [transformers], as usual --- - [prec], the operator's precedence --- - [assoc] for [infix] table, the operator's associativity, which --- can be "left", "right" or "flat" (default to left) --- --- In [p], useful fields are: --- * [transformers]: as usual --- * [name]: as usual --- * [primary]: the atomic expression parser, or a multisequence config --- table (mandatory) --- * [prefix]: prefix operators config table, see above. --- * [infix]: infix operators config table, see above. --- * [suffix]: suffix operators config table, see above. --- --- After creation, these fields are added: --- * [kind] == "expr" --- * [parse] as usual --- * each table is turned into a multisequence, and therefore has an --- [add] method --- -------------------------------------------------------------------------------- -function expr (p) - make_parser ("expr", p) - - ------------------------------------------------------------------- - -- parser method. - -- In addition to the lexer, it takes an optional precedence: - -- it won't read expressions whose precedence is lower or equal - -- to [prec]. - ------------------------------------------------------------------- - function p:parse (lx, prec) - prec = prec or 0 - - ------------------------------------------------------ - -- Extract the right parser and the corresponding - -- options table, for (pre|in|suff)fix operators. - -- Options include prec, assoc, transformers. - ------------------------------------------------------ - local function get_parser_info (tab) - local p2 = tab:get (lx:is_keyword (lx:peek())) - if p2 then -- keyword-based sequence found - local function parser(lx) return raw_parse_sequence(lx, p2) end - return parser, p2 - else -- Got to use the default parser - local d = tab.default - if d then return d.parse or d.parser, d - else return false, false end - end - end - - ------------------------------------------------------ - -- Look for a prefix sequence. Multiple prefixes are - -- handled through the recursive [p.parse] call. - -- Notice the double-transform: one for the primary - -- expr, and one for the one with the prefix op. - ------------------------------------------------------ - local function handle_prefix () - local fli = lx:lineinfo_right() - local p2_func, p2 = get_parser_info (self.prefix) - local op = p2_func and p2_func (lx) - if op then -- Keyword-based sequence found - local ili = lx:lineinfo_right() -- Intermediate LineInfo - local e = p2.builder (op, self:parse (lx, p2.prec)) - local lli = lx:lineinfo_left() - return transform (transform (e, p2, ili, lli), self, fli, lli) - else -- No prefix found, get a primary expression - local e = self.primary(lx) - local lli = lx:lineinfo_left() - return transform (e, self, fli, lli) - end - end --</expr.parse.handle_prefix> - - ------------------------------------------------------ - -- Look for an infix sequence+right-hand-side operand. - -- Return the whole binary expression result, - -- or false if no operator was found. - ------------------------------------------------------ - local function handle_infix (e) - local p2_func, p2 = get_parser_info (self.infix) - if not p2 then return false end - - ----------------------------------------- - -- Handle flattening operators: gather all operands - -- of the series in [list]; when a different operator - -- is found, stop, build from [list], [transform] and - -- return. - ----------------------------------------- - if (not p2.prec or p2.prec>prec) and p2.assoc=="flat" then - local fli = lx:lineinfo_right() - local pflat, list = p2, { e } - repeat - local op = p2_func(lx) - if not op then break end - table.insert (list, self:parse (lx, p2.prec)) - local _ -- We only care about checking that p2==pflat - _, p2 = get_parser_info (self.infix) - until p2 ~= pflat - local e2 = pflat.builder (list) - local lli = lx:lineinfo_left() - return transform (transform (e2, pflat, fli, lli), self, fli, lli) - - ----------------------------------------- - -- Handle regular infix operators: [e] the LHS is known, - -- just gather the operator and [e2] the RHS. - -- Result goes in [e3]. - ----------------------------------------- - elseif p2.prec and p2.prec>prec or - p2.prec==prec and p2.assoc=="right" then - local fli = e.lineinfo.first -- lx:lineinfo_right() - local op = p2_func(lx) - if not op then return false end - local e2 = self:parse (lx, p2.prec) - local e3 = p2.builder (e, op, e2) - local lli = lx:lineinfo_left() - return transform (transform (e3, p2, fli, lli), self, fli, lli) - - ----------------------------------------- - -- Check for non-associative operators, and complain if applicable. - ----------------------------------------- - elseif p2.assoc=="none" and p2.prec==prec then - parser_error (lx, "non-associative operator!") - - ----------------------------------------- - -- No infix operator suitable at that precedence - ----------------------------------------- - else return false end - - end --</expr.parse.handle_infix> - - ------------------------------------------------------ - -- Look for a suffix sequence. - -- Return the result of suffix operator on [e], - -- or false if no operator was found. - ------------------------------------------------------ - local function handle_suffix (e) - -- FIXME bad fli, must take e.lineinfo.first - local p2_func, p2 = get_parser_info (self.suffix) - if not p2 then return false end - if not p2.prec or p2.prec>=prec then - local fli = lx:lineinfo_right() - local op = p2_func(lx) - if not op then return false end - local lli = lx:lineinfo_left() - e = p2.builder (e, op) - e = transform (transform (e, p2, fli, lli), self, fli, lli) - return e - end - return false - end --</expr.parse.handle_suffix> - - ------------------------------------------------------ - -- Parser body: read suffix and (infix+operand) - -- extensions as long as we're able to fetch more at - -- this precedence level. - ------------------------------------------------------ - local e = handle_prefix() - repeat - local x = handle_suffix (e); e = x or e - local y = handle_infix (e); e = y or e - until not (x or y) - - -- No transform: it already happened in operators handling - return e - end --</expr.parse> - - ------------------------------------------------------------------- - -- Construction - ------------------------------------------------------------------- - if not p.primary then p.primary=p[1]; p[1]=nil end - for _, t in ipairs{ "primary", "prefix", "infix", "suffix" } do - if not p[t] then p[t] = { } end - if not is_parser(p[t]) then multisequence(p[t]) end - end - function p:add(...) return self.primary:add(...) end - return p -end --</expr> - - -------------------------------------------------------------------------------- --- --- List parser generator --- -------------------------------------------------------------------------------- --- In [p], the following fields can be provided in input: --- --- * [builder]: takes list of subparser results, returns AST --- * [transformers]: as usual --- * [name]: as usual --- --- * [terminators]: list of strings representing the keywords which --- might mark the end of the list. When non-empty, the list is --- allowed to be empty. A string is treated as a single-element --- table, whose element is that string, e.g. ["do"] is the same as --- [{"do"}]. --- --- * [separators]: list of strings representing the keywords which can --- separate elements of the list. When non-empty, one of these --- keyword has to be found between each element. Lack of a separator --- indicates the end of the list. A string is treated as a --- single-element table, whose element is that string, e.g. ["do"] --- is the same as [{"do"}]. If [terminators] is empty/nil, then --- [separators] has to be non-empty. --- --- After creation, the following fields are added: --- * [parse] the parsing function lexer->AST --- * [kind] == "list" --- -------------------------------------------------------------------------------- -function list (p) - make_parser ("list", p) - - ------------------------------------------------------------------- - -- Parsing method - ------------------------------------------------------------------- - function p:parse (lx) - - ------------------------------------------------------ - -- Used to quickly check whether there's a terminator - -- or a separator immediately ahead - ------------------------------------------------------ - local function peek_is_in (keywords) - return keywords and lx:is_keyword(lx:peek(), unpack(keywords)) end - - local x = { } - local fli = lx:lineinfo_right() - - -- if there's a terminator to start with, don't bother trying - if not peek_is_in (self.terminators) then - repeat table.insert (x, self.primary (lx)) -- read one element - until - -- First reason to stop: There's a separator list specified, - -- and next token isn't one. Otherwise, consume it with [lx:next()] - self.separators and not(peek_is_in (self.separators) and lx:next()) or - -- Other reason to stop: terminator token ahead - peek_is_in (self.terminators) or - -- Last reason: end of file reached - lx:peek().tag=="Eof" - end - - local lli = lx:lineinfo_left() - - -- Apply the builder. It can be a string, or a callable value, - -- or simply nothing. - local b = self.builder - if b then - if type(b)=="string" then x.tag = b -- b is a string, use it as a tag - elseif type(b)=="function" then x=b(x) - else - local bmt = getmetatable(b) - if bmt and bmt.__call then x=b(x) end - end - end - return transform (x, self, fli, lli) - end --</list.parse> - - ------------------------------------------------------------------- - -- Construction - ------------------------------------------------------------------- - if not p.primary then p.primary = p[1]; p[1] = nil end - if type(p.terminators) == "string" then p.terminators = { p.terminators } - elseif p.terminators and #p.terminators == 0 then p.terminators = nil end - if type(p.separators) == "string" then p.separators = { p.separators } - elseif p.separators and #p.separators == 0 then p.separators = nil end - - return p -end --</list> - - -------------------------------------------------------------------------------- --- --- Keyword-conditionned parser generator --- -------------------------------------------------------------------------------- --- --- Only apply a parser if a given keyword is found. The result of --- [gg.onkeyword] parser is the result of the subparser (modulo --- [transformers] applications). --- --- Input fields: --- --- * [name]: as usual --- --- * [transformers]: as usual --- --- * [peek]: if non-nil, the conditionning keyword is left in the lexeme --- stream instead of being consumed. --- --- * [primary]: the subparser. --- --- * [keywords]: list of strings representing triggering keywords. --- --- * Table-part entries can contain strings, and/or exactly one parser. --- Strings are put in [keywords], and the parser is put in [primary]. --- --- After the call, the following fields will be set: --- --- * [parse] the parsing method --- * [kind] == "onkeyword" --- * [primary] --- * [keywords] --- -------------------------------------------------------------------------------- -function onkeyword (p) - make_parser ("onkeyword", p) - - ------------------------------------------------------------------- - -- Parsing method - ------------------------------------------------------------------- - function p:parse(lx) - if lx:is_keyword (lx:peek(), unpack(self.keywords)) then - local fli = lx:lineinfo_right() - if not self.peek then lx:next() end - local lli = lx:lineinfo_left() - return transform (self.primary(lx), p, fli, lli) - else return false end - end - - ------------------------------------------------------------------- - -- Construction - ------------------------------------------------------------------- - if not p.keywords then p.keywords = { } end - for _, x in ipairs(p) do - if type(x)=="string" then table.insert (p.keywords, x) - else assert (not p.primary and is_parser (x)); p.primary = x end - end - assert (p.primary, 'no primary parser in gg.onkeyword') - return p -end --</onkeyword> - - -------------------------------------------------------------------------------- --- --- Optional keyword consummer pseudo-parser generator --- -------------------------------------------------------------------------------- --- --- This doesn't return a real parser, just a function. That function parses --- one of the keywords passed as parameters, and returns it. It returns --- [false] if no matching keyword is found. --- --- Notice that tokens returned by lexer already carry lineinfo, therefore --- there's no need to add them, as done usually through transform() calls. -------------------------------------------------------------------------------- -function optkeyword (...) - local args = {...} - if type (args[1]) == "table" then - assert (#args == 1) - args = args[1] - end - for _, v in ipairs(args) do assert (type(v)=="string") end - return function (lx) - local x = lx:is_keyword (lx:peek(), unpack (args)) - if x then lx:next(); return x - else return false end - end -end - - -------------------------------------------------------------------------------- --- --- Run a parser with a special lexer --- -------------------------------------------------------------------------------- --- --- This doesn't return a real parser, just a function. --- First argument is the lexer class to be used with the parser, --- 2nd is the parser itself. --- The resulting parser returns whatever the argument parser does. --- -------------------------------------------------------------------------------- -function with_lexer(new_lexer, parser) - - ------------------------------------------------------------------- - -- Most gg functions take their parameters in a table, so it's - -- better to silently accept when with_lexer{ } is called with - -- its arguments in a list: - ------------------------------------------------------------------- - if not parser and #new_lexer==2 and type(new_lexer[1])=='table' then - return with_lexer(unpack(new_lexer)) - end - - ------------------------------------------------------------------- - -- Save the current lexer, switch it for the new one, run the parser, - -- restore the previous lexer, even if the parser caused an error. - ------------------------------------------------------------------- - return function (lx) - local old_lexer = getmetatable(lx) - lx:sync() - setmetatable(lx, new_lexer) - local status, result = pcall(parser, lx) - lx:sync() - setmetatable(lx, old_lexer) - if status then return result else error(result) end - end -end +---------------------------------------------------------------------- +-- Metalua. +-- +-- Summary: parser generator. Collection of higher order functors, +-- which allow to build and combine parsers. Relies on a lexer +-- that supports the same API as the one exposed in mll.lua. +-- +---------------------------------------------------------------------- +-- +-- Copyright (c) 2006-2008, Fabien Fleutot <metalua@gmail.com>. +-- +-- This software is released under the MIT Licence, see licence.txt +-- for details. +-- +---------------------------------------------------------------------- + +-------------------------------------------------------------------------------- +-- +-- Exported API: +-- +-- Parser generators: +-- * [gg.sequence()] +-- * [gg.multisequence()] +-- * [gg.expr()] +-- * [gg.list()] +-- * [gg.onkeyword()] +-- * [gg.optkeyword()] +-- +-- Other functions: +-- * [gg.parse_error()] +-- * [gg.make_parser()] +-- * [gg.is_parser()] +-- +-------------------------------------------------------------------------------- + +module("gg", package.seeall) + +------------------------------------------------------------------------------- +-- parser metatable, which maps __call to method parse, and adds some +-- error tracing boilerplate. +------------------------------------------------------------------------------- +local parser_metatable = { } +function parser_metatable.__call (parser, lx, ...) + --printf ("Call parser %q of type %q", parser.name or "?", parser.kind) + if mlc.metabugs then + return parser:parse (lx, ...) + --local x = parser:parse (lx, ...) + --printf ("Result of parser %q: %s", + -- parser.name or "?", + -- _G.table.tostring(x, "nohash", 80)) + --return x + else + local li = lx:lineinfo_right() or { "?", "?", "?", "?" } + local status, ast = pcall (parser.parse, parser, lx, ...) + if status then return ast else + error (string.format ("%s\n - (l.%s, c.%s, k.%s) in parser %s", + ast:strmatch "gg.lua:%d+: (.*)" or ast, + li[1], li[2], li[3], parser.name or parser.kind)) + end + end +end + +------------------------------------------------------------------------------- +-- Turn a table into a parser, mainly by setting the metatable. +------------------------------------------------------------------------------- +function make_parser(kind, p) + p.kind = kind + if not p.transformers then p.transformers = { } end + function p.transformers:add (x) + table.insert (self, x) + end + setmetatable (p, parser_metatable) + return p +end + +------------------------------------------------------------------------------- +-- Return true iff [x] is a parser. +-- If it's a gg-generated parser, return the name of its kind. +------------------------------------------------------------------------------- +function is_parser (x) + return type(x)=="function" or getmetatable(x)==parser_metatable and x.kind +end + +------------------------------------------------------------------------------- +-- Parse a sequence, without applying builder nor transformers +------------------------------------------------------------------------------- +local function raw_parse_sequence (lx, p) + local r = { } + for i=1, #p do + e=p[i] + if type(e) == "string" then + if not lx:is_keyword (lx:next(), e) then + parse_error (lx, "Keyword '%s' expected", e) end + elseif is_parser (e) then + table.insert (r, e (lx)) + else + gg.parse_error (lx,"Sequence `%s': element #%i is not a string ".. + "nor a parser: %s", + p.name, i, table.tostring(e)) + end + end + return r +end + +------------------------------------------------------------------------------- +-- Parse a multisequence, without applying multisequence transformers. +-- The sequences are completely parsed. +------------------------------------------------------------------------------- +local function raw_parse_multisequence (lx, sequence_table, default) + local seq_parser = sequence_table[lx:is_keyword(lx:peek())] + if seq_parser then return seq_parser (lx) + elseif default then return default (lx) + else return false end +end + +------------------------------------------------------------------------------- +-- Applies all transformers listed in parser on ast. +------------------------------------------------------------------------------- +local function transform (ast, parser, fli, lli) + if parser.transformers then + for _, t in ipairs (parser.transformers) do ast = t(ast) or ast end + end + if type(ast) == 'table'then + local ali = ast.lineinfo + if not ali or ali.first~=fli or ali.last~=lli then + ast.lineinfo = { first = fli, last = lli } + end + end + return ast +end + +------------------------------------------------------------------------------- +-- Generate a tracable parsing error (not implemented yet) +------------------------------------------------------------------------------- +function parse_error(lx, fmt, ...) + local li = lx:lineinfo_left() or {-1,-1,-1, "<unknown file>"} + local msg = string.format("line %i, char %i: "..fmt, li[1], li[2], ...) + local src = lx.src + if li[3]>0 and src then + local i, j = li[3], li[3] + while src:sub(i,i) ~= '\n' and i>=0 do i=i-1 end + while src:sub(j,j) ~= '\n' and j<=#src do j=j+1 end + local srcline = src:sub (i+1, j-1) + local idx = string.rep (" ", li[2]).."^" + msg = string.format("%s\n>>> %s\n>>> %s", msg, srcline, idx) + end + error(msg) +end + +------------------------------------------------------------------------------- +-- +-- Sequence parser generator +-- +------------------------------------------------------------------------------- +-- Input fields: +-- +-- * [builder]: how to build an AST out of sequence parts. let [x] be the list +-- of subparser results (keywords are simply omitted). [builder] can be: +-- - [nil], in which case the result of parsing is simply [x] +-- - a string, which is then put as a tag on [x] +-- - a function, which takes [x] as a parameter and returns an AST. +-- +-- * [name]: the name of the parser. Used for debug messages +-- +-- * [transformers]: a list of AST->AST functions, applied in order on ASTs +-- returned by the parser. +-- +-- * Table-part entries corresponds to keywords (strings) and subparsers +-- (function and callable objects). +-- +-- After creation, the following fields are added: +-- * [parse] the parsing function lexer->AST +-- * [kind] == "sequence" +-- * [name] is set, if it wasn't in the input. +-- +------------------------------------------------------------------------------- +function sequence (p) + make_parser ("sequence", p) + + ------------------------------------------------------------------- + -- Parsing method + ------------------------------------------------------------------- + function p:parse (lx) + -- Raw parsing: + local fli = lx:lineinfo_right() + local seq = raw_parse_sequence (lx, self) + local lli = lx:lineinfo_left() + + -- Builder application: + local builder, tb = self.builder, type (self.builder) + if tb == "string" then seq.tag = builder + elseif tb == "function" or builder and builder.__call then seq = builder(seq) + elseif builder == nil then -- nothing + else error ("Invalid builder of type "..tb.." in sequence") end + seq = transform (seq, self, fli, lli) + assert (not seq or seq.lineinfo) + return seq + end + + ------------------------------------------------------------------- + -- Construction + ------------------------------------------------------------------- + -- Try to build a proper name + if not p.name and type(p[1])=="string" then + p.name = p[1].." ..." + if type(p[#p])=="string" then p.name = p.name .. " " .. p[#p] end + else + p.name = "<anonymous>" + end + + return p +end --</sequence> + + +------------------------------------------------------------------------------- +-- +-- Multiple, keyword-driven, sequence parser generator +-- +------------------------------------------------------------------------------- +-- in [p], useful fields are: +-- +-- * [transformers]: as usual +-- +-- * [name]: as usual +-- +-- * Table-part entries must be sequence parsers, or tables which can +-- be turned into a sequence parser by [gg.sequence]. These +-- sequences must start with a keyword, and this initial keyword +-- must be different for each sequence. The table-part entries will +-- be removed after [gg.multisequence] returns. +-- +-- * [default]: the parser to run if the next keyword in the lexer is +-- none of the registered initial keywords. If there's no default +-- parser and no suitable initial keyword, the multisequence parser +-- simply returns [false]. +-- +-- After creation, the following fields are added: +-- +-- * [parse] the parsing function lexer->AST +-- +-- * [sequences] the table of sequences, indexed by initial keywords. +-- +-- * [add] method takes a sequence parser or a config table for +-- [gg.sequence], and adds/replaces the corresponding sequence +-- parser. If the keyword was already used, the former sequence is +-- removed and a warning is issued. +-- +-- * [get] method returns a sequence by its initial keyword +-- +-- * [kind] == "multisequence" +-- +------------------------------------------------------------------------------- +function multisequence (p) + make_parser ("multisequence", p) + + ------------------------------------------------------------------- + -- Add a sequence (might be just a config table for [gg.sequence]) + ------------------------------------------------------------------- + function p:add (s) + -- compile if necessary: + local keyword = s[1] + if not is_parser(s) then sequence(s) end + if is_parser(s) ~= 'sequence' or type(keyword) ~= "string" then + if self.default then -- two defaults + error ("In a multisequence parser, all but one sequences ".. + "must start with a keyword") + else self.default = s end -- first default + elseif self.sequences[keyword] then -- duplicate keyword + eprintf (" *** Warning: keyword %q overloaded in multisequence ***", keyword) + self.sequences[keyword] = s + else -- newly caught keyword + self.sequences[keyword] = s + end + end -- </multisequence.add> + + ------------------------------------------------------------------- + -- Get the sequence starting with this keyword. [kw :: string] + ------------------------------------------------------------------- + function p:get (kw) return self.sequences [kw] end + + ------------------------------------------------------------------- + -- Remove the sequence starting with keyword [kw :: string] + ------------------------------------------------------------------- + function p:del (kw) + if not self.sequences[kw] then + eprintf("*** Warning: trying to delete sequence starting ".. + "with %q from a multisequence having no such ".. + "entry ***", kw) end + local removed = self.sequences[kw] + self.sequences[kw] = nil + return removed + end + + ------------------------------------------------------------------- + -- Parsing method + ------------------------------------------------------------------- + function p:parse (lx) + local fli = lx:lineinfo_right() + local x = raw_parse_multisequence (lx, self.sequences, self.default) + local lli = lx:lineinfo_left() + return transform (x, self, fli, lli) + end + + ------------------------------------------------------------------- + -- Construction + ------------------------------------------------------------------- + -- Register the sequences passed to the constructor. They're going + -- from the array part of the parser to the hash part of field + -- [sequences] + p.sequences = { } + for i=1, #p do p:add (p[i]); p[i] = nil end + + -- FIXME: why is this commented out? + --if p.default and not is_parser(p.default) then sequence(p.default) end + return p +end --</multisequence> + + +------------------------------------------------------------------------------- +-- +-- Expression parser generator +-- +------------------------------------------------------------------------------- +-- +-- Expression configuration relies on three tables: [prefix], [infix] +-- and [suffix]. Moreover, the primary parser can be replaced by a +-- table: in this case the [primary] table will be passed to +-- [gg.multisequence] to create a parser. +-- +-- Each of these tables is a modified multisequence parser: the +-- differences with respect to regular multisequence config tables are: +-- +-- * the builder takes specific parameters: +-- - for [prefix], it takes the result of the prefix sequence parser, +-- and the prefixed expression +-- - for [infix], it takes the left-hand-side expression, the results +-- of the infix sequence parser, and the right-hand-side expression. +-- - for [suffix], it takes the suffixed expression, and theresult +-- of the suffix sequence parser. +-- +-- * the default field is a list, with parameters: +-- - [parser] the raw parsing function +-- - [transformers], as usual +-- - [prec], the operator's precedence +-- - [assoc] for [infix] table, the operator's associativity, which +-- can be "left", "right" or "flat" (default to left) +-- +-- In [p], useful fields are: +-- * [transformers]: as usual +-- * [name]: as usual +-- * [primary]: the atomic expression parser, or a multisequence config +-- table (mandatory) +-- * [prefix]: prefix operators config table, see above. +-- * [infix]: infix operators config table, see above. +-- * [suffix]: suffix operators config table, see above. +-- +-- After creation, these fields are added: +-- * [kind] == "expr" +-- * [parse] as usual +-- * each table is turned into a multisequence, and therefore has an +-- [add] method +-- +------------------------------------------------------------------------------- +function expr (p) + make_parser ("expr", p) + + ------------------------------------------------------------------- + -- parser method. + -- In addition to the lexer, it takes an optional precedence: + -- it won't read expressions whose precedence is lower or equal + -- to [prec]. + ------------------------------------------------------------------- + function p:parse (lx, prec) + prec = prec or 0 + + ------------------------------------------------------ + -- Extract the right parser and the corresponding + -- options table, for (pre|in|suff)fix operators. + -- Options include prec, assoc, transformers. + ------------------------------------------------------ + local function get_parser_info (tab) + local p2 = tab:get (lx:is_keyword (lx:peek())) + if p2 then -- keyword-based sequence found + local function parser(lx) return raw_parse_sequence(lx, p2) end + return parser, p2 + else -- Got to use the default parser + local d = tab.default + if d then return d.parse or d.parser, d + else return false, false end + end + end + + ------------------------------------------------------ + -- Look for a prefix sequence. Multiple prefixes are + -- handled through the recursive [p.parse] call. + -- Notice the double-transform: one for the primary + -- expr, and one for the one with the prefix op. + ------------------------------------------------------ + local function handle_prefix () + local fli = lx:lineinfo_right() + local p2_func, p2 = get_parser_info (self.prefix) + local op = p2_func and p2_func (lx) + if op then -- Keyword-based sequence found + local ili = lx:lineinfo_right() -- Intermediate LineInfo + local e = p2.builder (op, self:parse (lx, p2.prec)) + local lli = lx:lineinfo_left() + return transform (transform (e, p2, ili, lli), self, fli, lli) + else -- No prefix found, get a primary expression + local e = self.primary(lx) + local lli = lx:lineinfo_left() + return transform (e, self, fli, lli) + end + end --</expr.parse.handle_prefix> + + ------------------------------------------------------ + -- Look for an infix sequence+right-hand-side operand. + -- Return the whole binary expression result, + -- or false if no operator was found. + ------------------------------------------------------ + local function handle_infix (e) + local p2_func, p2 = get_parser_info (self.infix) + if not p2 then return false end + + ----------------------------------------- + -- Handle flattening operators: gather all operands + -- of the series in [list]; when a different operator + -- is found, stop, build from [list], [transform] and + -- return. + ----------------------------------------- + if (not p2.prec or p2.prec>prec) and p2.assoc=="flat" then + local fli = lx:lineinfo_right() + local pflat, list = p2, { e } + repeat + local op = p2_func(lx) + if not op then break end + table.insert (list, self:parse (lx, p2.prec)) + local _ -- We only care about checking that p2==pflat + _, p2 = get_parser_info (self.infix) + until p2 ~= pflat + local e2 = pflat.builder (list) + local lli = lx:lineinfo_left() + return transform (transform (e2, pflat, fli, lli), self, fli, lli) + + ----------------------------------------- + -- Handle regular infix operators: [e] the LHS is known, + -- just gather the operator and [e2] the RHS. + -- Result goes in [e3]. + ----------------------------------------- + elseif p2.prec and p2.prec>prec or + p2.prec==prec and p2.assoc=="right" then + local fli = e.lineinfo.first -- lx:lineinfo_right() + local op = p2_func(lx) + if not op then return false end + local e2 = self:parse (lx, p2.prec) + local e3 = p2.builder (e, op, e2) + local lli = lx:lineinfo_left() + return transform (transform (e3, p2, fli, lli), self, fli, lli) + + ----------------------------------------- + -- Check for non-associative operators, and complain if applicable. + ----------------------------------------- + elseif p2.assoc=="none" and p2.prec==prec then + parser_error (lx, "non-associative operator!") + + ----------------------------------------- + -- No infix operator suitable at that precedence + ----------------------------------------- + else return false end + + end --</expr.parse.handle_infix> + + ------------------------------------------------------ + -- Look for a suffix sequence. + -- Return the result of suffix operator on [e], + -- or false if no operator was found. + ------------------------------------------------------ + local function handle_suffix (e) + -- FIXME bad fli, must take e.lineinfo.first + local p2_func, p2 = get_parser_info (self.suffix) + if not p2 then return false end + if not p2.prec or p2.prec>=prec then + --local fli = lx:lineinfo_right() + local fli = e.lineinfo.first + local op = p2_func(lx) + if not op then return false end + local lli = lx:lineinfo_left() + e = p2.builder (e, op) + e = transform (transform (e, p2, fli, lli), self, fli, lli) + return e + end + return false + end --</expr.parse.handle_suffix> + + ------------------------------------------------------ + -- Parser body: read suffix and (infix+operand) + -- extensions as long as we're able to fetch more at + -- this precedence level. + ------------------------------------------------------ + local e = handle_prefix() + repeat + local x = handle_suffix (e); e = x or e + local y = handle_infix (e); e = y or e + until not (x or y) + + -- No transform: it already happened in operators handling + return e + end --</expr.parse> + + ------------------------------------------------------------------- + -- Construction + ------------------------------------------------------------------- + if not p.primary then p.primary=p[1]; p[1]=nil end + for _, t in ipairs{ "primary", "prefix", "infix", "suffix" } do + if not p[t] then p[t] = { } end + if not is_parser(p[t]) then multisequence(p[t]) end + end + function p:add(...) return self.primary:add(...) end + return p +end --</expr> + + +------------------------------------------------------------------------------- +-- +-- List parser generator +-- +------------------------------------------------------------------------------- +-- In [p], the following fields can be provided in input: +-- +-- * [builder]: takes list of subparser results, returns AST +-- * [transformers]: as usual +-- * [name]: as usual +-- +-- * [terminators]: list of strings representing the keywords which +-- might mark the end of the list. When non-empty, the list is +-- allowed to be empty. A string is treated as a single-element +-- table, whose element is that string, e.g. ["do"] is the same as +-- [{"do"}]. +-- +-- * [separators]: list of strings representing the keywords which can +-- separate elements of the list. When non-empty, one of these +-- keyword has to be found between each element. Lack of a separator +-- indicates the end of the list. A string is treated as a +-- single-element table, whose element is that string, e.g. ["do"] +-- is the same as [{"do"}]. If [terminators] is empty/nil, then +-- [separators] has to be non-empty. +-- +-- After creation, the following fields are added: +-- * [parse] the parsing function lexer->AST +-- * [kind] == "list" +-- +------------------------------------------------------------------------------- +function list (p) + make_parser ("list", p) + + ------------------------------------------------------------------- + -- Parsing method + ------------------------------------------------------------------- + function p:parse (lx) + + ------------------------------------------------------ + -- Used to quickly check whether there's a terminator + -- or a separator immediately ahead + ------------------------------------------------------ + local function peek_is_in (keywords) + return keywords and lx:is_keyword(lx:peek(), unpack(keywords)) end + + local x = { } + local fli = lx:lineinfo_right() + + -- if there's a terminator to start with, don't bother trying + if not peek_is_in (self.terminators) then + repeat table.insert (x, self.primary (lx)) -- read one element + until + -- First reason to stop: There's a separator list specified, + -- and next token isn't one. Otherwise, consume it with [lx:next()] + self.separators and not(peek_is_in (self.separators) and lx:next()) or + -- Other reason to stop: terminator token ahead + peek_is_in (self.terminators) or + -- Last reason: end of file reached + lx:peek().tag=="Eof" + end + + local lli = lx:lineinfo_left() + + -- Apply the builder. It can be a string, or a callable value, + -- or simply nothing. + local b = self.builder + if b then + if type(b)=="string" then x.tag = b -- b is a string, use it as a tag + elseif type(b)=="function" then x=b(x) + else + local bmt = getmetatable(b) + if bmt and bmt.__call then x=b(x) end + end + end + return transform (x, self, fli, lli) + end --</list.parse> + + ------------------------------------------------------------------- + -- Construction + ------------------------------------------------------------------- + if not p.primary then p.primary = p[1]; p[1] = nil end + if type(p.terminators) == "string" then p.terminators = { p.terminators } + elseif p.terminators and #p.terminators == 0 then p.terminators = nil end + if type(p.separators) == "string" then p.separators = { p.separators } + elseif p.separators and #p.separators == 0 then p.separators = nil end + + return p +end --</list> + + +------------------------------------------------------------------------------- +-- +-- Keyword-conditionned parser generator +-- +------------------------------------------------------------------------------- +-- +-- Only apply a parser if a given keyword is found. The result of +-- [gg.onkeyword] parser is the result of the subparser (modulo +-- [transformers] applications). +-- +-- lineinfo: the keyword is *not* included in the boundaries of the +-- resulting lineinfo. A review of all usages of gg.onkeyword() in the +-- implementation of metalua has shown that it was the appropriate choice +-- in every case. +-- +-- Input fields: +-- +-- * [name]: as usual +-- +-- * [transformers]: as usual +-- +-- * [peek]: if non-nil, the conditionning keyword is left in the lexeme +-- stream instead of being consumed. +-- +-- * [primary]: the subparser. +-- +-- * [keywords]: list of strings representing triggering keywords. +-- +-- * Table-part entries can contain strings, and/or exactly one parser. +-- Strings are put in [keywords], and the parser is put in [primary]. +-- +-- After the call, the following fields will be set: +-- +-- * [parse] the parsing method +-- * [kind] == "onkeyword" +-- * [primary] +-- * [keywords] +-- +------------------------------------------------------------------------------- +function onkeyword (p) + make_parser ("onkeyword", p) + + ------------------------------------------------------------------- + -- Parsing method + ------------------------------------------------------------------- + function p:parse(lx) + if lx:is_keyword (lx:peek(), unpack(self.keywords)) then + --local fli = lx:lineinfo_right() + if not self.peek then lx:next() end + local content = self.primary (lx) + --local lli = lx:lineinfo_left() + local fli, lli = content.lineinfo.first, content.lineinfo.last + return transform (content, p, fli, lli) + else return false end + end + + ------------------------------------------------------------------- + -- Construction + ------------------------------------------------------------------- + if not p.keywords then p.keywords = { } end + for _, x in ipairs(p) do + if type(x)=="string" then table.insert (p.keywords, x) + else assert (not p.primary and is_parser (x)); p.primary = x end + end + if not next (p.keywords) then + eprintf("Warning, no keyword to trigger gg.onkeyword") end + assert (p.primary, 'no primary parser in gg.onkeyword') + return p +end --</onkeyword> + + +------------------------------------------------------------------------------- +-- +-- Optional keyword consummer pseudo-parser generator +-- +------------------------------------------------------------------------------- +-- +-- This doesn't return a real parser, just a function. That function parses +-- one of the keywords passed as parameters, and returns it. It returns +-- [false] if no matching keyword is found. +-- +-- Notice that tokens returned by lexer already carry lineinfo, therefore +-- there's no need to add them, as done usually through transform() calls. +------------------------------------------------------------------------------- +function optkeyword (...) + local args = {...} + if type (args[1]) == "table" then + assert (#args == 1) + args = args[1] + end + for _, v in ipairs(args) do assert (type(v)=="string") end + return function (lx) + local x = lx:is_keyword (lx:peek(), unpack (args)) + if x then lx:next(); return x + else return false end + end +end + + +------------------------------------------------------------------------------- +-- +-- Run a parser with a special lexer +-- +------------------------------------------------------------------------------- +-- +-- This doesn't return a real parser, just a function. +-- First argument is the lexer class to be used with the parser, +-- 2nd is the parser itself. +-- The resulting parser returns whatever the argument parser does. +-- +------------------------------------------------------------------------------- +function with_lexer(new_lexer, parser) + + ------------------------------------------------------------------- + -- Most gg functions take their parameters in a table, so it's + -- better to silently accept when with_lexer{ } is called with + -- its arguments in a list: + ------------------------------------------------------------------- + if not parser and #new_lexer==2 and type(new_lexer[1])=='table' then + return with_lexer(unpack(new_lexer)) + end + + ------------------------------------------------------------------- + -- Save the current lexer, switch it for the new one, run the parser, + -- restore the previous lexer, even if the parser caused an error. + ------------------------------------------------------------------- + return function (lx) + local old_lexer = getmetatable(lx) + lx:sync() + setmetatable(lx, new_lexer) + local status, result = pcall(parser, lx) + lx:sync() + setmetatable(lx, old_lexer) + if status then return result else error(result) end + end +end lib/gg.lua @@ -7,9 +7,6 @@ -- -- TODO: -- --- * Make it possible to change lexer on the fly. This implies the --- ability to easily undo any pre-extracted tokens; --- -- * Make it easy to define new flavors of strings. Replacing the -- lexer.patterns.long_string regexp by an extensible list, with -- customizable token tag, would probably be enough. Maybe add: @@ -50,12 +47,10 @@ lexer.patterns = { final_short_comment = "^%-%-([^\n]*)()$", long_comment = "^%-%-%[(=*)%[\n?(.-)%]%1%]()", long_string = "^%[(=*)%[\n?(.-)%]%1%]()", - number_mantissa = { - "^%d+%.?%d*()", - "^%d*%.%d+()" }, - number_exponant = "^[eE][%+%-]?%d+()", - number_hex = "^0[xX]%x+()", - word = "^([%a_][%w_]*)()" + number_mantissa = { "^%d+%.?%d*()", "^%d*%.%d+()" }, + number_exponant = "^[eE][%+%-]?%d+()", + number_hex = "^0[xX]%x+()", + word = "^([%a_][%w_]*)()" } ---------------------------------------------------------------------- @@ -66,10 +61,23 @@ local function unescape_string (s) -- Turn the digits of an escape sequence into the corresponding -- character, e.g. [unesc_digits("123") == string.char(123)]. - local function unesc_digits (x) - local k, j, i = x:reverse():byte(1, 3) + local function unesc_digits (backslashes, digits) + if #backslashes%2==0 then + -- Even number of backslashes, they escape each other, not the digits. + -- Return them so that unesc_letter() can treaat them + return backslashes..digits + else + -- Remove the odd backslash, which escapes the number sequence. + -- The rest will be returned and parsed by unesc_letter() + backslashes = backslashes :sub (1,-2) + end + local k, j, i = digits:reverse():byte(1, 3) local z = _G.string.byte "0" - return _G.string.char ((k or z) + 10*(j or z) + 100*(i or z) - 111*z) + local code = (k or z) + 10*(j or z) + 100*(i or z) - 111*z + if code > 255 then + error ("Illegal escape sequence '\\"..digits.."' in string: ASCII codes must be in [0..255]") + end + return backslashes .. string.char (code) end -- Take a letter [x], and returns the character represented by the @@ -83,8 +91,8 @@ local function unescape_string (s) end return s + :gsub ("(\\+)([0-9][0-9]?[0-9]?)", unesc_digits) :gsub ("\\(%D)",unesc_letter) - :gsub ("\\([0-9][0-9]?[0-9]?)", unesc_digits) end lexer.extractors = { @@ -96,7 +104,9 @@ lexer.token_metatable = { -- __tostring = function(a) -- return string.format ("`%s{'%s'}",a.tag, a[1]) -- end - } +} + +lexer.lineinfo_metatable = { } ---------------------------------------------------------------------- -- Really extract next token fron the raw string @@ -109,7 +119,7 @@ function lexer:extract () local loc = self.i local eof, token - -- Put line info, comments and metatable arount the tag and content + -- Put line info, comments and metatable around the tag and content -- provided by extractors, thus returning a complete lexer token. -- first_line: line # at the beginning of token -- first_column_offset: char # of the last '\n' before beginning of token @@ -135,6 +145,9 @@ function lexer:extract () -- lineinfo entries: [1]=line, [2]=column, [3]=char, [4]=filename local fli = { first_line, loc-first_column_offset, loc, self.src_name } local lli = { self.line, self.i-self.column_offset-1, self.i-1, self.src_name } + --Pluto barfes when the metatable is set:( + setmetatable(fli, lexer.lineinfo_metatable) + setmetatable(lli, lexer.lineinfo_metatable) local a = { tag = tag, lineinfo = { first=fli, last=lli }, content } if lli[2]==-1 then lli[1], lli[2] = lli[1]-1, previous_line_length-1 end if #self.attached_comments > 0 then lib/lexer.lua @@ -1,3 +1,3 @@ -require 'base' -require 'table2' -require 'string2' \ No newline at end of file +require 'metalua.base' +require 'metalua.table2' +require 'metalua.string2' lib/metalua/runtime.lua @@ -71,12 +71,12 @@ expr_list = gg.list{ _expr, separators = "," } -------------------------------------------------------------------------------- -- Helpers for function applications / method applications -------------------------------------------------------------------------------- -local func_args_content = gg.list { +func_args_content = gg.list { name = "function arguments", _expr, separators = ",", terminators = ")" } -- Used to parse methods -local method_args = gg.multisequence{ +method_args = gg.multisequence{ name = "function argument(s)", { "{", table_content, "}" }, { "(", func_args_content, ")", builder = fget(1) }, @@ -105,10 +105,11 @@ local _func_val = function (lx) return func_val(lx) end -------------------------------------------------------------------------------- -- Default parser for primary expressions -------------------------------------------------------------------------------- -local function id_or_literal (lx) +function id_or_literal (lx) local a = lx:next() if a.tag~="Id" and a.tag~="String" and a.tag~="Number" then - gg.parse_error (lx, "Unexpected expr token %s", _G.table.tostring(a)) + gg.parse_error (lx, "Unexpected expr token %s", + _G.table.tostring (a, 'nohash')) end return a end lib/mlp_expr.lua @@ -29,7 +29,10 @@ expr:add{ "`", adt, builder = fget(1) } local lambda_expr = gg.sequence{ "|", func_params_content, "|", expr, builder= function (x) - return {tag="Function", x[1], { {tag="Return", x[2] } } } end } + local li = x[2].lineinfo + return { tag="Function", x[1], + { {tag="Return", x[2], lineinfo=li }, lineinfo=li } } + end } -- In an earlier version, lambda_expr took an expr_list rather than an expr -- after the 2nd bar. However, it happened to be much more of a burden than an lib/mlp_ext.lua @@ -11,24 +11,6 @@ -- for details. -- ---------------------------------------------------------------------- --- History: --- $Log: mlp_meta.lua,v $ --- Revision 1.4 2006/11/15 09:07:50 fab13n --- debugged meta operators. --- --- Revision 1.2 2006/11/09 09:39:57 fab13n --- some cleanup --- --- Revision 1.1 2006/11/07 21:29:02 fab13n --- improved quasi-quoting --- --- Revision 1.3 2006/11/07 04:38:00 fab13n --- first bootstrapping version. --- --- Revision 1.2 2006/11/05 15:08:34 fab13n --- updated code generation, to be compliant with 5.1 --- ----------------------------------------------------------------------- -------------------------------------------------------------------------------- @@ -39,9 +21,6 @@ -- -------------------------------------------------------------------------------- ---require "compile" ---require "ldump" - module ("mlp", package.seeall) -------------------------------------------------------------------------------- @@ -51,12 +30,7 @@ module ("mlp", package.seeall) -------------------------------------------------------------------------------- function splice (ast) - --printf(" [SPLICE] Ready to compile:\n%s", _G.table.tostring (ast, "nohash", 60)) local f = mlc.function_of_ast(ast, '=splice') - --printf " [SPLICE] Splice Compiled." - --local status, result = pcall(f) - --printf " [SPLICE] Splice Evaled." - --if not status then print 'ERROR IN SPLICE' end local result=f() return result end @@ -106,12 +80,6 @@ function splice_content (lx) lx:next() -- skip ":" assert (a.tag=="Id", "Invalid splice parser name") parser_name = a[1] --- printf("this splice is a %s", parser_name) --- else --- printf("no splice specifier:\npeek(1)") --- _G.table.print(lx:peek(1)) --- printf("peek(2)") --- _G.table.print(lx:peek(2)) end local ast = mlp[parser_name](lx) if in_a_quote then @@ -132,25 +100,18 @@ end -------------------------------------------------------------------------------- function quote_content (lx) local parser - if lx:is_keyword (lx:peek(1), ":") then -- +{:parser: content } - lx:next() - errory "NOT IMPLEMENTED" - elseif lx:is_keyword (lx:peek(2), ":") then -- +{parser: content } + if lx:is_keyword (lx:peek(2), ":") then -- +{parser: content } parser = mlp[id(lx)[1]] lx:next() else -- +{ content } parser = mlp.expr end - --assert(not in_a_quote, "Nested quotes not handled yet") local prev_iq = in_a_quote in_a_quote = true --print("IN_A_QUOTE") local content = parser (lx) local q_content = quote (content) --- printf("/IN_A_QUOTE:\n* content=\n%s\n* q_content=\n%s\n", --- _G.table.tostring(content, "nohash", 60), --- _G.table.tostring(q_content, "nohash", 60)) in_a_quote = prev_iq return q_content end lib/mlp_meta.lua @@ -171,7 +171,7 @@ function skip_initial_sharp_comment (lx) if i then lx.i, lx.column_offset, lx.line = i, i, lx.line+1 end end -function chunk (lx) +local function _chunk (lx) if lx:peek().tag == 'Eof' then return { } -- handle empty files else skip_initial_sharp_comment (lx) @@ -179,4 +179,7 @@ function chunk (lx) if lx:peek().tag ~= "Eof" then error "End-of-file expected" end return chunk end -end \ No newline at end of file +end + +-- chunk is wrapped in a sequence so that it has a "transformer" field. +chunk = gg.sequence { _chunk, builder = unpack } \ No newline at end of file lib/mlp_misc.lua @@ -89,7 +89,7 @@ function for_header (lx) else -- Forin: there might be several vars local a = lx:is_keyword (lx:next(), ",", "in") - if a=="in" then var_list = { var } else + if a=="in" then var_list = { var, lineinfo = var.lineinfo } else -- several vars; first "," skipped, read other vars var_list = gg.list{ primary = id, separators = ",", terminators = "in" } (lx) @@ -179,7 +179,7 @@ local function assign_or_call_stat_parser (lx) end end -local local_stat_parser = gg.multisequence{ +local_stat_parser = gg.multisequence{ -- local function <name> <func_val> { "function", id, func_val, builder = function(x) lib/mlp_stat.lua lib/base.lua lib/string2.lua lib/table2.lua 4a865d301277d6a156369862bc1be072dbc442af David Manura dm.git@math2.org http://github.com/davidm/lua2c/commit/4f53e83c111d3608e29c80eb291ee4ed16cad8bb 4f53e83c111d3608e29c80eb291ee4ed16cad8bb 2009-02-12T22:33:58-08:00 2009-02-12T22:33:58-08:00 update to metalua 0.5 git 87e20a76770e6265677df1e9061befa115437549 David Manura dm.git@math2.org