@@ -34,24 +34,20 @@ end define_class("VBROp") function VBROp:initialize(bits) self.bits = bits - self.name = name end define_class("FixedOp") function FixedOp:initialize(bits) self.bits = bits - self.name = name end define_class("ArrayOp") function ArrayOp:initialize(elem_type) self.elem_type = elem_type - self.name = name end define_class("File") function File:initialize(filename, app_magic_number) - self.name = name self.file = io.open(filename, "w") self.current_byte = 0 self.current_bits = 0 compiler/bc.lua @@ -64,15 +64,23 @@ define_class("CharStream") return self.string:sub(self.offset, self.offset+amount-1) end + function CharStream:get_offset() + -- Inefficient, but this is only until we are self-hosting. + local lineno = 1 + for nl in self.string:sub(0, self.offset):gmatch("[\n\r]+") do lineno = lineno + 1 end + local first, last = self.string:sub(0, self.offset):find(".*[\n\r]") + local colno = self.offset - (last or 0) + return TextOffset:new(lineno, colno, self.offset) + end + function CharStream:consume(str) self:skip_ignored() local actual_str = self.string:sub(self.offset, self.offset+str:len()-1) if actual_str ~= str then - local lineno = 1 - for nl in self.string:sub(0, self.offset):gmatch("[\n\r]") do lineno = lineno + 1 end - local first, last = self.string:sub(0, self.offset):find(".*[\n\r]") - local colno = self.offset - (last or 0) - error(string.format("Error parsing grammar %s:\nat line %s, column %s expected '%s', got '%s'", input_filename, lineno, colno, str, actual_str)) + local offset = self:get_offset() + error(string.format("Error parsing grammar %s:\nat line %s, column %s " .. + "(expected '%s', got '%s')", + input_filename, offset.line, offset.column, str, actual_str)) end self.offset = self.offset + str:len() self:skip_ignored() @@ -109,155 +117,172 @@ define_class("CharStream") -- class TokenStream -- Parse the grammar file given in +chars+ and return a Grammar object. -function parse_grammar(chars) +define_class("RTNParser") +function RTNParser:initialize() + self.grammar = nil + self.chars = nil + self.slotnum = nil + self.current_rule_name = nil +end + +function RTNParser:parse(chars, grammar) + self.chars = chars + self.grammar = grammar + self:parse_grammar() + return grammar +end + +function RTNParser:get_next_slotnum() + local ret = self.slotnum + self.slotnum = self.slotnum + 1 + return ret +end + +function RTNParser:parse_grammar() + local chars = self.chars chars:ignore("whitespace") - local grammar = Grammar:new() - local attributes = {ignore={}, slot_counts={}, regex_text={}, grammar=grammar} while not chars:eof() do if chars:match(" *@start") then chars:consume_pattern(" *@start") - grammar.start = parse_nonterm(chars).name; + self.grammar.start = self:parse_ident() chars:consume(";") elseif chars:match(" *@allow") then chars:consume_pattern(" *@allow") - local what_to_allow = parse_nonterm(chars); - local start_nonterm = parse_nonterm(chars).name; + local what_to_allow = self:parse_ident() + local start_ident = self:parse_ident() chars:consume_pattern("%.%.%.") - local end_nonterms = Set:new() - end_nonterms:add(parse_nonterm(chars).name) + local end_idents = Set:new() + end_idents:add(self:parse_ident()) while chars:match(" *,") do chars:consume(",") - end_nonterms:add(parse_nonterm(chars).name) + end_idents:add(self:parse_ident()) end chars:consume(";") - grammar:add_allow(what_to_allow, start_nonterm, end_nonterms) + self.grammar:add_allow(what_to_allow, start_ident, end_idents) else local before_offset = chars.offset - local stmt = parse_statement(chars, attributes) + local offset = chars:get_offset() + local stmt = self:parse_statement() if not stmt then break elseif stmt.nonterm then stmt.derivations.final.final = "Final" local rule_text = chars.string:sub(before_offset, chars.offset-1) - grammar:add_nonterm(stmt.nonterm.name, stmt.derivations, stmt.slot_count, rule_text) + self.grammar:add_nonterm(stmt.nonterm, stmt.derivations, stmt.slot_count, rule_text, offset) elseif stmt.term then - grammar:add_terminal(stmt.term, stmt.regex) + self.grammar:add_terminal(stmt.term, stmt.regex, offset) end end end - - grammar.attributes = attributes - - -- start symbol defaults to the first symbol - if not grammar.start then - grammar.start = grammar.rtns:get_key_at_offset(1) - end - - return grammar end -function parse_statement(chars, attributes) - local old_ignore = chars:ignore("whitespace") +function RTNParser:parse_statement() + local old_ignore = self.chars:ignore("whitespace") local ret = {} - local ident = parse_nonterm(chars) + local ident = self:parse_ident() - if chars:match("->") then - attributes.nonterm = ident + if self.chars:match("->") then + self.current_rule_name = ident + -- Need to register the rule here, so that we catch conflicting references + -- *within* the rule, and so that ordering of symbols is right. + self.grammar:get_object(ident, {GrammarObj.RULE}) ret.nonterm = ident - chars:consume("->") - attributes.slotnum = 1 - ret.derivations = parse_derivations(chars, attributes) - ret.slot_count = attributes.slotnum - 1 + self.chars:consume("->") + self.slotnum = 1 + ret.derivations = self:parse_derivations() + ret.slot_count = self.slotnum - 1 else - ret.term = ident.name - chars:consume(":") - ret.regex = parse_regex(chars) + ret.term = ident + self.chars:consume(":") + ret.regex = self:parse_regex() end - chars:consume(";") - chars:ignore(old_ignore) + self.chars:consume(";") + self.chars:ignore(old_ignore) return ret end -function parse_derivations(chars, attributes) - local old_ignore = chars:ignore("whitespace") +function RTNParser:parse_derivations() + local old_ignore = self.chars:ignore("whitespace") local derivations = {} repeat - local derivation = parse_derivation(chars, attributes) + local derivation = self:parse_derivation() -- Any prioritized derivations we parse together as a group, then build -- an NFA of *prioritized* alternation. - if chars:lookahead(1) == "/" then - chars:consume("/") + if self.chars:lookahead(1) == "/" then + self.chars:consume("/") local prioritized_derivations = {derivation} repeat - local prioritized_derivation = parse_derivation(chars, attributes) + local prioritized_derivation = self:parse_derivation() table.insert(prioritized_derivations, prioritized_derivation) - until chars:lookahead(1) ~= "/" or not chars:consume("/") + until self.chars:lookahead(1) ~= "/" or not self.chars:consume("/") derivation = nfa_construct.alt(prioritized_derivations, true) end table.insert(derivations, derivation) - until chars:lookahead(1) ~= "|" or not chars:consume("|") + until self.chars:lookahead(1) ~= "|" or not self.chars:consume("|") - chars:ignore(old_ignore) + self.chars:ignore(old_ignore) return nfa_construct.alt(derivations) end -function parse_derivation(chars, attributes) - local old_ignore = chars:ignore("whitespace") - local ret = parse_term(chars, attributes) - while chars:lookahead(1) ~= "|" and chars:lookahead(1) ~= "/" and - chars:lookahead(1) ~= ";" and chars:lookahead(1) ~= ")" do - ret = nfa_construct.concat(ret, parse_term(chars, attributes)) +function RTNParser:parse_derivation() + local old_ignore = self.chars:ignore("whitespace") + local ret = self:parse_term() + while self.chars:lookahead(1) ~= "|" and self.chars:lookahead(1) ~= "/" and + self.chars:lookahead(1) ~= ";" and self.chars:lookahead(1) ~= ")" do + ret = nfa_construct.concat(ret, self:parse_term()) end - chars:ignore(old_ignore) + self.chars:ignore(old_ignore) return ret end -function parse_term(chars, attributes) - local old_ignore = chars:ignore("whitespace") +function RTNParser:parse_term() + local old_ignore = self.chars:ignore("whitespace") local name local ret - if chars:match(" *%.[%w_]+ *=") then - name = parse_name(chars) - chars:consume("=") + local offset = self.chars:get_offset() + if self.chars:match(" *%.[%w_]+ *=") then + name = self:parse_name() + self.chars:consume("=") end local symbol - if chars:lookahead(1) == "/" and name then - name = name or attributes.nonterm.name - intfa, text = parse_regex(chars) - attributes.grammar:add_terminal(name, intfa, text) - ret = fa.RTN:new{symbol=name, properties={name=name, slotnum=attributes.slotnum}} - attributes.slotnum = attributes.slotnum + 1 - elseif chars:lookahead(1) == "'" or chars:lookahead(1) == '"' then - local string = parse_string(chars) - attributes.grammar:add_terminal(string, string) + if self.chars:lookahead(1) == "/" and name then + local intfa, text = self:parse_regex() + local obj = self.grammar:add_terminal(name, intfa, text, offset) + ret = fa.RTN:new{ + symbol=obj, + properties={name=name, slotnum=self:get_next_slotnum()} + } + elseif self.chars:lookahead(1) == "'" or self.chars:lookahead(1) == '"' then + local string = self:parse_string() name = name or string - ret = fa.RTN:new{symbol=string, properties={name=name, slotnum=attributes.slotnum}} - attributes.slotnum = attributes.slotnum + 1 - elseif chars:lookahead(1) == "(" then + local obj = self.grammar:add_implicit_terminal(name, string, offset) + ret = fa.RTN:new{ + symbol=obj, + properties={name=name, slotnum=self:get_next_slotnum()} + } + elseif self.chars:lookahead(1) == "(" then if name then error("You cannot name a group") end - chars:consume("(") - ret = parse_derivations(chars, attributes) - chars:consume(")") + self.chars:consume("(") + ret = self:parse_derivations() + self.chars:consume(")") else - local nonterm = parse_nonterm(chars) - name = name or nonterm.name - if attributes.grammar.terminals[nonterm.name] then - ret = fa.RTN:new{symbol=nonterm.name, properties={name=nonterm.name, slotnum=attributes.slotnum}} - else - ret = fa.RTN:new{symbol=nonterm, properties={name=name, slotnum=attributes.slotnum}} - end - attributes.slotnum = attributes.slotnum + 1 + local ident = self:parse_ident() + name = name or ident + ret = fa.RTN:new{ + symbol=self.grammar:get_object(ident, {GrammarObj.RULE, GrammarObj.TERMINAL}), + properties={name=ident, slotnum=self:get_next_slotnum()} + } end - local one_ahead = chars:lookahead(1) + local one_ahead = self.chars:lookahead(1) if one_ahead == "?" or one_ahead == "*" or one_ahead == "+" then - local modifier, sep, favor_repeat = parse_modifier(chars, attributes) + local modifier, sep, favor_repeat = self:parse_modifier() -- foo +(bar) == foo (bar foo)* -- foo *(bar) == (foo (bar foo)*)? if sep then @@ -292,98 +317,99 @@ function parse_term(chars, attributes) end end - chars:ignore(old_ignore) + self.chars:ignore(old_ignore) return ret end -function parse_name(chars) - local old_ignore = chars:ignore() - chars:consume(".") - local ret = chars:consume_pattern("[%w_]+") - chars:ignore(old_ignore) +function RTNParser:parse_name() + local old_ignore = self.chars:ignore() + self.chars:consume(".") + local ret = self.chars:consume_pattern("[%w_]+") + self.chars:ignore(old_ignore) return ret end -function parse_modifier(chars, attributes) - local old_ignore = chars:ignore() +function RTNParser:parse_modifier() + local old_ignore = self.chars:ignore() local modifier, str, prefer_repeat - modifier = chars:consume_pattern("[?*+]") - if chars:lookahead(1) == "(" then - chars:consume("(") + modifier = self.chars:consume_pattern("[?*+]") + if self.chars:lookahead(1) == "(" then + self.chars:consume("(") + local offset = self.chars:get_offset() local sep_string - if chars:lookahead(1) == "'" or chars:lookahead(1) == '"' then - sep_string = parse_string(chars) + if self.chars:lookahead(1) == "'" or self.chars:lookahead(1) == '"' then + sep_string = self:parse_string() else - sep_string = chars:consume_pattern("[^)]*") + sep_string = self.chars:consume_pattern("[^)]*") end - str = fa.RTN:new{symbol=sep_string, properties={slotnum=attributes.slotnum, name=sep_string}} - attributes.grammar:add_terminal(sep_string, sep_string) - attributes.slotnum = attributes.slotnum + 1 - chars:consume(")") + local obj = self.grammar:add_implicit_terminal(sep_string, sep_string, offset) + str = fa.RTN:new{symbol=obj, properties={slotnum=self:get_next_slotnum(), name=sep_string}} + self.chars:consume(")") end - if chars:lookahead(1) == "+" or chars:lookahead(1) == "-" then - local prefer_repeat_ch = chars:consume_pattern("[+-]") + if self.chars:lookahead(1) == "+" or self.chars:lookahead(1) == "-" then + local prefer_repeat_ch = self.chars:consume_pattern("[+-]") if prefer_repeat_ch == "+" then prefer_repeat = true else prefer_repeat = false end end - chars:ignore(old_ignore) + self.chars:ignore(old_ignore) return modifier, str, prefer_repeat end -function parse_nonterm(chars) - local old_ignore = chars:ignore() - local ret = fa.nonterms:get(chars:consume_pattern("[%w_]+")) - chars:ignore(old_ignore) +function RTNParser:parse_ident() + local old_ignore = self.chars:ignore() + local ret = self.chars:consume_pattern("[%w_]+") + self.chars:ignore(old_ignore) return ret end -function parse_string(chars) - local old_ignore = chars:ignore() +function RTNParser:parse_string() + local old_ignore = self.chars:ignore() local str = "" - if chars:lookahead(1) == "'" then - chars:consume("'") - while chars:lookahead(1) ~= "'" do - if chars:lookahead(1) == "\\" then - chars:consume("\\") - str = str .. chars:consume_pattern(".") -- TODO: other backslash sequences + if self.chars:lookahead(1) == "'" then + self.chars:consume("'") + while self.chars:lookahead(1) ~= "'" do + if self.chars:lookahead(1) == "\\" then + self.chars:consume("\\") + str = str .. self.chars:consume_pattern(".") -- TODO: other backslash sequences else - str = str .. chars:consume_pattern(".") + str = str .. self.chars:consume_pattern(".") end end - chars:consume("'") + self.chars:consume("'") else - chars:consume('"') - while chars:lookahead(1) ~= '"' do - if chars:lookahead(1) == "\\" then - chars:consume("\\") - str = str .. chars:consume_pattern(".") -- TODO: other backslash sequences + self.chars:consume('"') + while self.chars:lookahead(1) ~= '"' do + if self.chars:lookahead(1) == "\\" then + self.chars:consume("\\") + str = str .. self.chars:consume_pattern(".") -- TODO: other backslash sequences else - str = str .. chars:consume_pattern(".") + str = str .. self.chars:consume_pattern(".") end end - chars:consume('"') + self.chars:consume('"') end - chars:ignore(old_ignore) + self.chars:ignore(old_ignore) return str end -function parse_regex(chars) - local old_ignore = chars:ignore() - chars:consume("/") +function RTNParser:parse_regex() + local old_ignore = self.chars:ignore() + self.chars:consume("/") local regex_text = "" - while chars:lookahead(1) ~= "/" do - if chars:lookahead(1) == "\\" then - regex_text = regex_text .. chars:consume_pattern("..") + while self.chars:lookahead(1) ~= "/" do + if self.chars:lookahead(1) == "\\" then + regex_text = regex_text .. self.chars:consume_pattern("..") else - regex_text = regex_text .. chars:consume_pattern(".") + regex_text = regex_text .. self.chars:consume_pattern(".") end end local regex = regex_parser.parse_regex(regex_parser.TokenStream:new(regex_text)) - chars:consume("/") - chars:ignore(old_ignore) + regex.regex_text = regex_text + self.chars:consume("/") + self.chars:ignore(old_ignore) return regex, regex_text end compiler/bootstrap/rtn.lua @@ -40,7 +40,7 @@ BC_GLA_STATE = 0 BC_GLA_FINAL_STATE = 1 BC_GLA_TRANSITION = 2 -if not print_verbose then +if not rawget(_G, "print_verbose") then function print_verbose(str) print(str) end @@ -48,8 +48,8 @@ end function write_bytecode(grammar, outfilename) -- write Bitcode header - bc_file = bc.File:new(outfilename, "GH") - abbrevs = define_abbrevs(bc_file) + local bc_file = bc.File:new(outfilename, "GH") + local abbrevs = define_abbrevs(bc_file) -- Obtain linearized representations of all the DFAs from the Grammar object. local strings = grammar:get_strings() @@ -143,7 +143,7 @@ function emit_intfa(intfa, strings, bc_file, abbrevs) -- emit the transitions for transition in each(intfa_transitions) do local range, target_state = unpack(transition) - target_state_offset = intfa_state_offsets[target_state] + local target_state_offset = intfa_state_offsets[target_state] if range.low == range.high then bc_file:write_abbreviated_record(abbrevs.bc_intfa_transition, range.low, target_state_offset) else @@ -275,7 +275,7 @@ end function define_abbrevs(bc_file) - abbrevs = {} + local abbrevs = {} -- Enter a BLOCKINFO record to define abbreviations for all our records. -- See FILEFORMAT for a description of what all the record types mean. compiler/bytecode.lua @@ -157,8 +157,8 @@ define_class("Set") function Set:intersection(x) local new_set = Set:new() - for element in self:each() do - if x:contains(element) then + for element in each(x) do + if self:contains(element) then new_set:add(element) end end @@ -213,7 +213,7 @@ define_class("Set") table.sort(arr) - str = "" + local str = "" for elem in each(arr) do str = str .. elem end return str end @@ -301,6 +301,7 @@ define_class("OrderedSet") end -- OrderedMap +-- NOTE: the map currently does not support deletion or replacement, only adding. define_class("OrderedMap") function OrderedMap:initialize() -- An ordered array of {key, value} pairs that are the members of the set. @@ -313,10 +314,18 @@ define_class("OrderedMap") end function OrderedMap:add(key, value) - if not self.key_offsets[key] then - table.insert(self.elements, {key, value}) - self.key_offsets[key] = #self.elements + if self.key_offsets[key] then + error(string.format("Attempted to insert duplicate key '%s'", key)) + end + table.insert(self.elements, {key, value}) + self.key_offsets[key] = #self.elements + end + + function OrderedMap:get_or_insert_new(key, creator) + if not self:contains(key) then + self:add(key, creator()) end + return self:get(key) end function OrderedMap:insert_front(key, value) @@ -329,11 +338,22 @@ define_class("OrderedMap") table.insert(self.elements, 1, {key, value}) end + function OrderedMap:subset(predicate) + local new_map = OrderedMap:new() + for key, value in self:each() do + if predicate(key, value) then + new_map:add(key, value) + end + end + return new_map + end + function OrderedMap:get(key) return self.elements[self.key_offsets[key]][2] end function OrderedMap:get_key_at_offset(offset) + assert(offset <= self:count()) return self.elements[offset][1] end @@ -507,7 +527,7 @@ define_class("IntSet") end function IntSet:toasciistring() - local convert_func = function (x) + local function convert_func(x) if x == math.huge then return "del" elseif x < 33 then compiler/data_structures.lua @@ -67,7 +67,7 @@ define_class("FAState") function FAState:tostring() local str = string.format("{%s, %d transitions", self.class.name, self:num_transitions()) -- TODO: rename .rtn to .fa, to be more generic - if self.rtn then + if self.class == RTN then str = str .. string.format(", from rule named %s", self.rtn.name) if self.rtn.start == self then str = str .. ", start" @@ -239,6 +239,7 @@ define_class("IntFA", FA) function IntFA:initialize(init) FA.initialize(self, init) self.termset = nil + self.regex_text = nil end function IntFA:new_graph(init) @@ -279,6 +280,10 @@ function IntFA:get_outgoing_edge_values(states) return values end +function IntFA:to_dot_edge_str() + return self.regex_text +end + function IntFA:to_dot() str = "" states = self:states():to_array() @@ -496,6 +501,10 @@ function RTN:get_outgoing_edge_values(states) return values end +function RTN:to_dot_edge_str() + return string.format("<%s>", self.name) +end + function escape(str) if not isobject(str) and str.gsub then return str:gsub("[\"\\]", "\\%1") @@ -540,21 +549,12 @@ function RTN:to_dot(indent, suffix, intfas, glas) indent, tostring(state) .. suffix, extra_label, peripheries, color) for edge_val, target_state in state:transitions() do - if fa.is_nonterm(edge_val) then - str = str .. string.format('%s"%s" -> "%s" [label="<%s>"]\n', - indent, tostring(state) .. suffix, tostring(target_state) .. suffix, - escape(edge_val.name)) - else - --if attributes.regex_text[edge_val] then - -- edge_val = "/" .. attributes.regex_text[edge_val] .. "/" - --end - if edge_val == fa.eof then - edge_val = "EOF" - end - str = str .. string.format('%s"%s" -> "%s" [label="%s"]\n', - indent, tostring(state) .. suffix, tostring(target_state) .. suffix, - escape(edge_val)) + if type(edge_val) == "table" then + edge_val = edge_val:to_dot_edge_str() end + str = str .. string.format('%s"%s" -> "%s" [label="%s"]\n', + indent, tostring(state) .. suffix, tostring(target_state) .. suffix, + escape(edge_val)) end end return str @@ -626,15 +626,8 @@ function RTNState:needs_intfa() end -define_class("NonTerm") -function NonTerm:initialize(name) - self.name = name -end - -nonterms = MemoizedObject:new(NonTerm) - function is_nonterm(thing) - return isobject(thing) and thing.class == NonTerm + return isobject(thing) and thing.class == RTN end -- vim:et:sts=2:sw=2 compiler/fa.lua @@ -341,7 +341,7 @@ function fa_longest_path(fa) local longest = 0 local current_depth = 0 local seen = Set:new() - function dfs_helper(state) + local function dfs_helper(state) seen:add(state) if state.final and current_depth > longest then longest = current_depth compiler/fa_algorithms.lua @@ -26,37 +26,262 @@ --------------------------------------------------------------------]]-- require "data_structures" +require "misc" require "fa_algorithms" require "intfa_combine" +define_class("TextOffset") + function TextOffset:initialize(line, column, offset) + self.line = line + self.column = column + self.offset = offset + end + + function TextOffset:tostring() + return string.format("line %d, column %d", self.line, self.column) + end +-- class TextOffset + +--[[-------------------------------------------------------------------- + + GrammarObj: a class for representing grammar objects (rules, + terminals, etc) that are seen or even just referenced in a Gazelle + grammar file. The symbol table maps key (names) to objects of this + type. + +--------------------------------------------------------------------]]-- + +define_class("GrammarObj") + +-- What types of grammar objects can exist in a grammar? +GrammarObj.RULE = "rule" +GrammarObj.TERMINAL = "terminal" +-- (more to come...) + + function GrammarObj:initialize(name) + self.name = name + self.definition = nil -- starts out undefined + self.explicit = false + self.type = nil + self.expected_types = nil + self.explicit_offset = nil + self.implicit_offsets = {} + end + + function GrammarObj:set_expected(types) + types = Set:new(types) + if not self.expected_types then + self.expected_types = types + else + local intersection = self.expected_types:intersection(types) + if intersection:isempty() then + if self.definition then + error(string.format("Symbol '%s' was previously defined as %s, but its " .. + "use here expects it to be %s.", + self.name, self:type_str(), self:expected_types_str())) + else + error(string.format("Symbol '%s' was previously referenced as a(n) %s, but " .. + "its use here expects it to be %s.", + self.name, self:expected_types_str(), + self:expected_types_str(types))) + end + end + end + end + + function GrammarObj:define(definition, _type, offset, implicit) + local explicit = not implicit + if self.definition then + if self.explicit == false and self.definition == definition and self.type == _type then + -- redefinition is ok here. + else + error(string.format("Redefinition of symbol '%s' at %s (previous definition was at %s).", + self.name, offset:tostring(), + (self.explicit_offset or self.implicit_offsets[1]):tostring())) + end + elseif self.expected_types and not self.expected_types:contains(_type) then + error(string.format("Symbol '%s' defined as %s, but previously used as if it were %s.", + self.name, self:type_str(_type), self:expected_types_str())) + else + self.definition = definition + self.type = _type + self.expected_types = Set:new({_type}) + end + self.explicit = self.explicit or explicit + if explicit then + self.explicit_offset = offset + else + table.insert(self.implicit_offsets, offset) + end + end + + function GrammarObj:type_str(_type) + _type = _type or self.type + return "a " .. _type + end + + function GrammarObj:expected_types_str(types) + types = types or self.expected_types + types = types:to_array() + local str = "a " .. types[1] + if #types > 1 then + str = str .. " or " .. types[2] + end + return str + end +-- class GrammarObj + +--[[-------------------------------------------------------------------- + + Grammar: the Grammar class itself, which represents a single Gazelle + grammar that is parsed from one or more input files, analzed and + translated, and finally emitted to output. + +--------------------------------------------------------------------]]-- + define_class("Grammar") function Grammar:initialize() - self.rtns = OrderedMap:new() - self.terminals = {} + -- The symbol table of objects that have been defined or referenced. + self.objects = MemoizedObject:new(GrammarObj) + + -- The master IntFAs we build once all IntFAs are combined. self.master_intfas = OrderedSet:new() - self.start = nil -- what rule the entire grammar starts on + + -- What rule the entire grammar starts on. + self.start = nil + + -- @allow definitions. self.allow = {} - self.attributes = nil + + -- Once we start processing, we assign all the rtns and terminals + -- to separate lists for processing. + self.rtns = nil + self.terminals = nil end --- Add a nonterminal and its associated RTN to the grammar. --- TODO: how should redefinition be caught and warned/errored? -function Grammar:add_nonterm(name, rtn, slot_count, text) +function Grammar:parse_source_string(string) + local parser = RTNParser:new() + parser:parse(CharStream:new(string), self) +end + +--[[-------------------------------------------------------------------- + + The next block of functions are intended for the Gazelle grammar + parser to call as they are parsing a Gazelle grammar file. + +--------------------------------------------------------------------]]-- + +function Grammar:get_object(name, expected) + local obj = self.objects:get(name) + obj:set_expected(expected) + return obj +end + +function Grammar:add_nonterm(name, rtn, slot_count, text, offset) rtn.name = name rtn.slot_count = slot_count rtn.text = text for state in each(rtn:states()) do state.rtn = rtn end - self.rtns:add(name, rtn) + self.objects:get(name):define(rtn, GrammarObj.RULE, offset) +end + +function Grammar:add_terminal(name, string_or_intfa, text, offset) + assert(type(name) == "string") + assert(string_or_intfa) + local obj = self.objects:get(name) + obj:define(string_or_intfa, GrammarObj.TERMINAL, offset) + return obj +end + +function Grammar:add_implicit_terminal(name, string, offset) + assert(type(name) == "string") + local obj = self.objects:get(name) + obj:define(string, GrammarObj.TERMINAL, offset, true) + return obj end --- Add a terminal and its associated IntFA to the grammar. --- TODO: how should redefinition be caught and warned/errored? -function Grammar:add_terminal(name, intfa) - self.terminals[name] = intfa +--[[-------------------------------------------------------------------- + + Methods for analyzing/translating the grammar once it has been + fully read from source files. + +--------------------------------------------------------------------]]-- + +function Grammar:process() + self:bind_symbols() + -- start symbol defaults to the first rule. + self.start = self.start or self.rtns:get_key_at_offset(1) + + -- assign priorities to RTN transitions + --print_verbose("Assigning RTN transition priorities...") + self:assign_priorities() + + -- make the RTNs in the grammar determistic and minimal + --print_verbose("Convering RTN NFAs to DFAs...") + self:determinize_rtns() + self:process_allow() + self:canonicalize_properties() +end + +function Grammar:compute_lookahead(max_k) + -- Generate GLAs by doing lookahead calculations. + -- This annotates every nontrivial state in the grammar with a GLA. + --print_verbose("Doing LL(*) lookahead calculations...") + compute_lookahead(self, max_k) +end + +-- we now have everything figured out at the RTN and GLA levels. Now we just +-- need to figure out how many IntFAs to generate, which terminals each one +-- should handle, and generate/determinize/minimize those IntFAs. + +function Grammar:bind_symbols() + -- Ensure that all referenced symbols were defined, and replace the GrammarObj + -- objects with the raw values. + local objects = {} + self.rtns = OrderedMap:new() + self.terminals = {} + for name, obj in self.objects:get_objects():each() do + local definition = obj.definition + if not definition then + error(string.format("Symbol '%s' was referenced but never defined.", name)) + end + assert(name == obj.name) + objects[name] = definition + if obj.type == GrammarObj.RULE then + self.rtns:add(name, definition) + else + self.terminals[name] = definition + end + end + + -- Replace all references to GrammarObj objects with the actual value. + for name, rtn in each(self.rtns) do + for rtn_state in each(rtn:states()) do + local new_transitions = {} + for edge_val, dest_state, properties in rtn_state:transitions() do + -- This is embarrassingly hacky. Need to have a more polymorphic way + -- of dealing with multiple kinds of edge values. + if edge_val == fa.eof or edge_val == fa.e then + -- do nothing + elseif edge_val.type == GrammarObj.TERMINAL then + edge_val = edge_val.name + else + edge_val = objects[edge_val.name] + end + table.insert(new_transitions, {edge_val, dest_state, properties}) + end + rtn_state:clear_transitions() + for tuple in each(new_transitions) do + local edge_val, dest_state, properties = unpack(tuple) + rtn_state:add_transition(edge_val, dest_state, properties) + end + end + end end + function Grammar:add_allow(what_to_allow, start_nonterm, end_nonterms) table.insert(self.allow, {what_to_allow, start_nonterm, end_nonterms}) end @@ -64,7 +289,7 @@ end function Grammar:process_allow() for allow in each(self.allow) do local what_to_allow, start_nonterm, end_nonterms = unpack(allow) - local children_func = function(rule_name) + local function children_func(rule_name) if not end_nonterms:contains(rule_name) then local rtn = self.rtns:get(rule_name) if not rtn then @@ -83,7 +308,8 @@ function Grammar:process_allow() -- add self-transitions for every state for state in each(rtn:states()) do - state:add_transition(what_to_allow, state, {name=what_to_allow.name, slotnum=-1}) + local allow_rtn = self.rtns:get(what_to_allow) + state:add_transition(allow_rtn, state, {name=allow_rtn.name, slotnum=-1}) end return subrules @@ -102,18 +328,6 @@ function Grammar:canonicalize_properties() end end -function Grammar:check_defined() - for name, rtn in each(self.rtns) do - for rtn_state in each(rtn:states()) do - for edge_val in rtn_state:transitions() do - if fa.is_nonterm(edge_val) and not self.rtns:contains(edge_val.name) then - error(string.format("Rule '%s' was referred to but never defined.", edge_val.name)) - end - end - end - end -end - function Grammar:get_rtn_states_needing_intfa() local states = Set:new() for name, rtn in each(self.rtns) do @@ -138,15 +352,6 @@ function Grammar:get_rtn_states_needing_gla() return states end -function copy_attributes(rtn, new_rtn) - new_rtn.name = rtn.name - new_rtn.slot_count = rtn.slot_count - new_rtn.text = rtn.text - for state in each(new_rtn:states()) do - state.rtn = new_rtn - end -end - function Grammar:assign_priorities() -- For each non-epsilon transition in the grammar, we want to find the epsilon -- closure (within the rule -- no following nonterminal or final transitions) @@ -175,7 +380,7 @@ function Grammar:assign_priorities() end local priorities = {} - local children = function(state, stack) + local function children(state, stack) local reverse_transitions = reverse_epsilon_transitions[state] if reverse_transitions then local child_states, priority_classes = unpack(reverse_transitions) @@ -227,6 +432,7 @@ function Grammar:minimize_rtns() end function Grammar:generate_intfas() + --print_verbose("Generating lexer DFAs...") -- first generate the set of states that need an IntFA: some RTN -- states and all nonfinal GLA states. local states = self:get_rtn_states_needing_intfa() @@ -253,6 +459,7 @@ function Grammar:generate_intfas() terms:add(edge_val) end end + assert(terms:count() > 0) table.insert(state_term_pairs, {state, terms}) end @@ -299,7 +506,7 @@ function Grammar:get_strings() -- sort the strings for deterministic output strings = strings:to_array() table.sort(strings) - strings_ordered_set = OrderedSet:new() + local strings_ordered_set = OrderedSet:new() for string in each(strings) do strings_ordered_set:add(string) end @@ -350,7 +557,7 @@ function Grammar:get_flattened_rtn_list() -- create a list of transitions for every state for name, rtn in each(rtns) do for state in each(rtn.states) do - transitions = {} + local transitions = {} for edge_val, target_state, properties in state:transitions() do table.insert(transitions, {edge_val, target_state, properties}) end @@ -361,7 +568,7 @@ function Grammar:get_flattened_rtn_list() -- 3. nonterminal transitions are sorted by the name of the nonterminal -- 4. transitions with the same edge value are sorted by their order -- in the list of states (which is stable) (TODO: no it's not, yet) - sort_func = function (a, b) + local function sort_func(a, b) if not fa.is_nonterm(a[1]) and fa.is_nonterm(b[1]) then return true elseif fa.is_nonterm(a[1]) and not fa.is_nonterm(b[1]) then @@ -403,4 +610,13 @@ function Grammar:get_flattened_gla_list() return glas end +function copy_attributes(rtn, new_rtn) + new_rtn.name = rtn.name + new_rtn.slot_count = rtn.slot_count + new_rtn.text = rtn.text + for state in each(new_rtn:states()) do + state.rtn = new_rtn + end +end + -- vim:et:sts=2:sw=2 compiler/grammar.lua @@ -140,33 +140,13 @@ end grm_str = input_file:read("*a") print_verbose("Parsing grammar...") -grammar = parse_grammar(CharStream:new(grm_str)) -grammar:check_defined() - --- assign priorities to RTN transitions -print_verbose("Assigning RTN transition priorities...") -grammar:assign_priorities() - --- make the RTNs in the grammar determistic and minimal -print_verbose("Convering RTN NFAs to DFAs...") -grammar:determinize_rtns() -grammar:process_allow() -grammar:canonicalize_properties() +grammar = Grammar:new() +grammar:parse_source_string(grm_str) +grammar:process() if minimize_rtns then - print_verbose("Minimizing RTN DFAs...") grammar:minimize_rtns() end - --- Generate GLAs by doing lookahead calculations. --- This annotates every nontrivial state in the grammar with a GLA. -print_verbose("Doing LL(*) lookahead calculations...") -compute_lookahead(grammar, k) - --- we now have everything figured out at the RTN and GLA levels. Now we just --- need to figure out how many IntFAs to generate, which terminals each one --- should handle, and generate/determinize/minimize those IntFAs. - -print_verbose("Generating lexer DFAs...") +grammar:compute_lookahead(k) grammar:generate_intfas() print_verbose(string.format("Writing to output file '%s'...", output_filename)) compiler/gzlc @@ -84,9 +84,11 @@ function create_or_reuse_termset_for(terminals, conflicts, termsets, nonterm) end -- add all the terminals for this phase of lookahead to the termset we found + local termset = termsets[found_termset] for term in each(terminals) do - termsets[found_termset]:add(term) + termset:add(term) end + assert(termset:count() > 0) return found_termset end @@ -100,6 +102,7 @@ function intfa_combine(all_terminals, state_term_pairs) local intfa_nums = {} for state_term_pair in each(state_term_pairs) do local state, terms = unpack(state_term_pair) + assert(terms:count() > 0) local nonterm if state.rtn == nil then nonterm = state.gla.rtn_state.rtn.name @@ -114,6 +117,7 @@ function intfa_combine(all_terminals, state_term_pairs) local nfas = {} for term in each(termset) do local target = all_terminals[term] + assert(target, string.format("No terminal for terminal '%s'", tostring(serialize(term)))) if type(target) == "string" then target = fa.intfa_for_string(target) end compiler/intfa_combine.lua @@ -80,7 +80,7 @@ function check_for_nonrecursive_alt(grammar) for name, rtn in each(grammar.rtns) do local all_paths_see local found_nonrecursive_alt = false - local child_states = function(state_tuple) + local function child_states(state_tuple) local state, seen_nonterms, seen_states = unpack(state_tuple) if state.final then if all_paths_see then @@ -115,7 +115,7 @@ function check_for_nonrecursive_alt(grammar) -- each other. Cycles indicate that there is a cycle of rules that -- can never be returned from. This indicates a bug in the grammar. for name, all_paths_see in pairs(always_recurses) do - local child_recurses = function(rtn_name, stack) + local function child_recurses(rtn_name, stack) local children = {} for child_rtn_name in each(always_recurses[rtn_name]) do if stack:contains(child_rtn_name) then @@ -140,7 +140,7 @@ function check_for_left_recursion(grammar) for name, rtn in each(grammar.rtns) do local states = Set:new() - local children = function(state, stack) + local function children(state, stack) local children = {} for edge_val, dest_state in state:transitions() do if fa.is_nonterm(edge_val) then @@ -490,7 +490,7 @@ end function remove_excess_states(gla) for state in each(gla:states()) do local seen_alts = Set:new() - local child_states = function(state) + local function child_states(state) if state.final then seen_alts:add(state.final) end @@ -915,7 +915,7 @@ end -- This method is a helper method for the depth-first search of -- get_rtn_state_closure. function get_rtn_state_closure_for_path(path, grammar, follow_states) - local child_epsilon_paths = function(path) + local function child_epsilon_paths(path) local child_paths = {} if path.current_state.transitions == nil then print(serialize(path.current_state, 4, " ")) compiler/ll.lua @@ -105,29 +105,75 @@ function define_class(name, superclass) -- Do this here so that :new above is a class method, but all others become -- object methods. setmetatable(class, class_mt) - _G[name] = class + rawset(_G, name, class) end define_class("Object") define_class("MemoizedObject") -function MemoizedObject:initialize(class) - self.memoized_class = class - self.cache = {} + function MemoizedObject:initialize(class) + self.memoized_class = class + self.objects = OrderedMap:new() + end + + function MemoizedObject:get(name) + return self.objects:get_or_insert_new( + name, function() return self.memoized_class:new(name) end) + end + + function MemoizedObject:get_objects() + return self.objects + end +-- class MemoizedObject + +--[[-------------------------------------------------------------------- + + strict.lua, from the Lua distribution. Forces all globals to be + assigned at global scope before they are referenced. This prevents: + - assigning to a global inside a function, if you have not previously + assigned to the global at global scope. + - referencing a global before it has been defined. + +--------------------------------------------------------------------]]-- + +local getinfo, error, rawset, rawget = debug.getinfo, error, rawset, rawget + +local mt = getmetatable(_G) +if mt == nil then + mt = {} + setmetatable(_G, mt) +end + +mt.__declared = {} + +local function what () + local d = getinfo(3, "S") + return d and d.what or "C" end -function MemoizedObject:get(name) - local obj = self.cache[name] - if not obj then - obj = self.memoized_class:new(name) - self.cache[name] = obj +mt.__newindex = function (t, n, v) + if not mt.__declared[n] then + local w = what() + if w ~= "main" and w ~= "C" then + error("assign to undeclared variable '"..n.."'", 2) + end + mt.__declared[n] = true + end + rawset(t, n, v) +end + +mt.__index = function (t, n) + if not mt.__declared[n] and what() ~= "C" then + error("variable '"..n.."' is not declared", 2) end - return obj + return rawget(t, n) end + -- each(foo): returns an iterator; if the object supports the each method, -- call that, otherwise return an iterator for a plain array. function each(array_or_eachable_obj) + assert(type(array_or_eachable_obj) == "table") if array_or_eachable_obj.each then return array_or_eachable_obj:each() else @@ -245,7 +291,7 @@ function equivalence_classes(int_sets) end end - local cmp_events = function(a, b) + local function cmp_events(a, b) if a[1] == b[1] then return b[2] < a[2] -- END events should go before BEGIN events else @@ -257,7 +303,7 @@ function equivalence_classes(int_sets) local nested_regions = Set:new() local last_offset = nil - classes = {} + local classes = {} for event in each(events) do local offset, event_type, int_set = unpack(event) compiler/misc.lua @@ -120,7 +120,7 @@ end function rep(nfa, favor_repeat) local new_nfa = nfa:new_graph() - local repeat_properties, finish_properties = get_repeating_properties(favor_repeating) + local repeat_properties, finish_properties = get_repeating_properties(favor_repeat) new_nfa.start:add_transition(fa.e, nfa.start) nfa.final:add_transition(fa.e, nfa.start, repeat_properties) nfa.final:add_transition(fa.e, new_nfa.final, finish_properties) compiler/nfa_construct.lua @@ -18,6 +18,6 @@ number -> .sign="-"? value -> string | number | "true" | "false" | "null" | object | array; -whitespace -> .whitespace=/[\r\n\s\t]+/; +whitespace -> .whitespace_str=/[\r\n\s\t]+/; @allow whitespace object ... number, string; sketches/json.gzl @@ -106,11 +106,11 @@ function orderedNext(t, state) if state == nil then -- the first time, generate the index t.__orderedIndex = __genOrderedIndex( t ) - key = t.__orderedIndex[1] + local key = t.__orderedIndex[1] return key, t[key] end -- fetch the next value - key = nil + local key = nil for i = 1,table.getn(t.__orderedIndex) do if t.__orderedIndex[i] == state then key = t.__orderedIndex[i+1] @@ -170,7 +170,7 @@ UnitResult = { -- class end function UnitResult:displayOneFailedTest( failure ) - testName, errorMsg = unpack( failure ) + local testName, errorMsg = unpack( failure ) print(">>> "..testName.." failed") print( errorMsg ) end @@ -255,8 +255,8 @@ LuaUnit = { end function LuaUnit.strip_luaunit_stack(stack_trace) - stack_list = LuaUnit.strsplit( "\n", stack_trace ) - strip_end = #stack_trace + local stack_list = LuaUnit.strsplit( "\n", stack_trace ) + local strip_end = #stack_trace for i = table.getn(stack_list),1,-1 do -- a bit rude but it works ! if string.find(stack_list[i],"[C]: in function 'xpcall'",0,true) @@ -350,7 +350,7 @@ LuaUnit = { -- create the list before. If you do not do it now, you -- get undefined result because you modify _G while iterating -- over it. - testClassList = {} + local testClassList = {} for key, val in pairs(_G) do if string.sub(key,1,4) == 'Test' then table.insert( testClassList, key ) tests/luaunit.lua @@ -16,11 +16,11 @@ require "grammar" require "ll" function assert_intfas(grammar_str, ...) - local grammar = parse_grammar(CharStream:new(grammar_str)) - grammar:assign_priorities() - grammar:determinize_rtns() + local grammar = Grammar:new() + grammar:parse_source_string(grammar_str) + grammar:process() grammar:minimize_rtns() - compute_lookahead(grammar, k) + grammar:compute_lookahead() grammar:generate_intfas() local intfa_strings = {...} tests/test_intfa.lua @@ -87,16 +87,15 @@ function parse_gla(str, rtn_state) end function assert_lookahead(grammar_str, rule_str, slotnum, expected_gla_str, k) - local grammar = parse_grammar(CharStream:new(grammar_str)) - grammar:assign_priorities() - grammar:determinize_rtns() - grammar:minimize_rtns() + local grammar = Grammar:new() + grammar:parse_source_string(grammar_str) + grammar:process() local rule = grammar.rtns:get(rule_str) local state = find_state(rule, slotnum) local expected_gla = parse_gla(expected_gla_str, state) - compute_lookahead(grammar, k) + grammar:compute_lookahead(k) if not fa_isequal(expected_gla, state.gla) then local bad = io.open("bad.dot", "w") @@ -578,12 +577,11 @@ function TestFollow:test2() end function assert_fails_with_error(grammar_str, error_string) - local grammar = parse_grammar(CharStream:new(grammar_str)) - grammar:assign_priorities() - grammar:determinize_rtns() - grammar:minimize_rtns() + local grammar = Grammar:new() + grammar:parse_source_string(grammar_str) + grammar:process() - local success, message = pcall(compute_lookahead, grammar) + local success, message = pcall(grammar.compute_lookahead, grammar) if success then error("Failed to fail!") elseif not message:find(error_string) then tests/test_ll.lua @@ -35,7 +35,7 @@ function TestObject:test1() self.bar = bar self.baz = nil end - obj = HasInitializer:new(1, 2) + local obj = HasInitializer:new(1, 2) assert_equals(1, obj.foo) assert_equals(2, obj.bar) obj.baz = 3 tests/test_misc.lua 338a003e541f73d9c22741174362447f56654d02 Joshua Haberman joshua@reverberate.org http://github.com/haberman/gazelle/commit/e3d2566a0f805cc67b44f1f065d57eb99b9ed141 e3d2566a0f805cc67b44f1f065d57eb99b9ed141 2009-02-21T18:43:54-08:00 2009-02-21T18:43:54-08:00 Major refactoring. Gazelle is closer to self-hosting. This is a major change to the design of the Grammar class, and the Grammar <-> Parser interface. - the interface between the Grammar class and the parser is cleaner -- less application logic lives in the parser now. This should make Gazelle closer to self-hosting. - we more robustly maintain a symbol table that tracks what symbols have been defined, what symbols have been referenced (but not defined), and what types we expect them to be. This allows us to better catch situations like redefinition, references to symbols that are never defined, etc. - strict mode is now enabled, which throws an error if a global is referenced before it is defined, or if it's assigned in a function without being first assigned at global scope. I can't believe I didn't turn this on long, long ago. It caught a ton of local variables that I neglected to declare local. 3886bc67dd20482d058aa425eebdb90772ee9b0e Joshua Haberman joshua@reverberate.org