state.rb

#--
#
#
#
# Copyright (c) 1999-2006 Minero Aoki
#
# This program is free software.
# You can distribute/modify this program under the same terms of ruby.
# see the file "COPYING".
#
#++

require 'racc/iset'
require 'racc/statetransitiontable'
require 'racc/exception'
require 'forwardable'

module Racc

  # A table of LALR states.
  class States

    include Enumerable

    def initialize(grammar, debug_flags = DebugFlags.new)
      @grammar = grammar
      @symboltable = grammar.symboltable
      @d_state = debug_flags.state
      @d_la    = debug_flags.la
      @d_prec  = debug_flags.prec
      @states = []
      @statecache = {}
      @actions = ActionTable.new(@grammar, self)
      @nfa_computed = false
      @dfa_computed = false
    end

    attr_reader :grammar
    attr_reader :actions

    def size
      @states.size
    end

    def inspect
      '#<state table>'
    end

    alias to_s inspect

    def [](i)
      @states[i]
    end

    def each_state(&block)
      @states.each(&block)
    end

    alias each each_state

    def each_index(&block)
      @states.each_index(&block)
    end

    extend Forwardable

    def_delegator "@actions", :shift_n
    def_delegator "@actions", :reduce_n
    def_delegator "@actions", :nt_base

    def should_report_srconflict?
      srconflict_exist? and
          (n_srconflicts() != @grammar.n_expected_srconflicts)
    end

    def srconflict_exist?
      n_srconflicts() != 0
    end

    def n_srconflicts
      @n_srconflicts ||= inject(0) {|sum, st| sum + st.n_srconflicts }
    end

    def rrconflict_exist?
      n_rrconflicts() != 0
    end

    def n_rrconflicts
      @n_rrconflicts ||= inject(0) {|sum, st| sum + st.n_rrconflicts }
    end

    def state_transition_table
      @state_transition_table ||= StateTransitionTable.generate(self.dfa)
    end

    #
    # NFA (Non-deterministic Finite Automaton) Computation
    #

    public

    def nfa
      return self if @nfa_computed
      compute_nfa
      @nfa_computed = true
      self
    end

    private

    def compute_nfa
      @grammar.init
      # add state 0
      core_to_state  [ @grammar[0].ptrs[0] ]
      # generate LALR states
      cur = 0
      @gotos = []
      while cur < @states.size
        generate_states @states[cur]   # state is added here
        cur += 1
      end
      @actions.init
    end

    def generate_states(state)
      puts "dstate: #{state}" if @d_state

      table = {}
      state.closure.each do |ptr|
        if sym = ptr.dereference
          addsym table, sym, ptr.next
        end
      end
      table.each do |sym, core|
        puts "dstate: sym=#{sym} ncore=#{core}" if @d_state

        dest = core_to_state(core.to_a)
        state.goto_table[sym] = dest
        id = sym.nonterminal?() ? @gotos.size : nil
        g = Goto.new(id, sym, state, dest)
        @gotos.push g if sym.nonterminal?
        state.gotos[sym] = g
        puts "dstate: #{state.ident} --#{sym}--> #{dest.ident}" if @d_state

        # check infinite recursion
        if state.ident == dest.ident and state.closure.size == 1
          raise CompileError,
              sprintf("Infinite recursion: state %d, with rule %d",
                      state.ident, state.ptrs[0].rule.ident)
        end
      end
    end

    def addsym(table, sym, ptr)
      unless s = table[sym]
        table[sym] = s = ISet.new
      end
      s.add ptr
    end

    def core_to_state(core)
      #
      # convert CORE to a State object.
      # If matching state does not exist, create it and add to the table.
      #

      k = fingerprint(core)
      unless dest = @statecache[k]
        # not registered yet
        dest = State.new(@states.size, core)
        @states.push dest

        @statecache[k] = dest

        puts "core_to_state: create state   ID #{dest.ident}" if @d_state
      else
        if @d_state
          puts "core_to_state: dest is cached ID #{dest.ident}"
          puts "core_to_state: dest core #{dest.core.join(' ')}"
        end
      end

      dest
    end

    def fingerprint(arr)
      arr.map {|i| i.ident }.pack('L*')
    end

    #
    # DFA (Deterministic Finite Automaton) Generation
    #

    public

    def dfa
      return self if @dfa_computed
      nfa
      compute_dfa
      @dfa_computed = true
      self
    end

    private

    def compute_dfa
      la = lookahead()
      @states.each do |state|
        state.la = la
        resolve state
      end
      set_accept
      @states.each do |state|
        pack state
      end
      check_useless
    end

    def lookahead
      #
      # lookahead algorithm ver.3 -- from bison 1.26
      #

      gotos = @gotos
      if @d_la
        puts "\n--- goto ---"
        gotos.each_with_index {|g, i| print i, ' '; p g }
      end

      ### initialize_LA()
      ### set_goto_map()
      la_rules = []
      @states.each do |state|
        state.check_la la_rules
      end

      ### initialize_F()
      f     = create_tmap(gotos.size)
      reads = []
      edge  = []
      gotos.each do |goto|
        goto.to_state.goto_table.each do |t, st|
          if t.terminal?
            f[goto.ident] |= (1 << t.ident)
          elsif t.nullable?
            edge.push goto.to_state.gotos[t].ident
          end
        end
        if edge.empty?
          reads.push nil
        else
          reads.push edge
          edge = []
        end
      end
      digraph f, reads
      if @d_la
        puts "\n--- F1 (reads) ---"
        print_tab gotos, reads, f
      end

      ### build_relations()
      ### compute_FOLLOWS
      path = nil
      edge = []
      lookback = Array.new(la_rules.size, nil)
      includes = []
      gotos.each do |goto|
        goto.symbol.heads.each do |ptr|
          path = record_path(goto.from_state, ptr.rule)
          lastgoto = path.last
          st = lastgoto ? lastgoto.to_state : goto.from_state
          if st.conflict?
            addrel lookback, st.rruleid(ptr.rule), goto
          end
          path.reverse_each do |g|
            break if     g.symbol.terminal?
            edge.push    g.ident
            break unless g.symbol.nullable?
          end
        end
        if edge.empty?
          includes.push nil
        else
          includes.push edge
          edge = []
        end
      end
      includes = transpose(includes)
      digraph f, includes
      if @d_la
        puts "\n--- F2 (includes) ---"
        print_tab gotos, includes, f
      end

      ### compute_lookaheads
      la = create_tmap(la_rules.size)
      lookback.each_with_index do |arr, i|
        if arr
          arr.each do |g|
            la[i] |= f[g.ident]
          end
        end
      end
      if @d_la
        puts "\n--- LA (lookback) ---"
        print_tab la_rules, lookback, la
      end

      la
    end

    def create_tmap(size)
      Array.new(size, 0)   # use Integer as bitmap
    end

    def addrel(tbl, i, item)
      if a = tbl[i]
        a.push item
      else
        tbl[i] = [item]
      end
    end

    def record_path(begst, rule)
      st = begst
      path = []
      rule.symbols.each do |t|
        goto = st.gotos[t]
        path.push goto
        st = goto.to_state
      end
      path
    end

    def transpose(rel)
      new = Array.new(rel.size, nil)
      rel.each_with_index do |arr, idx|
        if arr
          arr.each do |i|
            addrel new, i, idx
          end
        end
      end
      new
    end

    def digraph(map, relation)
      n = relation.size
      index    = Array.new(n, nil)
      vertices = []
      @infinity = n + 2

      index.each_index do |i|
        if not index[i] and relation[i]
          traverse i, index, vertices, map, relation
        end
      end
    end

    def traverse(i, index, vertices, map, relation)
      vertices.push i
      index[i] = height = vertices.size

      if rp = relation[i]
        rp.each do |proci|
          unless index[proci]
            traverse proci, index, vertices, map, relation
          end
          if index[i] > index[proci]
            # circulative recursion !!!
            index[i] = index[proci]
          end
          map[i] |= map[proci]
        end
      end

      if index[i] == height
        while true
          proci = vertices.pop
          index[proci] = @infinity
          break if i == proci

          map[proci] |= map[i]
        end
      end
    end

    # for debug
    def print_atab(idx, tab)
      tab.each_with_index do |i,ii|
        printf '%-20s', idx[ii].inspect
        p i
      end
    end

    def print_tab(idx, rel, tab)
      tab.each_with_index do |bin,i|
        print i, ' ', idx[i].inspect, ' << '; p rel[i]
        print '  '
        each_t(@symboltable, bin) {|t| print ' ', t }
        puts
      end
    end

    # for debug
    def print_tab_i(idx, rel, tab, i)
      bin = tab[i]
      print i, ' ', idx[i].inspect, ' << '; p rel[i]
      print '  '
      each_t(@symboltable, bin) {|t| print ' ', t }
    end

    # for debug
    def printb(i)
      each_t(@symboltable, i) do |t|
        print t, ' '
      end
      puts
    end

    def each_t(tbl, set)
      0.upto( set.size ) do |i|
        (0..7).each do |ii|
          if set[idx = i * 8 + ii] == 1
            yield tbl[idx]
          end
        end
      end
    end

    #
    # resolve
    #

    def resolve(state)
      if state.conflict?
        resolve_rr state, state.ritems
        resolve_sr state, state.stokens
      else
        if state.rrules.empty?
          # shift
          state.stokens.each do |t|
            state.action[t] = @actions.shift(state.goto_table[t])
          end
        else
          # reduce
          state.defact = @actions.reduce(state.rrules[0])
        end
      end
    end

    def resolve_rr(state, r)
      r.each do |item|
        item.each_la(@symboltable) do |t|
          act = state.action[t]
          if act
            unless act.kind_of?(Reduce)
              raise "racc: fatal: #{act.class} in action table"
            end
            # Cannot resolve R/R conflict (on t).
            # Reduce with upper rule as default.
            state.rr_conflict act.rule, item.rule, t
          else
            # No conflict.
            state.action[t] = @actions.reduce(item.rule)
          end
        end
      end
    end

    def resolve_sr(state, s)
      s.each do |stok|
        goto = state.goto_table[stok]
        act = state.action[stok]

        unless act
          # no conflict
          state.action[stok] = @actions.shift(goto)
        else
          unless act.kind_of?(Reduce)
            puts 'DEBUG -------------------------------'
            p stok
            p act
            state.action.each do |k,v|
              print k.inspect, ' ', v.inspect, "\n"
            end
            raise "racc: fatal: #{act.class} in action table"
          end

          # conflict on stok

          rtok = act.rule.precedence
          case do_resolve_sr(stok, rtok)
          when :Reduce
            # action is already set

          when :Shift
            # overwrite
            act.decref
            state.action[stok] = @actions.shift(goto)

          when :Error
            act.decref
            state.action[stok] = @actions.error

          when :CantResolve
            # shift as default
            act.decref
            state.action[stok] = @actions.shift(goto)
            state.sr_conflict stok, act.rule
          end
        end
      end
    end

    ASSOC = {
      :Left     => :Reduce,
      :Right    => :Shift,
      :Nonassoc => :Error
    }

    def do_resolve_sr(stok, rtok)
      puts "resolve_sr: s/r conflict: rtok=#{rtok}, stok=#{stok}" if @d_prec

      unless rtok and rtok.precedence
        puts "resolve_sr: no prec for #{rtok}(R)" if @d_prec
        return :CantResolve
      end
      rprec = rtok.precedence

      unless stok and stok.precedence
        puts "resolve_sr: no prec for #{stok}(S)" if @d_prec
        return :CantResolve
      end
      sprec = stok.precedence

      ret = if rprec == sprec
              ASSOC[rtok.assoc] or
                  raise "racc: fatal: #{rtok}.assoc is not Left/Right/Nonassoc"
            else
              (rprec > sprec) ? (:Reduce) : (:Shift)
            end

      puts "resolve_sr: resolved as #{ret.id2name}" if @d_prec
      ret
    end

    #
    # complete
    #

    def set_accept
      anch = @symboltable.anchor
      init_state = @states[0].goto_table[@grammar.start]
      targ_state = init_state.action[anch].goto_state
      acc_state  = targ_state.action[anch].goto_state

      acc_state.action.clear
      acc_state.goto_table.clear
      acc_state.defact = @actions.accept
    end

    def pack(state)
      ### find most frequently used reduce rule
      act = state.action
      arr = Array.new(@grammar.size, 0)
      act.each do |t, a|
        arr[a.ruleid] += 1  if a.kind_of?(Reduce)
      end
      i = arr.max
      s = (i > 0) ? arr.index(i) : nil

      ### set & delete default action
      if s
        r = @actions.reduce(s)
        if not state.defact or state.defact == r
          act.delete_if {|t, a| a == r }
          state.defact = r
        end
      else
        state.defact ||= @actions.error
      end
    end

    def check_useless
      used = []
      @actions.each_reduce do |act|
        if not act or act.refn == 0
          act.rule.useless = true
        else
          t = act.rule.target
          used[t.ident] = t
        end
      end
      @symboltable.nt_base.upto(@symboltable.nt_max - 1) do |n|
        unless used[n]
          @symboltable[n].useless = true
        end
      end
    end

  end   # class StateTable


  # A LALR state.
  class State

    def initialize(ident, core)
      @ident = ident
      @core = core
      @goto_table = {}
      @gotos = {}
      @stokens = nil
      @ritems = nil
      @action = {}
      @defact = nil
      @rrconf = nil
      @srconf = nil

      @closure = make_closure(@core)
    end

    attr_reader :ident
    alias stateid ident
    alias hash ident

    attr_reader :core
    attr_reader :closure

    attr_reader :goto_table
    attr_reader :gotos

    attr_reader :stokens
    attr_reader :ritems
    attr_reader :rrules

    attr_reader :action
    attr_accessor :defact   # default action

    attr_reader :rrconf
    attr_reader :srconf

    def inspect
      "<state #{@ident}>"
    end

    alias to_s inspect

    def ==(oth)
      @ident == oth.ident
    end

    alias eql? ==

    def make_closure(core)
      set = ISet.new
      core.each do |ptr|
        set.add ptr
        if t = ptr.dereference and t.nonterminal?
          set.update_a t.expand
        end
      end
      set.to_a
    end

    def check_la(la_rules)
      @conflict = false
      s = []
      r = []
      @closure.each do |ptr|
        if t = ptr.dereference
          if t.terminal?
            s[t.ident] = t
            if t.ident == 1    # $error
              @conflict = true
            end
          end
        else
          r.push ptr.rule
        end
      end
      unless r.empty?
        if not s.empty? or r.size > 1
          @conflict = true
        end
      end
      s.compact!
      @stokens  = s
      @rrules = r

      if @conflict
        @la_rules_i = la_rules.size
        @la_rules = r.map {|i| i.ident }
        la_rules.concat r
      else
        @la_rules_i = @la_rules = nil
      end
    end

    def conflict?
      @conflict
    end

    def rruleid(rule)
      if i = @la_rules.index(rule.ident)
        @la_rules_i + i
      else
        puts '/// rruleid'
        p self
        p rule
        p @rrules
        p @la_rules_i
        raise 'racc: fatal: cannot get reduce rule id'
      end
    end

    def la=(la)
      return unless @conflict
      i = @la_rules_i
      @ritems = r = []
      @rrules.each do |rule|
        r.push Item.new(rule, la[i])
        i += 1
      end
    end

    def rr_conflict(high, low, ctok)
      c = RRconflict.new(@ident, high, low, ctok)

      @rrconf ||= {}
      if a = @rrconf[ctok]
        a.push c
      else
        @rrconf[ctok] = [c]
      end
    end

    def sr_conflict(shift, reduce)
      c = SRconflict.new(@ident, shift, reduce)

      @srconf ||= {}
      if a = @srconf[shift]
        a.push c
      else
        @srconf[shift] = [c]
      end
    end

    def n_srconflicts
      @srconf ? @srconf.size : 0
    end

    def n_rrconflicts
      @rrconf ? @rrconf.size : 0
    end

  end   # class State


  #
  # Represents a transition on the grammar.
  # "Real goto" means a transition by nonterminal,
  # but this class treats also terminal's.
  # If one is a terminal transition, .ident returns nil.
  #
  class Goto
    def initialize(ident, sym, from, to)
      @ident      = ident
      @symbol     = sym
      @from_state = from
      @to_state   = to
    end

    attr_reader :ident
    attr_reader :symbol
    attr_reader :from_state
    attr_reader :to_state

    def inspect
      "(#{@from_state.ident}-#{@symbol}->#{@to_state.ident})"
    end
  end


  # LALR item.  A set of rule and its lookahead tokens.
  class Item
    def initialize(rule, la)
      @rule = rule
      @la  = la
    end

    attr_reader :rule
    attr_reader :la

    def each_la(tbl)
      la = @la
      0.upto(la.size - 1) do |i|
        (0..7).each do |ii|
          if la[idx = i * 8 + ii] == 1
            yield tbl[idx]
          end
        end
      end
    end
  end


  # The table of LALR actions. Actions are either of
  # Shift, Reduce, Accept and Error.
  class ActionTable

    def initialize(rt, st)
      @grammar = rt
      @statetable = st

      @reduce = []
      @shift = []
      @accept = nil
      @error = nil
    end

    def init
      @grammar.each do |rule|
        @reduce.push Reduce.new(rule)
      end
      @statetable.each do |state|
        @shift.push Shift.new(state)
      end
      @accept = Accept.new
      @error = Error.new
    end

    def reduce_n
      @reduce.size
    end

    def reduce(i)
      case i
      when Rule    then i = i.ident
      when Integer then ;
      else
        raise "racc: fatal: wrong class #{i.class} for reduce"
      end

      r = @reduce[i] or raise "racc: fatal: reduce action #{i.inspect} not exist"
      r.incref
      r
    end

    def each_reduce(&block)
      @reduce.each(&block)
    end

    def shift_n
      @shift.size
    end

    def shift(i)
      case i
      when State   then i = i.ident
      when Integer then ;
      else
        raise "racc: fatal: wrong class #{i.class} for shift"
      end

      @shift[i] or raise "racc: fatal: shift action #{i} does not exist"
    end

    def each_shift(&block)
      @shift.each(&block)
    end

    attr_reader :accept
    attr_reader :error

  end


  class Shift
    def initialize(goto)
      @goto_state = goto
    end

    attr_reader :goto_state

    def goto_id
      @goto_state.ident
    end

    def inspect
      "<shift #{@goto_state.ident}>"
    end
  end


  class Reduce
    def initialize(rule)
      @rule = rule
      @refn = 0
    end

    attr_reader :rule
    attr_reader :refn

    def ruleid
      @rule.ident
    end

    def inspect
      "<reduce #{@rule.ident}>"
    end

    def incref
      @refn += 1
    end

    def decref
      @refn -= 1
      raise 'racc: fatal: act.refn < 0' if @refn < 0
    end
  end

  class Accept
    def inspect
      "<accept>"
    end
  end

  class Error
    def inspect
      "<error>"
    end
  end

  class SRconflict
    def initialize(sid, shift, reduce)
      @stateid = sid
      @shift   = shift
      @reduce  = reduce
    end

    attr_reader :stateid
    attr_reader :shift
    attr_reader :reduce

    def to_s
      sprintf('state %d: S/R conflict rule %d reduce and shift %s',
              @stateid, @reduce.ruleid, @shift.to_s)
    end
  end

  class RRconflict
    def initialize(sid, high, low, tok)
      @stateid   = sid
      @high_prec = high
      @low_prec  = low
      @token     = tok
    end

    attr_reader :stateid
    attr_reader :high_prec
    attr_reader :low_prec
    attr_reader :token

    def to_s
      sprintf('state %d: R/R conflict with rule %d and %d on %s',
              @stateid, @high_prec.ident, @low_prec.ident, @token.to_s)
    end
  end

end