Fix a couple bugs that led to failures in test/osh2oil.sh.

- The generated ASDL Python code didn't allow optional integer values to
  be initialized to zero.  It uses Python's 'or' in initialization.

  It turns out we didn't need optional ints, so that worked around it.

- Don't add a line span for the initial Eol_Tok token.

We can do a basic bin/osh --fix -c 'ls', but some other tests are still
broken.

asdl/gen_python.py

@@ -68,6 +68,11 @@ def _GenClass(self, desc, name, super_name, depth, tag_num=None):
      elif f.seq:  # Array
        default = '[]'

      # PROBLEM: Optional ints can't be zero!
      # self.span_id = span_id or const.NO_INTEGER
      # I don't want to add if statements checking against None?
      # For now don't use optional ints.  We don't need it.

      default_str = (' or %s' % default) if default else ''
      self.Emit('    self.%s = %s%s' % (f.name, f.name, default_str), depth)

bin/oil.py

@@ -354,6 +354,7 @@ def OshMain(argv, login_shell):

    do_exec = True
    if opts.fix:
      #log('SPANS: %s', arena.spans)
      osh2oil.PrintAsOil(arena, node, opts.debug_spans)
      do_exec = False
    if exec_opts.noexec:

core/lexer.py

@@ -16,6 +16,8 @@
from core.id_kind import Id
from osh import ast_ as ast

from core.util import log


def C(pat, tok_type):
  """ Create a constant mapping like C('$*', VSub_Star) """
@@ -106,6 +108,8 @@ def Read(self, lex_mode):
    #assert self.line_pos <= len(self.line), (self.line, self.line_pos)
    tok_type, end_pos = self.match_func(lex_mode, self.line, self.line_pos)
    #assert end_pos <= len(self.line)
    if tok_type == Id.Eol_Tok:  # Do NOT add a span for this sentinel!
      return ast.token(tok_type, '', const.NO_INTEGER)

    tok_val = self.line[self.line_pos:end_pos]

@@ -129,6 +133,7 @@ def Read(self, lex_mode):
      span_id = self.arena.AddLineSpan(line_span)
      self.last_span_id = span_id

    #log('LineLexer.Read() span ID %d for %s', span_id, tok_type)
    t = ast.token(tok_type, tok_val, span_id)

    self.line_pos = end_pos
@@ -226,5 +231,5 @@ def Read(self, lex_mode):
      if t.id != Id.Ignored_LineCont:
        break

    #print("T", t, lex_mode)
    #log('Read() Returning %s', t)
    return t

osh/osh.asdl

@@ -45,7 +45,7 @@ module osh
  -- rewrite the parser in C++, we might care for memory footprint.  But for 
  -- now this is convenient.
  -- NOTE: identical strings can shared, if we care.
  token = (id id, string val, int? span_id)
  token = (id id, string val, int span_id)

  -- Optional step for {100..50..-15}
  braced_step = (int val, int negated)

tools/osh2oil.py

@@ -10,6 +10,7 @@
from core import util
from core import word
from core.id_kind import Id
from asdl import const

from osh import ast_ as ast

@@ -35,8 +36,14 @@ def __init__(self, arena, f):
    self.next_span_id = 0

  def PrintUntil(self, until_span_id):
    # Sometimes we add +1
    assert until_span_id < const.NO_INTEGER, 'Missing span ID, got %d' % until_span_id
    #log('PrintUntil %d', until_span_id)
    for span_id in range(self.next_span_id, until_span_id):
      #log('Looking up span id %d', span_id)
      span = self.arena.GetLineSpan(span_id)
      #log('SPAN %s', span)

      line = self.arena.GetLine(span.line_id)
      piece = line[span.col : span.col + span.length]
      self.f.write(piece)
@@ -50,7 +57,7 @@ def SkipUntil(self, next_span_id):
    """Skip everything before next_span_id.
    Printing will start at next_span_id
    """
    assert next_span_id >= 0, next_span_id
    assert next_span_id != const.NO_INTEGER, next_span_id
    self.next_span_id = next_span_id


@@ -663,7 +670,7 @@ def DoCommand(self, node, local_symbols, at_top_level=False):

      in_spid, semi_spid = node.spids

      if in_spid == -1:
      if in_spid == const.NO_INTEGER:
        #self.cursor.PrintUntil()  # 'for x' and then space
        self.f.write('for %s in @Argv ' % node.iter_name)
        self.cursor.SkipUntil(node.body.spids[0])
@@ -675,7 +682,7 @@ def DoCommand(self, node, local_symbols, at_top_level=False):
        self.f.write(']')
        #print("SKIPPING SEMI %d" % semi_spid, file=sys.stderr)

      if semi_spid != -1:
      if semi_spid != const.NO_INTEGER:
        self.cursor.PrintUntil(semi_spid)
        self.cursor.SkipUntil(semi_spid + 1)

@@ -701,7 +708,7 @@ def DoCommand(self, node, local_symbols, at_top_level=False):
      # elif foo; then -> } elif foo {
      for arm in node.arms:
        elif_spid, then_spid = arm.spids
        if elif_spid != -1:
        if elif_spid != const.NO_INTEGER:
          self.cursor.PrintUntil(elif_spid)
          self.f.write('} ')

@@ -781,13 +788,13 @@ def DoCommand(self, node, local_symbols, at_top_level=False):
        for child in arm.action:
          self.DoCommand(child, local_symbols)

        if dsemi_spid != -1:
        if dsemi_spid != const.NO_INTEGER:
          self.cursor.PrintUntil(dsemi_spid)
          self.cursor.SkipUntil(dsemi_spid + 1)
          # NOTE: indentation here will be off because ;; is likely indented
          # with body.
          self.f.write('}')
        elif last_spid != -1:
        elif last_spid != const.NO_INTEGER:
          self.cursor.PrintUntil(last_spid)
          # NOTE: Indentation is also off here.  Arbitrarily put 4 spaces.
          self.f.write('    }\n')
@@ -928,7 +935,7 @@ def DoWordInCommand(self, node, local_symbols):

        # TODO: Double quoted part needs left and right IDs
        left_spid, right_spid = dq_part.spids
        assert right_spid != -1, right_spid
        assert right_spid != const.NO_INTEGER, right_spid

        if len(dq_part.parts) == 1:
          part0 = dq_part.parts[0]
@@ -997,7 +1004,7 @@ def DoWordInCommand(self, node, local_symbols):

  def DoWordPart(self, node, local_symbols, quoted=False):
    span_id = word.LeftMostSpanForPart(node)
    if span_id is not None and span_id >= 0:
    if span_id is not None and span_id != const.NO_INTEGER:
      span = self.arena.GetLineSpan(span_id)
      #print(span)

@@ -1173,7 +1180,6 @@ def DoWordPart(self, node, local_symbols, quoted=False):
      self.cursor.SkipUntil(right_spid + 1)

    else:
      #log('WordPart not handled: %s', node)
      raise AssertionError(node.__class__.__name__)

  def DoArithExpr(self, node, local_symbols):
@@ -1192,7 +1198,6 @@ def DoArithExpr(self, node, local_symbols):
      self.DoWordInCommand(node.w, local_symbols)

    else:
      #log("Unhandled: %s", node)
      raise AssertionError(node.__class__.__name__)

  def DoBoolExpr(self, node):