Tweaks to store only unsigned integers in the AST.

This makes for simpler 'oheap' encoding.

- Make span IDs start from 1, so we can use 0 for undefined.
- Initialize ASDL int? type to 0 rather than None.

We don't want -1 to be undefined because oheap only has unsigned ints
right now.  We would lose half our bits.

asdl/arith.asdl

@@ -14,6 +14,9 @@ module arith {
  -- it has no tag?
  source_location = (string path, int line, int col, int length)

  -- Optional int
  token = (int id, string value, int? span_id)

  -- TODO:
  -- - Add optional.  For slicing maybe -- optional end.
  -- - Add repeated.  For function call maybe.

asdl/arith_ast_test.py

@@ -41,6 +41,11 @@ def testFieldDefaults(self):
    self.assertEqual([], func.args)
    print(func)

    t = arith_ast.token(5, 'x')
    self.assertEqual(5, t.id)
    self.assertEqual('x', t.value)
    self.assertEqual(0, t.span_id)

  def testTypeCheck(self):
    v = ArithVar('name')
    # Integer is not allowed

asdl/py_meta.py

@@ -161,11 +161,18 @@ def __ne__(self, other):

  def _SetDefaults(self):
    for name in self.FIELDS:
      #print("%r wasn't assigned" % name)
      desc = self.DESCRIPTOR_LOOKUP[name]
      # item_desc = desc.desc

      if isinstance(desc, asdl.MaybeType):
        self.__setattr__(name, None)  # Maybe values can be None
        child = desc.desc
        if isinstance(child, asdl.IntType):
          value = 0
        elif isinstance(child, asdl.StrType):
          value = ''
        else:
          value = None
        self.__setattr__(name, value)  # Maybe values can be None

      elif isinstance(desc, asdl.ArrayType):
        self.__setattr__(name, [])

core/alloc.py

@@ -32,8 +32,9 @@ def __init__(self, arena_id):
    self.lines = []
    self.next_line_id = 0

    self.spans = []
    self.next_span_id = 0
    # first real span is 1.  0 means undefined.
    self.spans = [None]
    self.next_span_id = 1

    # List of (src_path index, physical line number).  This is two integers for
    # every line read.  We could use a clever encoding of this.  (Although the
@@ -94,8 +95,8 @@ def AddLineSpan(self, line_span):
    return span_id

  def GetLineSpan(self, span_id):
    assert span_id >= 0, span_id
    return self.spans[span_id]
    assert span_id > 0, span_id
    return self.spans[span_id]  # span IDs start from 1

  def GetDebugInfo(self, line_id):
    """Get the path and physical line number, for parse errors."""

core/alloc_test.py

@@ -24,7 +24,7 @@ def testPool(self):
    self.assertEqual(1, line_id)

    span_id = arena.AddLineSpan(None)
    self.assertEqual(0, span_id)
    self.assertEqual(1, span_id)

    arena.PopSource()

core/lexer.py

@@ -43,7 +43,7 @@ def __init__(self, match_func, line, arena=None):
    self.arena = arena

    self.arena_skip = False  # For MaybeUnreadOne
    self.last_span_id = -1  # For MaybeUnreadOne
    self.last_span_id = 0  # For MaybeUnreadOne

    self.Reset(line, -1)  # Invalid arena index to start

@@ -88,7 +88,7 @@ def LookAhead(self, lex_mode):
        # would involve interacting with the line reader, and we never need
        # it.  In the OUTER mode, there is an explicit newline token, but
        # ARITH doesn't have it.
        t = ast.token(Id.Unknown_Tok, '', -1)  # no span ID
        t = ast.token(Id.Unknown_Tok, '', 0)  # no span ID
        return t

      tok_type, end_pos = self.match_func(lex_mode, self.line, pos)
@@ -99,7 +99,7 @@ def LookAhead(self, lex_mode):
        break
      pos = end_pos

    return ast.token(tok_type, tok_val)  # no location
    return ast.token(tok_type, tok_val, 0)  # no span ID

  def AtEnd(self):
    return self.line_pos == len(self.line)
@@ -125,7 +125,7 @@ def Read(self, lex_mode):

    if self.arena is not None:
      if self.arena_skip:
        assert self.last_span_id != -1
        assert self.last_span_id != 0
        span_id = self.last_span_id
        self.arena_skip = False
      else:
@@ -134,7 +134,7 @@ def Read(self, lex_mode):
    else:
      # Completion parser might not have arena?
      # We should probably get rid of this.
      span_id = -1
      span_id = 0

    t = ast.token(tok_type, tok_val, span_id)