Reduce object creation / memory usage by reusing ID instances.

The pure Python code just passes around pointers.  It never created more
than one instance of a given Id.

This should fix the regression in memory usage.

core/id_kind.py

@@ -12,18 +12,27 @@
from core import util


_ID_TO_KIND = {}  # type: dict
_ID_TO_KIND = {}  # int -> Kind

def LookupKind(id_):
  return _ID_TO_KIND[id_.enum_value]


_ID_NAMES = {}  # type: dict
_ID_NAMES = {}  # int -> string

def IdName(id_):
  return _ID_NAMES[id_.enum_value]


# Save memory by keeping one instance.
# TODO: Fold this into ASDL, which will enforce this?

_ID_INSTANCES = {}  # int -> Id

def IdInstance(i):
  return _ID_INSTANCES[i]


class Id(object):
  """Token and op type.

@@ -56,10 +65,11 @@ class Kind(object):
class IdSpec(object):
  """Identifiers that form the "spine" of the shell program representation."""

  def __init__(self, token_names, kind_lookup, bool_ops):
  def __init__(self, token_names, instance_lookup, kind_lookup, bool_ops):
    self.id_enum = Id
    self.kind_enum = Kind
    self.token_names = token_names  # integer -> string Id
    self.instance_lookup = instance_lookup
    self.kind_lookup = kind_lookup  # Id -> Kind

    self.kind_sizes = []  # stats
@@ -78,8 +88,11 @@ def _AddId(self, token_name):
    self.token_index += 1  # leave out 0 I guess?
    id_val = Id(self.token_index)
    setattr(self.id_enum, token_name, id_val)
    self.token_names[self.token_index] = token_name
    self.kind_lookup[self.token_index] = self.kind_index

    t = self.token_index
    self.token_names[t] = token_name
    self.instance_lookup[t] = id_val
    self.kind_lookup[t] = self.kind_index
    return id_val

  def _AddKind(self, kind_name):
@@ -465,7 +478,7 @@ def _SetupTestBuiltin(id_spec, unary_lookup, binary_lookup, other_lookup):
#


ID_SPEC = IdSpec(_ID_NAMES, _ID_TO_KIND, BOOL_OPS)
ID_SPEC = IdSpec(_ID_NAMES, _ID_INSTANCES, _ID_TO_KIND, BOOL_OPS)

_AddKinds(ID_SPEC)
_AddBoolKinds(ID_SPEC)  # must come second

osh/parse_lib.py

@@ -6,7 +6,7 @@

from core import lexer
from core import reader
from core.id_kind import Id
from core import id_kind

from osh import lex
from osh import word_parse
@@ -44,7 +44,9 @@ def __call__(self, lex_mode, line, start_pos):
def MatchToken_Fast(lex_mode, line, start_pos):
  """Returns (id, end_pos)."""
  tok_type, end_pos = fastlex.MatchToken(lex_mode.enum_id, line, start_pos)
  return Id(tok_type), end_pos
  # IMPORTANT: We're reusing Id instances here.  Ids are very common, so this
  # saves memory.
  return id_kind.IdInstance(tok_type), end_pos


def _MakeMatcher():