Slight optimization of Eol_Tok handling in last commit.

Just set end_pos and do an early return when you see NUL.

asdl/py_meta.py

@@ -248,7 +248,9 @@ def MakeTypes(module, root, type_lookup):
        cls = type(class_name, (SimpleObj, ), class_attr)
        setattr(root, class_name, cls)

        # TODO: cons needs ASDL_TYPE?
        # NOTE: Right now the ASDL_TYPE for for an enum value is the Sum type,
        # not the Constructor type.  We may want to change this if we need
        # reflection.
        for i, cons in enumerate(sum_type.types):
          enum_id = i + 1
          name = cons.name

core/lexer_gen.py

@@ -225,7 +225,11 @@ def TranslateLexer(lexer_def):
      from core import id_kind
      id_name = id_kind.IdName(token_id)
      print '      %-30s { *id = id__%s; break; }' % (re2_pat, id_name)
    print '      %-30s { *id = id__%s; break; }' % (r'"\x00"', 'Eol_Tok')

    # EARLY RETURN: Do NOT advance past the NUL terminator.
    print '      %-30s { *id = id__Eol_Tok; *end_pos = start_pos; return; }' % \
        r'"\x00"'

    print '      */'
    print '    }'
    print '    break;'
@@ -270,12 +274,7 @@ def TranslateLexer(lexer_def):
    assert(0);

  }
  if (*id == id__Eol_Tok) {
    /* don't move past if Eol_Tok */
    *end_pos = start_pos;
  } else {
    *end_pos = p - line;  /* relative */
  }
  *end_pos = p - line;  /* relative */
}
"""