Rename lex.c to fastlex.c so there's no confusion with osh/lex.py.

Also: Updates to line count script. Count code generators and generated
code.

asdl/gen_cpp.py

@@ -395,7 +395,7 @@ class CEnumVisitor(AsdlVisitor):
  def VisitSimpleSum(self, sum, name, depth):
    # Just use #define, since enums aren't namespaced.
    for i, variant in enumerate(sum.types):
      self.Emit('#define %s__%s %d;' % (name, variant.name, i + 1), depth)
      self.Emit('#define %s__%s %d' % (name, variant.name, i + 1), depth)
    self.Emit("", depth)

build/codegen.sh

@@ -60,7 +60,7 @@ all() {
  lex-gen-native

  # Why do we need this?
  rm _devbuild/pylibc/x86_64/lex.so
  rm -f _devbuild/pylibc/x86_64/lex.so

  # Note: This also does pylibc, which we don't want.
  build/dev.sh all

build/dev.sh

@@ -30,7 +30,7 @@ gen-help() {
  build/doc.sh oil-quick-ref
}

# TODO: should lex.c be part of the dev build?  It means you need re2c
# TODO: should fastlex.c be part of the dev build?  It means you need re2c
# installed?  I don't think it makes sense to have 3 builds, so yes I think we
# can put it here for simplicity.
# However one problem is that if the Python lexer definition is changed, then
@@ -46,18 +46,18 @@ pylibc() {
  local libc_so=$(echo _devbuild/pylibc/$arch/libc.so)
  ln -s -f -v $libc_so libc.so

  local lex_so=$(echo _devbuild/pylibc/$arch/lex.so)
  ln -s -f -v $lex_so lex.so
  local fastlex_so=$(echo _devbuild/pylibc/$arch/fastlex.so)
  ln -s -f -v $fastlex_so fastlex.so

  file libc.so lex.so
  file libc.so fastlex.so
}

# Also done by unit.sh.
test-pylibc() {
  export PYTHONPATH=.
  pylibc
  native/libc_test.py
  native/lex_test.py
  native/fastlex_test.py
}

clean-pylibc() {

build/setup.py

@@ -9,10 +9,10 @@
      description = 'Module for libc functions like fnmatch()',
      ext_modules = [module])

module = Extension('lex',
                    sources = ['native/lex.c'])
module = Extension('fastlex',
                    sources = ['native/fastlex.c'])

setup(name = 'lex',
setup(name = 'fastlex',
      version = '1.0',
      description = 'Module to speed up lexers',
      include_dirs = ['_build/gen'],

core/lexer.py

@@ -52,6 +52,24 @@ def FindLongestMatch(re_list, s, pos):
  return end_index, tok_type, tok_val


# TODO: LineLexer needs the matcher rather than lexer_def.

class MatchTokenSlow(object):
  """An abstract matcher that doesn't depend on OSH."""
  def __init__(self, lexer_def):
    self.lexer_def = lexer_def

  def __call__(self, lex_mode, line, start_index):
    """Returns (id, end_index)."""
    return FindLongestMatch(self.lexer_def[lex_mode], line, start_index)


def MatchTokenFast(lex_mode, line, start_index):
  """Returns (id, end_index)."""
  tok_type, end_index = lex.MatchToken(lex_mode.enum_id, line, start_index)
  return Id(tok_type), end_index


class LineLexer(object):
  def __init__(self, lexer_def, line, arena=None):
    # Compile all regexes

native/lex.c → native/fastlex.c

@@ -25,7 +25,7 @@ void debug(const char* fmt, ...) {
}

static PyObject *
lex_MatchToken(PyObject *self, PyObject *args) {
fastlex_MatchToken(PyObject *self, PyObject *args) {
  int lex_mode;
  const char* line;

@@ -57,11 +57,11 @@ lex_MatchToken(PyObject *self, PyObject *args) {
// FastTokenMatcher

PyMethodDef methods[] = {
  {"MatchToken", lex_MatchToken, METH_VARARGS,
  {"MatchToken", fastlex_MatchToken, METH_VARARGS,
   "(lexer mode, line, start_index) -> (id, end_index)."},
  {NULL, NULL},
};

void initlex(void) {
  Py_InitModule("lex", methods);
void initfastlex(void) {
  Py_InitModule("fastlex", methods);
}

native/lex_test.py → native/fastlex_test.py

@@ -14,20 +14,20 @@
from core.id_kind import Id
from osh import ast_ as ast

import lex  # module under test
import fastlex  # module under test

lex_mode_e = ast.lex_mode_e


def MatchToken(lex_mode, line, s):
  tok_type, end_index = lex.MatchToken(lex_mode.enum_id, line, s)
  tok_type, end_index = fastlex.MatchToken(lex_mode.enum_id, line, s)
  return Id(tok_type), end_index


class LexTest(unittest.TestCase):

  def testMatchToken(self):
    print(dir(lex))
    print(dir(fastlex))
    print lex_mode_e.COMMENT.enum_id
    result = MatchToken(lex_mode_e.COMMENT, 'line', 3)
    print result

osh/lex_gen.py

@@ -65,11 +65,21 @@ def main(argv):

  print """

inline void MatchToken(int lexer_mode, char* line, int line_len, int start_index,
inline void MatchToken(int lex_mode, char* line, int line_len, int start_index,
                int* id, int* end_index) {
  *id = id__Lit_Chars;
  //*id = id__Lit_Other;
  *end_index = 3;
  switch (lex_mode)  {
  case lex_mode__OUTER:
    *id = id__Lit_Chars;
    //*id = id__Lit_Other;
    *end_index = 3;
    break;
  case lex_mode__COMMENT:
    *id = id__Lit_Other;
    *end_index = 5;
    break;
  default:
    assert(0);
  }
}
"""
  return

scripts/count.sh

@@ -44,6 +44,14 @@ all() {
  wc -l asdl/{asdl_,py_meta,gen_cpp,encode,format}.py | sort --numeric
  echo

  echo 'CODE GENERATORS'
  wc -l */*_gen.py | sort --numeric
  echo

  echo 'GENERATED CODE'
  wc -l _devbuild/*.py _build/gen/* | sort --numeric
  echo

  echo 'TOOLS'
  ls tools/*.py | filter-py | xargs wc -l | sort --numeric
  echo