[performance] Automation / tools for instrumenting memory allocation.

- Instrument malloc and free (in addition to new and delete)
- Create an histogram of Id returned from the lexer
- Allow benchmarking against yash.  OSH is faster!

benchmarks/auto.sh

@@ -72,7 +72,7 @@ osh-parser-quick() {
   local osh_parse=_bin/osh_parse.opt.stripped
 
   local prov2
-  prov2=$(benchmarks/id.sh shell-provenance bash dash mksh $osh_parse)
+  prov2=$(benchmarks/id.sh shell-provenance bash dash mksh yash $osh_parse)
 
   benchmarks/osh-parser.sh measure $prov2 $base_dir/osh-parser
 }

benchmarks/id.sh

@@ -84,7 +84,7 @@ dump-shell-id() {
   esac
 
   case $name in
-    bash|zsh)
+    bash|zsh|yash)
       $sh --version > $out_dir/version.txt
       ;;
     osh)

build/mycpp.sh

@@ -163,6 +163,7 @@ compile() {
 
   # flags are split
   $CXX $flags \
+    -D INSTRUMENT_MALLOC_FREE \
     -I mycpp \
     -I cpp \
     -I _build/cpp \

core/ui.py

@@ -251,6 +251,15 @@ def PrintAst(nodes, opts):
 
     if opts.ast_format == 'none':
       print('AST not printed.', file=sys.stderr)
+      if 0:
+        from _devbuild.gen.id_kind_asdl import Id_str
+        from frontend.lexer import ID_HIST
+        total = 0
+        for id_, count in ID_HIST.most_common():
+          print('%8d %s' % (count, Id_str(id_)))
+          total += count
+        print()
+        print('%8d total tokens returned' % total)
 
     else:  # text output
       f = mylib.Stdout()

cpp/dumb_alloc.cc

@@ -5,11 +5,9 @@
 #include <stdio.h>
 
 // 100 MiB of memory
-//char kMem[100 << 20];
-// 1 GiB for running 10 times
-char kMem[1 << 30];
+char kMem[100 << 20];
 
-int gMemPos;
+int gMemPos = 0;
 int gNumNew = 0;
 int gNumDelete = 0;
 
@@ -30,6 +28,24 @@ void operator delete(void* p) noexcept {
   ++gNumDelete;
 }
 
+char kMem2[100 << 20];
+int gMemPos2 = 0;
+int gNumMalloc = 0;
+int gNumFree = 0;
+
+void* dumb_malloc(size_t size) noexcept {
+  char* p = &(kMem2[gMemPos2]);
+  //fprintf(stderr, "malloc %d\n", size);
+  gMemPos2 += size;
+  ++gNumMalloc;
+  return p;
+}
+
+void dumb_free(void* p) noexcept {
+  //fprintf(stderr, "free\n");
+  ++gNumFree;
+}
+
 namespace dumb_alloc {
 
 void Summarize() {
@@ -38,6 +54,10 @@ void Summarize() {
   fprintf(stderr, "\tgNumNew = %d\n", gNumNew);
   fprintf(stderr, "\tgNumDelete = %d\n", gNumDelete);
   fprintf(stderr, "\tgMemPos = %d\n", gMemPos);
+  fprintf(stderr, "\n");
+  fprintf(stderr, "\tgNumMalloc = %d\n", gNumMalloc);
+  fprintf(stderr, "\tgNumFree = %d\n", gNumFree);
+  fprintf(stderr, "\tgMemPos2 = %d\n", gMemPos2);
 }
 
 };

cpp/dumb_alloc.h

@@ -1,9 +1,13 @@
-
-// Replacement for asdl/pretty.py
+// Dumb Allocator
 
 #ifndef DUMB_ALLOC_H
 #define DUMB_ALLOC_H
 
+#include <cstddef>  // size_t
+
+void* dumb_malloc(size_t size) noexcept;
+void dumb_free(void* p) noexcept;
+
 namespace dumb_alloc {
 
 void Summarize();

cpp/frontend_match.cc

@@ -7,6 +7,12 @@
 #include "osh-types.h"
 #include "osh-lex.h"
 
+#if INSTRUMENT_MALLOC_FREE
+#include "dumb_alloc.h"
+#define malloc dumb_malloc
+#define free dumb_free
+#endif
+
 namespace match {
 
 Tuple2<Id_t, int>* OneToken(lex_mode_t lex_mode, Str* line, int start_pos) {

frontend/lexer.py

@@ -13,10 +13,11 @@
 from _devbuild.gen.id_kind_asdl import Id_t, Id, Kind
 from asdl import runtime
 from core.util import log
+from mycpp import mylib
 from frontend import lookup
 from frontend import match
 
-from typing import Callable, List, Tuple, Optional, TYPE_CHECKING
+from typing import Callable, List, Tuple, Optional, Counter, TYPE_CHECKING
 if TYPE_CHECKING:
   from core.alloc import Arena
   from frontend.reader import _Reader
@@ -255,5 +256,11 @@ def Read(self, lex_mode):
       if t.id != Id.Ignored_LineCont:
         break
 
+    #ID_HIST[t.id] += 1
     #log('> Read() Returning %s', t)
     return t
+
+
+if mylib.PYTHON:
+  import collections
+  ID_HIST = collections.Counter()  # type: Counter[Id_t]

mycpp/mylib.h

@@ -15,6 +15,12 @@
 #include <climits>  // CHAR_BIT
 #include <cstdint>
 
+#if INSTRUMENT_MALLOC_FREE
+#include "dumb_alloc.h"
+#define malloc dumb_malloc
+#define free dumb_free
+#endif
+
 // To reduce code size
 
 #define DISALLOW_COPY_AND_ASSIGN(TypeName) \
@@ -247,6 +253,8 @@ class StrIter {
 template <class T>
 class List {
  public:
+  // TODO: Shoudl we assume there are 2 or 5 or 10 elements at first? Measure
+  // speed?
   List() : v_() {
   }
 

oil_lang/grammar.pgen2

@@ -7,6 +7,9 @@
 #   mytuple,
 #   f(args,)
 #   func f(params,)
+# 
+# Kinds used:
+#   Left, Right, Expr, Op, Arith, Eof, Char
 
 # Oil patch: removed @= **= //=
 # We're missing div= and xor=, which now look weird.  ^= is

vendor/typing.py

@@ -24,6 +24,7 @@
 Any = None
 NoReturn = None
 Callable = None
+Counter = None  # for ID_HIST
 
 TYPE_CHECKING = False