[translation] Add GC header to ASDL objects.

And remove the Token/speck hack that "compressed" Id_t to 16 bits.  The
GC header invalidates that optimization unfortunately.

Metrics here:
https://oilshell.zulipchat.com/#narrow/stream/121539-oil-dev/topic/Back.20to.20Copying.20GC.20Idea/near/208994320

[release] Publish benchmarks of mycpp examples.

asdl/gen_cpp.py

@@ -362,6 +362,7 @@ def _GenClass(self, ast_node, attributes, class_name, base_classes, depth, tag):
     # Members
     #
     self.Emit('  uint16_t tag;')
+    self.Emit('  uint16_t how_to_trace;')  # for copying GC
     for field in all_fields:
       self.Emit("  %s %s;" % (_GetCppType(field.typ), field.name))
 

benchmarks/report.sh

@@ -82,9 +82,9 @@ compute() {
   stage3 $base_dir
 }
 
-mycpp() {
+mycpp-examples() {
   # ./run.sh benchmark-all  in the mycpp/ dir produces this
-  local base_dir=${1:-_tmp/mycpp-benchmarks}
+  local base_dir=${1:-_tmp/mycpp-examples}
   mkdir -p $base_dir
 
   local dir2=$base_dir/stage2
@@ -101,6 +101,9 @@ all() {
   vm-baseline
   ovm-build
   compute
+
+  # Only run on one machine
+  mycpp-examples
 }
 
 # For view

build/dev.sh

@@ -142,16 +142,6 @@ oil-asdl-to-cpp() {
   gen-asdl-cpp core/runtime.asdl
 
   gen-asdl-cpp frontend/syntax.asdl
-
-  # CRAZY SHARP HACK to make Token 16 bytes instead of 24!  (And speck is 8
-  # bytes rather than 12, although we might want to get rid of it.)
-  # Because of C's weak type system (typedef int Id_t), this is better than
-  # changing ASDL.
-  local orig=_build/cpp/syntax_asdl.h 
-  local tmp=_devbuild/tmp/syntax_asdl.h
-  sed 's/Id_t id;/uint16_t id;/g' $orig > $tmp
-  diff -u $orig $tmp || true
-  mv -v $tmp $orig
 }
 
 oil-cpp() {

build/mycpp.sh

@@ -294,7 +294,7 @@ all-variants() {
   ls -l _bin/$name*
 }
 
-readonly TMP=_tmp/mycpp
+readonly TMP=_devbuild/tmp
 
 osh-eval-manifest() {
   # _devbuild is ASDL stuff
@@ -347,10 +347,10 @@ asdl-runtime() {
   #   depending on Oil.
 
   local name=asdl_runtime
-  local raw=_tmp/mycpp/${name}_raw.cc 
+  local raw=$TMP/${name}_raw.cc 
 
   mycpp \
-    --header-out _tmp/mycpp/runtime.h \
+    --header-out $TMP/runtime.h \
     --to-header asdl.runtime \
     --to-header asdl.format \
     $REPO_ROOT/{asdl/runtime,asdl/format,core/ansi,pylib/cgi,qsn_/qsn}.py \
@@ -371,7 +371,7 @@ inline Str* repr(void* obj) {
 }
 
 EOF
-    cat _tmp/mycpp/runtime.h
+    cat $TMP/runtime.h
   } > asdl/runtime.h
 
   cat $raw > $cc
@@ -382,7 +382,7 @@ EOF
 port-os-path() {
 
   local name=os_path
-  local raw=_tmp/mycpp/${name}_raw.cc 
+  local raw=$TMP/${name}_raw.cc 
 
   mycpp \
     $REPO_ROOT/pylib/os_path.py \

devtools/release.sh

@@ -97,6 +97,8 @@ auto-machine1() {
   $0 test-opy
   $0 spec-all
   $0 benchmark-run
+
+  $0 mycpp-examples
 }
 
 # TODO:
@@ -474,18 +476,27 @@ compress-benchmarks() {
   # Technically we only need index.html.  But it's nice to have stage1 and
   # stage2 in case we need backup.
 
+  # note: mycpp-benchmarks only run on one machine
   pushd _tmp
   find \
     compute/{stage1,stage2,index.html} \
     osh-parser/{stage1,stage2,index.html} \
     osh-runtime/{stage1,stage2,index.html} \
     vm-baseline/{stage1,stage2,index.html} \
     ovm-build/{stage1,stage2,index.html} \
+    mycpp-examples/{stage2,index.html} \
     -type f \
     | xargs --verbose -- zip -q $out 
   popd
 }
 
+mycpp-examples() {
+  ### Single machine benchmarks that show our GC progress
+  mycpp/setup.sh build-examples
+  mycpp/setup.sh test-examples
+  mycpp/setup.sh benchmark-examples
+}
+
 line-counts() {
   local out=$1  # should be an absolute path
   mkdir -p $out

doc/release-index.md

@@ -122,6 +122,8 @@ of Oil's C++ translation.
   build Oil?  How big is the resulting binary?
 - [Virtual Memory Baseline](benchmarks.wwz/vm-baseline/).  How much memory do
   shells use at startup?
+- [mycpp](benchmarks.wwz/mycpp-examples/).  Compares Python and generated C++
+  on small examples.
 
 ## Metrics
 

frontend/syntax.asdl

@@ -92,10 +92,8 @@ module syntax
 
   -- After parsing, we often don't need the token 'string val', so use a
   -- 'speck'.  Notes:
-  -- * This structure could be packed into (2 + 2 + 4) = 8 bytes.
-  -- * If we had a step between parsing and execution (i.e. compilation), we could
-  --   just use specks as tokens.  We care about 'string val' when they're made
-  --   literal parts of words; variable, function, and parameter names; etc.
+  -- * It's 12 bytes with the GC header
+  -- * It's a "slab" to the GC (no pointers to follow)
   speck = (id id, int span_id)
 
   -- Note: like token -> speck, token -> strand is for when we don't need the id.
@@ -305,7 +303,7 @@ module syntax
   | VarDecl(Token? keyword, name_type* lhs, expr rhs)
     -- setvar/set, auto.  Note: op can be a speck
   | PlaceMutation(Token? keyword, place_expr* lhs, Token op, expr rhs)
-    -- do, pp, pass.  An expression for its side effects
+    -- = keyword
   | Expr(speck keyword, expr e)
     -- return an Obj, not an int-like string
   | Return(Token keyword, expr e)
@@ -404,7 +402,6 @@ module syntax
     Var(Token name)  -- a variable name to evaluate
     -- For null, Bool, Int, Float
     -- Python uses Num(object n), which doesn't respect our "LST" invariant.
-    -- speck?
   | Const(Token c)
     -- @(one 'two' "$three")
   | ShArrayLiteral %sh_array_literal

mycpp/harness.sh

@@ -227,7 +227,7 @@ benchmark-all() {
 
   export BENCHMARK=1
 
-  local out_dir=../_tmp/mycpp-benchmarks/raw
+  local out_dir=../_tmp/mycpp-examples/raw
   mkdir -p $out_dir
   local out=$out_dir/times.tsv
 

mycpp/setup.sh

@@ -62,5 +62,11 @@ test-examples() {
   ./run.sh test-all
 }
 
+benchmark-examples() {
+  ### Benchmark all mycpp/examples
+
+  cd $THIS_DIR
+  ./run.sh benchmark-all
+}
 
 "$@"