Preparing to benchmark compile time and binary size.

- Draft of benchmarks/ovm-build.sh
- Bump version to 0.5.alpha1.
- consolidate definitions of $CLANG and upgrade them to Clang 5
- use asdl.LoadSchema in asdl/gen_cpp.py

Other:

- count OPy lines more precisely

INSTALL.txt

@@ -12,8 +12,8 @@ Quick Start

If you haven't already done so, extract the tarball:
 
    tar -x --xz < oil-0.4.0.tar.xz
    cd oil-0.4.0
    tar -x --xz < oil-0.5.alpha1.tar.xz
    cd oil-0.5.alpha1

Either install as /usr/local/bin/osh:

@@ -31,7 +31,7 @@ The latter doesn't require root access, but it requires ~/bin to be in your
PATH.

NOTE: Out-of-tree builds are NOT currently supported, so you have to be in the
oil-0.4.0 directory.
oil-0.5.alpha1 directory.

Smoke Test
----------

asdl/gen_cpp.py

@@ -30,6 +30,8 @@
from asdl import asdl_ as asdl
from asdl import encode

from osh.meta import Id

TABSIZE = 2
MAX_COL = 80

@@ -419,19 +421,14 @@ def main(argv):
  if action == 'cpp':
    schema_path = argv[2]

    # TODO: Use asdl.LoadSchema here.
    app_types = {'id': asdl.UserType(Id)}
    with open(schema_path) as input_f:
      module = asdl.parse(input_f)
      module, type_lookup = asdl.LoadSchema(input_f, app_types)

    # TODO: gen_cpp.py should be a library and the application should add Id?
    # Or we should enable ASDL metaprogramming, and let Id be a metaprogrammed
    # simple sum type.

    from core.id_kind import Id
    app_types = {'id': asdl.UserType(Id)}

    type_lookup = asdl.ResolveTypes(module, app_types)

    f = sys.stdout

    # How do mutation of strings, arrays, etc.  work?  Are they like C++

asdl/run.sh

@@ -9,6 +9,8 @@ set -o nounset
set -o pipefail
set -o errexit

source build/common.sh  # for clang

export PYTHONPATH=.

asdl-arith-encode() {
@@ -92,11 +94,9 @@ osh-both() { py-cpp osh/osh.asdl; }
# Native Code
#

readonly CLANG=~/install/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang++

cxx() {
  #local CXX=c++ 
  local CXX=$CLANG
  local CXX=$CLANGXX
  local opt_flag='-O2'
  local opt_flag='-O0'

@@ -193,15 +193,15 @@ a4() {
disassemble() {
  local opt_flag=${1:-'-O0'}
  local out=_tmp/arith_demo$opt_flag.S 
  $CLANG -std='c++11' $opt_flag -I _tmp -o $out -S \
  $CLANGXX -std='c++11' $opt_flag -I _tmp -o $out -S \
    -mllvm --x86-asm-syntax=intel asdl/arith_demo.cc
  #cat $out
}

llvm() {
  local opt_flag=${1:-'-O0'}
  local out=_tmp/arith_demo$opt_flag.ll 
  $CLANG -std='c++11' $opt_flag -I _tmp -o $out -S \
  $CLANGXX -std='c++11' $opt_flag -I _tmp -o $out -S \
    -emit-llvm asdl/arith_demo.cc
  #cat $out
}

benchmarks/auto.sh

@@ -147,6 +147,11 @@ all() {
  record-provenance $job_id > $provenance

  measure-all $provenance

  # TODO:
  # record-compiler-provenance
  # benchmarks/ovm-build.sh measure $compiler_prov $base_dir/ovm-build
  # Note this has to happen AFTER a tarball is built.
}

#

benchmarks/ovm-build.sh

@@ -0,0 +1,146 @@
#!/bin/bash
#
# Measure the time it takes to build a binary, and the size of the binary.
#
# Usage:
#   ./ovm-build.sh <function name>

set -o nounset
set -o pipefail
set -o errexit

source build/common.sh  # for $CLANG

# NOTE: build/test.sh measures the time already.

# Coarse Size and Time Benchmarks
# --------------------------------
# 
# RUN:
#   compiler: CC=gcc or CC=clang
#   host: lisa or flanders
#   target: oil.ovm vs. oil.ovm-dbg
#   Then do benchmarks/time.py of "make CC=$CC"
#   make clean between?
# 
# Measure:
#   bytecode.zip size vs. ovm size
#   (Forget about individual files for now)
#   end-to-end build time in seconds
# 
# After optimization:
#   ovm should be a lot smaller
#   build time should be lower, as long as you did the #if 0
# 
# LATER:
#   reduce the amount of code.
#   do more fine-grained coverage?  I don't think you necessarily need it to
# reduce code.  You can do it by COMPILE TIME slicing, not runtime! 
# 
# I think doing it function-by-function at compile time is easier.  I need to
# modify Opy to spit out all references though?
#
# Other tools:
# - bloaty to look inside elf file
# - zipfile.py to look inside bytecode.zip

bytecode-size() {
  local zip=_build/oil/bytecode.zip

  # 242 files, 1.85 MB
  unzip -l $zip | tail -n 1 

  # 1.88 MB, so there's 30K of header overhead.
  ls -l $zip
}

# NOTE: ovm-dbg is not stripped, so it's not super meaningful.
binary-size() {
  make _build/oil/ovm{,-dbg}
  ls -l _build/oil/ovm{,-dbg}
}

# 6.8 seconds for debug build, instead of 8 seconds.
clang() {
  make clean
  CC=$CLANG make _build/oil/ovm-dbg
}

# TODO: Follow pattern in benchmarks/osh-{runtime,parser} ?
# You get provenance.txt, and
# But that is measuring against bash and dash BINARIES.  At the very least, we
# want to measure bash, dash, mksh compile times.

# target is _build/oil/ovm
#
# So we need another provenance function.  Instead of host/shell name/hash, we
# need host/compiler name/hash.

# Add --target-size?  Add that functionality to benchmarks/time.py?
#
# Should we add explicit targets?
#   - ovm-clang, ovm-clang-dbg
#   - ovm-gcc, ovm-gcc-dbg
#
# It would be possible, but it complicates the makefile.

readonly HEADER='status,elapsed_secs,host_name,host_hash,compiler_name,compiler_hash,tarball,target,target_num_bytes'

# I think I want to do this for every version.  Save it in
# ~/git/oil/benchmarks-data.

expand-tasks() {
  while read host_name compiler_name; do
    #local prefix="$job_id $host_name $host_hash $sh_path $shell_hash"
    local prefix="$host_name $compiler_name"

    # NOTE: it MUST be a tarball and not the git repo, because we do the build
    # of bytecode.zip!  We care about the "package experience".
    local tarball=_release/oil.tar

    echo "$prefix" $tarball _build/oil/ovm
    echo "$prefix" $tarball _build/oil/ovm-dbg
  done
}

# 5 releases: 0.0.0 to 0.4.0.  Or we could just do the 0.5.alpha1 release?
# Then you can show the drop.
oil-historical() {
  echo 
}

# maybe just ./configure and compile each shell with GCC?  Just for a ballpark
# comparison.  It doesn't have to get too detailed.
# Put it in benchmark-data/saved ?
# Multiple hosts are useful though.  I may change hosts.  I might want to run
# it again on Clang upgrades?  As a control?

other-shells() {
  echo
}

print-tasks() {
  local h=$(hostname) 
  { echo $h gcc;
    echo $h clang;
  } | expand-tasks
}

measure() {
  print-tasks
}

# TODO: Move to benchmarks/id.

gcc-hash() {
  #gcc --version
  gcc -v
}

clang-hash() {
  #$CLANG -v
  # -v has some output we don't want.
  $CLANG --version | grep -v InstalledDir
}

"$@"

build/common.sh

@@ -7,8 +7,9 @@ set -o errexit
# TODO: This changes depending on the version.  Maybe there should be a 'clang'
# function for things that really require clang, like code coverage and so
# forth.
readonly CLANG_DIR=~/install/clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-14.04
readonly CLANG_DIR=~/install/clang+llvm-5.0.1-x86_64-linux-gnu-ubuntu-16.04
readonly CLANG=$CLANG_DIR/bin/clang
readonly CLANGXX=$CLANG_DIR/bin/clang++

readonly CLANG_COV_FLAGS='-fprofile-instr-generate -fcoverage-mapping'
readonly CLANG_LINK_FLAGS=''

doc/osh-quick-ref-toc.txt

@@ -2,7 +2,7 @@ OSH Quick Reference

- Below is a list of topics, organized into [Sections].
- The X prefix means "unimplemented".  Oil features are all unimplemented!
- HTML version: https://www.oilshell.org/release/0.4.0/doc/osh-quick-ref.html
- HTML version: https://www.oilshell.org/release/0.5.alpha1/doc/osh-quick-ref.html

INTRO
  [Overview]      overview   osh-vs-oil   command-vs-expr

doc/release-index.md

@@ -1,6 +1,6 @@
<!-- NOTE: This file is at /release/$VERSION/index.html -->

Oil Version 0.4.0
Oil Version 0.5.alpha1
-----------------

### What's New

oil-version.txt

@@ -1,4 +1,4 @@
0.4.0
0.5.alpha1

# The first line of this file is the Oil version, and the rest is ignored.
# It's used at build time for the release tarball, and at runtime for oil 

opy/run.sh

@@ -337,12 +337,20 @@ for n in names:
# 8700 lines for tokenizer -> tokens -> parser -> homogeneous nodes ->
# transformer -> ast -> compiler -> byte code
count() {
  echo PARSER GENERATOR
  wc -l *.py pgen2/*.py | sort -n
  echo

  wc -l compiler2/*.py | sort -n
  # ast is generated
  echo COMPILER2
  ls compiler2/*.py | grep -v ast.py | xargs wc -l | sort -n
  echo

  echo GENERATED CODE
  wc -l compiler2/ast.py
  echo

  echo BYTERUN
  ls byterun/*.py | grep -v 'test' | xargs wc -l | sort -n
  echo
}