The ovm-build.sh benchmark now runs.

It times debug and release builds of Oil with Clang and GCC. It also measures the build time of dash and bash (with GCC only). It downloads tarballs from oilshell.org/blob/ovm-build. Also, fix 'make clean' in the release tarball. Addresses issue #41.
oilshell · Feb 16, 2018 · 297593f1b31040b5b2d8900374f1bdc4515d21af · 297593f
1 parent 0834861
commit 297593f1b31040b5b2d8900374f1bdc4515d21af
Unified Split

Showing with 151 additions and 61 deletions.

+6 −1 Makefile

+8 −14 benchmarks/osh-runtime.sh

+128 −44 benchmarks/ovm-build.sh

+9 −2 build/actions.sh
diff --git a/Makefile b/Makefile
@@ -85,8 +85,13 @@ all: \
 	_bin/hello.ovm-dbg _bin/oil.ovm-dbg \
 	_release/hello.tar _release/oil.tar

+# For the release tarball.
 clean:
-	$(ACTIONS_SH) clean
+	$(ACTIONS_SH) clean-source-tarball-build
+
+# For developers in a repo.
+clean-repo:
+	$(ACTIONS_SH) clean-repo

 .PHONY: default all clean install

diff --git a/benchmarks/osh-runtime.sh b/benchmarks/osh-runtime.sh
@@ -23,26 +23,20 @@ readonly OSH=$PWD/_bin/osh
 # Dependencies
 #

+readonly -a TAR_SUBDIRS=( ocaml-4.06.0 tcc-0.9.26 yash-2.46 )
+
 # NOTE: Same list in oilshell.org/blob/run.sh.
-files() {
+tarballs() {
  cat <<EOF
 tcc-0.9.26.tar.bz2
 yash-2.46.tar.xz
 ocaml-4.06.0.tar.xz
 EOF
 }

-conf-dirs() {
-  cat <<EOF
-$TAR_DIR/ocaml-4.06.0
-$TAR_DIR/tcc-0.9.26
-$TAR_DIR/yash-2.46
-EOF
-}
-
 download() {
  mkdir -p $TAR_DIR
-  files | xargs -n 1 -I {} --verbose -- \
+  tarballs | xargs -n 1 -I {} --verbose -- \
    wget --directory $TAR_DIR 'https://www.oilshell.org/blob/testdata/{}'
 }

@@ -97,9 +91,8 @@ runtime-task() {
  local files_out_dir="$PWD/$raw_dir/$host.$job_id.files/$task_label"
  mkdir -p $vm_out_dir $files_out_dir

-  local time_tool=$PWD/benchmarks/time.py
  local -a TIME_PREFIX=(
-    $time_tool \
+    $PWD/benchmarks/time.py \
    --output $times_out \
    --field "$host" --field "$host_hash" \
    --field "$shell_name" --field "$shell_hash" \
@@ -218,8 +211,9 @@ print-tasks() {
    echo "$prefix" abuild abuild-help
    echo "$prefix" cpython cpython-configure

-    conf-dirs | xargs -n 1 -- echo "$prefix" configure
-
+    for dir in "${TAR_SUBDIRS[@]}"; do
+      echo "$prefix" configure $TAR_DIR/$dir
+    done
  done
 }

diff --git a/benchmarks/ovm-build.sh b/benchmarks/ovm-build.sh
@@ -5,13 +5,64 @@
 # Usage:
 #   ./ovm-build.sh <function name>

+# Directories used:
+#
+# _tmp/
+#   ovm-build/  
+#     raw/     # output CSV
+#     stage1
+# _deps/
+#   ovm-build  # tarballs and extracted source
+
 set -o nounset
 set -o pipefail
 set -o errexit

+source benchmarks/common.sh  # for log, etc.
 source build/common.sh  # for $CLANG

 readonly BASE_DIR=_tmp/ovm-build
+readonly TAR_DIR=$PWD/_deps/ovm-build # Make it absolute
+
+#
+# Dependencies
+#
+
+readonly OIL_VERSION=$(head -n 1 oil-version.txt)
+
+# Leave out mksh for now, because it doesn't follow ./configure make.  It just
+# has Build.sh.
+readonly -a TAR_SUBDIRS=( bash-4.4 dash-0.5.9.1 )  # mksh )
+
+# NOTE: Same list in oilshell.org/blob/run.sh.
+tarballs() {
+  cat <<EOF
+bash-4.4.tar.gz
+dash-0.5.9.1.tar.gz
+mksh-R56c.tgz
+EOF
+}
+
+download() {
+  mkdir -p $TAR_DIR
+  tarballs | xargs -n 1 -I {} --verbose -- \
+    wget --directory $TAR_DIR 'https://www.oilshell.org/blob/ovm-build/{}'
+}
+
+extract-other() {
+  time for f in $TAR_DIR/*gz; do
+    tar -x --directory $TAR_DIR --file $f 
+  done
+  ls -l $TAR_DIR
+}
+
+extract-oil() {
+  local target=_release/oil.tar
+  rm -f -v $target
+  make $target
+  tar -x --directory $TAR_DIR --file $target
+  ls -l $TAR_DIR
+}

 # NOTE: build/test.sh measures the time already.

@@ -23,7 +74,6 @@ readonly BASE_DIR=_tmp/ovm-build
 #   host: lisa or flanders
 #   target: oil.ovm vs. oil.ovm-dbg
 #   Then do benchmarks/time.py of "make CC=$CC"
-#   make clean between?
 # 
 # Measure:
 #   bytecode.zip size vs. ovm size
@@ -63,21 +113,11 @@ binary-size() {
 }

 # 6.8 seconds for debug build, instead of 8 seconds.
-clang() {
+clang-oil-dbg() {
  make clean
  CC=$CLANG make _build/oil/ovm-dbg
 }

-# TODO: Follow pattern in benchmarks/osh-{runtime,parser} ?
-# You get provenance.txt, and
-# But that is measuring against bash and dash BINARIES.  At the very least, we
-# want to measure bash, dash, mksh compile times.
-
-# target is _build/oil/ovm
-#
-# So we need another provenance function.  Instead of host/shell name/hash, we
-# need host/compiler name/hash.
-
 # Add --target-size?  Add that functionality to benchmarks/time.py?
 #
 # Should we add explicit targets?
@@ -86,43 +126,67 @@ clang() {
 #
 # It would be possible, but it complicates the makefile.

-#readonly HEADER='status,elapsed_secs,host_name,host_hash,compiler_name,compiler_hash,tarball,target,target_num_bytes'
-
-# 5 releases: 0.0.0 to 0.4.0.  Or we could just do the 0.5.alpha1 release?
-# Then you can show the drop.
-oil-historical() {
-  echo 
-}
-
-# maybe just ./configure and compile each shell with GCC?  Just for a ballpark
-# comparison.  It doesn't have to get too detailed.
-# Put it in benchmark-data/saved ?
-# Multiple hosts are useful though.  I may change hosts.  I might want to run
-# it again on Clang upgrades?  As a control?
-
-other-shells() {
-  echo
-}
-
 build-task() {
  local raw_dir=$1  # output
  local job_id=$2
  local host=$3
  local host_hash=$4
  local compiler_path=$5
  local compiler_hash=$6
-  local tarball=$7
-  local target=$8
+  local src_dir=$7
+  local action=$8
+
+  local times_out="$PWD/$raw_dir/$host.$job_id.times.csv"
+
+  local -a TIME_PREFIX=(
+    $PWD/benchmarks/time.py \
+    --output $times_out \
+    --field "$host" --field "$host_hash" \
+    --field "$compiler_path" --field "$compiler_hash" \
+    --field "$src_dir" --field "$action"
+  )
+
+  pushd $src_dir >/dev/null
+
+  # NOTE: We're not saving the output anywhere.  We save the status, which
+  # protects against basic errors.
+
+  case $action in
+    configure)
+      # Cleaning here relies on the ORDER of tasks.txt.  configure happens
+      # before build.  The Clang build shouldn't reuse GCC objects!
+      make clean
+      "${TIME_PREFIX[@]}" -- ./configure
+      ;;
+    make)
+      "${TIME_PREFIX[@]}" -- make CC=$compiler_path
+      ;;
+    *)
+      # Assume it's a target.
+      "${TIME_PREFIX[@]}" -- make CC=$compiler_path $action
+      ;;
+  esac
+
+  popd >/dev/null
+}

-  # Really we should just measure "make", and then the ovm-dbg target can be
-  # separate?
-  # We also want to do ./configure.  Do that for bash/dash too.
+oil-tasks() {
+  local provenance=$1

-  # time them with benchmarks/time.py
-  echo TODO $tarball $target
+  # NOTE: it MUST be a tarball and not the git repo, because we do the build
+  # of bytecode.zip!  We care about the "package experience".
+  local dir="$TAR_DIR/oil-$OIL_VERSION"
+
+  # Add 1 field for each of 5 fields.
+  cat $provenance | while read line; do
+    # NOTE: configure is independent of compiler.
+    echo "$line" $dir configure
+    echo "$line" $dir _bin/oil.ovm
+    echo "$line" $dir _bin/oil.ovm-dbg
+  done
 }

-print-tasks() {
+other-shell-tasks() {
  local provenance=$1

  # NOTE: it MUST be a tarball and not the git repo, because we do the build
@@ -131,13 +195,28 @@ print-tasks() {

  # Add 1 field for each of 5 fields.
  cat $provenance | while read line; do
-    echo "$line" $tarball _build/oil/ovm
-    echo "$line" $tarball _build/oil/ovm-dbg
+    case $line in
+      # Skip clang for now.
+      *clang*)
+        continue
+        ;;
+    esac
+
+    for dir in "${TAR_SUBDIRS[@]}"; do
+      echo "$line" $TAR_DIR/$dir configure
+      echo "$line" $TAR_DIR/$dir make
+    done
  done
 }

+# 5 releases: 0.0.0 to 0.4.0.  For now, just do the 0.5.alpha1 release, and
+# show the drop.
+oil-historical-tasks() {
+  echo 
+}

-readonly HEADER='status,elapsed_secs,host_name,host_hash,compiler_path,compiler_hash,tarball,target'
+# action is 'configure', a target name, etc.
+readonly HEADER='status,elapsed_secs,host_name,host_hash,compiler_path,compiler_hash,src_dir,action'
 readonly NUM_COLUMNS=7  # 5 from provenence, then tarball/target

 measure() {
@@ -153,13 +232,18 @@ measure() {
  local times_out="$raw_dir/$prefix.times.csv"
  mkdir -p $BASE_DIR/{raw,stage1}

+  # TODO: the $times_out calculation is duplicated in build-task()0
+
  # Write Header of the CSV file that is appended to.
  echo $HEADER > $times_out

-  local tasks=$BASE_DIR/tasks.txt
-  print-tasks $provenance > $tasks
+  local t1=$BASE_DIR/oil-tasks.txt
+  local t2=$BASE_DIR/other-shell-tasks.txt
+
+  oil-tasks $provenance > $t1
+  other-shell-tasks $provenance > $t2

-  time cat $tasks |
+  time cat $t1 $t2 |
    xargs -n $NUM_COLUMNS -- $0 build-task $raw_dir ||
    die "*** Some tasks failed. ***"

diff --git a/build/actions.sh b/build/actions.sh
@@ -176,10 +176,17 @@ clean-pyc() {

 # NOTE: Not deleting cpython-full.  Maybe we should, or we should put it in a
 # diffent directory?
-clean() {
+clean-repo() {
+  clean-source-tarball-build
  rm -r -f _build _release
-  rm -f _bin/oil.* _bin/hello.*
+  rm -f _bin/hello.*
  clean-pyc
 }

+# 'make clean'.  NOTE: There are no object files written now.
+clean-source-tarball-build() {
+  rm -f -v _bin/oil.* 
+  rm -f -v _build/oil/ovm _build/oil/ovm-dbg
+}
+
 "$@"