Changes for release 0.5.alpha2, which now uses bytecode-opy.zip.

- oil-version.txt Bump version to 0.5.alpha2.
- Makefile: Switch to OPy bytecode.

- Consolidate release preparation and benchmark preparation in
  scripts/release.sh.
  - 'benchmarks/auto.sh all' and 'scripts/release.sh build-and-test' did
    a lot of the same things, with slight inconsistencies.  There are
    new functions 'benchmark-build', 'benchmark-run', and
    'benchmark-run-on-1-machine' (oheap).
  - The benchmarks can now be run with _bin/osh or
    _tmp/oil-tar-test/oil-$VERSION/_bin/osh.  I turned _bin/osh into a
    common $OSH_OVM constant.

- Fix various bugs in the ovm-build.sh benchmark
  - Measure the size of the right bytecode.
  - Fix bug with stale bytecode
  - Fix omission of writing stage2/times.tsv.

INSTALL.txt

@@ -12,8 +12,8 @@ Quick Start

If you haven't already done so, extract the tarball:
 
    tar -x --xz < oil-0.5.alpha1.tar.xz
    cd oil-0.5.alpha1
    tar -x --xz < oil-0.5.alpha2.tar.xz
    cd oil-0.5.alpha2

Either install as /usr/local/bin/osh:

@@ -31,7 +31,7 @@ The latter doesn't require root access, but it requires ~/bin to be in your
PATH.

NOTE: Out-of-tree builds are NOT currently supported, so you have to be in the
oil-0.5.alpha1 directory.
oil-0.5.alpha2 directory.

Smoke Test
----------

Makefile

@@ -80,8 +80,8 @@ ACTIONS_SH := build/actions.sh
COMPILE_SH := build/compile.sh

# Change the bytecode compiler here.
BYTECODE_ZIP := bytecode-cpython.zip
#BYTECODE_ZIP := bytecode-opy.zip
#BYTECODE_ZIP := bytecode-cpython.zip
BYTECODE_ZIP := bytecode-opy.zip

# For faster tesing of builds
#default: _bin/oil.ovm-dbg

benchmarks/auto.sh

@@ -54,33 +54,9 @@ measure-builds() {
}

# Run the whole benchmark from a clean git checkout.
#
# Similar to scripts/release.sh build-and-test.
all() {
  test/spec.sh install-shells

  # Technically we need build-essential too?
  sudo apt install python-dev

  # Ideally I wouldn't need this, but the build process is not great now.
  make clean

  build/dev.sh all

  _banner 'OSH dev build'
  bin/osh -c 'echo OSH dev build'

  build/prepare.sh configure
  build/prepare.sh build-python

  make _bin/oil.ovm
  # This does what 'install' does.
  scripts/run.sh make-bin-links

  _banner 'OSH production build'

  _bin/osh -c 'echo OSH production build'
# Before this, run scripts/release.sh benchmark-build.

all() {
  measure-shells
  measure-builds
}

benchmarks/common.sh

@@ -3,6 +3,9 @@
# Common functions for benchmarks.
#

# What binary the benchmarks will run.
readonly OSH_OVM=${OSH_OVM:-$PWD/_bin/osh}

# NOTE: This is in {build,test}/common.sh too.
die() {
  echo "FATAL: $@" 1>&2

benchmarks/id.sh

@@ -75,7 +75,7 @@ dump-shell-id() {

  # Add extra repository info for osh.
  case $sh in
    bin/osh|_bin/osh)
    */osh)
      local branch
      branch=$(git rev-parse --abbrev-ref HEAD)
      echo $branch > $out_dir/git-branch.txt
@@ -317,7 +317,7 @@ shell-provenance() {

  local shell_hash

  for sh_path in bash dash mksh zsh bin/osh _bin/osh; do
  for sh_path in bash dash mksh zsh bin/osh $OSH_OVM; do
    # There will be two different OSH
    local name=$(basename $sh_path)

benchmarks/oheap.sh

@@ -17,7 +17,7 @@ readonly BASE_DIR=_tmp/oheap
encode-one() {
  local script=$1
  local oheap_out=$2
  bin/osh -n --ast-format oheap "$script" > $oheap_out
  $OSH_OVM -n --ast-format oheap "$script" > $oheap_out
}

task-spec() {

benchmarks/osh-runtime.sh

@@ -15,10 +15,6 @@ source benchmarks/common.sh  # csv-concat
readonly BASE_DIR=_tmp/osh-runtime
readonly TAR_DIR=$PWD/_deps/osh-runtime  # Make it absolute

# Use the compiled version.  Otherwise /proc/self/exe is the Python
# interpreter, which matters for yash's configure script!
readonly OSH=$PWD/_bin/osh

#
# Dependencies
#
@@ -67,7 +63,7 @@ cpython-configure() {

# 18.9 seconds vs 11 seconds above.
osh-cpython-configure() {
  cpython-configure $PWD/_bin/osh $BASE_DIR/osh-cpython-configure
  cpython-configure $OSH_OVM $BASE_DIR/osh-cpython-configure
}

runtime-task() {
@@ -330,7 +326,7 @@ abuild-h() {
  local out=$out_dir/abuild-h-times.csv

  echo 'status,elapsed_secs,sh_path' > $out
  for sh_path in bash dash mksh zsh $OSH; do
  for sh_path in bash dash mksh zsh $OSH_OVM; do
    benchmarks/time.py --output $out --field "$sh_path" -- \
      $sh_path benchmarks/testdata/abuild -h
  done
@@ -344,7 +340,7 @@ abuild-h() {
qemu-old() {
  local out_dir=$PWD/_tmp/qemu-old
  mkdir -p $out_dir
  configure-and-copy ~/src/qemu-1.6.0 $OSH $out_dir
  configure-and-copy ~/src/qemu-1.6.0 $OSH_OVM $out_dir
}

# This doesn't work for ash either, because it uses the busybox pattern.  It

benchmarks/ovm-build.sh

@@ -60,22 +60,21 @@ download() {
    wget --directory $TAR_DIR 'https://www.oilshell.org/blob/ovm-build/{}'
}

# Done MANUALLY.
extract-other() {
  time for f in $TAR_DIR/*gz; do
    tar -x --directory $TAR_DIR --file $f 
  done
}

# Done automatically by 'measure' function.
#
# NOTE: We assume that _release/oil.tar exists.  It should be made by
# scripts/release.sh build-and-test or benchmark-build.
extract-oil() {
  local target=_release/oil.tar
  rm -f -v $target
  make $target
  tar -x --directory $TAR_DIR --file $target
}

extract() {
  extract-oil
  extract-other
  # This is different than the others tarballs.
  rm -r -f -v $TAR_DIR/oil-*
  tar -x --directory $TAR_DIR --file _release/oil.tar
}

#
@@ -102,7 +101,7 @@ measure-sizes() {
  # PROBLEM: Do I need provenance for gcc/clang here?  I can just join it later
  # in R.

  sizes-tsv $TAR_DIR/oil-$OIL_VERSION/_build/oil/bytecode.zip \
  sizes-tsv $TAR_DIR/oil-$OIL_VERSION/_build/oil/bytecode-opy.zip \
    > ${prefix}.bytecode-size.tsv

  sizes-tsv $BASE_DIR/bin/*/oil.* \
@@ -221,8 +220,8 @@ build-task() {
oil-tasks() {
  local provenance=$1

  # NOTE: it MUST be a tarball and not the git repo, because we do the build
  # of bytecode.zip!  We care about the "package experience".
  # NOTE: it MUST be a tarball and not the git repo, because we don't build
  # bytecode-*.zip!  We care about the "packager's experience".
  local dir="$TAR_DIR/oil-$OIL_VERSION"

  # Add 1 field for each of 5 fields.
@@ -271,7 +270,7 @@ measure() {
  local provenance=$1  # from benchmarks/id.sh compiler-provenance
  local raw_dir=${2:-$BASE_DIR/raw}

  #local base_dir=${2:-../benchmark-data/osh-parser}
  extract-oil

  # Job ID is everything up to the first dot in the filename.
  local name=$(basename $provenance)

benchmarks/report.R

@@ -417,13 +417,14 @@ OvmBuildReport = function(in_dir, out_dir) {
    bytecode_size

  bin_sizes %>%
	  # reorder
		select(c(host_label, path, num_bytes)) %>%
		left_join(bytecode_size, by = c('host_label')) %>%
		mutate(native_code_size = num_bytes - bytecode_size) ->
  	sizes
    # reorder
    select(c(host_label, path, num_bytes)) %>%
    left_join(bytecode_size, by = c('host_label')) %>%
    mutate(native_code_size = num_bytes - bytecode_size) ->
    sizes

  # NOTE: These don't have the host and compiler.
  writeTsv(times, file.path(out_dir, 'times'))
  writeTsv(bytecode_size, file.path(out_dir, 'bytecode-size'))
  writeTsv(sizes, file.path(out_dir, 'sizes'))

build/test.sh

@@ -34,7 +34,7 @@ _tarball() {
  local name=${1:-hello}
  local version=${2:-0.0.0}

  local tmp=_tmp/$name-tar-test
  local tmp=_tmp/${name}-tar-test
  rm -r -f $tmp
  mkdir -p $tmp
  cd $tmp

doc/osh-quick-ref-toc.txt

@@ -2,7 +2,7 @@ OSH Quick Reference

- Below is a list of topics, organized into [Sections].
- The X prefix means "unimplemented".  Oil features are all unimplemented!
- HTML version: https://www.oilshell.org/release/0.5.alpha1/doc/osh-quick-ref.html
- HTML version: https://www.oilshell.org/release/0.5.alpha2/doc/osh-quick-ref.html

INTRO
  [Overview]      overview   osh-vs-oil   command-vs-expr

doc/release-index.md

@@ -1,6 +1,6 @@
<!-- NOTE: This file is at /release/$VERSION/index.html -->

Oil Version 0.5.alpha1
Oil Version 0.5.alpha2
-----------------

### What's New

oil-version.txt

@@ -1,4 +1,4 @@
0.5.alpha1
0.5.alpha2

# The first line of this file is the Oil version, and the rest is ignored.
# It's used at build time for the release tarball, and at runtime for oil