Automation changes for the 0.5 release.

- Bump the version to 0.5
- Allow running benchmarks without root
- Install more R package dependencies
- Add a function to install Ubuntu packages that are only needed for the
release
- Change benchmark machine names
- shell functions for release notes

benchmarks/common.sh

@@ -6,6 +6,12 @@
# What binary the benchmarks will run.
readonly OSH_OVM=${OSH_OVM:-$PWD/_bin/osh}

#readonly MACHINE1=flanders
#readonly MACHINE2=lisa

readonly MACHINE1=broome
readonly MACHINE2=crosby

# NOTE: This is in {build,test}/common.sh too.
die() {
  echo "FATAL: $@" 1>&2

benchmarks/osh-parser.sh

@@ -150,14 +150,14 @@ stage1() {
  mkdir -p $out

  local vm_csv=$out/virtual-memory.csv
  local -a x=($raw_dir/flanders.*.virtual-memory)
  local -a y=($raw_dir/lisa.*.virtual-memory)
  local -a x=($raw_dir/$MACHINE1.*.virtual-memory)
  local -a y=($raw_dir/$MACHINE2.*.virtual-memory)
  benchmarks/virtual_memory.py osh-parser ${x[-1]} ${y[-1]} > $vm_csv

  local times_csv=$out/times.csv
  # Globs are in lexicographical order, which works for our dates.
  local -a a=($raw_dir/flanders.*.times.csv)
  local -a b=($raw_dir/lisa.*.times.csv)
  local -a a=($raw_dir/$MACHINE1.*.times.csv)
  local -a b=($raw_dir/$MACHINE2.*.times.csv)
  csv-concat ${a[-1]} ${b[-1]} > $times_csv

  # Construct a one-column CSV file
@@ -169,8 +169,8 @@ stage1() {

  # Verify that the files are equal, and pass one of them.
  local lines_csv=$out/lines.csv
  local -a c=($raw_dir/flanders.*.lines.csv)
  local -a d=($raw_dir/lisa.*.lines.csv)
  local -a c=($raw_dir/$MACHINE1.*.lines.csv)
  local -a d=($raw_dir/$MACHINE2.*.lines.csv)

  local left=${c[-1]}
  local right=${d[-1]}

benchmarks/osh-runtime.sh

@@ -252,14 +252,14 @@ stage1() {

  local times_csv=$out_dir/times.csv
  # Globs are in lexicographical order, which works for our dates.
  local -a a=($raw_dir/flanders.*.times.csv)
  local -a b=($raw_dir/lisa.*.times.csv)
  local -a a=($raw_dir/$MACHINE1.*.times.csv)
  local -a b=($raw_dir/$MACHINE2.*.times.csv)
  csv-concat ${a[-1]} ${b[-1]} > $times_csv

  local vm_csv=$out_dir/virtual-memory.csv

  local -a c=($raw_dir/flanders.*.virtual-memory)
  local -a d=($raw_dir/lisa.*.virtual-memory)
  local -a c=($raw_dir/$MACHINE1.*.virtual-memory)
  local -a d=($raw_dir/$MACHINE2.*.virtual-memory)
  benchmarks/virtual_memory.py osh-runtime ${c[-1]} ${d[-1]} > $vm_csv

  #local raw_dir=${1:-../benchmark-data/osh-parser}

benchmarks/ovm-build.sh

@@ -316,18 +316,18 @@ stage1() {

  # Globs are in lexicographical order, which works for our dates.
  x=$out/times.tsv
  a=($raw_dir/flanders.*.times.tsv)
  b=($raw_dir/lisa.*.times.tsv)
  a=($raw_dir/$MACHINE1.*.times.tsv)
  b=($raw_dir/$MACHINE2.*.times.tsv)
  tsv-concat ${a[-1]} ${b[-1]} > $x

  x=$out/bytecode-size.tsv
  a=($raw_dir/flanders.*.bytecode-size.tsv)
  b=($raw_dir/lisa.*.bytecode-size.tsv)
  a=($raw_dir/$MACHINE1.*.bytecode-size.tsv)
  b=($raw_dir/$MACHINE2.*.bytecode-size.tsv)
  tsv-concat ${a[-1]} ${b[-1]} > $x

  x=$out/bin-sizes.tsv
  a=($raw_dir/flanders.*.bin-sizes.tsv)
  b=($raw_dir/lisa.*.bin-sizes.tsv)
  a=($raw_dir/$MACHINE1.*.bin-sizes.tsv)
  b=($raw_dir/$MACHINE2.*.bin-sizes.tsv)
  tsv-concat ${a[-1]} ${b[-1]} > $x

  # Construct a one-column TSV file

benchmarks/report.R

@@ -6,7 +6,7 @@
#   osh-parser.R OUT_DIR [TIMES_CSV...]

library(dplyr)
library(tidyr)
library(tidyr)  # spread()
library(stringr)

source('benchmarks/common.R')

benchmarks/vm-baseline.sh

@@ -61,8 +61,8 @@ stage1() {
  mkdir -p $out

  # Globs are in lexicographical order, which works for our dates.
  local -a m1=(../benchmark-data/vm-baseline/flanders.*)
  local -a m2=(../benchmark-data/vm-baseline/lisa.*)
  local -a m1=(../benchmark-data/vm-baseline/$MACHINE1.*)
  local -a m2=(../benchmark-data/vm-baseline/$MACHINE2.*)

  # The last one
  local -a latest=(${m1[-1]} ${m2[-1]})

build/dev.sh

@@ -21,18 +21,25 @@ ubuntu-deps() {
  # gawk: used by spec-runner.sh for the special match() function.
  # time: used to collect the exit code and timing of a test
  # libreadline-dev: needed for the build/prepare.sh Python build.
  sudo apt-get install python-dev gawk time libreadline-dev
  sudo apt install python-dev gawk time libreadline-dev

  test/spec.sh install-shells

  # TODO: For the release to run test/report.R, you need r-base-core too.
}

# Needed for the release process, but not the dev process.
release-ubuntu-deps() {
  # For the release to run test/report.R, you need r-base-core too.
  # cloc is used for line counts
  # TODO: switch to CommonMark rather than using markdown.pl.
  sudo apt install r-base-core cloc markdown
}

r-packages() {
  # Install to a directory that doesn't require root.  This requires setting
  # R_LIBS_USER.  Or library(dplyr, lib.loc = "~/R", but the former is preferable.
  mkdir -p ~/R
  INSTALL_DEST=$R_PATH Rscript -e 'install.packages(c("dplyr"), lib=Sys.getenv("INSTALL_DEST"), repos="http://cran.us.r-project.org")'
  INSTALL_DEST=$R_PATH Rscript -e 'install.packages(c("dplyr", "tidyr", "stringr"), lib=Sys.getenv("INSTALL_DEST"), repos="http://cran.us.r-project.org")'
}

test-r-packages() {

scripts/release.sh

@@ -177,7 +177,9 @@ _install() {

# Run before benchmarks/auto.sh all.  We just build, and assume we tested.
benchmark-build() {
  _install
  if test -n "$HAVE_ROOT"; then
    _install
  fi
  _clean
  _dev-build

@@ -376,6 +378,11 @@ git-changelog-0.5.alpha3() {
    > _release/VERSION/changelog.html
}

git-changelog-0.5.0() {
  # NOTE: release/0.5 branch should be sync'd up with master squashes.
  _git-changelog release/0.5.0 release/0.5.alpha3 \
    > _release/VERSION/changelog.html
}

# For announcement.html
html-redirect() {
@@ -438,6 +445,10 @@ announcement-0.5.alpha3() {
  html-redirect '/blog/2018/04/30.html' > _release/VERSION/announcement.html
}

announcement-0.5() {
  html-redirect '/blog/2018/07/12.html' > _release/VERSION/announcement.html
}

_link() {
  ln -s -f -v --no-target-directory "$@"
}