[benchmarks/compute] Fold in benchmarks/parse-help

- Bubble sort can take different sizes - Try to tickle quadratic behavior in arrays, but failed. I think their randomness works. Somebody probably patched that.
oilshell · Aug 4, 2020 · f32bc9a · f32bc9a
1 parent 2d9e81a
commit f32bc9a
Show file tree

Hide file tree

Showing 3 changed files with 131 additions and 46 deletions.
diff --git a/benchmarks/compute.sh b/benchmarks/compute.sh
@@ -60,14 +60,40 @@ EOF
 }
 
 bubble_sort-tasks() {
+  # Note: this is quadratic, but bubble sort itself is quadratic!
+
+  cat <<EOF
+bubble_sort python   int   200
+bubble_sort python   bytes 200
+bubble_sort bash     int   200
+bubble_sort bash     bytes 200
+bubble_sort $OSH_CC  int   200
+bubble_sort $OSH_CC  bytes 200
+EOF
+}
+
+# Arrays are doubly linked lists in bash!  With a LASTREF hack to avoid being
+# quadratic.  
+#
+# See array_reference() in array.c in bash.  It searches both back and
+# forward.  Every cell has its index, a value, a forward pointer, and a back
+# pointer.
+
+array_ref-tasks() {
   cat <<EOF
-bubble_sort python   int
-bubble_sort python   bytes
-bubble_sort bash     int
-bubble_sort bash     bytes
-bubble_sort $OSH_CC  int
-bubble_sort $OSH_CC  bytes
+array_ref bash     seq    5000
+array_ref bash     seq    7000
+array_ref bash     seq    9000
+array_ref bash     random 5000
+array_ref bash     random 7000
+array_ref bash     random 9000
 EOF
+
+#array_ref $OSH_CC  seq    5000
+#array_ref $OSH_CC  seq    10000
+#array_ref $OSH_CC  random 5000
+#array_ref $OSH_CC  random 10000
+#EOF
 }
 
 palindrome-tasks() {
@@ -81,6 +107,17 @@ palindrome $OSH_CC  bytes
 EOF
 }
 
+parse-help-tasks() {
+  cat <<EOF
+parse-help bash     ls-short
+parse-help bash     ls
+parse-help bash     mypy
+parse-help $OSH_CC  ls-short
+parse-help $OSH_CC  ls
+parse-help $OSH_CC  mypy
+EOF
+}
+
 # We also want:
 #   status, elapsed
 #   host_name, host_hash (we don't want to test only fast machines)
@@ -131,9 +168,22 @@ bubble_sort-one() {
   local name=${1:-bubble_sort}
   local runtime=$2
   local mode=${3:-int}
+  local n=${4:-100}
 
   $runtime benchmarks/compute/bubble_sort.$(ext $runtime) $mode \
-     < _tmp/compute/$name/testdata.txt
+     < _tmp/compute/$name/testdata-$n.txt
+}
+
+# OSH is like 10x faster here!
+array_ref-one() {
+  ### Run one array_ref task (arrays)
+
+  local name=${1:-bubble_sort}
+  local runtime=$2
+  local mode=${3:-seq}
+  local n=${4:-100}
+
+  seq $n |shuf| $runtime benchmarks/compute/array_ref.$(ext $runtime) $mode
 }
 
 palindrome-one() {
@@ -147,6 +197,17 @@ palindrome-one() {
     < _tmp/compute/$name/testdata.txt
 }
 
+parse-help-one() {
+  ### Run one palindrome task (strings, real code)
+
+  local name=${1:-parse-help}
+  local runtime=$2
+  local workload=${3:-}
+
+  $runtime benchmarks/parse-help/pure-excerpt.sh _parse_help - \
+    < benchmarks/parse-help/$workload.txt
+}
+
 #
 # Helpers
 #
@@ -158,9 +219,14 @@ word_freq-all() { task-all word_freq; }
 # completes quickly.
 bubble_sort-all() { task-all bubble_sort; }
 
+# Array that is not quadratic
+array_ref-all() { task-all array_ref; }
+
 # Hm osh is a little slower here
 palindrome-all() { task-all palindrome; }
 
+parse-help-all() { task-all parse-help; }
+
 task-all() {
   local name=$1
 
@@ -211,8 +277,13 @@ task-all() {
 bubble_sort-testdata() {
   local out='_tmp/compute/bubble_sort'
   mkdir -p $out
-  seq 200 | shuf > $out/testdata.txt
-  wc -l $out/testdata.txt
+
+  # TODO: Make these deterministic for more stable benchmarks?
+  for n in 100 200 300 400; do
+    seq $n | shuf > $out/testdata-$n.txt
+  done
+
+  wc -l $out/testdata-*.txt
 }
 
 palindrome-testdata() {

diff --git a/benchmarks/compute/array_ref.sh b/benchmarks/compute/array_ref.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+#
+# Shows some slight superlinear behavior in bash ararys?
+#
+# Usage:
+#   ./reverse_sum.sh <function name>
+
+set -o nounset
+set -o pipefail
+set -o errexit
+
+main() {
+  local mode=$1
+
+  mapfile -t array
+
+  local n=${#array[@]}
+  local i=$((n-1))
+  local sum=0
+
+  case $mode in
+    linear)
+      while test $i -ge 0; do
+        sum=$((sum + array[i]))
+        i=$((i - 1))
+      done
+      ;;
+
+    random)
+      while test $i -ge 0; do
+        # Super linear
+        sum=$((sum + array[array[i]]))
+        i=$((i - 1))
+      done
+      ;;
+  esac
+  echo sum=$sum
+
+
+  # This doesn't seem to defeat LASTREF?
+  #array+=('X')
+  #unset 'array[-1]'
+
+  # neither does this
+  #array[i]=$i
+}
+
+main "$@"
diff --git a/benchmarks/parse-help.sh b/benchmarks/parse-help.sh
@@ -2,8 +2,10 @@
 #
 # A pure string-processing benchmark extracted from bash-completion.
 #
+# Note: most stuff moved to benchmarks/compute.
+#
 # Usage:
-#   ./parse-help.sh <function name>
+#   benchmarks/parse-help.sh <function name>
 
 set -o nounset
 set -o pipefail
@@ -28,18 +30,6 @@ shorten() {
 
 TIMEFORMAT='%U'
 
-run-cmd() {
-  local sh=$1
-  local cmd=$2
-  # read from stdin
-
-  local prog=$EXCERPT
-  # our pure fork
-  local prog=benchmarks/parse-help/pure-excerpt.sh
-
-  time cat $DATA_DIR/$cmd.txt | $sh $prog _parse_help -
-}
-
 # Geez:
 #        ls     mypy
 # bash   25ms   25ms
@@ -52,30 +42,6 @@ run-cmd() {
 # - count the number of printf invocations.  But you have to do it recursively!
 # - Turn this into a proper benchmark with an HTML page.
 
-all() {
-  wc -l $DATA_DIR/*
-
-  local dir=_tmp/parse-help
-  mkdir -p $dir
-
-  # Woohoo oil-native is faster!
-  OSH_CC=_bin/osh_eval.opt.stripped
-  for sh in bash bin/osh $OSH_CC; do
-    echo
-    echo "--- $sh --- "
-    echo
-
-    for cmd in ls-short ls mypy; do
-      local out=$dir/${cmd}__$(basename $sh).txt
-
-      printf '%10s ' $cmd
-      run-cmd $sh $cmd >$out
-    done
-  done
-
-  md5sum $dir/*
-}
-
 one() {
   local sh='bin/osh'
   local cmd='ls-short'