Publish all the new benchmarks.

- Put them in a .wwz file - Link to them from release/index.html - share benchmarks.css among all benchmarks
oilshell · Dec 9, 2017 · e081a57c2affc49d3df773045702a48cf1ccf10c · e081a57
1 parent 6a4683b
commit e081a57c2affc49d3df773045702a48cf1ccf10c
diff --git a/benchmarks/oheap.sh b/benchmarks/oheap.sh
@@ -98,16 +98,16 @@ stage1() {

 print-report() {
  local in_dir=$1
-  local base_url='../../web/table'
+  local base_url='../../web'

  cat <<EOF
 <!DOCTYPE html>
 <html>
  <head>
    <title>OHeap Encoding</title>
-    <script type="text/javascript" src="$base_url/table-sort.js"></script>
-    <link rel="stylesheet" type="text/css" href="$base_url/table-sort.css" />
-    <link rel="stylesheet" type="text/css" href="benchmarks.css" />
+    <script type="text/javascript" src="$base_url/table/table-sort.js"></script>
+    <link rel="stylesheet" type="text/css" href="$base_url/table/table-sort.css" />
+    <link rel="stylesheet" type="text/css" href="$base_url/benchmarks.css" />

  </head>
  <body>
diff --git a/benchmarks/osh-parser.sh b/benchmarks/osh-parser.sh
@@ -195,16 +195,16 @@ stage1() {

 print-report() {
  local in_dir=$1
-  local base_url='../../web/table'
+  local base_url='../../web'

  cat <<EOF
 <!DOCTYPE html>
 <html>
  <head>
    <title>OSH Parser Performance</title>
-    <script type="text/javascript" src="$base_url/table-sort.js"></script>
-    <link rel="stylesheet" type="text/css" href="$base_url/table-sort.css" />
-    <link rel="stylesheet" type="text/css" href="benchmarks.css" />
+    <script type="text/javascript" src="$base_url/table/table-sort.js"></script>
+    <link rel="stylesheet" type="text/css" href="$base_url/table/table-sort.css" />
+    <link rel="stylesheet" type="text/css" href="$base_url/benchmarks.css" />

  </head>
  <body>
diff --git a/benchmarks/osh-runtime.sh b/benchmarks/osh-runtime.sh
@@ -273,16 +273,16 @@ stage1() {

 print-report() {
  local in_dir=$1
-  local base_url='../../web/table'
+  local base_url='../../web'

  cat <<EOF
 <!DOCTYPE html>
 <html>
  <head>
    <title>OSH Runtime Performance</title>
-    <script type="text/javascript" src="$base_url/table-sort.js"></script>
-    <link rel="stylesheet" type="text/css" href="$base_url/table-sort.css" />
-    <link rel="stylesheet" type="text/css" href="benchmarks.css" />
+    <script type="text/javascript" src="$base_url/table/table-sort.js"></script>
+    <link rel="stylesheet" type="text/css" href="$base_url/table/table-sort.css" />
+    <link rel="stylesheet" type="text/css" href="$base_url/benchmarks.css" />

  </head>
  <body>
diff --git a/benchmarks/report.sh b/benchmarks/report.sh
@@ -38,7 +38,6 @@ stage3() {

  $script print-report $base_dir/stage2 > $out

-  cp -v benchmarks/benchmarks.css $base_dir
  echo "Wrote $out"
 }

@@ -76,6 +75,13 @@ oheap() {
  stage3 $base_dir
 }

+all() {
+  osh-parser
+  osh-runtime
+  vm-baseline
+  oheap
+}
+
 # For view
 dev-index() {
  local out=_tmp/benchmarks.html
diff --git a/benchmarks/vm-baseline.sh b/benchmarks/vm-baseline.sh
@@ -73,16 +73,16 @@ stage1() {

 print-report() {
  local in_dir=$1
-  local base_url='../../web/table'
+  local base_url='../../web'

  cat <<EOF
 <!DOCTYPE html>
 <html>
  <head>
    <title>Virtual Memory Baseline</title>
-    <script type="text/javascript" src="$base_url/table-sort.js"></script>
-    <link rel="stylesheet" type="text/css" href="$base_url/table-sort.css" />
-    <link rel="stylesheet" type="text/css" href="benchmarks.css" />
+    <script type="text/javascript" src="$base_url/table/table-sort.js"></script>
+    <link rel="stylesheet" type="text/css" href="$base_url/table/table-sort.css" />
+    <link rel="stylesheet" type="text/css" href="$base_url/benchmarks.css" />

  </head>
  <body>
diff --git a/doc/release-index.md b/doc/release-index.md
@@ -22,16 +22,25 @@ Oil Version 0.2.0

 - [Github Wiki for oilshell/oil](https://github.com/oilshell/oil/wiki)

-### Links
+### Tests and Metrics

 - [Spec Test Results](test/spec.wwz/).  Test OSH behavior against
  that of existing shells.
 - [Wild Test Results](test/wild.wwz/).  Parsing and translating thousands of
  shell scripts with OSH.
- [OSH Parser Performance](benchmarks/osh-parser.wwz/).  How fast does OSH
-  parse?
 - [Line Counts](metrics/line-counts.wwz/).  How much code is in Oil?

+### Benchmarks
+
+- [OSH Parser Performance](benchmarks.wwz/osh-parser/).  How fast does OSH
+  parse compared to other shells?
+- [OSH Runtime](benchmarks.wwz/osh-runtime/).  How fast does OSH
+  run compared to other shells?
+- [Virtual Memory Baseline](benchmarks.wwz/vm-baseline/).  How much memory to
+  shells use at startup?
+- [OHeap](benchmarks.wwz/oheap/).  Metrics for a possible AST encoding format.
+
+
 <!-- TODO: 
 benchmarks/
  virtual-memory/
diff --git a/scripts/release.sh b/scripts/release.sh
@@ -6,10 +6,8 @@
 # Steps:
 #   $0 build-and-test  (runs spec tests, etc.)
 #   test/wild.sh all
-#   benchmarks/osh-parser.sh
-#     - 'auto' on both flanders and lisa
-#     - summarize
-#     - report
+#   benchmarks/auto.sh all on both flanders and lisa
+#   benchmarks/report.sh all
 #   $0 line-counts
 #   $0 build-tree
 #   $0 compress
@@ -67,7 +65,7 @@ log() {
 #       coverage/  # coverage of all spec tests?  And gold tests maybe?
 #         python/  # python stdlib coverage  with pycoverage
 #         c/       # c coverage with gcc/clang
-#       benchmarks/
+#       benchmarks.wwz/
 #         machine-lisa/
 #           proc/meminfo etc.
 #           compile time on my machine (serial, optimized, etc.)
@@ -335,17 +333,33 @@ compress() {
  time zip -r -q $out .  # recursive, quiet
  popd

-  # TODO: _tmp/*/raw/ should't be included.  Raw data is in ../benchmarks-data.
-
-  log "--- benchmarks/osh-parser"
-  local out="$root/benchmarks/osh-parser.wwz"
-  pushd _tmp/osh-parser/
-  time zip -r -q $out .  # recursive, quiet
-  popd
+  compress-benchmarks

  tree _release/VERSION
 }

+compress-benchmarks() {
+  local root=$PWD/_release/VERSION/
+  mkdir -p $root
+
+  log "--- benchmarks"
+
+  local out="$root/benchmarks.wwz"
+
+  # Technically we only need index.html.  But it's nice to have stage1 and
+  # stage2 in case we need backup.
+
+  pushd _tmp
+  find \
+    osh-parser/{stage1,stage2,index.html} \
+    osh-runtime/{stage1,stage2,index.html} \
+    vm-baseline/{stage1,stage2,index.html} \
+    oheap/{stage1,stage2,index.html} \
+    -type f \
+    | xargs --verbose -- zip -q $out 
+  popd
+}
+
 metrics-index() {
  # TODO: Have to write index.html so it will serve out of .wwz.
  cat <<EOF
@@ -413,7 +427,7 @@ copy-web() {

 build-tree() {
  local root=_release/VERSION
-  mkdir -p $root/{doc,test,benchmarks,metrics}
+  mkdir -p $root/{doc,test,metrics}

  # Metadata
  cp -v _build/release-date.txt oil-version.txt $root
diff --git a/benchmarks/benchmarks.css → web/benchmarks.css b/benchmarks/benchmarks.css → web/benchmarks.css
@@ -26,7 +26,7 @@ h3, h4 {
 }

 /* columns */
-#osh-ovm, #osh-cpython {
+#osh, #osh-ovm, #osh-cpython {
  background-color: oldlace;
 }
 /* rows */