Generate an ovm-build report in HTML, via benchmarks/report.R.

Also, fix a bug in recording gcc provenance.
oilshell · Feb 16, 2018 · 5f601e626c8f24710409f61734da94a8b6263ee9 · 5f601e6
1 parent 6f86d6b
commit 5f601e626c8f24710409f61734da94a8b6263ee9
Unified Split

Showing with 138 additions and 3 deletions.

+2 −2 benchmarks/id.sh

+58 −0 benchmarks/ovm-build.sh

+69 −1 benchmarks/report.R

+9 −0 benchmarks/report.sh
diff --git a/benchmarks/id.sh b/benchmarks/id.sh
@@ -242,8 +242,8 @@ dump-compiler-id() {
  mkdir -p $out_dir

  case $cc in
-    gcc)
-      $cc --version 2>&1
+    */gcc)
+      $cc --version
      # -v has more details, but they might be overkill.
      ;;
    */clang)
diff --git a/benchmarks/ovm-build.sh b/benchmarks/ovm-build.sh
@@ -67,6 +67,11 @@ extract() {
  extract-other
 }

+#
+# Measure Size of Binaries.
+# NOTE: This only needs to be done on one machine?
+#
+
 # NOTE: build/test.sh measures the time already.

 # Coarse Size and Time Benchmarks
@@ -121,6 +126,10 @@ clang-oil-dbg() {
  CC=$CLANG make _build/oil/ovm-dbg
 }

+#
+# Measure Elapsed Time
+#
+
 # Add --target-size?  Add that functionality to benchmarks/time.py?
 #
 # Should we add explicit targets?
@@ -283,4 +292,53 @@ stage1() {
  wc -l $out/*
 }

+print-report() {
+  local in_dir=$1
+  local base_url='../../web'
+
+  cat <<EOF
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>OVM Build Performance</title>
+    <script type="text/javascript" src="$base_url/table/table-sort.js"></script>
+    <link rel="stylesheet" type="text/css" href="$base_url/table/table-sort.css" />
+    <link rel="stylesheet" type="text/css" href="$base_url/benchmarks.css" />
+
+  </head>
+  <body>
+    <p id="home-link">
+      <a href="/">oilshell.org</a>
+    </p>
+    <h2>OVM Build Performance</h2>
+
+    <h3>Elapsed Time by Host and Compiler</h3>
+
+    <p>Some benchmarks call many external tools, while some exercise the shell
+    interpreter itself.  Parse time is included.</p>
+EOF
+  csv2html $in_dir/times.csv
+
+  cat <<EOF
+    <h3>Binary Size</h3>
+
+    <p>Running under <code>osh-ovm</code>.  Memory usage is measured in MB
+    (powers of 10), not MiB (powers of 2).</p>
+
+EOF
+  #csv2html $in_dir/virtual-memory.csv
+
+  cat <<EOF
+
+    <h3>Host and Compiler Details</h3>
+EOF
+  csv2html $in_dir/hosts.csv
+  csv2html $in_dir/compilers.csv
+
+  cat <<EOF
+  </body>
+</html>
+EOF
+}
+
 "$@"
diff --git a/benchmarks/report.R b/benchmarks/report.R
@@ -221,7 +221,7 @@ RuntimeReport = function(in_dir, out_dir) {
  times %>% filter(status != 0) -> failed
  if (nrow(failed) != 0) {
    print(failed)
-    stop('Some tasks failed')
+    stop('Some osh-runtime tasks failed')
  }

  # Host label is the same as name
@@ -355,6 +355,71 @@ VmBaselineReport = function(in_dir, out_dir) {
  writeCsv(vm, file.path(out_dir, 'vm-baseline'))
 }

+WriteOvmBuildDetails = function(distinct_hosts, distinct_compilers, out_dir) {
+  host_table = data_frame(
+    host_label = distinct_hosts$host_label,
+    host_id = paste(distinct_hosts$host_name,
+                    distinct_hosts$host_hash, sep='-'),
+    host_id_HREF = benchmarkDataLink('host-id', host_id, '/')
+  )
+  print(host_table)
+
+  dc = distinct_compilers
+  compiler_table = data_frame(
+    compiler_label = dc$compiler_label,
+    compiler_id = paste(dc$compiler_label, dc$compiler_hash, sep='-'),
+    compiler_id_HREF = benchmarkDataLink('compiler-id', compiler_id, '/')
+  )
+  print(compiler_table)
+
+  writeCsv(host_table, file.path(out_dir, 'hosts'))
+  writeCsv(compiler_table, file.path(out_dir, 'compilers'))
+}
+
+OvmBuildReport = function(in_dir, out_dir) {
+  times = read.csv(file.path(in_dir, 'times.csv'))
+  raw_data = read.csv(file.path(in_dir, 'raw-data.csv'))
+
+  times %>% filter(status != 0) -> failed
+  if (nrow(failed) != 0) {
+    print(failed)
+    stop('Some ovm-build tasks failed')
+  }
+  #print(vm)
+
+  #times %>%
+  #  arrange(host, VmPeak_MB) ->
+  #  vm
+
+  times %>% distinct(host_name, host_hash) -> distinct_hosts
+  distinct_hosts$host_label = distinct_hosts$host_name
+
+  times %>% distinct(compiler_path, compiler_hash) -> distinct_compilers
+  distinct_compilers$compiler_label = basename(distinct_compilers$compiler_path)
+
+  print(distinct_hosts)
+  print(distinct_compilers)
+
+  WriteOvmBuildDetails(distinct_hosts, distinct_compilers, out_dir)
+
+  times %>%
+    select(-c(status)) %>%
+    left_join(distinct_hosts, by = c('host_name', 'host_hash')) %>%
+    left_join(distinct_compilers, by = c('compiler_path', 'compiler_hash')) %>%
+    select(-c(host_name, host_hash, compiler_path, compiler_hash)) %>%
+    mutate(src_dir = basename(src_dir)) %>%
+    arrange(host_label, src_dir) %>%
+    select(host_label, compiler_label, src_dir, action, elapsed_secs) ->
+    times
+
+  #print(times)
+
+  writeCsv(times, file.path(out_dir, 'times'))
+
+  # TODO: I want a size report too
+  #writeCsv(sizes, file.path(out_dir, 'sizes'))
+}
+
 main = function(argv) {
  action = argv[[1]]
  in_dir = argv[[2]]
@@ -369,6 +434,9 @@ main = function(argv) {
  } else if (action == 'vm-baseline') {
    VmBaselineReport(in_dir, out_dir)

+  } else if (action == 'ovm-build') {
+    OvmBuildReport(in_dir, out_dir)
+
  } else if (action == 'oheap') {
    OheapReport(in_dir, out_dir)

diff --git a/benchmarks/report.sh b/benchmarks/report.sh
@@ -66,6 +66,14 @@ vm-baseline() {
  stage3 $base_dir
 }

+ovm-build() {
+  local base_dir=_tmp/ovm-build
+
+  benchmarks/ovm-build.sh stage1 ../benchmark-data/ovm-build
+  stage2 $base_dir
+  stage3 $base_dir
+}
+
 # This is one is specific to a particular machine.
 oheap() {
  local base_dir=_tmp/oheap
@@ -79,6 +87,7 @@ all() {
  osh-parser
  osh-runtime
  vm-baseline
+  ovm-build
  oheap
 }