Permalink
Browse files

add skeleton of a performance regression detector

  • Loading branch information...
1 parent 5e2cd4d commit de0d6f3b68ba32be3bf942bd46b7305f798c0ee1 Txus committed Sep 13, 2011
Showing with 178 additions and 0 deletions.
  1. +86 −0 perf.sh
  2. +12 −0 perf/assignment_bench.rb
  3. +17 −0 perf/conditional_bench.rb
  4. +48 −0 perf/nobench.rb
  5. +15 −0 perf/recursive_method_bench.rb
View
86 perf.sh
@@ -0,0 +1,86 @@
+#!/bin/bash
+#
+# This script runs a given command over a range of Git revisions. Note that it
+# will check past revisions out! Exercise caution if there are important
+# untracked files in your working tree.
+#
+# This came from Gary Bernhardt's dotfiles:
+# https://github.com/garybernhardt/dotfiles
+#
+# Example usage:
+# $ run-command-on-git-revisions origin/master master 'python runtests.py'
+
+set -e
+
+start_ref=$1
+end_ref=$2
+
+run_benchmarks() {
+ # Get git revsA
+
+ revs=`log_command git rev-list --reverse --abbrev-commit ${start_ref}..${end_ref}`
+
+ # Backup current benchmarks
+
+ if [ -e "../__perf" ]; then
+ rm -fR ../__perf
+ fi
+
+ mkdir -p ../__perf/perf
+ cp -r perf/* ../__perf/perf/
+ cp -r perf.sh ../__perf/perf.sh
+
+ # Remove stats file if they exist
+ if [ -e "../__perf/*.csv" ]; then
+ log_command rm ../__perf/*.csv
+ fi
+
+ for rev in $revs; do
+ echo "Checking out: $(git log --oneline -1 $rev)"
+ log_command git checkout --quiet $rev
+
+ for f in ../__perf/perf/*_bench.rb
+ do
+ name=`echo "$f" | sed -e "s/\// /g" | sed -e 's/_bench\.rb/ /g' | awk '{ print $4 }'`
+ echo "$name benchmarks..."
+
+ # Replace perf directory with our own
+
+ if [ -e "perf" ]; then
+ log_command rm -fR perf
+ fi
+ log_command cp -r ../__perf/perf perf
+
+ stats_filename=${name}_stats.csv
+
+ output=`ruby perf/${name}_bench.rb`
+
+ lexer_time=`echo "$output" | grep '^LEXER' | awk '{print $2}'`
+ parser_time=`echo "$output" | grep '^PARSER' | awk '{print $2}'`
+ runtime=`echo "$output" | grep '^RUNTIME' | awk '{print $2}'`
+ total_time=`echo "$output" | grep '^TOTAL' | awk '{print $2}'`
+
+ timestamp=`git log -1 --pretty="format:%ai" $rev`
+
+ log_command echo $timestamp,$rev,$lexer_time,$parser_time,$runtime,$total_time >> ../__perf/$stats_filename
+ done
+
+ # Cleanup the mess
+ git clean -fd
+ done
+ log_command git checkout $end_ref
+
+ echo "Restoring benchmarks..."
+ log_command cp -r ../__perf/perf perf
+ log_command cp -r ../__perf/perf.sh perf.sh
+
+ echo "OK for all revisions!"
+ echo "Stats are in: `ls ../__perf/*.csv`"
+}
+
+log_command() {
+ echo "=> $*" >&2
+ eval $*
+}
+
+run_benchmarks
View
12 perf/assignment_bench.rb
@@ -0,0 +1,12 @@
+require_relative 'nobench'
+
+benchmark(<<CODE)
+
+ a = 34
+ b = 91
+ c = 'hello'
+ d = true
+ e = false
+ f = nil
+
+CODE
View
17 perf/conditional_bench.rb
@@ -0,0 +1,17 @@
+require_relative 'nobench'
+
+benchmark(<<CODE)
+
+ if 3 == 3
+ true
+ else
+ false
+ end
+
+ if 3 != 3
+ false
+ else
+ true
+ end
+
+CODE
View
48 perf/nobench.rb
@@ -0,0 +1,48 @@
+class Array; def mean; inject(:+) / size.to_f; end; end
+require 'benchmark'
+require_relative '../lib/noscript'
+
+TIMES = 10000
+
+def benchmark(code)
+ code.strip!
+
+ Noscript::Parser.new.scan_str(code).compile(Noscript::Context.generate)
+ # Do a first run to eliminate random GC effects
+
+ # LEXER
+
+ lexer = Noscript::Parser.new
+ lexer_time = (0..TIMES).to_a.map do
+ Benchmark.realtime {
+ lexer.scan_setup(code)
+ while token = lexer.next_token; end
+ }
+ end.mean * 1000
+ puts "LEXER: #{lexer_time}"
+
+ # PARSER
+
+ parser = Noscript::Parser.new
+ parser_time = (0..TIMES).to_a.map do
+ Benchmark.realtime {
+ parser.scan_str(code)
+ }
+ end.mean * 1000
+ puts "PARSER: #{parser_time - lexer_time}"
+
+ # RUNTIME
+
+ parser = Noscript::Parser.new
+ ast = parser.scan_str(code)
+ runtime = (0..TIMES).to_a.map do
+ Benchmark.realtime {
+ ast.compile(Noscript::Context.generate)
+ }
+ end.mean * 1000
+
+ puts "RUNTIME: #{runtime}"
+ puts "TOTAL: #{lexer_time + parser_time + runtime}"
+rescue
+ puts "ERROR"
+end
View
15 perf/recursive_method_bench.rb
@@ -0,0 +1,15 @@
+require_relative 'nobench'
+
+benchmark(<<CODE)
+
+ bar = 100
+
+ def foo(bar)
+ if bar == 0
+
+ else
+ foo(bar - 1)
+ end
+ end
+
+CODE

0 comments on commit de0d6f3

Please sign in to comment.