Merge pull request #319 from ujh/flexible-benchmark

Flexible benchmark

bin/benchmark.rb

@@ -27,6 +27,8 @@
 require 'fileutils'
 require 'optparse'
 
+require_relative "../misc/lib/benchmark_results"
+
 OptionParser.new do |opts|
   # Use this when running on a 36 core machine. Then gogui-twogtp will start 4 games in parallel and
   # Iomrascálaí will use 8 threads instead of all 36.
@@ -51,25 +53,23 @@
 case SIZE
 when "9"
   TIME = "2m"
-  GAMES = "500"
 when "13"
   TIME = "10m"
-  GAMES = "700"
 when "15"
   TIME="17m"
-  GAMES="200"
 when "17"
   TIME="24m"
-  GAMES="100"
 when "19"
   TIME = "30m"
-  GAMES = "100"
 else
   raise "Size #{SIZE} isn't supported!"
 end
 
-GOGUI_TWOGTP = %Q/gogui-twogtp -auto -black "#{GNUGO}" -white "#{IOMRASCALAI}" -size #{SIZE} -alternate -games #{GAMES} -sgffile #{FILENAME} -time #{TIME} -referee "#{REFEREE}" -verbose/
-GOGUI_TWOGTP += " -threads 4" if $ec2
+def gogui_twogtp(games)
+  cmd = %Q/gogui-twogtp -auto -black "#{GNUGO}" -white "#{IOMRASCALAI}" -size #{SIZE} -alternate -games #{games} -sgffile #{FILENAME} -time #{TIME} -referee "#{REFEREE}" -verbose/
+  cmd += " -threads 4" if $ec2
+  cmd
+end
 
 DAT_FILE = "#{FILENAME}.dat"
 
@@ -79,7 +79,12 @@ def run(cmd)
 end
 
 def run_benchmark
-  run(GOGUI_TWOGTP)
+  games = if File.exists?(DAT_FILE)
+    BenchmarkResults.new(DAT_FILE).games
+  else
+    0
+  end
+  run(gogui_twogtp(games+10))
 end
 
 def data
@@ -152,9 +157,11 @@ def remove_first_error
   # shift following rows down (change GAME IDs)
   contents = shift_game_ids(error_id, contents)
   # save updated file to disk
-  FileUtils.mv(DAT_FILE, "#{DAT_FILE}-#{Time.now.to_f}")
+  backup = "#{DAT_FILE}-#{Time.now.to_f}"
+  FileUtils.mv(DAT_FILE, backup)
   File.open(DAT_FILE, 'w') {|f| f.write header }
   CSV.open(DAT_FILE, 'a', col_sep: "\t") {|csv| contents.each {|row| csv << row }}
+  File.unlink(backup)
 end
 
 def check_for_crashes
@@ -167,11 +174,18 @@ def check_for_crashes
 end
 
 def done?
-  return true unless File.exists?(DAT_FILE)
-  _, contents = parse_file
-  contents.length == GAMES.to_i
+  return false unless File.exists?(DAT_FILE)
+  br = BenchmarkResults.new(DAT_FILE)
+  # Continue if less than 100 games were played
+  return false if br.games < 100
+  # Stop at 1000 games
+  return true if br.games >= 1000
+  # Stop if the error is below 3 percent
+  br.error95 <= 3.0
 end
 
+exit if done?
+
 loop do
   check_for_crashes
   run_benchmark

bin/wins-from-benchmark-results.rb

@@ -22,44 +22,7 @@
 # ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-require 'csv'
-
-def parse_file(fn)
-  contents = File.read(fn)
-  relevant_lines = contents.each_line.find_all {|l| l !~ /^#/ }
-  CSV.parse(relevant_lines.map(&:strip).join("\n"), col_sep: "\t")
-end
-
-RES_B = 1
-RES_W = 2
-RES_R = 3
-
-def wins(fn)
-  data = parse_file(fn)
-  white = data.find_all {|row| row[RES_R] =~ /W\+/ }.count
-  black = data.find_all {|row| row[RES_R] =~ /B\+/ }.count
-  n = white + black
-  p = white.to_f/n
-  "#{(p*100).round(2)}% wins (#{white} games of #{n}, ± #{error(p, n, 0.95).round(2)} at 95%, ± #{error(p, n, 0.99).round(2)} at 99%)"
-end
-
-def scoring(fn)
-  data = parse_file(fn)
-  relevant = data.find_all {|row| row[RES_R] !~ /[BW]\+R/ }
-  agreeing = relevant.find_all {|row| row[RES_W] == row[RES_B] }.count
-  n = relevant.length
-  p = agreeing.to_f/n
-  "#{(p*100).round(2)}% same score as GnuGo (#{agreeing} of #{n}, ± #{error(p, n, 0.95).round(2)} at 95%, ± #{error(p, n, 0.99).round(2)} at 99%)"
-end
-
-def z(confidence)
-  alpha = 1 - confidence
-  (1 - 0.5*alpha)*2
-end
-
-def error(p, n, confidence)
-  (z(confidence) * Math.sqrt((1.0/n)*p*(1-p)))*100
-end
+require_relative "../misc/lib/benchmark_results"
 
 files = Dir["*.dat"].sort do |a, b|
   a =~ /(.*)-(\d+)x\d+/
@@ -79,6 +42,9 @@ def error(p, n, confidence)
 files.each do |fn|
   next if fn =~ /summary\.dat/
   puts "#{fn}:"
-  puts "\t\t#{wins(fn)}"
-  puts "\t\t#{scoring(fn)}"
+  br = BenchmarkResults.new(fn)
+  puts "\t\t#{(br.win_percentage*100).round(2)}% wins (#{br.wins} games of #{br.games}, ± #{br.error95.round(2)} at 95%, ± #{br.error99.round(2)} at 99%)"
+
+  scoring = br.scoring
+  puts "\t\t#{(scoring[:same_score_percentage]*100).round(2)}% same score as GnuGo (#{scoring[:same_score]} of #{scoring[:games]}, ± #{scoring[:error95].round(2)} at 95%, ± #{scoring[:error99].round(2)} at 99%)"
 end

misc/lib/benchmark_results.rb

@@ -0,0 +1,83 @@
+#
+# Copyright (c) 2016 Urban Hafner
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation files
+# (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+require 'csv'
+
+class BenchmarkResults
+
+  attr_reader :wins, :games, :error95, :error99, :win_percentage, :scoring
+
+  def initialize(filename)
+    @filename = filename
+    process
+  end
+
+  private
+
+  RES_B = 1
+  RES_W = 2
+  RES_R = 3
+
+  def process
+    @data ||= parse_file
+    process_win_percentage
+    process_score
+  end
+
+  def process_win_percentage
+    white = @data.find_all {|row| row[RES_R] =~ /W\+/ }.count
+    black = @data.find_all {|row| row[RES_R] =~ /B\+/ }.count
+    @games = white + black
+    @wins = white
+    @win_percentage = white.to_f/@games
+    @error95 = error(@win_percentage, @games, 0.95)
+    @error99 = error(@win_percentage, @games, 0.99)
+  end
+
+  def process_score
+    @scoring = {}
+    relevant = @data.find_all {|row| row[RES_R] !~ /[BW]\+R/ }
+    agreeing = relevant.find_all {|row| row[RES_W] == row[RES_B] }.count
+    @scoring[:games] = relevant.length
+    @scoring[:same_score] = agreeing
+    @scoring[:same_score_percentage] = agreeing.to_f/@scoring[:games]
+    p = agreeing.to_f/@scoring[:games]
+    @scoring[:error95] = error(p, @scoring[:games], 0.95)
+    @scoring[:error99] = error(p, @scoring[:games], 0.99)
+  end
+
+  def parse_file
+    contents = File.read(@filename)
+    relevant_lines = contents.each_line.find_all {|l| l !~ /^#/ }
+    CSV.parse(relevant_lines.map(&:strip).join("\n"), col_sep: "\t")
+  end
+
+  def z(confidence)
+    alpha = 1 - confidence
+    (1 - 0.5*alpha)*2
+  end
+
+  def error(p, n, confidence)
+    (z(confidence) * Math.sqrt((1.0/n)*p*(1-p)))*100
+  end
+
+end