Initial version.

README

@@ -0,0 +1,61 @@
+A set of Python scripts to fetch benchmarks spec.org, analyze them, and output some text & PNG files.
+Original version by Jeff Preshing in February 2012.
+Released to the public domain.
+
+For more information:
+http://preshing.com/ [TODO]
+
+
+Requirements
+------------
+
+* Python 2.6 - 2.7 is required.
+* lxml is required if you want to fetch all the data from SPEC's website. Otherwise, you can download aggregated data from [TODO]. You could probably rewrite the lxml part using one of Python's built-in modules; I didn't bother.
+* pycairo is optional if you want to generate the PNG files.
+* PIL is optional if you want those PNG files to have high-quality anti-aliasing.
+* If you are going to republish any results, you need to abide by SPEC's fair use policy. http://www.spec.org/fairuse.html
+
+
+Collecting SPEC's data
+----------------------
+
+These scripts work with the individual benchmarks in each SPEC result, and not the geometric average which SPEC lists in their CSV downloads. Currently, the only way to access these individual benchmark results is to scrape each result page from their website as text and/or HTML.
+
+If you want to skip the steps in this section, you can simply download the aggregated result files from [TODO] and extract them to this folder.
+
+If you want to scrape & aggregate the results yourself, proceed as follows:
+
+1. Run fetch-pages.py. As of this writing, this script downloads 30715 individual pages from SPEC's website and stores them to a folder named "scraped". It's about 383 MB of data, but may be more in the future. The script launches a pool of 20 subprocesses to speed up the download process, so it completes in a matter of minutes. Some requests may time out and break the script; if that happens, simply run the script again. All previously downloaded pages will not be downloaded again. Note that if SPEC changes their website in the future, the script will need to be updated.
+
+2. Run analyze-pages.py. This will scan all the pages downloaded by the previous script, and output two CSV files: summaries.txt and benchmarks.txt. These files will be used as inputs for the remaining scripts.
+
+
+Determining which benchmarks took advantage of autoparallel, and disqualifying them
+-----------------------------------------------------------------------------------
+
+Run check-autoparallel.py to see which benchmark tests had the highest multiple of the geometric average. [TODO] I've assumed that the highest scores definitely benefit from autoparallelization.
+
+I took the top six benchmark tests from both the INT and FP categories, and excluded them from the results in make-graphs.py. Search for DISQUALIFIED_BENCHMARKS in make-graphs.py.
+
+
+Generating the graphs
+---------------------
+
+Run make-graphs.py. It outputs the following:
+
+* identified_cpus.txt
+	The right column contains a list of all processor names encountered in the input, along with the source filename and line number. The left column contains the recognized brand name, model name and MHz. I used this file to develop & debug the identifyCPU() function found in the script. If new processors are introduced, this function may need to adapt. It might be a good idea to do a diff of this file generated from the latest SPEC data against a copy generated from older data.
+
+* int_report.txt
+	The first two lines show the automatically computed conversion ratios between CINT95, CINT2000 and CINT2006. The rest of the file groups all the results by family, then sorts them by hardware release date and normalized SPECint2006 result value. Each line shows the benchmark suite and line number. You should be able to pick out certain points on the PNG graph, find them in this text file, locate the corresponding line in the CSV, and use that to find the detailed html/PDF result page on SPEC's website.
+
+* fp_report.txt
+	Same thing as int_report.txt, but for floating-point benchmarks.
+
+* int_graph.png
+* fp_graph.png
+	Graphs similar to the ones you'll find on [TODO].
+
+
+-----
+SPECint(R) and SPECfp(R) are registered trademarks of the Standard Performance Evaluation Corporation (SPEC).

analyze-pages.py

@@ -0,0 +1,249 @@
+import csv
+import re
+import os
+import bz2
+import cPickle
+import sys
+from collections import namedtuple, defaultdict
+from datetime import datetime
+from pprint import pprint
+
+TestRecord = namedtuple('TestRecord', 'testID tester machine cpu mhz hwAvail os compiler autoParallel benchType base peak')
+BenchRecord = namedtuple('BenchRecord', 'testID benchName base peak')
+
+def scanUntilLine(lineIter, pattern):
+    for line in lineIter:
+        m = re.search(pattern, line)
+        if m:
+            g = m.groups()
+            if len(g) == 1:
+                return g[0].strip()            
+            return [x.strip() for x in g]
+
+MHzExp = re.compile('[(/]?(\\d+(?:\\.\\d+)?)a? ?([mg]hz)\\)?')
+
+def ExtractMHzFromName(name):
+    name = name.lower()
+    m = MHzExp.search(name)
+    value, units = m.groups()
+    value = float(value)
+    if units == 'ghz':
+        value *= 1000
+    return value
+
+def parse95(path):
+    testID = os.path.splitext(os.path.basename(path))[0]    
+    lineIter = iter(open(path))
+    for line in lineIter:
+        if line.startswith('   ------------  --------  --------  --------  --------  --------  --------'):
+            break
+        if 'SPEC has determined that this result was not in' in line:
+            return [], []
+    benches = []
+    for line in lineIter:
+        m = re.match('   (SPEC.{32}) ', line)
+        if m:
+            benchType = m.group(1).strip()
+            break
+        benchName = line[:15].strip()
+        base = line[35:45].strip()
+        peak = line[65:75].strip()
+        benches.append(BenchRecord(testID, benchName, base, peak))
+    if '_rate' in benchType:
+        return [], []
+    benchType = {
+        'SPECint_base95 (Geom. Mean)' : 'CINT95',
+        'SPECfp_base95 (Geom. Mean)' : 'CFP95'
+    }[benchType]
+    base = line[35:45].strip()
+    peak = lineIter.next()[65:75].strip()
+    properties = {}
+    label = ''
+    for line in lineIter:
+        l = line.strip()
+        if l in ['HARDWARE', 'SOFTWARE', 'TESTER INFORMATION', '------------------', '--------']:
+            continue
+        if l == 'NOTES':
+            break
+        if line[19:20] == ':':
+            label = line[:19].strip()
+        desc = line[21:].strip()
+        if label and desc:
+            if label in properties:
+                properties[label] += ' ' + desc
+            else:
+                properties[label] = desc
+    cpu = properties['CPU']
+    mhz = ExtractMHzFromName(cpu)
+    opSys = properties['Operating System']
+    compiler = properties['Compiler']
+    if 'Hardware Avail' not in properties:
+        html = open(path[:-4] + '.html').read()
+        m = re.search('Hardware Avail:\\s+<TD align=left>([^\\s]+)\\s', html)
+        hwAvail = m.group(1).strip()
+        m = re.search('Tested By:\\s+<TD align=left>(.+)$', html, re.MULTILINE)
+        testedBy = m.group(1).strip()
+    else:
+        hwAvail = properties['Hardware Avail']
+        testedBy = properties['Tested By']
+    try:
+        hwAvail = datetime.strptime(hwAvail, '%b-%y').strftime('%b-%Y')
+    except ValueError:
+        pass
+    model = properties['Model Name']
+
+    testRecord = TestRecord(testID, testedBy, model, cpu, mhz, hwAvail, opSys, compiler, 'No', benchType, base, peak)
+    return [testRecord], benches
+
+
+def parse2000(path):
+    testID = os.path.splitext(os.path.basename(path))[0]    
+    lineIter = iter(open(path))
+    lineIter.next()
+    hwAvail = scanUntilLine(lineIter, 'Hardware availability: (.*)')
+    tester = scanUntilLine(lineIter, 'Tester: (.*?) *Software availability')
+    for line in lineIter:
+        if line.startswith('   ========================================================================'):
+            break
+        if 'SPEC has determined that this result was not in' in line:
+            return [], []
+    benches = []
+    for line in lineIter:
+        m = re.match('   (SPEC.{24})    ', line)
+        if m:
+            benchType = m.group(1).strip()
+            break
+        benchName = line[:15].strip()
+        base = line[35:45].strip()
+        peak = line[65:75].strip()
+        benches.append(BenchRecord(testID, benchName, base, peak))
+    if '_rate_' in benchType:
+        return [], []
+    benchType = {
+        'SPECint_base2000' : 'CINT2000',
+        'SPECfp_base2000' : 'CFP2000'
+    }[benchType]
+    base = line[35:45].strip()
+    peak = lineIter.next()[65:75].strip()
+    properties = {}
+    label = ''
+    for line in lineIter:
+        l = line.strip()
+        if l in ['HARDWARE', 'SOFTWARE', '--------']:
+            continue
+        if l == 'NOTES':
+            break
+        if line[20:21] == ':':
+            label = line[:20].strip()
+        desc = line[22:].strip()
+        if label and desc:
+            if label in properties:
+                properties[label] += ' ' + desc
+            else:
+                properties[label] = desc
+    cpu = properties['CPU']
+    mhz = float(properties['CPU MHz'])
+    opSys = properties['Operating System']
+    compiler = properties['Compiler']
+    model = properties['Model Name']
+
+    testRecord = TestRecord(testID, tester, model, cpu, mhz, hwAvail, opSys, compiler, 'No', benchType, base, peak)
+    return [testRecord], benches
+
+
+def parse2006(path):
+    testID = os.path.splitext(os.path.basename(path))[0]    
+    lineIter = iter(open(path))
+    if '######################' in lineIter.next():
+        return [], []
+    model = lineIter.next().strip()
+    hwAvail = scanUntilLine(lineIter, 'Hardware availability: (.*)')
+    tester = scanUntilLine(lineIter, 'Tested by:    (.*?) *Software availability')
+    if model.startswith(tester):
+        model = model[len(tester):].strip()
+    for line in lineIter:
+        if line.startswith('=============================================================================='):
+            break
+        if 'SPEC has determined that this result was not in' in line:
+            return [], []
+        if 'SPEC has determined that this result is not in' in line:
+            return [], []
+    benches = []
+    for line in lineIter:
+        m = re.match(' (SPEC.{27})    ', line)
+        if m:
+            benchType = m.group(1).strip()
+            break
+        benchName = line[:15].strip()
+        base = line[33:43].strip()
+        peak = line[65:75].strip()
+        benches.append(BenchRecord(testID, benchName, base, peak))
+    if '_rate_' in benchType:
+        return [], []
+    benchType = {
+        'SPECint(R)_base2006' : 'CINT2006',
+        'SPECfp(R)_base2006' : 'CFP2006'
+    }[benchType]
+    base = line[33:43].strip()
+    peak = lineIter.next()[65:75].strip()
+    properties = {}
+    label = ''
+    for line in lineIter:
+        l = line.strip()
+        if l in ['HARDWARE', 'SOFTWARE', '--------']:
+            continue
+        if l == 'Submit Notes':
+            break
+        if line[20:21] == ':':
+            label = line[:20].strip()
+        desc = line[22:].strip()
+        if label and desc:
+            if label in properties:
+                properties[label] += ' ' + desc
+            else:
+                properties[label] = desc
+    cpu = properties['CPU Name']
+    mhz = float(properties['CPU MHz'])
+    opSys = properties['Operating System']
+    compiler = properties['Compiler']
+    autoParallel = properties['Auto Parallel']
+
+    testRecord = TestRecord(testID, tester, model, cpu, mhz, hwAvail, opSys, compiler, autoParallel, benchType, base, peak)
+    return [testRecord], benches
+
+def iterRecords():
+    allTests = []
+
+    for fn in os.listdir(os.path.join('scraped', 'cpu95')):
+        if fn.lower().endswith('.asc'):
+            allTests.append((parse95, os.path.join('scraped', 'cpu95', fn)))
+    for fn in os.listdir(os.path.join('scraped', 'cpu2000')):
+        allTests.append((parse2000, os.path.join('scraped', 'cpu2000', fn)))
+    for fn in os.listdir(os.path.join('scraped', 'cpu2006')):
+        allTests.append((parse2006, os.path.join('scraped', 'cpu2006', fn)))
+
+    tests = []
+    benches = []
+    for i, pair in enumerate(allTests):
+        if i % 100 == 0:
+            print 'Analyzing %d/%d ...' % (i, len(allTests))
+        func, arg = pair
+        t, b = func(arg)
+        tests += t
+        benches += b
+
+    print 'Writing summaries.txt ...'
+    with open('summaries.txt', 'w') as f:
+        w = csv.writer(f)
+        w.writerow(TestRecord._fields)
+        for t in tests:
+            w.writerow(t)
+
+    print 'Writing benchmarks.txt ...'
+    with open('benchmarks.txt', 'w') as f:
+        w = csv.writer(f)
+        w.writerow(BenchRecord._fields)
+        for b in benches:
+            w.writerow(b)
+
+iterRecords()

check-autoparallel.py

@@ -0,0 +1,40 @@
+import csv
+import collections
+import math
+from pprint import pprint
+
+def iterCsvRecords(path, className):
+    with open(path, 'rb') as f:
+        reader = csv.reader(f)
+        clazz = None
+        for row in reader:
+            if clazz is None:
+                clazz = collections.namedtuple(className, row)
+            else:
+                yield clazz(*row)
+
+benchTable = collections.defaultdict(dict)
+for brec in iterCsvRecords('benchmarks.txt', 'BenchmarkRecord'):
+    benchTable[brec.testID][brec.benchName] = brec
+
+def geomAverage(values):
+    averageExp = sum([math.log(x) for x in values]) / len(values)
+    return math.exp(averageExp)
+
+for benchType in ['INT', 'FP']:
+    print 'Top contributing benchmarks to %s results, by maximum multiple of the geometric average:' % benchType
+    topBenchResults = {}
+    summaryTable = {}
+    for srec in iterCsvRecords('summaries.txt', 'SummaryRecord'):
+        summaryTable[srec.testID] = srec
+        if srec.autoParallel == 'Yes' and benchType in srec.benchType:
+            base = geomAverage([float(brec.base) for brec in benchTable[srec.testID].itervalues()])
+            for brec in benchTable[srec.testID].itervalues():
+                r = (float(brec.base) / base, brec)
+                topBenchResults[brec.benchName] = max(r, topBenchResults.get(brec.benchName, (0, None)))
+    for v, k in sorted([(v, k) for k, v in topBenchResults.iteritems()], reverse=True):
+        benchValue, brec = v
+        print benchValue, k, brec.testID, summaryTable[brec.testID].machine
+    print
+
+