Permalink
Browse files

update scripts.

  • Loading branch information...
1 parent d9ac856 commit f662604b72f23cb74aced539b1fe1fae15db0586 HOSHINO Takashi committed Mar 19, 2012
Showing with 154 additions and 74 deletions.
  1. +1 −1 scripts/analyze.sh
  2. +45 −42 scripts/make_report.py
  3. +46 −14 scripts/performance_plot.py
  4. +4 −2 scripts/plot_bps.py
  5. +28 −14 scripts/plot_h.py
  6. +12 −0 scripts/relation.py
  7. +18 −1 scripts/util.py
View
@@ -51,7 +51,7 @@ get_histogram()
|grep '^threadId' \
|teee 1 |filter.py -g 1,2 -r 'threadId' '[0-9]+' \
|teee 2 |project.py -g 10 \
-|teee 3 |./histogram.py ${width} \
+|teee 3 |histogram.py ${width} \
|teee 4 > ${d}/histogram_${width}
}
View
@@ -1,15 +1,15 @@
#!/usr/bin/env python
"""
-
+Make report template with mediawiki format from a parameter file.
"""
import sys
import os
-from param import *
from relation import *
+from util import *
def createTableStr1(headerX, itemMatrix):
"""
@@ -83,19 +83,23 @@ def sep():
+ end()
-def renderStatistics(f, chartTypes, patterns, blockSizeUs, prefix=''):
+def renderStatistics(f, chartTypes, patterns, blockSizeUs, names=['Default'], prefixes=[''], widthPx=640, heightPx=(640 * 3 / 4)):
"""
f :: file
file object to write rendered text.
chartTypes :: [str]
chart types like 'res', 'iops', 'bps'.
patterns :: [str]
pattern like 'rnd', 'seq'.
- blockSIzeUs :: [str]
+ blockSizeUs :: [str]
block size with unit like '512', '4k', '32k', '256k'.
- prefix :: str
- prefix of filepath.
+ names :: [str]
+ list of name.
+ prefixes :: [str]
+ list of prefix of filepath
return :: None
+
+ len(names) == len(prefixes) is required.
"""
#paramsStat = getParamsRelForStatistics()
@@ -120,9 +124,11 @@ def mapperY(paramY):
matrix = []
for bsU in paramsY:
line = []
- line.append('[[%s%s_%s_bs%s.png|width=%dpx|height=%dpx]]' % (prefix, chartType, pattern, bsU, 640, 640 * 3 / 4))
+ for prefix in prefixes:
+ line.append('[[%s%s_%s_bs%s.png|width=%dpx|height=%dpx]]' % \
+ (prefix, chartType, pattern, bsU, widthPx, heightPx))
matrix.append(line)
- print >>f, createTableStr2([mapperC(paramC)], map(mapperY, paramsY), matrix)
+ print >>f, createTableStr2(names, map(mapperY, paramsY), matrix)
#print createTableStr2(map(mapperX, paramsX), map(mapperY, paramsY), matrix)
@@ -184,12 +190,10 @@ def mapperY0((pattern, mode)):
#print createTableStr2(paramsX, paramsY, matrix)
#paramsHist = getParamsRelForHistogram()
-def renderIorethReport(f, exprName, params, goals=None, settings=None, conclusion=None):
+def renderIorethReport(f, params, goals=None, settings=None, conclusion=None):
"""
Render ioreth report in mediawiki format.
- exprName :: str
- Experiment name like '20120227a'
params :: dict(paramName :: str, [str] | [(str, str)])
Parameter data.
goals :: str
@@ -200,6 +204,23 @@ def renderIorethReport(f, exprName, params, goals=None, settings=None, conclusio
Conclusion text in mediawiki format.
"""
+ exprName = params['exprName']
+ nameList = params['nameList']
+ statisticsDirList = params['statisticsDirectories']
+ if 'histogramDirectory' in params:
+ histogramDir = params['histogramDirectory']
+ else:
+ histogramDir = None
+
+ if 'widthPx' in params:
+ widthPx = int(params['widthPx'])
+ else:
+ widthPx = 640
+ if 'heightPx' in params:
+ heightPx = int(params['heightPx'])
+ else:
+ heightPx = widthPx * 3 / 4
+
print >>f, '== %s ==' % exprName
print >>f, '== Goals =='
@@ -220,51 +241,33 @@ def renderIorethReport(f, exprName, params, goals=None, settings=None, conclusio
widthList = params['widthList']
print >>f, '=== Performance Statistics ==='
-
renderStatistics(f, chartTypeList, patternList,
blockSizeUnitList,
- prefix='%s/charts/' % exprName)
-
- print >>f, '=== Response Histogram ==='
-
- renderHistogram(f, patternModeList, blockSizeUnitList,
- nThreadsList, widthList,
- prefix=('%s/histogram/' % exprName))
+ names=nameList,
+ prefixes=map(lambda x: '%s/' % x, statisticsDirList),
+ widthPx=widthPx, heightPx=heightPx)
+
+ if histogramDir is not None:
+ print >>f, '=== Response Histogram ==='
+ renderHistogram(f, patternModeList, blockSizeUnitList,
+ nThreadsList, widthList,
+ prefix='%s/' % histogramDir)
print >>f, '== Conclusion =='
if conclusion is not None:
print >>f, conclusion
-def getParams(fileName):
- """
- fileName :: str
- parameter file name.
-
- params :: dict(paramName :: str, [str] | [(str, str)])
- Parameter data.
-
- """
- f = open(fileName, 'r')
- s = ''
- for line in f:
- s += line
- f.close()
- params = eval(s)
- return params
-
def doMain():
if len(sys.argv) == 1:
- print "Specify expreiment name and prepare parameter file."
+ print "Usage %s [param file]" % sys.argv[0]
exit(1)
- paramName = sys.argv[1]
- paramFileName = '%s.param' % paramName
- wikiFileName = '%s.mediawiki' % paramName
-
+ paramFileName = sys.argv[1]
params = getParams(paramFileName)
+ wikiFileName = params['exprName'] + '.mediawiki'
f = open(wikiFileName, 'w')
- renderIorethReport(f, paramName, params)
+ renderIorethReport(f, params)
f.close()
if __name__ == '__main__':
@@ -7,6 +7,7 @@
import re
import time
import itertools
+from decimal import Decimal
import Gnuplot
from relation import *
@@ -21,7 +22,8 @@ class PerformancePlot:
GroupedRelations :: [(mode :: (str,), plot :: Relation)]
"""
- def __init__(self, rel, pattern, blockSize, target, ylabel, title='', output='output.png'):
+ def __init__(self, rel, pattern, blockSize, target, ylabel, title='', output='output.png',
+ scale=Decimal(1.0), xRange='*:*', yRange='0:*'):
"""
rel :: Relation
pattern :: str
@@ -33,6 +35,12 @@ def __init__(self, rel, pattern, blockSize, target, ylabel, title='', output='ou
title :: str
output :: str
Output file name.
+ scale :: Decimal
+ Scale of the plot data.
+ xRange :: str
+ xrange.
+ yRange :: str
+ yrange.
"""
assert(isinstance(rel, Relation))
@@ -42,13 +50,19 @@ def __init__(self, rel, pattern, blockSize, target, ylabel, title='', output='ou
assert(isinstance(ylabel, str))
assert(isinstance(title, str))
assert(isinstance(output, str))
+ assert(isinstance(scale, Decimal))
+ assert(isinstance(xRange, str))
+ assert(isinstance(yRange, str))
self.__rel = rel
self.__pattern = pattern
self.__blockSize = blockSize
self.__target = target
self.__ylabel = ylabel
self.__title = title
self.__output = output
+ self.__scale = scale
+ self.__xrange = xRange
+ self.__yrange = yRange
def plot(self):
"""
@@ -57,13 +71,14 @@ def plot(self):
"""
#print "relation size: ", self.__rel.size(), self.__pattern, self.__blockSize #debug
groupedRelations = self.__filterAndGroupByMode(
- self.__rel, self.__pattern, self.__blockSize, self.__target)
+ self.__rel, self.__pattern, self.__blockSize, self.__target, self.__scale)
#print "groupedRelations: ", len(groupedRelations) #debug
plotData = self.__toPlotData(groupedRelations)
#print "plotData: ", plotData #debug
- self.__doPlot(plotData, self.__output, self.__title, self.__ylabel)
+ self.__doPlot(plotData, self.__output, self.__title, self.__ylabel,
+ self.__xrange, self.__yrange)
- def __filterAndGroupByMode(self, rel, pattern, blockSize, target):
+ def __filterAndGroupByMode(self, rel, pattern, blockSize, target, scale=Decimal(1.0)):
"""
Get plot data for 'number of threads' vs 'response time'.
@@ -76,10 +91,14 @@ def __filterAndGroupByMode(self, rel, pattern, blockSize, target):
Number of threads. (This is X axis.)
blockSize :: int
Block size [bytes].
- 'target' :: Decimal
- Something to plot. (This is Y axis.)
+ 'target' :: str
+ Column name of data. (This is Y axis.)
pattern :: str
blockSize :: str(int)
+ target :: str
+ Target column name.
+ scale :: Decimal
+ Scale of the Y data.
return :: GroupedRelations
Schema of each relation: (nThreads, target)
@@ -94,9 +113,14 @@ def pred(rec):
#print "rel: ", rel.show() #debug
rel2 = rel.filter(pred)
#print "rel2: ", rel2.show() #debug
- rel3 = rel2.groupbyAsRelation(['mode'], ['nThreads', target])
- #print "rel3: ", rel3 #debug
- return rel3
+ def mapper((pattern, mode, nThreads, blockSize, target)):
+ return (pattern, mode, nThreads, blockSize, str(Decimal(target) * Decimal(scale)))
+ cols = ['pattern', 'mode', 'nThreads', 'blockSize', target]
+ rel3 = rel2.map(cols, cols, mapper)
+ #print "rel3: ", rel3.show() #debug
+ rel4 = rel3.groupbyAsRelation(['mode'], ['nThreads', target])
+ #print "rel4: ", rel4 #debug
+ return rel4
def __toPlotData(self, groupedRelation):
"""
@@ -112,13 +136,16 @@ def __toPlotData(self, groupedRelation):
Gnuplot.PlotItems.Data(rel.getL(), with_="lp", title=','.join(rawKey)))
return plots
- def __doPlot(self, plotData, outputFileName, title, ylabel):
+ def __doPlot(self, plotData, outputFileName, title, ylabel,
+ xRange='*:*', yRange='0:*'):
"""
plotData :: [Gnuplot.PlotItems.Data]
outputFileName :: str
title :: str
ylabel :: str
Label of Y axis.
+ xRange :: str
+ yRange :: yRange
return :: None
"""
@@ -128,13 +155,15 @@ def __doPlot(self, plotData, outputFileName, title, ylabel):
g.ylabel(ylabel)
g('set key top left')
g('set terminal png')
- g('set yrange[0:*]')
+ g('set xrange [%s]' % xRange)
+ g('set yrange [%s]' % yRange)
g('set output "%s"' % outputFileName)
g.plot(*plotData)
def plotPerformanceData(csvLike, titleTemplate, outputTemplate, targetColumn, ylabel,
- patternMap, keyPairs):
+ patternMap, keyPairs, scale=Decimal(1.0), xRange='*:*', yRange='0:*'):
+
"""
"""
assert(isinstance(csvLike, CsvLike))
@@ -144,12 +173,15 @@ def plotPerformanceData(csvLike, titleTemplate, outputTemplate, targetColumn, yl
assert(isinstance(ylabel, str))
assert(isinstance(patternMap, dict))
assert(isinstance(keyPairs, list))
+ assert(isinstance(scale, Decimal))
+ assert(isinstance(xRange, str))
+ assert(isinstance(yRange, str))
for pattern, blockSizeU in keyPairs:
title = titleTemplate % (patternMap[pattern], blockSizeU)
output = outputTemplate % (pattern, blockSizeU)
blockSizeS = str(util.u2s(blockSizeU))
- rp = PerformancePlot(csvLike, pattern, blockSizeS, targetColumn, ylabel, title, output)
+ rp = PerformancePlot(csvLike, pattern, blockSizeS, targetColumn, ylabel, title, output,
+ scale, xRange, yRange)
rp.plot()
-
View
@@ -5,6 +5,7 @@
import re
import time
import itertools
+from decimal import Decimal
import Gnuplot
import util
@@ -32,9 +33,10 @@ def doMain():
titleTemplate = 'Throughput with pattern %s, blocksize %s'
targetColumn = 'Bps'
- ylabel = 'Throughput [bytes/sec]'
+ ylabel = 'Throughput [MB/sec]'
- plotPerformanceData(c, titleTemplate, outputTemplate, targetColumn, ylabel, patternMap, pairs)
+ plotPerformanceData(c, titleTemplate, outputTemplate, targetColumn, ylabel, patternMap, pairs,
+ scale=Decimal('1')/Decimal('1000000'), xRange='*:*', yRange='0:5000')
f.close()
if __name__ == "__main__":
Oops, something went wrong.

0 comments on commit f662604

Please sign in to comment.