Permalink
Browse files

Check in benchmark suite.

  • Loading branch information...
1 parent d22bc51 commit de4a6c752e623f113b09ffc91ec05f00d12cb904 @gregorycollins committed Apr 26, 2012
View
@@ -1,4 +1,4 @@
-Copyright (c) 2011, Google, Inc.
+Copyright (c) 2011-2012, Google, Inc.
All rights reserved.
View
@@ -0,0 +1,28 @@
+Copyright (c) 2011-2012, Google, Inc.
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+ * Neither the name of Google, Inc. nor the names of other contributors may
+ be used to endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,46 @@
+Name: hashtable-benchmark
+Version: 0.1
+Synopsis: Benchmarks for hashtables
+License: BSD3
+License-file: LICENSE
+Author: Gregory Collins
+Maintainer: greg@gregorycollins.net
+Category: Data
+Build-type: Simple
+Cabal-version: >=1.2
+
+Flag chart
+ Default: False
+
+Executable hashtable-benchmark
+ main-is: Main.hs
+ hs-source-dirs: src ../src
+
+ build-depends: base == 4.*,
+ base16-bytestring == 0.1.*,
+ bytestring == 0.9.*,
+ containers == 0.4.*,
+ criterion >= 0.5 && <0.7,
+ csv == 0.1.*,
+ deepseq >= 1.1 && <1.4,
+ filepath == 1.*,
+ hashable >= 1.1 && <2,
+ hashtables >= 1.0.1.3 && <1.1,
+ mtl == 2.*,
+ mwc-random >= 0.8 && <0.13,
+ primitive,
+ statistics >= 0.8 && <0.11,
+ threads >= 0.4 && <0.6,
+ unordered-containers >= 0.2 && <0.3,
+ vector >= 0.7 && <0.10,
+ vector-algorithms >= 0.5 && <0.6
+
+ if flag(chart)
+ Build-depends: Chart == 0.14.*,
+ colour == 2.3.*,
+ data-accessor == 0.2.*
+ Cpp-options: -DCHART
+
+ ghc-options: -O2 -Wall -fwarn-tabs -funbox-strict-fields -threaded
+ -fno-warn-unused-do-bind -rtsopts
+ -with-rtsopts="-A4M -H2G"
@@ -0,0 +1,101 @@
+module Criterion.Collection.Chart
+ ( errBarChart
+ , defaultColors
+ ) where
+
+
+import Criterion.Measurement
+import Data.Accessor
+import Data.Colour
+import Data.Colour.Names
+import Graphics.Rendering.Chart hiding (Vector)
+
+import Criterion.Collection.Sample
+
+
+defaultColors :: [AlphaColour Double]
+defaultColors = cycle $ map opaque [
+ blue,
+ red,
+ brown,
+ black,
+ darkgoldenrod,
+ coral,
+ cyan,
+ darkcyan,
+ darkkhaki,
+ darkmagenta,
+ darkslategrey
+ ]
+
+
+plotErrBars :: String
+ -> CairoLineStyle
+ -> [SampleData]
+ -> Plot Double Double
+plotErrBars name lineStyle samples = toPlot plot
+ where
+ value sd = symErrPoint size m 0 s
+ where
+ size = fromIntegral $ sdInputSize sd
+ (m,s) = computeMeanAndStddev sd
+
+ plot = plot_errbars_values ^= map value samples
+ $ plot_errbars_line_style ^= lineStyle
+ $ plot_errbars_title ^= name
+ $ defaultPlotErrBars
+
+
+plotPoints :: String
+ -> CairoPointStyle
+ -> [SampleData]
+ -> Plot Double Double
+plotPoints name pointStyle samples = toPlot plot
+ where
+ value sd = (fromIntegral size, m)
+ where
+ size = sdInputSize sd
+ (m,_) = computeMeanAndStddev sd
+
+ plot = plot_points_values ^= map value samples
+ $ plot_points_style ^= pointStyle
+ $ plot_points_title ^= name
+ $ defaultPlotPoints
+
+
+errBarChart :: Bool
+ -> Double
+ -> String
+ -> [(AlphaColour Double, String, [SampleData])]
+ -> Renderable ()
+errBarChart logPlot lineWidth plotTitle plotData = toRenderable layout
+ where
+ mkPlot (colour, plotName, samples) = joinPlot eb pts
+ where
+ lStyle = line_width ^= lineWidth
+ $ line_color ^= colour
+ $ defaultPlotErrBars ^. plot_errbars_line_style
+
+ pStyle = filledCircles (1.5 * lineWidth) colour
+
+ eb = plotErrBars plotName lStyle samples
+ pts = plotPoints plotName pStyle samples
+
+ remapLabels = axis_labels ^: f
+ where
+ f labels = map (map g) labels
+ g (x,_) = (x, secs x)
+
+ axisfn = if logPlot
+ then autoScaledLogAxis defaultLogAxis
+ else autoScaledAxis defaultLinearAxis
+
+ layout = layout1_title ^= plotTitle
+ $ layout1_background ^= solidFillStyle (opaque white)
+ $ layout1_left_axis ^: laxis_generate ^= axisfn
+ $ layout1_left_axis ^: laxis_override ^= remapLabels
+ $ layout1_left_axis ^: laxis_title ^= "Time (seconds)"
+ $ layout1_bottom_axis ^: laxis_generate ^= axisfn
+ $ layout1_bottom_axis ^: laxis_title ^= "# of items in collection"
+ $ layout1_plots ^= (map (Left . mkPlot) plotData)
+ $ defaultLayout1
@@ -0,0 +1,100 @@
+{-# LANGUAGE BangPatterns #-}
+{-# LANGUAGE ExistentialQuantification #-}
+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
+
+module Criterion.Collection.Internal.Types
+ ( Workload(..)
+ , WorkloadGenerator
+ , WorkloadMonad(..)
+ , runWorkloadMonad
+ , getRNG
+ , DataStructure(..)
+ , setupData
+ , setupDataIO
+ ) where
+
+------------------------------------------------------------------------------
+import Control.DeepSeq
+import Control.Monad.Reader
+import Data.Vector (Vector)
+import System.Random.MWC
+
+------------------------------------------------------------------------------
+-- Some thoughts on benchmarking modes
+--
+-- * pre-fill data structure, test an operation workload without modifying the
+-- data structure, measure time for each operation
+--
+-- ---> allows you to get fine-grained per-operation times with distributions
+--
+-- * pre-fill data structure, get a bunch of work to do (cumulatively modifying
+-- the data structure), measure time per-operation OR for the whole batch and
+-- divide out
+--
+--
+-- Maybe it will look like this?
+-- > data MeasurementMode = PerBatch | PerOperation
+-- > data WorkloadMode = Pure | Mutating
+
+------------------------------------------------------------------------------
+newtype WorkloadMonad a = WM (ReaderT GenIO IO a)
+ deriving (Monad, MonadIO)
+
+
+------------------------------------------------------------------------------
+runWorkloadMonad :: WorkloadMonad a -> GenIO -> IO a
+runWorkloadMonad (WM m) gen = runReaderT m gen
+
+
+------------------------------------------------------------------------------
+getRNG :: WorkloadMonad GenIO
+getRNG = WM ask
+
+
+------------------------------------------------------------------------------
+-- | Given an 'Int' representing \"input size\", a 'WorkloadGenerator' makes a
+-- 'Workload'. @Workload@s generate operations to prepopulate data structures
+-- with /O(n)/ data items, then generate operations on-demand to benchmark your
+-- data structure according to some interesting distribution.
+type WorkloadGenerator op = Int -> WorkloadMonad (Workload op)
+
+
+------------------------------------------------------------------------------
+data (NFData op) => Workload op = Workload {
+ -- | \"Setup work\" is work that you do to prepopulate a data structure
+ -- to a certain size before testing begins.
+ setupWork :: !(Vector op)
+
+ -- | Given the number of operations to produce, 'genWorkload' spits out a
+ -- randomly-distributed workload simulation to be used in the benchmark.
+ --
+ -- | Some kinds of skewed workload distributions (the canonical example
+ -- being \"frequent lookups for a small set of keys and infrequent
+ -- lookups for the others\") need a certain minimum number of operations
+ -- to be generated to be statistically valid, which only the
+ -- 'WorkloadGenerator' would know how to decide. In these cases, you are
+ -- free to return more than @N@ samples from 'genWorkload', and
+ -- @criterion-collection@ will run them all for you.
+ --
+ -- Otherwise, @criterion-collection@ is free to bootstrap your benchmark
+ -- using as many sample points as it would take to make the results
+ -- statistically relevant.
+ , genWorkload :: !(Int -> WorkloadMonad (Vector op))
+}
+
+
+------------------------------------------------------------------------------
+data DataStructure op = forall m . DataStructure {
+ emptyData :: !(Int -> IO m)
+ , runOperation :: !(m -> op -> IO m)
+}
+
+
+------------------------------------------------------------------------------
+setupData :: m -> (m -> op -> m) -> DataStructure op
+setupData e r = DataStructure (const $ return e) (\m o -> return $ r m o)
+
+
+------------------------------------------------------------------------------
+setupDataIO :: (Int -> IO m) -> (m -> op -> IO m) -> DataStructure op
+setupDataIO = DataStructure
Oops, something went wrong.

0 comments on commit de4a6c7

Please sign in to comment.