Permalink
Browse files

An Accelerate build-bot for automated regression testing

Ignore-this: 10cec1488d29034cc37cca8631e24c4d
- runs the accelerate-examples programs and emails performance results
- sends blame emails on failure

darcs-hash:20110228024126-dcabc-2cc8e907054ff4a4e0613d5074d7587c5282c0ac.gz
  • Loading branch information...
0 parents commit 7d17cf66be181cb0d7f854c1c7ab2b5b97a92ccb @tmcdonell tmcdonell committed Feb 28, 2011
Showing with 751 additions and 0 deletions.
  1. +30 −0 LICENSE
  2. +2 −0 Setup.hs
  3. +34 −0 accelerate-buildbot.cabal
  4. +6 −0 misc/banner.txt
  5. +116 −0 src/Args.hs
  6. +126 −0 src/Benchmarks.hs
  7. +174 −0 src/Build.hs
  8. +105 −0 src/Config.hs
  9. +56 −0 src/Darcs.hs
  10. +76 −0 src/Main.hs
  11. +26 −0 src/Util.hs
30 LICENSE
@@ -0,0 +1,30 @@
+Copyright (c)2011, Trevor L. McDonell
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+
+ * Neither the name of Trevor L. McDonell nor the names of other
+ contributors may be used to endorse or promote products derived
+ from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,2 @@
+import Distribution.Simple
+main = defaultMain
@@ -0,0 +1,34 @@
+Name: accelerate-buildbot
+Version: 0.1.0.0
+Synopsis: Accelerate performance & regression build bot
+Description: Accelerate performance & regression build bot
+License: BSD3
+License-file: LICENSE
+Author: The Accelerate Team
+Maintainer: Trevor L. McDonell <tmcdonell@cse.unsw.edu.au>
+Category: Compilers/Interpreters, Concurrency, Data
+Build-type: Simple
+Cabal-version: >=1.6
+
+Extra-source-files: misc/banner.txt
+
+
+Executable accelerate-buildbot
+ Main-is: Main.hs
+ hs-source-dirs: src
+ ghc-options: -Wall
+
+ Build-depends:
+ base == 4.*,
+ buildbox == 1.4.*,
+ bytestring == 0.9.*,
+ bytestring-csv == 0.1.*,
+ bytestring-lexing == 0.2.*,
+ cmdargs == 0.6.*,
+ containers >= 0.3 && < 0.5,
+ directory >= 1.0 && < 1.2,
+ filepath >= 1.1 && < 1.3,
+ mtl >= 1.1 && < 3.0,
+ old-locale == 1.0.*,
+ random == 1.0.*
+
@@ -0,0 +1,6 @@
+ ______ _____
+______ ____________________ /_________________ __ /_____
+_ __ `/ ___/ ___/ _ \_ /_ _ \_ ___/ __ `/ __/ _ \
+/ /_/ // /__ / /__ / __/ / / __/ / / /_/ // /_ / __/
+\__,_/ \___/ \___/ \___//_/ \___//_/ \__,_/ \__/ \___/
+
@@ -0,0 +1,116 @@
+{-# LANGUAGE DeriveDataTypeable #-}
+-- |
+-- Module : Args
+-- Copyright : [2011] The Accelerate Team
+-- License : BSD3
+--
+-- Maintainer : Trevor L. McDonell <tmcdonell@cse.unsw.edu.au>
+-- Stability : experimental
+-- Portability : non-partable (GHC extensions)
+--
+
+module Args where
+
+import Data.Version
+import Paths_accelerate_buildbot
+
+import System.Console.CmdArgs
+
+data Args = Args
+ {
+ accelerate_repo :: String
+ , daily_at :: Maybe String
+ , with_ghc :: String
+ , output :: Maybe FilePath
+ , with_timestamp :: Bool
+ , upload_to :: Maybe String
+ , compare_to :: Maybe FilePath
+ , compare_swing :: Maybe Double
+ , mail_from :: Maybe String
+ , mail_to :: [String]
+ , mail_fail_to :: [String]
+ , mail_banner :: Maybe FilePath
+ , history :: Maybe FilePath
+ }
+ deriving (Show, Data, Typeable)
+
+defaultArgs :: Args
+defaultArgs = Args
+ {
+ accelerate_repo
+ = "http://code.haskell.org/accelerate"
+ &= name "r"
+ &= help "The Accelerate repository to test"
+ &= typ "DARCS_PATH"
+
+ , with_ghc
+ = "ghc"
+ &= name "w"
+ &= help "Give path to a particular compiler"
+ &= typ "PATH"
+
+ , daily_at
+ = def
+ &= name "a"
+ &= help "Run every day at this time, else once now"
+ &= typ "HH:MM:SS"
+
+ , output
+ = def
+ &= help "Write results to this file"
+ &= typFile
+
+ , with_timestamp
+ = def
+ &= name "t"
+ &= help "Append time stamp to result files"
+
+ , upload_to
+ = def
+ &= help "Upload result files to this address"
+ &= typ "SCP_PATH"
+
+ , compare_to
+ = def
+ &= name "c"
+ &= help "Compare to results in this file"
+ &= typFile
+
+ , compare_swing
+ = def
+ &= name "s"
+ &= help "Treat this fractional swing as interesting"
+ &= typ "DOUBLE"
+
+ , mail_from
+ = def
+ &= help "Send test results from this address"
+ &= typ "ADDRESS"
+
+ , mail_to
+ = def
+ &= help "Send test results to this address"
+ &= typ "ADDRESS"
+
+ , mail_fail_to
+ = def
+ &= help "Send failure notifications to this address"
+ &= typ "ADDRESS"
+
+ , mail_banner
+ = def
+ &= help "Append a banner to the result email"
+ &= typFile
+
+ , history
+ = def
+ &= explicit
+ &= name "history"
+ &= help "Where to stash the buildbot history"
+ &= typFile
+
+ } &= program "accelerate-buildbot"
+ &= summary "accelerate-buildbot (c) 2011 The Accelerate Team"
+ &= versionArg [summary $ "accelerate-buildbot-" ++ showVersion version]
+ &= verbosityArgs [help "Verbose logging of build commands"] [ignore]
+
@@ -0,0 +1,126 @@
+{-# LANGUAGE OverloadedStrings #-}
+-- |
+-- Module : Benchmarks
+-- Copyright : [2011] The Accelerate Team
+-- License : BSD3
+--
+-- Maintainer : Trevor L. McDonell <tmcdonell@cse.unsw.edu.au>
+-- Stability : experimental
+-- Portability : non-partable (GHC extensions)
+--
+
+module Benchmarks (runAccBenchmarks) where
+
+import Config
+import BuildBox
+
+import Control.Applicative
+import Data.Char
+import Data.Maybe
+import Text.CSV.ByteString
+import System.FilePath
+import qualified Data.Sequence as Seq
+import qualified BuildBox.Data.Log as Log
+import qualified Data.ByteString.Char8 as B
+import qualified Data.ByteString.Lex.Double as B
+
+
+
+-- The list of benchmarks. In most cases it is expected that the program write
+-- timing information from criterion (or similar) to the given file. It should
+-- also be linked with RTS options to query heap usage.
+--
+-- Note that we require the criterion name for the benchmark to match that below
+--
+benchmarks :: Config -> FilePath -> [Benchmark]
+benchmarks _config summaryLog =
+ let simple name args =
+ Benchmark
+ { benchmarkName = name
+ , benchmarkSetup = return ()
+ , benchmarkCommand = runAcc summaryLog ("accelerate-examples/dist/build" </> name </> name) args
+ , benchmarkCheck = return [] }
+ in
+ [ simple "acc-saxpy" [] ]
+
+
+-- | Execute all benchmarks and return statistics for each.
+--
+runAccBenchmarks :: Config -> Build [BenchResult Stats]
+runAccBenchmarks config = withTempFile $ \f -> do
+ runs <- mapM run (benchmarks config f)
+ crit <- readCriterionStats f
+ return $ combineBenchStats runs crit
+ where
+ unit x = [x]
+ run x = statBenchResult
+ . BenchResult (benchmarkName x) . unit <$> runBenchmarkOnce 1 x
+
+ combineBenchStats = zipWith . liftBenchRunResult2 $ \b1 b2 ->
+ [BenchRunResult 0
+ (concatMap benchRunResultQuirks b1 ++ concatMap benchRunResultQuirks b2)
+ (concatMap benchRunResultAspects b1 ++ concatMap benchRunResultAspects b2) ]
+
+
+-- Read criterion statistics. The file contains the following header, which we
+-- ignore, and corresponding format:
+--
+-- "Name,Mean,MeanLB,MeanUB,Stddev,StddevLB,StddevUB"
+--
+readCriterionStats :: FilePath -> Build [BenchResult Stats]
+readCriterionStats f = do
+ contents <- io $ B.readFile f
+ case parseCSV contents of
+ Nothing -> throw $ ErrorOther "failed to parse criterion results"
+ Just csv -> return $ map parse (tail csv)
+
+ where
+ seconds = Seconds . fst . fromJust . B.readDouble
+ stats avg lb ub = WithSeconds (Time KernelWall Stats { statsMin = seconds lb
+ , statsMax = seconds ub
+ , statsAvg = seconds avg })
+
+ parse (n:avg:lb:ub:_) = BenchResult (B.unpack n) [BenchRunResult 0 [] [ stats avg lb ub ]]
+ parse _ = error "we should not be here..."
+
+
+-- Run a standard accelerate-examples benchmark. The executable is expected to
+-- write kernel statistics to the given file, which will be read later.
+--
+-- TLM: return the wall time for the entire program? Includes random number
+-- generation and the criterion machinery, but also important Accelerate
+-- operations such as sharing recovery and code generation.
+--
+runAcc :: FilePath -> FilePath -> [String] -> Build [WithUnits (Aspect Single)]
+runAcc tlog exe args = do
+ let cmd = unwords [exe, "--summary=" ++ tlog, unwords args, "+RTS -t"]
+ (status, logOut, logErr) <- systemTeeLog False cmd Log.empty
+ case status of
+ ExitSuccess -> return $ parseGCLine logErr
+ _ -> throw $ ErrorSystemCmdFailed cmd status logOut logErr
+
+
+-- Parse one-line GC summary statistics and return heap allocation and maximum
+-- residency aspects.
+--
+-- Example (from ghc-7.0.1):
+--
+-- <<ghc: 3074525896 bytes, 4949 GCs, 2674563/6977652 avg/max bytes residency (8 samples), ...
+-- 15M in use, 0.01 INIT (0.00 elapsed), 5.56 MUT (6.61 elapsed), 0.19 GC (0.21 elapsed) :ghc>>
+--
+parseGCLine :: Log.Log -> [WithUnits (Aspect Single)]
+parseGCLine gc =
+ case index of
+ Nothing -> []
+ Just i ->
+ let toks = B.words $ Seq.index gc i
+ in
+ [ Used HeapAlloc `bytes` read' (toks !! 1)
+ , Used HeapMax `bytes` read' (snd . B.spanEnd isDigit $ toks !! 5) ]
+
+ where
+ read' = fst . fromJust . B.readInteger
+ index = Seq.findIndexR (\l -> B.isPrefixOf "<<ghc:" l && B.isSuffixOf ":ghc>>" l) gc
+ -- TLM: could reasonably assume that this is the last line of the log,
+ -- since it is printed by the RTS after program termination.
+
Oops, something went wrong.

0 comments on commit 7d17cf6

Please sign in to comment.