Permalink
Browse files

totally revamped how benchmarks work

  • Loading branch information...
1 parent a7f8761 commit 42b9c43dbc88f37616474b6001df094201e03a36 Onsi Fakhouri committed Sep 18, 2013
Showing with 415 additions and 239 deletions.
  1. +86 −0 benchmarker.go
  2. +90 −0 benchmarker_test.go
  3. +4 −4 config/config.go
  4. +39 −12 default_reporter.go
  5. +3 −3 enums_and_types.go
  6. +16 −79 example.go
  7. +4 −4 example_collection.go
  8. +14 −14 example_collection_test.go
  9. +23 −60 example_test.go
  10. +8 −8 ginkgo.go
  11. +47 −0 measure_node.go
  12. +44 −0 measure_node_test.go
  13. +21 −11 reporter_interface.go
  14. +0 −25 runnable_node.go
  15. +0 −13 runnable_node_test.go
  16. +2 −2 suite.go
  17. +14 −4 suite_test.go
View
@@ -0,0 +1,86 @@
+package ginkgo
+
+import (
+ "math"
+ "time"
+)
+
+type Benchmarker interface {
+ Time(name string, body func(), info ...interface{}) (elapsedTime time.Duration)
+ RecordValue(name string, value float64, info ...interface{})
+}
+
+type benchmarker struct {
+ measurements map[string]*ExampleMeasurement
+}
+
+func newBenchmarker() *benchmarker {
+ return &benchmarker{
+ measurements: make(map[string]*ExampleMeasurement, 0),
+ }
+}
+
+func (b *benchmarker) Time(name string, body func(), info ...interface{}) (elapsedTime time.Duration) {
+ t := time.Now()
+ body()
+ elapsedTime = time.Since(t)
+
+ measurement := b.getMeasurement(name, "Fastest Time", "Slowest Time", "Average Time", "s", info...)
+ measurement.Results = append(measurement.Results, elapsedTime.Seconds())
+
+ return
+}
+
+func (b *benchmarker) RecordValue(name string, value float64, info ...interface{}) {
+ measurement := b.getMeasurement(name, "Smallest", " Largest", " Average", "", info...)
+ measurement.Results = append(measurement.Results, value)
+}
+
+func (b *benchmarker) getMeasurement(name string, smallestLabel string, largestLabel string, averageLabel string, units string, info ...interface{}) *ExampleMeasurement {
+ measurement, ok := b.measurements[name]
+ if !ok {
+ var computedInfo interface{}
+ computedInfo = nil
+ if len(info) > 0 {
+ computedInfo = info[0]
+ }
+ measurement = &ExampleMeasurement{
+ Name: name,
+ Info: computedInfo,
+ SmallestLabel: smallestLabel,
+ LargestLabel: largestLabel,
+ AverageLabel: averageLabel,
+ Units: units,
+ Results: make([]float64, 0),
+ }
+ b.measurements[name] = measurement
+ }
+
+ return measurement
+}
+
+func (b *benchmarker) measurementsReport() map[string]*ExampleMeasurement {
+ for _, measurement := range b.measurements {
+ measurement.Smallest = math.MaxFloat64
+ measurement.Largest = -math.MaxFloat64
+ sum := float64(0)
+ sumOfSquares := float64(0)
+
+ for _, result := range measurement.Results {
+ if result > measurement.Largest {
+ measurement.Largest = result
+ }
+ if result < measurement.Smallest {
+ measurement.Smallest = result
+ }
+ sum += result
+ sumOfSquares += result * result
+ }
+
+ n := float64(len(measurement.Results))
+ measurement.Average = sum / n
+ measurement.StdDeviation = math.Sqrt(sumOfSquares/n - (sum/n)*(sum/n))
+ }
+
+ return b.measurements
+}
View
@@ -0,0 +1,90 @@
+package ginkgo
+
+import (
+ . "github.com/onsi/gomega"
+ "time"
+)
+
+func init() {
+ var benchmarker *benchmarker
+
+ Describe("Benchmarker", func() {
+ BeforeEach(func() {
+ benchmarker = newBenchmarker()
+ })
+
+ Describe("Value", func() {
+ BeforeEach(func() {
+ benchmarker.RecordValue("foo", 7, "info!")
+ benchmarker.RecordValue("foo", 2)
+ benchmarker.RecordValue("foo", 3)
+ benchmarker.RecordValue("bar", 0.3)
+ benchmarker.RecordValue("bar", 0.1)
+ benchmarker.RecordValue("bar", 0.5)
+ benchmarker.RecordValue("bar", 0.7)
+ })
+
+ It("records passed in values and reports on them", func() {
+ report := benchmarker.measurementsReport()
+ Ω(report).Should(HaveLen(2))
+ Ω(report["foo"].Name).Should(Equal("foo"))
+ Ω(report["foo"].Info).Should(Equal("info!"))
+ Ω(report["foo"].SmallestLabel).Should(Equal("Smallest"))
+ Ω(report["foo"].LargestLabel).Should(Equal(" Largest"))
+ Ω(report["foo"].AverageLabel).Should(Equal(" Average"))
+ Ω(report["foo"].Units).Should(Equal(""))
+ Ω(report["foo"].Results).Should(Equal([]float64{7, 2, 3}))
+ Ω(report["foo"].Smallest).Should(BeNumerically("==", 2))
+ Ω(report["foo"].Largest).Should(BeNumerically("==", 7))
+ Ω(report["foo"].Average).Should(BeNumerically("==", 4))
+ Ω(report["foo"].StdDeviation).Should(BeNumerically("~", 2.16, 0.01))
+
+ Ω(report["bar"].Name).Should(Equal("bar"))
+ Ω(report["bar"].Info).Should(BeNil())
+ Ω(report["bar"].SmallestLabel).Should(Equal("Smallest"))
+ Ω(report["bar"].LargestLabel).Should(Equal(" Largest"))
+ Ω(report["bar"].AverageLabel).Should(Equal(" Average"))
+ Ω(report["foo"].Units).Should(Equal(""))
+ Ω(report["bar"].Results).Should(Equal([]float64{0.3, 0.1, 0.5, 0.7}))
+ Ω(report["bar"].Smallest).Should(BeNumerically("==", 0.1))
+ Ω(report["bar"].Largest).Should(BeNumerically("==", 0.7))
+ Ω(report["bar"].Average).Should(BeNumerically("==", 0.4))
+ Ω(report["bar"].StdDeviation).Should(BeNumerically("~", 0.22, 0.01))
+ })
+ })
+
+ Describe("Time", func() {
+ BeforeEach(func() {
+ benchmarker.Time("foo", func() {
+ time.Sleep(100 * time.Millisecond)
+ }, "info!")
+ benchmarker.Time("foo", func() {
+ time.Sleep(200 * time.Millisecond)
+ })
+ benchmarker.Time("foo", func() {
+ time.Sleep(170 * time.Millisecond)
+ })
+ })
+
+ It("records passed in values and reports on them", func() {
+ report := benchmarker.measurementsReport()
+ Ω(report).Should(HaveLen(1))
+ Ω(report["foo"].Name).Should(Equal("foo"))
+ Ω(report["foo"].Info).Should(Equal("info!"))
+ Ω(report["foo"].SmallestLabel).Should(Equal("Fastest Time"))
+ Ω(report["foo"].LargestLabel).Should(Equal("Slowest Time"))
+ Ω(report["foo"].AverageLabel).Should(Equal("Average Time"))
+ Ω(report["foo"].Units).Should(Equal("s"))
+ Ω(report["foo"].Results).Should(HaveLen(3))
+ Ω(report["foo"].Results[0]).Should(BeNumerically("~", 0.1, 0.01))
+ Ω(report["foo"].Results[1]).Should(BeNumerically("~", 0.2, 0.01))
+ Ω(report["foo"].Results[2]).Should(BeNumerically("~", 0.17, 0.01))
+ Ω(report["foo"].Smallest).Should(BeNumerically("~", 0.1, 0.01))
+ Ω(report["foo"].Largest).Should(BeNumerically("~", 0.2, 0.01))
+ Ω(report["foo"].Average).Should(BeNumerically("~", 0.16, 0.01))
+ Ω(report["foo"].StdDeviation).Should(BeNumerically("~", 0.04, 0.01))
+ })
+ })
+
+ })
+}
View
@@ -15,7 +15,7 @@ type GinkgoConfigType struct {
FocusString string
ParallelNode int
ParallelTotal int
- SkipBenchmarks bool
+ SkipMeasurements bool
FailOnPending bool
}
@@ -40,7 +40,7 @@ func Flags(prefix string, includeParallelFlags bool) {
prefix = processPrefix(prefix)
flag.Int64Var(&(GinkgoConfig.RandomSeed), prefix+"seed", time.Now().Unix(), "The seed used to randomize the spec suite.")
flag.BoolVar(&(GinkgoConfig.RandomizeAllSpecs), prefix+"randomizeAllSpecs", false, "If set, ginkgo will randomize all specs together. By default, ginkgo only randomizes the top level Describe/Context groups.")
- flag.BoolVar(&(GinkgoConfig.SkipBenchmarks), prefix+"skipBenchmarks", false, "If set, ginkgo will skip any benchmark specs.")
+ flag.BoolVar(&(GinkgoConfig.SkipMeasurements), prefix+"skipMeasurements", false, "If set, ginkgo will skip any measurement specs.")
flag.BoolVar(&(GinkgoConfig.FailOnPending), prefix+"failOnPending", false, "If set, ginkgo will mark the test suite as failed if any specs are pending.")
flag.StringVar(&(GinkgoConfig.FocusString), prefix+"focus", "", "If set, ginkgo will only run specs that match this regular expression.")
@@ -66,8 +66,8 @@ func BuildFlagArgs(prefix string, ginkgo GinkgoConfigType, reporter DefaultRepor
result = append(result, fmt.Sprintf("--%srandomizeAllSpecs", prefix))
}
- if ginkgo.SkipBenchmarks {
- result = append(result, fmt.Sprintf("--%sskipBenchmarks", prefix))
+ if ginkgo.SkipMeasurements {
+ result = append(result, fmt.Sprintf("--%sskipMeasurements", prefix))
}
if ginkgo.FailOnPending {
View
@@ -174,8 +174,8 @@ func (reporter *defaultReporter) printBlockWithMessage(header string, message st
func (reporter *defaultReporter) printStatus(color string, message string, exampleSummary *ExampleSummary) {
if exampleSummary.State == ExampleStatePending && reporter.config.NoisyPendings {
reporter.printBlockWithMessage(reporter.colorize(color, "%s [PENDING]", message), "", exampleSummary)
- } else if exampleSummary.State == ExampleStatePassed && exampleSummary.Benchmark.IsBenchmark {
- reporter.printBlockWithMessage(reporter.colorize(color, "%s [BENCHMARK]", message), reporter.benchmarkReport(exampleSummary.Benchmark), exampleSummary)
+ } else if exampleSummary.State == ExampleStatePassed && exampleSummary.IsMeasurement {
+ reporter.printBlockWithMessage(reporter.colorize(color, "%s [MEASUREMENT]", message), reporter.measurementReport(exampleSummary), exampleSummary)
} else if exampleSummary.RunTime.Seconds() >= reporter.config.SlowSpecThreshold {
reporter.printBlockWithMessage(reporter.colorize(color, "%s [SLOW TEST:%.3f seconds]", message, exampleSummary.RunTime.Seconds()), "", exampleSummary)
} else {
@@ -184,14 +184,41 @@ func (reporter *defaultReporter) printStatus(color string, message string, examp
}
}
-func (reporter *defaultReporter) benchmarkReport(benchmark ExampleBenchmark) string {
- return fmt.Sprintf("Ran %s samples:\n Fastest Sample: %ss\n Slowest Sample: %ss\n Average Time: %ss ± %ss",
- reporter.colorize(boldStyle, "%d", benchmark.NumberOfSamples),
- reporter.colorize(greenColor, "%.3f", benchmark.FastestTime.Seconds()),
- reporter.colorize(redColor, "%.3f", benchmark.SlowestTime.Seconds()),
- reporter.colorize(cyanColor, "%.3f", benchmark.AverageTime.Seconds()),
- reporter.colorize(cyanColor, "%.3f", benchmark.StdDeviation.Seconds()),
- )
+func (reporter *defaultReporter) measurementReport(exampleSummary *ExampleSummary) (message string) {
+ if len(exampleSummary.Measurements) == 0 {
+ return "Found no measurements"
+ }
+
+ message = fmt.Sprintf("Ran %s samples:\n", reporter.colorize(boldStyle, "%d", exampleSummary.NumberOfSamples))
+ i := 0
+ for _, measurement := range exampleSummary.Measurements {
+ if i > 0 {
+ message += "\n"
+ }
+ info := ""
+ if measurement.Info != nil {
+ info = fmt.Sprintf("%v\n", measurement.Info)
+ }
+
+ message += fmt.Sprintf("%s:\n%s %s: %s%s\n %s: %s%s\n %s: %s%s ± %s%s\n",
+ reporter.colorize(boldStyle, "%s", measurement.Name),
+ info,
+ measurement.SmallestLabel,
+ reporter.colorize(greenColor, "%.3f", measurement.Smallest),
+ measurement.Units,
+ measurement.LargestLabel,
+ reporter.colorize(redColor, "%.3f", measurement.Largest),
+ measurement.Units,
+ measurement.AverageLabel,
+ reporter.colorize(cyanColor, "%.3f", measurement.Average),
+ measurement.Units,
+ reporter.colorize(cyanColor, "%.3f", measurement.StdDeviation),
+ measurement.Units,
+ )
+ i++
+ }
+
+ return
}
func (reporter *defaultReporter) printFailure(message string, exampleSummary *ExampleSummary) {
@@ -219,8 +246,8 @@ func (reporter *defaultReporter) printFailure(message string, exampleSummary *Ex
blockType = "AfterEach"
case ExampleComponentTypeIt:
blockType = "It"
- case ExampleComponentTypeBenchmark:
- blockType = "Benchmark"
+ case ExampleComponentTypeMeasure:
+ blockType = "Measurement"
}
reporter.println(i+offset, reporter.colorize(redColor+boldStyle, "%s [%s]", exampleSummary.ComponentTexts[i], blockType))
reporter.println(i+offset, reporter.colorize(grayColor, "(%s)", exampleSummary.ComponentCodeLocations[i]))
View
@@ -8,8 +8,8 @@ type node interface {
type exampleSubject interface {
node
- getFlag() flagType
run() (runOutcome, failureData)
+ getFlag() flagType
getCodeLocation() CodeLocation
}
@@ -36,7 +36,7 @@ const (
nodeTypeInvalid nodeType = iota
nodeTypeContainer
nodeTypeIt
- nodeTypeBenchmark
+ nodeTypeMeasure
)
type ExampleState uint
@@ -61,5 +61,5 @@ const (
ExampleComponentTypeJustBeforeEach
ExampleComponentTypeAfterEach
ExampleComponentTypeIt
- ExampleComponentTypeBenchmark
+ ExampleComponentTypeMeasure
)
Oops, something went wrong.

0 comments on commit 42b9c43

Please sign in to comment.