Skip to content

Commit

Permalink
Add detailed human-readable report option
Browse files Browse the repository at this point in the history
Output now includes a text/human-readable output as well as the previous JSON output. Additional stats were added to both output types included network stats and response latency percentiles.

* Simplify human readable reporting

* Replace JSON only reporting with text (human readable) and JSON output.

Text format is optimized for readability, JSON
is optimized for automated processing.

Also refactored some packages and updated the README.
  • Loading branch information
youngkin committed Jun 2, 2020
1 parent fb93b9e commit 6fc69be
Show file tree
Hide file tree
Showing 23 changed files with 794 additions and 634 deletions.
6 changes: 5 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@
# Test binary, built with `go test -c`
*.test

# Output of the go coverage tool, specifically when used with LiteIDE
# Output of the go coverage and profile tools
*.out
*.prof

# Dependency directories (remove the comment below to include it)
# vendor/
Expand All @@ -24,3 +25,6 @@ heyyall-linux-amd64
heyyall-linux-arm
internal/coverage.html
internal/testserver/testserver

// VSCode history
.history
181 changes: 125 additions & 56 deletions README.md

Large diffs are not rendered by default.

4 changes: 0 additions & 4 deletions api/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,6 @@ type LoadTestConfig struct {
// both RunDuration and NumRequests is an error. See RunDuration
// above for a bit more info.
NumRequests int
// OutputType specifies if the output will be written in JSON or
// CSV format. Acceptable values are "JSON" and "CSV". If not
// specified output will be in JSON format.
OutputType string
// KeyFile is the name of a file, in PEM format, that contains an SSL private
// key. It will only be used if it has a non-empty value. It can be overridden
// at the Endpoint level.
Expand Down
85 changes: 85 additions & 0 deletions api/report.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
// Copyright (c) 2020 Richard Youngkin. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package api

import "time"

// RqstStats contains a set of common runtime stats reported at both the
// Summary and Endpoint level
type RqstStats struct {
// TimingResultsNanos contains the duration of each request.
TimingResultsNanos []time.Duration
// TotalRqsts is the overall number of requests made during the run
TotalRqsts int64
// TotalRequestDurationNanos is the sum of all request run durations
TotalRequestDurationNanos time.Duration
// MaxRqstDurationNanos is the longest request duration
MaxRqstDurationNanos time.Duration
// NormalizedMaxRqstDurationNanos is the longest request duration rejecting outlier
// durations more than 'x' times the MinRqstDuration
NormalizedMaxRqstDurationNanos time.Duration
// MinRqstDurationNanos is the smallest request duration for an endpoint
MinRqstDurationNanos time.Duration
// AvgRqstDurationNanos is the average duration of a request for an endpoint
AvgRqstDurationNanos time.Duration
}

// EndpointDetail is used to report an overview of the results of
// a load test run for a given endpoint.
type EndpointDetail struct {
// URL is the endpoint URL
URL string
// HTTPMethodStatusDist summarizes, by HTTP method, the number of times a
// given status was returned (e.g., 200, 201, 404, etc). More specifically,
// it is a map keyed by HTTP method containing a map keyed by HTTP status
// referencing the number of times that status was returned.
HTTPMethodStatusDist map[string]map[int]int
// HTTPMethodRqstStats provides summary request statistics by HTTP Method. It is
// map of RqstStats keyed by HTTP method.
HTTPMethodRqstStats map[string]*RqstStats
}

// RunResults is used to report an overview of the results of a
// load test run
type RunResults struct {
// RunSummary is a roll-up of the detailed run results
RunSummary RunSummary
// EndpointSummary describes how often each endpoint was called.
// It is a map keyed by URL of a map keyed by HTTP verb with a value of
// number of requests. So it's a summary of how often each HTTP verb
// was called on each endpoint.
EndpointSummary map[string]map[string]int
// EndpointDetails is the per endpoint summary of results keyed by URL
EndpointDetails map[string]*EndpointDetail `json:",omitempty"`
}

// RunSummary is a roll-up of the detailed run results
type RunSummary struct {
// RqstRatePerSec is the overall request rate per second
// rounded to the nearest integer
RqstRatePerSec float64
// RunDurationNanos is the wall clock duration of the test
RunDurationNanos time.Duration

// MaxRqstRatePerSec is the maximum request rate per second
// over 1/10th of the run duration or number of requests
//MaxRqstRatePerSec int
// MinRqstRatePerSec is the maximum request rate per second
// over 1/10th of the run duration or number of requests
//MinRqstRatePerSec int

// RqstStats is a summary of runtime statistics
RqstStats RqstStats
// DNSLookupNanos records how long it took to resolve the hostname to an IP Address
DNSLookupNanos []time.Duration
// TCPConnSetupNanos records how long it took to setup the TCP connection
TCPConnSetupNanos []time.Duration
// RqstRoundTripNanos records duration from the time the TCP connection was setup
// until the response was received
RqstRoundTripNanos []time.Duration
// TLSHandshakeNanos records the time it took to complete the TLS negotiation with
// the server. It's only meaningful for HTTPS connections
TLSHandshakeNanos []time.Duration
}
22 changes: 0 additions & 22 deletions buildexes.sh

This file was deleted.

33 changes: 21 additions & 12 deletions heyyall.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"os"
"os/signal"
"runtime"
"runtime/pprof"
"syscall"
"time"

Expand All @@ -30,7 +31,7 @@ Usage: heyyall -config <ConfigFileLocation> [flags...]
Options:
-loglevel Logging level. Default is 'WARN' (2). 0 is DEBUG, 1 INFO, up to 4 FATAL
-detail Detail level of output report, 'short' or 'long'. Default is 'long'
-out Type of output report, 'text' or 'json'. Default is 'text'
-nf Normalization factor used to compress the output histogram by eliminating long tails.
Lower values provide a finer grained view of the data at the expense of dropping data
associated with the tail of the latency distribution. The latter is partly mitigated by
Expand All @@ -49,13 +50,23 @@ Options:

configFile := flag.String("config", "", "path and filename containing the runtime configuration")
logLevel := flag.Int("loglevel", int(zerolog.WarnLevel), "log level, 0 for debug, 1 info, 2 warn, ...")
reportDetailFlag := flag.String("detail", "long", "what level of report detail is desired, 'short' or 'long'")
outputType := flag.String("out", "text", "what type of report is desired, 'text' or 'json'")
normalizationFactor := flag.Int("nf", 0, "normalization factor used to compress the output histogram by eliminating long tails. If provided, the value must be at least 10. The default is 0 which signifies no normalization will be done")
cpus := flag.Int("cpus", 0, "number of CPUs to use for the test run. Default is 0 which specifies all CPUs are to be used.")
help := flag.Bool("help", false, "help will emit detailed usage instructions and exit")
cpuprofile := flag.String("cpuprofile", "", "write cpu profile to file")

flag.Parse()

if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal().Err(err).Msg("unable to create cpuprofile file")
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}

if *help {
fmt.Println(usage)
return
Expand Down Expand Up @@ -94,20 +105,18 @@ Options:
responseC := make(chan internal.Response, config.MaxConcurrentRqsts)
doneC := make(chan struct{})

var reportDetail internal.ReportDetail = internal.Long
if *reportDetailFlag == "short" {
reportDetail = internal.Short
var reportDetail internal.OutputType = internal.JSON
if *outputType == "text" {
reportDetail = internal.Text
}
responseHandler := &internal.ResponseHandler{
ReportDetail: reportDetail,
ResponseC: responseC,
DoneC: doneC,
NumRqsts: config.NumRequests,
NormFactor: *normalizationFactor,
OutputType: reportDetail,
ResponseC: responseC,
DoneC: doneC,
NumRqsts: config.NumRequests,
NormFactor: *normalizationFactor,
}
go responseHandler.Start()
// Give responseHandler a bit of time to start
time.Sleep(time.Millisecond * 20)

var cert tls.Certificate
if config.CertFile != "" && config.KeyFile != "" {
Expand Down
Loading

0 comments on commit 6fc69be

Please sign in to comment.