Skip to content

Commit

Permalink
WIP > Added "category" and "environment" to benchmark metadata and re…
Browse files Browse the repository at this point in the history 
…moved the overall run duration
  • Loading branch information
@karmi
karmi committed May 15, 2020
1 parent a2a5c3d commit db3b41b
Show file tree
Hide file tree
Showing 2 changed files with 87 additions and 42 deletions.
43 changes: 32 additions & 11 deletions _benchmarks/runner/main.go
View file
Expand Up @@ -45,16 +45,17 @@ func main() {

start := time.Now().UTC()
config := map[string]string{
"BUILD_ID": "",
"DATA_SOURCE": "",
"CLIENT_BRANCH": "",
"CLIENT_COMMIT": "",
"ELASTICSEARCH_TARGET_URL": "",
"ELASTICSEARCH_REPORT_URL": "",
"TARGET_SERVICE_TYPE": "",
"TARGET_SERVICE_NAME": "",
"TARGET_SERVICE_VERSION": "",
"TARGET_SERVICE_OS_FAMILY": "",
"BUILD_ID": "",
"DATA_SOURCE": "",
"CLIENT_BRANCH": "",
"CLIENT_COMMIT": "",
"CLIENT_BENCHMARK_ENVIRONMENT": "",
"ELASTICSEARCH_TARGET_URL": "",
"ELASTICSEARCH_REPORT_URL": "",
"TARGET_SERVICE_TYPE": "",
"TARGET_SERVICE_NAME": "",
"TARGET_SERVICE_VERSION": "",
"TARGET_SERVICE_OS_FAMILY": "",
}

log.Printf(boldUnderline("Running benchmarks for go-elasticsearch@%s; %s/go%s"), elasticsearch.Version, runner.RuntimeOS, runner.RuntimeVersion)
Expand Down Expand Up @@ -95,7 +96,9 @@ func main() {
RunnerClient: runnerClient,
ReportClient: reportClient,

BuildID: config["BUILD_ID"],
BuildID: config["BUILD_ID"],
Category: os.Getenv("CLIENT_BENCHMARK_CATEGORY"),
Environment: config["CLIENT_BENCHMARK_ENVIRONMENT"],
Target: struct {
OS runner.ConfigOS
Service runner.ConfigService
Expand All @@ -122,6 +125,8 @@ func main() {

operations := []struct {
Action string
Category string
Environment string
NumWarmups int
NumRepetitions int
SetupFunc runner.RunnerFunc
Expand All @@ -130,6 +135,7 @@ func main() {
// ----- Ping() -------------------------------------------------------------------------------
{
Action: "ping",
Category: "core",
NumWarmups: 0,
NumRepetitions: defaultRepetitions,
RunnerFunc: func(n int, c runner.Config) (*esapi.Response, error) {
Expand All @@ -144,6 +150,7 @@ func main() {
// ----- Info() -------------------------------------------------------------------------------
{
Action: "info",
Category: "core",
NumWarmups: 0,
NumRepetitions: defaultRepetitions,
RunnerFunc: func(n int, c runner.Config) (*esapi.Response, error) {
Expand All @@ -158,6 +165,7 @@ func main() {
// ----- Get() --------------------------------------------------------------------------------
{
Action: "get",
Category: "core",
NumWarmups: 100,
NumRepetitions: defaultRepetitions,
SetupFunc: func(n int, c runner.Config) (*esapi.Response, error) {
Expand Down Expand Up @@ -205,6 +213,7 @@ func main() {
// ----- Index() --------------------------------------------------------------------------------
{
Action: "index",
Category: "core",
NumWarmups: 100,
NumRepetitions: defaultRepetitions,
SetupFunc: func(n int, c runner.Config) (*esapi.Response, error) {
Expand Down Expand Up @@ -269,6 +278,18 @@ func main() {
runnerConfig.SetupFunc = operation.SetupFunc
runnerConfig.RunnerFunc = operation.RunnerFunc

if operation.Category != "" {
runnerConfig.Category = operation.Category
} else {
runnerConfig.Category = os.Getenv("CLIENT_BENCHMARK_CATEGORY")
}

if operation.Environment != "" {
runnerConfig.Environment = operation.Environment
} else {
runnerConfig.Environment = os.Getenv("CLIENT_BENCHMARK_ENVIRONMENT")
}

run, err := runner.NewRunner(runnerConfig)
if err != nil {
log.Fatalf("ERROR: %s", err)
Expand Down
86 changes: 55 additions & 31 deletions _benchmarks/runner/runner/runner.go
View file
Expand Up @@ -40,24 +40,8 @@ func init() {
// NewRunner returns new benchmarking runner.
//
func NewRunner(cfg Config) (*Runner, error) {
if cfg.RunnerClient == nil {
return nil, fmt.Errorf("missing cfg.RunnerClient")
}

if cfg.ReportClient == nil {
return nil, fmt.Errorf("missing cfg.ReportClient")
}

if cfg.RunnerFunc == nil {
return nil, fmt.Errorf("missing cfg.RunnerFunc")
}

if cfg.Action == "" {
return nil, fmt.Errorf("missing cfg.Action")
}

if cfg.BuildID == "" {
return nil, fmt.Errorf("missing cfg.BuildID")
if err := validateConfig(cfg); err != nil {
return nil, err
}

indexer, _ := esutil.NewBulkIndexer(
Expand Down Expand Up @@ -89,7 +73,9 @@ type Runner struct {
type Config struct {
BuildID string

Action string
Action string
Category string
Environment string

NumWarmups int
NumRepetitions int
Expand Down Expand Up @@ -169,6 +155,10 @@ func (e *Error) Errs() string {
// Run executes the benchmark runs.
//
func (r *Runner) Run() error {
if err := validateConfig(r.config); err != nil {
return err
}

var errs []error

r.stats = r.stats[:]
Expand All @@ -185,7 +175,6 @@ func (r *Runner) Run() error {
}
}

start := time.Now().UTC()
for n := 1; n <= r.config.NumRepetitions; n++ {
stat := Stats{Start: time.Now().UTC()}
res, err := r.config.RunnerFunc(n, r.config)
Expand All @@ -205,8 +194,7 @@ func (r *Runner) Run() error {
}
}

duration := time.Since(start)
if err := r.SaveStats(duration); err != nil {
if err := r.SaveStats(); err != nil {
return err
}

Expand All @@ -230,13 +218,17 @@ func (r *Runner) Errs() []error {

// SaveStats stores runner statistics in Elasticsearch.
//
func (r *Runner) SaveStats(dur time.Duration) error {
func (r *Runner) SaveStats() error {
var errs []error

for _, s := range r.stats {
record := record{
Timestamp: s.Start,
Tags: []string{"bench", "go-elasticsearch"},
Labels: map[string]string{
"build_id": r.config.BuildID,
"environment": r.config.Environment,
},
Event: recordEvent{
Action: r.config.Action,
Duration: s.Duration.Nanoseconds(),
Expand All @@ -248,9 +240,9 @@ func (r *Runner) SaveStats(dur time.Duration) error {
},
Benchmark: recordBenchmark{
BuildID: r.config.BuildID,
Warmups: r.config.NumWarmups,
Category: r.config.Category,
Environment: r.config.Environment,
Repetitions: r.config.NumRepetitions,
Duration: dur,
Runner: recordRunner{
Service: recordService{
Type: "client",
Expand Down Expand Up @@ -325,12 +317,12 @@ type recordEvent struct {

// recordBenchmark represents the benchmark information for a single iteration.
type recordBenchmark struct {
BuildID string `json:"build_id"`
Warmups int `json:"warmups"`
Repetitions int `json:"repetitions"`
Duration time.Duration `json:"duration"`
Runner recordRunner `json:"runner"`
Target recordTarget `json:"target"`
BuildID string `json:"build_id"`
Repetitions int `json:"repetitions"`
Runner recordRunner `json:"runner"`
Target recordTarget `json:"target"`
Category string `json:"category,omitempty"`
Environment string `json:"environment,omitempty"`
}

// recordRunner represents the information about the runner.
Expand Down Expand Up @@ -382,3 +374,35 @@ type recordGit struct {
Branch string `json:"branch,omitempty"`
Commit string `json:"commit,omitempty"`
}

func validateConfig(cfg Config) error {
if cfg.BuildID == "" {
return fmt.Errorf("missing cfg.BuildID")
}

if cfg.Action == "" {
return fmt.Errorf("missing cfg.Action")
}

if cfg.Category == "" {
return fmt.Errorf("missing cfg.Category")
}

if cfg.Environment == "" {
return fmt.Errorf("missing cfg.Environment")
}

if cfg.RunnerClient == nil {
return fmt.Errorf("missing cfg.RunnerClient")
}

if cfg.ReportClient == nil {
return fmt.Errorf("missing cfg.ReportClient")
}

if cfg.RunnerFunc == nil {
return fmt.Errorf("missing cfg.RunnerFunc")
}

return nil
}

0 comments on commit db3b41b

Please sign in to comment.