From 5bdae700c22195a69f36c009d4c53b4ff361d570 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 8 Mar 2024 17:03:29 +0100 Subject: [PATCH 01/10] first version --- cmd/benchmark.go | 47 +++++++++++++ internal/benchrunner/runners/stream/runner.go | 66 +++++++++++++++---- 2 files changed, 102 insertions(+), 11 deletions(-) diff --git a/cmd/benchmark.go b/cmd/benchmark.go index 9675e37858..0bac552e26 100644 --- a/cmd/benchmark.go +++ b/cmd/benchmark.go @@ -85,6 +85,9 @@ func setupBenchmarkCommand() *cobraext.Command { systemCmd := getSystemCommand() cmd.AddCommand(systemCmd) + validateCmd := getValidateCommand() + cmd.AddCommand(validateCmd) + return cobraext.NewCommand(cmd, cobraext.ContextPackage) } @@ -519,6 +522,50 @@ func streamCommandAction(cmd *cobra.Command, args []string) error { return nil } +func getValidateCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "validate", + Short: "Validate benchmarks assets", + Long: "Statically validate benchmarks assets for the package", + Args: cobra.NoArgs, + RunE: validateCommandAction, + } + + // cmd.Flags().StringP(cobraext.BenchNameFlagName, "", "", cobraext.BenchNameFlagDescription) + // cmd.Flags().String(cobraext.VariantFlagName, "", cobraext.VariantFlagDescription) + // cmd.Flags().DurationP(cobraext.BenchStreamBackFillFlagName, "", 15*time.Minute, cobraext.BenchStreamBackFillFlagDescription) + // cmd.Flags().Uint64P(cobraext.BenchStreamEventsPerPeriodFlagName, "", 10, cobraext.BenchStreamEventsPerPeriodFlagDescription) + // cmd.Flags().DurationP(cobraext.BenchStreamPeriodDurationFlagName, "", 10*time.Second, cobraext.BenchStreamPeriodDurationFlagDescription) + // cmd.Flags().BoolP(cobraext.BenchStreamPerformCleanupFlagName, "", false, cobraext.BenchStreamPerformCleanupFlagDescription) + // cmd.Flags().StringP(cobraext.BenchStreamTimestampFieldFlagName, "", "timestamp", cobraext.BenchStreamTimestampFieldFlagDescription) + + return cmd +} + +func validateCommandAction(cmd *cobra.Command, args []string) error { + cmd.Println("Run benchmark asset validation") + + packageRootPath, found, err := packages.FindPackageRoot() + if !found { + return errors.New("package root not found") + } + if err != nil { + return fmt.Errorf("error finding package root: %w", err) + } + + withOpts := []stream.OptionFunc{ + stream.WithPackageRootPath(packageRootPath), + } + + err = stream.StaticValidation(stream.NewOptions(withOpts...)) + + if err != nil { + return fmt.Errorf("validation failed: %w", err) + } + + return nil +} + func getSystemCommand() *cobra.Command { cmd := &cobra.Command{ Use: "system", diff --git a/internal/benchrunner/runners/stream/runner.go b/internal/benchrunner/runners/stream/runner.go index ce91b004b6..cc184e4698 100644 --- a/internal/benchrunner/runners/stream/runner.go +++ b/internal/benchrunner/runners/stream/runner.go @@ -63,6 +63,16 @@ func (r *runner) SetUp() error { return r.setUp() } +func StaticValidation(opts Options) error { + runner := runner{options: opts} + err := runner.initialize() + if err != nil { + return err + } + err = runner.validateGenerators() + return err +} + // Run runs the system benchmarks defined under the given folder func (r *runner) Run() (reporters.Reportable, error) { return nil, r.run() @@ -99,15 +109,9 @@ func (r *runner) TearDown() error { return merr } -func (r *runner) setUp() error { +func (r *runner) initialize() error { r.generators = make(map[string]genlib.Generator) r.backFillGenerators = make(map[string]genlib.Generator) - r.errChanGenerators = make(chan error) - r.done = make(chan struct{}) - - r.runtimeDataStreams = make(map[string]string) - - r.ctxt.Test.RunID = common.NewRunID() pkgManifest, err := packages.ReadPackageManifestFromPackageRoot(r.options.PackageRootPath) if err != nil { @@ -120,15 +124,55 @@ func (r *runner) setUp() error { } r.scenarios = scenarios - if err = r.installPackage(); err != nil { - return fmt.Errorf("error installing package: %w", err) - } - err = r.collectGenerators() if err != nil { return fmt.Errorf("can't initialize generator: %w", err) } + return nil +} + +const NumberOfEvents = 10 + +func (r *runner) validateGenerators() error { + for scenarioName, generator := range r.generators { + // run for NumberOfEvents times and check whether the generated event is valid json + for i := 0; i < NumberOfEvents; i++ { + buf := bytes.NewBufferString("") + err := generator.Emit(buf) + if err != nil { + return fmt.Errorf("[%s] error while generating event: %w", scenarioName, err) + } + // check whether the generated event is valid json + var event map[string]any + err = json.Unmarshal(buf.Bytes(), &event) + if err != nil { + return fmt.Errorf("[%s] failed to unmarshal json event: %w, generated output: %s", scenarioName, err, buf.String()) + } + } + } + + return nil +} + +func (r *runner) setUp() error { + r.initialize() + r.errChanGenerators = make(chan error) + r.done = make(chan struct{}) + + r.runtimeDataStreams = make(map[string]string) + + r.ctxt.Test.RunID = common.NewRunID() + + pkgManifest, err := packages.ReadPackageManifestFromPackageRoot(r.options.PackageRootPath) + if err != nil { + return fmt.Errorf("reading package manifest failed: %w", err) + } + + if err = r.installPackage(); err != nil { + return fmt.Errorf("error installing package: %w", err) + } + for scenarioName, scenario := range r.scenarios { var err error dataStreamManifest, err := packages.ReadDataStreamManifest( From 68679c2b45c6afe2d20ec1ef7bcac2933a9582ad Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 8 Mar 2024 17:06:22 +0100 Subject: [PATCH 02/10] remove unncessary stuff@ --- cmd/benchmark.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cmd/benchmark.go b/cmd/benchmark.go index 0bac552e26..077d418634 100644 --- a/cmd/benchmark.go +++ b/cmd/benchmark.go @@ -531,14 +531,6 @@ func getValidateCommand() *cobra.Command { RunE: validateCommandAction, } - // cmd.Flags().StringP(cobraext.BenchNameFlagName, "", "", cobraext.BenchNameFlagDescription) - // cmd.Flags().String(cobraext.VariantFlagName, "", cobraext.VariantFlagDescription) - // cmd.Flags().DurationP(cobraext.BenchStreamBackFillFlagName, "", 15*time.Minute, cobraext.BenchStreamBackFillFlagDescription) - // cmd.Flags().Uint64P(cobraext.BenchStreamEventsPerPeriodFlagName, "", 10, cobraext.BenchStreamEventsPerPeriodFlagDescription) - // cmd.Flags().DurationP(cobraext.BenchStreamPeriodDurationFlagName, "", 10*time.Second, cobraext.BenchStreamPeriodDurationFlagDescription) - // cmd.Flags().BoolP(cobraext.BenchStreamPerformCleanupFlagName, "", false, cobraext.BenchStreamPerformCleanupFlagDescription) - // cmd.Flags().StringP(cobraext.BenchStreamTimestampFieldFlagName, "", "timestamp", cobraext.BenchStreamTimestampFieldFlagDescription) - return cmd } From 4ff5e2afe6785784ac006207bd9eca46eaaf70f8 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 8 Mar 2024 17:14:37 +0100 Subject: [PATCH 03/10] terminate setUp early in case of initialize error --- internal/benchrunner/runners/stream/runner.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/benchrunner/runners/stream/runner.go b/internal/benchrunner/runners/stream/runner.go index cc184e4698..2f1665467c 100644 --- a/internal/benchrunner/runners/stream/runner.go +++ b/internal/benchrunner/runners/stream/runner.go @@ -156,7 +156,10 @@ func (r *runner) validateGenerators() error { } func (r *runner) setUp() error { - r.initialize() + err := r.initialize() + if err != nil { + return err + } r.errChanGenerators = make(chan error) r.done = make(chan struct{}) From 3410aafa7dc0f4444f18dca9b521b83f7eb26eda Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 8 Mar 2024 17:22:42 +0100 Subject: [PATCH 04/10] update readme --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index dca14997ec..2a027d600f 100644 --- a/README.md +++ b/README.md @@ -199,6 +199,12 @@ _Context: package_ Run system benchmarks for the package. +### `elastic-package benchmark validate` + +_Context: package_ + +Statically validate benchmarks assets for the package. + ### `elastic-package build` _Context: package_ From fa87a81c20e8c5a9b428b48833fd898f6a206e88 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 13 Mar 2024 13:03:51 +0100 Subject: [PATCH 05/10] review comments --- internal/benchrunner/runners/stream/runner.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/internal/benchrunner/runners/stream/runner.go b/internal/benchrunner/runners/stream/runner.go index 2f1665467c..5a2eaaf561 100644 --- a/internal/benchrunner/runners/stream/runner.go +++ b/internal/benchrunner/runners/stream/runner.go @@ -37,6 +37,8 @@ import ( "github.com/elastic/elastic-package/internal/signal" ) +const numberOfEvents = 10 + type runner struct { options Options scenarios map[string]*scenario @@ -132,12 +134,9 @@ func (r *runner) initialize() error { return nil } -const NumberOfEvents = 10 - func (r *runner) validateGenerators() error { for scenarioName, generator := range r.generators { - // run for NumberOfEvents times and check whether the generated event is valid json - for i := 0; i < NumberOfEvents; i++ { + for i := 0; i < numberOfEvents; i++ { buf := bytes.NewBufferString("") err := generator.Emit(buf) if err != nil { @@ -342,17 +341,17 @@ func (r *runner) collectGenerators() error { for scenarioName, scenario := range r.scenarios { config, err := r.getGeneratorConfig(scenario) if err != nil { - return err + return fmt.Errorf("[%s]: %w", scenarioName, err) } fields, err := r.getGeneratorFields(scenario) if err != nil { - return err + return fmt.Errorf("[%s]: %w", scenarioName, err) } tpl, err := r.getGeneratorTemplate(scenario) if err != nil { - return err + return fmt.Errorf("[%s]: %w", scenarioName, err) } genlib.InitGeneratorTimeNow(time.Now()) @@ -360,7 +359,7 @@ func (r *runner) collectGenerators() error { generator, err := r.initializeGenerator(tpl, *config, fields, scenario, 0, 0) if err != nil { - return err + return fmt.Errorf("[%s]: %w", scenarioName, err) } r.generators[scenarioName] = generator @@ -374,7 +373,7 @@ func (r *runner) collectGenerators() error { generator, err = r.initializeGenerator(tpl, *config, fields, scenario, r.options.BackFill, totEvents) if err != nil { - return err + return fmt.Errorf("[%s]: %w", scenarioName, err) } r.backFillGenerators[scenarioName] = generator From c8038ce749dc80cac92451f41155af4ac1cca91a Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 22 Mar 2024 10:54:49 +0100 Subject: [PATCH 06/10] bump number of generated events --- internal/benchrunner/runners/stream/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/benchrunner/runners/stream/runner.go b/internal/benchrunner/runners/stream/runner.go index 83f2c1a073..d2a6c0a7ee 100644 --- a/internal/benchrunner/runners/stream/runner.go +++ b/internal/benchrunner/runners/stream/runner.go @@ -35,7 +35,7 @@ import ( "github.com/elastic/elastic-package/internal/wait" ) -const numberOfEvents = 10 +const numberOfEvents = 100 type runner struct { options Options From b6c36eb0b251932d44cee6fe7968f8a73836bc62 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 22 Mar 2024 11:12:18 +0100 Subject: [PATCH 07/10] review comments --- internal/benchrunner/runners/stream/runner.go | 10 +++++----- internal/benchrunner/runners/stream/scenario.go | 4 ++++ internal/testrunner/runners/static/runner.go | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/internal/benchrunner/runners/stream/runner.go b/internal/benchrunner/runners/stream/runner.go index d2a6c0a7ee..df550b57b0 100644 --- a/internal/benchrunner/runners/stream/runner.go +++ b/internal/benchrunner/runners/stream/runner.go @@ -311,17 +311,17 @@ func (r *runner) collectGenerators(ctx context.Context) error { for scenarioName, scenario := range r.scenarios { config, err := r.getGeneratorConfig(scenario) if err != nil { - return fmt.Errorf("[%s]: %w", scenarioName, err) + return fmt.Errorf("failed to obtain generator config for scenario %q: %w", scenarioName, err) } fields, err := r.getGeneratorFields(ctx, scenario) if err != nil { - return fmt.Errorf("[%s]: %w", scenarioName, err) + return fmt.Errorf("failed to obtain fields from generator for scenario %q: %w", scenarioName, err) } tpl, err := r.getGeneratorTemplate(scenario) if err != nil { - return fmt.Errorf("[%s]: %w", scenarioName, err) + return fmt.Errorf("failed to obtain template from for scenario %q: %w", scenarioName, err) } genlib.InitGeneratorTimeNow(time.Now()) @@ -329,7 +329,7 @@ func (r *runner) collectGenerators(ctx context.Context) error { generator, err := r.initializeGenerator(tpl, *config, fields, scenario, 0, 0) if err != nil { - return fmt.Errorf("[%s]: %w", scenarioName, err) + return fmt.Errorf("failed to initialize backfill generator for scenario %q: %w", scenarioName, err) } r.generators[scenarioName] = generator @@ -343,7 +343,7 @@ func (r *runner) collectGenerators(ctx context.Context) error { generator, err = r.initializeGenerator(tpl, *config, fields, scenario, r.options.BackFill, totEvents) if err != nil { - return fmt.Errorf("[%s]: %w", scenarioName, err) + return fmt.Errorf("failed to initialize backfill generator for scenario %q: %w", scenarioName, err) } r.backFillGenerators[scenarioName] = generator diff --git a/internal/benchrunner/runners/stream/scenario.go b/internal/benchrunner/runners/stream/scenario.go index 36abf5f840..495910cf67 100644 --- a/internal/benchrunner/runners/stream/scenario.go +++ b/internal/benchrunner/runners/stream/scenario.go @@ -85,6 +85,10 @@ func readScenarios(path, scenarioName, packageName, packageVersion string) (map[ } scenarios[scenarioName] = scenario } else { + if _, err := os.Stat(filepath.Join(path, devPath)); os.IsNotExist(err) { + // if the dev path doesn't exist, treat that as no scenarios found + return nil, nil + } err := filepath.Walk(filepath.Join(path, devPath), func(_ string, info os.FileInfo, err error) error { if err != nil { return err diff --git a/internal/testrunner/runners/static/runner.go b/internal/testrunner/runners/static/runner.go index 7eb7400eee..b25ca1d657 100644 --- a/internal/testrunner/runners/static/runner.go +++ b/internal/testrunner/runners/static/runner.go @@ -80,7 +80,7 @@ func (r runner) run(ctx context.Context) ([]testrunner.TestResult, error) { func (r runner) verifyStreamConfig(ctx context.Context, packageRootPath string) []testrunner.TestResult { resultComposer := testrunner.NewResultComposer(testrunner.TestResult{ - Name: "Verify stream config", + Name: "Verify benchmark config (if available)", TestType: TestType, Package: r.options.TestFolder.Package, DataStream: r.options.TestFolder.DataStream, From bec28e4f919382f648b1bcfde3a24a951aa4b009 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 5 Apr 2024 17:39:24 +0200 Subject: [PATCH 08/10] comment reviews --- cmd/benchmark.go | 42 ------- internal/benchrunner/runners/stream/runner.go | 108 +++++++++++------- internal/testrunner/runners/static/runner.go | 8 +- .../_dev/benchmark/rally/logs-benchmark.yml | 14 +++ .../benchmark/rally/logs-benchmark/config.yml | 22 ++++ .../benchmark/rally/logs-benchmark/fields.yml | 23 ++++ .../rally/logs-benchmark/template.ndjson | 14 +++ .../changelog.yml | 6 + .../_dev/benchmark/pipeline/access-raw.log | 1 + .../testds/_dev/benchmark/pipeline/config.yml | 1 + .../elasticsearch/ingest_pipeline/default.yml | 23 ++++ .../data_stream/testds/fields/base-fields.yml | 38 ++++++ .../data_stream/testds/manifest.yml | 3 + .../docs/README.md | 3 + .../manifest.yml | 13 +++ 15 files changed, 231 insertions(+), 88 deletions(-) create mode 100644 test/packages/false_positives/rally_benchmark_malformed_template/_dev/benchmark/rally/logs-benchmark.yml create mode 100644 test/packages/false_positives/rally_benchmark_malformed_template/_dev/benchmark/rally/logs-benchmark/config.yml create mode 100644 test/packages/false_positives/rally_benchmark_malformed_template/_dev/benchmark/rally/logs-benchmark/fields.yml create mode 100644 test/packages/false_positives/rally_benchmark_malformed_template/_dev/benchmark/rally/logs-benchmark/template.ndjson create mode 100644 test/packages/false_positives/rally_benchmark_malformed_template/changelog.yml create mode 100644 test/packages/false_positives/rally_benchmark_malformed_template/data_stream/testds/_dev/benchmark/pipeline/access-raw.log create mode 100644 test/packages/false_positives/rally_benchmark_malformed_template/data_stream/testds/_dev/benchmark/pipeline/config.yml create mode 100644 test/packages/false_positives/rally_benchmark_malformed_template/data_stream/testds/elasticsearch/ingest_pipeline/default.yml create mode 100644 test/packages/false_positives/rally_benchmark_malformed_template/data_stream/testds/fields/base-fields.yml create mode 100644 test/packages/false_positives/rally_benchmark_malformed_template/data_stream/testds/manifest.yml create mode 100644 test/packages/false_positives/rally_benchmark_malformed_template/docs/README.md create mode 100644 test/packages/false_positives/rally_benchmark_malformed_template/manifest.yml diff --git a/cmd/benchmark.go b/cmd/benchmark.go index a306002e3f..37fc4ae512 100644 --- a/cmd/benchmark.go +++ b/cmd/benchmark.go @@ -84,9 +84,6 @@ func setupBenchmarkCommand() *cobraext.Command { systemCmd := getSystemCommand() cmd.AddCommand(systemCmd) - validateCmd := getValidateCommand() - cmd.AddCommand(validateCmd) - return cobraext.NewCommand(cmd, cobraext.ContextPackage) } @@ -520,45 +517,6 @@ func streamCommandAction(cmd *cobra.Command, args []string) error { return nil } -func getValidateCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "validate", - Short: "Validate benchmarks assets", - Long: "Statically validate benchmarks assets for the package", - Args: cobra.NoArgs, - RunE: validateCommandAction, - } - - return cmd -} - -func validateCommandAction(cmd *cobra.Command, args []string) error { - cmd.Println("Run benchmark asset validation") - - packageRootPath, found, err := packages.FindPackageRoot() - if !found { - return errors.New("package root not found") - } - if err != nil { - return fmt.Errorf("error finding package root: %w", err) - } - - withOpts := []stream.OptionFunc{ - stream.WithPackageRootPath(packageRootPath), - } - - ctx, stop := signal.Enable(cmd.Context(), logger.Info) - defer stop() - - err = stream.StaticValidation(ctx, stream.NewOptions(withOpts...)) - - if err != nil { - return fmt.Errorf("validation failed: %w", err) - } - - return nil -} - func getSystemCommand() *cobra.Command { cmd := &cobra.Command{ Use: "system", diff --git a/internal/benchrunner/runners/stream/runner.go b/internal/benchrunner/runners/stream/runner.go index 35e96c4939..8d507cda11 100644 --- a/internal/benchrunner/runners/stream/runner.go +++ b/internal/benchrunner/runners/stream/runner.go @@ -59,14 +59,14 @@ func (r *runner) SetUp(ctx context.Context) error { return r.setUp(ctx) } -func StaticValidation(ctx context.Context, opts Options) error { +func StaticValidation(ctx context.Context, opts Options, scenarioName string) (bool, error) { runner := runner{options: opts} - err := runner.initialize(ctx) + err := runner.initialize() if err != nil { - return err + return false, err } - err = runner.validateGenerators() - return err + hasBenchmark, err := runner.validateScenario(ctx, scenarioName) + return hasBenchmark, err } // Run runs the system benchmarks defined under the given folder @@ -106,7 +106,7 @@ func (r *runner) TearDown(ctx context.Context) error { return merr } -func (r *runner) initialize(ctx context.Context) error { +func (r *runner) initialize() error { r.generators = make(map[string]genlib.Generator) r.backFillGenerators = make(map[string]genlib.Generator) @@ -121,40 +121,49 @@ func (r *runner) initialize(ctx context.Context) error { } r.scenarios = scenarios - err = r.collectGenerators(ctx) - if err != nil { - return fmt.Errorf("can't initialize generator: %w", err) - } - return nil } -func (r *runner) validateGenerators() error { - for scenarioName, generator := range r.generators { +func (r *runner) validateScenario(ctx context.Context, targetScenarioName string) (bool, error) { + for scenarioName, scenario := range r.scenarios { + println(scenarioName, targetScenarioName) + if scenario.DataStream.Name != targetScenarioName { + continue + } + generator, _, err := r.createGenerator(ctx, scenarioName, scenario) + if err != nil { + return true, err + } for i := 0; i < numberOfEvents; i++ { buf := bytes.NewBufferString("") err := generator.Emit(buf) if err != nil { - return fmt.Errorf("[%s] error while generating event: %w", scenarioName, err) + return true, fmt.Errorf("[%s] error while generating event: %w", scenarioName, err) } // check whether the generated event is valid json var event map[string]any err = json.Unmarshal(buf.Bytes(), &event) if err != nil { - return fmt.Errorf("[%s] failed to unmarshal json event: %w, generated output: %s", scenarioName, err, buf.String()) + return true, fmt.Errorf("[%s] failed to unmarshal json event: %w, generated output: %s", scenarioName, err, buf.String()) } } + return true, nil } - return nil + return false, nil } func (r *runner) setUp(ctx context.Context) error { - err := r.initialize(ctx) + err := r.initialize() if err != nil { return err } + err = r.collectGenerators(ctx) + if err != nil { + return fmt.Errorf("can't initialize generator: %w", err) + } + r.runtimeDataStreams = make(map[string]string) r.svcInfo.Test.RunID = common.NewRunID() @@ -309,47 +318,58 @@ func (r *runner) initializeGenerator(tpl []byte, config genlib.Config, fields ge } func (r *runner) collectGenerators(ctx context.Context) error { for scenarioName, scenario := range r.scenarios { - config, err := r.getGeneratorConfig(scenario) + generator, backfillGenerator, err := r.createGenerator(ctx, scenarioName, scenario) if err != nil { - return fmt.Errorf("failed to obtain generator config for scenario %q: %w", scenarioName, err) + return err } - fields, err := r.getGeneratorFields(ctx, scenario) - if err != nil { - return fmt.Errorf("failed to obtain fields from generator for scenario %q: %w", scenarioName, err) - } + r.generators[scenarioName] = generator - tpl, err := r.getGeneratorTemplate(scenario) - if err != nil { - return fmt.Errorf("failed to obtain template from for scenario %q: %w", scenarioName, err) + if backfillGenerator != nil { + r.backFillGenerators[scenarioName] = backfillGenerator } + } - genlib.InitGeneratorTimeNow(time.Now()) - genlib.InitGeneratorRandSeed(time.Now().UnixNano()) + return nil +} - generator, err := r.initializeGenerator(tpl, *config, fields, scenario, 0, 0) - if err != nil { - return fmt.Errorf("failed to initialize backfill generator for scenario %q: %w", scenarioName, err) - } +func (r *runner) createGenerator(ctx context.Context, scenarioName string, scenario *scenario) (genlib.Generator, genlib.Generator, error) { + config, err := r.getGeneratorConfig(scenario) + if err != nil { + return nil, nil, fmt.Errorf("failed to obtain generator config for scenario %q: %w", scenarioName, err) + } - r.generators[scenarioName] = generator + fields, err := r.getGeneratorFields(ctx, scenario) + if err != nil { + return nil, nil, fmt.Errorf("failed to obtain fields from generator for scenario %q: %w", scenarioName, err) + } - if r.options.BackFill >= 0 { - continue - } + tpl, err := r.getGeneratorTemplate(scenario) + if err != nil { + return nil, nil, fmt.Errorf("failed to obtain template from for scenario %q: %w", scenarioName, err) + } - // backfill is a negative duration, make it positive, find how many periods in the backfill and multiply by events for periodk - totEvents := uint64((-1*r.options.BackFill)/r.options.PeriodDuration) * r.options.EventsPerPeriod + genlib.InitGeneratorTimeNow(time.Now()) + genlib.InitGeneratorRandSeed(time.Now().UnixNano()) - generator, err = r.initializeGenerator(tpl, *config, fields, scenario, r.options.BackFill, totEvents) - if err != nil { - return fmt.Errorf("failed to initialize backfill generator for scenario %q: %w", scenarioName, err) - } + generator, err := r.initializeGenerator(tpl, *config, fields, scenario, 0, 0) + if err != nil { + return nil, nil, fmt.Errorf("failed to initialize backfill generator for scenario %q: %w", scenarioName, err) + } - r.backFillGenerators[scenarioName] = generator + if r.options.BackFill >= 0 { + return generator, nil, nil } - return nil + // backfill is a negative duration, make it positive, find how many periods in the backfill and multiply by events for periodk + totEvents := uint64((-1*r.options.BackFill)/r.options.PeriodDuration) * r.options.EventsPerPeriod + + backFillGenerator, err := r.initializeGenerator(tpl, *config, fields, scenario, r.options.BackFill, totEvents) + if err != nil { + return nil, nil, fmt.Errorf("failed to initialize backfill generator for scenario %q: %w", scenarioName, err) + } + + return generator, backFillGenerator, nil } func (r *runner) getGeneratorConfig(scenario *scenario) (*config.Config, error) { diff --git a/internal/testrunner/runners/static/runner.go b/internal/testrunner/runners/static/runner.go index b25ca1d657..b6580d8647 100644 --- a/internal/testrunner/runners/static/runner.go +++ b/internal/testrunner/runners/static/runner.go @@ -80,7 +80,7 @@ func (r runner) run(ctx context.Context) ([]testrunner.TestResult, error) { func (r runner) verifyStreamConfig(ctx context.Context, packageRootPath string) []testrunner.TestResult { resultComposer := testrunner.NewResultComposer(testrunner.TestResult{ - Name: "Verify benchmark config (if available)", + Name: "Verify benchmark config", TestType: TestType, Package: r.options.TestFolder.Package, DataStream: r.options.TestFolder.DataStream, @@ -93,12 +93,16 @@ func (r runner) verifyStreamConfig(ctx context.Context, packageRootPath string) ctx, stop := signal.Enable(ctx, logger.Info) defer stop() - err := stream.StaticValidation(ctx, stream.NewOptions(withOpts...)) + hasBenchmark, err := stream.StaticValidation(ctx, stream.NewOptions(withOpts...), r.options.TestFolder.DataStream) if err != nil { results, _ := resultComposer.WithError(err) return results } + if !hasBenchmark { + return []testrunner.TestResult{} + } + results, _ := resultComposer.WithSuccess() return results } diff --git a/test/packages/false_positives/rally_benchmark_malformed_template/_dev/benchmark/rally/logs-benchmark.yml b/test/packages/false_positives/rally_benchmark_malformed_template/_dev/benchmark/rally/logs-benchmark.yml new file mode 100644 index 0000000000..16e45f7782 --- /dev/null +++ b/test/packages/false_positives/rally_benchmark_malformed_template/_dev/benchmark/rally/logs-benchmark.yml @@ -0,0 +1,14 @@ +--- +description: Benchmark 20000 events ingested +data_stream: + name: testds +corpora: + generator: + total_events: 20000 + template: + type: gotext + path: ./logs-benchmark/template.ndjson + config: + path: ./logs-benchmark/config.yml + fields: + path: ./logs-benchmark/fields.yml diff --git a/test/packages/false_positives/rally_benchmark_malformed_template/_dev/benchmark/rally/logs-benchmark/config.yml b/test/packages/false_positives/rally_benchmark_malformed_template/_dev/benchmark/rally/logs-benchmark/config.yml new file mode 100644 index 0000000000..03b8ad10a3 --- /dev/null +++ b/test/packages/false_positives/rally_benchmark_malformed_template/_dev/benchmark/rally/logs-benchmark/config.yml @@ -0,0 +1,22 @@ +fields: + - name: '@timestamp' + period: 1s + - name: container.id + - name: log.flags + type: keyword + - name: log.offset + cardinality: 10000 + - name: tags + enum: ["production", "env2"] + - name: IP + cardinality: 100 + - name: StatusCode + enum: ["200", "400", "404"] + - name: Size + range: + min: 1 + max: 1000 + - name: Port + range: + min: 8000 + max: 8080 diff --git a/test/packages/false_positives/rally_benchmark_malformed_template/_dev/benchmark/rally/logs-benchmark/fields.yml b/test/packages/false_positives/rally_benchmark_malformed_template/_dev/benchmark/rally/logs-benchmark/fields.yml new file mode 100644 index 0000000000..32a82b632d --- /dev/null +++ b/test/packages/false_positives/rally_benchmark_malformed_template/_dev/benchmark/rally/logs-benchmark/fields.yml @@ -0,0 +1,23 @@ +- name: timestamp + type: date +- name: container.id + type: keyword +- name: log.file.path + example: /var/log/fun-times.log + type: keyword +- name: log.flags + type: keyword +- name: log.offset + type: long +- name: tags + type: keyword +- name: IP + type: ip +- name: StatusCode + type: keyword +- name: Size + type: long +- name: Hostname + type: keyword +- name: Port + type: long diff --git a/test/packages/false_positives/rally_benchmark_malformed_template/_dev/benchmark/rally/logs-benchmark/template.ndjson b/test/packages/false_positives/rally_benchmark_malformed_template/_dev/benchmark/rally/logs-benchmark/template.ndjson new file mode 100644 index 0000000000..0d1de34e72 --- /dev/null +++ b/test/packages/false_positives/rally_benchmark_malformed_template/_dev/benchmark/rally/logs-benchmark/template.ndjson @@ -0,0 +1,14 @@ +{{- $timestamp := generate "timestamp" -}} +{ +"@timestamp": "{{ $timestamp.Format "2006-01-02T15:04:05.999999Z07:00" }}", +"data_stream.type": "logs", +"data_stream.dataset": "rally_benchmarks.testds", +"data_stream.namespace": "ep", +"container.id": "{{ generate "container.id" }}", +"input.type": "filestream", +"log.file.path": "{{ generate "log.file.path" }}", +"log.flags": "{{ generate "log.flags" }}", +"log.offset": {{ generate "log.offset" }}, +"tags": ["rally_benchmark.testds", "forwarded", "{{ generate "tags" }" ], +"message": "{{ generate "IP" }} - - [{{ $timestamp.Format "02/Jan/2006:15:04:05.999999 -0700" }}] \"GET /favicon.ico HTTP/1.1\" {{ generate "StatusCode" }} {{ generate "Size" }} \"http://{{ generate "Hostname" }}:{{ generate "Port" }}/\" \"skip-this-one/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"" +} \ No newline at end of file diff --git a/test/packages/false_positives/rally_benchmark_malformed_template/changelog.yml b/test/packages/false_positives/rally_benchmark_malformed_template/changelog.yml new file mode 100644 index 0000000000..dde678acef --- /dev/null +++ b/test/packages/false_positives/rally_benchmark_malformed_template/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "999.999.999" + changes: + - description: initial release + type: enhancement # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/elastic-package/pull/1522 diff --git a/test/packages/false_positives/rally_benchmark_malformed_template/data_stream/testds/_dev/benchmark/pipeline/access-raw.log b/test/packages/false_positives/rally_benchmark_malformed_template/data_stream/testds/_dev/benchmark/pipeline/access-raw.log new file mode 100644 index 0000000000..c8c9ffe960 --- /dev/null +++ b/test/packages/false_positives/rally_benchmark_malformed_template/data_stream/testds/_dev/benchmark/pipeline/access-raw.log @@ -0,0 +1 @@ +1.2.3.4 - - [25/Oct/2016:14:49:34 +0200] "GET /favicon.ico HTTP/1.1" 404 571 "http://localhost:8080/" "skip-this-one/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36" \ No newline at end of file diff --git a/test/packages/false_positives/rally_benchmark_malformed_template/data_stream/testds/_dev/benchmark/pipeline/config.yml b/test/packages/false_positives/rally_benchmark_malformed_template/data_stream/testds/_dev/benchmark/pipeline/config.yml new file mode 100644 index 0000000000..30a2b50cf6 --- /dev/null +++ b/test/packages/false_positives/rally_benchmark_malformed_template/data_stream/testds/_dev/benchmark/pipeline/config.yml @@ -0,0 +1 @@ +num_docs: 10000 diff --git a/test/packages/false_positives/rally_benchmark_malformed_template/data_stream/testds/elasticsearch/ingest_pipeline/default.yml b/test/packages/false_positives/rally_benchmark_malformed_template/data_stream/testds/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 0000000000..f39b8ee231 --- /dev/null +++ b/test/packages/false_positives/rally_benchmark_malformed_template/data_stream/testds/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,23 @@ +--- +description: Pipeline for parsing Nginx access logs. Requires the geoip and user_agent + plugins. +processors: + - grok: + field: message + patterns: + - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address}) + - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.access.time}\] "%{DATA:nginx.access.info}" + %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} + "(-|%{DATA:http.request.referrer})" "(-|%{DATA:user_agent.original})" + pattern_definitions: + NGINX_HOST: (?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})? + NGINX_NOTSEPARATOR: "[^\t ,:]+" + NGINX_ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))* + ignore_missing: true + - user_agent: + field: user_agent.original + ignore_missing: true +on_failure: + - set: + field: error.message + value: '{{ _ingest.on_failure_message }}' \ No newline at end of file diff --git a/test/packages/false_positives/rally_benchmark_malformed_template/data_stream/testds/fields/base-fields.yml b/test/packages/false_positives/rally_benchmark_malformed_template/data_stream/testds/fields/base-fields.yml new file mode 100644 index 0000000000..0ec2cc7e01 --- /dev/null +++ b/test/packages/false_positives/rally_benchmark_malformed_template/data_stream/testds/fields/base-fields.yml @@ -0,0 +1,38 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: container.id + description: Unique container id. + ignore_above: 1024 + type: keyword +- name: input.type + description: Type of Filebeat input. + type: keyword +- name: log.file.path + description: Full path to the log file this event came from. + example: /var/log/fun-times.log + ignore_above: 1024 + type: keyword +- name: log.source.address + description: Source address from which the log event was read / sent from. + type: keyword +- name: log.flags + description: Flags for the log file. + type: keyword +- name: log.offset + description: Offset of the entry in the log file. + type: long +- name: tags + description: List of keywords used to tag each event. + example: '["production", "env2"]' + ignore_above: 1024 + type: keyword diff --git a/test/packages/false_positives/rally_benchmark_malformed_template/data_stream/testds/manifest.yml b/test/packages/false_positives/rally_benchmark_malformed_template/data_stream/testds/manifest.yml new file mode 100644 index 0000000000..250726a37b --- /dev/null +++ b/test/packages/false_positives/rally_benchmark_malformed_template/data_stream/testds/manifest.yml @@ -0,0 +1,3 @@ +title: Test +release: experimental +type: logs diff --git a/test/packages/false_positives/rally_benchmark_malformed_template/docs/README.md b/test/packages/false_positives/rally_benchmark_malformed_template/docs/README.md new file mode 100644 index 0000000000..a98577ec42 --- /dev/null +++ b/test/packages/false_positives/rally_benchmark_malformed_template/docs/README.md @@ -0,0 +1,3 @@ +# Test integration + +There is a syntax error in `_dev/benchmark/rally/logs-benchmark/template.ndjson` \ No newline at end of file diff --git a/test/packages/false_positives/rally_benchmark_malformed_template/manifest.yml b/test/packages/false_positives/rally_benchmark_malformed_template/manifest.yml new file mode 100644 index 0000000000..7cbe79f7a0 --- /dev/null +++ b/test/packages/false_positives/rally_benchmark_malformed_template/manifest.yml @@ -0,0 +1,13 @@ +format_version: 3.0.1 +name: rally_benchmarks +title: Rally benchmarks +version: 999.999.999 +description: Test for rally benchmark runner +categories: ["network"] +type: integration +conditions: + kibana: + version: '^8.0.0' +owner: + github: elastic/integrations + type: elastic From 9a6ad9f3ac03e362ad631c6c774b02a3b90bba0c Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 5 Apr 2024 17:45:38 +0200 Subject: [PATCH 09/10] update readme --- README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.md b/README.md index 2a027d600f..dca14997ec 100644 --- a/README.md +++ b/README.md @@ -199,12 +199,6 @@ _Context: package_ Run system benchmarks for the package. -### `elastic-package benchmark validate` - -_Context: package_ - -Statically validate benchmarks assets for the package. - ### `elastic-package build` _Context: package_ From ca081eea94bcf9233cd3ae2a886c6c97e5e11d2f Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 9 Apr 2024 11:53:38 +0200 Subject: [PATCH 10/10] review comment --- internal/benchrunner/runners/stream/runner.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/internal/benchrunner/runners/stream/runner.go b/internal/benchrunner/runners/stream/runner.go index 8d507cda11..e7c33b0608 100644 --- a/internal/benchrunner/runners/stream/runner.go +++ b/internal/benchrunner/runners/stream/runner.go @@ -59,13 +59,13 @@ func (r *runner) SetUp(ctx context.Context) error { return r.setUp(ctx) } -func StaticValidation(ctx context.Context, opts Options, scenarioName string) (bool, error) { +func StaticValidation(ctx context.Context, opts Options, dataStreamName string) (bool, error) { runner := runner{options: opts} err := runner.initialize() if err != nil { return false, err } - hasBenchmark, err := runner.validateScenario(ctx, scenarioName) + hasBenchmark, err := runner.validateScenario(ctx, dataStreamName) return hasBenchmark, err } @@ -124,10 +124,9 @@ func (r *runner) initialize() error { return nil } -func (r *runner) validateScenario(ctx context.Context, targetScenarioName string) (bool, error) { +func (r *runner) validateScenario(ctx context.Context, dataStreamName string) (bool, error) { for scenarioName, scenario := range r.scenarios { - println(scenarioName, targetScenarioName) - if scenario.DataStream.Name != targetScenarioName { + if scenario.DataStream.Name != dataStreamName { continue } generator, _, err := r.createGenerator(ctx, scenarioName, scenario)