Support system benchmarking with the logstash-output in the benchmarking agent policies

internal/benchrunner/runners/system/runner.go

@@ -362,6 +362,11 @@ func (r *runner) createBenchmarkPolicy(pkgManifest *packages.PackageManifest) (*
 		MonitoringEnabled: []string{"logs", "metrics"},
 	}
 
+	// Assign the data_output_id to the agent policy to configure the output to logstash. The value is inferred from stack/_static/kibana.yml.tmpl
+	if r.options.Profile.Config("stack.logstash_enabled", "false") == "true" {
+		p.DataOutputID = "fleet-logstash-output"
+	}
+
 	policy, err := r.options.KibanaClient.CreatePolicy(p)
 	if err != nil {
 		return nil, err

scripts/test-check-packages.sh

@@ -102,6 +102,8 @@ for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do
       if [ "${package_to_test}" == "system_benchmark" ]; then
         elastic-package benchmark system --benchmark logs-benchmark -v --defer-cleanup 1s
       fi
+    elif [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ] && [ "${PACKAGE_UNDER_TEST:-*}" == "system_benchmark" ]; then
+        elastic-package benchmark system --benchmark logs-benchmark -v --defer-cleanup 1s
     else
       # defer-cleanup is set to a short period to verify that the option is available
       elastic-package test -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage

test/packages/with-logstash/system_benchmark/_dev/benchmark/system/logs-benchmark.yml

@@ -0,0 +1,20 @@
+---
+description: Benchmark 20000 events ingested
+input: filestream
+vars: ~
+data_stream:
+  name: testds
+  vars:
+    paths:
+      - "{{SERVICE_LOGS_DIR}}/corpus-*"
+warmup_time_period: 10s
+wait_for_data_timeout: 10m
+corpora:
+  generator:
+    total_events: 20000
+    template:
+      path: ./logs-benchmark/template.log
+    config:
+      path: ./logs-benchmark/config.yml
+    fields:
+      path: ./logs-benchmark/fields.yml

test/packages/with-logstash/system_benchmark/_dev/benchmark/system/logs-benchmark/config.yml

@@ -0,0 +1,39 @@
+fields:
+  - name: IP
+    cardinality: 100
+  - name: Day
+    range:
+      min: 1
+      max: 28
+  - name: H
+    range:
+      min: 10
+      max: 23
+  - name: MS
+    range:
+      min: 10
+      max: 59
+  - name: Mon
+    enum:
+      - "Jan"
+      - "Feb"
+      - "Mar"
+      - "Apr"
+      - "May"
+      - "Jun"
+      - "Jul"
+      - "Aug"
+      - "Sep"
+      - "Oct"
+      - "Nov"
+      - "Dec"
+  - name: StatusCode
+    enum: ["200", "400", "404"]
+  - name: Size
+    range:
+      min: 1
+      max: 1000
+  - name: Port
+    range:
+      min: 8000
+      max: 8080

test/packages/with-logstash/system_benchmark/_dev/benchmark/system/logs-benchmark/fields.yml

@@ -0,0 +1,18 @@
+- name: IP
+  type: ip
+- name: Day
+  type: long
+- name: Mon
+  type: keyword
+- name: H
+  type: long
+- name: MS
+  type: long
+- name: StatusCode
+  type: keyword
+- name: Size
+  type: long
+- name: Hostname
+  type: keyword
+- name: Port
+  type: long

test/packages/with-logstash/system_benchmark/_dev/benchmark/system/logs-benchmark/template.log

@@ -0,0 +1 @@
+{{.IP}} - - [{{.Day}}/{{.Mon}}/2022:{{.H}}:{{.MS}}:{{.MS}} +0200] "GET /favicon.ico HTTP/1.1" {{.StatusCode}} {{.Size}} "http://{{.Hostname}}:{{.Port}}/" "skip-this-one/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36"

test/packages/with-logstash/system_benchmark/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "999.999.999"
+  changes:
+    - description: initial release
+      type: enhancement # can be one of: enhancement, bugfix, breaking-change
+      link: https://github.com/elastic/elastic-package/pull/906

test/packages/with-logstash/system_benchmark/data_stream/testds/_dev/benchmark/pipeline/access-raw.log

@@ -0,0 +1 @@
+1.2.3.4 - - [25/Oct/2016:14:49:34 +0200] "GET /favicon.ico HTTP/1.1" 404 571 "http://localhost:8080/" "skip-this-one/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36"

test/packages/with-logstash/system_benchmark/data_stream/testds/_dev/benchmark/pipeline/config.yml

@@ -0,0 +1 @@
+num_docs: 10000

test/packages/with-logstash/system_benchmark/data_stream/testds/agent/stream/filestream.yml.hbs

@@ -0,0 +1,4 @@
+paths:
+{{#each paths as |path i|}}
+  - {{path}}
+{{/each}}

test/packages/with-logstash/system_benchmark/data_stream/testds/agent/stream/udp.yml.hbs

@@ -0,0 +1 @@
+host: "{{host}}:{{port}}"

test/packages/with-logstash/system_benchmark/data_stream/testds/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,23 @@
+---
+description: Pipeline for parsing Nginx access logs. Requires the geoip and user_agent
+  plugins.
+processors:
+  - grok:
+      field: message
+      patterns:
+        - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address})
+          - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.access.time}\] "%{DATA:nginx.access.info}"
+          %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long}
+          "(-|%{DATA:http.request.referrer})" "(-|%{DATA:user_agent.original})"
+      pattern_definitions:
+        NGINX_HOST: (?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})?
+        NGINX_NOTSEPARATOR: "[^\t ,:]+"
+        NGINX_ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))*
+      ignore_missing: true
+  - user_agent:
+      field: user_agent.original
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: '{{ _ingest.on_failure_message }}'

test/packages/with-logstash/system_benchmark/data_stream/testds/fields/base-fields.yml

@@ -0,0 +1,38 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.
+- name: container.id
+  description: Unique container id.
+  ignore_above: 1024
+  type: keyword
+- name: input.type
+  description: Type of Filebeat input.
+  type: keyword
+- name: log.file.path
+  description: Full path to the log file this event came from.
+  example: /var/log/fun-times.log
+  ignore_above: 1024
+  type: keyword
+- name: log.source.address
+  description: Source address from which the log event was read / sent from.
+  type: keyword
+- name: log.flags
+  description: Flags for the log file.
+  type: keyword
+- name: log.offset
+  description: Offset of the entry in the log file.
+  type: long
+- name: tags
+  description: List of keywords used to tag each event.
+  example: '["production", "env2"]'
+  ignore_above: 1024
+  type: keyword

test/packages/with-logstash/system_benchmark/data_stream/testds/manifest.yml

@@ -0,0 +1,36 @@
+title: Test
+release: experimental
+type: logs
+streams:
+  - input: udp
+    title: UDP logs
+    enabled: false
+    description: Collect UDP logs
+    template_path: udp.yml.hbs
+    vars:
+      - name: host
+        type: text
+        title: UDP host to listen on
+        multi: false
+        required: true
+        show_user: true
+        default: localhost
+      - name: port
+        type: integer
+        title: UDP port to listen on
+        multi: false
+        required: true
+        show_user: true
+        default: 9511
+  - input: filestream
+    enabled: false
+    title: Logs
+    description: Collect logs
+    template_path: filestream.yml.hbs
+    vars:
+      - name: paths
+        type: text
+        title: Paths
+        multi: true
+        required: true
+        show_user: true

test/packages/with-logstash/system_benchmark/docs/README.md

@@ -0,0 +1,2 @@
+# Test integration
+

test/packages/with-logstash/system_benchmark/manifest.yml

@@ -0,0 +1,24 @@
+format_version: 3.0.0
+name: system_benchmarks
+title: System benchmarks
+version: 999.999.999
+description: Test for system benchmark runner
+categories: ["network"]
+type: integration
+conditions:
+  kibana:
+    version: '^8.0.0'
+policy_templates:
+  - name: testpo
+    title: Test
+    description: Description
+    inputs:
+      - type: udp
+        title: Foo bar
+        description: Foo bar
+      - type: filestream
+        title: Collect logs
+        description: Collecting logs
+owner:
+  github: elastic/integrations
+  type: elastic