diff --git a/internal/benchrunner/runners/system/runner.go b/internal/benchrunner/runners/system/runner.go index 34e7d014e1..3393949587 100644 --- a/internal/benchrunner/runners/system/runner.go +++ b/internal/benchrunner/runners/system/runner.go @@ -362,6 +362,11 @@ func (r *runner) createBenchmarkPolicy(pkgManifest *packages.PackageManifest) (* MonitoringEnabled: []string{"logs", "metrics"}, } + // Assign the data_output_id to the agent policy to configure the output to logstash. The value is inferred from stack/_static/kibana.yml.tmpl + if r.options.Profile.Config("stack.logstash_enabled", "false") == "true" { + p.DataOutputID = "fleet-logstash-output" + } + policy, err := r.options.KibanaClient.CreatePolicy(p) if err != nil { return nil, err diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh index 2e013ed676..544fc327b8 100755 --- a/scripts/test-check-packages.sh +++ b/scripts/test-check-packages.sh @@ -102,6 +102,8 @@ for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do if [ "${package_to_test}" == "system_benchmark" ]; then elastic-package benchmark system --benchmark logs-benchmark -v --defer-cleanup 1s fi + elif [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ] && [ "${PACKAGE_UNDER_TEST:-*}" == "system_benchmark" ]; then + elastic-package benchmark system --benchmark logs-benchmark -v --defer-cleanup 1s else # defer-cleanup is set to a short period to verify that the option is available elastic-package test -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage diff --git a/test/packages/with-logstash/system_benchmark/_dev/benchmark/system/logs-benchmark.yml b/test/packages/with-logstash/system_benchmark/_dev/benchmark/system/logs-benchmark.yml new file mode 100644 index 0000000000..06e97ee383 --- /dev/null +++ b/test/packages/with-logstash/system_benchmark/_dev/benchmark/system/logs-benchmark.yml @@ -0,0 +1,20 @@ +--- +description: Benchmark 20000 events ingested +input: filestream +vars: ~ +data_stream: + name: testds + vars: + paths: + - "{{SERVICE_LOGS_DIR}}/corpus-*" +warmup_time_period: 10s +wait_for_data_timeout: 10m +corpora: + generator: + total_events: 20000 + template: + path: ./logs-benchmark/template.log + config: + path: ./logs-benchmark/config.yml + fields: + path: ./logs-benchmark/fields.yml diff --git a/test/packages/with-logstash/system_benchmark/_dev/benchmark/system/logs-benchmark/config.yml b/test/packages/with-logstash/system_benchmark/_dev/benchmark/system/logs-benchmark/config.yml new file mode 100644 index 0000000000..bbceb2681c --- /dev/null +++ b/test/packages/with-logstash/system_benchmark/_dev/benchmark/system/logs-benchmark/config.yml @@ -0,0 +1,39 @@ +fields: + - name: IP + cardinality: 100 + - name: Day + range: + min: 1 + max: 28 + - name: H + range: + min: 10 + max: 23 + - name: MS + range: + min: 10 + max: 59 + - name: Mon + enum: + - "Jan" + - "Feb" + - "Mar" + - "Apr" + - "May" + - "Jun" + - "Jul" + - "Aug" + - "Sep" + - "Oct" + - "Nov" + - "Dec" + - name: StatusCode + enum: ["200", "400", "404"] + - name: Size + range: + min: 1 + max: 1000 + - name: Port + range: + min: 8000 + max: 8080 diff --git a/test/packages/with-logstash/system_benchmark/_dev/benchmark/system/logs-benchmark/fields.yml b/test/packages/with-logstash/system_benchmark/_dev/benchmark/system/logs-benchmark/fields.yml new file mode 100644 index 0000000000..4ed5ea81a4 --- /dev/null +++ b/test/packages/with-logstash/system_benchmark/_dev/benchmark/system/logs-benchmark/fields.yml @@ -0,0 +1,18 @@ +- name: IP + type: ip +- name: Day + type: long +- name: Mon + type: keyword +- name: H + type: long +- name: MS + type: long +- name: StatusCode + type: keyword +- name: Size + type: long +- name: Hostname + type: keyword +- name: Port + type: long diff --git a/test/packages/with-logstash/system_benchmark/_dev/benchmark/system/logs-benchmark/template.log b/test/packages/with-logstash/system_benchmark/_dev/benchmark/system/logs-benchmark/template.log new file mode 100644 index 0000000000..f38c5b861e --- /dev/null +++ b/test/packages/with-logstash/system_benchmark/_dev/benchmark/system/logs-benchmark/template.log @@ -0,0 +1 @@ +{{.IP}} - - [{{.Day}}/{{.Mon}}/2022:{{.H}}:{{.MS}}:{{.MS}} +0200] "GET /favicon.ico HTTP/1.1" {{.StatusCode}} {{.Size}} "http://{{.Hostname}}:{{.Port}}/" "skip-this-one/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36" \ No newline at end of file diff --git a/test/packages/with-logstash/system_benchmark/changelog.yml b/test/packages/with-logstash/system_benchmark/changelog.yml new file mode 100644 index 0000000000..1ced0b8d36 --- /dev/null +++ b/test/packages/with-logstash/system_benchmark/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "999.999.999" + changes: + - description: initial release + type: enhancement # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/elastic-package/pull/906 diff --git a/test/packages/with-logstash/system_benchmark/data_stream/testds/_dev/benchmark/pipeline/access-raw.log b/test/packages/with-logstash/system_benchmark/data_stream/testds/_dev/benchmark/pipeline/access-raw.log new file mode 100644 index 0000000000..c8c9ffe960 --- /dev/null +++ b/test/packages/with-logstash/system_benchmark/data_stream/testds/_dev/benchmark/pipeline/access-raw.log @@ -0,0 +1 @@ +1.2.3.4 - - [25/Oct/2016:14:49:34 +0200] "GET /favicon.ico HTTP/1.1" 404 571 "http://localhost:8080/" "skip-this-one/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36" \ No newline at end of file diff --git a/test/packages/with-logstash/system_benchmark/data_stream/testds/_dev/benchmark/pipeline/config.yml b/test/packages/with-logstash/system_benchmark/data_stream/testds/_dev/benchmark/pipeline/config.yml new file mode 100644 index 0000000000..30a2b50cf6 --- /dev/null +++ b/test/packages/with-logstash/system_benchmark/data_stream/testds/_dev/benchmark/pipeline/config.yml @@ -0,0 +1 @@ +num_docs: 10000 diff --git a/test/packages/with-logstash/system_benchmark/data_stream/testds/agent/stream/filestream.yml.hbs b/test/packages/with-logstash/system_benchmark/data_stream/testds/agent/stream/filestream.yml.hbs new file mode 100644 index 0000000000..cc801fea22 --- /dev/null +++ b/test/packages/with-logstash/system_benchmark/data_stream/testds/agent/stream/filestream.yml.hbs @@ -0,0 +1,4 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} diff --git a/test/packages/with-logstash/system_benchmark/data_stream/testds/agent/stream/udp.yml.hbs b/test/packages/with-logstash/system_benchmark/data_stream/testds/agent/stream/udp.yml.hbs new file mode 100644 index 0000000000..b4a46979ed --- /dev/null +++ b/test/packages/with-logstash/system_benchmark/data_stream/testds/agent/stream/udp.yml.hbs @@ -0,0 +1 @@ +host: "{{host}}:{{port}}" diff --git a/test/packages/with-logstash/system_benchmark/data_stream/testds/elasticsearch/ingest_pipeline/default.yml b/test/packages/with-logstash/system_benchmark/data_stream/testds/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 0000000000..f39b8ee231 --- /dev/null +++ b/test/packages/with-logstash/system_benchmark/data_stream/testds/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,23 @@ +--- +description: Pipeline for parsing Nginx access logs. Requires the geoip and user_agent + plugins. +processors: + - grok: + field: message + patterns: + - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address}) + - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.access.time}\] "%{DATA:nginx.access.info}" + %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} + "(-|%{DATA:http.request.referrer})" "(-|%{DATA:user_agent.original})" + pattern_definitions: + NGINX_HOST: (?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})? + NGINX_NOTSEPARATOR: "[^\t ,:]+" + NGINX_ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))* + ignore_missing: true + - user_agent: + field: user_agent.original + ignore_missing: true +on_failure: + - set: + field: error.message + value: '{{ _ingest.on_failure_message }}' \ No newline at end of file diff --git a/test/packages/with-logstash/system_benchmark/data_stream/testds/fields/base-fields.yml b/test/packages/with-logstash/system_benchmark/data_stream/testds/fields/base-fields.yml new file mode 100644 index 0000000000..0ec2cc7e01 --- /dev/null +++ b/test/packages/with-logstash/system_benchmark/data_stream/testds/fields/base-fields.yml @@ -0,0 +1,38 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: container.id + description: Unique container id. + ignore_above: 1024 + type: keyword +- name: input.type + description: Type of Filebeat input. + type: keyword +- name: log.file.path + description: Full path to the log file this event came from. + example: /var/log/fun-times.log + ignore_above: 1024 + type: keyword +- name: log.source.address + description: Source address from which the log event was read / sent from. + type: keyword +- name: log.flags + description: Flags for the log file. + type: keyword +- name: log.offset + description: Offset of the entry in the log file. + type: long +- name: tags + description: List of keywords used to tag each event. + example: '["production", "env2"]' + ignore_above: 1024 + type: keyword diff --git a/test/packages/with-logstash/system_benchmark/data_stream/testds/manifest.yml b/test/packages/with-logstash/system_benchmark/data_stream/testds/manifest.yml new file mode 100644 index 0000000000..34c28ea4a3 --- /dev/null +++ b/test/packages/with-logstash/system_benchmark/data_stream/testds/manifest.yml @@ -0,0 +1,36 @@ +title: Test +release: experimental +type: logs +streams: + - input: udp + title: UDP logs + enabled: false + description: Collect UDP logs + template_path: udp.yml.hbs + vars: + - name: host + type: text + title: UDP host to listen on + multi: false + required: true + show_user: true + default: localhost + - name: port + type: integer + title: UDP port to listen on + multi: false + required: true + show_user: true + default: 9511 + - input: filestream + enabled: false + title: Logs + description: Collect logs + template_path: filestream.yml.hbs + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true diff --git a/test/packages/with-logstash/system_benchmark/docs/README.md b/test/packages/with-logstash/system_benchmark/docs/README.md new file mode 100644 index 0000000000..e0ef7b4a18 --- /dev/null +++ b/test/packages/with-logstash/system_benchmark/docs/README.md @@ -0,0 +1,2 @@ +# Test integration + diff --git a/test/packages/with-logstash/system_benchmark/manifest.yml b/test/packages/with-logstash/system_benchmark/manifest.yml new file mode 100644 index 0000000000..86f739daf7 --- /dev/null +++ b/test/packages/with-logstash/system_benchmark/manifest.yml @@ -0,0 +1,24 @@ +format_version: 3.0.0 +name: system_benchmarks +title: System benchmarks +version: 999.999.999 +description: Test for system benchmark runner +categories: ["network"] +type: integration +conditions: + kibana: + version: '^8.0.0' +policy_templates: + - name: testpo + title: Test + description: Description + inputs: + - type: udp + title: Foo bar + description: Foo bar + - type: filestream + title: Collect logs + description: Collecting logs +owner: + github: elastic/integrations + type: elastic