diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 21ab4da9b9d..cbe5a438a71 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -414,6 +414,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add fileset to the Zeek module for the intel.log. {pull}14404[14404] - New fileset googlecloud/firewall for ingesting Google Cloud Firewall logs. {pull}14553[14553] - Add document for Filebeat input httpjson. {pull}14602[14602] +- Add more configuration options to the Netflow module. {pull}14628{14628} *Heartbeat* - Add non-privileged icmp on linux and darwin(mac). {pull}13795[13795] {issue}11498[11498] diff --git a/filebeat/docs/inputs/input-common-udp-options.asciidoc b/filebeat/docs/inputs/input-common-udp-options.asciidoc index 5500ee97190..5a9dbd7e324 100644 --- a/filebeat/docs/inputs/input-common-udp-options.asciidoc +++ b/filebeat/docs/inputs/input-common-udp-options.asciidoc @@ -18,6 +18,12 @@ The host and UDP port to listen on for event streams. [float] [id="{beatname_lc}-input-{type}-udp-read-buffer"] -=== `read_buffer` +==== `read_buffer` The size of the read buffer on the UDP socket. + +[float] +[id="{beatname_lc}-input-{type}-udp-timeout"] +==== `timeout` + +The read and write timeout for socket operations. diff --git a/filebeat/docs/modules/netflow.asciidoc b/filebeat/docs/modules/netflow.asciidoc index c0d698ff8fc..934c6e456d0 100644 --- a/filebeat/docs/modules/netflow.asciidoc +++ b/filebeat/docs/modules/netflow.asciidoc @@ -51,6 +51,10 @@ traffic from network devices. `var.max_message_size`:: The maximum size of the message received over UDP. The default is `10KiB`. +`var.read_buffer`:: The size of the read buffer on the UDP socket. + +`var.timeout`:: The read and write timeout for socket operations. + `var.expiration_timeout`:: The time before an idle session or unused template is expired. Only applicable to v9 and IPFIX protocols. A value of zero disables expiration. @@ -59,6 +63,11 @@ expiration. processing. Use this setting to avoid packet-loss when dealing with occasional bursts of traffic. +`var.custom_definitions`:: A list of paths to field definitions YAML files. +These allow to update the NetFlow/IPFIX fields with vendor extensions and to +override existing fields. See <> for +details. + :has-dashboards!: :fileset_ex!: diff --git a/filebeat/fileset/fileset.go b/filebeat/fileset/fileset.go index f8291d0b0c8..edd8a72bd20 100644 --- a/filebeat/fileset/fileset.go +++ b/filebeat/fileset/fileset.go @@ -173,10 +173,9 @@ func (fs *Fileset) evaluateVars(beatVersion string) (map[string]interface{}, err return nil, fmt.Errorf("Variable doesn't have a string 'name' key") } - value, exists := vals["default"] - if !exists { - return nil, fmt.Errorf("Variable %s doesn't have a 'default' key", name) - } + // Variables are not required to have a default. Templates should + // handle null default values as necessary. + value := vals["default"] // evaluate OS specific vars osVals, exists := vals["os"].(map[string]interface{}) @@ -268,7 +267,7 @@ func resolveVariable(vars map[string]interface{}, value interface{}) (interface{ // the delimiters are set to `{<` and `>}` instead of `{{` and `}}`. These are easier to use // in pipeline definitions. func applyTemplate(vars map[string]interface{}, templateString string, specialDelims bool) (string, error) { - tpl := template.New("text") + tpl := template.New("text").Option("missingkey=error") if specialDelims { tpl = tpl.Delims("{<", ">}") } diff --git a/x-pack/filebeat/module/netflow/_meta/docs.asciidoc b/x-pack/filebeat/module/netflow/_meta/docs.asciidoc index 44d4e4abbae..abb548233e4 100644 --- a/x-pack/filebeat/module/netflow/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/netflow/_meta/docs.asciidoc @@ -46,6 +46,10 @@ traffic from network devices. `var.max_message_size`:: The maximum size of the message received over UDP. The default is `10KiB`. +`var.read_buffer`:: The size of the read buffer on the UDP socket. + +`var.timeout`:: The read and write timeout for socket operations. + `var.expiration_timeout`:: The time before an idle session or unused template is expired. Only applicable to v9 and IPFIX protocols. A value of zero disables expiration. @@ -54,6 +58,11 @@ expiration. processing. Use this setting to avoid packet-loss when dealing with occasional bursts of traffic. +`var.custom_definitions`:: A list of paths to field definitions YAML files. +These allow to update the NetFlow/IPFIX fields with vendor extensions and to +override existing fields. See <> for +details. + :has-dashboards!: :fileset_ex!: diff --git a/x-pack/filebeat/module/netflow/log/config/netflow.yml b/x-pack/filebeat/module/netflow/log/config/netflow.yml index b6d045353db..e4956454707 100644 --- a/x-pack/filebeat/module/netflow/log/config/netflow.yml +++ b/x-pack/filebeat/module/netflow/log/config/netflow.yml @@ -4,3 +4,18 @@ host: '{{.netflow_host}}:{{.netflow_port}}' max_message_size: '{{.max_message_size}}' expiration_timeout: '{{.expiration_timeout}}' queue_size: {{.queue_size}} + +{{if .timeout}} +timeout: '{{.timeout}}' +{{end}} + +{{if .read_buffer}} +read_buffer: '{{.read_buffer}}' +{{end}} + +{{ if .custom_definitions}} +custom_definitions: +{{range .custom_definitions}} +- '{{ . }}' +{{end}} +{{end}} diff --git a/x-pack/filebeat/module/netflow/log/manifest.yml b/x-pack/filebeat/module/netflow/log/manifest.yml index 956ddc5e187..f263b485bb2 100644 --- a/x-pack/filebeat/module/netflow/log/manifest.yml +++ b/x-pack/filebeat/module/netflow/log/manifest.yml @@ -11,6 +11,9 @@ var: default: 30m - name: queue_size default: 8192 + - name: read_buffer + - name: timeout + - name: custom_definitions ingest_pipeline: ingest/pipeline.yml input: config/netflow.yml diff --git a/x-pack/filebeat/module/zeek/dhcp/manifest.yml b/x-pack/filebeat/module/zeek/dhcp/manifest.yml index 9803f8b2341..a09038725e3 100644 --- a/x-pack/filebeat/module/zeek/dhcp/manifest.yml +++ b/x-pack/filebeat/module/zeek/dhcp/manifest.yml @@ -10,6 +10,8 @@ var: - /usr/local/var/logs/current/dhcp.log - name: tags default: [zeek.dhcp] + - name: community_id + default: true ingest_pipeline: ingest/pipeline.json input: config/dhcp.yml