filebeat/config: set filebeat.registry.flush default to 1s (#31473)

This will reduce the CPU usage and disk I/O from Filebeat, while
keeping the chances of data duplication in the event of a crash to a
minimum.

Linter issues are also fixed.

Co-authored-by: Craig MacKenzie <craig.mackenzie@elastic.co>
Co-authored-by: Noémi Ványi <kvch@users.noreply.github.com>

CHANGELOG.next.asciidoc

@@ -110,6 +110,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 - Add `application/zip` decoder to the `httpsjon` input. {issue}31282[31282] {pull}31304[31304]
 - Sanitize the Azure storage account container names with underscores (_). {pull}31384[31384]
 - Add missing docs for the `delegated_account` option in the `httpjson` input. {pull}31498[31498]
+- Default value of `filebeat.registry.flush` increased from 0s to 1s. CPU and disk I/O usage are reduced because the registry is not written to disk for each ingested log line. {issue}30279[30279]
 - Cisco ASA/FTD: Add support for messages 434001 and 434003. {pull}31533[31533]
 
 *Auditbeat*

filebeat/_meta/config/filebeat.global.reference.yml.tmpl

@@ -12,8 +12,8 @@
 # The timeout value that controls when registry entries are written to disk
 # (flushed). When an unwritten update exceeds this value, it triggers a write
 # to disk. When flush is set to 0s, the registry is written to disk after each
-# batch of events has been published successfully. The default value is 0s.
-#filebeat.registry.flush: 0s
+# batch of events has been published successfully. The default value is 1s.
+#filebeat.registry.flush: 1s
 
 
 # Starting with Filebeat 7.0, the registry uses a new directory format to store

filebeat/config/config.go

@@ -64,6 +64,7 @@ var DefaultConfig = Config{
 		Permissions:   0o600,
 		MigrateFile:   "",
 		CleanInterval: 5 * time.Minute,
+		FlushTimeout:  time.Second,
 	},
 	ShutdownTimeout:    0,
 	OverwritePipelines: false,
@@ -106,9 +107,10 @@ func mergeConfigFiles(configFiles []string, config *Config) error {
 		tmpConfig := struct {
 			Filebeat Config
 		}{}
+		//nolint:staticcheck // Let's keep the logic here
 		err := cfgfile.Read(&tmpConfig, file)
 		if err != nil {
-			return fmt.Errorf("Failed to read %s: %s", file, err)
+			return fmt.Errorf("failed to read %s: %w", file, err)
 		}
 
 		config.Inputs = append(config.Inputs, tmpConfig.Filebeat.Inputs...)
@@ -157,7 +159,7 @@ func (config *Config) ListEnabledInputs() []string {
 	var inputs []string
 	for _, input := range config.Inputs {
 		if input.Enabled() {
-			input.Unpack(&t)
+			_ = input.Unpack(&t)
 			inputs = append(inputs, t.Type)
 		}
 	}

filebeat/docs/filebeat-general-options.asciidoc

@@ -59,7 +59,7 @@ filebeat.registry.file_permissions: 0600
 The timeout value that controls when registry entries are written to disk
 (flushed). When an unwritten update exceeds this value, it triggers a write to
 disk. When `registry.flush` is set to 0s, the registry is written to disk after
-each batch of events has been published successfully. The default value is 0s.
+each batch of events has been published successfully. The default value is 1s.
 
 NOTE: The registry is always updated when Filebeat shuts down normally. After an
 abnormal shutdown, the registry will not be up-to-date if the `registry.flush`

filebeat/filebeat.reference.yml

@@ -1203,8 +1203,8 @@ filebeat.inputs:
 # The timeout value that controls when registry entries are written to disk
 # (flushed). When an unwritten update exceeds this value, it triggers a write
 # to disk. When flush is set to 0s, the registry is written to disk after each
-# batch of events has been published successfully. The default value is 0s.
-#filebeat.registry.flush: 0s
+# batch of events has been published successfully. The default value is 1s.
+#filebeat.registry.flush: 1s
 
 
 # Starting with Filebeat 7.0, the registry uses a new directory format to store

filebeat/tests/system/test_harvester.py

@@ -66,8 +66,10 @@ def test_close_renamed(self):
             lambda: self.output_has(lines=iterations1 + 1), max_timeout=10)
 
         # Wait until registry file is created
+        # Because the default flush timeout is 1s, we might have only one message
+        # and need to wait at least 60s
         self.wait_until(
-            lambda: self.log_contains_count("Registry file updated") > 1)
+            lambda: self.log_contains_count("Registry file updated") >= 1, max_timeout=2)
 
         # Make sure new file was picked up. As it has the same file name,
         # one entry for the new and one for the old should exist

filebeat/tests/system/test_publisher.py

@@ -46,7 +46,7 @@ def test_registrar_file_content(self):
         # Wait until registry file is written
         self.wait_until(
             lambda: self.log_contains_count(
-                "Registry file updated.") > 1,
+                "Registry file updated.") >= 1,
             max_timeout=15)
 
         filebeat.check_kill_and_wait()

filebeat/tests/system/test_registrar.py

@@ -795,7 +795,9 @@ def test_clean_inactive(self):
 
         # Wait until registry file is created
         self.wait_until(lambda: self.registry.exists(), max_timeout=15)
-        assert self.registry.count() == 2
+
+        # registry_flush is now set to 1s, so we need to wait more than 60s
+        self.wait_until(lambda: self.registry.count() == 2, max_timeout=2)
 
         # Wait until states are removed from inputs
         self.wait_until(self.logs.nextCheck("State removed for", count=2), max_timeout=15)

x-pack/filebeat/filebeat.reference.yml

@@ -3403,8 +3403,8 @@ filebeat.inputs:
 # The timeout value that controls when registry entries are written to disk
 # (flushed). When an unwritten update exceeds this value, it triggers a write
 # to disk. When flush is set to 0s, the registry is written to disk after each
-# batch of events has been published successfully. The default value is 0s.
-#filebeat.registry.flush: 0s
+# batch of events has been published successfully. The default value is 1s.
+#filebeat.registry.flush: 1s
 
 
 # Starting with Filebeat 7.0, the registry uses a new directory format to store