Merge pull request elastic#3894 from tsg/automatic_merge_from_5.3_to_…

…5.x_branch

Automatic merge from 5.3 to 5.x branch

CHANGELOG.asciidoc

@@ -31,10 +31,14 @@ https://github.com/elastic/beats/compare/v5.3.0...master[Check the HEAD diff]
 
 *Filebeat*
 
+- Fix modules default file permissions. {pull}3879[3879]
+
 *Heartbeat*
 
 *Metricbeat*
 
+- Avoid errors when some Apache status fields are missing. {issue}3074[3074]
+
 *Packetbeat*
 
 *Winlogbeat*

dev-tools/package_test.go

@@ -21,13 +21,15 @@ import (
 )
 
 const (
-	expectedConfigMode = os.FileMode(0600)
-	expectedConfigUID  = 0
-	expectedConfigGID  = 0
+	expectedConfigMode   = os.FileMode(0600)
+	expectedManifestMode = os.FileMode(0644)
+	expectedConfigUID    = 0
+	expectedConfigGID    = 0
 )
 
 var (
-	configFilePattern = regexp.MustCompile(`.*beat\.yml`)
+	configFilePattern   = regexp.MustCompile(`.*beat\.yml`)
+	manifestFilePattern = regexp.MustCompile(`manifest.yml`)
 )
 
 var (
@@ -73,6 +75,9 @@ func checkRPM(t *testing.T, file string) {
 	}
 
 	checkConfigPermissions(t, p)
+	checkConfigOwner(t, p)
+	checkManifestPermissions(t, p)
+	checkManifestOwner(t, p)
 }
 
 func checkDeb(t *testing.T, file string, buf *bytes.Buffer) {
@@ -84,6 +89,8 @@ func checkDeb(t *testing.T, file string, buf *bytes.Buffer) {
 
 	checkConfigPermissions(t, p)
 	checkConfigOwner(t, p)
+	checkManifestPermissions(t, p)
+	checkManifestOwner(t, p)
 }
 
 func checkTar(t *testing.T, file string) {
@@ -95,6 +102,7 @@ func checkTar(t *testing.T, file string) {
 
 	checkConfigPermissions(t, p)
 	checkConfigOwner(t, p)
+	checkManifestPermissions(t, p)
 }
 
 func checkZip(t *testing.T, file string) {
@@ -105,6 +113,7 @@ func checkZip(t *testing.T, file string) {
 	}
 
 	checkConfigPermissions(t, p)
+	checkManifestPermissions(t, p)
 }
 
 // Verify that the main configuration file is installed with a 0600 file mode.
@@ -115,7 +124,7 @@ func checkConfigPermissions(t *testing.T, p *packageFile) {
 				mode := entry.Mode.Perm()
 				if expectedConfigMode != mode {
 					t.Errorf("file %v has wrong permissions: expected=%v actual=%v",
-						entry.Mode, expectedConfigMode, mode)
+						entry.File, expectedConfigMode, mode)
 				}
 				return
 			}
@@ -141,6 +150,37 @@ func checkConfigOwner(t *testing.T, p *packageFile) {
 	})
 }
 
+// Verify that the modules manifest.yml files are installed with a 0644 file mode.
+func checkManifestPermissions(t *testing.T, p *packageFile) {
+	t.Run(p.Name+" manifest file permissions", func(t *testing.T) {
+		for _, entry := range p.Contents {
+			if manifestFilePattern.MatchString(entry.File) {
+				mode := entry.Mode.Perm()
+				if expectedManifestMode != mode {
+					t.Errorf("file %v has wrong permissions: expected=%v actual=%v",
+						entry.File, expectedManifestMode, mode)
+				}
+			}
+		}
+	})
+}
+
+// Verify that the manifest owner is root
+func checkManifestOwner(t *testing.T, p *packageFile) {
+	t.Run(p.Name+" manifest file owner", func(t *testing.T) {
+		for _, entry := range p.Contents {
+			if manifestFilePattern.MatchString(entry.File) {
+				if expectedConfigUID != entry.UID {
+					t.Errorf("file %v should be owned by user %v, owner=%v", entry.File, expectedConfigGID, entry.UID)
+				}
+				if expectedConfigGID != entry.GID {
+					t.Errorf("file %v should be owned by group %v, group=%v", entry.File, expectedConfigGID, entry.GID)
+				}
+			}
+		}
+	})
+}
+
 // Helpers
 
 type packageFile struct {

filebeat/docs/configuring-howto.asciidoc

@@ -11,6 +11,10 @@ To configure {beatname_uc}, you edit the configuration file. For rpm and deb, yo
 +/etc/{beatname_lc}/{beatname_lc}.yml+. There's also a full example configuration file at
 +/etc/{beatname_lc}/{beatname_lc}.full.yml+ that shows all non-deprecated options.  For mac and win, look in the archive that you extracted.
 
+See the
+{libbeat}/config-file-format.html[Config File Format] section of the
+_Beats Platform Reference_ for more about the structure of the config file.
+
 The following topics describe how to configure Filebeat:
 
 * <<filebeat-configuration-details>>

filebeat/docs/faq.asciidoc

@@ -33,7 +33,7 @@ it's publishing events successfully:
 
 [float]
 [[open-file-handlers]]
-== Too many open file handlers?
+=== Too many open file handlers?
 
 Filebeat keeps the file handler open in case it reaches the end of a file so that it can read new log lines in near real time. If Filebeat is harvesting a large number of files, the number of open files can become an issue. In most environments, the number of files that are actively updated is low. The `close_inactive` configuration option should be set accordingly to close files that are no longer active.
 

filebeat/docs/getting-started.asciidoc

@@ -153,33 +153,71 @@ See our https://www.elastic.co/downloads/beats/filebeat[download page] for other
 [[deb]]
 *deb:*
 
+ifeval::["{release-state}"=="unreleased"]
+
+Version {version} of {beatname_uc} has not yet been released.
+
+endif::[]
+
+ifeval::["{release-state}"!="unreleased"]
+
 ["source","sh",subs="attributes,callouts"]
 ------------------------------------------------
 curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-{version}-amd64.deb
 sudo dpkg -i filebeat-{version}-amd64.deb
 ------------------------------------------------
 
+endif::[]
+
 [[rpm]]
 *rpm:*
 
+ifeval::["{release-state}"=="unreleased"]
+
+Version {version} of {beatname_uc} has not yet been released.
+
+endif::[]
+
+ifeval::["{release-state}"!="unreleased"]
+
 ["source","sh",subs="attributes,callouts"]
 ------------------------------------------------
 curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-{version}-x86_64.rpm
 sudo rpm -vi filebeat-{version}-x86_64.rpm
 ------------------------------------------------
 
+endif::[]
+
 [[mac]]
 *mac:*
 
+ifeval::["{release-state}"=="unreleased"]
+
+Version {version} of {beatname_uc} has not yet been released.
+
+endif::[]
+
+ifeval::["{release-state}"!="unreleased"]
+
 ["source","sh",subs="attributes,callouts"]
 ------------------------------------------------
 curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-{version}-darwin-x86_64.tar.gz
 tar xzvf filebeat-{version}-darwin-x86_64.tar.gz
 ------------------------------------------------
 
+endif::[]
+
 [[win]]
 *win:*
 
+ifeval::["{release-state}"=="unreleased"]
+
+Version {version} of {beatname_uc} has not yet been released.
+
+endif::[]
+
+ifeval::["{release-state}"!="unreleased"]
+
 . Download the Filebeat Windows zip file from the
 https://www.elastic.co/downloads/beats/filebeat[downloads page].
 
@@ -199,6 +237,8 @@ PS C:\Program Files\Filebeat> .\install-service-filebeat.ps1
 
 NOTE: If script execution is disabled on your system, you need to set the execution policy for the current session to allow the script to run. For example: `PowerShell.exe -ExecutionPolicy UnRestricted -File .\install-service-filebeat.ps1`.
 
+endif::[]
+
 If you're using modules to get started with Filebeat, go back to the
 <<filebeat-modules-quickstart>> page.
 
@@ -220,6 +260,10 @@ find the configuration file at `/etc/filebeat/filebeat.yml`. For mac and win, lo
 the archive that you just extracted. There’s also a full example configuration file
 called `filebeat.full.yml` that shows all non-deprecated options.
 
+See the
+{libbeat}/config-file-format.html[Config File Format] section of the
+_Beats Platform Reference_ for more about the structure of the config file.
+
 Here is a sample of the `filebeat` section of the `filebeat.yml` file. Filebeat uses predefined
 default values for most configuration options.
 

filebeat/docs/modules-overview.asciidoc

@@ -92,10 +92,10 @@ production setup. The equivalent of the above in the configuration file is:
 
 [source,yaml]
 ----------------------------------------------------------------------
-modules:
-- name: nginx
-- name: mysql
-- name: syslog
+filebeat.modules:
+- module: nginx
+- module: mysql
+- module: syslog
 ----------------------------------------------------------------------
 
 Then you can start Filebeat simply with: `./filebeat -e`.
@@ -116,17 +116,17 @@ files are in a custom location:
 
 [source,shell]
 ----------------------------------------------------------------------
-$ filebeat -e -modules=nginx -M "nginx.access.var.paths=[/opt/apache2/logs/access.log*]"
+$ filebeat -e -modules=nginx -M "nginx.access.var.paths=[/var/log/nginx/access.log*]"
 ----------------------------------------------------------------------
 
 Or via the configuration file:
 
 [source,yaml]
 ----------------------------------------------------------------------
-modules:
-- name: nginx
+filebeat.modules:
+- module: nginx
   access:
-    var.paths = ["/opt/apache2/logs/access.log*"]
+    var.paths = ["/var/log/nginx/access.log*"]
 ----------------------------------------------------------------------
 
 The Nginx `access` fileset also has a `pipeline` variables which allows
@@ -150,8 +150,8 @@ example, enabling <<close-eof,close_eof>> can be done like this:
 
 [source,yaml]
 ----------------------------------------------------------------------
-modules:
-- name: nginx
+filebeat.modules:
+- module: nginx
   access:
     prospector:
       close_eof: true
@@ -181,4 +181,3 @@ created by any of the modules:
 ----------------------------------------------------------------------
 filebeat -e -modules=nginx,mysql -M "*.*.prospector.close_eof=true"
 ----------------------------------------------------------------------
-

filebeat/docs/reference/configuration.asciidoc

@@ -5,7 +5,10 @@
 Before modifying configuration settings, make sure you've completed the
 <<filebeat-configuration,configuration steps>> in the Getting Started.
 
-The {beatname_uc} configuration file, +{beatname_lc}.yml+, uses http://yaml.org/[YAML] for its syntax.
+The {beatname_uc} configuration file, +{beatname_lc}.yml+, uses http://yaml.org/[YAML] for its syntax.  See the
+{libbeat}/config-file-format.html[Config File Format] section of the
+_Beats Platform Reference_ for more about the structure of the config file.
+
 The configuration options are described in the following sections. After changing
 configuration settings, you need to restart {beatname_uc} to pick up the changes.
 

heartbeat/docs/configuring-howto.asciidoc

@@ -14,6 +14,10 @@ configuration file at +/etc/heartbeat/heartbeat.full.yml+ that shows
 all non-deprecated options. For mac and win, look in the archive that you
 extracted.
 
+See the
+{libbeat}/config-file-format.html[Config File Format] section of the
+_Beats Platform Reference_ for more about the structure of the config file.
+
 The following topics describe how to configure Heartbeat:
 
 * <<heartbeat-configuration-details>>