[Filebeat] Enable journald input in releases (#27351)

Use the main-debian8 variant of golang-crossbuilder to build Filebeat
packages for linux/amd64 and linux/368. Previously a debian7 image was
used, but debian7 does not have the systemd libraries needed for journald.

Debian 8 uses glibc 2.19 and debian 7 uses 2.13. We used debian7 because
the binaries produced worked on RHEL 6 which has glibc 2.12.

To ensure that the Filebeat binaries for linux/amd64 and linux/386 continue
to work on RHEL 6 the build process will check the glibc requirement to
ensure it is <=2.12.

To keep the linked glibc version compatible with RHEL 6 I had to upgrade to
github.com/dgraph-io/badger/v3 from v2. v3 removed a bunch of cgo usages which
reduced the need on various libc functions.

badger adopted golang/glog for logging. This library is adds global CLI
flags that conflict with Beats own logging CLI flags (like '-v'). This is
a common problem for glog (k8s encountered this). To address the problem
I forked golang/glog and made the flag.FlagSet used by the library configurable.

CHANGELOG.next.asciidoc

@@ -489,6 +489,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add dashboard for pubsub metricset in googlecloud module. {pull}21326[21326] {issue}17137[17137]
 - Enrich events of `state_service` metricset with kubernetes services' metadata. {pull}23730[23730]
 - Check fields are documented in aws metricsets. {pull}23887[23887]
+- Enable `journald` input type in Filebeat. {issue}7955[7955] {pull}27351[27351]
 - Move openmetrics module to oss. {pull}26561[26561]
 - Fix release state of kubernetes metricsets. {pull}26864[26864]
 - Add `gke` metricset collection to `gcp` module {pull}26824[26824]

NOTICE.txt

@@ -4440,12 +4440,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 --------------------------------------------------------------------------------
-Dependency : github.com/dgraph-io/badger/v2
-Version: v2.2007.3-0.20201012072640-f5a7e0a1c83b
+Dependency : github.com/dgraph-io/badger/v3
+Version: v3.2103.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/dgraph-io/badger/v2@v2.2007.3-0.20201012072640-f5a7e0a1c83b/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/dgraph-io/badger/v3@v3.2103.1/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -9792,11 +9792,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/gogo/protobuf
-Version: v1.3.1
+Version: v1.3.2
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/gogo/protobuf@v1.3.1/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/gogo/protobuf@v1.3.2/LICENSE:
 
 Copyright (c) 2013, The GoGo Authors. All rights reserved.
 
@@ -10087,11 +10087,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/golang/snappy
-Version: v0.0.1
+Version: v0.0.3
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/golang/snappy@v0.0.1/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/golang/snappy@v0.0.3/LICENSE:
 
 Copyright (c) 2011 The Snappy-Go Authors. All rights reserved.
 
@@ -10311,11 +10311,11 @@ Contents of probable licence file $GOMODCACHE/github.com/gomodule/redigo@v1.8.3/
 
 --------------------------------------------------------------------------------
 Dependency : github.com/google/flatbuffers
-Version: v1.7.2-0.20170925184458-7a6b2bf521e9
+Version: v1.12.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/google/flatbuffers@v1.7.2-0.20170925184458-7a6b2bf521e9/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/google/flatbuffers@v1.12.0/LICENSE.txt:
 
 
                                  Apache License
@@ -10506,7 +10506,7 @@ Contents of probable licence file $GOMODCACHE/github.com/google/flatbuffers@v1.7
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2014 Google Inc.
+   Copyright [yyyy] [name of copyright owner]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -10523,11 +10523,11 @@ Contents of probable licence file $GOMODCACHE/github.com/google/flatbuffers@v1.7
 
 --------------------------------------------------------------------------------
 Dependency : github.com/google/go-cmp
-Version: v0.5.2
+Version: v0.5.4
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/google/go-cmp@v0.5.2/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/google/go-cmp@v0.5.4/LICENSE:
 
 Copyright (c) 2017 The Go Authors. All rights reserved.
 
@@ -21798,43 +21798,6 @@ Contents of probable licence file $GOMODCACHE/github.com/!burnt!sushi/xgb@v0.0.0
 // such litigation is filed.
 
 
---------------------------------------------------------------------------------
-Dependency : github.com/DataDog/zstd
-Version: v1.4.1
-Licence type (autodetected): BSD-3-Clause
---------------------------------------------------------------------------------
-
-Contents of probable licence file $GOMODCACHE/github.com/!data!dog/zstd@v1.4.1/LICENSE:
-
-Simplified BSD License
-
-Copyright (c) 2016, Datadog <info@datadoghq.com>
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-    * Redistributions of source code must retain the above copyright notice,
-      this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above copyright notice,
-      this list of conditions and the following disclaimer in the documentation
-      and/or other materials provided with the distribution.
-    * Neither the name of the copyright holder nor the names of its contributors
-      may be used to endorse or promote products derived from this software
-      without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
 --------------------------------------------------------------------------------
 Dependency : github.com/Microsoft/hcsshim
 Version: v0.8.7
@@ -26435,11 +26398,11 @@ SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/dgraph-io/ristretto
-Version: v0.0.3-0.20200630154024-f66de99634de
+Version: v0.1.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/dgraph-io/ristretto@v0.0.3-0.20200630154024-f66de99634de/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/dgraph-io/ristretto@v0.1.0/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -29856,12 +29819,12 @@ Contents of probable licence file $GOMODCACHE/github.com/golang-sql/civil@v0.0.0
    limitations under the License.
 
 --------------------------------------------------------------------------------
-Dependency : github.com/golang/glog
-Version: v0.0.0-20160126235308-23def4e6c14b
+Dependency : github.com/elastic/glog
+Version: v1.0.1-0.20210831205241-7d8b5c89dfc4
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/golang/glog@v0.0.0-20160126235308-23def4e6c14b/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/glog@v1.0.1-0.20210831205241-7d8b5c89dfc4/LICENSE:
 
 Apache License
 Version 2.0, January 2004
@@ -34800,11 +34763,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/kisielk/errcheck
-Version: v1.2.0
+Version: v1.5.0
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/kisielk/errcheck@v1.2.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/kisielk/errcheck@v1.5.0/LICENSE:
 
 Copyright (c) 2013 Kamil Kisiel
 
@@ -34874,11 +34837,11 @@ match.go, match_test.go:
 
 --------------------------------------------------------------------------------
 Dependency : github.com/klauspost/compress
-Version: v1.11.0
+Version: v1.12.3
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/klauspost/compress@v1.11.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/klauspost/compress@v1.12.3/LICENSE:
 
 Copyright (c) 2012 The Go Authors. All rights reserved.
 Copyright (c) 2019 Klaus Post. All rights reserved.
@@ -40553,11 +40516,11 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 --------------------------------------------------------------------------------
 Dependency : go.opencensus.io
-Version: v0.22.2
+Version: v0.22.5
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.opencensus.io@v0.22.2/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.opencensus.io@v0.22.5/LICENSE:
 
 
                                  Apache License

dev-tools/mage/common.go

@@ -26,6 +26,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"crypto/sha512"
+	"debug/elf"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
@@ -38,6 +39,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -913,3 +915,48 @@ func IntegrationTestEnvVars() []string {
 	}
 	return vars
 }
+
+// ReadGLIBCRequirement returns the required glibc version for a dynamically
+// linked ELF binary. The target machine must have a version equal to or
+// greater than (newer) the returned value.
+func ReadGLIBCRequirement(elfFile string) (*SemanticVersion, error) {
+	e, err := elf.Open(elfFile)
+	if err != nil {
+		return nil, err
+	}
+
+	symbols, err := e.DynamicSymbols()
+	if err != nil {
+		return nil, err
+	}
+
+	versionSet := map[SemanticVersion]struct{}{}
+	for _, sym := range symbols {
+		if strings.HasPrefix(sym.Version, "GLIBC_") {
+			semver, err := NewSemanticVersion(strings.TrimPrefix(sym.Version, "GLIBC_"))
+			if err != nil {
+				continue
+			}
+
+			versionSet[*semver] = struct{}{}
+		}
+	}
+
+	if len(versionSet) == 0 {
+		return nil, errors.New("no GLIBC symbols found in binary (is this a static binary?)")
+	}
+
+	var versions []SemanticVersion
+	for ver := range versionSet {
+		versions = append(versions, ver)
+	}
+
+	sort.Slice(versions, func(i, j int) bool {
+		a := versions[i]
+		b := versions[j]
+		return a.LessThan(&b)
+	})
+
+	max := versions[len(versions)-1]
+	return &max, nil
+}

dev-tools/mage/crossbuild.go

@@ -129,7 +129,7 @@ type crossBuildParams struct {
 
 // CrossBuild executes a given build target once for each target platform.
 func CrossBuild(options ...CrossBuildOption) error {
-	params := crossBuildParams{Platforms: Platforms, Target: defaultCrossBuildTarget, ImageSelector: crossBuildImage}
+	params := crossBuildParams{Platforms: Platforms, Target: defaultCrossBuildTarget, ImageSelector: CrossBuildImage}
 	for _, opt := range options {
 		opt(&params)
 	}
@@ -193,7 +193,7 @@ func buildMage() error {
 		"-compile", CreateDir(filepath.Join("build", "mage-linux-"+arch)))
 }
 
-func crossBuildImage(platform string) (string, error) {
+func CrossBuildImage(platform string) (string, error) {
 	tagSuffix := "main"
 
 	switch {

dev-tools/mage/pkg.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"path/filepath"
 	"runtime"
 	"strconv"
 
@@ -242,3 +243,38 @@ func TestPackages(options ...TestPackagesOption) error {
 
 	return nil
 }
+
+// TestLinuxForCentosGLIBC checks the GLIBC requirements of linux/amd64 and
+// linux/386 binaries to ensure they meet the requirements for RHEL 6 which has
+// glibc 2.12.
+func TestLinuxForCentosGLIBC() error {
+	switch Platform.Name {
+	case "linux/amd64", "linux/386":
+		return TestBinaryGLIBCVersion(filepath.Join("build/golang-crossbuild", BeatName+"-linux-"+Platform.GOARCH), "2.12")
+	default:
+		return nil
+	}
+}
+
+func TestBinaryGLIBCVersion(elfPath, maxGlibcVersion string) error {
+	requiredGlibc, err := ReadGLIBCRequirement(elfPath)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil
+		}
+		return err
+	}
+
+	upperBound, err := NewSemanticVersion(maxGlibcVersion)
+	if err != nil {
+		return err
+	}
+
+	if !requiredGlibc.LessThanOrEqual(upperBound) {
+		return fmt.Errorf("dynamically linked binary %q requires glibc "+
+			"%v, but maximum allowed glibc is %v",
+			elfPath, requiredGlibc, upperBound)
+	}
+	fmt.Printf(">> testBinaryGLIBCVersion: %q requires glibc %v or greater\n", elfPath, requiredGlibc)
+	return nil
+}