Cherry-pick #18596 to 7.x: [Auditbeat] Update auditbeat ECS mappings

CHANGELOG.next.asciidoc

@@ -317,6 +317,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add system module socket dataset ECS categorization fields. {pull}18036[18036]
 - Add file integrity module ECS categorization fields. {pull}18012[18012]
 - Add `file.mime_type`, `file.extension`, and `file.drive_letter` for file integrity module. {pull}18012[18012]
+- Add ECS categorization info for auditd module {pull}18596[18596]
 
 *Filebeat*
 

NOTICE.txt

@@ -1681,10 +1681,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 
 --------------------------------------------------------------------
-Dependency: github.com/elastic/go-libaudit
-Version: v0.4.0
+Dependency: github.com/elastic/go-libaudit/v2
+Version: v2.0.0
+Revision: 92371bef3fb8
 License type (autodetected): Apache-2.0
-./vendor/github.com/elastic/go-libaudit/LICENSE.txt:
+./vendor/github.com/elastic/go-libaudit/v2/LICENSE.txt:
 --------------------------------------------------------------------
 Apache License 2.0
 
@@ -8430,7 +8431,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------
 Dependency: gopkg.in/yaml.v2
-Version: v2.2.8
+Version: v2.3.0
 License type (autodetected): Apache-2.0
 ./vendor/gopkg.in/yaml.v2/LICENSE:
 --------------------------------------------------------------------
@@ -8453,7 +8454,7 @@ limitations under the License.
 
 --------------------------------------------------------------------
 Dependency: gopkg.in/yaml.v2
-Version: v2.2.8
+Version: v2.3.0
 License type (autodetected): MIT
 ./vendor/gopkg.in/yaml.v2/LICENSE.libyaml:
 --------------------------------------------------------------------

auditbeat/module/auditd/audit_linux.go

@@ -35,10 +35,10 @@ import (
 	"github.com/elastic/beats/v7/libbeat/monitoring"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 	"github.com/elastic/beats/v7/metricbeat/mb/parse"
-	"github.com/elastic/go-libaudit"
-	"github.com/elastic/go-libaudit/aucoalesce"
-	"github.com/elastic/go-libaudit/auparse"
-	"github.com/elastic/go-libaudit/rule"
+	"github.com/elastic/go-libaudit/v2"
+	"github.com/elastic/go-libaudit/v2/aucoalesce"
+	"github.com/elastic/go-libaudit/v2/auparse"
+	"github.com/elastic/go-libaudit/v2/rule"
 )
 
 const (
@@ -539,10 +539,10 @@ func buildMetricbeatEvent(msgs []*auparse.AuditMessage, config Config) mb.Event
 		m.Put("paths", auditEvent.Paths)
 	}
 
+	normalizeEventFields(auditEvent, out.RootFields)
+
 	switch auditEvent.Category {
 	case aucoalesce.EventTypeUserLogin:
-		// Customize event.type / event.category to match unified values.
-		normalizeEventFields(out.RootFields)
 		// Set ECS user fields from the attempted login account.
 		if usernameOrID := auditEvent.Summary.Actor.Secondary; usernameOrID != "" {
 			if usr, err := resolveUsernameOrID(usernameOrID); err == nil {
@@ -572,25 +572,39 @@ func resolveUsernameOrID(userOrID string) (usr *user.User, err error) {
 	return user.LookupId(userOrID)
 }
 
-func normalizeEventFields(m common.MapStr) {
-	getFieldAsStr := func(key string) (s string, found bool) {
-		iface, err := m.GetValue(key)
-		if err != nil {
+func normalizeEventFields(event *aucoalesce.Event, m common.MapStr) {
+	// we need to merge types for backwards compatibility
+	types := event.ECS.Event.Type
+
+	// Remove this block in 8.x
+	{
+		getFieldAsStr := func(key string) (s string, found bool) {
+			iface, err := m.GetValue(key)
+			if err != nil {
+				return
+			}
+			s, found = iface.(string)
 			return
 		}
-		s, found = iface.(string)
-		return
+		oldCategory, ok1 := getFieldAsStr("event.category")
+		oldAction, ok2 := getFieldAsStr("event.action")
+		oldOutcome, ok3 := getFieldAsStr("event.outcome")
+		if ok1 && ok2 && ok3 {
+			if oldCategory == "user-login" && oldAction == "logged-in" { // USER_LOGIN
+				types = append(types, fmt.Sprintf("authentication_%s", oldOutcome))
+			}
+		}
 	}
 
-	category, ok1 := getFieldAsStr("event.category")
-	action, ok2 := getFieldAsStr("event.action")
-	outcome, ok3 := getFieldAsStr("event.outcome")
-	if !ok1 || !ok2 || !ok3 {
-		return
+	m.Put("event.kind", "event")
+	if len(event.ECS.Event.Category) > 0 {
+		m.Put("event.category", event.ECS.Event.Category)
+	}
+	if len(types) > 0 {
+		m.Put("event.type", types)
 	}
-	if category == "user-login" && action == "logged-in" { // USER_LOGIN
-		m.Put("event.category", "authentication")
-		m.Put("event.type", fmt.Sprintf("authentication_%s", outcome))
+	if event.ECS.Event.Outcome != "" {
+		m.Put("event.outcome", event.ECS.Event.Outcome)
 	}
 }
 

auditbeat/module/auditd/audit_linux_test.go

@@ -40,8 +40,8 @@ import (
 	"github.com/elastic/beats/v7/libbeat/mapping"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
-	"github.com/elastic/go-libaudit"
-	"github.com/elastic/go-libaudit/auparse"
+	"github.com/elastic/go-libaudit/v2"
+	"github.com/elastic/go-libaudit/v2/auparse"
 )
 
 // Specify the -audit flag when running these tests to interact with the real
@@ -141,23 +141,24 @@ func TestLoginType(t *testing.T) {
 
 	for idx, expected := range []common.MapStr{
 		{
-			"event.category": "authentication",
-			"event.type":     "authentication_failure",
+			"event.category": []string{"authentication"},
+			"event.type":     []string{"start", "authentication_failure"},
 			"event.outcome":  "failure",
 			"user.name":      "(invalid user)",
 			"user.id":        nil,
 			"session":        nil,
 		},
 		{
-			"event.category": "authentication",
-			"event.type":     "authentication_success",
+			"event.category": []string{"authentication"},
+			"event.type":     []string{"start", "authentication_success"},
 			"event.outcome":  "success",
 			"user.name":      "adrian",
 			"user.audit.id":  nil,
 			"auditd.session": nil,
 		},
 		{
-			"event.category": "user-login",
+			"event.category": []string{"authentication"},
+			"event.type":     []string{"info"},
 			"event.outcome":  "success",
 			"user.name":      "root",
 			"user.id":        "0",

auditbeat/module/auditd/config.go

@@ -31,8 +31,8 @@ import (
 	"github.com/joeshaw/multierror"
 	"github.com/pkg/errors"
 
-	"github.com/elastic/go-libaudit/rule"
-	"github.com/elastic/go-libaudit/rule/flags"
+	"github.com/elastic/go-libaudit/v2/rule"
+	"github.com/elastic/go-libaudit/v2/rule/flags"
 )
 
 const (

auditbeat/module/auditd/mock_linux_test.go

@@ -23,8 +23,8 @@ import (
 	"errors"
 	"syscall"
 
-	"github.com/elastic/go-libaudit"
-	"github.com/elastic/go-libaudit/auparse"
+	"github.com/elastic/go-libaudit/v2"
+	"github.com/elastic/go-libaudit/v2/auparse"
 )
 
 type MockNetlinkSendReceiver struct {

auditbeat/module/auditd/show_linux.go

@@ -24,8 +24,8 @@ import (
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 
-	"github.com/elastic/go-libaudit"
-	"github.com/elastic/go-libaudit/rule"
+	"github.com/elastic/go-libaudit/v2"
+	"github.com/elastic/go-libaudit/v2/rule"
 
 	"github.com/elastic/beats/v7/auditbeat/cmd"
 )

go.mod

@@ -57,7 +57,7 @@ require (
 	github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4
 	github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2
 	github.com/elastic/ecs v1.5.0
-	github.com/elastic/go-libaudit v0.4.0
+	github.com/elastic/go-libaudit/v2 v2.0.0-20200515221334-92371bef3fb8
 	github.com/elastic/go-licenser v0.2.1
 	github.com/elastic/go-lookslike v0.3.0
 	github.com/elastic/go-lumber v0.1.0
@@ -164,7 +164,7 @@ require (
 	gopkg.in/inf.v0 v0.9.0
 	gopkg.in/jcmturner/gokrb5.v7 v7.3.0
 	gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528
-	gopkg.in/yaml.v2 v2.2.8
+	gopkg.in/yaml.v2 v2.3.0
 	howett.net/plist v0.0.0-20181124034731-591f970eefbb
 	k8s.io/api v0.0.0-20190722141453-b90922c02518
 	k8s.io/apimachinery v0.0.0-20190719140911-bfcf53abc9f8

go.sum

@@ -89,6 +89,7 @@ github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
+github.com/Sirupsen/logrus v1.0.1-0.20170608221441-85b1699d5056/go.mod h1:rmk17hk6i8ZSAJkSDa7nOxamrG+SP4P0mm+DAvExv4U=
 github.com/StackExchange/wmi v0.0.0-20170221213301-9f32b5905fd6 h1:2Gl9Tray0NEjP9KC0FjdGWlszbmTIsBP3JYzgyFdL4E=
 github.com/StackExchange/wmi v0.0.0-20170221213301-9f32b5905fd6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/adriansr/fsnotify v0.0.0-20180417234312-c9bbe1f46f1d h1:g0M6kedfjDpyAAuxqBvJzMNjFzlrQ7Av6LCDFqWierk=
@@ -227,8 +228,8 @@ github.com/elastic/ecs v1.5.0 h1:/VEIBsRU4ecq2+U3RPfKNc6bFyomP6qnthYEcQZu8GU=
 github.com/elastic/ecs v1.5.0/go.mod h1:pgiLbQsijLOJvFR8OTILLu0Ni/R/foUNg0L+T6mU9b4=
 github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQVCpGSRXmLqjEHpJKbR60rxh1nQZY4=
 github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270/go.mod h1:Msl1pdboCbArMF/nSCDUXgQuWTeoMmE/z8607X+k7ng=
-github.com/elastic/go-libaudit v0.4.0 h1:pxLCycMJKW91W8ZmZT74DQmryTZuXryKESo6sXdu1XY=
-github.com/elastic/go-libaudit v0.4.0/go.mod h1:lNJ7gX+arohEQTwqinAc8xycVuFNqsaunba1mwcBdvE=
+github.com/elastic/go-libaudit/v2 v2.0.0-20200515221334-92371bef3fb8 h1:Jcnojiuok7Ea5hitJK9VWmBigganE2MMETOH0VZasEA=
+github.com/elastic/go-libaudit/v2 v2.0.0-20200515221334-92371bef3fb8/go.mod h1:j2CZcVcluWDGhQTnq1SOPy1NKEIa74FtQ39Nnz87Jxk=
 github.com/elastic/go-licenser v0.2.1 h1:K76YI6XR2LRpewLGwhrTqasXZcNJG2yHY4/jit/IXGY=
 github.com/elastic/go-licenser v0.2.1/go.mod h1:D8eNQk70FOCVBl3smCGQt/lv7meBeQno2eI1S5apiHQ=
 github.com/elastic/go-lookslike v0.3.0 h1:HDI/DQ65V85ZqM7D/sbxcK2wFFnh3+7iFvBk2v2FTHs=
@@ -555,6 +556,7 @@ github.com/pierrec/lz4 v2.2.6+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi
 github.com/pierrre/gotestcover v0.0.0-20160113212533-7b94f124d338 h1:/VAZ3an4jHXs+61iNHugNR1mG25MSpaxtMnwOJVEAQM=
 github.com/pierrre/gotestcover v0.0.0-20160113212533-7b94f124d338/go.mod h1:4xpMLz7RBWyB+ElzHu8Llua96TRCB3YwX+l5EP1wmHk=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1-0.20170505043639-c605e284fe17/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -631,6 +633,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.1.5-0.20170601210322-f6abca593680/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -778,6 +781,7 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20170608164803-0b25a408a500/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -920,6 +924,8 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=