Add DNS header_flags, registered_domain, resolved_ip

 Add DNS fields for ECS. This adds three fields:

- dns.question.registered_domain
- dns.header_flags
- dns.resolved_ip

Relates #13320

CHANGELOG.next.asciidoc

@@ -333,6 +333,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Packetbeat*
 
+- Update DNS protocol plugin to produce events with ECS fields for DNS. {issue}13320[13320] {pull}13354[13354]
+
 *Functionbeat*
 
 - New options to configure roles and VPC. {pull}11779[11779]

packetbeat/protos/dns/dns.go

@@ -459,6 +459,31 @@ func addDNSToMapStr(m common.MapStr, dns *mkdns.Msg, authority bool, additional
 	}
 	m["response_code"] = dnsResponseCodeToString(dns.Rcode)
 
+	// Add a list of header flags.
+	var hf []string
+	if dns.Authoritative {
+		hf = append(hf, "AA")
+	}
+	if dns.Truncated {
+		hf = append(hf, "TC")
+	}
+	if dns.RecursionDesired {
+		hf = append(hf, "RD")
+	}
+	if dns.RecursionAvailable {
+		hf = append(hf, "RA")
+	}
+	if dns.AuthenticatedData {
+		hf = append(hf, "AD")
+	}
+	if dns.CheckingDisabled {
+		hf = append(hf, "CD")
+	}
+	if opt := dns.IsEdns0(); opt != nil && opt.Do() {
+		hf = append(hf, "DO")
+	}
+	m["header_flags"] = hf
+
 	if len(dns.Question) > 0 {
 		q := dns.Question[0]
 		qMapStr := common.MapStr{
@@ -470,7 +495,9 @@ func addDNSToMapStr(m common.MapStr, dns *mkdns.Msg, authority bool, additional
 
 		eTLDPlusOne, err := publicsuffix.EffectiveTLDPlusOne(q.Name)
 		if err == nil {
+			// etld_plus_one should be removed for 8.0.0.
 			qMapStr["etld_plus_one"] = eTLDPlusOne
+			qMapStr["registered_domain"] = eTLDPlusOne
 		}
 	}
 
@@ -481,12 +508,16 @@ func addDNSToMapStr(m common.MapStr, dns *mkdns.Msg, authority bool, additional
 
 	m["answers_count"] = len(dns.Answer)
 	if len(dns.Answer) > 0 {
-		m["answers"] = rrsToMapStrs(dns.Answer)
+		var resolvedIPs []string
+		m["answers"], resolvedIPs = rrsToMapStrs(dns.Answer, true)
+		if len(resolvedIPs) > 0 {
+			m["resolved_ip"] = resolvedIPs
+		}
 	}
 
 	m["authorities_count"] = len(dns.Ns)
 	if authority && len(dns.Ns) > 0 {
-		m["authorities"] = rrsToMapStrs(dns.Ns)
+		m["authorities"], _ = rrsToMapStrs(dns.Ns, false)
 	}
 
 	if rrOPT != nil {
@@ -495,7 +526,7 @@ func addDNSToMapStr(m common.MapStr, dns *mkdns.Msg, authority bool, additional
 		m["additionals_count"] = len(dns.Extra)
 	}
 	if additional && len(dns.Extra) > 0 {
-		rrsMapStrs := rrsToMapStrs(dns.Extra)
+		rrsMapStrs, _ := rrsToMapStrs(dns.Extra, false)
 		// We do not want OPT RR to appear in the 'additional' section,
 		// that's why rrsMapStrs could be empty even though len(dns.Extra) > 0
 		if len(rrsMapStrs) > 0 {
@@ -538,23 +569,27 @@ func optToMapStr(rrOPT *mkdns.OPT) common.MapStr {
 	return optMapStr
 }
 
-// rrsToMapStr converts an slice of RR's to an slice of MapStr's.
-func rrsToMapStrs(records []mkdns.RR) []common.MapStr {
+// rrsToMapStr converts an slice of RR's to an slice of MapStr's and optionally
+// returns a list of the IP addresses found in the resource records.
+func rrsToMapStrs(records []mkdns.RR, ipList bool) ([]common.MapStr, []string) {
+	var allIPs []string
 	mapStrSlice := make([]common.MapStr, 0, len(records))
 	for _, rr := range records {
 		rrHeader := rr.Header()
 
-		mapStr := rrToMapStr(rr)
+		mapStr, ips := rrToMapStr(rr, ipList)
 		if len(mapStr) == 0 { // OPT pseudo-RR returns an empty MapStr
 			continue
 		}
+		allIPs = append(allIPs, ips...)
+
 		mapStr["name"] = trimRightDot(rrHeader.Name)
 		mapStr["type"] = dnsTypeToString(rrHeader.Rrtype)
 		mapStr["class"] = dnsClassToString(rrHeader.Class)
 		mapStr["ttl"] = strconv.FormatInt(int64(rrHeader.Ttl), 10)
 		mapStrSlice = append(mapStrSlice, mapStr)
 	}
-	return mapStrSlice
+	return mapStrSlice, allIPs
 }
 
 // Convert all RDATA fields of a RR to a single string
@@ -566,7 +601,7 @@ func rrToString(rr mkdns.RR) string {
 	var st string
 	var keys []string
 
-	mapStr := rrToMapStr(rr)
+	mapStr, _ := rrToMapStr(rr, false)
 	data, ok := mapStr["data"]
 	delete(mapStr, "data")
 
@@ -599,10 +634,18 @@ func rrToString(rr mkdns.RR) string {
 	return b.String()
 }
 
-func rrToMapStr(rr mkdns.RR) common.MapStr {
+func rrToMapStr(rr mkdns.RR, ipList bool) (common.MapStr, []string) {
 	mapStr := common.MapStr{}
 	rrType := rr.Header().Rrtype
 
+	var ips []string
+	appendIP := func(ip string) string {
+		if ipList {
+			ips = append(ips, ip)
+		}
+		return ip
+	}
+
 	switch x := rr.(type) {
 	default:
 		// We don't have special handling for this type
@@ -619,9 +662,9 @@ func rrToMapStr(rr mkdns.RR) common.MapStr {
 			debugf("Rdata for the unhandled RR type %s could not be fetched", dnsTypeToString(rrType))
 		}
 	case *mkdns.A:
-		mapStr["data"] = x.A.String()
+		mapStr["data"] = appendIP(x.A.String())
 	case *mkdns.AAAA:
-		mapStr["data"] = x.AAAA.String()
+		mapStr["data"] = appendIP(x.AAAA.String())
 	case *mkdns.CNAME:
 		mapStr["data"] = trimRightDot(x.Target)
 	case *mkdns.DNSKEY:
@@ -656,7 +699,7 @@ func rrToMapStr(rr mkdns.RR) common.MapStr {
 		mapStr["data"] = dnsSaltToString(x.Salt)
 	case *mkdns.OPT: // EDNS [RFC6891]
 		// OPT pseudo-RR is managed in addDnsToMapStr function
-		return nil
+		return nil, nil
 	case *mkdns.PTR:
 		mapStr["data"] = trimRightDot(x.Ptr)
 	case *mkdns.RFC3597:
@@ -694,7 +737,7 @@ func rrToMapStr(rr mkdns.RR) common.MapStr {
 		mapStr["data"] = strings.Join(x.Txt, " ")
 	}
 
-	return mapStr
+	return mapStr, ips
 }
 
 // dnsQuestionToString converts a Question to a string.

packetbeat/protos/dns/dns_test.go

@@ -264,6 +264,7 @@ func assertRequest(t testing.TB, m common.MapStr, q dnsTestMessage) {
 	assert.Equal(t, q.qType, mapValue(t, m, "dns.question.type"))
 	assert.Equal(t, q.qName, mapValue(t, m, "dns.question.name"))
 	assert.Equal(t, q.qEtld, mapValue(t, m, "dns.question.etld_plus_one"))
+	assert.Equal(t, q.qEtld, mapValue(t, m, "dns.question.registered_domain"))
 }
 
 // Assert that the specified flags are set.
@@ -310,7 +311,7 @@ func TestRRsToMapStrsWithOPTRecord(t *testing.T) {
 
 	// The OPT record is a pseudo-record so it doesn't become a real record
 	// in our conversion, and there will be 1 entry instead of 2.
-	mapStrs := rrsToMapStrs([]mkdns.RR{o, r})
+	mapStrs, _ := rrsToMapStrs([]mkdns.RR{o, r}, false)
 	assert.Len(t, mapStrs, 1)
 
 	mapStr := mapStrs[0]