diff --git a/integration_tests/output-templates/dns_ipv4_tcp.Darwin.json b/integration_tests/output-templates/dns_ipv4_tcp.Darwin.json index c528d1604..eb40af603 100644 --- a/integration_tests/output-templates/dns_ipv4_tcp.Darwin.json +++ b/integration_tests/output-templates/dns_ipv4_tcp.Darwin.json @@ -1 +1 @@ -{"5m":{"dns":{"period":{"length":7,"start_ts":1567706433},"top_nxdomain":[],"top_qname2":[{"estimate":420,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":"wWnyafx.test.com"},{"estimate":2,"name":".6RL.test.com"},{"estimate":2,"name":".LdmqNq.test.com"},{"estimate":2,"name":".UbuZpGD.test.com"},{"estimate":2,"name":".rZ4.test.com"},{"estimate":2,"name":".mTIoCZeUb8.test.com"},{"estimate":2,"name":"UlUoHV.test.com"},{"estimate":2,"name":".de0xnQUa3.test.com"},{"estimate":2,"name":".bp5MFj.test.com"},{"estimate":2,"name":".Sl2l.test.com"}],"top_qtype":[{"estimate":100,"name":"AAAA"},{"estimate":66,"name":"NS"},{"estimate":60,"name":"A"},{"estimate":60,"name":"SOA"},{"estimate":56,"name":"MX"},{"estimate":46,"name":"TXT"},{"estimate":32,"name":"CNAME"}],"top_rcode":[{"estimate":210,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":2,"name":"44495"},{"estimate":2,"name":"39127"},{"estimate":2,"name":"42695"},{"estimate":2,"name":"40281"},{"estimate":2,"name":"46399"},{"estimate":2,"name":"35805"},{"estimate":2,"name":"37805"},{"estimate":2,"name":"42397"},{"estimate":2,"name":"45847"},{"estimate":2,"name":"35287"}],"wire_packets":{"deep_samples":420,"ipv4":420,"ipv6":0,"noerror":0,"nxdomain":0,"queries":210,"refused":0,"replies":210,"srvfail":0,"tcp":420,"total":420,"udp":0},"xact":{"counts":{"timed_out":0,"total":210},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":210}}},"packets":{"deep_samples":2100,"in":2100,"ipv4":2100,"ipv6":0,"other_l4":0,"out":0,"period":{"length":7,"start_ts":1567706433},"tcp":2100,"top_ASN":[{"estimate":2100,"name":"Unknown"}],"top_geoLoc":[{"estimate":2100,"name":"Unknown"}],"top_ipv4":[{"estimate":2100,"name":"127.0.0.1"}],"top_ipv6":[],"total":2100,"udp":0}}} +{"5m":{"dns":{"period":{"length":7,"start_ts":1567706433},"top_nxdomain":[],"top_qname2":[{"estimate":420,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":"wWnyafx.test.com"},{"estimate":2,"name":".6RL.test.com"},{"estimate":2,"name":".LdmqNq.test.com"},{"estimate":2,"name":".UbuZpGD.test.com"},{"estimate":2,"name":".rZ4.test.com"},{"estimate":2,"name":".mTIoCZeUb8.test.com"},{"estimate":2,"name":"UlUoHV.test.com"},{"estimate":2,"name":".de0xnQUa3.test.com"},{"estimate":2,"name":".bp5MFj.test.com"},{"estimate":2,"name":".Sl2l.test.com"}],"top_qtype":[{"estimate":100,"name":"AAAA"},{"estimate":66,"name":"NS"},{"estimate":60,"name":"A"},{"estimate":60,"name":"SOA"},{"estimate":56,"name":"MX"},{"estimate":46,"name":"TXT"},{"estimate":32,"name":"CNAME"}],"top_rcode":[{"estimate":210,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":2,"name":"44495"},{"estimate":2,"name":"39127"},{"estimate":2,"name":"42695"},{"estimate":2,"name":"40281"},{"estimate":2,"name":"46399"},{"estimate":2,"name":"35805"},{"estimate":2,"name":"37805"},{"estimate":2,"name":"42397"},{"estimate":2,"name":"45847"},{"estimate":2,"name":"35287"}],"wire_packets":{"deep_samples":420,"filtered":0,"ipv4":420,"ipv6":0,"noerror":0,"nxdomain":0,"queries":210,"refused":0,"replies":210,"srvfail":0,"tcp":420,"total":420,"udp":0},"xact":{"counts":{"timed_out":0,"total":210},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":210}}},"packets":{"deep_samples":2100,"in":2100,"ipv4":2100,"ipv6":0,"other_l4":0,"out":0,"period":{"length":7,"start_ts":1567706433},"tcp":2100,"top_ASN":[{"estimate":2100,"name":"Unknown"}],"top_geoLoc":[{"estimate":2100,"name":"Unknown"}],"top_ipv4":[{"estimate":2100,"name":"127.0.0.1"}],"top_ipv6":[],"total":2100,"udp":0}}} diff --git a/integration_tests/output-templates/dns_ipv4_tcp.Linux.json b/integration_tests/output-templates/dns_ipv4_tcp.Linux.json index dbb9c770f..379508775 100644 --- a/integration_tests/output-templates/dns_ipv4_tcp.Linux.json +++ b/integration_tests/output-templates/dns_ipv4_tcp.Linux.json @@ -1 +1 @@ -{"5m":{"dns":{"period":{"length":7,"start_ts":1567706433},"top_nxdomain":[],"top_qname2":[{"estimate":420,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".lMw.test.com"},{"estimate":2,"name":".rZ4.test.com"},{"estimate":2,"name":"ZX0A4K.test.com"},{"estimate":2,"name":".6cR5fxGGfF.test.com"},{"estimate":2,"name":".lk06eYI2ck.test.com"},{"estimate":2,"name":".2at.test.com"},{"estimate":2,"name":".gHsieFqf.test.com"},{"estimate":2,"name":".uuteIwE.test.com"},{"estimate":2,"name":"Bao7Qb.test.com"},{"estimate":2,"name":".ZguNCWgzm.test.com"}],"top_qtype":[{"estimate":100,"name":"AAAA"},{"estimate":66,"name":"NS"},{"estimate":60,"name":"A"},{"estimate":60,"name":"SOA"},{"estimate":56,"name":"MX"},{"estimate":46,"name":"TXT"},{"estimate":32,"name":"CNAME"}],"top_rcode":[{"estimate":210,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":2,"name":"43151"},{"estimate":2,"name":"41799"},{"estimate":2,"name":"35007"},{"estimate":2,"name":"43313"},{"estimate":2,"name":"33341"},{"estimate":2,"name":"46517"},{"estimate":2,"name":"46063"},{"estimate":2,"name":"44273"},{"estimate":2,"name":"43615"},{"estimate":2,"name":"39577"}],"wire_packets":{"deep_samples":420,"ipv4":420,"ipv6":0,"noerror":0,"nxdomain":0,"queries":210,"refused":0,"replies":210,"srvfail":0,"tcp":420,"total":420,"udp":0},"xact":{"counts":{"timed_out":0,"total":210},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":210}}},"packets":{"deep_samples":2100,"in":2100,"ipv4":2100,"ipv6":0,"other_l4":0,"out":0,"period":{"length":7,"start_ts":1567706433},"tcp":2100,"top_ASN":[{"estimate":2100,"name":"Unknown"}],"top_geoLoc":[{"estimate":2100,"name":"Unknown"}],"top_ipv4":[{"estimate":2100,"name":"127.0.0.1"}],"top_ipv6":[],"total":2100,"udp":0}}} +{"5m":{"dns":{"period":{"length":7,"start_ts":1567706433},"top_nxdomain":[],"top_qname2":[{"estimate":420,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".lMw.test.com"},{"estimate":2,"name":".rZ4.test.com"},{"estimate":2,"name":"ZX0A4K.test.com"},{"estimate":2,"name":".6cR5fxGGfF.test.com"},{"estimate":2,"name":".lk06eYI2ck.test.com"},{"estimate":2,"name":".2at.test.com"},{"estimate":2,"name":".gHsieFqf.test.com"},{"estimate":2,"name":".uuteIwE.test.com"},{"estimate":2,"name":"Bao7Qb.test.com"},{"estimate":2,"name":".ZguNCWgzm.test.com"}],"top_qtype":[{"estimate":100,"name":"AAAA"},{"estimate":66,"name":"NS"},{"estimate":60,"name":"A"},{"estimate":60,"name":"SOA"},{"estimate":56,"name":"MX"},{"estimate":46,"name":"TXT"},{"estimate":32,"name":"CNAME"}],"top_rcode":[{"estimate":210,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":2,"name":"43151"},{"estimate":2,"name":"41799"},{"estimate":2,"name":"35007"},{"estimate":2,"name":"43313"},{"estimate":2,"name":"33341"},{"estimate":2,"name":"46517"},{"estimate":2,"name":"46063"},{"estimate":2,"name":"44273"},{"estimate":2,"name":"43615"},{"estimate":2,"name":"39577"}],"wire_packets":{"deep_samples":420,"filtered":0,"ipv4":420,"ipv6":0,"noerror":0,"nxdomain":0,"queries":210,"refused":0,"replies":210,"srvfail":0,"tcp":420,"total":420,"udp":0},"xact":{"counts":{"timed_out":0,"total":210},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":210}}},"packets":{"deep_samples":2100,"in":2100,"ipv4":2100,"ipv6":0,"other_l4":0,"out":0,"period":{"length":7,"start_ts":1567706433},"tcp":2100,"top_ASN":[{"estimate":2100,"name":"Unknown"}],"top_geoLoc":[{"estimate":2100,"name":"Unknown"}],"top_ipv4":[{"estimate":2100,"name":"127.0.0.1"}],"top_ipv6":[],"total":2100,"udp":0}}} diff --git a/integration_tests/output-templates/dns_ipv4_udp.Darwin.json b/integration_tests/output-templates/dns_ipv4_udp.Darwin.json index 043013c83..1f30f12c4 100644 --- a/integration_tests/output-templates/dns_ipv4_udp.Darwin.json +++ b/integration_tests/output-templates/dns_ipv4_udp.Darwin.json @@ -1 +1 @@ -{"5m":{"dns":{"period":{"length":6,"start_ts":1567706414},"top_nxdomain":[],"top_qname2":[{"estimate":140,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".yhsx.test.com"},{"estimate":2,"name":".rEdwaRK.test.com"},{"estimate":2,"name":".-cSCWAO.test.com"},{"estimate":2,"name":"ik4R35VBCx.test.com"},{"estimate":2,"name":".-q-SZlz.test.com"},{"estimate":2,"name":".1gGN.test.com"},{"estimate":2,"name":".J-r.test.com"},{"estimate":2,"name":".cx.test.com"},{"estimate":2,"name":".N5V-rUqc.test.com"},{"estimate":2,"name":".TQHgrpzDe.test.com"}],"top_qtype":[{"estimate":34,"name":"AAAA"},{"estimate":24,"name":"NS"},{"estimate":18,"name":"SOA"},{"estimate":18,"name":"TXT"},{"estimate":16,"name":"A"},{"estimate":16,"name":"CNAME"},{"estimate":14,"name":"MX"}],"top_rcode":[{"estimate":70,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":14,"name":"38352"},{"estimate":14,"name":"59755"},{"estimate":14,"name":"45478"},{"estimate":14,"name":"55945"},{"estimate":14,"name":"37253"},{"estimate":14,"name":"46297"},{"estimate":14,"name":"37371"},{"estimate":14,"name":"53819"},{"estimate":14,"name":"39485"},{"estimate":14,"name":"38302"}],"wire_packets":{"deep_samples":140,"ipv4":140,"ipv6":0,"noerror":0,"nxdomain":0,"queries":70,"refused":0,"replies":70,"srvfail":0,"tcp":0,"total":140,"udp":140},"xact":{"counts":{"timed_out":0,"total":70},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":70}}},"packets":{"deep_samples":140,"in":140,"ipv4":140,"ipv6":0,"other_l4":0,"out":0,"period":{"length":6,"start_ts":1567706414},"tcp":0,"top_ASN":[{"estimate":140,"name":"Unknown"}],"top_geoLoc":[{"estimate":140,"name":"Unknown"}],"top_ipv4":[{"estimate":140,"name":"127.0.0.1"}],"top_ipv6":[],"total":140,"udp":140}}} +{"5m":{"dns":{"period":{"length":6,"start_ts":1567706414},"top_nxdomain":[],"top_qname2":[{"estimate":140,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".yhsx.test.com"},{"estimate":2,"name":".rEdwaRK.test.com"},{"estimate":2,"name":".-cSCWAO.test.com"},{"estimate":2,"name":"ik4R35VBCx.test.com"},{"estimate":2,"name":".-q-SZlz.test.com"},{"estimate":2,"name":".1gGN.test.com"},{"estimate":2,"name":".J-r.test.com"},{"estimate":2,"name":".cx.test.com"},{"estimate":2,"name":".N5V-rUqc.test.com"},{"estimate":2,"name":".TQHgrpzDe.test.com"}],"top_qtype":[{"estimate":34,"name":"AAAA"},{"estimate":24,"name":"NS"},{"estimate":18,"name":"SOA"},{"estimate":18,"name":"TXT"},{"estimate":16,"name":"A"},{"estimate":16,"name":"CNAME"},{"estimate":14,"name":"MX"}],"top_rcode":[{"estimate":70,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":14,"name":"38352"},{"estimate":14,"name":"59755"},{"estimate":14,"name":"45478"},{"estimate":14,"name":"55945"},{"estimate":14,"name":"37253"},{"estimate":14,"name":"46297"},{"estimate":14,"name":"37371"},{"estimate":14,"name":"53819"},{"estimate":14,"name":"39485"},{"estimate":14,"name":"38302"}],"wire_packets":{"deep_samples":140,"filtered":0,"ipv4":140,"ipv6":0,"noerror":0,"nxdomain":0,"queries":70,"refused":0,"replies":70,"srvfail":0,"tcp":0,"total":140,"udp":140},"xact":{"counts":{"timed_out":0,"total":70},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":70}}},"packets":{"deep_samples":140,"in":140,"ipv4":140,"ipv6":0,"other_l4":0,"out":0,"period":{"length":6,"start_ts":1567706414},"tcp":0,"top_ASN":[{"estimate":140,"name":"Unknown"}],"top_geoLoc":[{"estimate":140,"name":"Unknown"}],"top_ipv4":[{"estimate":140,"name":"127.0.0.1"}],"top_ipv6":[],"total":140,"udp":140}}} diff --git a/integration_tests/output-templates/dns_ipv4_udp.Linux.json b/integration_tests/output-templates/dns_ipv4_udp.Linux.json index d28ef7561..5a5723bc3 100644 --- a/integration_tests/output-templates/dns_ipv4_udp.Linux.json +++ b/integration_tests/output-templates/dns_ipv4_udp.Linux.json @@ -1 +1 @@ -{"5m":{"dns":{"period":{"length":6,"start_ts":1567706414},"top_nxdomain":[],"top_qname2":[{"estimate":140,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".gTjGva7m-8.test.com"},{"estimate":2,"name":".pWK6.test.com"},{"estimate":2,"name":".TQHgrpzDe.test.com"},{"estimate":2,"name":".vmTwD0v30.test.com"},{"estimate":2,"name":".2MT.test.com"},{"estimate":2,"name":".GnKHW4hgN.test.com"},{"estimate":2,"name":"ik4R35VBCx.test.com"},{"estimate":2,"name":".cx.test.com"},{"estimate":2,"name":"G2s1RNfdkE.test.com"},{"estimate":2,"name":".yGa.test.com"}],"top_qtype":[{"estimate":34,"name":"AAAA"},{"estimate":24,"name":"NS"},{"estimate":18,"name":"SOA"},{"estimate":18,"name":"TXT"},{"estimate":16,"name":"A"},{"estimate":16,"name":"CNAME"},{"estimate":14,"name":"MX"}],"top_rcode":[{"estimate":70,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":14,"name":"38352"},{"estimate":14,"name":"59755"},{"estimate":14,"name":"45478"},{"estimate":14,"name":"55945"},{"estimate":14,"name":"37253"},{"estimate":14,"name":"46297"},{"estimate":14,"name":"37371"},{"estimate":14,"name":"53819"},{"estimate":14,"name":"39485"},{"estimate":14,"name":"38302"}],"wire_packets":{"deep_samples":140,"ipv4":140,"ipv6":0,"noerror":0,"nxdomain":0,"queries":70,"refused":0,"replies":70,"srvfail":0,"tcp":0,"total":140,"udp":140},"xact":{"counts":{"timed_out":0,"total":70},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":70}}},"packets":{"deep_samples":140,"in":140,"ipv4":140,"ipv6":0,"other_l4":0,"out":0,"period":{"length":6,"start_ts":1567706414},"tcp":0,"top_ASN":[{"estimate":140,"name":"Unknown"}],"top_geoLoc":[{"estimate":140,"name":"Unknown"}],"top_ipv4":[{"estimate":140,"name":"127.0.0.1"}],"top_ipv6":[],"total":140,"udp":140}}} +{"5m":{"dns":{"period":{"length":6,"start_ts":1567706414},"top_nxdomain":[],"top_qname2":[{"estimate":140,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".gTjGva7m-8.test.com"},{"estimate":2,"name":".pWK6.test.com"},{"estimate":2,"name":".TQHgrpzDe.test.com"},{"estimate":2,"name":".vmTwD0v30.test.com"},{"estimate":2,"name":".2MT.test.com"},{"estimate":2,"name":".GnKHW4hgN.test.com"},{"estimate":2,"name":"ik4R35VBCx.test.com"},{"estimate":2,"name":".cx.test.com"},{"estimate":2,"name":"G2s1RNfdkE.test.com"},{"estimate":2,"name":".yGa.test.com"}],"top_qtype":[{"estimate":34,"name":"AAAA"},{"estimate":24,"name":"NS"},{"estimate":18,"name":"SOA"},{"estimate":18,"name":"TXT"},{"estimate":16,"name":"A"},{"estimate":16,"name":"CNAME"},{"estimate":14,"name":"MX"}],"top_rcode":[{"estimate":70,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":14,"name":"38352"},{"estimate":14,"name":"59755"},{"estimate":14,"name":"45478"},{"estimate":14,"name":"55945"},{"estimate":14,"name":"37253"},{"estimate":14,"name":"46297"},{"estimate":14,"name":"37371"},{"estimate":14,"name":"53819"},{"estimate":14,"name":"39485"},{"estimate":14,"name":"38302"}],"wire_packets":{"deep_samples":140,"filtered":0,"ipv4":140,"ipv6":0,"noerror":0,"nxdomain":0,"queries":70,"refused":0,"replies":70,"srvfail":0,"tcp":0,"total":140,"udp":140},"xact":{"counts":{"timed_out":0,"total":70},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":70}}},"packets":{"deep_samples":140,"in":140,"ipv4":140,"ipv6":0,"other_l4":0,"out":0,"period":{"length":6,"start_ts":1567706414},"tcp":0,"top_ASN":[{"estimate":140,"name":"Unknown"}],"top_geoLoc":[{"estimate":140,"name":"Unknown"}],"top_ipv4":[{"estimate":140,"name":"127.0.0.1"}],"top_ipv6":[],"total":140,"udp":140}}} diff --git a/integration_tests/output-templates/dns_ipv6_tcp.Darwin.json b/integration_tests/output-templates/dns_ipv6_tcp.Darwin.json index 50a471052..9bb68f5e7 100644 --- a/integration_tests/output-templates/dns_ipv6_tcp.Darwin.json +++ b/integration_tests/output-templates/dns_ipv6_tcp.Darwin.json @@ -1 +1 @@ -{"5m":{"dns":{"period":{"length":7,"start_ts":1567706308},"top_nxdomain":[],"top_qname2":[{"estimate":360,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".8Zd.test.com"},{"estimate":2,"name":".7iF.test.com"},{"estimate":2,"name":".eXz.test.com"},{"estimate":2,"name":".j_dakR7.test.com"},{"estimate":2,"name":"2Z.test.com"},{"estimate":2,"name":".Sl1D.test.com"},{"estimate":2,"name":".V_o.test.com"},{"estimate":2,"name":".bmbEQvNBk.test.com"},{"estimate":2,"name":".DU5Q4ir-4A.test.com"},{"estimate":2,"name":".QD.test.com"}],"top_qtype":[{"estimate":94,"name":"AAAA"},{"estimate":58,"name":"CNAME"},{"estimate":48,"name":"SOA"},{"estimate":48,"name":"NS"},{"estimate":40,"name":"TXT"},{"estimate":38,"name":"MX"},{"estimate":34,"name":"A"}],"top_rcode":[{"estimate":180,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":2,"name":"34743"},{"estimate":2,"name":"40389"},{"estimate":2,"name":"43851"},{"estimate":2,"name":"40027"},{"estimate":2,"name":"33933"},{"estimate":2,"name":"39459"},{"estimate":2,"name":"43355"},{"estimate":2,"name":"40595"},{"estimate":2,"name":"38367"},{"estimate":2,"name":"36465"}],"wire_packets":{"deep_samples":360,"ipv4":0,"ipv6":360,"noerror":0,"nxdomain":0,"queries":180,"refused":0,"replies":180,"srvfail":0,"tcp":360,"total":360,"udp":0},"xact":{"counts":{"timed_out":0,"total":180},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":180}}},"packets":{"deep_samples":1800,"in":0,"ipv4":0,"ipv6":1800,"other_l4":0,"out":0,"period":{"length":7,"start_ts":1567706308},"tcp":1800,"top_ASN":[],"top_geoLoc":[],"top_ipv4":[],"top_ipv6":[],"total":1800,"udp":0}}} +{"5m":{"dns":{"period":{"length":7,"start_ts":1567706308},"top_nxdomain":[],"top_qname2":[{"estimate":360,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".8Zd.test.com"},{"estimate":2,"name":".7iF.test.com"},{"estimate":2,"name":".eXz.test.com"},{"estimate":2,"name":".j_dakR7.test.com"},{"estimate":2,"name":"2Z.test.com"},{"estimate":2,"name":".Sl1D.test.com"},{"estimate":2,"name":".V_o.test.com"},{"estimate":2,"name":".bmbEQvNBk.test.com"},{"estimate":2,"name":".DU5Q4ir-4A.test.com"},{"estimate":2,"name":".QD.test.com"}],"top_qtype":[{"estimate":94,"name":"AAAA"},{"estimate":58,"name":"CNAME"},{"estimate":48,"name":"SOA"},{"estimate":48,"name":"NS"},{"estimate":40,"name":"TXT"},{"estimate":38,"name":"MX"},{"estimate":34,"name":"A"}],"top_rcode":[{"estimate":180,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":2,"name":"34743"},{"estimate":2,"name":"40389"},{"estimate":2,"name":"43851"},{"estimate":2,"name":"40027"},{"estimate":2,"name":"33933"},{"estimate":2,"name":"39459"},{"estimate":2,"name":"43355"},{"estimate":2,"name":"40595"},{"estimate":2,"name":"38367"},{"estimate":2,"name":"36465"}],"wire_packets":{"deep_samples":360,"filtered":0,"ipv4":0,"ipv6":360,"noerror":0,"nxdomain":0,"queries":180,"refused":0,"replies":180,"srvfail":0,"tcp":360,"total":360,"udp":0},"xact":{"counts":{"timed_out":0,"total":180},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":180}}},"packets":{"deep_samples":1800,"in":0,"ipv4":0,"ipv6":1800,"other_l4":0,"out":0,"period":{"length":7,"start_ts":1567706308},"tcp":1800,"top_ASN":[],"top_geoLoc":[],"top_ipv4":[],"top_ipv6":[],"total":1800,"udp":0}}} diff --git a/integration_tests/output-templates/dns_ipv6_tcp.Linux.json b/integration_tests/output-templates/dns_ipv6_tcp.Linux.json index eced2313c..5dfff6631 100644 --- a/integration_tests/output-templates/dns_ipv6_tcp.Linux.json +++ b/integration_tests/output-templates/dns_ipv6_tcp.Linux.json @@ -1 +1 @@ -{"5m":{"dns":{"period":{"length":7,"start_ts":1567706308},"top_nxdomain":[],"top_qname2":[{"estimate":360,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".NW5zpCG.test.com"},{"estimate":2,"name":".qeg0fit.test.com"},{"estimate":2,"name":"0Hb.test.com"},{"estimate":2,"name":".z8Pa.test.com"},{"estimate":2,"name":".YCOpO3bN.test.com"},{"estimate":2,"name":"._ST.test.com"},{"estimate":2,"name":".90ylh4E.test.com"},{"estimate":2,"name":".mD2GMXLqA.test.com"},{"estimate":2,"name":".pO.test.com"},{"estimate":2,"name":".bzj6xe.test.com"}],"top_qtype":[{"estimate":94,"name":"AAAA"},{"estimate":58,"name":"CNAME"},{"estimate":48,"name":"SOA"},{"estimate":48,"name":"NS"},{"estimate":40,"name":"TXT"},{"estimate":38,"name":"MX"},{"estimate":34,"name":"A"}],"top_rcode":[{"estimate":180,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":2,"name":"45561"},{"estimate":2,"name":"46571"},{"estimate":2,"name":"36263"},{"estimate":2,"name":"44467"},{"estimate":2,"name":"43471"},{"estimate":2,"name":"43035"},{"estimate":2,"name":"35661"},{"estimate":2,"name":"46011"},{"estimate":2,"name":"37159"},{"estimate":2,"name":"44697"}],"wire_packets":{"deep_samples":360,"ipv4":0,"ipv6":360,"noerror":0,"nxdomain":0,"queries":180,"refused":0,"replies":180,"srvfail":0,"tcp":360,"total":360,"udp":0},"xact":{"counts":{"timed_out":0,"total":180},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":180}}},"packets":{"deep_samples":1800,"in":0,"ipv4":0,"ipv6":1800,"other_l4":0,"out":0,"period":{"length":7,"start_ts":1567706308},"tcp":1800,"top_ASN":[],"top_geoLoc":[],"top_ipv4":[],"top_ipv6":[],"total":1800,"udp":0}}} +{"5m":{"dns":{"period":{"length":7,"start_ts":1567706308},"top_nxdomain":[],"top_qname2":[{"estimate":360,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".NW5zpCG.test.com"},{"estimate":2,"name":".qeg0fit.test.com"},{"estimate":2,"name":"0Hb.test.com"},{"estimate":2,"name":".z8Pa.test.com"},{"estimate":2,"name":".YCOpO3bN.test.com"},{"estimate":2,"name":"._ST.test.com"},{"estimate":2,"name":".90ylh4E.test.com"},{"estimate":2,"name":".mD2GMXLqA.test.com"},{"estimate":2,"name":".pO.test.com"},{"estimate":2,"name":".bzj6xe.test.com"}],"top_qtype":[{"estimate":94,"name":"AAAA"},{"estimate":58,"name":"CNAME"},{"estimate":48,"name":"SOA"},{"estimate":48,"name":"NS"},{"estimate":40,"name":"TXT"},{"estimate":38,"name":"MX"},{"estimate":34,"name":"A"}],"top_rcode":[{"estimate":180,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":2,"name":"45561"},{"estimate":2,"name":"46571"},{"estimate":2,"name":"36263"},{"estimate":2,"name":"44467"},{"estimate":2,"name":"43471"},{"estimate":2,"name":"43035"},{"estimate":2,"name":"35661"},{"estimate":2,"name":"46011"},{"estimate":2,"name":"37159"},{"estimate":2,"name":"44697"}],"wire_packets":{"deep_samples":360,"filtered":0,"ipv4":0,"ipv6":360,"noerror":0,"nxdomain":0,"queries":180,"refused":0,"replies":180,"srvfail":0,"tcp":360,"total":360,"udp":0},"xact":{"counts":{"timed_out":0,"total":180},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":180}}},"packets":{"deep_samples":1800,"in":0,"ipv4":0,"ipv6":1800,"other_l4":0,"out":0,"period":{"length":7,"start_ts":1567706308},"tcp":1800,"top_ASN":[],"top_geoLoc":[],"top_ipv4":[],"top_ipv6":[],"total":1800,"udp":0}}} diff --git a/integration_tests/output-templates/dns_ipv6_udp.Darwin.json b/integration_tests/output-templates/dns_ipv6_udp.Darwin.json index 69ae7f248..58a003155 100644 --- a/integration_tests/output-templates/dns_ipv6_udp.Darwin.json +++ b/integration_tests/output-templates/dns_ipv6_udp.Darwin.json @@ -1 +1 @@ -{"5m":{"dns":{"period":{"length":6,"start_ts":1567706365},"top_nxdomain":[],"top_qname2":[{"estimate":140,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".8a.test.com"},{"estimate":2,"name":"Il9NL4Uvb.test.com"},{"estimate":2,"name":"aC6.test.com"},{"estimate":2,"name":".DNn1O5NnK.test.com"},{"estimate":2,"name":"-aQ2.test.com"},{"estimate":2,"name":".g7k.test.com"},{"estimate":2,"name":".OGL8.test.com"},{"estimate":2,"name":"at1s2IrEO.test.com"},{"estimate":2,"name":".FiKKi.test.com"},{"estimate":2,"name":".ZOtW.test.com"}],"top_qtype":[{"estimate":34,"name":"AAAA"},{"estimate":22,"name":"A"},{"estimate":22,"name":"SOA"},{"estimate":22,"name":"NS"},{"estimate":14,"name":"MX"},{"estimate":14,"name":"CNAME"},{"estimate":12,"name":"TXT"}],"top_rcode":[{"estimate":70,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":14,"name":"46590"},{"estimate":14,"name":"58571"},{"estimate":14,"name":"36484"},{"estimate":14,"name":"43798"},{"estimate":14,"name":"46567"},{"estimate":14,"name":"59640"},{"estimate":14,"name":"44741"},{"estimate":14,"name":"51811"},{"estimate":14,"name":"38920"},{"estimate":14,"name":"52826"}],"wire_packets":{"deep_samples":140,"ipv4":0,"ipv6":140,"noerror":0,"nxdomain":0,"queries":70,"refused":0,"replies":70,"srvfail":0,"tcp":0,"total":140,"udp":140},"xact":{"counts":{"timed_out":0,"total":70},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":0}}},"packets":{"deep_samples":140,"in":0,"ipv4":0,"ipv6":140,"other_l4":0,"out":0,"period":{"length":6,"start_ts":1567706365},"tcp":0,"top_ASN":[],"top_geoLoc":[],"top_ipv4":[],"top_ipv6":[],"total":140,"udp":140}}} +{"5m":{"dns":{"period":{"length":6,"start_ts":1567706365},"top_nxdomain":[],"top_qname2":[{"estimate":140,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".8a.test.com"},{"estimate":2,"name":"Il9NL4Uvb.test.com"},{"estimate":2,"name":"aC6.test.com"},{"estimate":2,"name":".DNn1O5NnK.test.com"},{"estimate":2,"name":"-aQ2.test.com"},{"estimate":2,"name":".g7k.test.com"},{"estimate":2,"name":".OGL8.test.com"},{"estimate":2,"name":"at1s2IrEO.test.com"},{"estimate":2,"name":".FiKKi.test.com"},{"estimate":2,"name":".ZOtW.test.com"}],"top_qtype":[{"estimate":34,"name":"AAAA"},{"estimate":22,"name":"A"},{"estimate":22,"name":"SOA"},{"estimate":22,"name":"NS"},{"estimate":14,"name":"MX"},{"estimate":14,"name":"CNAME"},{"estimate":12,"name":"TXT"}],"top_rcode":[{"estimate":70,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":14,"name":"46590"},{"estimate":14,"name":"58571"},{"estimate":14,"name":"36484"},{"estimate":14,"name":"43798"},{"estimate":14,"name":"46567"},{"estimate":14,"name":"59640"},{"estimate":14,"name":"44741"},{"estimate":14,"name":"51811"},{"estimate":14,"name":"38920"},{"estimate":14,"name":"52826"}],"wire_packets":{"deep_samples":140,"filtered":0,"ipv4":0,"ipv6":140,"noerror":0,"nxdomain":0,"queries":70,"refused":0,"replies":70,"srvfail":0,"tcp":0,"total":140,"udp":140},"xact":{"counts":{"timed_out":0,"total":70},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":0}}},"packets":{"deep_samples":140,"in":0,"ipv4":0,"ipv6":140,"other_l4":0,"out":0,"period":{"length":6,"start_ts":1567706365},"tcp":0,"top_ASN":[],"top_geoLoc":[],"top_ipv4":[],"top_ipv6":[],"total":140,"udp":140}}} diff --git a/integration_tests/output-templates/dns_ipv6_udp.Linux.json b/integration_tests/output-templates/dns_ipv6_udp.Linux.json index b7c8bddbe..dcdf080cd 100644 --- a/integration_tests/output-templates/dns_ipv6_udp.Linux.json +++ b/integration_tests/output-templates/dns_ipv6_udp.Linux.json @@ -1 +1 @@ -{"5m":{"dns":{"period":{"length":6,"start_ts":1567706365},"top_nxdomain":[],"top_qname2":[{"estimate":140,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".2pmr.test.com"},{"estimate":2,"name":".uL8E2T.test.com"},{"estimate":2,"name":".kGuiH.test.com"},{"estimate":2,"name":".ddU3i_v.test.com"},{"estimate":2,"name":".nU.test.com"},{"estimate":2,"name":"lfV2-uf4P9.test.com"},{"estimate":2,"name":".eW_.test.com"},{"estimate":2,"name":".FiKKi.test.com"},{"estimate":2,"name":".MA_b.test.com"},{"estimate":2,"name":".SmCeRI.test.com"}],"top_qtype":[{"estimate":34,"name":"AAAA"},{"estimate":22,"name":"A"},{"estimate":22,"name":"SOA"},{"estimate":22,"name":"NS"},{"estimate":14,"name":"MX"},{"estimate":14,"name":"CNAME"},{"estimate":12,"name":"TXT"}],"top_rcode":[{"estimate":70,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":14,"name":"46590"},{"estimate":14,"name":"58571"},{"estimate":14,"name":"36484"},{"estimate":14,"name":"43798"},{"estimate":14,"name":"46567"},{"estimate":14,"name":"59640"},{"estimate":14,"name":"44741"},{"estimate":14,"name":"51811"},{"estimate":14,"name":"38920"},{"estimate":14,"name":"52826"}],"wire_packets":{"deep_samples":140,"ipv4":0,"ipv6":140,"noerror":0,"nxdomain":0,"queries":70,"refused":0,"replies":70,"srvfail":0,"tcp":0,"total":140,"udp":140},"xact":{"counts":{"timed_out":0,"total":70},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":0}}},"packets":{"deep_samples":140,"in":0,"ipv4":0,"ipv6":140,"other_l4":0,"out":0,"period":{"length":6,"start_ts":1567706365},"tcp":0,"top_ASN":[],"top_geoLoc":[],"top_ipv4":[],"top_ipv6":[],"total":140,"udp":140}}} +{"5m":{"dns":{"period":{"length":6,"start_ts":1567706365},"top_nxdomain":[],"top_qname2":[{"estimate":140,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".2pmr.test.com"},{"estimate":2,"name":".uL8E2T.test.com"},{"estimate":2,"name":".kGuiH.test.com"},{"estimate":2,"name":".ddU3i_v.test.com"},{"estimate":2,"name":".nU.test.com"},{"estimate":2,"name":"lfV2-uf4P9.test.com"},{"estimate":2,"name":".eW_.test.com"},{"estimate":2,"name":".FiKKi.test.com"},{"estimate":2,"name":".MA_b.test.com"},{"estimate":2,"name":".SmCeRI.test.com"}],"top_qtype":[{"estimate":34,"name":"AAAA"},{"estimate":22,"name":"A"},{"estimate":22,"name":"SOA"},{"estimate":22,"name":"NS"},{"estimate":14,"name":"MX"},{"estimate":14,"name":"CNAME"},{"estimate":12,"name":"TXT"}],"top_rcode":[{"estimate":70,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":14,"name":"46590"},{"estimate":14,"name":"58571"},{"estimate":14,"name":"36484"},{"estimate":14,"name":"43798"},{"estimate":14,"name":"46567"},{"estimate":14,"name":"59640"},{"estimate":14,"name":"44741"},{"estimate":14,"name":"51811"},{"estimate":14,"name":"38920"},{"estimate":14,"name":"52826"}],"wire_packets":{"deep_samples":140,"filtered":0,"ipv4":0,"ipv6":140,"noerror":0,"nxdomain":0,"queries":70,"refused":0,"replies":70,"srvfail":0,"tcp":0,"total":140,"udp":140},"xact":{"counts":{"timed_out":0,"total":70},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":0}}},"packets":{"deep_samples":140,"in":0,"ipv4":0,"ipv6":140,"other_l4":0,"out":0,"period":{"length":6,"start_ts":1567706365},"tcp":0,"top_ASN":[],"top_geoLoc":[],"top_ipv4":[],"top_ipv6":[],"total":140,"udp":140}}} diff --git a/src/AbstractMetricsManager.h b/src/AbstractMetricsManager.h index 5a59137ed..12295896a 100644 --- a/src/AbstractMetricsManager.h +++ b/src/AbstractMetricsManager.h @@ -284,16 +284,18 @@ class AbstractMetricsManager protected: /** * the "base" event method that should be called on every event before specialized event functionality. sampling will be - * chosen, and the time window will be maintained + * (optionally) chosen, and the time window will be maintained * * @param stamp time stamp of the event */ - void new_event(timespec stamp) + void new_event(timespec stamp, bool sample = true) { // CRITICAL EVENT PATH - _deep_sampling_now.store(true, std::memory_order_relaxed); - if (_deep_sample_rate != 100) { - _deep_sampling_now.store((_rng.uniform(0U, 100U) <= _deep_sample_rate), std::memory_order_relaxed); + if (sample) { + _deep_sampling_now.store(true, std::memory_order_relaxed); + if (_deep_sample_rate != 100) { + _deep_sampling_now.store((_rng.uniform(0U, 100U) <= _deep_sample_rate), std::memory_order_relaxed); + } } std::shared_lock rlb(_base_mutex); bool will_shift = _num_periods > 1 && stamp.tv_sec >= _next_shift_tstamp.tv_sec; diff --git a/src/Configurable.h b/src/Configurable.h index ec64e6ac6..24daee946 100644 --- a/src/Configurable.h +++ b/src/Configurable.h @@ -13,6 +13,7 @@ #include #include #include +#include #include namespace visor { @@ -30,8 +31,11 @@ class ConfigException : public std::runtime_error class Configurable { +public: + typedef std::vector StringList; + private: - std::unordered_map> _config; + std::unordered_map> _config; mutable std::shared_mutex _config_mutex; public: diff --git a/src/handlers/dns/DnsLayer.cpp b/src/handlers/dns/DnsLayer.cpp index 2f9d567a2..ccfe3b239 100644 --- a/src/handlers/dns/DnsLayer.cpp +++ b/src/handlers/dns/DnsLayer.cpp @@ -120,6 +120,11 @@ bool DnsLayer::shortenLayer(int offsetInLayer, size_t numOfBytesToShorten, IDnsR bool DnsLayer::parseResources(bool queryOnly) { + + if (m_ResourcesParsed) { + return m_ResourcesParseResult; + } + size_t offsetInPacket = sizeof(dnshdr); IDnsResource *curResource = m_ResourceList; @@ -132,7 +137,9 @@ bool DnsLayer::parseResources(bool queryOnly) if (numOfOtherResources > 100) { // probably bad packet - return false; + m_ResourcesParsed = true; + m_ResourcesParseResult = false; + return m_ResourcesParseResult; } for (uint32_t i = 0; i < numOfOtherResources; i++) { @@ -174,7 +181,9 @@ bool DnsLayer::parseResources(bool queryOnly) if (offsetInPacket > m_DataLen) { //Parse packet failed, DNS resource is out of bounds. Probably a bad packet delete newGenResource; - return false; + m_ResourcesParsed = true; + m_ResourcesParseResult = false; + return m_ResourcesParseResult; } // this resource is the first resource @@ -201,7 +210,9 @@ bool DnsLayer::parseResources(bool queryOnly) m_FirstAdditional = newResource; } - return true; + m_ResourcesParsed = true; + m_ResourcesParseResult = true; + return m_ResourcesParseResult; } IDnsResource* DnsLayer::getResourceByName(IDnsResource* startFrom, size_t resourceCount, const std::string& name, bool exactMatch) const diff --git a/src/handlers/dns/DnsLayer.h b/src/handlers/dns/DnsLayer.h index e6466c9cb..8a5c10e5d 100644 --- a/src/handlers/dns/DnsLayer.h +++ b/src/handlers/dns/DnsLayer.h @@ -435,28 +435,33 @@ struct dnshdr { bool parseResources(bool queryOnly); - pcpp::OsiModelLayer getOsiModelLayer() const { return pcpp::OsiModelApplicationLayer; } + pcpp::OsiModelLayer getOsiModelLayer() const + { + return pcpp::OsiModelApplicationLayer; + } - /** + /** * A static method that checks whether the port is considered as DNS * @param[in] port The port number to be checked */ - static inline bool isDnsPort(uint16_t port); + static inline bool isDnsPort(uint16_t port); - private: - IDnsResource* m_ResourceList; - DnsQuery* m_FirstQuery; - DnsResource* m_FirstAnswer; - DnsResource* m_FirstAuthority; - DnsResource* m_FirstAdditional; + private: + bool m_ResourcesParsed{false}; + bool m_ResourcesParseResult{false}; + IDnsResource *m_ResourceList; + DnsQuery *m_FirstQuery; + DnsResource *m_FirstAnswer; + DnsResource *m_FirstAuthority; + DnsResource *m_FirstAdditional; - IDnsResource* getFirstResource(DnsResourceType resType) const; - void setFirstResource(DnsResourceType resType, IDnsResource* resource); + IDnsResource *getFirstResource(DnsResourceType resType) const; + void setFirstResource(DnsResourceType resType, IDnsResource *resource); - using Layer::extendLayer; - bool extendLayer(int offsetInLayer, size_t numOfBytesToExtend, IDnsResource* resource); + using Layer::extendLayer; + bool extendLayer(int offsetInLayer, size_t numOfBytesToExtend, IDnsResource *resource); - using Layer::shortenLayer; + using Layer::shortenLayer; bool shortenLayer(int offsetInLayer, size_t numOfBytesToShorten, IDnsResource* resource); IDnsResource* getResourceByName(IDnsResource* startFrom, size_t resourceCount, const std::string& name, bool exactMatch) const; diff --git a/src/handlers/dns/DnsStreamHandler.cpp b/src/handlers/dns/DnsStreamHandler.cpp index be044cf5d..9e10e8d92 100644 --- a/src/handlers/dns/DnsStreamHandler.cpp +++ b/src/handlers/dns/DnsStreamHandler.cpp @@ -37,6 +37,37 @@ void DnsStreamHandler::start() return; } + // Setup Filters + if (config_exists("filter_exclude_noerror") && config_get("filter_exclude_noerror")) { + _f_enabled.set(Filters::ExcludingRCode); + _f_rcode = NoError; + } else if (config_exists("filter_only_rcode")) { + auto want_code = config_get("filter_only_rcode"); + switch (want_code) { + case NoError: + case NXDomain: + case SrvFail: + case Refused: + _f_enabled.set(Filters::OnlyRCode); + _f_rcode = want_code; + break; + default: + throw ConfigException("filter_only_rcode contained an invalid/unsupported rcode"); + } + } + if (config_exists("filter_only_qname_suffix")) { + _f_enabled.set(Filters::OnlyQNameSuffix); + for (const auto &qname : config_get("filter_only_qname_suffix")) { + // note, this currently copies the strings, meaning there could be a big list that is duplicated + // we can work on trying to make this a string_view instead + // we copy it out so that we don't have to hit the config mutex + std::string qname_ci{qname}; + std::transform(qname_ci.begin(), qname_ci.end(), qname_ci.begin(), + [](unsigned char c) { return std::tolower(c); }); + _f_qnames.emplace_back(std::move(qname_ci)); + } + } + if (config_exists("recorded_stream")) { _metrics->set_recorded_stream(); } @@ -71,10 +102,6 @@ void DnsStreamHandler::stop() _running = false; } -DnsStreamHandler::~DnsStreamHandler() -{ -} - // callback from input module void DnsStreamHandler::process_udp_packet_cb(pcpp::Packet &payload, PacketDirection dir, pcpp::ProtocolType l3, uint32_t flowkey, timespec stamp) { @@ -93,7 +120,9 @@ void DnsStreamHandler::process_udp_packet_cb(pcpp::Packet &payload, PacketDirect } if (metric_port) { DnsLayer dnsLayer(udpLayer, &payload); - _metrics->process_dns_layer(dnsLayer, dir, l3, pcpp::UDP, flowkey, metric_port, stamp); + if (!_filtering(dnsLayer, dir, l3, pcpp::UDP, metric_port, stamp)) { + _metrics->process_dns_layer(dnsLayer, dir, l3, pcpp::UDP, flowkey, metric_port, stamp); + } } } @@ -169,7 +198,9 @@ void DnsStreamHandler::tcp_message_ready_cb(int8_t side, const pcpp::TcpStreamDa // instead using the packet meta data we pass in pcpp::Packet dummy_packet; DnsLayer dnsLayer(data.get(), size, nullptr, &dummy_packet); - _metrics->process_dns_layer(dnsLayer, dir, l3Type, pcpp::TCP, flowKey, port, stamp); + if (!_filtering(dnsLayer, dir, l3Type, pcpp::UDP, port, stamp)) { + _metrics->process_dns_layer(dnsLayer, dir, l3Type, pcpp::TCP, flowKey, port, stamp); + } // data is freed upon return }; @@ -225,6 +256,40 @@ void DnsStreamHandler::info_json(json &j) const common_info_json(j); j[schema_key()]["xact"]["open"] = _metrics->num_open_transactions(); } +static inline bool endsWith(std::string_view str, std::string_view suffix) +{ + return str.size() >= suffix.size() && 0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix); +} +bool DnsStreamHandler::_filtering(DnsLayer &payload, [[maybe_unused]] PacketDirection dir, [[maybe_unused]] pcpp::ProtocolType l3, [[maybe_unused]] pcpp::ProtocolType l4, [[maybe_unused]] uint16_t port, timespec stamp) +{ + if (_f_enabled.test(Filters::ExcludingRCode) && payload.getDnsHeader()->responseCode == _f_rcode) { + goto will_filter; + } else if (_f_enabled.test(Filters::OnlyRCode) && payload.getDnsHeader()->responseCode != _f_rcode) { + goto will_filter; + } + if (_f_enabled.test(Filters::OnlyQNameSuffix)) { + if (!payload.parseResources(true) || payload.getFirstQuery() == nullptr) { + goto will_filter; + } + // we need an all lower case version of this, we can't get away without making a copy + std::string qname_ci{payload.getFirstQuery()->getName()}; + std::transform(qname_ci.begin(), qname_ci.end(), qname_ci.begin(), + [](unsigned char c) { return std::tolower(c); }); + for (auto fqn : _f_qnames) { + // if it matched, we know we are not filtering + if (endsWith(qname_ci, fqn)) { + goto will_not_filter; + } + } + // checked the whole list and none of them matched: filter + goto will_filter; + } +will_not_filter: + return false; +will_filter: + _metrics->process_filtered(stamp); + return true; +} void DnsMetricsBucket::specialized_merge(const AbstractMetricsBucket &o) { @@ -249,6 +314,8 @@ void DnsMetricsBucket::specialized_merge(const AbstractMetricsBucket &o) _counters.SRVFAIL += other._counters.SRVFAIL; _counters.NOERROR += other._counters.NOERROR; + _counters.filtered += other._counters.filtered; + _dnsXactFromTimeUs.merge(other._dnsXactFromTimeUs); _dnsXactToTimeUs.merge(other._dnsXactToTimeUs); @@ -289,6 +356,8 @@ void DnsMetricsBucket::to_json(json &j) const _counters.SRVFAIL.to_json(j); _counters.NOERROR.to_json(j); + _counters.filtered.to_json(j); + _dns_qnameCard.to_json(j); _counters.xacts_total.to_json(j); _counters.xacts_timed_out.to_json(j); @@ -406,7 +475,6 @@ void DnsMetricsBucket::process_dns_layer(bool deep, DnsLayer &payload, pcpp::Pro _dns_topQname3.update(std::string(aggDomain.second)); } } - } void DnsMetricsBucket::new_dns_transaction(bool deep, float to90th, float from90th, DnsLayer &dns, PacketDirection dir, DnsTransaction xact) @@ -466,6 +534,8 @@ void DnsMetricsBucket::to_prometheus(std::stringstream &out, Metric::LabelMap ad _counters.SRVFAIL.to_prometheus(out, add_labels); _counters.NOERROR.to_prometheus(out, add_labels); + _counters.filtered.to_prometheus(out, add_labels); + _dns_qnameCard.to_prometheus(out, add_labels); _counters.xacts_total.to_prometheus(out, add_labels); _counters.xacts_timed_out.to_prometheus(out, add_labels); @@ -500,6 +570,11 @@ void DnsMetricsBucket::to_prometheus(std::stringstream &out, Metric::LabelMap ad } }); } +void DnsMetricsBucket::process_filtered() +{ + std::unique_lock lock(_mutex); + ++_counters.filtered; +} // the general metrics manager entry point (both UDP and TCP) void DnsMetricsManager::process_dns_layer(DnsLayer &payload, PacketDirection dir, pcpp::ProtocolType l3, pcpp::ProtocolType l4, uint32_t flowkey, uint16_t port, timespec stamp) @@ -518,5 +593,10 @@ void DnsMetricsManager::process_dns_layer(DnsLayer &payload, PacketDirection dir _qr_pair_manager.start_transaction(flowkey, payload.getDnsHeader()->transactionID, stamp); } } - +void DnsMetricsManager::process_filtered(timespec stamp) +{ + // base event, no sample + new_event(stamp, false); + live_bucket()->process_filtered(); +} } \ No newline at end of file diff --git a/src/handlers/dns/DnsStreamHandler.h b/src/handlers/dns/DnsStreamHandler.h index e00005635..d96adf24e 100644 --- a/src/handlers/dns/DnsStreamHandler.h +++ b/src/handlers/dns/DnsStreamHandler.h @@ -11,6 +11,8 @@ #include "dns.h" #include "querypairmgr.h" #include +#include +#include #include namespace visor::handler::dns { @@ -54,6 +56,7 @@ class DnsMetricsBucket final : public visor::AbstractMetricsBucket Counter REFUSED; Counter SRVFAIL; Counter NOERROR; + Counter filtered; counters() : xacts_total("dns", {"xact", "counts", "total"}, "Total DNS transactions (query/reply pairs)") , xacts_in("dns", {"xact", "in", "total"}, "Total ingress DNS transactions (host is server)") @@ -69,6 +72,7 @@ class DnsMetricsBucket final : public visor::AbstractMetricsBucket , REFUSED("dns", {"wire_packets", "refused"}, "Total DNS wire packets flagged as reply with return code REFUSED (ingress and egress)") , SRVFAIL("dns", {"wire_packets", "srvfail"}, "Total DNS wire packets flagged as reply with return code SRVFAIL (ingress and egress)") , NOERROR("dns", {"wire_packets", "noerror"}, "Total DNS wire packets flagged as reply with return code NOERROR (ingress and egress)") + , filtered("dns", {"wire_packets", "filtered"}, "Total DNS wire packets seen that did not match the configured filter(s) (if any)") { } }; @@ -124,6 +128,7 @@ class DnsMetricsBucket final : public visor::AbstractMetricsBucket void to_json(json &j) const override; void to_prometheus(std::stringstream &out, Metric::LabelMap add_labels = {}) const override; + void process_filtered(); void process_dns_layer(bool deep, DnsLayer &payload, pcpp::ProtocolType l3, pcpp::ProtocolType l4, uint16_t port); void new_dns_transaction(bool deep, float to90th, float from90th, DnsLayer &dns, PacketDirection dir, DnsTransaction xact); @@ -133,8 +138,8 @@ class DnsMetricsManager final : public visor::AbstractMetricsManager _f_enabled; + uint16_t _f_rcode{0}; + std::vector _f_qnames; + + bool _filtering(DnsLayer &payload, PacketDirection dir, pcpp::ProtocolType l3, pcpp::ProtocolType l4, uint16_t port, timespec stamp); + public: DnsStreamHandler(const std::string &name, InputStream *stream, const Configurable *window_config); - ~DnsStreamHandler() override; + ~DnsStreamHandler() = default; // visor::AbstractModule std::string schema_key() const override diff --git a/src/handlers/dns/tests/test_dns_layer.cpp b/src/handlers/dns/tests/test_dns_layer.cpp index a63ecd0f3..ca183c4c1 100644 --- a/src/handlers/dns/tests/test_dns_layer.cpp +++ b/src/handlers/dns/tests/test_dns_layer.cpp @@ -252,3 +252,124 @@ TEST_CASE("Parse DNS random UDP/TCP tests", "[pcap][net]") CHECK(j["top_qtype"][6]["name"] == "TXT"); CHECK(j["top_qtype"][6]["estimate"] == 620); } + +TEST_CASE("DNS Filters: filter_exclude_noerror", "[pcap][net]") +{ + + PcapInputStream stream{"pcap-test"}; + stream.config_set("pcap_file", "tests/fixtures/dns_udp_mixed_rcode.pcap"); + stream.config_set("bpf", ""); + stream.config_set("host_spec", "192.168.0.0/24"); + stream.parse_host_spec(); + + visor::Config c; + c.config_set("num_periods", 1); + DnsStreamHandler dns_handler{"dns-test", &stream, &c}; + + dns_handler.config_set("filter_exclude_noerror", true); + + dns_handler.start(); + stream.start(); + stream.stop(); + dns_handler.stop(); + + auto counters = dns_handler.metrics()->bucket(0)->counters(); + REQUIRE(counters.NOERROR.value() == 0); + REQUIRE(counters.SRVFAIL.value() == 0); + REQUIRE(counters.REFUSED.value() == 1); + REQUIRE(counters.NX.value() == 1); + REQUIRE(counters.filtered.value() == 22); + nlohmann::json j; + dns_handler.metrics()->bucket(0)->to_json(j); + REQUIRE(j["wire_packets"]["filtered"] == 22); +} + +TEST_CASE("DNS Filters: filter_only_rcode nx", "[pcap][net]") +{ + + PcapInputStream stream{"pcap-test"}; + stream.config_set("pcap_file", "tests/fixtures/dns_udp_mixed_rcode.pcap"); + stream.config_set("bpf", ""); + stream.config_set("host_spec", "192.168.0.0/24"); + stream.parse_host_spec(); + + visor::Config c; + c.config_set("num_periods", 1); + DnsStreamHandler dns_handler{"dns-test", &stream, &c}; + + dns_handler.config_set("filter_only_rcode", NXDomain); + + dns_handler.start(); + stream.start(); + stream.stop(); + dns_handler.stop(); + + auto counters = dns_handler.metrics()->bucket(0)->counters(); + REQUIRE(counters.NOERROR.value() == 0); + REQUIRE(counters.SRVFAIL.value() == 0); + REQUIRE(counters.REFUSED.value() == 0); + REQUIRE(counters.NX.value() == 1); + REQUIRE(counters.filtered.value() == 23); + nlohmann::json j; + dns_handler.metrics()->bucket(0)->to_json(j); + REQUIRE(j["wire_packets"]["filtered"] == 23); +} + +TEST_CASE("DNS Filters: filter_only_rcode refused", "[pcap][net]") +{ + + PcapInputStream stream{"pcap-test"}; + stream.config_set("pcap_file", "tests/fixtures/dns_udp_mixed_rcode.pcap"); + stream.config_set("bpf", ""); + stream.config_set("host_spec", "192.168.0.0/24"); + stream.parse_host_spec(); + + visor::Config c; + c.config_set("num_periods", 1); + DnsStreamHandler dns_handler{"dns-test", &stream, &c}; + + dns_handler.config_set("filter_only_rcode", Refused); + + dns_handler.start(); + stream.start(); + stream.stop(); + dns_handler.stop(); + + auto counters = dns_handler.metrics()->bucket(0)->counters(); + REQUIRE(counters.NOERROR.value() == 0); + REQUIRE(counters.SRVFAIL.value() == 0); + REQUIRE(counters.REFUSED.value() == 1); + REQUIRE(counters.NX.value() == 0); + REQUIRE(counters.filtered.value() == 23); + nlohmann::json j; + dns_handler.metrics()->bucket(0)->to_json(j); + REQUIRE(j["wire_packets"]["filtered"] == 23); +} + +TEST_CASE("DNS Filters: filter_only_qname_suffix", "[pcap][net]") +{ + + PcapInputStream stream{"pcap-test"}; + stream.config_set("pcap_file", "tests/fixtures/dns_udp_mixed_rcode.pcap"); + stream.config_set("bpf", ""); + stream.config_set("host_spec", "192.168.0.0/24"); + stream.parse_host_spec(); + + visor::Config c; + c.config_set("num_periods", 1); + DnsStreamHandler dns_handler{"dns-test", &stream, &c}; + + // notice, case insensitive + dns_handler.config_set("filter_only_qname_suffix", {"GooGle.com"}); + dns_handler.start(); + stream.start(); + stream.stop(); + dns_handler.stop(); + + auto counters = dns_handler.metrics()->bucket(0)->counters(); + CHECK(counters.NOERROR.value() == 4); + CHECK(counters.SRVFAIL.value() == 0); + CHECK(counters.REFUSED.value() == 0); + CHECK(counters.NX.value() == 1); + CHECK(counters.filtered.value() == 14); +} diff --git a/src/handlers/dns/tests/window-schema.json b/src/handlers/dns/tests/window-schema.json index f2d9da1e2..1d08f68f0 100644 --- a/src/handlers/dns/tests/window-schema.json +++ b/src/handlers/dns/tests/window-schema.json @@ -841,7 +841,8 @@ "srvfail", "tcp", "total", - "udp" + "udp", + "filtered" ], "properties": { "deep_samples": { @@ -963,6 +964,16 @@ "examples": [ 2971 ] + }, + "filtered": { + "$id": "#/properties/dns/properties/wire_packets/properties/filtered", + "type": "integer", + "title": "The filtered schema", + "description": "An explanation about the purpose of this instance.", + "default": 0, + "examples": [ + 2971 + ] } }, "additionalProperties": false diff --git a/src/tests/fixtures/dns_udp_mixed_rcode.pcap b/src/tests/fixtures/dns_udp_mixed_rcode.pcap new file mode 100644 index 000000000..7799e031c Binary files /dev/null and b/src/tests/fixtures/dns_udp_mixed_rcode.pcap differ