From 2f71d4b60bdef130dd912da900baa43e640e83ae Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Mon, 14 Jun 2021 17:44:31 -0400 Subject: [PATCH 1/6] initial dns filter implementation --- src/handlers/dns/DnsStreamHandler.cpp | 61 ++++++++++++++++++++--- src/handlers/dns/DnsStreamHandler.h | 17 +++++-- src/handlers/dns/tests/test_dns_layer.cpp | 27 ++++++++++ src/handlers/dns/tests/window-schema.json | 13 ++++- 4 files changed, 107 insertions(+), 11 deletions(-) diff --git a/src/handlers/dns/DnsStreamHandler.cpp b/src/handlers/dns/DnsStreamHandler.cpp index be044cf5d..7cff25d1b 100644 --- a/src/handlers/dns/DnsStreamHandler.cpp +++ b/src/handlers/dns/DnsStreamHandler.cpp @@ -37,6 +37,25 @@ void DnsStreamHandler::start() return; } + // Setup Filters + if (config_exists("filter_exclude_noerror") && config_get("filter_exclude_noerror")) { + _f_excluding_rcode = true; + _f_rcode = NoError; + } else if (config_exists("filter_only_rcode")) { + auto want_code = config_get("filter_only_rcode"); + switch (want_code) { + case NoError: + case NXDomain: + case SrvFail: + case Refused: + _f_only_rcode = true; + _f_rcode = want_code; + break; + default: + throw ConfigException("filter_only_rcode contained an invalid/unsupported rcode"); + } + } + if (config_exists("recorded_stream")) { _metrics->set_recorded_stream(); } @@ -71,10 +90,6 @@ void DnsStreamHandler::stop() _running = false; } -DnsStreamHandler::~DnsStreamHandler() -{ -} - // callback from input module void DnsStreamHandler::process_udp_packet_cb(pcpp::Packet &payload, PacketDirection dir, pcpp::ProtocolType l3, uint32_t flowkey, timespec stamp) { @@ -93,7 +108,9 @@ void DnsStreamHandler::process_udp_packet_cb(pcpp::Packet &payload, PacketDirect } if (metric_port) { DnsLayer dnsLayer(udpLayer, &payload); - _metrics->process_dns_layer(dnsLayer, dir, l3, pcpp::UDP, flowkey, metric_port, stamp); + if (!_filtering(dnsLayer, dir, l3, pcpp::UDP, metric_port, stamp)) { + _metrics->process_dns_layer(dnsLayer, dir, l3, pcpp::UDP, flowkey, metric_port, stamp); + } } } @@ -169,7 +186,9 @@ void DnsStreamHandler::tcp_message_ready_cb(int8_t side, const pcpp::TcpStreamDa // instead using the packet meta data we pass in pcpp::Packet dummy_packet; DnsLayer dnsLayer(data.get(), size, nullptr, &dummy_packet); - _metrics->process_dns_layer(dnsLayer, dir, l3Type, pcpp::TCP, flowKey, port, stamp); + if (!_filtering(dnsLayer, dir, l3Type, pcpp::UDP, port, stamp)) { + _metrics->process_dns_layer(dnsLayer, dir, l3Type, pcpp::TCP, flowKey, port, stamp); + } // data is freed upon return }; @@ -225,6 +244,18 @@ void DnsStreamHandler::info_json(json &j) const common_info_json(j); j[schema_key()]["xact"]["open"] = _metrics->num_open_transactions(); } +bool DnsStreamHandler::_filtering(DnsLayer &payload, [[maybe_unused]] PacketDirection dir, [[maybe_unused]] pcpp::ProtocolType l3, [[maybe_unused]] pcpp::ProtocolType l4, [[maybe_unused]] uint16_t port, timespec stamp) +{ + if (_f_excluding_rcode && payload.getDnsHeader()->responseCode == _f_rcode) { + goto will_filter; + } else if (_f_only_rcode && payload.getDnsHeader()->responseCode != _f_rcode) { + goto will_filter; + } + return false; +will_filter: + _metrics->process_filtered(stamp); + return true; +} void DnsMetricsBucket::specialized_merge(const AbstractMetricsBucket &o) { @@ -249,6 +280,8 @@ void DnsMetricsBucket::specialized_merge(const AbstractMetricsBucket &o) _counters.SRVFAIL += other._counters.SRVFAIL; _counters.NOERROR += other._counters.NOERROR; + _counters.filtered += other._counters.filtered; + _dnsXactFromTimeUs.merge(other._dnsXactFromTimeUs); _dnsXactToTimeUs.merge(other._dnsXactToTimeUs); @@ -289,6 +322,8 @@ void DnsMetricsBucket::to_json(json &j) const _counters.SRVFAIL.to_json(j); _counters.NOERROR.to_json(j); + _counters.filtered.to_json(j); + _dns_qnameCard.to_json(j); _counters.xacts_total.to_json(j); _counters.xacts_timed_out.to_json(j); @@ -406,7 +441,6 @@ void DnsMetricsBucket::process_dns_layer(bool deep, DnsLayer &payload, pcpp::Pro _dns_topQname3.update(std::string(aggDomain.second)); } } - } void DnsMetricsBucket::new_dns_transaction(bool deep, float to90th, float from90th, DnsLayer &dns, PacketDirection dir, DnsTransaction xact) @@ -466,6 +500,8 @@ void DnsMetricsBucket::to_prometheus(std::stringstream &out, Metric::LabelMap ad _counters.SRVFAIL.to_prometheus(out, add_labels); _counters.NOERROR.to_prometheus(out, add_labels); + _counters.filtered.to_prometheus(out, add_labels); + _dns_qnameCard.to_prometheus(out, add_labels); _counters.xacts_total.to_prometheus(out, add_labels); _counters.xacts_timed_out.to_prometheus(out, add_labels); @@ -500,6 +536,11 @@ void DnsMetricsBucket::to_prometheus(std::stringstream &out, Metric::LabelMap ad } }); } +void DnsMetricsBucket::process_filtered() +{ + std::unique_lock lock(_mutex); + ++_counters.filtered; +} // the general metrics manager entry point (both UDP and TCP) void DnsMetricsManager::process_dns_layer(DnsLayer &payload, PacketDirection dir, pcpp::ProtocolType l3, pcpp::ProtocolType l4, uint32_t flowkey, uint16_t port, timespec stamp) @@ -518,5 +559,11 @@ void DnsMetricsManager::process_dns_layer(DnsLayer &payload, PacketDirection dir _qr_pair_manager.start_transaction(flowkey, payload.getDnsHeader()->transactionID, stamp); } } +void DnsMetricsManager::process_filtered(timespec stamp) +{ + // base event + new_event(stamp); + live_bucket()->process_filtered(); +} } \ No newline at end of file diff --git a/src/handlers/dns/DnsStreamHandler.h b/src/handlers/dns/DnsStreamHandler.h index e00005635..49d89fce4 100644 --- a/src/handlers/dns/DnsStreamHandler.h +++ b/src/handlers/dns/DnsStreamHandler.h @@ -11,6 +11,7 @@ #include "dns.h" #include "querypairmgr.h" #include +#include #include namespace visor::handler::dns { @@ -54,6 +55,7 @@ class DnsMetricsBucket final : public visor::AbstractMetricsBucket Counter REFUSED; Counter SRVFAIL; Counter NOERROR; + Counter filtered; counters() : xacts_total("dns", {"xact", "counts", "total"}, "Total DNS transactions (query/reply pairs)") , xacts_in("dns", {"xact", "in", "total"}, "Total ingress DNS transactions (host is server)") @@ -69,6 +71,7 @@ class DnsMetricsBucket final : public visor::AbstractMetricsBucket , REFUSED("dns", {"wire_packets", "refused"}, "Total DNS wire packets flagged as reply with return code REFUSED (ingress and egress)") , SRVFAIL("dns", {"wire_packets", "srvfail"}, "Total DNS wire packets flagged as reply with return code SRVFAIL (ingress and egress)") , NOERROR("dns", {"wire_packets", "noerror"}, "Total DNS wire packets flagged as reply with return code NOERROR (ingress and egress)") + , filtered("dns", {"wire_packets", "filtered"}, "Total DNS wire packets seen that did not match the configured filter(s) (if any)") { } }; @@ -124,6 +127,7 @@ class DnsMetricsBucket final : public visor::AbstractMetricsBucket void to_json(json &j) const override; void to_prometheus(std::stringstream &out, Metric::LabelMap add_labels = {}) const override; + void process_filtered(); void process_dns_layer(bool deep, DnsLayer &payload, pcpp::ProtocolType l3, pcpp::ProtocolType l4, uint16_t port); void new_dns_transaction(bool deep, float to90th, float from90th, DnsLayer &dns, PacketDirection dir, DnsTransaction xact); @@ -133,8 +137,8 @@ class DnsMetricsManager final : public visor::AbstractMetricsManager("num_periods", 1); + DnsStreamHandler dns_handler{"dns-test", &stream, &c}; + + dns_handler.config_set("filter_exclude_noerror", true); + + dns_handler.start(); + stream.start(); + stream.stop(); + dns_handler.stop(); + + auto counters = dns_handler.metrics()->bucket(0)->counters(); + REQUIRE(counters.filtered.value() == 5851); + nlohmann::json j; + dns_handler.metrics()->bucket(0)->to_json(j); + REQUIRE(j["wire_packets"]["filtered"] == 5851); +} \ No newline at end of file diff --git a/src/handlers/dns/tests/window-schema.json b/src/handlers/dns/tests/window-schema.json index f2d9da1e2..1d08f68f0 100644 --- a/src/handlers/dns/tests/window-schema.json +++ b/src/handlers/dns/tests/window-schema.json @@ -841,7 +841,8 @@ "srvfail", "tcp", "total", - "udp" + "udp", + "filtered" ], "properties": { "deep_samples": { @@ -963,6 +964,16 @@ "examples": [ 2971 ] + }, + "filtered": { + "$id": "#/properties/dns/properties/wire_packets/properties/filtered", + "type": "integer", + "title": "The filtered schema", + "description": "An explanation about the purpose of this instance.", + "default": 0, + "examples": [ + 2971 + ] } }, "additionalProperties": false From 3002f4b7edec18b27a8837a2fddf44f4fc38940a Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Tue, 15 Jun 2021 10:57:22 -0400 Subject: [PATCH 2/6] test rcode filters --- src/handlers/dns/tests/test_dns_layer.cpp | 72 +++++++++++++++++++- src/tests/fixtures/dns_udp_mixed_rcode.pcap | Bin 0 -> 3221 bytes 2 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 src/tests/fixtures/dns_udp_mixed_rcode.pcap diff --git a/src/handlers/dns/tests/test_dns_layer.cpp b/src/handlers/dns/tests/test_dns_layer.cpp index fd7f48827..fe291f734 100644 --- a/src/handlers/dns/tests/test_dns_layer.cpp +++ b/src/handlers/dns/tests/test_dns_layer.cpp @@ -257,7 +257,7 @@ TEST_CASE("DNS Filters: filter_exclude_noerror", "[pcap][net]") { PcapInputStream stream{"pcap-test"}; - stream.config_set("pcap_file", "tests/fixtures/dns_udp_tcp_random.pcap"); + stream.config_set("pcap_file", "tests/fixtures/dns_udp_mixed_rcode.pcap"); stream.config_set("bpf", ""); stream.config_set("host_spec", "192.168.0.0/24"); stream.parse_host_spec(); @@ -274,8 +274,74 @@ TEST_CASE("DNS Filters: filter_exclude_noerror", "[pcap][net]") dns_handler.stop(); auto counters = dns_handler.metrics()->bucket(0)->counters(); - REQUIRE(counters.filtered.value() == 5851); + REQUIRE(counters.NOERROR.value() == 0); + REQUIRE(counters.SRVFAIL.value() == 0); + REQUIRE(counters.REFUSED.value() == 1); + REQUIRE(counters.NX.value() == 1); + REQUIRE(counters.filtered.value() == 22); + nlohmann::json j; + dns_handler.metrics()->bucket(0)->to_json(j); + REQUIRE(j["wire_packets"]["filtered"] == 22); +} + +TEST_CASE("DNS Filters: filter_only_rcode nx", "[pcap][net]") +{ + + PcapInputStream stream{"pcap-test"}; + stream.config_set("pcap_file", "tests/fixtures/dns_udp_mixed_rcode.pcap"); + stream.config_set("bpf", ""); + stream.config_set("host_spec", "192.168.0.0/24"); + stream.parse_host_spec(); + + visor::Config c; + c.config_set("num_periods", 1); + DnsStreamHandler dns_handler{"dns-test", &stream, &c}; + + dns_handler.config_set("filter_only_rcode", NXDomain); + + dns_handler.start(); + stream.start(); + stream.stop(); + dns_handler.stop(); + + auto counters = dns_handler.metrics()->bucket(0)->counters(); + REQUIRE(counters.NOERROR.value() == 0); + REQUIRE(counters.SRVFAIL.value() == 0); + REQUIRE(counters.REFUSED.value() == 0); + REQUIRE(counters.NX.value() == 1); + REQUIRE(counters.filtered.value() == 23); + nlohmann::json j; + dns_handler.metrics()->bucket(0)->to_json(j); + REQUIRE(j["wire_packets"]["filtered"] == 23); +} + +TEST_CASE("DNS Filters: filter_only_rcode refused", "[pcap][net]") +{ + + PcapInputStream stream{"pcap-test"}; + stream.config_set("pcap_file", "tests/fixtures/dns_udp_mixed_rcode.pcap"); + stream.config_set("bpf", ""); + stream.config_set("host_spec", "192.168.0.0/24"); + stream.parse_host_spec(); + + visor::Config c; + c.config_set("num_periods", 1); + DnsStreamHandler dns_handler{"dns-test", &stream, &c}; + + dns_handler.config_set("filter_only_rcode", Refused); + + dns_handler.start(); + stream.start(); + stream.stop(); + dns_handler.stop(); + + auto counters = dns_handler.metrics()->bucket(0)->counters(); + REQUIRE(counters.NOERROR.value() == 0); + REQUIRE(counters.SRVFAIL.value() == 0); + REQUIRE(counters.REFUSED.value() == 1); + REQUIRE(counters.NX.value() == 0); + REQUIRE(counters.filtered.value() == 23); nlohmann::json j; dns_handler.metrics()->bucket(0)->to_json(j); - REQUIRE(j["wire_packets"]["filtered"] == 5851); + REQUIRE(j["wire_packets"]["filtered"] == 23); } \ No newline at end of file diff --git a/src/tests/fixtures/dns_udp_mixed_rcode.pcap b/src/tests/fixtures/dns_udp_mixed_rcode.pcap new file mode 100644 index 0000000000000000000000000000000000000000..7799e031cffefdb5ec3c8fbe5f28030fff7be2d0 GIT binary patch literal 3221 zcmcImZA=_x5T3og+vBT*1q-x7Lo7i^DdFHgN($w$f+AKeV6h=sj&el=?&vK9Xl*?x zNTUg+ekEFxN;DM}i2^D~gKok{F_omW5)n;-Uz#R_8a2^k(V5*lSniJFk8YT|ec8Er z=6Pn`+4sBAfgTm8Vf|7AgA4V@Wce!x1ECD>$;=fxRrq*}-P)J&RRH7yJek*S1~adJ zNbCjU>~ca-r6DPqUV;}s(FgRWz5Cj1o`MbA&`w6>BU?D$gz=99RytF79V9l_PpZbD;WGxUb z8*F7=txg0(VZFh`5Z4nC)O|k$r?|vNSJ;h!cMfoH6z|ErDqUc@LzRBw7=cZs@6A~$ zM0Td3i`E)+Cpi36qntXnK2@i&30QO1W^JHn_;jtx1k_W@K{fi5c`+;9o@)jpf9fuQ z$jlJxk=5s|HCP%Kv(12944K1DP*0>MD}f(G;EDZOKMZf*C5GL_GQ;GY3zv!E;Kf@r z#0_mThA|r$63`3vT4+RnGB1WzuPvCt#NR8#3|$NnswjH+*31?R5eOL)95D<_h=W6B zs5lB*DLqfNrqfzK-FmN!y7imCU2K`|Bh;;f$0AQMac;I-bZc6h?(+ECiJ`>!{ywdr zZe8po#{K_LP2`#)Ty}FyYr)lWz**Vv-%YSez8wJ17u7PJTYcj8W_85l7%zJ|=>#3E|k%@B{+*mlj;6!&h zb7qsw72Aw(QeX0m!$>&NASImmQW+=Zs(&152d-^r!kM4YKo9KpJBjF_=pj%dCfo;hl4szml0@b5@=?pnZ%u}ynWEt$l69xx3qS=jVmEy>#KE(eBAM6Fd5XJ%|K-JDB1bd2^BN9=%)#=Tsf@fy zvFo)C>xo9eTB$s{DbFsGd~Pgn-nL3;g+Fsph)86fy$cV={z7j9NB{cU0P#}$CvT>< zYTP5v7aE-u>B>NaKi_Ad0Po3U1Xy_dxzKWH4`+3b5uf^Aa0DI5yrg=xxzmiHRe2;89Yjk646PSqmzvOB4$b%st(eY) za^E%;oX4M#c}aAt`y3s0|A<;Y>LSjGMAcGLyNrTuD864rh#gV-;DYZr`w0i`W%`*eDFeK`I{dHoA0q!48Q literal 0 HcmV?d00001 From 27c3536f068076db18faa80ab26a3d69a8e133d4 Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Tue, 15 Jun 2021 11:03:57 -0400 Subject: [PATCH 3/6] make sample optional for base event --- src/AbstractMetricsManager.h | 12 +++++++----- src/handlers/dns/DnsStreamHandler.cpp | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/AbstractMetricsManager.h b/src/AbstractMetricsManager.h index 5a59137ed..12295896a 100644 --- a/src/AbstractMetricsManager.h +++ b/src/AbstractMetricsManager.h @@ -284,16 +284,18 @@ class AbstractMetricsManager protected: /** * the "base" event method that should be called on every event before specialized event functionality. sampling will be - * chosen, and the time window will be maintained + * (optionally) chosen, and the time window will be maintained * * @param stamp time stamp of the event */ - void new_event(timespec stamp) + void new_event(timespec stamp, bool sample = true) { // CRITICAL EVENT PATH - _deep_sampling_now.store(true, std::memory_order_relaxed); - if (_deep_sample_rate != 100) { - _deep_sampling_now.store((_rng.uniform(0U, 100U) <= _deep_sample_rate), std::memory_order_relaxed); + if (sample) { + _deep_sampling_now.store(true, std::memory_order_relaxed); + if (_deep_sample_rate != 100) { + _deep_sampling_now.store((_rng.uniform(0U, 100U) <= _deep_sample_rate), std::memory_order_relaxed); + } } std::shared_lock rlb(_base_mutex); bool will_shift = _num_periods > 1 && stamp.tv_sec >= _next_shift_tstamp.tv_sec; diff --git a/src/handlers/dns/DnsStreamHandler.cpp b/src/handlers/dns/DnsStreamHandler.cpp index 7cff25d1b..ed234e17a 100644 --- a/src/handlers/dns/DnsStreamHandler.cpp +++ b/src/handlers/dns/DnsStreamHandler.cpp @@ -561,8 +561,8 @@ void DnsMetricsManager::process_dns_layer(DnsLayer &payload, PacketDirection dir } void DnsMetricsManager::process_filtered(timespec stamp) { - // base event - new_event(stamp); + // base event, no sample + new_event(stamp, false); live_bucket()->process_filtered(); } From c83f543d7e14dc86d50fcc9441ac0b755db27de0 Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Tue, 15 Jun 2021 11:16:47 -0400 Subject: [PATCH 4/6] add filtered to integration tests --- integration_tests/output-templates/dns_ipv4_tcp.Darwin.json | 2 +- integration_tests/output-templates/dns_ipv4_tcp.Linux.json | 2 +- integration_tests/output-templates/dns_ipv4_udp.Darwin.json | 2 +- integration_tests/output-templates/dns_ipv4_udp.Linux.json | 2 +- integration_tests/output-templates/dns_ipv6_tcp.Darwin.json | 2 +- integration_tests/output-templates/dns_ipv6_tcp.Linux.json | 2 +- integration_tests/output-templates/dns_ipv6_udp.Darwin.json | 2 +- integration_tests/output-templates/dns_ipv6_udp.Linux.json | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/integration_tests/output-templates/dns_ipv4_tcp.Darwin.json b/integration_tests/output-templates/dns_ipv4_tcp.Darwin.json index c528d1604..eb40af603 100644 --- a/integration_tests/output-templates/dns_ipv4_tcp.Darwin.json +++ b/integration_tests/output-templates/dns_ipv4_tcp.Darwin.json @@ -1 +1 @@ -{"5m":{"dns":{"period":{"length":7,"start_ts":1567706433},"top_nxdomain":[],"top_qname2":[{"estimate":420,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":"wWnyafx.test.com"},{"estimate":2,"name":".6RL.test.com"},{"estimate":2,"name":".LdmqNq.test.com"},{"estimate":2,"name":".UbuZpGD.test.com"},{"estimate":2,"name":".rZ4.test.com"},{"estimate":2,"name":".mTIoCZeUb8.test.com"},{"estimate":2,"name":"UlUoHV.test.com"},{"estimate":2,"name":".de0xnQUa3.test.com"},{"estimate":2,"name":".bp5MFj.test.com"},{"estimate":2,"name":".Sl2l.test.com"}],"top_qtype":[{"estimate":100,"name":"AAAA"},{"estimate":66,"name":"NS"},{"estimate":60,"name":"A"},{"estimate":60,"name":"SOA"},{"estimate":56,"name":"MX"},{"estimate":46,"name":"TXT"},{"estimate":32,"name":"CNAME"}],"top_rcode":[{"estimate":210,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":2,"name":"44495"},{"estimate":2,"name":"39127"},{"estimate":2,"name":"42695"},{"estimate":2,"name":"40281"},{"estimate":2,"name":"46399"},{"estimate":2,"name":"35805"},{"estimate":2,"name":"37805"},{"estimate":2,"name":"42397"},{"estimate":2,"name":"45847"},{"estimate":2,"name":"35287"}],"wire_packets":{"deep_samples":420,"ipv4":420,"ipv6":0,"noerror":0,"nxdomain":0,"queries":210,"refused":0,"replies":210,"srvfail":0,"tcp":420,"total":420,"udp":0},"xact":{"counts":{"timed_out":0,"total":210},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":210}}},"packets":{"deep_samples":2100,"in":2100,"ipv4":2100,"ipv6":0,"other_l4":0,"out":0,"period":{"length":7,"start_ts":1567706433},"tcp":2100,"top_ASN":[{"estimate":2100,"name":"Unknown"}],"top_geoLoc":[{"estimate":2100,"name":"Unknown"}],"top_ipv4":[{"estimate":2100,"name":"127.0.0.1"}],"top_ipv6":[],"total":2100,"udp":0}}} +{"5m":{"dns":{"period":{"length":7,"start_ts":1567706433},"top_nxdomain":[],"top_qname2":[{"estimate":420,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":"wWnyafx.test.com"},{"estimate":2,"name":".6RL.test.com"},{"estimate":2,"name":".LdmqNq.test.com"},{"estimate":2,"name":".UbuZpGD.test.com"},{"estimate":2,"name":".rZ4.test.com"},{"estimate":2,"name":".mTIoCZeUb8.test.com"},{"estimate":2,"name":"UlUoHV.test.com"},{"estimate":2,"name":".de0xnQUa3.test.com"},{"estimate":2,"name":".bp5MFj.test.com"},{"estimate":2,"name":".Sl2l.test.com"}],"top_qtype":[{"estimate":100,"name":"AAAA"},{"estimate":66,"name":"NS"},{"estimate":60,"name":"A"},{"estimate":60,"name":"SOA"},{"estimate":56,"name":"MX"},{"estimate":46,"name":"TXT"},{"estimate":32,"name":"CNAME"}],"top_rcode":[{"estimate":210,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":2,"name":"44495"},{"estimate":2,"name":"39127"},{"estimate":2,"name":"42695"},{"estimate":2,"name":"40281"},{"estimate":2,"name":"46399"},{"estimate":2,"name":"35805"},{"estimate":2,"name":"37805"},{"estimate":2,"name":"42397"},{"estimate":2,"name":"45847"},{"estimate":2,"name":"35287"}],"wire_packets":{"deep_samples":420,"filtered":0,"ipv4":420,"ipv6":0,"noerror":0,"nxdomain":0,"queries":210,"refused":0,"replies":210,"srvfail":0,"tcp":420,"total":420,"udp":0},"xact":{"counts":{"timed_out":0,"total":210},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":210}}},"packets":{"deep_samples":2100,"in":2100,"ipv4":2100,"ipv6":0,"other_l4":0,"out":0,"period":{"length":7,"start_ts":1567706433},"tcp":2100,"top_ASN":[{"estimate":2100,"name":"Unknown"}],"top_geoLoc":[{"estimate":2100,"name":"Unknown"}],"top_ipv4":[{"estimate":2100,"name":"127.0.0.1"}],"top_ipv6":[],"total":2100,"udp":0}}} diff --git a/integration_tests/output-templates/dns_ipv4_tcp.Linux.json b/integration_tests/output-templates/dns_ipv4_tcp.Linux.json index dbb9c770f..379508775 100644 --- a/integration_tests/output-templates/dns_ipv4_tcp.Linux.json +++ b/integration_tests/output-templates/dns_ipv4_tcp.Linux.json @@ -1 +1 @@ -{"5m":{"dns":{"period":{"length":7,"start_ts":1567706433},"top_nxdomain":[],"top_qname2":[{"estimate":420,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".lMw.test.com"},{"estimate":2,"name":".rZ4.test.com"},{"estimate":2,"name":"ZX0A4K.test.com"},{"estimate":2,"name":".6cR5fxGGfF.test.com"},{"estimate":2,"name":".lk06eYI2ck.test.com"},{"estimate":2,"name":".2at.test.com"},{"estimate":2,"name":".gHsieFqf.test.com"},{"estimate":2,"name":".uuteIwE.test.com"},{"estimate":2,"name":"Bao7Qb.test.com"},{"estimate":2,"name":".ZguNCWgzm.test.com"}],"top_qtype":[{"estimate":100,"name":"AAAA"},{"estimate":66,"name":"NS"},{"estimate":60,"name":"A"},{"estimate":60,"name":"SOA"},{"estimate":56,"name":"MX"},{"estimate":46,"name":"TXT"},{"estimate":32,"name":"CNAME"}],"top_rcode":[{"estimate":210,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":2,"name":"43151"},{"estimate":2,"name":"41799"},{"estimate":2,"name":"35007"},{"estimate":2,"name":"43313"},{"estimate":2,"name":"33341"},{"estimate":2,"name":"46517"},{"estimate":2,"name":"46063"},{"estimate":2,"name":"44273"},{"estimate":2,"name":"43615"},{"estimate":2,"name":"39577"}],"wire_packets":{"deep_samples":420,"ipv4":420,"ipv6":0,"noerror":0,"nxdomain":0,"queries":210,"refused":0,"replies":210,"srvfail":0,"tcp":420,"total":420,"udp":0},"xact":{"counts":{"timed_out":0,"total":210},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":210}}},"packets":{"deep_samples":2100,"in":2100,"ipv4":2100,"ipv6":0,"other_l4":0,"out":0,"period":{"length":7,"start_ts":1567706433},"tcp":2100,"top_ASN":[{"estimate":2100,"name":"Unknown"}],"top_geoLoc":[{"estimate":2100,"name":"Unknown"}],"top_ipv4":[{"estimate":2100,"name":"127.0.0.1"}],"top_ipv6":[],"total":2100,"udp":0}}} +{"5m":{"dns":{"period":{"length":7,"start_ts":1567706433},"top_nxdomain":[],"top_qname2":[{"estimate":420,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".lMw.test.com"},{"estimate":2,"name":".rZ4.test.com"},{"estimate":2,"name":"ZX0A4K.test.com"},{"estimate":2,"name":".6cR5fxGGfF.test.com"},{"estimate":2,"name":".lk06eYI2ck.test.com"},{"estimate":2,"name":".2at.test.com"},{"estimate":2,"name":".gHsieFqf.test.com"},{"estimate":2,"name":".uuteIwE.test.com"},{"estimate":2,"name":"Bao7Qb.test.com"},{"estimate":2,"name":".ZguNCWgzm.test.com"}],"top_qtype":[{"estimate":100,"name":"AAAA"},{"estimate":66,"name":"NS"},{"estimate":60,"name":"A"},{"estimate":60,"name":"SOA"},{"estimate":56,"name":"MX"},{"estimate":46,"name":"TXT"},{"estimate":32,"name":"CNAME"}],"top_rcode":[{"estimate":210,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":2,"name":"43151"},{"estimate":2,"name":"41799"},{"estimate":2,"name":"35007"},{"estimate":2,"name":"43313"},{"estimate":2,"name":"33341"},{"estimate":2,"name":"46517"},{"estimate":2,"name":"46063"},{"estimate":2,"name":"44273"},{"estimate":2,"name":"43615"},{"estimate":2,"name":"39577"}],"wire_packets":{"deep_samples":420,"filtered":0,"ipv4":420,"ipv6":0,"noerror":0,"nxdomain":0,"queries":210,"refused":0,"replies":210,"srvfail":0,"tcp":420,"total":420,"udp":0},"xact":{"counts":{"timed_out":0,"total":210},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":210}}},"packets":{"deep_samples":2100,"in":2100,"ipv4":2100,"ipv6":0,"other_l4":0,"out":0,"period":{"length":7,"start_ts":1567706433},"tcp":2100,"top_ASN":[{"estimate":2100,"name":"Unknown"}],"top_geoLoc":[{"estimate":2100,"name":"Unknown"}],"top_ipv4":[{"estimate":2100,"name":"127.0.0.1"}],"top_ipv6":[],"total":2100,"udp":0}}} diff --git a/integration_tests/output-templates/dns_ipv4_udp.Darwin.json b/integration_tests/output-templates/dns_ipv4_udp.Darwin.json index 043013c83..1f30f12c4 100644 --- a/integration_tests/output-templates/dns_ipv4_udp.Darwin.json +++ b/integration_tests/output-templates/dns_ipv4_udp.Darwin.json @@ -1 +1 @@ -{"5m":{"dns":{"period":{"length":6,"start_ts":1567706414},"top_nxdomain":[],"top_qname2":[{"estimate":140,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".yhsx.test.com"},{"estimate":2,"name":".rEdwaRK.test.com"},{"estimate":2,"name":".-cSCWAO.test.com"},{"estimate":2,"name":"ik4R35VBCx.test.com"},{"estimate":2,"name":".-q-SZlz.test.com"},{"estimate":2,"name":".1gGN.test.com"},{"estimate":2,"name":".J-r.test.com"},{"estimate":2,"name":".cx.test.com"},{"estimate":2,"name":".N5V-rUqc.test.com"},{"estimate":2,"name":".TQHgrpzDe.test.com"}],"top_qtype":[{"estimate":34,"name":"AAAA"},{"estimate":24,"name":"NS"},{"estimate":18,"name":"SOA"},{"estimate":18,"name":"TXT"},{"estimate":16,"name":"A"},{"estimate":16,"name":"CNAME"},{"estimate":14,"name":"MX"}],"top_rcode":[{"estimate":70,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":14,"name":"38352"},{"estimate":14,"name":"59755"},{"estimate":14,"name":"45478"},{"estimate":14,"name":"55945"},{"estimate":14,"name":"37253"},{"estimate":14,"name":"46297"},{"estimate":14,"name":"37371"},{"estimate":14,"name":"53819"},{"estimate":14,"name":"39485"},{"estimate":14,"name":"38302"}],"wire_packets":{"deep_samples":140,"ipv4":140,"ipv6":0,"noerror":0,"nxdomain":0,"queries":70,"refused":0,"replies":70,"srvfail":0,"tcp":0,"total":140,"udp":140},"xact":{"counts":{"timed_out":0,"total":70},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":70}}},"packets":{"deep_samples":140,"in":140,"ipv4":140,"ipv6":0,"other_l4":0,"out":0,"period":{"length":6,"start_ts":1567706414},"tcp":0,"top_ASN":[{"estimate":140,"name":"Unknown"}],"top_geoLoc":[{"estimate":140,"name":"Unknown"}],"top_ipv4":[{"estimate":140,"name":"127.0.0.1"}],"top_ipv6":[],"total":140,"udp":140}}} +{"5m":{"dns":{"period":{"length":6,"start_ts":1567706414},"top_nxdomain":[],"top_qname2":[{"estimate":140,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".yhsx.test.com"},{"estimate":2,"name":".rEdwaRK.test.com"},{"estimate":2,"name":".-cSCWAO.test.com"},{"estimate":2,"name":"ik4R35VBCx.test.com"},{"estimate":2,"name":".-q-SZlz.test.com"},{"estimate":2,"name":".1gGN.test.com"},{"estimate":2,"name":".J-r.test.com"},{"estimate":2,"name":".cx.test.com"},{"estimate":2,"name":".N5V-rUqc.test.com"},{"estimate":2,"name":".TQHgrpzDe.test.com"}],"top_qtype":[{"estimate":34,"name":"AAAA"},{"estimate":24,"name":"NS"},{"estimate":18,"name":"SOA"},{"estimate":18,"name":"TXT"},{"estimate":16,"name":"A"},{"estimate":16,"name":"CNAME"},{"estimate":14,"name":"MX"}],"top_rcode":[{"estimate":70,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":14,"name":"38352"},{"estimate":14,"name":"59755"},{"estimate":14,"name":"45478"},{"estimate":14,"name":"55945"},{"estimate":14,"name":"37253"},{"estimate":14,"name":"46297"},{"estimate":14,"name":"37371"},{"estimate":14,"name":"53819"},{"estimate":14,"name":"39485"},{"estimate":14,"name":"38302"}],"wire_packets":{"deep_samples":140,"filtered":0,"ipv4":140,"ipv6":0,"noerror":0,"nxdomain":0,"queries":70,"refused":0,"replies":70,"srvfail":0,"tcp":0,"total":140,"udp":140},"xact":{"counts":{"timed_out":0,"total":70},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":70}}},"packets":{"deep_samples":140,"in":140,"ipv4":140,"ipv6":0,"other_l4":0,"out":0,"period":{"length":6,"start_ts":1567706414},"tcp":0,"top_ASN":[{"estimate":140,"name":"Unknown"}],"top_geoLoc":[{"estimate":140,"name":"Unknown"}],"top_ipv4":[{"estimate":140,"name":"127.0.0.1"}],"top_ipv6":[],"total":140,"udp":140}}} diff --git a/integration_tests/output-templates/dns_ipv4_udp.Linux.json b/integration_tests/output-templates/dns_ipv4_udp.Linux.json index d28ef7561..5a5723bc3 100644 --- a/integration_tests/output-templates/dns_ipv4_udp.Linux.json +++ b/integration_tests/output-templates/dns_ipv4_udp.Linux.json @@ -1 +1 @@ -{"5m":{"dns":{"period":{"length":6,"start_ts":1567706414},"top_nxdomain":[],"top_qname2":[{"estimate":140,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".gTjGva7m-8.test.com"},{"estimate":2,"name":".pWK6.test.com"},{"estimate":2,"name":".TQHgrpzDe.test.com"},{"estimate":2,"name":".vmTwD0v30.test.com"},{"estimate":2,"name":".2MT.test.com"},{"estimate":2,"name":".GnKHW4hgN.test.com"},{"estimate":2,"name":"ik4R35VBCx.test.com"},{"estimate":2,"name":".cx.test.com"},{"estimate":2,"name":"G2s1RNfdkE.test.com"},{"estimate":2,"name":".yGa.test.com"}],"top_qtype":[{"estimate":34,"name":"AAAA"},{"estimate":24,"name":"NS"},{"estimate":18,"name":"SOA"},{"estimate":18,"name":"TXT"},{"estimate":16,"name":"A"},{"estimate":16,"name":"CNAME"},{"estimate":14,"name":"MX"}],"top_rcode":[{"estimate":70,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":14,"name":"38352"},{"estimate":14,"name":"59755"},{"estimate":14,"name":"45478"},{"estimate":14,"name":"55945"},{"estimate":14,"name":"37253"},{"estimate":14,"name":"46297"},{"estimate":14,"name":"37371"},{"estimate":14,"name":"53819"},{"estimate":14,"name":"39485"},{"estimate":14,"name":"38302"}],"wire_packets":{"deep_samples":140,"ipv4":140,"ipv6":0,"noerror":0,"nxdomain":0,"queries":70,"refused":0,"replies":70,"srvfail":0,"tcp":0,"total":140,"udp":140},"xact":{"counts":{"timed_out":0,"total":70},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":70}}},"packets":{"deep_samples":140,"in":140,"ipv4":140,"ipv6":0,"other_l4":0,"out":0,"period":{"length":6,"start_ts":1567706414},"tcp":0,"top_ASN":[{"estimate":140,"name":"Unknown"}],"top_geoLoc":[{"estimate":140,"name":"Unknown"}],"top_ipv4":[{"estimate":140,"name":"127.0.0.1"}],"top_ipv6":[],"total":140,"udp":140}}} +{"5m":{"dns":{"period":{"length":6,"start_ts":1567706414},"top_nxdomain":[],"top_qname2":[{"estimate":140,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".gTjGva7m-8.test.com"},{"estimate":2,"name":".pWK6.test.com"},{"estimate":2,"name":".TQHgrpzDe.test.com"},{"estimate":2,"name":".vmTwD0v30.test.com"},{"estimate":2,"name":".2MT.test.com"},{"estimate":2,"name":".GnKHW4hgN.test.com"},{"estimate":2,"name":"ik4R35VBCx.test.com"},{"estimate":2,"name":".cx.test.com"},{"estimate":2,"name":"G2s1RNfdkE.test.com"},{"estimate":2,"name":".yGa.test.com"}],"top_qtype":[{"estimate":34,"name":"AAAA"},{"estimate":24,"name":"NS"},{"estimate":18,"name":"SOA"},{"estimate":18,"name":"TXT"},{"estimate":16,"name":"A"},{"estimate":16,"name":"CNAME"},{"estimate":14,"name":"MX"}],"top_rcode":[{"estimate":70,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":14,"name":"38352"},{"estimate":14,"name":"59755"},{"estimate":14,"name":"45478"},{"estimate":14,"name":"55945"},{"estimate":14,"name":"37253"},{"estimate":14,"name":"46297"},{"estimate":14,"name":"37371"},{"estimate":14,"name":"53819"},{"estimate":14,"name":"39485"},{"estimate":14,"name":"38302"}],"wire_packets":{"deep_samples":140,"filtered":0,"ipv4":140,"ipv6":0,"noerror":0,"nxdomain":0,"queries":70,"refused":0,"replies":70,"srvfail":0,"tcp":0,"total":140,"udp":140},"xact":{"counts":{"timed_out":0,"total":70},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":70}}},"packets":{"deep_samples":140,"in":140,"ipv4":140,"ipv6":0,"other_l4":0,"out":0,"period":{"length":6,"start_ts":1567706414},"tcp":0,"top_ASN":[{"estimate":140,"name":"Unknown"}],"top_geoLoc":[{"estimate":140,"name":"Unknown"}],"top_ipv4":[{"estimate":140,"name":"127.0.0.1"}],"top_ipv6":[],"total":140,"udp":140}}} diff --git a/integration_tests/output-templates/dns_ipv6_tcp.Darwin.json b/integration_tests/output-templates/dns_ipv6_tcp.Darwin.json index 50a471052..9bb68f5e7 100644 --- a/integration_tests/output-templates/dns_ipv6_tcp.Darwin.json +++ b/integration_tests/output-templates/dns_ipv6_tcp.Darwin.json @@ -1 +1 @@ -{"5m":{"dns":{"period":{"length":7,"start_ts":1567706308},"top_nxdomain":[],"top_qname2":[{"estimate":360,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".8Zd.test.com"},{"estimate":2,"name":".7iF.test.com"},{"estimate":2,"name":".eXz.test.com"},{"estimate":2,"name":".j_dakR7.test.com"},{"estimate":2,"name":"2Z.test.com"},{"estimate":2,"name":".Sl1D.test.com"},{"estimate":2,"name":".V_o.test.com"},{"estimate":2,"name":".bmbEQvNBk.test.com"},{"estimate":2,"name":".DU5Q4ir-4A.test.com"},{"estimate":2,"name":".QD.test.com"}],"top_qtype":[{"estimate":94,"name":"AAAA"},{"estimate":58,"name":"CNAME"},{"estimate":48,"name":"SOA"},{"estimate":48,"name":"NS"},{"estimate":40,"name":"TXT"},{"estimate":38,"name":"MX"},{"estimate":34,"name":"A"}],"top_rcode":[{"estimate":180,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":2,"name":"34743"},{"estimate":2,"name":"40389"},{"estimate":2,"name":"43851"},{"estimate":2,"name":"40027"},{"estimate":2,"name":"33933"},{"estimate":2,"name":"39459"},{"estimate":2,"name":"43355"},{"estimate":2,"name":"40595"},{"estimate":2,"name":"38367"},{"estimate":2,"name":"36465"}],"wire_packets":{"deep_samples":360,"ipv4":0,"ipv6":360,"noerror":0,"nxdomain":0,"queries":180,"refused":0,"replies":180,"srvfail":0,"tcp":360,"total":360,"udp":0},"xact":{"counts":{"timed_out":0,"total":180},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":180}}},"packets":{"deep_samples":1800,"in":0,"ipv4":0,"ipv6":1800,"other_l4":0,"out":0,"period":{"length":7,"start_ts":1567706308},"tcp":1800,"top_ASN":[],"top_geoLoc":[],"top_ipv4":[],"top_ipv6":[],"total":1800,"udp":0}}} +{"5m":{"dns":{"period":{"length":7,"start_ts":1567706308},"top_nxdomain":[],"top_qname2":[{"estimate":360,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".8Zd.test.com"},{"estimate":2,"name":".7iF.test.com"},{"estimate":2,"name":".eXz.test.com"},{"estimate":2,"name":".j_dakR7.test.com"},{"estimate":2,"name":"2Z.test.com"},{"estimate":2,"name":".Sl1D.test.com"},{"estimate":2,"name":".V_o.test.com"},{"estimate":2,"name":".bmbEQvNBk.test.com"},{"estimate":2,"name":".DU5Q4ir-4A.test.com"},{"estimate":2,"name":".QD.test.com"}],"top_qtype":[{"estimate":94,"name":"AAAA"},{"estimate":58,"name":"CNAME"},{"estimate":48,"name":"SOA"},{"estimate":48,"name":"NS"},{"estimate":40,"name":"TXT"},{"estimate":38,"name":"MX"},{"estimate":34,"name":"A"}],"top_rcode":[{"estimate":180,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":2,"name":"34743"},{"estimate":2,"name":"40389"},{"estimate":2,"name":"43851"},{"estimate":2,"name":"40027"},{"estimate":2,"name":"33933"},{"estimate":2,"name":"39459"},{"estimate":2,"name":"43355"},{"estimate":2,"name":"40595"},{"estimate":2,"name":"38367"},{"estimate":2,"name":"36465"}],"wire_packets":{"deep_samples":360,"filtered":0,"ipv4":0,"ipv6":360,"noerror":0,"nxdomain":0,"queries":180,"refused":0,"replies":180,"srvfail":0,"tcp":360,"total":360,"udp":0},"xact":{"counts":{"timed_out":0,"total":180},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":180}}},"packets":{"deep_samples":1800,"in":0,"ipv4":0,"ipv6":1800,"other_l4":0,"out":0,"period":{"length":7,"start_ts":1567706308},"tcp":1800,"top_ASN":[],"top_geoLoc":[],"top_ipv4":[],"top_ipv6":[],"total":1800,"udp":0}}} diff --git a/integration_tests/output-templates/dns_ipv6_tcp.Linux.json b/integration_tests/output-templates/dns_ipv6_tcp.Linux.json index eced2313c..5dfff6631 100644 --- a/integration_tests/output-templates/dns_ipv6_tcp.Linux.json +++ b/integration_tests/output-templates/dns_ipv6_tcp.Linux.json @@ -1 +1 @@ -{"5m":{"dns":{"period":{"length":7,"start_ts":1567706308},"top_nxdomain":[],"top_qname2":[{"estimate":360,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".NW5zpCG.test.com"},{"estimate":2,"name":".qeg0fit.test.com"},{"estimate":2,"name":"0Hb.test.com"},{"estimate":2,"name":".z8Pa.test.com"},{"estimate":2,"name":".YCOpO3bN.test.com"},{"estimate":2,"name":"._ST.test.com"},{"estimate":2,"name":".90ylh4E.test.com"},{"estimate":2,"name":".mD2GMXLqA.test.com"},{"estimate":2,"name":".pO.test.com"},{"estimate":2,"name":".bzj6xe.test.com"}],"top_qtype":[{"estimate":94,"name":"AAAA"},{"estimate":58,"name":"CNAME"},{"estimate":48,"name":"SOA"},{"estimate":48,"name":"NS"},{"estimate":40,"name":"TXT"},{"estimate":38,"name":"MX"},{"estimate":34,"name":"A"}],"top_rcode":[{"estimate":180,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":2,"name":"45561"},{"estimate":2,"name":"46571"},{"estimate":2,"name":"36263"},{"estimate":2,"name":"44467"},{"estimate":2,"name":"43471"},{"estimate":2,"name":"43035"},{"estimate":2,"name":"35661"},{"estimate":2,"name":"46011"},{"estimate":2,"name":"37159"},{"estimate":2,"name":"44697"}],"wire_packets":{"deep_samples":360,"ipv4":0,"ipv6":360,"noerror":0,"nxdomain":0,"queries":180,"refused":0,"replies":180,"srvfail":0,"tcp":360,"total":360,"udp":0},"xact":{"counts":{"timed_out":0,"total":180},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":180}}},"packets":{"deep_samples":1800,"in":0,"ipv4":0,"ipv6":1800,"other_l4":0,"out":0,"period":{"length":7,"start_ts":1567706308},"tcp":1800,"top_ASN":[],"top_geoLoc":[],"top_ipv4":[],"top_ipv6":[],"total":1800,"udp":0}}} +{"5m":{"dns":{"period":{"length":7,"start_ts":1567706308},"top_nxdomain":[],"top_qname2":[{"estimate":360,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".NW5zpCG.test.com"},{"estimate":2,"name":".qeg0fit.test.com"},{"estimate":2,"name":"0Hb.test.com"},{"estimate":2,"name":".z8Pa.test.com"},{"estimate":2,"name":".YCOpO3bN.test.com"},{"estimate":2,"name":"._ST.test.com"},{"estimate":2,"name":".90ylh4E.test.com"},{"estimate":2,"name":".mD2GMXLqA.test.com"},{"estimate":2,"name":".pO.test.com"},{"estimate":2,"name":".bzj6xe.test.com"}],"top_qtype":[{"estimate":94,"name":"AAAA"},{"estimate":58,"name":"CNAME"},{"estimate":48,"name":"SOA"},{"estimate":48,"name":"NS"},{"estimate":40,"name":"TXT"},{"estimate":38,"name":"MX"},{"estimate":34,"name":"A"}],"top_rcode":[{"estimate":180,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":2,"name":"45561"},{"estimate":2,"name":"46571"},{"estimate":2,"name":"36263"},{"estimate":2,"name":"44467"},{"estimate":2,"name":"43471"},{"estimate":2,"name":"43035"},{"estimate":2,"name":"35661"},{"estimate":2,"name":"46011"},{"estimate":2,"name":"37159"},{"estimate":2,"name":"44697"}],"wire_packets":{"deep_samples":360,"filtered":0,"ipv4":0,"ipv6":360,"noerror":0,"nxdomain":0,"queries":180,"refused":0,"replies":180,"srvfail":0,"tcp":360,"total":360,"udp":0},"xact":{"counts":{"timed_out":0,"total":180},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":180}}},"packets":{"deep_samples":1800,"in":0,"ipv4":0,"ipv6":1800,"other_l4":0,"out":0,"period":{"length":7,"start_ts":1567706308},"tcp":1800,"top_ASN":[],"top_geoLoc":[],"top_ipv4":[],"top_ipv6":[],"total":1800,"udp":0}}} diff --git a/integration_tests/output-templates/dns_ipv6_udp.Darwin.json b/integration_tests/output-templates/dns_ipv6_udp.Darwin.json index 69ae7f248..58a003155 100644 --- a/integration_tests/output-templates/dns_ipv6_udp.Darwin.json +++ b/integration_tests/output-templates/dns_ipv6_udp.Darwin.json @@ -1 +1 @@ -{"5m":{"dns":{"period":{"length":6,"start_ts":1567706365},"top_nxdomain":[],"top_qname2":[{"estimate":140,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".8a.test.com"},{"estimate":2,"name":"Il9NL4Uvb.test.com"},{"estimate":2,"name":"aC6.test.com"},{"estimate":2,"name":".DNn1O5NnK.test.com"},{"estimate":2,"name":"-aQ2.test.com"},{"estimate":2,"name":".g7k.test.com"},{"estimate":2,"name":".OGL8.test.com"},{"estimate":2,"name":"at1s2IrEO.test.com"},{"estimate":2,"name":".FiKKi.test.com"},{"estimate":2,"name":".ZOtW.test.com"}],"top_qtype":[{"estimate":34,"name":"AAAA"},{"estimate":22,"name":"A"},{"estimate":22,"name":"SOA"},{"estimate":22,"name":"NS"},{"estimate":14,"name":"MX"},{"estimate":14,"name":"CNAME"},{"estimate":12,"name":"TXT"}],"top_rcode":[{"estimate":70,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":14,"name":"46590"},{"estimate":14,"name":"58571"},{"estimate":14,"name":"36484"},{"estimate":14,"name":"43798"},{"estimate":14,"name":"46567"},{"estimate":14,"name":"59640"},{"estimate":14,"name":"44741"},{"estimate":14,"name":"51811"},{"estimate":14,"name":"38920"},{"estimate":14,"name":"52826"}],"wire_packets":{"deep_samples":140,"ipv4":0,"ipv6":140,"noerror":0,"nxdomain":0,"queries":70,"refused":0,"replies":70,"srvfail":0,"tcp":0,"total":140,"udp":140},"xact":{"counts":{"timed_out":0,"total":70},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":0}}},"packets":{"deep_samples":140,"in":0,"ipv4":0,"ipv6":140,"other_l4":0,"out":0,"period":{"length":6,"start_ts":1567706365},"tcp":0,"top_ASN":[],"top_geoLoc":[],"top_ipv4":[],"top_ipv6":[],"total":140,"udp":140}}} +{"5m":{"dns":{"period":{"length":6,"start_ts":1567706365},"top_nxdomain":[],"top_qname2":[{"estimate":140,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".8a.test.com"},{"estimate":2,"name":"Il9NL4Uvb.test.com"},{"estimate":2,"name":"aC6.test.com"},{"estimate":2,"name":".DNn1O5NnK.test.com"},{"estimate":2,"name":"-aQ2.test.com"},{"estimate":2,"name":".g7k.test.com"},{"estimate":2,"name":".OGL8.test.com"},{"estimate":2,"name":"at1s2IrEO.test.com"},{"estimate":2,"name":".FiKKi.test.com"},{"estimate":2,"name":".ZOtW.test.com"}],"top_qtype":[{"estimate":34,"name":"AAAA"},{"estimate":22,"name":"A"},{"estimate":22,"name":"SOA"},{"estimate":22,"name":"NS"},{"estimate":14,"name":"MX"},{"estimate":14,"name":"CNAME"},{"estimate":12,"name":"TXT"}],"top_rcode":[{"estimate":70,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":14,"name":"46590"},{"estimate":14,"name":"58571"},{"estimate":14,"name":"36484"},{"estimate":14,"name":"43798"},{"estimate":14,"name":"46567"},{"estimate":14,"name":"59640"},{"estimate":14,"name":"44741"},{"estimate":14,"name":"51811"},{"estimate":14,"name":"38920"},{"estimate":14,"name":"52826"}],"wire_packets":{"deep_samples":140,"filtered":0,"ipv4":0,"ipv6":140,"noerror":0,"nxdomain":0,"queries":70,"refused":0,"replies":70,"srvfail":0,"tcp":0,"total":140,"udp":140},"xact":{"counts":{"timed_out":0,"total":70},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":0}}},"packets":{"deep_samples":140,"in":0,"ipv4":0,"ipv6":140,"other_l4":0,"out":0,"period":{"length":6,"start_ts":1567706365},"tcp":0,"top_ASN":[],"top_geoLoc":[],"top_ipv4":[],"top_ipv6":[],"total":140,"udp":140}}} diff --git a/integration_tests/output-templates/dns_ipv6_udp.Linux.json b/integration_tests/output-templates/dns_ipv6_udp.Linux.json index b7c8bddbe..dcdf080cd 100644 --- a/integration_tests/output-templates/dns_ipv6_udp.Linux.json +++ b/integration_tests/output-templates/dns_ipv6_udp.Linux.json @@ -1 +1 @@ -{"5m":{"dns":{"period":{"length":6,"start_ts":1567706365},"top_nxdomain":[],"top_qname2":[{"estimate":140,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".2pmr.test.com"},{"estimate":2,"name":".uL8E2T.test.com"},{"estimate":2,"name":".kGuiH.test.com"},{"estimate":2,"name":".ddU3i_v.test.com"},{"estimate":2,"name":".nU.test.com"},{"estimate":2,"name":"lfV2-uf4P9.test.com"},{"estimate":2,"name":".eW_.test.com"},{"estimate":2,"name":".FiKKi.test.com"},{"estimate":2,"name":".MA_b.test.com"},{"estimate":2,"name":".SmCeRI.test.com"}],"top_qtype":[{"estimate":34,"name":"AAAA"},{"estimate":22,"name":"A"},{"estimate":22,"name":"SOA"},{"estimate":22,"name":"NS"},{"estimate":14,"name":"MX"},{"estimate":14,"name":"CNAME"},{"estimate":12,"name":"TXT"}],"top_rcode":[{"estimate":70,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":14,"name":"46590"},{"estimate":14,"name":"58571"},{"estimate":14,"name":"36484"},{"estimate":14,"name":"43798"},{"estimate":14,"name":"46567"},{"estimate":14,"name":"59640"},{"estimate":14,"name":"44741"},{"estimate":14,"name":"51811"},{"estimate":14,"name":"38920"},{"estimate":14,"name":"52826"}],"wire_packets":{"deep_samples":140,"ipv4":0,"ipv6":140,"noerror":0,"nxdomain":0,"queries":70,"refused":0,"replies":70,"srvfail":0,"tcp":0,"total":140,"udp":140},"xact":{"counts":{"timed_out":0,"total":70},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":0}}},"packets":{"deep_samples":140,"in":0,"ipv4":0,"ipv6":140,"other_l4":0,"out":0,"period":{"length":6,"start_ts":1567706365},"tcp":0,"top_ASN":[],"top_geoLoc":[],"top_ipv4":[],"top_ipv6":[],"total":140,"udp":140}}} +{"5m":{"dns":{"period":{"length":6,"start_ts":1567706365},"top_nxdomain":[],"top_qname2":[{"estimate":140,"name":".test.com"}],"top_qname3":[{"estimate":2,"name":".2pmr.test.com"},{"estimate":2,"name":".uL8E2T.test.com"},{"estimate":2,"name":".kGuiH.test.com"},{"estimate":2,"name":".ddU3i_v.test.com"},{"estimate":2,"name":".nU.test.com"},{"estimate":2,"name":"lfV2-uf4P9.test.com"},{"estimate":2,"name":".eW_.test.com"},{"estimate":2,"name":".FiKKi.test.com"},{"estimate":2,"name":".MA_b.test.com"},{"estimate":2,"name":".SmCeRI.test.com"}],"top_qtype":[{"estimate":34,"name":"AAAA"},{"estimate":22,"name":"A"},{"estimate":22,"name":"SOA"},{"estimate":22,"name":"NS"},{"estimate":14,"name":"MX"},{"estimate":14,"name":"CNAME"},{"estimate":12,"name":"TXT"}],"top_rcode":[{"estimate":70,"name":"NOTIMP"}],"top_refused":[],"top_srvfail":[],"top_udp_ports":[{"estimate":14,"name":"46590"},{"estimate":14,"name":"58571"},{"estimate":14,"name":"36484"},{"estimate":14,"name":"43798"},{"estimate":14,"name":"46567"},{"estimate":14,"name":"59640"},{"estimate":14,"name":"44741"},{"estimate":14,"name":"51811"},{"estimate":14,"name":"38920"},{"estimate":14,"name":"52826"}],"wire_packets":{"deep_samples":140,"filtered":0,"ipv4":0,"ipv6":140,"noerror":0,"nxdomain":0,"queries":70,"refused":0,"replies":70,"srvfail":0,"tcp":0,"total":140,"udp":140},"xact":{"counts":{"timed_out":0,"total":70},"in":{"top_slow":[],"total":0},"out":{"top_slow":[],"total":0}}},"packets":{"deep_samples":140,"in":0,"ipv4":0,"ipv6":140,"other_l4":0,"out":0,"period":{"length":6,"start_ts":1567706365},"tcp":0,"top_ASN":[],"top_geoLoc":[],"top_ipv4":[],"top_ipv6":[],"total":140,"udp":140}}} From 08885bfd9fcf3cc1fdd511a9b63ff99d9003d81c Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Tue, 15 Jun 2021 13:55:04 -0400 Subject: [PATCH 5/6] add filter_only_qname_suffix filter. add StringList to Configurable. make parseResources idempotent --- src/Configurable.h | 6 +++- src/handlers/dns/DnsLayer.cpp | 17 +++++++-- src/handlers/dns/DnsLayer.h | 33 +++++++++-------- src/handlers/dns/DnsStreamHandler.cpp | 43 ++++++++++++++++++++--- src/handlers/dns/DnsStreamHandler.h | 11 ++++-- src/handlers/dns/tests/test_dns_layer.cpp | 30 +++++++++++++++- 6 files changed, 114 insertions(+), 26 deletions(-) diff --git a/src/Configurable.h b/src/Configurable.h index ec64e6ac6..24daee946 100644 --- a/src/Configurable.h +++ b/src/Configurable.h @@ -13,6 +13,7 @@ #include #include #include +#include #include namespace visor { @@ -30,8 +31,11 @@ class ConfigException : public std::runtime_error class Configurable { +public: + typedef std::vector StringList; + private: - std::unordered_map> _config; + std::unordered_map> _config; mutable std::shared_mutex _config_mutex; public: diff --git a/src/handlers/dns/DnsLayer.cpp b/src/handlers/dns/DnsLayer.cpp index 2f9d567a2..ccfe3b239 100644 --- a/src/handlers/dns/DnsLayer.cpp +++ b/src/handlers/dns/DnsLayer.cpp @@ -120,6 +120,11 @@ bool DnsLayer::shortenLayer(int offsetInLayer, size_t numOfBytesToShorten, IDnsR bool DnsLayer::parseResources(bool queryOnly) { + + if (m_ResourcesParsed) { + return m_ResourcesParseResult; + } + size_t offsetInPacket = sizeof(dnshdr); IDnsResource *curResource = m_ResourceList; @@ -132,7 +137,9 @@ bool DnsLayer::parseResources(bool queryOnly) if (numOfOtherResources > 100) { // probably bad packet - return false; + m_ResourcesParsed = true; + m_ResourcesParseResult = false; + return m_ResourcesParseResult; } for (uint32_t i = 0; i < numOfOtherResources; i++) { @@ -174,7 +181,9 @@ bool DnsLayer::parseResources(bool queryOnly) if (offsetInPacket > m_DataLen) { //Parse packet failed, DNS resource is out of bounds. Probably a bad packet delete newGenResource; - return false; + m_ResourcesParsed = true; + m_ResourcesParseResult = false; + return m_ResourcesParseResult; } // this resource is the first resource @@ -201,7 +210,9 @@ bool DnsLayer::parseResources(bool queryOnly) m_FirstAdditional = newResource; } - return true; + m_ResourcesParsed = true; + m_ResourcesParseResult = true; + return m_ResourcesParseResult; } IDnsResource* DnsLayer::getResourceByName(IDnsResource* startFrom, size_t resourceCount, const std::string& name, bool exactMatch) const diff --git a/src/handlers/dns/DnsLayer.h b/src/handlers/dns/DnsLayer.h index e6466c9cb..8a5c10e5d 100644 --- a/src/handlers/dns/DnsLayer.h +++ b/src/handlers/dns/DnsLayer.h @@ -435,28 +435,33 @@ struct dnshdr { bool parseResources(bool queryOnly); - pcpp::OsiModelLayer getOsiModelLayer() const { return pcpp::OsiModelApplicationLayer; } + pcpp::OsiModelLayer getOsiModelLayer() const + { + return pcpp::OsiModelApplicationLayer; + } - /** + /** * A static method that checks whether the port is considered as DNS * @param[in] port The port number to be checked */ - static inline bool isDnsPort(uint16_t port); + static inline bool isDnsPort(uint16_t port); - private: - IDnsResource* m_ResourceList; - DnsQuery* m_FirstQuery; - DnsResource* m_FirstAnswer; - DnsResource* m_FirstAuthority; - DnsResource* m_FirstAdditional; + private: + bool m_ResourcesParsed{false}; + bool m_ResourcesParseResult{false}; + IDnsResource *m_ResourceList; + DnsQuery *m_FirstQuery; + DnsResource *m_FirstAnswer; + DnsResource *m_FirstAuthority; + DnsResource *m_FirstAdditional; - IDnsResource* getFirstResource(DnsResourceType resType) const; - void setFirstResource(DnsResourceType resType, IDnsResource* resource); + IDnsResource *getFirstResource(DnsResourceType resType) const; + void setFirstResource(DnsResourceType resType, IDnsResource *resource); - using Layer::extendLayer; - bool extendLayer(int offsetInLayer, size_t numOfBytesToExtend, IDnsResource* resource); + using Layer::extendLayer; + bool extendLayer(int offsetInLayer, size_t numOfBytesToExtend, IDnsResource *resource); - using Layer::shortenLayer; + using Layer::shortenLayer; bool shortenLayer(int offsetInLayer, size_t numOfBytesToShorten, IDnsResource* resource); IDnsResource* getResourceByName(IDnsResource* startFrom, size_t resourceCount, const std::string& name, bool exactMatch) const; diff --git a/src/handlers/dns/DnsStreamHandler.cpp b/src/handlers/dns/DnsStreamHandler.cpp index ed234e17a..9e10e8d92 100644 --- a/src/handlers/dns/DnsStreamHandler.cpp +++ b/src/handlers/dns/DnsStreamHandler.cpp @@ -39,7 +39,7 @@ void DnsStreamHandler::start() // Setup Filters if (config_exists("filter_exclude_noerror") && config_get("filter_exclude_noerror")) { - _f_excluding_rcode = true; + _f_enabled.set(Filters::ExcludingRCode); _f_rcode = NoError; } else if (config_exists("filter_only_rcode")) { auto want_code = config_get("filter_only_rcode"); @@ -48,13 +48,25 @@ void DnsStreamHandler::start() case NXDomain: case SrvFail: case Refused: - _f_only_rcode = true; + _f_enabled.set(Filters::OnlyRCode); _f_rcode = want_code; break; default: throw ConfigException("filter_only_rcode contained an invalid/unsupported rcode"); } } + if (config_exists("filter_only_qname_suffix")) { + _f_enabled.set(Filters::OnlyQNameSuffix); + for (const auto &qname : config_get("filter_only_qname_suffix")) { + // note, this currently copies the strings, meaning there could be a big list that is duplicated + // we can work on trying to make this a string_view instead + // we copy it out so that we don't have to hit the config mutex + std::string qname_ci{qname}; + std::transform(qname_ci.begin(), qname_ci.end(), qname_ci.begin(), + [](unsigned char c) { return std::tolower(c); }); + _f_qnames.emplace_back(std::move(qname_ci)); + } + } if (config_exists("recorded_stream")) { _metrics->set_recorded_stream(); @@ -244,13 +256,35 @@ void DnsStreamHandler::info_json(json &j) const common_info_json(j); j[schema_key()]["xact"]["open"] = _metrics->num_open_transactions(); } +static inline bool endsWith(std::string_view str, std::string_view suffix) +{ + return str.size() >= suffix.size() && 0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix); +} bool DnsStreamHandler::_filtering(DnsLayer &payload, [[maybe_unused]] PacketDirection dir, [[maybe_unused]] pcpp::ProtocolType l3, [[maybe_unused]] pcpp::ProtocolType l4, [[maybe_unused]] uint16_t port, timespec stamp) { - if (_f_excluding_rcode && payload.getDnsHeader()->responseCode == _f_rcode) { + if (_f_enabled.test(Filters::ExcludingRCode) && payload.getDnsHeader()->responseCode == _f_rcode) { + goto will_filter; + } else if (_f_enabled.test(Filters::OnlyRCode) && payload.getDnsHeader()->responseCode != _f_rcode) { goto will_filter; - } else if (_f_only_rcode && payload.getDnsHeader()->responseCode != _f_rcode) { + } + if (_f_enabled.test(Filters::OnlyQNameSuffix)) { + if (!payload.parseResources(true) || payload.getFirstQuery() == nullptr) { + goto will_filter; + } + // we need an all lower case version of this, we can't get away without making a copy + std::string qname_ci{payload.getFirstQuery()->getName()}; + std::transform(qname_ci.begin(), qname_ci.end(), qname_ci.begin(), + [](unsigned char c) { return std::tolower(c); }); + for (auto fqn : _f_qnames) { + // if it matched, we know we are not filtering + if (endsWith(qname_ci, fqn)) { + goto will_not_filter; + } + } + // checked the whole list and none of them matched: filter goto will_filter; } +will_not_filter: return false; will_filter: _metrics->process_filtered(stamp); @@ -565,5 +599,4 @@ void DnsMetricsManager::process_filtered(timespec stamp) new_event(stamp, false); live_bucket()->process_filtered(); } - } \ No newline at end of file diff --git a/src/handlers/dns/DnsStreamHandler.h b/src/handlers/dns/DnsStreamHandler.h index 49d89fce4..d96adf24e 100644 --- a/src/handlers/dns/DnsStreamHandler.h +++ b/src/handlers/dns/DnsStreamHandler.h @@ -11,6 +11,7 @@ #include "dns.h" #include "querypairmgr.h" #include +#include #include #include @@ -231,9 +232,15 @@ class DnsStreamHandler final : public visor::StreamMetricsHandler _f_enabled; uint16_t _f_rcode{0}; + std::vector _f_qnames; + bool _filtering(DnsLayer &payload, PacketDirection dir, pcpp::ProtocolType l3, pcpp::ProtocolType l4, uint16_t port, timespec stamp); public: diff --git a/src/handlers/dns/tests/test_dns_layer.cpp b/src/handlers/dns/tests/test_dns_layer.cpp index fe291f734..d2f1d70fd 100644 --- a/src/handlers/dns/tests/test_dns_layer.cpp +++ b/src/handlers/dns/tests/test_dns_layer.cpp @@ -344,4 +344,32 @@ TEST_CASE("DNS Filters: filter_only_rcode refused", "[pcap][net]") nlohmann::json j; dns_handler.metrics()->bucket(0)->to_json(j); REQUIRE(j["wire_packets"]["filtered"] == 23); -} \ No newline at end of file +} + +TEST_CASE("DNS Filters: filter_only_qname_suffix", "[pcap][net]") +{ + + PcapInputStream stream{"pcap-test"}; + stream.config_set("pcap_file", "tests/fixtures/dns_udp_mixed_rcode.pcap"); + stream.config_set("bpf", ""); + stream.config_set("host_spec", "192.168.0.0/24"); + stream.parse_host_spec(); + + visor::Config c; + c.config_set("num_periods", 1); + DnsStreamHandler dns_handler{"dns-test", &stream, &c}; + + // notice, case insensitive + dns_handler.config_set("filter_only_qname_suffix", {"GooGle.com"}); + / dns_handler.start(); + stream.start(); + stream.stop(); + dns_handler.stop(); + + auto counters = dns_handler.metrics()->bucket(0)->counters(); + CHECK(counters.NOERROR.value() == 4); + CHECK(counters.SRVFAIL.value() == 0); + CHECK(counters.REFUSED.value() == 0); + CHECK(counters.NX.value() == 1); + CHECK(counters.filtered.value() == 14); +} From 2728f22866b9df43968d377c63c731e833203423 Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Tue, 15 Jun 2021 14:05:34 -0400 Subject: [PATCH 6/6] typo --- src/handlers/dns/tests/test_dns_layer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/handlers/dns/tests/test_dns_layer.cpp b/src/handlers/dns/tests/test_dns_layer.cpp index d2f1d70fd..ca183c4c1 100644 --- a/src/handlers/dns/tests/test_dns_layer.cpp +++ b/src/handlers/dns/tests/test_dns_layer.cpp @@ -361,7 +361,7 @@ TEST_CASE("DNS Filters: filter_only_qname_suffix", "[pcap][net]") // notice, case insensitive dns_handler.config_set("filter_only_qname_suffix", {"GooGle.com"}); - / dns_handler.start(); + dns_handler.start(); stream.start(); stream.stop(); dns_handler.stop();