From ad222fa09e1604527d3dcf8226379f428c0175ac Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Mon, 4 May 2020 15:05:12 -0400 Subject: [PATCH 1/7] layout --- src/main.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 823242ace..7b16c35b6 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -429,11 +429,11 @@ int main(int argc, char *argv[]) openPcap(args["TARGET"].asString(), tcpDnsReassembly, bpf); if (args["--summary"].asBool()) { // in summary mode we output a single summary of stats - std::cout << metricsManager->getMetrics() << std::endl; + std::cout << std::endl << metricsManager->getMetrics() << std::endl; } else { // otherwise, merge the max time window available - std::cout << metricsManager->getMetricsMerged(periods) << std::endl; + std::cout << std::endl << metricsManager->getMetricsMerged(periods) << std::endl; } } catch (const std::exception &e) { std::cerr << e.what() << std::endl; From e733edfb625860876da4083e80233c96e9ef6695 Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Mon, 4 May 2020 17:47:54 -0400 Subject: [PATCH 2/7] move to parsing dns resources only in deep sample (needs upstream commitin pcpp). switch to buster-slim in docker --- Dockerfile | 2 +- src/metrics.cpp | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ba9150a18..b7a54081f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:disco AS build +FROM debian:buster-slim AS build ENV BUILD_DEPS "g++ cmake make git libpcap-dev pkgconf golang ca-certificates libmaxminddb-dev jq" diff --git a/src/metrics.cpp b/src/metrics.cpp index c58b9bbd4..b4c3bb6d0 100644 --- a/src/metrics.cpp +++ b/src/metrics.cpp @@ -133,6 +133,8 @@ void Metrics::newDNSPacket(pcpp::DnsLayer *dns, Direction dir, pcpp::ProtocolTyp return; } + dns->parseResources(); + // lock for write std::unique_lock lock(_sketchMutex); From 595b5b3d6179e308e8918e970ca91ac18ba4c10d Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Fri, 12 Jun 2020 11:05:33 -0400 Subject: [PATCH 3/7] edns parse --- src/dns.h | 2 ++ src/metrics.cpp | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/dns.h b/src/dns.h index bd8301dce..7a6331250 100644 --- a/src/dns.h +++ b/src/dns.h @@ -5,6 +5,8 @@ namespace pktvisor { +const uint16_t EDNS0_CSUBNET = 8; + static std::unordered_map DNS_QTYPES({ {1, "A"}, {2, "NS"}, diff --git a/src/metrics.cpp b/src/metrics.cpp index b4c3bb6d0..69d19b1a7 100644 --- a/src/metrics.cpp +++ b/src/metrics.cpp @@ -92,6 +92,24 @@ void Metrics::merge(Metrics &other) _sketches->_dns_slowXactOut.merge(other._sketches->_dns_slowXactOut); _sketches->_net_topGeoLoc.merge(other._sketches->_net_topGeoLoc); _sketches->_net_topASN.merge(other._sketches->_net_topASN); +} + +static int parseCSubnet(const uint8_t* ednsData, size_t ednsDataLen) { + + if (ednsDataLen < 4) + return -1; + + auto ednsCode = (uint16_t*)ednsData; + ednsData += 2; + auto optLen = (uint16_t*)ednsData; + ednsData += 2; + + if (*ednsCode != EDNS0_CSUBNET) + return -1; + + + + } void Metrics::newDNSPacket(pcpp::DnsLayer *dns, Direction dir, pcpp::ProtocolType l3, pcpp::ProtocolType l4) @@ -135,6 +153,21 @@ void Metrics::newDNSPacket(pcpp::DnsLayer *dns, Direction dir, pcpp::ProtocolTyp dns->parseResources(); + // look for ECS + auto optRR = dns->getFirstAdditionalRecord(); + if (optRR && optRR->getDnsType() == pcpp::DNS_TYPE_OPT) { + auto optData = optRR->getData(); + if (optData) { + auto genericRRData = optData->castAs(); + if (genericRRData) { + // decode EDNS, look for CSUBNET + auto ednsData = genericRRData->getData(); + auto ednsDataLen = genericRRData->getDataLen(); + auto scope = parseCSubnet(ednsData, ednsDataLen); + } + } + } + // lock for write std::unique_lock lock(_sketchMutex); From 16c7a54f022b72cd6dcbc4c7fdfff1b268fa822c Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Wed, 29 Jul 2020 14:43:11 -0400 Subject: [PATCH 4/7] back out initial csubnet, fix version --- src/config.h.in | 4 ++-- src/dns.h | 2 -- src/metrics.cpp | 33 --------------------------------- 3 files changed, 2 insertions(+), 37 deletions(-) diff --git a/src/config.h.in b/src/config.h.in index f7f0ac17d..9b578ec13 100644 --- a/src/config.h.in +++ b/src/config.h.in @@ -1,5 +1,5 @@ #pragma once #cmakedefine MMDB_ENABLE -#cmakedefine PKTVISOR_VERSION_NUM "@PROJECT_VERSION@" -#cmakedefine PKTVISOR_VERSION "@FLAME_VERSION@" +#cmakedefine PKTVISOR_VERSION_NUM "@PKTVISOR_VERSION_NUM@" +#cmakedefine PKTVISOR_VERSION "@PKTVISOR_VERSION@" diff --git a/src/dns.h b/src/dns.h index 7a6331250..bd8301dce 100644 --- a/src/dns.h +++ b/src/dns.h @@ -5,8 +5,6 @@ namespace pktvisor { -const uint16_t EDNS0_CSUBNET = 8; - static std::unordered_map DNS_QTYPES({ {1, "A"}, {2, "NS"}, diff --git a/src/metrics.cpp b/src/metrics.cpp index 69d19b1a7..b4c3bb6d0 100644 --- a/src/metrics.cpp +++ b/src/metrics.cpp @@ -92,24 +92,6 @@ void Metrics::merge(Metrics &other) _sketches->_dns_slowXactOut.merge(other._sketches->_dns_slowXactOut); _sketches->_net_topGeoLoc.merge(other._sketches->_net_topGeoLoc); _sketches->_net_topASN.merge(other._sketches->_net_topASN); -} - -static int parseCSubnet(const uint8_t* ednsData, size_t ednsDataLen) { - - if (ednsDataLen < 4) - return -1; - - auto ednsCode = (uint16_t*)ednsData; - ednsData += 2; - auto optLen = (uint16_t*)ednsData; - ednsData += 2; - - if (*ednsCode != EDNS0_CSUBNET) - return -1; - - - - } void Metrics::newDNSPacket(pcpp::DnsLayer *dns, Direction dir, pcpp::ProtocolType l3, pcpp::ProtocolType l4) @@ -153,21 +135,6 @@ void Metrics::newDNSPacket(pcpp::DnsLayer *dns, Direction dir, pcpp::ProtocolTyp dns->parseResources(); - // look for ECS - auto optRR = dns->getFirstAdditionalRecord(); - if (optRR && optRR->getDnsType() == pcpp::DNS_TYPE_OPT) { - auto optData = optRR->getData(); - if (optData) { - auto genericRRData = optData->castAs(); - if (genericRRData) { - // decode EDNS, look for CSUBNET - auto ednsData = genericRRData->getData(); - auto ednsDataLen = genericRRData->getDataLen(); - auto scope = parseCSubnet(ednsData, ednsDataLen); - } - } - } - // lock for write std::unique_lock lock(_sketchMutex); From 9f889d2508bcf5a028ffdfe3b08ff776549e2fd9 Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Thu, 27 Aug 2020 16:04:43 -0400 Subject: [PATCH 5/7] fix tests --- tests/test_parse_pcap.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_parse_pcap.cpp b/tests/test_parse_pcap.cpp index fa8d1225f..ec0f3ecc5 100644 --- a/tests/test_parse_pcap.cpp +++ b/tests/test_parse_pcap.cpp @@ -25,6 +25,7 @@ TEST_CASE("Parse DNS UDP IPv4 tests", "[pcap][ipv4][udp][dns]") numUDP++; if (dnsRequest.isPacketOfType(pcpp::DNS)) { pcpp::DnsLayer *dnsLayer = dnsRequest.getLayerOfType(); + dnsLayer->parseResources(); if (numDNS == 0) { CHECK(dnsLayer->getFirstQuery()->getName() == "utadwnME.POJwOc9R.KtfO.test.com"); CHECK(dnsLayer->getFirstQuery()->getDnsType() == pcpp::DNS_TYPE_AAAA); @@ -107,6 +108,7 @@ TEST_CASE("Parse DNS UDP IPv6 tests", "[pcap][ipv6][udp][dns]") numUDP++; if (dnsRequest.isPacketOfType(pcpp::DNS)) { pcpp::DnsLayer *dnsLayer = dnsRequest.getLayerOfType(); + dnsLayer->parseResources(); if (numDNS == 0) { CHECK(dnsLayer->getFirstQuery()->getName() == "LOJ5Pq2._EmpLuAPR.PPLIop.1F8J2R1.eMVq5.test.com"); CHECK(dnsLayer->getFirstQuery()->getDnsType() == pcpp::DNS_TYPE_AAAA); From a399a20ab58cd538770a698cc58a10cc369758d1 Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Thu, 27 Aug 2020 17:28:06 -0400 Subject: [PATCH 6/7] add -l to specify metrics web server bind host --- src/main.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 7b16c35b6..2f9ab57c4 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -23,7 +23,7 @@ static const char USAGE[] = R"(pktvisord. Usage: - pktvisord [-b BPF] [-p PORT] [-H HOSTSPEC] [--periods P] [--summary] [--geo-city FILE] [--geo-asn FILE] + pktvisord [-b BPF] [-l HOST] [-p PORT] [-H HOSTSPEC] [--periods P] [--summary] [--geo-city FILE] [--geo-asn FILE] [--max-deep-sample N] TARGET pktvisord (-h | --help) @@ -34,7 +34,8 @@ static const char USAGE[] = TARGET is either a network interface, an IP address (4 or 6) or a pcap file (ending in .pcap or .cap) Options: - -p PORT Run metrics webserver on the given localhost port [default: 10853] + -l HOST Run metrics webserver on the given host or IP [default: localhost] + -p PORT Run metrics webserver on the given port [default: 10853] -b BPF Filter packets using the given BPF string --geo-city FILE GeoLite2 City database to use for IP to Geo mapping (if enabled) --geo-asn FILE GeoLite2 ASN database to use for IP to ASN mapping (if enabled) @@ -465,9 +466,13 @@ int main(int argc, char *argv[]) } httplib::Server svr; setupRoutes(svr); + auto host = args["-l"].asString(); auto port = args["-p"].asLong(); - std::thread httpThread([&svr, port] { - svr.listen("localhost", port); + std::thread httpThread([&svr, host, port] { + std::cerr << "Metrics web server listening on " << host << ":" << port << std::endl; + if (!svr.listen(host.c_str(), port)) { + throw std::runtime_error("unable to listen"); + } }); try { std::cerr << "Interface " << dev->getName() << std::endl; From a98e3df3d7ce85ba216eaed29073081de141d9ef Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Tue, 1 Sep 2020 12:06:22 -0400 Subject: [PATCH 7/7] fix runtime image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b7a54081f..d62b11c52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,7 +32,7 @@ RUN \ go get github.com/docopt/docopt-go && \ go build /src/cmd/pktvisor/pktvisor.go -FROM ubuntu:disco AS runtime +FROM debian:buster-slim AS runtime ENV RUNTIME_DEPS "curl libpcap0.8 libmaxminddb0"