From daa313113f3c9132820cc24912a99df0727fc797 Mon Sep 17 00:00:00 2001 From: xiaoshuai <49056817+xiaoshuai7038@users.noreply.github.com> Date: Wed, 20 May 2026 17:26:36 +0800 Subject: [PATCH 1/2] feat: track DNS record history --- README.md | 25 ++++ dooked/CMakeLists.txt | 7 +- dooked/include/cli_preprocessor.hpp | 5 + dooked/include/utils/exceptions.hpp | 1 + dooked/include/utils/io_utils.hpp | 30 ++++ dooked/include/utils/probe_result.hpp | 3 + dooked/source/cli_preprocessor.cpp | 188 ++++++++++++++++++++++-- dooked/source/http/requests_handler.cpp | 21 ++- dooked/source/main.cpp | 6 + dooked/source/utils/io_utils.cpp | 5 +- 10 files changed, 271 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index f1a761c..58026fe 100644 --- a/README.md +++ b/README.md @@ -39,3 +39,28 @@ make ## Usage For comprehensive help, use `dooked --help` + +### DNS history tracking + +JSON output now records DNS history on each `dns_probe` entry: + +- `first_seen`: first date the record was observed (`YYYY-MM-DD`) +- `last_seen`: latest date the record was observed (`YYYY-MM-DD`) +- `seen`: number of runs where the record was observed + +When a previous JSON result is used as input, dooked preserves old records that are +not seen in the current run. This makes load-balanced targets easier to track, +because a rotating IP is not lost just because it disappeared from one scan. + +Useful flags: + +```bash +# Print records discovered for the first time in this run +dooked --fs -i domains.txt -o current.json + +# Re-scan a previous result and report records not seen in the last 30 days +dooked --ls 30 -i current.json -o next.json + +# Report records last seen before a specific US date +dooked --lsd 01/31/2026 -i current.json -o next.json +``` diff --git a/dooked/CMakeLists.txt b/dooked/CMakeLists.txt index c43ff38..65e31ef 100644 --- a/dooked/CMakeLists.txt +++ b/dooked/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.0 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5.0 FATAL_ERROR) # Project get_filename_component(PROJECT_DIR "${CMAKE_CURRENT_SOURCE_DIR}" ABSOLUTE) @@ -23,8 +23,8 @@ include_directories(${PROJECT_DIR}/json/single_include) include_directories(${PROJECT_DIR}/spdlog/include) -link_directories(/usr/lib) -link_libraries(pthread ssl crypto) +find_package(OpenSSL REQUIRED) + # Outputs set(OUTPUT_DEBUG ${PROJECT_DIR}/bin) @@ -102,6 +102,7 @@ source_group("Headers" FILES ${HEADERS_FILES}) add_executable(${PROJECT_NAME} ${SRC_FILES} ${HEADERS_FILES} ) +target_link_libraries(${PROJECT_NAME} PRIVATE pthread OpenSSL::SSL OpenSSL::Crypto) if(NOT MSVC) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3") diff --git a/dooked/include/cli_preprocessor.hpp b/dooked/include/cli_preprocessor.hpp index 43fa1ba..39a83ea 100644 --- a/dooked/include/cli_preprocessor.hpp +++ b/dooked/include/cli_preprocessor.hpp @@ -24,7 +24,10 @@ struct cli_args_t { int post_http_request{}; int thread_count{}; int content_length{-1}; + int last_seen_days{-1}; bool include_date{false}; + bool first_seen_log{false}; + std::string last_seen_date{}; }; struct runtime_args_t { @@ -36,6 +39,8 @@ struct runtime_args_t { http_process_e http_request_time_{}; int thread_count{}; int content_length{-1}; + bool first_seen_log{false}; + std::string last_seen_threshold{}; }; void run_program(cli_args_t const &cli_args); diff --git a/dooked/include/utils/exceptions.hpp b/dooked/include/utils/exceptions.hpp index a749a1b..846d544 100644 --- a/dooked/include/utils/exceptions.hpp +++ b/dooked/include/utils/exceptions.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include namespace dooked { diff --git a/dooked/include/utils/io_utils.hpp b/dooked/include/utils/io_utils.hpp index 829b09e..c292297 100644 --- a/dooked/include/utils/io_utils.hpp +++ b/dooked/include/utils/io_utils.hpp @@ -29,7 +29,10 @@ struct json_data_t { int ttl{}; int http_code{}; int content_length{}; + std::string first_seen{}; + std::string last_seen{}; dns_record_type_e type{}; + int seen{1}; static json_data_t serialize(std::string const &d, int const len, int const http_code, @@ -42,8 +45,35 @@ struct json_data_t { data.ttl = json_object["ttl"].get(); data.content_length = len; data.http_code = http_code; + data.first_seen = get_history_string(json_object, "first_seen", + "first-seen"); + data.last_seen = + get_history_string(json_object, "last_seen", "last-seen"); + data.seen = get_history_count(json_object); return data; } + +private: + static std::string get_history_string(json::object_t const &json_object, + char const *primary_key, + char const *legacy_key) { + auto iter = json_object.find(primary_key); + if (iter == json_object.cend()) { + iter = json_object.find(legacy_key); + } + if (iter != json_object.cend() && iter->second.is_string()) { + return iter->second.get(); + } + return {}; + } + + static int get_history_count(json::object_t const &json_object) { + auto const iter = json_object.find("seen"); + if (iter != json_object.cend() && iter->second.is_number_integer()) { + return iter->second.get(); + } + return 1; + } }; struct jd_domain_comparator_t { diff --git a/dooked/include/utils/probe_result.hpp b/dooked/include/utils/probe_result.hpp index 07211c6..6d2adf9 100644 --- a/dooked/include/utils/probe_result.hpp +++ b/dooked/include/utils/probe_result.hpp @@ -10,8 +10,11 @@ bool case_insensitive_compare(std::string const &, std::string const &); struct probe_result_t { std::string rdata{}; + std::string first_seen{}; + std::string last_seen{}; dns_record_type_e type{}; // RR TYPE (2 octets) std::uint32_t ttl{}; // time to live(4 octets) + int seen{}; friend bool operator==(probe_result_t const &a, probe_result_t const &b) { return case_insensitive_compare(a.rdata, b.rdata) && (a.type == b.type); diff --git a/dooked/source/cli_preprocessor.cpp b/dooked/source/cli_preprocessor.cpp index c08d7fb..1756517 100644 --- a/dooked/source/cli_preprocessor.cpp +++ b/dooked/source/cli_preprocessor.cpp @@ -6,8 +6,13 @@ #include "utils/string_utils.hpp" #include #include +#include +#include +#include #include #include +#include +#include // defined (and assigned to) in main.cpp extern bool silent; @@ -18,6 +23,158 @@ namespace dooked { namespace net = boost::asio; using namespace fmt::v7::literals; +std::string today_iso_date() { + std::string today{}; + if (!timet_to_string(today, std::time(nullptr), "%Y-%m-%d")) { + return {}; + } + return today; +} + +std::string date_days_ago(int const days) { + if (days < 0) { + return {}; + } + std::time_t threshold = std::time(nullptr) - + static_cast(days) * 24 * 60 * 60; + std::string date{}; + if (!timet_to_string(date, threshold, "%Y-%m-%d")) { + return {}; + } + return date; +} + +std::string parse_us_date(std::string date) { + trim(date); + if (date.size() != 10 || !std::isdigit(date[0]) || !std::isdigit(date[1]) || + !std::isdigit(date[3]) || !std::isdigit(date[4]) || + !std::isdigit(date[6]) || !std::isdigit(date[7]) || + !std::isdigit(date[8]) || !std::isdigit(date[9]) || + !((date[2] == '/' && date[5] == '/') || + (date[2] == '-' && date[5] == '-'))) { + return {}; + } + int const month = std::stoi(date.substr(0, 2)); + int const day = std::stoi(date.substr(3, 2)); + int const year = std::stoi(date.substr(6, 4)); + if (month < 1 || month > 12 || day < 1 || day > 31) { + return {}; + } + std::tm tm{}; + tm.tm_year = year - 1900; + tm.tm_mon = month - 1; + tm.tm_mday = day; + tm.tm_isdst = -1; + auto const normalized = std::mktime(&tm); + if (normalized == static_cast(-1) || tm.tm_year != year - 1900 || + tm.tm_mon != month - 1 || tm.tm_mday != day) { + return {}; + } + char output[11]{}; + std::strftime(output, sizeof(output), "%Y-%m-%d", &tm); + return std::string{output}; +} + +using history_key_t = std::tuple; + +history_key_t history_key(std::string const &domain, dns_record_type_e const type, + std::string const &rdata) { + return {domain, type, rdata}; +} + +history_key_t history_key(json_data_t const &record) { + return history_key(record.domain_name, record.type, record.rdata); +} + +history_key_t history_key(std::string const &domain, probe_result_t const &record) { + return history_key(domain, record.type, record.rdata); +} + +probe_result_t json_to_probe_result(json_data_t const &record) { + probe_result_t result{}; + result.rdata = record.rdata; + result.type = record.type; + result.ttl = static_cast(record.ttl); + result.first_seen = record.first_seen; + result.last_seen = record.last_seen; + result.seen = record.seen > 0 ? record.seen : 1; + return result; +} + +void log_first_seen(std::string const &domain, probe_result_t const &record) { + spdlog::info("[FIRST-SEEN][{}][{}] `{}`", domain, + dns_record_type_to_str(record.type), record.rdata); +} + +void log_last_seen(std::string const &domain, probe_result_t const &record) { + spdlog::info("[LAST-SEEN][{}][{}] `{}` last seen `{}`", domain, + dns_record_type_to_str(record.type), record.rdata, + record.last_seen); +} + +void merge_history(map_container_t &result_map, + std::optional> const &previous_data, + runtime_args_t const &rt_args) { + auto const today = today_iso_date(); + std::map previous_by_key{}; + if (previous_data) { + for (auto const &record : *previous_data) { + previous_by_key.emplace(history_key(record), record); + } + } + + std::set seen_this_run{}; + for (auto &result_pair : result_map.result()) { + for (auto &record : result_pair.second.dns_result_list_) { + auto const key = history_key(result_pair.first, record); + seen_this_run.insert(key); + auto const prev_iter = previous_by_key.find(key); + if (prev_iter == previous_by_key.end()) { + record.first_seen = today; + record.last_seen = today; + record.seen = 1; + if (rt_args.first_seen_log) { + log_first_seen(result_pair.first, record); + } + } else { + auto const &previous = prev_iter->second; + record.first_seen = previous.first_seen.empty() ? today : previous.first_seen; + record.last_seen = today; + record.seen = (previous.seen > 0 ? previous.seen : 1) + 1; + } + } + } + + if (previous_data) { + for (auto const &previous : *previous_data) { + if (seen_this_run.find(history_key(previous)) != seen_this_run.end()) { + continue; + } + auto preserved = json_to_probe_result(previous); + if (preserved.first_seen.empty()) { + preserved.first_seen = today; + } + if (preserved.last_seen.empty()) { + preserved.last_seen = preserved.first_seen; + } + result_map.append(previous.domain_name, preserved); + result_map.insert(previous.domain_name, previous.content_length, + previous.http_code); + } + } + + if (!rt_args.last_seen_threshold.empty()) { + for (auto const &result_pair : result_map.cresult()) { + for (auto const &record : result_pair.second.dns_result_list_) { + if (!record.last_seen.empty() && + record.last_seen < rt_args.last_seen_threshold) { + log_last_seen(result_pair.first, record); + } + } + } + } +} + void compare_http_result(int const base_cl, json_data_t const &prev_http_result, http_response_t const ¤t_result) { auto const current_req_cl = current_result.content_length_; @@ -339,7 +496,7 @@ void start_name_checking(runtime_args_t &&rt_args) { // if we deferred HTTP/S "probe", now is the time to get to it if (deferring) { - io_context.reset(); + io_context.restart(); thread_pool.emplace(thread_count); rt_args.names.emplace(std::move(*deferred_names_)); for (std::size_t index = 0; index < thread_count; ++index) { @@ -350,12 +507,7 @@ void start_name_checking(runtime_args_t &&rt_args) { } thread_pool->join(); } - if (!silent) { - spdlog::info("Writing JSON output"); - } - write_json_result(result_map, rt_args); - - // compare old with new result -- only if we had previous record + // compare old with new result before preserving missing history records. if (rt_args.previous_data) { auto &previous_data = *rt_args.previous_data; @@ -373,9 +525,14 @@ void start_name_checking(runtime_args_t &&rt_args) { return std::tie(a.type, a.rdata) < std::tie(b.type, b.rdata); }); } - return compare_results(*rt_args.previous_data, result_map, - rt_args.content_length); + compare_results(*rt_args.previous_data, result_map, rt_args.content_length); + } + + if (!silent) { + spdlog::info("Writing JSON output"); } + merge_history(result_map, rt_args.previous_data, rt_args); + write_json_result(result_map, rt_args); } void run_program(cli_args_t const &cli_args) { @@ -477,6 +634,19 @@ void run_program(cli_args_t const &cli_args) { static_cast(cli_args.post_http_request); rt_args.thread_count = cli_args.thread_count; rt_args.content_length = cli_args.content_length; + rt_args.first_seen_log = cli_args.first_seen_log; + if (!cli_args.last_seen_date.empty()) { + rt_args.last_seen_threshold = parse_us_date(cli_args.last_seen_date); + if (rt_args.last_seen_threshold.empty()) { + return spdlog::error("invalid --lsd date `{}`; use MM/DD/YYYY or MM-DD-YYYY", + cli_args.last_seen_date); + } + } else if (cli_args.last_seen_days >= 0) { + rt_args.last_seen_threshold = date_days_ago(cli_args.last_seen_days); + if (rt_args.last_seen_threshold.empty()) { + return spdlog::error("invalid --ls value `{}`", cli_args.last_seen_days); + } + } return start_name_checking(std::move(rt_args)); } diff --git a/dooked/source/http/requests_handler.cpp b/dooked/source/http/requests_handler.cpp index d21a592..39d69fc 100644 --- a/dooked/source/http/requests_handler.cpp +++ b/dooked/source/http/requests_handler.cpp @@ -139,7 +139,8 @@ void http_request_handler_t::on_data_received( if (status_code_simple == 2) { response_int = response_type_e::ok; } else if (status_code_simple == 3) { // redirected - response_string = (*response_)[http::field::location].to_string(); + auto const location = (*response_)[http::field::location]; + response_string = std::string{location.data(), location.size()}; if (response_string.empty()) { response_int = response_type_e::unknown_response; } else { @@ -171,7 +172,9 @@ void http_request_handler_t::on_data_received( int content_length{}; if (response_->has_content_length()) { try { - auto const cl_str = (*response_)[http::field::content_length].to_string(); + auto const content_length_header = (*response_)[http::field::content_length]; + auto const cl_str = std::string{content_length_header.data(), + content_length_header.size()}; content_length = std::stoi(cl_str); } catch (std::exception const &) { } @@ -246,9 +249,10 @@ void https_request_handler_t::on_ssl_handshake( void https_request_handler_t::send_https_data() { beast::get_lowest_layer(*ssl_stream_) .expires_after(std::chrono::seconds(DOOKED_MAX_HTTP_WAIT_TIME)); - http::async_write( - *ssl_stream_, *get_request_, - beast::bind_front_handler(&https_request_handler_t::on_data_sent, this)); + http::async_write(*ssl_stream_, *get_request_, + [this](beast::error_code ec, std::size_t bytes_sent) { + on_data_sent(ec, bytes_sent); + }); } void https_request_handler_t::on_data_sent(beast::error_code ec, std::size_t) { @@ -365,7 +369,8 @@ void https_request_handler_t::on_data_received( if (status_code_simple == 2) { response_int = response_type_e::ok; } else if (status_code_simple == 3) { // redirected - response_string = (*response_)[http::field::location].to_string(); + auto const location = (*response_)[http::field::location]; + response_string = std::string{location.data(), location.size()}; if (response_string.empty()) { response_int = response_type_e::unknown_response; } else { @@ -392,7 +397,9 @@ void https_request_handler_t::on_data_received( int content_length = 0; if (response_->has_content_length()) { try { - auto const cl_str = (*response_)[http::field::content_length].to_string(); + auto const content_length_header = (*response_)[http::field::content_length]; + auto const cl_str = std::string{content_length_header.data(), + content_length_header.size()}; content_length = std::stoi(cl_str); } catch (std::exception const &) { } diff --git a/dooked/source/main.cpp b/dooked/source/main.cpp index cf29460..b273ba4 100644 --- a/dooked/source/main.cpp +++ b/dooked/source/main.cpp @@ -34,8 +34,14 @@ int main(int argc, char **argv) { app.add_option( "-c,--content-length", cli_args.content_length, "show content lengths that changed more than --content-length"); + app.add_option("--ls", cli_args.last_seen_days, + "show DNS records last seen before today minus DAYS"); + app.add_option("--lsd", cli_args.last_seen_date, + "show DNS records last seen before MM/DD/YYYY or MM-DD-YYYY"); app.add_flag("-d,--include-date", cli_args.include_date, "append present datetime(-ddMMyyyy_hhmmss) in output name"); + app.add_flag("--fs", cli_args.first_seen_log, + "show DNS records first seen in this run"); app.add_flag( "--defer", cli_args.post_http_request, "defers http request until after all DNS requests have been completed"); diff --git a/dooked/source/utils/io_utils.cpp b/dooked/source/utils/io_utils.cpp index a1bd5d3..9ed2127 100644 --- a/dooked/source/utils/io_utils.cpp +++ b/dooked/source/utils/io_utils.cpp @@ -5,7 +5,10 @@ namespace dooked { void to_json(json &j, probe_result_t const &record) { j = json{{"ttl", record.ttl}, {"type", dns_record_type_to_str(record.type)}, - {"info", record.rdata}}; + {"info", record.rdata}, + {"first_seen", record.first_seen}, + {"last_seen", record.last_seen}, + {"seen", record.seen}}; } bool is_text_file(std::string const &file_extension) { From 51ac00471b8a47de8c8797357193e82a0ed8d727 Mon Sep 17 00:00:00 2001 From: xiaoshuai <49056817+xiaoshuai7038@users.noreply.github.com> Date: Wed, 20 May 2026 17:40:11 +0800 Subject: [PATCH 2/2] fix: output dashed DNS history keys --- dooked/source/utils/io_utils.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dooked/source/utils/io_utils.cpp b/dooked/source/utils/io_utils.cpp index 9ed2127..0772911 100644 --- a/dooked/source/utils/io_utils.cpp +++ b/dooked/source/utils/io_utils.cpp @@ -6,8 +6,8 @@ void to_json(json &j, probe_result_t const &record) { j = json{{"ttl", record.ttl}, {"type", dns_record_type_to_str(record.type)}, {"info", record.rdata}, - {"first_seen", record.first_seen}, - {"last_seen", record.last_seen}, + {"first-seen", record.first_seen}, + {"last-seen", record.last_seen}, {"seen", record.seen}}; }