feat: track DNS record history

README.md

@@ -39,3 +39,28 @@ make
 ## Usage
 
 For comprehensive help, use `dooked --help`
+
+### DNS history tracking
+
+JSON output now records DNS history on each `dns_probe` entry:
+
+- `first_seen`: first date the record was observed (`YYYY-MM-DD`)
+- `last_seen`: latest date the record was observed (`YYYY-MM-DD`)
+- `seen`: number of runs where the record was observed
+
+When a previous JSON result is used as input, dooked preserves old records that are
+not seen in the current run. This makes load-balanced targets easier to track,
+because a rotating IP is not lost just because it disappeared from one scan.
+
+Useful flags:
+
+```bash
+# Print records discovered for the first time in this run
+dooked --fs -i domains.txt -o current.json
+
+# Re-scan a previous result and report records not seen in the last 30 days
+dooked --ls 30 -i current.json -o next.json
+
+# Report records last seen before a specific US date
+dooked --lsd 01/31/2026 -i current.json -o next.json
+```

dooked/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.0.0 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.5.0 FATAL_ERROR)
 
 # Project
 get_filename_component(PROJECT_DIR "${CMAKE_CURRENT_SOURCE_DIR}" ABSOLUTE)
@@ -23,8 +23,8 @@ include_directories(${PROJECT_DIR}/json/single_include)
 include_directories(${PROJECT_DIR}/spdlog/include)
 
 
-link_directories(/usr/lib)
-link_libraries(pthread ssl crypto)
+find_package(OpenSSL REQUIRED)
+
 
 # Outputs
 set(OUTPUT_DEBUG ${PROJECT_DIR}/bin)
@@ -102,6 +102,7 @@ source_group("Headers" FILES ${HEADERS_FILES})
 add_executable(${PROJECT_NAME} 
    ${SRC_FILES} ${HEADERS_FILES}
 )
+target_link_libraries(${PROJECT_NAME} PRIVATE pthread OpenSSL::SSL OpenSSL::Crypto)
 
 if(NOT MSVC)
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3")

dooked/include/cli_preprocessor.hpp

@@ -24,7 +24,10 @@ struct cli_args_t {
   int post_http_request{};
   int thread_count{};
   int content_length{-1};
+  int last_seen_days{-1};
   bool include_date{false};
+  bool first_seen_log{false};
+  std::string last_seen_date{};
 };
 
 struct runtime_args_t {
@@ -36,6 +39,8 @@ struct runtime_args_t {
   http_process_e http_request_time_{};
   int thread_count{};
   int content_length{-1};
+  bool first_seen_log{false};
+  std::string last_seen_threshold{};
 };
 
 void run_program(cli_args_t const &cli_args);

dooked/include/utils/exceptions.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <stdexcept>
+#include <string>
 
 namespace dooked {
 

dooked/include/utils/io_utils.hpp

@@ -29,7 +29,10 @@ struct json_data_t {
   int ttl{};
   int http_code{};
   int content_length{};
+  std::string first_seen{};
+  std::string last_seen{};
   dns_record_type_e type{};
+  int seen{1};
 
   static json_data_t serialize(std::string const &d, int const len,
                                int const http_code,
@@ -42,8 +45,35 @@ struct json_data_t {
     data.ttl = json_object["ttl"].get<json::number_integer_t>();
     data.content_length = len;
     data.http_code = http_code;
+    data.first_seen = get_history_string(json_object, "first_seen",
+                                         "first-seen");
+    data.last_seen =
+        get_history_string(json_object, "last_seen", "last-seen");
+    data.seen = get_history_count(json_object);
     return data;
   }
+
+private:
+  static std::string get_history_string(json::object_t const &json_object,
+                                        char const *primary_key,
+                                        char const *legacy_key) {
+    auto iter = json_object.find(primary_key);
+    if (iter == json_object.cend()) {
+      iter = json_object.find(legacy_key);
+    }
+    if (iter != json_object.cend() && iter->second.is_string()) {
+      return iter->second.get<json::string_t>();
+    }
+    return {};
+  }
+
+  static int get_history_count(json::object_t const &json_object) {
+    auto const iter = json_object.find("seen");
+    if (iter != json_object.cend() && iter->second.is_number_integer()) {
+      return iter->second.get<json::number_integer_t>();
+    }
+    return 1;
+  }
 };
 
 struct jd_domain_comparator_t {

dooked/include/utils/probe_result.hpp

@@ -10,8 +10,11 @@ bool case_insensitive_compare(std::string const &, std::string const &);
 
 struct probe_result_t {
   std::string rdata{};
+  std::string first_seen{};
+  std::string last_seen{};
   dns_record_type_e type{}; // RR TYPE (2 octets)
   std::uint32_t ttl{};      // time to live(4 octets)
+  int seen{};
 
   friend bool operator==(probe_result_t const &a, probe_result_t const &b) {
     return case_insensitive_compare(a.rdata, b.rdata) && (a.type == b.type);

dooked/source/cli_preprocessor.cpp

@@ -6,8 +6,13 @@
 #include "utils/string_utils.hpp"
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/thread_pool.hpp>
+#include <cctype>
+#include <ctime>
+#include <map>
 #include <set>
 #include <spdlog/spdlog.h>
+#include <string>
+#include <tuple>
 
 // defined (and assigned to) in main.cpp
 extern bool silent;
@@ -18,6 +23,158 @@ namespace dooked {
 namespace net = boost::asio;
 using namespace fmt::v7::literals;
 
+std::string today_iso_date() {
+  std::string today{};
+  if (!timet_to_string(today, std::time(nullptr), "%Y-%m-%d")) {
+    return {};
+  }
+  return today;
+}
+
+std::string date_days_ago(int const days) {
+  if (days < 0) {
+    return {};
+  }
+  std::time_t threshold = std::time(nullptr) -
+                          static_cast<std::time_t>(days) * 24 * 60 * 60;
+  std::string date{};
+  if (!timet_to_string(date, threshold, "%Y-%m-%d")) {
+    return {};
+  }
+  return date;
+}
+
+std::string parse_us_date(std::string date) {
+  trim(date);
+  if (date.size() != 10 || !std::isdigit(date[0]) || !std::isdigit(date[1]) ||
+      !std::isdigit(date[3]) || !std::isdigit(date[4]) ||
+      !std::isdigit(date[6]) || !std::isdigit(date[7]) ||
+      !std::isdigit(date[8]) || !std::isdigit(date[9]) ||
+      !((date[2] == '/' && date[5] == '/') ||
+        (date[2] == '-' && date[5] == '-'))) {
+    return {};
+  }
+  int const month = std::stoi(date.substr(0, 2));
+  int const day = std::stoi(date.substr(3, 2));
+  int const year = std::stoi(date.substr(6, 4));
+  if (month < 1 || month > 12 || day < 1 || day > 31) {
+    return {};
+  }
+  std::tm tm{};
+  tm.tm_year = year - 1900;
+  tm.tm_mon = month - 1;
+  tm.tm_mday = day;
+  tm.tm_isdst = -1;
+  auto const normalized = std::mktime(&tm);
+  if (normalized == static_cast<std::time_t>(-1) || tm.tm_year != year - 1900 ||
+      tm.tm_mon != month - 1 || tm.tm_mday != day) {
+    return {};
+  }
+  char output[11]{};
+  std::strftime(output, sizeof(output), "%Y-%m-%d", &tm);
+  return std::string{output};
+}
+
+using history_key_t = std::tuple<std::string, dns_record_type_e, std::string>;
+
+history_key_t history_key(std::string const &domain, dns_record_type_e const type,
+                          std::string const &rdata) {
+  return {domain, type, rdata};
+}
+
+history_key_t history_key(json_data_t const &record) {
+  return history_key(record.domain_name, record.type, record.rdata);
+}
+
+history_key_t history_key(std::string const &domain, probe_result_t const &record) {
+  return history_key(domain, record.type, record.rdata);
+}
+
+probe_result_t json_to_probe_result(json_data_t const &record) {
+  probe_result_t result{};
+  result.rdata = record.rdata;
+  result.type = record.type;
+  result.ttl = static_cast<std::uint32_t>(record.ttl);
+  result.first_seen = record.first_seen;
+  result.last_seen = record.last_seen;
+  result.seen = record.seen > 0 ? record.seen : 1;
+  return result;
+}
+
+void log_first_seen(std::string const &domain, probe_result_t const &record) {
+  spdlog::info("[FIRST-SEEN][{}][{}] `{}`", domain,
+               dns_record_type_to_str(record.type), record.rdata);
+}
+
+void log_last_seen(std::string const &domain, probe_result_t const &record) {
+  spdlog::info("[LAST-SEEN][{}][{}] `{}` last seen `{}`", domain,
+               dns_record_type_to_str(record.type), record.rdata,
+               record.last_seen);
+}
+
+void merge_history(map_container_t<probe_result_t> &result_map,
+                   std::optional<std::vector<json_data_t>> const &previous_data,
+                   runtime_args_t const &rt_args) {
+  auto const today = today_iso_date();
+  std::map<history_key_t, json_data_t> previous_by_key{};
+  if (previous_data) {
+    for (auto const &record : *previous_data) {
+      previous_by_key.emplace(history_key(record), record);
+    }
+  }
+
+  std::set<history_key_t> seen_this_run{};
+  for (auto &result_pair : result_map.result()) {
+    for (auto &record : result_pair.second.dns_result_list_) {
+      auto const key = history_key(result_pair.first, record);
+      seen_this_run.insert(key);
+      auto const prev_iter = previous_by_key.find(key);
+      if (prev_iter == previous_by_key.end()) {
+        record.first_seen = today;
+        record.last_seen = today;
+        record.seen = 1;
+        if (rt_args.first_seen_log) {
+          log_first_seen(result_pair.first, record);
+        }
+      } else {
+        auto const &previous = prev_iter->second;
+        record.first_seen = previous.first_seen.empty() ? today : previous.first_seen;
+        record.last_seen = today;
+        record.seen = (previous.seen > 0 ? previous.seen : 1) + 1;
+      }
+    }
+  }
+
+  if (previous_data) {
+    for (auto const &previous : *previous_data) {
+      if (seen_this_run.find(history_key(previous)) != seen_this_run.end()) {
+        continue;
+      }
+      auto preserved = json_to_probe_result(previous);
+      if (preserved.first_seen.empty()) {
+        preserved.first_seen = today;
+      }
+      if (preserved.last_seen.empty()) {
+        preserved.last_seen = preserved.first_seen;
+      }
+      result_map.append(previous.domain_name, preserved);
+      result_map.insert(previous.domain_name, previous.content_length,
+                        previous.http_code);
+    }
+  }
+
+  if (!rt_args.last_seen_threshold.empty()) {
+    for (auto const &result_pair : result_map.cresult()) {
+      for (auto const &record : result_pair.second.dns_result_list_) {
+        if (!record.last_seen.empty() &&
+            record.last_seen < rt_args.last_seen_threshold) {
+          log_last_seen(result_pair.first, record);
+        }
+      }
+    }
+  }
+}
+
 void compare_http_result(int const base_cl, json_data_t const &prev_http_result,
                          http_response_t const &current_result) {
   auto const current_req_cl = current_result.content_length_;
@@ -339,7 +496,7 @@ void start_name_checking(runtime_args_t &&rt_args) {
 
   // if we deferred HTTP/S "probe", now is the time to get to it
   if (deferring) {
-    io_context.reset();
+    io_context.restart();
     thread_pool.emplace(thread_count);
     rt_args.names.emplace(std::move(*deferred_names_));
     for (std::size_t index = 0; index < thread_count; ++index) {
@@ -350,12 +507,7 @@ void start_name_checking(runtime_args_t &&rt_args) {
     }
     thread_pool->join();
   }
-  if (!silent) {
-    spdlog::info("Writing JSON output");
-  }
-  write_json_result(result_map, rt_args);
-
-  // compare old with new result -- only if we had previous record
+  // compare old with new result before preserving missing history records.
   if (rt_args.previous_data) {
     auto &previous_data = *rt_args.previous_data;
 
@@ -373,9 +525,14 @@ void start_name_checking(runtime_args_t &&rt_args) {
                   return std::tie(a.type, a.rdata) < std::tie(b.type, b.rdata);
                 });
     }
-    return compare_results(*rt_args.previous_data, result_map,
-                           rt_args.content_length);
+    compare_results(*rt_args.previous_data, result_map, rt_args.content_length);
+  }
+
+  if (!silent) {
+    spdlog::info("Writing JSON output");
   }
+  merge_history(result_map, rt_args.previous_data, rt_args);
+  write_json_result(result_map, rt_args);
 }
 
 void run_program(cli_args_t const &cli_args) {
@@ -477,6 +634,19 @@ void run_program(cli_args_t const &cli_args) {
       static_cast<http_process_e>(cli_args.post_http_request);
   rt_args.thread_count = cli_args.thread_count;
   rt_args.content_length = cli_args.content_length;
+  rt_args.first_seen_log = cli_args.first_seen_log;
+  if (!cli_args.last_seen_date.empty()) {
+    rt_args.last_seen_threshold = parse_us_date(cli_args.last_seen_date);
+    if (rt_args.last_seen_threshold.empty()) {
+      return spdlog::error("invalid --lsd date `{}`; use MM/DD/YYYY or MM-DD-YYYY",
+                           cli_args.last_seen_date);
+    }
+  } else if (cli_args.last_seen_days >= 0) {
+    rt_args.last_seen_threshold = date_days_ago(cli_args.last_seen_days);
+    if (rt_args.last_seen_threshold.empty()) {
+      return spdlog::error("invalid --ls value `{}`", cli_args.last_seen_days);
+    }
+  }
   return start_name_checking(std::move(rt_args));
 }