From aa8b068f0dde040ea835d6487f9b784d84027602 Mon Sep 17 00:00:00 2001 From: bogdanpatenko Date: Sun, 31 Aug 2025 23:06:43 +0200 Subject: [PATCH 1/5] implement heidpi-logger-cpp --- .dockerignore | 16 +- .github/workflows/docker-publish-consumer.yml | 4 +- .github/workflows/docker-publish-producer.yml | 4 +- Dockerfile.consumer | 53 +++-- README.consumer.md | 12 +- README.md | 16 +- README.producer.md | 12 +- config.yml | 8 +- docker-compose.yml | 2 +- heidpi-logger-cpp/CMakeLists.txt | 51 +++++ heidpi-logger-cpp/include/Config.hpp | 44 ++++ heidpi-logger-cpp/include/EventProcessor.hpp | 20 ++ heidpi-logger-cpp/include/GeoIP.hpp | 26 +++ heidpi-logger-cpp/include/Logger.hpp | 19 ++ heidpi-logger-cpp/include/NDPIClient.hpp | 20 ++ heidpi-logger-cpp/src/Config.cpp | 36 ++++ heidpi-logger-cpp/src/EventProcessor.cpp | 55 +++++ heidpi-logger-cpp/src/GeoIP.cpp | 122 +++++++++++ heidpi-logger-cpp/src/Logger.cpp | 38 ++++ heidpi-logger-cpp/src/NDPIClient.cpp | 66 ++++++ heidpi-logger-cpp/src/main.cpp | 204 ++++++++++++++++++ {heidpi => heidpi-logger-py}/__init__.py | 0 {heidpi => heidpi-logger-py}/heiDPI_env.py | 0 {heidpi => heidpi-logger-py}/heiDPI_logger.py | 0 {heidpi => heidpi-logger-py}/heiDPIsrvd.py | 0 {heidpi => heidpi-logger-py}/schema/README.md | 0 .../schema/daemon_event_schema.json | 0 .../schema/error_event_schema.json | 0 .../schema/flow_event_schema.json | 0 .../schema/geoip2_schema.json | 0 .../schema/packet_event_schema.json | 0 {heidpi => heidpi-logger-py}/version.py | 0 32 files changed, 787 insertions(+), 41 deletions(-) create mode 100644 heidpi-logger-cpp/CMakeLists.txt create mode 100644 heidpi-logger-cpp/include/Config.hpp create mode 100644 heidpi-logger-cpp/include/EventProcessor.hpp create mode 100644 heidpi-logger-cpp/include/GeoIP.hpp create mode 100644 heidpi-logger-cpp/include/Logger.hpp create mode 100644 heidpi-logger-cpp/include/NDPIClient.hpp create mode 100644 heidpi-logger-cpp/src/Config.cpp create mode 100644 heidpi-logger-cpp/src/EventProcessor.cpp create mode 100644 heidpi-logger-cpp/src/GeoIP.cpp create mode 100644 heidpi-logger-cpp/src/Logger.cpp create mode 100644 heidpi-logger-cpp/src/NDPIClient.cpp create mode 100644 heidpi-logger-cpp/src/main.cpp rename {heidpi => heidpi-logger-py}/__init__.py (100%) rename {heidpi => heidpi-logger-py}/heiDPI_env.py (100%) rename {heidpi => heidpi-logger-py}/heiDPI_logger.py (100%) rename {heidpi => heidpi-logger-py}/heiDPIsrvd.py (100%) rename {heidpi => heidpi-logger-py}/schema/README.md (100%) rename {heidpi => heidpi-logger-py}/schema/daemon_event_schema.json (100%) rename {heidpi => heidpi-logger-py}/schema/error_event_schema.json (100%) rename {heidpi => heidpi-logger-py}/schema/flow_event_schema.json (100%) rename {heidpi => heidpi-logger-py}/schema/geoip2_schema.json (100%) rename {heidpi => heidpi-logger-py}/schema/packet_event_schema.json (100%) rename {heidpi => heidpi-logger-py}/version.py (100%) diff --git a/.dockerignore b/.dockerignore index 412c257..1a23d60 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,15 @@ -docker-compose.yml \ No newline at end of file +docker-compose.yml +.git +.github +build +cmake-build-* +**/__pycache__ +**/*.o +**/*.a +**/*.log +tests/ +nDPId/ +heidpi-logger-py/ +heidpi-rust/ +*.swp +.DS_Store \ No newline at end of file diff --git a/.github/workflows/docker-publish-consumer.yml b/.github/workflows/docker-publish-consumer.yml index 1b1d627..d4ecdb9 100644 --- a/.github/workflows/docker-publish-consumer.yml +++ b/.github/workflows/docker-publish-consumer.yml @@ -12,7 +12,7 @@ on: env: REGISTRY: docker.io - IMAGE_NAME: stefan96/heidpi-consumer + IMAGE_NAME: stefan96/heidpi-logger-py-consumer jobs: @@ -84,7 +84,7 @@ jobs: with: context: . push: ${{ github.event_name != 'pull_request' }} - tags: stefan96/heidpi-consumer:latest + tags: stefan96/heidpi-logger-py-consumer:latest labels: ${{ steps.meta.outputs.labels }} file: ./Dockerfile.consumer cache-from: type=gha diff --git a/.github/workflows/docker-publish-producer.yml b/.github/workflows/docker-publish-producer.yml index 01a90b5..d043691 100644 --- a/.github/workflows/docker-publish-producer.yml +++ b/.github/workflows/docker-publish-producer.yml @@ -12,7 +12,7 @@ on: env: REGISTRY: docker.io - IMAGE_NAME: stefan96/heidpi-producer + IMAGE_NAME: stefan96/heidpi-logger-py-producer jobs: @@ -84,7 +84,7 @@ jobs: with: context: . push: ${{ github.event_name != 'pull_request' }} - tags: stefan96/heidpi-producer:latest + tags: stefan96/heidpi-logger-py-producer:latest labels: ${{ steps.meta.outputs.labels }} file: ./Dockerfile.producer cache-from: type=gha diff --git a/Dockerfile.consumer b/Dockerfile.consumer index 693edd6..424006f 100644 --- a/Dockerfile.consumer +++ b/Dockerfile.consumer @@ -1,4 +1,40 @@ -FROM pypy:3.9-slim-bookworm +# ---------- Build stage ---------- +FROM debian:bookworm AS build +ARG CMAKE_BUILD_TYPE=Release + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential cmake pkg-config git ca-certificates \ + # <-- hier dev abhängigkeiten + && rm -rf /var/lib/apt/lists/* + +WORKDIR /src + +COPY heidpi-logger-cpp/ ./heidpi-logger-cpp/ + +# CMake Build +RUN cmake -S heidpi-logger-cpp -B /build -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ + && cmake --build /build --target heidpi-logger-cpp -- -j$(nproc) +# OPTIONAL +# RUN ctest --test-dir /build --output-on-failure || true + + +# ---------- Runtime stage ---------- +FROM debian:bookworm-slim AS runtime + +# minimale Runtime-Libs +RUN apt-get update && apt-get install -y --no-install-recommends \ + libstdc++6 ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Non-root user +RUN useradd -r -u 10001 appuser +WORKDIR /app + +# Configs (aus Repo-Root) +COPY config.yml /app/config.yml + +# Binary aus Build +COPY --from=build /build/heidpi-logger-cpp /usr/local/bin/heidpi-logger-cpp ENV WRITE="/var/log" \ SHOW_FLOW_EVENTS=1 \ @@ -9,16 +45,9 @@ ENV WRITE="/var/log" \ PORT=7000 \ HOST="" -WORKDIR /usr/src/app - -COPY heidpi ./ \ - config.yml ./ \ - LICENSE ./ \ - README.md ./ \ - requirements.txt ./ \ - pyproject.toml ./ +USER appuser -RUN pip install . \ - pip install -r requirements.txt +# Optional Healthcheck, wenn Binary das zukünftig unterstützt +# HEALTHCHECK --interval=30s --timeout=5s CMD ["/usr/local/bin/heidpi-logger-cpp","--healthcheck"] -CMD [ "heiDPI"] +CMD ["--config","/app/config.yml"] diff --git a/README.consumer.md b/README.consumer.md index 200999b..aa7856d 100644 --- a/README.consumer.md +++ b/README.consumer.md @@ -101,15 +101,15 @@ In order to run this container you'll need docker installed. Pull images: ```sh -docker pull stefan96/heidpi-producer:main -docker pull stefan96/heidpi-consumer:main +docker pull stefan96/heidpi-logger-py-producer:main +docker pull stefan96/heidpi-logger-py-consumer:main ``` Run producer and consumer separately from each other using UDP socket: ```sh -docker run -p 127.0.0.1:7000:7000 --net host stefan96/heidpi-producer:main -docker run -e HOST=127.0.0.1 --net host stefan96/heidpi-consumer:main +docker run -p 127.0.0.1:7000:7000 --net host stefan96/heidpi-logger-py-producer:main +docker run -e HOST=127.0.0.1 --net host stefan96/heidpi-logger-py-consumer:main ``` or use the `docker-compose.yml`: @@ -121,8 +121,8 @@ docker-compose up Additionally, you use a UNIX socket: ```sh -docker run -v ${PWD}/heidpi-data:/tmp/ --net host stefan96/heidpi-producer:main -docker run -v ${PWD}/heidpi-data:/tmp/ -v ${PWD}/heidpi-logs:/var/log -e UNIX=/tmp/nDPIsrvd-daemon-distributor.sock --net host stefan96/heidpi-consumer:main +docker run -v ${PWD}/heidpi-logger-py-data:/tmp/ --net host stefan96/heidpi-logger-py-producer:main +docker run -v ${PWD}/heidpi-logger-py-data:/tmp/ -v ${PWD}/heidpi-logger-py-logs:/var/log -e UNIX=/tmp/nDPIsrvd-daemon-distributor.sock --net host stefan96/heidpi-logger-py-consumer:main ``` ## Environment Variables diff --git a/README.md b/README.md index cafb570..8a4a6dc 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -![heiFIP Logo](https://raw.githubusercontent.com/stefanDeveloper/heiDPI/main/assets/heidpi_logo.png?raw=true) +[include](../heiDPI_benchmark/heidpi_logger_cpp_port/include) +[src](../heiDPI_benchmark/heidpi_logger_cpp_port/src) +[CMakeLists.txt](../heiDPI_benchmark/heidpi_logger_cpp_port/CMakeLists.txt)![heiFIP Logo](https://raw.githubusercontent.com/stefanDeveloper/heiDPI/main/assets/heidpi_logo.png?raw=true) -------------------------------------------------------------------------------- @@ -86,15 +88,15 @@ In order to run this container you'll need docker installed. Pull images: ```sh -docker pull stefan96/heidpi-producer:main -docker pull stefan96/heidpi-consumer:main +docker pull stefan96/heidpi-logger-py-producer:main +docker pull stefan96/heidpi-logger-py-consumer:main ``` Run producer and consumer separately from each other using UDP socket: ```sh -docker run -p 127.0.0.1:7000:7000 --net host stefan96/heidpi-producer:main -docker run -e HOST=127.0.0.1 --net host stefan96/heidpi-consumer:main +docker run -p 127.0.0.1:7000:7000 --net host stefan96/heidpi-logger-py-producer:main +docker run -e HOST=127.0.0.1 --net host stefan96/heidpi-logger-py-consumer:main ``` or use the `docker-compose.yml`: @@ -106,8 +108,8 @@ docker-compose up Additionally, you use a UNIX socket: ```sh -docker run -v ${PWD}/heidpi-data:/tmp/ --net host stefan96/heidpi-producer:main -docker run -v ${PWD}/heidpi-data:/tmp/ -v ${PWD}/heidpi-logs:/var/log -e UNIX=/tmp/nDPIsrvd-daemon-distributor.sock --net host stefan96/heidpi-consumer:main +docker run -v ${PWD}/heidpi-logger-py-data:/tmp/ --net host stefan96/heidpi-logger-py-producer:main +docker run -v ${PWD}/heidpi-logger-py-data:/tmp/ -v ${PWD}/heidpi-logger-py-logs:/var/log -e UNIX=/tmp/nDPIsrvd-daemon-distributor.sock --net host stefan96/heidpi-logger-py-consumer:main ``` ## Configuration diff --git a/README.producer.md b/README.producer.md index ca4edc4..0e669fc 100644 --- a/README.producer.md +++ b/README.producer.md @@ -32,15 +32,15 @@ In order to run this container you'll need docker installed. Pull images: ```sh -docker pull stefan96/heidpi-producer:main -docker pull stefan96/heidpi-consumer:main +docker pull stefan96/heidpi-logger-py-producer:main +docker pull stefan96/heidpi-logger-py-consumer:main ``` Run producer and consumer separately from each other using UDP socket: ```sh -docker run -p 127.0.0.1:7000:7000 --net host stefan96/heidpi-producer:main -docker run -e HOST=127.0.0.1 --net host stefan96/heidpi-consumer:main +docker run -p 127.0.0.1:7000:7000 --net host stefan96/heidpi-logger-py-producer:main +docker run -e HOST=127.0.0.1 --net host stefan96/heidpi-logger-py-consumer:main ``` or use the `docker-compose.yml`: @@ -52,8 +52,8 @@ docker-compose up Additionally, you use a UNIX socket: ```sh -docker run -v ${PWD}/heidpi-data:/tmp/ --net host stefan96/heidpi-producer:main -docker run -v ${PWD}/heidpi-data:/tmp/ -v ${PWD}/heidpi-logs:/var/log -e UNIX=/tmp/nDPIsrvd-daemon-distributor.sock --net host stefan96/heidpi-consumer:main +docker run -v ${PWD}/heidpi-logger-py-data:/tmp/ --net host stefan96/heidpi-logger-py-producer:main +docker run -v ${PWD}/heidpi-logger-py-data:/tmp/ -v ${PWD}/heidpi-logger-py-logs:/var/log -e UNIX=/tmp/nDPIsrvd-daemon-distributor.sock --net host stefan96/heidpi-logger-py-consumer:main ``` ## Environment Variables diff --git a/config.yml b/config.yml index 7d63f43..b8c9cfa 100644 --- a/config.yml +++ b/config.yml @@ -28,7 +28,7 @@ flow_event: # - city # - traits # - postal - threads: 4 +# threads: 4 daemon_event: ignore_fields: [] @@ -36,18 +36,18 @@ daemon_event: - init - status filename: daemon_event - threads: 4 +# threads: 4 packet_event: ignore_fields: [] packet_event_name: - packet-flow filename: packet_event - threads: 4 +# threads: 4 error_event: ignore_fields: [] error_event_name: - error-flow filename: error_event - threads: 4 +# threads: 4 diff --git a/docker-compose.yml b/docker-compose.yml index aed9dfe..3bfe125 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,7 +25,7 @@ services: image: stefan96/heidpi-consumer:latest container_name: heidpi_consumer volumes: - - ./heidpi-logs:/var/log/:rw + - ./heidpi-logger-py-logs:/var/log/:rw - ./config.yml:/usr/src/app/config.yml:ro network_mode: host security_opt: diff --git a/heidpi-logger-cpp/CMakeLists.txt b/heidpi-logger-cpp/CMakeLists.txt new file mode 100644 index 0000000..1c8f653 --- /dev/null +++ b/heidpi-logger-cpp/CMakeLists.txt @@ -0,0 +1,51 @@ +cmake_minimum_required(VERSION 3.10) +project(heidpi_cpp VERSION 0.1 LANGUAGES CXX) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_POLICY_VERSION_MINIMUM 3.5) + +include(FetchContent) + +FetchContent_Declare( + yaml-cpp + GIT_REPOSITORY https://github.com/jbeder/yaml-cpp.git + GIT_TAG f732014 +) +FetchContent_MakeAvailable(yaml-cpp) + +FetchContent_Declare( + json + GIT_REPOSITORY https://github.com/nlohmann/json.git + GIT_TAG v3.11.3 +) +FetchContent_MakeAvailable(json) + +FetchContent_Declare( + json-schema-validator + GIT_REPOSITORY https://github.com/pboettch/json-schema-validator.git + GIT_TAG main +) +FetchContent_MakeAvailable(json-schema-validator) + +FetchContent_Declare( + maxminddb + GIT_REPOSITORY https://github.com/maxmind/libmaxminddb.git + GIT_TAG 1.12.2 + CMAKE_ARGS + -DBUILD_SHARED_LIBS=OFF + -DBUILD_TESTING=OFF + -DMAXMINDDB_BUILD_BINARIES=OFF + -DMAXMINDDB_INSTALL=OFF +) +FetchContent_MakeAvailable(maxminddb) + +file(GLOB SOURCES src/*.cpp) +add_executable(heidpi_cpp ${SOURCES}) +target_include_directories(heidpi_cpp PRIVATE include) +target_link_libraries(heidpi_cpp PRIVATE + yaml-cpp + nlohmann_json::nlohmann_json + nlohmann_json_schema_validator + maxminddb::maxminddb +) + diff --git a/heidpi-logger-cpp/include/Config.hpp b/heidpi-logger-cpp/include/Config.hpp new file mode 100644 index 0000000..46fa75a --- /dev/null +++ b/heidpi-logger-cpp/include/Config.hpp @@ -0,0 +1,44 @@ +#pragma once +#include +#include +#include +#include + +/** + * @brief Loads application configuration from a YAML file. + */ +struct LoggingConfig { + std::string level{"INFO"}; + std::string format{"%Y-%m-%dT%H:%M:%S"}; + std::string datefmt{"%Y-%m-%dT%H:%M:%S"}; + std::string filename{}; // optional log file +}; + +struct EventConfig { + std::vector ignore_fields; + std::vector ignore_risks; + std::vector event_names; // empty -> allow all event names + std::string filename{"event"}; + int threads{1}; + // GeoIP configuration (flow events only) + bool geoip_enabled{false}; + std::string geoip_path{}; + std::vector geoip_keys; +}; + +class Config { +public: + explicit Config(const std::string &path); + const LoggingConfig &logging() const { return logging_cfg; } + const EventConfig &flowEvent() const { return flow_cfg; } + const EventConfig &packetEvent() const { return packet_cfg; } + const EventConfig &daemonEvent() const { return daemon_cfg; } + const EventConfig &errorEvent() const { return error_cfg; } +private: + LoggingConfig logging_cfg; + EventConfig flow_cfg; + EventConfig packet_cfg; + EventConfig daemon_cfg; + EventConfig error_cfg; +}; + diff --git a/heidpi-logger-cpp/include/EventProcessor.hpp b/heidpi-logger-cpp/include/EventProcessor.hpp new file mode 100644 index 0000000..7138f0d --- /dev/null +++ b/heidpi-logger-cpp/include/EventProcessor.hpp @@ -0,0 +1,20 @@ +#pragma once +#include +#include "Config.hpp" +#include "GeoIP.hpp" +#include "Logger.hpp" +#include + +/** + * @brief Processes events based on configuration and writes them as JSON lines. + */ +class EventProcessor { +public: + EventProcessor(const EventConfig &cfg, const std::string &outDir); + void process(const nlohmann::json &j); +private: + EventConfig config; + std::string directory; + std::unique_ptr geo; +}; + diff --git a/heidpi-logger-cpp/include/GeoIP.hpp b/heidpi-logger-cpp/include/GeoIP.hpp new file mode 100644 index 0000000..2264c68 --- /dev/null +++ b/heidpi-logger-cpp/include/GeoIP.hpp @@ -0,0 +1,26 @@ +#pragma once +#include +#include +#include +#include + +/** + * @brief Performs GeoIP lookups using a MaxMind DB and enriches events. + */ +class GeoIP { +public: + GeoIP() = default; + GeoIP(const std::string &path, const std::vector &keys); + ~GeoIP(); + + void enrich(const std::string &src_ip, const std::string &dst_ip, + nlohmann::json &out) const; + +private: + nlohmann::json lookup(const std::string &ip) const; + + MMDB_s mmdb{}; + bool loaded{false}; + std::vector keys; +}; + diff --git a/heidpi-logger-cpp/include/Logger.hpp b/heidpi-logger-cpp/include/Logger.hpp new file mode 100644 index 0000000..b3b9ad7 --- /dev/null +++ b/heidpi-logger-cpp/include/Logger.hpp @@ -0,0 +1,19 @@ +#pragma once +#include +#include +#include +#include "Config.hpp" + +/** + * @brief Very small logger writing to stdout and optional file. + */ +class Logger { +public: + static void init(const LoggingConfig &cfg); + static void info(const std::string &msg); + static void error(const std::string &msg); +private: + static std::mutex mtx; + static std::ofstream file; +}; + diff --git a/heidpi-logger-cpp/include/NDPIClient.hpp b/heidpi-logger-cpp/include/NDPIClient.hpp new file mode 100644 index 0000000..98f9254 --- /dev/null +++ b/heidpi-logger-cpp/include/NDPIClient.hpp @@ -0,0 +1,20 @@ +#pragma once +#include +#include +#include + +/** + * @brief Simple client for nDPIsrvd server. + * Messages are length-prefixed JSON blobs. + */ +class NDPIClient { +public: + NDPIClient(); + ~NDPIClient(); + void connectTcp(const std::string &host, unsigned short port); + void connectUnix(const std::string &path); + void loop(const std::function &cb, const std::string &filter=""); +private: + int fd{-1}; +}; + diff --git a/heidpi-logger-cpp/src/Config.cpp b/heidpi-logger-cpp/src/Config.cpp new file mode 100644 index 0000000..d5be9d0 --- /dev/null +++ b/heidpi-logger-cpp/src/Config.cpp @@ -0,0 +1,36 @@ +#include "Config.hpp" + +Config::Config(const std::string &path) { + YAML::Node config = YAML::LoadFile(path); + auto logNode = config["logging"]; + if (logNode) { + logging_cfg.level = logNode["level"].as("INFO"); + logging_cfg.format = logNode["format"].as("%Y-%m-%dT%H:%M:%S"); + logging_cfg.datefmt = logNode["datefmt"].as("%Y-%m-%dT%H:%M:%S"); + if (logNode["filename"]) logging_cfg.filename = logNode["filename"].as(); + } + + auto parseEvent = [](const YAML::Node &node, EventConfig &cfg) { + if (!node) return; + if (node["ignore_fields"]) cfg.ignore_fields = node["ignore_fields"].as>(); + if (node["ignore_risks"]) cfg.ignore_risks = node["ignore_risks"].as>(); + if (node["flow_event_name"]) cfg.event_names = node["flow_event_name"].as>(); + if (node["packet_event_name"]) cfg.event_names = node["packet_event_name"].as>(); + if (node["daemon_event_name"]) cfg.event_names = node["daemon_event_name"].as>(); + if (node["error_event_name"]) cfg.event_names = node["error_event_name"].as>(); + if (node["filename"]) cfg.filename = node["filename"].as(); + if (node["threads"]) cfg.threads = node["threads"].as(); + if (node["geoip2_city"]) { + auto geo = node["geoip2_city"]; + cfg.geoip_enabled = geo["enabled"].as(false); + if (geo["filepath"]) cfg.geoip_path = geo["filepath"].as(); + if (geo["keys"]) cfg.geoip_keys = geo["keys"].as>(); + } + }; + + parseEvent(config["flow_event"], flow_cfg); + parseEvent(config["packet_event"], packet_cfg); + parseEvent(config["daemon_event"], daemon_cfg); + parseEvent(config["error_event"], error_cfg); +} + diff --git a/heidpi-logger-cpp/src/EventProcessor.cpp b/heidpi-logger-cpp/src/EventProcessor.cpp new file mode 100644 index 0000000..9662b23 --- /dev/null +++ b/heidpi-logger-cpp/src/EventProcessor.cpp @@ -0,0 +1,55 @@ +#include "EventProcessor.hpp" +#include +#include +#include +#include +#include + +EventProcessor::EventProcessor(const EventConfig &cfg, const std::string &outDir) + : config(cfg), directory(outDir) { + if (cfg.geoip_enabled && !cfg.geoip_path.empty()) { + geo = std::make_unique(cfg.geoip_path, cfg.geoip_keys); + } else { + // optional, aber hilfreich zur Diagnose: + Logger::info(std::string("GeoIP disabled for '") + cfg.filename + + "' (enabled=" + (cfg.geoip_enabled ? "true" : "false") + + ", path=" + (cfg.geoip_path.empty() ? "" : cfg.geoip_path) + ")"); + } +} + +static std::string nowTs() { + auto now = std::chrono::system_clock::now(); + std::time_t tt = std::chrono::system_clock::to_time_t(now); + std::tm tm = *std::localtime(&tt); + char buf[64]; + std::strftime(buf, sizeof(buf), "%FT%T", &tm); + return std::string(buf); +} + +void EventProcessor::process(const nlohmann::json &j) { + nlohmann::json out = j; + out["timestamp"] = nowTs(); + + if (geo) { // statt config.geoip_enabled + std::string src = j.value("src_ip", ""); + std::string dst = j.value("dst_ip", ""); + geo->enrich(src, dst, out); + } + for (const auto &field : config.ignore_fields) { + out.erase(field); + } + if (!config.ignore_risks.empty() && out.contains("ndpi") && out["ndpi"].contains("flow_risk")) { + for (const auto &risk : config.ignore_risks) { + out["ndpi"]["flow_risk"].erase(risk); + } + } + std::filesystem::create_directories(directory); + auto path = std::filesystem::path(directory) / (config.filename + ".json"); + std::ofstream ofs(path, std::ios::app); + if (!ofs.is_open()) { + Logger::error("Failed to open output file: " + path.string()); + return; + } + ofs << out.dump() << std::endl; +} + diff --git a/heidpi-logger-cpp/src/GeoIP.cpp b/heidpi-logger-cpp/src/GeoIP.cpp new file mode 100644 index 0000000..80944a3 --- /dev/null +++ b/heidpi-logger-cpp/src/GeoIP.cpp @@ -0,0 +1,122 @@ +#include "GeoIP.hpp" +#include "Logger.hpp" +#include + +namespace { +nlohmann::json entryToJson(const MMDB_s &db, const MMDB_entry_data_s &entry) { + switch (entry.type) { + case MMDB_DATA_TYPE_UTF8_STRING: + return std::string(entry.utf8_string, entry.data_size); + case MMDB_DATA_TYPE_DOUBLE: + return entry.double_value; + case MMDB_DATA_TYPE_FLOAT: + return entry.float_value; + case MMDB_DATA_TYPE_UINT16: + return entry.uint16; + case MMDB_DATA_TYPE_UINT32: + return entry.uint32; + case MMDB_DATA_TYPE_INT32: + return entry.int32; + case MMDB_DATA_TYPE_UINT64: + return entry.uint64; + case MMDB_DATA_TYPE_BOOLEAN: + return static_cast(entry.boolean); + case MMDB_DATA_TYPE_MAP: { + MMDB_entry_s sub{&db, entry.offset}; + MMDB_entry_data_list_s *list = nullptr; + if (MMDB_get_entry_data_list(&sub, &list) == MMDB_SUCCESS && list) { + nlohmann::json obj = nlohmann::json::object(); + MMDB_entry_data_list_s *ptr = list; + while (ptr && ptr->next) { + auto key = ptr->entry_data; + ptr = ptr->next; + auto val = ptr->entry_data; + ptr = ptr->next; + if (key.type != MMDB_DATA_TYPE_UTF8_STRING) continue; + std::string k(key.utf8_string, key.data_size); + obj[k] = entryToJson(db, val); + } + MMDB_free_entry_data_list(list); + return obj; + } + break; + } + case MMDB_DATA_TYPE_ARRAY: { + MMDB_entry_s sub{&db, entry.offset}; + MMDB_entry_data_list_s *list = nullptr; + if (MMDB_get_entry_data_list(&sub, &list) == MMDB_SUCCESS && list) { + nlohmann::json arr = nlohmann::json::array(); + MMDB_entry_data_list_s *ptr = list; + while (ptr) { + arr.push_back(entryToJson(db, ptr->entry_data)); + ptr = ptr->next; + } + MMDB_free_entry_data_list(list); + return arr; + } + break; + } + default: + break; + } + return {}; +} +} // namespace + +GeoIP::GeoIP(const std::string &path, const std::vector &k) + : keys(k) { + int status = MMDB_open(path.c_str(), MMDB_MODE_MMAP, &mmdb); + if (status != MMDB_SUCCESS) { + Logger::error(std::string("GeoIP open failed: ") + path + " " + MMDB_strerror(status)); + loaded = false; + } else { + loaded = true; + } +} + +GeoIP::~GeoIP() { + if (loaded) { + MMDB_close(&mmdb); + } +} + +nlohmann::json GeoIP::lookup(const std::string &ip) const { + nlohmann::json result; + if (!loaded || ip.empty()) return result; + int gai_error = 0, mmdb_error = 0; + MMDB_lookup_result_s res = MMDB_lookup_string(&mmdb, ip.c_str(), &gai_error, &mmdb_error); + if (gai_error != 0 || mmdb_error != MMDB_SUCCESS || !res.found_entry) { + return result; + } + for (const auto &key : keys) { + std::vector parts; + std::stringstream ss(key); + std::string part; + while (std::getline(ss, part, '.')) parts.push_back(part); + std::vector path; + for (auto &p : parts) path.push_back(p.c_str()); + path.push_back(nullptr); + MMDB_entry_data_s entry{}; + int status = MMDB_aget_value(&res.entry, &entry, path.data()); + if (status != MMDB_SUCCESS || !entry.has_data) continue; + const std::string &field = parts.back(); + nlohmann::json value = entryToJson(mmdb, entry); + if (!value.is_null() && !(value.is_object() && value.empty())) { + result[field] = value; + } + } + return result; +} + +void GeoIP::enrich(const std::string &src_ip, const std::string &dst_ip, + nlohmann::json &out) const { + if (!loaded) return; + auto src = lookup(src_ip); + if (!src.empty()) { + out["src_geoip2_city"] = src; + } + auto dst = lookup(dst_ip); + if (!dst.empty()) { + out["dst_geoip2_city"] = dst; + } +} \ No newline at end of file diff --git a/heidpi-logger-cpp/src/Logger.cpp b/heidpi-logger-cpp/src/Logger.cpp new file mode 100644 index 0000000..cf17ebc --- /dev/null +++ b/heidpi-logger-cpp/src/Logger.cpp @@ -0,0 +1,38 @@ +#include "Logger.hpp" +#include +#include +#include +#include + +std::mutex Logger::mtx; +std::ofstream Logger::file; + +static std::string timestamp() { + auto now = std::chrono::system_clock::now(); + std::time_t tt = std::chrono::system_clock::to_time_t(now); + std::tm tm = *std::localtime(&tt); + char buf[64]; + std::strftime(buf, sizeof(buf), "%FT%T", &tm); + return std::string(buf); +} + +void Logger::init(const LoggingConfig &cfg) { + if (!cfg.filename.empty()) { + file.open(cfg.filename, std::ios::app); + } +} + +void Logger::info(const std::string &msg) { + std::lock_guard lock(mtx); + std::string line = timestamp() + " INFO: " + msg + "\n"; + std::cout << line; + if (file.is_open()) file << line; +} + +void Logger::error(const std::string &msg) { + std::lock_guard lock(mtx); + std::string line = timestamp() + " ERROR: " + msg + "\n"; + std::cerr << line; + if (file.is_open()) file << line; +} + diff --git a/heidpi-logger-cpp/src/NDPIClient.cpp b/heidpi-logger-cpp/src/NDPIClient.cpp new file mode 100644 index 0000000..20cd60b --- /dev/null +++ b/heidpi-logger-cpp/src/NDPIClient.cpp @@ -0,0 +1,66 @@ +#include "NDPIClient.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +NDPIClient::NDPIClient() {} +NDPIClient::~NDPIClient() { if (fd >= 0) ::close(fd); } + +void NDPIClient::connectTcp(const std::string &host, unsigned short port) { + fd = ::socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) throw std::runtime_error("socket"); + sockaddr_in addr{}; + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + ::inet_pton(AF_INET, host.c_str(), &addr.sin_addr); + if (::connect(fd, (sockaddr*)&addr, sizeof(addr)) < 0) + throw std::runtime_error("connect"); +} + +void NDPIClient::connectUnix(const std::string &path) { + fd = ::socket(AF_UNIX, SOCK_STREAM, 0); + if (fd < 0) throw std::runtime_error("socket"); + sockaddr_un addr{}; + addr.sun_family = AF_UNIX; + std::strncpy(addr.sun_path, path.c_str(), sizeof(addr.sun_path)-1); + if (::connect(fd, (sockaddr*)&addr, sizeof(addr)) < 0) + throw std::runtime_error("connect"); +} + +void NDPIClient::loop(const std::function &cb, const std::string &filter) { + // send optional filter expression before starting the receive loop + if (!filter.empty()) { + std::ostringstream ss; + ss << std::setw(6) << std::setfill('0') << filter.size() << filter; + std::string msg = ss.str(); + ssize_t sent = ::send(fd, msg.c_str(), msg.size(), 0); + if (sent < 0 || static_cast(sent) != msg.size()) + throw std::runtime_error("send"); + } + + while (true) { + char lenbuf[6]; + // Der Generator verwendet immer fünf Ziffern für die Länge + ssize_t n = ::recv(fd, lenbuf, 5, MSG_WAITALL); + if (n <= 0) break; + lenbuf[5] = '\0'; + size_t len = std::stoul(lenbuf); + // anschließend die JSON‑Nutzlast lesen (inklusive '{') + std::string payload(len, '\0'); + n = ::recv(fd, payload.data(), len, MSG_WAITALL); + if (n <= 0) break; + try { + auto j = nlohmann::json::parse(payload); + cb(j); + } catch (...) { + // JSON‑Fehler ignorieren + } + } +} + diff --git a/heidpi-logger-cpp/src/main.cpp b/heidpi-logger-cpp/src/main.cpp new file mode 100644 index 0000000..6eb63d6 --- /dev/null +++ b/heidpi-logger-cpp/src/main.cpp @@ -0,0 +1,204 @@ +#include "Config.hpp" +#include "Logger.hpp" +#include "NDPIClient.hpp" +#include "EventProcessor.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +struct CLIOptions { + std::string host{"127.0.0.1"}; + std::string unix_path{}; + int port{7000}; + std::string write_path{"/var/log"}; + std::string config_path{"config.yml"}; + std::string filter{}; + bool show_daemon{false}; + bool show_packet{false}; + bool show_error{false}; + bool show_flow{false}; +}; + +static std::string envOrDefault(const char *env, const std::string &def) { + const char *v = std::getenv(env); + return v ? std::string(v) : def; +} + +CLIOptions parse(int argc, char **argv) { + CLIOptions o; + o.host = envOrDefault("HOST", o.host); + o.unix_path = envOrDefault("UNIX", o.unix_path); + o.port = std::stoi(envOrDefault("PORT", std::to_string(o.port))); + o.write_path = envOrDefault("WRITE", o.write_path); + o.config_path = envOrDefault("CONFIG", o.config_path); + o.filter = envOrDefault("FILTER", o.filter); + o.show_daemon = envOrDefault("SHOW_DAEMON_EVENTS", "0") == "1"; + o.show_packet = envOrDefault("SHOW_PACKET_EVENTS", "0") == "1"; + o.show_error = envOrDefault("SHOW_ERROR_EVENTS", "0") == "1"; + o.show_flow = envOrDefault("SHOW_FLOW_EVENTS", "0") == "1"; + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + auto next = [&](int &i){ return std::string(argv[++i]); }; + if (a == "--host" && i+1 < argc) o.host = next(i); + else if (a == "--unix" && i+1 < argc) o.unix_path = next(i); + else if (a == "--port" && i+1 < argc) o.port = std::stoi(next(i)); + else if (a == "--write" && i+1 < argc) o.write_path = next(i); + else if (a == "--config" && i+1 < argc) o.config_path = next(i); + else if (a == "--filter" && i+1 < argc) o.filter = next(i); + else if (a == "--show-daemon-events") o.show_daemon = !o.show_daemon; + else if (a == "--show-packet-events") o.show_packet = !o.show_packet; + else if (a == "--show-error-events") o.show_error = !o.show_error; + else if (a == "--show-flow-events") o.show_flow = !o.show_flow; + else if (a == "--help" || a == "-h") { + std::cout << "Usage: " << argv[0] << " [options]\n" + << " --host Set host\n" + << " --unix Set unix socket path\n" + << " --port Set port\n" + << " --write Set write path\n" + << " --config Set config path\n" + << " --filter Filter expression\n" + << " --show-daemon-events Toggle daemon events\n" + << " --show-packet-events Toggle packet events\n" + << " --show-error-events Toggle error events\n" + << " --show-flow-events Toggle flow events\n" + << " -h, --help Show this help message\n"; + std::exit(0); + } + } + return o; +} + +struct Worker { + std::string eventKey; + EventConfig config; + EventProcessor processor; + Worker(const std::string &k, const EventConfig &c, const std::string &dir) + : eventKey(k), config(c), processor(c, dir) {} +}; + +int main(int argc, char **argv) { + // Help kurz vorher abfangen (wie im Original) + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + if (a == "-h" || a == "--help") { + std::string name = std::filesystem::path(argv[0]).filename(); + std::cout << "usage: " << name + << " [-h] [--host HOST | --unix UNIX] [--port PORT] [--write WRITE]\n" + " [--config CONFIG] [--filter FILTER]\n" + " [--show-daemon-events]\n" + " [--show-packet-events]\n" + " [--show-error-events]\n" + " [--show-flow-events]\n"; + return 0; + } + } + + CLIOptions opts = parse(argc, argv); + Config cfg(opts.config_path); + Logger::init(cfg.logging()); + + std::vector workers; + workers.reserve(4); + if (opts.show_flow) workers.emplace_back("flow_event_name", cfg.flowEvent(), opts.write_path); + if (opts.show_packet) workers.emplace_back("packet_event_name", cfg.packetEvent(), opts.write_path); + if (opts.show_daemon) workers.emplace_back("daemon_event_name", cfg.daemonEvent(), opts.write_path); + if (opts.show_error) workers.emplace_back("error_event_name", cfg.errorEvent(), opts.write_path); + + if (workers.empty()) { + Logger::error("No event types enabled. Use --show-*_events flags to enable processing."); + return 1; + } + + NDPIClient client; + try { + if (!opts.unix_path.empty()) + client.connectUnix(opts.unix_path); + else + client.connectTcp(opts.host, static_cast(opts.port)); // FIX + } catch (const std::exception &ex) { + Logger::error(std::string("Failed to connect: ") + ex.what()); + return 1; + } + + // ------------------------- + // NEU: FIFO-Queue + Dispatcher + // ------------------------- + std::queue eventQueue; // FIX: Typ-Parameter + std::mutex mtx; + std::condition_variable cv; + std::atomic done{false}; + + // Dispatcher-Thread (arbeitet streng nacheinander ab) + std::thread dispatcher([&]{ + while (true) { + nlohmann::json event; + { + std::unique_lock lk(mtx); + cv.wait(lk, [&]{ return done || !eventQueue.empty(); }); + if (done && eventQueue.empty()) break; + event = std::move(eventQueue.front()); + eventQueue.pop(); + } + + // Event-Typ ermitteln & Namen lesen + std::string key; + std::string name; + if (event.contains("flow_event_name")) { + key = "flow_event_name"; + name = event["flow_event_name"].get(); // FIX: get() + } else if (event.contains("packet_event_name")) { + key = "packet_event_name"; + name = event["packet_event_name"].get(); // FIX: get() + } else if (event.contains("daemon_event_name")) { + key = "daemon_event_name"; + name = event["daemon_event_name"].get(); // FIX: get() + } else if (event.contains("error_event_name")) { + key = "error_event_name"; + name = event["error_event_name"].get(); // FIX: get() + } else { + Logger::info("Received unknown event: missing event name"); + continue; + } + + bool handled = false; + for (auto &w : workers) { + if (w.eventKey != key) continue; + w.processor.process(event); + handled = true; + } + if (!handled) { + Logger::info("No handler enabled for event '" + name + "' of type " + key); + } + } + }); + + // Reader: liest nonstop und füttert nur die Queue + client.loop([&](const nlohmann::json &j) { + { + std::lock_guard lk(mtx); + eventQueue.push(j); + } + cv.notify_one(); + }, opts.filter); + + // Nach Abbruch der Verbindung: Queue leeren lassen und Thread beenden + { + std::lock_guard lk(mtx); + done = true; + } + cv.notify_all(); + dispatcher.join(); + + return 0; +} diff --git a/heidpi/__init__.py b/heidpi-logger-py/__init__.py similarity index 100% rename from heidpi/__init__.py rename to heidpi-logger-py/__init__.py diff --git a/heidpi/heiDPI_env.py b/heidpi-logger-py/heiDPI_env.py similarity index 100% rename from heidpi/heiDPI_env.py rename to heidpi-logger-py/heiDPI_env.py diff --git a/heidpi/heiDPI_logger.py b/heidpi-logger-py/heiDPI_logger.py similarity index 100% rename from heidpi/heiDPI_logger.py rename to heidpi-logger-py/heiDPI_logger.py diff --git a/heidpi/heiDPIsrvd.py b/heidpi-logger-py/heiDPIsrvd.py similarity index 100% rename from heidpi/heiDPIsrvd.py rename to heidpi-logger-py/heiDPIsrvd.py diff --git a/heidpi/schema/README.md b/heidpi-logger-py/schema/README.md similarity index 100% rename from heidpi/schema/README.md rename to heidpi-logger-py/schema/README.md diff --git a/heidpi/schema/daemon_event_schema.json b/heidpi-logger-py/schema/daemon_event_schema.json similarity index 100% rename from heidpi/schema/daemon_event_schema.json rename to heidpi-logger-py/schema/daemon_event_schema.json diff --git a/heidpi/schema/error_event_schema.json b/heidpi-logger-py/schema/error_event_schema.json similarity index 100% rename from heidpi/schema/error_event_schema.json rename to heidpi-logger-py/schema/error_event_schema.json diff --git a/heidpi/schema/flow_event_schema.json b/heidpi-logger-py/schema/flow_event_schema.json similarity index 100% rename from heidpi/schema/flow_event_schema.json rename to heidpi-logger-py/schema/flow_event_schema.json diff --git a/heidpi/schema/geoip2_schema.json b/heidpi-logger-py/schema/geoip2_schema.json similarity index 100% rename from heidpi/schema/geoip2_schema.json rename to heidpi-logger-py/schema/geoip2_schema.json diff --git a/heidpi/schema/packet_event_schema.json b/heidpi-logger-py/schema/packet_event_schema.json similarity index 100% rename from heidpi/schema/packet_event_schema.json rename to heidpi-logger-py/schema/packet_event_schema.json diff --git a/heidpi/version.py b/heidpi-logger-py/version.py similarity index 100% rename from heidpi/version.py rename to heidpi-logger-py/version.py From 23d5b8983d66d4295775fc6af5d0e29ef1b33d2f Mon Sep 17 00:00:00 2001 From: bogdanpatenko Date: Mon, 1 Sep 2025 00:18:36 +0200 Subject: [PATCH 2/5] implement heidpi-logger-cpp --- Dockerfile.consumer | 28 ++++++++++++---------------- config.yml | 2 +- docker-compose.yml | 10 +++++++--- heidpi-logger-cpp/CMakeLists.txt | 5 +++++ 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/Dockerfile.consumer b/Dockerfile.consumer index 424006f..6a97f6e 100644 --- a/Dockerfile.consumer +++ b/Dockerfile.consumer @@ -4,41 +4,39 @@ ARG CMAKE_BUILD_TYPE=Release RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential cmake pkg-config git ca-certificates \ - # <-- hier dev abhängigkeiten && rm -rf /var/lib/apt/lists/* WORKDIR /src - COPY heidpi-logger-cpp/ ./heidpi-logger-cpp/ -# CMake Build -RUN cmake -S heidpi-logger-cpp -B /build -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - && cmake --build /build --target heidpi-logger-cpp -- -j$(nproc) -# OPTIONAL -# RUN ctest --test-dir /build --output-on-failure || true - +RUN cmake -S heidpi-logger-cpp -B /build \ + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ + -DBUILD_TESTING=OFF \ + -DMAXMINDDB_BUILD_BINARIES=OFF \ + -DMAXMINDDB_TOOLS=OFF \ + -DMAXMINDDB_INSTALL=OFF \ + && cmake --build /build --target heidpi_cpp -- -j"$(nproc)" \ + && strip /build/heidpi_cpp || true # ---------- Runtime stage ---------- FROM debian:bookworm-slim AS runtime -# minimale Runtime-Libs RUN apt-get update && apt-get install -y --no-install-recommends \ libstdc++6 ca-certificates \ && rm -rf /var/lib/apt/lists/* -# Non-root user +# non-root RUN useradd -r -u 10001 appuser WORKDIR /app # Configs (aus Repo-Root) COPY config.yml /app/config.yml -# Binary aus Build -COPY --from=build /build/heidpi-logger-cpp /usr/local/bin/heidpi-logger-cpp +COPY --from=build /build/heidpi_cpp /usr/local/bin/app ENV WRITE="/var/log" \ SHOW_FLOW_EVENTS=1 \ - SHOW_PACKET_EVENTS=0 \ + SHOW_PACKET_EVENTS=1 \ SHOW_ERROR_EVENTS=0 \ SHOW_DAEMON_EVENTS=0 \ UNIX="" \ @@ -47,7 +45,5 @@ ENV WRITE="/var/log" \ USER appuser -# Optional Healthcheck, wenn Binary das zukünftig unterstützt -# HEALTHCHECK --interval=30s --timeout=5s CMD ["/usr/local/bin/heidpi-logger-cpp","--healthcheck"] - +ENTRYPOINT ["/usr/local/bin/app"] CMD ["--config","/app/config.yml"] diff --git a/config.yml b/config.yml index b8c9cfa..ee8cdee 100644 --- a/config.yml +++ b/config.yml @@ -1,7 +1,7 @@ appName: heiDPI logging: - level: INFO + level: ERROR encoding: utf-8 format: "%(asctime)s %(levelname)s:%(message)s" datefmt: "%Y-%m-%dT%I:%M:%S" diff --git a/docker-compose.yml b/docker-compose.yml index 3bfe125..36184f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: network_mode: host security_opt: - no-new-privileges - pids_limit: 8192 +# pids_limit: 8192 restart: on-failure:5 deploy: resources: @@ -22,7 +22,10 @@ services: - SSL_SHA1_URL=https://sslbl.abuse.ch/blacklist/sslblacklist.csv consumer: - image: stefan96/heidpi-consumer:latest + build: + context: . + dockerfile: Dockerfile.consumer + #image: stefan96/heidpi-consumer:latest container_name: heidpi_consumer volumes: - ./heidpi-logger-py-logs:/var/log/:rw @@ -30,7 +33,7 @@ services: network_mode: host security_opt: - no-new-privileges - pids_limit: 8192 +# pids_limit: 8192 restart: on-failure:5 deploy: resources: @@ -41,3 +44,4 @@ services: - producer environment: - SHOW_DAEMON_EVENTS=1 + - SHOW_PACKET_EVENTS=1 diff --git a/heidpi-logger-cpp/CMakeLists.txt b/heidpi-logger-cpp/CMakeLists.txt index 1c8f653..7d81f51 100644 --- a/heidpi-logger-cpp/CMakeLists.txt +++ b/heidpi-logger-cpp/CMakeLists.txt @@ -49,3 +49,8 @@ target_link_libraries(heidpi_cpp PRIVATE maxminddb::maxminddb ) +include(GNUInstallDirs) + +install(TARGETS heidpi_cpp + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + From f9e0aaff11996823bbc43eef5459c8b7cc6f22d7 Mon Sep 17 00:00:00 2001 From: bogdanpatenko Date: Mon, 1 Sep 2025 09:08:26 +0200 Subject: [PATCH 3/5] show all Events by default --- docker-compose.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 36184f5..a5eada6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,7 @@ services: deploy: resources: limits: - cpus: '5' + cpus: '1' memory: 32G environment: - HOSTNAME=test @@ -25,10 +25,10 @@ services: build: context: . dockerfile: Dockerfile.consumer - #image: stefan96/heidpi-consumer:latest + #image: stefan96/heidpi-consumer:latest container_name: heidpi_consumer volumes: - - ./heidpi-logger-py-logs:/var/log/:rw + - ./heidpi-logs:/var/log/:rw - ./config.yml:/usr/src/app/config.yml:ro network_mode: host security_opt: @@ -45,3 +45,5 @@ services: environment: - SHOW_DAEMON_EVENTS=1 - SHOW_PACKET_EVENTS=1 + - SHOW_ERROR_EVENTS=1 + - SHOW_FLOW_EVENTS=1 From 78dad690084057ed2a58980688f7b4f328a9a164 Mon Sep 17 00:00:00 2001 From: bogdanpatenko Date: Mon, 1 Sep 2025 16:40:15 +0200 Subject: [PATCH 4/5] delete heidpi_logger python version --- Dockerfile.consumer | 4 +- heidpi-logger-py/__init__.py | 18 - heidpi-logger-py/heiDPI_env.py | 27 - heidpi-logger-py/heiDPI_logger.py | 307 ----- heidpi-logger-py/heiDPIsrvd.py | 493 ------- heidpi-logger-py/schema/README.md | 5 - .../schema/daemon_event_schema.json | 166 --- .../schema/error_event_schema.json | 186 --- .../schema/flow_event_schema.json | 472 ------- heidpi-logger-py/schema/geoip2_schema.json | 302 ----- .../schema/packet_event_schema.json | 116 -- heidpi-logger-py/version.py | 8 - .../CMakeLists.txt | 0 .../include/Config.hpp | 0 .../include/EventProcessor.hpp | 0 .../include/GeoIP.hpp | 0 .../include/Logger.hpp | 0 .../include/NDPIClient.hpp | 0 .../src/Config.cpp | 0 .../src/EventProcessor.cpp | 0 .../src/GeoIP.cpp | 0 .../src/Logger.cpp | 0 .../src/NDPIClient.cpp | 0 .../src/main.cpp | 0 heidpi-rust/Cargo.lock | 1180 ----------------- heidpi-rust/Cargo.toml | 26 - heidpi-rust/Dockerfile | 12 - heidpi-rust/src/cli.rs | 171 --- heidpi-rust/src/config.rs | 66 - heidpi-rust/src/geoip.rs | 61 - heidpi-rust/src/logging.rs | 45 - heidpi-rust/src/main.rs | 28 - heidpi-rust/src/process.rs | 69 - heidpi-rust/src/schema/README.md | 5 - .../src/schema/daemon_event_schema.json | 166 --- .../src/schema/error_event_schema.json | 186 --- heidpi-rust/src/schema/flow_event_schema.json | 472 ------- heidpi-rust/src/schema/geoip2_schema.json | 302 ----- .../src/schema/packet_event_schema.json | 116 -- heidpi-rust/src/stream.rs | 132 -- pyproject.toml | 42 - shell.nix | 57 - 42 files changed, 2 insertions(+), 5238 deletions(-) delete mode 100644 heidpi-logger-py/__init__.py delete mode 100644 heidpi-logger-py/heiDPI_env.py delete mode 100644 heidpi-logger-py/heiDPI_logger.py delete mode 100644 heidpi-logger-py/heiDPIsrvd.py delete mode 100644 heidpi-logger-py/schema/README.md delete mode 100644 heidpi-logger-py/schema/daemon_event_schema.json delete mode 100644 heidpi-logger-py/schema/error_event_schema.json delete mode 100644 heidpi-logger-py/schema/flow_event_schema.json delete mode 100644 heidpi-logger-py/schema/geoip2_schema.json delete mode 100644 heidpi-logger-py/schema/packet_event_schema.json delete mode 100644 heidpi-logger-py/version.py rename {heidpi-logger-cpp => heidpi-logger}/CMakeLists.txt (100%) rename {heidpi-logger-cpp => heidpi-logger}/include/Config.hpp (100%) rename {heidpi-logger-cpp => heidpi-logger}/include/EventProcessor.hpp (100%) rename {heidpi-logger-cpp => heidpi-logger}/include/GeoIP.hpp (100%) rename {heidpi-logger-cpp => heidpi-logger}/include/Logger.hpp (100%) rename {heidpi-logger-cpp => heidpi-logger}/include/NDPIClient.hpp (100%) rename {heidpi-logger-cpp => heidpi-logger}/src/Config.cpp (100%) rename {heidpi-logger-cpp => heidpi-logger}/src/EventProcessor.cpp (100%) rename {heidpi-logger-cpp => heidpi-logger}/src/GeoIP.cpp (100%) rename {heidpi-logger-cpp => heidpi-logger}/src/Logger.cpp (100%) rename {heidpi-logger-cpp => heidpi-logger}/src/NDPIClient.cpp (100%) rename {heidpi-logger-cpp => heidpi-logger}/src/main.cpp (100%) delete mode 100644 heidpi-rust/Cargo.lock delete mode 100644 heidpi-rust/Cargo.toml delete mode 100644 heidpi-rust/Dockerfile delete mode 100644 heidpi-rust/src/cli.rs delete mode 100644 heidpi-rust/src/config.rs delete mode 100644 heidpi-rust/src/geoip.rs delete mode 100644 heidpi-rust/src/logging.rs delete mode 100644 heidpi-rust/src/main.rs delete mode 100644 heidpi-rust/src/process.rs delete mode 100644 heidpi-rust/src/schema/README.md delete mode 100644 heidpi-rust/src/schema/daemon_event_schema.json delete mode 100644 heidpi-rust/src/schema/error_event_schema.json delete mode 100644 heidpi-rust/src/schema/flow_event_schema.json delete mode 100644 heidpi-rust/src/schema/geoip2_schema.json delete mode 100644 heidpi-rust/src/schema/packet_event_schema.json delete mode 100644 heidpi-rust/src/stream.rs delete mode 100644 pyproject.toml delete mode 100644 shell.nix diff --git a/Dockerfile.consumer b/Dockerfile.consumer index 6a97f6e..787860a 100644 --- a/Dockerfile.consumer +++ b/Dockerfile.consumer @@ -7,9 +7,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* WORKDIR /src -COPY heidpi-logger-cpp/ ./heidpi-logger-cpp/ +COPY heidpi-logger/ ./heidpi-logger/ -RUN cmake -S heidpi-logger-cpp -B /build \ +RUN cmake -S heidpi-logger -B /build \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DBUILD_TESTING=OFF \ -DMAXMINDDB_BUILD_BINARIES=OFF \ diff --git a/heidpi-logger-py/__init__.py b/heidpi-logger-py/__init__.py deleted file mode 100644 index 66f4025..0000000 --- a/heidpi-logger-py/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -import os -import confuse -import logging - -from confuse.core import ConfigView - - -class App: - __conf = None - def __init__(self, path) -> None: - source = confuse.YamlSource(path) - App.__conf = confuse.RootView([source]) - - logging.basicConfig(**App.config()["logging"].get()) - - @staticmethod - def config() -> ConfigView: - return App.__conf diff --git a/heidpi-logger-py/heiDPI_env.py b/heidpi-logger-py/heiDPI_env.py deleted file mode 100644 index 9c6aed2..0000000 --- a/heidpi-logger-py/heiDPI_env.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Provides a utility to inject environment variables into argparse definitions. -Currently requires explicit naming of env vars to check for""" - -import argparse -import os - - -class EnvDefault(argparse.Action): - """An argparse action class that auto-sets missing default values from env - vars. Defaults to requiring the argument.""" - - def __init__(self, envvar, required=True, default=None, *args, **kwargs): - if not default and envvar: - if envvar in os.environ: - default = os.environ[envvar] - if required and default: - required = False - argparse.Action.__init__(self, default=default, required=required, *args, **kwargs) - - def __call__(self, parser, namespace, values, option_string=None): - setattr(namespace, self.dest, values) - -def env_default(envvar): - def wrapper(*args, **kwargs): - return EnvDefault(envvar, *args, **kwargs) - return wrapper - diff --git a/heidpi-logger-py/heiDPI_logger.py b/heidpi-logger-py/heiDPI_logger.py deleted file mode 100644 index 135d303..0000000 --- a/heidpi-logger-py/heiDPI_logger.py +++ /dev/null @@ -1,307 +0,0 @@ -import argparse -import multiprocessing -import geoip2.database -import geoip2.errors -import os -import json -import stat -import logging -import datetime -import copy -import gc -from concurrent.futures import ThreadPoolExecutor - -from heidpi import App -from heidpi import heiDPIsrvd -from heidpi import heiDPI_env - -DEFAULT_HOST = '127.0.0.1' -DEFAULT_PORT = 7000 -DEFAULT_UNIX = '/tmp/ndpid-distributor.sock' - -def dir_path(string): - if os.path.isdir(string): - return string - else: - raise NotADirectoryError(string) - -def file_path(string): - if os.path.isfile(string): - return string - else: - raise FileNotFoundError(string) - -def get_timestamp(): - date_time = datetime.datetime.fromtimestamp(datetime.datetime.now().timestamp()) - return date_time.strftime(LOGGING_CONFIG["datefmt"]) - -def heidpi_log_event(config_dict, json_dict, additional_processing): - json_dict_copy = copy.deepcopy(json_dict) - json_dict_copy['timestamp'] = get_timestamp() - - if additional_processing != None: - additional_processing(config_dict, json_dict_copy) - - ignore_fields = config_dict["ignore_fields"] - if ignore_fields != []: - list(map(json_dict_copy.pop, ignore_fields, [None] * len(ignore_fields))) - - with open(f'{JSON_PATH}/{config_dict["filename"]}.json', "a") as f: - json.dump(json_dict_copy, f) - f.write("\n") - - del json_dict_copy - gc.collect() - -def heidpi_flow_processing(config_dict: dict, json_dict: dict): - if bool(config_dict["geoip2_city"]["enabled"]): - response = {} - try: - reader = geoip2.database.Reader(config_dict['geoip2_city']["filepath"]) - - response = reader.city(str(json_dict["src_ip"])).raw - - json_dict["src_geoip2_city"] = {} - - for keys in config_dict["geoip2_city"]["keys"]: - if "." in keys: - current_data = response - try: - for subkey in keys.split("."): - if not subkey in current_data: - raise geoip2.errors.AddressNotFoundError(f"Error in key: {subkey} for {current_data}") - current_data = current_data[subkey] - json_dict["src_geoip2_city"][subkey] = current_data - except geoip2.errors.AddressNotFoundError: - logging.debug(f"No record found for src_ip: {json_dict['src_ip']}") - except Exception as e: - logging.exception(f"Exception: {e}") - finally: - del current_data - else: - try: - if not keys in response: - raise geoip2.errors.AddressNotFoundError(f"Error in key: {keys}") - json_dict["src_geoip2_city"][keys] = response[keys] - except geoip2.errors.AddressNotFoundError: - logging.debug(f"No record found for src_ip: {json_dict['src_ip']}") - except Exception as e: - logging.exception(f"Exception: {e}") - except geoip2.errors.AddressNotFoundError: - logging.debug(f"No record found for dst_ip:{json_dict['src_ip']}") - except Exception as e: - logging.exception(f"Exception: {e}") - - try: - response = reader.city(str(json_dict["dst_ip"])).raw - - json_dict["dst_geoip2_city"] = {} - - for keys in config_dict["geoip2_city"]["keys"]: - if "." in keys: - current_data = response - try: - for subkey in keys.split("."): - if not subkey in current_data: - raise geoip2.errors.AddressNotFoundError(f"Error in key: {subkey} for {current_data}") - current_data = current_data[subkey] - json_dict["dst_geoip2_city"][subkey] = current_data - except Exception as e: - logging.exception(f"Exception: {e}") - finally: - del current_data - else: - try: - if not keys in response: - raise geoip2.errors.AddressNotFoundError(f"Error in key: {keys}") - json_dict["dst_geoip2_city"][keys] = response[keys] - except Exception as e: - logging.exception(f"Exception: {e}") - except geoip2.errors.AddressNotFoundError: - logging.debug(f"No record found for dst_ip:{json_dict['dst_ip']}") - except Exception as e: - logging.exception(f"Exception: {e}") - - del response - del reader - gc.collect() - - # Filter risks, normally applied to flow events - if "ndpi" in json_dict and "flow_risk" in json_dict["ndpi"] and config_dict["ignore_risks"] != []: - list(map(json_dict["ndpi"]["flow_risk"].pop, config_dict["ignore_risks"], [None] * len(config_dict["ignore_risks"]))) - -def heidpi_worker(address, function, filter): - nsock = heiDPIsrvd.nDPIsrvdSocket() - nsock.connect(address) - nsock.loop(function, None, None) - if filter != "": - nsock.addFilter(filter_str=filter) - - -def heidpi_process_packet_events(json_dict, instance, current_flow, global_user_data): - if SHOW_PACKET_EVENTS and ("packet_event_id" in json_dict): - if json_dict["packet_event_name"] in PACKET_CONFIG["packet_event_name"]: - POOL_PACKET.submit(heidpi_log_event, PACKET_CONFIG, json_dict, None) - return True - -def heidpi_process_flow_events(json_dict, instance, current_flow, global_user_data): - if SHOW_FLOW_EVENTS and ("flow_event_id" in json_dict): - if json_dict["flow_event_name"] in FLOW_CONFIG["flow_event_name"]: - POOL_FLOW.submit(heidpi_log_event, FLOW_CONFIG, json_dict, heidpi_flow_processing) - return True - -def heidpi_process_daemon_events(json_dict, instance, current_flow, global_user_data): - if SHOW_DAEMON_EVENTS and ("daemon_event_id" in json_dict): - if json_dict["daemon_event_name"] in DAEMON_CONFIG["daemon_event_name"]: - POOL_DAEMON.submit(heidpi_log_event, DAEMON_CONFIG, json_dict, None) - return True - -def heidpi_process_error_events(json_dict, instance, current_flow, global_user_data): - if SHOW_ERROR_EVENTS and ("error_event_id" in json_dict): - if json_dict["error_event_name"] in ERROR_CONFIG["error_event_name"]: - POOL_ERROR.submit(heidpi_log_event, ERROR_CONFIG, json_dict, None) - return True - -def heidpi_type_analyzer(json_dict, instance, current_flow, global_user_data): - if SHOW_FLOW_EVENTS and ("flow_event_id" in json_dict): - if json_dict["flow_event_name"] in FLOW_CONFIG["flow_event_name"]: - POOL_FLOW.submit(heidpi_log_event, FLOW_CONFIG, json_dict, heidpi_flow_processing) - elif SHOW_PACKET_EVENTS and ("packet_event_id" in json_dict): - if json_dict["packet_event_name"] in PACKET_CONFIG["packet_event_name"]: - POOL_PACKET.submit(heidpi_log_event, PACKET_CONFIG, json_dict, None) - elif SHOW_DAEMON_EVENTS and ("daemon_event_id" in json_dict): - if json_dict["daemon_event_name"] in DAEMON_CONFIG["daemon_event_name"]: - POOL_DAEMON.submit(heidpi_log_event, DAEMON_CONFIG, json_dict, None) - elif SHOW_ERROR_EVENTS and ("error_event_id" in json_dict): - if json_dict["error_event_name"] in ERROR_CONFIG["error_event_name"]: - POOL_ERROR.submit(heidpi_log_event, ERROR_CONFIG, json_dict, None) - return True - -def heidpi_validateAddress(args): - tcp_addr_set = False - address = None - - if args.host is None: - address_tcpip = (DEFAULT_HOST, args.port) - else: - address_tcpip = (args.host, args.port) - tcp_addr_set = True - if args.unix is None: - address_unix = DEFAULT_UNIX - else: - address_unix = args.unix - - possible_sock_mode = 0 - try: - possible_sock_mode = os.stat(address_unix).st_mode - except: - pass - if tcp_addr_set == False and stat.S_ISSOCK(possible_sock_mode): - address = address_unix - else: - address = address_tcpip - - return address - -def main(): - parser = argparse.ArgumentParser(description='heiDPI Python Interface', formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - group = parser.add_mutually_exclusive_group() - group.add_argument('--host', type=str, action=heiDPI_env.env_default('HOST'), required=False, help='nDPIsrvd host IP') - group.add_argument('--unix', type=str, action=heiDPI_env.env_default('UNIX'), required=False, help='nDPIsrvd unix socket path') - - parser.add_argument('--port', type=int, action=heiDPI_env.env_default('PORT'), default=DEFAULT_PORT, help='nDPIsrvd TCP port') - - parser.add_argument('--write', type=dir_path, action=heiDPI_env.env_default('WRITE'), default='/var/log', help='heiDPI write path for logs') - - parser.add_argument('--config', type=file_path, action=heiDPI_env.env_default('CONFIG'), default=f'{os.getcwd()}/config.yml', help='heiDPI write path for logs') - - parser.add_argument('--filter', type=str, action=heiDPI_env.env_default('FILTER'), required=False, default="", help="nDPId filter string, e.g. --filter 'ndpi' in json_dict and 'proto' in json_dict['ndpi']") - - parser.add_argument('--show-daemon-events', type=int, action=heiDPI_env.env_default('SHOW_DAEMON_EVENTS'), default=0, required=False, help='heiDPI shows daemon events') - parser.add_argument('--show-packet-events', type=int, action=heiDPI_env.env_default('SHOW_PACKET_EVENTS'), default=0, required=False, help='heiDPI shows packet events') - parser.add_argument('--show-error-events', type=int, action=heiDPI_env.env_default('SHOW_ERROR_EVENTS'), default=0, required=False, help='heiDPI shows error events') - parser.add_argument('--show-flow-events', type=int, action=heiDPI_env.env_default('SHOW_FLOW_EVENTS'), default=0, required=False, help='heiDPI shows flow events') - - args = parser.parse_args() - address = heidpi_validateAddress(args) - - App(args.config) - - global SHOW_ERROR_EVENTS - global SHOW_PACKET_EVENTS - global SHOW_FLOW_EVENTS - global SHOW_DAEMON_EVENTS - global JSON_PATH - - SHOW_ERROR_EVENTS = args.show_error_events - SHOW_PACKET_EVENTS = args.show_packet_events - SHOW_FLOW_EVENTS = args.show_flow_events - SHOW_DAEMON_EVENTS = args.show_daemon_events - JSON_PATH = args.write - - global PACKET_CONFIG - global FLOW_CONFIG - global DAEMON_CONFIG - global ERROR_CONFIG - - PACKET_CONFIG = App.config()["packet_event"].get() - FLOW_CONFIG = App.config()["flow_event"].get() - DAEMON_CONFIG = App.config()["daemon_event"].get() - ERROR_CONFIG = App.config()["error_event"].get() - - global LOGGING_CONFIG - LOGGING_CONFIG = App.config()["logging"].get() - - logging.info('Recv buffer size: {}'.format( - heiDPIsrvd.NETWORK_BUFFER_MAX_SIZE)) - logging.info('Connecting to {} ..'.format( - address[0]+':'+str(address[1]) if type(address) is tuple else address)) - - if SHOW_FLOW_EVENTS: - global POOL_FLOW - - POOL_FLOW = ThreadPoolExecutor(max_workers=PACKET_CONFIG['threads']) - - heidpi_daemon_job = multiprocessing.Process( - target=heidpi_worker, - args=(address, heidpi_process_flow_events, args.filter)) - heidpi_daemon_job.start() - - - ####################################################################################### - if SHOW_PACKET_EVENTS: - global POOL_PACKET - - POOL_PACKET = ThreadPoolExecutor(max_workers=PACKET_CONFIG['threads']) - - heidpi_packet_job = multiprocessing.Process( - target=heidpi_worker, - args=(address, heidpi_process_packet_events, args.filter)) - heidpi_packet_job.start() - - ####################################################################################### - if SHOW_DAEMON_EVENTS: - global POOL_DAEMON - - POOL_DAEMON = ThreadPoolExecutor(max_workers=DAEMON_CONFIG['threads']) - - heidpi_daemon_job = multiprocessing.Process( - target=heidpi_worker, - args=(address, heidpi_process_daemon_events, args.filter)) - heidpi_daemon_job.start() - - - ####################################################################################### - if SHOW_ERROR_EVENTS: - global POOL_ERROR - - POOL_ERROR = ThreadPoolExecutor(max_workers=ERROR_CONFIG['threads']) - - heidpi_error_job = multiprocessing.Process( - target=heidpi_worker, - args=(address, heidpi_process_error_events, args.filter)) - heidpi_error_job.start() - -if __name__ == '__main__': - main() diff --git a/heidpi-logger-py/heiDPIsrvd.py b/heidpi-logger-py/heiDPIsrvd.py deleted file mode 100644 index fe086c8..0000000 --- a/heidpi-logger-py/heiDPIsrvd.py +++ /dev/null @@ -1,493 +0,0 @@ -#!/usr/bin/env python3 - -import json -import re -import os -import socket -import sys - -NETWORK_BUFFER_MIN_SIZE = 6 # NETWORK_BUFFER_LENGTH_DIGITS + 1 -NETWORK_BUFFER_MAX_SIZE = 33792 # Please keep this value in sync with the one in config.h -nDPId_PACKETS_PLEN_MAX = 8192 # Please keep this value in sync with the one in config.h - -PKT_TYPE_ETH_IP4 = 0x0800 -PKT_TYPE_ETH_IP6 = 0x86DD - -class ThreadData: - pass - -class Instance: - - def __init__(self, alias, source): - self.alias = str(alias) - self.source = str(source) - self.flows = dict() - self.thread_data = dict() - - def __str__(self): - return '<%s.%s object at %s with alias %s, source %s>' % ( - self.__class__.__module__, - self.__class__.__name__, - hex(id(self)), - self.alias, - self.source - ) - - def getThreadData(self, thread_id): - if thread_id not in self.thread_data: - return None - return self.thread_data[thread_id] - - def getThreadDataFromJSON(self, json_dict): - if 'thread_id' not in json_dict: - return None - return self.getThreadData(json_dict['thread_id']) - - def getMostRecentFlowTime(self, thread_id): - return self.thread_data[thread_id].most_recent_flow_time - - def setMostRecentFlowTime(self, thread_id, most_recent_flow_time): - if thread_id in self.thread_data: - return self.thread_data[thread_id] - - self.thread_data[thread_id] = ThreadData() - self.thread_data[thread_id].most_recent_flow_time = most_recent_flow_time - return self.thread_data[thread_id] - - def getMostRecentFlowTimeFromJSON(self, json_dict): - if 'thread_id' not in json_dict: - return 0 - return self.getThreadData(json_dict['thread_id']).most_recent_flow_time - - def setMostRecentFlowTimeFromJSON(self, json_dict): - if 'thread_id' not in json_dict: - return - thread_id = json_dict['thread_id'] - if 'thread_ts_usec' in json_dict: - mrtf = self.getMostRecentFlowTime(thread_id) if thread_id in self.thread_data else 0 - self.setMostRecentFlowTime(thread_id, max(json_dict['thread_ts_usec'], mrtf)) - -class Flow: - - def __init__(self, flow_id, thread_id): - self.flow_id = flow_id - self.thread_id = thread_id - self.flow_last_seen = -1 - self.flow_idle_time = -1 - self.cleanup_reason = -1 - - def __str__(self): - return '<%s.%s object at %s with flow id %d>' % ( - self.__class__.__module__, - self.__class__.__name__, - hex(id(self)), - self.flow_id - ) - -class FlowManager: - CLEANUP_REASON_INVALID = 0 - CLEANUP_REASON_DAEMON_INIT = 1 # can happen if kill -SIGKILL $(pidof nDPId) or restart after SIGSEGV - CLEANUP_REASON_DAEMON_SHUTDOWN = 2 # graceful shutdown e.g. kill -SIGTERM $(pidof nDPId) - CLEANUP_REASON_FLOW_END = 3 - CLEANUP_REASON_FLOW_IDLE = 4 - CLEANUP_REASON_FLOW_TIMEOUT = 5 # nDPId died a long time ago w/o restart? - CLEANUP_REASON_APP_SHUTDOWN = 6 # your python app called FlowManager.doShutdown() - - def __init__(self): - self.instances = dict() - - def getInstance(self, json_dict): - if 'alias' not in json_dict or \ - 'source' not in json_dict: - return None - - alias = json_dict['alias'] - source = json_dict['source'] - - if alias not in self.instances: - self.instances[alias] = dict() - if source not in self.instances[alias]: - self.instances[alias][source] = dict() - self.instances[alias][source] = Instance(alias, source) - - self.instances[alias][source].setMostRecentFlowTimeFromJSON(json_dict) - - return self.instances[alias][source] - - @staticmethod - def getLastPacketTime(instance, flow_id, json_dict): - return max(int(json_dict['flow_src_last_pkt_time']), int(json_dict['flow_dst_last_pkt_time']), instance.flows[flow_id].flow_last_seen) - - def getFlow(self, instance, json_dict): - if 'flow_id' not in json_dict: - return None - - flow_id = int(json_dict['flow_id']) - - if flow_id in instance.flows: - instance.flows[flow_id].flow_last_seen = FlowManager.getLastPacketTime(instance, flow_id, json_dict) - instance.flows[flow_id].flow_idle_time = int(json_dict['flow_idle_time']) - return instance.flows[flow_id] - - thread_id = int(json_dict['thread_id']) - instance.flows[flow_id] = Flow(flow_id, thread_id) - instance.flows[flow_id].flow_last_seen = FlowManager.getLastPacketTime(instance, flow_id, json_dict) - instance.flows[flow_id].flow_idle_time = int(json_dict['flow_idle_time']) - instance.flows[flow_id].cleanup_reason = FlowManager.CLEANUP_REASON_INVALID - - return instance.flows[flow_id] - - def getFlowsToCleanup(self, instance, json_dict): - flows = dict() - - if 'daemon_event_name' in json_dict: - if json_dict['daemon_event_name'].lower() == 'init' or \ - json_dict['daemon_event_name'].lower() == 'shutdown': - # invalidate all existing flows with that alias/source/thread_id - for flow_id in instance.flows: - flow = instance.flows[flow_id] - if flow.thread_id != int(json_dict['thread_id']): - continue - if json_dict['daemon_event_name'].lower() == 'init': - flow.cleanup_reason = FlowManager.CLEANUP_REASON_DAEMON_INIT - else: - flow.cleanup_reason = FlowManager.CLEANUP_REASON_DAEMON_SHUTDOWN - flows[flow_id] = flow - for flow_id in flows: - del instance.flows[flow_id] - if len(instance.flows) == 0: - del self.instances[instance.alias][instance.source] - - elif 'flow_event_name' in json_dict and \ - (json_dict['flow_event_name'].lower() == 'end' or \ - json_dict['flow_event_name'].lower() == 'idle' or \ - json_dict['flow_event_name'].lower() == 'guessed' or \ - json_dict['flow_event_name'].lower() == 'not-detected' or \ - json_dict['flow_event_name'].lower() == 'detected'): - flow_id = json_dict['flow_id'] - if json_dict['flow_event_name'].lower() == 'end': - instance.flows[flow_id].cleanup_reason = FlowManager.CLEANUP_REASON_FLOW_END - elif json_dict['flow_event_name'].lower() == 'idle': - instance.flows[flow_id].cleanup_reason = FlowManager.CLEANUP_REASON_FLOW_IDLE - # TODO: Flow Guessing/Detection can happen right before an idle event. - # We need to prevent that it results in a CLEANUP_REASON_FLOW_TIMEOUT. - # This may cause inconsistency and needs to be handled in another way. - if json_dict['flow_event_name'].lower() != 'guessed' and \ - json_dict['flow_event_name'].lower() != 'not-detected' and \ - json_dict['flow_event_name'].lower() != 'detected': - flows[flow_id] = instance.flows.pop(flow_id) - - elif 'flow_last_seen' in json_dict: - if int(json_dict['flow_last_seen']) + int(json_dict['flow_idle_time']) < \ - instance.getMostRecentFlowTimeFromJSON(json_dict): - flow_id = json_dict['flow_id'] - instance.flows[flow_id].cleanup_reason = FlowManager.CLEANUP_REASON_FLOW_TIMEOUT - flows[flow_id] = instance.flows.pop(flow_id) - - return flows - - def doShutdown(self): - flows = dict() - - for alias in self.instances: - for source in self.instances[alias]: - for flow_id in self.instances[alias][source].flows: - flow = self.instances[alias][source].flows[flow_id] - flow.cleanup_reason = FlowManager.CLEANUP_REASON_APP_SHUTDOWN - flows[flow_id] = flow - - del self.instances - - return flows - - def verifyFlows(self): - invalid_flows = list() - - for alias in self.instances: - for source in self.instances[alias]: - for flow_id in self.instances[alias][source].flows: - thread_id = self.instances[alias][source].flows[flow_id].thread_id - if self.instances[alias][source].flows[flow_id].flow_last_seen + \ - self.instances[alias][source].flows[flow_id].flow_idle_time < \ - self.instances[alias][source].getMostRecentFlowTime(thread_id): - invalid_flows += [flow_id] - - return invalid_flows - -class nDPIsrvdException(Exception): - UNSUPPORTED_ADDRESS_TYPE = 1 - BUFFER_CAPACITY_REACHED = 2 - SOCKET_CONNECTION_BROKEN = 3 - INVALID_LINE_RECEIVED = 4 - CALLBACK_RETURNED_FALSE = 5 - SOCKET_TIMEOUT = 6 - - def __init__(self, etype): - self.etype = etype - def __str__(self): - return 'nDPIsrvdException type {}'.format(self.etype) - -class UnsupportedAddressType(nDPIsrvdException): - def __init__(self, addr): - super().__init__(nDPIsrvdException.UNSUPPORTED_ADDRESS_TYPE) - self.addr = addr - def __str__(self): - return '{}'.format(str(self.addr)) - -class BufferCapacityReached(nDPIsrvdException): - def __init__(self, current_length, max_length): - super().__init__(nDPIsrvdException.BUFFER_CAPACITY_REACHED) - self.current_length = current_length - self.max_length = max_length - def __str__(self): - return '{} of {} bytes'.format(self.current_length, self.max_length) - -class SocketConnectionBroken(nDPIsrvdException): - def __init__(self): - super().__init__(nDPIsrvdException.SOCKET_CONNECTION_BROKEN) - def __str__(self): - return 'Disconnected.' - -class InvalidLineReceived(nDPIsrvdException): - def __init__(self, packet_buffer): - super().__init__(nDPIsrvdException.INVALID_LINE_RECEIVED) - self.packet_buffer = packet_buffer - def __str__(self): - return 'Received JSON line is invalid.' - -class CallbackReturnedFalse(nDPIsrvdException): - def __init__(self): - super().__init__(nDPIsrvdException.CALLBACK_RETURNED_FALSE) - def __str__(self): - return 'Callback returned False, abort.' - -class SocketTimeout(nDPIsrvdException): - def __init__(self): - super().__init__(nDPIsrvdException.SOCKET_TIMEOUT) - def __str__(self): - return 'Socket timeout.' - -class JsonFilter(): - def __init__(self, filter_string): - self.filter_string = filter_string - self.filter = compile(filter_string, '', 'eval') - def evaluate(self, json_dict): - if type(json_dict) is not dict: - raise nDPIsrvdException('Could not evaluate JSON Filter: expected dictionary, got {}'.format(type(json_dict))) - return eval(self.filter, {'json_dict': json_dict}) - -class nDPIsrvdSocket: - def __init__(self): - self.sock_family = None - self.flow_mgr = FlowManager() - self.received_bytes = 0 - self.json_filter = list() - - def addFilter(self, filter_str): - self.json_filter.append(JsonFilter(filter_str)) - - def evalFilters(self, json_dict): - for jf in self.json_filter: - try: - json_filter_retval = jf.evaluate(json_dict) - except Exception as err: - print() - sys.stderr.write('Error while evaluating expression "{}"\n'.format(jf.filter_string)) - raise err - - if not isinstance(json_filter_retval, bool): - print() - sys.stderr.write('Error while evaluating expression "{}"\n'.format(jf.filter_string)) - raise nDPIsrvdException('JSON Filter returned an invalid type: expected bool, got {}'.format(type(json_filter_retval))) - - if json_filter_retval is False: - return False - - return True - - def connect(self, addr): - if type(addr) is tuple: - self.sock_family = socket.AF_INET - elif type(addr) is str: - self.sock_family = socket.AF_UNIX - else: - raise UnsupportedAddressType(addr) - - self.sock = socket.socket(self.sock_family, socket.SOCK_STREAM) - self.sock.connect(addr) - self.buffer = bytes() - self.msglen = 0 - self.digitlen = 0 - self.lines = [] - self.failed_lines = [] - self.filtered_lines = 0 - - def timeout(self, timeout): - self.sock.settimeout(timeout) - - def receive(self): - if len(self.buffer) == NETWORK_BUFFER_MAX_SIZE: - raise BufferCapacityReached(len(self.buffer), NETWORK_BUFFER_MAX_SIZE) - - connection_finished = False - try: - recvd = self.sock.recv(NETWORK_BUFFER_MAX_SIZE - len(self.buffer)) - except ConnectionResetError: - connection_finished = True - recvd = bytes() - except TimeoutError: - raise SocketTimeout() - except socket.timeout: - raise SocketTimeout() - - if len(recvd) == 0: - connection_finished = True - - self.buffer += recvd - - new_data_avail = False - while self.msglen + self.digitlen <= len(self.buffer): - - if self.msglen == 0: - starts_with_digits = re.match(r'(^\d+){', self.buffer[:NETWORK_BUFFER_MIN_SIZE].decode(errors='strict')) - if starts_with_digits is None: - if len(self.buffer) < NETWORK_BUFFER_MIN_SIZE: - break - raise InvalidLineReceived(self.buffer) - self.msglen = int(starts_with_digits.group(1)) - self.digitlen = len(starts_with_digits.group(1)) - - if len(self.buffer) >= self.msglen + self.digitlen: - recvd = self.buffer[self.digitlen:self.msglen + self.digitlen] - self.buffer = self.buffer[self.msglen + self.digitlen:] - self.lines += [(recvd,self.msglen,self.digitlen)] - new_data_avail = True - - self.received_bytes += self.msglen + self.digitlen - self.msglen = 0 - self.digitlen = 0 - - if connection_finished is True: - raise SocketConnectionBroken() - - return new_data_avail - - def parse(self, callback_json, callback_flow_cleanup, global_user_data): - retval = True - - for received_line in self.lines: - try: - json_dict = json.loads(received_line[0].decode('ascii', errors='replace'), strict=True) - except json.decoder.JSONDecodeError as e: - json_dict = dict() - self.failed_lines += [received_line] - self.lines = self.lines[1:] - raise(e) - - instance = self.flow_mgr.getInstance(json_dict) - if instance is None: - self.failed_lines += [received_line] - retval = False - continue - - current_flow = self.flow_mgr.getFlow(instance, json_dict) - filter_eval = self.evalFilters(json_dict) - if filter_eval is True: - try: - if callback_json(json_dict, instance, current_flow, global_user_data) is not True: - self.failed_lines += [received_line] - retval = False - except Exception as e: - self.failed_lines += [received_line] - self.lines = self.lines[1:] - raise(e) - else: - self.filtered_lines += 1 - - for _, flow in self.flow_mgr.getFlowsToCleanup(instance, json_dict).items(): - if callback_flow_cleanup is None: - pass - elif filter_eval is True and callback_flow_cleanup(instance, flow, global_user_data) is not True: - self.failed_lines += [received_line] - self.lines = self.lines[1:] - retval = False - - self.lines = self.lines[1:] - - return retval - - def loop(self, callback_json, callback_flow_cleanup, global_user_data): - throw_ex = None - - while True: - bytes_recv = 0 - try: - bytes_recv = self.receive() - except Exception as err: - throw_ex = err - - if self.parse(callback_json, callback_flow_cleanup, global_user_data) is False: - raise CallbackReturnedFalse() - - if throw_ex is not None: - raise throw_ex - - def shutdown(self): - return self.flow_mgr.doShutdown().items() - - def verify(self): - if len(self.failed_lines) > 0: - raise nDPIsrvdException('Failed lines > 0: {}'.format(len(self.failed_lines))) - return self.flow_mgr.verifyFlows() - - -def toSeconds(usec): - return usec / (1000 * 1000) - -global schema -schema = {'packet_event_schema' : None, 'error_event_schema' : None, 'daemon_event_schema' : None, 'flow_event_schema' : None} - -def initSchemaValidator(schema_dirs=[]): - if len(schema_dirs) == 0: - schema_dirs += [os.path.dirname(sys.argv[0]) + '/schema'] - - for key in schema: - for schema_dir in schema_dirs: - try: - with open(schema_dir + '/' + str(key) + '.json', 'r') as schema_file: - schema[key] = json.load(schema_file) - except FileNotFoundError: - print(f"No schema in {schema_dir}") - continue - else: - break - -def validateAgainstSchema(json_dict): - import jsonschema - - if 'packet_event_id' in json_dict: - try: - jsonschema.Draft7Validator(schema=schema['packet_event_schema']).validate(instance=json_dict) - except AttributeError: - jsonschema.validate(instance=json_dict, schema=schema['packet_event_schema']) - return True - if 'error_event_id' in json_dict: - try: - jsonschema.Draft7Validator(schema=schema['error_event_schema']).validate(instance=json_dict) - except AttributeError: - jsonschema.validate(instance=json_dict, schema=schema['error_event_schema']) - return True - if 'daemon_event_id' in json_dict: - try: - jsonschema.Draft7Validator(schema=schema['daemon_event_schema']).validate(instance=json_dict) - except AttributeError: - jsonschema.validate(instance=json_dict, schema=schema['daemon_event_schema']) - return True - if 'flow_event_id' in json_dict: - try: - jsonschema.Draft7Validator(schema=schema['flow_event_schema']).validate(instance=json_dict) - except AttributeError: - jsonschema.validate(instance=json_dict, schema=schema['flow_event_schema']) - return True - - return False diff --git a/heidpi-logger-py/schema/README.md b/heidpi-logger-py/schema/README.md deleted file mode 100644 index 186dc4d..0000000 --- a/heidpi-logger-py/schema/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# schema - -All schema's placed in here are nDPId exclusive, meaning that they are not necessarily representing a "real-world" JSON string received by e.g. `./example/py-json-stdout`. -This is due to the fact that libnDPI itself add's some JSON information to the serializer of which we have no control over. -IMHO it makes no sense to include stuff here that is part of libnDPI. diff --git a/heidpi-logger-py/schema/daemon_event_schema.json b/heidpi-logger-py/schema/daemon_event_schema.json deleted file mode 100644 index 91f3875..0000000 --- a/heidpi-logger-py/schema/daemon_event_schema.json +++ /dev/null @@ -1,166 +0,0 @@ -{ - "type": "object", - "required": [ - "alias", - "source", - "thread_id", - "packet_id", - "daemon_event_id", - "daemon_event_name", - "global_ts_usec" - ], - "if": { - "properties": { "daemon_event_name": { "enum": [ "init", "reconnect" ] } } - }, - "then": { - "required": [ "max-flows-per-thread", "max-idle-flows-per-thread", "reader-thread-count", "flow-scan-interval", "generic-max-idle-time", "icmp-max-idle-time", "udp-max-idle-time", "tcp-max-idle-time", "max-packets-per-flow-to-send", "max-packets-per-flow-to-process", "max-packets-per-flow-to-analyse" ] - }, - "if": { - "properties": { "daemon_event_name": { "enum": [ "status", "shutdown" ] } } - }, - "then": { - "required": [ "packets-captured", "packets-processed", "total-skipped-flows", "total-l4-payload-len", "total-not-detected-flows", "total-guessed-flows", "total-detected-flows", "total-detection-updates", "total-updates", "current-active-flows", "total-active-flows", "total-idle-flows", "total-compressions", "total-compression-diff", "current-compression-diff", "total-events-serialized" ] - }, - "properties": { - "alias": { - "type": "string" - }, - "source": { - "type": "string" - }, - "thread_id": { - "type": "number", - "minimum": 0, - "maximum": 31 - }, - "packet_id": { - "type": "number", - "minimum": 0 - }, - "daemon_event_id": { - "type": "number", - "minimum": 0, - "maximum": 4 - }, - "daemon_event_name": { - "type": "string", - "enum": [ - "invalid", - "init", - "reconnect", - "shutdown", - "status" - ] - }, - - "max-flows-per-thread": { - "type": "number" - }, - "max-idle-flows-per-thread": { - "type": "number" - }, - "reader-thread-count": { - "type": "number" - }, - "flow-scan-interval": { - "type": "number" - }, - "generic-max-idle-time": { - "type": "number" - }, - "icmp-max-idle-time": { - "type": "number" - }, - "udp-max-idle-time": { - "type": "number" - }, - "tcp-max-idle-time": { - "type": "number" - }, - "max-packets-per-flow-to-process": { - "type": "number" - }, - "max-packets-per-flow-to-send": { - "type": "number" - }, - "max-packets-per-flow-to-analyse": { - "type": "number" - }, - - "packets-captured": { - "type": "number", - "minimum": 0 - }, - "packets-processed": { - "type": "number", - "minimum": 0 - }, - "total-skipped-flows": { - "type": "number", - "minimum": 0 - }, - "total-l4-payload-len": { - "type": "number", - "minimum": 0 - }, - "total-not-detected-flows": { - "type": "number", - "minimum": 0 - }, - "total-guessed-flows": { - "type": "number", - "minimum": 0 - }, - "total-detected-flows": { - "type": "number", - "minimum": 0 - }, - "total-detection-updates": { - "type": "number", - "minimum": 0 - }, - "total-updates": { - "type": "number", - "minimum": 0 - }, - "current-active-flows": { - "type": "number", - "minimum": 0 - }, - "total-active-flows": { - "type": "number", - "minimum": 0 - }, - "total-idle-flows": { - "type": "number", - "minimum": 0 - }, - "total-compressions": { - "type": "number", - "minimum": 0 - }, - "total-compression-diff": { - "type": "number", - "minimum": 0 - }, - "current-compression-diff": { - "type": "number", - "minimum": 0 - }, - "total-events-serialized": { - "type": "number", - "minimum": 1 - }, - "global_ts_usec": { - "type": "number", - "if": { - "properties": { "daemon_event_name": { "enum": [ "init" ] } } - }, - "then" : true, - "else" : { - "minimum": 1000000 - } - } - }, - "additionalProperties": false -} diff --git a/heidpi-logger-py/schema/error_event_schema.json b/heidpi-logger-py/schema/error_event_schema.json deleted file mode 100644 index b81140e..0000000 --- a/heidpi-logger-py/schema/error_event_schema.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "type": "object", - "required": [ - "alias", - "source", - "packet_id", - "error_event_id", - "error_event_name", - "datalink", - "threshold_n", - "threshold_n_max", - "threshold_time", - "threshold_ts_usec", - "global_ts_usec" - ], - - "if": { - "properties": { "error_event_name": { "enum": [ "Unknown datalink layer packet", "Unknown packet type" ] } } - }, - "then": { - "anyOf": [ - { "required": [ "layer_type" ] }, - { "not": { "required": [ "thread_id" ] } } - ] - }, - - "if": { - "properties": { "error_event_name": { "enum": [ "Unknown L3 protocol" ] } } - }, - "then": { - "anyOf": [ - { "required": [ "protocol" ] }, - { "not": { "required": [ "thread_id" ] } } - ] - }, - - "if": { - "properties": { "error_event_name": { "enum": [ "Packet too short", "IP4 packet too short", - "IP6 packet too short", "TCP packet smaller than expected", - "UDP packet smaller than expected", - "Captured packet size is smaller than expected packet size" ] } } - }, - "then": { - "anyOf": [ - { "required": [ "size", "expected" ] }, - { "not": { "required": [ "thread_id" ] } } - ] - }, - - "if": { - "properties": { "error_event_name": { "enum": [ "Packet header invalid" ] } } - }, - "then": { - "anyOf": [ - { "required": [ "raeson" ] }, - { "not": { "required": [ "thread_id" ] } } - ] - }, - - "if": { - "properties": { "error_event_name": { "enum": [ "Flow memory allocation failed" ] } } - }, - "then": { - "required": [ "thread_id", "size" ] - }, - - "if": { - "properties": { "error_event_name": { "enum": [ "Max flows to track reached" ] } } - }, - "then": { - "required": [ "thread_id", "current_active", "current_idle", "max_active", "max_idle" ] - }, - - "properties": { - "alias": { - "type": "string" - }, - "source": { - "type": "string" - }, - "thread_id": { - "type": "number" - }, - "packet_id": { - "type": "number", - "minimum": 0 - }, - "error_event_id": { - "type": "number", - "minimum": 0, - "maximum": 16 - }, - "error_event_name": { - "type": "string", - "enum": [ - "Unknown datalink layer packet", - "Unknown L3 protocol", - "Unsupported datalink layer", - "Packet too short", - "Unknown packet type", - "Packet header invalid", - "IP4 packet too short", - "Packet smaller than IP4 header", - "nDPI IPv4/L4 payload detection failed", - "IP6 packet too short", - "Packet smaller than IP6 header", - "nDPI IPv6/L4 payload detection failed", - "TCP packet smaller than expected", - "UDP packet smaller than expected", - "Captured packet size is smaller than expected packet size", - "Max flows to track reached", - "Flow memory allocation failed" - ] - }, - - "datalink": { - "type": "number", - "minimum": 0, - "maximum": 292 - }, - - "threshold_n": { - "type": "number", - "minimum": 1 - }, - - "threshold_n_max": { - "type": "number", - "minimum": 1, - "maximum": 65535 - }, - - "threshold_time": { - "type": "number" - }, - - "threshold_ts_usec": { - "type": "number" - }, - - "layer_type": { - "type": "number", - "minimum": 0 - }, - - "l4_data_len": { - "type": "number", - "minimum": 0 - }, - - "reason": { - "type": "string" - }, - - "protocol": { - "type": "number", - "minimum": 0, - "maximum": 65535 - }, - - "size": { - "type": "number" - }, - "expected": { - "type": "number" - }, - - "current_active": { - "type": "number" - }, - "current_idle": { - "type": "number" - }, - "max_active": { - "type": "number" - }, - "max_idle": { - "type": "number" - }, - "global_ts_usec": { - "type": "number", - "minimum": 0 - } - }, - "additionalProperties": false -} diff --git a/heidpi-logger-py/schema/flow_event_schema.json b/heidpi-logger-py/schema/flow_event_schema.json deleted file mode 100644 index 01cc9d9..0000000 --- a/heidpi-logger-py/schema/flow_event_schema.json +++ /dev/null @@ -1,472 +0,0 @@ -{ - "type": "object", - "required": [ - "alias", - "source", - "thread_id", - "packet_id", - "flow_event_id", - "flow_event_name", - "flow_id", - "flow_state", - "flow_src_packets_processed", - "flow_dst_packets_processed", - "flow_first_seen", - "flow_src_last_pkt_time", - "flow_dst_last_pkt_time", - "flow_idle_time", - "flow_src_min_l4_payload_len", - "flow_dst_min_l4_payload_len", - "flow_src_max_l4_payload_len", - "flow_dst_max_l4_payload_len", - "flow_src_tot_l4_payload_len", - "flow_dst_tot_l4_payload_len", - "l3_proto", - "l4_proto", - "midstream", - "thread_ts_usec", - "src_ip", - "dst_ip" - ], - - "if": { - "properties": { "flow_event_name": { "enum": [ "new", "end", "idle", "update" ] } } - }, - "then": { - "required": [ "flow_datalink", "flow_max_packets" ] - }, - - "if": { - "properties": { "flow_event_name": { "enum": [ "analyse" ] } } - }, - "then": { - "required": [ "data_analysis" ] - }, - - "if": { - "properties": { "flow_state": { "enum": [ "finished" ] } } - }, - "then": { - "required": [ "ndpi" ] - }, - - "if": { - "properties": { "flow_event_name": { "enum": [ "guessed", "detected", - "detection-update", "not-detected" ] } } - }, - "then": { - "required": [ "ndpi" ] - }, - - "properties": { - "alias": { - "type": "string" - }, - "source": { - "type": "string" - }, - "thread_id": { - "type": "number", - "minimum": 0, - "maximum": 31 - }, - "packet_id": { - "type": "number", - "minimum": 0 - }, - "flow_event_id": { - "type": "number", - "minimum": 0, - "maximum": 9 - }, - "flow_event_name": { - "type": "string", - "enum": [ - "invalid", - "new", - "end", - "idle", - "update", - "analyse", - "guessed", - "detected", - "detection-update", - "not-detected" - ] - }, - "flow_id": { - "type": "number", - "minimum": 1 - }, - "flow_state": { - "type": "string", - "enum": [ - "finished", - "info" - ] - }, - "flow_datalink": { - "type": "number", - "minimum": 0, - "maximum": 292 - }, - "flow_src_packets_processed": { - "type": "number", - "minimum": 0 - }, - "flow_dst_packets_processed": { - "type": "number", - "minimum": 0 - }, - "flow_max_packets": { - "type": "number", - "minimum": 0 - }, - "flow_first_seen": { - "type": "number", - "minimum": 0 - }, - "flow_src_last_pkt_time": { - "type": "number", - "minimum": 0 - }, - "flow_dst_last_pkt_time": { - "type": "number", - "minimum": 0 - }, - "flow_idle_time": { - "type": "number", - "minimum": 1 - }, - "flow_src_min_l4_payload_len": { - "type": "number", - "minimum": 0 - }, - "flow_dst_min_l4_payload_len": { - "type": "number", - "minimum": 0 - }, - "flow_src_max_l4_payload_len": { - "type": "number", - "minimum": 0 - }, - "flow_dst_max_l4_payload_len": { - "type": "number", - "minimum": 0 - }, - "flow_src_tot_l4_payload_len": { - "type": "number", - "minimum": 0 - }, - "flow_dst_tot_l4_payload_len": { - "type": "number", - "minimum": 0 - }, - "l3_proto": { - "type": "string", - "enum": [ - "ip4", - "ip6", - "unknown" - ] - }, - "l4_proto": { - "oneOf": [ - { - "type": "number" - }, - { - "type": "string", - "enum": [ - "tcp", - "udp", - "icmp", - "icmp6" - ] - } - ] - }, - "midstream": { - "type": "number", - "minimum": 0, - "maximum": 1 - }, - "thread_ts_usec": { - "type": "number", - "minimum": 0 - }, - "src_ip": { - "type": "string", - "anyOf" : [ - { "format": "ipv4" }, - { "format": "ipv6" } - ] - }, - "dst_ip": { - "type": "string", - "anyOf" : [ - { "format": "ipv4" }, - { "format": "ipv6" } - ] - }, - "src_port": { - "type": "number", - "minimum": 1, - "maximum": 65535 - }, - "dst_port": { - "type": "number", - "minimum": 1, - "maximum": 65535 - }, - "ndpi": { - "type": "object", - "required": [ "proto", "proto_id", "breed", "encrypted" ], - - "properties": { - "proto": { - "type": "string" - }, - "proto_id": { - "type": "string" - }, - "proto_by_ip": { - "type": "string" - }, - "proto_by_ip_id": { - "type": "number" - }, - "category": { - "type": "string" - }, - "category_id": { - "type": "number" - }, - "encrypted": { - "type": "number", - "enum": [ - 0, - 1 - ] - }, - "breed": { - "type": "string" - }, - "flow_risk": { - "type": "object" - }, - "confidence": { - "type": "object", - "properties": { - "0": { - "type": "string" - }, - "1": { - "type": "string" - }, - "2": { - "type": "string" - }, - "3": { - "type": "string" - }, - "4": { - "type": "string" - }, - "5": { - "type": "string" - }, - "6": { - "type": "string" - } - }, - "additionalProperties": false - }, - "entropy": { - "type": "number" - }, - "hostname": { - "type": "string" - }, - "collectd": { - "type": "object" - }, - "dhcp": { - "type": "object" - }, - "discord": { - "type": "object" - }, - "bittorrent": { - "type": "object" - }, - "mdns": { - "type": "object" - }, - "natpmp": { - "type": "object" - }, - "ntp": { - "type": "object" - }, - "ubntac2": { - "type": "object" - }, - "kerberos": { - "type": "object" - }, - "telnet": { - "type": "object" - }, - "tls": { - "type": "object" - }, - "quic": { - "type": "object" - }, - "imap": { - "type": "object" - }, - "http": { - "type": "object" - }, - "pop": { - "type": "object" - }, - "smtp": { - "type": "object" - }, - "dns": { - "type": "object" - }, - "ftp": { - "type": "object" - }, - "snmp": { - "type": "object" - }, - "ssh": { - "type": "object" - }, - "stun": { - "type": "object" - }, - "softether": { - "type": "object" - }, - "tftp": { - "type": "object" - }, - "tivoconnect": { - "type": "object" - }, - "rsh": { - "type": "object" - } - }, - "additionalProperties": false - }, - "data_analysis": { - "type": "object", - "required": [ "iat", "pktlen", "bins", "directions" ], - - "properties": { - "iat": { - "type": "object", - - "properties": { - "min": { - "type": "number" - }, - "avg": { - "type": "number" - }, - "max": { - "type": "number" - }, - "stddev": { - "type": "number" - }, - "var": { - "type": "number" - }, - "ent": { - "type": "number" - }, - "data": { - "type": "array", - "items": { - "type": "number" - } - } - }, - "additionalProperties": false - }, - "pktlen": { - "type": "object", - - "properties": { - "min": { - "type": "number" - }, - "avg": { - "type": "number" - }, - "max": { - "type": "number" - }, - "stddev": { - "type": "number" - }, - "var": { - "type": "number" - }, - "ent": { - "type": "number" - }, - "data": { - "type": "array", - "items": { - "type": "number" - } - } - }, - "additionalProperties": false - }, - "bins": { - "type": "object", - - "properties": { - "c_to_s": { - "type": "array", - "items": { - "type": "number" - } - }, - "s_to_c": { - "type": "array", - "items": { - "type": "number" - } - } - }, - "additionalProperties": false - }, - "directions": { - "type": "array", - "items": { - "type": "number" - } - }, - "entropies": { - "type": "array", - "items": { - "type": "number" - } - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false -} diff --git a/heidpi-logger-py/schema/geoip2_schema.json b/heidpi-logger-py/schema/geoip2_schema.json deleted file mode 100644 index b8d7847..0000000 --- a/heidpi-logger-py/schema/geoip2_schema.json +++ /dev/null @@ -1,302 +0,0 @@ -{ - "type": "object", - "properties": { - "city": { - "type": "object", - "properties": { - "geoname_id": { - "type": "integer" - }, - "names": { - "type": "object", - "properties": { - "de": { - "type": "string" - }, - "en": { - "type": "string" - }, - "es": { - "type": "string" - }, - "fr": { - "type": "string" - }, - "ja": { - "type": "string" - }, - "pt-BR": { - "type": "string" - }, - "ru": { - "type": "string" - }, - "zh-CN": { - "type": "string" - } - }, - "additionalProperties": true - } - }, - "required": [ - "geoname_id", - "names" - ] - }, - "continent": { - "type": "object", - "properties": { - "code": { - "type": "string" - }, - "geoname_id": { - "type": "integer" - }, - "names": { - "type": "object", - "properties": { - "de": { - "type": "string" - }, - "en": { - "type": "string" - }, - "es": { - "type": "string" - }, - "fr": { - "type": "string" - }, - "ja": { - "type": "string" - }, - "pt-BR": { - "type": "string" - }, - "ru": { - "type": "string" - }, - "zh-CN": { - "type": "string" - } - }, - "additionalProperties": true - } - }, - "required": [ - "code", - "geoname_id", - "names" - ] - }, - "country": { - "type": "object", - "properties": { - "geoname_id": { - "type": "integer" - }, - "iso_code": { - "type": "string" - }, - "names": { - "type": "object", - "properties": { - "de": { - "type": "string" - }, - "en": { - "type": "string" - }, - "es": { - "type": "string" - }, - "fr": { - "type": "string" - }, - "ja": { - "type": "string" - }, - "pt-BR": { - "type": "string" - }, - "ru": { - "type": "string" - }, - "zh-CN": { - "type": "string" - } - }, - "additionalProperties": true - } - }, - "required": [ - "geoname_id", - "iso_code", - "names" - ] - }, - "location": { - "type": "object", - "properties": { - "accuracy_radius": { - "type": "integer" - }, - "latitude": { - "type": "number" - }, - "longitude": { - "type": "number" - }, - "metro_code": { - "type": "integer" - }, - "time_zone": { - "type": "string" - } - }, - "additionalProperties": true, - "required": [ - "accuracy_radius", - "latitude", - "longitude", - "metro_code", - "time_zone" - ] - }, - "postal": { - "type": "object", - "properties": { - "code": { - "type": "string" - } - }, - "additionalProperties": true, - "required": [ - "code" - ] - }, - "registered_country": { - "type": "object", - "properties": { - "geoname_id": { - "type": "integer" - }, - "iso_code": { - "type": "string" - }, - "names": { - "type": "object", - "properties": { - "de": { - "type": "string" - }, - "en": { - "type": "string" - }, - "es": { - "type": "string" - }, - "fr": { - "type": "string" - }, - "ja": { - "type": "string" - }, - "pt-BR": { - "type": "string" - }, - "ru": { - "type": "string" - }, - "zh-CN": { - "type": "string" - } - }, - "additionalProperties": true - } - }, - "required": [ - "geoname_id", - "iso_code", - "names" - ] - }, - "subdivisions": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "geoname_id": { - "type": "integer" - }, - "iso_code": { - "type": "string" - }, - "names": { - "type": "object", - "properties": { - "de": { - "type": "string" - }, - "en": { - "type": "string" - }, - "es": { - "type": "string" - }, - "fr": { - "type": "string" - }, - "ja": { - "type": "string" - }, - "pt-BR": { - "type": "string" - }, - "ru": { - "type": "string" - }, - "zh-CN": { - "type": "string" - } - }, - "additionalProperties": true - } - }, - "required": [ - "geoname_id", - "iso_code", - "names" - ] - } - ] - }, - "traits": { - "type": "object", - "properties": { - "ip_address": { - "type": "string" - }, - "prefix_len": { - "type": "integer" - } - }, - "required": [ - "ip_address", - "prefix_len" - ] - } - }, - "required": [ - "city", - "continent", - "country", - "location", - "postal", - "registered_country", - "subdivisions", - "traits" - ], - "additionalProperties": false -} \ No newline at end of file diff --git a/heidpi-logger-py/schema/packet_event_schema.json b/heidpi-logger-py/schema/packet_event_schema.json deleted file mode 100644 index ad7416a..0000000 --- a/heidpi-logger-py/schema/packet_event_schema.json +++ /dev/null @@ -1,116 +0,0 @@ -{ - "type": "object", - "required": [ - "alias", - "source", - "packet_id", - "packet_event_id", - "packet_event_name", - "pkt_caplen", - "pkt_type", - "pkt_l3_offset", - "pkt_l4_offset", - "pkt_len", - "pkt_l4_len", - "thread_ts_usec" - ], - - "dependencies" : { - "flow_id" : [ "flow_packet_id", "flow_src_last_pkt_time", "flow_dst_last_pkt_time", "flow_idle_time" ] - }, - - "if": { - "properties": { "packet_event_name": { "enum": ["packet-flow"] } } - }, - "then": { - "required": [ "thread_id", "flow_id", "flow_packet_id", "flow_src_last_pkt_time", "flow_dst_last_pkt_time", "flow_idle_time" ] - }, - "else": { - "not": { "required": [ "thread_id", "flow_id", "flow_packet_id", "flow_src_last_pkt_time", "flow_dst_last_pkt_time", "flow_idle_time" ] } - }, - - "properties": { - "alias": { - "type": "string" - }, - "source": { - "type": "string" - }, - "thread_id": { - "type": "number" - }, - "packet_id": { - "type": "number", - "minimum": 0 - }, - "packet_event_id": { - "type": "number", - "minimum": 0, - "maximum": 2 - }, - "packet_event_name": { - "type": "string", - "enum": [ - "invalid", - "packet", - "packet-flow" - ] - }, - "flow_id": { - "type": "number", - "minimum": 1 - }, - "flow_packet_id": { - "type": "number" - }, - "flow_src_last_pkt_time": { - "type": "number", - "minimum": 0 - }, - "flow_dst_last_pkt_time": { - "type": "number", - "minimum": 0 - }, - "flow_idle_time": { - "type": "number", - "minimum": 1 - }, - "pkt_caplen": { - "type": "number", - "minimum": 1, - "maximum": 65535 - }, - "pkt_type": { - "type": "number", - "minimum": 0, - "maximum": 65535 - }, - "pkt_l3_offset": { - "type": "number", - "minimum": 0, - "maximum": 65535 - }, - "pkt_l4_len": { - "type": "number", - "minimum": 0, - "maximum": 65535 - }, - "thread_ts_usec": { - "type": "number", - "minimum": 0 - }, - "pkt_l4_offset": { - "type": "number", - "minimum": 0, - "maximum": 65535 - }, - "pkt_len": { - "type": "number", - "minimum": 0 - }, - "pkt": { - "type": "string" - } - }, - "additionalProperties": false -} diff --git a/heidpi-logger-py/version.py b/heidpi-logger-py/version.py deleted file mode 100644 index 4818072..0000000 --- a/heidpi-logger-py/version.py +++ /dev/null @@ -1,8 +0,0 @@ -""" Store the version here so: -# 1) we don't load dependencies by storing it in __init__.py -# 2) we can import it in setup.py for the same reason -# 3) we can import it into your module module -""" - -__version_info__ = ("1", "3", "0") -__version__ = ".".join(__version_info__) diff --git a/heidpi-logger-cpp/CMakeLists.txt b/heidpi-logger/CMakeLists.txt similarity index 100% rename from heidpi-logger-cpp/CMakeLists.txt rename to heidpi-logger/CMakeLists.txt diff --git a/heidpi-logger-cpp/include/Config.hpp b/heidpi-logger/include/Config.hpp similarity index 100% rename from heidpi-logger-cpp/include/Config.hpp rename to heidpi-logger/include/Config.hpp diff --git a/heidpi-logger-cpp/include/EventProcessor.hpp b/heidpi-logger/include/EventProcessor.hpp similarity index 100% rename from heidpi-logger-cpp/include/EventProcessor.hpp rename to heidpi-logger/include/EventProcessor.hpp diff --git a/heidpi-logger-cpp/include/GeoIP.hpp b/heidpi-logger/include/GeoIP.hpp similarity index 100% rename from heidpi-logger-cpp/include/GeoIP.hpp rename to heidpi-logger/include/GeoIP.hpp diff --git a/heidpi-logger-cpp/include/Logger.hpp b/heidpi-logger/include/Logger.hpp similarity index 100% rename from heidpi-logger-cpp/include/Logger.hpp rename to heidpi-logger/include/Logger.hpp diff --git a/heidpi-logger-cpp/include/NDPIClient.hpp b/heidpi-logger/include/NDPIClient.hpp similarity index 100% rename from heidpi-logger-cpp/include/NDPIClient.hpp rename to heidpi-logger/include/NDPIClient.hpp diff --git a/heidpi-logger-cpp/src/Config.cpp b/heidpi-logger/src/Config.cpp similarity index 100% rename from heidpi-logger-cpp/src/Config.cpp rename to heidpi-logger/src/Config.cpp diff --git a/heidpi-logger-cpp/src/EventProcessor.cpp b/heidpi-logger/src/EventProcessor.cpp similarity index 100% rename from heidpi-logger-cpp/src/EventProcessor.cpp rename to heidpi-logger/src/EventProcessor.cpp diff --git a/heidpi-logger-cpp/src/GeoIP.cpp b/heidpi-logger/src/GeoIP.cpp similarity index 100% rename from heidpi-logger-cpp/src/GeoIP.cpp rename to heidpi-logger/src/GeoIP.cpp diff --git a/heidpi-logger-cpp/src/Logger.cpp b/heidpi-logger/src/Logger.cpp similarity index 100% rename from heidpi-logger-cpp/src/Logger.cpp rename to heidpi-logger/src/Logger.cpp diff --git a/heidpi-logger-cpp/src/NDPIClient.cpp b/heidpi-logger/src/NDPIClient.cpp similarity index 100% rename from heidpi-logger-cpp/src/NDPIClient.cpp rename to heidpi-logger/src/NDPIClient.cpp diff --git a/heidpi-logger-cpp/src/main.cpp b/heidpi-logger/src/main.cpp similarity index 100% rename from heidpi-logger-cpp/src/main.cpp rename to heidpi-logger/src/main.cpp diff --git a/heidpi-rust/Cargo.lock b/heidpi-rust/Cargo.lock deleted file mode 100644 index 9d5b31e..0000000 --- a/heidpi-rust/Cargo.lock +++ /dev/null @@ -1,1180 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "addr" -version = "0.15.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a93b8a41dbe230ad5087cc721f8d41611de654542180586b315d9f4cf6b72bef" -dependencies = [ - "psl", - "psl-types", -] - -[[package]] -name = "addr2line" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - -[[package]] -name = "aho-corasick" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" -dependencies = [ - "memchr", -] - -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] -name = "anstream" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" - -[[package]] -name = "anstyle-parse" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" -dependencies = [ - "windows-sys 0.52.0", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" -dependencies = [ - "anstyle", - "windows-sys 0.52.0", -] - -[[package]] -name = "anyhow" -version = "1.0.79" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" - -[[package]] -name = "autocfg" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - -[[package]] -name = "backtrace" -version = "0.3.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" -dependencies = [ - "addr2line", - "cc", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", -] - -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - -[[package]] -name = "bit-set" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bumpalo" -version = "3.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d32a994c2b3ca201d9b263612a374263f05e7adde37c4707f693dcd375076d1f" - -[[package]] -name = "bytes" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" - -[[package]] -name = "cc" -version = "1.0.83" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" -dependencies = [ - "libc", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "chrono" -version = "0.4.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" -dependencies = [ - "android-tzdata", - "iana-time-zone", - "num-traits", - "windows-targets 0.52.0", -] - -[[package]] -name = "clap" -version = "4.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80c21025abd42669a92efc996ef13cfb2c5c627858421ea58d5c3b331a6c134f" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458bf1f341769dfcf849846f65dffdf9146daa56bcd2a47cb4e1de9915567c99" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim", -] - -[[package]] -name = "clap_derive" -version = "4.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "307bc0538d5f0f83b8248db3087aa92fe504e4691294d0c96c0eabc33f47ba47" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "clap_lex" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" - -[[package]] -name = "colorchoice" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" - -[[package]] -name = "core-foundation-sys" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" - -[[package]] -name = "downcast-rs" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" - -[[package]] -name = "env_filter" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" -dependencies = [ - "log", - "regex", -] - -[[package]] -name = "env_logger" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c012a26a7f605efc424dd53697843a72be7dc86ad2d01f7814337794a12231d" -dependencies = [ - "anstream", - "anstyle", - "env_filter", - "humantime", - "log", -] - -[[package]] -name = "equivalent" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" - -[[package]] -name = "erased-serde" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c138974f9d5e7fe373eb04df7cae98833802ae4b11c24ac7039a21d5af4b26c" -dependencies = [ - "serde", -] - -[[package]] -name = "fancy-regex" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b95f7c0680e4142284cf8b22c14a476e87d61b004a3a0861872b32ef7ead40a2" -dependencies = [ - "bit-set", - "regex", -] - -[[package]] -name = "form_urlencoded" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "getrandom" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "gimli" -version = "0.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" - -[[package]] -name = "hashbrown" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - -[[package]] -name = "heidpi" -version = "0.1.0" -dependencies = [ - "anyhow", - "clap", - "env_logger", - "log", - "maxminddb", - "serde", - "serde_derive", - "serde_json", - "tokio", - "toml", - "valico", -] - -[[package]] -name = "hermit-abi" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd5256b483761cd23699d0da46cc6fd2ee3be420bbe6d020ae4a091e70b7e9fd" - -[[package]] -name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - -[[package]] -name = "iana-time-zone" -version = "0.1.60" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "idna" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - -[[package]] -name = "indexmap" -version = "2.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" -dependencies = [ - "equivalent", - "hashbrown", -] - -[[package]] -name = "ipnetwork" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e" -dependencies = [ - "serde", -] - -[[package]] -name = "itoa" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" - -[[package]] -name = "js-sys" -version = "0.3.68" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" -dependencies = [ - "wasm-bindgen", -] - -[[package]] -name = "json-pointer" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fe841b94e719a482213cee19dd04927cf412f26d8dc84c5a446c081e49c2997" -dependencies = [ - "serde_json", -] - -[[package]] -name = "jsonway" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "effcb749443c905fbaef49d214f8b1049c240e0adb7af9baa0e201e625e4f9de" -dependencies = [ - "serde", - "serde_json", -] - -[[package]] -name = "libc" -version = "0.2.153" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" - -[[package]] -name = "lock_api" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" - -[[package]] -name = "maxminddb" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6087e5d8ea14861bb7c7f573afbc7be3798d3ef0fae87ec4fd9a4de9a127c3c" -dependencies = [ - "ipnetwork", - "log", - "memchr", - "serde", -] - -[[package]] -name = "memchr" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" - -[[package]] -name = "miniz_oxide" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" -dependencies = [ - "adler", -] - -[[package]] -name = "mio" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" -dependencies = [ - "libc", - "wasi", - "windows-sys 0.48.0", -] - -[[package]] -name = "num-traits" -version = "0.2.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" -dependencies = [ - "autocfg", -] - -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi", - "libc", -] - -[[package]] -name = "object" -version = "0.32.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" -dependencies = [ - "memchr", -] - -[[package]] -name = "once_cell" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" - -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets 0.48.5", -] - -[[package]] -name = "percent-encoding" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" - -[[package]] -name = "phf" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" -dependencies = [ - "phf_shared", -] - -[[package]] -name = "phf_codegen" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" -dependencies = [ - "phf_generator", - "phf_shared", -] - -[[package]] -name = "phf_generator" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" -dependencies = [ - "phf_shared", - "rand", -] - -[[package]] -name = "phf_shared" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" -dependencies = [ - "siphasher", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" - -[[package]] -name = "proc-macro2" -version = "1.0.78" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "psl" -version = "2.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cc7ffe15173de4c22def678345bb2ac7e1646b83a37c1484572732b5a79a49e" -dependencies = [ - "psl-types", -] - -[[package]] -name = "psl-types" -version = "2.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" - -[[package]] -name = "quote" -version = "1.0.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" - -[[package]] -name = "redox_syscall" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" -dependencies = [ - "bitflags", -] - -[[package]] -name = "regex" -version = "1.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" - -[[package]] -name = "rustc-demangle" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" - -[[package]] -name = "ryu" -version = "1.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "serde" -version = "1.0.196" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.196" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.113" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" -dependencies = [ - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_spanned" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" -dependencies = [ - "serde", -] - -[[package]] -name = "signal-hook-registry" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" -dependencies = [ - "libc", -] - -[[package]] -name = "siphasher" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" - -[[package]] -name = "smallvec" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" - -[[package]] -name = "socket2" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" -dependencies = [ - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "strsim" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" - -[[package]] -name = "syn" -version = "2.0.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "tinyvec" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokio" -version = "1.36.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" -dependencies = [ - "backtrace", - "bytes", - "libc", - "mio", - "num_cpus", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "socket2", - "tokio-macros", - "windows-sys 0.48.0", -] - -[[package]] -name = "tokio-macros" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "toml" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a9aad4a3066010876e8dcf5a8a06e70a558751117a145c6ce2b82c2e2054290" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - -[[package]] -name = "toml_datetime" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c1b5fd4128cc8d3e0cb74d4ed9a9cc7c7284becd4df68f5f940e1ad123606f6" -dependencies = [ - "indexmap", - "serde", - "serde_spanned", - "toml_datetime", - "winnow", -] - -[[package]] -name = "unicode-bidi" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" - -[[package]] -name = "unicode-ident" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" - -[[package]] -name = "unicode-normalization" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "uritemplate-next" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcde98d1fc3f528255b1ecb22fb688ee0d23deb672a8c57127df10b98b4bd18c" -dependencies = [ - "regex", -] - -[[package]] -name = "url" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", -] - -[[package]] -name = "utf8parse" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" - -[[package]] -name = "uuid" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" -dependencies = [ - "getrandom", -] - -[[package]] -name = "valico" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8a0a4df97f827fcbcbe69c65364acddddf3a4bb50e6507f63361177a7ea7a4" -dependencies = [ - "addr", - "base64", - "chrono", - "downcast-rs", - "erased-serde", - "fancy-regex", - "json-pointer", - "jsonway", - "percent-encoding", - "phf", - "phf_codegen", - "serde", - "serde_json", - "uritemplate-next", - "url", - "uuid", -] - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "wasm-bindgen" -version = "0.2.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" -dependencies = [ - "cfg-if", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" -dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" - -[[package]] -name = "windows-core" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" -dependencies = [ - "windows-targets 0.52.0", -] - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets 0.52.0", -] - -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - -[[package]] -name = "windows-targets" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" -dependencies = [ - "windows_aarch64_gnullvm 0.52.0", - "windows_aarch64_msvc 0.52.0", - "windows_i686_gnu 0.52.0", - "windows_i686_msvc 0.52.0", - "windows_x86_64_gnu 0.52.0", - "windows_x86_64_gnullvm 0.52.0", - "windows_x86_64_msvc 0.52.0", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" - -[[package]] -name = "winnow" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d90f4e0f530c4c69f62b80d839e9ef3855edc9cba471a160c4d692deed62b401" -dependencies = [ - "memchr", -] diff --git a/heidpi-rust/Cargo.toml b/heidpi-rust/Cargo.toml deleted file mode 100644 index 75ae2c3..0000000 --- a/heidpi-rust/Cargo.toml +++ /dev/null @@ -1,26 +0,0 @@ -[package] -name = "heidpi" -authors = ["Stefan Machmeier"] -license = "EUPL" -license-file = "LICENSE" -readme = "README.md" -repository = "https://github.com/stefanDeveloper/heidpi" -description = "nDPId Docker Image for packet inspection" -version = "0.1.0" -edition = "2021" - -[dependencies] -tokio = { version = "*", features = ["full"] } -serde = { version = "*", features = ["derive"] } -serde_derive = "*" -serde_json = { version = "*" } -log = { version = "*" } -env_logger = { version = "*" } -toml = { version = "*" } -clap = { version = "*", features = ["derive"] } -anyhow = { version = "*" } -maxminddb = { version = "*" } -valico = { version = "*" } - -[build-dependencies] -anyhow = { version = "*" } \ No newline at end of file diff --git a/heidpi-rust/Dockerfile b/heidpi-rust/Dockerfile deleted file mode 100644 index 6603369..0000000 --- a/heidpi-rust/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -# Build the Rust application -FROM rust:1.76.0 as builder -WORKDIR /usr/src/heidpi-rust -COPY . . -RUN cargo install --path . - -# Set up the runtime environment -FROM debian:bookworm-slim -RUN apt-get update && apt-get install -y extra-runtime-dependencies glibc & rm -rf /var/lib/apt/lists/* -COPY --from=builder /usr/local/cargo/bin/heidpi /usr/local/bin/heidpi - -CMD ["heidpi", "start", "--host", "127.0.0.1", "--port", "7000"] \ No newline at end of file diff --git a/heidpi-rust/src/cli.rs b/heidpi-rust/src/cli.rs deleted file mode 100644 index 71d232e..0000000 --- a/heidpi-rust/src/cli.rs +++ /dev/null @@ -1,171 +0,0 @@ -/** - * Handles user input from the command line, parses it, and routes it to the appropriate functions. - * It supports operations like starting the application with specific configurations or running in different modes - * (e.g. server mode, pcap file processing mode) - */ - -use anyhow::{bail, ensure}; // Error handling, in this case: return early with an error and if a condition is not met -use clap::Parser; // Parse command-line arguments into Self -use std::path::PathBuf; // path manipulation - -use tokio::sync::OnceCell; // Thread safe cell that can be written to only once - -use crate::stream; -use crate::geoip::GeoIP; - -use crate::config::Config; -use crate::config::LoggingConfig; -use crate::config::Event; -use crate::logging::Logging; - -static ONCON: OnceCell = OnceCell::const_new(); - - -#[derive(Parser, Debug)] -#[command(author, version, about, long_about)] -pub enum Cli { - #[allow(rustdoc::broken_intra_doc_links)] - #[allow(rustdoc::invalid_html_tags)] - Start { - #[clap(short, long)] - config_file: Option, - - /// nDPIsrvd host IP - #[clap(long)] - host: String, - /// nDPIsrvd TCP port - #[clap(long)] - port: String, - - /// where to write log files - #[clap(short, long)] - write: Option, - - /// Enable daemon events - #[clap(long, default_value_t=false)] - daemon_events: bool, - /// Enable packet events - #[clap(long, default_value_t=false)] - packet_events: bool, - /// Enable error events - #[clap(long, default_value_t=false)] - error_events: bool, - /// Enable flow events - #[clap(long, default_value_t=true)] - flow_events: bool, - }, - - Man, -} - -impl Cli { - pub async fn run() -> anyhow::Result<()> { - let cli = Self::parse(); - use Cli::*; - match cli { - Man => { - let man_cmd = std::process::Command::new("man") - .args(["1", "heidpi"]) - .status(); - - // if !(man_cmd.is_ok() && man_cmd.unwrap().success()) { - // println!(include_str!(env!("HEIDPI_MAN"))); - // } - } - Start { - config_file, - - host, - port, - - write, - - daemon_events, - packet_events, - error_events, - flow_events, - } => { - // TODO Handle parameters - // TODO Handle config, should be global accessible - //let mut v = stream::connect("127.0.0.1:7000").await; // Because it should be global accessible, we're not using this hard coded version - - // if a configuration file is provided, it is read and parsed. Otherwise default values are used. - let config = if let Some(config_path) = config_file{ - Config::from_file(&config_path)? - } else{ - Config::new( - LoggingConfig { - level: "info".to_string(), - encoding: "utf-8".to_string(), - format: "plain".to_string(), - datefmt: "%Y-%m-%d %H:%M:%S".to_string(), - }, - Event { - ignore_fields: vec![], - ignore_risks: vec![], - flow_event_name: vec![], - geoip: None, - filename: "flow.log".to_string(), - }, - Event { - ignore_fields: vec![], - ignore_risks: vec![], - flow_event_name: vec![], - geoip: None, - filename: "daemon.log".to_string(), - }, - Event { - ignore_fields: vec![], - ignore_risks: vec![], - flow_event_name: vec![], - geoip: None, - filename: "packet.log".to_string(), - }, - Event { - ignore_fields: vec![], - ignore_risks: vec![], - flow_event_name: vec![], - geoip: None, - filename: "error.log".to_string(), - }, - ) - }; - - ONCON.set(config).unwrap(); - - //Access global configuration - let config = ONCON.get().unwrap(); - println!("Using configuration: {:?}", config); - - // Create loggers for each event type - let flow_logger = Logging::new("flow".to_string(), PathBuf::from(&config.flow_event.filename)); - let daemon_logger = Logging::new("daemon".to_string(), PathBuf::from(&config.daemon_event.filename)); - let packet_logger = Logging::new("packet".to_string(), PathBuf::from(&config.packet_event.filename)); - let error_logger = Logging::new("error".to_string(), PathBuf::from(&config.error_event.filename)); - - // Initialize GeoIP if necessary - let geoip = if let Some(geoip_config) = &config.flow_event.geoip { - if geoip_config.enabled { - Some(GeoIP::new(&geoip_config.filepath)?) - } else { - None - } - } else { - None - }; - - // Connect to the stream and process data - stream::connect( - &format!("{}:{}", host, port), - geoip, - flow_logger, - daemon_logger, - packet_logger, - error_logger, - ).await?; - } - } - - Ok(()) - } -} diff --git a/heidpi-rust/src/config.rs b/heidpi-rust/src/config.rs deleted file mode 100644 index 6d6a7e6..0000000 --- a/heidpi-rust/src/config.rs +++ /dev/null @@ -1,66 +0,0 @@ - -/** - * Defines configuration structures and functions to load configurations from files. - * This file ensures that all parts of the application can access configuration settings in a structured manner. -*/ -use serde::Deserialize; - -use anyhow::Result; -use std::fs; - -#[derive(Debug, Deserialize)] -pub struct Config { - pub logging: LoggingConfig, - pub flow_event: Event, - pub daemon_event: Event, - pub packet_event: Event, - pub error_event: Event, -} - -#[derive(Debug, Deserialize)] -pub struct LoggingConfig { - pub level: String, - pub encoding: String, - pub format: String, - pub datefmt: String, -} - -#[derive(Debug, Deserialize)] -pub struct Event { - pub ignore_fields: Vec, - pub ignore_risks: Vec, - pub flow_event_name: Vec, - pub geoip: Option, - pub filename: String, -} - -#[derive(Debug, Deserialize)] -pub struct GeoIP { - pub enabled: bool, - pub filepath: String, - pub keys: Vec, -} - -impl Config { - pub fn from_file(path: &std::path::Path) -> Result { - let config_str = fs::read_to_string(path)?; - let config = toml::from_str(&config_str)?; - Ok(config) - } - - pub fn new( - logging: LoggingConfig, - flow_event: Event, - daemon_event: Event, - packet_event: Event, - error_event: Event, - ) -> Self { - Config { - logging, - flow_event, - daemon_event, - packet_event, - error_event, - } - } -} diff --git a/heidpi-rust/src/geoip.rs b/heidpi-rust/src/geoip.rs deleted file mode 100644 index 5ed2ff9..0000000 --- a/heidpi-rust/src/geoip.rs +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Implements functionality to read from a GeoIP database and fetch geographical information based on IP addresses. - */ - -use std::net::IpAddr; -use anyhow::{Result, Ok}; -use maxminddb::{geoip2, Reader}; // Reader for the MaxMind DB format. Reading the contents of the database files -use serde::{Deserialize, Serialize}; - -use std::sync::{Arc, Mutex}; // For Mutex - -/// . -// TODO create geoip2 object, be aware multiple threads will read this object, check Tokio how to do it! (Mutex...) -pub struct GeoIP{ - reader: Arc>>>, -} - -#[derive(Debug, Deserialize, Serialize)] -pub struct CityInfo { - pub city_name: Option, - pub country_name: Option, - pub latitude: Option, - pub longitude: Option, -} - - -impl GeoIP { - - // Create a new GeoIP object - pub fn new(db_path: &str)-> anyhow::Result { - let db_reader = Reader::open_readfile(db_path)?; - Ok(Self{ - reader: Arc::new(Mutex::new(db_reader)), - } - - ) - } - - // Get geolocation information for a given IP address - pub fn get_geoip(&self, ip: &IpAddr) -> anyhow::Result { - //let reader = maxminddb::Reader::open_readfile("test-data/test-data/GeoIP2-City-Test.mmdb").unwrap(); - let reader = self.reader.lock().unwrap(); - let city: geoip2::City = reader.lookup(*ip)?; - - // Extract and copy the relevant data to CityInfo - let city_name = city.city.as_ref() - .and_then(|city| city.names.as_ref().and_then(|names| names.get("en").map(|s| s.to_string()))); - let country_name = city.country.as_ref() - .and_then(|country| country.names.as_ref().and_then(|names| names.get("en").map(|s| s.to_string()))); - let location = city.location; - let latitude = location.as_ref().and_then(|loc| loc.latitude); - let longitude = location.as_ref().and_then(|loc| loc.longitude); - - Ok(CityInfo { - city_name, - country_name, - latitude, - longitude, - }) - } -} \ No newline at end of file diff --git a/heidpi-rust/src/logging.rs b/heidpi-rust/src/logging.rs deleted file mode 100644 index 0d49479..0000000 --- a/heidpi-rust/src/logging.rs +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Provides a logging mechanism that writes logs to files. it defines a Logging struct and implements methods to write logs safely, - * ensuring that log files are correctly managed and closed after writing. - */ - -use anyhow::bail; -use std::io::prelude::*; -use std::{fs::OpenOptions, path::PathBuf}; - -// TODO Generate logging struct (Be aware it should generic to be used for daemon, packet, flow and error) -// Generate logging struct (being generic to be used for daemon, packet, flow, and error) -#[derive(Clone)] -pub struct Logging { - log_type: String, - filepath: PathBuf, -} - - -// TODO Write file (Be aware when file is opened, don't forget to close it) -impl Logging { - - //Create new Logging instance - pub fn new(log_type: String, filepath: PathBuf) -> Self { - Logging {log_type, filepath} - } - - // Write data to the log file (Be aware when the file is opened, don't forget to close it) - pub fn write(&self, data: &str, filepath: PathBuf) -> anyhow::Result<()> { - let mut file = OpenOptions::new() - .write(true) - .append(true) - .create(true) - .open(filepath) - .unwrap(); - - if let Err(e) = writeln!(file, "{}", data) { - bail!("Couldn't write to file: {}", e); - } - - file.flush()?; - // File is automatically closed here when file goes out of scope - - Ok(()) - } -} diff --git a/heidpi-rust/src/main.rs b/heidpi-rust/src/main.rs deleted file mode 100644 index e69a418..0000000 --- a/heidpi-rust/src/main.rs +++ /dev/null @@ -1,28 +0,0 @@ -pub mod cli; -pub mod logging; -pub mod stream; -pub mod process; -pub mod geoip; -pub mod config; - -use log::{error, info}; -use cli::Cli; -use std::process::exit; -use geoip::GeoIP; - -/// Catches errors, prints them through the logger, then exits -#[tokio::main] -pub async fn main() { - //default to displaying warning and error log messages only - env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("trace")).init(); - - match Cli::run().await { - Ok(_) => { - info!("Application exited successfully."); - } - Err(e) => { - error!("{e}"); - exit(1); - } - } -} \ No newline at end of file diff --git a/heidpi-rust/src/process.rs b/heidpi-rust/src/process.rs deleted file mode 100644 index 9286a05..0000000 --- a/heidpi-rust/src/process.rs +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Contains the core logic got processing data, such as filtering JSON objects or adding geolocation information. - * It provides functions to transform and filter data according to the configuration settings. - */ -use std::net::IpAddr; - -use serde_json::Value; - -use crate::geoip::GeoIP; -use crate::config::Event; - -pub fn process(json: Value, geoip: Option<&GeoIP>, event_config: &Event) -> Value { - let mut json = json; - - // TODO remove risk in ignore - json = remove_risks(json, &event_config.ignore_risks); - // TODO remove attributes - json = remove_attributes(json, &event_config.ignore_fields); - // TODO remove event types - json = remove_event_types(json, &event_config.flow_event_name); - // TODO get geoip of string (if boolean is set) - if let Some(geoip) = geoip { - json = add_geoip_info(json, geoip); - } - - // return processed string - json -} - -fn remove_risks(mut json: Value, ignore_risks: &[String]) -> Value { - if let Some(obj) = json.as_object_mut() { - for risk in ignore_risks { - obj.remove(risk); - } - } - json -} - -fn remove_attributes(mut json: Value, ignore_fields: &[String]) -> Value { - if let Some(obj) = json.as_object_mut() { - for field in ignore_fields { - obj.remove(field); - } - } - json -} - -fn remove_event_types(mut json: Value, ignore_event_types: &[String]) -> Value { - if let Some(obj) = json.as_object_mut() { - for event_type in ignore_event_types { - obj.remove(event_type); - } - } - json -} - -fn add_geoip_info(mut json: Value, geoip: &GeoIP) -> Value { - if let Some(obj) = json.as_object_mut() { - if let Some(ip_str) = obj.get("ip").and_then(|ip| ip.as_str()) { - if let Ok(ip) = ip_str.parse::() { - if let Ok(city) = geoip.get_geoip(&ip) { - obj.insert("geoip".to_string(), serde_json::json!(city)); - } - } - } - } - json -} - diff --git a/heidpi-rust/src/schema/README.md b/heidpi-rust/src/schema/README.md deleted file mode 100644 index 186dc4d..0000000 --- a/heidpi-rust/src/schema/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# schema - -All schema's placed in here are nDPId exclusive, meaning that they are not necessarily representing a "real-world" JSON string received by e.g. `./example/py-json-stdout`. -This is due to the fact that libnDPI itself add's some JSON information to the serializer of which we have no control over. -IMHO it makes no sense to include stuff here that is part of libnDPI. diff --git a/heidpi-rust/src/schema/daemon_event_schema.json b/heidpi-rust/src/schema/daemon_event_schema.json deleted file mode 100644 index 91f3875..0000000 --- a/heidpi-rust/src/schema/daemon_event_schema.json +++ /dev/null @@ -1,166 +0,0 @@ -{ - "type": "object", - "required": [ - "alias", - "source", - "thread_id", - "packet_id", - "daemon_event_id", - "daemon_event_name", - "global_ts_usec" - ], - "if": { - "properties": { "daemon_event_name": { "enum": [ "init", "reconnect" ] } } - }, - "then": { - "required": [ "max-flows-per-thread", "max-idle-flows-per-thread", "reader-thread-count", "flow-scan-interval", "generic-max-idle-time", "icmp-max-idle-time", "udp-max-idle-time", "tcp-max-idle-time", "max-packets-per-flow-to-send", "max-packets-per-flow-to-process", "max-packets-per-flow-to-analyse" ] - }, - "if": { - "properties": { "daemon_event_name": { "enum": [ "status", "shutdown" ] } } - }, - "then": { - "required": [ "packets-captured", "packets-processed", "total-skipped-flows", "total-l4-payload-len", "total-not-detected-flows", "total-guessed-flows", "total-detected-flows", "total-detection-updates", "total-updates", "current-active-flows", "total-active-flows", "total-idle-flows", "total-compressions", "total-compression-diff", "current-compression-diff", "total-events-serialized" ] - }, - "properties": { - "alias": { - "type": "string" - }, - "source": { - "type": "string" - }, - "thread_id": { - "type": "number", - "minimum": 0, - "maximum": 31 - }, - "packet_id": { - "type": "number", - "minimum": 0 - }, - "daemon_event_id": { - "type": "number", - "minimum": 0, - "maximum": 4 - }, - "daemon_event_name": { - "type": "string", - "enum": [ - "invalid", - "init", - "reconnect", - "shutdown", - "status" - ] - }, - - "max-flows-per-thread": { - "type": "number" - }, - "max-idle-flows-per-thread": { - "type": "number" - }, - "reader-thread-count": { - "type": "number" - }, - "flow-scan-interval": { - "type": "number" - }, - "generic-max-idle-time": { - "type": "number" - }, - "icmp-max-idle-time": { - "type": "number" - }, - "udp-max-idle-time": { - "type": "number" - }, - "tcp-max-idle-time": { - "type": "number" - }, - "max-packets-per-flow-to-process": { - "type": "number" - }, - "max-packets-per-flow-to-send": { - "type": "number" - }, - "max-packets-per-flow-to-analyse": { - "type": "number" - }, - - "packets-captured": { - "type": "number", - "minimum": 0 - }, - "packets-processed": { - "type": "number", - "minimum": 0 - }, - "total-skipped-flows": { - "type": "number", - "minimum": 0 - }, - "total-l4-payload-len": { - "type": "number", - "minimum": 0 - }, - "total-not-detected-flows": { - "type": "number", - "minimum": 0 - }, - "total-guessed-flows": { - "type": "number", - "minimum": 0 - }, - "total-detected-flows": { - "type": "number", - "minimum": 0 - }, - "total-detection-updates": { - "type": "number", - "minimum": 0 - }, - "total-updates": { - "type": "number", - "minimum": 0 - }, - "current-active-flows": { - "type": "number", - "minimum": 0 - }, - "total-active-flows": { - "type": "number", - "minimum": 0 - }, - "total-idle-flows": { - "type": "number", - "minimum": 0 - }, - "total-compressions": { - "type": "number", - "minimum": 0 - }, - "total-compression-diff": { - "type": "number", - "minimum": 0 - }, - "current-compression-diff": { - "type": "number", - "minimum": 0 - }, - "total-events-serialized": { - "type": "number", - "minimum": 1 - }, - "global_ts_usec": { - "type": "number", - "if": { - "properties": { "daemon_event_name": { "enum": [ "init" ] } } - }, - "then" : true, - "else" : { - "minimum": 1000000 - } - } - }, - "additionalProperties": false -} diff --git a/heidpi-rust/src/schema/error_event_schema.json b/heidpi-rust/src/schema/error_event_schema.json deleted file mode 100644 index b81140e..0000000 --- a/heidpi-rust/src/schema/error_event_schema.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "type": "object", - "required": [ - "alias", - "source", - "packet_id", - "error_event_id", - "error_event_name", - "datalink", - "threshold_n", - "threshold_n_max", - "threshold_time", - "threshold_ts_usec", - "global_ts_usec" - ], - - "if": { - "properties": { "error_event_name": { "enum": [ "Unknown datalink layer packet", "Unknown packet type" ] } } - }, - "then": { - "anyOf": [ - { "required": [ "layer_type" ] }, - { "not": { "required": [ "thread_id" ] } } - ] - }, - - "if": { - "properties": { "error_event_name": { "enum": [ "Unknown L3 protocol" ] } } - }, - "then": { - "anyOf": [ - { "required": [ "protocol" ] }, - { "not": { "required": [ "thread_id" ] } } - ] - }, - - "if": { - "properties": { "error_event_name": { "enum": [ "Packet too short", "IP4 packet too short", - "IP6 packet too short", "TCP packet smaller than expected", - "UDP packet smaller than expected", - "Captured packet size is smaller than expected packet size" ] } } - }, - "then": { - "anyOf": [ - { "required": [ "size", "expected" ] }, - { "not": { "required": [ "thread_id" ] } } - ] - }, - - "if": { - "properties": { "error_event_name": { "enum": [ "Packet header invalid" ] } } - }, - "then": { - "anyOf": [ - { "required": [ "raeson" ] }, - { "not": { "required": [ "thread_id" ] } } - ] - }, - - "if": { - "properties": { "error_event_name": { "enum": [ "Flow memory allocation failed" ] } } - }, - "then": { - "required": [ "thread_id", "size" ] - }, - - "if": { - "properties": { "error_event_name": { "enum": [ "Max flows to track reached" ] } } - }, - "then": { - "required": [ "thread_id", "current_active", "current_idle", "max_active", "max_idle" ] - }, - - "properties": { - "alias": { - "type": "string" - }, - "source": { - "type": "string" - }, - "thread_id": { - "type": "number" - }, - "packet_id": { - "type": "number", - "minimum": 0 - }, - "error_event_id": { - "type": "number", - "minimum": 0, - "maximum": 16 - }, - "error_event_name": { - "type": "string", - "enum": [ - "Unknown datalink layer packet", - "Unknown L3 protocol", - "Unsupported datalink layer", - "Packet too short", - "Unknown packet type", - "Packet header invalid", - "IP4 packet too short", - "Packet smaller than IP4 header", - "nDPI IPv4/L4 payload detection failed", - "IP6 packet too short", - "Packet smaller than IP6 header", - "nDPI IPv6/L4 payload detection failed", - "TCP packet smaller than expected", - "UDP packet smaller than expected", - "Captured packet size is smaller than expected packet size", - "Max flows to track reached", - "Flow memory allocation failed" - ] - }, - - "datalink": { - "type": "number", - "minimum": 0, - "maximum": 292 - }, - - "threshold_n": { - "type": "number", - "minimum": 1 - }, - - "threshold_n_max": { - "type": "number", - "minimum": 1, - "maximum": 65535 - }, - - "threshold_time": { - "type": "number" - }, - - "threshold_ts_usec": { - "type": "number" - }, - - "layer_type": { - "type": "number", - "minimum": 0 - }, - - "l4_data_len": { - "type": "number", - "minimum": 0 - }, - - "reason": { - "type": "string" - }, - - "protocol": { - "type": "number", - "minimum": 0, - "maximum": 65535 - }, - - "size": { - "type": "number" - }, - "expected": { - "type": "number" - }, - - "current_active": { - "type": "number" - }, - "current_idle": { - "type": "number" - }, - "max_active": { - "type": "number" - }, - "max_idle": { - "type": "number" - }, - "global_ts_usec": { - "type": "number", - "minimum": 0 - } - }, - "additionalProperties": false -} diff --git a/heidpi-rust/src/schema/flow_event_schema.json b/heidpi-rust/src/schema/flow_event_schema.json deleted file mode 100644 index 01cc9d9..0000000 --- a/heidpi-rust/src/schema/flow_event_schema.json +++ /dev/null @@ -1,472 +0,0 @@ -{ - "type": "object", - "required": [ - "alias", - "source", - "thread_id", - "packet_id", - "flow_event_id", - "flow_event_name", - "flow_id", - "flow_state", - "flow_src_packets_processed", - "flow_dst_packets_processed", - "flow_first_seen", - "flow_src_last_pkt_time", - "flow_dst_last_pkt_time", - "flow_idle_time", - "flow_src_min_l4_payload_len", - "flow_dst_min_l4_payload_len", - "flow_src_max_l4_payload_len", - "flow_dst_max_l4_payload_len", - "flow_src_tot_l4_payload_len", - "flow_dst_tot_l4_payload_len", - "l3_proto", - "l4_proto", - "midstream", - "thread_ts_usec", - "src_ip", - "dst_ip" - ], - - "if": { - "properties": { "flow_event_name": { "enum": [ "new", "end", "idle", "update" ] } } - }, - "then": { - "required": [ "flow_datalink", "flow_max_packets" ] - }, - - "if": { - "properties": { "flow_event_name": { "enum": [ "analyse" ] } } - }, - "then": { - "required": [ "data_analysis" ] - }, - - "if": { - "properties": { "flow_state": { "enum": [ "finished" ] } } - }, - "then": { - "required": [ "ndpi" ] - }, - - "if": { - "properties": { "flow_event_name": { "enum": [ "guessed", "detected", - "detection-update", "not-detected" ] } } - }, - "then": { - "required": [ "ndpi" ] - }, - - "properties": { - "alias": { - "type": "string" - }, - "source": { - "type": "string" - }, - "thread_id": { - "type": "number", - "minimum": 0, - "maximum": 31 - }, - "packet_id": { - "type": "number", - "minimum": 0 - }, - "flow_event_id": { - "type": "number", - "minimum": 0, - "maximum": 9 - }, - "flow_event_name": { - "type": "string", - "enum": [ - "invalid", - "new", - "end", - "idle", - "update", - "analyse", - "guessed", - "detected", - "detection-update", - "not-detected" - ] - }, - "flow_id": { - "type": "number", - "minimum": 1 - }, - "flow_state": { - "type": "string", - "enum": [ - "finished", - "info" - ] - }, - "flow_datalink": { - "type": "number", - "minimum": 0, - "maximum": 292 - }, - "flow_src_packets_processed": { - "type": "number", - "minimum": 0 - }, - "flow_dst_packets_processed": { - "type": "number", - "minimum": 0 - }, - "flow_max_packets": { - "type": "number", - "minimum": 0 - }, - "flow_first_seen": { - "type": "number", - "minimum": 0 - }, - "flow_src_last_pkt_time": { - "type": "number", - "minimum": 0 - }, - "flow_dst_last_pkt_time": { - "type": "number", - "minimum": 0 - }, - "flow_idle_time": { - "type": "number", - "minimum": 1 - }, - "flow_src_min_l4_payload_len": { - "type": "number", - "minimum": 0 - }, - "flow_dst_min_l4_payload_len": { - "type": "number", - "minimum": 0 - }, - "flow_src_max_l4_payload_len": { - "type": "number", - "minimum": 0 - }, - "flow_dst_max_l4_payload_len": { - "type": "number", - "minimum": 0 - }, - "flow_src_tot_l4_payload_len": { - "type": "number", - "minimum": 0 - }, - "flow_dst_tot_l4_payload_len": { - "type": "number", - "minimum": 0 - }, - "l3_proto": { - "type": "string", - "enum": [ - "ip4", - "ip6", - "unknown" - ] - }, - "l4_proto": { - "oneOf": [ - { - "type": "number" - }, - { - "type": "string", - "enum": [ - "tcp", - "udp", - "icmp", - "icmp6" - ] - } - ] - }, - "midstream": { - "type": "number", - "minimum": 0, - "maximum": 1 - }, - "thread_ts_usec": { - "type": "number", - "minimum": 0 - }, - "src_ip": { - "type": "string", - "anyOf" : [ - { "format": "ipv4" }, - { "format": "ipv6" } - ] - }, - "dst_ip": { - "type": "string", - "anyOf" : [ - { "format": "ipv4" }, - { "format": "ipv6" } - ] - }, - "src_port": { - "type": "number", - "minimum": 1, - "maximum": 65535 - }, - "dst_port": { - "type": "number", - "minimum": 1, - "maximum": 65535 - }, - "ndpi": { - "type": "object", - "required": [ "proto", "proto_id", "breed", "encrypted" ], - - "properties": { - "proto": { - "type": "string" - }, - "proto_id": { - "type": "string" - }, - "proto_by_ip": { - "type": "string" - }, - "proto_by_ip_id": { - "type": "number" - }, - "category": { - "type": "string" - }, - "category_id": { - "type": "number" - }, - "encrypted": { - "type": "number", - "enum": [ - 0, - 1 - ] - }, - "breed": { - "type": "string" - }, - "flow_risk": { - "type": "object" - }, - "confidence": { - "type": "object", - "properties": { - "0": { - "type": "string" - }, - "1": { - "type": "string" - }, - "2": { - "type": "string" - }, - "3": { - "type": "string" - }, - "4": { - "type": "string" - }, - "5": { - "type": "string" - }, - "6": { - "type": "string" - } - }, - "additionalProperties": false - }, - "entropy": { - "type": "number" - }, - "hostname": { - "type": "string" - }, - "collectd": { - "type": "object" - }, - "dhcp": { - "type": "object" - }, - "discord": { - "type": "object" - }, - "bittorrent": { - "type": "object" - }, - "mdns": { - "type": "object" - }, - "natpmp": { - "type": "object" - }, - "ntp": { - "type": "object" - }, - "ubntac2": { - "type": "object" - }, - "kerberos": { - "type": "object" - }, - "telnet": { - "type": "object" - }, - "tls": { - "type": "object" - }, - "quic": { - "type": "object" - }, - "imap": { - "type": "object" - }, - "http": { - "type": "object" - }, - "pop": { - "type": "object" - }, - "smtp": { - "type": "object" - }, - "dns": { - "type": "object" - }, - "ftp": { - "type": "object" - }, - "snmp": { - "type": "object" - }, - "ssh": { - "type": "object" - }, - "stun": { - "type": "object" - }, - "softether": { - "type": "object" - }, - "tftp": { - "type": "object" - }, - "tivoconnect": { - "type": "object" - }, - "rsh": { - "type": "object" - } - }, - "additionalProperties": false - }, - "data_analysis": { - "type": "object", - "required": [ "iat", "pktlen", "bins", "directions" ], - - "properties": { - "iat": { - "type": "object", - - "properties": { - "min": { - "type": "number" - }, - "avg": { - "type": "number" - }, - "max": { - "type": "number" - }, - "stddev": { - "type": "number" - }, - "var": { - "type": "number" - }, - "ent": { - "type": "number" - }, - "data": { - "type": "array", - "items": { - "type": "number" - } - } - }, - "additionalProperties": false - }, - "pktlen": { - "type": "object", - - "properties": { - "min": { - "type": "number" - }, - "avg": { - "type": "number" - }, - "max": { - "type": "number" - }, - "stddev": { - "type": "number" - }, - "var": { - "type": "number" - }, - "ent": { - "type": "number" - }, - "data": { - "type": "array", - "items": { - "type": "number" - } - } - }, - "additionalProperties": false - }, - "bins": { - "type": "object", - - "properties": { - "c_to_s": { - "type": "array", - "items": { - "type": "number" - } - }, - "s_to_c": { - "type": "array", - "items": { - "type": "number" - } - } - }, - "additionalProperties": false - }, - "directions": { - "type": "array", - "items": { - "type": "number" - } - }, - "entropies": { - "type": "array", - "items": { - "type": "number" - } - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false -} diff --git a/heidpi-rust/src/schema/geoip2_schema.json b/heidpi-rust/src/schema/geoip2_schema.json deleted file mode 100644 index b8d7847..0000000 --- a/heidpi-rust/src/schema/geoip2_schema.json +++ /dev/null @@ -1,302 +0,0 @@ -{ - "type": "object", - "properties": { - "city": { - "type": "object", - "properties": { - "geoname_id": { - "type": "integer" - }, - "names": { - "type": "object", - "properties": { - "de": { - "type": "string" - }, - "en": { - "type": "string" - }, - "es": { - "type": "string" - }, - "fr": { - "type": "string" - }, - "ja": { - "type": "string" - }, - "pt-BR": { - "type": "string" - }, - "ru": { - "type": "string" - }, - "zh-CN": { - "type": "string" - } - }, - "additionalProperties": true - } - }, - "required": [ - "geoname_id", - "names" - ] - }, - "continent": { - "type": "object", - "properties": { - "code": { - "type": "string" - }, - "geoname_id": { - "type": "integer" - }, - "names": { - "type": "object", - "properties": { - "de": { - "type": "string" - }, - "en": { - "type": "string" - }, - "es": { - "type": "string" - }, - "fr": { - "type": "string" - }, - "ja": { - "type": "string" - }, - "pt-BR": { - "type": "string" - }, - "ru": { - "type": "string" - }, - "zh-CN": { - "type": "string" - } - }, - "additionalProperties": true - } - }, - "required": [ - "code", - "geoname_id", - "names" - ] - }, - "country": { - "type": "object", - "properties": { - "geoname_id": { - "type": "integer" - }, - "iso_code": { - "type": "string" - }, - "names": { - "type": "object", - "properties": { - "de": { - "type": "string" - }, - "en": { - "type": "string" - }, - "es": { - "type": "string" - }, - "fr": { - "type": "string" - }, - "ja": { - "type": "string" - }, - "pt-BR": { - "type": "string" - }, - "ru": { - "type": "string" - }, - "zh-CN": { - "type": "string" - } - }, - "additionalProperties": true - } - }, - "required": [ - "geoname_id", - "iso_code", - "names" - ] - }, - "location": { - "type": "object", - "properties": { - "accuracy_radius": { - "type": "integer" - }, - "latitude": { - "type": "number" - }, - "longitude": { - "type": "number" - }, - "metro_code": { - "type": "integer" - }, - "time_zone": { - "type": "string" - } - }, - "additionalProperties": true, - "required": [ - "accuracy_radius", - "latitude", - "longitude", - "metro_code", - "time_zone" - ] - }, - "postal": { - "type": "object", - "properties": { - "code": { - "type": "string" - } - }, - "additionalProperties": true, - "required": [ - "code" - ] - }, - "registered_country": { - "type": "object", - "properties": { - "geoname_id": { - "type": "integer" - }, - "iso_code": { - "type": "string" - }, - "names": { - "type": "object", - "properties": { - "de": { - "type": "string" - }, - "en": { - "type": "string" - }, - "es": { - "type": "string" - }, - "fr": { - "type": "string" - }, - "ja": { - "type": "string" - }, - "pt-BR": { - "type": "string" - }, - "ru": { - "type": "string" - }, - "zh-CN": { - "type": "string" - } - }, - "additionalProperties": true - } - }, - "required": [ - "geoname_id", - "iso_code", - "names" - ] - }, - "subdivisions": { - "type": "array", - "items": [ - { - "type": "object", - "properties": { - "geoname_id": { - "type": "integer" - }, - "iso_code": { - "type": "string" - }, - "names": { - "type": "object", - "properties": { - "de": { - "type": "string" - }, - "en": { - "type": "string" - }, - "es": { - "type": "string" - }, - "fr": { - "type": "string" - }, - "ja": { - "type": "string" - }, - "pt-BR": { - "type": "string" - }, - "ru": { - "type": "string" - }, - "zh-CN": { - "type": "string" - } - }, - "additionalProperties": true - } - }, - "required": [ - "geoname_id", - "iso_code", - "names" - ] - } - ] - }, - "traits": { - "type": "object", - "properties": { - "ip_address": { - "type": "string" - }, - "prefix_len": { - "type": "integer" - } - }, - "required": [ - "ip_address", - "prefix_len" - ] - } - }, - "required": [ - "city", - "continent", - "country", - "location", - "postal", - "registered_country", - "subdivisions", - "traits" - ], - "additionalProperties": false -} \ No newline at end of file diff --git a/heidpi-rust/src/schema/packet_event_schema.json b/heidpi-rust/src/schema/packet_event_schema.json deleted file mode 100644 index ad7416a..0000000 --- a/heidpi-rust/src/schema/packet_event_schema.json +++ /dev/null @@ -1,116 +0,0 @@ -{ - "type": "object", - "required": [ - "alias", - "source", - "packet_id", - "packet_event_id", - "packet_event_name", - "pkt_caplen", - "pkt_type", - "pkt_l3_offset", - "pkt_l4_offset", - "pkt_len", - "pkt_l4_len", - "thread_ts_usec" - ], - - "dependencies" : { - "flow_id" : [ "flow_packet_id", "flow_src_last_pkt_time", "flow_dst_last_pkt_time", "flow_idle_time" ] - }, - - "if": { - "properties": { "packet_event_name": { "enum": ["packet-flow"] } } - }, - "then": { - "required": [ "thread_id", "flow_id", "flow_packet_id", "flow_src_last_pkt_time", "flow_dst_last_pkt_time", "flow_idle_time" ] - }, - "else": { - "not": { "required": [ "thread_id", "flow_id", "flow_packet_id", "flow_src_last_pkt_time", "flow_dst_last_pkt_time", "flow_idle_time" ] } - }, - - "properties": { - "alias": { - "type": "string" - }, - "source": { - "type": "string" - }, - "thread_id": { - "type": "number" - }, - "packet_id": { - "type": "number", - "minimum": 0 - }, - "packet_event_id": { - "type": "number", - "minimum": 0, - "maximum": 2 - }, - "packet_event_name": { - "type": "string", - "enum": [ - "invalid", - "packet", - "packet-flow" - ] - }, - "flow_id": { - "type": "number", - "minimum": 1 - }, - "flow_packet_id": { - "type": "number" - }, - "flow_src_last_pkt_time": { - "type": "number", - "minimum": 0 - }, - "flow_dst_last_pkt_time": { - "type": "number", - "minimum": 0 - }, - "flow_idle_time": { - "type": "number", - "minimum": 1 - }, - "pkt_caplen": { - "type": "number", - "minimum": 1, - "maximum": 65535 - }, - "pkt_type": { - "type": "number", - "minimum": 0, - "maximum": 65535 - }, - "pkt_l3_offset": { - "type": "number", - "minimum": 0, - "maximum": 65535 - }, - "pkt_l4_len": { - "type": "number", - "minimum": 0, - "maximum": 65535 - }, - "thread_ts_usec": { - "type": "number", - "minimum": 0 - }, - "pkt_l4_offset": { - "type": "number", - "minimum": 0, - "maximum": 65535 - }, - "pkt_len": { - "type": "number", - "minimum": 0 - }, - "pkt": { - "type": "string" - } - }, - "additionalProperties": false -} diff --git a/heidpi-rust/src/stream.rs b/heidpi-rust/src/stream.rs deleted file mode 100644 index ab6f063..0000000 --- a/heidpi-rust/src/stream.rs +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Manages the network aspects, including establishing connections and reading data from sockets. - */ -use log::{info, trace, warn}; -use serde_json::Value; -use std::io::{self}; -use std::str; -use std::time::Duration; -use std::{thread, time}; -use tokio::net::TcpStream; - -use tokio::sync::mpsc; -use tokio::task; - -use crate::process::process; -use crate::geoip::GeoIP; -use crate::logging::Logging; - -const NETWORK_BUFFER_LENGTH_DIGITS: usize = 5; -const NETWORK_BUFFER_MAX_SIZE: usize = 33792; -const EOL: &str = "\n"; - -#[derive(Debug)] -pub struct HeiDPITcpstream { - event_type: HeiDPIEventType, - data: Value, -} - -#[derive(Debug)] -pub enum HeiDPIEventType { - PACKET, - FLOW, - DAEMON, - ERROR, -} - -pub async fn connect( - connection: &str, - geoip: Option, - flow_logger: Logging, - daemon_logger: Logging, - packet_logger: Logging, - error_logger: Logging, -) -> anyhow::Result<()> { - loop { - match std::net::TcpStream::connect(connection) { - Err(_e) => { - warn!("Could not connect to Server"); - - // We don't want to hammer the server with reconnection attempts, so we wait for 5 seconds. - let five_seconds = time::Duration::from_millis(5000); - thread::sleep(five_seconds); - - continue; - } - Ok(std_stream) => { - info!("Connected"); - - match std_stream.set_nonblocking(true) { - Ok(..) => info!("Non-blocking State"), - Err(..) => panic!("Non-blocking State Failed"), - }; - - match std_stream.set_read_timeout(Some(Duration::new(60, 0))) { - Ok(..) => info!("Set Read Timeout"), - Err(..) => panic!("Setting Read Timeout Failed"), - }; - - let stream = TcpStream::from_std(std_stream)?; - let (tx, mut rx) = mpsc::channel::<(Value, Logging)>(100); - let mut buf = vec![0u8; NETWORK_BUFFER_MAX_SIZE]; - - loop { - match stream.try_read(&mut buf) { - Ok(data) => { - trace!("read {} bytes", data); - match std::str::from_utf8(&buf[..data]) { - Ok(json_str) => { - for s_plit_n in json_str.split(EOL).into_iter() { - if s_plit_n.len() > NETWORK_BUFFER_LENGTH_DIGITS { - let v: Value = match serde_json::from_str( - &s_plit_n[NETWORK_BUFFER_LENGTH_DIGITS..], - ) { - Ok(json) => { - trace!("Converted result: {}", json); - json - } - Err(_e) => { - warn!("Invalid JSON object: '{}'.", _e); - serde_json::Value::Null - } - }; - // TODO Multithreading? - // TODO Call processing (geoip2, remove risk, remove attributes, ignore event types, ...) and save to file - if let Some(event_type) = v.get("event_type").and_then(|e| e.as_str()) { - let logger = match event_type { - "flow" => flow_logger.clone(), - "daemon" => daemon_logger.clone(), - "packet" => packet_logger.clone(), - "error" => error_logger.clone(), - _ => continue, - }; - - if let Err(e) = tx.send((v, logger)).await { - warn!("Failed to send data for processing: {}", e); - } - //process(v); - } - - } - } - } - Err(_) => { - warn!("BUG: Invalid UTF-8 in buffer"); - } - } - } - Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { - continue; - } - Err(..) => { - let five_seconds = time::Duration::from_millis(10); - thread::sleep(five_seconds); - - continue; - } - } - } - } - } - } -} diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 7e6f555..0000000 --- a/pyproject.toml +++ /dev/null @@ -1,42 +0,0 @@ - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[project] -name = "heiDPI" -version = "1.3.0" -authors = [ - { name="Stefan Machmeier", email="stefan.machmeier@uni-heidelberg.de" }, -] -description = "nDPId consumer implementation" -readme = "README.consumer.md" -license = { file="LICENSE" } -requires-python = ">=3.6" -classifiers = [ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: European Union Public Licence 1.2 (EUPL 1.2)", - "Operating System :: OS Independent", - "Operating System :: OS Independent", - 'Intended Audience :: Telecommunications Industry', - 'Intended Audience :: Information Technology', - 'Intended Audience :: System Administrators', - 'Intended Audience :: Science/Research', - 'Intended Audience :: Developers', - 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Topic :: Security', - 'Topic :: Internet :: Log Analysis', - 'Topic :: System :: Networking :: Monitoring', -] - -[project.scripts] -heiDPI = "heidpi.heiDPI_logger:main" - -[project.urls] -"Homepage" = "https://github.com/stefanDeveloper/heiDPI" -"Bug Tracker" = "https://github.com/stefanDeveloper/heiDPI/issues" \ No newline at end of file diff --git a/shell.nix b/shell.nix deleted file mode 100644 index 1101fb2..0000000 --- a/shell.nix +++ /dev/null @@ -1,57 +0,0 @@ -with import { }; - -let - pythonPackages = python39Packages; - pypyPackages = pypy3Packages; -in pkgs.mkShell rec { - venvDir = "./.venv"; - requirements = "requirements.txt"; - - name = "heiDPI"; - - buildInputs = [ - pythonPackages.setuptools - pythonPackages.virtualenv # run virtualenv . - pythonPackages.pip - pythonPackages.pyqt5 # avoid installing via pip - # This execute some shell code to initialize a venv in $venvDir before - # dropping into the shell - pythonPackages.venvShellHook - - # Without setting the zlib in LD_LIBRARY_PATH we get the following error: - # Original error was: libz.so.1: cannot open shared object file: No such file or directory - zlib - ]; - shellHook = '' - # fixes libstdc++ issues and libgl.so issues - LD_LIBRARY_PATH=${zlib}/lib/:${stdenv.cc.cc.lib}/lib/:/run/opengl-driver/lib/ - - # fixes xcb issues : - QT_PLUGIN_PATH=${qt5.qtbase}/${qt5.qtbase.qtPluginPrefix} - SOURCE_DATE_EPOCH=$(date +%s) - QT_XCB_GL_INTEGRATION="none" - - if [ -d "${venvDir}" ]; then - echo "Skipping venv creation, '${venvDir}' already exists" - else - echo "Creating new venv environment in path: '${venvDir}'" - # Note that the module venv was only introduced in python 3, so for 2.7 - # this needs to be replaced with a call to virtualenv - ${pythonPackages.python.interpreter} -m venv "${venvDir}" - fi - - # Under some circumstances it might be necessary to add your virtual - # environment to PYTHONPATH, which you can do here too; - PYTHONPATH=$PWD/${venvDir}/${pythonPackages.python.sitePackages}/:${pypy}:$PYTHONPATH - - source "${venvDir}/bin/activate" - - echo "Upgrading pip to latest version" - python -m pip install --upgrade pip - - if [ -f "./${requirements}" ]; then - echo "Install '${requirements}'" - pip install -r ${requirements} - fi - ''; -} \ No newline at end of file From 0ccef10d7c8a4b83f0dc6e0f13b3aa7bed1474df Mon Sep 17 00:00:00 2001 From: bogdanpatenko Date: Mon, 1 Sep 2025 16:40:36 +0200 Subject: [PATCH 5/5] restored README.md --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 8a4a6dc..bb5489a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,4 @@ -[include](../heiDPI_benchmark/heidpi_logger_cpp_port/include) -[src](../heiDPI_benchmark/heidpi_logger_cpp_port/src) -[CMakeLists.txt](../heiDPI_benchmark/heidpi_logger_cpp_port/CMakeLists.txt)![heiFIP Logo](https://raw.githubusercontent.com/stefanDeveloper/heiDPI/main/assets/heidpi_logo.png?raw=true) +![heiFIP Logo](https://raw.githubusercontent.com/stefanDeveloper/heiDPI/main/assets/heidpi_logo.png?raw=true) --------------------------------------------------------------------------------