Implement heidpi logger cpp version

.dockerignore

@@ -1 +1,15 @@
-docker-compose.yml
+docker-compose.yml
+.git
+.github
+build
+cmake-build-*
+**/__pycache__
+**/*.o
+**/*.a
+**/*.log
+tests/
+nDPId/
+heidpi-logger-py/
+heidpi-rust/
+*.swp
+.DS_Store

.github/workflows/docker-publish-consumer.yml

@@ -12,7 +12,7 @@ on:
 
 env:
   REGISTRY: docker.io 
-  IMAGE_NAME: stefan96/heidpi-consumer
+  IMAGE_NAME: stefan96/heidpi-logger-py-consumer
 
 
 jobs:
@@ -84,7 +84,7 @@ jobs:
         with:
           context: .
           push: ${{ github.event_name != 'pull_request' }}
-          tags: stefan96/heidpi-consumer:latest
+          tags: stefan96/heidpi-logger-py-consumer:latest
           labels: ${{ steps.meta.outputs.labels }}
           file: ./Dockerfile.consumer
           cache-from: type=gha

.github/workflows/docker-publish-producer.yml

@@ -12,7 +12,7 @@ on:
 
 env:
   REGISTRY: docker.io 
-  IMAGE_NAME: stefan96/heidpi-producer
+  IMAGE_NAME: stefan96/heidpi-logger-py-producer
 
 
 jobs:
@@ -84,7 +84,7 @@ jobs:
         with:
           context: .
           push: ${{ github.event_name != 'pull_request' }}
-          tags: stefan96/heidpi-producer:latest
+          tags: stefan96/heidpi-logger-py-producer:latest
           labels: ${{ steps.meta.outputs.labels }}
           file: ./Dockerfile.producer
           cache-from: type=gha

Dockerfile.consumer

@@ -1,24 +1,49 @@
-FROM pypy:3.9-slim-bookworm
+# ---------- Build stage ----------
+FROM debian:bookworm AS build
+ARG CMAKE_BUILD_TYPE=Release
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential cmake pkg-config git ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /src
+COPY heidpi-logger/ ./heidpi-logger/
+
+RUN cmake -S heidpi-logger -B /build \
+      -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
+      -DBUILD_TESTING=OFF \
+      -DMAXMINDDB_BUILD_BINARIES=OFF \
+      -DMAXMINDDB_TOOLS=OFF \
+      -DMAXMINDDB_INSTALL=OFF \
+ && cmake --build /build --target heidpi_cpp -- -j"$(nproc)" \
+ && strip /build/heidpi_cpp || true
+
+# ---------- Runtime stage ----------
+FROM debian:bookworm-slim AS runtime
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libstdc++6 ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+# non-root
+RUN useradd -r -u 10001 appuser
+WORKDIR /app
+
+# Configs (aus Repo-Root)
+COPY config.yml /app/config.yml
+
+COPY --from=build /build/heidpi_cpp /usr/local/bin/app
 
 ENV WRITE="/var/log" \
     SHOW_FLOW_EVENTS=1 \
-    SHOW_PACKET_EVENTS=0 \
+    SHOW_PACKET_EVENTS=1 \
     SHOW_ERROR_EVENTS=0 \
     SHOW_DAEMON_EVENTS=0 \
     UNIX="" \
     PORT=7000 \
     HOST=""
 
-WORKDIR /usr/src/app
-
-COPY heidpi ./ \
-     config.yml ./ \
-     LICENSE ./ \
-     README.md ./ \
-     requirements.txt ./ \
-     pyproject.toml ./
-
-RUN pip install . \
-    pip install -r requirements.txt
+USER appuser
 
-CMD [ "heiDPI"]
+ENTRYPOINT ["/usr/local/bin/app"]
+CMD ["--config","/app/config.yml"]

README.consumer.md

@@ -101,15 +101,15 @@ In order to run this container you'll need docker installed.
 Pull images:
 
 ```sh
-docker pull stefan96/heidpi-producer:main
-docker pull stefan96/heidpi-consumer:main
+docker pull stefan96/heidpi-logger-py-producer:main
+docker pull stefan96/heidpi-logger-py-consumer:main
 ```
 
 Run producer and consumer separately from each other using UDP socket:
 
 ```sh
-docker run -p 127.0.0.1:7000:7000 --net host stefan96/heidpi-producer:main
-docker run -e HOST=127.0.0.1 --net host stefan96/heidpi-consumer:main
+docker run -p 127.0.0.1:7000:7000 --net host stefan96/heidpi-logger-py-producer:main
+docker run -e HOST=127.0.0.1 --net host stefan96/heidpi-logger-py-consumer:main
 ```
 
 or use the `docker-compose.yml`:
@@ -121,8 +121,8 @@ docker-compose up
 Additionally, you use a UNIX socket:
 
 ```sh
-docker run -v ${PWD}/heidpi-data:/tmp/ --net host stefan96/heidpi-producer:main
-docker run -v ${PWD}/heidpi-data:/tmp/ -v ${PWD}/heidpi-logs:/var/log -e UNIX=/tmp/nDPIsrvd-daemon-distributor.sock --net host stefan96/heidpi-consumer:main
+docker run -v ${PWD}/heidpi-logger-py-data:/tmp/ --net host stefan96/heidpi-logger-py-producer:main
+docker run -v ${PWD}/heidpi-logger-py-data:/tmp/ -v ${PWD}/heidpi-logger-py-logs:/var/log -e UNIX=/tmp/nDPIsrvd-daemon-distributor.sock --net host stefan96/heidpi-logger-py-consumer:main
 ```
 
 ## Environment Variables

README.md

@@ -86,15 +86,15 @@ In order to run this container you'll need docker installed.
 Pull images:
 
 ```sh
-docker pull stefan96/heidpi-producer:main
-docker pull stefan96/heidpi-consumer:main
+docker pull stefan96/heidpi-logger-py-producer:main
+docker pull stefan96/heidpi-logger-py-consumer:main
 ```
 
 Run producer and consumer separately from each other using UDP socket:
 
 ```sh
-docker run -p 127.0.0.1:7000:7000 --net host stefan96/heidpi-producer:main
-docker run -e HOST=127.0.0.1 --net host stefan96/heidpi-consumer:main
+docker run -p 127.0.0.1:7000:7000 --net host stefan96/heidpi-logger-py-producer:main
+docker run -e HOST=127.0.0.1 --net host stefan96/heidpi-logger-py-consumer:main
 ```
 
 or use the `docker-compose.yml`:
@@ -106,8 +106,8 @@ docker-compose up
 Additionally, you use a UNIX socket:
 
 ```sh
-docker run -v ${PWD}/heidpi-data:/tmp/ --net host stefan96/heidpi-producer:main
-docker run -v ${PWD}/heidpi-data:/tmp/ -v ${PWD}/heidpi-logs:/var/log -e UNIX=/tmp/nDPIsrvd-daemon-distributor.sock --net host stefan96/heidpi-consumer:main
+docker run -v ${PWD}/heidpi-logger-py-data:/tmp/ --net host stefan96/heidpi-logger-py-producer:main
+docker run -v ${PWD}/heidpi-logger-py-data:/tmp/ -v ${PWD}/heidpi-logger-py-logs:/var/log -e UNIX=/tmp/nDPIsrvd-daemon-distributor.sock --net host stefan96/heidpi-logger-py-consumer:main
 ```
 
 ## Configuration

README.producer.md

@@ -32,15 +32,15 @@ In order to run this container you'll need docker installed.
 Pull images:
 
 ```sh
-docker pull stefan96/heidpi-producer:main
-docker pull stefan96/heidpi-consumer:main
+docker pull stefan96/heidpi-logger-py-producer:main
+docker pull stefan96/heidpi-logger-py-consumer:main
 ```
 
 Run producer and consumer separately from each other using UDP socket:
 
 ```sh
-docker run -p 127.0.0.1:7000:7000 --net host stefan96/heidpi-producer:main
-docker run -e HOST=127.0.0.1 --net host stefan96/heidpi-consumer:main
+docker run -p 127.0.0.1:7000:7000 --net host stefan96/heidpi-logger-py-producer:main
+docker run -e HOST=127.0.0.1 --net host stefan96/heidpi-logger-py-consumer:main
 ```
 
 or use the `docker-compose.yml`:
@@ -52,8 +52,8 @@ docker-compose up
 Additionally, you use a UNIX socket:
 
 ```sh
-docker run -v ${PWD}/heidpi-data:/tmp/ --net host stefan96/heidpi-producer:main
-docker run -v ${PWD}/heidpi-data:/tmp/ -v ${PWD}/heidpi-logs:/var/log -e UNIX=/tmp/nDPIsrvd-daemon-distributor.sock --net host stefan96/heidpi-consumer:main
+docker run -v ${PWD}/heidpi-logger-py-data:/tmp/ --net host stefan96/heidpi-logger-py-producer:main
+docker run -v ${PWD}/heidpi-logger-py-data:/tmp/ -v ${PWD}/heidpi-logger-py-logs:/var/log -e UNIX=/tmp/nDPIsrvd-daemon-distributor.sock --net host stefan96/heidpi-logger-py-consumer:main
 ```
 
 ## Environment Variables

config.yml

@@ -1,7 +1,7 @@
 appName: heiDPI
 
 logging:
-  level: INFO
+  level: ERROR
   encoding: utf-8
   format: "%(asctime)s %(levelname)s:%(message)s"
   datefmt: "%Y-%m-%dT%I:%M:%S"
@@ -28,26 +28,26 @@ flow_event:
       # - city
       # - traits
       # - postal
-  threads: 4
+#  threads: 4
 
 daemon_event:
   ignore_fields: []
   daemon_event_name:
     - init
     - status
   filename: daemon_event
-  threads: 4
+#  threads: 4
 
 packet_event:
   ignore_fields: []
   packet_event_name:
     - packet-flow
   filename: packet_event
-  threads: 4
+#  threads: 4
 
 error_event:
   ignore_fields: []
   error_event_name:
     - error-flow
   filename: error_event
-  threads: 4
+#  threads: 4

docker-compose.yml

@@ -7,12 +7,12 @@ services:
     network_mode: host
     security_opt:
       - no-new-privileges
-    pids_limit: 8192
+#    pids_limit: 8192
     restart: on-failure:5
     deploy:
       resources:
         limits:
-          cpus: '5'
+          cpus: '1'
           memory: 32G
     environment:
       - HOSTNAME=test
@@ -22,15 +22,18 @@ services:
       - SSL_SHA1_URL=https://sslbl.abuse.ch/blacklist/sslblacklist.csv
 
   consumer:
-    image: stefan96/heidpi-consumer:latest
+    build:
+      context: .
+      dockerfile: Dockerfile.consumer
+    #image: stefan96/heidpi-consumer:latest
     container_name: heidpi_consumer
     volumes:
       - ./heidpi-logs:/var/log/:rw
       - ./config.yml:/usr/src/app/config.yml:ro
     network_mode: host
     security_opt:
       - no-new-privileges
-    pids_limit: 8192
+#    pids_limit: 8192
     restart: on-failure:5
     deploy:
       resources:
@@ -41,3 +44,6 @@ services:
       - producer
     environment:
       - SHOW_DAEMON_EVENTS=1
+      - SHOW_PACKET_EVENTS=1
+      - SHOW_ERROR_EVENTS=1
+      - SHOW_FLOW_EVENTS=1

heidpi-logger/CMakeLists.txt

@@ -0,0 +1,56 @@
+cmake_minimum_required(VERSION 3.10)
+project(heidpi_cpp VERSION 0.1 LANGUAGES CXX)
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_POLICY_VERSION_MINIMUM 3.5)
+
+include(FetchContent)
+
+FetchContent_Declare(
+        yaml-cpp
+        GIT_REPOSITORY https://github.com/jbeder/yaml-cpp.git
+        GIT_TAG        f732014
+)
+FetchContent_MakeAvailable(yaml-cpp)
+
+FetchContent_Declare(
+        json
+        GIT_REPOSITORY https://github.com/nlohmann/json.git
+        GIT_TAG v3.11.3
+)
+FetchContent_MakeAvailable(json)
+
+FetchContent_Declare(
+        json-schema-validator
+        GIT_REPOSITORY https://github.com/pboettch/json-schema-validator.git
+        GIT_TAG main
+)
+FetchContent_MakeAvailable(json-schema-validator)
+
+FetchContent_Declare(
+        maxminddb
+        GIT_REPOSITORY https://github.com/maxmind/libmaxminddb.git
+        GIT_TAG        1.12.2
+        CMAKE_ARGS
+        -DBUILD_SHARED_LIBS=OFF
+        -DBUILD_TESTING=OFF
+        -DMAXMINDDB_BUILD_BINARIES=OFF
+        -DMAXMINDDB_INSTALL=OFF
+)
+FetchContent_MakeAvailable(maxminddb)
+
+file(GLOB SOURCES src/*.cpp)
+add_executable(heidpi_cpp ${SOURCES})
+target_include_directories(heidpi_cpp PRIVATE include)
+target_link_libraries(heidpi_cpp PRIVATE
+        yaml-cpp
+        nlohmann_json::nlohmann_json
+        nlohmann_json_schema_validator
+        maxminddb::maxminddb
+)
+
+include(GNUInstallDirs)
+
+install(TARGETS heidpi_cpp
+        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+

heidpi-logger/include/Config.hpp

@@ -0,0 +1,44 @@
+#pragma once
+#include <string>
+#include <vector>
+#include <nlohmann/json.hpp>
+#include <yaml-cpp/yaml.h>
+
+/**
+ * @brief Loads application configuration from a YAML file.
+ */
+struct LoggingConfig {
+    std::string level{"INFO"};
+    std::string format{"%Y-%m-%dT%H:%M:%S"};
+    std::string datefmt{"%Y-%m-%dT%H:%M:%S"};
+    std::string filename{}; // optional log file
+};
+
+struct EventConfig {
+    std::vector<std::string> ignore_fields;
+    std::vector<std::string> ignore_risks;
+    std::vector<std::string> event_names; // empty -> allow all event names
+    std::string filename{"event"};
+    int threads{1};
+    // GeoIP configuration (flow events only)
+    bool geoip_enabled{false};
+    std::string geoip_path{};
+    std::vector<std::string> geoip_keys;
+};
+
+class Config {
+public:
+    explicit Config(const std::string &path);
+    const LoggingConfig &logging() const { return logging_cfg; }
+    const EventConfig &flowEvent() const { return flow_cfg; }
+    const EventConfig &packetEvent() const { return packet_cfg; }
+    const EventConfig &daemonEvent() const { return daemon_cfg; }
+    const EventConfig &errorEvent() const { return error_cfg; }
+private:
+    LoggingConfig logging_cfg;
+    EventConfig flow_cfg;
+    EventConfig packet_cfg;
+    EventConfig daemon_cfg;
+    EventConfig error_cfg;
+};
+