Log parser

.github/workflows/release.yml

@@ -91,9 +91,43 @@ jobs:
           name: fcore-auth
           path: target/release/auth
 
+  build-pixel-agent:
+    needs: [tests]
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@nightly
+      - uses: Swatinem/rust-cache@v2
+      - name: Build Pixel Agent
+        run: cargo build --release --bin pixel-agent --no-default-features
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: fcore-pixel-agent
+          path: target/release/pixel-agent
+
+  build-pixel-agent-backfill:
+    needs: [tests]
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@nightly
+      - uses: Swatinem/rust-cache@v2
+      - name: Build Pixel Agent Backfill
+        run: cargo build --release --bin pixel-agent-backfill --no-default-features
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: fcore-pixel-agent-backfill
+          path: target/release/pixel-agent-backfill
+
   release:
     name: Create Release
-    needs: [build-api, build-auth, build-node]
+    needs: [build-api, build-auth, build-node, build-pixel-agent, build-pixel-agent-backfill]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -113,6 +147,8 @@ jobs:
           cp artifacts/fcore-node-armv7-unknown-linux-gnueabihf/node release_dist/node-armv7
           cp artifacts/fcore-api/api release_dist/api-x86_64
           cp artifacts/fcore-auth/auth release_dist/auth-x86_64
+          cp artifacts/fcore-pixel-agent/pixel-agent release_dist/pixel-agent-x86_64
+          cp artifacts/fcore-pixel-agent-backfill/pixel-agent-backfill release_dist/pixel-agent-backfill-x86_64
           chmod +x release_dist/*
 
       - name: Upload to GitHub Release
@@ -125,5 +161,6 @@ jobs:
             config-node-example.toml
             config-api-example.toml
             config-auth-example.toml
+            config-pixel-agent-example.toml
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/rust.yml

@@ -135,9 +135,83 @@ jobs:
           name: fcore-auth
           path: target/x86_64-unknown-linux-musl/release/auth
 
+  build-pixel-agent:
+    needs: [tests]
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@nightly
+        with:
+          targets: x86_64-unknown-linux-musl
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          key: pixel-agent
+
+      - name: Install cross
+        run: cargo install cross
+
+      - name: Build Pixel Agent
+        run: |
+          cross build --release \
+            --bin pixel-agent \
+            --no-default-features \
+            --target x86_64-unknown-linux-musl
+
+      - name: Verify binary
+        run: |
+          file target/x86_64-unknown-linux-musl/release/pixel-agent
+          ldd target/x86_64-unknown-linux-musl/release/pixel-agent || true
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: fcore-pixel-agent
+          path: target/x86_64-unknown-linux-musl/release/pixel-agent
+
+  build-pixel-agent-backfill:
+    needs: [tests]
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@nightly
+        with:
+          targets: x86_64-unknown-linux-musl
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          key: pixel-agent-backfill
+
+      - name: Install cross
+        run: cargo install cross
+
+      - name: Build Pixel Agent Backfill
+        run: |
+          cross build --release \
+            --bin pixel-agent-backfill \
+            --no-default-features \
+            --target x86_64-unknown-linux-musl
+
+      - name: Verify binary
+        run: |
+          file target/x86_64-unknown-linux-musl/release/pixel-agent-backfill
+          ldd target/x86_64-unknown-linux-musl/release/pixel-agent-backfill || true
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: fcore-pixel-agent-backfill
+          path: target/x86_64-unknown-linux-musl/release/pixel-agent-backfill
+
   collect-binaries:
     runs-on: ubuntu-latest
-    needs: [build-node, build-api, build-auth]
+    needs: [build-node, build-api, build-auth, build-pixel-agent, build-pixel-agent-backfill]
 
     steps:
       - name: Download all artifacts

.gitignore

@@ -4,6 +4,7 @@
 dev/config-agent.toml
 dev/config-api.toml
 dev/config-auth.toml
+dev/pixel-logs
 target
 snapshots/*.bin
 dev/data/*

Cargo.toml

@@ -61,6 +61,8 @@ netlink-packet-amnezia-wireguard = { version = "0.1.2", optional = true }
 netlink-packet-core = { version = "0.8", optional = true }
 netlink-packet-generic = { version = "0.4", optional = true }
 netlink-sys = { version = "0.8", optional = true }
+maxminddb = "0.24"
+nom = "7"
 
 
 [build-dependencies]
@@ -93,3 +95,11 @@ path = "src/bin/api/main.rs"
 name = "auth"
 path = "src/bin/auth/main.rs"
 
+[[bin]]
+name = "pixel-agent"
+path = "src/bin/pixel_agent/main.rs"
+
+[[bin]]
+name = "pixel-agent-backfill"
+path = "src/bin/pixel_agent/backfill.rs"
+

config-api-example.toml

@@ -32,6 +32,7 @@ admin_token = "change-me-in-production"
 reciever = "tcp://0.0.0.0:3002"
 max_points = 10000
 retention_seconds = 604800 # 7 days
+snapshot_path = "snapshots/metrics.bin"
 
 [metrics.log]
 enabled = true

config-pixel-agent-example.toml

@@ -0,0 +1,14 @@
+log_level = "info"
+log_path = "/var/log/nginx/pixel.log"
+offset_path = "/var/lib/pixel-agent/offset.dat"
+snapshot_path = "/var/lib/pixel-agent/metrics.snapshot"
+geoip_db = "/usr/share/GeoIP/GeoLite2-Country.mmdb"
+admin_listen = "0.0.0.0"
+admin_port = 9102
+poll_interval_sec = 30
+flush_interval_sec = 300
+snapshot_interval_sec = 300
+bucket_minutes = 5
+retention_hours = 168
+max_points = 10000
+retention_seconds = 604800

dev/analytics.js

@@ -0,0 +1,36 @@
+(function () {
+  var img = new Image();
+  var params = [];
+
+  params.push('page=' + encodeURIComponent(location.pathname + location.search));
+  params.push('host=' + encodeURIComponent(location.host));
+
+  if (document.referrer) {
+    params.push('ref=' + encodeURIComponent(document.referrer));
+  }
+
+  var lang = navigator.language || navigator.userLanguage || '';
+  if (lang) {
+    params.push('lang=' + encodeURIComponent(lang));
+  }
+
+  function getQueryParam(name) {
+    var match = new RegExp('[?&]' + name + '=([^&]*)').exec(location.search);
+    return match ? decodeURIComponent(match[1].replace(/\+/g, ' ')) : '';
+  }
+
+  [
+    'utm_source',
+    'utm_medium',
+    'utm_campaign',
+    'utm_content',
+    'utm_term'
+  ].forEach(function (key) {
+    var value = getQueryParam(key);
+    if (value) {
+      params.push(key + '=' + encodeURIComponent(value));
+    }
+  });
+
+  img.src = 'https://media.frkn.org/pixel?' + params.join('&');
+})();

dev/backfill-pixel-logs.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Backfills pixel-agent storage from rotated nginx pixel logs.
+# Usage: sudo ./backfill-pixel-logs.sh /etc/pixel-agent/config.toml
+
+CONFIG_PATH="${1:-/etc/pixel-agent/config.toml}"
+LOG_DIR="/var/log/nginx"
+BACKFILL_LOG="/tmp/pixel-backfill.log"
+
+if [ ! -f "$CONFIG_PATH" ]; then
+    echo "Config not found: $CONFIG_PATH"
+    exit 1
+fi
+
+echo "Preparing backfill log at $BACKFILL_LOG..."
+: > "$BACKFILL_LOG"
+
+# Append older gzipped logs in reverse chronological order (oldest first).
+# Adjust glob if your logrotate naming differs.
+for gz in $(ls -1 "$LOG_DIR"/pixel.log.*.gz 2>/dev/null | sort -V); do
+    echo "Unpacking $gz..."
+    zcat "$gz" >> "$BACKFILL_LOG"
+done
+
+# Append uncompressed rotated logs if any.
+for rotated in $(ls -1 "$LOG_DIR"/pixel.log.[0-9] 2>/dev/null | sort -V); do
+    echo "Appending $rotated..."
+    cat "$rotated" >> "$BACKFILL_LOG"
+done
+
+# Append current log.
+if [ -f "$LOG_DIR/pixel.log" ]; then
+    echo "Appending current $LOG_DIR/pixel.log..."
+    cat "$LOG_DIR/pixel.log" >> "$BACKFILL_LOG"
+fi
+
+echo "Running backfill..."
+pixel-agent-backfill "$CONFIG_PATH" "$BACKFILL_LOG"
+
+echo "Cleaning up..."
+rm -f "$BACKFILL_LOG"
+
+echo "Backfill complete. Restart pixel-agent to pick up the updated snapshot."

dev/deploy-pixel-agent.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Build and deploy the pixel-agent binary to a remote server.
+# Usage: ./deploy-pixel-agent.sh user@host
+
+HOST="${1:?usage: $0 user@host}"
+REMOTE_DIR="/usr/local/bin"
+CONFIG_DIR="/etc/pixel-agent"
+DATA_DIR="/var/lib/pixel-agent"
+
+echo "Building release binary..."
+cargo build --release --bin pixel-agent
+
+echo "Uploading binary..."
+scp target/release/pixel-agent "${HOST}:${REMOTE_DIR}/pixel-agent.new"
+ssh "${HOST}" "mv ${REMOTE_DIR}/pixel-agent.new ${REMOTE_DIR}/pixel-agent && chmod +x ${REMOTE_DIR}/pixel-agent"
+
+echo "Ensuring directories and user..."
+ssh "${HOST}" "id -u pixel-agent &>/dev/null || useradd --system --no-create-home pixel-agent"
+ssh "${HOST}" "mkdir -p ${CONFIG_DIR} ${DATA_DIR} && chown pixel-agent:pixel-agent ${DATA_DIR}"
+
+echo "Uploading config if missing..."
+ssh "${HOST}" "test -f ${CONFIG_DIR}/config.toml || echo 'Config not present, copy config-pixel-agent-example.toml manually'"
+
+echo "Uploading systemd unit..."
+scp dev/pixel-agent.service "${HOST}:/etc/systemd/system/pixel-agent.service"
+
+echo "Reloading and restarting..."
+ssh "${HOST}" "systemctl daemon-reload && systemctl enable --now pixel-agent && systemctl status pixel-agent --no-pager"
+
+echo "Done."