agent: refactor; use simpler config abstraction

The agent will now also monitor for new log files that may show up after
agent start.

Cargo.toml

@@ -61,3 +61,5 @@ thiserror = "1.0.19"
 ulid = "0.5.0"
 uuid = { version = "0.8.1", features = ["v4"] }
 webbrowser = "0.5.2"
+yaml-rust = "0.4.5"
+futures = "0.3.21"

LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright 2014-2021 Jason Ish
+Copyright 2014-2022 Jason Ish
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

doc/agent.rst

@@ -14,5 +14,5 @@ Command Line Options
 Configuration File
 ------------------
 
-.. literalinclude:: ../agent.yaml.example
+.. literalinclude:: ../examples/agent.yaml
    :language: yaml

agent.yaml.example → examples/agent.yaml

@@ -2,7 +2,7 @@
 
 # Server information.
 server:
-  url: http://localhost:5636
+  url: http://10.16.1.10:5636
 
   # Username and password. Note that at this time even with
   # authentication enabled on the EveBox server, agents can still
@@ -12,37 +12,35 @@ server:
   #username: username
   #password: password
 
-# Directory to store bookmark information. This is optional and not
-# required if the agent has write access to the directory of the log
-# file being reader.
-#bookmark-directory: "/var/lib/evebox"
+# Directory to store data and state information required by the agent. This
+# isn't always required. If the agent has write access to the log directory it
+# can store bookmark information along side the eve log files.
+#data-directory: "/var/lib/evebox"
 
 # If the EveBox server is running behind TLS and the certificate is
 # self signed, certificate validation can be disabled.
 #disable-certificate-check: true
 
 # Path to log file. Only a single path is allowed at this time.
 input:
-  filename: "/var/log/suricata/eve.json"
+  #filename: "/var/log/suricata/eve.json"
 
   # The filename parameter can also contain a wildcard.
-  #filename: "/var/log/suricata/eve.*.json"
+  filename: "/var/log/suricata/eve*.json"
 
   # If multiple paths are required, use "paths" instead of filename.
   #paths:
   #  - "/var/log/suricata/sensor1/eve.json"
   #  - "/var/log/suricata/sensor2/eve.*.json"
 
-  # Custom fields to add to the event. Only top level fields can be set,
-  # and only simple values (string, integer) can be set.
-  custom-fields:
-    # Set a host field. This will override the "host" field set by
-    # Suricata if the Suricata "sensor-name" option is set.
-    #host: "evebox-agent"
-
-  # The event reader can also add the rule to alert events. Do not enable
-  # if you already have Suricata logging the rule.
-  #rules:
-  #  - /var/lib/suricata/rules/*.rules
-  #  - /usr/share/suricata/rules/*.rules
-  #  - /etc/suricata/rules/*.rules
+# Additional fields that will be added to each event. This is currently limited
+# to strings at this time.
+additional-fields:
+  #sensor-name: "my super secret sensor"
+
+# The event reader can also add the rule to alert events. Do not enable
+# if you already have Suricata logging the rule.
+#rules:
+#  - /var/lib/suricata/rules/*.rules
+#  - /usr/share/suricata/rules/*.rules
+#  - /etc/suricata/rules/*.rules

rustfmt.toml

@@ -1 +0,0 @@
-#fn_args_layout = "compressed"

src/agent/client.rs

@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright (C) 2020-2022 Jason Ish
+
+// EveBox agent client (to EveBox server)
+#[derive(Clone, Debug)]
+pub struct Client {
+    url: String,
+    disable_certificate_validation: bool,
+    username: Option<String>,
+    password: Option<String>,
+}
+
+impl Client {
+    pub fn new(
+        url: &str,
+        username: Option<String>,
+        password: Option<String>,
+        disable_certificate_validation: bool,
+    ) -> Self {
+        Self {
+            url: url.to_string(),
+            disable_certificate_validation,
+            username,
+            password,
+        }
+    }
+
+    pub fn get_http_client(&self) -> Result<reqwest::Client, reqwest::Error> {
+        let mut builder = reqwest::Client::builder();
+        if self.disable_certificate_validation {
+            builder = builder.danger_accept_invalid_certs(true);
+        }
+        builder.build()
+    }
+
+    pub fn post(&self, path: &str) -> Result<reqwest::RequestBuilder, reqwest::Error> {
+        let url = format!("{}/{}", self.url, path);
+        let request = self
+            .get_http_client()?
+            .post(&url)
+            .header("Content-Type", "application/json");
+        let request = if let Some(username) = &self.username {
+            request.basic_auth(username, self.password.clone())
+        } else {
+            request
+        };
+        Ok(request)
+    }
+}

src/agent/importer.rs

@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright (C) 2020-2022 Jason Ish
+// EveBox agent import. For importing events to an EveBox server.
+
+use crate::agent::client::Client;
+use crate::eve::eve::EveJson;
+use tracing::trace;
+
+#[derive(Debug, Clone)]
+pub struct EveboxImporter {
+    pub client: Client,
+    pub queue: Vec<String>,
+}
+
+impl EveboxImporter {
+    pub fn new(client: Client) -> Self {
+        Self {
+            queue: Vec::new(),
+            client: client,
+        }
+    }
+
+    pub async fn submit(
+        &mut self,
+        event: EveJson,
+    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        self.queue.push(event.to_string());
+        Ok(())
+    }
+
+    pub fn pending(&self) -> usize {
+        self.queue.len()
+    }
+
+    pub async fn commit(&mut self) -> anyhow::Result<usize> {
+        let n = self.queue.len();
+        let body = self.queue.join("\n");
+        let size = body.len();
+        trace!("Committing {} events (bytes: {})", n, size);
+        let r = self.client.post("api/1/submit")?.body(body).send().await?;
+        let status_code = r.status();
+        if status_code != 200 {
+            let response_body = r.text().await?;
+            if !response_body.is_empty() {
+                if let Ok(error) = serde_json::from_str::<serde_json::Value>(&response_body) {
+                    if let serde_json::Value::String(error) = &error["error"] {
+                        return Err(anyhow!("{}", error));
+                    }
+                }
+                return Err(anyhow!("{}", response_body));
+            }
+            return Err(anyhow!("Server returned status code {}", status_code));
+        }
+        self.queue.truncate(0);
+        Ok(n)
+    }
+}