feat(vrl): Support DataDog grok parser

Cargo.toml

@@ -89,7 +89,9 @@ members = [
   "lib/vrl/stdlib",
   "lib/vrl/tests",
   "lib/vrl/proptests",
-  "lib/datadog/search-syntax"
+  "lib/datadog/search-syntax",
+  "lib/datadog/grok",
+  "lib/parsing"
 ]
 
 [dependencies]
@@ -193,6 +195,7 @@ vrl-compiler = { path = "lib/vrl/compiler", optional = true }
 # Lookup
 lookup = { path = "lib/lookup" }
 
+
 # External libs
 anyhow = { version = "1.0.41", default-features = false }
 async-compression = { version = "0.3.7", default-features = false, features = ["tokio", "gzip", "zstd"] }

lib/datadog/grok/Cargo.toml

@@ -0,0 +1,42 @@
+[package]
+name = "datadog-grok"
+version = "0.1.0"
+authors = ["Vector Contributors <vector@timber.io>"]
+edition = "2018"
+build = "src/build.rs" # LALRPOP preprocessing
+
+[dependencies]
+serde = { version = "1.0.126", default-features = false, features = ["derive"] }
+serde_yaml = "0.8"
+lalrpop-util = "0.19"
+thiserror = "1"
+lazy_static = "1.3.0"
+regex = "1"
+grok = "1"
+indoc = { version = "1.0.3", default-features = false }
+inventory = { version = "0.1.10", default-features = false }
+async-trait = "0.1.50"
+typetag = { version = "0.1.7", default-features = false }
+derivative = { version = "2.2.0", default-features = false }
+itertools = { version = "0.10.1", default-features = false }
+serde_json = { version = "1.0.64", default-features = false }
+percent-encoding = { version = "2.1.0", default-features = false }
+bytes = "1.0.0"
+enum_derive = "0.1.3"
+custom_derive = "0.1.2"
+strum = "0.21.0"
+strum_macros = "0.21"
+tracing = { version = "0.1.26", default-features = false }
+tokio = { version = "1", features = ["sync"] } # is it the right way to enable sync?
+
+# Internal
+shared = { path = "../../shared" }
+lookup = { path = "../../lookup" }
+vector_core = { path = "../../vector-core"}
+parsing = { path = "../../parsing"}
+
+[build-dependencies]
+lalrpop = "0.19"
+
+[dev-dependencies]
+anyhow = "1"

lib/datadog/grok/patterns/core

@@ -0,0 +1,64 @@
+# Basic constructs
+numberStr [+-]?(?>\d+(?:\.(?:\d*)?)?|\.\d+)
+numberExtStr [+-]?(?>\d+(?:\.(?:\d*)?)?|\.\d+)(?:[eE][+-]?\d+)?
+integerStr [+-]?\d+
+integerExtStr [+-]?\d+(?:[eE][+-]?\d+)?
+word \b\w+\b
+
+doubleQuotedString "[^"]*"
+singleQuotedString '[^']*'
+quotedString (?>%{doubleQuotedString}|%{singleQuotedString})
+qs %{quotedString}
+
+uuid [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
+
+notSpace \S+
+data .*?
+greedyData .*
+space \s+
+
+# Username
+user [\w.-]+
+
+# Networking
+# MAC addresses
+ciscoMac (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
+windowsMac (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
+commonMac (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
+mac (?:%{ciscoMac}|%{windowsMac}|%{commonMac})
+
+# IP addresses
+ipv6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
+ipv4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])
+ip (?:%{ipv6}|%{ipv4})
+
+# Hostname
+# We allow underscores in hostnames (https://issues.apache.org/bugzilla/show_bug.cgi?id=21133)
+hostname \b(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62}))*(\.?|\b)
+host %{hostname}
+
+# IP or host
+ipOrHost (?:%{ip}|%{host})
+
+# A port can be any value from 1 to 65535
+port [1-9]\d{0,4}
+hostPort %{ipOrHost}:%{port}
+
+# Paths
+path (?:%{unixPath}|%{winPath})
+unixPath (?>/(?>[\w_%!$@:.,~-]+|\\.)*)+
+tty (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
+winPath (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
+uriProto [A-Za-z]+(?:\+[A-Za-z+]+)?
+uriHost %{ipOrHost}(?::%{port})?
+
+# Uripath comes loosely from RFC1738, but mostly from what Firefox
+# doesn't turn into %XX
+uriPath (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+
+#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
+uriParam \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]*
+uriPathParam %{uriPath}(?:%{uriParam})?
+uri %{uriProto}://(?:%{user}(?::[^@]*)?@)?(?:%{uriHost})?(?:%{uriPathParam})?
+
+# Log Levels
+# LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)

lib/datadog/grok/src/ast.rs

@@ -0,0 +1,26 @@
+use lookup::LookupBuf;
+use vector_core::event::Value;
+
+#[derive(Clone, Debug)]
+pub struct GrokPattern {
+    pub match_fn: Function,
+    pub destination: Option<Destination>,
+}
+
+#[derive(Clone, Debug)]
+pub struct Destination {
+    pub path: LookupBuf,
+    pub filter_fn: Option<Function>,
+}
+
+#[derive(Clone, Debug)]
+pub struct Function {
+    pub name: String,
+    pub args: Option<Vec<FunctionArgument>>,
+}
+
+#[derive(Clone, Debug)]
+pub enum FunctionArgument {
+    Function(Function),
+    Arg(Value),
+}

lib/datadog/grok/src/build.rs

@@ -0,0 +1,41 @@
+extern crate lalrpop;
+
+use std::fmt::Write as fmt_write;
+use std::fs::File;
+use std::io::{BufRead, BufReader};
+use std::path::Path;
+use std::{env, fs};
+
+fn main() {
+    lalrpop::Configuration::new()
+        .always_use_colors()
+        .process_current_dir()
+        .unwrap();
+
+    println!("cargo:rerun-if-changed=src/parser.lalrpop");
+
+    read_grok_patterns();
+}
+
+/// Reads grok patterns defined in the `patterns` folder into the static `PATTERNS` variable
+fn read_grok_patterns() {
+    let mut output = "static PATTERNS: &[(&str, &str)] = &[\n".to_string();
+
+    fs::read_dir(Path::new("patterns"))
+        .expect("can read 'patterns' dir")
+        .filter_map(|path| File::open(path.expect("can read 'patterns' dir").path()).ok())
+        .flat_map(|f| BufReader::new(f).lines().filter_map(|l| l.ok()))
+        .filter(|line| !line.starts_with('#') && !line.is_empty())
+        .for_each(|line| {
+            let (key, value) =
+                line.split_at(line.find(' ').expect("pattern is 'ruleName definition'"));
+            write!(output, "\t(\"{}\", r#\"{}\"#),", key, &value[1..])
+                .expect("can append patterns");
+        });
+
+    output.push_str("];\n");
+
+    let out_dir = env::var("OUT_DIR").expect("OUT_DIR is defined");
+    let dest_path = Path::new(&out_dir).join("patterns.rs");
+    fs::write(dest_path, output).expect("'patterns.rs' is created");
+}