enhancement(remap): Add base64 encoding and decoding functions (#5768)

* Add encode_base64 function Signed-off-by: Luc Perkins <luc@timber.io> * Add decode_base64 function Signed-off-by: Luc Perkins <luc@timber.io> * Add padding parameter to encode function plus behvaior tests and docs Signed-off-by: Luc Perkins <luc@timber.io> * Fix TypeDef for encoding function Signed-off-by: Luc Perkins <luc@timber.io> * Fix test name Signed-off-by: Luc Perkins <luc@timber.io> * Update docs to include padding option Signed-off-by: Luc Perkins <luc@timber.io> * Fix misleading test name Signed-off-by: Luc Perkins <luc@timber.io> * Fix errant example in docs Signed-off-by: Luc Perkins <luc@timber.io> * Add test for decoding padded vs non padded Signed-off-by: Luc Perkins <luc@timber.io> * Incorporate @JeanMertz suggestions Signed-off-by: Luc Perkins <luc@timber.io> * Make decode_base64 always fallible Signed-off-by: Luc Perkins <luc@timber.io> * Refactor padding defaulting and parsing Signed-off-by: Jesse Szwedko <jesse@szwedko.me> * Fix CUE formatting Signed-off-by: Luc Perkins <luc@timber.io> Co-authored-by: Jesse Szwedko <jesse@szwedko.me>
vectordotdev · Jan 11, 2021 · 1b9880c · 1b9880c
1 parent 02c327a
commit 1b9880c
Show file tree

Hide file tree

Showing 10 changed files with 367 additions and 1 deletion.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/docs/reference/remap.cue b/docs/reference/remap.cue
@@ -48,7 +48,7 @@ remap: {
 
 			arguments: [...#Argument] // Allow for empty list
 			return: [#RemapReturnTypes, ...#RemapReturnTypes]
-			category:    "Array" | "Check" | "Coerce" | "Encode" | "Enumerate" | "Event" | "Hash" | "IP" | "Map" | "Number" | "Parse" | "Random" | "String" | "Test" | "Timestamp"
+			category:    "Array" | "Check" | "Coerce" | "Decode" | "Encode" | "Enumerate" | "Event" | "Hash" | "IP" | "Map" | "Number" | "Parse" | "Random" | "String" | "Test" | "Timestamp"
 			description: string
 			examples?: [#RemapExample, ...#RemapExample]
 			name: Name

diff --git a/docs/reference/remap/decode_base64.cue b/docs/reference/remap/decode_base64.cue
@@ -0,0 +1,28 @@
+package metadata
+
+remap: functions: decode_base64: {
+	arguments: [
+		{
+			name:        "value"
+			description: "The [Base64](\(urls.base64)) data to decode."
+			required:    true
+			type: ["string"]
+		},
+	]
+	return: ["string"]
+	category:    "Decode"
+	description: "Decodes the provided [Base64](\(urls.base64)) data to a string."
+	examples: [
+		{
+			title: "Decode Base64 data"
+			input: {
+				message: "eW91IGhhdmUgc3VjY2Vzc2Z1bGx5IGRlY29kZWQgbWU="
+			}
+			source: ".decoded = decode_base64(.message)"
+			output: {
+				message: "eW91IGhhdmUgc3VjY2Vzc2Z1bGx5IGRlY29kZWQgbWU="
+				decoded: "you have successfully decoded me"
+			}
+		},
+	]
+}
diff --git a/docs/reference/remap/encode_base64.cue b/docs/reference/remap/encode_base64.cue
@@ -0,0 +1,46 @@
+package metadata
+
+remap: functions: encode_base64: {
+	arguments: [
+		{
+			name:        "value"
+			description: "The string to encode."
+			required:    true
+			type: ["string"]
+		},
+		{
+			name:        "padding"
+			description: "Whether the Base64 output is [padded](\(urls.base64_padding))."
+			required:    false
+			type: ["boolean"]
+			default: true
+		},
+	]
+	return: ["string"]
+	category:    "Encode"
+	description: "Encodes the provided string to [Base64](\(urls.base64))."
+	examples: [
+		{
+			title: "Encode string"
+			input: {
+				message: "please encode me"
+			}
+			source: ".encoded = encode_base64(.message)"
+			output: {
+				message: "please encode me"
+				encoded: "cGxlYXNlIGVuY29kZSBtZQ=="
+			}
+		},
+		{
+			title: "Encode string without padding"
+			input: {
+				message: "please encode me, no padding though"
+			}
+			source: ".encoded = encode_base64(.message, padding = false)"
+			output: {
+				message: "please encode me, no padding though"
+				encoded: "cGxlYXNlIGVuY29kZSBtZSwgbm8gcGFkZGluZyB0aG91Z2g"
+			}
+		},
+	]
+}
diff --git a/docs/reference/urls.cue b/docs/reference/urls.cue
@@ -75,6 +75,8 @@ urls: {
 	aws_vpc_flow_logs:                                        "https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html"
 	azure_monitor:                                            "https://azure.microsoft.com/en-us/services/monitor/"
 	azure_monitor_logs_endpoints:                             "https://docs.microsoft.com/en-us/rest/api/monitor/"
+	base64:                                                   "https://en.wikipedia.org/wiki/Base64"
+	base64_padding:                                           "https://en.wikipedia.org/wiki/Base64#Output_padding"
 	basic_auth:                                               "https://en.wikipedia.org/wiki/Basic_access_authentication"
 	big_query_streaming:                                      "https://cloud.google.com/bigquery/streaming-data-into-bigquery"
 	cargo_audit:                                              "https://github.com/RustSec/cargo-audit"

diff --git a/lib/remap-functions/Cargo.toml b/lib/remap-functions/Cargo.toml
@@ -8,6 +8,7 @@ publish = false
 [dependencies]
 remap = { package = "remap-lang", path = "../remap-lang" }
 
+base64 = { version = "0.13.0", optional = true }
 bytes = { version = "0.5.6", optional = true }
 chrono = { version = "0.4", optional = true }
 cidr-utils = { version = "0.5", optional = true }
@@ -40,8 +41,10 @@ default = [
     "ceil",
     "compact",
     "contains",
+    "decode_base64",
     "del",
     "downcase",
+    "encode_base64",
     "encode_json",
     "ends_with",
     "exists",
@@ -106,8 +109,10 @@ assert = []
 ceil = []
 compact = []
 contains = []
+decode_base64 = ["base64"]
 del = []
 downcase = []
+encode_base64 = ["base64"]
 encode_json = ["serde_json"]
 ends_with = []
 exists = []

diff --git a/lib/remap-functions/src/decode_base64.rs b/lib/remap-functions/src/decode_base64.rs
@@ -0,0 +1,89 @@
+use remap::prelude::*;
+
+#[derive(Clone, Copy, Debug)]
+pub struct DecodeBase64;
+
+impl Function for DecodeBase64 {
+    fn identifier(&self) -> &'static str {
+        "decode_base64"
+    }
+
+    fn parameters(&self) -> &'static [Parameter] {
+        &[Parameter {
+            keyword: "value",
+            accepts: |v| matches!(v, Value::Bytes(_)),
+            required: true,
+        }]
+    }
+
+    fn compile(&self, mut arguments: ArgumentList) -> Result<Box<dyn Expression>> {
+        let value = arguments.required("value")?.boxed();
+
+        Ok(Box::new(DecodeBase64Fn { value }))
+    }
+}
+
+#[derive(Clone, Debug)]
+struct DecodeBase64Fn {
+    value: Box<dyn Expression>,
+}
+
+impl Expression for DecodeBase64Fn {
+    fn execute(&self, state: &mut state::Program, object: &mut dyn Object) -> Result<Value> {
+        let value = self.value.execute(state, object)?.try_bytes()?;
+
+        base64::decode(value)
+            .map(Into::into)
+            .map_err(|_| "unable to decode value to base64".into())
+    }
+
+    fn type_def(&self, state: &state::Compiler) -> TypeDef {
+        // Always fallible due to the possibility of decoding errors that VRL can't detect in
+        // advance: https://docs.rs/base64/0.13.0/base64/enum.DecodeError.html
+        self.value
+            .type_def(state)
+            .into_fallible(true)
+            .with_constraint(value::Kind::Bytes)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use value::Kind;
+
+    test_type_def![
+        value_string_fallible {
+            expr: |_| DecodeBase64Fn {
+                value: Literal::from("foo").boxed(),
+            },
+            def: TypeDef { fallible: true, kind: Kind::Bytes, ..Default::default() },
+        }
+
+        value_non_string_fallible {
+            expr: |_| DecodeBase64Fn {
+                value: Literal::from(127).boxed(),
+            },
+            def: TypeDef { fallible: true, kind: Kind::Bytes, ..Default::default() },
+        }
+    ];
+
+    test_function![
+        decode_base64 => DecodeBase64;
+
+        string_value_with_padding {
+            args: func_args![value: value!("c29tZSBzdHJpbmcgdmFsdWU=")],
+            want: Ok(value!("some string value")),
+        }
+
+        string_value_no_padding {
+            args: func_args![value: value!("c29tZSBzdHJpbmcgdmFsdWU")],
+            want: Ok(value!("some string value")),
+        }
+
+        empty_string_value {
+            args: func_args![value: value!("")],
+            want: Ok(value!("")),
+        }
+    ];
+}
diff --git a/lib/remap-functions/src/encode_base64.rs b/lib/remap-functions/src/encode_base64.rs
@@ -0,0 +1,149 @@
+use remap::prelude::*;
+
+#[derive(Clone, Copy, Debug)]
+pub struct EncodeBase64;
+
+impl Function for EncodeBase64 {
+    fn identifier(&self) -> &'static str {
+        "encode_base64"
+    }
+
+    fn parameters(&self) -> &'static [Parameter] {
+        &[
+            Parameter {
+                keyword: "value",
+                accepts: |v| matches!(v, Value::Bytes(_)),
+                required: true,
+            },
+            Parameter {
+                keyword: "padding",
+                accepts: |v| matches!(v, Value::Boolean(_)),
+                required: false,
+            },
+        ]
+    }
+
+    fn compile(&self, mut arguments: ArgumentList) -> Result<Box<dyn Expression>> {
+        let value = arguments.required("value")?.boxed();
+        let padding = arguments.optional("padding").map(Expr::boxed);
+
+        Ok(Box::new(EncodeBase64Fn { value, padding }))
+    }
+}
+
+#[derive(Clone, Debug)]
+struct EncodeBase64Fn {
+    value: Box<dyn Expression>,
+    padding: Option<Box<dyn Expression>>,
+}
+
+impl Expression for EncodeBase64Fn {
+    fn execute(&self, state: &mut state::Program, object: &mut dyn Object) -> Result<Value> {
+        let value = self.value.execute(state, object)?.try_bytes()?;
+
+        let padding = self
+            .padding
+            .as_ref()
+            .map(|p| {
+                p.execute(state, object)
+                    .and_then(|v| Value::try_boolean(v).map_err(Into::into))
+            })
+            .transpose()?
+            .unwrap_or(true);
+
+        let config = if padding {
+            base64::STANDARD
+        } else {
+            base64::STANDARD_NO_PAD
+        };
+
+        Ok(base64::encode_config(value, config).into())
+    }
+
+    fn type_def(&self, state: &state::Compiler) -> TypeDef {
+        use value::Kind;
+
+        let padding_def = self
+            .padding
+            .as_ref()
+            .map(|padding| padding.type_def(state).fallible_unless(Kind::Boolean));
+
+        self.value
+            .type_def(state)
+            .fallible_unless(Kind::Bytes)
+            .merge_optional(padding_def)
+            .with_constraint(Kind::Bytes)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use value::Kind;
+
+    test_type_def![
+        value_string_padding_unspecified_infallible {
+            expr: |_| EncodeBase64Fn {
+                value: Literal::from("foo").boxed(),
+                padding: None,
+            },
+            def: TypeDef { kind: Kind::Bytes, ..Default::default() },
+        }
+
+        value_string_padding_boolean_infallible {
+            expr: |_| EncodeBase64Fn {
+                value: Literal::from("foo").boxed(),
+                padding: Some(Literal::from(false).boxed()),
+            },
+            def: TypeDef { kind: Kind::Bytes, ..Default::default() },
+        }
+
+        value_string_padding_non_boolean_fallible {
+            expr: |_| EncodeBase64Fn {
+                value: Literal::from("foo").boxed(),
+                padding: Some(Literal::from("foo").boxed()),
+            },
+            def: TypeDef { fallible: true, kind: Kind::Bytes, ..Default::default() },
+        }
+
+        value_non_string_fallible {
+            expr: |_| EncodeBase64Fn {
+                value: Literal::from(127).boxed(),
+                padding: Some(Literal::from(true).boxed()),
+            },
+            def: TypeDef { fallible: true, kind: Kind::Bytes, ..Default::default() },
+        }
+
+        both_types_wrong_fallible {
+            expr: |_| EncodeBase64Fn {
+                value: Literal::from(127).boxed(),
+                padding: Some(Literal::from("foo").boxed()),
+            },
+            def: TypeDef { fallible: true, kind: Kind::Bytes, ..Default::default() },
+        }
+    ];
+
+    test_function![
+        encode_base64 => EncodeBase64;
+
+        string_value_with_padding {
+            args: func_args![value: value!("some string value"), padding: value!(true)],
+            want: Ok(value!("c29tZSBzdHJpbmcgdmFsdWU=")),
+        }
+
+        string_value_with_default_padding {
+            args: func_args![value: value!("some string value")],
+            want: Ok(value!("c29tZSBzdHJpbmcgdmFsdWU=")),
+        }
+
+        string_value_no_padding {
+            args: func_args![value: value!("some string value"), padding: value!(false)],
+            want: Ok(value!("c29tZSBzdHJpbmcgdmFsdWU")),
+        }
+
+        empty_string_value {
+            args: func_args![value: value!("")],
+            want: Ok(value!("")),
+        }
+    ];
+}