enhancement(remap): Add join function (#6313)

* Add basic function scaffolding

Signed-off-by: Luc Perkins <luc@timber.io>

* Add initial implementation of join logic plus tests

Signed-off-by: Luc Perkins <luc@timber.io>

* Add more tests

Signed-off-by: Luc Perkins <luc@timber.io>

* Add comment explaining vector length comparison

Signed-off-by: Luc Perkins <luc@timber.io>

* Re-use array variable

Signed-off-by: Luc Perkins <luc@timber.io>

* Fix formatting issue

Signed-off-by: Luc Perkins <luc@timber.io>

* Adopt Clippy suggestion

Signed-off-by: Luc Perkins <luc@timber.io>

* Simplify type def

Signed-off-by: Luc Perkins <luc@timber.io>

* Refactor string collection logic

Signed-off-by: Luc Perkins <luc@timber.io>

* Remove string conversion logic

Signed-off-by: Luc Perkins <luc@timber.io>

* Remove unnecessary type signature

Signed-off-by: Luc Perkins <luc@timber.io>

* Add inner type def for value array

Signed-off-by: Luc Perkins <luc@timber.io>

* Add mixed type array test

Signed-off-by: Luc Perkins <luc@timber.io>

* Add inner type constraint for join array

Signed-off-by: Luc Perkins <luc@timber.io>

* Change function name and add docs

Signed-off-by: Luc Perkins <luc@timber.io>

* Add behavior test

Signed-off-by: Luc Perkins <luc@timber.io>

* Add tests for new array type constraint function

Signed-off-by: Luc Perkins <luc@timber.io>

* Set fallible to true for non-arrays

Signed-off-by: Luc Perkins <luc@timber.io>

* Add more tests

Signed-off-by: Luc Perkins <luc@timber.io>

* Fix behavior test

Signed-off-by: Luc Perkins <luc@timber.io>

* Make maps fallible in array type function

Signed-off-by: Luc Perkins <luc@timber.io>

* Add test for maps

Signed-off-by: Luc Perkins <luc@timber.io>

* Set fallible to true unless array + mismatched kinds

Signed-off-by: Luc Perkins <luc@timber.io>

* Revamp inner type logic

Signed-off-by: Luc Perkins <luc@timber.io>

lib/remap-functions/Cargo.toml

@@ -62,6 +62,7 @@ default = [
     "ip_to_ipv6",
     "ipv6_to_ipv4",
     "is_nullish",
+    "join",
     "length",
     "log",
     "match",
@@ -132,6 +133,7 @@ ip_subnet = ["lazy_static", "regex"]
 ip_to_ipv6 = []
 ipv6_to_ipv4 = []
 is_nullish = []
+join = []
 length = []
 log = ["tracing"]
 match = ["regex"]

lib/remap-functions/src/join.rs

@@ -0,0 +1,174 @@
+use remap::prelude::*;
+use std::borrow::Cow;
+
+#[derive(Clone, Copy, Debug)]
+pub struct Join;
+
+impl Function for Join {
+    fn identifier(&self) -> &'static str {
+        "join"
+    }
+
+    fn parameters(&self) -> &'static [Parameter] {
+        &[
+            Parameter {
+                keyword: "value",
+                accepts: |v| matches!(v, Value::Array(_)),
+                required: true,
+            },
+            Parameter {
+                keyword: "separator",
+                accepts: |v| matches!(v, Value::Bytes(_)),
+                required: false,
+            },
+        ]
+    }
+
+    fn compile(&self, mut arguments: ArgumentList) -> Result<Box<dyn Expression>> {
+        let value = arguments.required("value")?.boxed();
+        let separator = arguments.optional("separator").map(Expr::boxed);
+
+        Ok(Box::new(JoinFn { value, separator }))
+    }
+}
+
+#[derive(Clone, Debug)]
+struct JoinFn {
+    value: Box<dyn Expression>,
+    separator: Option<Box<dyn Expression>>,
+}
+
+impl Expression for JoinFn {
+    fn execute(&self, state: &mut state::Program, object: &mut dyn Object) -> Result<Value> {
+        let array = self.value.execute(state, object)?.try_array()?;
+
+        let string_vec = array
+            .iter()
+            .map(|s| s.try_bytes_utf8_lossy().map_err(Into::into))
+            .collect::<Result<Vec<Cow<'_, str>>>>()
+            .map_err(|_| "all array items must be strings")?;
+
+        let separator: String = self
+            .separator
+            .as_ref()
+            .map(|s| {
+                s.execute(state, object)
+                    .and_then(|v| Value::try_bytes(v).map_err(Into::into))
+            })
+            .transpose()?
+            .map(|s| String::from_utf8_lossy(&s).to_string())
+            .unwrap_or_else(|| "".into());
+
+        let joined = string_vec.join(&separator);
+
+        Ok(Value::from(joined))
+    }
+
+    fn type_def(&self, state: &state::Compiler) -> TypeDef {
+        use value::Kind;
+
+        let separator_type = self
+            .separator
+            .as_ref()
+            .map(|separator| separator.type_def(state).fallible_unless(Kind::Bytes));
+
+        self.value
+            .type_def(state)
+            .fallible_unless(Kind::Array)
+            .merge_optional(separator_type)
+            .fallible_unless_array_has_inner_type(Kind::Bytes)
+            .with_constraint(Kind::Bytes)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use value::Kind;
+
+    test_type_def![
+        value_string_array_infallible {
+            expr: |_| JoinFn {
+                value: array!["one", "two", "three"].boxed(),
+                separator: Some(lit!(", ").boxed()),
+            },
+            def: TypeDef {
+                fallible: false,
+                kind: Kind::Bytes,
+                ..Default::default()
+            },
+        }
+
+        value_mixed_array_fallible {
+            expr: |_| JoinFn {
+                value: array!["one", 1].boxed(),
+                separator: Some(lit!(", ").boxed()),
+            },
+            def: TypeDef {
+                fallible: true,
+                kind: Kind::Bytes,
+                ..Default::default()
+            },
+        }
+
+        value_literal_fallible {
+            expr: |_| JoinFn {
+                value: lit!(427).boxed(),
+                separator: None,
+            },
+            def: TypeDef {
+                fallible: true,
+                kind: Kind::Bytes,
+                ..Default::default()
+            },
+        }
+
+        separator_integer_fallible {
+            expr: |_| JoinFn {
+                value: array!["one", "two", "three"].boxed(),
+                separator: Some(lit!(427).boxed()),
+            },
+            def: TypeDef {
+                fallible: true,
+                kind: Kind::Bytes,
+                ..Default::default()
+            },
+        }
+
+        both_types_wrong_fallible {
+            expr: |_| JoinFn {
+                value: lit!(true).boxed(),
+                separator: Some(lit!(427).boxed()),
+            },
+            def: TypeDef {
+                fallible: true,
+                kind: Kind::Bytes,
+                ..Default::default()
+            },
+        }
+    ];
+
+    test_function![
+        join => Join;
+
+        with_comma_separator {
+            args: func_args![value: array!["one", "two", "three"], separator: lit!(", ")],
+            want: Ok(value!("one, two, three")),
+        }
+
+        with_space_separator {
+            args: func_args![value: array!["one", "two", "three"], separator: lit!(" ")],
+            want: Ok(value!("one two three")),
+        }
+
+        without_separator {
+            args: func_args![value: array!["one", "two", "three"]],
+            want: Ok(value!("onetwothree")),
+        }
+
+        non_string_array_item_throws_error {
+            args: func_args![value: array!["one", "two", 3]],
+            want: Err("function call error: all array items must be strings"),
+        }
+    ];
+}

lib/remap-functions/src/lib.rs

@@ -48,6 +48,8 @@ mod ip_to_ipv6;
 mod ipv6_to_ipv4;
 #[cfg(feature = "is_nullish")]
 mod is_nullish;
+#[cfg(feature = "join")]
+mod join;
 #[cfg(feature = "length")]
 mod length;
 #[cfg(feature = "log")]
@@ -191,6 +193,8 @@ pub use ip_to_ipv6::IpToIpv6;
 pub use ipv6_to_ipv4::Ipv6ToIpV4;
 #[cfg(feature = "is_nullish")]
 pub use is_nullish::IsNullish;
+#[cfg(feature = "join")]
+pub use join::Join;
 #[cfg(feature = "length")]
 pub use length::Length;
 #[cfg(feature = "log")]
@@ -330,6 +334,8 @@ pub fn all() -> Vec<Box<dyn remap::Function>> {
         Box::new(Ipv6ToIpV4),
         #[cfg(feature = "is_nullish")]
         Box::new(IsNullish),
+        #[cfg(feature = "join")]
+        Box::new(Join),
         #[cfg(feature = "length")]
         Box::new(Length),
         #[cfg(feature = "log")]

lib/remap-lang/src/type_def.rs

@@ -206,6 +206,20 @@ impl TypeDef {
         self
     }
 
+    /// Applies a type constraint to the items in an array. If you need all items in the array to
+    /// be integers, for example, set `Kind::Integer`; if items can be either integers or Booleans,
+    /// set `Kind::Integer | Kind::Boolean`; and so on.
+    pub fn fallible_unless_array_has_inner_type(mut self, kind: impl Into<value::Kind>) -> Self {
+        match &self.inner_type_def {
+            Some(InnerTypeDef::Array(inner_kind)) if kind.into() == inner_kind.kind => (),
+            _ => {
+                self.fallible = true;
+            }
+        }
+
+        self
+    }
+
     pub fn merge(self, other: Self) -> Self {
         self | other
     }
@@ -412,4 +426,63 @@ mod tests {
 
         assert_eq!(expected, type_def_a | type_def_b);
     }
+
+    #[test]
+    fn array_inner_type() {
+        // All items are strings + all must be strings -> infallible
+        let non_mixed_array = TypeDef {
+            inner_type_def: Some(inner_type_def!([Kind::Bytes])),
+            ..Default::default()
+        }
+        .fallible_unless_array_has_inner_type(Kind::Bytes);
+
+        assert!(!non_mixed_array.is_fallible());
+
+        // Items are strings or Booleans + all must be strings -> fallible
+        let mixed_array_mismatched = TypeDef {
+            inner_type_def: Some(inner_type_def!([Kind::Bytes | Kind::Boolean])),
+            ..Default::default()
+        }
+        .fallible_unless_array_has_inner_type(Kind::Bytes);
+
+        assert!(mixed_array_mismatched.is_fallible());
+
+        // Items are integers or floats + all must be integers or floats -> infallible
+        let mixed_array_matched = TypeDef {
+            inner_type_def: Some(inner_type_def!([Kind::Integer | Kind::Float])),
+            ..Default::default()
+        }
+        .fallible_unless_array_has_inner_type(Kind::Integer | Kind::Float);
+
+        assert!(!mixed_array_matched.is_fallible());
+
+        // Items are Booleans or maps + must be floats -> fallible
+        let mismatched_array = TypeDef {
+            inner_type_def: Some(inner_type_def!([Kind::Boolean | Kind::Map])),
+            ..Default::default()
+        }
+        .fallible_unless_array_has_inner_type(Kind::Float);
+
+        assert!(mismatched_array.is_fallible());
+
+        // Setting a required array type on a map -> fallible
+        let map_type = TypeDef {
+            kind: Kind::Map,
+            inner_type_def: Some(inner_type_def!([Kind::Map])),
+            ..Default::default()
+        }
+        .fallible_unless_array_has_inner_type(Kind::Bytes);
+
+        assert!(map_type.is_fallible());
+
+        // Any non-array should be fallible if an inner type constraint is
+        // applied
+        let non_array = TypeDef {
+            kind: Kind::Bytes | Kind::Float | Kind::Boolean,
+            ..Default::default()
+        }
+        .fallible_unless_array_has_inner_type(Kind::Bytes);
+
+        assert!(non_array.is_fallible());
+    }
 }

tests/behavior/transforms/remap.toml

@@ -1979,3 +1979,30 @@
       source = '''
         .a != ""
       '''
+
+[transforms.remap_function_join]
+  inputs = []
+  type = "remap"
+  source = """
+    items = ["foo", "bar", "baz"]
+    .comma = join(items, ", ")
+    .space = join(items, " ")
+    .none = join(items)
+    .from_split = join!(split("big bad booper", " "), "__")
+  """
+[[tests]]
+  name = "remap_function_join"
+  [tests.input]
+    insert_at = "remap_function_join"
+    type = "log"
+    [tests.input.log_fields]
+  [[tests.outputs]]
+    extract_from = "remap_function_join"
+    [[tests.outputs.conditions]]
+      type = "remap"
+      source = '''
+        .comma == "foo, bar, baz" && \
+        .space == "foo bar baz" && \
+        .none == "foobarbaz" && \
+        .from_split == "big__bad__booper"
+      '''