diff --git a/lib/remap-lang/src/error.rs b/lib/remap-lang/src/error.rs index dac484d7a7c47..311edccf5d428 100644 --- a/lib/remap-lang/src/error.rs +++ b/lib/remap-lang/src/error.rs @@ -1,12 +1,15 @@ -use crate::{expression, function, parser::Rule, value}; +use crate::{expression, function, parser::Rule, program, value}; use std::error::Error as StdError; use std::fmt; -#[derive(thiserror::Error, Debug, PartialEq)] +#[derive(thiserror::Error, Clone, Debug, PartialEq)] pub enum Error { #[error("parser error: {0}")] Parser(String), + #[error("program error")] + Program(#[from] program::Error), + #[error("unexpected token sequence")] Rule(#[from] Rule), @@ -107,7 +110,7 @@ impl fmt::Display for Rule { } } -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub struct RemapError(pub(crate) Error); impl StdError for RemapError { @@ -130,6 +133,12 @@ impl fmt::Display for RemapError { } } +impl From for RemapError { + fn from(error: Error) -> Self { + RemapError(error) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/lib/remap-lang/src/expression.rs b/lib/remap-lang/src/expression.rs index c91db1e32c68b..9315de39b2094 100644 --- a/lib/remap-lang/src/expression.rs +++ b/lib/remap-lang/src/expression.rs @@ -1,29 +1,28 @@ -use crate::{Object, Result, State, Value}; +use crate::{state, Object, Result, TypeDef, Value}; -pub(super) mod arithmetic; -pub(super) mod assignment; +mod arithmetic; +mod assignment; mod block; -pub(super) mod function; -pub(super) mod if_statement; +pub(crate) mod function; +mod if_statement; mod literal; mod noop; -pub(super) mod not; -pub(super) mod path; -pub(super) mod variable; - -pub(super) use arithmetic::Arithmetic; -pub(super) use assignment::{Assignment, Target}; -pub(super) use block::Block; -pub(super) use function::Function; -pub(super) use if_statement::IfStatement; -pub(super) use not::Not; -pub(super) use variable::Variable; - +mod not; +pub(crate) mod path; +mod variable; + +pub use arithmetic::Arithmetic; +pub use assignment::{Assignment, Target}; +pub use block::Block; +pub use function::Function; +pub use if_statement::IfStatement; pub use literal::Literal; pub use noop::Noop; +pub use not::Not; pub use path::Path; +pub use variable::Variable; -#[derive(thiserror::Error, Debug, PartialEq)] +#[derive(thiserror::Error, Clone, Debug, PartialEq)] pub enum Error { #[error("expected expression, got none")] Missing, @@ -48,7 +47,9 @@ pub enum Error { } pub trait Expression: Send + Sync + std::fmt::Debug + dyn_clone::DynClone { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result>; + fn execute(&self, state: &mut state::Program, object: &mut dyn Object) + -> Result>; + fn type_def(&self, state: &state::Compiler) -> TypeDef; } dyn_clone::clone_trait_object!(Expression); @@ -66,16 +67,22 @@ macro_rules! expression_dispatch { /// Any expression that stores other expressions internally will still /// have to box this enum, to avoid infinite recursion. #[derive(Debug, Clone)] - pub(crate) enum Expr { + pub enum Expr { $($expr($expr)),+ } impl Expression for Expr { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute(&self, state: &mut state::Program, object: &mut dyn Object) -> Result> { match self { $(Expr::$expr(expression) => expression.execute(state, object)),+ } } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + match self { + $(Expr::$expr(expression) => expression.type_def(state)),+ + } + } } $( @@ -100,3 +107,53 @@ expression_dispatch![ Path, Variable, ]; + +#[cfg(test)] +mod tests { + use crate::value; + + #[test] + fn test_contains() { + use value::Constraint::*; + use value::Kind::*; + + let cases = vec![ + (true, Any, Any), + (true, Any, Exact(String)), + (true, Any, Exact(Integer)), + (true, Any, OneOf(vec![Float, Boolean])), + (true, Any, OneOf(vec![Map])), + (true, Exact(String), Exact(String)), + (true, Exact(String), OneOf(vec![String])), + (false, Exact(String), Exact(Array)), + (false, Exact(String), OneOf(vec![Integer])), + (false, Exact(String), OneOf(vec![Integer, Float])), + ]; + + for (expect, this, other) in cases { + assert_eq!(this.contains(&other), expect); + } + } + + #[test] + fn test_merge() { + use value::Constraint::*; + use value::Kind::*; + + let cases = vec![ + (Any, Any, Any), + (Any, OneOf(vec![Integer, String]), Any), + (OneOf(vec![Integer, Float]), Exact(Integer), Exact(Float)), + (Exact(Integer), Exact(Integer), Exact(Integer)), + ( + OneOf(vec![String, Integer, Float, Boolean]), + OneOf(vec![Integer, String]), + OneOf(vec![Float, Boolean]), + ), + ]; + + for (expect, this, other) in cases { + assert_eq!(this.merge(&other), expect); + } + } +} diff --git a/lib/remap-lang/src/expression/arithmetic.rs b/lib/remap-lang/src/expression/arithmetic.rs index 16c60adf24eb0..4706acf86635d 100644 --- a/lib/remap-lang/src/expression/arithmetic.rs +++ b/lib/remap-lang/src/expression/arithmetic.rs @@ -1,5 +1,5 @@ -use super::{Expr, Expression, Object, Result, State, Value}; -use crate::Operator; +use super::{Expr, Expression, Object, Result, TypeDef, Value}; +use crate::{state, value, Operator}; #[derive(Debug, Clone)] pub struct Arithmetic { @@ -9,13 +9,17 @@ pub struct Arithmetic { } impl Arithmetic { - pub(crate) fn new(lhs: Box, rhs: Box, op: Operator) -> Self { + pub fn new(lhs: Box, rhs: Box, op: Operator) -> Self { Self { lhs, rhs, op } } } impl Expression for Arithmetic { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let lhs = self .lhs .execute(state, object)? @@ -32,6 +36,9 @@ impl Expression for Arithmetic { Divide => lhs.try_div(rhs), Add => lhs.try_add(rhs), Subtract => lhs.try_sub(rhs), + + // TODO: make `Or` infallible, `Null`, `false` and `None` resolve to + // rhs, everything else resolves to lhs Or => lhs.try_or(rhs), And => lhs.try_and(rhs), Remainder => lhs.try_rem(rhs), @@ -45,4 +52,223 @@ impl Expression for Arithmetic { result.map(Some).map_err(Into::into) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + use value::{Constraint::*, Kind::*}; + use Operator::*; + + let constraint = match self.op { + Or => self + .lhs + .type_def(state) + .constraint + .merge(&self.rhs.type_def(state).constraint), + Multiply | Add => OneOf(vec![String, Integer, Float]), + Remainder | Subtract | Divide => OneOf(vec![Integer, Float]), + And | Equal | NotEqual | Greater | GreaterOrEqual | Less | LessOrEqual => { + Exact(Boolean) + } + }; + + TypeDef { + fallible: true, + optional: false, + constraint, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + expression::{Literal, Noop}, + test_type_def, + value::Constraint::*, + value::Kind::*, + }; + + test_type_def![ + or_exact { + expr: |_| Arithmetic::new( + Box::new(Literal::from("foo").into()), + Box::new(Literal::from(true).into()), + Operator::Or, + ), + def: TypeDef { + fallible: true, + optional: false, + constraint: OneOf(vec![String, Boolean]) + }, + } + + or_any { + expr: |_| Arithmetic::new( + Box::new(Noop.into()), + Box::new(Literal::from(true).into()), + Operator::Or, + ), + def: TypeDef { + fallible: true, + optional: false, + constraint: Any, + }, + } + + multiply { + expr: |_| Arithmetic::new( + Box::new(Noop.into()), + Box::new(Noop.into()), + Operator::Multiply, + ), + def: TypeDef { + fallible: true, + optional: false, + constraint: OneOf(vec![String, Integer, Float]), + }, + } + + add { + expr: |_| Arithmetic::new( + Box::new(Noop.into()), + Box::new(Noop.into()), + Operator::Add, + ), + def: TypeDef { + fallible: true, + optional: false, + constraint: OneOf(vec![String, Integer, Float]), + }, + } + + remainder { + expr: |_| Arithmetic::new( + Box::new(Noop.into()), + Box::new(Noop.into()), + Operator::Remainder, + ), + def: TypeDef { + fallible: true, + optional: false, + constraint: OneOf(vec![Integer, Float]), + }, + } + + subtract { + expr: |_| Arithmetic::new( + Box::new(Noop.into()), + Box::new(Noop.into()), + Operator::Subtract, + ), + def: TypeDef { + fallible: true, + optional: false, + constraint: OneOf(vec![Integer, Float]), + }, + } + + divide { + expr: |_| Arithmetic::new( + Box::new(Noop.into()), + Box::new(Noop.into()), + Operator::Divide, + ), + def: TypeDef { + fallible: true, + optional: false, + constraint: OneOf(vec![Integer, Float]), + }, + } + + and { + expr: |_| Arithmetic::new( + Box::new(Noop.into()), + Box::new(Noop.into()), + Operator::And, + ), + def: TypeDef { + fallible: true, + optional: false, + constraint: Exact(Boolean), + }, + } + + equal { + expr: |_| Arithmetic::new( + Box::new(Noop.into()), + Box::new(Noop.into()), + Operator::Equal, + ), + def: TypeDef { + fallible: true, + optional: false, + constraint: Exact(Boolean), + }, + } + + not_equal { + expr: |_| Arithmetic::new( + Box::new(Noop.into()), + Box::new(Noop.into()), + Operator::NotEqual, + ), + def: TypeDef { + fallible: true, + optional: false, + constraint: Exact(Boolean), + }, + } + + greater { + expr: |_| Arithmetic::new( + Box::new(Noop.into()), + Box::new(Noop.into()), + Operator::Greater, + ), + def: TypeDef { + fallible: true, + optional: false, + constraint: Exact(Boolean), + }, + } + + greater_or_equal { + expr: |_| Arithmetic::new( + Box::new(Noop.into()), + Box::new(Noop.into()), + Operator::GreaterOrEqual, + ), + def: TypeDef { + fallible: true, + optional: false, + constraint: Exact(Boolean), + }, + } + + less { + expr: |_| Arithmetic::new( + Box::new(Noop.into()), + Box::new(Noop.into()), + Operator::Less, + ), + def: TypeDef { + fallible: true, + optional: false, + constraint: Exact(Boolean), + }, + } + + less_or_equal { + expr: |_| Arithmetic::new( + Box::new(Noop.into()), + Box::new(Noop.into()), + Operator::LessOrEqual, + ), + def: TypeDef { + fallible: true, + optional: false, + constraint: Exact(Boolean), + }, + } + ]; } diff --git a/lib/remap-lang/src/expression/assignment.rs b/lib/remap-lang/src/expression/assignment.rs index 7df05103936cb..78d6d946c4667 100644 --- a/lib/remap-lang/src/expression/assignment.rs +++ b/lib/remap-lang/src/expression/assignment.rs @@ -1,32 +1,46 @@ use super::Error as E; -use crate::{Expr, Expression, Object, Result, State, Value}; +use crate::{state, Expr, Expression, Object, Result, TypeDef, Value}; -#[derive(thiserror::Error, Debug, PartialEq)] +#[derive(thiserror::Error, Clone, Debug, PartialEq)] pub enum Error { #[error("unable to insert value in path: {0}")] PathInsertion(String), } #[derive(Debug, Clone)] -pub(crate) enum Target { +pub enum Target { Path(Vec>), Variable(String), } #[derive(Debug, Clone)] -pub(crate) struct Assignment { +pub struct Assignment { target: Target, value: Box, } impl Assignment { - pub fn new(target: Target, value: Box) -> Self { + pub fn new(target: Target, value: Box, state: &mut state::Compiler) -> Self { + let type_def = value.type_def(state); + + match &target { + Target::Variable(ident) => state.variable_types_mut().insert(ident.clone(), type_def), + Target::Path(segments) => { + let path = crate::expression::path::segments_to_path(segments); + state.path_query_types_mut().insert(path, type_def) + } + }; + Self { target, value } } } impl Expression for Assignment { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let value = self.value.execute(state, object)?; match value { @@ -45,4 +59,54 @@ impl Expression for Assignment { } } } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + match &self.target { + Target::Variable(ident) => state + .variable_type(ident.clone()) + .cloned() + .expect("variable must be assigned via Assignment::new"), + Target::Path(segments) => { + let path = crate::expression::path::segments_to_path(segments); + state + .path_query_type(&path) + .cloned() + .expect("variable must be assigned via Assignment::new") + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{expression::Literal, test_type_def, value::Constraint::*, value::Kind::*}; + + test_type_def![ + variable { + expr: |state: &mut state::Compiler| { + let target = Target::Variable("foo".to_owned()); + let value = Box::new(Literal::from(true).into()); + + Assignment::new(target, value, state) + }, + def: TypeDef { + constraint: Exact(Boolean), + ..Default::default() + }, + } + + path { + expr: |state: &mut state::Compiler| { + let target = Target::Path(vec![vec!["foo".to_owned()]]); + let value = Box::new(Literal::from("foo").into()); + + Assignment::new(target, value, state) + }, + def: TypeDef { + constraint: Exact(String), + ..Default::default() + }, + } + ]; } diff --git a/lib/remap-lang/src/expression/block.rs b/lib/remap-lang/src/expression/block.rs index 6796adea6a8c0..976777f5612fc 100644 --- a/lib/remap-lang/src/expression/block.rs +++ b/lib/remap-lang/src/expression/block.rs @@ -1,7 +1,7 @@ -use crate::{Expr, Expression, Object, Result, State, Value}; +use crate::{state, Expr, Expression, Object, Result, TypeDef, Value}; #[derive(Debug, Clone)] -pub(crate) struct Block { +pub struct Block { expressions: Vec, } @@ -12,7 +12,11 @@ impl Block { } impl Expression for Block { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let mut value = None; for expr in &self.expressions { @@ -21,4 +25,91 @@ impl Expression for Block { Ok(value) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + let mut type_defs = self + .expressions + .iter() + .map(|e| e.type_def(state)) + .collect::>(); + + // If any of the stored expressions is fallible, the entire block is + // fallible. + let fallible = type_defs.iter().any(TypeDef::is_fallible); + + // The last expression determines the resulting value of the block. + let mut type_def = type_defs.pop().unwrap_or(TypeDef { + optional: true, + ..Default::default() + }); + + type_def.fallible = fallible; + type_def + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + expression::{Arithmetic, Literal}, + test_type_def, + value::Constraint::*, + value::Kind::*, + Operator, + }; + + test_type_def![ + no_expression { + expr: |_| Block::new(vec![]), + def: TypeDef { optional: true, ..Default::default() }, + } + + one_expression { + expr: |_| Block::new(vec![Literal::from(true).into()]), + def: TypeDef { constraint: Exact(Boolean), ..Default::default() }, + } + + multiple_expressions { + expr: |_| Block::new(vec![ + Literal::from("foo").into(), + Literal::from(true).into(), + Literal::from(1234).into(), + ]), + def: TypeDef { constraint: Exact(Integer), ..Default::default() }, + } + + last_one_fallible { + expr: |_| Block::new(vec![ + Literal::from(true).into(), + Arithmetic::new( + Box::new(Literal::from(12).into()), + Box::new(Literal::from(true).into()), + Operator::Multiply, + ).into(), + ]), + def: TypeDef { + fallible: true, + constraint: OneOf(vec![String, Integer, Float]), + ..Default::default() + }, + } + + any_fallible { + expr: |_| Block::new(vec![ + Literal::from(true).into(), + Arithmetic::new( + Box::new(Literal::from(12).into()), + Box::new(Literal::from(true).into()), + Operator::Multiply, + ).into(), + Literal::from(vec![1]).into(), + ]), + def: TypeDef { + fallible: true, + constraint: Exact(Array), + ..Default::default() + }, + } + ]; } diff --git a/lib/remap-lang/src/expression/function.rs b/lib/remap-lang/src/expression/function.rs index d3daa08b618bc..d9dbd3919db6a 100644 --- a/lib/remap-lang/src/expression/function.rs +++ b/lib/remap-lang/src/expression/function.rs @@ -1,7 +1,10 @@ use super::Error as E; -use crate::{Argument, ArgumentList, Expression, Function as Fn, Object, Result, State, Value}; +use crate::{ + function::{Argument, ArgumentList}, + state, value, Expression, Function as Fn, Object, Result, TypeDef, Value, +}; -#[derive(thiserror::Error, Debug, PartialEq)] +#[derive(thiserror::Error, Clone, Debug, PartialEq)] pub enum Error { #[error("undefined")] Undefined, @@ -19,16 +22,16 @@ pub enum Error { Expression(&'static str), #[error(r#"incorrect value type for argument "{0}" (got "{0}")"#)] - Value(&'static str, &'static str), + Value(&'static str, value::Kind), } #[derive(Debug, Clone)] -pub(crate) struct Function { +pub struct Function { function: Box, } impl Function { - pub(crate) fn new( + pub fn new( ident: String, arguments: Vec<(Option, Argument)>, definitions: &[Box], @@ -120,9 +123,17 @@ impl Function { } impl Expression for Function { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { self.function.execute(state, object) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + self.function.type_def(state) + } } #[derive(Clone)] @@ -161,7 +172,11 @@ impl ArgumentValidator { } impl Expression for ArgumentValidator { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let value = self .expression .execute(state, object)? @@ -177,4 +192,26 @@ impl Expression for ArgumentValidator { Ok(Some(value)) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + self.expression.type_def(state) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{expression::Noop, test_type_def, value::Constraint::*}; + + test_type_def![pass_through { + expr: |_| { + let function = Box::new(Noop); + Function { function } + }, + def: TypeDef { + fallible: false, + optional: true, + constraint: Any + }, + }]; } diff --git a/lib/remap-lang/src/expression/if_statement.rs b/lib/remap-lang/src/expression/if_statement.rs index 2698f688c7019..7d6d2c36cfe3c 100644 --- a/lib/remap-lang/src/expression/if_statement.rs +++ b/lib/remap-lang/src/expression/if_statement.rs @@ -1,14 +1,14 @@ use super::Error as E; -use crate::{value, Expr, Expression, Object, Result, State, Value}; +use crate::{state, value, Expr, Expression, Object, Result, TypeDef, Value}; -#[derive(thiserror::Error, Debug, PartialEq)] +#[derive(thiserror::Error, Clone, Debug, PartialEq)] pub enum Error { #[error("invalid value kind")] Value(#[from] value::Error), } #[derive(Debug, Clone)] -pub(crate) struct IfStatement { +pub struct IfStatement { conditional: Box, true_expression: Box, false_expression: Box, @@ -29,15 +29,70 @@ impl IfStatement { } impl Expression for IfStatement { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { match self.conditional.execute(state, object)? { Some(Value::Boolean(true)) => self.true_expression.execute(state, object), Some(Value::Boolean(false)) | None => self.false_expression.execute(state, object), Some(v) => Err(E::from(Error::from(value::Error::Expected( - Value::Boolean(true).kind(), + value::Kind::Boolean, v.kind(), ))) .into()), } } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + self.conditional + .type_def(state) + .fallible_unless(value::Kind::Boolean) + .merge(self.true_expression.type_def(state)) + .merge(self.false_expression.type_def(state)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + expression::{Literal, Noop}, + test_type_def, + value::Constraint::*, + value::Kind::*, + }; + + test_type_def![ + concrete_type_def { + expr: |_| { + let conditional = Box::new(Literal::from(true).into()); + let true_expression = Box::new(Literal::from(true).into()); + let false_expression = Box::new(Literal::from(true).into()); + + IfStatement::new(conditional, true_expression, false_expression) + }, + def: TypeDef { + fallible: false, + optional: false, + constraint: Exact(Boolean), + }, + } + + optional_any { + expr: |_| { + let conditional = Box::new(Literal::from(true).into()); + let true_expression = Box::new(Literal::from(true).into()); + let false_expression = Box::new(Noop.into()); + + IfStatement::new(conditional, true_expression, false_expression) + }, + def: TypeDef { + fallible: false, + optional: true, + constraint: Any, + }, + } + ]; } diff --git a/lib/remap-lang/src/expression/literal.rs b/lib/remap-lang/src/expression/literal.rs index 22083c1c5eb2b..6329abc98f72b 100644 --- a/lib/remap-lang/src/expression/literal.rs +++ b/lib/remap-lang/src/expression/literal.rs @@ -1,8 +1,14 @@ -use crate::{Expression, Object, Result, State, Value}; +use crate::{state, value, Expression, Object, Result, TypeDef, Value}; #[derive(Debug, Clone)] pub struct Literal(Value); +impl Literal { + pub fn boxed(self) -> Box { + Box::new(self) + } +} + impl> From for Literal { fn from(value: T) -> Self { Self(value.into()) @@ -10,7 +16,63 @@ impl> From for Literal { } impl Expression for Literal { - fn execute(&self, _: &mut State, _: &mut dyn Object) -> Result> { + fn execute(&self, _: &mut state::Program, _: &mut dyn Object) -> Result> { Ok(Some(self.0.clone())) } + + fn type_def(&self, _: &state::Compiler) -> TypeDef { + TypeDef { + constraint: value::Constraint::Exact(self.0.kind()), + ..Default::default() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{test_type_def, value::Constraint::*, value::Kind::*}; + use std::collections::BTreeMap; + + test_type_def![ + boolean { + expr: |_| Literal::from(true), + def: TypeDef { constraint: Exact(Boolean), ..Default::default() }, + } + + string { + expr: |_| Literal::from("foo"), + def: TypeDef { constraint: Exact(String), ..Default::default() }, + } + + integer { + expr: |_| Literal::from(123), + def: TypeDef { constraint: Exact(Integer), ..Default::default() }, + } + + float { + expr: |_| Literal::from(123.456), + def: TypeDef { constraint: Exact(Float), ..Default::default() }, + } + + array { + expr: |_| Literal::from(vec!["foo"]), + def: TypeDef { constraint: Exact(Array), ..Default::default() }, + } + + map { + expr: |_| Literal::from(BTreeMap::default()), + def: TypeDef { constraint: Exact(Map), ..Default::default() }, + } + + timestamp { + expr: |_| Literal::from(chrono::Utc::now()), + def: TypeDef { constraint: Exact(Timestamp), ..Default::default() }, + } + + null { + expr: |_| Literal::from(()), + def: TypeDef { constraint: Exact(Null), ..Default::default() }, + } + ]; } diff --git a/lib/remap-lang/src/expression/noop.rs b/lib/remap-lang/src/expression/noop.rs index 48216ee917e06..fe0dd9546f43d 100644 --- a/lib/remap-lang/src/expression/noop.rs +++ b/lib/remap-lang/src/expression/noop.rs @@ -1,10 +1,31 @@ -use crate::{Expression, Object, Result, State, Value}; +use crate::{state, Expression, Object, Result, TypeDef, Value}; #[derive(Debug, Clone)] pub struct Noop; impl Expression for Noop { - fn execute(&self, _: &mut State, _: &mut dyn Object) -> Result> { + fn execute(&self, _: &mut state::Program, _: &mut dyn Object) -> Result> { Ok(None) } + + fn type_def(&self, _: &state::Compiler) -> TypeDef { + TypeDef { + optional: true, + ..Default::default() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_type_def; + + test_type_def![noop { + expr: |_| Noop, + def: TypeDef { + optional: true, + ..Default::default() + }, + }]; } diff --git a/lib/remap-lang/src/expression/not.rs b/lib/remap-lang/src/expression/not.rs index cddf52b661848..97a993905145b 100644 --- a/lib/remap-lang/src/expression/not.rs +++ b/lib/remap-lang/src/expression/not.rs @@ -1,14 +1,14 @@ use super::Error as E; -use crate::{value, Expr, Expression, Object, Result, State, Value}; +use crate::{state, value, Expr, Expression, Object, Result, TypeDef, Value}; -#[derive(thiserror::Error, Debug, PartialEq)] +#[derive(thiserror::Error, Clone, Debug, PartialEq)] pub enum Error { #[error("invalid value kind")] Value(#[from] value::Error), } #[derive(Debug, Clone)] -pub(crate) struct Not { +pub struct Not { expression: Box, } @@ -19,12 +19,16 @@ impl Not { } impl Expression for Not { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { self.expression.execute(state, object).and_then(|opt| { opt.map(|v| match v { Value::Boolean(b) => Ok(Value::Boolean(!b)), _ => Err(E::from(Error::from(value::Error::Expected( - Value::Boolean(true).kind(), + value::Kind::Boolean, v.kind(), ))) .into()), @@ -32,34 +36,43 @@ impl Expression for Not { .transpose() }) } + + fn type_def(&self, _: &state::Compiler) -> TypeDef { + TypeDef { + fallible: true, + optional: true, + constraint: value::Constraint::Exact(value::Kind::Boolean), + } + } } #[cfg(test)] mod tests { use super::*; + use crate::{expression::*, test_type_def, value::Constraint::*, value::Kind::*}; #[test] fn not() { let cases = vec![ ( Err("path error".to_string()), - Not::new(Box::new(crate::Path::from("foo").into())), + Not::new(Box::new(Path::from("foo").into())), ), ( Ok(Some(false.into())), - Not::new(Box::new(crate::Literal::from(true).into())), + Not::new(Box::new(Literal::from(true).into())), ), ( Ok(Some(true.into())), - Not::new(Box::new(crate::Literal::from(false).into())), + Not::new(Box::new(Literal::from(false).into())), ), ( Err("not operation error".to_string()), - Not::new(Box::new(crate::Literal::from("not a bool").into())), + Not::new(Box::new(Literal::from("not a bool").into())), ), ]; - let mut state = crate::State::default(); + let mut state = state::Program::default(); let mut object = std::collections::HashMap::default(); for (exp, func) in cases { @@ -70,4 +83,13 @@ mod tests { assert_eq!(got, exp); } } + + test_type_def![boolean { + expr: |_| Not::new(Box::new(Noop.into())), + def: TypeDef { + fallible: true, + optional: true, + constraint: Exact(Boolean), + }, + }]; } diff --git a/lib/remap-lang/src/expression/path.rs b/lib/remap-lang/src/expression/path.rs index b030c213aa3b7..7342ec9a777e0 100644 --- a/lib/remap-lang/src/expression/path.rs +++ b/lib/remap-lang/src/expression/path.rs @@ -1,7 +1,7 @@ use super::Error as E; -use crate::{Expression, Object, Result, State, Value}; +use crate::{state, Expression, Object, Result, TypeDef, Value}; -#[derive(thiserror::Error, Debug, PartialEq)] +#[derive(thiserror::Error, Clone, Debug, PartialEq)] pub enum Error { #[error("missing path: {0}")] Missing(String), @@ -31,16 +31,29 @@ impl Path { } impl Expression for Path { - fn execute(&self, _: &mut State, object: &mut dyn Object) -> Result> { + fn execute(&self, _: &mut state::Program, object: &mut dyn Object) -> Result> { object .find(&self.segments) .map_err(|e| E::from(Error::Resolve(e)))? .ok_or_else(|| E::from(Error::Missing(segments_to_path(&self.segments))).into()) .map(Some) } + + /// A path resolves to `Any` by default, but the script might assign + /// specific values to paths during its execution, which increases our exact + /// understanding of the value kind the path contains. + fn type_def(&self, state: &state::Compiler) -> TypeDef { + state + .path_query_type(&segments_to_path(&self.segments)) + .cloned() + .unwrap_or(TypeDef { + fallible: true, + ..Default::default() + }) + } } -fn segments_to_path(segments: &[Vec]) -> String { +pub(crate) fn segments_to_path(segments: &[Vec]) -> String { segments .iter() .map(|c| { @@ -52,3 +65,59 @@ fn segments_to_path(segments: &[Vec]) -> String { .collect::>() .join(".") } + +#[cfg(test)] +mod tests { + use super::*; + use crate::{test_type_def, value::Constraint::*, value::Kind::*}; + + test_type_def![ + ident_match { + expr: |state: &mut state::Compiler| { + state.path_query_types_mut().insert("foo".to_owned(), TypeDef::default()); + Path::from("foo") + }, + def: TypeDef::default(), + } + + exact_match { + expr: |state: &mut state::Compiler| { + state.path_query_types_mut().insert("foo".to_owned(), TypeDef { + fallible: true, + optional: false, + constraint: Exact(String) + }); + + Path::from("foo") + }, + def: TypeDef { + fallible: true, + optional: false, + constraint: Exact(String), + }, + } + + ident_mismatch { + expr: |state: &mut state::Compiler| { + state.path_query_types_mut().insert("foo".to_owned(), TypeDef { + fallible: true, + ..Default::default() + }); + + Path::from("bar") + }, + def: TypeDef { + fallible: true, + ..Default::default() + }, + } + + empty_state { + expr: |_| Path::from("foo"), + def: TypeDef { + fallible: true, + ..Default::default() + }, + } + ]; +} diff --git a/lib/remap-lang/src/expression/variable.rs b/lib/remap-lang/src/expression/variable.rs index c26b9007a046c..429a6b76ab01f 100644 --- a/lib/remap-lang/src/expression/variable.rs +++ b/lib/remap-lang/src/expression/variable.rs @@ -1,14 +1,14 @@ use super::Error as E; -use crate::{Expression, Object, Result, State, Value}; +use crate::{state, Expression, Object, Result, TypeDef, Value}; -#[derive(thiserror::Error, Debug, PartialEq)] +#[derive(thiserror::Error, Clone, Debug, PartialEq)] pub enum Error { #[error("undefined variable: {0}")] Undefined(String), } #[derive(Debug, Clone)] -pub(crate) struct Variable { +pub struct Variable { ident: String, } @@ -16,14 +16,86 @@ impl Variable { pub fn new(ident: String) -> Self { Self { ident } } + + pub fn boxed(self) -> Box { + Box::new(self) + } } impl Expression for Variable { - fn execute(&self, state: &mut State, _: &mut dyn Object) -> Result> { + fn execute(&self, state: &mut state::Program, _: &mut dyn Object) -> Result> { state .variable(&self.ident) .cloned() .ok_or_else(|| E::from(Error::Undefined(self.ident.to_owned())).into()) .map(Some) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + state + .variable_type(&self.ident) + .cloned() + // TODO: we can make it so this can never happen, by making it a + // compile-time error to reference a variable before it is assigned. + .unwrap_or(TypeDef { + fallible: true, + ..Default::default() + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{test_type_def, value::Constraint::*, value::Kind::*}; + + test_type_def![ + ident_match { + expr: |state: &mut state::Compiler| { + state.variable_types_mut().insert("foo".to_owned(), TypeDef::default()); + Variable::new("foo".to_owned()) + }, + def: TypeDef::default(), + } + + exact_match { + expr: |state: &mut state::Compiler| { + state.variable_types_mut().insert("foo".to_owned(), TypeDef { + fallible: true, + optional: false, + constraint: Exact(String) + }); + + Variable::new("foo".to_owned()) + }, + def: TypeDef { + fallible: true, + optional: false, + constraint: Exact(String), + }, + } + + ident_mismatch { + expr: |state: &mut state::Compiler| { + state.variable_types_mut().insert("foo".to_owned(), TypeDef { + fallible: true, + ..Default::default() + }); + + Variable::new("bar".to_owned()) + }, + def: TypeDef { + fallible: true, + ..Default::default() + }, + } + + empty_state { + expr: |_| Variable::new("foo".to_owned()), + def: TypeDef { + fallible: true, + ..Default::default() + }, + } + ]; } diff --git a/lib/remap-lang/src/function.rs b/lib/remap-lang/src/function.rs index a4c2be8ca7602..20f0ed22475e8 100644 --- a/lib/remap-lang/src/function.rs +++ b/lib/remap-lang/src/function.rs @@ -2,7 +2,7 @@ use crate::{Expression, Result, Value}; use core::convert::TryInto; use std::collections::HashMap; -#[derive(thiserror::Error, Debug, PartialEq)] +#[derive(thiserror::Error, Clone, Debug, PartialEq)] pub enum Error { #[error(r#"expected expression argument, got regex"#)] ArgumentExprRegex, @@ -88,6 +88,24 @@ pub enum Argument { Regex(regex::Regex), } +impl From> for Argument { + fn from(expr: Box) -> Self { + Argument::Expression(expr) + } +} + +impl From> for Argument { + fn from(expr: Box) -> Self { + Argument::Expression(expr) + } +} + +impl From for Argument { + fn from(regex: regex::Regex) -> Self { + Argument::Regex(regex) + } +} + impl TryInto> for Argument { type Error = Error; diff --git a/lib/remap-lang/src/lib.rs b/lib/remap-lang/src/lib.rs index 9dda57e659314..33a06646cd598 100644 --- a/lib/remap-lang/src/lib.rs +++ b/lib/remap-lang/src/lib.rs @@ -1,23 +1,24 @@ mod error; -mod expression; -mod function; mod operator; mod parser; mod program; mod runtime; -mod state; -mod value; - -use expression::Expr; -use operator::Operator; +mod test_util; +mod type_def; +pub mod expression; +pub mod function; pub mod prelude; +pub mod state; +pub mod value; + pub use error::{Error, RemapError}; -pub use expression::{Expression, Literal, Noop, Path}; -pub use function::{Argument, ArgumentList, Function, Parameter}; +pub use expression::{Expr, Expression}; +pub use function::{Function, Parameter}; +pub use operator::Operator; pub use program::Program; pub use runtime::Runtime; -pub use state::State; +pub use type_def::TypeDef; pub use value::Value; pub type Result = std::result::Result; @@ -129,6 +130,7 @@ fn vec_path_to_string(path: &[Vec]) -> String { #[cfg(test)] mod tests { use super::*; + use crate::function::ArgumentList; use std::collections::HashMap; #[derive(Debug, Clone)] @@ -154,9 +156,13 @@ mod tests { #[derive(Debug, Clone)] struct RegexPrinterFn(regex::Regex); impl Expression for RegexPrinterFn { - fn execute(&self, _: &mut State, _: &mut dyn Object) -> Result> { + fn execute(&self, _: &mut state::Program, _: &mut dyn Object) -> Result> { Ok(Some(format!("regex: {:?}", self.0).into())) } + + fn type_def(&self, _: &state::Compiler) -> TypeDef { + TypeDef::default() + } } #[test] @@ -234,8 +240,14 @@ mod tests { ]; for (script, expectation) in cases { - let program = Program::new(script, &[Box::new(RegexPrinter)]).unwrap(); - let mut runtime = Runtime::new(State::default()); + let accept = TypeDef { + fallible: true, + optional: true, + constraint: value::Constraint::Any, + }; + + let program = Program::new(script, &[Box::new(RegexPrinter)], accept).unwrap(); + let mut runtime = Runtime::new(state::Program::default()); let mut event = HashMap::default(); let result = runtime.execute(&mut event, &program).map_err(|e| e.0); diff --git a/lib/remap-lang/src/operator.rs b/lib/remap-lang/src/operator.rs index 7b97eef23ec59..f4c6ae3402edf 100644 --- a/lib/remap-lang/src/operator.rs +++ b/lib/remap-lang/src/operator.rs @@ -2,7 +2,7 @@ use std::convert::AsRef; use std::str::FromStr; #[derive(Debug, Clone)] -pub(crate) enum Operator { +pub enum Operator { Multiply, Divide, Remainder, diff --git a/lib/remap-lang/src/parser.rs b/lib/remap-lang/src/parser.rs index d946a4ca9be6d..9ec9f5cd6e7dc 100644 --- a/lib/remap-lang/src/parser.rs +++ b/lib/remap-lang/src/parser.rs @@ -5,7 +5,8 @@ use crate::{ Arithmetic, Assignment, Block, Function, IfStatement, Literal, Noop, Not, Path, Target, Variable, }, - Argument, Error, Expr, Function as Fn, Operator, Result, Value, + function::Argument, + state, Error, Expr, Function as Fn, Operator, Result, Value, }; use pest::iterators::{Pair, Pairs}; use regex::{Regex, RegexBuilder}; @@ -15,6 +16,7 @@ use std::str::FromStr; #[grammar = "../grammar.pest"] pub(super) struct Parser<'a> { pub function_definitions: &'a [Box], + pub compiler_state: state::Compiler, } type R = Rule; @@ -24,7 +26,7 @@ macro_rules! operation_fns { (@impl $($rule:tt => { op: [$head_op:path, $($tail_op:path),+ $(,)?], next: $next:tt, })+) => ( $( paste::paste! { - fn [<$rule _from_pairs>](&self, mut pairs: Pairs) -> Result { + fn [<$rule _from_pairs>](&mut self, mut pairs: Pairs) -> Result { let inner = pairs.next().ok_or(e(R::$rule))?.into_inner(); let mut lhs = self.[<$next _from_pairs>](inner)?; let mut op = Operator::$head_op; @@ -58,7 +60,7 @@ macro_rules! operation_fns { impl Parser<'_> { /// Converts the set of known "root" rules into boxed [`Expression`] trait /// objects. - pub(crate) fn pairs_to_expressions(&self, pairs: Pairs) -> Result> { + pub(crate) fn pairs_to_expressions(&mut self, pairs: Pairs) -> Result> { let mut expressions = vec![]; for pair in pairs { @@ -75,7 +77,7 @@ impl Parser<'_> { } /// Given a `Pair`, build a boxed [`Expression`] trait object from it. - fn expression_from_pair(&self, pair: Pair) -> Result { + fn expression_from_pair(&mut self, pair: Pair) -> Result { match pair.as_rule() { R::assignment => { let mut inner = pair.into_inner(); @@ -83,7 +85,11 @@ impl Parser<'_> { let expression = self.expression_from_pair(inner.next().ok_or(e(R::expression))?)?; - Ok(Expr::from(Assignment::new(target, Box::new(expression)))) + Ok(Expr::from(Assignment::new( + target, + Box::new(expression), + &mut self.compiler_state, + ))) } R::boolean_expr => self.boolean_expr_from_pairs(pair.into_inner()), R::block => self.block_from_pairs(pair.into_inner()), @@ -96,7 +102,7 @@ impl Parser<'_> { /// /// This can either return a `variable` or a `target_path` target, depending /// on the parser rule being processed. - fn target_from_pair(&self, pair: Pair) -> Result { + fn target_from_pair(&mut self, pair: Pair) -> Result { match pair.as_rule() { R::variable => Ok(Target::Variable( pair.into_inner() @@ -113,7 +119,7 @@ impl Parser<'_> { } /// Parse block expressions. - fn block_from_pairs(&self, pairs: Pairs) -> Result { + fn block_from_pairs(&mut self, pairs: Pairs) -> Result { let mut expressions = vec![]; for pair in pairs { @@ -124,7 +130,7 @@ impl Parser<'_> { } /// Parse if-statement expressions. - fn if_statement_from_pairs(&self, mut pairs: Pairs) -> Result { + fn if_statement_from_pairs(&mut self, mut pairs: Pairs) -> Result { // if condition let conditional = self.expression_from_pair(pairs.next().ok_or(e(R::if_statement))?)?; let true_expression = self.expression_from_pair(pairs.next().ok_or(e(R::if_statement))?)?; @@ -173,7 +179,7 @@ impl Parser<'_> { } /// Parse not operator, or fall-through to primary values or function calls. - fn not_from_pairs(&self, pairs: Pairs) -> Result { + fn not_from_pairs(&mut self, pairs: Pairs) -> Result { let mut count = 0; let mut expression = Expr::from(Noop); @@ -194,7 +200,7 @@ impl Parser<'_> { } /// Parse one of possible primary expressions. - fn primary_from_pair(&self, pair: Pair) -> Result { + fn primary_from_pair(&mut self, pair: Pair) -> Result { let pair = pair.into_inner().next().ok_or(e(R::primary))?; match pair.as_rule() { @@ -222,7 +228,7 @@ impl Parser<'_> { } /// Parse function call expressions. - fn call_from_pair(&self, pair: Pair) -> Result { + fn call_from_pair(&mut self, pair: Pair) -> Result { let mut inner = pair.into_inner(); let ident = inner.next().ok_or(e(R::call))?.as_str().to_owned(); @@ -236,14 +242,14 @@ impl Parser<'_> { } /// Parse into a vector of argument properties. - fn arguments_from_pair(&self, pair: Pair) -> Result, Argument)>> { + fn arguments_from_pair(&mut self, pair: Pair) -> Result, Argument)>> { pair.into_inner() .map(|pair| self.argument_from_pair(pair)) .collect::>() } /// Parse optional argument keyword and [`Argument`] value. - fn argument_from_pair(&self, pair: Pair) -> Result<(Option, Argument)> { + fn argument_from_pair(&mut self, pair: Pair) -> Result<(Option, Argument)> { let mut ident = None; for pair in pair.into_inner() { diff --git a/lib/remap-lang/src/prelude.rs b/lib/remap-lang/src/prelude.rs index 3a969be0f6013..0b445c1cc8223 100644 --- a/lib/remap-lang/src/prelude.rs +++ b/lib/remap-lang/src/prelude.rs @@ -1,4 +1,11 @@ -pub use crate::{ - Argument, ArgumentList, Error, Expression, Function, Literal, Noop, Object, Parameter, Path, - Result, State, Value, -}; +// commonly used modules +pub use crate::{expression, function, state, value}; + +// commonly used top-level crate types +pub use crate::{Error, Expression, Function, Object, Result, TypeDef, Value}; + +// commonly used expressions +pub use crate::expression::{Literal, Noop, Path, Variable}; + +// commonly used function types +pub use crate::function::{Argument, ArgumentList, Parameter}; diff --git a/lib/remap-lang/src/program.rs b/lib/remap-lang/src/program.rs index 81e2a84045b7a..687205efc1249 100644 --- a/lib/remap-lang/src/program.rs +++ b/lib/remap-lang/src/program.rs @@ -1,5 +1,74 @@ -use crate::{parser, Error, Expr, Function, RemapError}; +use crate::{parser, state, Error as E, Expr, Expression, Function, RemapError, TypeDef}; use pest::Parser; +use std::fmt; + +#[derive(thiserror::Error, Clone, Debug, PartialEq)] +pub enum Error { + #[error(transparent)] + ResolvesTo(#[from] ResolvesToError), + + #[error("expected to be infallible, but is not")] + Fallible, +} + +#[derive(thiserror::Error, Clone, Debug, PartialEq)] +pub struct ResolvesToError(TypeDef, TypeDef); + +impl fmt::Display for ResolvesToError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let want = &self.0; + let got = &self.1; + + let fallible_diff = want.is_fallible() != got.is_fallible(); + let optional_diff = want.is_optional() != got.is_optional(); + + let mut want_str = "".to_owned(); + let mut got_str = "".to_owned(); + + if fallible_diff { + if want.is_fallible() { + want_str.push_str("an error, or "); + } + + if got.is_fallible() { + got_str.push_str("an error, or "); + } + } + + want_str.push_str(&want.constraint.to_string()); + got_str.push_str(&got.constraint.to_string()); + + if optional_diff { + if want.is_optional() { + want_str.push_str(" or no"); + } + + if got.is_optional() { + got_str.push_str(" or no"); + } + } + + want_str.push_str(" value"); + got_str.push_str(" value"); + + let want_kinds = want.constraint.value_kinds(); + let got_kinds = got.constraint.value_kinds(); + + if !want.constraint.is_any() && want_kinds.len() > 1 { + want_str.push('s'); + } + + if !got.constraint.is_any() && got_kinds.len() > 1 { + got_str.push('s'); + } + + write!( + f, + "expected to resolve to {}, but instead resolves to {}", + want_str, got_str + ) + } +} /// The program to execute. /// @@ -16,16 +85,103 @@ impl Program { pub fn new( source: &str, function_definitions: &[Box], + expected_result: TypeDef, ) -> Result { let pairs = parser::Parser::parse(parser::Rule::program, source) - .map_err(|s| Error::Parser(s.to_string())) + .map_err(|s| E::Parser(s.to_string())) .map_err(RemapError)?; - let parser = parser::Parser { + let compiler_state = state::Compiler::default(); + + let mut parser = parser::Parser { function_definitions, + compiler_state, }; + let expressions = parser.pairs_to_expressions(pairs).map_err(RemapError)?; + let mut type_defs = expressions + .iter() + .map(|e| e.type_def(&parser.compiler_state)) + .collect::>(); + + let computed_result = type_defs.pop().unwrap_or(TypeDef { + optional: true, + fallible: true, + ..Default::default() + }); + + if !expected_result.contains(&computed_result) { + return Err(RemapError::from(E::from(Error::ResolvesTo( + ResolvesToError(expected_result, computed_result), + )))); + } + + if !expected_result.is_fallible() && type_defs.iter().any(TypeDef::is_fallible) { + return Err(RemapError::from(E::from(Error::Fallible))); + } + Ok(Self { expressions }) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::value; + use std::error::Error; + + #[test] + fn program_test() { + use value::Constraint::*; + use value::Kind::*; + + let cases = vec![ + (".foo", TypeDef { fallible: true, ..Default::default()}, Ok(())), + + // The final expression is infallible, but the first one isn't, so + // this isn't allowed. + ( + ".foo\ntrue", + TypeDef { fallible: false, ..Default::default()}, + Err("expected to be infallible, but is not".to_owned()), + ), + ( + ".foo", + TypeDef::default(), + Err("expected to resolve to any value, but instead resolves to an error, or any value".to_owned()), + ), + ( + ".foo", + TypeDef { + fallible: false, + optional: false, + constraint: Exact(String), + }, + Err("expected to resolve to string value, but instead resolves to an error, or any value".to_owned()), + ), + ( + "false || 2", + + TypeDef { + fallible: false, + optional: false, + constraint: OneOf(vec![String, Float]), + }, + Err("expected to resolve to string or float values, but instead resolves to an error, or integer or boolean values".to_owned()), + ), + ]; + + for (source, expected_result, expect) in cases { + let program = Program::new(source, &[], expected_result) + .map(|_| ()) + .map_err(|e| { + e.source() + .and_then(|e| e.source().map(|e| e.to_string())) + .unwrap() + }); + + assert_eq!(program, expect); + } + } +} diff --git a/lib/remap-lang/src/runtime.rs b/lib/remap-lang/src/runtime.rs index 70fa1c4438329..e21a9629763c9 100644 --- a/lib/remap-lang/src/runtime.rs +++ b/lib/remap-lang/src/runtime.rs @@ -1,12 +1,12 @@ -use crate::{Expression, Object, Program, RemapError, State, Value}; +use crate::{state, Expression, Object, Program, RemapError, Value}; #[derive(Debug, Default)] pub struct Runtime { - state: State, + state: state::Program, } impl Runtime { - pub fn new(state: State) -> Self { + pub fn new(state: state::Program) -> Self { Self { state } } diff --git a/lib/remap-lang/src/state.rs b/lib/remap-lang/src/state.rs index f5453239e9b2f..11d182bb1c7ca 100644 --- a/lib/remap-lang/src/state.rs +++ b/lib/remap-lang/src/state.rs @@ -1,12 +1,12 @@ -use crate::Value; +use crate::{TypeDef, Value}; use std::collections::HashMap; #[derive(Debug, Default)] -pub struct State { +pub struct Program { variables: HashMap, } -impl State { +impl Program { pub fn variable(&self, key: impl AsRef) -> Option<&Value> { self.variables.get(key.as_ref()) } @@ -15,3 +15,44 @@ impl State { &mut self.variables } } + +/// State held by the compiler as it parses the program source. +#[derive(Debug, Default)] +pub struct Compiler { + /// The [`Constraint`] each variable is expected to have. + /// + /// This allows assignment operations to tell the compiler what kinds each + /// variable will have at runtime, so that the compiler can then check the + /// variable kinds at compile-time when a variable is called. + variable_types: HashMap, + + /// The [`Constraint`] each path query is expected to have. + /// + /// By default, the first time a path is queried, it resolves to `Any`, but + /// when a path is used to assign a value to, we can potentially narrow down + /// the list of values the path will resolve to. + /// + /// FIXME: this won't work for coalesced paths. We're either going to + /// disallow those in assignments, which makes this easier to fix, or we're + /// going to always return `Any` for coalesced paths. Either way, this is a + /// known bug that we need to fix soon. + path_query_types: HashMap, +} + +impl Compiler { + pub fn variable_type(&self, key: impl AsRef) -> Option<&TypeDef> { + self.variable_types.get(key.as_ref()) + } + + pub fn variable_types_mut(&mut self) -> &mut HashMap { + &mut self.variable_types + } + + pub fn path_query_type(&self, key: impl AsRef) -> Option<&TypeDef> { + self.path_query_types.get(key.as_ref()) + } + + pub fn path_query_types_mut(&mut self) -> &mut HashMap { + &mut self.path_query_types + } +} diff --git a/lib/remap-lang/src/test_util.rs b/lib/remap-lang/src/test_util.rs new file mode 100644 index 0000000000000..a532d8d4a57de --- /dev/null +++ b/lib/remap-lang/src/test_util.rs @@ -0,0 +1,18 @@ +#[macro_export] +macro_rules! test_type_def { + ($($name:ident { expr: $expr:expr, def: $def:expr, })+) => { + mod type_def { + use super::*; + + $( + #[test] + fn $name() { + let mut state = state::Compiler::default(); + let expression: Box = Box::new($expr(&mut state)); + + assert_eq!(expression.type_def(&state), $def); + } + )+ + } + }; +} diff --git a/lib/remap-lang/src/type_def.rs b/lib/remap-lang/src/type_def.rs new file mode 100644 index 0000000000000..a3f8c0c3dc8b3 --- /dev/null +++ b/lib/remap-lang/src/type_def.rs @@ -0,0 +1,127 @@ +use crate::value; + +/// Properties for a given expression that express the expected outcome of the +/// expression. +/// +/// This includes whether the expression is fallible, whether it can return +/// "nothing", and a list of values the expression can resolve to. +#[derive(Debug, Clone, Eq, PartialEq, Default)] +pub struct TypeDef { + /// True, if an expression can return an error. + /// + /// Some expressions are infallible (e.g. the [`Literal`] expression, or any + /// custom function designed to be infallible). + pub fallible: bool, + + /// True, if an expression can resolve to "nothing". + /// + /// For example, and if-statement without an else-condition can resolve to + /// nothing if the if-condition does not match. + pub optional: bool, + + /// The [`value::Constraint`] applied to this type check. + /// + /// This resolves to a list of [`value::Kind`]s the expression is expected + /// to return. + pub constraint: value::Constraint, +} + +impl TypeDef { + pub fn is_fallible(&self) -> bool { + self.fallible + } + + pub fn into_fallible(mut self, fallible: bool) -> Self { + self.fallible = fallible; + self + } + + pub fn is_optional(&self) -> bool { + self.optional + } + + pub fn into_optional(mut self, optional: bool) -> Self { + self.optional = optional; + self + } + + /// Returns `true` if the _other_ [`TypeDef`] is contained within the + /// current one. + /// + /// That is to say, its constraints must be more strict or equal to the + /// constraints of the current one. + pub fn contains(&self, other: &Self) -> bool { + // If we don't expect none, but the other does, the other's requirement + // is less strict than ours. + if !self.is_optional() && other.is_optional() { + return false; + } + + // The same applies to fallible checks. + if !self.is_fallible() && other.is_fallible() { + return false; + } + + self.constraint.contains(&other.constraint) + } + + pub fn fallible_unless(mut self, constraint: impl Into) -> Self { + if !constraint.into().contains(&self.constraint) { + self.fallible = true + } + + self + } + + pub fn with_constraint(mut self, constraint: impl Into) -> Self { + self.constraint = constraint.into(); + self + } + + pub fn merge(self, other: Self) -> Self { + let TypeDef { + fallible, + optional, + constraint, + } = other; + + // TODO: take `self` + let constraint = self.constraint.merge(&constraint); + + Self { + fallible: self.is_fallible() || fallible, + optional: self.is_optional() || optional, + constraint, + } + } + + pub fn merge_optional(self, other: Option) -> Self { + match other { + Some(other) => self.merge(other), + None => self, + } + } + + /// Similar to `merge_optional`, except that the optional `TypeDef` is + /// considered to be the "default" for the `self` `TypeDef`. + /// + /// The implication of this is that the resulting `TypeDef` will be equal to + /// `self` or `other`, if either of the two is infallible and non-optional. + /// + /// If neither are, the two type definitions are merged as usual. + pub fn merge_with_default_optional(self, other: Option) -> Self { + if !self.is_fallible() && !self.is_optional() { + return self; + } + + match other { + None => self, + + // If `self` isn't exact, see if `other` is. + Some(other) if !other.is_fallible() && !other.is_optional() => other, + + // Otherwise merge the optional as usual. + Some(other) => self.merge(other), + } + } +} diff --git a/lib/remap-lang/src/value.rs b/lib/remap-lang/src/value.rs index abd8ec052fa2b..c6948bd535e5b 100644 --- a/lib/remap-lang/src/value.rs +++ b/lib/remap-lang/src/value.rs @@ -1,9 +1,15 @@ +mod constraint; +mod kind; + use bytes::Bytes; use chrono::{DateTime, Utc}; use std::collections::BTreeMap; use std::convert::{TryFrom, TryInto}; use std::string::String as StdString; +pub use constraint::Constraint; +pub use kind::Kind; + #[derive(Debug, Clone, PartialEq)] pub enum Value { String(Bytes), @@ -16,46 +22,46 @@ pub enum Value { Null, } -#[derive(thiserror::Error, Debug, PartialEq)] +#[derive(thiserror::Error, Clone, Debug, PartialEq)] pub enum Error { #[error(r#"expected "{0}", got "{1}""#)] - Expected(&'static str, &'static str), + Expected(Kind, Kind), #[error(r#"unable to coerce "{0}" into "{1}""#)] - Coerce(&'static str, &'static str), + Coerce(Kind, Kind), #[error("unable to calculate remainder of values type {0} and {1}")] - Rem(&'static str, &'static str), + Rem(Kind, Kind), #[error("unable to multiply value type {0} by {1}")] - Mul(&'static str, &'static str), + Mul(Kind, Kind), #[error("unable to divide value type {0} by {1}")] - Div(&'static str, &'static str), + Div(Kind, Kind), #[error("unable to add value type {1} to {0}")] - Add(&'static str, &'static str), + Add(Kind, Kind), #[error("unable to subtract value type {1} from {0}")] - Sub(&'static str, &'static str), + Sub(Kind, Kind), #[error("unable to OR value type {0} with {1}")] - Or(&'static str, &'static str), + Or(Kind, Kind), #[error("unable to AND value type {0} with {1}")] - And(&'static str, &'static str), + And(Kind, Kind), #[error("unable to compare {0} > {1}")] - Gt(&'static str, &'static str), + Gt(Kind, Kind), #[error("unable to compare {0} >= {1}")] - Ge(&'static str, &'static str), + Ge(Kind, Kind), #[error("unable to compare {0} < {1}")] - Lt(&'static str, &'static str), + Lt(Kind, Kind), #[error("unable to compare {0} <= {1}")] - Le(&'static str, &'static str), + Le(Kind, Kind), } impl From for Value { @@ -118,6 +124,12 @@ impl From<&str> for Value { } } +impl From<()> for Value { + fn from(_: ()) -> Self { + Value::Null + } +} + impl From> for Value { fn from(value: BTreeMap) -> Self { Value::Map(value) @@ -137,7 +149,7 @@ impl TryFrom<&Value> for f64 { match value { Value::Integer(v) => Ok(*v as f64), Value::Float(v) => Ok(*v), - _ => Err(Error::Coerce(value.kind(), Value::Float(0.0).kind())), + _ => Err(Error::Coerce(value.kind(), Kind::Float)), } } } @@ -149,7 +161,7 @@ impl TryFrom<&Value> for i64 { match value { Value::Integer(v) => Ok(*v), Value::Float(v) => Ok(*v as i64), - _ => Err(Error::Coerce(value.kind(), Value::Integer(0).kind())), + _ => Err(Error::Coerce(value.kind(), Kind::Integer)), } } } @@ -166,7 +178,7 @@ impl TryFrom<&Value> for String { Float(v) => Ok(format!("{}", v)), Boolean(v) => Ok(format!("{}", v)), Null => Ok("".to_owned()), - _ => Err(Error::Coerce(value.kind(), Value::String("".into()).kind())), + _ => Err(Error::Coerce(value.kind(), Kind::String)), } } } @@ -187,20 +199,72 @@ impl TryFrom for i64 { } } -impl Value { - pub fn kind(&self) -> &'static str { - use Value::*; - - match self { - String(_) => "string", - Integer(_) => "integer", - Float(_) => "float", - Boolean(_) => "boolean", - Map(_) => "map", - Array(_) => "array", - Timestamp(_) => "timestamp", - Null => "null", +macro_rules! value_impl { + ($(($func:expr, $variant:expr, $ret:ty)),+ $(,)*) => { + impl Value { + $(paste::paste! { + pub fn [](&self) -> Option<&$ret> { + match self { + Value::$variant(v) => Some(v), + _ => None, + } + } + + pub fn [](&mut self) -> Option<&mut $ret> { + match self { + Value::$variant(v) => Some(v), + _ => None, + } + } + + pub fn [](self) -> Result<$ret, Error> { + match self { + Value::$variant(v) => Ok(v), + v => Err(Error::Expected(Kind::$variant, v.kind())), + } + } + + pub fn [](self) -> $ret { + self.[]().expect(stringify!($func)) + } + })+ + + pub fn as_null(&self) -> Option<()> { + match self { + Value::Null => Some(()), + _ => None, + } + } + + pub fn try_null(self) -> Result<(), Error> { + match self { + Value::Null => Ok(()), + v => Err(Error::Expected(Kind::Null, v.kind())), + } + } + + pub fn unwrap_null(self) -> () { + self.try_null().expect("null") + } } + }; +} + +value_impl! { + (string, String, Bytes), + (integer, Integer, i64), + (float, Float, f64), + (boolean, Boolean, bool), + (map, Map, BTreeMap), + (array, Array, Vec), + (timestamp, Timestamp, DateTime), + // manually implemented due to no variant value + // (null, Null, ()), +} + +impl Value { + pub fn kind(&self) -> Kind { + self.into() } /// Similar to [`std::ops::Mul`], but fallible (e.g. `TryMul`). diff --git a/lib/remap-lang/src/value/constraint.rs b/lib/remap-lang/src/value/constraint.rs new file mode 100644 index 0000000000000..99156f526f1ef --- /dev/null +++ b/lib/remap-lang/src/value/constraint.rs @@ -0,0 +1,134 @@ +use crate::value; +use std::fmt; + +/// The constraint of a set of [`value::Kind`]s. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum Constraint { + /// Any value kind is accepted. + Any, + + /// Exactly one value kind is accepted + Exact(value::Kind), + + /// A subset of value kinds is accepted. + OneOf(Vec), +} + +impl Default for Constraint { + fn default() -> Self { + Constraint::Any + } +} + +impl> From for Constraint { + fn from(kind: T) -> Self { + Constraint::Exact(kind.into()) + } +} + +impl From> for Constraint { + fn from(kinds: Vec) -> Self { + debug_assert!(kinds.len() > 1); + + Constraint::OneOf(kinds) + } +} + +impl Constraint { + /// Returns `true` if this is a [`Constraint::Exact`]. + pub fn is_exact(&self) -> bool { + matches!(self, Self::Exact(_)) + } + + /// Returns `true` if this is a [`Constraint::Any`]. + pub fn is_any(&self) -> bool { + matches!(self, Self::Any) + } + + /// Returns `true` if this constraint exactly matches `other`. + pub fn is(&self, other: impl Into) -> bool { + self == &other.into() + } + + /// Get a collection of [`value::Kind`]s accepted by this [`Constraint`]. + pub fn value_kinds(&self) -> Vec { + use Constraint::*; + + match self { + Any => value::Kind::all(), + OneOf(v) => v.clone(), + Exact(v) => vec![*v], + } + } + + /// Merge two [`Constraint`]s, such that the new `Constraint` provides the + /// most constraint possible value constraint. + pub fn merge(&self, other: &Self) -> Self { + use Constraint::*; + + if self.is_any() || other.is_any() { + return Any; + } + + let mut kinds: Vec<_> = self + .value_kinds() + .into_iter() + .chain(other.value_kinds().into_iter()) + .collect(); + + kinds.sort(); + kinds.dedup(); + + if kinds.len() == 1 { + Exact(kinds[0]) + } else { + OneOf(kinds) + } + } + + /// Returns `true` if the _other_ [`Constraint`] is contained within the + /// current one. + /// + /// That is to say, its constraints must be more strict or equal to the + /// constraints of the current one. + pub fn contains(&self, other: &Self) -> bool { + let self_kinds = self.value_kinds(); + let other_kinds = other.value_kinds(); + + for kind in other_kinds { + if !self_kinds.contains(&kind) { + return false; + } + } + + true + } +} + +impl fmt::Display for Constraint { + /// Print a human readable version of the value constraint. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use Constraint::*; + + match self { + Any => f.write_str("any"), + OneOf(v) => { + let mut kinds = v.iter().map(|v| v.as_str()).collect::>(); + + let last = kinds.pop(); + let mut string = kinds.join(", "); + + if let Some(last) = last { + if !string.is_empty() { + string.push_str(" or ") + } + + string.push_str(last); + } + + f.write_str(&string) + } + Exact(v) => f.write_str(&v), + } + } +} diff --git a/lib/remap-lang/src/value/kind.rs b/lib/remap-lang/src/value/kind.rs new file mode 100644 index 0000000000000..a1011d24ccf34 --- /dev/null +++ b/lib/remap-lang/src/value/kind.rs @@ -0,0 +1,69 @@ +use super::Value; +use std::fmt; +use std::ops::Deref; + +#[derive(Eq, PartialEq, Hash, Debug, Clone, Copy, Ord, PartialOrd)] +pub enum Kind { + String, + Integer, + Float, + Boolean, + Map, + Array, + Timestamp, + Null, +} + +impl fmt::Display for Kind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self) + } +} + +impl Kind { + pub fn all() -> Vec { + use Kind::*; + + vec![String, Integer, Float, Boolean, Map, Array, Timestamp, Null] + } + + pub fn as_str(&self) -> &'static str { + use Kind::*; + + match self { + String => "string", + Integer => "integer", + Float => "float", + Boolean => "boolean", + Map => "map", + Array => "array", + Timestamp => "timestamp", + Null => "null", + } + } +} + +impl Deref for Kind { + type Target = str; + + fn deref(&self) -> &Self::Target { + self.as_str() + } +} + +impl From<&Value> for Kind { + fn from(value: &Value) -> Self { + use Kind::*; + + match value { + Value::String(_) => String, + Value::Integer(_) => Integer, + Value::Float(_) => Float, + Value::Boolean(_) => Boolean, + Value::Map(_) => Map, + Value::Array(_) => Array, + Value::Timestamp(_) => Timestamp, + Value::Null => Null, + } + } +} diff --git a/src/conditions/remap.rs b/src/conditions/remap.rs index f727dd2de7307..217f2bb20510e 100644 --- a/src/conditions/remap.rs +++ b/src/conditions/remap.rs @@ -1,9 +1,10 @@ use crate::{ conditions::{Condition, ConditionConfig, ConditionDescription}, emit, - internal_events::{RemapConditionExecutionFailed, RemapConditionNonBooleanReturned}, + internal_events::RemapConditionExecutionFailed, Event, }; +use remap::{value, Program, RemapError, Runtime, TypeDef, Value}; use serde::{Deserialize, Serialize}; #[derive(Deserialize, Serialize, Debug, Default, Clone)] @@ -20,7 +21,14 @@ impl_generate_config_from_default!(RemapConfig); #[typetag::serde(name = "remap")] impl ConditionConfig for RemapConfig { fn build(&self) -> crate::Result> { - let program = remap::Program::new(&self.source, &crate::remap::FUNCTIONS)?; + let expected_result = TypeDef { + fallible: true, + optional: true, + constraint: value::Constraint::Exact(value::Kind::Boolean), + }; + + let program = Program::new(&self.source, &crate::remap::FUNCTIONS, expected_result) + .map_err(|e| e.to_string())?; Ok(Box::new(Remap { program })) } @@ -30,11 +38,11 @@ impl ConditionConfig for RemapConfig { #[derive(Clone)] pub struct Remap { - program: remap::Program, + program: Program, } impl Remap { - fn execute(&self, event: &Event) -> Result, remap::RemapError> { + fn execute(&self, event: &Event) -> Result, RemapError> { // TODO(jean): This clone exists until remap-lang has an "immutable" // mode. // @@ -47,39 +55,38 @@ impl Remap { // program wants to mutate its events. // // see: https://github.com/timberio/vector/issues/4744 - remap::Runtime::default().execute(&mut event.clone(), &self.program) + Runtime::default().execute(&mut event.clone(), &self.program) } } impl Condition for Remap { fn check(&self, event: &Event) -> bool { self.execute(&event) - .unwrap_or_else(|_| { - emit!(RemapConditionExecutionFailed); - None + .map(|opt| match opt { + Some(value) => value, + None => Value::Boolean(false), }) .map(|value| match value { - remap::Value::Boolean(boolean) => boolean, - _ => { - emit!(RemapConditionNonBooleanReturned); - false - } + Value::Boolean(boolean) => boolean, + _ => unreachable!("boolean type constraint set"), }) - .unwrap_or_else(|| { - emit!(RemapConditionNonBooleanReturned); + .unwrap_or_else(|_| { + emit!(RemapConditionExecutionFailed); false }) } fn check_with_context(&self, event: &Event) -> Result<(), String> { - self.execute(event) + let result = self + .execute(event) .map_err(|err| format!("source execution failed: {:#}", err))? - .ok_or_else(|| "source execution resolved to no value".into()) - .and_then(|value| match value { - remap::Value::Boolean(v) if v => Ok(()), - remap::Value::Boolean(v) if !v => Err("source execution resolved to false".into()), - _ => Err("source execution resolved to non-boolean value".into()), - }) + .unwrap_or_else(|| Value::Boolean(false)); + + match result { + Value::Boolean(v) if v => Ok(()), + Value::Boolean(v) if !v => Err("source execution resolved to false".into()), + _ => unreachable!("boolean type constraint set"), + } } } @@ -104,7 +111,7 @@ mod test { ), ( log_event!["foo" => true, "bar" => false], - ".bar || .foo", + "to_bool(.bar || .foo)", Ok(()), Ok(()), ), @@ -117,14 +124,14 @@ mod test { ( log_event![], "", + Err("remap error: program error: expected to resolve to boolean value, but instead resolves to any value"), Ok(()), - Err("source execution resolved to no value"), ), ( log_event!["foo" => "string"], ".foo", + Err("remap error: program error: expected to resolve to boolean or no value, but instead resolves to any value"), Ok(()), - Err("source execution resolved to non-boolean value"), ), ( log_event![], diff --git a/src/internal_events/remap.rs b/src/internal_events/remap.rs index df3bfd55fb05d..e4e04f8b7462a 100644 --- a/src/internal_events/remap.rs +++ b/src/internal_events/remap.rs @@ -49,15 +49,3 @@ impl InternalEvent for RemapConditionExecutionFailed { ) } } - -#[derive(Debug, Copy, Clone)] -pub struct RemapConditionNonBooleanReturned; - -impl InternalEvent for RemapConditionNonBooleanReturned { - fn emit_logs(&self) { - warn!( - message = "Remap condition non-boolean value returned.", - rate_limit_secs = 120 - ) - } -} diff --git a/src/remap/function.rs b/src/remap/function.rs index 8299c4e2b5667..284fcb028d3bf 100644 --- a/src/remap/function.rs +++ b/src/remap/function.rs @@ -122,8 +122,8 @@ fn is_scalar_value(value: &Value) -> bool { use Value::*; match value { - Integer(_) | Float(_) | String(_) | Boolean(_) => true, - Timestamp(_) | Map(_) | Array(_) | Null => false, + Integer(_) | Float(_) | String(_) | Boolean(_) | Null => true, + Timestamp(_) | Map(_) | Array(_) => false, } } diff --git a/src/remap/function/ceil.rs b/src/remap/function/ceil.rs index 26e40a7b3951b..b6d09bce564e4 100644 --- a/src/remap/function/ceil.rs +++ b/src/remap/function/ceil.rs @@ -46,7 +46,11 @@ impl CeilFn { } impl Expression for CeilFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let precision = optional!(state, object, self.precision, Value::Integer(v) => v).unwrap_or(0); let res = required!(state, object, self.value, @@ -58,12 +62,68 @@ impl Expression for CeilFn { Ok(res.into()) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + use value::Kind::*; + + let value_def = self + .value + .type_def(state) + .fallible_unless(vec![Integer, Float]); + let precision_def = self + .precision + .as_ref() + .map(|precision| precision.type_def(state).fallible_unless(Integer)); + + value_def + .clone() + .merge_optional(precision_def) + .with_constraint(match value_def.constraint { + v if v.is(Float) || v.is(Integer) => v, + _ => vec![Integer, Float].into(), + }) + } } #[cfg(test)] mod tests { use super::*; use crate::map; + use value::Kind::*; + + remap::test_type_def![ + value_float { + expr: |_| CeilFn { + value: Literal::from(1.0).boxed(), + precision: None, + }, + def: TypeDef { constraint: Float.into(), ..Default::default() }, + } + + value_integer { + expr: |_| CeilFn { + value: Literal::from(1).boxed(), + precision: None, + }, + def: TypeDef { constraint: Integer.into(), ..Default::default() }, + } + + value_float_or_integer { + expr: |_| CeilFn { + value: Variable::new("foo".to_owned()).boxed(), + precision: None, + }, + def: TypeDef { fallible: true, constraint: vec![Integer, Float].into(), ..Default::default() }, + } + + fallible_precision { + expr: |_| CeilFn { + value: Literal::from(1).boxed(), + precision: Some(Variable::new("foo".to_owned()).boxed()), + }, + def: TypeDef { fallible: true, constraint: Integer.into(), ..Default::default() }, + } + ]; #[test] fn ceil() { @@ -118,7 +178,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/contains.rs b/src/remap/function/contains.rs index e18c46f48d00e..a1cf7ccdbd39b 100644 --- a/src/remap/function/contains.rs +++ b/src/remap/function/contains.rs @@ -63,7 +63,11 @@ impl ContainsFn { } impl Expression for ContainsFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let substring = { let bytes = required!(state, object, self.substring, Value::String(v) => v); String::from_utf8_lossy(&bytes).into_owned() @@ -82,6 +86,23 @@ impl Expression for ContainsFn { Ok(Some(contains.into())) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + self.value + .type_def(state) + .fallible_unless(value::Kind::String) + .merge( + self.substring + .type_def(state) + .fallible_unless(value::Kind::String), + ) + .merge_optional(self.case_sensitive.as_ref().map(|case_sensitive| { + case_sensitive + .type_def(state) + .fallible_unless(value::Kind::Boolean) + })) + .with_constraint(value::Kind::Boolean) + } } #[cfg(test)] @@ -144,7 +165,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/del.rs b/src/remap/function/del.rs index 05481fe22baa8..40022beef0f0c 100644 --- a/src/remap/function/del.rs +++ b/src/remap/function/del.rs @@ -52,7 +52,11 @@ pub struct DelFn { } impl Expression for DelFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let paths = self .paths .iter() @@ -66,4 +70,35 @@ impl Expression for DelFn { Ok(None) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + self.paths + .iter() + .fold(TypeDef::default(), |acc, expression| { + acc.merge( + expression + .type_def(state) + .fallible_unless(value::Kind::String), + ) + }) + .with_constraint(value::Constraint::Any) + .into_optional(true) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + remap::test_type_def![ + value_string { + expr: |_| DelFn { paths: vec![Literal::from("foo").boxed()] }, + def: TypeDef { optional: true, constraint: value::Constraint::Any, ..Default::default() }, + } + + fallible_expression { + expr: |_| DelFn { paths: vec![Variable::new("foo".to_owned()).boxed(), Literal::from("foo").boxed()] }, + def: TypeDef { fallible: true, optional: true, constraint: value::Constraint::Any }, + } + ]; } diff --git a/src/remap/function/downcase.rs b/src/remap/function/downcase.rs index 883e04a130aac..26648b5cccd99 100644 --- a/src/remap/function/downcase.rs +++ b/src/remap/function/downcase.rs @@ -37,7 +37,11 @@ impl DowncaseFn { } impl Expression for DowncaseFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { self.value .execute(state, object)? .map(String::try_from) @@ -47,6 +51,13 @@ impl Expression for DowncaseFn { .map(Ok) .transpose() } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + self.value + .type_def(state) + .fallible_unless(value::Kind::String) + .with_constraint(value::Kind::String) + } } #[cfg(test)] @@ -69,7 +80,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func @@ -79,4 +90,16 @@ mod tests { assert_eq!(got, exp); } } + + remap::test_type_def![ + string { + expr: |_| DowncaseFn { value: Literal::from("foo").boxed() }, + def: TypeDef { constraint: value::Kind::String.into(), ..Default::default() }, + } + + non_string { + expr: |_| DowncaseFn { value: Literal::from(true).boxed() }, + def: TypeDef { fallible: true, constraint: value::Kind::String.into(), ..Default::default() }, + } + ]; } diff --git a/src/remap/function/ends_with.rs b/src/remap/function/ends_with.rs index b83676ddee223..e2617e05a5021 100644 --- a/src/remap/function/ends_with.rs +++ b/src/remap/function/ends_with.rs @@ -63,7 +63,11 @@ impl EndsWithFn { } impl Expression for EndsWithFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let substring = { let bytes = required!(state, object, self.substring, Value::String(v) => v); String::from_utf8_lossy(&bytes).into_owned() @@ -74,13 +78,32 @@ impl Expression for EndsWithFn { String::from_utf8_lossy(&bytes).into_owned() }; - let starts_with = value.ends_with(&substring) + let ends_with = value.ends_with(&substring) || optional!(state, object, self.case_sensitive, Value::Boolean(b) => b) .iter() .filter(|&case_sensitive| !case_sensitive) .any(|_| value.to_lowercase().ends_with(&substring.to_lowercase())); - Ok(Some(starts_with.into())) + Ok(Some(ends_with.into())) + } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + let substring_def = self + .substring + .type_def(state) + .fallible_unless(value::Kind::String); + + let case_sensitive_def = self + .case_sensitive + .as_ref() + .map(|cs| cs.type_def(state).fallible_unless(value::Kind::Boolean)); + + self.value + .type_def(state) + .fallible_unless(value::Kind::String) + .merge(substring_def) + .merge_optional(case_sensitive_def) + .with_constraint(value::Kind::Boolean) } } @@ -88,6 +111,45 @@ impl Expression for EndsWithFn { mod tests { use super::*; use crate::map; + use value::Kind::*; + + remap::test_type_def![ + value_string { + expr: |_| EndsWithFn { + value: Literal::from("foo").boxed(), + substring: Literal::from("foo").boxed(), + case_sensitive: None, + }, + def: TypeDef { constraint: Boolean.into(), ..Default::default() }, + } + + value_non_string { + expr: |_| EndsWithFn { + value: Literal::from(true).boxed(), + substring: Literal::from("foo").boxed(), + case_sensitive: None, + }, + def: TypeDef { fallible: true, constraint: Boolean.into(), ..Default::default() }, + } + + substring_non_string { + expr: |_| EndsWithFn { + value: Literal::from("foo").boxed(), + substring: Literal::from(true).boxed(), + case_sensitive: None, + }, + def: TypeDef { fallible: true, constraint: Boolean.into(), ..Default::default() }, + } + + case_sensitive_non_boolean { + expr: |_| EndsWithFn { + value: Literal::from("foo").boxed(), + substring: Literal::from("foo").boxed(), + case_sensitive: Some(Literal::from(1).boxed()), + }, + def: TypeDef { fallible: true, constraint: Boolean.into(), ..Default::default() }, + } + ]; #[test] fn ends_with() { @@ -144,7 +206,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/floor.rs b/src/remap/function/floor.rs index 41ee55bce825b..79754daf8b527 100644 --- a/src/remap/function/floor.rs +++ b/src/remap/function/floor.rs @@ -46,7 +46,11 @@ impl FloorFn { } impl Expression for FloorFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let precision = optional!(state, object, self.precision, Value::Integer(v) => v).unwrap_or(0); let res = required!(state, object, self.value, @@ -58,12 +62,68 @@ impl Expression for FloorFn { Ok(res.into()) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + use value::Kind::*; + + let value_def = self + .value + .type_def(state) + .fallible_unless(vec![Integer, Float]); + let precision_def = self + .precision + .as_ref() + .map(|precision| precision.type_def(state).fallible_unless(Integer)); + + value_def + .clone() + .merge_optional(precision_def) + .with_constraint(match value_def.constraint { + v if v.is(Float) || v.is(Integer) => v, + _ => vec![Integer, Float].into(), + }) + } } #[cfg(test)] mod tests { use super::*; use crate::map; + use value::Kind::*; + + remap::test_type_def![ + value_float { + expr: |_| FloorFn { + value: Literal::from(1.0).boxed(), + precision: None, + }, + def: TypeDef { constraint: Float.into(), ..Default::default() }, + } + + value_integer { + expr: |_| FloorFn { + value: Literal::from(1).boxed(), + precision: None, + }, + def: TypeDef { constraint: Integer.into(), ..Default::default() }, + } + + value_float_or_integer { + expr: |_| FloorFn { + value: Variable::new("foo".to_owned()).boxed(), + precision: None, + }, + def: TypeDef { fallible: true, constraint: vec![Integer, Float].into(), ..Default::default() }, + } + + fallible_precision { + expr: |_| FloorFn { + value: Literal::from(1).boxed(), + precision: Some(Variable::new("foo".to_owned()).boxed()), + }, + def: TypeDef { fallible: true, constraint: Integer.into(), ..Default::default() }, + } + ]; #[test] fn floor() { @@ -118,7 +178,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/format_number.rs b/src/remap/function/format_number.rs index 5bb69ec696768..cf75d13348bf9 100644 --- a/src/remap/function/format_number.rs +++ b/src/remap/function/format_number.rs @@ -80,7 +80,11 @@ impl FormatNumberFn { } impl Expression for FormatNumberFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let value = required!(state, object, self.value, Value::Integer(v) => Decimal::from_i64(v), Value::Float(v) => Decimal::from_f64(v), @@ -149,12 +153,75 @@ impl Expression for FormatNumberFn { .into(), )) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + use value::Kind::*; + + let scale_def = self + .scale + .as_ref() + .map(|scale| scale.type_def(state).fallible_unless(Integer)); + + let decimal_separator_def = self + .decimal_separator + .as_ref() + .map(|decimal_separator| decimal_separator.type_def(state).fallible_unless(String)); + + let grouping_separator_def = self + .grouping_separator + .as_ref() + .map(|grouping_separator| grouping_separator.type_def(state).fallible_unless(String)); + + self.value + .type_def(state) + .fallible_unless(vec![Integer, Float]) + .merge_optional(scale_def) + .merge_optional(decimal_separator_def) + .merge_optional(grouping_separator_def) + .into_fallible(true) // `Decimal::from` can theoretically fail. + .with_constraint(String) + } } #[cfg(test)] mod tests { use super::*; use crate::map; + use value::Kind::*; + + remap::test_type_def![ + value_integer { + expr: |_| FormatNumberFn { + value: Literal::from(1).boxed(), + scale: None, + decimal_separator: None, + grouping_separator: None, + }, + def: TypeDef { fallible: true, constraint: String.into(), ..Default::default() }, + } + + value_float { + expr: |_| FormatNumberFn { + value: Literal::from(1.0).boxed(), + scale: None, + decimal_separator: None, + grouping_separator: None, + }, + def: TypeDef { fallible: true, constraint: String.into(), ..Default::default() }, + } + + // TODO(jean): we should update the function to ignore `None` values, + // instead of aborting. + optional_scale { + expr: |_| FormatNumberFn { + value: Literal::from(1.0).boxed(), + scale: Some(Box::new(Noop)), + decimal_separator: None, + grouping_separator: None, + }, + def: TypeDef { fallible: true, optional: true, constraint: String.into() }, + } + ]; #[test] fn format_number() { @@ -221,7 +288,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/format_timestamp.rs b/src/remap/function/format_timestamp.rs index 1720a640d3ca6..f2f74d4b9116a 100644 --- a/src/remap/function/format_timestamp.rs +++ b/src/remap/function/format_timestamp.rs @@ -49,12 +49,30 @@ impl FormatTimestampFn { } impl Expression for FormatTimestampFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let format = required!(state, object, self.format, Value::String(b) => String::from_utf8_lossy(&b).into_owned()); let ts = required!(state, object, self.value, Value::Timestamp(ts) => ts); try_format(&ts, &format).map(Into::into).map(Some) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + let format_def = self + .format + .type_def(state) + .fallible_unless(value::Kind::String); + + self.value + .type_def(state) + .fallible_unless(value::Kind::Timestamp) + .merge(format_def) + .into_fallible(true) // due to `try_format` + .with_constraint(value::Kind::String) + } } fn try_format(dt: &DateTime, format: &str) -> Result { @@ -73,6 +91,25 @@ mod tests { use super::*; use crate::map; use chrono::TimeZone; + use value::Kind::*; + + remap::test_type_def![ + value_and_format { + expr: |_| FormatTimestampFn { + value: Literal::from(chrono::Utc::now()).boxed(), + format: Literal::from("%s").boxed(), + }, + def: TypeDef { fallible: true, constraint: String.into(), ..Default::default() }, + } + + optional_value { + expr: |_| FormatTimestampFn { + value: Box::new(Noop), + format: Literal::from("%s").boxed(), + }, + def: TypeDef { fallible: true, optional: true, constraint: String.into() }, + } + ]; #[test] fn format_timestamp() { @@ -108,7 +145,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/match.rs b/src/remap/function/match.rs index 75b54cce803ee..7e803365dfaef 100644 --- a/src/remap/function/match.rs +++ b/src/remap/function/match.rs @@ -46,7 +46,11 @@ impl MatchFn { } impl Expression for MatchFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { required!( state, object, self.value, @@ -56,12 +60,46 @@ impl Expression for MatchFn { } ) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + self.value + .type_def(state) + .fallible_unless(value::Kind::String) + .with_constraint(value::Kind::Boolean) + } } #[cfg(test)] mod tests { use super::*; use crate::map; + use value::Kind::*; + + remap::test_type_def![ + value_string { + expr: |_| MatchFn { + value: Literal::from("foo").boxed(), + pattern: Regex::new("").unwrap(), + }, + def: TypeDef { constraint: Boolean.into(), ..Default::default() }, + } + + value_non_string { + expr: |_| MatchFn { + value: Literal::from(1).boxed(), + pattern: Regex::new("").unwrap(), + }, + def: TypeDef { fallible: true, constraint: Boolean.into(), ..Default::default() }, + } + + value_optional { + expr: |_| MatchFn { + value: Box::new(Noop), + pattern: Regex::new("").unwrap(), + }, + def: TypeDef { fallible: true, optional: true, constraint: Boolean.into() }, + } + ]; #[test] fn r#match() { @@ -92,7 +130,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/md5.rs b/src/remap/function/md5.rs index fb874ad03c4d7..fdeb29b20f8f3 100644 --- a/src/remap/function/md5.rs +++ b/src/remap/function/md5.rs @@ -36,7 +36,11 @@ impl Md5Fn { } impl Expression for Md5Fn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { use md5::{Digest, Md5}; self.value.execute(state, object).map(|r| { @@ -46,12 +50,37 @@ impl Expression for Md5Fn { }) }) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + self.value + .type_def(state) + .fallible_unless(value::Kind::String) + .with_constraint(value::Kind::String) + } } #[cfg(test)] mod tests { use super::*; use crate::map; + use value::Kind::*; + + remap::test_type_def![ + value_string { + expr: |_| Md5Fn { value: Literal::from("foo").boxed() }, + def: TypeDef { constraint: String.into(), ..Default::default() }, + } + + value_non_string { + expr: |_| Md5Fn { value: Literal::from(1).boxed() }, + def: TypeDef { fallible: true, constraint: String.into(), ..Default::default() }, + } + + value_optional { + expr: |_| Md5Fn { value: Box::new(Noop) }, + def: TypeDef { fallible: true, optional: true, constraint: String.into() }, + } + ]; #[test] fn md5() { @@ -68,7 +97,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/now.rs b/src/remap/function/now.rs index 6859e037e41c2..1a8845e8f98ec 100644 --- a/src/remap/function/now.rs +++ b/src/remap/function/now.rs @@ -18,7 +18,27 @@ impl Function for Now { struct NowFn; impl Expression for NowFn { - fn execute(&self, _: &mut State, _: &mut dyn Object) -> Result> { + fn execute(&self, _: &mut state::Program, _: &mut dyn Object) -> Result> { Ok(Some(Utc::now().into())) } + + fn type_def(&self, _: &state::Compiler) -> TypeDef { + TypeDef { + constraint: value::Kind::Timestamp.into(), + ..Default::default() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + remap::test_type_def![static_def { + expr: |_| NowFn, + def: TypeDef { + constraint: value::Kind::Timestamp.into(), + ..Default::default() + }, + }]; } diff --git a/src/remap/function/only_fields.rs b/src/remap/function/only_fields.rs index 360a794f67d8d..eefa5a57e8927 100644 --- a/src/remap/function/only_fields.rs +++ b/src/remap/function/only_fields.rs @@ -52,7 +52,11 @@ pub struct OnlyFieldsFn { } impl Expression for OnlyFieldsFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let paths = self .paths .iter() @@ -68,4 +72,35 @@ impl Expression for OnlyFieldsFn { Ok(None) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + self.paths + .iter() + .fold(TypeDef::default(), |acc, expression| { + acc.merge( + expression + .type_def(state) + .fallible_unless(value::Kind::String), + ) + }) + .with_constraint(value::Constraint::Any) + .into_optional(true) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + remap::test_type_def![ + value_string { + expr: |_| OnlyFieldsFn { paths: vec![Literal::from("foo").boxed()] }, + def: TypeDef { optional: true, constraint: value::Constraint::Any, ..Default::default() }, + } + + fallible_expression { + expr: |_| OnlyFieldsFn { paths: vec![Variable::new("foo".to_owned()).boxed(), Literal::from("foo").boxed()] }, + def: TypeDef { fallible: true, optional: true, constraint: value::Constraint::Any }, + } + ]; } diff --git a/src/remap/function/parse_duration.rs b/src/remap/function/parse_duration.rs index 2711cb3a92816..60f022b6bdd48 100644 --- a/src/remap/function/parse_duration.rs +++ b/src/remap/function/parse_duration.rs @@ -79,7 +79,11 @@ impl ParseDurationFn { } impl Expression for ParseDurationFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let value = { let bytes = required!(state, object, self.value, Value::String(v) => v); String::from_utf8_lossy(&bytes).into_owned() @@ -112,6 +116,20 @@ impl Expression for ParseDurationFn { Ok(Some(number.into())) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + let output_def = self + .output + .type_def(state) + .fallible_unless(value::Kind::String); + + self.value + .type_def(state) + .fallible_unless(value::Kind::String) + .merge(output_def) + .into_fallible(true) // parsing errors + .with_constraint(value::Kind::Float) + } } #[cfg(test)] @@ -119,6 +137,24 @@ mod tests { use super::*; use crate::map; + remap::test_type_def![ + value_string { + expr: |_| ParseDurationFn { + value: Literal::from("foo").boxed(), + output: Literal::from("foo").boxed(), + }, + def: TypeDef { fallible: true, constraint: value::Kind::Float.into(), ..Default::default() }, + } + + optional_expression { + expr: |_| ParseDurationFn { + value: Box::new(Noop), + output: Literal::from("foo").boxed(), + }, + def: TypeDef { fallible: true, optional: true, constraint: value::Kind::Float.into() }, + } + ]; + #[test] fn parse_duration() { let cases = vec![ @@ -184,7 +220,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/parse_json.rs b/src/remap/function/parse_json.rs index 1a6430fde9bd6..9dd0d181b9997 100644 --- a/src/remap/function/parse_json.rs +++ b/src/remap/function/parse_json.rs @@ -47,7 +47,11 @@ impl ParseJsonFn { } impl Expression for ParseJsonFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let to_json = |value| match value { Value::String(bytes) => serde_json::from_slice(&bytes) .map(|v: serde_json::Value| { @@ -64,12 +68,79 @@ impl Expression for ParseJsonFn { to_json, ) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + use value::Kind::*; + + let default_def = self + .default + .as_ref() + .map(|default| default.type_def(state).fallible_unless(String)); + + self.value + .type_def(state) + .fallible_unless(String) + .merge_with_default_optional(default_def) + .into_fallible(true) // JSON parsing errors + .with_constraint(vec![String, Boolean, Integer, Float, Array, Map, Null]) + } } #[cfg(test)] mod tests { use super::*; use crate::map; + use value::Kind::*; + + remap::test_type_def![ + value_string { + expr: |_| ParseJsonFn { + value: Literal::from("foo").boxed(), + default: None, + }, + def: TypeDef { + fallible: true, + constraint: vec![String, Boolean, Integer, Float, Array, Map, Null].into(), + ..Default::default() + }, + } + + optional_default { + expr: |_| ParseJsonFn { + value: Literal::from("foo").boxed(), + default: Some(Box::new(Noop)), + }, + def: TypeDef { + fallible: true, + constraint: vec![String, Boolean, Integer, Float, Array, Map, Null].into(), + ..Default::default() + }, + } + + optional_value { + expr: |_| ParseJsonFn { + value: Box::new(Noop), + default: Some(Literal::from("foo").boxed()), + }, + def: TypeDef { + fallible: true, + constraint: vec![String, Boolean, Integer, Float, Array, Map, Null].into(), + ..Default::default() + }, + } + + optional_value_and_default { + expr: |_| ParseJsonFn { + value: Box::new(Noop), + default: Some(Box::new(Noop)), + }, + def: TypeDef { + fallible: true, + optional: true, + constraint: vec![String, Boolean, Integer, Float, Array, Map, Null].into(), + }, + } + ]; #[test] fn parse_json() { @@ -106,7 +177,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/parse_syslog.rs b/src/remap/function/parse_syslog.rs index b039f1ba28e14..78c97a7a09827 100644 --- a/src/remap/function/parse_syslog.rs +++ b/src/remap/function/parse_syslog.rs @@ -100,13 +100,24 @@ fn message_to_value(message: Message<&str>) -> Value { } impl Expression for ParseSyslogFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let message = required!(state, object, self.value, Value::String(v) => String::from_utf8_lossy(&v).into_owned()); let parsed = syslog_loose::parse_message_with_year(&message, resolve_year); Ok(Some(message_to_value(parsed))) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + self.value + .type_def(state) + .fallible_unless(value::Kind::String) + .with_constraint(value::Kind::Map) + } } #[cfg(test)] @@ -115,6 +126,23 @@ mod tests { use crate::map; use chrono::prelude::*; + remap::test_type_def![ + value_string { + expr: |_| ParseSyslogFn { value: Literal::from("foo").boxed() }, + def: TypeDef { constraint: value::Kind::Map.into(), ..Default::default() }, + } + + value_non_string { + expr: |_| ParseSyslogFn { value: Literal::from(1).boxed() }, + def: TypeDef { fallible: true, constraint: value::Kind::Map.into(), ..Default::default() }, + } + + value_optional { + expr: |_| ParseSyslogFn { value: Box::new(Noop) }, + def: TypeDef { fallible: true, optional: true, constraint: value::Kind::Map.into() }, + } + ]; + #[test] fn parses() { let cases = vec![ @@ -160,7 +188,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func @@ -182,7 +210,7 @@ mod tests { } } - let mut state = remap::State::default(); + let mut state = state::Program::default(); let mut object = map![]; let msg = format!( diff --git a/src/remap/function/parse_timestamp.rs b/src/remap/function/parse_timestamp.rs index 21495e805a117..053078df78773 100644 --- a/src/remap/function/parse_timestamp.rs +++ b/src/remap/function/parse_timestamp.rs @@ -64,16 +64,20 @@ impl ParseTimestampFn { } impl Expression for ParseTimestampFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { - let format = { - let bytes = required!(state, object, self.format, Value::String(v) => v); - format!("timestamp|{}", String::from_utf8_lossy(&bytes)) - }; - - let conversion: Conversion = format.parse().map_err(|e| format!("{}", e))?; + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { + let format = self.format.execute(state, object); let to_timestamp = |value| match value { - Value::String(_) => conversion + Value::String(_) => format + .clone()? + .ok_or_else(|| Error::from(function::Error::Required("format".to_owned()))) + .map(|v| format!("timestamp|{}", String::from_utf8_lossy(&v.unwrap_string())))? + .parse::() + .map_err(|e| format!("{}", e))? .convert(value.into()) .map(Into::into) .map_err(|e| e.to_string().into()), @@ -87,6 +91,43 @@ impl Expression for ParseTimestampFn { to_timestamp, ) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + let value_def = self + .value + .type_def(state) + .fallible_unless(value::Kind::Timestamp); + + let default_def = self + .default + .as_ref() + .map(|v| v.type_def(state).fallible_unless(value::Kind::Timestamp)); + + // The `format` type definition is only relevant if: + // + // 1. `value` can resolve to a string, AND: + // 1. `default` is not defined, OR + // 2. `default` can also resolve to a string. + // + // The `format` type is _always_ fallible, because its string has to be + // parsed into a valid timestamp format. + let format_def = if value_def.constraint.contains(&value::Kind::String.into()) { + match &default_def { + Some(def) if def.constraint.contains(&value::Kind::String.into()) => { + Some(self.format.type_def(state).into_fallible(true)) + } + Some(_) => None, + None => Some(self.format.type_def(state).into_fallible(true)), + } + } else { + None + }; + + value_def + .merge_with_default_optional(default_def) + .merge_optional(format_def) + .with_constraint(value::Kind::Timestamp) + } } #[cfg(test)] @@ -95,6 +136,82 @@ mod tests { use crate::map; use chrono::{DateTime, Utc}; + remap::test_type_def![ + value_fallible_no_default { + expr: |_| ParseTimestampFn { + value: Literal::from("").boxed(), + format: Literal::from("").boxed(), + default: None, + }, + def: TypeDef { + fallible: true, + constraint: value::Kind::Timestamp.into(), + ..Default::default() + }, + } + + value_fallible_default_fallible { + expr: |_| ParseTimestampFn { + value: Literal::from("").boxed(), + format: Literal::from("").boxed(), + default: Some(Literal::from("").boxed()), + }, + def: TypeDef { + fallible: true, + constraint: value::Kind::Timestamp.into(), + ..Default::default() + }, + } + + value_fallible_default_infallible { + expr: |_| ParseTimestampFn { + value: Literal::from("").boxed(), + format: Literal::from("").boxed(), + default: Some(Literal::from(chrono::Utc::now()).boxed()), + }, + def: TypeDef { + constraint: value::Kind::Timestamp.into(), + ..Default::default() + }, + } + + value_infallible_no_default { + expr: |_| ParseTimestampFn { + value: Literal::from(chrono::Utc::now()).boxed(), + format: Literal::from("").boxed(), + default: None, + }, + def: TypeDef { + constraint: value::Kind::Timestamp.into(), + ..Default::default() + }, + } + + value_infallible_default_fallible { + expr: |_| ParseTimestampFn { + value: Literal::from(chrono::Utc::now()).boxed(), + format: Literal::from("").boxed(), + default: Some(Literal::from("").boxed()), + }, + def: TypeDef { + constraint: value::Kind::Timestamp.into(), + ..Default::default() + }, + } + + value_infallible_default_infallible { + expr: |_| ParseTimestampFn { + value: Literal::from(chrono::Utc::now()).boxed(), + format: Literal::from("").boxed(), + default: Some(Literal::from(chrono::Utc::now()).boxed()), + }, + def: TypeDef { + constraint: value::Kind::Timestamp.into(), + ..Default::default() + }, + } + ]; + #[test] fn parse_timestamp() { let cases = vec![ @@ -145,7 +262,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/parse_url.rs b/src/remap/function/parse_url.rs index e9ceddfc75026..603d15bb69bf1 100644 --- a/src/remap/function/parse_url.rs +++ b/src/remap/function/parse_url.rs @@ -40,7 +40,11 @@ impl ParseUrlFn { } impl Expression for ParseUrlFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let bytes = required!(state, object, self.value, Value::String(v) => v); Url::parse(&String::from_utf8_lossy(&bytes)) @@ -49,6 +53,14 @@ impl Expression for ParseUrlFn { .map(Into::into) .map(Some) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + self.value + .type_def(state) + .fallible_unless(value::Kind::String) + .into_fallible(true) // URL parsing error + .with_constraint(value::Kind::Map) + } } impl From for event::Value { @@ -86,6 +98,18 @@ mod tests { use super::*; use crate::map; + remap::test_type_def![ + value_string { + expr: |_| ParseUrlFn { value: Literal::from("foo").boxed() }, + def: TypeDef { fallible: true, constraint: value::Kind::Map.into(), ..Default::default() }, + } + + value_optional { + expr: |_| ParseUrlFn { value: Box::new(Noop) }, + def: TypeDef { fallible: true, optional: true, constraint: value::Kind::Map.into() }, + } + ]; + #[test] fn parse_url() { let cases = vec![ @@ -132,7 +156,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/replace.rs b/src/remap/function/replace.rs index a27e5267e8ff7..3b0e5e2d703e3 100644 --- a/src/remap/function/replace.rs +++ b/src/remap/function/replace.rs @@ -72,7 +72,11 @@ impl ReplaceFn { } impl Expression for ReplaceFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let value = required!(state, object, self.value, Value::String(b) => String::from_utf8_lossy(&b).into_owned()); let with = required!(state, object, self.with, Value::String(b) => String::from_utf8_lossy(&b).into_owned()); let count = optional!(state, object, self.count, Value::Integer(v) => v).unwrap_or(-1); @@ -102,6 +106,30 @@ impl Expression for ReplaceFn { } } } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + use value::Kind::*; + + let with_def = self.with.type_def(state).fallible_unless(String); + + let count_def = self + .count + .as_ref() + .map(|count| count.type_def(state).fallible_unless(Integer)); + + let pattern_def = match &self.pattern { + Argument::Expression(expr) => Some(expr.type_def(state).fallible_unless(String)), + Argument::Regex(_) => None, // regex is a concrete infallible type + }; + + self.value + .type_def(state) + .fallible_unless(String) + .merge(with_def) + .merge_optional(pattern_def) + .merge_optional(count_def) + .with_constraint(String) + } } #[cfg(test)] @@ -109,6 +137,103 @@ mod test { use super::*; use crate::map; + remap::test_type_def![ + infallible { + expr: |_| ReplaceFn { + value: Literal::from("foo").boxed(), + pattern: regex::Regex::new("foo").unwrap().into(), + with: Literal::from("foo").boxed(), + count: None, + }, + def: TypeDef { + constraint: value::Kind::String.into(), + ..Default::default() + }, + } + + value_fallible { + expr: |_| ReplaceFn { + value: Literal::from(10).boxed(), + pattern: regex::Regex::new("foo").unwrap().into(), + with: Literal::from("foo").boxed(), + count: None, + }, + def: TypeDef { + fallible: true, + constraint: value::Kind::String.into(), + ..Default::default() + }, + } + + pattern_expression_infallible { + expr: |_| ReplaceFn { + value: Literal::from("foo").boxed(), + pattern: Literal::from("foo").boxed().into(), + with: Literal::from("foo").boxed(), + count: None, + }, + def: TypeDef { + constraint: value::Kind::String.into(), + ..Default::default() + }, + } + + pattern_expression_fallible { + expr: |_| ReplaceFn { + value: Literal::from("foo").boxed(), + pattern: Literal::from(10).boxed().into(), + with: Literal::from("foo").boxed(), + count: None, + }, + def: TypeDef { + fallible: true, + constraint: value::Kind::String.into(), + ..Default::default() + }, + } + + with_fallible { + expr: |_| ReplaceFn { + value: Literal::from("foo").boxed(), + pattern: regex::Regex::new("foo").unwrap().into(), + with: Literal::from(10).boxed(), + count: None, + }, + def: TypeDef { + fallible: true, + constraint: value::Kind::String.into(), + ..Default::default() + }, + } + + count_infallible { + expr: |_| ReplaceFn { + value: Literal::from("foo").boxed(), + pattern: regex::Regex::new("foo").unwrap().into(), + with: Literal::from("foo").boxed(), + count: Some(Literal::from(10).boxed()), + }, + def: TypeDef { + constraint: value::Kind::String.into(), + ..Default::default() + }, + } + + count_fallible { + expr: |_| ReplaceFn { + value: Literal::from("foo").boxed(), + pattern: regex::Regex::new("foo").unwrap().into(), + with: Literal::from("foo").boxed(), + count: Some(Literal::from("foo").boxed()), + }, + def: TypeDef { + fallible: true, + constraint: value::Kind::String.into(), + ..Default::default() + }, + } + ]; + #[test] fn check_replace_string() { let cases = vec![ @@ -117,7 +242,7 @@ mod test { Ok(Some("I like opples ond bononos".into())), ReplaceFn::new( Box::new(Literal::from("I like apples and bananas")), - Argument::Expression(Box::new(Literal::from("a"))), + Box::new(Literal::from("a")).into(), "o", None, ), @@ -127,7 +252,7 @@ mod test { Ok(Some("I like opples ond bononos".into())), ReplaceFn::new( Box::new(Literal::from("I like apples and bananas")), - Argument::Expression(Box::new(Literal::from("a"))), + Box::new(Literal::from("a")).into(), "o", Some(-1), ), @@ -137,7 +262,7 @@ mod test { Ok(Some("I like apples and bananas".into())), ReplaceFn::new( Box::new(Literal::from("I like apples and bananas")), - Argument::Expression(Box::new(Literal::from("a"))), + Box::new(Literal::from("a")).into(), "o", Some(0), ), @@ -147,7 +272,7 @@ mod test { Ok(Some("I like opples and bananas".into())), ReplaceFn::new( Box::new(Literal::from("I like apples and bananas")), - Argument::Expression(Box::new(Literal::from("a"))), + Box::new(Literal::from("a")).into(), "o", Some(1), ), @@ -157,14 +282,14 @@ mod test { Ok(Some("I like opples ond bananas".into())), ReplaceFn::new( Box::new(Literal::from("I like apples and bananas")), - Argument::Expression(Box::new(Literal::from("a"))), + Box::new(Literal::from("a")).into(), "o", Some(2), ), ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func @@ -183,7 +308,7 @@ mod test { Ok(Some("I like opples ond bononos".into())), ReplaceFn::new( Box::new(Literal::from("I like apples and bananas")), - Argument::Regex(regex::Regex::new("a").unwrap()), + regex::Regex::new("a").unwrap().into(), "o", None, ), @@ -193,7 +318,7 @@ mod test { Ok(Some("I like opples ond bononos".into())), ReplaceFn::new( Box::new(Literal::from("I like apples and bananas")), - Argument::Regex(regex::Regex::new("a").unwrap()), + regex::Regex::new("a").unwrap().into(), "o", Some(-1), ), @@ -203,7 +328,7 @@ mod test { Ok(Some("I like apples and bananas".into())), ReplaceFn::new( Box::new(Literal::from("I like apples and bananas")), - Argument::Regex(regex::Regex::new("a").unwrap()), + regex::Regex::new("a").unwrap().into(), "o", Some(0), ), @@ -213,7 +338,7 @@ mod test { Ok(Some("I like opples and bananas".into())), ReplaceFn::new( Box::new(Literal::from("I like apples and bananas")), - Argument::Regex(regex::Regex::new("a").unwrap()), + regex::Regex::new("a").unwrap().into(), "o", Some(1), ), @@ -223,14 +348,14 @@ mod test { Ok(Some("I like opples ond bananas".into())), ReplaceFn::new( Box::new(Literal::from("I like apples and bananas")), - Argument::Regex(regex::Regex::new("a").unwrap()), + regex::Regex::new("a").unwrap().into(), "o", Some(2), ), ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func @@ -249,7 +374,7 @@ mod test { Ok(Some("I like biscuits and bananas".into())), ReplaceFn::new( Box::new(Literal::from("I like apples and bananas")), - Argument::Expression(Box::new(Literal::from("apples"))), + Box::new(Literal::from("apples")).into(), "biscuits", None, ), @@ -259,7 +384,7 @@ mod test { Ok(Some("I like opples and bananas".into())), ReplaceFn::new( Box::new(Path::from("foo")), - Argument::Regex(regex::Regex::new("a").unwrap()), + regex::Regex::new("a").unwrap().into(), "o", Some(1), ), @@ -269,14 +394,14 @@ mod test { Ok(Some("I like biscuits and bananas".into())), ReplaceFn::new( Box::new(Path::from("foo")), - Argument::Regex(regex::Regex::new("\\[apples\\]").unwrap()), + regex::Regex::new("\\[apples\\]").unwrap().into(), "biscuits", None, ), ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/round.rs b/src/remap/function/round.rs index ff622aaef7b84..b7d558cf9e1b9 100644 --- a/src/remap/function/round.rs +++ b/src/remap/function/round.rs @@ -46,7 +46,11 @@ impl RoundFn { } impl Expression for RoundFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let precision = optional!(state, object, self.precision, Value::Integer(v) => v).unwrap_or(0); let res = required!(state, object, self.value, @@ -58,12 +62,68 @@ impl Expression for RoundFn { Ok(res.into()) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + use value::Kind::*; + + let value_def = self + .value + .type_def(state) + .fallible_unless(vec![Integer, Float]); + let precision_def = self + .precision + .as_ref() + .map(|precision| precision.type_def(state).fallible_unless(Integer)); + + value_def + .clone() + .merge_optional(precision_def) + .with_constraint(match value_def.constraint { + v if v.is(Float) || v.is(Integer) => v, + _ => vec![Integer, Float].into(), + }) + } } #[cfg(test)] mod tests { use super::*; use crate::map; + use value::Kind::*; + + remap::test_type_def![ + value_float { + expr: |_| RoundFn { + value: Literal::from(1.0).boxed(), + precision: None, + }, + def: TypeDef { constraint: Float.into(), ..Default::default() }, + } + + value_integer { + expr: |_| RoundFn { + value: Literal::from(1).boxed(), + precision: None, + }, + def: TypeDef { constraint: Integer.into(), ..Default::default() }, + } + + value_float_or_integer { + expr: |_| RoundFn { + value: Variable::new("foo".to_owned()).boxed(), + precision: None, + }, + def: TypeDef { fallible: true, constraint: vec![Integer, Float].into(), ..Default::default() }, + } + + fallible_precision { + expr: |_| RoundFn { + value: Literal::from(1).boxed(), + precision: Some(Variable::new("foo".to_owned()).boxed()), + }, + def: TypeDef { fallible: true, constraint: Integer.into(), ..Default::default() }, + } + ]; #[test] fn round() { @@ -118,7 +178,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/sha1.rs b/src/remap/function/sha1.rs index abea600120c81..38c29ff727f94 100644 --- a/src/remap/function/sha1.rs +++ b/src/remap/function/sha1.rs @@ -36,7 +36,11 @@ impl Sha1Fn { } impl Expression for Sha1Fn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { use ::sha1::{Digest, Sha1}; self.value.execute(state, object).map(|r| { @@ -46,12 +50,37 @@ impl Expression for Sha1Fn { }) }) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + self.value + .type_def(state) + .fallible_unless(value::Kind::String) + .with_constraint(value::Kind::String) + } } #[cfg(test)] mod tests { use super::*; use crate::map; + use value::Kind::*; + + remap::test_type_def![ + value_string { + expr: |_| Sha1Fn { value: Literal::from("foo").boxed() }, + def: TypeDef { constraint: String.into(), ..Default::default() }, + } + + value_non_string { + expr: |_| Sha1Fn { value: Literal::from(1).boxed() }, + def: TypeDef { fallible: true, constraint: String.into(), ..Default::default() }, + } + + value_optional { + expr: |_| Sha1Fn { value: Box::new(Noop) }, + def: TypeDef { fallible: true, optional: true, constraint: String.into() }, + } + ]; #[test] fn sha1() { @@ -70,7 +99,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/sha2.rs b/src/remap/function/sha2.rs index 81a9bc98a21ca..885e68f44122b 100644 --- a/src/remap/function/sha2.rs +++ b/src/remap/function/sha2.rs @@ -41,7 +41,11 @@ impl Sha2Fn { } impl Expression for Sha2Fn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let value = required!(state, object, self.value, Value::String(v) => v); let variant = optional!(state, object, self.variant, Value::String(v) => v); @@ -63,6 +67,19 @@ impl Expression for Sha2Fn { Ok(Some(hash.into())) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + self.value + .type_def(state) + .fallible_unless(value::Kind::String) + .merge_optional( + self.variant + .as_ref() + .map(|variant| variant.type_def(state).fallible_unless(value::Kind::String)), + ) + .into_fallible(true) // unknown variant enum + .with_constraint(value::Kind::String) + } } #[inline] @@ -74,6 +91,41 @@ fn encode(value: &[u8]) -> String { mod tests { use super::*; use crate::map; + use value::Kind::*; + + remap::test_type_def![ + value_string { + expr: |_| Sha2Fn { + value: Literal::from("foo").boxed(), + variant: None, + }, + def: TypeDef { fallible: true, constraint: String.into(), ..Default::default() }, + } + + value_non_string { + expr: |_| Sha2Fn { + value: Literal::from(1).boxed(), + variant: None, + }, + def: TypeDef { fallible: true, constraint: String.into(), ..Default::default() }, + } + + value_optional { + expr: |_| Sha2Fn { + value: Box::new(Noop), + variant: None, + }, + def: TypeDef { fallible: true, optional: true, constraint: String.into() }, + } + + variant_fallible { + expr: |_| Sha2Fn { + value: Literal::from("foo").boxed(), + variant: Some(Variable::new("foo".to_string()).boxed()), + }, + def: TypeDef { fallible: true, constraint: String.into(), ..Default::default() }, + } + ]; #[test] fn sha2() { @@ -139,7 +191,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/sha3.rs b/src/remap/function/sha3.rs index f8dfdd779e91f..8d0821dca3c90 100644 --- a/src/remap/function/sha3.rs +++ b/src/remap/function/sha3.rs @@ -41,7 +41,11 @@ impl Sha3Fn { } impl Expression for Sha3Fn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let value = required!(state, object, self.value, Value::String(v) => v); let variant = optional!(state, object, self.variant, Value::String(v) => v); @@ -61,6 +65,19 @@ impl Expression for Sha3Fn { Ok(Some(hash.into())) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + self.value + .type_def(state) + .fallible_unless(value::Kind::String) + .merge_optional( + self.variant + .as_ref() + .map(|variant| variant.type_def(state).fallible_unless(value::Kind::String)), + ) + .into_fallible(true) // unknown variant enum + .with_constraint(value::Kind::String) + } } #[inline] @@ -72,6 +89,41 @@ fn encode(value: &[u8]) -> String { mod tests { use super::*; use crate::map; + use value::Kind::*; + + remap::test_type_def![ + value_string { + expr: |_| Sha3Fn { + value: Literal::from("foo").boxed(), + variant: None, + }, + def: TypeDef { fallible: true, constraint: String.into(), ..Default::default() }, + } + + value_non_string { + expr: |_| Sha3Fn { + value: Literal::from(1).boxed(), + variant: None, + }, + def: TypeDef { fallible: true, constraint: String.into(), ..Default::default() }, + } + + value_optional { + expr: |_| Sha3Fn { + value: Box::new(Noop), + variant: None, + }, + def: TypeDef { fallible: true, optional: true, constraint: String.into() }, + } + + variant_fallible { + expr: |_| Sha3Fn { + value: Literal::from("foo").boxed(), + variant: Some(Variable::new("foo".to_string()).boxed()), + }, + def: TypeDef { fallible: true, constraint: String.into(), ..Default::default() }, + } + ]; #[test] fn sha3() { @@ -123,7 +175,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/slice.rs b/src/remap/function/slice.rs index a9554d52c561f..ed1dbc5cdbdfb 100644 --- a/src/remap/function/slice.rs +++ b/src/remap/function/slice.rs @@ -55,7 +55,11 @@ impl SliceFn { } impl Expression for SliceFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let start = required!(state, object, self.start, Value::Integer(v) => v); let end = optional!(state, object, self.end, Value::Integer(v) => v); @@ -93,12 +97,65 @@ impl Expression for SliceFn { .map(Some), } } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + use value::Kind::*; + + let value_def = self + .value + .type_def(state) + .fallible_unless(vec![String, Array]); + let end_def = self + .end + .as_ref() + .map(|end| end.type_def(state).fallible_unless(Integer)); + + value_def + .clone() + .merge(self.start.type_def(state).fallible_unless(Integer)) + .merge_optional(end_def) + .with_constraint(match value_def.constraint { + v if v.is(String) || v.is(Array) => v, + _ => vec![String, Array].into(), + }) + .into_fallible(true) // can fail for invalid start..end ranges + } } #[cfg(test)] mod tests { use super::*; use crate::map; + use value::Kind::*; + + remap::test_type_def![ + value_string { + expr: |_| SliceFn { + value: Literal::from("foo").boxed(), + start: Literal::from(0).boxed(), + end: None, + }, + def: TypeDef { fallible: true, constraint: String.into(), ..Default::default() }, + } + + value_array { + expr: |_| SliceFn { + value: Literal::from(vec!["foo"]).boxed(), + start: Literal::from(0).boxed(), + end: None, + }, + def: TypeDef { fallible: true, constraint: Array.into(), ..Default::default() }, + } + + value_unknown { + expr: |_| SliceFn { + value: Variable::new("foo".to_owned()).boxed(), + start: Literal::from(0).boxed(), + end: None, + }, + def: TypeDef { fallible: true, constraint: vec![String, Array].into(), ..Default::default() }, + } + ]; #[test] fn bytes() { @@ -163,7 +220,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func @@ -212,7 +269,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func @@ -248,7 +305,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/split.rs b/src/remap/function/split.rs index 5d7aa6d93cf89..3b3d96243e904 100644 --- a/src/remap/function/split.rs +++ b/src/remap/function/split.rs @@ -1,5 +1,5 @@ use remap::prelude::*; -use std::convert::{TryFrom, TryInto}; +use std::convert::TryFrom; #[derive(Clone, Copy, Debug)] pub struct Split; @@ -50,13 +50,12 @@ pub(crate) struct SplitFn { } impl Expression for SplitFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { - let string: String = self - .value - .execute(state, object)? - .ok_or_else(|| Error::from("argument missing"))? - .try_into()?; - + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { + let value = required!(state, object, self.value, Value::String(b) => String::from_utf8_lossy(&b).into_owned()); let limit: usize = self .limit .as_ref() @@ -69,19 +68,119 @@ impl Expression for SplitFn { let value = match &self.pattern { Argument::Regex(pattern) => pattern - .splitn(&string, limit as usize) + .splitn(&value, limit as usize) .collect::>() .into(), Argument::Expression(expr) => { - let pattern: String = expr - .execute(state, object)? - .ok_or_else(|| Error::from("argument missing"))? - .try_into()?; + let pattern = required!(state, object, expr, Value::String(b) => String::from_utf8_lossy(&b).into_owned()); - string.splitn(limit, &pattern).collect::>().into() + value.splitn(limit, &pattern).collect::>().into() } }; Ok(Some(value)) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + use value::Kind::*; + + let limit_def = self + .limit + .as_ref() + .map(|limit| limit.type_def(state).fallible_unless(vec![Integer, Float])); + + let pattern_def = match &self.pattern { + Argument::Expression(expr) => Some(expr.type_def(state).fallible_unless(String)), + Argument::Regex(_) => None, // regex is a concrete infallible type + }; + + self.value + .type_def(state) + .fallible_unless(String) + .merge_optional(limit_def) + .merge_optional(pattern_def) + .with_constraint(Array) + } +} + +#[cfg(test)] +mod test { + use super::*; + + remap::test_type_def![ + infallible { + expr: |_| SplitFn { + value: Literal::from("foo").boxed(), + pattern: regex::Regex::new("foo").unwrap().into(), + limit: None, + }, + def: TypeDef { + constraint: value::Kind::Array.into(), + ..Default::default() + }, + } + + value_fallible { + expr: |_| SplitFn { + value: Literal::from(10).boxed(), + pattern: regex::Regex::new("foo").unwrap().into(), + limit: None, + }, + def: TypeDef { + fallible: true, + constraint: value::Kind::Array.into(), + ..Default::default() + }, + } + + pattern_expression_infallible { + expr: |_| SplitFn { + value: Literal::from("foo").boxed(), + pattern: Literal::from("foo").boxed().into(), + limit: None, + }, + def: TypeDef { + constraint: value::Kind::Array.into(), + ..Default::default() + }, + } + + pattern_expression_fallible { + expr: |_| SplitFn { + value: Literal::from("foo").boxed(), + pattern: Literal::from(10).boxed().into(), + limit: None, + }, + def: TypeDef { + fallible: true, + constraint: value::Kind::Array.into(), + ..Default::default() + }, + } + + limit_infallible { + expr: |_| SplitFn { + value: Literal::from("foo").boxed(), + pattern: regex::Regex::new("foo").unwrap().into(), + limit: Some(Literal::from(10).boxed()), + }, + def: TypeDef { + constraint: value::Kind::Array.into(), + ..Default::default() + }, + } + + limit_fallible { + expr: |_| SplitFn { + value: Literal::from("foo").boxed(), + pattern: regex::Regex::new("foo").unwrap().into(), + limit: Some(Literal::from("foo").boxed()), + }, + def: TypeDef { + fallible: true, + constraint: value::Kind::Array.into(), + ..Default::default() + }, + } + ]; } diff --git a/src/remap/function/starts_with.rs b/src/remap/function/starts_with.rs index 54a211f3764df..c95c9399ff2d4 100644 --- a/src/remap/function/starts_with.rs +++ b/src/remap/function/starts_with.rs @@ -63,7 +63,11 @@ impl StartsWithFn { } impl Expression for StartsWithFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let substring = { let bytes = required!(state, object, self.substring, Value::String(v) => v); String::from_utf8_lossy(&bytes).into_owned() @@ -82,12 +86,68 @@ impl Expression for StartsWithFn { Ok(Some(starts_with.into())) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + self.value + .type_def(state) + .fallible_unless(value::Kind::String) + .merge( + self.substring + .type_def(state) + .fallible_unless(value::Kind::String), + ) + .merge_optional(self.case_sensitive.as_ref().map(|case_sensitive| { + case_sensitive + .type_def(state) + .fallible_unless(value::Kind::Boolean) + })) + .with_constraint(value::Kind::Boolean) + } } #[cfg(test)] mod tests { use super::*; use crate::map; + use value::Kind::*; + + remap::test_type_def![ + value_string { + expr: |_| StartsWithFn { + value: Literal::from("foo").boxed(), + substring: Literal::from("foo").boxed(), + case_sensitive: None, + }, + def: TypeDef { constraint: Boolean.into(), ..Default::default() }, + } + + value_non_string { + expr: |_| StartsWithFn { + value: Literal::from(true).boxed(), + substring: Literal::from("foo").boxed(), + case_sensitive: None, + }, + def: TypeDef { fallible: true, constraint: Boolean.into(), ..Default::default() }, + } + + substring_non_string { + expr: |_| StartsWithFn { + value: Literal::from("foo").boxed(), + substring: Literal::from(true).boxed(), + case_sensitive: None, + }, + def: TypeDef { fallible: true, constraint: Boolean.into(), ..Default::default() }, + } + + case_sensitive_non_boolean { + expr: |_| StartsWithFn { + value: Literal::from("foo").boxed(), + substring: Literal::from("foo").boxed(), + case_sensitive: Some(Literal::from(1).boxed()), + }, + def: TypeDef { fallible: true, constraint: Boolean.into(), ..Default::default() }, + } + ]; #[test] fn starts_with() { @@ -144,7 +204,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/strip_ansi_escape_codes.rs b/src/remap/function/strip_ansi_escape_codes.rs index b0a68004663f7..d0fef7dc5a2be 100644 --- a/src/remap/function/strip_ansi_escape_codes.rs +++ b/src/remap/function/strip_ansi_escape_codes.rs @@ -37,7 +37,11 @@ impl StripAnsiEscapeCodesFn { } impl Expression for StripAnsiEscapeCodesFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let bytes = required!(state, object, self.value, Value::String(v) => v); strip_ansi_escapes::strip(&bytes) @@ -46,6 +50,16 @@ impl Expression for StripAnsiEscapeCodesFn { .map(Into::into) .map_err(|e| e.to_string().into()) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + self.value + .type_def(state) + .fallible_unless(value::Kind::String) + // TODO: Can probably remove this, as it only fails if writing to + // the buffer fails. + .into_fallible(true) + .with_constraint(value::Kind::String) + } } #[cfg(test)] @@ -53,6 +67,18 @@ mod tests { use super::*; use crate::map; + remap::test_type_def![ + value_string { + expr: |_| StripAnsiEscapeCodesFn { value: Literal::from("foo").boxed() }, + def: TypeDef { fallible: true, constraint: value::Kind::String.into(), ..Default::default() }, + } + + fallible_expression { + expr: |_| StripAnsiEscapeCodesFn { value: Literal::from(10).boxed() }, + def: TypeDef { fallible: true, constraint: value::Kind::String.into(), ..Default::default() }, + } + ]; + #[test] fn strip_ansi_escape_codes() { let cases = vec![ @@ -83,7 +109,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/strip_whitespace.rs b/src/remap/function/strip_whitespace.rs index ed732b4ab90e1..f7d3b75483ee7 100644 --- a/src/remap/function/strip_whitespace.rs +++ b/src/remap/function/strip_whitespace.rs @@ -36,11 +36,22 @@ impl StripWhitespaceFn { } impl Expression for StripWhitespaceFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let value = required!(state, object, self.value, Value::String(b) => String::from_utf8_lossy(&b).into_owned()); Ok(Some(value.trim().into())) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + self.value + .type_def(state) + .fallible_unless(value::Kind::String) + .with_constraint(value::Kind::String) + } } #[cfg(test)] @@ -48,6 +59,18 @@ mod tests { use super::*; use crate::map; + remap::test_type_def![ + value_string { + expr: |_| StripWhitespaceFn { value: Literal::from("foo").boxed() }, + def: TypeDef { constraint: value::Kind::String.into(), ..Default::default() }, + } + + fallible_expression { + expr: |_| StripWhitespaceFn { value: Literal::from(10).boxed() }, + def: TypeDef { fallible: true, constraint: value::Kind::String.into(), ..Default::default() }, + } + ]; + #[test] fn strip_whitespace() { let cases = vec![ @@ -83,7 +106,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/to_bool.rs b/src/remap/function/to_bool.rs index 4a15ab5ac5a4f..421d29253dd68 100644 --- a/src/remap/function/to_bool.rs +++ b/src/remap/function/to_bool.rs @@ -47,18 +47,23 @@ impl ToBoolFn { } impl Expression for ToBoolFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { use Value::*; let to_bool = |value| match value { Boolean(_) => Ok(value), Integer(v) => Ok(Boolean(v != 0)), Float(v) => Ok(Boolean(v != 0.0)), + Null => Ok(Boolean(false)), String(_) => Conversion::Boolean .convert(value.into()) .map(Into::into) .map_err(|e| e.to_string().into()), - _ => Err("unable to convert value to boolean".into()), + Array(_) | Map(_) | Timestamp(_) => Err("unable to convert value to boolean".into()), }; super::convert_value_or_default( @@ -67,12 +72,127 @@ impl Expression for ToBoolFn { to_bool, ) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + use value::Kind::*; + + self.value + .type_def(state) + .fallible_unless(vec![Boolean, Integer, Float, Null]) + .merge_with_default_optional(self.default.as_ref().map(|default| { + default + .type_def(state) + .fallible_unless(vec![Boolean, Integer, Float, Null]) + })) + .with_constraint(Boolean) + } } #[cfg(test)] mod tests { use super::*; use crate::map; + use std::collections::BTreeMap; + use value::Kind::*; + + remap::test_type_def![ + boolean_infallible { + expr: |_| ToBoolFn { value: Literal::from(true).boxed(), default: None }, + def: TypeDef { constraint: Boolean.into(), ..Default::default() }, + } + + integer_infallible { + expr: |_| ToBoolFn { value: Literal::from(1).boxed(), default: None }, + def: TypeDef { constraint: Boolean.into(), ..Default::default() }, + } + + float_infallible { + expr: |_| ToBoolFn { value: Literal::from(1.0).boxed(), default: None }, + def: TypeDef { constraint: Boolean.into(), ..Default::default() }, + } + + null_infallible { + expr: |_| ToBoolFn { value: Literal::from(()).boxed(), default: None }, + def: TypeDef { constraint: Boolean.into(), ..Default::default() }, + } + + string_fallible { + expr: |_| ToBoolFn { value: Literal::from("foo").boxed(), default: None }, + def: TypeDef { fallible: true, constraint: Boolean.into(), ..Default::default() }, + } + + map_fallible { + expr: |_| ToBoolFn { value: Literal::from(BTreeMap::new()).boxed(), default: None }, + def: TypeDef { fallible: true, constraint: Boolean.into(), ..Default::default() }, + } + + array_fallible { + expr: |_| ToBoolFn { value: Literal::from(vec![0]).boxed(), default: None }, + def: TypeDef { fallible: true, constraint: Boolean.into(), ..Default::default() }, + } + + timestamp_fallible { + expr: |_| ToBoolFn { value: Literal::from(chrono::Utc::now()).boxed(), default: None }, + def: TypeDef { fallible: true, constraint: Boolean.into(), ..Default::default() }, + } + + fallible_value_without_default { + expr: |_| ToBoolFn { value: Literal::from("foo".to_owned()).boxed(), default: None }, + def: TypeDef { + fallible: true, + optional: false, + constraint: Boolean.into(), + }, + } + + fallible_value_with_fallible_default { + expr: |_| ToBoolFn { + value: Literal::from(vec![0]).boxed(), + default: Some(Literal::from(vec![0]).boxed()), + }, + def: TypeDef { + fallible: true, + optional: false, + constraint: Boolean.into(), + }, + } + + fallible_value_with_infallible_default { + expr: |_| ToBoolFn { + value: Literal::from(vec![0]).boxed(), + default: Some(Literal::from(true).boxed()), + }, + def: TypeDef { + fallible: false, + optional: false, + constraint: Boolean.into(), + }, + } + + infallible_value_with_fallible_default { + expr: |_| ToBoolFn { + value: Literal::from(true).boxed(), + default: Some(Literal::from(vec![0]).boxed()), + }, + def: TypeDef { + fallible: false, + optional: false, + constraint: Boolean.into(), + }, + } + + infallible_value_with_infallible_default { + expr: |_| ToBoolFn { + value: Literal::from(true).boxed(), + default: Some(Literal::from(true).boxed()), + }, + def: TypeDef { + fallible: false, + optional: false, + constraint: Boolean.into(), + }, + } + ]; #[test] fn to_bool() { @@ -99,7 +219,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/to_float.rs b/src/remap/function/to_float.rs index 89e2270fc6888..991625e97fb89 100644 --- a/src/remap/function/to_float.rs +++ b/src/remap/function/to_float.rs @@ -47,18 +47,23 @@ impl ToFloatFn { } impl Expression for ToFloatFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { use Value::*; let to_float = |value| match value { Float(_) => Ok(value), Integer(v) => Ok(Float(v as f64)), Boolean(v) => Ok(Float(if v { 1.0 } else { 0.0 })), + Null => Ok(0.0.into()), String(_) => Conversion::Float .convert(value.into()) .map(Into::into) .map_err(|e| e.to_string().into()), - _ => Err("unable to convert value to float".into()), + Array(_) | Map(_) | Timestamp(_) => Err("unable to convert value to float".into()), }; super::convert_value_or_default( @@ -67,12 +72,127 @@ impl Expression for ToFloatFn { to_float, ) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + use value::Kind::*; + + self.value + .type_def(state) + .fallible_unless(vec![Float, Integer, Boolean, Null]) + .merge_with_default_optional(self.default.as_ref().map(|default| { + default + .type_def(state) + .fallible_unless(vec![Float, Integer, Boolean, Null]) + })) + .with_constraint(Float) + } } #[cfg(test)] mod tests { use super::*; use crate::map; + use std::collections::BTreeMap; + use value::Kind::*; + + remap::test_type_def![ + boolean_infallible { + expr: |_| ToFloatFn { value: Literal::from(true).boxed(), default: None }, + def: TypeDef { constraint: Float.into(), ..Default::default() }, + } + + integer_infallible { + expr: |_| ToFloatFn { value: Literal::from(1).boxed(), default: None }, + def: TypeDef { constraint: Float.into(), ..Default::default() }, + } + + float_infallible { + expr: |_| ToFloatFn { value: Literal::from(1.0).boxed(), default: None }, + def: TypeDef { constraint: Float.into(), ..Default::default() }, + } + + null_infallible { + expr: |_| ToFloatFn { value: Literal::from(()).boxed(), default: None }, + def: TypeDef { constraint: Float.into(), ..Default::default() }, + } + + string_fallible { + expr: |_| ToFloatFn { value: Literal::from("foo").boxed(), default: None }, + def: TypeDef { fallible: true, constraint: Float.into(), ..Default::default() }, + } + + map_fallible { + expr: |_| ToFloatFn { value: Literal::from(BTreeMap::new()).boxed(), default: None }, + def: TypeDef { fallible: true, constraint: Float.into(), ..Default::default() }, + } + + array_fallible { + expr: |_| ToFloatFn { value: Literal::from(vec![0]).boxed(), default: None }, + def: TypeDef { fallible: true, constraint: Float.into(), ..Default::default() }, + } + + timestamp_infallible { + expr: |_| ToFloatFn { value: Literal::from(chrono::Utc::now()).boxed(), default: None }, + def: TypeDef { fallible: true, constraint: Float.into(), ..Default::default() }, + } + + fallible_value_without_default { + expr: |_| ToFloatFn { value: Variable::new("foo".to_owned()).boxed(), default: None }, + def: TypeDef { + fallible: true, + optional: false, + constraint: Float.into(), + }, + } + + fallible_value_with_fallible_default { + expr: |_| ToFloatFn { + value: Literal::from(vec![0]).boxed(), + default: Some(Literal::from(vec![0]).boxed()), + }, + def: TypeDef { + fallible: true, + optional: false, + constraint: Float.into(), + }, + } + + fallible_value_with_infallible_default { + expr: |_| ToFloatFn { + value: Literal::from(vec![0]).boxed(), + default: Some(Literal::from(1).boxed()), + }, + def: TypeDef { + fallible: false, + optional: false, + constraint: Float.into(), + }, + } + + infallible_value_with_fallible_default { + expr: |_| ToFloatFn { + value: Literal::from(1).boxed(), + default: Some(Literal::from(vec![0]).boxed()), + }, + def: TypeDef { + fallible: false, + optional: false, + constraint: Float.into(), + }, + } + + infallible_value_with_infallible_default { + expr: |_| ToFloatFn { + value: Literal::from(1).boxed(), + default: Some(Literal::from(1).boxed()), + }, + def: TypeDef { + fallible: false, + optional: false, + constraint: Float.into(), + }, + } + ]; #[test] fn to_float() { @@ -99,7 +219,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/to_int.rs b/src/remap/function/to_int.rs index 7eb2689c12751..6be6a5a923743 100644 --- a/src/remap/function/to_int.rs +++ b/src/remap/function/to_int.rs @@ -47,18 +47,23 @@ impl ToIntFn { } impl Expression for ToIntFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { use Value::*; let to_int = |value| match value { Integer(_) => Ok(value), Float(v) => Ok(Integer(v as i64)), Boolean(v) => Ok(Integer(if v { 1 } else { 0 })), + Null => Ok(0.into()), String(_) => Conversion::Integer .convert(value.into()) .map(Into::into) .map_err(|e| e.to_string().into()), - _ => Err("unable to convert value to integer".into()), + Array(_) | Map(_) | Timestamp(_) => Err("unable to convert value to integer".into()), }; super::convert_value_or_default( @@ -67,12 +72,127 @@ impl Expression for ToIntFn { to_int, ) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + use value::Kind::*; + + self.value + .type_def(state) + .fallible_unless(vec![Integer, Float, Boolean, Null]) + .merge_with_default_optional(self.default.as_ref().map(|default| { + default + .type_def(state) + .fallible_unless(vec![Integer, Float, Boolean, Null]) + })) + .with_constraint(Integer) + } } #[cfg(test)] mod tests { use super::*; use crate::map; + use std::collections::BTreeMap; + use value::Kind::*; + + remap::test_type_def![ + boolean_infallible { + expr: |_| ToIntFn { value: Literal::from(true).boxed(), default: None }, + def: TypeDef { constraint: Integer.into(), ..Default::default() }, + } + + integer_infallible { + expr: |_| ToIntFn { value: Literal::from(1).boxed(), default: None }, + def: TypeDef { constraint: Integer.into(), ..Default::default() }, + } + + float_infallible { + expr: |_| ToIntFn { value: Literal::from(1.0).boxed(), default: None }, + def: TypeDef { constraint: Integer.into(), ..Default::default() }, + } + + null_infallible { + expr: |_| ToIntFn { value: Literal::from(()).boxed(), default: None }, + def: TypeDef { constraint: Integer.into(), ..Default::default() }, + } + + string_fallible { + expr: |_| ToIntFn { value: Literal::from("foo").boxed(), default: None }, + def: TypeDef { fallible: true, constraint: Integer.into(), ..Default::default() }, + } + + map_fallible { + expr: |_| ToIntFn { value: Literal::from(BTreeMap::new()).boxed(), default: None }, + def: TypeDef { fallible: true, constraint: Integer.into(), ..Default::default() }, + } + + array_fallible { + expr: |_| ToIntFn { value: Literal::from(vec![0]).boxed(), default: None }, + def: TypeDef { fallible: true, constraint: Integer.into(), ..Default::default() }, + } + + timestamp_infallible { + expr: |_| ToIntFn { value: Literal::from(chrono::Utc::now()).boxed(), default: None }, + def: TypeDef { fallible: true, constraint: Integer.into(), ..Default::default() }, + } + + fallible_value_without_default { + expr: |_| ToIntFn { value: Variable::new("foo".to_owned()).boxed(), default: None }, + def: TypeDef { + fallible: true, + optional: false, + constraint: Integer.into(), + }, + } + + fallible_value_with_fallible_default { + expr: |_| ToIntFn { + value: Literal::from(vec![0]).boxed(), + default: Some(Literal::from(vec![0]).boxed()), + }, + def: TypeDef { + fallible: true, + optional: false, + constraint: Integer.into(), + }, + } + + fallible_value_with_infallible_default { + expr: |_| ToIntFn { + value: Literal::from(vec![0]).boxed(), + default: Some(Literal::from(1).boxed()), + }, + def: TypeDef { + fallible: false, + optional: false, + constraint: Integer.into(), + }, + } + + infallible_value_with_fallible_default { + expr: |_| ToIntFn { + value: Literal::from(1).boxed(), + default: Some(Literal::from(vec![0]).boxed()), + }, + def: TypeDef { + fallible: false, + optional: false, + constraint: Integer.into(), + }, + } + + infallible_value_with_infallible_default { + expr: |_| ToIntFn { + value: Literal::from(1).boxed(), + default: Some(Literal::from(1).boxed()), + }, + def: TypeDef { + fallible: false, + optional: false, + constraint: Integer.into(), + }, + } + ]; #[test] fn to_int() { @@ -99,7 +219,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/to_string.rs b/src/remap/function/to_string.rs index eb6f90998ad30..1ea077cb2fabf 100644 --- a/src/remap/function/to_string.rs +++ b/src/remap/function/to_string.rs @@ -46,7 +46,11 @@ impl ToStringFn { } impl Expression for ToStringFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let to_string = |value| match value { Value::String(_) => Ok(value), _ => Ok(value.as_string_lossy()), @@ -58,12 +62,122 @@ impl Expression for ToStringFn { to_string, ) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + self.value + .type_def(state) + .merge_with_default_optional( + self.default.as_ref().map(|default| default.type_def(state)), + ) + .with_constraint(value::Kind::String) + } } #[cfg(test)] mod tests { use super::*; use crate::map; + use std::collections::BTreeMap; + use value::Kind::*; + + remap::test_type_def![ + boolean_infallible { + expr: |_| ToStringFn { value: Literal::from(true).boxed(), default: None}, + def: TypeDef { constraint: String.into(), ..Default::default() }, + } + + integer_infallible { + expr: |_| ToStringFn { value: Literal::from(1).boxed(), default: None}, + def: TypeDef { constraint: String.into(), ..Default::default() }, + } + + float_infallible { + expr: |_| ToStringFn { value: Literal::from(1.0).boxed(), default: None}, + def: TypeDef { constraint: String.into(), ..Default::default() }, + } + + null_infallible { + expr: |_| ToStringFn { value: Literal::from(()).boxed(), default: None}, + def: TypeDef { constraint: String.into(), ..Default::default() }, + } + + string_infallible { + expr: |_| ToStringFn { value: Literal::from("foo").boxed(), default: None}, + def: TypeDef { constraint: String.into(), ..Default::default() }, + } + + map_infallible { + expr: |_| ToStringFn { value: Literal::from(BTreeMap::new()).boxed(), default: None}, + def: TypeDef { constraint: String.into(), ..Default::default() }, + } + + array_infallible { + expr: |_| ToStringFn { value: Literal::from(vec![0]).boxed(), default: None}, + def: TypeDef { constraint: String.into(), ..Default::default() }, + } + + timestamp_infallible { + expr: |_| ToStringFn { value: Literal::from(chrono::Utc::now()).boxed(), default: None}, + def: TypeDef { constraint: String.into(), ..Default::default() }, + } + + fallible_value_without_default { + expr: |_| ToStringFn { value: Variable::new("foo".to_owned()).boxed(), default: None}, + def: TypeDef { + fallible: true, + optional: false, + constraint: String.into(), + }, + } + + fallible_value_with_fallible_default { + expr: |_| ToStringFn { + value: Variable::new("foo".to_owned()).boxed(), + default: Some(Variable::new("foo".to_owned()).boxed()), + }, + def: TypeDef { + fallible: true, + optional: false, + constraint: String.into(), + }, + } + + fallible_value_with_infallible_default { + expr: |_| ToStringFn { + value: Variable::new("foo".to_owned()).boxed(), + default: Some(Literal::from(true).boxed()), + }, + def: TypeDef { + fallible: false, + optional: false, + constraint: String.into(), + }, + } + + infallible_value_with_fallible_default { + expr: |_| ToStringFn { + value: Literal::from(true).boxed(), + default: Some(Variable::new("foo".to_owned()).boxed()), + }, + def: TypeDef { + fallible: false, + optional: false, + constraint: String.into(), + }, + } + + infallible_value_with_infallible_default { + expr: |_| ToStringFn { + value: Literal::from(true).boxed(), + default: Some(Literal::from(true).boxed()), + }, + def: TypeDef { + fallible: false, + optional: false, + constraint: String.into(), + }, + } + ]; #[test] fn to_string() { @@ -90,7 +204,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/to_timestamp.rs b/src/remap/function/to_timestamp.rs index a7f7bf4ce564d..8c33e1448e3bf 100644 --- a/src/remap/function/to_timestamp.rs +++ b/src/remap/function/to_timestamp.rs @@ -64,18 +64,24 @@ impl ToTimestampFn { } impl Expression for ToTimestampFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { use Value::*; let to_timestamp = |value| match value { Timestamp(_) => Ok(value), + Integer(v) => Ok(Timestamp(Utc.timestamp(v, 0))), + Float(v) => Ok(Timestamp(Utc.timestamp(v.round() as i64, 0))), String(_) => Conversion::Timestamp .convert(value.into()) .map(Into::into) .map_err(|e| e.to_string().into()), - Integer(v) => Ok(Timestamp(Utc.timestamp(v, 0))), - Float(v) => Ok(Timestamp(Utc.timestamp(v.round() as i64, 0))), - _ => Err("unable to convert value to timestamp".into()), + Boolean(_) | Array(_) | Map(_) | Null => { + Err("unable to convert value to timestamp".into()) + } }; super::convert_value_or_default( @@ -84,12 +90,127 @@ impl Expression for ToTimestampFn { to_timestamp, ) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + use value::Kind::*; + + self.value + .type_def(state) + .fallible_unless(vec![Timestamp, Integer, Float]) + .merge_with_default_optional(self.default.as_ref().map(|default| { + default + .type_def(state) + .fallible_unless(vec![Timestamp, Integer, Float]) + })) + .with_constraint(Timestamp) + } } #[cfg(test)] mod tests { use super::*; use crate::map; + use std::collections::BTreeMap; + use value::Kind::*; + + remap::test_type_def![ + timestamp_infallible { + expr: |_| ToTimestampFn { value: Literal::from(chrono::Utc::now()).boxed(), default: None}, + def: TypeDef { constraint: Timestamp.into(), ..Default::default() }, + } + + integer_infallible { + expr: |_| ToTimestampFn { value: Literal::from(1).boxed(), default: None}, + def: TypeDef { constraint: Timestamp.into(), ..Default::default() }, + } + + float_infallible { + expr: |_| ToTimestampFn { value: Literal::from(1.0).boxed(), default: None}, + def: TypeDef { constraint: Timestamp.into(), ..Default::default() }, + } + + null_fallible { + expr: |_| ToTimestampFn { value: Literal::from(()).boxed(), default: None}, + def: TypeDef { fallible: true, constraint: Timestamp.into(), ..Default::default() }, + } + + string_fallible { + expr: |_| ToTimestampFn { value: Literal::from("foo").boxed(), default: None}, + def: TypeDef { fallible: true, constraint: Timestamp.into(), ..Default::default() }, + } + + map_fallible { + expr: |_| ToTimestampFn { value: Literal::from(BTreeMap::new()).boxed(), default: None}, + def: TypeDef { fallible: true, constraint: Timestamp.into(), ..Default::default() }, + } + + array_fallible { + expr: |_| ToTimestampFn { value: Literal::from(vec![0]).boxed(), default: None}, + def: TypeDef { fallible: true, constraint: Timestamp.into(), ..Default::default() }, + } + + boolean_fallible { + expr: |_| ToTimestampFn { value: Literal::from(true).boxed(), default: None}, + def: TypeDef { fallible: true, constraint: Timestamp.into(), ..Default::default() }, + } + + fallible_value_without_default { + expr: |_| ToTimestampFn { value: Variable::new("foo".to_owned()).boxed(), default: None}, + def: TypeDef { + fallible: true, + optional: false, + constraint: Timestamp.into(), + }, + } + + fallible_value_with_fallible_default { + expr: |_| ToTimestampFn { + value: Literal::from(vec![0]).boxed(), + default: Some(Literal::from(vec![0]).boxed()), + }, + def: TypeDef { + fallible: true, + optional: false, + constraint: Timestamp.into(), + }, + } + + fallible_value_with_infallible_default { + expr: |_| ToTimestampFn { + value: Literal::from(vec![0]).boxed(), + default: Some(Literal::from(1).boxed()), + }, + def: TypeDef { + fallible: false, + optional: false, + constraint: Timestamp.into(), + }, + } + + infallible_value_with_fallible_default { + expr: |_| ToTimestampFn { + value: Literal::from(1).boxed(), + default: Some(Literal::from(vec![0]).boxed()), + }, + def: TypeDef { + fallible: false, + optional: false, + constraint: Timestamp.into(), + }, + } + + infallible_value_with_infallible_default { + expr: |_| ToTimestampFn { + value: Literal::from(1).boxed(), + default: Some(Literal::from(1).boxed()), + }, + def: TypeDef { + fallible: false, + optional: false, + constraint: Timestamp.into(), + }, + } + ]; #[test] fn to_timestamp() { @@ -124,7 +245,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/tokenize.rs b/src/remap/function/tokenize.rs index 107f8f8e6a104..eca940ac62c57 100644 --- a/src/remap/function/tokenize.rs +++ b/src/remap/function/tokenize.rs @@ -37,7 +37,11 @@ impl TokenizeFn { } impl Expression for TokenizeFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let value = { let bytes = required!(state, object, self.value, Value::String(v) => v); String::from_utf8_lossy(&bytes).into_owned() @@ -54,6 +58,13 @@ impl Expression for TokenizeFn { Ok(Some(tokens)) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + self.value + .type_def(state) + .fallible_unless(value::Kind::String) + .with_constraint(value::Kind::Array) + } } #[cfg(test)] @@ -61,6 +72,18 @@ mod tests { use super::*; use crate::map; + remap::test_type_def![ + value_string { + expr: |_| TokenizeFn { value: Literal::from("foo").boxed() }, + def: TypeDef { constraint: value::Kind::Array.into(), ..Default::default() }, + } + + value_non_string { + expr: |_| TokenizeFn { value: Literal::from(10).boxed() }, + def: TypeDef { fallible: true, constraint: value::Kind::Array.into(), ..Default::default() }, + } + ]; + #[test] fn tokenize() { let cases = vec![( @@ -78,7 +101,7 @@ mod tests { TokenizeFn::new(Box::new(Literal::from("217.250.207.207 - - [07/Sep/2020:16:38:00 -0400] \"DELETE /deliverables/next-generation/user-centric HTTP/1.1\" 205 11881"))), )]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/truncate.rs b/src/remap/function/truncate.rs index cbf037c266add..588507b141649 100644 --- a/src/remap/function/truncate.rs +++ b/src/remap/function/truncate.rs @@ -66,7 +66,11 @@ impl TruncateFn { } impl Expression for TruncateFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { let mut value = { let bytes = required!(state, object, self.value, Value::String(v) => v); String::from_utf8_lossy(&bytes).into_owned() @@ -74,10 +78,12 @@ impl Expression for TruncateFn { let limit = required!( state, object, self.limit, - Value::Float(f) if f >= 0.0 => f.floor() as usize, - Value::Integer(i) if i >= 0 => i as usize, + Value::Float(f) => f.floor() as i64, + Value::Integer(i) => i as i64, ); + let limit = if limit < 0 { 0 } else { limit as usize }; + let ellipsis = optional!(state, object, self.ellipsis, Value::Boolean(v) => v).unwrap_or_default(); @@ -100,6 +106,25 @@ impl Expression for TruncateFn { Ok(Some(value.into())) } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + use value::Kind::*; + + self.value + .type_def(state) + .fallible_unless(String) + .merge( + self.limit + .type_def(state) + .fallible_unless(vec![Integer, Float]), + ) + .merge_optional( + self.ellipsis + .as_ref() + .map(|ellipsis| ellipsis.type_def(state).fallible_unless(Boolean)), + ) + .with_constraint(String) + } } #[cfg(test)] @@ -107,6 +132,62 @@ mod tests { use super::*; use crate::map; + remap::test_type_def![ + infallible { + expr: |_| TruncateFn { + value: Literal::from("foo").boxed(), + limit: Literal::from(1).boxed(), + ellipsis: None, + }, + def: TypeDef { constraint: value::Kind::String.into(), ..Default::default() }, + } + + value_non_string { + expr: |_| TruncateFn { + value: Literal::from(false).boxed(), + limit: Literal::from(1).boxed(), + ellipsis: None, + }, + def: TypeDef { fallible: true, constraint: value::Kind::String.into(), ..Default::default() }, + } + + limit_float { + expr: |_| TruncateFn { + value: Literal::from("foo").boxed(), + limit: Literal::from(1.0).boxed(), + ellipsis: None, + }, + def: TypeDef { constraint: value::Kind::String.into(), ..Default::default() }, + } + + limit_non_number { + expr: |_| TruncateFn { + value: Literal::from("foo").boxed(), + limit: Literal::from("bar").boxed(), + ellipsis: None, + }, + def: TypeDef { fallible: true, constraint: value::Kind::String.into(), ..Default::default() }, + } + + ellipsis_boolean { + expr: |_| TruncateFn { + value: Literal::from("foo").boxed(), + limit: Literal::from(10).boxed(), + ellipsis: Some(Literal::from(true).boxed()), + }, + def: TypeDef { constraint: value::Kind::String.into(), ..Default::default() }, + } + + ellipsis_non_boolean { + expr: |_| TruncateFn { + value: Literal::from("foo").boxed(), + limit: Literal::from("bar").boxed(), + ellipsis: Some(Literal::from("baz").boxed()), + }, + def: TypeDef { fallible: true, constraint: value::Kind::String.into(), ..Default::default() }, + } + ]; + #[test] fn truncate() { let cases = vec![ @@ -184,7 +265,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/upcase.rs b/src/remap/function/upcase.rs index 4f1eebf04f58d..234ff94ab08c1 100644 --- a/src/remap/function/upcase.rs +++ b/src/remap/function/upcase.rs @@ -37,7 +37,11 @@ impl UpcaseFn { } impl Expression for UpcaseFn { - fn execute(&self, state: &mut State, object: &mut dyn Object) -> Result> { + fn execute( + &self, + state: &mut state::Program, + object: &mut dyn Object, + ) -> Result> { self.value .execute(state, object)? .map(String::try_from) @@ -47,6 +51,13 @@ impl Expression for UpcaseFn { .map(Ok) .transpose() } + + fn type_def(&self, state: &state::Compiler) -> TypeDef { + self.value + .type_def(state) + .fallible_unless(value::Kind::String) + .with_constraint(value::Kind::String) + } } #[cfg(test)] @@ -54,6 +65,18 @@ mod tests { use super::*; use crate::map; + remap::test_type_def![ + string { + expr: |_| UpcaseFn { value: Literal::from("foo").boxed() }, + def: TypeDef { constraint: value::Kind::String.into(), ..Default::default() }, + } + + non_string { + expr: |_| UpcaseFn { value: Literal::from(true).boxed() }, + def: TypeDef { fallible: true, constraint: value::Kind::String.into(), ..Default::default() }, + } + ]; + #[test] fn upcase() { let cases = vec![ @@ -69,7 +92,7 @@ mod tests { ), ]; - let mut state = remap::State::default(); + let mut state = state::Program::default(); for (mut object, exp, func) in cases { let got = func diff --git a/src/remap/function/uuid_v4.rs b/src/remap/function/uuid_v4.rs index be668e08467da..eccae1bd6188d 100644 --- a/src/remap/function/uuid_v4.rs +++ b/src/remap/function/uuid_v4.rs @@ -18,12 +18,19 @@ impl Function for UuidV4 { struct UuidV4Fn; impl Expression for UuidV4Fn { - fn execute(&self, _: &mut State, _: &mut dyn Object) -> Result> { + fn execute(&self, _: &mut state::Program, _: &mut dyn Object) -> Result> { let mut buf = [0; 36]; let uuid = uuid::Uuid::new_v4().to_hyphenated().encode_lower(&mut buf); Ok(Some(Bytes::copy_from_slice(uuid.as_bytes()).into())) } + + fn type_def(&self, _: &state::Compiler) -> TypeDef { + TypeDef { + constraint: value::Kind::String.into(), + ..Default::default() + } + } } #[cfg(test)] @@ -32,9 +39,17 @@ mod tests { use crate::map; use std::convert::TryFrom; + remap::test_type_def![static_def { + expr: |_| UuidV4Fn, + def: TypeDef { + constraint: value::Kind::String.into(), + ..Default::default() + }, + }]; + #[test] fn uuid_v4() { - let mut state = remap::State::default(); + let mut state = state::Program::default(); let mut object = map![]; let value = UuidV4Fn.execute(&mut state, &mut object).unwrap().unwrap(); diff --git a/src/transforms/remap.rs b/src/transforms/remap.rs index 0977969e91951..2655b7e206d5d 100644 --- a/src/transforms/remap.rs +++ b/src/transforms/remap.rs @@ -5,7 +5,7 @@ use crate::{ transforms::{FunctionTransform, Transform}, Result, }; -use remap::{Program, Runtime}; +use remap::{value, Program, Runtime, TypeDef}; use serde::{Deserialize, Serialize}; #[derive(Deserialize, Serialize, Debug, Clone, Derivative)] @@ -50,8 +50,16 @@ pub struct Remap { impl Remap { pub fn new(config: RemapConfig) -> crate::Result { + let accepts = TypeDef { + fallible: true, + optional: true, + constraint: value::Constraint::Any, + }; + + let program = Program::new(&config.source, &crate::remap::FUNCTIONS_MUT, accepts)?; + Ok(Remap { - program: Program::new(&config.source, &crate::remap::FUNCTIONS_MUT)?, + program, drop_on_err: config.drop_on_err, }) }