First-order recursive filters

jaq-core/tests/tests.rs

@@ -7,6 +7,8 @@ use jaq_interpret::error::{Error, Type};
 use jaq_interpret::Val;
 use serde_json::json;
 
+yields!(repeat, "def r(f): f, r(f); [limit(3; r(1, 2))]", [1, 2, 1]);
+
 yields!(nested_rec, "def f: def g: 0, g; g; def h: h; first(f)", 0);
 
 yields!(

jaq-interpret/src/filter.rs

@@ -2,7 +2,7 @@ use crate::box_iter::{box_once, flat_map_with, map_with};
 use crate::path::{self, Path};
 use crate::results::{fold, recurse, then, Results};
 use crate::val::{Val, ValR, ValRs};
-use crate::{rc_lazy_list, Ctx, Error};
+use crate::{rc_lazy_list, Bind, Ctx, Error};
 use alloc::{boxed::Box, string::String, vec::Vec};
 use dyn_clone::DynClone;
 use jaq_syn::filter::FoldType;
@@ -16,7 +16,8 @@ pub struct Owned(Ast, Vec<Def>);
 pub struct Ref<'a>(&'a Ast, &'a [Def]);
 
 #[derive(Debug, Clone)]
-pub struct Def {
+pub(crate) struct Def {
+    pub rec: bool,
     pub rhs: Ast,
 }
 
@@ -28,7 +29,7 @@ impl Owned {
 
 /// Function from a value to a stream of value results.
 #[derive(Clone, Debug, Default)]
-pub enum Ast {
+pub(crate) enum Ast {
     #[default]
     Id,
     ToString,
@@ -86,7 +87,7 @@ pub enum Ast {
     Call {
         skip: usize,
         id: usize,
-        args: Vec<Self>,
+        args: Vec<Bind<Self, Self>>,
     },
 
     Native(Native, Vec<Self>),
@@ -104,22 +105,25 @@ dyn_clone::clone_trait_object!(<'a> Update<'a>);
 /// and return the enhanced contexts together with the original value of `cv`.
 ///
 /// This is used when we call filters with variable arguments.
-fn bind_vars<'a, F, I>(mut args: I, ctx: Ctx<'a>, cv: Cv<'a>) -> Results<'a, Cv<'a>, Error>
+fn bind_vars<'a, I>(mut args: I, ctx: Ctx<'a>, cv: Cv<'a>) -> Results<'a, Cv<'a>, Error>
 where
-    F: FilterT<'a>,
-    I: Iterator<Item = F> + Clone + 'a,
+    I: Iterator<Item = Bind<Ref<'a>, Ref<'a>>> + Clone + 'a,
 {
     match args.next() {
-        Some(arg) => flat_map_with(
+        Some(Bind::Var(arg)) => flat_map_with(
             arg.run(cv.clone()),
             (ctx, cv, args),
             |y, (ctx, cv, args)| then(y, |y| bind_vars(args, ctx.cons_var(y), cv)),
         ),
+        Some(Bind::Fun(Ref(arg, _defs))) => bind_vars(args, ctx.cons_fun((arg, cv.0.clone())), cv),
         None => box_once(Ok((ctx, cv.1))),
     }
 }
 
-fn reduce<'a>(xs: ValRs<'a>, init: Val, f: impl Fn(Val, Val) -> ValRs<'a> + 'a) -> ValRs {
+fn reduce<'a, T: Clone + 'a, F>(xs: Results<'a, T, Error>, init: Val, f: F) -> ValRs
+where
+    F: Fn(T, Val) -> ValRs<'a> + 'a,
+{
     let xs = rc_lazy_list::List::from_iter(xs);
     Box::new(fold(false, xs, box_once(Ok(init)), f))
 }
@@ -281,13 +285,21 @@ impl<'a> FilterT<'a> for Ref<'a> {
             }
             Ast::Recurse(f) => w(f).recurse(true, true, cv),
 
-            Ast::Var(v) => box_once(Ok(cv.0.vars.get(*v).unwrap().clone())),
-            Ast::Call { skip, id, args } => Box::new(crate::LazyIter::new(move || {
+            Ast::Var(v) => match cv.0.vars.get(*v).unwrap() {
+                Bind::Var(v) => box_once(Ok(v.clone())),
+                Bind::Fun(f) => w(f.0).run((f.1.clone(), cv.1)),
+            },
+            Ast::Call { skip, id, args } => {
                 let def = &self.1[*id];
                 let ctx = cv.0.clone().skip_vars(*skip);
-                let ctxs = bind_vars(args.iter().map(w), ctx, cv);
-                ctxs.flat_map(move |cv| then(cv, |cv| w(&def.rhs).run(cv)))
-            })),
+                let cvs = bind_vars(args.iter().map(move |a| a.as_ref().map(w)), ctx, cv);
+                let f = move || cvs.flat_map(move |cv| then(cv, |cv| w(&def.rhs).run(cv)));
+                if def.rec {
+                    Box::new(crate::LazyIter::new(f))
+                } else {
+                    Box::new(f())
+                }
+            }
 
             Ast::Native(Native { run, .. }, args) => (run)(Args(args, self.1), cv),
         }
@@ -334,15 +346,17 @@ impl<'a> FilterT<'a> for Ref<'a> {
             }),
             Ast::Recurse(l) => w(l).recurse_update(cv, f),
 
-            Ast::Var(_) => err,
+            Ast::Var(v) => match cv.0.vars.get(*v).unwrap() {
+                Bind::Var(_) => err,
+                Bind::Fun(l) => w(l.0).update((l.1.clone(), cv.1), f),
+            },
             Ast::Call { skip, id, args } => {
                 let def = &self.1[*id];
+                let rhs = w(&def.rhs);
+                let init = cv.1.clone();
                 let ctx = cv.0.clone().skip_vars(*skip);
-                let init = box_once(Ok(cv.1.clone()));
-                let ctxs = rc_lazy_list::List::from_iter(bind_vars(args.iter().map(w), ctx, cv));
-                Box::new(fold(false, ctxs, init, move |cv, v| {
-                    w(&def.rhs).update((cv.0, v), f.clone())
-                }))
+                let cvs = bind_vars(args.iter().map(move |a| a.as_ref().map(w)), ctx, cv);
+                reduce(cvs, init, move |cv, v| rhs.update((cv.0, v), f.clone()))
             }
 
             Ast::Native(Native { update, .. }, args) => (update)(Args(args, self.1), cv, f),

jaq-interpret/src/lib.rs

@@ -61,11 +61,11 @@ mod val;
 
 pub use error::Error;
 pub use filter::{Args, FilterT, Native, Owned as Filter, RunPtr, UpdatePtr};
-pub use mir::Ctx as ParseCtx;
 pub use rc_iter::RcIter;
 pub use val::{Val, ValR, ValRs};
 
-use alloc::string::String;
+use alloc::{string::String, vec::Vec};
+use jaq_syn::Arg as Bind;
 use lazy_iter::LazyIter;
 use rc_list::List as RcList;
 
@@ -75,20 +75,26 @@ type Inputs<'i> = RcIter<dyn Iterator<Item = Result<Val, String>> + 'i>;
 #[derive(Clone)]
 pub struct Ctx<'a> {
     /// variable bindings
-    vars: RcList<Val>,
+    vars: RcList<Bind<Val, (&'a filter::Ast, Self)>>,
     inputs: &'a Inputs<'a>,
 }
 
 impl<'a> Ctx<'a> {
     /// Construct a context.
     pub fn new(vars: impl IntoIterator<Item = Val>, inputs: &'a Inputs<'a>) -> Self {
-        let vars = RcList::new().extend(vars);
+        let vars = RcList::new().extend(vars.into_iter().map(Bind::Var));
         Self { vars, inputs }
     }
 
     /// Add a new variable binding.
     pub(crate) fn cons_var(mut self, x: Val) -> Self {
-        self.vars = self.vars.cons(x);
+        self.vars = self.vars.cons(Bind::Var(x));
+        self
+    }
+
+    /// Add a new filter binding.
+    pub(crate) fn cons_fun(mut self, f: (&'a filter::Ast, Self)) -> Self {
+        self.vars = self.vars.cons(Bind::Fun(f));
         self
     }
 
@@ -104,17 +110,76 @@ impl<'a> Ctx<'a> {
     }
 }
 
+/// Combined MIR/LIR compilation.
+///
+/// This allows to go from a parsed filter to a filter executable by this crate.
+pub struct ParseCtx {
+    /// errors occurred during transformation
+    // TODO for v2.0: remove this and make it a function
+    pub errs: Vec<jaq_syn::Spanned<mir::Error>>,
+    native: Vec<(String, usize, filter::Native)>,
+    def: jaq_syn::Def,
+}
+
 impl ParseCtx {
+    /// Initialise new context with list of global variables.
+    ///
+    /// When running a filter produced by this context,
+    /// values corresponding to the variables have to be supplied in the execution context.
+    pub fn new(vars: Vec<String>) -> Self {
+        use alloc::string::ToString;
+        let def = jaq_syn::Def {
+            lhs: jaq_syn::Call {
+                name: "$".to_string(),
+                args: vars.into_iter().map(Bind::Var).collect(),
+            },
+            rhs: jaq_syn::Main {
+                defs: Vec::new(),
+                body: (jaq_syn::filter::Filter::Id, 0..0),
+            },
+        };
+
+        Self {
+            errs: Vec::new(),
+            native: Vec::new(),
+            def,
+        }
+    }
+
+    /// Add a native filter with given name and arity.
+    pub fn insert_native(&mut self, name: String, arity: usize, f: filter::Native) {
+        self.native.push((name, arity, f))
+    }
+
+    /// Add native filters with given names and arities.
+    pub fn insert_natives<I>(&mut self, natives: I)
+    where
+        I: IntoIterator<Item = (String, usize, filter::Native)>,
+    {
+        self.native.extend(natives)
+    }
+
+    /// Import parsed definitions, such as obtained from the standard library.
+    ///
+    /// Errors that might occur include undefined variables, for example.
+    pub fn insert_defs(&mut self, defs: impl IntoIterator<Item = jaq_syn::Def>) {
+        self.def.rhs.defs.extend(defs);
+    }
+
     /// Given a main filter (consisting of definitions and a body), return a finished filter.
     pub fn compile(&mut self, main: jaq_syn::Main) -> Filter {
-        self.insert_defs(main.defs);
-        self.root_filter(main.body);
+        let mut mctx = mir::Ctx::default();
+        mctx.native = self.native.clone();
+        self.def.rhs.defs.extend(main.defs);
+        self.def.rhs.body = main.body;
+        let def = mctx.def(self.def.clone());
+        self.errs = mctx.errs;
 
         if !self.errs.is_empty() {
             return Default::default();
         }
-        //std::dbg!("before LIR");
-        lir::root_def(&self.defs)
+
+        lir::root_def(def)
     }
 
     /// Compile and run a filter on given input, panic if it does not compile or yield the given output.