diff --git a/.github/workflows/msrv.yml b/.github/workflows/msrv.yml index e1ac70a7..fbb39fc6 100644 --- a/.github/workflows/msrv.yml +++ b/.github/workflows/msrv.yml @@ -16,6 +16,12 @@ jobs: - name: Check jaq-core working-directory: jaq-core run: cargo check + - name: Check jaq-syn + working-directory: jaq-syn + run: cargo check + - name: Check jaq-interpret + working-directory: jaq-interpret + run: cargo check - uses: dtolnay/rust-toolchain@1.64 - name: Check jaq-std diff --git a/Cargo.lock b/Cargo.lock index 1b7b3625..53f9dcc6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,6 +148,12 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68b0cf012f1230e43cd00ebb729c6bb58707ecfa8ad08b52ef3a4ccd2697fc30" +[[package]] +name = "either" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" + [[package]] name = "env_logger" version = "0.10.0" @@ -255,6 +261,7 @@ dependencies = [ "jaq-parse", "jaq-std", "jaq-syn", + "log", "memmap2", "mimalloc", "serde_json", @@ -283,7 +290,9 @@ name = "jaq-interpret" version = "1.2.1" dependencies = [ "ahash", + "chumsky", "dyn-clone", + "either", "hifijson", "indexmap", "jaq-parse", @@ -297,6 +306,7 @@ name = "jaq-parse" version = "1.0.2" dependencies = [ "chumsky", + "either", "jaq-syn", ] diff --git a/README.md b/README.md index d12fc69c..f5ddaa58 100644 --- a/README.md +++ b/README.md @@ -203,7 +203,7 @@ Here is an overview that summarises: ## Basics - [x] Identity (`.`) -- [x] Recursion (`..`) +- [x] Recursive descent (`..`) - [x] Basic data types (null, boolean, number, string, array, object) - [x] if-then-else (`if .a < .b then .a else .b end`) - [x] Folding (`reduce .[] as $x (0; . + $x)`, `foreach .[] as $x (0; . + $x; . + .)`) @@ -229,9 +229,9 @@ Here is an overview that summarises: - [x] Plain assignment (`=`) - [x] Update assignment (`|=`, `+=`, `-=`) - [x] Alternation (`//`) -- [x] Logic (`or`, `and`) +- [x] Logical (`or`, `and`) - [x] Equality and comparison (`.a == .b`, `.a < .b`) -- [x] Arithmetic (`+`, `-`, `*`, `/`, `%`) +- [x] Arithmetical (`+`, `-`, `*`, `/`, `%`) - [x] Negation (`-`) - [x] Error suppression (`?`) @@ -277,7 +277,7 @@ Their definitions are at [`std.jq`](jaq-std/src/std.jq). - [x] Array filters (`transpose`, `first`, `last`, `nth(10)`, `flatten`, `min`, `max`) - [x] Object-array conversion (`to_entries`, `from_entries`, `with_entries`) - [x] Universal/existential (`all`, `any`) -- [x] Recursion (`walk`) +- [x] Recursive application (`walk`) - [x] I/O (`input`) - [x] Regular expressions (`test`, `scan`, `match`, `capture`, `splits`, `sub`, `gsub`) - [x] Time (`fromdate`, `todate`) diff --git a/jaq-core/Cargo.toml b/jaq-core/Cargo.toml index 82767df9..34b474f6 100644 --- a/jaq-core/Cargo.toml +++ b/jaq-core/Cargo.toml @@ -12,6 +12,8 @@ rust-version = "1.63" [features] default = ["std", "format", "log", "math", "parse_json", "regex", "time"] +unstable = ["unstable-flag", "jaq-interpret/unstable"] +unstable-flag = ["jaq-interpret/unstable-flag"] std = [] format = ["aho-corasick", "base64", "urlencoding"] math = ["libm"] diff --git a/jaq-core/src/lib.rs b/jaq-core/src/lib.rs index e22eaca3..737eb4c9 100644 --- a/jaq-core/src/lib.rs +++ b/jaq-core/src/lib.rs @@ -24,7 +24,11 @@ use jaq_interpret::{Error, FilterT, Native, RunPtr, UpdatePtr, Val, ValR, ValRs} /// but not `now`, `debug`, `fromdateiso8601`, ... /// /// Does not return filters from the standard library, such as `map`. -pub fn minimal() -> impl Iterator { +pub fn minimal( + #[cfg(feature = "unstable-flag")] + #[allow(unused_variables)] + unstable: bool, +) -> impl Iterator { run(CORE_RUN).chain(upd(CORE_UPDATE)) } @@ -42,15 +46,20 @@ pub fn minimal() -> impl Iterator { feature = "regex", feature = "time", ))] -pub fn core() -> impl Iterator { - minimal() - .chain(run(STD)) - .chain(run(FORMAT)) - .chain(upd(LOG)) - .chain(run(MATH)) - .chain(run(PARSE_JSON)) - .chain(run(REGEX)) - .chain(run(TIME)) +pub fn core( + #[cfg(feature = "unstable-flag")] unstable: bool, +) -> impl Iterator { + minimal( + #[cfg(feature = "unstable-flag")] + unstable, + ) + .chain(run(STD)) + .chain(run(FORMAT)) + .chain(upd(LOG)) + .chain(run(MATH)) + .chain(run(PARSE_JSON)) + .chain(run(REGEX)) + .chain(run(TIME)) } fn run<'a>(fs: &'a [(&str, usize, RunPtr)]) -> impl Iterator + 'a { @@ -86,6 +95,8 @@ fn length(v: &Val) -> ValR { Val::Str(s) => Ok(Val::Int(s.chars().count() as isize)), Val::Arr(a) => Ok(Val::Int(a.len() as isize)), Val::Obj(o) => Ok(Val::Int(o.len() as isize)), + #[cfg(feature = "unstable-flag")] + _ => unimplemented!(), } } diff --git a/jaq-core/tests/common/mod.rs b/jaq-core/tests/common/mod.rs index e6936a57..4bc53599 100644 --- a/jaq-core/tests/common/mod.rs +++ b/jaq-core/tests/common/mod.rs @@ -1,32 +1,57 @@ use serde_json::Value; -fn yields(x: jaq_interpret::Val, f: &str, ys: impl Iterator) { +#[cfg(not(feature = "unstable-flag"))] +pub const UNSTABLE: bool = false; +#[cfg(feature = "unstable-flag")] +pub const UNSTABLE: bool = false; + +#[track_caller] +fn yields( + #[cfg_attr(not(feature = "unstable-flag"), allow(unused_variables))] unstable: bool, + x: jaq_interpret::Val, + f: &str, + ys: impl Iterator, +) { let mut ctx = jaq_interpret::ParseCtx::new(Vec::new()); - ctx.insert_natives(jaq_core::core()); + ctx.insert_natives(jaq_core::core( + #[cfg(feature = "unstable-flag")] + unstable, + )); - let (f, errs) = jaq_parse::parse(f, jaq_parse::main()); + let (f, errs) = jaq_parse::parse( + #[cfg(feature = "unstable-flag")] + unstable, + f, + jaq_parse::main( + #[cfg(feature = "unstable-flag")] + unstable, + ), + ); assert!(errs.is_empty()); ctx.yields(x, f.unwrap(), ys) } -pub fn fail(x: Value, f: &str, err: jaq_interpret::Error) { - yields(x.into(), f, core::iter::once(Err(err))) +#[track_caller] +pub fn fail(unstable: bool, x: Value, f: &str, err: jaq_interpret::Error) { + yields(unstable, x.into(), f, core::iter::once(Err(err))) } -pub fn give(x: Value, f: &str, y: Value) { - yields(x.into(), f, core::iter::once(Ok(y.into()))) +#[track_caller] +pub fn give(unstable: bool, x: Value, f: &str, y: Value) { + yields(unstable, x.into(), f, core::iter::once(Ok(y.into()))) } -pub fn gives(x: Value, f: &str, ys: [Value; N]) { - yields(x.into(), f, ys.into_iter().map(|y| Ok(y.into()))) +#[track_caller] +pub fn gives(unstable: bool, x: Value, f: &str, ys: [Value; N]) { + yields(unstable, x.into(), f, ys.into_iter().map(|y| Ok(y.into()))) } #[macro_export] macro_rules! yields { - ($func_name:ident, $filter:expr, $output: expr) => { + ($func_name:ident, $unstable:expr, $filter:expr, $output: expr) => { #[test] fn $func_name() { - give(json!(null), $filter, json!($output)) + give($unstable, json!(null), $filter, json!($output)) } }; } diff --git a/jaq-core/tests/tests.rs b/jaq-core/tests/tests.rs index d7b8dac2..24c59b55 100644 --- a/jaq-core/tests/tests.rs +++ b/jaq-core/tests/tests.rs @@ -2,55 +2,81 @@ pub mod common; -use common::{fail, give, gives}; +use common::{fail, give, gives, UNSTABLE}; use jaq_interpret::error::{Error, Type}; use jaq_interpret::Val; use serde_json::json; -yields!(repeat, "def r(f): f, r(f); [limit(3; r(1, 2))]", [1, 2, 1]); +yields!( + repeat, + UNSTABLE, + "def r(f): f, r(f); [limit(3; r(1, 2))]", + [1, 2, 1] +); -yields!(lazy_array, "def f: 1, [f]; limit(1; f)", 1); +yields!(lazy_array, UNSTABLE, "def f: 1, [f]; limit(1; f)", 1); yields!( lazy_foreach, + UNSTABLE, "def f: f; limit(1; foreach (1, f) as $x (0; .))", 0 ); -yields!(nested_rec, "def f: def g: 0, g; g; def h: h; first(f)", 0); +yields!( + nested_rec, + UNSTABLE, + "def f: def g: 0, g; g; def h: h; first(f)", + 0 +); yields!( rec_two_var_args, + UNSTABLE, "def f($a; $b): [$a, $b], f($a+1; $b+1); [limit(3; f(0; 1))]", [[0, 1], [1, 2], [2, 3]] ); yields!( rec_update, + UNSTABLE, "def upto($x): .[$x], (if $x > 0 then upto($x-1) else empty end); [1, 2, 3, 4] | upto(1) |= .+1", [2, 3, 3, 4] ); #[test] fn ascii() { - give(json!("aAaAäの"), "ascii_upcase", json!("AAAAäの")); - give(json!("aAaAäの"), "ascii_downcase", json!("aaaaäの")); + give(UNSTABLE, json!("aAaAäの"), "ascii_upcase", json!("AAAAäの")); + give( + UNSTABLE, + json!("aAaAäの"), + "ascii_downcase", + json!("aaaaäの"), + ); } #[test] fn dateiso8601() { give( + UNSTABLE, json!("1970-01-02T00:00:00Z"), "fromdateiso8601", json!(86400), ); give( + UNSTABLE, json!("1970-01-02T00:00:00.123456789Z"), "fromdateiso8601", json!(86400.123456789), ); - give(json!(86400), "todateiso8601", json!("1970-01-02T00:00:00Z")); give( + UNSTABLE, + json!(86400), + "todateiso8601", + json!("1970-01-02T00:00:00Z"), + ); + give( + UNSTABLE, json!(86400.123456789), "todateiso8601", json!("1970-01-02T00:00:00.123456789Z"), @@ -59,78 +85,100 @@ fn dateiso8601() { #[test] fn explode_implode() { - give(json!("❤ の"), "explode", json!([10084, 32, 12398])); - give(json!("y̆"), "explode", json!([121, 774])); + give( + UNSTABLE, + json!("❤ の"), + "explode", + json!([10084, 32, 12398]), + ); + give(UNSTABLE, json!("y̆"), "explode", json!([121, 774])); - give(json!("❤ の"), "explode | implode", json!("❤ の")); - give(json!("y̆"), "explode | implode", json!("y̆")); + give(UNSTABLE, json!("❤ の"), "explode | implode", json!("❤ の")); + give(UNSTABLE, json!("y̆"), "explode | implode", json!("y̆")); - give(json!([1114112]), "try implode catch -1", json!(-1)); + give( + UNSTABLE, + json!([1114112]), + "try implode catch -1", + json!(-1), + ); } -yields!(first_empty, "[first({}[])]", json!([])); -yields!(first_some, "first(1, 2, 3)", 1); +yields!(first_empty, UNSTABLE, "[first({}[])]", json!([])); +yields!(first_some, UNSTABLE, "first(1, 2, 3)", 1); yields!( format_text, + UNSTABLE, r#"[0, 0 == 0, {}.a, "hello", {}, [] | @text]"#, ["0", "true", "null", "hello", "{}", "[]"] ); yields!( format_json, + UNSTABLE, r#"[0, 0 == 0, {}.a, "hello", {}, [] | @json]"#, ["0", "true", "null", "\"hello\"", "{}", "[]"] ); yields!( format_html, + UNSTABLE, r#""

sneaky

" | @html"#, "<p style='visibility: hidden'>sneaky</p>" ); yields!( format_uri, + UNSTABLE, r#""abc123 ?#+&[]" | @uri"#, "abc123%20%3F%23%2B%26%5B%5D" ); yields!( format_csv, + UNSTABLE, r#"[0, 0 == 0, {}.a, "hello \"quotes\" and, commas"] | @csv"#, r#"0,true,,"hello ""quotes"" and, commas""# ); yields!( format_tsv, + UNSTABLE, r#"[0, 0 == 0, {}.a, "hello \"quotes\" and \n\r\t\\ escapes"] | @tsv"#, "0\ttrue\t\thello \"quotes\" and \\n\\r\\t\\\\ escapes" ); yields!( format_base64, + UNSTABLE, r#""hello cruel world" | @base64"#, "aGVsbG8gY3J1ZWwgd29ybGQ=" ); yields!( format_unformat_base64, + UNSTABLE, r#""hello cruel world" | @base64 | @base64d"#, "hello cruel world" ); yields!( format_sh, + UNSTABLE, r#"[0, 0 == 0, {}.a, "O'Hara!", ["Here", "there"] | @sh]"#, ["0", "true", "null", r#"'O'\''Hara!'"#, r#"'Here' 'there'"#,] ); yields!( format_sh_rejects_objects, + UNSTABLE, r#"{a: "b"} | try @sh catch -1"#, -1 ); yields!( format_sh_rejects_nested_arrays, + UNSTABLE, r#"["fine, but", []] | try @sh catch -1"#, -1 ); #[test] fn group_by() { - gives(json!([]), "group_by(.)", [json!([])]); + gives(UNSTABLE, json!([]), "group_by(.)", [json!([])]); gives( + UNSTABLE, json!([{"key":1, "value": "foo"},{"key":2, "value":"bar"},{"key":1,"value":"baz"}]), "group_by(.key)", [json!([[{"key":1,"value":"foo"}, {"key":1,"value":"baz"}],[{"key":2,"value":"bar"}]])], @@ -140,105 +188,141 @@ fn group_by() { #[test] fn has() { let err = Error::Index(Val::Null, Val::Int(0)); - fail(json!(null), "has(0)", err); + fail(UNSTABLE, json!(null), "has(0)", err); let err = Error::Index(Val::Int(0), Val::Null); - fail(json!(0), "has([][0])", err); + fail(UNSTABLE, json!(0), "has([][0])", err); let err = Error::Index(Val::Int(0), Val::Int(1)); - fail(json!(0), "has(1)", err); + fail(UNSTABLE, json!(0), "has(1)", err); let err = Error::Index(Val::Str("a".to_string().into()), Val::Int(0)); - fail(json!("a"), "has(0)", err); + fail(UNSTABLE, json!("a"), "has(0)", err); - give(json!([0, null]), "has(0)", json!(true)); - give(json!([0, null]), "has(1)", json!(true)); - give(json!([0, null]), "has(2)", json!(false)); + give(UNSTABLE, json!([0, null]), "has(0)", json!(true)); + give(UNSTABLE, json!([0, null]), "has(1)", json!(true)); + give(UNSTABLE, json!([0, null]), "has(2)", json!(false)); - give(json!({"a": 1, "b": null}), r#"has("a")"#, json!(true)); - give(json!({"a": 1, "b": null}), r#"has("b")"#, json!(true)); - give(json!({"a": 1, "b": null}), r#"has("c")"#, json!(false)); + give( + UNSTABLE, + json!({"a": 1, "b": null}), + r#"has("a")"#, + json!(true), + ); + give( + UNSTABLE, + json!({"a": 1, "b": null}), + r#"has("b")"#, + json!(true), + ); + give( + UNSTABLE, + json!({"a": 1, "b": null}), + r#"has("c")"#, + json!(false), + ); } #[test] fn json() { // TODO: correct this - give(json!(1.0), "tojson", json!("1.0")); - give(json!(0), "1.0 | tojson", json!("1.0")); - give(json!(0), "1.1 | tojson", json!("1.1")); - give(json!(0), "0.0 / 0.0 | tojson", json!("null")); - give(json!(0), "1.0 / 0.0 | tojson", json!("null")); + give(UNSTABLE, json!(1.0), "tojson", json!("1.0")); + give(UNSTABLE, json!(0), "1.0 | tojson", json!("1.0")); + give(UNSTABLE, json!(0), "1.1 | tojson", json!("1.1")); + give(UNSTABLE, json!(0), "0.0 / 0.0 | tojson", json!("null")); + give(UNSTABLE, json!(0), "1.0 / 0.0 | tojson", json!("null")); } #[test] fn keys_unsorted() { - give(json!([0, null, "a"]), "keys_unsorted", json!([0, 1, 2])); - give(json!({"a": 1, "b": 2}), "keys_unsorted", json!(["a", "b"])); + give( + UNSTABLE, + json!([0, null, "a"]), + "keys_unsorted", + json!([0, 1, 2]), + ); + give( + UNSTABLE, + json!({"a": 1, "b": 2}), + "keys_unsorted", + json!(["a", "b"]), + ); let err = |v| Error::Type(v, Type::Iter); - fail(json!(0), "keys_unsorted", err(Val::Int(0))); - fail(json!(null), "keys_unsorted", err(Val::Null)); + fail(UNSTABLE, json!(0), "keys_unsorted", err(Val::Int(0))); + fail(UNSTABLE, json!(null), "keys_unsorted", err(Val::Null)); } #[test] fn length() { - give(json!("ƒoo"), "length", json!(3)); - give(json!("नमस्ते"), "length", json!(6)); - give(json!({"a": 5, "b": 3}), "length", json!(2)); - give(json!(2), "length", json!(2)); - give(json!(-2), "length", json!(2)); - give(json!(2.5), "length", json!(2.5)); - give(json!(-2.5), "length", json!(2.5)); + give(UNSTABLE, json!("ƒoo"), "length", json!(3)); + give(UNSTABLE, json!("नमस्ते"), "length", json!(6)); + give(UNSTABLE, json!({"a": 5, "b": 3}), "length", json!(2)); + give(UNSTABLE, json!(2), "length", json!(2)); + give(UNSTABLE, json!(-2), "length", json!(2)); + give(UNSTABLE, json!(2.5), "length", json!(2.5)); + give(UNSTABLE, json!(-2.5), "length", json!(2.5)); } #[test] fn utf8bytelength() { - give(json!("foo"), "utf8bytelength", json!(3)); - give(json!("ƒoo"), "utf8bytelength", json!(4)); - give(json!("नमस्ते"), "utf8bytelength", json!(18)); + give(UNSTABLE, json!("foo"), "utf8bytelength", json!(3)); + give(UNSTABLE, json!("ƒoo"), "utf8bytelength", json!(4)); + give(UNSTABLE, json!("नमस्ते"), "utf8bytelength", json!(18)); } #[test] fn limit() { // a big WTF: jq outputs "1" here! that looks like another bug ... - gives(json!(null), "limit(0; 1,2)", []); - give(json!(null), "[limit(1, 0, 3; 0, 1)]", json!([0, 0, 1])); + gives(UNSTABLE, json!(null), "limit(0; 1,2)", []); + give( + UNSTABLE, + json!(null), + "[limit(1, 0, 3; 0, 1)]", + json!([0, 0, 1]), + ); // here, jaq diverges from jq, which returns `[0, 1]` - give(json!(null), "[limit(-1; 0, 1)]", json!([])); + give(UNSTABLE, json!(null), "[limit(-1; 0, 1)]", json!([])); } -yields!(min_empty, "[] | min_by(.)", json!(null)); +yields!(min_empty, UNSTABLE, "[] | min_by(.)", json!(null)); // when output is equal, min_by selects the left element and max_by the right one yields!( min_max_eq, + UNSTABLE, "[{a: 1, b: 3}, {a: 1, b: 2}] | [(min_by(.a), max_by(.a)) | .b]", [3, 2] ); // multiple-output functions can be used to differentiate elements yields!( max_mult, + UNSTABLE, "[{a: 1, b: 3}, {a: 1, b: 2}] | max_by(.a, .b) | .b", 3 ); yields!( math_0_argument_scalar_filters, + UNSTABLE, "[-2.2, -1.1, 0, 1.1, 2.2 | sin as $s | cos as $c | $s * $s + $c * $c]", [1.0, 1.0, 1.0, 1.0, 1.0] ); yields!( math_0_argument_vector_filters, + UNSTABLE, "[3, 3.25, 3.5 | modf]", [[0.0, 3.0], [0.25, 3.0], [0.5, 3.0]] ); yields!( math_2_argument_filters, + UNSTABLE, "[pow(0.25, 4, 9; 1, 0.5, 2)]", [0.25, 0.5, 0.0625, 4.0, 2.0, 16.0, 9.0, 3.0, 81.0] ); yields!( math_3_argument_filters, + UNSTABLE, "[fma(2, 1; 3, 4; 4, 5)]", [10.0, 11.0, 12.0, 13.0, 7.0, 8.0, 8.0, 9.0] ); @@ -250,7 +334,7 @@ fn regex() { let f = |f, re, flags| format!("{f}(\"{re}\"; \"{flags}\")"); let out = json!(["", ", ", " and ", ""]); - give(json!(s), &f("split_", date, "g"), out); + give(UNSTABLE, json!(s), &f("split_", date, "g"), out); let c = |o: usize, s: &str| { json!({ @@ -263,39 +347,50 @@ fn regex() { let d2 = json!([c(12, "2013-01-01"), c(12, "2013"), c(17, "01"), c(20, "01")]); let d3 = json!([c(27, "2014-07-05"), c(27, "2014"), c(32, "07"), c(35, "05")]); - give(json!(s), &f("matches", date, "g"), json!([d1, d2, d3])); + give( + UNSTABLE, + json!(s), + &f("matches", date, "g"), + json!([d1, d2, d3]), + ); let out = json!(["", d1, ", ", d2, " and ", d3, ""]); - give(json!(s), &f("split_matches", date, "g"), out); + give(UNSTABLE, json!(s), &f("split_matches", date, "g"), out); let out = json!(["", d1, ", 2013-01-01 and 2014-07-05"]); - give(json!(s), &f("split_matches", date, ""), out); + give(UNSTABLE, json!(s), &f("split_matches", date, ""), out); } #[test] fn round() { - give(json!(1), "round", json!(1)); - give(json!(1.0), "round", json!(1)); - give(json!(-1.0), "round", json!(-1)); - give(json!(-1), "round", json!(-1)); + give(UNSTABLE, json!(1), "round", json!(1)); + give(UNSTABLE, json!(1.0), "round", json!(1)); + give(UNSTABLE, json!(-1.0), "round", json!(-1)); + give(UNSTABLE, json!(-1), "round", json!(-1)); - give(json!(-1.5), "round", json!(-2)); - give(json!(-1.5), "floor", json!(-2)); - give(json!(-1.5), "ceil", json!(-1)); + give(UNSTABLE, json!(-1.5), "round", json!(-2)); + give(UNSTABLE, json!(-1.5), "floor", json!(-2)); + give(UNSTABLE, json!(-1.5), "ceil", json!(-1)); - give(json!(-1.4), "round", json!(-1)); - give(json!(-1.4), "floor", json!(-2)); - give(json!(-1.4), "ceil", json!(-1)); + give(UNSTABLE, json!(-1.4), "round", json!(-1)); + give(UNSTABLE, json!(-1.4), "floor", json!(-2)); + give(UNSTABLE, json!(-1.4), "ceil", json!(-1)); let err = |v| Error::Type(Val::from(v), Type::Num); - fail(json!([]), "round", err(json!([]))); - fail(json!({}), "round", err(json!({}))); + fail(UNSTABLE, json!([]), "round", err(json!([]))); + fail(UNSTABLE, json!({}), "round", err(json!({}))); } #[test] fn split() { - give(json!("aöß"), r#"split("")"#, json!(["a", "ö", "ß"])); give( + UNSTABLE, + json!("aöß"), + r#"split("")"#, + json!(["a", "ö", "ß"]), + ); + give( + UNSTABLE, json!("abcabcdab"), r#"split("ab")"#, json!(["", "c", "cd", ""]), @@ -304,32 +399,87 @@ fn split() { #[test] fn startswith() { - give(json!("foobar"), r#"startswith("")"#, json!(true)); - give(json!("foobar"), r#"startswith("bar")"#, json!(false)); - give(json!("foobar"), r#"startswith("foo")"#, json!(true)); - give(json!(""), r#"startswith("foo")"#, json!(false)); + give(UNSTABLE, json!("foobar"), r#"startswith("")"#, json!(true)); + give( + UNSTABLE, + json!("foobar"), + r#"startswith("bar")"#, + json!(false), + ); + give( + UNSTABLE, + json!("foobar"), + r#"startswith("foo")"#, + json!(true), + ); + give(UNSTABLE, json!(""), r#"startswith("foo")"#, json!(false)); } #[test] fn endswith() { - give(json!("foobar"), r#"endswith("")"#, json!(true)); - give(json!("foobar"), r#"endswith("foo")"#, json!(false)); - give(json!("foobar"), r#"endswith("bar")"#, json!(true)); - give(json!(""), r#"endswith("foo")"#, json!(false)); + give(UNSTABLE, json!("foobar"), r#"endswith("")"#, json!(true)); + give( + UNSTABLE, + json!("foobar"), + r#"endswith("foo")"#, + json!(false), + ); + give(UNSTABLE, json!("foobar"), r#"endswith("bar")"#, json!(true)); + give(UNSTABLE, json!(""), r#"endswith("foo")"#, json!(false)); } #[test] fn ltrimstr() { - give(json!("foobar"), r#"ltrimstr("")"#, json!("foobar")); - give(json!("foobar"), r#"ltrimstr("foo")"#, json!("bar")); - give(json!("foobar"), r#"ltrimstr("bar")"#, json!("foobar")); - give(json!("اَلْعَرَبِيَّةُ"), r#"ltrimstr("ا")"#, json!("َلْعَرَبِيَّةُ")); + give( + UNSTABLE, + json!("foobar"), + r#"ltrimstr("")"#, + json!("foobar"), + ); + give( + UNSTABLE, + json!("foobar"), + r#"ltrimstr("foo")"#, + json!("bar"), + ); + give( + UNSTABLE, + json!("foobar"), + r#"ltrimstr("bar")"#, + json!("foobar"), + ); + give( + UNSTABLE, + json!("اَلْعَرَبِيَّةُ"), + r#"ltrimstr("ا")"#, + json!("َلْعَرَبِيَّةُ"), + ); } #[test] fn rtrimstr() { - give(json!("foobar"), r#"rtrimstr("")"#, json!("foobar")); - give(json!("foobar"), r#"rtrimstr("bar")"#, json!("foo")); - give(json!("foobar"), r#"rtrimstr("foo")"#, json!("foobar")); - give(json!("اَلْعَرَبِيَّةُ"), r#"rtrimstr("ا")"#, json!("اَلْعَرَبِيَّةُ")); + give( + UNSTABLE, + json!("foobar"), + r#"rtrimstr("")"#, + json!("foobar"), + ); + give( + UNSTABLE, + json!("foobar"), + r#"rtrimstr("bar")"#, + json!("foo"), + ); + give( + UNSTABLE, + json!("foobar"), + r#"rtrimstr("foo")"#, + json!("foobar"), + ); + give( + UNSTABLE, + json!("اَلْعَرَبِيَّةُ"), + r#"rtrimstr("ا")"#, + json!("اَلْعَرَبِيَّةُ"), + ); } diff --git a/jaq-interpret/Cargo.toml b/jaq-interpret/Cargo.toml index e0fceb4f..4bf134d7 100644 --- a/jaq-interpret/Cargo.toml +++ b/jaq-interpret/Cargo.toml @@ -13,16 +13,20 @@ rust-version = "1.63" [features] default = ["std", "hifijson", "serde_json"] -std = [] +unstable = ["unstable-flag", "jaq-syn/unstable"] +unstable-flag = ["jaq-syn/unstable-flag"] +std = ["either/use_std"] [dependencies] jaq-syn = { version = "1.1.0", path = "../jaq-syn" } ahash = "0.8.6" dyn-clone = "1.0" +either = { version = "1.10.0", default-features = false } hifijson = { version = "0.2.0", optional = true } indexmap = "2.0" once_cell = "1.16.0" serde_json = { version = "1.0.81", optional = true } [dev-dependencies] +chumsky = { version = "0.9.0", default-features = false } jaq-parse = { version = "1.0.0", path = "../jaq-parse" } diff --git a/jaq-interpret/src/filter.rs b/jaq-interpret/src/filter.rs index 3700c390..bc5b17a4 100644 --- a/jaq-interpret/src/filter.rs +++ b/jaq-interpret/src/filter.rs @@ -54,21 +54,34 @@ impl Owned { /// Function from a value to a stream of value results. #[derive(Clone, Debug, Default)] pub(crate) enum Ast { + /// Nullary identity operation (`.`) #[default] Id, ToString, + /// Integer value literal Int(isize), + /// Floating point value literal Float(f64), + /// String value literal Str(String), + /// Array value literal (`[f]`) Array(Id), + /// Object value literal (`{}`, `{(f): g, …}`) Object(Box<[(Id, Id)]>), + /// Try-catch (`try f catch g`) Try(Id, Id), + /// Unary negation operation (`-f`) Neg(Id), + /// Binary binding operation (`f as $VAR | g`) if identifier (`VAR`) is + /// given, otherwise binary application operation (`f | g`) Pipe(Id, bool, Id), + /// Binary concatenation operation (`f, g`) Comma(Id, Id), + /// Binary alternation operation (`f // g`) Alt(Id, Id), + /// If-then-else (`if f then g else h end`) Ite(Id, Id, Id), /// `reduce`, `for`, and `foreach` /// @@ -95,17 +108,31 @@ pub(crate) enum Ast { /// ~~~ Fold(FoldType, Id, Id, Id), + /// Path Path(Id, crate::path::Path), + /// Assignment operation (`f = g`) + Assign(Id, Id), + /// Update-assignment operation (`f |= g`) Update(Id, Id), + /// Alternation update-assignment operation (`f //= g`) + AltUpdate(Id, Id), + /// Arithmetical update-assignment operation (`f += g`, `f -= g`, `f *= g`, + /// `f /= g`, `f %= g`, …) UpdateMath(Id, MathOp, Id), - Assign(Id, Id), + /// Binary logical operation (`f and g`, `f or g`) Logic(Id, bool, Id), + /// Binary arithmetical operation (`f + g`, `f - g`, `f * g`, `f / g`, + /// `f % g`, …) Math(Id, MathOp, Id), + /// Binary comparative operation (`f < g`, `f <= g`, `f > g`, `f >= g`, + /// `f == g`, `f != g`, …) Ord(Id, OrdOp, Id), + /// Bound variable reference (`$x`) Var(usize), + /// Call to a filter (`filter`, `filter(…)`) Call(Call), Native(Native, Box<[Id]>), @@ -134,6 +161,8 @@ where |y, (ctx, cv, args)| then(y, |y| bind_vars(args, ctx.cons_var(y), cv)), ), Some(Bind::Fun(Ref(arg, _defs))) => bind_vars(args, ctx.cons_fun((arg, cv.0.clone())), cv), + #[cfg(feature = "unstable-flag")] + Some(_) => unimplemented!(), None => box_once(Ok((ctx, cv.1))), } } @@ -290,16 +319,23 @@ impl<'a> FilterT<'a> for Ref<'a> { }) } + Ast::Assign(path, f) => w(f).pipe(cv, move |cv, y| { + w(path).update(cv, Box::new(move |_| box_once(Ok(y.clone())))) + }), Ast::Update(path, f) => w(path).update( (cv.0.clone(), cv.1), Box::new(move |v| w(f).run((cv.0.clone(), v))), ), + #[cfg(feature = "unstable-flag")] + Ast::AltUpdate(path, f) => w(f).pipe(cv, move |cv, y| { + w(path).update( + cv, + Box::new(move |x| box_once(Ok(if x.as_bool() { x } else { y.clone() }))), + ) + }), Ast::UpdateMath(path, op, f) => w(f).pipe(cv, move |cv, y| { w(path).update(cv, Box::new(move |x| box_once(op.run(x, y.clone())))) }), - Ast::Assign(path, f) => w(f).pipe(cv, move |cv, y| { - w(path).update(cv, Box::new(move |_| box_once(Ok(y.clone())))) - }), Ast::Logic(l, stop, r) => w(l).pipe(cv, move |cv, l| { if l.as_bool() == *stop { box_once(Ok(Val::Bool(*stop))) @@ -325,12 +361,16 @@ impl<'a> FilterT<'a> for Ref<'a> { FoldType::Foreach => flat_map_with(init, xs, move |i, xs| { then(i, |i| Box::new(fold(true, xs, Input(i), f.clone()))) }), + #[cfg(feature = "unstable-flag")] + _ => unimplemented!(), } } Ast::Var(v) => match cv.0.vars.get(*v).unwrap() { Bind::Var(v) => box_once(Ok(v.clone())), Bind::Fun(f) => w(&f.0).run((cv.0.with_vars(f.1.clone()), cv.1)), + #[cfg(feature = "unstable-flag")] + _ => unimplemented!(), }, Ast::Call(call) => { let def = w(&call.id); @@ -366,11 +406,12 @@ impl<'a> FilterT<'a> for Ref<'a> { Ast::Int(_) | Ast::Float(_) | Ast::Str(_) => err, Ast::Array(_) | Ast::Object(_) => err, Ast::Neg(_) | Ast::Logic(..) | Ast::Math(..) | Ast::Ord(..) => err, - Ast::Update(..) | Ast::UpdateMath(..) | Ast::Assign(..) => err, + Ast::Assign(..) | Ast::Update(..) | Ast::UpdateMath(..) => err, + #[cfg(feature = "unstable-flag")] + Ast::AltUpdate(..) => err, - // these are up for grabs to implement :) - Ast::Try(..) | Ast::Alt(..) => todo!(), - Ast::Fold(..) => todo!(), + Ast::Try(..) | Ast::Alt(..) => todo!("these are up for grabs to implement :)"), + Ast::Fold(..) => todo!("these are up for grabs to implement :)"), Ast::Id => f(cv.1), Ast::Path(l, path) => { @@ -404,6 +445,8 @@ impl<'a> FilterT<'a> for Ref<'a> { Ast::Var(v) => match cv.0.vars.get(*v).unwrap() { Bind::Var(_) => err, Bind::Fun(l) => w(&l.0).update((cv.0.with_vars(l.1.clone()), cv.1), f), + #[cfg(feature = "unstable-flag")] + _ => unimplemented!(), }, Ast::Call(call) => { let def = w(&call.id); diff --git a/jaq-interpret/src/hir.rs b/jaq-interpret/src/hir.rs index d8598745..e1831799 100644 --- a/jaq-interpret/src/hir.rs +++ b/jaq-interpret/src/hir.rs @@ -55,6 +55,8 @@ impl fmt::Display for Error { Self::Undefined(Bind::Fun(_)) => "undefined filter", Self::Num(Num::Float(_)) => "cannot interpret as floating-point number", Self::Num(Num::Int(_)) => "cannot interpret as machine-size integer", + #[cfg(feature = "unstable-flag")] + _ => unimplemented!(), } .fmt(f) } @@ -126,7 +128,7 @@ impl Ctx { .drain(self.callable.len() - defs.len()..) .for_each(|callable| assert_eq!(callable.typ, Relative::Sibling)); - jaq_syn::Main { defs, body } + jaq_syn::Main::new(defs, body) } pub fn def(&mut self, def: jaq_syn::Def) -> Def { @@ -136,7 +138,7 @@ impl Ctx { }); let rhs = self.main(def.rhs); self.callable.last_mut().unwrap().typ = Relative::Sibling; - jaq_syn::Def { lhs: def.lhs, rhs } + jaq_syn::Def::new(def.lhs, rhs) } fn expr(&mut self, f: Spanned) -> Spanned { @@ -168,12 +170,12 @@ impl Ctx { assert!(self.vars.pop().as_ref() == Some(&x)); Expr::Binary(l, BinaryOp::Pipe(Some(x)), r) } - Expr::Fold(typ, Fold { xs, x, init, f }) => { + Expr::Fold(typ, Fold { xs, x, init, f, .. }) => { let (xs, init) = (get(self, *xs), get(self, *init)); self.vars.push(x.clone()); let f = get(self, *f); assert!(self.vars.pop().as_ref() == Some(&x)); - Expr::Fold(typ, Fold { xs, x, init, f }) + Expr::Fold(typ, Fold::new(xs, x, init, f)) } Expr::Id => Expr::Id, Expr::Num(n) => Expr::Num(Num::parse(&n).unwrap_or_else(|n| { @@ -206,6 +208,8 @@ impl Ctx { .map(|(p, opt)| (p.map(|p| self.expr(p)), opt)); Expr::Path(f, path.collect()) } + #[cfg(feature = "unstable-flag")] + _ => unimplemented!(), }; (result, f.1) } diff --git a/jaq-interpret/src/into_iter.rs b/jaq-interpret/src/into_iter.rs index 0953df90..3cd1fe8c 100644 --- a/jaq-interpret/src/into_iter.rs +++ b/jaq-interpret/src/into_iter.rs @@ -12,25 +12,22 @@ impl I> IntoIterator for Delay { } #[derive(Clone)] -pub enum Either { - L(L), - R(R), -} +pub struct Either(pub either::Either); -pub struct EitherIter(Either); +pub struct EitherIter(either::Either); impl> Iterator for EitherIter { type Item = L::Item; fn next(&mut self) -> Option { match &mut self.0 { - Either::L(l) => l.next(), - Either::R(r) => r.next(), + either::Left(l) => l.next(), + either::Right(r) => r.next(), } } fn size_hint(&self) -> (usize, Option) { match &self.0 { - Either::L(l) => l.size_hint(), - Either::R(r) => r.size_hint(), + either::Left(l) => l.size_hint(), + either::Right(r) => r.size_hint(), } } } @@ -39,10 +36,7 @@ impl> IntoIterator for Either; fn into_iter(self) -> Self::IntoIter { - EitherIter(match self { - Self::L(l) => Either::L(l.into_iter()), - Self::R(r) => Either::R(r.into_iter()), - }) + EitherIter(self.0.into_iter()) } } @@ -53,8 +47,8 @@ pub fn collect_if_once I + Clone>( if iter.size_hint().1 == Some(1) { if let Some(x) = iter.next() { assert!(iter.next().is_none()); - return Either::L(core::iter::once(x)); + return Either(either::Left(core::iter::once(x))); } } - Either::R(Delay(f)) + Either(either::Right(Delay(f))) } diff --git a/jaq-interpret/src/lib.rs b/jaq-interpret/src/lib.rs index 02d82e30..0ad95680 100644 --- a/jaq-interpret/src/lib.rs +++ b/jaq-interpret/src/lib.rs @@ -22,7 +22,11 @@ //! let mut defs = ParseCtx::new(Vec::new()); //! //! // parse the filter -//! let (f, errs) = jaq_parse::parse(filter, jaq_parse::main()); +//! let (f, errs) = jaq_parse::parse( +//! #[cfg(feature = "unstable-flag")] false, +//! filter, +//! jaq_parse::main(#[cfg(feature = "unstable-flag")] false), +//! ); //! assert_eq!(errs, Vec::new()); //! //! // compile the filter in the context of the given definitions @@ -117,8 +121,10 @@ impl<'a> Ctx<'a> { } fn with_vars(&self, vars: Vars) -> Self { - let inputs = self.inputs; - Self { vars, inputs } + Self { + vars, + inputs: self.inputs, + } } /// Return remaining input values. @@ -145,16 +151,10 @@ impl ParseCtx { /// values corresponding to the variables have to be supplied in the execution context. pub fn new(vars: Vec) -> Self { use alloc::string::ToString; - let def = jaq_syn::Def { - lhs: jaq_syn::Call { - name: "$".to_string(), - args: vars.into_iter().map(Bind::Var).collect(), - }, - rhs: jaq_syn::Main { - defs: Vec::new(), - body: (jaq_syn::filter::Filter::Id, 0..0), - }, - }; + let def = jaq_syn::Def::new( + jaq_syn::Call::new("$".to_string(), vars.into_iter().map(Bind::Var).collect()), + jaq_syn::Main::new(Vec::new(), (jaq_syn::filter::Filter::Id, 0..0)), + ); Self { errs: Vec::new(), diff --git a/jaq-interpret/src/lir.rs b/jaq-interpret/src/lir.rs index d4ee88af..45fceca3 100644 --- a/jaq-interpret/src/lir.rs +++ b/jaq-interpret/src/lir.rs @@ -140,6 +140,8 @@ impl Ctx { let iter = s.parts.into_iter().map(|part| match part { Part::Str(s) => Filter::Str(s), Part::Fun(f) => Filter::Pipe(get(f, ctx), false, fmt), + #[cfg(feature = "unstable-flag")] + _ => unimplemented!(), }); let mut iter = iter.collect::>().into_iter().rev(); let last = iter.next(); @@ -203,6 +205,8 @@ impl Ctx { }; (k, v) } + #[cfg(feature = "unstable-flag")] + _ => unimplemented!(), }); Filter::Object(kvs.collect()) } @@ -222,7 +226,12 @@ impl Ctx { BinaryOp::Ord(op) => Filter::Ord(l, op, r), BinaryOp::Assign(AssignOp::Assign) => Filter::Assign(l, r), BinaryOp::Assign(AssignOp::Update) => Filter::Update(l, r), + #[cfg(feature = "unstable-flag")] + #[allow(deprecated)] + BinaryOp::Assign(AssignOp::AltUpdate) => Filter::AltUpdate(l, r), BinaryOp::Assign(AssignOp::UpdateWith(op)) => Filter::UpdateMath(l, op, r), + #[cfg(feature = "unstable-flag")] + _ => unimplemented!(), } } @@ -245,9 +254,13 @@ impl Ctx { let upper = upper.map(|f| get(f, self)); (path::Part::Range(lower, upper), opt) } + #[cfg(feature = "unstable-flag")] + _ => unimplemented!(), }); Filter::Path(f, Path(path.collect())) } + #[cfg(feature = "unstable-flag")] + _ => unimplemented!(), } } } diff --git a/jaq-interpret/src/mir.rs b/jaq-interpret/src/mir.rs index bd635648..6715aebb 100644 --- a/jaq-interpret/src/mir.rs +++ b/jaq-interpret/src/mir.rs @@ -126,11 +126,11 @@ impl Ctx { Expr::Binary(get(self, *l, notr()), op, get(self, *r, notr())) } - Expr::Fold(typ, Fold { xs, x, init, f }) => { + Expr::Fold(typ, Fold { xs, x, init, f, .. }) => { let xs = get(self, *xs, notr()); let init = get(self, *init, notr()); let f = get(self, *f, notr()); - Expr::Fold(typ, Fold { xs, x, init, f }) + Expr::Fold(typ, Fold::new(xs, x, init, f)) } Expr::Id => Expr::Id, Expr::Recurse => Expr::Recurse, @@ -161,6 +161,8 @@ impl Ctx { .map(|(p, opt)| (p.map(|p| self.expr(p, notr())), opt)); Expr::Path(f, path.collect()) } + #[cfg(feature = "unstable-flag")] + _ => unimplemented!(), }; (result, f.1) } diff --git a/jaq-interpret/src/val.rs b/jaq-interpret/src/val.rs index 5e3178fc..015902eb 100644 --- a/jaq-interpret/src/val.rs +++ b/jaq-interpret/src/val.rs @@ -20,6 +20,7 @@ use jaq_syn::MathOp; /// * The sum, difference, product, and remainder of two integers is integer. /// * Any other operation between two numbers yields a float. #[derive(Clone, Debug)] +#[cfg_attr(feature = "unstable-flag", non_exhaustive)] pub enum Val { /// Null Null, diff --git a/jaq-interpret/tests/common/mod.rs b/jaq-interpret/tests/common/mod.rs index 20d0b92a..929fea29 100644 --- a/jaq-interpret/tests/common/mod.rs +++ b/jaq-interpret/tests/common/mod.rs @@ -1,30 +1,71 @@ use serde_json::Value; -fn yields(x: jaq_interpret::Val, f: &str, ys: impl Iterator) { +#[cfg(not(feature = "unstable-flag"))] +pub const UNSTABLE: bool = false; +#[cfg(feature = "unstable-flag")] +pub const UNSTABLE: bool = false; + +#[track_caller] +pub fn parse_fail( + #[cfg_attr(not(feature = "unstable-flag"), allow(unused_variables))] unstable: bool, + f: &str, + errs_f: impl FnOnce(Vec>), +) { + let (_f, errs) = jaq_parse::parse( + #[cfg(feature = "unstable-flag")] + unstable, + f, + jaq_parse::main( + #[cfg(feature = "unstable-flag")] + unstable, + ), + ); + assert!(!errs.is_empty()); + errs_f(errs) +} + +#[track_caller] +fn yields( + #[cfg_attr(not(feature = "unstable-flag"), allow(unused_variables))] unstable: bool, + x: jaq_interpret::Val, + f: &str, + ys: impl Iterator, +) { let mut ctx = jaq_interpret::ParseCtx::new(Vec::new()); - let (f, errs) = jaq_parse::parse(f, jaq_parse::main()); + let (f, errs) = jaq_parse::parse( + #[cfg(feature = "unstable-flag")] + unstable, + f, + jaq_parse::main( + #[cfg(feature = "unstable-flag")] + unstable, + ), + ); assert!(errs.is_empty()); ctx.yields(x, f.unwrap(), ys) } -pub fn fail(x: Value, f: &str, err: jaq_interpret::Error) { - yields(x.into(), f, core::iter::once(Err(err))) +#[track_caller] +pub fn fail(unstable: bool, x: Value, f: &str, err: jaq_interpret::Error) { + yields(unstable, x.into(), f, core::iter::once(Err(err))) } -pub fn give(x: Value, f: &str, y: Value) { - yields(x.into(), f, core::iter::once(Ok(y.into()))) +#[track_caller] +pub fn give(unstable: bool, x: Value, f: &str, y: Value) { + yields(unstable, x.into(), f, core::iter::once(Ok(y.into()))) } -pub fn gives(x: Value, f: &str, ys: [Value; N]) { - yields(x.into(), f, ys.into_iter().map(|y| Ok(y.into()))) +#[track_caller] +pub fn gives(unstable: bool, x: Value, f: &str, ys: [Value; N]) { + yields(unstable, x.into(), f, ys.into_iter().map(|y| Ok(y.into()))) } #[macro_export] macro_rules! yields { - ($func_name:ident, $filter:expr, $output:expr) => { + ($func_name:ident, $unstable:expr, $filter:expr, $output:expr) => { #[test] fn $func_name() { - give(json!(null), $filter, json!($output)) + give($unstable, json!(null), $filter, json!($output)) } }; } diff --git a/jaq-interpret/tests/path.rs b/jaq-interpret/tests/path.rs index 63402d3a..df85b5de 100644 --- a/jaq-interpret/tests/path.rs +++ b/jaq-interpret/tests/path.rs @@ -1,37 +1,43 @@ pub mod common; -use common::{fail, give, gives}; +use common::{fail, give, gives, UNSTABLE}; use jaq_interpret::Error; use serde_json::json; #[test] fn index_access() { - give(json!([0, 1, 2]), ".[-4]", json!(null)); - give(json!([0, 1, 2]), ".[-3]", json!(0)); - give(json!([0, 1, 2]), ".[-1]", json!(2)); - give(json!([0, 1, 2]), ".[ 0]", json!(0)); - give(json!([0, 1, 2]), ".[ 2]", json!(2)); - give(json!([0, 1, 2]), ".[ 3]", json!(null)); - - give(json!({"a": 1}), ".a", json!(1)); - give(json!({"a": 1}), ".a?", json!(1)); - give(json!({"a": 1}), ".a ?", json!(1)); - give(json!({"a": 1}), r#"."a""#, json!(1)); - give(json!({"a": 1}), r#". "a""#, json!(1)); - give(json!({"a": 1}), r#".["a"]"#, json!(1)); - give(json!({"a": 1}), r#". ["a"]"#, json!(1)); - give(json!({"a_": 1}), ".a_", json!(1)); - give(json!({"_a": 1}), "._a", json!(1)); - give(json!({"_0": 1}), "._0", json!(1)); - - give(json!({"a": 1}), r#".[0, "a", 0 == 0]?"#, json!(1)); - give(json!([0, 1, 2]), r#".["a", 0, 0 == 0]?"#, json!(0)); - give(json!([0, 1, 2]), r#".[3]?"#, json!(null)); - gives(json!("asdf"), ".[0]?", []); - - give(json!(1), "[1, 2, 3][.]", json!(2)); + give(UNSTABLE, json!([0, 1, 2]), ".[-4]", json!(null)); + give(UNSTABLE, json!([0, 1, 2]), ".[-3]", json!(0)); + give(UNSTABLE, json!([0, 1, 2]), ".[-1]", json!(2)); + give(UNSTABLE, json!([0, 1, 2]), ".[ 0]", json!(0)); + give(UNSTABLE, json!([0, 1, 2]), ".[ 2]", json!(2)); + give(UNSTABLE, json!([0, 1, 2]), ".[ 3]", json!(null)); + + give(UNSTABLE, json!({"a": 1}), ".a", json!(1)); + give(UNSTABLE, json!({"a": 1}), ".a?", json!(1)); + give(UNSTABLE, json!({"a": 1}), ".a ?", json!(1)); + give(UNSTABLE, json!({"a": 1}), r#"."a""#, json!(1)); + give(UNSTABLE, json!({"a": 1}), r#". "a""#, json!(1)); + give(UNSTABLE, json!({"a": 1}), r#".["a"]"#, json!(1)); + give(UNSTABLE, json!({"a": 1}), r#". ["a"]"#, json!(1)); + give(UNSTABLE, json!({"a_": 1}), ".a_", json!(1)); + give(UNSTABLE, json!({"_a": 1}), "._a", json!(1)); + give(UNSTABLE, json!({"_0": 1}), "._0", json!(1)); + + give(UNSTABLE, json!({"a": 1}), r#".[0, "a", 0 == 0]?"#, json!(1)); + give( + UNSTABLE, + json!([0, 1, 2]), + r#".["a", 0, 0 == 0]?"#, + json!(0), + ); + give(UNSTABLE, json!([0, 1, 2]), r#".[3]?"#, json!(null)); + gives(UNSTABLE, json!("asdf"), ".[0]?", []); + + give(UNSTABLE, json!(1), "[1, 2, 3][.]", json!(2)); gives( + UNSTABLE, json!({"a": 1, "b": 2}), r#".["b", "a"]"#, [json!(2), json!(1)], @@ -40,34 +46,55 @@ fn index_access() { #[test] fn iter_access() { - gives(json!([0, 1, 2]), ".[]", [json!(0), json!(1), json!(2)]); - gives(json!({"a": [1, 2]}), ".a[]", [json!(1), json!(2)]); - gives(json!({"a": [1, 2]}), ".a.[]", [json!(1), json!(2)]); - gives(json!({"a": 1, "b": 2}), ".[]", [json!(1), json!(2)]); + gives( + UNSTABLE, + json!([0, 1, 2]), + ".[]", + [json!(0), json!(1), json!(2)], + ); + gives(UNSTABLE, json!({"a": [1, 2]}), ".a[]", [json!(1), json!(2)]); + gives( + UNSTABLE, + json!({"a": [1, 2]}), + ".a.[]", + [json!(1), json!(2)], + ); + gives( + UNSTABLE, + json!({"a": 1, "b": 2}), + ".[]", + [json!(1), json!(2)], + ); // TODO: correct this - //gives(json!({"b": 2, "a": 1}), ".[]", [json!(2), json!(1)]); - gives(json!("asdf"), ".[]?", []); + //gives( + // UNSTABLE, + // json!({"b": 2, "a": 1}), + // ".[]", + // [json!(2), json!(1)], + //); + gives(UNSTABLE, json!("asdf"), ".[]?", []); } #[test] fn range_access() { - give(json!("Möwe"), ".[1:-1]", json!("öw")); - give(json!("नमस्ते"), ".[1:5]", json!("मस्त")); + give(UNSTABLE, json!("Möwe"), ".[1:-1]", json!("öw")); + give(UNSTABLE, json!("नमस्ते"), ".[1:5]", json!("मस्त")); - give(json!([0, 1, 2]), ".[-4:4]", json!([0, 1, 2])); - give(json!([0, 1, 2]), ".[0:3]", json!([0, 1, 2])); - give(json!([0, 1, 2]), ".[1:]", json!([1, 2])); - give(json!([0, 1, 2]), ".[:-1]", json!([0, 1])); - give(json!([0, 1, 2]), ".[1:0]", json!([])); - give(json!([0, 1, 2]), ".[4:5]", json!([])); + give(UNSTABLE, json!([0, 1, 2]), ".[-4:4]", json!([0, 1, 2])); + give(UNSTABLE, json!([0, 1, 2]), ".[0:3]", json!([0, 1, 2])); + give(UNSTABLE, json!([0, 1, 2]), ".[1:]", json!([1, 2])); + give(UNSTABLE, json!([0, 1, 2]), ".[:-1]", json!([0, 1])); + give(UNSTABLE, json!([0, 1, 2]), ".[1:0]", json!([])); + give(UNSTABLE, json!([0, 1, 2]), ".[4:5]", json!([])); - give(json!([0, 1, 2]), ".[0:2,3.14]?", json!([0, 1])); + give(UNSTABLE, json!([0, 1, 2]), ".[0:2,3.14]?", json!([0, 1])); } #[test] fn iter_assign() { - give(json!([1, 2]), ".[] = .", json!([[1, 2], [1, 2]])); + give(UNSTABLE, json!([1, 2]), ".[] = .", json!([[1, 2], [1, 2]])); give( + UNSTABLE, json!({"a": [1,2], "b": 3}), ".a[] = .b+.b", json!({"a": [6,6], "b": 3}), @@ -76,61 +103,119 @@ fn iter_assign() { #[test] fn index_update() { - give(json!({"a": 1}), ".b |= .", json!({"a": 1, "b": null})); - give(json!({"a": 1}), ".b |= 1", json!({"a": 1, "b": 1})); - give(json!({"a": 1}), ".b |= .+1", json!({"a": 1, "b": 1})); - give(json!({"a": 1, "b": 2}), ".b |= {}[]", json!({"a": 1})); - give(json!({"a": 1, "b": 2}), ".a += 1", json!({"a": 2, "b": 2})); + give( + UNSTABLE, + json!({"a": 1}), + ".b |= .", + json!({"a": 1, "b": null}), + ); + give( + UNSTABLE, + json!({"a": 1}), + ".b |= 1", + json!({"a": 1, "b": 1}), + ); + give( + UNSTABLE, + json!({"a": 1}), + ".b |= .+1", + json!({"a": 1, "b": 1}), + ); + give( + UNSTABLE, + json!({"a": 1, "b": 2}), + ".b |= {}[]", + json!({"a": 1}), + ); + give( + UNSTABLE, + json!({"a": 1, "b": 2}), + ".a += 1", + json!({"a": 2, "b": 2}), + ); - give(json!([0, 1, 2]), ".[1] |= .+2", json!([0, 3, 2])); - give(json!([0, 1, 2]), ".[-1,-1] |= {}[]", json!([0])); - give(json!([0, 1, 2]), ".[ 0, 0] |= {}[]", json!([2])); + give(UNSTABLE, json!([0, 1, 2]), ".[1] |= .+2", json!([0, 3, 2])); + give(UNSTABLE, json!([0, 1, 2]), ".[-1,-1] |= {}[]", json!([0])); + give(UNSTABLE, json!([0, 1, 2]), ".[ 0, 0] |= {}[]", json!([2])); use Error::IndexOutOfBounds as Oob; - fail(json!([0, 1, 2]), ".[ 3] |= 3", Oob(3)); - fail(json!([0, 1, 2]), ".[-4] |= -1", Oob(-4)); + fail(UNSTABLE, json!([0, 1, 2]), ".[ 3] |= 3", Oob(3)); + fail(UNSTABLE, json!([0, 1, 2]), ".[-4] |= -1", Oob(-4)); - give(json!({"a": 1}), r#".[0, "a"]? |= .+1"#, json!({"a": 2})); - give(json!([0, 1, 2]), r#".["a", 0]? |= .+1"#, json!([1, 1, 2])); - give(json!([0, 1, 2]), r#".[3]? |= .+1"#, json!([0, 1, 2])); - give(json!("asdf"), ".[0]? |= .+1", json!("asdf")); + give( + UNSTABLE, + json!({"a": 1}), + r#".[0, "a"]? |= .+1"#, + json!({"a": 2}), + ); + give( + UNSTABLE, + json!([0, 1, 2]), + r#".["a", 0]? |= .+1"#, + json!([1, 1, 2]), + ); + give( + UNSTABLE, + json!([0, 1, 2]), + r#".[3]? |= .+1"#, + json!([0, 1, 2]), + ); + give(UNSTABLE, json!("asdf"), ".[0]? |= .+1", json!("asdf")); } #[test] fn iter_update() { // precedence tests - give(json!([]), ".[] |= . or 0", json!([])); - gives(json!([]), ".[] |= .,.", [json!([]), json!([])]); - give(json!([]), ".[] |= (.,.)", json!([])); - give(json!([0]), ".[] |= .+1 | .+[2]", json!([1, 2])); + give(UNSTABLE, json!([]), ".[] |= . or 0", json!([])); + gives(UNSTABLE, json!([]), ".[] |= .,.", [json!([]), json!([])]); + give(UNSTABLE, json!([]), ".[] |= (.,.)", json!([])); + give(UNSTABLE, json!([0]), ".[] |= .+1 | .+[2]", json!([1, 2])); // this yields a syntax error in jq, but it is consistent to permit this - give(json!([[1]]), ".[] |= .[] |= .+1", json!([[2]])); + give(UNSTABLE, json!([[1]]), ".[] |= .[] |= .+1", json!([[2]])); // ditto - give(json!([[1]]), ".[] |= .[] += 1", json!([[2]])); + give(UNSTABLE, json!([[1]]), ".[] |= .[] += 1", json!([[2]])); - give(json!([1]), ".[] |= .+1", json!([2])); - give(json!([[1]]), ".[][] |= .+1", json!([[2]])); + give(UNSTABLE, json!([1]), ".[] |= .+1", json!([2])); + give(UNSTABLE, json!([[1]]), ".[][] |= .+1", json!([[2]])); give( + UNSTABLE, json!({"a": 1, "b": 2}), ".[] |= ((if .>1 then . else {}[] end) | .+1)", json!({"b": 3}), ); - give(json!([[0, 1], "a"]), ".[][]? |= .+1", json!([[1, 2], "a"])); + give( + UNSTABLE, + json!([[0, 1], "a"]), + ".[][]? |= .+1", + json!([[1, 2], "a"]), + ); } #[test] fn range_update() { - give(json!([0, 1, 2]), ".[:2] |= [.[] | .+5]", json!([5, 6, 2])); - give(json!([0, 1, 2]), ".[-2:-1] |= [5]+.", json!([0, 5, 1, 2])); give( + UNSTABLE, + json!([0, 1, 2]), + ".[:2] |= [.[] | .+5]", + json!([5, 6, 2]), + ); + give( + UNSTABLE, + json!([0, 1, 2]), + ".[-2:-1] |= [5]+.", + json!([0, 5, 1, 2]), + ); + give( + UNSTABLE, json!([0, 1, 2]), ".[-2:-1,-1] |= [5,6]+.", json!([0, 5, 6, 5, 6, 1, 2]), ); give( + UNSTABLE, json!([0, 1, 2]), ".[:2,3.0]? |= [.[] | .+1]", json!([1, 2, 2]), @@ -145,19 +230,24 @@ fn range_update() { #[test] fn update_mult() { // first the cases where jaq and jq agree - give(json!({"a": 1}), ".a |= (.,.+1)", json!({"a": 1})); + give(UNSTABLE, json!({"a": 1}), ".a |= (.,.+1)", json!({"a": 1})); // jq returns null here - gives(json!(1), ". |= {}[]", []); + gives(UNSTABLE, json!(1), ". |= {}[]", []); // jq returns just 1 here - gives(json!(1), ". |= (.,.)", [json!(1), json!(1)]); + gives(UNSTABLE, json!(1), ". |= (.,.)", [json!(1), json!(1)]); // jq returns just [1] here - give(json!([1]), ".[] |= (., .+1)", json!([1, 2])); + give(UNSTABLE, json!([1]), ".[] |= (., .+1)", json!([1, 2])); // jq returns just [1,2] here - give(json!([1, 3]), ".[] |= (., .+1)", json!([1, 2, 3, 4])); + give( + UNSTABLE, + json!([1, 3]), + ".[] |= (., .+1)", + json!([1, 2, 3, 4]), + ); // here comes a huge WTF: jq returns [2,4] here // this is a known bug: - give(json!([1, 2, 3, 4, 5]), ".[] |= {}[]", json!([])); + give(UNSTABLE, json!([1, 2, 3, 4, 5]), ".[] |= {}[]", json!([])); } #[test] @@ -166,5 +256,5 @@ fn update_complex() { // in general, `a | a |= .` // works in jq when `a` is either null, a number, or a boolean --- it // does *not* work when `a` is a string, an array, or an object! - fail(json!(0), "0 |= .+1", Error::PathExp); + fail(UNSTABLE, json!(0), "0 |= .+1", Error::PathExp); } diff --git a/jaq-interpret/tests/tests.rs b/jaq-interpret/tests/tests.rs index 462cf5c4..2effece0 100644 --- a/jaq-interpret/tests/tests.rs +++ b/jaq-interpret/tests/tests.rs @@ -2,59 +2,104 @@ pub mod common; -use common::{give, gives}; +use chumsky::error::{Simple, SimpleReason}; +use common::{give, gives, parse_fail, UNSTABLE}; use serde_json::json; #[test] fn update_assign() { let ab = |v| json!({"a": v, "b": 2}); - gives(ab(1), ".a = (.a, .b)", [ab(1), ab(2)]); - gives(ab(1), ".a += (.a, .b)", [ab(2), ab(3)]); - gives(ab(1), ".a |= (.+1, .)", [ab(2)]); + gives( + UNSTABLE, + ab(Some(1)), + ".a = (.a, .b)", + [ab(Some(1)), ab(Some(2))], + ); + gives(UNSTABLE, ab(Some(1)), ".a |= (.+1, .)", [ab(Some(2))]); + gives( + UNSTABLE, + ab(None), + ".a = .a+.b | ., .a = .a+.b", + [ab(Some(2)), ab(Some(4))], + ); + if UNSTABLE { + gives( + UNSTABLE, + ab(None), + ".a //= .a+.b | ., .a //= .a+.b", + [ab(Some(2)), ab(Some(2))], + ); + } else { + parse_fail( + UNSTABLE, + ".a //= .a+.b | ., .a //= .a+.b", + |errs| match &errs[..] { + [e @ Simple { .. }] => { + assert_eq!(e.span(), 5..7); + assert_eq!(e.reason(), &SimpleReason::Unexpected); + assert_eq!(e.found().map(|i| &**i), Some("=")); + } + _ => panic!("errs: {:?}", errs), + }, + ); + } + gives( + UNSTABLE, + ab(Some(1)), + ".a += (.a, .b)", + [ab(Some(2)), ab(Some(3))], + ); } // here, jaq diverges from jq, which returns [3,6,4,8]! // idem for other arithmetic operations -yields!(cartesian_arith, "[(1,2) * (3,4)]", [3, 4, 6, 8]); +yields!(cartesian_arith, UNSTABLE, "[(1,2) * (3,4)]", [3, 4, 6, 8]); #[test] fn add() { - give(json!(1), ". + 2", json!(3)); - give(json!(1.0), ". + 2.", json!(3.0)); - give(json!(1), "2.0 + .", json!(3.0)); - give(json!(null), "1.e1 + 2.1e2", json!(220.0)); + give(UNSTABLE, json!(1), ". + 2", json!(3)); + give(UNSTABLE, json!(1.0), ". + 2.", json!(3.0)); + give(UNSTABLE, json!(1), "2.0 + .", json!(3.0)); + give(UNSTABLE, json!(null), "1.e1 + 2.1e2", json!(220.0)); - give(json!("Hello "), ". + \"world\"", json!("Hello world")); - give(json!([1, 2]), ". + [3, 4]", json!([1, 2, 3, 4])); give( + UNSTABLE, + json!("Hello "), + ". + \"world\"", + json!("Hello world"), + ); + give(UNSTABLE, json!([1, 2]), ". + [3, 4]", json!([1, 2, 3, 4])); + give( + UNSTABLE, json!({"a": 1, "b": 2}), ". + {c: 3, a: 4}", json!({"a": 4, "b": 2, "c": 3}), ); - give(json!({}), ". + {}", json!({})); - give(json!({"a": 1}), ". + {}", json!({"a": 1})); + give(UNSTABLE, json!({}), ". + {}", json!({})); + give(UNSTABLE, json!({"a": 1}), ". + {}", json!({"a": 1})); } #[test] fn sub() { - give(json!(1), ". - -2", json!(3)); - give(json!(1.0), ". - 0.1", json!(0.9)); - give(json!(1.0), ". - 1", json!(0.0)); + give(UNSTABLE, json!(1), ". - -2", json!(3)); + give(UNSTABLE, json!(1.0), ". - 0.1", json!(0.9)); + give(UNSTABLE, json!(1.0), ". - 1", json!(0.0)); } #[test] fn mul() { - give(json!(1), ". * 2", json!(2)); - give(json!(1.0), ". * 2.", json!(2.0)); - give(json!(1), "2.0 * .", json!(2.0)); + give(UNSTABLE, json!(1), ". * 2", json!(2)); + give(UNSTABLE, json!(1.0), ". * 2.", json!(2.0)); + give(UNSTABLE, json!(1), "2.0 * .", json!(2.0)); - give(json!("Hello"), "2 * .", json!("HelloHello")); - give(json!(2), ". * \"Hello\"", json!("HelloHello")); + give(UNSTABLE, json!("Hello"), "2 * .", json!("HelloHello")); + give(UNSTABLE, json!(2), ". * \"Hello\"", json!("HelloHello")); - give(json!("Hello"), "0 * .", json!(null)); - give(json!(-1), ". * \"Hello\"", json!(null)); + give(UNSTABLE, json!("Hello"), "0 * .", json!(null)); + give(UNSTABLE, json!(-1), ". * \"Hello\"", json!(null)); give( + UNSTABLE, json!({"k": {"a": 1, "b": 2}}), ". * {k: {a: 0, c: 3}}", json!({"k": {"a": 0, "b": 2, "c": 3}}), @@ -64,138 +109,181 @@ fn mul() { #[test] fn logic() { let tf = json!([true, false]); - give(tf.clone(), "[.[] and .[]]", json!([true, false, false])); - give(tf, "[.[] or .[]]", json!([true, true, false])); + give( + UNSTABLE, + tf.clone(), + "[.[] and .[]]", + json!([true, false, false]), + ); + give(UNSTABLE, tf, "[.[] or .[]]", json!([true, true, false])); } #[test] fn alt() { - give(json!([]), ".[] // 0", json!(0)); - give(json!([null, false]), ".[] // 0", json!(0)); - give(json!([null, 1, false, 2]), "[.[] // 0]", json!([1, 2])); - give(json!([1, 2]), "[.[] // 0]", json!([1, 2])); - give(json!([1, 2]), r#"[.[] // -"a"]"#, json!([1, 2])); + give(UNSTABLE, json!([]), ".[] // 0", json!(0)); + give(UNSTABLE, json!([null, false]), ".[] // 0", json!(0)); + give( + UNSTABLE, + json!([null, 1, false, 2]), + "[.[] // 0]", + json!([1, 2]), + ); + give(UNSTABLE, json!([1, 2]), "[.[] // 0]", json!([1, 2])); + give(UNSTABLE, json!([1, 2]), r#"[.[] // -"a"]"#, json!([1, 2])); } #[test] fn try_() { - give(json!(0), ".?", json!(0)); - give(json!(0), r#"(-"a")?, 1"#, json!(1)); - give(json!(0), r#"[(1, -"a", 2)?]"#, json!([1, 2])); + give(UNSTABLE, json!(0), ".?", json!(0)); + give(UNSTABLE, json!(0), r#"(-"a")?, 1"#, json!(1)); + give(UNSTABLE, json!(0), r#"[(1, -"a", 2)?]"#, json!([1, 2])); } #[test] fn precedence() { // concatenation binds stronger than application - give(json!(null), "[0, 1 | . + 1]", json!([1, 2])); - give(json!(null), "[0, (1 | . + 1)]", json!([0, 2])); + give(UNSTABLE, json!(null), "[0, 1 | . + 1]", json!([1, 2])); + give(UNSTABLE, json!(null), "[0, (1 | . + 1)]", json!([0, 2])); // assignment binds stronger than concatenation - give(json!(1), "[. += 1, 2]", json!([2, 2])); - give(json!(1), "[. += (1, 2)]", json!([2, 3])); + give(UNSTABLE, json!(1), "[. += 1, 2]", json!([2, 2])); + give(UNSTABLE, json!(1), "[. += (1, 2)]", json!([2, 3])); // alternation binds stronger than assignment - give(json!(false), "[(., .) | . = . // 0]", json!([0, 0])); - give(json!(false), "((., .) | . = .) // 0", json!(0)); + give( + UNSTABLE, + json!(false), + "[(., .) | . = . // 0]", + json!([0, 0]), + ); + give(UNSTABLE, json!(false), "((., .) | . = .) // 0", json!(0)); // disjunction binds stronger than alternation - give(json!(false), ". or . // 0", json!(0)); - give(json!(false), ". or (. // 0)", json!(true)); + give(UNSTABLE, json!(false), ". or . // 0", json!(0)); + give(UNSTABLE, json!(false), ". or (. // 0)", json!(true)); // conjunction binds stronger than disjunction - give(json!(true), "(0 != 0) and . or .", json!(true)); - give(json!(true), "(0 != 0) and (. or .)", json!(false)); + give(UNSTABLE, json!(true), "(0 != 0) and . or .", json!(true)); + give(UNSTABLE, json!(true), "(0 != 0) and (. or .)", json!(false)); - give(json!(null), "1 + 2 * 3", json!(7)); - give(json!(null), "2 * 3 + 1", json!(7)); + give(UNSTABLE, json!(null), "1 + 2 * 3", json!(7)); + give(UNSTABLE, json!(null), "2 * 3 + 1", json!(7)); } -yields!(interpolation, r#"1 | "yields \(.+1)!""#, "yields 2!"); +yields!( + interpolation, + UNSTABLE, + r#"1 | "yields \(.+1)!""#, + "yields 2!" +); // this diverges from jq, which yields ["2 2", "3 2", "2 4", "3 4"], // probably due to different order of evaluation addition yields!( interpolation_many, + UNSTABLE, r#"2 | ["\(., .+1) \(., .*2)"]"#, ["2 2", "2 4", "3 2", "3 4"] ); // this does not work in jq, because jq does not allow for defining formatters yields!( interpolation_fmt, + UNSTABLE, r#"def @say: "say " + .; @say "I \("disco"), you \("party")""#, "I say disco, you say party" ); yields!( interpolation_nested, + UNSTABLE, r#""Here \("be \("nestings")")""#, "Here be nestings" ); yields!( obj_trailing_comma, + UNSTABLE, "{a:1, b:2, c:3,}", json!({"a": 1, "b": 2, "c": 3}) ); yields!( obj_complex_key, + UNSTABLE, r#""c" | {a: 1, "b": 2, (.): 3}"#, json!({"a": 1, "b": 2, "c": 3}) ); -yields!(obj_proj, "{a: 1, b: 2} | {a,}", json!({"a": 1})); +yields!(obj_proj, UNSTABLE, "{a: 1, b: 2} | {a,}", json!({"a": 1})); yields!( obj_proj_set, + UNSTABLE, "{a: 1, b: 2} | {a, c: 3}", json!({"a": 1, "c": 3}) ); yields!( obj_multi_keys, + UNSTABLE, r#"[{("a", "b"): 1}]"#, json!([{"a": 1}, {"b": 1}]) ); yields!( obj_multi_vals, + UNSTABLE, "[{a: (1,2), b: (3,4)}]", json!([{"a": 1, "b": 3}, {"a": 1, "b": 4}, {"a": 2, "b": 3}, {"a": 2, "b": 4}]) ); #[test] fn if_then_else() { - gives(json!([]), "if . | .[] then 0 else 0 end", []); + gives(UNSTABLE, json!([]), "if . | .[] then 0 else 0 end", []); let f = r#".[] | if . < 0 then "n" else "p" end"#; - gives(json!([-1, 1, -1]), f, [json!("n"), json!("p"), json!("n")]); + gives( + UNSTABLE, + json!([-1, 1, -1]), + f, + [json!("n"), json!("p"), json!("n")], + ); let f = r#".[] | if .<0 then "n" elif .>0 then "p" else "z" end"#; - gives(json!([-1, 0, 1]), f, [json!("n"), json!("z"), json!("p")]); + gives( + UNSTABLE, + json!([-1, 0, 1]), + f, + [json!("n"), json!("z"), json!("p")], + ); let f = r#"if .>0, .<0 then 0 elif .>0, .<0 then 1 else 2 end"#; - gives(json!(1), f, [json!(0), json!(1), json!(2)]); + gives(UNSTABLE, json!(1), f, [json!(0), json!(1), json!(2)]); } // This behaviour diverges from jq. In jaq, a `try` will propagate all // errors in the stream to the `catch` filter. yields!( try_catch_does_not_short_circuit, + UNSTABLE, "[try (\"1\", \"2\", {}[0], \"4\") catch .]", ["1", "2", "cannot index {} with 0", "4"] ); yields!( try_catch_nested, + UNSTABLE, "try try {}[0] catch {}[1] catch .", "cannot index {} with 1" ); yields!( try_catch_multi_valued, + UNSTABLE, "[(try (1,2,3[0]) catch (3,4)) | . - 1]", [0, 1, 2, 3] ); -yields!(try_without_catch, "[try (1,2,3[0],4)]", [1, 2, 4]); +yields!(try_without_catch, UNSTABLE, "[try (1,2,3[0],4)]", [1, 2, 4]); yields!( try_catch_prefix_operation, + UNSTABLE, r#"(try -[] catch .) | . > "" and . < []"#, true ); yields!( try_catch_postfix_operation, + UNSTABLE, "[try 0[0]? catch .]", json!([]) ); @@ -206,89 +294,122 @@ yields!( // empty stream yields!( try_parsing_isnt_greedy_wrt_comma, + UNSTABLE, "try (try 0[0], 1[1]) catch . == try 1[1] catch .", true ); yields!( try_parsing_isnt_greedy_wrt_pipe, + UNSTABLE, "try (try 0 | 1[1]) catch . == try 1[1] catch .", true ); yields!( try_parsing_isnt_greedy_wrt_plus, + UNSTABLE, "try (try 0 + 1[1]) catch . == try 1[1] catch .", true ); #[test] fn ord() { - give(json!(null), ". < (0 != 0)", json!(true)); - give(json!(false), ". < (0 == 0)", json!(true)); - give(json!(1), ". > 0.0", json!(true)); - give(json!(1), ". < 1.5", json!(true)); - give(json!(1.1), ". < 1.5", json!(true)); - give(json!(1.5), ". > 1.1", json!(true)); - give(json!("ab"), ". < \"b\"", json!(true)); - give(json!("a"), ". < \"ab\"", json!(true)); - give(json!({"a": 2}), r#". < {"a": 1, "b": 0}"#, json!(true)); + give(UNSTABLE, json!(null), ". < (0 != 0)", json!(true)); + give(UNSTABLE, json!(false), ". < (0 == 0)", json!(true)); + give(UNSTABLE, json!(1), ". > 0.0", json!(true)); + give(UNSTABLE, json!(1), ". < 1.5", json!(true)); + give(UNSTABLE, json!(1.1), ". < 1.5", json!(true)); + give(UNSTABLE, json!(1.5), ". > 1.1", json!(true)); + give(UNSTABLE, json!("ab"), ". < \"b\"", json!(true)); + give(UNSTABLE, json!("a"), ". < \"ab\"", json!(true)); + give( + UNSTABLE, + json!({"a": 2}), + r#". < {"a": 1, "b": 0}"#, + json!(true), + ); } #[test] fn eq() { - give(json!(1), ". == 1", json!(true)); - give(json!(1), "0 == . - .", json!(true)); - give(json!(1), ". == -1 * -1", json!(true)); - give(json!(1), ". == 2 / 2", json!(true)); - - give(json!(0), ". == -.", json!(true)); - give(json!(0), "-. == .", json!(true)); - give(json!(0.0), ". == -.", json!(true)); - give(json!(0.0), "-. == .", json!(true)); - - gives(json!([0, 1]), ".[] == 0", [json!(true), json!(false)]); + give(UNSTABLE, json!(1), ". == 1", json!(true)); + give(UNSTABLE, json!(1), "0 == . - .", json!(true)); + give(UNSTABLE, json!(1), ". == -1 * -1", json!(true)); + give(UNSTABLE, json!(1), ". == 2 / 2", json!(true)); + + give(UNSTABLE, json!(0), ". == -.", json!(true)); + give(UNSTABLE, json!(0), "-. == .", json!(true)); + give(UNSTABLE, json!(0.0), ". == -.", json!(true)); + give(UNSTABLE, json!(0.0), "-. == .", json!(true)); + + gives( + UNSTABLE, + json!([0, 1]), + ".[] == 0", + [json!(true), json!(false)], + ); - give(json!(1), ". == 1.0", json!(true)); - give(json!(1), ". == 2 / 2.0", json!(true)); + give(UNSTABLE, json!(1), ". == 1.0", json!(true)); + give(UNSTABLE, json!(1), ". == 2 / 2.0", json!(true)); - give(json!({"a": 1, "b": 2}), ". == {b: 2, a: 1}", json!(true)); + give( + UNSTABLE, + json!({"a": 1, "b": 2}), + ". == {b: 2, a: 1}", + json!(true), + ); } -yields!(def_var_filter, "def f($a; b): $a+b; f(1; 2)", 3); +yields!(def_var_filter, UNSTABLE, "def f($a; b): $a+b; f(1; 2)", 3); #[test] fn vars() { - give(json!(1), " 2 as $x | . + $x", json!(3)); - give(json!(1), ".+1 as $x | . + $x", json!(3)); - give(json!(1), ". as $x | (2 as $y | 3) | $x", json!(1)); + give(UNSTABLE, json!(1), " 2 as $x | . + $x", json!(3)); + give(UNSTABLE, json!(1), ".+1 as $x | . + $x", json!(3)); + give(UNSTABLE, json!(1), ". as $x | (2 as $y | 3) | $x", json!(1)); - give(json!(1), "def g(f): f; . as $x | g($x)", json!(1)); + give(UNSTABLE, json!(1), "def g(f): f; . as $x | g($x)", json!(1)); let f = r#"def g(f): "z" as $z | f | .+$z; "x" as $x | g("y" as $y | $x+$y)"#; - give(json!(null), f, json!("xyz")); + give(UNSTABLE, json!(null), f, json!("xyz")); let f = r#"def g(f): "a" as $a | "b" as $b | $a + $b + f; "c" as $c | g($c)"#; - give(json!(null), f, json!("abc")); + give(UNSTABLE, json!(null), f, json!("abc")); let f = r#". as $x | ("y" as $y | "z") | $x"#; - give(json!("x"), f, json!("x")); + give(UNSTABLE, json!("x"), f, json!("x")); let out = || json!([[1, 4], [1, 5], [2, 4], [2, 5]]); let f = "def f($a; b; $c; d): [$a+b, $c+d]; [f((1, 2); 0; (3, 4); 1)]"; - give(json!(null), f, out()); + give(UNSTABLE, json!(null), f, out()); let f = "def f($a; b; $c; d): [$a+b, $c+d]; 0 as $a | 1 as $b | [f((1, 2); $a; (3, 4); $b)]"; - give(json!(null), f, out()); + give(UNSTABLE, json!(null), f, out()); } -yields!(shadow_funs, "def a: 1; def b: a; def a: 2; a + b", 3); -yields!(shadow_vars, "1 as $x | 2 as $x | $x", 2); +yields!( + shadow_funs, + UNSTABLE, + "def a: 1; def b: a; def a: 2; a + b", + 3 +); +yields!(shadow_vars, UNSTABLE, "1 as $x | 2 as $x | $x", 2); // arguments from the right are stronger than from the left -yields!(shadow_args, "def f(g; g): g; f(1; 2)", 2); +yields!(shadow_args, UNSTABLE, "def f(g; g): g; f(1; 2)", 2); -yields!(id_var, "def f($a): $a; f(0)", 0); -yields!(id_arg, "def f( a): a; f(0)", 0); -yields!(args_mixed, "def f(a; $b): a + $b; 1 as $a | f($a; 2)", 3); +yields!(id_var, UNSTABLE, "def f($a): $a; f(0)", 0); +yields!(id_arg, UNSTABLE, "def f( a): a; f(0)", 0); +yields!( + args_mixed, + UNSTABLE, + "def f(a; $b): a + $b; 1 as $a | f($a; 2)", + 3 +); -yields!(nested_comb_args, "def f(a): def g(b): a + b; g(1); f(2)", 3); +yields!( + nested_comb_args, + UNSTABLE, + "def f(a): def g(b): a + b; g(1); f(2)", + 3 +); const ACKERMANN: &str = "def ack($m; $n): if $m == 0 then $n + 1 @@ -296,26 +417,33 @@ const ACKERMANN: &str = "def ack($m; $n): else ack($m-1; ack($m; $n-1)) end;"; -yields!(ackermann, &(ACKERMANN.to_owned() + "ack(3; 4)"), 125); +yields!( + ackermann, + UNSTABLE, + &(ACKERMANN.to_owned() + "ack(3; 4)"), + 125 +); #[test] fn reduce() { let ff = |s| format!(". as $x | reduce 2 as $y (4; {}) | . + $x", s); let f = ff("3 as $z | . + $x + $y + $z"); - give(json!(1), &f, json!(11)); + give(UNSTABLE, json!(1), &f, json!(11)); let f = "def g(x; y): 3 as $z | . + x + y + $z; ".to_owned() + &ff("g($x; $y)"); - give(json!(1), &f, json!(11)); + give(UNSTABLE, json!(1), &f, json!(11)); } yields!( foreach_cumulative_sum, + UNSTABLE, "[1, 2, 3] | [foreach .[] as $x (0; .+$x)]", [1, 3, 6] ); yields!( for_cumulative_sum, + UNSTABLE, "[1, 2, 3] | [for .[] as $x (0; .+$x)]", [0, 1, 3, 6] ); @@ -325,11 +453,13 @@ yields!( // jaq keeps all output values as input values yields!( foreach_many_outputs, + UNSTABLE, "[foreach (3,4) as $x (1; .+$x, .*$x)]", [4, 8, 16, 3, 7, 12] ); yields!( for_many_outputs, + UNSTABLE, "[for (3,4) as $x (1; .+$x, .*$x)]", [1, 4, 8, 16, 3, 7, 12] ); diff --git a/jaq-parse/Cargo.toml b/jaq-parse/Cargo.toml index 9c216026..1c5a3dae 100644 --- a/jaq-parse/Cargo.toml +++ b/jaq-parse/Cargo.toml @@ -11,6 +11,12 @@ keywords = ["json", "query", "jq"] categories = ["parser-implementations"] rust-version = "1.64" +[features] +default = [] +unstable = ["unstable-flag", "jaq-syn/unstable"] +unstable-flag = ["jaq-syn/unstable-flag"] + [dependencies] chumsky = { version = "0.9.0", default-features = false } +either = { version = "1.10.0", default-features = false } jaq-syn = { version = "1.0.0", path = "../jaq-syn" } diff --git a/jaq-parse/src/def.rs b/jaq-parse/src/def.rs index 286a9399..c341b955 100644 --- a/jaq-parse/src/def.rs +++ b/jaq-parse/src/def.rs @@ -4,7 +4,10 @@ use chumsky::prelude::*; use jaq_syn::{Arg, Call, Def, Main}; /// A (potentially empty) parenthesised and `;`-separated sequence of arguments. -fn args(arg: P) -> impl Parser, Error = P::Error> + Clone +fn args( + #[allow(unused_variables)] unstable: bool, + arg: P, +) -> impl Parser, Error = P::Error> + Clone where P: Parser + Clone, { @@ -14,7 +17,7 @@ where .map(Option::unwrap_or_default) } -pub fn call(expr: P) -> impl Parser, Error = P::Error> + Clone +pub fn call(unstable: bool, expr: P) -> impl Parser, Error = P::Error> + Clone where P: Parser> + Clone, { @@ -22,12 +25,12 @@ where Token::Ident(ident) => ident, } .labelled("filter name") - .then(args(expr).labelled("filter args")) - .map(|(name, args)| Call { name, args }) + .then(args(unstable, expr).labelled("filter args")) + .map(|(name, args)| Call::new(name, args)) } /// Parser for a single definition. -fn def

(def: P) -> impl Parser> + Clone +fn def

(unstable: bool, def: P) -> impl Parser> + Clone where P: Parser> + Clone, { @@ -39,22 +42,40 @@ where let defs = def.repeated().collect(); just(Token::Def) - .ignore_then(call(arg)) + .ignore_then(call(unstable, arg)) .then_ignore(just(Token::Colon)) - .then(defs.then(filter()).map(|(defs, body)| Main { defs, body })) + .then( + defs.then(filter( + #[cfg(feature = "unstable-flag")] + unstable, + )) + .map(|(defs, body)| Main::new(defs, body)), + ) .then_ignore(just(Token::Semicolon)) - .map(|(lhs, rhs)| Def { lhs, rhs }) + .map(|(lhs, rhs)| Def::new(lhs, rhs)) .labelled("definition") } /// Parser for a sequence of definitions. -pub fn defs() -> impl Parser, Error = Simple> + Clone { - recursive(def).repeated().collect() +pub fn defs( + #[cfg(feature = "unstable-flag")] unstable: bool, +) -> impl Parser, Error = Simple> + Clone { + #[cfg(not(feature = "unstable-flag"))] + let unstable = false; + recursive(|p| def(unstable, p)).repeated().collect() } /// Parser for a (potentially empty) sequence of definitions, followed by a filter. -pub fn main() -> impl Parser> + Clone { - defs() - .then(filter()) - .map(|(defs, body)| Main { defs, body }) +pub fn main( + #[cfg(feature = "unstable-flag")] unstable: bool, +) -> impl Parser> + Clone { + defs( + #[cfg(feature = "unstable-flag")] + unstable, + ) + .then(filter( + #[cfg(feature = "unstable-flag")] + unstable, + )) + .map(|(defs, body)| Main::new(defs, body)) } diff --git a/jaq-parse/src/either.rs b/jaq-parse/src/either.rs new file mode 100644 index 00000000..dedec8f7 --- /dev/null +++ b/jaq-parse/src/either.rs @@ -0,0 +1,135 @@ +pub(crate) use ::either::*; +use alloc::vec::Vec; +use chumsky::{ + debug::{Silent, Verbose}, + error::Located, + Error, Parser, Span, Stream, +}; + +macro_rules! map_either { + ($value:expr, $pattern:pat => $result:expr) => { + match $value { + $crate::either::Left($pattern) => $crate::either::Left($result), + $crate::either::Right($pattern) => $crate::either::Right($result), + } + }; +} + +// ([], Ok((out, alt_err))) => parsing successful, +// alt_err = potential alternative error should a different number of optional patterns be parsed +// ([x, ...], Ok((out, alt_err)) => parsing failed, but recovery occurred so parsing may continue +// ([...], Err(err)) => parsing failed, recovery failed, and one or more errors were produced +type PResult = ( + Vec>, + Result<(O, Option>), Located>, +); + +// Shorthand for a stream with the given input and error type. +type StreamOf<'a, I, E> = Stream<'a, I, >::Span>; + +#[derive(Clone, Copy, Debug)] +#[repr(transparent)] +pub struct ChumskyEither(Either); + +impl From> for ChumskyEither { + fn from(value: Either) -> Self { + Self(value) + } +} + +impl From> for Either { + fn from(value: ChumskyEither) -> Self { + value.0 + } +} + +impl Parser for ChumskyEither +where + L: Parser, + R: Parser, +{ + type Error = L::Error; + + #[allow(deprecated)] + fn parse_inner( + &self, + debugger: &mut D, + stream: &mut StreamOf, + ) -> PResult + where + Self: Sized, + { + either::for_both!(&self.0, parser => Parser::parse_inner(parser, debugger, stream)) + } + + #[allow(deprecated)] + fn parse_inner_verbose( + &self, + debugger: &mut Verbose, + stream: &mut StreamOf, + ) -> PResult { + either::for_both!(&self.0, parser => Parser::parse_inner_verbose(parser, debugger, stream)) + } + + #[allow(deprecated)] + fn parse_inner_silent( + &self, + debugger: &mut Silent, + stream: &mut StreamOf, + ) -> PResult { + either::for_both!(&self.0, parser => Parser::parse_inner_silent(parser, debugger, stream)) + } + + fn parse_recovery<'a, Iter, S>(&self, stream: S) -> (Option, Vec) + where + Self: Sized, + Iter: Iterator>::Span)> + 'a, + S: Into>::Span, Iter>>, + { + either::for_both!(&self.0, parser => Parser::parse_recovery(parser, stream)) + } + + fn parse_recovery_verbose<'a, Iter, S>(&self, stream: S) -> (Option, Vec) + where + Self: Sized, + Iter: Iterator>::Span)> + 'a, + S: Into>::Span, Iter>>, + { + either::for_both!(&self.0, parser => Parser::parse_recovery_verbose(parser, stream)) + } + + fn parse<'a, Iter, S>(&self, stream: S) -> Result> + where + Self: Sized, + Iter: Iterator>::Span)> + 'a, + S: Into>::Span, Iter>>, + { + either::for_both!(&self.0, parser => Parser::parse(parser, stream)) + } +} + +impl Span for ChumskyEither +where + L: Span, + R: Span, +{ + type Context = Either; + + type Offset = L::Offset; + + fn new(context: Self::Context, range: core::ops::Range) -> Self { + Self::from(map_either!(context, context => Span::new(context, range))) + } + + fn context(&self) -> Self::Context { + map_either!(&self.0, span => Span::context(span)) + } + + fn start(&self) -> Self::Offset { + either::for_both!(&self.0, s => Span::start(s)) + } + + fn end(&self) -> Self::Offset { + either::for_both!(&self.0, s => Span::end(s)) + } +} diff --git a/jaq-parse/src/filter.rs b/jaq-parse/src/filter.rs index 88ec5e30..67f6dc8e 100644 --- a/jaq-parse/src/filter.rs +++ b/jaq-parse/src/filter.rs @@ -1,6 +1,8 @@ use super::{prec_climb, Delim, Token}; use alloc::{boxed::Box, string::String, string::ToString, vec::Vec}; use chumsky::prelude::*; +use crate::either; +use either::{ChumskyEither, Left, Right}; use jaq_syn::filter::{AssignOp, BinaryOp, Filter, Fold, FoldType, KeyVal}; use jaq_syn::{MathOp, OrdOp, Spanned}; @@ -42,7 +44,7 @@ where .then_ignore(just(Token::As)) .then(variable()) .then(Delim::Paren.around(args)) - .map(|(((inner, xs), x), (init, f))| (inner, Fold { xs, x, init, f })) + .map(|(((inner, xs), x), (init, f))| (inner, Fold::new(xs, x, init, f))) .map_with_span(|(inner, fold), span| (Filter::Fold(inner, fold), span)) } @@ -57,7 +59,11 @@ where } // 'Atoms' are filters that contain no ambiguity -fn atom

(filter: P, no_comma: P) -> impl Parser, Error = P::Error> + Clone +fn atom

( + unstable: bool, + filter: P, + no_comma: P, +) -> impl Parser, Error = P::Error> + Clone where P: Parser, Error = Simple> + Clone, { @@ -66,8 +72,8 @@ where } .labelled("number"); - let str_ = super::string::str_(filter.clone()); - let call = super::def::call(filter.clone()); + let str_ = super::string::str_(unstable, filter.clone()); + let call = super::def::call(unstable, filter.clone()); // Atoms can also just be normal filters, but surrounded with parentheses let parenthesised = Delim::Paren.around(filter.clone()); @@ -77,7 +83,7 @@ where let array = Delim::Brack.around(filter.clone().or_not()); let is_val = just(Token::Colon).ignore_then(no_comma); - let key_str = super::path::key(filter) + let key_str = super::path::key(unstable, filter) .then(is_val.clone().or_not()) .map(|(key, val)| KeyVal::Str(key, val)); let key_filter = parenthesised @@ -128,6 +134,8 @@ impl prec_climb::Op for BinaryOp { Self::Math(MathOp::Add | MathOp::Sub) => Self::And.prec() + 3, Self::Math(MathOp::Mul | MathOp::Div) => Self::Math(MathOp::Add).prec() + 1, Self::Math(MathOp::Rem) => Self::Math(MathOp::Mul).prec() + 1, + #[cfg(feature = "unstable-flag")] + _ => unimplemented!(), } } @@ -142,7 +150,7 @@ impl prec_climb::Output for Spanned { } } -fn binary_op() -> impl Parser> + Clone { +fn binary_op(unstable: bool) -> impl Parser> + Clone { let as_var = just(Token::As).ignore_then(variable()).or_not(); let pipe = as_var .then_ignore(just(Token::Op("|".to_string()))) @@ -162,6 +170,13 @@ fn binary_op() -> impl Parser> + Clone { // therefore, we add `,` later assign(AssignOp::Assign), assign(AssignOp::Update), + #[cfg(feature = "unstable-flag")] + ChumskyEither::from(if unstable { + #[allow(deprecated)] + Right(assign(AssignOp::AltUpdate)) + } else { + Left(crate::fail()) + }), update_with(MathOp::Add), update_with(MathOp::Sub), update_with(MathOp::Mul), @@ -195,20 +210,24 @@ where .map(|(f, ops)| f.parse(ops)) } -pub fn filter() -> impl Parser, Error = Simple> + Clone { +pub fn filter( + #[cfg(feature = "unstable-flag")] unstable: bool, +) -> impl Parser, Error = Simple> + Clone { + #[cfg(not(feature = "unstable-flag"))] + let unstable = false; // filters that may or may not contain commas on the toplevel, // i.e. not inside parentheses let mut with_comma = Recursive::declare(); let mut sans_comma = Recursive::declare(); // e.g. `keys[]` - let atom = atom(with_comma.clone(), sans_comma.clone()); - let atom_path = || super::path::path(with_comma.clone()); + let atom = atom(unstable, with_comma.clone(), sans_comma.clone()); + let atom_path = || super::path::path(unstable, with_comma.clone()); let atom_with_path = atom.then(atom_path().collect()); // e.g. `.[].a` or `.a` let id = just(Token::Dot).map_with_span(|_, span| (Filter::Id, span)); - let id_path = super::path::part(with_comma.clone()).chain(atom_path()); + let id_path = super::path::part(unstable, with_comma.clone()).chain(atom_path()); let id_with_path = id.then(id_path.or_not().flatten()); let path = atom_with_path.or(id_with_path); @@ -234,7 +253,7 @@ pub fn filter() -> impl Parser, Error = Simple> + let neg = neg(try_).boxed(); let tc = recursive(|f| try_catch(f).or(neg)); - let op = binary_op().boxed(); + let op = binary_op(unstable).boxed(); let comma = just(Token::Comma).to(BinaryOp::Comma); sans_comma.define(climb(tc.clone(), op.clone())); diff --git a/jaq-parse/src/lib.rs b/jaq-parse/src/lib.rs index 9971cbc1..fe450698 100644 --- a/jaq-parse/src/lib.rs +++ b/jaq-parse/src/lib.rs @@ -1,11 +1,12 @@ //! JSON query language parser. #![no_std] -#![forbid(unsafe_code)] +#![deny(unsafe_code)] #![warn(missing_docs)] extern crate alloc; mod def; +pub(crate) mod either; mod filter; mod path; mod prec_climb; @@ -18,28 +19,74 @@ pub use def::{defs, main}; use token::{Delim, Token}; use alloc::{string::String, string::ToString, vec::Vec}; -use chumsky::prelude::*; +use chumsky::{prelude::*, primitive::custom}; use syn::Spanned; +fn silent() -> chumsky::debug::Silent { + struct Silent { + phantom: core::marker::PhantomData<()>, + } + let silent = Silent { + phantom: Default::default(), + }; + #[allow(unsafe_code)] + unsafe { + core::mem::transmute(silent) + } +} + +fn fail>() -> impl chumsky::Parser + Clone { + fn f>(stream: &mut chumsky::Stream>::Span>) -> ( + Vec>, + Result<(O, Option>), chumsky::error::Located>, + ) { + let (errors, res): (Vec>, _) = + empty().not().parse_inner_silent(&mut silent(), stream); + ( + errors, + match res { + Err(err) => Err(err), + Ok((_out, Some(alt_err))) => Err(alt_err), + Ok((_out, None)) => unimplemented!(), + }, + ) + } + custom(f) +} + /// Lex/parse error. pub type Error = Simple; -fn lex() -> impl Parser>, Error = Simple> { - recursive(token::tree) - .map_with_span(|tree, span| tree.tokens(span)) - .repeated() - .flatten() - .collect() +fn lex( + #[cfg_attr(not(feature = "unstable-flag"), allow(unused_variables))] unstable: bool, +) -> impl Parser>, Error = Simple> { + recursive(|tree| { + token::tree( + #[cfg(feature = "unstable-flag")] + unstable, + tree, + ) + }) + .map_with_span(|tree, span| tree.tokens(span)) + .repeated() + .flatten() + .collect() } /// Parse a string with a given parser. /// /// May produce `Some` output even if there were errors. -pub fn parse(src: &str, parser: P) -> (Option, Vec) +pub fn parse( + #[cfg(feature = "unstable-flag")] unstable: bool, + src: &str, + parser: P, +) -> (Option, Vec) where P: Parser> + Clone, { - let (tokens, lex_errs) = lex() + #[cfg(not(feature = "unstable-flag"))] + let unstable = false; + let (tokens, lex_errs) = lex(unstable) .then_ignore(end()) .recover_with(skip_then_retry_until([])) .parse_recovery(src); diff --git a/jaq-parse/src/path.rs b/jaq-parse/src/path.rs index 5cd151e0..0fa2cbe9 100644 --- a/jaq-parse/src/path.rs +++ b/jaq-parse/src/path.rs @@ -10,7 +10,10 @@ fn opt() -> impl Parser> + Clone { }) } -pub fn key(expr: P) -> impl Parser>, Error = P::Error> + Clone +pub fn key( + unstable: bool, + expr: P, +) -> impl Parser>, Error = P::Error> + Clone where T: From>>, P: Parser, Error = Simple> + Clone, @@ -18,16 +21,19 @@ where select! { Token::Ident(s) => Str::from(s), } - .or(super::string::str_(expr)) + .or(super::string::str_(unstable, expr)) .labelled("object key") } -fn index(expr: P) -> impl Parser>, Error = P::Error> + Clone +fn index( + unstable: bool, + expr: P, +) -> impl Parser>, Error = P::Error> + Clone where T: From>> + From>>, P: Parser, Error = Simple> + Clone, { - key(expr).map_with_span(|id, span| Part::Index((T::from(id), span))) + key(unstable, expr).map_with_span(|id, span| Part::Index((T::from(id), span))) } /// Match `[]`, `[e]`, `[e:]`, `[e:e]`, `[:e]` (all without brackets). @@ -51,23 +57,26 @@ where } /// A path after an atomic filter (that is not the identity filter). -pub fn path(expr: P) -> impl Parser, Error = P::Error> + Clone +pub fn path(unstable: bool, expr: P) -> impl Parser, Error = P::Error> + Clone where T: From>> + From>>, P: Parser, Error = Simple> + Clone, { let range = Delim::Brack.around(range(expr.clone())); - let dot_index = just(Token::Dot).ignore_then(index(expr)); + let dot_index = just(Token::Dot).ignore_then(index(unstable, expr)); let dot_range = just(Token::Dot).or_not().ignore_then(range); dot_index.or(dot_range).then(opt()).repeated() } /// The first part of a path after an identity filter. -pub fn part(expr: P) -> impl Parser>, Opt), Error = P::Error> + Clone +pub fn part( + unstable: bool, + expr: P, +) -> impl Parser>, Opt), Error = P::Error> + Clone where T: From>> + From>>, P: Parser, Error = Simple> + Clone, { let range = Delim::Brack.around(range(expr.clone())); - range.or(index(expr)).then(opt()) + range.or(index(unstable, expr)).then(opt()) } diff --git a/jaq-parse/src/string.rs b/jaq-parse/src/string.rs index 2c089486..c2230efb 100644 --- a/jaq-parse/src/string.rs +++ b/jaq-parse/src/string.rs @@ -2,15 +2,15 @@ use super::{Delim, Token}; use chumsky::prelude::*; use jaq_syn::{Call, Spanned, Str}; -pub fn str_(expr: P) -> impl Parser>, Error = P::Error> + Clone +pub fn str_( + #[allow(unused_variables)] unstable: bool, + expr: P, +) -> impl Parser>, Error = P::Error> + Clone where T: From>>, P: Parser, Error = Simple> + Clone, { - let call = |name| Call { - name, - args: Default::default(), - }; + let call = |name| Call::new(name, Default::default()); let ident = select! { Token::Ident(ident) if ident.starts_with('@') => ident, }; @@ -29,6 +29,6 @@ where }); fmt.or_not() .then(parts.delimited_by(just(Token::Quote), just(Token::Quote))) - .map(|(fmt, parts)| Str { fmt, parts }) + .map(|(fmt, parts)| Str::new(fmt, parts)) .labelled("string") } diff --git a/jaq-parse/src/token.rs b/jaq-parse/src/token.rs index 18ab296e..a51e527d 100644 --- a/jaq-parse/src/token.rs +++ b/jaq-parse/src/token.rs @@ -175,6 +175,7 @@ fn char_() -> impl Parser> { } pub fn tree( + #[cfg(feature = "unstable-flag")] unstable: bool, tree: impl Parser> + Clone, ) -> impl Parser> { let trees = || tree.clone().map_with_span(|t, span| (t, span)).repeated(); @@ -204,7 +205,11 @@ pub fn tree( brack.map(|t| Tree::Delim(Delim::Brack, t)), brace.map(|t| Tree::Delim(Delim::Brace, t)), string.map(|(s, interpol)| Tree::String(s, interpol)), - token().map(Tree::Token), + token( + #[cfg(feature = "unstable-flag")] + unstable, + ) + .map(Tree::Token), )) .recover_with(strategy('(', ')', [('[', ']'), ('{', '}')])) .recover_with(strategy('[', ']', [('{', '}'), ('(', ')')])) @@ -213,8 +218,22 @@ pub fn tree( .padded() } -pub fn token() -> impl Parser> { +pub fn token( + #[cfg(feature = "unstable-flag")] unstable: bool, +) -> impl Parser> { // A parser for operators + #[cfg(feature = "unstable-flag")] + let op = crate::either::ChumskyEither::from(if unstable { + crate::either::Left( + one_of("|=!<>+-*/%") + .chain::(just('/').or_not()) + .chain::(just('=').or_not()) + .collect(), + ) + } else { + crate::either::Right(one_of("|=!<>+-*/%").chain(one_of("=/").or_not()).collect()) + }); + #[cfg(not(feature = "unstable-flag"))] let op = one_of("|=!<>+-*/%").chain(one_of("=/").or_not()).collect(); let var = just('$').ignore_then(text::ident()); diff --git a/jaq-std/Cargo.toml b/jaq-std/Cargo.toml index 79606d60..78a6f76a 100644 --- a/jaq-std/Cargo.toml +++ b/jaq-std/Cargo.toml @@ -12,6 +12,8 @@ rust-version = "1.64" [features] default = ["bincode"] +unstable = ["unstable-flag", "jaq-syn/unstable", "jaq-parse/unstable"] +unstable-flag = ["jaq-syn/unstable-flag", "jaq-parse/unstable-flag"] [build-dependencies] jaq-parse = { version = "1.0.0", path = "../jaq-parse" } diff --git a/jaq-std/build.rs b/jaq-std/build.rs index 54373c68..cd123e8c 100644 --- a/jaq-std/build.rs +++ b/jaq-std/build.rs @@ -3,15 +3,40 @@ #[cfg(feature = "bincode")] fn main() { let out_dir = std::env::var_os("OUT_DIR").unwrap(); - let dest_path = std::path::Path::new(&out_dir).join("std.bin"); - let buffer = std::fs::File::create(dest_path).unwrap(); - let std = include_str!("src/std.jq"); - let (std, errs) = jaq_parse::parse(std, jaq_parse::defs()); - assert_eq!(errs, Vec::new()); - let std = std.unwrap(); - bincode::serialize_into(buffer, &std).unwrap(); + + { + let dest_path = std::path::Path::new(&out_dir).join("std.bin"); + let buffer = std::fs::File::create(dest_path).unwrap(); + serialize_std(std, false, buffer); + } + #[cfg(feature = "unstable-flag")] + { + let dest_path = std::path::Path::new(&out_dir).join("std-unstable.bin"); + let buffer = std::fs::File::create(dest_path).unwrap(); + serialize_std(std, true, buffer); + } } #[cfg(not(feature = "bincode"))] fn main() {} + +#[cfg(feature = "bincode")] +fn serialize_std( + src: &str, + #[cfg_attr(not(feature = "unstable-flag"), allow(unused))] unstable: bool, + buffer: std::fs::File, +) { + let (std, errs) = jaq_parse::parse( + #[cfg(feature = "unstable-flag")] + unstable, + src, + jaq_parse::defs( + #[cfg(feature = "unstable-flag")] + unstable, + ), + ); + assert_eq!(errs, Vec::new()); + let std = std.unwrap(); + bincode::serialize_into(buffer, &std).unwrap(); +} diff --git a/jaq-std/src/lib.rs b/jaq-std/src/lib.rs index 34a5f482..d3640783 100644 --- a/jaq-std/src/lib.rs +++ b/jaq-std/src/lib.rs @@ -17,16 +17,22 @@ extern crate alloc; use alloc::vec::Vec; /// Return the standard library. -pub fn std() -> Vec { +pub fn std(#[cfg(feature = "unstable-flag")] unstable: bool) -> Vec { #[cfg(feature = "bincode")] { // use preparsed standard library let std = include_bytes!(concat!(env!("OUT_DIR"), "/std.bin")); + #[cfg(feature = "unstable-flag")] + let std_unstable = include_bytes!(concat!(env!("OUT_DIR"), "/std-unstable.bin")); + #[cfg(feature = "unstable-flag")] + let std = if unstable { std } else { std_unstable }; bincode::deserialize(std).unwrap() } #[cfg(not(feature = "bincode"))] { let std = include_str!("std.jq"); - jaq_parse::parse(std, jaq_parse::defs()).0.unwrap() + jaq_parse::parse(unstable, std, jaq_parse::defs(unstable)) + .0 + .unwrap() } } diff --git a/jaq-std/tests/common/mod.rs b/jaq-std/tests/common/mod.rs index d8c78f26..138844e9 100644 --- a/jaq-std/tests/common/mod.rs +++ b/jaq-std/tests/common/mod.rs @@ -1,33 +1,61 @@ use serde_json::Value; -fn yields(x: jaq_interpret::Val, f: &str, ys: impl Iterator) { +#[cfg(not(feature = "unstable-flag"))] +pub const UNSTABLE: bool = false; +#[cfg(feature = "unstable-flag")] +pub const UNSTABLE: bool = false; + +#[track_caller] +fn yields( + #[cfg_attr(not(feature = "unstable-flag"), allow(unused_variables))] unstable: bool, + x: jaq_interpret::Val, + f: &str, + ys: impl Iterator, +) { let mut ctx = jaq_interpret::ParseCtx::new(Vec::new()); - ctx.insert_natives(jaq_core::core()); - ctx.insert_defs(jaq_std::std()); + ctx.insert_natives(jaq_core::core( + #[cfg(feature = "unstable-flag")] + unstable, + )); + ctx.insert_defs(jaq_std::std( + #[cfg(feature = "unstable-flag")] + unstable, + )); - let (f, errs) = jaq_parse::parse(f, jaq_parse::main()); + let (f, errs) = jaq_parse::parse( + #[cfg(feature = "unstable-flag")] + unstable, + f, + jaq_parse::main( + #[cfg(feature = "unstable-flag")] + unstable, + ), + ); assert!(errs.is_empty()); ctx.yields(x, f.unwrap(), ys) } -pub fn fail(x: Value, f: &str, err: jaq_interpret::Error) { - yields(x.into(), f, core::iter::once(Err(err))) +#[track_caller] +pub fn fail(unstable: bool, x: Value, f: &str, err: jaq_interpret::Error) { + yields(unstable, x.into(), f, core::iter::once(Err(err))) } -pub fn give(x: Value, f: &str, y: Value) { - yields(x.into(), f, core::iter::once(Ok(y.into()))) +#[track_caller] +pub fn give(unstable: bool, x: Value, f: &str, y: Value) { + yields(unstable, x.into(), f, core::iter::once(Ok(y.into()))) } -pub fn gives(x: Value, f: &str, ys: [Value; N]) { - yields(x.into(), f, ys.into_iter().map(|y| Ok(y.into()))) +#[track_caller] +pub fn gives(unstable: bool, x: Value, f: &str, ys: [Value; N]) { + yields(unstable, x.into(), f, ys.into_iter().map(|y| Ok(y.into()))) } #[macro_export] macro_rules! yields { - ($func_name:ident, $filter:expr, $output: expr) => { + ($func_name:ident, $unstable:expr, $filter:expr, $output: expr) => { #[test] fn $func_name() { - give(json!(null), $filter, json!($output)) + give($unstable, json!(null), $filter, json!($output)) } }; } diff --git a/jaq-std/tests/std.rs b/jaq-std/tests/std.rs index 513d3a91..3e1436c0 100644 --- a/jaq-std/tests/std.rs +++ b/jaq-std/tests/std.rs @@ -2,40 +2,62 @@ pub mod common; -use common::{give, gives}; +use common::{give, gives, UNSTABLE}; use serde_json::json; #[test] fn add() { - give(json!({"a": 1, "b": 2}), "add", json!(3)); - give(json!([[0, 1], [2, 3]]), "add", json!([0, 1, 2, 3])); + give(UNSTABLE, json!({"a": 1, "b": 2}), "add", json!(3)); + give( + UNSTABLE, + json!([[0, 1], [2, 3]]), + "add", + json!([0, 1, 2, 3]), + ); } #[test] fn all() { - give(json!({"a": false, "b": true}), "all", json!(false)); - give(json!({"a": 1, "b": 2}), "all", json!(true)); + give( + UNSTABLE, + json!({"a": false, "b": true}), + "all", + json!(false), + ); + give(UNSTABLE, json!({"a": 1, "b": 2}), "all", json!(true)); let f = "def positive(f): all(f; . > 0); positive(.[])"; - give(json!([1, 2]), f, json!(true)); + give(UNSTABLE, json!([1, 2]), f, json!(true)); } #[test] fn any() { - give(json!({"a": false, "b": true}), "any", json!(true)); + give(UNSTABLE, json!({"a": false, "b": true}), "any", json!(true)); } #[test] fn date() { // aliases for fromdateiso8601 and todateiso8601 - give(json!("1970-01-02T00:00:00Z"), "fromdate", json!(86400)); give( + UNSTABLE, + json!("1970-01-02T00:00:00Z"), + "fromdate", + json!(86400), + ); + give( + UNSTABLE, json!("1970-01-02T00:00:00.123456789Z"), "fromdate", json!(86400.123456789), ); - give(json!(86400), "todate", json!("1970-01-02T00:00:00Z")); give( + UNSTABLE, + json!(86400), + "todate", + json!("1970-01-02T00:00:00Z"), + ); + give( + UNSTABLE, json!(86400.123456789), "todate", json!("1970-01-02T00:00:00.123456789Z"), @@ -45,23 +67,30 @@ fn date() { #[test] fn date_roundtrip() { let epoch = 946684800; - give(json!(epoch), "todate|fromdate", json!(epoch)); + give(UNSTABLE, json!(epoch), "todate|fromdate", json!(epoch)); let epoch_ns = 946684800.123456; - give(json!(epoch_ns), "todate|fromdate", json!(epoch_ns)); + give( + UNSTABLE, + json!(epoch_ns), + "todate|fromdate", + json!(epoch_ns), + ); let iso = "2000-01-01T00:00:00Z"; - give(json!(iso), "fromdate|todate", json!(iso)); + give(UNSTABLE, json!(iso), "fromdate|todate", json!(iso)); let iso_ns = "2000-01-01T00:00:00.123456000Z"; - give(json!(iso_ns), "fromdate|todate", json!(iso_ns)); + give(UNSTABLE, json!(iso_ns), "fromdate|todate", json!(iso_ns)); } yields!( drem_nan, + UNSTABLE, "[drem(nan, 1; nan, 1)] == [nan, nan, nan, 0.0]", true ); yields!( drem_range, + UNSTABLE, "[drem(3.5, -4; 6, 1.75, 2)]", [-2.5, 0.0, -0.5, 2.0, -0.5, -0.0] ); @@ -72,15 +101,15 @@ fn entries() { let entries = json!([{"key": "a", "value": 1}, {"key": "b", "value": 2}]); let objk = json!({"ak": 1, "bk": 2}); - give(obj.clone(), "to_entries", entries.clone()); - give(entries, "from_entries", obj.clone()); - give(obj, r#"with_entries(.key += "k")"#, objk); + give(UNSTABLE, obj.clone(), "to_entries", entries.clone()); + give(UNSTABLE, entries, "from_entries", obj.clone()); + give(UNSTABLE, obj, r#"with_entries(.key += "k")"#, objk); let arr = json!([null, 0]); let entries = json!([{"key": 0, "value": null}, {"key": 1, "value": 0}]); - give(arr, "to_entries", entries); + give(UNSTABLE, arr, "to_entries", entries); - give(json!([]), "from_entries", json!({})); + give(UNSTABLE, json!([]), "from_entries", json!({})); } #[test] @@ -88,106 +117,136 @@ fn flatten() { let a0 = || json!([1, [{"a": 2}, [3]]]); let a1 = || json!([1, {"a": 2}, [3]]); let a2 = || json!([1, {"a": 2}, 3]); - give(a0(), "flatten", json!(a2())); + give(UNSTABLE, a0(), "flatten", json!(a2())); let f = "[flatten(0, 1, 2, 3)]"; - give(a0(), f, json!([a0(), a1(), a2(), a2()])); + give(UNSTABLE, a0(), f, json!([a0(), a1(), a2(), a2()])); } yields!( flatten_deep, + UNSTABLE, "[[[0], 1], 2, [3, [4]]] | flatten", [0, 1, 2, 3, 4] ); // here, we diverge from jq, which returns just 1 -yields!(flatten_obj, "{a: 1} | flatten", json!([{"a": 1}])); +yields!(flatten_obj, UNSTABLE, "{a: 1} | flatten", json!([{"a": 1}])); // jq gives an error here -yields!(flatten_num, "0 | flatten", [0]); +yields!(flatten_num, UNSTABLE, "0 | flatten", [0]); #[test] fn inside() { give( + UNSTABLE, json!(["foo", "bar"]), r#"map(in({"foo": 42}))"#, json!([true, false]), ); - give(json!([2, 0]), r#"map(in([0,1]))"#, json!([false, true])); + give( + UNSTABLE, + json!([2, 0]), + r#"map(in([0,1]))"#, + json!([false, true]), + ); - give(json!("bar"), r#"inside("foobar")"#, json!(true)); + give(UNSTABLE, json!("bar"), r#"inside("foobar")"#, json!(true)); let f = r#"inside(["foobar", "foobaz", "blarp"])"#; - give(json!(["baz", "bar"]), f, json!(true)); - give(json!(["bazzzz", "bar"]), f, json!(false)); + give(UNSTABLE, json!(["baz", "bar"]), f, json!(true)); + give(UNSTABLE, json!(["bazzzz", "bar"]), f, json!(false)); let f = r#"inside({"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]})"#; - give(json!({"foo": 12, "bar": [{"barp": 12}]}), f, json!(true)); - give(json!({"foo": 12, "bar": [{"barp": 15}]}), f, json!(false)); + give( + UNSTABLE, + json!({"foo": 12, "bar": [{"barp": 12}]}), + f, + json!(true), + ); + give( + UNSTABLE, + json!({"foo": 12, "bar": [{"barp": 15}]}), + f, + json!(false), + ); } -yields!(isfinite_true, "all((0, 1, nan); isfinite)", true); +yields!(isfinite_true, UNSTABLE, "all((0, 1, nan); isfinite)", true); yields!( isfinite_false, + UNSTABLE, "any((infinite, -infinite, []); isfinite)", false ); -yields!(isnormal_true, "1 | isnormal", true); +yields!(isnormal_true, UNSTABLE, "1 | isnormal", true); yields!( isnormal_false, + UNSTABLE, "any(0, nan, infinite, -infinite, []; isnormal)", false ); -yields!(join_empty, r#"[] | join(" ")"#, json!(null)); +yields!(join_empty, UNSTABLE, r#"[] | join(" ")"#, json!(null)); yields!( join_strs, + UNSTABLE, r#"["Hello", "world"] | join(" ")"#, "Hello world" ); // 2 + 1 + 3 + 1 + 4 + 1 + 5 -yields!(join_nums, r#"[2, 3, 4, 5] | join(1)"#, 17); +yields!(join_nums, UNSTABLE, r#"[2, 3, 4, 5] | join(1)"#, 17); -yields!(map, "[1, 2] | map(.+1)", [2, 3]); +yields!(map, UNSTABLE, "[1, 2] | map(.+1)", [2, 3]); yields!( keys, + UNSTABLE, r#"{"foo":null,"abc":null,"fax":null,"az":null} | keys"#, ["abc", "az", "fax", "foo"] ); // this diverges from jq, which returns [null] -yields!(last_empty, "[last({}[])]", json!([])); -yields!(last_some, "last(1, 2, 3)", 3); +yields!(last_empty, UNSTABLE, "[last({}[])]", json!([])); +yields!(last_some, UNSTABLE, "last(1, 2, 3)", 3); -yields!(logb_inf, "infinite | logb | . == infinite", true); -yields!(logb_nan, "nan | logb | isnan", true); -yields!(logb_neg_inf, "-infinite | logb | . == infinite", true); +yields!(logb_inf, UNSTABLE, "infinite | logb | . == infinite", true); +yields!(logb_nan, UNSTABLE, "nan | logb | isnan", true); +yields!( + logb_neg_inf, + UNSTABLE, + "-infinite | logb | . == infinite", + true +); yields!( logb_range, + UNSTABLE, "[-2.2, -2, -1, 1, 2, 2.2] | map(logb)", [1.0, 1.0, 0.0, 0.0, 1.0, 1.0] ); -yields!(logb_zero, "0 | logb | . == -infinite", true); +yields!(logb_zero, UNSTABLE, "0 | logb | . == -infinite", true); // here we diverge from jq, which returns ["a", "b", "A", "B"] yields!( match_many, + UNSTABLE, r#""ABab" | [match("a", "b"; "", "i") | .string]"#, ["a", "A", "b", "B"] ); #[test] fn min_max() { - give(json!([1, 4, 2]), "min", json!(1)); - give(json!([1, 4, 2]), "max", json!(4)); + give(UNSTABLE, json!([1, 4, 2]), "min", json!(1)); + give(UNSTABLE, json!([1, 4, 2]), "max", json!(4)); // TODO: find examples where `min_by(f)` yields output different from `min` // (and move it then to jaq-core/tests/tests.rs) give( + UNSTABLE, json!([{"a": {"b": {"c": 1}}}, {"a": {"b": {"c": 4}}}, {"a": {"b": {"c": 2}}}]), "min_by(.a.b.c)", json!({"a": {"b": {"c": 1}}}), ); give( + UNSTABLE, json!([{"a": {"b": {"c": 1}}}, {"a": {"b": {"c": 4}}}, {"a": {"b": {"c": 2}}}]), "max_by(.a.b.c)", json!({"a": {"b": {"c": 4}}}), @@ -197,77 +256,106 @@ fn min_max() { #[test] fn nth() { let fib = "[0,1] | recurse([.[1], add]) | .[0]"; - give(json!(10), &format!("nth(.; {})", fib), json!(55)); + give(UNSTABLE, json!(10), &format!("nth(.; {})", fib), json!(55)); let fib = "[0,1] | recurse([.[1], add])[0]"; - give(json!(10), &format!("nth(.; {})", fib), json!(55)); + give(UNSTABLE, json!(10), &format!("nth(.; {})", fib), json!(55)); } -yields!(paths_num, "1 | [paths]", json!([])); -yields!(paths_null, "null | [paths]", json!([])); -yields!(paths_arr, "[1, 2] | [paths]", [[0], [1]]); +yields!(paths_num, UNSTABLE, "1 | [paths]", json!([])); +yields!(paths_null, UNSTABLE, "null | [paths]", json!([])); +yields!(paths_arr, UNSTABLE, "[1, 2] | [paths]", [[0], [1]]); yields!( paths_arr_obj, + UNSTABLE, "{a: [1, [2]], b: {c: 3}} | [paths]", json!([["a"], ["a", 0], ["a", 1], ["a", 1, 0], ["b"], ["b", "c"]]) ); -yields!(range_many, "[range(-1, 1; 0, 2)]", json!([-1, -1, 0, 1, 1])); +yields!( + range_many, + UNSTABLE, + "[range(-1, 1; 0, 2)]", + json!([-1, -1, 0, 1, 1]) +); #[test] fn range_reverse() { - give(json!(null), "[range(1, 2)]", json!([0, 0, 1])); + give(UNSTABLE, json!(null), "[range(1, 2)]", json!([0, 0, 1])); - give(json!(3), "[range(.)] | reverse", json!([2, 1, 0])); + give(UNSTABLE, json!(3), "[range(.)] | reverse", json!([2, 1, 0])); } yields!( recurse_update, + UNSTABLE, "[0, [1, 2], 3] | recurse |= (.+1)? // .", json!([1, [2, 3], 4]) ); // the following tests show that sums are evaluated lazily // (otherwise this would not terminate) -yields!(limit_inf_suml, "[limit(3; recurse(.+1) + 0)]", [0, 1, 2]); -yields!(limit_inf_sumr, "[limit(3; 0 + recurse(.+1))]", [0, 1, 2]); +yields!( + limit_inf_suml, + UNSTABLE, + "[limit(3; recurse(.+1) + 0)]", + [0, 1, 2] +); +yields!( + limit_inf_sumr, + UNSTABLE, + "[limit(3; 0 + recurse(.+1))]", + [0, 1, 2] +); -yields!(limit_inf_path, "[limit(2; [1] | .[repeat(0)])]", [1, 1]); +yields!( + limit_inf_path, + UNSTABLE, + "[limit(2; [1] | .[repeat(0)])]", + [1, 1] +); #[test] fn recurse() { let x = json!({"a":0,"b":[1]}); - gives(x.clone(), "recurse", [x, json!(0), json!([1]), json!(1)]); + gives( + UNSTABLE, + x.clone(), + "recurse", + [x, json!(0), json!([1]), json!(1)], + ); let y = [json!(1), json!(2), json!(3)]; - gives(json!(1), "recurse(.+1; . < 4)", y); + gives(UNSTABLE, json!(1), "recurse(.+1; . < 4)", y); let y = [json!(2), json!(4), json!(16)]; - gives(json!(2), "recurse(. * .; . < 20)", y); + gives(UNSTABLE, json!(2), "recurse(. * .; . < 20)", y); let x = json!([[[0], 1], 2, [3, [4]]]); let y = json!([[[1], 2], 3, [4, [5]]]); - give(x.clone(), "(.. | scalars) |= .+1", y); + give(UNSTABLE, x.clone(), "(.. | scalars) |= .+1", y); let f = ".. |= if . < [] then .+1 else . + [42] end"; let y = json!([[[1, 43], 2, 43], 3, [4, [5, 43], 43], 43]); // jq gives: `[[[1, 42], 2, 42], 3, [4, [5, 42], 42], 42]` - give(x.clone(), f, y); + give(UNSTABLE, x.clone(), f, y); let f = ".. |= if . < [] then .+1 else [42] + . end"; let y = json!([43, [43, [43, 1], 2], 3, [43, 4, [43, 5]]]); // jq fails here with: "Cannot index number with number" - give(x.clone(), f, y); + give(UNSTABLE, x.clone(), f, y); } yields!( recurse3, + UNSTABLE, "[1 | recurse(if . < 3 then .+1 else empty end)]", [1, 2, 3] ); yields!( reduce_recurse, + UNSTABLE, "reduce recurse(if . == 1000 then empty else .+1 end) as $x (0; . + $x)", 500500 ); @@ -279,6 +367,7 @@ const RECURSE_PATHS: &str = "def paths: yields!( recurse_paths, + UNSTABLE, &(RECURSE_PATHS.to_owned() + "{a: [1, [2]], b: {c: 3}} | [paths]"), json!([["a"], ["a", 0], ["a", 1], ["a", 1, 0], ["b"], ["b", "c"]]) ); @@ -291,6 +380,7 @@ const RECURSE_FLATTEN: &str = "def flatten($d): yields!( recurse_flatten, + UNSTABLE, &(RECURSE_FLATTEN.to_owned() + "[[[1], 2], 3] | flatten(1)"), json!([[1], 2, 3]) ); @@ -298,7 +388,7 @@ yields!( #[test] fn repeat() { let y = json!([0, 1, 0, 1]); - give(json!([0, 1]), "[limit(4; repeat(.[]))]", y); + give(UNSTABLE, json!([0, 1]), "[limit(4; repeat(.[]))]", y); } // the implementation of scalb in jq (or the libm.a library) doesn't @@ -306,16 +396,19 @@ fn repeat() { // and rejects them in scalbln yields!( scalb_eqv_pow2, + UNSTABLE, "[-2.2, -1.1, -0.01, 0, 0.01, 1.1, 2.2] | [scalb(1.0; .[])] == [pow(2.0; .[])]", true ); yields!( scalb_nan, + UNSTABLE, "[scalb(nan, 1; nan, 1)] == [nan, nan, nan, 2.0]", true ); yields!( scalb_range, + UNSTABLE, "[scalb(-2.5, 0, 2.5; 2, 2.5, 3) * 1000 | round]", [-10000, -14142, -20000, 0, 0, 0, 10000, 14142, 20000] ); @@ -323,6 +416,7 @@ yields!( // here we diverge from jq, which returns ["a", "b", "a", "A", "b", "B"] yields!( scan, + UNSTABLE, r#""abAB" | [scan("a", "b"; "g", "gi")]"#, // TODO: is this order really desired? json!(["a", "a", "A", "b", "b", "B"]) @@ -330,37 +424,55 @@ yields!( #[test] fn select() { - give(json!([1, 2]), ".[] | select(.>1)", json!(2)); - give(json!([0, 1, 2]), "map(select(.<1, 1<.))", json!([0, 2])); + give(UNSTABLE, json!([1, 2]), ".[] | select(.>1)", json!(2)); + give( + UNSTABLE, + json!([0, 1, 2]), + "map(select(.<1, 1<.))", + json!([0, 2]), + ); let v = json!([null, false, true, 1, 1.0, "", "a", [], [0], {}, {"a": 1}]); let iterables = json!([[], [0], {}, {"a": 1}]); let scalars = json!([null, false, true, 1, 1.0, "", "a"]); let values = json!([false, true, 1, 1.0, "", "a", [], [0], {}, {"a": 1}]); - give(v.clone(), ".[] | nulls", json!(null)); - give(v.clone(), "[.[] | booleans]", json!([false, true])); - give(v.clone(), "[.[] | numbers]", json!([1, 1.0])); - give(v.clone(), "[.[] | strings]", json!(["", "a"])); - give(v.clone(), "[.[] | arrays]", json!([[], [0]])); - give(v.clone(), "[.[] | objects]", json!([{}, {"a": 1}])); - give(v.clone(), "[.[] | iterables]", iterables); - give(v.clone(), "[.[] | scalars]", scalars); - give(v.clone(), "[.[] | values]", values); + give(UNSTABLE, v.clone(), ".[] | nulls", json!(null)); + give( + UNSTABLE, + v.clone(), + "[.[] | booleans]", + json!([false, true]), + ); + give(UNSTABLE, v.clone(), "[.[] | numbers]", json!([1, 1.0])); + give(UNSTABLE, v.clone(), "[.[] | strings]", json!(["", "a"])); + give(UNSTABLE, v.clone(), "[.[] | arrays]", json!([[], [0]])); + give( + UNSTABLE, + v.clone(), + "[.[] | objects]", + json!([{}, {"a": 1}]), + ); + give(UNSTABLE, v.clone(), "[.[] | iterables]", iterables); + give(UNSTABLE, v.clone(), "[.[] | scalars]", scalars); + give(UNSTABLE, v.clone(), "[.[] | values]", values); } yields!( significand_inf, + UNSTABLE, "infinite | significand | . == infinite", true ); -yields!(significand_nan, "nan | significand | isnan", true); +yields!(significand_nan, UNSTABLE, "nan | significand | isnan", true); yields!( significand_neg_inf, + UNSTABLE, "-infinite | significand | . == -infinite", true ); yields!( significand_range, + UNSTABLE, "[-123.456, -2.2, -2, -1, 0, 0.00001, 1, 2, 2.2, 123.456] | map(significand)", [-1.929, -1.1, -1.0, -1.0, 0.0, 1.31072, 1.0, 1.0, 1.1, 1.929] ); @@ -368,32 +480,34 @@ yields!( #[test] fn transpose() { let y = json!([[1, 2], [3, null]]); - give(json!([[1, 3], [2]]), "transpose", y); + give(UNSTABLE, json!([[1, 3], [2]]), "transpose", y); let y = json!([[1, 2], [3, 4]]); - give(json!([[1, 3], [2, 4]]), "transpose", y); + give(UNSTABLE, json!([[1, 3], [2, 4]]), "transpose", y); } #[test] fn typ() { - give(json!({"a": 1, "b": 2}), "type", json!("object")); - give(json!([0, 1]), "type", json!("array")); - give(json!("Hello"), "type", json!("string")); - give(json!(1), "type", json!("number")); - give(json!(1.0), "type", json!("number")); - give(json!(true), "type", json!("boolean")); - give(json!(null), "type", json!("null")); + give(UNSTABLE, json!({"a": 1, "b": 2}), "type", json!("object")); + give(UNSTABLE, json!([0, 1]), "type", json!("array")); + give(UNSTABLE, json!("Hello"), "type", json!("string")); + give(UNSTABLE, json!(1), "type", json!("number")); + give(UNSTABLE, json!(1.0), "type", json!("number")); + give(UNSTABLE, json!(true), "type", json!("boolean")); + give(UNSTABLE, json!(null), "type", json!("null")); } #[test] fn walk() { give( + UNSTABLE, json!([[4, 1, 7], [8, 5, 2], [3, 6, 9]]), r#"walk(if . < [] then . else sort end)"#, json!([[1, 4, 7], [2, 5, 8], [3, 6, 9]]), ); give( + UNSTABLE, json!({"a": {"b": 1, "c": 2}}), r#"walk(if . < {} then . + 1 else . + {"l": length} end)"#, json!({"a": {"b": 2, "c": 3, "l": 2}, "l": 1}), @@ -403,47 +517,64 @@ fn walk() { #[test] fn while_until() { give( + UNSTABLE, json!(1), "[while(. < 100; . * 2)]", json!([1, 2, 4, 8, 16, 32, 64]), ); give( + UNSTABLE, json!("a"), "[while(length < 4; . + \"a\")]", json!(["a", "aa", "aaa"]), ); give( + UNSTABLE, json!([1, 2, 3]), "[while(length > 0; .[1:])]", json!([[1, 2, 3], [2, 3], [3]]), ); - give(json!(50), "until(. > 100; . * 2)", json!(200)); + give(UNSTABLE, json!(50), "until(. > 100; . * 2)", json!(200)); give( + UNSTABLE, json!([1, 2, 3]), "until(length == 1; .[1:]) | .[0]", json!(3), ); give( + UNSTABLE, json!(5), "[.,1] | until(.[0] < 1; [.[0] - 1, .[1] * .[0]]) | .[1]", json!(120), ); } -yields!(sub, r#""XYxyXYxy" | sub("x";"Q")"#, "XYQyXYxy"); -yields!(gsub, r#""XYxyXYxy" | gsub("x";"Q")"#, "XYQyXYQy"); -yields!(isub, r#""XYxyXYxy" | sub("x";"Q";"i")"#, "QYxyXYxy"); -yields!(gisub, r#""XYxyXYxy" | gsub("x";"Q";"i")"#, "QYQyQYQy"); +yields!(sub, UNSTABLE, r#""XYxyXYxy" | sub("x";"Q")"#, "XYQyXYxy"); +yields!(gsub, UNSTABLE, r#""XYxyXYxy" | gsub("x";"Q")"#, "XYQyXYQy"); +yields!( + isub, + UNSTABLE, + r#""XYxyXYxy" | sub("x";"Q";"i")"#, + "QYxyXYxy" +); +yields!( + gisub, + UNSTABLE, + r#""XYxyXYxy" | gsub("x";"Q";"i")"#, + "QYQyQYQy" +); // swap adjacent occurrences of upper- and lower-case characters yields!( gsub_swap, + UNSTABLE, r#""XYxyXYxy" | gsub("(?[A-Z])(?[a-z])"; .lower + .upper)"#, "XxYyXxYy" ); // this diverges from jq, which yields ["XxYy", "!XxYy", "Xx!Yy", "!Xx!Yy"] yields!( gsub_many, + UNSTABLE, r#""XxYy" | [gsub("(?[A-Z])"; .upper, "!" + .upper)]"#, ["XxYy", "Xx!Yy", "!XxYy", "!Xx!Yy"] ); diff --git a/jaq-syn/Cargo.toml b/jaq-syn/Cargo.toml index e1afbcc5..9601b922 100644 --- a/jaq-syn/Cargo.toml +++ b/jaq-syn/Cargo.toml @@ -8,9 +8,12 @@ readme = "../README.md" description = "Syntax of the jaq language" repository = "https://github.com/01mf02/jaq" keywords = ["json", "query", "jq"] +rust-version = "1.63" [features] default = ["serde"] +unstable = ["unstable-flag"] +unstable-flag = [] [dependencies] serde = { version = "1.0.137", features = ["derive"], optional = true } diff --git a/jaq-syn/src/def.rs b/jaq-syn/src/def.rs index 24158fe9..73b1e0ef 100644 --- a/jaq-syn/src/def.rs +++ b/jaq-syn/src/def.rs @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; /// Call to a filter identified by a name type `N` with arguments of type `A`. #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "unstable-flag", non_exhaustive)] pub struct Call { /// Name of the filter, e.g. `map` pub name: N, @@ -16,6 +17,11 @@ pub struct Call { } impl Call { + /// Create a call to a filter identified by a name type `N` with arguments of type `A`. + pub fn new(name: N, args: Vec) -> Self { + Self { name, args } + } + /// Apply a function to the call arguments. pub fn map_args(self, f: impl FnMut(A) -> B) -> Call { Call { @@ -28,6 +34,7 @@ impl Call { /// A definition, such as `def map(f): [.[] | f];`. #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Clone, Debug)] +#[cfg_attr(feature = "unstable-flag", non_exhaustive)] pub struct Def { /// left-hand side, i.e. what shall be defined, e.g. `map(f)` pub lhs: Call, @@ -35,6 +42,13 @@ pub struct Def { pub rhs: Rhs, } +impl Def { + /// Create a definition. + pub fn new(lhs: Call, rhs: Rhs) -> Self { + Self { lhs, rhs } + } +} + /// Argument of a definition, such as `$v` or `f` in `def foo($v; f): ...`. /// /// In jq, we can bind filters in three different ways: @@ -47,6 +61,7 @@ pub struct Def { /// In the third case, we bind `f` to a filter `fx` #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "unstable-flag", non_exhaustive)] pub enum Arg { /// binding to a variable Var(V), @@ -124,9 +139,17 @@ impl Arg { /// (Potentially empty) sequence of definitions, followed by a filter. #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Clone, Debug)] +#[cfg_attr(feature = "unstable-flag", non_exhaustive)] pub struct Main { /// Definitions at the top of the filter pub defs: Vec>, /// Body of the filter, e.g. `[.[] | f`. pub body: Spanned, } + +impl Main { + /// Construct a (potentially empty) sequence of definitions, followed by a filter. + pub fn new(defs: Vec>, body: Spanned) -> Self { + Self { defs, body } + } +} diff --git a/jaq-syn/src/filter.rs b/jaq-syn/src/filter.rs index 8c16ef70..8e78a872 100644 --- a/jaq-syn/src/filter.rs +++ b/jaq-syn/src/filter.rs @@ -5,15 +5,20 @@ use core::fmt; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; -/// Assignment operators, such as `=`, `|=` (update), and `+=`, `-=`, ... +/// Assignment operators (`=`, `|=`, `//=`, `+=`, …) #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Clone, Debug)] +#[cfg_attr(feature = "unstable-flag", non_exhaustive)] pub enum AssignOp { - /// `=` + /// Assignment operator (`=`) Assign, - /// `|=` + /// Update-assignment operator (`|=`) Update, - /// `+=`, `-=`, `*=`, `/=`, `%=` + /// Alternation update-assignment operator (`//=`) + #[cfg(feature = "unstable-flag")] + #[deprecated(note = "Unstable feature")] + AltUpdate, + /// Arithmetic update-assignment operator (`+=`, `-=`, `*=`, `/=`, `%=`, …) UpdateWith(MathOp), } @@ -22,30 +27,35 @@ impl fmt::Display for AssignOp { match self { Self::Assign => "=".fmt(f), Self::Update => "|=".fmt(f), + #[cfg(feature = "unstable-flag")] + #[allow(deprecated)] + Self::AltUpdate => "//=".fmt(f), Self::UpdateWith(op) => write!(f, "{op}="), } } } -/// Binary operators, such as `|`, `,`, `//`, ... +/// Binary operators (`|`, `,`, `//`, `or`, `and`, `+=`, …, `=`, …, `<`, …) #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Clone, Debug)] +#[cfg_attr(feature = "unstable-flag", non_exhaustive)] pub enum BinaryOp { - /// Application, i.e. `l | r` if no string is given, else `l as $x | r` + /// Binding operator (`EXP as $x | EXP`) if identifier (`x`) is given, + /// otherwise application operator (`EXP | EXP`) Pipe(Option), - /// Concatenation, i.e. `l, r` + /// Concatenation operator (`,`) Comma, - /// Alternation, i.e. `l // r` + /// Alternation operator (`,`) Alt, - /// Logical disjunction, i.e. `l or r` + /// Logical disjunction operator (`or`) Or, - /// Logical conjunction, i.e. `l and r` + /// Logical conjunction operator (`and`) And, - /// Arithmetic operation, e.g. `l + r`, `l - r`, ... + /// Arithmetical operator (`+`, `-`, `*`, `/`, `%`, …) Math(MathOp), - /// Assignment, i.e. `l = r`, `l |= r`, `l += r`, `l -= r`, ... + /// Assignment operator (`=`, `|=`, `//=`, `+=`, …) Assign(AssignOp), - /// Ordering operation, e.g. `l == r`, `l <= r`, ... + /// Comparative operator (`<`, `<=`, `>`, `>=`, `==`, `!=`, …) Ord(OrdOp), } @@ -55,6 +65,7 @@ pub enum BinaryOp { /// consists of two elements, namely `(.): 1` and `b: 2`. #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Clone, Debug)] +#[cfg_attr(feature = "unstable-flag", non_exhaustive)] pub enum KeyVal { /// Both key and value are proper filters, e.g. `{(.+1): .+2}` Filter(T, T), @@ -76,6 +87,7 @@ impl KeyVal { /// Common information for folding filters (such as `reduce` and `foreach`) #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Clone, Debug)] +#[cfg_attr(feature = "unstable-flag", non_exhaustive)] pub struct Fold { /// Generator pub xs: F, @@ -87,59 +99,72 @@ pub struct Fold { pub f: F, } +impl Fold { + /// Create common information for folding filters. + pub fn new(xs: F, x: String, init: F, f: F) -> Self { + Self { xs, x, init, f } + } +} + /// Type of folding filter. #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Copy, Clone, Debug)] +#[cfg_attr(feature = "unstable-flag", non_exhaustive)] pub enum FoldType { - /// return only the final value of fold + /// Return only the final value of fold Reduce, - /// return initial, intermediate, and final values of fold + /// Return initial, intermediate, and final values of fold For, - /// return intermediate and final values of fold + /// Return intermediate and final values of fold Foreach, } /// Function from value to stream of values, such as `.[] | add / length`. #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Clone, Debug)] +#[cfg_attr(feature = "unstable-flag", non_exhaustive)] pub enum Filter { - /// Call to another filter, e.g. `map(.+1)` + /// Call to another filter (`FILTER(…)`), e.g. `map(.+1)` Call(C, Vec>), - /// Variable, such as $x (without leading '$') + /// Variable (`$x`), only storing identifier (`x`) Var(V), /// Integer or floating-point number. Num(Num), /// String Str(Box>>), - /// Array, empty if `None` + /// Array (`[…]`), empty if `None` (`[]`) Array(Option>>), - /// Object, specifying its key-value pairs + /// Object (`{…}`), specifying its key-value pairs Object(Vec>>), - /// Identity, i.e. `.` + /// Nullary identity operation (`.`) Id, - /// Path such as `.`, `.a`, `.[][]."b"` + /// Path, e.g. `.`, `.a`, and `.[][]."b"` Path(Box>, Path), - /// If-then-else + /// If-then-else (`if EXP then EXP else EXP end`) if alternative expression + /// is given, otherwise if-then (`if EXP then EXP end`) Ite( Vec<(Spanned, Spanned)>, Option>>, ), - /// `reduce` and `foreach`, e.g. `reduce .[] as $x (0; .+$x)` + /// Folding filters, such as `reduce` (`reduce f as $x (g; h)`) and + /// `foreach` (`foreach f as $x (g; h; i)`), e.g. + /// `reduce .[] as $x (0; .+$x)` /// /// The first field indicates whether to yield intermediate results /// (`false` for `reduce` and `true` for `foreach`). Fold(FoldType, Fold>>), - /// `try` and optional `catch` + /// Try-catch (`try EXP catch EXP`) if handler expression is given, + /// otherwise try (`try EXP`) TryCatch(Box>, Option>>), - /// Error suppression, e.g. `keys?` + /// Error suppression (`EXP?`), e.g. `keys?` Try(Box>), - /// Negation + /// Unary negation operation (`-EXP`) Neg(Box>), - /// Recursion (`..`) + /// Nullary recursive descent operation (`..`) Recurse, - /// Binary operation, such as `0, 1`, `[] | .[]`, `.[] += 1`, `0 == 0`, ... + /// Binary operation, e.g. `0, 1`, `[] | .[]`, `.[] += 1`, `0 == 0`, … Binary(Box>, BinaryOp, Box>), } @@ -156,13 +181,13 @@ impl From>> for Filter { } impl Filter { - /// Create a binary expression, such as `1 + 2`. + /// Create a binary operation expression, e.g. `1 + 2`, … pub fn binary(a: Spanned, op: BinaryOp, b: Spanned) -> Spanned { let span = a.1.start..b.1.end; (Self::Binary(Box::new(a), op, Box::new(b)), span) } - /// Create a path expression, such as `keys[]` or `.a.b`. + /// Create a path expression, e.g. `keys[]`, `.a.b`, … /// /// Here, `f` is a filter on whose output the path is executed on, /// such as `keys` and `.` in the example above. diff --git a/jaq-syn/src/ops.rs b/jaq-syn/src/ops.rs index f980dc06..069b5fd7 100644 --- a/jaq-syn/src/ops.rs +++ b/jaq-syn/src/ops.rs @@ -3,24 +3,25 @@ use core::ops::{Add, Div, Mul, Rem, Sub}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; -/// Arithmetic operation, such as `+`, `-`, `*`, `/`, `%`. +/// Binary arithmetical operators (`+`, `-`, `*`, `/`, `%`, …) #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "unstable-flag", non_exhaustive)] pub enum MathOp { - /// Addition + /// Addition operator (`+`) Add, - /// Subtraction + /// Subtraction operator (`-`) Sub, - /// Multiplication + /// Multiplication operator (`*`) Mul, - /// Division + /// Division operator (`/`) Div, - /// Remainder + /// Remainder operator (`%`) Rem, } impl MathOp { - /// Perform the arithmetic operation on the given inputs. + /// Perform the arithmetical operation on the given inputs. pub fn run(&self, l: I, r: I) -> O where I: Add + Sub + Mul + Div + Rem, @@ -47,26 +48,27 @@ impl fmt::Display for MathOp { } } -/// An operation that orders two values, such as `<`, `<=`, `>`, `>=`, `==`, `!=`. +/// Binary comparative operators (`<`, `<=`, `>`, `>=`, `==`, `!=`, …) #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "unstable-flag", non_exhaustive)] pub enum OrdOp { - /// Less-than (<). + /// Less-than operation (`<`). Lt, - /// Less-than or equal (<=). + /// Less-than or equal-to operation (`<=`). Le, - /// Greater-than (>). + /// Greater-than operation (`>`). Gt, - /// Greater-than or equal (>=). + /// Greater-than or equal-to operation (`>=`). Ge, - /// Equals (=). + /// Equal-to operation (`=`). Eq, - /// Not equals (!=). + /// Not equal-to operation (`!=`). Ne, } impl OrdOp { - /// Perform the ordering operation on the given inputs. + /// Perform the comparative operation on the given inputs. pub fn run(&self, l: &I, r: &I) -> bool { match self { Self::Gt => l > r, diff --git a/jaq-syn/src/path.rs b/jaq-syn/src/path.rs index f491a109..20f21441 100644 --- a/jaq-syn/src/path.rs +++ b/jaq-syn/src/path.rs @@ -9,6 +9,7 @@ pub type Path = Vec<(Part>, Opt)>; /// A part of a path, such as `[]`, `a`, and `[1:]` in `.[].a?[1:]`. #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Clone, Debug)] +#[cfg_attr(feature = "unstable-flag", non_exhaustive)] pub enum Part { /// Access arrays with integer and objects with string indices Index(I), @@ -23,6 +24,7 @@ pub enum Part { /// Annotating them with `?` makes them *optional*. #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Copy, Clone, Debug)] +#[cfg_attr(feature = "unstable-flag", non_exhaustive)] pub enum Opt { /// Return nothing if the input cannot be accessed with the path Optional, diff --git a/jaq-syn/src/string.rs b/jaq-syn/src/string.rs index 94709a03..86ba5cf6 100644 --- a/jaq-syn/src/string.rs +++ b/jaq-syn/src/string.rs @@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize}; /// A part of an interpolated string. #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Clone, Debug)] +#[cfg_attr(feature = "unstable-flag", non_exhaustive)] pub enum Part { /// constant string Str(String), @@ -34,6 +35,7 @@ impl Part { /// A possibly interpolated string. #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Clone, Debug)] +#[cfg_attr(feature = "unstable-flag", non_exhaustive)] pub struct Str { /// optional filter that is applied to the output of interpolated filters /// (`tostring` if not given) @@ -43,6 +45,11 @@ pub struct Str { } impl Str { + /// Create a possibly interpolated string. + pub fn new(fmt: Option>, parts: Vec>) -> Self { + Self { fmt, parts } + } + /// Apply a function to the contained filters. pub fn map(self, mut f: impl FnMut(T) -> U) -> Str { Str { diff --git a/jaq-syn/src/test.rs b/jaq-syn/src/test.rs index 271bb820..094d94c5 100644 --- a/jaq-syn/src/test.rs +++ b/jaq-syn/src/test.rs @@ -3,7 +3,11 @@ use alloc::vec::Vec; /// A single jq unit test. +#[cfg_attr(feature = "unstable-flag", non_exhaustive)] pub struct Test { + /// Unstable flag + #[cfg(feature = "unstable-flag")] + pub unstable: bool, /// jq filter pub filter: S, /// input value in JSON format @@ -13,12 +17,19 @@ pub struct Test { } /// Parser for a jq unit test. -pub struct Parser(I); +#[cfg_attr(feature = "unstable-flag", non_exhaustive)] +pub struct Parser { + #[cfg_attr(not(feature = "unstable-flag"), allow(unused))] + unstable: bool, + lines: I, +} impl Parser { /// Create a parser from an iterator over lines. - pub fn new(lines: I) -> Self { - Self(lines) + pub fn new(#[cfg(feature = "unstable-flag")] unstable: bool, lines: I) -> Self { + #[cfg(not(feature = "unstable-flag"))] + let unstable = false; + Self { unstable, lines } } } @@ -26,8 +37,10 @@ impl, I: Iterator> Iterator for Pars type Item = Test; fn next(&mut self) -> Option { - let lines = &mut self.0; + let lines = &mut self.lines; Some(Test { + #[cfg(feature = "unstable-flag")] + unstable: self.unstable, filter: lines.find(|l| !(l.is_empty() || l.starts_with('#')))?, input: lines.next()?, output: lines.take_while(|l| !l.is_empty()).collect(), diff --git a/jaq/Cargo.toml b/jaq/Cargo.toml index fe49d9b1..984328c5 100644 --- a/jaq/Cargo.toml +++ b/jaq/Cargo.toml @@ -12,7 +12,25 @@ categories = ["command-line-utilities", "compilers", "parser-implementations"] rust-version = "1.64" [features] -default = ["mimalloc"] +default = [ + "mimalloc", + "unstable-flag", +] +unstable = [ + "unstable-flag", + "jaq-syn/unstable", + "jaq-parse/unstable", + "jaq-interpret/unstable", + "jaq-core/unstable", + "jaq-std/unstable", +] +unstable-flag = [ + "jaq-syn/unstable-flag", + "jaq-parse/unstable-flag", + "jaq-interpret/unstable-flag", + "jaq-core/unstable-flag", + "jaq-std/unstable-flag", +] [dependencies] jaq-syn = { version = "1.1.0", path = "../jaq-syn" } @@ -27,6 +45,7 @@ clap = { version = "4.0.0", features = ["derive"] } colored_json = "3.0.1" env_logger = { version = "0.10.0", default-features = false } hifijson = "0.2.0" +log = { version = "0.4.17" } memmap2 = "0.9" mimalloc = { version = "0.1.29", default-features = false, optional = true } serde_json = { version = "1.0.81", features = [ "arbitrary_precision", "preserve_order" ] } diff --git a/jaq/src/main.rs b/jaq/src/main.rs index e4d1a86f..61efebfe 100644 --- a/jaq/src/main.rs +++ b/jaq/src/main.rs @@ -12,6 +12,11 @@ static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; #[derive(Parser)] #[command(version)] struct Cli { + /// Enable unstable features. + #[cfg(feature = "unstable-flag")] + #[arg(long)] + unstable: bool, + /// Use null as single input value #[arg(short, long)] null_input: bool, @@ -126,18 +131,28 @@ fn real_main() -> Result { .format_target(false) .init(); + #[cfg(feature = "unstable")] + log::info!("jaq: unstable features enabled"); + #[cfg(feature = "unstable-flag")] + let unstable = cli.unstable; + #[cfg(not(feature = "unstable-flag"))] + let unstable = false; + if unstable { + log::info!("jaq: unstable feature flag enabled"); + } + if let Some(test_file) = &cli.run_tests { - return Ok(run_tests(std::fs::File::open(test_file)?)); + return Ok(run_tests(unstable, std::fs::File::open(test_file)?)); } let (vars, ctx) = binds(&cli)?.into_iter().unzip(); let mut args = cli.args.iter(); let filter = match &cli.from_file { - Some(file) => parse(&std::fs::read_to_string(file)?, vars)?, + Some(file) => parse(unstable, &std::fs::read_to_string(file)?, vars)?, None => { if let Some(filter) = args.next() { - parse(filter, vars)? + parse(unstable, filter, vars)? } else { Filter::default() } @@ -232,12 +247,30 @@ fn args_named(var_val: &[(String, Val)]) -> Val { Val::obj(args.collect()) } -fn parse(filter_str: &str, vars: Vec) -> Result> { +fn parse( + #[cfg_attr(not(feature = "unstable-flag"), allow(unused_variables))] unstable: bool, + filter_str: &str, + vars: Vec, +) -> Result> { let mut defs = ParseCtx::new(vars); - defs.insert_natives(jaq_core::core()); - defs.insert_defs(jaq_std::std()); + defs.insert_natives(jaq_core::core( + #[cfg(feature = "unstable-flag")] + unstable, + )); + defs.insert_defs(jaq_std::std( + #[cfg(feature = "unstable-flag")] + unstable, + )); assert!(defs.errs.is_empty()); - let (filter, errs) = jaq_parse::parse(filter_str, jaq_parse::main()); + let (filter, errs) = jaq_parse::parse( + #[cfg(feature = "unstable-flag")] + unstable, + filter_str, + jaq_parse::main( + #[cfg(feature = "unstable-flag")] + unstable, + ), + ); if !errs.is_empty() { return Err(errs .into_iter() @@ -533,10 +566,14 @@ fn report<'a>(e: chumsky::error::Simple) -> ariadne::Report<'a> { } fn run_test(test: jaq_syn::test::Test) -> Result<(Val, Val), Error> { + #[cfg(feature = "unstable-flag")] + let unstable = test.unstable; + #[cfg(not(feature = "unstable-flag"))] + let unstable = false; let inputs = RcIter::new(Box::new(core::iter::empty())); let ctx = Ctx::new(Vec::new(), &inputs); - let filter = parse(&test.filter, Vec::new())?; + let filter = parse(unstable, &test.filter, Vec::new())?; use hifijson::token::Lex; let json = |s: String| { @@ -550,9 +587,16 @@ fn run_test(test: jaq_syn::test::Test) -> Result<(Val, Val), Error> { Ok((Val::arr(expect?), Val::arr(obtain.map_err(Error::Jaq)?))) } -fn run_tests(file: std::fs::File) -> ExitCode { +fn run_tests( + #[cfg_attr(not(feature = "unstable-flag"), allow(unused_variables))] unstable: bool, + file: std::fs::File, +) -> ExitCode { let lines = io::BufReader::new(file).lines().map(|l| l.unwrap()); - let tests = jaq_syn::test::Parser::new(lines); + let tests = jaq_syn::test::Parser::new( + #[cfg(feature = "unstable-flag")] + unstable, + lines, + ); let (mut passed, mut total) = (0, 0); for test in tests { diff --git a/jaq/tests/golden.rs b/jaq/tests/golden.rs index 7420f36b..8f41b5cc 100644 --- a/jaq/tests/golden.rs +++ b/jaq/tests/golden.rs @@ -1,4 +1,4 @@ -use std::{env, fs, io, path, process, str}; +use std::{env, io, process, str}; fn golden_test(args: &[&str], input: &str, out_ex: &str) -> io::Result<()> { let mut child = process::Command::new(env!("CARGO_BIN_EXE_jaq"))