Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions examples/jpar-stream.ilo
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
-- JSONL streaming: rdjl reads a JSONL file line by line and returns
-- L (R _ t) — a list of per-line parse results. Empty lines are skipped,
-- so log files with stray blank lines parse cleanly. Each entry is
-- wrapped in a Result so a single malformed line doesn't poison the
-- whole stream.

-- Hydrate a JSONL fixture, then count the parsed entries. In a real
-- pipeline rdjl is used on a file produced by a log shipper or another
-- process; here we generate one in-place so the example is hermetic.
prepare p:t>R t t;wrl p ["{\"k\":1}", "{\"k\":2}", "{\"k\":3}"]
n-lines p:t>n;w=prepare p;es=rdjl p;len es

-- run: n-lines /tmp/ilo-jpar-stream-example.jsonl
-- out: 3
4 changes: 4 additions & 0 deletions src/builtins.rs
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ pub enum Builtin {
Jpth,
Jdmp,
Jpar,
Rdjl,

// HTTP
Get,
Expand Down Expand Up @@ -221,6 +222,7 @@ impl Builtin {
"jpth" => Some(Builtin::Jpth),
"jdmp" => Some(Builtin::Jdmp),
"jpar" => Some(Builtin::Jpar),
"rdjl" => Some(Builtin::Rdjl),
"get" => Some(Builtin::Get),
"post" => Some(Builtin::Post),
"get-many" => Some(Builtin::GetMany),
Expand Down Expand Up @@ -328,6 +330,7 @@ impl Builtin {
Builtin::Jpth => "jpth",
Builtin::Jdmp => "jdmp",
Builtin::Jpar => "jpar",
Builtin::Rdjl => "rdjl",
Builtin::Get => "get",
Builtin::Post => "post",
Builtin::GetMany => "get-many",
Expand Down Expand Up @@ -456,6 +459,7 @@ mod tests {
"solve",
"inv",
"det",
"rdjl",
];
for name in &all {
let b = Builtin::from_name(name).unwrap_or_else(|| panic!("missing builtin: {name}"));
Expand Down
28 changes: 28 additions & 0 deletions src/interpreter/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2197,6 +2197,34 @@ fn call_function(env: &mut Env, name: &str, args: Vec<Value>) -> Result<Value> {
)),
};
}
if builtin == Some(Builtin::Rdjl) && args.len() == 1 {
return match &args[0] {
Value::Text(path) => match std::fs::read_to_string(path) {
Ok(content) => {
let mut items: Vec<Value> = Vec::new();
for line in content.split('\n') {
if line.is_empty() {
continue;
}
let parsed = match serde_json::from_str::<serde_json::Value>(line) {
Ok(v) => Value::Ok(Box::new(serde_json_to_value(v))),
Err(e) => Value::Err(Box::new(Value::Text(e.to_string()))),
};
items.push(parsed);
}
Ok(Value::List(items))
}
Err(e) => Err(RuntimeError::new(
"ILO-R009",
format!("rdjl failed to read '{}': {}", path, e),
)),
},
other => Err(RuntimeError::new(
"ILO-R009",
format!("rdjl requires text path, got {:?}", other),
)),
};
}

if builtin == Some(Builtin::Env) && args.len() == 1 {
return match &args[0] {
Expand Down
23 changes: 23 additions & 0 deletions src/verify.rs
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,7 @@ const BUILTINS: &[(&str, &[&str], &str)] = &[
("fmt", &["t"], "t"), // variadic: fmt template arg1 arg2 … — checked specially
("fmt2", &["n", "n"], "t"),
("jpar", &["t"], "R ? t"),
("rdjl", &["t"], "L (R ? t)"),
// Higher-order: map/flt/fld take a function ref as first arg (special-cased in builtin_check_args)
("map", &["fn", "list"], "list"),
("flt", &["fn", "list"], "list"),
Expand Down Expand Up @@ -1429,6 +1430,28 @@ fn builtin_check_args(
errors,
)
}
"rdjl" => {
if let Some(arg) = arg_types.first()
&& !compatible(arg, &Ty::Text)
{
errors.push(VerifyError {
code: "ILO-T013",
function: func_ctx.to_string(),
message: format!("'rdjl' expects t (path), got {arg}"),
hint: None,
span,
is_warning: false,
});
}
// rdjl path → L (R _ t): list of per-line parse results
(
Ty::List(Box::new(Ty::Result(
Box::new(Ty::Unknown),
Box::new(Ty::Text),
))),
errors,
)
}
"map" => {
// map fn:F a b xs:L a → L b
// map fn:F a c b ctx:c xs:L a → L b (closure-bind variant)
Expand Down
24 changes: 17 additions & 7 deletions src/vm/compile_cranelift.rs
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ struct HelperFuncs {
jpth: FuncId,
jdmp: FuncId,
jpar: FuncId,
rdjl: FuncId,
call: FuncId,
// Type predicates
isnum: FuncId,
Expand Down Expand Up @@ -273,6 +274,7 @@ fn declare_all_helpers(module: &mut ObjectModule) -> HelperFuncs {
jpth: declare_helper(module, "jit_jpth", 2, 1),
jdmp: declare_helper(module, "jit_jdmp", 1, 1),
jpar: declare_helper(module, "jit_jpar", 1, 1),
rdjl: declare_helper(module, "jit_rdjl", 1, 1),
call: declare_helper(module, "jit_call", 4, 1),
// Type predicates
isnum: declare_helper(module, "jit_isnum", 1, 1),
Expand Down Expand Up @@ -993,13 +995,14 @@ fn compile_function_body(
| OP_UNWRAP | OP_RECFLD | OP_RECFLD_NAME | OP_LISTGET | OP_INDEX | OP_STR
| OP_HD | OP_AT | OP_FMT2 | OP_TL | OP_REV | OP_SRT | OP_SRTDESC | OP_SLC
| OP_TAKE | OP_DROP | OP_SPL | OP_CAT | OP_GET | OP_POST | OP_GETH | OP_POSTH
| OP_GETMANY | OP_ENV | OP_JPTH | OP_JDMP | OP_JPAR | OP_MAPNEW | OP_MGET
| OP_MSET | OP_MDEL | OP_MKEYS | OP_MVALS | OP_LISTNEW | OP_LISTAPPEND
| OP_RECNEW | OP_RECWITH | OP_PRT | OP_RD | OP_RDL | OP_WR | OP_WRL | OP_TRM
| OP_UPR | OP_LWR | OP_CAP | OP_PADL | OP_PADR | OP_UNQ | OP_UNIQBY
| OP_PARTITION | OP_FRQ | OP_NUM | OP_RGXSUB | OP_ZIP | OP_ENUMERATE | OP_RANGE
| OP_WINDOW | OP_CHUNKS | OP_CUMSUM | OP_SETUNION | OP_SETINTER | OP_SETDIFF
| OP_FFT | OP_IFFT | OP_TRANSPOSE | OP_MATMUL | OP_INV | OP_SOLVE => {
| OP_GETMANY | OP_ENV | OP_JPTH | OP_JDMP | OP_JPAR | OP_RDJL | OP_MAPNEW
| OP_MGET | OP_MSET | OP_MDEL | OP_MKEYS | OP_MVALS | OP_LISTNEW
| OP_LISTAPPEND | OP_RECNEW | OP_RECWITH | OP_PRT | OP_RD | OP_RDL | OP_WR
| OP_WRL | OP_TRM | OP_UPR | OP_LWR | OP_CAP | OP_PADL | OP_PADR | OP_UNQ
| OP_UNIQBY | OP_PARTITION | OP_FRQ | OP_NUM | OP_RGXSUB | OP_ZIP
| OP_ENUMERATE | OP_RANGE | OP_WINDOW | OP_CHUNKS | OP_CUMSUM | OP_SETUNION
| OP_SETINTER | OP_SETDIFF | OP_FFT | OP_IFFT | OP_TRANSPOSE | OP_MATMUL
| OP_INV | OP_SOLVE => {
non_num_write[a] = true;
non_bool_write[a] = true;
}
Expand Down Expand Up @@ -3055,6 +3058,13 @@ fn compile_function_body(
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_RDJL => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, module, helpers.rdjl);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
// ── Type predicates ──
OP_ISNUM => {
let bv = builder.use_var(vars[b_idx]);
Expand Down
12 changes: 11 additions & 1 deletion src/vm/jit_cranelift.rs
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ struct HelperFuncs {
jpth: FuncId,
jdmp: FuncId,
jpar: FuncId,
rdjl: FuncId,
call: FuncId,
// Type predicates
isnum: FuncId,
Expand Down Expand Up @@ -271,6 +272,7 @@ fn register_helpers(builder: &mut JITBuilder) {
("jit_jpth", jit_jpth as *const u8),
("jit_jdmp", jit_jdmp as *const u8),
("jit_jpar", jit_jpar as *const u8),
("jit_rdjl", jit_rdjl as *const u8),
("jit_call", jit_call as *const u8),
// Type predicates
("jit_isnum", jit_isnum as *const u8),
Expand Down Expand Up @@ -411,6 +413,7 @@ fn declare_all_helpers(module: &mut JITModule) -> HelperFuncs {
jpth: declare_helper(module, "jit_jpth", 2, 1),
jdmp: declare_helper(module, "jit_jdmp", 1, 1),
jpar: declare_helper(module, "jit_jpar", 1, 1),
rdjl: declare_helper(module, "jit_rdjl", 1, 1),
call: declare_helper(module, "jit_call", 4, 1),
// Type predicates
isnum: declare_helper(module, "jit_isnum", 1, 1),
Expand Down Expand Up @@ -1016,7 +1019,7 @@ fn compile_function_body(
| OP_SETUNION | OP_SETINTER | OP_SETDIFF
| OP_INV | OP_SOLVE
| OP_SPL | OP_CAT | OP_GET | OP_POST | OP_GETH | OP_POSTH | OP_GETMANY
| OP_ENV | OP_JPTH | OP_JDMP | OP_JPAR
| OP_ENV | OP_JPTH | OP_JDMP | OP_JPAR | OP_RDJL
| OP_MAPNEW | OP_MGET | OP_MSET | OP_MDEL | OP_MKEYS | OP_MVALS
| OP_LISTNEW | OP_LISTAPPEND
| OP_RECNEW | OP_RECWITH
Expand Down Expand Up @@ -3602,6 +3605,13 @@ fn compile_function_body(
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_RDJL => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, module, helpers.rdjl);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
// ── Type predicates (1-arg → 1 return) ──
OP_ISNUM => {
let bv = builder.use_var(vars[b_idx]);
Expand Down
88 changes: 84 additions & 4 deletions src/vm/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,7 @@ pub(crate) const OP_PADL: u8 = 121; // R[A] = pad_left(R[B], R[C]) (text, width
pub(crate) const OP_PADR: u8 = 122; // R[A] = pad_right(R[B], R[C]) (text, width → text)

pub(crate) const OP_GETMANY: u8 = 136; // R[A] = get_many(R[B]) (L t → L (R t t), concurrent fan-out)
pub(crate) const OP_RDJL: u8 = 135; // R[A] = rdjl(R[B]) (read JSONL file → L (R _ t))

// Linear algebra advanced.
pub(crate) const OP_SOLVE: u8 = 126; // R[A] = solve(R[B], R[C]) — solve Ax = b
Expand Down Expand Up @@ -2271,6 +2272,15 @@ impl RegCompiler {
}
return ra;
}
(Builtin::Rdjl, 1) => {
// rdjl path → L (R _ t). Not a Result-returning op, so `!`
// is unsupported here; the verifier rejects it via the
// standard return-type check.
let rb = self.compile_expr(&args[0]);
let ra = self.alloc_reg();
self.emit_abc(OP_RDJL, ra, rb, 0);
return ra;
}
// Map builtins
(Builtin::Mmap, 0) => {
let ra = self.alloc_reg();
Expand Down Expand Up @@ -2786,10 +2796,10 @@ fn chunk_is_all_numeric(chunk: &Chunk) -> bool {
OP_RECNEW | OP_LISTNEW | OP_RECWITH | OP_WRAPOK | OP_WRAPERR | OP_STR | OP_CAT
| OP_SPL | OP_REV | OP_SRT | OP_SRTDESC | OP_SLC | OP_TAKE | OP_DROP | OP_UNQ
| OP_UNIQBY | OP_FRQ | OP_PARTITION | OP_LISTAPPEND | OP_JPAR | OP_JDMP | OP_ENV
| OP_GET | OP_GETH | OP_GETMANY | OP_POST | OP_POSTH | OP_RD | OP_RDL | OP_WR
| OP_WRL | OP_MAPNEW | OP_MGET | OP_MSET | OP_MKEYS | OP_MVALS | OP_HD | OP_AT
| OP_LST | OP_TL | OP_FMT2 | OP_RGXSUB | OP_ZIP | OP_ENUMERATE | OP_WINDOW | OP_FFT
| OP_IFFT | OP_RANGE | OP_CHUNKS | OP_CUMSUM | OP_SETUNION | OP_SETINTER
| OP_GET | OP_GETH | OP_GETMANY | OP_POST | OP_POSTH | OP_RD | OP_RDL | OP_RDJL
| OP_WR | OP_WRL | OP_MAPNEW | OP_MGET | OP_MSET | OP_MKEYS | OP_MVALS | OP_HD
| OP_AT | OP_LST | OP_TL | OP_FMT2 | OP_RGXSUB | OP_ZIP | OP_ENUMERATE | OP_WINDOW
| OP_FFT | OP_IFFT | OP_RANGE | OP_CHUNKS | OP_CUMSUM | OP_SETUNION | OP_SETINTER
| OP_SETDIFF | OP_TRANSPOSE | OP_MATMUL | OP_INV | OP_SOLVE => {
return false;
}
Expand Down Expand Up @@ -6169,6 +6179,40 @@ impl<'a> VM<'a> {
};
reg_set!(a, result);
}
OP_RDJL => {
let a = ((inst >> 16) & 0xFF) as usize + base;
let b = ((inst >> 8) & 0xFF) as usize + base;
let v = reg!(b);
if !v.is_string() {
vm_err!(VmError::Type("rdjl requires a string path"));
}
// SAFETY: is_string() confirmed heap-tagged string with live RC.
let path = unsafe {
match v.as_heap_ref() {
HeapObj::Str(s) => s.as_str().to_owned(),
_ => unreachable!(),
}
};
match std::fs::read_to_string(&path) {
Ok(content) => {
let mut items: Vec<NanVal> = Vec::new();
for line in content.split('\n') {
if line.is_empty() {
continue;
}
let entry = match serde_json::from_str::<serde_json::Value>(line) {
Ok(parsed) => NanVal::heap_ok(serde_json_to_nanval(parsed)),
Err(e) => NanVal::heap_err(NanVal::heap_string(e.to_string())),
};
items.push(entry);
}
reg_set!(a, NanVal::heap_list(items));
}
Err(_) => {
vm_err!(VmError::Type("rdjl failed to read file"));
}
}
}
OP_SPL => {
let a = ((inst >> 16) & 0xFF) as usize + base;
let b = ((inst >> 8) & 0xFF) as usize + base;
Expand Down Expand Up @@ -10083,6 +10127,42 @@ pub(crate) extern "C" fn jit_jpar(a: u64) -> u64 {
}
}

#[cfg(feature = "cranelift")]
#[unsafe(no_mangle)]
pub(crate) extern "C" fn jit_rdjl(a: u64) -> u64 {
let v = NanVal(a);
if !v.is_string() {
return TAG_NIL;
}
let path = unsafe {
match v.as_heap_ref() {
HeapObj::Str(s) => s.as_str().to_owned(),
_ => unreachable!(),
}
};
match std::fs::read_to_string(&path) {
Ok(content) => {
let mut items: Vec<NanVal> = Vec::new();
for line in content.split('\n') {
if line.is_empty() {
continue;
}
let entry = match serde_json::from_str::<serde_json::Value>(line) {
Ok(parsed) => NanVal::heap_ok(serde_json_to_nanval(parsed)),
Err(e) => NanVal::heap_err(NanVal::heap_string(e.to_string())),
};
items.push(entry);
}
NanVal::heap_list(items).0
}
// On file-read failure JIT returns nil; the VM dispatch path raises a
// typed runtime error. JIT helper-callers don't have a runtime-error
// channel, so nil is the conventional signal here (matches jit_rd /
// jit_jpar conventions for non-string inputs).
Err(_) => TAG_NIL,
}
}

/// Call a VM function from JIT code. `func_idx` is the chunk index,
/// `regs` points to `n_args` u64 values. Returns the result as u64.
#[cfg(feature = "cranelift")]
Expand Down
Loading
Loading