Add code generation for cranelift

Next, debugging.

src/codegen/clir.rs → src/codegen/clif.rs

@@ -8,17 +8,15 @@ use smallvec::{smallvec, SmallVec};
 
 use crate::builtins;
 use crate::bytecode::Accum;
-use crate::codegen::{intrinsics, Backend, CodeGenerator, Config, Op, Ref, Sig, StrReg};
+use crate::codegen::{intrinsics, Backend, CodeGenerator, Config, Jit, Op, Ref, Sig, StrReg};
 use crate::common::{traverse, CompileError, Either, FileSpec, NodeIx, NumTy, Result, Stage};
 use crate::compile::{self, Typer};
 use crate::runtime::{self, UniqueStr};
 
 use std::convert::TryFrom;
+use std::io;
 use std::mem;
 
-// TODO:
-// * initialize data-structures, call main
-//
 // TODO (cleanup; after tests are passing):
 // * move floatfunc/bitwise stuff into llvm module
 // * move llvm module under codegen
@@ -34,6 +32,9 @@ struct FuncInfo {
 
 const PLACEHOLDER: Ref = (compile::UNUSED, compile::Ty::Null);
 
+// for debugging
+const DUMP_IR: bool = false;
+
 /// After a function is declared, some additional information is required to map parameteres to
 /// variables. `Prelude` contains that information.
 struct Prelude {
@@ -89,7 +90,7 @@ struct Shared {
 }
 
 /// Toplevel information
-struct GlobalContext {
+pub(crate) struct Generator {
     shared: Shared,
     ctx: FunctionBuilderContext,
     cctx: codegen::Context,
@@ -136,8 +137,16 @@ fn ty_to_clifty(ty: compile::Ty, ptr_ty: Type) -> Result<Type> {
     }
 }
 
-impl GlobalContext {
-    pub(crate) fn init(typer: &mut Typer, config: Config) -> Result<GlobalContext> {
+impl Jit for Generator {
+    fn main_pointers(&mut self) -> Result<Stage<*const u8>> {
+        Ok(self
+            .mains
+            .map_ref(|id| self.shared.module.get_finalized_function(*id)))
+    }
+}
+
+impl Generator {
+    pub(crate) fn init(typer: &mut Typer, config: Config) -> Result<Generator> {
         let builder = JITBuilder::new(default_libcall_names());
         let mut regstate = RegistrationState { builder };
         intrinsics::register_all(&mut regstate)?;
@@ -150,7 +159,7 @@ impl GlobalContext {
             sig: cctx.func.signature.clone(),
         };
         // TODO: define codegen-specific data and only export that
-        let mut global = GlobalContext {
+        let mut global = Generator {
             shared,
             ctx: FunctionBuilderContext::new(),
             cctx,
@@ -176,7 +185,7 @@ impl GlobalContext {
             },
         };
         global.mains = stage;
-        // TODO: finalize_definitions, finish, on module
+        global.shared.module.finalize_definitions();
         Ok(global)
     }
 
@@ -488,6 +497,9 @@ impl<'a> View<'a> {
         self.builder.ins().jump(bbs[0], &[]);
         self.builder.seal_all_blocks();
         self.builder.finalize();
+        if DUMP_IR {
+            eprintln!("{}", self.builder.display(None));
+        }
         Ok(())
     }
 

src/codegen/mod.rs

@@ -3,11 +3,15 @@
 use crate::{
     builtins,
     bytecode::{self, Accum},
-    common::{FileSpec, NumTy, Result},
+    common::{FileSpec, NumTy, Result, Stage},
     compile,
+    pushdown::FieldSet,
     runtime::{self, UniqueStr},
 };
 
+use std::marker::PhantomData;
+use std::mem;
+
 /// Options used to configure a code-generating backend.
 #[derive(Copy, Clone)]
 pub struct Config {
@@ -17,9 +21,9 @@ pub struct Config {
 
 #[macro_use]
 pub(crate) mod intrinsics;
-pub(crate) mod clir;
+pub(crate) mod clif;
 
-// TODO: start on clir
+use intrinsics::Runtime;
 
 pub(crate) type Ref = (NumTy, compile::Ty);
 pub(crate) type StrReg<'a> = bytecode::Reg<runtime::Str<'a>>;
@@ -86,6 +90,149 @@ pub enum FunctionAttr {
     ArgmemOnly,
 }
 
+/// A handle around a generated main function, potentially allocated dynamically with the given
+/// lifetime.
+#[derive(Copy, Clone)]
+pub(crate) struct MainFunction<'a> {
+    fn_ptr: *const u8,
+    _marker: PhantomData<&'a ()>,
+}
+
+unsafe impl<'a> Send for MainFunction<'a> {}
+
+impl<'a> MainFunction<'a> {
+    /// Create a `MainFunction` of arbitrary lifetime.
+    ///
+    /// This is only called from the default impl in the `Jit` trait. General use is extra unsafe
+    /// as it allows the caller to "cook up" whatever lifetime suits them.
+    fn from_ptr(fn_ptr: *const u8) -> MainFunction<'a> {
+        MainFunction {
+            fn_ptr,
+            _marker: PhantomData,
+        }
+    }
+
+    /// Call the generated function on the given runtime.
+    ///
+    /// Unsafe because it pretends an arbitrary memory location contains code, and then runs that
+    /// code.
+    pub(crate) unsafe fn invoke<'b>(&self, rt: &mut Runtime<'b>) {
+        mem::transmute::<*const u8, unsafe extern "C" fn(*mut Runtime<'b>)>(self.fn_ptr)(rt)
+    }
+}
+
+pub(crate) trait Jit: Sized {
+    fn main_pointers(&mut self) -> Result<Stage<*const u8>>;
+    fn main_functions<'a>(&'a mut self) -> Result<Stage<MainFunction<'a>>> {
+        Ok(self.main_pointers()?.map(MainFunction::from_ptr))
+    }
+}
+
+/// Run the main function (or functions, for parallel scripts) given a [`Jit`] and the various
+/// other parameters required to construct a runtime.
+pub(crate) unsafe fn run_main<R, FF, J>(
+    mut jit: J,
+    stdin: R,
+    ff: FF,
+    used_fields: &FieldSet,
+    named_columns: Option<Vec<&[u8]>>,
+    num_workers: usize,
+) -> Result<()>
+where
+    R: intrinsics::IntoRuntime,
+    FF: runtime::writers::FileFactory,
+    J: Jit,
+{
+    let mut rt = stdin.into_runtime(ff, used_fields, named_columns);
+    let main = jit.main_functions()?;
+    match main {
+        Stage::Main(m) => Ok(m.invoke(&mut rt)),
+        Stage::Par {
+            begin,
+            main_loop,
+            end,
+        } => {
+            rt.concurrent = true;
+            // This triply-nested macro is here to allow mutable access to a "runtime" struct
+            // as well as mutable access to the same "read_files" value. The generated code is
+            // pretty awful; It may be worth a RefCell just to clean up.
+            with_input!(&mut rt.input_data, |(_, read_files)| {
+                let reads = read_files.try_resize(num_workers.saturating_sub(1));
+                if num_workers <= 1 || reads.len() == 0 || main_loop.is_none() {
+                    // execute serially.
+                    for main in begin.into_iter().chain(main_loop).chain(end) {
+                        main.invoke(&mut rt);
+                    }
+                    return Ok(());
+                }
+                #[cfg(not(debug_assertions))]
+                {
+                    std::panic::set_hook(Box::new(|pi| {
+                        if let Some(s) = pi.payload().downcast_ref::<&str>() {
+                            if s.len() > 0 {
+                                eprintln_ignore!("{}", s);
+                            }
+                        }
+                    }));
+                }
+                if let Some(begin) = begin {
+                    begin.invoke(&mut rt);
+                }
+                if let Err(_) = rt.core.write_files.flush_stdout() {
+                    return Ok(());
+                }
+                let (sender, receiver) = crossbeam_channel::bounded(reads.len());
+                let launch_data: Vec<_> = reads
+                    .into_iter()
+                    .enumerate()
+                    .map(|(i, reader)| {
+                        (
+                            reader,
+                            sender.clone(),
+                            rt.core.shuttle(i as runtime::Int + 2),
+                        )
+                    })
+                    .collect();
+                with_input!(&mut rt.input_data, |(_, read_files)| {
+                    let old_read_files = mem::replace(&mut read_files.inputs, Default::default());
+                    let main_loop_fn = main_loop.unwrap();
+                    let scope_res = crossbeam::scope(|s| {
+                        for (reader, sender, shuttle) in launch_data.into_iter() {
+                            s.spawn(move |_| {
+                                let mut runtime = Runtime {
+                                    concurrent: true,
+                                    core: shuttle(),
+                                    input_data: reader().into(),
+                                };
+                                main_loop_fn.invoke(&mut runtime);
+                                sender.send(runtime.core.extract_result()).unwrap();
+                            });
+                        }
+                        rt.core.vars.pid = 1;
+                        main_loop_fn.invoke(&mut rt);
+                        rt.core.vars.pid = 0;
+                        mem::drop(sender);
+                        with_input!(&mut rt.input_data, |(_, read_files)| {
+                            while let Ok(res) = receiver.recv() {
+                                rt.core.combine(res);
+                            }
+                            rt.concurrent = false;
+                            if let Some(end) = end {
+                                read_files.inputs = old_read_files;
+                                end.invoke(&mut rt);
+                            }
+                        });
+                    });
+                    if let Err(_) = scope_res {
+                        return err!("failed to execute parallel script");
+                    }
+                });
+            });
+            Ok(())
+        }
+    }
+}
+
 pub(crate) trait Backend {
     type Ty: Clone;
     // mappings from compile::Ty to Self::Ty

src/common.rs

@@ -348,11 +348,11 @@ impl Default for FileSpec {
 }
 
 pub(crate) fn traverse<T>(o: Option<Result<T>>) -> Result<Option<T>> {
-            match o {
-                Some(e) => Ok(Some(e?)),
-                None => Ok(None),
-            }
-        }
+    match o {
+        Some(e) => Ok(Some(e?)),
+        None => Ok(None),
+    }
+}
 
 #[cfg(test)]
 mod tests {

src/compile.rs

@@ -216,8 +216,24 @@ pub(crate) fn run_llvm<'a>(
     let used_fields = typer.used_fields.clone();
     let named_cols = typer.named_columns.take();
     unsafe {
-        let mut gen = Generator::init(&mut typer, cfg)?;
-        gen.run_main(reader, ff, &used_fields, named_cols, cfg.num_workers)
+        let gen = Generator::init(&mut typer, cfg)?;
+        codegen::run_main(gen, reader, ff, &used_fields, named_cols, cfg.num_workers)
+    }
+}
+
+pub(crate) fn run_cranelift<'a>(
+    ctx: &mut cfg::ProgramContext<'a, &'a str>,
+    reader: impl codegen::intrinsics::IntoRuntime,
+    ff: impl runtime::writers::FileFactory,
+    cfg: codegen::Config,
+) -> Result<()> {
+    use codegen::clif::Generator;
+    let mut typer = Typer::init_from_ctx(ctx)?;
+    let used_fields = typer.used_fields.clone();
+    let named_cols = typer.named_columns.take();
+    unsafe {
+        let gen = Generator::init(&mut typer, cfg)?;
+        codegen::run_main(gen, reader, ff, &used_fields, named_cols, cfg.num_workers)
     }
 }