refactor(kclvm-runner): encapsulate dylib generating, linking and exe…

…cuting in kclvm/lib.rs into kclvm-runner

Encapsulate dylibs generating and executing in kclvm/lib.rs into kclvm-runner.
Add struct "KclvmAssembler" in kclvm-runner/runner.rs to provide method "gen_dylibs" for dylibs generating.
Add struct "KclvmLinker" in kclvm-runner/runner.rs to provide method "link_all_dylibs" for dylib linking.
Add method "execute" in kclvm-runner/lib.rs to encapsulate dylibs generating(gen_dylib), dylib linking(link_all_dylib) and running(runner.run) together
The main purpose of separating the three parts from kclvm/lib.rs and encapsulating them into kclvm-runner is to support reuse in kcl-vet.

fix #67

kclvm/runner/Cargo.toml

@@ -15,6 +15,7 @@ libc = "0.2.112"
 indexmap = "1.0"
 fslock = "0.2.1"
 libloading = "0.7.3"
+threadpool = "1.0"
 
 kclvm-ast = {path = "../ast", version = "0.1.0"}
 kclvm-parser = {path = "../parser", version = "0.1.0"}

kclvm/runner/src/command.rs

@@ -423,7 +423,6 @@ impl Command {
         } else {
             "clang"
         };
-
 
         if let Some(s) = Self::find_it(clang_exe) {
             return s.to_str().unwrap().to_string();

kclvm/runner/src/lib.rs

@@ -1,2 +1,42 @@
+use kclvm_ast::ast::Program;
+use kclvm_sema::resolver::scope::ProgramScope;
+use runner::{ExecProgramArgs, KclvmAssembler, KclvmLinker, KclvmRunner, KclvmRunnerOptions};
+
 pub mod command;
 pub mod runner;
+
+#[cfg(test)]
+mod tests;
+
+/// Returns the KCL program executing result as Result< a_json_string, an_err_string >.
+/// program: ast.Program returned by kclvm-parser.
+/// scope: ProgramScope returned by kclvm-sema
+///
+/// In the method, dylibs is generated by KclvmAssembler, and method "KclvmAssembler::gen_dylibs"
+/// will return dylibs path in a "Vec<String>";
+///
+/// After linking all dylibs by KclvmLinker, method "KclvmLinker::link_all_dylibs" will return a path
+/// for dylib.
+///
+/// At last, KclvmRunner will be constructed and call method "run" to execute the kcl program.
+pub fn execute(
+    program: Program,
+    scope: ProgramScope,
+    plugin_agent: u64,
+    args: &ExecProgramArgs,
+) -> Result<String, String> {
+    // generate dylibs
+    let dylib_paths = KclvmAssembler::gen_dylibs(program, scope, plugin_agent);
+
+    // link dylibs
+    let dylib_path = KclvmLinker::link_all_dylibs(dylib_paths, plugin_agent);
+
+    // run
+    let runner = KclvmRunner::new(
+        dylib_path.as_str(),
+        Some(KclvmRunnerOptions {
+            plugin_agent_ptr: plugin_agent,
+        }),
+    );
+    runner.run(&args)
+}

kclvm/runner/src/runner.rs

@@ -1,6 +1,19 @@
+use std::{
+    collections::HashMap,
+    path::{Path, PathBuf},
+    sync::mpsc::channel,
+};
+
+use indexmap::IndexMap;
+use kclvm_compiler::codegen::{llvm::emit_code, EmitOptions};
+use kclvm_config::cache::{load_pkg_cache, save_pkg_cache, CacheOption};
+use kclvm_sema::resolver::scope::ProgramScope;
 use serde::{Deserialize, Serialize};
 
 use kclvm_ast::ast;
+use threadpool::ThreadPool;
+
+use crate::command::Command;
 
 #[allow(non_camel_case_types)]
 pub type kclvm_char_t = i8;
@@ -79,6 +92,158 @@ pub struct KclvmRunnerOptions {
     pub plugin_agent_ptr: u64,
 }
 
+/// KclvmAssembler is mainly responsible for generating bytecode,
+/// LLVM IR or dylib, and take the result of kclvm-parser and kclvm-sema
+/// as input.
+pub struct KclvmAssembler {}
+const LL_FILE: &str = "_a.out";
+impl KclvmAssembler {
+    /// Generate the dylibs and return file paths from .
+    pub fn gen_dylibs(
+        program: ast::Program,
+        scope: ProgramScope,
+        plugin_agent: u64,
+    ) -> Vec<String> {
+        // gen bc or ll_file
+        let path = std::path::Path::new(LL_FILE);
+        if path.exists() {
+            std::fs::remove_file(path).unwrap();
+        }
+        for entry in glob::glob(&format!("{}*.ll", LL_FILE)).unwrap() {
+            match entry {
+                Ok(path) => {
+                    if path.exists() {
+                        std::fs::remove_file(path).unwrap();
+                    }
+                }
+                Err(e) => println!("{:?}", e),
+            };
+        }
+
+        let cache_dir = Path::new(&program.root)
+            .join(".kclvm")
+            .join("cache")
+            .join(kclvm_version::get_full_version());
+        if !cache_dir.exists() {
+            std::fs::create_dir_all(&cache_dir).unwrap();
+        }
+        let mut compile_progs: IndexMap<
+            String,
+            (
+                ast::Program,
+                IndexMap<String, IndexMap<String, String>>,
+                PathBuf,
+            ),
+        > = IndexMap::default();
+        for (pkgpath, modules) in program.pkgs {
+            let mut pkgs = HashMap::new();
+            pkgs.insert(pkgpath.clone(), modules);
+            let compile_prog = ast::Program {
+                root: program.root.clone(),
+                main: program.main.clone(),
+                pkgs,
+                cmd_args: vec![],
+                cmd_overrides: vec![],
+            };
+            compile_progs.insert(
+                pkgpath,
+                (compile_prog, scope.import_names.clone(), cache_dir.clone()),
+            );
+        }
+        let pool = ThreadPool::new(4);
+        let (tx, rx) = channel();
+        let prog_count = compile_progs.len();
+        for (pkgpath, (compile_prog, import_names, cache_dir)) in compile_progs {
+            let tx = tx.clone();
+            pool.execute(move || {
+                let root = &compile_prog.root;
+                let is_main_pkg = pkgpath == kclvm_ast::MAIN_PKG;
+                let file = if is_main_pkg {
+                    PathBuf::from(&pkgpath)
+                } else {
+                    cache_dir.join(&pkgpath)
+                };
+                let ll_file = file.to_str().unwrap();
+                let ll_path = format!("{}.ll", ll_file);
+                let dylib_path = format!("{}{}", ll_file, Command::get_lib_suffix());
+                let mut ll_path_lock =
+                    fslock::LockFile::open(&format!("{}.lock", ll_path)).unwrap();
+                ll_path_lock.lock().unwrap();
+                if Path::new(&ll_path).exists() {
+                    std::fs::remove_file(&ll_path).unwrap();
+                }
+                let dylib_path = if is_main_pkg {
+                    emit_code(
+                        &compile_prog,
+                        import_names,
+                        &EmitOptions {
+                            from_path: None,
+                            emit_path: Some(&ll_file),
+                            no_link: true,
+                        },
+                    )
+                    .expect("Compile KCL to LLVM error");
+                    let mut cmd = Command::new(plugin_agent);
+                    cmd.run_clang_single(&ll_path, &dylib_path)
+                } else {
+                    // If AST module has been modified, ignore the dylib cache
+                    let dylib_relative_path: Option<String> =
+                        load_pkg_cache(root, &pkgpath, CacheOption::default());
+                    match dylib_relative_path {
+                        Some(dylib_relative_path) => {
+                            if dylib_relative_path.starts_with('.') {
+                                dylib_relative_path.replacen(".", root, 1)
+                            } else {
+                                dylib_relative_path
+                            }
+                        }
+                        None => {
+                            emit_code(
+                                &compile_prog,
+                                import_names,
+                                &EmitOptions {
+                                    from_path: None,
+                                    emit_path: Some(&ll_file),
+                                    no_link: true,
+                                },
+                            )
+                            .expect("Compile KCL to LLVM error");
+                            let mut cmd = Command::new(plugin_agent);
+                            let dylib_path = cmd.run_clang_single(&ll_path, &dylib_path);
+                            let dylib_relative_path = dylib_path.replacen(root, ".", 1);
+
+                            save_pkg_cache(
+                                root,
+                                &pkgpath,
+                                dylib_relative_path,
+                                CacheOption::default(),
+                            );
+                            dylib_path
+                        }
+                    }
+                };
+                if Path::new(&ll_path).exists() {
+                    std::fs::remove_file(&ll_path).unwrap();
+                }
+                ll_path_lock.unlock().unwrap();
+                tx.send(dylib_path)
+                    .expect("channel will be there waiting for the pool");
+            });
+        }
+        rx.iter().take(prog_count).collect::<Vec<String>>()
+    }
+}
+
+/// KclvmLinker is mainly responsible for linking the dylibs.
+pub struct KclvmLinker {}
+impl KclvmLinker {
+    /// Link the dylibs generated by method "gen_bc_or_ll_file".
+    pub fn link_all_dylibs(dylib_paths: Vec<String>, plugin_agent: u64) -> String {
+        let mut cmd = Command::new(plugin_agent);
+        cmd.link_dylibs(&dylib_paths, "")
+    }
+}
+
 pub struct KclvmRunner {
     opts: KclvmRunnerOptions,
     lib: libloading::Library,

kclvm/runner/src/test_datas/init_check_order_0/main.k

@@ -0,0 +1,11 @@
+schema Person:
+    name: str
+    age: int
+    gender: str
+    info: str = "{}, {}, {} years old".format(name, gender, age)
+
+alice = Person {
+    "name": "alice",
+    "age": 10,
+    "gender": "female"
+}

kclvm/runner/src/test_datas/init_check_order_0/stdout.golden.json

@@ -0,0 +1 @@
+{"alice": {"name": "alice", "age": 10, "gender": "female", "info": "alice, female, 10 years old", "__settings__": {"output_type": "INLINE", "__schema_type__": "__main__.Person"}}}

kclvm/runner/src/test_datas/init_check_order_1/main.k

@@ -0,0 +1,61 @@
+schema Name:
+    mixin [UpperMixin]
+    firstName: str
+    lastName: str
+    upper: str
+
+    # print("init name")
+
+schema Person(Name):
+    gender: str
+    title: str
+    info: str
+
+    # print("init person")
+
+schema Girl(Person):
+    mixin [TitleMixin, InfoMixin]
+    gender: str = "female"
+    added: str = "some girl attr"
+
+    # print("init girl")
+
+    check:
+        gender == "female", "gender should be female in Girl"
+
+schema Boy(Person):
+    mixin [TitleMixin, InfoMixin]
+    gender: str = "male"
+    added: str = "some boy attr"
+
+    # print("init boy")
+
+    check:
+        gender == "male", "gender should be male in Boy"
+
+schema UpperMixin:
+    # print("init upperMixin")
+    upper: str = lastName.upper()
+
+schema TitleMixin:
+    # print("init title mixin")
+    if gender == "female":
+        title = "Ms.{}".format(lastName)
+    else:
+        title = "Mr.{}".format(lastName)
+
+schema InfoMixin:
+    # print("init info mixin")
+    info = "{}, {}".format(title, gender)
+
+alice = Girl {
+    "firstName": "Alice",
+    "lastName": "Smith"
+}
+
+# print(" ===")
+
+bob = Boy {
+    "firstName": "Bob",
+    "lastName": "Green"
+}

kclvm/runner/src/test_datas/init_check_order_1/stdout.golden.json

@@ -0,0 +1 @@
+{"alice": {"firstName": "Alice", "lastName": "Smith", "upper": "SMITH", "__settings__": {"output_type": "INLINE", "__schema_type__": "__main__.InfoMixin"}, "gender": "female", "title": "Ms.Smith", "info": "Ms.Smith, female", "added": "some girl attr"}, "bob": {"firstName": "Bob", "lastName": "Green", "upper": "GREEN", "__settings__": {"output_type": "INLINE", "__schema_type__": "__main__.InfoMixin"}, "gender": "male", "title": "Mr.Green", "info": "Mr.Green, male", "added": "some boy attr"}}

kclvm/runner/src/test_datas/normal_2/main.k

@@ -0,0 +1,11 @@
+
+schema NumberMap:
+    [num: str]: int
+
+    check:
+        int(num) % 2 == 0
+
+numMap = NumberMap {
+    str(0): 0
+    str(2): 2
+}

kclvm/runner/src/test_datas/normal_2/stdout.golden.json

@@ -0,0 +1 @@
+{"numMap":{"0":0,"2":2,"__settings__":{"__schema_type__":"__main__.NumberMap","output_type":"INLINE"}}}

kclvm/runner/src/test_datas/type_annotation_not_full_2/main.k

@@ -0,0 +1,6 @@
+schema A[arg=1]:
+    a: int = arg
+
+a1 = A()
+a2 = A(2)
+a3 = A(arg=3)

kclvm/runner/src/test_datas/type_annotation_not_full_2/stdout.golden.json

@@ -0,0 +1 @@
+{"a1":{"__settings__":{"__schema_type__":"__main__.A","output_type":"INLINE"},"a":1},"a2":{"__settings__":{"__schema_type__":"__main__.A","output_type":"INLINE"},"a":2},"a3":{"__settings__":{"__schema_type__":"__main__.A","output_type":"INLINE"},"a":3}}