From 6409a6d5dc4e1f86dcc6ea3e8b13198bc8ccacb8 Mon Sep 17 00:00:00 2001 From: zong-zhe Date: Mon, 4 Jul 2022 21:13:45 +0800 Subject: [PATCH] refactor(kclvm-runner): encapsulate lib generating, linking and executing into kclvm-runner. 1. Encapsulate generating of libs in "kclvm/src/lib.rs" and "kclvm/src/main.rs" into "kclvm/runner/assembler.rs". 2. Encapsulate linking of libs in "kclvm/src/lib.rs" and "kclvm/src/main.rs" into "kclvm/runner/linker.rs" 3. Encapsulate executing of libs in "kclvm/src/lib.rs" and "kclvm/src/main.rs" into "kclvm/runner/lib.rs". 4. A timer is added during the concurrent multi-file compilation to prevent KCLVM locked due to child thread panic. fix #67 #106 #82 --- .github/workflows/github-actions.yaml | 2 +- internal/kclvm_py/scripts/update-kclvm.sh | 1 - kclvm/runner/benches/bench_runner.rs | 10 +- kclvm/runner/src/assembler.rs | 304 ++++-------------- kclvm/runner/src/command.rs | 183 +---------- kclvm/runner/src/lib.rs | 17 +- kclvm/runner/src/linker.rs | 4 +- .../test_datas/exec_prog_args/default.json | 1 + .../import_abs_path/app-main/main.k | 7 + .../app-main/some1/pkg1/pkg1.k | 1 + .../import_abs_path/kcl.mod | 0 .../import_abs_path/some0/pkg1/pkg1.k | 1 + .../import_abs_path/some1/pkg1/pkg1.k | 1 + .../import_regular_module/kcl.mod | 0 .../import_regular_module/main.k | 3 + .../import_regular_module/mymodule.k | 6 + .../import_regular_module_as/kcl.mod | 0 .../import_regular_module_as/main.k | 3 + .../import_regular_module_as/mymodule.k | 6 + .../no_kcl_mod_file/main.k | 7 + .../no_kcl_mod_file/pkg1/pkg.k | 1 + .../no_kcl_mod_file/pkg2.k | 1 + .../no_kcl_mod_file/stdout.golden | 2 + .../relative_import/main.k | 4 + .../relative_import/mydir/mydir2/mymodule2.k | 6 + .../relative_import/mydir/mymodule.k | 6 + .../relative_import_as/main.k | 4 + .../mydir/mydir2/mymodule2.k | 6 + .../relative_import_as/mydir/mymodule.k | 6 + .../test_datas/settings_file/settings.json | 1 + .../test_datas/settings_file/settings.yaml | 14 + kclvm/runner/src/tests.rs | 281 +++++++++++++++- kclvm/src/main.rs | 1 + 33 files changed, 439 insertions(+), 451 deletions(-) create mode 100644 kclvm/runner/src/test_datas/exec_prog_args/default.json create mode 100644 kclvm/runner/src/test_datas/multi_file_compilation/import_abs_path/app-main/main.k create mode 100644 kclvm/runner/src/test_datas/multi_file_compilation/import_abs_path/app-main/some1/pkg1/pkg1.k create mode 100644 kclvm/runner/src/test_datas/multi_file_compilation/import_abs_path/kcl.mod create mode 100644 kclvm/runner/src/test_datas/multi_file_compilation/import_abs_path/some0/pkg1/pkg1.k create mode 100644 kclvm/runner/src/test_datas/multi_file_compilation/import_abs_path/some1/pkg1/pkg1.k create mode 100644 kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module/kcl.mod create mode 100644 kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module/main.k create mode 100644 kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module/mymodule.k create mode 100644 kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module_as/kcl.mod create mode 100644 kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module_as/main.k create mode 100644 kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module_as/mymodule.k create mode 100644 kclvm/runner/src/test_datas/multi_file_compilation/no_kcl_mod_file/main.k create mode 100644 kclvm/runner/src/test_datas/multi_file_compilation/no_kcl_mod_file/pkg1/pkg.k create mode 100644 kclvm/runner/src/test_datas/multi_file_compilation/no_kcl_mod_file/pkg2.k create mode 100644 kclvm/runner/src/test_datas/multi_file_compilation/no_kcl_mod_file/stdout.golden create mode 100644 kclvm/runner/src/test_datas/multi_file_compilation/relative_import/main.k create mode 100644 kclvm/runner/src/test_datas/multi_file_compilation/relative_import/mydir/mydir2/mymodule2.k create mode 100644 kclvm/runner/src/test_datas/multi_file_compilation/relative_import/mydir/mymodule.k create mode 100644 kclvm/runner/src/test_datas/multi_file_compilation/relative_import_as/main.k create mode 100644 kclvm/runner/src/test_datas/multi_file_compilation/relative_import_as/mydir/mydir2/mymodule2.k create mode 100644 kclvm/runner/src/test_datas/multi_file_compilation/relative_import_as/mydir/mymodule.k create mode 100644 kclvm/runner/src/test_datas/settings_file/settings.json create mode 100644 kclvm/runner/src/test_datas/settings_file/settings.yaml diff --git a/.github/workflows/github-actions.yaml b/.github/workflows/github-actions.yaml index a269223c5..96ddb3aeb 100644 --- a/.github/workflows/github-actions.yaml +++ b/.github/workflows/github-actions.yaml @@ -35,7 +35,7 @@ jobs: components: clippy, rustfmt - name: Rust unit test working-directory: ./kclvm - run: export PATH=$PATH:$(PWD)/_build/dist/ubuntu/kclvm/bin/ && make && make codecov-lcov + run: export PATH=$PATH:$PWD/../_build/dist/ubuntu/kclvm/bin && make install-rustc-wasm && make && make codecov-lcov shell: bash - name: Coveralls upload uses: coverallsapp/github-action@master diff --git a/internal/kclvm_py/scripts/update-kclvm.sh b/internal/kclvm_py/scripts/update-kclvm.sh index 9fedf3251..b29c2fda3 100755 --- a/internal/kclvm_py/scripts/update-kclvm.sh +++ b/internal/kclvm_py/scripts/update-kclvm.sh @@ -122,7 +122,6 @@ cargo build --release --target wasm32-unknown-unknown cp $topdir/kclvm/target/wasm32-unknown-unknown/release/libkclvm.a $kclvm_install_dir/lib/libkclvm_wasm32.a cp src/_kclvm_undefined_wasm.txt $kclvm_install_dir/lib/_kclvm_undefined_wasm.txt - cp src/_kclvm.bc $kclvm_install_dir/include/_kclvm.bc cp src/_kclvm.h $kclvm_install_dir/include/_kclvm.h diff --git a/kclvm/runner/benches/bench_runner.rs b/kclvm/runner/benches/bench_runner.rs index f560bd64f..5ec706209 100644 --- a/kclvm/runner/benches/bench_runner.rs +++ b/kclvm/runner/benches/bench_runner.rs @@ -3,18 +3,12 @@ use kclvm_parser::load_program; use kclvm_runner::{execute, runner::ExecProgramArgs}; use kclvm_tools::query::apply_overrides; -const TEST_CASE_PATH: &str = "/src/test_datas/init_check_order_0/main.k"; +const TEST_CASE_PATH: &str = "./src/test_datas/init_check_order_0/main.k"; pub fn criterion_benchmark(c: &mut Criterion) { - let kcl_path = &format!( - "{}{}", - std::env::current_dir().unwrap().to_str().unwrap(), - TEST_CASE_PATH - ); - c.bench_function("refactor kclvm-runner", |b| { b.iter(|| { - after_refactor(kcl_path.to_string()); + after_refactor(TEST_CASE_PATH.to_string()); }) }); } diff --git a/kclvm/runner/src/assembler.rs b/kclvm/runner/src/assembler.rs index 4ccefdb4e..bf974d43a 100644 --- a/kclvm/runner/src/assembler.rs +++ b/kclvm/runner/src/assembler.rs @@ -14,7 +14,7 @@ use std::{ use threadpool::ThreadPool; /// IR code file suffix. -const IR_FILE: &str = "_a.out"; +const DEFAULT_IR_FILE: &str = "_a.out"; /// Default codegen timeout. const DEFAULT_TIME_OUT: u64 = 5; @@ -25,31 +25,30 @@ const DEFAULT_TIME_OUT: u64 = 5; /// Note: LibAssembler is only for single file kcl program. For multi-file kcl programs, /// KclvmAssembler is provided to support for multi-file parallel compilation to improve /// the performance of the compiler. -pub trait LibAssembler { - /// Add a suffix to the file name according to the file suffix of different intermediate codes. - /// e.g. LLVM IR - /// code_file : "/test_dir/test_code_file" - /// return : "/test_dir/test_code_file.ll" +pub(crate) trait LibAssembler { + /// Add a suffix to the file name according to the file suffix of different intermediate code files. + /// e.g. LLVM IR -> code_file : "/test_dir/test_code_file" -> return : "/test_dir/test_code_file.ll" fn add_code_file_suffix(&self, code_file: &str) -> String; - /// Return the file suffix of different intermediate codes. - /// e.g. LLVM IR - /// return : ".ll" - fn get_code_file_suffix(&self) -> &str; + /// Return the file suffix of different intermediate code files. + /// e.g. LLVM IR -> return : ".ll" + fn get_code_file_suffix(&self) -> String; /// Assemble different intermediate codes into dynamic link libraries for single file kcl program. + /// Returns the path of the dynamic link library. /// /// Inputs: /// compile_prog: Reference of kcl program ast. /// /// "import_names" is import pkgpath and name of kcl program. /// Type of import_names is "IndexMap>". + /// /// "kcl_file_name" is the kcl file name string. /// "import_name" is the name string of import stmt. - /// e.g. "import test/main_pkg as main", "main" is an import_name. /// "import_path" is the path string of import stmt. + /// + /// e.g. "import test/main_pkg as main", "main" is an "import_name". /// e.g. "import test/main_pkg as main", "test/main_pkg" is an import_path. - /// import_names is from "ProgramScope.import_names" returned by "resolve_program" after resolving kcl ast by kclvm-sema. /// /// "code_file" is the filename of the generated intermediate code file. /// e.g. code_file : "/test_dir/test_code_file" @@ -61,10 +60,6 @@ pub trait LibAssembler { /// e.g. lib_path : "/test_dir/test_code_file.ll.dylib" (mac) /// e.g. lib_path : "/test_dir/test_code_file.ll.dll.lib" (windows) /// e.g. lib_path : "/test_dir/test_code_file.ll.so" (ubuntu) - /// - /// "plugin_agent" is a pointer to the plugin address. - /// - /// Returns the path of the dynamic link library. fn assemble_lib( &self, compile_prog: &Program, @@ -72,7 +67,6 @@ pub trait LibAssembler { code_file: &str, code_file_path: &str, lib_path: &str, - plugin_agent: &u64, ) -> String; /// This method is prepared for concurrent compilation in KclvmAssembler. @@ -87,12 +81,7 @@ pub trait LibAssembler { compile_prog: &Program, import_names: IndexMap>, file: &Path, - plugin_agent: &u64, ) -> String { - // e.g. LLVM IR - // code_file: file_name - // code_file_path: file_name.ll - // lock_file_path: file_name.dll.lib or file_name.lib or file_name.so let code_file = file.to_str().unwrap(); let code_file_path = &self.add_code_file_suffix(code_file); let lock_file_path = &format!("{}.lock", code_file_path); @@ -110,7 +99,6 @@ pub trait LibAssembler { code_file, code_file_path, &lib_path, - plugin_agent, ); // Unlock file @@ -119,8 +107,6 @@ pub trait LibAssembler { gen_lib_path } - // Clean file path - // Delete the file in "path". #[inline] fn clean_path(&self, path: &str) { if Path::new(path).exists() { @@ -128,8 +114,6 @@ pub trait LibAssembler { } } - // Clean lock file - // Clear the lock files generated during concurrent compilation. #[inline] fn clean_lock_file(&self, path: &str) { let lock_path = &format!("{}.lock", self.add_code_file_suffix(path)); @@ -140,8 +124,8 @@ pub trait LibAssembler { /// This enum lists all the intermediate code assemblers currently supported by kclvm. /// Currently only supports assemble llvm intermediate code into dynamic link library. #[derive(Clone)] -pub enum KclvmLibAssembler { - LLVM(LlvmLibAssembler), +pub(crate) enum KclvmLibAssembler { + LLVM, } /// KclvmLibAssembler is a dispatcher, responsible for calling corresponding methods @@ -158,16 +142,14 @@ impl LibAssembler for KclvmLibAssembler { code_file: &str, code_file_path: &str, lib_path: &str, - plugin_agent: &u64, ) -> String { match &self { - KclvmLibAssembler::LLVM(llvm_a) => llvm_a.assemble_lib( + KclvmLibAssembler::LLVM => LlvmLibAssembler::default().assemble_lib( compile_prog, import_names, code_file, code_file_path, lib_path, - plugin_agent, ), } } @@ -175,14 +157,14 @@ impl LibAssembler for KclvmLibAssembler { #[inline] fn add_code_file_suffix(&self, code_file: &str) -> String { match &self { - KclvmLibAssembler::LLVM(llvm_a) => llvm_a.add_code_file_suffix(code_file), + KclvmLibAssembler::LLVM => LlvmLibAssembler::default().add_code_file_suffix(code_file), } } #[inline] - fn get_code_file_suffix(&self) -> &str { + fn get_code_file_suffix(&self) -> String { match &self { - KclvmLibAssembler::LLVM(llvm_a) => llvm_a.get_code_file_suffix(), + KclvmLibAssembler::LLVM => LlvmLibAssembler::default().get_code_file_suffix(), } } @@ -192,11 +174,10 @@ impl LibAssembler for KclvmLibAssembler { compile_prog: &Program, import_names: IndexMap>, file: &Path, - plugin_agent: &u64, ) -> String { match &self { - KclvmLibAssembler::LLVM(llvm_a) => { - llvm_a.lock_file_and_gen_lib(compile_prog, import_names, file, plugin_agent) + KclvmLibAssembler::LLVM => { + LlvmLibAssembler::default().lock_file_and_gen_lib(compile_prog, import_names, file) } } } @@ -204,7 +185,21 @@ impl LibAssembler for KclvmLibAssembler { /// LlvmLibAssembler is mainly responsible for assembling the generated LLVM IR into a dynamic link library. #[derive(Clone)] -pub struct LlvmLibAssembler; +pub(crate) struct LlvmLibAssembler; + +impl LlvmLibAssembler { + #[inline] + fn new() -> Self { + Self {} + } +} + +impl Default for LlvmLibAssembler { + #[inline] + fn default() -> Self { + Self::new() + } +} /// KclvmLibAssembler implements the LibAssembler trait, impl LibAssembler for LlvmLibAssembler { @@ -214,47 +209,6 @@ impl LibAssembler for LlvmLibAssembler { /// And then assemble the dynamic link library based on the LLVM IR, /// /// At last remove the codegen temp files and return the dynamic link library path. - /// # Examples - /// - /// ``` - /// use kclvm_runner::runner::ExecProgramArgs; - /// use kclvm_parser::load_program; - /// use kclvm_sema::resolver::resolve_program; - /// use kclvm_runner::assembler::LlvmLibAssembler; - /// use crate::kclvm_runner::assembler::LibAssembler; - /// - /// // default args and configuration - /// let mut args = ExecProgramArgs::default(); - /// let k_path = "./src/test_datas/init_check_order_0/main.k"; - /// args.k_filename_list.push(k_path.to_string()); - /// let plugin_agent = 0; - /// let files = args.get_files(); - /// let opts = args.get_load_program_options(); - /// - /// // parse and resolve kcl - /// let mut program = load_program(&files, Some(opts)).unwrap(); - /// let scope = resolve_program(&mut program); - /// - /// // tmp file - /// let temp_entry_file = "test_entry_file"; - /// let temp_entry_file_path = &format!("{}.ll", temp_entry_file); - /// let temp_entry_file_lib = &format!("{}.dylib", temp_entry_file); - /// - /// // assemble libs - /// let llvm_assembler = LlvmLibAssembler{}; - /// let lib_file = llvm_assembler.assemble_lib( - /// &program, - /// scope.import_names.clone(), - /// temp_entry_file, - /// temp_entry_file_path, - /// temp_entry_file_lib, - /// &plugin_agent - /// ); - /// let lib_path = std::path::Path::new(&lib_file); - /// assert_eq!(lib_path.exists(), true); - /// llvm_assembler.clean_path(&lib_file); - /// assert_eq!(lib_path.exists(), false); - /// ``` #[inline] fn assemble_lib( &self, @@ -263,7 +217,6 @@ impl LibAssembler for LlvmLibAssembler { code_file: &str, code_file_path: &str, lib_path: &str, - plugin_agent: &u64, ) -> String { // clean "*.ll" file path. self.clean_path(&code_file_path.to_string()); @@ -280,25 +233,21 @@ impl LibAssembler for LlvmLibAssembler { ) .expect("Compile KCL to LLVM error"); - // assemble lib - let mut cmd = Command::new(*plugin_agent); + let mut cmd = Command::new(); let gen_lib_path = cmd.run_clang_single(code_file_path, lib_path); - // clean "*.ll" file path self.clean_path(&code_file_path.to_string()); gen_lib_path } - /// Add ".ll" suffix to a file path. #[inline] fn add_code_file_suffix(&self, code_file: &str) -> String { format!("{}.ll", code_file) } - /// Get String ".ll" #[inline] - fn get_code_file_suffix(&self) -> &str { - ".ll" + fn get_code_file_suffix(&self) -> String { + ".ll".to_string() } } @@ -311,39 +260,29 @@ impl LibAssembler for LlvmLibAssembler { /// /// KclvmAssembler provides an atomic operation for generating a dynamic link library for a single file /// through KclvmLibAssembler for each thread. -pub struct KclvmAssembler { +pub(crate) struct KclvmAssembler { thread_count: usize, } impl KclvmAssembler { + /// get the number of threads used in parallel multi-file compilation. + pub(crate) fn get_thread_count(self) -> usize { + return self.thread_count; + } + /// Constructs an KclvmAssembler instance with a default value 4 /// for the number of threads in multi-file compilation. - /// - /// # Examples - /// - /// ``` - /// use kclvm_runner::assembler::KclvmAssembler; - /// - /// let assembler = KclvmAssembler::new(); - /// ``` #[inline] - pub fn new() -> Self { + pub(crate) fn new() -> Self { Self { thread_count: 4 } } /// Constructs an KclvmAssembler instance with a value /// for the number of threads in multi-file compilation. - /// - /// # Examples - /// - /// ``` - /// use kclvm_runner::assembler::KclvmAssembler; - /// - /// let assembler = KclvmAssembler::new_with_thread_count(5); - /// ``` + /// The number of threads must be greater than to 0. #[inline] - pub fn new_with_thread_count(thread_count: usize) -> Self { - if thread_count == 0 { + pub(crate) fn new_with_thread_count(thread_count: usize) -> Self { + if thread_count <= 0 { bug!("Illegal thread count in multi-file compilation"); } Self { thread_count } @@ -351,47 +290,8 @@ impl KclvmAssembler { /// Clean up the path of the dynamic link libraries generated. /// It will remove the file in "file_path" and all the files in file_path end with ir code file suffix. - /// - /// # Examples - /// - /// ``` - /// use std::fs; - /// use std::fs::File; - /// use kclvm_runner::assembler::KclvmAssembler; - /// - /// // create test dir - /// std::fs::create_dir_all("./src/test_datas/test_clean").unwrap(); - /// - /// // File name and suffix for test - /// let test_path = "./src/test_datas/test_clean/test.out"; - /// let file_suffix = ".ll"; - /// - /// // Create file "./src/test_datas/test_clean/test.out" - /// File::create(test_path); - /// let path = std::path::Path::new(test_path); - /// assert_eq!(path.exists(), true); - /// - /// // Delete file "./src/test_datas/test_clean/test.out" and "./src/test_datas/test_clean/test.out*.ll" - /// KclvmAssembler::new().clean_path_for_genlibs(test_path, file_suffix); - /// assert_eq!(path.exists(), false); - /// - /// // Delete files whose filename end with "*.ll" - /// let test1 = &format!("{}{}", test_path, ".test1.ll"); - /// let test2 = &format!("{}{}", test_path, ".test2.ll"); - /// File::create(test1); - /// File::create(test2); - /// let path1 = std::path::Path::new(test1); - /// let path2 = std::path::Path::new(test2); - /// assert_eq!(path1.exists(), true); - /// assert_eq!(path2.exists(), true); - /// - /// // Delete file "./src/test_datas/test_clean/test.out" and "./src/test_datas/test_clean/test.out*.ll" - /// KclvmAssembler::new().clean_path_for_genlibs(test_path, file_suffix); - /// assert_eq!(path1.exists(), false); - /// assert_eq!(path2.exists(), false); - /// ``` #[inline] - pub fn clean_path_for_genlibs(&self, file_path: &str, suffix: &str) { + pub(crate) fn clean_path_for_genlibs(&self, file_path: &str, suffix: &str) { let path = std::path::Path::new(file_path); if path.exists() { std::fs::remove_file(path).unwrap(); @@ -408,39 +308,25 @@ impl KclvmAssembler { } } - /// Generate cache dir from the program root path. Create cache dir if it doesn't exist. - /// - /// # Examples - /// - /// ``` - /// use std::fs; - /// use kclvm_runner::assembler::KclvmAssembler; - /// - /// let expected_dir = "test_prog_name/.kclvm/cache/0.4.2-e07ed7af0d9bd1e86a3131714e4bd20c"; - /// let path = std::path::Path::new(expected_dir); - /// assert_eq!(path.exists(), false); - /// - /// let cache_dir = KclvmAssembler::new().load_cache_dir("test_prog_name"); - /// assert_eq!(cache_dir.display().to_string(), expected_dir); - /// - /// let path = std::path::Path::new(expected_dir); - /// assert_eq!(path.exists(), true); - /// - /// fs::remove_dir(expected_dir); - /// assert_eq!(path.exists(), false); - /// ``` + /// Generate cache dir from the program root path. + /// Create cache dir if it doesn't exist. #[inline] - pub fn load_cache_dir(&self, prog_root_name: &str) -> PathBuf { - let cache_dir = Path::new(prog_root_name) - .join(".kclvm") - .join("cache") - .join(kclvm_version::get_full_version()); + pub(crate) fn load_cache_dir(&self, prog_root_name: &str) -> PathBuf { + let cache_dir = self.construct_cache_dir(prog_root_name); if !cache_dir.exists() { std::fs::create_dir_all(&cache_dir).unwrap(); } cache_dir } + #[inline] + pub(crate) fn construct_cache_dir(&self, prog_root_name: &str) -> PathBuf { + Path::new(prog_root_name) + .join(".kclvm") + .join("cache") + .join(kclvm_version::get_full_version()) + } + /// Generate the dynamic link libraries and return file paths. /// /// In the method, multiple threads will be created to concurrently generate dynamic link libraries @@ -451,61 +337,18 @@ impl KclvmAssembler { /// /// `gen_libs` will create multiple threads and call the method provided by [KclvmLibAssembler] in each thread /// to generate the dynamic link library in parallel. - /// - /// # Examples - /// - /// ``` - /// use std::fs; - /// use kclvm_parser::load_program; - /// use kclvm_runner::runner::ExecProgramArgs; - /// use kclvm_runner::assembler::KclvmAssembler; - /// use kclvm_runner::assembler::KclvmLibAssembler; - /// use kclvm_runner::assembler::LlvmLibAssembler; - /// use kclvm_sema::resolver::resolve_program; - /// use kclvm_runner::command::Command; - /// - /// let plugin_agent = 0; - /// - /// let args = ExecProgramArgs::default(); - /// let opts = args.get_load_program_options(); - /// - /// let kcl_path = fs::canonicalize("./src/test_datas/init_check_order_0/main.k").unwrap().display().to_string(); - /// - /// let mut prog = load_program(&[&kcl_path], Some(opts)).unwrap(); - /// let scope = resolve_program(&mut prog); - /// - /// let lib_paths = KclvmAssembler::new().gen_libs( - /// prog, - /// scope, - /// plugin_agent, - /// &("test_entry_file_name".to_string()), - /// KclvmLibAssembler::LLVM(LlvmLibAssembler {})); - /// assert_eq!(lib_paths.len(), 1); - /// println!("kcl path: {}", *lib_paths.get(0).unwrap()); - /// - /// let lib_path = format!("./test_entry_file_name{}", Command::get_lib_suffix()); - /// let expected_lib_path = fs::canonicalize(lib_path).unwrap().display().to_string(); - /// assert_eq!(*lib_paths.get(0).unwrap(), expected_lib_path); - /// - /// let path = std::path::Path::new(&expected_lib_path); - /// assert_eq!(path.exists(), true); - /// - /// KclvmAssembler::new().clean_path_for_genlibs(&expected_lib_path, &Command::get_lib_suffix()); - /// assert_eq!(path.exists(), false); - ///``` - pub fn gen_libs( + pub(crate) fn gen_libs( &self, program: ast::Program, scope: ProgramScope, - plugin_agent: u64, entry_file: &String, single_file_assembler: KclvmLibAssembler, ) -> Vec { - // Clean the code generated path. - self.clean_path_for_genlibs(IR_FILE, single_file_assembler.get_code_file_suffix()); - // Load cache + self.clean_path_for_genlibs( + DEFAULT_IR_FILE, + &single_file_assembler.get_code_file_suffix(), + ); let cache_dir = self.load_cache_dir(&program.root); - let mut compile_progs: IndexMap< String, ( @@ -549,12 +392,7 @@ impl KclvmAssembler { let lib_path = if is_main_pkg { let file = PathBuf::from(&temp_entry_file); // generate dynamic link library for single file kcl program - assembler.lock_file_and_gen_lib( - &compile_prog, - import_names, - &file, - &plugin_agent, - ) + assembler.lock_file_and_gen_lib(&compile_prog, import_names, &file) } else { let file = cache_dir.join(&pkgpath); // Read the lib path cache @@ -579,12 +417,8 @@ impl KclvmAssembler { Some(path) => path, None => { // generate dynamic link library for single file kcl program - let lib_path = assembler.lock_file_and_gen_lib( - &compile_prog, - import_names, - &file, - &plugin_agent, - ); + let lib_path = + assembler.lock_file_and_gen_lib(&compile_prog, import_names, &file); let lib_relative_path = lib_path.replacen(root, ".", 1); save_pkg_cache( root, @@ -612,13 +446,13 @@ impl KclvmAssembler { .unwrap(); lib_paths.push(lib_path); } - // Clean the lock file. single_file_assembler.clean_lock_file(entry_file); lib_paths } } impl Default for KclvmAssembler { + #[inline] fn default() -> Self { Self::new() } diff --git a/kclvm/runner/src/command.rs b/kclvm/runner/src/command.rs index 0c957be9c..e115c08e1 100644 --- a/kclvm/runner/src/command.rs +++ b/kclvm/runner/src/command.rs @@ -1,19 +1,14 @@ use std::{env, path::PathBuf}; -use super::runner::*; -use kclvm::ValueRef; -use kclvm_config::settings::SettingsFile; - #[derive(Debug)] pub struct Command { clang_path: String, rust_stdlib: String, executable_root: String, - plugin_method_ptr: u64, } impl Command { - pub fn new(plugin_method_ptr: u64) -> Self { + pub fn new() -> Self { let executable_root = Self::get_executable_root(); let rust_stdlib = Self::get_rust_stdlib(executable_root.as_str()); let clang_path = Self::get_clang_path(); @@ -22,183 +17,7 @@ impl Command { clang_path, rust_stdlib, executable_root, - plugin_method_ptr, - } - } - - pub fn run_lib(&self, lib_path: &str) -> Result { - unsafe { - let lib = libloading::Library::new(lib_path).unwrap(); - - // get kclvm_plugin_init - let kclvm_plugin_init: libloading::Symbol< - unsafe extern "C" fn( - fn_ptr: extern "C" fn( - method: *const i8, - args_json: *const i8, - kwargs_json: *const i8, - ) -> *const i8, - ), - > = lib.get(b"kclvm_plugin_init").unwrap(); - - // get _kcl_run - let kcl_run: libloading::Symbol< - unsafe extern "C" fn( - kclvm_main_ptr: u64, // main.k => kclvm_main - option_len: kclvm_size_t, - option_keys: *const *const kclvm_char_t, - option_values: *const *const kclvm_char_t, - strict_range_check: i32, - disable_none: i32, - disable_schema_check: i32, - list_option_mode: i32, - debug_mode: i32, - result_buffer_len: kclvm_size_t, - result_buffer: *mut kclvm_char_t, - warn_buffer_len: kclvm_size_t, - warn_buffer: *mut kclvm_char_t, - ) -> kclvm_size_t, - > = lib.get(b"_kcl_run").unwrap(); - - // get kclvm_main - let kclvm_main: libloading::Symbol = lib.get(b"kclvm_main").unwrap(); - let kclvm_main_ptr = kclvm_main.into_raw().into_raw() as u64; - - // get plugin_method - let plugin_method_ptr = self.plugin_method_ptr; - let plugin_method_ptr = (plugin_method_ptr as *const u64) as *const () - as *const extern "C" fn( - method: *const i8, - args: *const i8, - kwargs: *const i8, - ) -> *const i8; - let plugin_method: extern "C" fn( - method: *const i8, - args: *const i8, - kwargs: *const i8, - ) -> *const i8 = std::mem::transmute(plugin_method_ptr); - - // register plugin agent - kclvm_plugin_init(plugin_method); - - let option_len = 0; - let option_keys = std::ptr::null(); - let option_values = std::ptr::null(); - let strict_range_check = 0; - let disable_none = 0; - let disable_schema_check = 0; - let list_option_mode = 0; - let debug_mode = 0; - - let mut result = vec![0u8; 1024 * 1024]; - let result_buffer_len = result.len() as i32 - 1; - let result_buffer = result.as_mut_ptr() as *mut i8; - - let mut warn_buffer = vec![0u8; 1024 * 1024]; - let warn_buffer_len = warn_buffer.len() as i32 - 1; - let warn_buffer = warn_buffer.as_mut_ptr() as *mut i8; - - let n = kcl_run( - kclvm_main_ptr, - option_len, - option_keys, - option_values, - strict_range_check, - disable_none, - disable_schema_check, - list_option_mode, - debug_mode, - result_buffer_len, - result_buffer, - warn_buffer_len, - warn_buffer, - ); - - let s = std::str::from_utf8(&result[0..n as usize]).unwrap(); - Ok(s.to_string()) - } - } - - pub fn run_lib_with_settings( - &self, - lib_path: &str, - settings: SettingsFile, - ) -> Result { - unsafe { - let lib = libloading::Library::new(lib_path).unwrap(); - - let kcl_run: libloading::Symbol< - unsafe extern "C" fn( - kclvm_main_ptr: u64, // main.k => kclvm_main - option_len: kclvm_size_t, - option_keys: *const *const kclvm_char_t, - option_values: *const *const kclvm_char_t, - strict_range_check: i32, - disable_none: i32, - disable_schema_check: i32, - list_option_mode: i32, - debug_mode: i32, - result_buffer_len: kclvm_size_t, - result_buffer: *mut kclvm_char_t, - warn_buffer_len: kclvm_size_t, - warn_buffer: *mut kclvm_char_t, - ) -> kclvm_size_t, - > = lib.get(b"_kcl_run").unwrap(); - - let kclvm_main: libloading::Symbol = lib.get(b"kclvm_main").unwrap(); - let kclvm_main_ptr = kclvm_main.into_raw().into_raw() as u64; - - let option_len = 0; - let option_keys = std::ptr::null(); - let option_values = std::ptr::null(); - let strict_range_check = 0; - let disable_none = settings - .kcl_cli_configs - .as_ref() - .map_or(0, |c| c.disable_none.map_or(0, |v| v as i32)); - let disable_schema_check = 0; - let list_option_mode = 0; - let debug_mode = settings - .kcl_cli_configs - .as_ref() - .map_or(0, |c| c.debug.map_or(0, |v| v as i32)); - - let mut result = vec![0u8; 1024 * 1024]; - let result_buffer_len = result.len() as i32 - 1; - let result_buffer = result.as_mut_ptr() as *mut i8; - - let mut warn_buffer = vec![0u8; 1024 * 1024]; - let warn_buffer_len = warn_buffer.len() as i32 - 1; - let warn_buffer = warn_buffer.as_mut_ptr() as *mut i8; - - let n = kcl_run( - kclvm_main_ptr, - option_len, - option_keys, - option_values, - strict_range_check, - disable_none, - disable_schema_check, - list_option_mode, - debug_mode, - result_buffer_len, - result_buffer, - warn_buffer_len, - warn_buffer, - ); - - let ctx = kclvm::Context::current_context_mut(); - ctx.cfg.debug_mode = debug_mode > 0; - ctx.cfg.disable_none = disable_none > 0; - let s = std::str::from_utf8(&result[0..n as usize]).unwrap(); - if s.is_empty() { - println!() - } else { - println!("{}", ValueRef::from_json(s).unwrap().plan_to_yaml_string()); - } } - - Ok("".to_string()) } pub fn link_libs(&mut self, libs: &[String], lib_path: &str) -> String { diff --git a/kclvm/runner/src/lib.rs b/kclvm/runner/src/lib.rs index ebcd7d70b..9a2d153ef 100644 --- a/kclvm/runner/src/lib.rs +++ b/kclvm/runner/src/lib.rs @@ -1,6 +1,6 @@ use std::path::Path; -use assembler::{KclvmLibAssembler, LlvmLibAssembler}; +use assembler::KclvmLibAssembler; use command::Command; use kclvm_ast::ast::Program; use kclvm_sema::resolver::resolve_program; @@ -25,7 +25,6 @@ pub mod tests; /// It returns the KCL program executing result as Result, /// and mainly takes "program" (ast.Program returned by kclvm-parser) as input. /// -/// "plugin_agent" is related to KCLVM plugin. /// "args" is the items selected by the user in the KCLVM CLI. /// /// This method will first resolve “program” (ast.Program) and save the result to the "scope" (ProgramScope). @@ -43,18 +42,14 @@ pub mod tests; /// /// At last, KclvmRunner will be constructed and call method "run" to execute the kcl program. /// -/// TODO: Need to be a better backend abstraction -/// /// # Examples /// /// ``` /// use kclvm_runner::{execute, runner::ExecProgramArgs}; /// use kclvm_parser::load_program; /// use kclvm_ast::ast::Program; -/// -/// // Default plugin agent +/// // plugin_agent is the address of plugin. /// let plugin_agent = 0; -/// /// // Get default args /// let args = ExecProgramArgs::default(); /// let opts = args.get_load_program_options(); @@ -80,19 +75,19 @@ pub fn execute( let temp_dir = tempdir().unwrap(); let temp_dir_path = temp_dir.path().to_str().unwrap(); let temp_entry_file = temp_file(temp_dir_path); + // Generate libs - let lib_paths = assembler::KclvmAssembler::new().gen_libs( + let lib_paths = assembler::KclvmAssembler::default().gen_libs( program, scope, - plugin_agent, &temp_entry_file, - KclvmLibAssembler::LLVM(LlvmLibAssembler {}), + KclvmLibAssembler::LLVM, ); // Link libs let lib_suffix = Command::get_lib_suffix(); let temp_out_lib_file = format!("{}.out{}", temp_entry_file, lib_suffix); - let lib_path = linker::KclvmLinker::link_all_libs(lib_paths, temp_out_lib_file, plugin_agent); + let lib_path = linker::KclvmLinker::link_all_libs(lib_paths, temp_out_lib_file); // Run let runner = KclvmRunner::new( diff --git a/kclvm/runner/src/linker.rs b/kclvm/runner/src/linker.rs index 4be7a8492..571598689 100644 --- a/kclvm/runner/src/linker.rs +++ b/kclvm/runner/src/linker.rs @@ -4,8 +4,8 @@ use crate::command::Command; pub struct KclvmLinker; impl KclvmLinker { /// Link the libs generated by method "gen_bc_or_ll_file". - pub fn link_all_libs(lib_paths: Vec, lib_path: String, plugin_agent: u64) -> String { - let mut cmd = Command::new(plugin_agent); + pub fn link_all_libs(lib_paths: Vec, lib_path: String) -> String { + let mut cmd = Command::new(); cmd.link_libs(&lib_paths, &lib_path) } } diff --git a/kclvm/runner/src/test_datas/exec_prog_args/default.json b/kclvm/runner/src/test_datas/exec_prog_args/default.json new file mode 100644 index 000000000..bf05556a6 --- /dev/null +++ b/kclvm/runner/src/test_datas/exec_prog_args/default.json @@ -0,0 +1 @@ +{"work_dir":null,"k_filename_list":[],"k_code_list":[],"args":[],"overrides":[],"disable_yaml_result":false,"print_override_ast":false,"strict_range_check":false,"disable_none":false,"verbose":0,"debug":0,"sort_keys":false,"include_schema_type_path":false} \ No newline at end of file diff --git a/kclvm/runner/src/test_datas/multi_file_compilation/import_abs_path/app-main/main.k b/kclvm/runner/src/test_datas/multi_file_compilation/import_abs_path/app-main/main.k new file mode 100644 index 000000000..912eb0857 --- /dev/null +++ b/kclvm/runner/src/test_datas/multi_file_compilation/import_abs_path/app-main/main.k @@ -0,0 +1,7 @@ +import some0.pkg1 as some00 +import some1.pkg1 as some10 +import ..some1.pkg1 as some11 + +Name1 = some00.Name # some0.pkg1.name +Name2 = some10.Name # some1.pkg1.name +Name3 = some11.Name # some1.pkg1.name diff --git a/kclvm/runner/src/test_datas/multi_file_compilation/import_abs_path/app-main/some1/pkg1/pkg1.k b/kclvm/runner/src/test_datas/multi_file_compilation/import_abs_path/app-main/some1/pkg1/pkg1.k new file mode 100644 index 000000000..3fcc664c9 --- /dev/null +++ b/kclvm/runner/src/test_datas/multi_file_compilation/import_abs_path/app-main/some1/pkg1/pkg1.k @@ -0,0 +1 @@ +Name = '.some1.pkg1.name' diff --git a/kclvm/runner/src/test_datas/multi_file_compilation/import_abs_path/kcl.mod b/kclvm/runner/src/test_datas/multi_file_compilation/import_abs_path/kcl.mod new file mode 100644 index 000000000..e69de29bb diff --git a/kclvm/runner/src/test_datas/multi_file_compilation/import_abs_path/some0/pkg1/pkg1.k b/kclvm/runner/src/test_datas/multi_file_compilation/import_abs_path/some0/pkg1/pkg1.k new file mode 100644 index 000000000..6ddf6bada --- /dev/null +++ b/kclvm/runner/src/test_datas/multi_file_compilation/import_abs_path/some0/pkg1/pkg1.k @@ -0,0 +1 @@ +Name = 'some0.pkg1.name' diff --git a/kclvm/runner/src/test_datas/multi_file_compilation/import_abs_path/some1/pkg1/pkg1.k b/kclvm/runner/src/test_datas/multi_file_compilation/import_abs_path/some1/pkg1/pkg1.k new file mode 100644 index 000000000..6e08e1de8 --- /dev/null +++ b/kclvm/runner/src/test_datas/multi_file_compilation/import_abs_path/some1/pkg1/pkg1.k @@ -0,0 +1 @@ +Name = 'some1.pkg1.name' diff --git a/kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module/kcl.mod b/kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module/kcl.mod new file mode 100644 index 000000000..e69de29bb diff --git a/kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module/main.k b/kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module/main.k new file mode 100644 index 000000000..a12cf1ce8 --- /dev/null +++ b/kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module/main.k @@ -0,0 +1,3 @@ +import mymodule + +result = mymodule.data.num diff --git a/kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module/mymodule.k b/kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module/mymodule.k new file mode 100644 index 000000000..db9135a5f --- /dev/null +++ b/kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module/mymodule.k @@ -0,0 +1,6 @@ +schema Data: + num: int = 1 + +data = Data { + "num" = 100 +} diff --git a/kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module_as/kcl.mod b/kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module_as/kcl.mod new file mode 100644 index 000000000..e69de29bb diff --git a/kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module_as/main.k b/kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module_as/main.k new file mode 100644 index 000000000..c6ff5e75a --- /dev/null +++ b/kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module_as/main.k @@ -0,0 +1,3 @@ +import mymodule as mm + +result = mm.data.num diff --git a/kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module_as/mymodule.k b/kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module_as/mymodule.k new file mode 100644 index 000000000..db9135a5f --- /dev/null +++ b/kclvm/runner/src/test_datas/multi_file_compilation/import_regular_module_as/mymodule.k @@ -0,0 +1,6 @@ +schema Data: + num: int = 1 + +data = Data { + "num" = 100 +} diff --git a/kclvm/runner/src/test_datas/multi_file_compilation/no_kcl_mod_file/main.k b/kclvm/runner/src/test_datas/multi_file_compilation/no_kcl_mod_file/main.k new file mode 100644 index 000000000..cd9eda3dd --- /dev/null +++ b/kclvm/runner/src/test_datas/multi_file_compilation/no_kcl_mod_file/main.k @@ -0,0 +1,7 @@ +import pkg1 +import pkg2 + +# no kcl.mod, use dirname(main.k) as root + +Path1 = pkg1.Path +Path2 = pkg2.Path diff --git a/kclvm/runner/src/test_datas/multi_file_compilation/no_kcl_mod_file/pkg1/pkg.k b/kclvm/runner/src/test_datas/multi_file_compilation/no_kcl_mod_file/pkg1/pkg.k new file mode 100644 index 000000000..b6ec63378 --- /dev/null +++ b/kclvm/runner/src/test_datas/multi_file_compilation/no_kcl_mod_file/pkg1/pkg.k @@ -0,0 +1 @@ +Path = "pkg1" \ No newline at end of file diff --git a/kclvm/runner/src/test_datas/multi_file_compilation/no_kcl_mod_file/pkg2.k b/kclvm/runner/src/test_datas/multi_file_compilation/no_kcl_mod_file/pkg2.k new file mode 100644 index 000000000..d296fe1ab --- /dev/null +++ b/kclvm/runner/src/test_datas/multi_file_compilation/no_kcl_mod_file/pkg2.k @@ -0,0 +1 @@ +Path = "pkg2" \ No newline at end of file diff --git a/kclvm/runner/src/test_datas/multi_file_compilation/no_kcl_mod_file/stdout.golden b/kclvm/runner/src/test_datas/multi_file_compilation/no_kcl_mod_file/stdout.golden new file mode 100644 index 000000000..87b7c93f3 --- /dev/null +++ b/kclvm/runner/src/test_datas/multi_file_compilation/no_kcl_mod_file/stdout.golden @@ -0,0 +1,2 @@ +Path1: pkg1 +Path2: pkg2 diff --git a/kclvm/runner/src/test_datas/multi_file_compilation/relative_import/main.k b/kclvm/runner/src/test_datas/multi_file_compilation/relative_import/main.k new file mode 100644 index 000000000..346d8034b --- /dev/null +++ b/kclvm/runner/src/test_datas/multi_file_compilation/relative_import/main.k @@ -0,0 +1,4 @@ +import .mydir.mymodule +import .mydir.mydir2.mymodule2 + +result = mymodule2.data1.num \ No newline at end of file diff --git a/kclvm/runner/src/test_datas/multi_file_compilation/relative_import/mydir/mydir2/mymodule2.k b/kclvm/runner/src/test_datas/multi_file_compilation/relative_import/mydir/mydir2/mymodule2.k new file mode 100644 index 000000000..efa3e35aa --- /dev/null +++ b/kclvm/runner/src/test_datas/multi_file_compilation/relative_import/mydir/mydir2/mymodule2.k @@ -0,0 +1,6 @@ +schema data: + num : int = 1 + +data1 = data { + "num" = 10 +} diff --git a/kclvm/runner/src/test_datas/multi_file_compilation/relative_import/mydir/mymodule.k b/kclvm/runner/src/test_datas/multi_file_compilation/relative_import/mydir/mymodule.k new file mode 100644 index 000000000..72367c5c3 --- /dev/null +++ b/kclvm/runner/src/test_datas/multi_file_compilation/relative_import/mydir/mymodule.k @@ -0,0 +1,6 @@ +schema data: + num : int = 1 + +data0 = data { + "num" = 100 +} diff --git a/kclvm/runner/src/test_datas/multi_file_compilation/relative_import_as/main.k b/kclvm/runner/src/test_datas/multi_file_compilation/relative_import_as/main.k new file mode 100644 index 000000000..5ba0e0e4f --- /dev/null +++ b/kclvm/runner/src/test_datas/multi_file_compilation/relative_import_as/main.k @@ -0,0 +1,4 @@ +import .mydir.mymodule as m1 +import .mydir.mydir2.mymodule2 as m2 + +result = m2.data1.num diff --git a/kclvm/runner/src/test_datas/multi_file_compilation/relative_import_as/mydir/mydir2/mymodule2.k b/kclvm/runner/src/test_datas/multi_file_compilation/relative_import_as/mydir/mydir2/mymodule2.k new file mode 100644 index 000000000..7fc22a7b6 --- /dev/null +++ b/kclvm/runner/src/test_datas/multi_file_compilation/relative_import_as/mydir/mydir2/mymodule2.k @@ -0,0 +1,6 @@ +schema data: + num: int = 1 + +data1 = data { + "num" = 10 +} diff --git a/kclvm/runner/src/test_datas/multi_file_compilation/relative_import_as/mydir/mymodule.k b/kclvm/runner/src/test_datas/multi_file_compilation/relative_import_as/mydir/mymodule.k new file mode 100644 index 000000000..5832188b9 --- /dev/null +++ b/kclvm/runner/src/test_datas/multi_file_compilation/relative_import_as/mydir/mymodule.k @@ -0,0 +1,6 @@ +schema data: + num: int = 1 + +data0 = data { + "num" = 100 +} diff --git a/kclvm/runner/src/test_datas/settings_file/settings.json b/kclvm/runner/src/test_datas/settings_file/settings.json new file mode 100644 index 000000000..724ad05b1 --- /dev/null +++ b/kclvm/runner/src/test_datas/settings_file/settings.json @@ -0,0 +1 @@ +{"work_dir":null,"k_filename_list":["../main.k","./before/base.k","./main.k","./sub/sub.k"],"k_code_list":[],"args":[{"name":"app-name","value":"kclvm"},{"name":"image","value":"kclvm:v0.0.1"}],"overrides":[],"disable_yaml_result":false,"print_override_ast":false,"strict_range_check":false,"disable_none":false,"verbose":0,"debug":0,"sort_keys":false,"include_schema_type_path":false} \ No newline at end of file diff --git a/kclvm/runner/src/test_datas/settings_file/settings.yaml b/kclvm/runner/src/test_datas/settings_file/settings.yaml new file mode 100644 index 000000000..8b7a8a399 --- /dev/null +++ b/kclvm/runner/src/test_datas/settings_file/settings.yaml @@ -0,0 +1,14 @@ +kcl_cli_configs: + files: + - ../main.k + - ./before/base.k + - ./main.k + - ./sub/sub.k + disable_none: false + strict_range_check: false + debug: false +kcl_options: + - key: app-name + value: kclvm + - key: image + value: kclvm:v0.0.1 diff --git a/kclvm/runner/src/tests.rs b/kclvm/runner/src/tests.rs index 46e259b2e..78a86dc30 100644 --- a/kclvm/runner/src/tests.rs +++ b/kclvm/runner/src/tests.rs @@ -1,14 +1,22 @@ use crate::assembler::KclvmAssembler; -use crate::KclvmLibAssembler; -use crate::LlvmLibAssembler; +use crate::assembler::KclvmLibAssembler; +use crate::assembler::LibAssembler; +use crate::temp_file; +use crate::Command; use crate::{execute, runner::ExecProgramArgs}; use kclvm_ast::ast::{Module, Program}; +use kclvm_config::settings::load_file; use kclvm_parser::load_program; use kclvm_sema::resolver::resolve_program; +use std::fs::create_dir_all; +use std::panic::catch_unwind; +use std::panic::set_hook; +use std::path::PathBuf; use std::{ collections::HashMap, fs::{self, File}, }; +use tempfile::tempdir; const TEST_CASES: &[&'static str; 5] = &[ "init_check_order_0", @@ -17,19 +25,34 @@ const TEST_CASES: &[&'static str; 5] = &[ "type_annotation_not_full_2", "multi_vars_0", ]; -const EXPECTED_FILE_NAME: &str = "stdout.golden.json"; + +const MULTI_FILE_TEST_CASES: &[&'static str; 6] = &[ + "multi_file_compilation/no_kcl_mod_file", + "multi_file_compilation/relative_import", + "multi_file_compilation/relative_import_as", + "multi_file_compilation/import_abs_path/app-main", + "multi_file_compilation/import_regular_module", + "multi_file_compilation/import_regular_module_as", +]; + +const EXEC_PROG_ARGS_TEST_CASE: &[&'static str; 1] = &["exec_prog_args/default.json"]; + +const SETTINGS_FILE_TEST_CASE: &[&'static (&str, &str); 1] = + &[&("settings_file/settings.yaml", "settings_file/settings.json")]; + +const EXPECTED_JSON_FILE_NAME: &str = "stdout.golden.json"; const TEST_CASE_PATH: &str = "./src/test_datas"; const KCL_FILE_NAME: &str = "main.k"; const MAIN_PKG_NAME: &str = "__main__"; /// Load test kcl file to ast.Program -pub fn load_test_program(filename: String) -> Program { +fn load_test_program(filename: String) -> Program { let module = load_module(filename); construct_program(module) } /// Load test kcl file to ast.Module -pub fn load_module(filename: String) -> Module { +fn load_module(filename: String) -> Module { kclvm_parser::parse_file(&filename, None).unwrap() } @@ -40,7 +63,7 @@ pub fn load_module(filename: String) -> Module { /// Program.main = "__main__" /// Program.cmd_args = [] /// Program.cmd_overrides = [] -pub fn construct_program(mut module: Module) -> Program { +fn construct_program(mut module: Module) -> Program { module.pkg = MAIN_PKG_NAME.to_string(); let mut pkgs_ast = HashMap::new(); pkgs_ast.insert(MAIN_PKG_NAME.to_string(), vec![module]); @@ -53,20 +76,38 @@ pub fn construct_program(mut module: Module) -> Program { } } +fn construct_pkg_lib_path( + prog: &Program, + assembler: &KclvmAssembler, + main_path: &str, + suffix: String, +) -> Vec { + let cache_dir = assembler.construct_cache_dir(&prog.root); + let mut result = vec![]; + for (pkgpath, _) in &prog.pkgs { + if pkgpath == "__main__" { + result.push(fs::canonicalize(format!("{}{}", main_path.to_string(), suffix)).unwrap()); + } else { + result.push(cache_dir.join(format!("{}{}", pkgpath.clone(), suffix))); + } + } + return result; +} + /// Load the expect result from stdout.golden.json -pub fn load_expect_file(filename: String) -> String { +fn load_expect_file(filename: String) -> String { let f = File::open(filename).unwrap(); let v: serde_json::Value = serde_json::from_reader(f).unwrap(); v.to_string() } /// Format str by json str -pub fn format_str_by_json(str: String) -> String { +fn format_str_by_json(str: String) -> String { let v: serde_json::Value = serde_json::from_str(&str).unwrap(); v.to_string() } -pub fn execute_for_test(kcl_path: &String) -> String { +fn execute_for_test(kcl_path: &String) -> String { let plugin_agent = 0; let args = ExecProgramArgs::default(); // Parse kcl file @@ -75,14 +116,232 @@ pub fn execute_for_test(kcl_path: &String) -> String { execute(program, plugin_agent, &args).unwrap() } -// TODO: need to fix issue #79 +fn gen_libs_for_test(entry_file: &str, test_kcl_case_path: &str) { + let args = ExecProgramArgs::default(); + let opts = args.get_load_program_options(); + + let mut prog = load_program(&[&test_kcl_case_path], Some(opts)).unwrap(); + let scope = resolve_program(&mut prog); + + let assembler = KclvmAssembler::default(); + let prog_for_cache = prog.clone(); + + let lib_paths = assembler.gen_libs( + prog, + scope, + &(entry_file.to_string()), + KclvmLibAssembler::LLVM, + ); + + let expected_pkg_paths = construct_pkg_lib_path( + &prog_for_cache, + &assembler, + PathBuf::from(entry_file).to_str().unwrap(), + Command::get_lib_suffix(), + ); + assert_eq!(lib_paths.len(), expected_pkg_paths.len()); + for pkg_path in &expected_pkg_paths { + assert_eq!(pkg_path.exists(), true); + } + + let tmp_main_lib_path = fs::canonicalize(format!( + "{}{}", + entry_file.to_string(), + Command::get_lib_suffix() + )) + .unwrap(); + assert_eq!(tmp_main_lib_path.exists(), true); + + KclvmLibAssembler::LLVM.clean_path(&tmp_main_lib_path.to_str().unwrap()); + assert_eq!(tmp_main_lib_path.exists(), false); +} + +fn assemble_lib_for_test( + entry_file: &str, + test_kcl_case_path: &str, + assembler: &KclvmLibAssembler, +) -> String { + // default args and configuration + let mut args = ExecProgramArgs::default(); + + args.k_filename_list.push(test_kcl_case_path.to_string()); + let files = args.get_files(); + let opts = args.get_load_program_options(); + + // parse and resolve kcl + let mut program = load_program(&files, Some(opts)).unwrap(); + let scope = resolve_program(&mut program); + + // tmp file + let temp_entry_file_path = &format!("{}.ll", entry_file); + let temp_entry_file_lib = &format!("{}.{}", entry_file, Command::get_lib_suffix()); + + // assemble libs + assembler.assemble_lib( + &program, + scope.import_names.clone(), + entry_file, + temp_entry_file_path, + temp_entry_file_lib, + ) +} + #[test] fn test_kclvm_runner_execute() { for case in TEST_CASES { let kcl_path = &format!("{}/{}/{}", TEST_CASE_PATH, case, KCL_FILE_NAME); - let expected_path = &format!("{}/{}/{}", TEST_CASE_PATH, case, EXPECTED_FILE_NAME); + let expected_path = &format!("{}/{}/{}", TEST_CASE_PATH, case, EXPECTED_JSON_FILE_NAME); let result = execute_for_test(kcl_path); let expected_result = load_expect_file(expected_path.to_string()); assert_eq!(expected_result, format_str_by_json(result)); } } + +#[test] +fn test_kclvm_runner_execute_timeout() { + set_hook(Box::new(|_| {})); + let result_time_out = catch_unwind(|| { + gen_libs_for_test( + "test/no_exist_path/", + "./src/test_datas/multi_file_compilation/import_abs_path/app-main/main.k", + ); + }); + let timeout_panic_msg = "called `Result::unwrap()` on an `Err` value: Timeout"; + match result_time_out { + Err(panic_err) => { + if let Some(s) = panic_err.downcast_ref::() { + assert_eq!(s, timeout_panic_msg) + } + } + _ => { + unreachable!() + } + } +} + +#[test] +fn test_assemble_lib_llvm() { + for case in TEST_CASES { + let temp_dir = tempdir().unwrap(); + let temp_dir_path = temp_dir.path().to_str().unwrap(); + let temp_entry_file = temp_file(temp_dir_path); + + let kcl_path = &format!("{}/{}/{}", TEST_CASE_PATH, case, KCL_FILE_NAME); + let assembler = &KclvmLibAssembler::LLVM; + + let lib_file = assemble_lib_for_test( + &format!("{}{}", temp_entry_file, "4assemble_lib"), + kcl_path, + assembler, + ); + + let lib_path = std::path::Path::new(&lib_file); + assert_eq!(lib_path.exists(), true); + assembler.clean_path(&lib_file); + assert_eq!(lib_path.exists(), false); + } +} + +#[test] +fn test_gen_libs() { + for case in MULTI_FILE_TEST_CASES { + let temp_dir = tempdir().unwrap(); + let temp_dir_path = temp_dir.path().to_str().unwrap(); + let temp_entry_file = temp_file(temp_dir_path); + + let kcl_path = &format!("{}/{}/{}", TEST_CASE_PATH, case, KCL_FILE_NAME); + gen_libs_for_test(&format!("{}{}", temp_entry_file, "4gen_libs"), kcl_path); + } +} + +#[test] +fn test_new_assembler_with_thread_count() { + let assembler = KclvmAssembler::new_with_thread_count(5); + assert_eq!(assembler.get_thread_count(), 5); +} + +#[test] +fn test_new_assembler_with_thread_count_invalid() { + set_hook(Box::new(|_| {})); + let result_new = catch_unwind(|| { + let _assembler_err = KclvmAssembler::new_with_thread_count(0); + }); + let err_msg = "Internal error, please report a bug to us. The error message is: Illegal thread count in multi-file compilation"; + match result_new { + Err(panic_err) => { + if let Some(s) = panic_err.downcast_ref::() { + assert_eq!(s, err_msg); + } + } + _ => { + unreachable!() + } + } +} + +#[test] +fn test_clean_path_for_genlibs() { + let temp_dir = tempdir().unwrap(); + let temp_dir_path = temp_dir.path().to_str().unwrap(); + let tmp_file_path = &temp_file(temp_dir_path); + + create_dir_all(tmp_file_path).unwrap(); + + let file_name = &format!("{}/{}", tmp_file_path, "test"); + let file_suffix = ".ll"; + + File::create(file_name).unwrap(); + let path = std::path::Path::new(file_name); + assert_eq!(path.exists(), true); + + KclvmAssembler::new().clean_path_for_genlibs(file_name, file_suffix); + assert_eq!(path.exists(), false); + + let test1 = &format!("{}{}", file_name, ".test1.ll"); + let test2 = &format!("{}{}", file_name, ".test2.ll"); + File::create(test1).unwrap(); + File::create(test2).unwrap(); + let path1 = std::path::Path::new(test1); + + let path2 = std::path::Path::new(test2); + assert_eq!(path1.exists(), true); + assert_eq!(path2.exists(), true); + + KclvmAssembler::new().clean_path_for_genlibs(file_name, file_suffix); + assert_eq!(path1.exists(), false); + assert_eq!(path2.exists(), false); +} + +#[test] +fn test_to_json_program_arg() { + for case in EXEC_PROG_ARGS_TEST_CASE { + let test_case_json_file = &format!("{}/{}", TEST_CASE_PATH, case); + let expected_json_str = fs::read_to_string(test_case_json_file).unwrap(); + let exec_prog_args = ExecProgramArgs::default(); + assert_eq!(expected_json_str.trim(), exec_prog_args.to_json().trim()); + } +} + +#[test] +fn test_from_str_program_arg() { + for case in EXEC_PROG_ARGS_TEST_CASE { + let test_case_json_file = &format!("{}/{}", TEST_CASE_PATH, case); + let expected_json_str = fs::read_to_string(test_case_json_file).unwrap(); + let exec_prog_args = ExecProgramArgs::from_str(&expected_json_str); + assert_eq!(expected_json_str.trim(), exec_prog_args.to_json().trim()); + } +} + +#[test] +fn test_from_setting_file_program_arg() { + for (case_yaml, case_json) in SETTINGS_FILE_TEST_CASE { + let test_case_yaml_file = &format!("{}/{}", TEST_CASE_PATH, case_yaml); + let settings_file = load_file(test_case_yaml_file); + + let test_case_json_file = &format!("{}/{}", TEST_CASE_PATH, case_json); + let expected_json_str = fs::read_to_string(test_case_json_file).unwrap(); + + let exec_prog_args = ExecProgramArgs::from(settings_file); + assert_eq!(expected_json_str.trim(), exec_prog_args.to_json().trim()); + } +} diff --git a/kclvm/src/main.rs b/kclvm/src/main.rs index 49037d9ea..d35de7a48 100644 --- a/kclvm/src/main.rs +++ b/kclvm/src/main.rs @@ -35,6 +35,7 @@ fn main() { // Parse AST program. let program = load_program(&files, Some(args.get_load_program_options())).unwrap(); // Resolve AST program, generate libs, link libs and execute. + // TODO: The argument "plugin_agent" need to be read from python3. execute(program, 0, &ExecProgramArgs::default()).unwrap(); } else { println!("{}", matches.usage());