From a1f67666276b943e52ae9114aa0cb7e81ae5d8a0 Mon Sep 17 00:00:00 2001 From: Michael Bleuez Date: Sun, 5 Dec 2021 22:51:17 +0100 Subject: [PATCH 1/5] contributing to repl --- CHANGELOG.md | 4 ++ CHECKLIST.md | 1 + CONTRIBUTING.md | 2 +- Cargo.lock | 2 +- ressources/CONTRIBUTING_REPL.md | 122 ++++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 ressources/CONTRIBUTING_REPL.md diff --git a/CHANGELOG.md b/CHANGELOG.md index a3851128..871f81e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v1.0.5 +- fix issue with REPL interpreters staying active after nvim exit +- isolate backend REPL from different neovim instances + ## v1.0.4 - fix python3 fifo and sage interpreters empty line in indented bloc bug diff --git a/CHECKLIST.md b/CHECKLIST.md index 7b593818..2a0d023d 100644 --- a/CHECKLIST.md +++ b/CHECKLIST.md @@ -4,6 +4,7 @@ - update Cargo.lock: `cargo update` - Bump Cargo.toml to next version - cargo fmt --all / cargo check / cargo clippy + - update the changelog - create a version bump commit - merge - create a new tag vX.Y.Z on master diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0c29a4ee..d3a090ad 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -101,7 +101,7 @@ I think I've done a good job, but am I ready to submit a PR? --- REPL - based ? -Mathematica\_original has a pipe (UNIX fifo) - based ReplLikeInterpreter implementation, that may be useful if your language has an interpreter with proper stdio support. Building everything from Rust doesn't quite work yet, so there is a bash script in src/interpreters/Mathematica\_original/init\_repl.sh which can be of some use. +Python3\_fifo has a pipe (UNIX fifo) - based ReplLikeInterpreter implementation, that may be useful if your language has an interpreter with proper stdio support. See [ressources/CONTRIBUTING_REPL.md](CONTRIBUTING_REPL.md) for more info. ### What's the deal with... diff --git a/Cargo.lock b/Cargo.lock index bcc26edd..dc91a80e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -336,7 +336,7 @@ checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309" [[package]] name = "sniprun" -version = "1.0.4" +version = "1.0.5" dependencies = [ "dirs", "libc", diff --git a/ressources/CONTRIBUTING_REPL.md b/ressources/CONTRIBUTING_REPL.md new file mode 100644 index 00000000..a4df68cf --- /dev/null +++ b/ressources/CONTRIBUTING_REPL.md @@ -0,0 +1,122 @@ +# Making a REPL-capable intepreter for sniprun + +## Is it possible ? + +Yes, most of the time, if the language already has an available intepreter. It _could_ be possible otherwise but has yet to be really done. + +To avoid confusion, we'll call the language interpreter 'interpreter', and sniprun's part (implementing the Interpreter trait) the runner. + +## How ? + Two ways, mostly. Either: + - your language has 'quirks' (like for R and Python with the klepto module, see R\_original and Python3\_original) that allow current variables and stuff to be 'saved' to a file then loaded + + or + + - you make use of a named pipe (fifo) and pipe what sniprun says into it. the pipe is connected to a live, running, interpreter for your language. Its output is written to a file and sniprun waits for landmarks (start, end) to be printed. + + + I strongly advise the latter methodology, which has several advantages that I won't discuss here, but can be harder to implement if your language's interpreter has weird stdin/stdou/stderr behavior. Like non-disablable prompts printed to stdout. + + + ## How to implement a pipe-based repl-capable runner + +The best example I'm going to discuss is Python3\_fifo, even if it's a bit bloated from python-specific things. + +Just like you implemented the Intepreter trait for a conventional runner, you'll have to implement the ReplLikeInterpreter trait. Another trait (InterpreterUtils) is automatically implemented and provides features & data persistency to help you survive across different/independent runs. + +1. Running something in the background: + + Unfortunately, this require a first workaround. It's mainly due to how sniprun can't really launch a background process that stays alive, even when the thread executing the user's command exits, and sorta re-launch itself some time later (the interpreters needs some time to launch) to execute the input. The first user command will always fail with a message ("launching .. interpreter in the background, please re-run last snippet"). + ```rust + fn fetch_code_repl(&mut self) -> Result<(), SniprunError> { + if !self.read_previous_code().is_empty() { + // nothing to do, kernel already running + + .... + + self.fetch_code()?; + Ok(()) + } else { + + let init_repl_cmd = self.data.sniprun_root_dir.clone() + "/ressources/init_repl.sh"; + + match daemon() { + Ok(Fork::Child) => { // background child, launch interpreter + let _res = Command::new("....."); // bash init_repl_cmd args + + let pause = std::time::Duration::from_millis(36_000_000); + std::thread::sleep(pause); + + return Err(SniprunError::CustomError("Timeout expired for python3 REPL".to_owned())); + } + Ok(Fork::Parent(_)) => {} // do nothing + Err(_) => info!( + "Python3_fifo could not fork itself to the background to launch the kernel" + ), + }; + + let pause = std::time::Duration::from_millis(100); + std::thread::sleep(pause); + self.save_code("kernel_launched\nimport sys".to_owned()); + + Err(SniprunError::CustomError( + "Python3 kernel launched, re-run your snippet".to_owned(), + )) + } + ``` + The important thing to note is that `self.read_previous_code()` is used to determine whether a kernel was already launched; (`self.get_pid()/set_pid()` can be used to store an incrementing number of 'runs' or the child's PID, or whatever. + +2. Landmarks + +```rust +fn add_boilerplate_repl(&mut self) -> Result<(), SniprunError> { + self.add_boilerplate()?; + let start_mark = String::from("\nprint(\"sniprun_started_id=") + + &self.current_output_id.to_string() + + "\")\n"; + let end_mark = String::from("\nprint(\"sniprun_finished_id=") + + &self.current_output_id.to_string() + + "\")\n"; + let start_mark_err = String::from("\nprint(\"sniprun_started_id=") + + &self.current_output_id.to_string() + + "\", file=sys.stderr)\n"; + let end_mark_err = String::from("\nprint(\"sniprun_finished_id=") + + &self.current_output_id.to_string() + + "\", file=sys.stderr)\n"; + .... +``` + +the user's code has to be wrapped with 4 landmarks that prints 'start run°X', 'end run n°X' messages. Snipruns uses them to determine when the user's code has finished executing. It's then displayed. Note that things can't be displayed 'live', and if someone launches an infinite loop, they won't have any output. + + +3. Waiting for output +``` rust +fn wait_out_file (....){ + loop { + std::thread::sleep( 50 ms); + + //check for content matching the current ID in file for stderr + + //check for content matching the current ID in file for stdout + + //break when something found & finished + } +} +``` +is executed & returned at the end of `execute_repl` that firsts send the user's snippet (wrapped with landmarks) to the FIFO pipe. + +4. Helper scripts +Though not very documented, the `ressources/init_repl.sh` and `ressources/launcher.sh` script are resuable for other runners than Python3\_fifo (see Mathematica that has its own similar scripts in `src/interpreters/Mathematica_original/`. They take care of plugging together the fifo, stdout, stderr files and the interpreter's process. They also take care of closing the intepreter (and free the ressources) when nvim exits + + +### End notes: +- you should take care of separating the working directories for repl-capable interpreters from different neovim sessions, to avoid having nonsense because of mixed fifo and output files content: + +``` +fn new_with_level(...) + Box::new(Python3_fifo { + cache_dir: data.work_dir.clone() + "/python3_fifo/" + &Python3_fifo::get_nvim_pid(&data), + .... +``` + +- disable prompts for your interpreter. They'll pollute stdout From 78bf10ee9d0873d0e4afdd5b440add354c37ced0 Mon Sep 17 00:00:00 2001 From: Michael Bleuez Date: Sun, 5 Dec 2021 22:52:10 +0100 Subject: [PATCH 2/5] fix contributing repl link --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d3a090ad..7078bd0e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -101,7 +101,7 @@ I think I've done a good job, but am I ready to submit a PR? --- REPL - based ? -Python3\_fifo has a pipe (UNIX fifo) - based ReplLikeInterpreter implementation, that may be useful if your language has an interpreter with proper stdio support. See [ressources/CONTRIBUTING_REPL.md](CONTRIBUTING_REPL.md) for more info. +Python3\_fifo has a pipe (UNIX fifo) - based ReplLikeInterpreter implementation, that may be useful if your language has an interpreter with proper stdio support. See [CONTRIBUTING\_REPL.md](ressources/CONTRIBUTING_REPL.md) for more info. ### What's the deal with... From 766afe50330a4ebca8f1f362ce47629abc7154ce Mon Sep 17 00:00:00 2001 From: Michael Bleuez Date: Sat, 11 Dec 2021 10:24:30 +0100 Subject: [PATCH 3/5] fix escaping issues --- src/display.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/display.rs b/src/display.rs index 6f6dc687..cd0f3d31 100644 --- a/src/display.rs +++ b/src/display.rs @@ -328,17 +328,24 @@ fn shorten_err(message: &str) -> String { } fn cleanup_and_escape(message: &str) -> String { - let answer_str = message.replace("\\", "\\\\"); - let answer_str = answer_str.replace("\\\"", "\""); - let answer_str = answer_str.replace("\"", "\\\""); + let mut escaped = String::with_capacity(message.len()); + for c in message.chars() { + match c { + '\x08' => escaped += "\\b", + '\x0c' => escaped += "\\f", + '\t' => escaped += "\\t", + '"' => escaped += "\\\"", + '\\' => escaped += "\\\\", + c => escaped += &c.to_string(), + } + } //remove trailing /starting newlines - let answer_str = answer_str + let answer_str = escaped .trim_start_matches('\n') .trim_end_matches('\n') .to_string(); - - answer_str.replace("\n", "\\\n") + answer_str } fn no_output_wrap(message: &str, data: &DataHolder, current_type: &DisplayType) -> String { From 6b3296579b9b0455accf826f215ee43d11fa0abf Mon Sep 17 00:00:00 2001 From: Michael Bleuez Date: Sat, 11 Dec 2021 10:28:57 +0100 Subject: [PATCH 4/5] version bump --- CHANGELOG.md | 3 +++ Cargo.lock | 24 ++++++++++++------------ Cargo.toml | 2 +- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 871f81e7..6ecf91b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## v1.0.6 +- fix output with escape sequences + ## v1.0.5 - fix issue with REPL interpreters staying active after nvim exit - isolate backend REPL from different neovim instances diff --git a/Cargo.lock b/Cargo.lock index dc91a80e..e5eb5053 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -101,9 +101,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.107" +version = "0.2.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbe5e23404da5b4f555ef85ebed98fb4083e55a00c317800bc2a50ede9f3d219" +checksum = "f98a04dce437184842841303488f70d0188c5f51437d2a834dc097eafa909a01" [[package]] name = "lock_api" @@ -177,9 +177,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.32" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba508cc11742c0dc5c1659771673afbab7a0efab23aa17e854cbab0837ed0b43" +checksum = "fb37d2df5df740e582f28f8560cf425f52bb267d872fe58358eadb554909f07a" dependencies = [ "unicode-xid", ] @@ -259,9 +259,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.5" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +checksum = "254df5081ce98661a883445175e52efe99d1cb2a5552891d965d2f5d0cad1c16" [[package]] name = "scopeguard" @@ -271,9 +271,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "serde" -version = "1.0.130" +version = "1.0.131" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913" +checksum = "b4ad69dfbd3e45369132cc64e6748c2d65cdfb001a2b1c232d128b4ad60561c1" [[package]] name = "serde_bytes" @@ -286,9 +286,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.70" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e277c495ac6cd1a01a58d0a0c574568b4d1ddf14f59965c6a58b8d96400b54f3" +checksum = "d0ffa0837f2dfa6fb90868c2b5468cad482e175f7dad97e7421951e663f2b527" dependencies = [ "itoa", "ryu", @@ -362,9 +362,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.81" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2afee18b8beb5a596ecb4a2dce128c719b4ba399d34126b9e4396e3f9860966" +checksum = "8daf5dd0bb60cbd4137b1b587d2fc0ae729bc07cf01cd70b36a1ed5ade3b9d59" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 8df3503b..3745a343 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sniprun" -version = "1.0.5" +version = "1.0.6" authors = ["michaelb "] edition = "2018" From 9628f995924f582040e68a5df49f5aff66287668 Mon Sep 17 00:00:00 2001 From: Michael Bleuez Date: Sat, 11 Dec 2021 10:41:58 +0100 Subject: [PATCH 5/5] add codecov sane target --- .codecov.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.codecov.yml b/.codecov.yml index d561cd54..47f6f748 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -5,3 +5,9 @@ coverage: target: 1% threshold: 1% path: "src" + + project: + default: + target: 50% + threshold: 50% + path: "src"