diff --git a/Makefile b/Makefile index 4eea08c9..7a9077c4 100644 --- a/Makefile +++ b/Makefile @@ -156,6 +156,9 @@ lang-javascript-test: clean_test lang-go-test: clean_test @$(python_bin) test/r.py @test/lang-go.env +lang-rust-test: clean_test + @$(python_bin) test/r.py @test/lang-rust.env + # not supported lang-elixir-test: clean_test @$(python_bin) test/r.py @test/lang-elixir.env diff --git a/README.md b/README.md index 9478e8a9..a1504d3b 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,7 @@ Currently we support: - [iasm](https://byexamples.github.io/byexample/languages/iasm) - [PowerShell](https://byexamples.github.io/byexample/languages/powershell) - [Go](https://byexamples.github.io/byexample/languages/go) + - [Rust](https://byexamples.github.io/byexample/languages/rust) More languages will be supported in the future. Stay tuned. diff --git a/byexample/cmdline.py b/byexample/cmdline.py index 09aa56db..4ec2d144 100644 --- a/byexample/cmdline.py +++ b/byexample/cmdline.py @@ -422,6 +422,14 @@ def parse_args(args=None): help= "delay in seconds before sending a line to an runner/interpreter; 0 disable this (default)." ) + g.add_argument( + "-x-delayafterprompt", + metavar="", + default=None, + type=lambda n: None if n == 0 else float(n), + help= + "delay in seconds after the prompt to capture more output; 0 disable this (default)." + ) g.add_argument( "-x-min-rcount", metavar="", diff --git a/byexample/modules/cpp.py b/byexample/modules/cpp.py index 7bca02b1..1deecc5c 100644 --- a/byexample/modules/cpp.py +++ b/byexample/modules/cpp.py @@ -128,7 +128,7 @@ def run(self, example, options): # so we disable this: options['geometry'] = self._terminal_default_geometry - # cling's output requeries to be emulated by an ANSI Terminal + # cling's output requires to be emulated by an ANSI Terminal # so we force this (see _get_output()) options['term'] = 'ansi' diff --git a/byexample/modules/rust.py b/byexample/modules/rust.py new file mode 100644 index 00000000..dcb078f5 --- /dev/null +++ b/byexample/modules/rust.py @@ -0,0 +1,306 @@ +r""" +Example: + + >> 1 + 2 + 3 + + >> fn hello() { + :: println!("hello bla world"); // classic + :: } + + >> hello(); // byexample: +norm-ws + hello <...> world + + >> let mut j = 2; + >> for i in 0..4 { + :: j += i; + :: }; // extra ; to suppress output check + + >> j + 3 + 11 + + >> println!("{}", "this\n + :: is a multiline\n + :: string\n"); + this + is a multiline + string + + >> /* this + :: is a multiline + :: comment */ + + >> #[derive(Debug)] + :: struct Point { + :: x: f32, + :: y: f32, + :: } + + >> let p1 = Point { x: 2.0, y: 3.0 }; + >> p1 + Point { x: 2.0, y: 3.0 } + + >> #[derive(Debug)] + :: struct Child(i32); + + >> #[derive(Debug)] + :: struct Parent(Child); + + >> let c1 = Child(42); + >> let c2 = Parent(Child(33)); + + >> c1 + Child(42) + >> c2 + Parent(Child(33)) + + Pretty print for arrays and tuple are supported but only + up to 12 elements. This is a restriction of Rust. + + >> let array = [1, 2, 3]; + >> let tuple = (1, true, 2.3); + + >> array + [1, 2, 3] + >> tuple + (1, true, 2.3) + + Slices are not supported in the main space but they are okay + in a function + >> let slice: &[i32] = &array[0..2]; // byexample: +skip + + >> fn bar(slice: &[i32]) { + :: println!("{:?}", slice); + :: } + + >> bar(&array[0..2]); + [1, 2] + + >> const PI : f64 = 3.1416; + >> PI + 3.1416 + + Closure are not supported in the main space but they are okay + in a function + >> let i = 4; + >> let closure_explicit = |j: i32| -> i32 { i + j }; // byexample: +skip + >> let closure_implicit = |j | i + j ; // byexample: +skip + >> let one = || 1; // byexample: +skip + + >> let k = Box::new(42); // byexample: +skip + >> let stealer = move || { k }; // byexample: +skip + >> println!("{:?}", stealer()); // byexample: +skip + + >> fn baz() { + :: let i = 4; + :: let closure_explicit = |j: i32| -> i32 { i + j }; + :: let closure_implicit = |j | i + j ; + :: let one = || 1; + :: + :: let k = Box::new(42); + :: let stealer = move || { k }; + :: println!("{:?} {:?} {:?} {:?}", closure_explicit(2), closure_implicit(2), one(), stealer()); + :: } + + >> baz(); + 6 6 1 42 + + + Types + >> type Nanosecs = u64; + >> let a : Nanosecs = 2; + >> a + 2 + + Scopes + >> let y = { + :: let z = 11; + :: i + z // this is an *expression*, the result of the block + :: }; + >> y + 15 + + Flow control + >> let mut m = if i < 1 { + :: i + 2 + :: } else { + :: i + 8 + :: }; + + >> loop { + :: if m >= 20 { + :: break; + :: } + :: + :: m += 1; + :: if m % 2 == 0 { + :: continue; + :: } + :: + :: println!("m is odd: {}", m); + :: }; + m is odd: 13 + m is odd: 15 + m is odd: 17 + m is odd: 19 + + >> 'theloop : while true { + :: m -= 1; + :: if m == 0 { + :: break 'theloop; + :: } + :: }; + + >> m + 0 + +""" + +from __future__ import unicode_literals +import sys, time +import byexample.regex as re +from byexample.common import constant +from byexample.parser import ExampleParser +from byexample.runner import ExampleRunner, PexpectMixin, ShebangTemplate +from byexample.finder import ExampleFinder + +stability = 'experimental' + + +class RustPromptFinder(ExampleFinder): + target = 'rust-prompt' + + @constant + def example_regex(self): + return re.compile( + r''' + (?P + (?:^(?P [ ]*) (?:>>)[ ] .*) # PS1 line + (?:\n [ ]* :: .*)*) # PS2 lines + \n? + ## Want consists of any non-blank lines that do not start with PS1 + (?P (?:(?![ ]*$) # Not a blank line + (?![ ]*(?:>>)) # Not a line starting with PS1 + .+$\n? # But any other line + )*) + ''', re.MULTILINE | re.VERBOSE + ) + + def get_language_of(self, *args, **kargs): + return 'rust' + + def get_snippet_and_expected(self, match, where): + snippet, expected = ExampleFinder.get_snippet_and_expected( + self, match, where + ) + + snippet = self._remove_prompts(snippet) + return snippet, expected + + def _remove_prompts(self, snippet): + lines = snippet.split("\n") + return '\n'.join(line[3:] for line in lines) + + +class RustParser(ExampleParser): + language = 'rust' + + @constant + def example_options_string_regex(self): + # anything of the form: + # // byexample: +FOO -BAR +ZAZ=42 + return re.compile(r'//\s*byexample:\s*([^\n\'"]*)$', re.MULTILINE) + + +class RustInterpreter(ExampleRunner, PexpectMixin): + language = 'rust' + + def __init__(self, verbosity, encoding, **unused): + self.encoding = encoding + + PexpectMixin.__init__(self, PS1_re=r'>> ', any_PS_re=r'>> ') + + def get_default_cmd(self, *args, **kargs): + return "%e %p %a", { + 'e': + "/usr/bin/env", + 'p': + "evcxr", + 'a': [ + "--disable-readline", # no readline + "--opt", + "0", # disable optimizations (reduce exec time) + ] + } + + def run(self, example, options): + # evcxr's output requeries to be emulated by an ANSI Terminal + # so we force this (see _get_output()) + options['term'] = 'ansi' + options['timeout'] = (max(options['timeout'], 8)) + return PexpectMixin._run(self, example, options) + + def _run_impl(self, example, options): + src = example.source + src = self._strip_and_join_lines_into_one(src, strip=True) + return self._exec_and_wait(src, options, from_example=example) + + _SINGLE_LINE_COMMENT_RE = re.compile(r'//[^\n]*$') + + def _strip_and_join_lines_into_one(self, src, strip): + # evcxr doesn't support multiline code if the readline is disabled + # so the simplest thing to do is to collaps all the lines into one + # Rust is a language which syntax should not be affected by this + # in contrast to Python **except** when a single-line comment "//" + # is used. + # + # Valid code like this: + # fn foo() { // super + # 42 + # } + # It will not work as it will be seen as: + # fn foo () { // super 42 } + # + # The workaround is to strip those comments. + # + _RE = self._SINGLE_LINE_COMMENT_RE + lines = src.split('\n') + if strip: + lines = (_RE.sub('', line) for line in lines) + + return ''.join(lines) + + def interact(self, example, options): + PexpectMixin.interact(self) + + def initialize(self, options): + shebang, tokens = self.get_default_cmd() + shebang = options['shebangs'].get(self.language, shebang) + + cmd = ShebangTemplate(shebang).quote_and_substitute(tokens) + + options.up() + # evcxr can be quite slow so we increase the timeout by default + options['x']['dfl_timeout'] = (max(options['x']['dfl_timeout'], 30)) + self._spawn_interpreter(cmd, options) + + # enable sccache (https://github.com/mozilla/sccache) so evcxr + # can speed up the compilation of the examples + self._exec_and_wait( + ':sccache 1', options, timeout=options['x']['dfl_timeout'] + ) + options.down() + + def _expect_delayed_output(self, options): + # evcxr runs the example in background so it may return before + # the example finishes. In those cases we want to wait a little + # more to capture all the output + delay = options['x']['delayafterprompt'] or 0 + options['x']['delayafterprompt'] = max(delay, 0.25) + return super()._expect_delayed_output(options) + + def shutdown(self): + self._shutdown_interpreter() + + def cancel(self, example, options): + return self._abort(example, options) diff --git a/byexample/runner.py b/byexample/runner.py index 73bb9ea9..47feb128 100644 --- a/byexample/runner.py +++ b/byexample/runner.py @@ -437,6 +437,7 @@ def _exec_and_wait(self, source, options, *, from_example=None, **kargs): self._expect_prompt_or_type( options, countdown, prompt_re=self._PS1_re, input_list=input_list ) + self._expect_delayed_output(options) if input_list: s = short_string(input_list[0][-1]) @@ -451,6 +452,27 @@ def _exec_and_wait(self, source, options, *, from_example=None, **kargs): return self._get_output(options) + @profile + def _expect_delayed_output(self, options): + ''' Some interpreters may output text *after* printing the prompt. + This method is called to do a last output recollection before + processing the output and returning it to the user. + ''' + timeout = options['x']['delayafterprompt'] + if not timeout: + return + + expect = [pexpect.TIMEOUT, pexpect.EOF] + Timeout, EOF = range(len(expect)) + + what = self._interpreter.expect(expect, timeout=timeout) + + output = self._interpreter.before + self._add_output(output) + + if what == EOF: + self._interpreter_closed_unexpectedly_error(options) + def _create_terminal(self, options): rows, cols = options['geometry'] @@ -613,10 +635,7 @@ def _expect_prompt( what, output = self._expect_and_read(expect, timeout, expect_kinds) countdown.stop() - if self._last_output_may_be_incomplete: - self._output_between_prompts[-1] += output - else: - self._output_between_prompts.append(output) + self._add_output(output) if what == Timeout: msg = "Prompt not found: the code is taking too long to finish or there is a syntax error.\n\nLast 1000 bytes read:\n%s" @@ -629,15 +648,32 @@ def _expect_prompt( return False elif what == EOF: - msg = "Interpreter closed unexpectedly.\nThis could happen because the example triggered a close/shutdown/exit action,\nthe interpreter was killed by someone else or because the interpreter just crashed.\n\nLast 1000 bytes read:\n%s" - msg = msg % ''.join(self._output_between_prompts)[-1000:] - out = self._get_output(options) - raise InterpreterClosedUnexpectedly(msg, out) + self._interpreter_closed_unexpectedly_error(options) assert what == PS_found self._last_output_may_be_incomplete = False return True + def _interpreter_closed_unexpectedly_error(self, options): + msg = "Interpreter closed unexpectedly.\nThis could happen because the example triggered a close/shutdown/exit action,\nthe interpreter was killed by someone else or because the interpreter just crashed.\n\nLast 1000 bytes read:\n%s" + msg = msg % ''.join(self._output_between_prompts)[-1000:] + out = self._get_output(options) + raise InterpreterClosedUnexpectedly(msg, out) + + def _add_output(self, output): + ''' Add the given output to the output between prompts. + + If _last_output_may_be_incomplete is set, assume that the lastest + output was incomplete and the given output is a continuation + of it (like part of the same line). + + Otherwise assume that it is a new chunk/line. + ''' + if self._last_output_may_be_incomplete: + self._output_between_prompts[-1] += output + else: + self._output_between_prompts.append(output) + @profile def _expect_and_read(self, expect_list, timeout, expect_kinds): ''' Interact with the Pexpect instance, expect one of the expect @@ -799,6 +835,7 @@ def _recover_prompt_sync(self, example, options, cnt=5): This algorithm is not bug-free, just a best-effort one. ''' + err_msg = "Interpreter closed unexpectedly during the recovering. May be it is timming issue. Try to increase the timeout for the example." try: # wait for the prompt, ignore any extra output self._expect_prompt( @@ -811,6 +848,9 @@ def _recover_prompt_sync(self, example, options, cnt=5): except TimeoutException as ex: self._drop_output() good = False + except InterpreterClosedUnexpectedly: + clog().warn(err_msg) + good = False if good: try: @@ -830,5 +870,8 @@ def _recover_prompt_sync(self, example, options, cnt=5): good = False # we cannot ensure that we are in sync except TimeoutException as ex: self._drop_output() + except InterpreterClosedUnexpectedly: + clog().warn(err_msg) + good = False return good diff --git a/docs/_includes/idx.html b/docs/_includes/idx.html index 5ee3f047..48231156 100644 --- a/docs/_includes/idx.html +++ b/docs/_includes/idx.html @@ -57,6 +57,7 @@

Languages Supported

  • iasm
  • PowerShell
  • Go
    • +
  • Rust

    • Recipes and Tricks

    diff --git a/docs/index.md b/docs/index.md index c1b81a58..ae8a0af9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -52,18 +52,24 @@ Currently ``byexample`` supports the following languages:
    -
    +
    iasm Logo

    iasm

    -
    +
    PowerShell Logo

    PowerShell

    -
    +
    +
    +
    Go Logo

    Go

    +
    + Rust Logo +

    Rust

    +
    diff --git a/docs/languages/rust.md b/docs/languages/rust.md new file mode 100644 index 00000000..40cd7847 --- /dev/null +++ b/docs/languages/rust.md @@ -0,0 +1,174 @@ +# Rust + +Run the `Rust` examples calling `byexample` as: + +```shell +$ byexample -l rust your-file-here # byexample: +skip +``` + +You need the have installed `evcxr`, an interactive interpreter +for `Rust`. + +Check its [download page](https://github.com/google/evcxr) + +> **Note**: current versions of `evcxr` (0.10.0) has a high run time: +> around 20 seconds for starting up the runner and around 2 seconds per +> example. There is [an ongoing work](https://github.com/google/evcxr/issues/184) +> to improve this. + +> **Stability**: ``experimental`` - non backward compatibility changes are +> possible or even removal between versions (even patch versions). + +> *New* in ``byexample 10.2.0``. + +## Find interactive examples + +For ``Rust``, ``byexample`` uses the ``>>`` string as the primary prompt +and ``::`` as the secondary prompt. + + +```rust +>> 1 + 2 +3 + +>> fn hello() { +:: println!("hello bla world"); // classic +:: } + +>> hello(); // byexample: +norm-ws +hello <...> world +``` + +> Currently the flags/options can only be set in the single-line +> comments (`//`) + +### The object returned + +As you may know in `Rust` almost everything is an expression and +`byexample` will take and print the value of the expression. + +```rust +>> 1 + 2 +3 +``` + +These expressions can be turn into a statement appending a semicolon +in which case `byexample` will not print anything. + +```rust +>> 1 + 2; +``` + +## Pretty print + +`byexample` uses the default *pretty print* of Rust with the format +`"{:?}"`. + +This works quite well out of the box for native objects and objects with +the `#[derive(Debug)]`: + +```rust +>> #[derive(Debug)] +:: struct Point { +:: x: f32, +:: y: f32, +:: } + +>> let p1 = Point { x: 2.0, y: 3.0 }; +>> p1 +Point { x: 2.0, y: 3.0 } + +>> let array = [1, 2, 3]; +>> let tuple = (1, true, 2.3); + +>> array +[1, 2, 3] +>> tuple +(1, true, 2.3) +``` + +> **Note**: pretty print for arrays and tuple are supported but only +> up to 12 elements. This is a restriction of Rust. + +## Known limitations + + +### Runtime performance + +`evcxr` has a high runtime cost. `byexample` waits up to 30 seconds for +the interpreter to be up and up to 8 seconds per example. + +If you want to increase these timeouts you can do it with +`-x-dfl-timeout` and `--timeout`. + +### Output arrives late + +`evcxr` *may* tell `byexample` that an example finished +*before* it really did. `byexample` works around this and waits a little +after each example execution. + +You can control how much time `byexample` will wait with +`-x-delayafterprompt`. The default is a quarter of a second. + +If you run an example and this fails because the last part of the +expected output is missing **and** that output appears at the begin +of the *next* example, you are hitting this limitation. + +Try to increment the wait time with `-x-delayafterprompt`. + +### Slices and closures + +Slices are not supported if they are written in the main scope +but there is no problem if the slices are defined in the scope of a +function + +```rust +>> let array = [1, 2, 3]; + +>> // this will not work +>> let slice: &[i32] = &array[0..2]; // byexample: +skip + +>> // but this is perfectly fine +>> fn bar(slice: &[i32]) { +:: println!("{:?}", slice); +:: } + +>> bar(&array[0..2]); +[1, 2] +``` + +The same limitation happens with closures: they are not supported in the +main scope but in the functions are okay. + +```rust +>> let i = 4; + +>> // this will not work +>> let closure_implicit = |j| i + j; // byexample: +skip + +>> // but this is perfectly fine +>> fn baz() { +:: let i = 4; +:: let closure_implicit = |j| i + j; +:: println!("{:?}", closure_implicit(2)); +:: } + +>> baz(); +6 +``` + +### Type text + +The [type](/{{ site.uprefix }}/basic/input) +feature (`+type`) is not supported. + +## Rust specific options + +```shell +$ byexample -l rust --show-options # byexample: +norm-ws +<...> +rust's specific options +----------------------- + None. +<...> +``` diff --git a/media/logos/README.md b/media/logos/README.md index db56eddb..5276a55a 100644 --- a/media/logos/README.md +++ b/media/logos/README.md @@ -13,3 +13,4 @@ The ByExample logo created by R.V. Facultad de Ingeniería Equipo de Quimera Tal The Microsoft's PowerShell logo. [13](https://en.wikipedia.org/wiki/PowerShell) The iasm logo. [14](https://github.com/bad-address/iasm) The Go logo. [15](https://golang.org/) +The Rust logo. [16](https://www.rust-lang.org/) diff --git a/media/logos/rust_logo.png b/media/logos/rust_logo.png new file mode 100644 index 00000000..3536d8ab Binary files /dev/null and b/media/logos/rust_logo.png differ diff --git a/test/lang-rust.env b/test/lang-rust.env new file mode 100644 index 00000000..1f848579 --- /dev/null +++ b/test/lang-rust.env @@ -0,0 +1,6 @@ +--pretty=all +--ff +--timeout=8 +--language=rust,shell,python +docs/languages/rust.md +byexample/modules/rust.py