From 62b3cf2871512b1c50e019fcf1487b36ba2439b6 Mon Sep 17 00:00:00 2001 From: Asocia Date: Sun, 19 Jul 2020 15:55:52 +0300 Subject: [PATCH 1/5] Yield full keyword instead of the shortcut This is a preparation step that will be used by the new feature. --- calico/base.py | 2 +- tests/test_parse.py | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/calico/base.py b/calico/base.py index 8de3d83..f7b78a5 100644 --- a/calico/base.py +++ b/calico/base.py @@ -64,7 +64,7 @@ def __init__(self, type_, data, timeout=-1): def __iter__(self): """Get components of this action as a sequence.""" - yield self.type_.value[0] + yield self.type_.value[1] yield self.data if self.data != pexpect.EOF else "_EOF_" yield self.timeout diff --git a/tests/test_parse.py b/tests/test_parse.py index 616bb6f..27216ad 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -287,7 +287,7 @@ def test_case_with_no_script_should_expect_eof(): run: echo 1 """ runner = parse_spec(source) - assert [tuple(s) for s in runner["c1"].script] == [("e", "_EOF_", -1)] + assert [tuple(s) for s in runner["c1"].script] == [("expect", "_EOF_", -1)] def test_case_run_with_timeout_should_generate_expect_eof_with_timeout(): @@ -296,7 +296,7 @@ def test_case_run_with_timeout_should_generate_expect_eof_with_timeout(): run: echo 1 # timeout: 5 """ runner = parse_spec(source) - assert [tuple(s) for s in runner["c1"].script] == [("e", "_EOF_", 5)] + assert [tuple(s) for s in runner["c1"].script] == [("expect", "_EOF_", 5)] def test_case_run_with_non_numeric_timeout_value_should_raise_error(): @@ -329,7 +329,7 @@ def test_case_script_with_string_action_data_should_be_ok(): - expect: "1" """ runner = parse_spec(source) - assert [tuple(s) for s in runner["c1"].script] == [("e", "1", -1)] + assert [tuple(s) for s in runner["c1"].script] == [("expect", "1", -1)] def test_case_script_with_numeric_action_data_should_raise_error(): @@ -352,7 +352,7 @@ def test_case_script_with_action_data_eof_should_be_ok(): - expect: _EOF_ """ runner = parse_spec(source) - assert [tuple(s) for s in runner["c1"].script] == [("e", "_EOF_", -1)] + assert [tuple(s) for s in runner["c1"].script] == [("expect", "_EOF_", -1)] def test_case_script_with_multiple_action_data_should_raise_error(): @@ -377,7 +377,7 @@ def test_case_script_with_expect_shortcut_should_be_ok(): - e: "1" """ runner = parse_spec(source) - assert [tuple(s) for s in runner["c1"].script] == [("e", "1", -1)] + assert [tuple(s) for s in runner["c1"].script] == [("expect", "1", -1)] def test_case_script_with_send_shortcut_should_be_ok(): @@ -388,7 +388,7 @@ def test_case_script_with_send_shortcut_should_be_ok(): - s: "1" """ runner = parse_spec(source) - assert [tuple(s) for s in runner["c1"].script] == [("s", "1", -1)] + assert [tuple(s) for s in runner["c1"].script] == [("send", "1", -1)] def test_case_script_order_should_be_preserved(): @@ -402,9 +402,9 @@ def test_case_script_order_should_be_preserved(): """ runner = parse_spec(source) assert [tuple(s) for s in runner["c1"].script] == [ - ("e", "foo", -1), - ("s", "1", -1), - ("e", "_EOF_", -1), + ("expect", "foo", -1), + ("send", "1", -1), + ("expect", "_EOF_", -1), ] @@ -416,7 +416,7 @@ def test_case_script_action_with_integer_timeout_value_should_be_ok(): - expect: "foo" # timeout: 5 """ runner = parse_spec(source) - assert [tuple(s) for s in runner["c1"].script] == [("e", "foo", 5)] + assert [tuple(s) for s in runner["c1"].script] == [("expect", "foo", 5)] def test_case_script_action_with_fractional_timeout_value_should_raise_error(): From aca76441728eedcc9940484b0053e69afe3b76d1 Mon Sep 17 00:00:00 2001 From: Asocia Date: Mon, 20 Jul 2020 19:28:17 +0300 Subject: [PATCH 2/5] Get unparsed versions of objects with str function --- calico/base.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/calico/base.py b/calico/base.py index f7b78a5..d396cb4 100644 --- a/calico/base.py +++ b/calico/base.py @@ -22,6 +22,7 @@ import sys from collections import OrderedDict from enum import Enum +from json import dumps import pexpect @@ -68,6 +69,14 @@ def __iter__(self): yield self.data if self.data != pexpect.EOF else "_EOF_" yield self.timeout + def __str__(self): + """Get an str which produces this action when parsed.""" + action_type, action_data, timeout = self + timeout = "" if timeout == -1 else f"# timeout: {timeout}" + if action_data != "_EOF_": + action_data = dumps(action_data) + return f"- {action_type}: {action_data} {timeout}\n" + def run_script(command, script, defs=None, g_timeout=None): """Run a command and check whether it follows a script. @@ -192,6 +201,22 @@ def add_action(self, action): """ self.script.append(action) + def __str__(self): + """Get an str which produces this test case when parsed.""" + script = "\n" + for action in self.script: + script += str(action) + script = textwrap.indent(script, " " * 16) + + spec = f""" + - {self.name}: + run: {self.command} + script: {script} + return: {self.exits} + points: {self.points} + """ + return textwrap.dedent(spec) + def run(self, defs=None, jailed=False, g_timeout=None): """Run this test and produce a report. @@ -255,6 +280,10 @@ def add_case(self, case): super().__setitem__(case.name, case) self.points += case.points if case.points is not None else 0 + def __str__(self): + """Get an str which produces this test suite when parsed.""" + return "\n".join(str(test_case) for test_case in self.values()) + def run(self, tests=None, quiet=False, g_timeout=None): """Run this test suite. From d244fb7f25b40e826b85a5b3e6e000f6ea783ead Mon Sep 17 00:00:00 2001 From: Asocia Date: Mon, 20 Jul 2020 23:08:30 +0300 Subject: [PATCH 3/5] Implement the test spec generator class --- calico/base.py | 85 +++++++++++++++++++++++++++++++++++++++++++++++++ calico/base.pyi | 4 +++ calico/cli.py | 12 +++++-- 3 files changed, 98 insertions(+), 3 deletions(-) diff --git a/calico/base.py b/calico/base.py index d396cb4..9035dec 100644 --- a/calico/base.py +++ b/calico/base.py @@ -19,10 +19,16 @@ import logging import os +import pty import sys +import textwrap +import termios +import tty from collections import OrderedDict from enum import Enum +from itertools import count from json import dumps +from re import escape import pexpect @@ -328,3 +334,82 @@ def run(self, tests=None, quiet=False, g_timeout=None): report["points"] = earned_points return report + + +class Clioc: + """A class that is able to generate test specifications from a reference program's runs.""" + + def __init__(self, argv): + """Initialize this test specification generator. + + :sig: (List[str]) -> None + :param argv: List of command line arguments to run the reference program. + """ + self.argv = argv + """""" + self.__current_test_case = None + + def generate_test_spec(self): + """ + Run the reference program and return the generated test specification. + + :sig: () -> str + :return: The specification that is generated from reference program's run. + """ + test_suite = self.__create_test_suite() + return str(test_suite) + + def __create_test_suite(self): + calico = Calico() + case_number = count(1) + while True: + os.system("clear") + calico.add_case(self.__create_test_case(next(case_number))) + if input("Do you want to continue? [Y/n] ").lower() not in ("y", ""): + print("Abort.") + break + return calico + + def __create_test_case(self, case_num): + case_name = f"case_{case_num}" + print(f"Running for {case_name}...") + self.__current_test_case = TestCase(case_name, " ".join(self.argv), None) + exit_code = self.__spawn() + self.__current_test_case.add_action(Action(ActionType.EXPECT, "_EOF_")) + print(f"{case_name} ended") + points = int(input("Assign points for this run: ")) + self.__current_test_case.points = points + self.__current_test_case.exits = exit_code + return self.__current_test_case + + def __spawn(self): + pid, master_fd = pty.fork() + if pid == pty.CHILD: + os.execlp(self.argv[0], *self.argv) + try: + mode = tty.tcgetattr(pty.STDIN_FILENO) + tty.setraw(master_fd, termios.TCSANOW) + except tty.error: + restore = False + else: + restore = True + + try: + pty._copy(master_fd, self.__read_write_handler, self.__read_write_handler) + except OSError: + if restore: + tty.tcsetattr(pty.STDIN_FILENO, tty.TCSAFLUSH, mode) + + os.close(master_fd) + return os.waitpid(pid, 0)[1] >> 8 + + def __read_write_handler(self, fd): + data = os.read(fd, 1024).decode("utf8") + if fd == 0: # read from stdin + action = Action(ActionType.SEND, data[:-1]) # omit the end line character + self.__current_test_case.add_action(action) + else: # write to stdout + for line in data.splitlines(keepends=True): + action = Action(ActionType.EXPECT, escape(line).replace("\n", "\r\n")) # escape metacharacters + self.__current_test_case.add_action(action) + return data.encode("utf8") diff --git a/calico/base.pyi b/calico/base.pyi index 2cc230a..455e1a3 100644 --- a/calico/base.pyi +++ b/calico/base.pyi @@ -64,3 +64,7 @@ class Calico(OrderedDict): quiet: Optional[List[str]] = ..., g_timeout: Optional[int] = ..., ) -> Mapping[str, Any]: ... + +class Clioc: + def __init__(self, argv: List[str]) -> None: ... + def generate_test_spec(self) -> str: ... diff --git a/calico/cli.py b/calico/cli.py index e9b3afa..1fcde83 100644 --- a/calico/cli.py +++ b/calico/cli.py @@ -21,10 +21,9 @@ import os import sys from argparse import ArgumentParser - from calico import __version__ from calico.parse import parse_spec - +from calico.base import Clioc _logger = logging.getLogger("calico") @@ -40,7 +39,9 @@ def make_parser(prog): """ parser = ArgumentParser(prog=prog) parser.add_argument("--version", action="version", version="%(prog)s " + __version__) - + parser.add_argument( + "-g", "--generate-test-file", nargs="+", + help="generate a test file from run(s)") parser.add_argument("spec", help="test specifications file") parser.add_argument("-d", "--directory", help="change to directory before doing anything") parser.add_argument( @@ -91,6 +92,11 @@ def main(argv=None): arguments = parser.parse_args(argv[1:]) try: spec_filename = os.path.abspath(arguments.spec) + if arguments.generate_test_file: + test_suite_generator = Clioc(arguments.generate_test_file) + with open(spec_filename, "w") as f: + f.write(test_suite_generator.generate_test_spec()) + print("A test file named '%s' has been generated." % arguments.spec) with open(spec_filename) as f: content = f.read() From eb191a2bd187b392ecee47920b5067f51f97a970 Mon Sep 17 00:00:00 2001 From: Asocia Date: Tue, 21 Jul 2020 10:41:10 +0300 Subject: [PATCH 4/5] Don't escape spaces as it causes less readable specs --- calico/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/calico/base.py b/calico/base.py index 9035dec..5de70f2 100644 --- a/calico/base.py +++ b/calico/base.py @@ -410,6 +410,7 @@ def __read_write_handler(self, fd): self.__current_test_case.add_action(action) else: # write to stdout for line in data.splitlines(keepends=True): - action = Action(ActionType.EXPECT, escape(line).replace("\n", "\r\n")) # escape metacharacters + line = escape(line).replace("\\ ", " ").replace("\n", "\r\n") # escape metacharacters, reformat string + action = Action(ActionType.EXPECT, line) self.__current_test_case.add_action(action) return data.encode("utf8") From 5f881c900178f4dc22a98a6a3851b5847dfb60b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=9Eahin=20Akkaya?= Date: Sun, 2 Aug 2020 12:20:58 +0300 Subject: [PATCH 5/5] Simplify spawn function See https://github.com/itublg/calico/pull/13#discussion_r464052578 for more info. --- calico/base.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/calico/base.py b/calico/base.py index 5de70f2..66a7397 100644 --- a/calico/base.py +++ b/calico/base.py @@ -383,23 +383,18 @@ def __create_test_case(self, case_num): return self.__current_test_case def __spawn(self): + # see https://github.com/itublg/calico/pull/13#discussion_r464052578 pid, master_fd = pty.fork() if pid == pty.CHILD: os.execlp(self.argv[0], *self.argv) - try: - mode = tty.tcgetattr(pty.STDIN_FILENO) - tty.setraw(master_fd, termios.TCSANOW) - except tty.error: - restore = False - else: - restore = True + mode = tty.tcgetattr(master_fd) + mode[3] &= ~termios.ECHO # 3 corresponds to lflag, disable echoing + tty.tcsetattr(master_fd, termios.TCSANOW, mode) try: pty._copy(master_fd, self.__read_write_handler, self.__read_write_handler) except OSError: - if restore: - tty.tcsetattr(pty.STDIN_FILENO, tty.TCSAFLUSH, mode) - + pass os.close(master_fd) return os.waitpid(pid, 0)[1] >> 8 @@ -410,7 +405,7 @@ def __read_write_handler(self, fd): self.__current_test_case.add_action(action) else: # write to stdout for line in data.splitlines(keepends=True): - line = escape(line).replace("\\ ", " ").replace("\n", "\r\n") # escape metacharacters, reformat string + line = escape(line).replace("\\ ", " ") # escape metacharacters, reformat string action = Action(ActionType.EXPECT, line) self.__current_test_case.add_action(action) return data.encode("utf8")