-
Notifications
You must be signed in to change notification settings - Fork 4
Generate test specification from a reference program's runs #13
base: master
Are you sure you want to change the base?
Changes from 3 commits
62b3cf2
aca7644
d244fb7
eb191a2
5f881c9
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -19,9 +19,16 @@ | |
|
||
import logging | ||
import os | ||
import pty | ||
import sys | ||
import textwrap | ||
import termios | ||
import tty | ||
from collections import OrderedDict | ||
from enum import Enum | ||
from itertools import count | ||
from json import dumps | ||
from re import escape | ||
|
||
import pexpect | ||
|
||
|
@@ -64,10 +71,18 @@ def __init__(self, type_, data, timeout=-1): | |
|
||
def __iter__(self): | ||
"""Get components of this action as a sequence.""" | ||
yield self.type_.value[0] | ||
yield self.type_.value[1] | ||
yield self.data if self.data != pexpect.EOF else "_EOF_" | ||
yield self.timeout | ||
|
||
def __str__(self): | ||
"""Get an str which produces this action when parsed.""" | ||
action_type, action_data, timeout = self | ||
timeout = "" if timeout == -1 else f"# timeout: {timeout}" | ||
if action_data != "_EOF_": | ||
action_data = dumps(action_data) | ||
return f"- {action_type}: {action_data} {timeout}\n" | ||
|
||
|
||
def run_script(command, script, defs=None, g_timeout=None): | ||
"""Run a command and check whether it follows a script. | ||
|
@@ -192,6 +207,22 @@ def add_action(self, action): | |
""" | ||
self.script.append(action) | ||
|
||
def __str__(self): | ||
"""Get an str which produces this test case when parsed.""" | ||
script = "\n" | ||
for action in self.script: | ||
script += str(action) | ||
script = textwrap.indent(script, " " * 16) | ||
|
||
spec = f""" | ||
- {self.name}: | ||
run: {self.command} | ||
script: {script} | ||
return: {self.exits} | ||
points: {self.points} | ||
""" | ||
return textwrap.dedent(spec) | ||
|
||
def run(self, defs=None, jailed=False, g_timeout=None): | ||
"""Run this test and produce a report. | ||
|
||
|
@@ -255,6 +286,10 @@ def add_case(self, case): | |
super().__setitem__(case.name, case) | ||
self.points += case.points if case.points is not None else 0 | ||
|
||
def __str__(self): | ||
"""Get an str which produces this test suite when parsed.""" | ||
return "\n".join(str(test_case) for test_case in self.values()) | ||
|
||
def run(self, tests=None, quiet=False, g_timeout=None): | ||
"""Run this test suite. | ||
|
||
|
@@ -299,3 +334,82 @@ def run(self, tests=None, quiet=False, g_timeout=None): | |
|
||
report["points"] = earned_points | ||
return report | ||
|
||
|
||
class Clioc: | ||
"""A class that is able to generate test specifications from a reference program's runs.""" | ||
|
||
def __init__(self, argv): | ||
"""Initialize this test specification generator. | ||
|
||
:sig: (List[str]) -> None | ||
:param argv: List of command line arguments to run the reference program. | ||
""" | ||
self.argv = argv | ||
"""""" | ||
self.__current_test_case = None | ||
|
||
def generate_test_spec(self): | ||
""" | ||
Run the reference program and return the generated test specification. | ||
|
||
:sig: () -> str | ||
:return: The specification that is generated from reference program's run. | ||
""" | ||
test_suite = self.__create_test_suite() | ||
return str(test_suite) | ||
|
||
def __create_test_suite(self): | ||
calico = Calico() | ||
case_number = count(1) | ||
while True: | ||
os.system("clear") | ||
calico.add_case(self.__create_test_case(next(case_number))) | ||
if input("Do you want to continue? [Y/n] ").lower() not in ("y", ""): | ||
print("Abort.") | ||
break | ||
return calico | ||
|
||
def __create_test_case(self, case_num): | ||
case_name = f"case_{case_num}" | ||
print(f"Running for {case_name}...") | ||
self.__current_test_case = TestCase(case_name, " ".join(self.argv), None) | ||
exit_code = self.__spawn() | ||
self.__current_test_case.add_action(Action(ActionType.EXPECT, "_EOF_")) | ||
print(f"{case_name} ended") | ||
points = int(input("Assign points for this run: ")) | ||
self.__current_test_case.points = points | ||
self.__current_test_case.exits = exit_code | ||
return self.__current_test_case | ||
|
||
def __spawn(self): | ||
pid, master_fd = pty.fork() | ||
if pid == pty.CHILD: | ||
os.execlp(self.argv[0], *self.argv) | ||
try: | ||
mode = tty.tcgetattr(pty.STDIN_FILENO) | ||
tty.setraw(master_fd, termios.TCSANOW) | ||
except tty.error: | ||
restore = False | ||
else: | ||
restore = True | ||
|
||
try: | ||
pty._copy(master_fd, self.__read_write_handler, self.__read_write_handler) | ||
except OSError: | ||
if restore: | ||
tty.tcsetattr(pty.STDIN_FILENO, tty.TCSAFLUSH, mode) | ||
|
||
os.close(master_fd) | ||
return os.waitpid(pid, 0)[1] >> 8 | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This function is almost exact copy of mode[OFLAG] = mode[OFLAG] & ~(OPOST)
mode[LFLAG] = mode[LFLAG] & ~(ECHO | ICANON | IEXTEN | ISIG) # and only `~ECHO` is enough here actually
This, for example when enabled puts a carriage return,
This will make the pseudo-terminal device to echo it's stdin when enabled. We already capture it through our stdin, so we should disable echoing of the same characters. This line is needed. The other flags, ( I also realized that The purpose of this review was to mention |
||
|
||
def __read_write_handler(self, fd): | ||
data = os.read(fd, 1024).decode("utf8") | ||
if fd == 0: # read from stdin | ||
action = Action(ActionType.SEND, data[:-1]) # omit the end line character | ||
self.__current_test_case.add_action(action) | ||
else: # write to stdout | ||
for line in data.splitlines(keepends=True): | ||
action = Action(ActionType.EXPECT, escape(line).replace("\n", "\r\n")) # escape metacharacters | ||
self.__current_test_case.add_action(action) | ||
return data.encode("utf8") |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I have tried
ruamel.yaml.dump
but it's not doing what I want while dumping strings:There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is no longer a problem. I'm able to generate the yaml file with
ruamel.yaml
. I can work on it if this is OK to merge.