From 53cb9701ec11a5bdd74959f2b79742f50e91fd74 Mon Sep 17 00:00:00 2001 From: Konstantin Osipov Date: Tue, 14 Dec 2010 14:09:18 +0300 Subject: [PATCH] Test-runner: next increment. Add test/lib/ for python modules. Create the first module that implements admin connection. Automatically read host and port and pidfile from tarantool configuration file, and thus remove them from suite.ini. Change .gitignore and remove a too broad ignore mask (it ignored __init__.py, which is mandatory file name for a module in python). Fix a bug in config reader for tarantool that would leave "box.pid" in quotes. Move TestSuite and Test to a lib/ module. Patch cmdline.py and ./admin.py to read input in line-buffered fashion (used to be block-buffered), regardless of whether input is a terminal or a pipe. This allows to work with these tools interactively. --- .gitignore | 6 +- test/admin.py | 53 +---- test/box/suite.ini | 1 - test/cmd/suite.ini | 1 - test/cmdline.py | 7 +- test/lib/__init__.py | 0 test/lib/admin.py | 66 ++++++ test/lib/tarantool_silverbox_server.py | 124 ++++++++++ test/lib/test_suite.py | 208 +++++++++++++++++ test/test-run.py | 300 +------------------------ 10 files changed, 422 insertions(+), 344 deletions(-) create mode 100644 test/lib/__init__.py create mode 100755 test/lib/admin.py create mode 100644 test/lib/tarantool_silverbox_server.py create mode 100644 test/lib/test_suite.py diff --git a/.gitignore b/.gitignore index d9f849d028cb..e28629f86410 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,10 @@ .gitignore .gdb_history TAGS -_* +_debug_box +_release_box +_debug_feeder +_release_feeder config.mk lcov *.o @@ -10,3 +13,4 @@ lcov *.xlog tarantool_version.h test/var +test/lib/*.pyc diff --git a/test/admin.py b/test/admin.py index 1f171e03ef9a..c4dbd944b57d 100755 --- a/test/admin.py +++ b/test/admin.py @@ -33,13 +33,14 @@ import socket import sys import string +import lib.admin class Options: def __init__(self): """Add all program options, with their defaults.""" parser = argparse.ArgumentParser( - description = "Tarantool regression test suite client.") + description = "Tarantool administrative console client.") parser.add_argument( "--host", @@ -76,55 +77,20 @@ def __init__(self): self.args = parser.parse_args() -class Connection: - def __init__(self, host, port): - self.host = host - self.port = port - self.is_connected = False - - def connect(self): - self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.socket.connect((self.host, self.port)) - self.is_connected = True - - def disconnect(self): - if self.is_connected: - self.socket.close() - self.is_connected = False - - def execute(self, command): - self.socket.sendall(command) - - bufsiz = 4096 - res = "" - - while True: - buf = self.socket.recv(bufsiz) - if not buf: - break - res+= buf; - if res.rfind("---\n"): - break - - return res - - def __enter__(self): - self.connect() - return self - - def __exit__(self, type, value, tb): - self.disconnect() - - def main(): options = Options() try: - with Connection(options.args.host, options.args.port) as con: + with lib.admin.Connection(options.args.host, options.args.port) as con: result_prefix = options.args.result_prefix prompt = options.args.prompt if prompt != "": sys.stdout.write(prompt) - for line in iter(sys.stdin.readline, ""): +# We need line-buffering, and thus don't use 'for' loop + while True: + line = sys.stdin.readline() + sys.stdout.flush() + if not line: + break; if result_prefix != None and line.find(result_prefix) == 0: continue output = con.execute(line) @@ -134,6 +100,7 @@ def main(): else: sys.stdout.write(output) sys.stdout.write(prompt) + sys.stdout.flush() return 0 except (RuntimeError, socket.error, KeyboardInterrupt) as e: diff --git a/test/box/suite.ini b/test/box/suite.ini index 5d7d6ddca153..97d27f50155d 100644 --- a/test/box/suite.ini +++ b/test/box/suite.ini @@ -2,4 +2,3 @@ description = tarantool/silverbox, minimal configuration client = admin.py --prompt "" --result-prefix "r> " config = tarantool.cfg -pidfile = box.pid diff --git a/test/cmd/suite.ini b/test/cmd/suite.ini index 25ddabc50755..33adae646fb2 100644 --- a/test/cmd/suite.ini +++ b/test/cmd/suite.ini @@ -2,4 +2,3 @@ description = tarantool/silverbox, command line options client = cmdline.py $server --result-prefix "r> " config = tarantool.cfg -pidfile = box.pid diff --git a/test/cmdline.py b/test/cmdline.py index 3e8a6a1d3468..5315ed105fe7 100755 --- a/test/cmdline.py +++ b/test/cmdline.py @@ -61,7 +61,11 @@ def main(): try: result_prefix = args.result_prefix - for line in iter(sys.stdin.readline, ""): +# We need line-buffering, and thus don't use 'for' loop + while True: + line = sys.stdin.readline() + if not line: + break; if result_prefix != None and line.find(result_prefix) == 0: continue path = string.join([args.bin, line]) @@ -73,6 +77,7 @@ def main(): "\n" + result_prefix) else: sys.stdout.write(output) + sys.stdout.flush() return 0 except RuntimeError as e: diff --git a/test/lib/__init__.py b/test/lib/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/lib/admin.py b/test/lib/admin.py new file mode 100755 index 000000000000..f42b0f64d972 --- /dev/null +++ b/test/lib/admin.py @@ -0,0 +1,66 @@ +__author__ = "Konstantin Osipov " + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +import socket +import sys +import string + +class Connection: + def __init__(self, host, port): + self.host = host + self.port = port + self.is_connected = False + + def connect(self): + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.socket.connect((self.host, self.port)) + self.is_connected = True + + def disconnect(self): + if self.is_connected: + self.socket.close() + self.is_connected = False + + def execute(self, command): + self.socket.sendall(command) + + bufsiz = 4096 + res = "" + + while True: + buf = self.socket.recv(bufsiz) + if not buf: + break + res+= buf; + if res.rfind("---\n"): + break + + return res + + def __enter__(self): + self.connect() + return self + + def __exit__(self, type, value, tb): + self.disconnect() + diff --git a/test/lib/tarantool_silverbox_server.py b/test/lib/tarantool_silverbox_server.py new file mode 100644 index 000000000000..be76a0d3e6f6 --- /dev/null +++ b/test/lib/tarantool_silverbox_server.py @@ -0,0 +1,124 @@ +import os +import stat +import shutil +import subprocess +import pexpect +import sys +import signal +import time + +class TarantoolSilverboxServer: + """Server represents a single server instance. Normally, the + program operates with only one server, but in future we may add + replication slaves. The server is started once at the beginning + of each suite, and stopped at the end.""" + + def __init__(self, args, config, pidfile): + """Set server options: path to configuration file, pid file, exe, etc.""" + self.args = args + self.path_to_config = config + self.path_to_pidfile = os.path.join(args.vardir, pidfile) + self.path_to_exe = None + self.abspath_to_exe = None + self.is_started = False + + def start(self): + """Start server instance: check if the old one exists, kill it + if necessary, create necessary directories and files, start + the server. The server working directory is taken from 'vardir', + specified in the prgoram options. + Currently this is implemented for tarantool_silverbox only.""" + + if not self.is_started: + print "Starting the server..." + + if self.path_to_exe == None: + self.path_to_exe = self.find_exe() + self.abspath_to_exe = os.path.abspath(self.path_to_exe) + + print " Found executable at " + self.path_to_exe + "." + + print " Creating and populating working directory in " +\ + self.args.vardir + "..." + + if os.access(self.args.vardir, os.F_OK): + print " Found old vardir, deleting..." + self.kill_old_server() + shutil.rmtree(self.args.vardir, ignore_errors = True) + + os.mkdir(self.args.vardir) + shutil.copy(self.path_to_config, self.args.vardir) + + subprocess.check_call([self.abspath_to_exe, "--init_storage"], + cwd = self.args.vardir, +# catch stdout/stderr to not clutter output + stdout = subprocess.PIPE, + stderr = subprocess.PIPE) + + if self.args.start_and_exit: + subprocess.check_call([self.abspath_to_exe, "--daemonize"], + cwd = self.args.vardir, + stdout = subprocess.PIPE, + stderr = subprocess.PIPE) + else: + self.server = pexpect.spawn(self.abspath_to_exe, + cwd = self.args.vardir) + self.logfile_read = sys.stdout + self.server.expect_exact("entering event loop") + + version = subprocess.Popen([self.abspath_to_exe, "--version"], + cwd = self.args.vardir, + stdout = subprocess.PIPE).stdout.read().rstrip() + + print "Started {0} {1}.".format(os.path.basename(self.abspath_to_exe), + version) + +# Set is_started flag, to nicely support cleanup during an exception. + self.is_started = True + else: + print "The server is already started." + + def stop(self): + """Stop server instance. Do nothing if the server is not started, + to properly shut down the server in case of an exception during + start up.""" + if self.is_started: + print "Stopping the server..." + self.server.terminate() + self.server.expect(pexpect.EOF) + self.is_started = False + else: + print "The server is not started." + + def find_exe(self): + """Locate server executable in the bindir. We just take + the first thing looking like an exe in there.""" + + if (os.access(self.args.bindir, os.F_OK) == False or + stat.S_ISDIR(os.stat(self.args.bindir).st_mode) == False): + raise TestRunException("Directory " + self.args.bindir + + " doesn't exist") + + for f in os.listdir(self.args.bindir): + f = os.path.join(self.args.bindir, f) + st_mode = os.stat(f).st_mode + if stat.S_ISREG(st_mode) and st_mode & stat.S_IXUSR: + return f + raise TestRunException("Can't find server executable in " + + self.args.bindir) + + def kill_old_server(self): + """Kill old server instance if it exists.""" + if os.access(self.path_to_pidfile, os.F_OK) == False: + return # Nothing to do + pid = 0 + with open(self.path_to_pidfile) as f: + pid = int(f.read()) + print " Found old server, pid {0}, killing...".format(pid) + try: + os.kill(pid, signal.SIGTERM) + while os.kill(pid, 0) != -1: + time.sleep(0.01) + except OSError: + pass + diff --git a/test/lib/test_suite.py b/test/lib/test_suite.py new file mode 100644 index 000000000000..ce39841f094b --- /dev/null +++ b/test/lib/test_suite.py @@ -0,0 +1,208 @@ +import os +import os.path +import sys +import stat +import glob +import ConfigParser +import subprocess +import collections +import difflib +import filecmp +import shlex +from tarantool_silverbox_server import TarantoolSilverboxServer + +class TestRunException(RuntimeError): + """A common exception to use across the program.""" + def __init__(self, message): + self.message = message + def __str__(self): + return self.message + +class Test: + """An individual test file. A test can run itself, and remembers + its completion state.""" + def __init__(self, name, suite_ini): + """Initialize test properties: path to test file, path to + temporary result file, path to the client program, test status.""" + self.name = name + self.result = name.replace(".test", ".result") + self.suite_ini = suite_ini + self.is_executed = False + self.is_client_ok = None + self.is_equal_result = None + + def passed(self): + """Return true if this test was run successfully.""" + return self.is_executed and self.is_client_ok and self.is_equal_result + + def run(self): + """Execute the client program, giving it test as stdin, + result as stdout. If the client program aborts, print + its output to stdout, and raise an exception. Else, comprare + result and reject files. If there is a difference, print it to + stdout and raise an exception. The exception is raised only + if is_force flag is not set.""" + + def subst_test_env(arg): + if len(arg) and arg[0] == '$': + return self.suite_ini[arg[1:]] + else: + return arg + + client = os.path.join(".", self.suite_ini["client"]) + args = map(subst_test_env, shlex.split(client)) + + sys.stdout.write(self.name) +# for better diagnostics in case of a long-running test + sys.stdout.flush() + + with open(self.name, "r") as test: + with open(self.result, "w+") as result: + self.is_client_ok = \ + subprocess.call(args, + stdin = test, stdout = result) == 0 + + self.is_executed = True + + if self.is_client_ok: + self.is_equal_result = filecmp.cmp(self.name, self.result) + + if self.is_client_ok and self.is_equal_result: + print "\t\t\t[ pass ]" + os.remove(self.result) + else: + print "\t\t\t[ fail ]" + where = "" + if not self.is_client_ok: + self.print_diagnostics() + where = ": client execution aborted" + else: + self.print_unidiff() + where = ": wrong test output" + if not self.suite_ini["is_force"]: + raise TestRunException("Failed to run test " + self.name + where) + + + def print_diagnostics(self): + """Print 10 lines of client program output leading to test + failure. Used to diagnose a failure of the client program""" + + print "Test failed! Last 10 lines of the result file:" + with open(self.result, "r+") as result: + tail_10 = collections.deque(result, 10) + for line in tail_10: + sys.stdout.write(line) + + def print_unidiff(self): + """Print a unified diff between .test and .result files. Used + to establish the cause of a failure when .test differs + from .result.""" + + print "Test failed! Result content mismatch:" + with open(self.name, "r") as test: + with open(self.result, "r") as result: + test_time = time.ctime(os.stat(self.name).st_mtime) + result_time = time.ctime(os.stat(self.result).st_mtime) + diff = difflib.unified_diff(test.readlines(), + result.readlines(), + self.name, + self.result, + test_time, + result_time) + for line in diff: + sys.stdout.write(line) + +class TarantoolConfigFile: + """ConfigParser can't read files without sections, work it around""" + def __init__(self, fp, section_name): + self.fp = fp + self.section_name = "[" + section_name + "]" + def readline(self): + if self.section_name: + section_name = self.section_name + self.section_name = None + return section_name + # tarantool.cfg puts string values in quotes + return self.fp.readline().replace("\"", '') + + +class TestSuite: + """Each test suite contains a number of related tests files, + located in the same directory on disk. Each test file has + extention .test and contains a listing of server commands, + followed by their output. The commands are executed, and + obtained results are compared with pre-recorded output. In case + of a comparision difference, an exception is raised. A test suite + must also contain suite.ini, which describes how to start the + server for this suite, the client program to execute individual + tests and other suite properties. The server is started once per + suite.""" + + def __init__(self, suite_path, args): + """Initialize a test suite: check that it exists and contains + a syntactically correct configuration file. Then create + a test instance for each found test.""" + self.path = suite_path + self.args = args + self.tests = [] + + if os.access(self.path, os.F_OK) == False: + raise TestRunException("Suite \"" + self.path + "\" doesn't exist") + +# read the suite config + config = ConfigParser.ConfigParser() + config.read(os.path.join(self.path, "suite.ini")) + self.ini = dict(config.items("default")) +# import the necessary module for test suite client + + self.ini["host"] = "localhost" + self.ini["is_force"] = self.args.is_force + +# now read the server config, we need some properties from it + + with open(os.path.join(self.path, self.ini["config"])) as fp: + dummy_section_name = "tarantool_silverbox" + config.readfp(TarantoolConfigFile(fp, dummy_section_name)) + self.ini["pidfile"] = config.get(dummy_section_name, "pid_file") + self.ini["port"] = config.get(dummy_section_name, "admin_port") + + print "Collecting tests in \"" + self.path + "\": " +\ + self.ini["description"] + "." + + for test_name in glob.glob(os.path.join(self.path, "*.test")): + for test_pattern in self.args.tests: + if test_name.find(test_pattern) != -1: + self.tests.append(Test(test_name, self.ini)) + print "Found " + str(len(self.tests)) + " tests." + + def run_all(self): + """For each file in the test suite, run client program + assuming each file represents an individual test.""" + server = TarantoolSilverboxServer(self.args, + os.path.join(self.path, + self.ini["config"]), + self.ini["pidfile"]) + server.start() + if self.args.start_and_exit: + print " Start and exit requested, exiting..." + exit(0) + + longsep = "==============================================================================" + shortsep = "------------------------------------------------------------" + print longsep + print "TEST\t\t\t\tRESULT" + print shortsep + failed_tests = [] + self.ini["server"] = server.abspath_to_exe + + for test in self.tests: + test.run() + if not test.passed(): + failed_tests.append(test.name) + + print shortsep + if len(failed_tests): + print "Failed {0} tests: {1}.".format(len(failed_tests), + ", ".join(failed_tests)) + server.stop(); + diff --git a/test/test-run.py b/test/test-run.py index bfa77eea7e58..e5fbb6b52570 100755 --- a/test/test-run.py +++ b/test/test-run.py @@ -25,21 +25,9 @@ # SUCH DAMAGE. import argparse -import os import os.path -import signal import sys -import stat -import glob -import shutil -import ConfigParser -import subprocess -import pexpect -import time -import collections -import difflib -import filecmp -import shlex +from lib.test_suite import TestSuite, TestRunException # # Run a collection of tests. @@ -48,17 +36,6 @@ # --gdb # put class definitions into separate files -############################################################################ -# Class definition -############################################################################ - -class TestRunException(RuntimeError): - """A common exception to use across the program.""" - def __init__(self, message): - self.message = message - def __str__(self): - return self.message - class Options: """Handle options of test-runner""" def __init__(self): @@ -91,7 +68,8 @@ def __init__(self): metavar = "suite", nargs="*", default = ["box", "cmd"], - help = """List of tests suites to look for tests in. Default: "box".""") + help = """List of tests suites to look for tests in. Default: "box" + and "cmd".""") parser.add_argument( "--force", @@ -138,278 +116,6 @@ def check(self, parser): exit(-1) -class Server: - """Server represents a single server instance. Normally, the - program operates with only one server, but in future we may add - replication slaves. The server is started once at the beginning - of each suite, and stopped at the end.""" - - def __init__(self, args, config, pidfile): - """Set server options: path to configuration file, pid file, exe, etc.""" - self.args = args - self.path_to_config = config - self.path_to_pidfile = os.path.join(args.vardir, pidfile) - self.path_to_exe = None - self.abspath_to_exe = None - self.is_started = False - - def start(self): - """Start server instance: check if the old one exists, kill it - if necessary, create necessary directories and files, start - the server. The server working directory is taken from 'vardir', - specified in the prgoram options. - Currently this is implemented for tarantool_silverbox only.""" - - if not self.is_started: - print "Starting the server..." - - if self.path_to_exe == None: - self.path_to_exe = self.find_exe() - self.abspath_to_exe = os.path.abspath(self.path_to_exe) - - print " Found executable at " + self.path_to_exe + "." - - print " Creating and populating working directory in " +\ - self.args.vardir + "..." - - if os.access(self.args.vardir, os.F_OK): - print " Found old vardir, deleting..." - self.kill_old_server() - shutil.rmtree(self.args.vardir, ignore_errors = True) - - os.mkdir(self.args.vardir) - shutil.copy(self.path_to_config, self.args.vardir) - - subprocess.check_call([self.abspath_to_exe, "--init_storage"], - cwd = self.args.vardir, -# catch stdout/stderr to not clutter output - stdout = subprocess.PIPE, - stderr = subprocess.PIPE) - - if self.args.start_and_exit: - subprocess.check_call([self.abspath_to_exe, "--daemonize"], - cwd = self.args.vardir, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE) - else: - self.server = pexpect.spawn(self.abspath_to_exe, - cwd = self.args.vardir) - self.logfile_read = sys.stdout - self.server.expect_exact("entering event loop") - - version = subprocess.Popen([self.abspath_to_exe, "--version"], - cwd = self.args.vardir, - stdout = subprocess.PIPE).stdout.read().rstrip() - - print "Started {0} {1}.".format(os.path.basename(self.abspath_to_exe), - version) - -# Set is_started flag, to nicely support cleanup during an exception. - self.is_started = True - else: - print "The server is already started." - - def stop(self): - """Stop server instance. Do nothing if the server is not started, - to properly shut down the server in case of an exception during - start up.""" - if self.is_started: - print "Stopping the server..." - self.server.terminate() - self.server.expect(pexpect.EOF) - self.is_started = False - else: - print "The server is not started." - - def find_exe(self): - """Locate server executable in the bindir. We just take - the first thing looking like an exe in there.""" - - if (os.access(self.args.bindir, os.F_OK) == False or - stat.S_ISDIR(os.stat(self.args.bindir).st_mode) == False): - raise TestRunException("Directory " + self.args.bindir + - " doesn't exist") - - for f in os.listdir(self.args.bindir): - f = os.path.join(self.args.bindir, f) - st_mode = os.stat(f).st_mode - if stat.S_ISREG(st_mode) and st_mode & stat.S_IXUSR: - return f - raise TestRunException("Can't find server executable in " + - self.args.bindir) - - def kill_old_server(self): - """Kill old server instance if it exists.""" - if os.access(self.path_to_pidfile, os.F_OK) == False: - return # Nothing to do - pid = 0 - with open(self.path_to_pidfile) as f: - pid = int(f.read()) - print " Found old server, pid {0}, killing...".format(pid) - try: - os.kill(pid, signal.SIGTERM) - while os.kill(pid, 0) != -1: - time.sleep(0.01) - except OSError: - pass - -class Test: - """An individual test file. A test can run itself, and remembers - its completion state.""" - def __init__(self, name, client): - """Initialize test properties: path to test file, path to - temporary result file, path to the client program, test status.""" - self.name = name - self.client = os.path.join(".", client) - self.result = name.replace(".test", ".result") - self.is_executed = False - self.is_client_ok = None - self.is_equal_result = None - - def passed(self): - """Return true if this test was run successfully.""" - return self.is_executed and self.is_client_ok and self.is_equal_result - - def run(self, test_env): - """Execute the client program, giving it test as stdin, - result as stdout. If the client program aborts, print - its output to stdout, and raise an exception. Else, comprare - result and reject files. If there is a difference, print it to - stdout and raise an exception. The exception is raised only - if is_force flag is not set.""" - - def subst_test_env(arg): - if len(arg) and arg[0] == '$': - return test_env[arg[1:]] - else: - return arg - - args = map(subst_test_env, shlex.split(self.client)) - - sys.stdout.write("{0}".format(self.name)) - - with open(self.name, "r") as test: - with open(self.result, "w+") as result: - self.is_client_ok = \ - subprocess.call(args, - stdin = test, stdout = result) == 0 - - self.is_executed = True - - if self.is_client_ok: - self.is_equal_result = filecmp.cmp(self.name, self.result) - - if self.is_client_ok and self.is_equal_result: - print "\t\t\t[ pass ]" - os.remove(self.result) - else: - print "\t\t\t[ fail ]" - where = "" - if not self.is_client_ok: - self.print_diagnostics() - where = ": client execution aborted" - else: - self.print_unidiff() - where = ": wrong test output" - if not test_env["is_force"]: - raise TestRunException("Failed to run test " + self.name + where) - - - def print_diagnostics(self): - """Print 10 lines of client program output leading to test - failure. Used to diagnose a failure of the client program""" - - print "Test failed! Last 10 lines of the result file:" - with open(self.result, "r+") as result: - tail_10 = collections.deque(result, 10) - for line in tail_10: - sys.stdout.write(line) - - def print_unidiff(self): - """Print a unified diff between .test and .result files. Used - to establish the cause of a failure when .test differs - from .result.""" - - print "Test failed! Result content mismatch:" - with open(self.name, "r") as test: - with open(self.result, "r") as result: - test_time = time.ctime(os.stat(self.name).st_mtime) - result_time = time.ctime(os.stat(self.result).st_mtime) - diff = difflib.unified_diff(test.readlines(), - result.readlines(), - self.name, - self.result, - test_time, - result_time) - for line in diff: - sys.stdout.write(line) - - -class TestSuite: - """Each test suite contains a number of related tests files, - located in the same directory on disk. Each test file has - extention .test and contains a listing of server commands, - followed by their output. The commands are executed, and - obtained results are compared with pre-recorded output. In case - of a comparision difference, an exception is raised. A test suite - must also contain suite.ini, which describes how to start the - server for this suite, the client program to execute individual - tests and other suite properties. The server is started once per - suite.""" - - def __init__(self, suite_path, args): - """Initialize a test suite: check that it exists and contains - a syntactically correct configuration file. Then create - a test instance for each found test.""" - self.path = suite_path - self.args = args - self.tests = [] - - if os.access(self.path, os.F_OK) == False: - raise TestRunException("Suite \"" + self.path + "\" doesn't exist") - - config = ConfigParser.ConfigParser() - config.read(os.path.join(self.path, "suite.ini")) - self.ini = dict(config.items("default")) - print "Collecting tests in \"" + self.path + "\": " +\ - self.ini["description"] + "." - - for test_name in glob.glob(os.path.join(self.path, "*.test")): - for test_pattern in self.args.tests: - if test_name.find(test_pattern) != -1: - self.tests.append(Test(test_name, self.ini["client"])) - print "Found " + str(len(self.tests)) + " tests." - - def run_all(self): - """For each file in the test suite, run client program - assuming each file represents an individual test.""" - server = Server(self.args, os.path.join(self.path, self.ini["config"]), - self.ini["pidfile"]) - server.start() - if self.args.start_and_exit: - print " Start and exit requested, exiting..." - exit(0) - - longsep = "==============================================================================" - shortsep = "------------------------------------------------------------" - print longsep - print "TEST\t\t\t\tRESULT" - print shortsep - failed_tests = [] - test_env = { "is_force" : self.args.is_force, - "server" : server.abspath_to_exe } - - for test in self.tests: - test.run(test_env) - if not test.passed(): - failed_tests.append(test.name) - - print shortsep - if len(failed_tests): - print "Failed {0} tests: {1}.".format(len(failed_tests), - ", ".join(failed_tests)) - server.stop(); - ####################################################################### # Program body #######################################################################