Merge pull request #180 from FND/runner-files

gabbi-run support for multiple input files (take two)

docs/source/runner.rst

@@ -5,8 +5,9 @@ If there is a running web service that needs to be tested and
 creating a test loader with :meth:`~gabbi.driver.build_tests` is
 either inconvenient or overkill it is possible to run YAML test
 files directly from the command line with the console-script
-``gabbi-run``. It accepts YAML on ``stdin``, generates and runs
-tests and outputs a summary of the results.
+``gabbi-run``. It accepts YAML on ``stdin`` or as multiple file
+arguments, and generates and runs tests and outputs a summary of
+the results.
 
 The provided YAML may not use custom :doc:`fixtures` but otherwise
 uses the default :doc:`format`. :doc:`host` information is either
@@ -19,6 +20,15 @@ or::
 
     gabbi-run http://host:port < /my/test.yaml
 
+To test with one or more files the following command syntax may be
+used::
+
+    gabbi-run http://host:port -- /my/test.yaml /my/other.yaml
+
+.. note:: The filename arguments must come after a ``--`` and all
+          other arguments (host, port, prefix, failfast) must come
+          before the ``--``.
+
 To facilitate using the same tests against the same application mounted
 in different locations in a WSGI server, a ``prefix`` may be provided
 as a second argument::

gabbi/runner.py

@@ -10,7 +10,7 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
-"""Implementation of a command-line runner of single gabbi files."""
+"""Implementation of a command-line runner for gabbi files (AKA suites)."""
 
 import argparse
 from importlib import import_module
@@ -53,62 +53,65 @@ def run():
 
         gabbi-run http://example.com:9999/mountpoint < mytest.yaml
 
-    Use `-x` or `--failfast` to abort after the first error or failure:
+    Use `-x` or `--failfast` to abort after the first error or failure::
 
         gabbi-run -x example.com:9999 /mountpoint < mytest.yaml
 
+    Multiple files may be named as arguments, separated from other arguments
+    by a ``--``. Each file will be run as a separate test suite::
+
+        gabbi-run http://example.com -- /path/to/x.yaml /path/to/y.yaml
+
     Output is formatted as unittest summary information.
     """
+    parser = _make_argparser()
 
-    parser = argparse.ArgumentParser(description='Run gabbi tests from STDIN')
-    parser.add_argument(
-        'target',
-        nargs='?', default='stub',
-        help='A fully qualified URL (with optional path as prefix) '
-             'to the primary target or a host and port, : separated. '
-             'If using an IPV6 address for the host in either form, '
-             'wrap it in \'[\' and \']\'.'
-    )
-    parser.add_argument(
-        'prefix',
-        nargs='?', default=None,
-        help='Path prefix where target app is mounted. Only used when '
-             'target is of the form host[:port]'
-    )
-    parser.add_argument(
-        '-x', '--failfast',
-        action='store_true',
-        help='Exit on first failure'
-    )
-    parser.add_argument(
-        '-r', '--response-handler',
-        nargs='?', default=None,
-        dest='response_handlers',
-        action='append',
-        help='Custom response handler. Should be an import path of the '
-             'form package.module or package.module:class.'
-    )
+    argv, input_files = extract_file_paths(sys.argv)
+    args = parser.parse_args(argv[1:])
 
-    args = parser.parse_args()
     host, port, prefix, force_ssl = utils.host_info_from_target(
         args.target, args.prefix)
 
-    # Initialize response handlers.
     handler_objects = initialize_handlers(args.response_handlers)
 
-    data = utils.load_yaml(handle=sys.stdin)
+    failfast = args.failfast
+    failure = False
+
+    if not input_files:
+        success = run_suite(sys.stdin, handler_objects, host, port,
+                            prefix, force_ssl, failfast)
+        failure = not success
+    else:
+        for input_file in input_files:
+            with open(input_file, 'r') as fh:
+                success = run_suite(fh, handler_objects, host, port,
+                                    prefix, force_ssl, failfast)
+            if not failure:  # once failed, this is considered immutable
+                failure = not success
+            if failure and failfast:
+                break
+
+    sys.exit(failure)
+
+
+def run_suite(handle, handler_objects, host, port, prefix, force_ssl=False,
+              failfast=False):
+    """Run the tests from the YAML in handle."""
+    data = utils.load_yaml(handle)
     if force_ssl:
         if 'defaults' in data:
             data['defaults']['ssl'] = True
         else:
             data['defaults'] = {'ssl': True}
+
     loader = unittest.defaultTestLoader
     test_suite = suitemaker.test_suite_from_dict(
         loader, 'input', data, '.', host, port, None, None, prefix=prefix,
         handlers=handler_objects)
+
     result = ConciseTestRunner(
-        verbosity=2, failfast=args.failfast).run(test_suite)
-    sys.exit(not result.wasSuccessful())
+        verbosity=2, failfast=failfast).run(test_suite)
+    return result.wasSuccessful()
 
 
 def initialize_handlers(response_handlers):
@@ -147,5 +150,52 @@ def load_response_handlers(import_path):
     return custom_handlers
 
 
+def extract_file_paths(argv):
+    """Extract command-line arguments following the `--` end-of-options
+    delimiter, if any.
+    """
+    try:  # extract file paths, separated by `--`
+        i = argv.index("--")
+        input_files = argv[i + 1:]
+        argv = argv[:i]
+    except ValueError:
+        input_files = None
+
+    return argv, input_files
+
+
+def _make_argparser():
+    """Set up the argparse.ArgumentParser."""
+    parser = argparse.ArgumentParser(description='Run gabbi tests from STDIN')
+    parser.add_argument(
+        'target',
+        nargs='?', default='stub',
+        help='A fully qualified URL (with optional path as prefix) '
+             'to the primary target or a host and port, : separated. '
+             'If using an IPV6 address for the host in either form, '
+             'wrap it in \'[\' and \']\'.'
+    )
+    parser.add_argument(
+        'prefix',
+        nargs='?', default=None,
+        help='Path prefix where target app is mounted. Only used when '
+             'target is of the form host[:port]'
+    )
+    parser.add_argument(
+        '-x', '--failfast',
+        action='store_true',
+        help='Exit on first failure'
+    )
+    parser.add_argument(
+        '-r', '--response-handler',
+        nargs='?', default=None,
+        dest='response_handlers',
+        action='append',
+        help='Custom response handler. Should be an import path of the '
+             'form package.module or package.module:class.'
+    )
+    return parser
+
+
 if __name__ == '__main__':
     run()

gabbi/tests/gabbits_runner/failure.yaml

@@ -0,0 +1,6 @@
+tests:
+
+- name: expected failure
+  GET: /
+
+  status: 666

gabbi/tests/gabbits_runner/success.yaml

@@ -0,0 +1,6 @@
+tests:
+
+- name: expected success
+  GET: /baz
+
+  status: 200

gabbi/tests/gabbits_runner/success_alt.yaml

@@ -0,0 +1,6 @@
+tests:
+
+- name: expected success
+  GET: /baz
+
+  status: 200

gabbi/tests/test_runner.py

@@ -56,6 +56,34 @@ def tearDown(self):
         sys.stderr = self._stderr
         sys.argv = self._argv
 
+    def test_input_files(self):
+        sys.argv = ['gabbi-run', 'http://%s:%s/foo' % (self.host, self.port)]
+
+        sys.argv.append('--')
+        sys.argv.append('gabbi/tests/gabbits_runner/success.yaml')
+
+        with self.server():
+            try:
+                runner.run()
+            except SystemExit as err:
+                self.assertSuccess(err)
+
+        sys.argv.append('gabbi/tests/gabbits_runner/failure.yaml')
+
+        with self.server():
+            try:
+                runner.run()
+            except SystemExit as err:
+                self.assertFailure(err)
+
+        sys.argv.append('gabbi/tests/gabbits_runner/success_alt.yaml')
+
+        with self.server():
+            try:
+                runner.run()
+            except SystemExit as err:
+                self.assertFailure(err)
+
     def test_target_url_parsing(self):
         sys.argv = ['gabbi-run', 'http://%s:%s/foo' % (self.host, self.port)]