Add CLI tool for running schemathesis test cases

- Fix: #30

README.rst

@@ -130,6 +130,21 @@ To narrow down the scope of the schemathesis tests it is possible to filter by m
 
 The acceptable values are regexps or list of regexps (matched with ``re.search``).
 
+CLI
+~~~
+
+The ``schemathesis`` command can be used to perform Schemathesis test cases::
+
+.. code:: bash
+
+    schemathesis run https://example.com/api/swagger.json
+
+For the full list of options, run::
+
+.. code:: bash
+
+    schemathesis --help
+
 Explicit examples
 ~~~~~~~~~~~~~~~~~
 

docs/changelog.rst

@@ -6,6 +6,11 @@ Changelog
 `Unreleased`_
 -------------
 
+Added
+~~~~~
+
+- CLI tool invoked by the ``schemathesis`` command
+
 `0.7.3`_ - 2019-09-30
 ---------------------
 

pyproject.toml

@@ -33,15 +33,20 @@ pytest = ">4.6.4"
 pyyaml = "^5.1"
 pytest-subtests = "^0.2.1"
 requests = "^2.22"
+click = "^7.0"
 
 [tool.poetry.dev-dependencies]
 coverage = "^4.5"
 pytest = ">4.6.4"
 aiohttp = "^3.6"
+pytest-mock = "^1.11.0"
 
 [tool.poetry.plugins]
 pytest11 = {schemathesis = "schemathesis.extra.pytest_plugin"}
 
+[tool.poetry.scripts]
+schemathesis = "schemathesis.commands:main"
+
 [tool.black]
 line-length = 120
 target_version = ["py37"]

src/schemathesis/commands.py

@@ -0,0 +1,43 @@
+from typing import Iterable
+from urllib.parse import urlparse
+
+import click
+
+from . import runner
+
+CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
+
+DEFAULT_CHECKS_NAMES = tuple(check.__name__ for check in runner.DEFAULT_CHECKS)
+
+
+@click.group(context_settings=CONTEXT_SETTINGS)
+@click.version_option()
+def main() -> None:
+    """Command line tool for testing your web application built with Open API / Swagger specifications."""
+
+
+@main.command(short_help="Perform schemathesis test.")
+@click.argument("schema", type=str)
+@click.option(
+    "--checks",
+    "-c",
+    multiple=True,
+    help="List of checks to run.",
+    type=click.Choice(DEFAULT_CHECKS_NAMES),
+    default=DEFAULT_CHECKS_NAMES,
+)
+def run(schema: str, checks: Iterable[str] = DEFAULT_CHECKS_NAMES) -> None:
+    """Perform schemathesis test against an API specified by SCHEMA.
+
+    SCHEMA must be a valid URL pointing to an Open API / Swagger specification.
+    """
+    if not urlparse(schema).netloc:
+        raise click.UsageError("Invalid SCHEMA, must be a valid URL.")
+
+    selected_checks = tuple(check for check in runner.DEFAULT_CHECKS if check.__name__ in checks)
+
+    click.echo("Running schemathesis test cases ...")
+
+    runner.execute(schema, checks=selected_checks)
+
+    click.echo("Done.")

src/schemathesis/runner.py

@@ -5,22 +5,30 @@
 
 from .loaders import from_uri
 from .models import Case
+from .schemas import BaseSchema
 
 
 def not_a_server_error(response: requests.Response) -> None:
     """A check to verify that the response is not a server-side error."""
     assert response.status_code < 500
 
 
-def execute(schema_uri: str, checks: Iterable[Callable] = (not_a_server_error,)) -> None:
-    """Generate and run test cases against the given API definition."""
+DEFAULT_CHECKS = (not_a_server_error,)
+
+
+def _execute_all_tests(schema: BaseSchema, base_url: str, checks: Iterable[Callable]) -> None:
     with requests.Session() as session:
-        schema = from_uri(schema_uri)
-        base_url = get_base_url(schema_uri)
         for _, test in schema.get_all_tests(single_test):
             test(session, base_url, checks)
 
 
+def execute(schema_uri: str, base_url: str = "", checks: Iterable[Callable] = DEFAULT_CHECKS) -> None:
+    """Generate and run test cases against the given API definition."""
+    schema = from_uri(schema_uri)
+    base_url = base_url or get_base_url(schema_uri)
+    _execute_all_tests(schema, base_url, checks)
+
+
 def get_base_url(uri: str) -> str:
     """Remove the path part off the given uri."""
     parts = urlsplit(uri)[:2] + ("", "", "")

test/conftest.py

@@ -6,7 +6,7 @@
 
 from .utils import make_schema
 
-pytest_plugins = ["pytester", "aiohttp.pytest_plugin"]
+pytest_plugins = ["pytester", "aiohttp.pytest_plugin", "pytest_mock"]
 
 
 @pytest.fixture
@@ -46,6 +46,11 @@ def maker(**kwargs):
     return maker
 
 
+@pytest.fixture()
+def testcmd(testdir):
+    return testdir.run
+
+
 @pytest.fixture()
 def testdir(testdir):
     def maker(content, method=None, endpoint=None, **kwargs):

test/test_commands.py

@@ -0,0 +1,73 @@
+import tempfile
+from functools import partial
+
+import pytest
+from click.testing import CliRunner
+
+from schemathesis import commands, runner
+
+
+@pytest.fixture()
+def schemathesis_cmd(testcmd):
+    return partial(testcmd, "schemathesis")
+
+
+def test_commands_help(schemathesis_cmd):
+    result = schemathesis_cmd()
+
+    assert result.ret == 0
+    assert result.stdout.get_lines_after("Commands:") == ["  run  Perform schemathesis test."]
+
+    result_help = schemathesis_cmd("--help")
+    result_h = schemathesis_cmd("-h")
+
+    assert result.stdout.lines == result_h.stdout.lines == result_help.stdout.lines
+
+
+def test_commands_version(schemathesis_cmd):
+    result = schemathesis_cmd("--version")
+
+    assert result.ret == 0
+    assert "version" in result.stdout.lines[0]
+
+
+def test_commands_run_errors(schemathesis_cmd):
+    result_no_args = schemathesis_cmd("run")
+
+    assert result_no_args.ret == 2
+    assert "Missing argument" in result_no_args.stderr.lines[-1]
+
+    result_invalid = schemathesis_cmd("run", "not-url")
+
+    assert result_invalid.ret == 2
+    assert "Invalid SCHEMA" in result_invalid.stderr.lines[-1]
+
+
+def test_commands_run_help(schemathesis_cmd):
+    result_help = schemathesis_cmd("run", "--help")
+
+    assert result_help.ret == 0
+    assert result_help.stdout.lines == [
+        "Usage: schemathesis run [OPTIONS] SCHEMA",
+        "",
+        "  Perform schemathesis test against an API specified by SCHEMA.",
+        "",
+        "  SCHEMA must be a valid URL pointing to an Open API / Swagger",
+        "  specification.",
+        "",
+        "Options:",
+        "  -c, --checks [not_a_server_error]",
+        "                                  List of checks to run.",
+        "  -h, --help                      Show this message and exit.",
+    ]
+
+
+def test_commands_run_schema_uri(mocker):
+    m_execute = mocker.patch("schemathesis.runner.execute")
+    cli = CliRunner()
+
+    schema_uri = "https://example.com/swagger.json"
+    result_schema_uri = cli.invoke(commands.run, [schema_uri])
+
+    assert result_schema_uri.exit_code == 0
+    m_execute.assert_called_once_with(schema_uri, checks=runner.DEFAULT_CHECKS)

tox.ini

@@ -6,6 +6,7 @@ envlist = pylint,mypy,py{36,37,38},coverage-report
 deps =
   aiohttp
   coverage
+  pytest-mock
 commands =
   coverage run --source=schemathesis -m pytest {posargs:} --junitxml=reports/xunit-tests.xml test