Adds CLI and Python invocations for the device test suite (#733)

* Adds CLI and Python invocations for the device test suite

* linting

* docs

* docs

* fixes

* Update pennylane/plugins/tests/pytest.ini

Co-authored-by: Maria Schuld <mariaschuld@gmail.com>

* suggested changes

* blacking

* update error codes

* update

* update

* add support for optional device keywords

* linting

* include pytest.ini

* suggested changes

* add comment

Co-authored-by: Maria Schuld <mariaschuld@gmail.com>
Co-authored-by: antalszava <antalszava@gmail.com>

pennylane/plugins/__init__.py

@@ -27,6 +27,7 @@
     default_gaussian
     tf_ops
     autograd_ops
+    tests
 """
 from .default_qubit import DefaultQubit
 from .default_gaussian import DefaultGaussian

pennylane/plugins/tests/__init__.py

@@ -15,27 +15,12 @@
 This subpackage provides integration tests for the devices with PennyLane's core
 functionalities. At the moment, the tests only run on devices based on the 'qubit' model.
 
-To run the tests against a particular device (i.e., for 'default.qubit'):
+The tests require that ``pytest``, ``pytest-mock``, and ``flaky`` be installed.
+These can be installed using ``pip``:
 
 .. code-block:: console
 
-    python3 -m pytest path_to_pennylane/plugins/tests --device default.qubit --shots 1234 --analytic False
-
-The location of your PennyLane installation may differ depending on installation method and operating
-system. To find the location, you can execute the following Python code:
-
->>> import os
->>> import pennylane as qml
->>> print(os.path.dirname(qml.__file__))
-
-The command line arguments are optional:
-
-* If `--device` is not given, the tests are run on the qubit core devices that ship with PennyLane.
-
-* If `--shots` is not given, a default of 10000 is used. The shots argument is ignored for devices running in
-  analytic mode.
-
-* If `--analytic` is not given, the device's default is used.
+    pip install pytest pytest-mock flaky
 
 The tests can also be run on an external device from a PennyLane plugin, such as
 ``'qiskit.aer'``. For this, make sure you have the correct dependencies installed.
@@ -47,4 +32,209 @@
 For non-analytic tests, the tolerance of the assert statements
 is set to a high enough value to account for stochastic fluctuations. Flaky is used to automatically
 repeat failed tests.
+
+There are several methods for running the tests against a particular device (i.e., for
+``'default.qubit'``), detailed below.
+
+Using pytest
+------------
+
+.. code-block:: console
+
+    pytest path_to_pennylane_src/plugins/tests --device=default.qubit --shots=10000 --analytic=False
+
+The location of your PennyLane installation may differ depending on installation method and
+operating system. To find the location, you can use the :func:`~.get_device_tests` function:
+
+>>> from pennylane.plugins.tests import get_device_tests
+>>> get_device_tests()
+
+The pl-device-test CLI
+----------------------
+
+Alternatively, PennyLane provides a command line interface for invoking the device tests.
+
+.. code-block:: console
+
+    pl-device-test --device default.qubit --shots 10000 --analytic False
+
+Within Python
+-------------
+
+Finally, the tests can be invoked within a Python session via the :func:`~.test_device`
+function:
+
+>>> from pennylane.plugins.tests import test_device
+>>> test_device("default.qubit")
+
+For more details on the available arguments, see the :func:`~.test_device` documentation.
+
+Functions
+---------
 """
+# pylint: disable=import-outside-toplevel,too-many-arguments
+import argparse
+import pathlib
+import subprocess
+import sys
+
+
+# determine if running in an interactive environment
+import __main__
+
+interactive = False
+
+try:
+    __main__.__file__
+except AttributeError:
+    interactive = True
+
+
+def get_device_tests():
+    """Returns the location of the device integration tests."""
+    return str(pathlib.Path(__file__).parent.absolute())
+
+
+def test_device(
+    device, analytic=None, shots=None, skip_ops=True, flaky_report=False, pytest_args=None, **kwargs
+):
+    """Run the device integration tests using an installed PennyLane device.
+
+    Args:
+        device (str): the name of the device to test
+        analytic (bool): Whether to run the device in analytic mode (where
+            expectation values and probabilities are computed exactly from the quantum state)
+            or non-analytic/"stochastic" mode (where probabilities and expectation
+            values are *estimated* using a finite number of shots.)
+            If not provided, the device default is used.
+        shots (int): The number of shots/samples used to estimate expectation
+            values and probability. Only takes affect if ``analytic=False``. If not
+            provided, the device default is used.
+        skip_ops (bool): whether to skip tests that use operations not supported
+            by the device
+        pytest_args (list[str]): additional PyTest arguments and flags
+        **kwargs: Additional device keyword args
+
+    **Example**
+
+    >>> from pennylane.plugins.tests import test_device
+    >>> test_device("default.qubit")
+    ================================ test session starts =======================================
+    platform linux -- Python 3.7.7, pytest-5.4.2, py-1.8.1, pluggy-0.13.1
+    rootdir: /home/josh/xanadu/pennylane/pennylane/plugins/tests, inifile: pytest.ini
+    plugins: flaky-3.6.1, cov-2.8.1, mock-3.1.0
+    collected 86 items
+    xanadu/pennylane/pennylane/plugins/tests/test_gates.py ..............................
+    ...............................                                                       [ 70%]
+    xanadu/pennylane/pennylane/plugins/tests/test_measurements.py .......sss...sss..sss   [ 95%]
+    xanadu/pennylane/pennylane/plugins/tests/test_properties.py ....                      [100%]
+    ================================= 77 passed, 9 skipped in 0.78s ============================
+
+    """
+    try:
+        import pytest  # pylint: disable=unused-import
+        import pytest_mock  # pylint: disable=unused-import
+        import flaky  # pylint: disable=unused-import
+    except ImportError:
+        raise ImportError(
+            "The device tests requires the following Python packages:"
+            "\npytest pytest_mock flaky"
+            "\nThese can be installed using pip."
+        )
+
+    pytest_args = pytest_args or []
+    test_dir = get_device_tests()
+
+    cmds = ["pytest"]
+    cmds.append(test_dir)
+    cmds.append(f"--device={device}")
+
+    if shots is not None:
+        cmds.append(f"--shots={shots}")
+
+    if analytic is not None:
+        cmds.append(f"--analytic={analytic}")
+
+    if skip_ops:
+        cmds.append("--skip-ops")
+
+    if not flaky_report:
+        cmds.append("--no-flaky-report")
+
+    if kwargs:
+        device_kwargs = " ".join([f"{k}={v}" for k, v in kwargs.items()])
+        cmds += ["--device-kwargs", device_kwargs]
+
+    try:
+        subprocess.run(cmds + pytest_args, check=not interactive)
+    except subprocess.CalledProcessError as e:
+        # pytest return codes:
+        #   Exit code 0:    All tests were collected and passed successfully
+        #   Exit code 1:    Tests were collected and run but some of the tests failed
+        #   Exit code 2:    Test execution was interrupted by the user
+        #   Exit code 3:    Internal error happened while executing tests
+        #   Exit code 4:    pytest command line usage error
+        #   Exit code 5:    No tests were collected
+        if e.returncode in range(1, 6):
+            # If a known pytest error code is returned, exit gracefully without
+            # an error message to avoid the user seeing duplicated tracebacks
+            sys.exit(1)
+
+        # otherwise raise the exception
+        raise e
+
+
+def cli():
+    """The PennyLane device test command line interface.
+
+    The ``pl-device-test`` CLI is a convenience wrapper that calls
+    pytest for a particular device.
+
+    .. code-block:: console
+
+        $ pl-device-test --help
+        usage: pl-device-test [-h] [--device DEVICE] [--shots SHOTS]
+                              [--analytic ANALYTIC] [--skip-ops]
+
+        See below for available options and commands for working with the PennyLane
+        device tests.
+
+        General Options:
+          -h, --help           show this help message and exit
+          --device DEVICE      The device to test.
+          --shots SHOTS        Number of shots to use in stochastic mode.
+          --analytic ANALYTIC  Whether to run the tests in stochastic or exact mode.
+          --skip-ops           Skip tests that use unsupported device operations.
+          --flaky-report       Show the flaky report in the terminal
+          --device-kwargs KEY=VAL [KEY=VAL ...]
+                               Additional device kwargs.
+
+    Note that additional pytest command line arguments and flags can also be passed:
+
+    .. code-block:: console
+
+        $ pl-device-test --device default.qubit --shots 1234 --analytic False --tb=short -x
+    """
+    from .conftest import pytest_addoption
+
+    parser = argparse.ArgumentParser(
+        description="See below for available options and commands for working with the PennyLane device tests."
+    )
+    parser._optionals.title = "General Options"  # pylint: disable=protected-access
+    pytest_addoption(parser)
+    args, pytest_args = parser.parse_known_args()
+
+    flaky = False
+    if "--flaky-report" in pytest_args:
+        pytest_args.remove("--flaky-report")
+        flaky = True
+
+    test_device(
+        args.device,
+        analytic=args.analytic,
+        shots=args.shots,
+        skip_ops=args.skip_ops,
+        flaky_report=flaky,
+        pytest_args=pytest_args,
+        **args.device_kwargs,
+    )

pennylane/plugins/tests/conftest.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Contains shared fixtures for the device tests."""
+import argparse
 import os
 
 import numpy as np
@@ -127,31 +128,75 @@ def pytest_runtest_setup(item):
 # These functions are required to define the device name to run the tests for
 
 
+class StoreDictKeyPair(argparse.Action):
+    """Argparse action for storing key-value pairs as a dictionary.
+
+    For example, calling a CLI program with ``--mydict v1=k1 v2=5``:
+
+    >>> parser.add_argument("--mydict", dest="my_dict", action=StoreDictKeyPair, nargs="+")
+    >>> args = parser.parse()
+    >>> args.my_dict
+    {"v1": "k1", "v2": "5"}
+
+    Note that all keys will be strings.
+    """
+
+    # pylint: disable=too-few-public-methods
+
+    def __init__(self, option_strings, dest, nargs=None, **kwargs):
+        self._nargs = nargs
+        super(StoreDictKeyPair, self).__init__(option_strings, dest, nargs=nargs, **kwargs)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        my_dict = {}
+        for kv in values:
+            k, v = kv.split("=")
+            my_dict[k] = v
+        setattr(namespace, self.dest, my_dict)
+
+
 def pytest_addoption(parser):
     """Add command line option to pytest."""
 
+    if hasattr(parser, "add_argument"):
+        # parser is a argparse.Parser object
+        addoption = parser.add_argument
+    else:
+        # parser is a pytest.config.Parser object
+        addoption = parser.addoption
+
     # The options are the three arguments every device takes
-    parser.addoption("--device", action="store", default=None, help="The device to test.")
-    parser.addoption(
+    addoption("--device", action="store", default=None, help="The device to test.")
+    addoption(
         "--shots",
         action="store",
         default=None,
         type=int,
         help="Number of shots to use in stochastic mode.",
     )
-    parser.addoption(
+    addoption(
         "--analytic",
         action="store",
         default=None,
         help="Whether to run the tests in stochastic or exact mode.",
     )
-    parser.addoption(
+    addoption(
         "--skip-ops",
         action="store_true",
         default=False,
         help="Skip tests that use unsupported device operations.",
     )
 
+    addoption(
+        "--device-kwargs",
+        dest="device_kwargs",
+        action=StoreDictKeyPair,
+        default={},
+        nargs="+",
+        metavar="KEY=VAL",
+        help="Additional device kwargs.",
+    )
+
 
 def pytest_generate_tests(metafunc):
     """Set up fixtures from command line options. """
@@ -161,6 +206,7 @@ def pytest_generate_tests(metafunc):
         "name": opt.device,
         "shots": opt.shots,
         "analytic": opt.analytic,
+        **opt.device_kwargs,
     }
 
     # ===========================================
@@ -209,7 +255,7 @@ def pytest_runtest_makereport(item, call):
             # and those using not implemented features
             if (
                 call.excinfo.type == qml.DeviceError
-                and "not supported on device" in str(call.excinfo.value)
+                and "supported" in str(call.excinfo.value)
                 or call.excinfo.type == NotImplementedError
             ):
                 tr.wasxfail = "reason:" + str(call.excinfo.value)

pennylane/plugins/tests/pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+markers =
+    skip_unsupported: skip a test if it uses an operation unsupported on a device
+

setup.py

@@ -44,15 +44,16 @@
             'default.tensor = pennylane.beta.plugins.default_tensor:DefaultTensor',
             'default.tensor.tf = pennylane.beta.plugins.default_tensor_tf:DefaultTensorTF',
             ],
+        'console_scripts': [
+                'pl-device-test=pennylane.plugins.tests:cli'
+            ]
         },
     'description': 'PennyLane is a Python quantum machine learning library by Xanadu Inc.',
     'long_description': open('README.rst').read(),
     'provides': ["pennylane"],
     'install_requires': requirements,
-    'command_options': {
-        'build_sphinx': {
-            'version': ('setup.py', version),
-            'release': ('setup.py', version)}}
+    'package_data': {'pennylane': ['plugins/tests/pytest.ini']},
+    'include_package_data': True
 }
 
 classifiers = [