diff --git a/pennylane/plugins/__init__.py b/pennylane/plugins/__init__.py index b5be5b692f4..98d11233957 100644 --- a/pennylane/plugins/__init__.py +++ b/pennylane/plugins/__init__.py @@ -27,6 +27,7 @@ default_gaussian tf_ops autograd_ops + tests """ from .default_qubit import DefaultQubit from .default_gaussian import DefaultGaussian diff --git a/pennylane/plugins/tests/__init__.py b/pennylane/plugins/tests/__init__.py index 26a375a45aa..4321461ff60 100644 --- a/pennylane/plugins/tests/__init__.py +++ b/pennylane/plugins/tests/__init__.py @@ -15,27 +15,12 @@ This subpackage provides integration tests for the devices with PennyLane's core functionalities. At the moment, the tests only run on devices based on the 'qubit' model. -To run the tests against a particular device (i.e., for 'default.qubit'): +The tests require that ``pytest``, ``pytest-mock``, and ``flaky`` be installed. +These can be installed using ``pip``: .. code-block:: console - python3 -m pytest path_to_pennylane/plugins/tests --device default.qubit --shots 1234 --analytic False - -The location of your PennyLane installation may differ depending on installation method and operating -system. To find the location, you can execute the following Python code: - ->>> import os ->>> import pennylane as qml ->>> print(os.path.dirname(qml.__file__)) - -The command line arguments are optional: - -* If `--device` is not given, the tests are run on the qubit core devices that ship with PennyLane. - -* If `--shots` is not given, a default of 10000 is used. The shots argument is ignored for devices running in - analytic mode. - -* If `--analytic` is not given, the device's default is used. + pip install pytest pytest-mock flaky The tests can also be run on an external device from a PennyLane plugin, such as ``'qiskit.aer'``. For this, make sure you have the correct dependencies installed. @@ -47,4 +32,209 @@ For non-analytic tests, the tolerance of the assert statements is set to a high enough value to account for stochastic fluctuations. Flaky is used to automatically repeat failed tests. + +There are several methods for running the tests against a particular device (i.e., for +``'default.qubit'``), detailed below. + +Using pytest +------------ + +.. code-block:: console + + pytest path_to_pennylane_src/plugins/tests --device=default.qubit --shots=10000 --analytic=False + +The location of your PennyLane installation may differ depending on installation method and +operating system. To find the location, you can use the :func:`~.get_device_tests` function: + +>>> from pennylane.plugins.tests import get_device_tests +>>> get_device_tests() + +The pl-device-test CLI +---------------------- + +Alternatively, PennyLane provides a command line interface for invoking the device tests. + +.. code-block:: console + + pl-device-test --device default.qubit --shots 10000 --analytic False + +Within Python +------------- + +Finally, the tests can be invoked within a Python session via the :func:`~.test_device` +function: + +>>> from pennylane.plugins.tests import test_device +>>> test_device("default.qubit") + +For more details on the available arguments, see the :func:`~.test_device` documentation. + +Functions +--------- """ +# pylint: disable=import-outside-toplevel,too-many-arguments +import argparse +import pathlib +import subprocess +import sys + + +# determine if running in an interactive environment +import __main__ + +interactive = False + +try: + __main__.__file__ +except AttributeError: + interactive = True + + +def get_device_tests(): + """Returns the location of the device integration tests.""" + return str(pathlib.Path(__file__).parent.absolute()) + + +def test_device( + device, analytic=None, shots=None, skip_ops=True, flaky_report=False, pytest_args=None, **kwargs +): + """Run the device integration tests using an installed PennyLane device. + + Args: + device (str): the name of the device to test + analytic (bool): Whether to run the device in analytic mode (where + expectation values and probabilities are computed exactly from the quantum state) + or non-analytic/"stochastic" mode (where probabilities and expectation + values are *estimated* using a finite number of shots.) + If not provided, the device default is used. + shots (int): The number of shots/samples used to estimate expectation + values and probability. Only takes affect if ``analytic=False``. If not + provided, the device default is used. + skip_ops (bool): whether to skip tests that use operations not supported + by the device + pytest_args (list[str]): additional PyTest arguments and flags + **kwargs: Additional device keyword args + + **Example** + + >>> from pennylane.plugins.tests import test_device + >>> test_device("default.qubit") + ================================ test session starts ======================================= + platform linux -- Python 3.7.7, pytest-5.4.2, py-1.8.1, pluggy-0.13.1 + rootdir: /home/josh/xanadu/pennylane/pennylane/plugins/tests, inifile: pytest.ini + plugins: flaky-3.6.1, cov-2.8.1, mock-3.1.0 + collected 86 items + xanadu/pennylane/pennylane/plugins/tests/test_gates.py .............................. + ............................... [ 70%] + xanadu/pennylane/pennylane/plugins/tests/test_measurements.py .......sss...sss..sss [ 95%] + xanadu/pennylane/pennylane/plugins/tests/test_properties.py .... [100%] + ================================= 77 passed, 9 skipped in 0.78s ============================ + + """ + try: + import pytest # pylint: disable=unused-import + import pytest_mock # pylint: disable=unused-import + import flaky # pylint: disable=unused-import + except ImportError: + raise ImportError( + "The device tests requires the following Python packages:" + "\npytest pytest_mock flaky" + "\nThese can be installed using pip." + ) + + pytest_args = pytest_args or [] + test_dir = get_device_tests() + + cmds = ["pytest"] + cmds.append(test_dir) + cmds.append(f"--device={device}") + + if shots is not None: + cmds.append(f"--shots={shots}") + + if analytic is not None: + cmds.append(f"--analytic={analytic}") + + if skip_ops: + cmds.append("--skip-ops") + + if not flaky_report: + cmds.append("--no-flaky-report") + + if kwargs: + device_kwargs = " ".join([f"{k}={v}" for k, v in kwargs.items()]) + cmds += ["--device-kwargs", device_kwargs] + + try: + subprocess.run(cmds + pytest_args, check=not interactive) + except subprocess.CalledProcessError as e: + # pytest return codes: + # Exit code 0: All tests were collected and passed successfully + # Exit code 1: Tests were collected and run but some of the tests failed + # Exit code 2: Test execution was interrupted by the user + # Exit code 3: Internal error happened while executing tests + # Exit code 4: pytest command line usage error + # Exit code 5: No tests were collected + if e.returncode in range(1, 6): + # If a known pytest error code is returned, exit gracefully without + # an error message to avoid the user seeing duplicated tracebacks + sys.exit(1) + + # otherwise raise the exception + raise e + + +def cli(): + """The PennyLane device test command line interface. + + The ``pl-device-test`` CLI is a convenience wrapper that calls + pytest for a particular device. + + .. code-block:: console + + $ pl-device-test --help + usage: pl-device-test [-h] [--device DEVICE] [--shots SHOTS] + [--analytic ANALYTIC] [--skip-ops] + + See below for available options and commands for working with the PennyLane + device tests. + + General Options: + -h, --help show this help message and exit + --device DEVICE The device to test. + --shots SHOTS Number of shots to use in stochastic mode. + --analytic ANALYTIC Whether to run the tests in stochastic or exact mode. + --skip-ops Skip tests that use unsupported device operations. + --flaky-report Show the flaky report in the terminal + --device-kwargs KEY=VAL [KEY=VAL ...] + Additional device kwargs. + + Note that additional pytest command line arguments and flags can also be passed: + + .. code-block:: console + + $ pl-device-test --device default.qubit --shots 1234 --analytic False --tb=short -x + """ + from .conftest import pytest_addoption + + parser = argparse.ArgumentParser( + description="See below for available options and commands for working with the PennyLane device tests." + ) + parser._optionals.title = "General Options" # pylint: disable=protected-access + pytest_addoption(parser) + args, pytest_args = parser.parse_known_args() + + flaky = False + if "--flaky-report" in pytest_args: + pytest_args.remove("--flaky-report") + flaky = True + + test_device( + args.device, + analytic=args.analytic, + shots=args.shots, + skip_ops=args.skip_ops, + flaky_report=flaky, + pytest_args=pytest_args, + **args.device_kwargs, + ) diff --git a/pennylane/plugins/tests/conftest.py b/pennylane/plugins/tests/conftest.py index 46feb3c410a..f34c68643eb 100755 --- a/pennylane/plugins/tests/conftest.py +++ b/pennylane/plugins/tests/conftest.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Contains shared fixtures for the device tests.""" +import argparse import os import numpy as np @@ -127,31 +128,75 @@ def pytest_runtest_setup(item): # These functions are required to define the device name to run the tests for +class StoreDictKeyPair(argparse.Action): + """Argparse action for storing key-value pairs as a dictionary. + + For example, calling a CLI program with ``--mydict v1=k1 v2=5``: + + >>> parser.add_argument("--mydict", dest="my_dict", action=StoreDictKeyPair, nargs="+") + >>> args = parser.parse() + >>> args.my_dict + {"v1": "k1", "v2": "5"} + + Note that all keys will be strings. + """ + + # pylint: disable=too-few-public-methods + + def __init__(self, option_strings, dest, nargs=None, **kwargs): + self._nargs = nargs + super(StoreDictKeyPair, self).__init__(option_strings, dest, nargs=nargs, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + my_dict = {} + for kv in values: + k, v = kv.split("=") + my_dict[k] = v + setattr(namespace, self.dest, my_dict) + + def pytest_addoption(parser): """Add command line option to pytest.""" + if hasattr(parser, "add_argument"): + # parser is a argparse.Parser object + addoption = parser.add_argument + else: + # parser is a pytest.config.Parser object + addoption = parser.addoption + # The options are the three arguments every device takes - parser.addoption("--device", action="store", default=None, help="The device to test.") - parser.addoption( + addoption("--device", action="store", default=None, help="The device to test.") + addoption( "--shots", action="store", default=None, type=int, help="Number of shots to use in stochastic mode.", ) - parser.addoption( + addoption( "--analytic", action="store", default=None, help="Whether to run the tests in stochastic or exact mode.", ) - parser.addoption( + addoption( "--skip-ops", action="store_true", default=False, help="Skip tests that use unsupported device operations.", ) + addoption( + "--device-kwargs", + dest="device_kwargs", + action=StoreDictKeyPair, + default={}, + nargs="+", + metavar="KEY=VAL", + help="Additional device kwargs.", + ) + def pytest_generate_tests(metafunc): """Set up fixtures from command line options. """ @@ -161,6 +206,7 @@ def pytest_generate_tests(metafunc): "name": opt.device, "shots": opt.shots, "analytic": opt.analytic, + **opt.device_kwargs, } # =========================================== @@ -209,7 +255,7 @@ def pytest_runtest_makereport(item, call): # and those using not implemented features if ( call.excinfo.type == qml.DeviceError - and "not supported on device" in str(call.excinfo.value) + and "supported" in str(call.excinfo.value) or call.excinfo.type == NotImplementedError ): tr.wasxfail = "reason:" + str(call.excinfo.value) diff --git a/pennylane/plugins/tests/pytest.ini b/pennylane/plugins/tests/pytest.ini new file mode 100644 index 00000000000..e609de1e14c --- /dev/null +++ b/pennylane/plugins/tests/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +markers = + skip_unsupported: skip a test if it uses an operation unsupported on a device + diff --git a/setup.py b/setup.py index dba3729f386..09db3862763 100644 --- a/setup.py +++ b/setup.py @@ -44,15 +44,16 @@ 'default.tensor = pennylane.beta.plugins.default_tensor:DefaultTensor', 'default.tensor.tf = pennylane.beta.plugins.default_tensor_tf:DefaultTensorTF', ], + 'console_scripts': [ + 'pl-device-test=pennylane.plugins.tests:cli' + ] }, 'description': 'PennyLane is a Python quantum machine learning library by Xanadu Inc.', 'long_description': open('README.rst').read(), 'provides': ["pennylane"], 'install_requires': requirements, - 'command_options': { - 'build_sphinx': { - 'version': ('setup.py', version), - 'release': ('setup.py', version)}} + 'package_data': {'pennylane': ['plugins/tests/pytest.ini']}, + 'include_package_data': True } classifiers = [