diff --git a/ckan/pastertemplates/template/README.rst_tmpl b/ckan/pastertemplates/template/README.rst_tmpl index a6783ba15dd..f0528fa7e11 100644 --- a/ckan/pastertemplates/template/README.rst_tmpl +++ b/ckan/pastertemplates/template/README.rst_tmpl @@ -100,12 +100,12 @@ Tests To run the tests, do:: - nosetests --nologcapture --with-pylons=test.ini + pytest --ckan-ini=test.ini To run the tests and produce a coverage report, first make sure you have coverage installed in your virtualenv (``pip install coverage``) then run:: - nosetests --nologcapture --with-pylons=test.ini --with-coverage --cover-package=ckanext.{{ project_shortname }} --cover-inclusive --cover-erase --cover-tests + pytest --ckan-ini=test.ini --cov=ckanext.{{ project_shortname }} ---------------------------------------- diff --git a/ckan/pastertemplates/template/bin/travis-run.sh_tmpl b/ckan/pastertemplates/template/bin/travis-run.sh_tmpl index 56e9548c07d..6a899cd8842 100755 --- a/ckan/pastertemplates/template/bin/travis-run.sh_tmpl +++ b/ckan/pastertemplates/template/bin/travis-run.sh_tmpl @@ -5,14 +5,8 @@ flake8 --version # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan,{{ project }} -nosetests --ckan \ - --nologcapture \ - --with-pylons=subdir/test.ini \ - --with-coverage \ - --cover-package=ckanext.{{ project_shortname }} \ - --cover-inclusive \ - --cover-erase \ - --cover-tests +pytest --ckan-ini=subdir/test.ini \ + --cov=ckanext.{{ project_shortname }} # strict linting flake8 . --count --max-complexity=10 --max-line-length=127 --statistics --exclude ckan,{{ project }} diff --git a/ckan/tests/factories.py b/ckan/tests/factories.py index 28619cf5af7..40aa4b5d408 100644 --- a/ckan/tests/factories.py +++ b/ckan/tests/factories.py @@ -171,15 +171,10 @@ class ResourceView(factory.Factory): Example:: - class TestSomethingWithResourceViews(object): - @classmethod - def setup_class(cls): - if not p.plugin_loaded('image_view'): - p.load('image_view') - - @classmethod - def teardown_class(cls): - p.unload('image_view') + @pytest.mark.ckan_config("ckan.plugins", "image_view") + @pytest.mark.usefixtures("with_plugins") + def test_resource_view_factory(): + ... """ diff --git a/ckan/tests/helpers.py b/ckan/tests/helpers.py index 402bbbe30e5..fce1384ba91 100644 --- a/ckan/tests/helpers.py +++ b/ckan/tests/helpers.py @@ -3,9 +3,8 @@ """This is a collection of helper functions for use in tests. We want to avoid sharing test helper functions between test modules as -much as possible, and we definitely don't want to share test fixtures between -test modules, or to introduce a complex hierarchy of test class subclasses, -etc. +much as possible, and we definitely don't want to introduce a complex +hierarchy of test class subclasses, etc. We want to reduce the amount of "travel" that a reader needs to undertake to understand a test method -- reducing the number of other files they need to go @@ -16,6 +15,9 @@ and make writing tests so much easier, that it's worth having them despite the potential drawbacks. +Consider using :ref:`fixtures` whenever is possible for setting up initial +state of a test or creating related objects. + This module is reserved for these very useful functions. """ @@ -43,7 +45,7 @@ def reset_db(): """Reset CKAN's database. - If a test class uses the database, then it should call this function in its + If a test class uses the database, then it may call this function in its ``setup()`` method to make sure that it has a clean database to start with (nothing left over from other test classes or from previous test runs). diff --git a/ckan/tests/logic/action/test_update.py b/ckan/tests/logic/action/test_update.py index 168c82cbff0..52a12a84827 100644 --- a/ckan/tests/logic/action/test_update.py +++ b/ckan/tests/logic/action/test_update.py @@ -126,33 +126,17 @@ def test_user_update_with_no_id(self): with pytest.raises(logic.ValidationError): helpers.call_action("user_update", **user_dict) - # START-FOR-LOOP-EXAMPLE - @pytest.mark.usefixtures("clean_db") - def test_user_update_with_invalid_name(self): + @pytest.mark.parametrize('name', ( + "", "a", False, 0, -1, 23, "new", "edit", "search", "a" * 200, "Hi!", + "i++%", + )) + def test_user_update_with_invalid_name(self, name): user = factories.User() + user["name"] = name + with pytest.raises(logic.ValidationError): - invalid_names = ( - "", - "a", - False, - 0, - -1, - 23, - "new", - "edit", - "search", - "a" * 200, - "Hi!", - "i++%", - ) - for name in invalid_names: - user["name"] = name - with pytest.raises(logic.ValidationError): - - helpers.call_action("user_update", **user) - - # END-FOR-LOOP-EXAMPLE + helpers.call_action("user_update", **user) @pytest.mark.usefixtures("clean_db") def test_user_update_to_name_that_already_exists(self): diff --git a/ckan/tests/pytest_ckan/fixtures.py b/ckan/tests/pytest_ckan/fixtures.py index 11295a6b908..cc8521f56db 100644 --- a/ckan/tests/pytest_ckan/fixtures.py +++ b/ckan/tests/pytest_ckan/fixtures.py @@ -1,4 +1,33 @@ # -*- coding: utf-8 -*- +"""This is a collection of pytest fixtures for use in tests. + +All fixtures bellow available anywhere under the root of CKAN +repository. Any external CKAN extension should be able to include them +by adding next lines under root `conftest.py` + +.. literalinclude:: /../conftest.py + +There are three type of fixtures available in CKAN: + +* Fixtures that have some side-effect. They don't return any useful + value and generally should be injected via + ``pytest.mark.usefixtures``. Ex.: `with_plugins`, `clean_db`, + `clean_index`. + +* Fixtures that provide value. Ex. `app` + +* Fixtures that provide factory function. They are rarely needed, so + prefer using 'side-effect' or 'value' fixtures. Main use-case when + one may use function-fixture - late initialization or repeatable + execution(ex.: cleaning database more than once in a single + test). But presence of these fixtures in test usually signals that + is's a good time to refactor this test. + +Deeper expanation can be found in `official documentation +`_ + +""" + import pytest import ckan.tests.helpers as test_helpers import ckan.plugins @@ -11,7 +40,20 @@ def ckan_config(request, monkeypatch): """Configuration object used by application. Takes into account config patches introduced by `ckan_config` - mark. + mark. For using custom config in the whole test, apply + `ckan_config` mark to it and inject `ckan_config` fixture: + + .. literalinclude:: /../ckan/tests/pytest_ckan/test_fixtures.py + :start-after: # START-CONFIG-OVERRIDE + :end-before: # END-CONFIG-OVERRIDE + + Otherwise, when change only need to be applied locally, use + ``monkeypatch`` fixture + + .. literalinclude:: /../ckan/tests/test_common.py + :start-after: # START-CONFIG-OVERRIDE + :end-before: # END-CONFIG-OVERRIDE + """ _original = config.copy() for mark in request.node.own_markers: @@ -26,7 +68,7 @@ def ckan_config(request, monkeypatch): def make_app(ckan_config): """Factory for client app. - Prefer using `app` instead if you have no need in lazy instantiation. + Prefer using ``app`` instead if you have no need in lazy instantiation. """ return test_helpers._get_test_app @@ -40,14 +82,17 @@ def app(make_app): @pytest.fixture(scope=u"session") def reset_db(): - """Callable for setting DB into initial state. + """Callable for setting DB into initial state. Prefer using + ``clean_db``. + """ return test_helpers.reset_db @pytest.fixture(scope=u"session") def reset_index(): - """Callable for cleaning search index. + """Callable for cleaning search index. Prefer using ``clean_index``. + """ return search.clear_all @@ -61,13 +106,22 @@ def clean_db(reset_db): @pytest.fixture def clean_index(reset_index): - """Start test with empty index. + """Start test with empty search index. """ reset_index() @pytest.fixture def with_plugins(ckan_config): + """Load all plugins specified by ``ckan.plugins`` config option in the + beginning of the test. When test ends (event with fail) unload all + those plugins in reverse order. + + .. literalinclude:: /../ckan/tests/test_factories.py + :start-after: # START-CONFIG-OVERRIDE + :end-before: # END-CONFIG-OVERRIDE + + """ plugins = ckan_config["ckan.plugins"].split() for plugin in plugins: if not ckan.plugins.plugin_loaded(plugin): diff --git a/ckan/tests/pytest_ckan/test_fixtures.py b/ckan/tests/pytest_ckan/test_fixtures.py index 70cd8026e33..1d1a21cac44 100644 --- a/ckan/tests/pytest_ckan/test_fixtures.py +++ b/ckan/tests/pytest_ckan/test_fixtures.py @@ -15,10 +15,11 @@ def test_ckan_config_do_not_have_some_new_config(ckan_config): assert u"some.new.config" not in ckan_config +# START-CONFIG-OVERRIDE @pytest.mark.ckan_config(u"some.new.config", u"exists") def test_ckan_config_mark(ckan_config): assert ckan_config[u"some.new.config"] == u"exists" - +# END-CONFIG-OVERRIDE @pytest.mark.ckan_config(u"some.new.config", u"exists") @pytest.mark.usefixtures(u"ckan_config") diff --git a/ckan/tests/test_common.py b/ckan/tests/test_common.py index c4fd9ac5c61..bdd17287a7e 100644 --- a/ckan/tests/test_common.py +++ b/ckan/tests/test_common.py @@ -126,12 +126,15 @@ def test_deleting_a_key_deletes_it_on_pylons_config(): assert u"ckan.site_title" not in ckan_config -def test_deleting_a_key_delets_it_on_flask_config(app, monkeypatch): +# START-CONFIG-OVERRIDE +def test_deleting_a_key_delets_it_on_flask_config( + app, monkeypatch, ckan_config +): with app.flask_app.app_context(): monkeypatch.setitem(ckan_config, u"ckan.site_title", u"Example title") del ckan_config[u"ckan.site_title"] assert u"ckan.site_title" not in flask.current_app.config - +# END-CONFIG-OVERRIDE @pytest.mark.ckan_config(u"ckan.site_title", u"Example title") def test_update_works_on_pylons_config(): diff --git a/ckan/tests/test_factories.py b/ckan/tests/test_factories.py index bda1857a2bc..b9d7e3785e9 100644 --- a/ckan/tests/test_factories.py +++ b/ckan/tests/test_factories.py @@ -30,13 +30,14 @@ def test_id_uniqueness(entity): assert first[u"id"] != second[u"id"] +# START-CONFIG-OVERRIDE @pytest.mark.ckan_config(u"ckan.plugins", u"image_view") @pytest.mark.usefixtures(u"with_plugins") def test_resource_view_factory(): resource_view1 = factories.ResourceView() resource_view2 = factories.ResourceView() assert resource_view1[u"id"] != resource_view2[u"id"] - +# END-CONFIG-OVERRIDE def test_dataset_factory_allows_creation_by_anonymous_user(): dataset = factories.Dataset(user=None) diff --git a/contrib/cookiecutter/ckan_extension/{{cookiecutter.project}}/README.rst b/contrib/cookiecutter/ckan_extension/{{cookiecutter.project}}/README.rst index f0159aee0d7..7f3497d9b38 100644 --- a/contrib/cookiecutter/ckan_extension/{{cookiecutter.project}}/README.rst +++ b/contrib/cookiecutter/ckan_extension/{{cookiecutter.project}}/README.rst @@ -100,12 +100,12 @@ Tests To run the tests, do:: - nosetests --nologcapture --with-pylons=test.ini + pytest --ckan-ini=test.ini To run the tests and produce a coverage report, first make sure you have -coverage installed in your virtualenv (``pip install coverage``) then run:: +``pytest-cov`` installed in your virtualenv (``pip install pytest-cov``) then run:: - nosetests --nologcapture --with-pylons=test.ini --with-coverage --cover-package=ckanext.{{ cookiecutter.project_shortname }} --cover-inclusive --cover-erase --cover-tests + pytest --ckan-ini=test.ini --cov=ckanext.{{ cookiecutter.project_shortname }} ---------------------------------------- diff --git a/contrib/cookiecutter/ckan_extension/{{cookiecutter.project}}/bin/travis-run.sh b/contrib/cookiecutter/ckan_extension/{{cookiecutter.project}}/bin/travis-run.sh index 1da694d8d48..376e0c92066 100755 --- a/contrib/cookiecutter/ckan_extension/{{cookiecutter.project}}/bin/travis-run.sh +++ b/contrib/cookiecutter/ckan_extension/{{cookiecutter.project}}/bin/travis-run.sh @@ -5,14 +5,8 @@ flake8 --version # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan,{{ cookiecutter.project }} -nosetests --ckan \ - --nologcapture \ - --with-pylons=subdir/test.ini \ - --with-coverage \ - --cover-package=ckanext.{{ cookiecutter.project_shortname }} \ - --cover-inclusive \ - --cover-erase \ - --cover-tests +pytest --ckan-ini=subdir/test.ini \ + --cov=ckanext.{{ cookiecutter.project_shortname }} # strict linting flake8 . --count --max-complexity=10 --max-line-length=127 --statistics --exclude ckan,{{ cookiecutter.project }} diff --git a/doc/contributing/database-migrations.rst b/doc/contributing/database-migrations.rst index 2cae6253cf3..00c8b0a26b9 100644 --- a/doc/contributing/database-migrations.rst +++ b/doc/contributing/database-migrations.rst @@ -66,8 +66,6 @@ made: 6. Do a dump again, then a diff again to see if the the only thing left are drop index statements. -7. run nosetests with ``--ckan-migration`` flag. - It's that simple. Well almost. * If you are doing any table/field renaming adding that to your new migrate diff --git a/doc/contributing/release-process.rst b/doc/contributing/release-process.rst index 39b6ff90650..72ff7154fb6 100644 --- a/doc/contributing/release-process.rst +++ b/doc/contributing/release-process.rst @@ -31,7 +31,7 @@ and the release is tagged in the form ``ckan-M.m.p``. All backports are cherry-p | | +-----+-------------+------> dev-v2.6 +-------> dev-v2.7 | | - ckan-2.6.0 ckan-2.6.1 + ckan-2.6.0 ckan-2.6.1 Additionally, the ``release-vM.m-latest`` branches always contain the latest @@ -41,7 +41,7 @@ published release for that version (eg ``2.6.1`` on the example above). .. note:: Prior to CKAN 2.6, release branches were named ``release-vM.m.p``, after the - :ref:`major, minor and patch versions ` they included, and patch releases + :ref:`major, minor and patch versions ` they included, and patch releases were always branched from the most recent tip of the previous patch release branch (tags were created with the same convention). Starting from CKAN 2.6, the convention is the one described above. @@ -115,7 +115,7 @@ Turn this file into a github issue with a checklist using this command:: #. Create the documentation branch from the release branch. This branch should be named just with the minor version and nothing else (eg ``2.7``, ``2.8``, etc). We will use - this branch to build the documentation in Read the Docs on all patch releases for + this branch to build the documentation in Read the Docs on all patch releases for this version. #. Make latest translation strings available on Transifex. @@ -334,7 +334,7 @@ a release. #. Run the most thorough tests:: - nosetests ckan/tests --ckan --ckan-migration --with-pylons=test-core.ini + pytest --ckan-ini=test-core.ini ckan/tests #. Do a final build of the front-end, add the generated files to the repo and commit the changes:: @@ -413,7 +413,7 @@ a release. (You will need an admin account.) - a. Make sure the documentation branch is up to date with the latest changes in the + a. Make sure the documentation branch is up to date with the latest changes in the corresponding ``dev-vX.Y`` branch. b. If this is the first time a minor version is released, go to the @@ -473,7 +473,7 @@ Preparing patch releases These are usually marked on Github using the ``Backport Pending`` `labels`_ and the relevant labels for the versions they should be cherry-picked to (eg ``Backport 2.5.3``). - Remember to look for PRs that are closed i.e. merged. Remove the ``Backport Pending`` label once the + Remember to look for PRs that are closed i.e. merged. Remove the ``Backport Pending`` label once the cherry-picking has been done (but leave the version ones). #. Ask the tech team if there are security fixes or other fixes to include. @@ -522,7 +522,7 @@ Doing the patch releases python setup.py sdist upload -#. Make sure the documentation branch (``X.Y``) is up to date with the latest changes in the +#. Make sure the documentation branch (``X.Y``) is up to date with the latest changes in the corresponding ``dev-vX.Y`` branch. #. Write a CKAN blog post and announce it to ckan-announce & ckan-dev & twitter. diff --git a/doc/contributing/testing.rst b/doc/contributing/testing.rst index d6918af7339..bb95364e065 100644 --- a/doc/contributing/testing.rst +++ b/doc/contributing/testing.rst @@ -53,12 +53,12 @@ Fast ``setup_class()`` methods, saved against the ``self`` attribute of test classes, or in test helper modules). - Instead write helper functions that create test objects and return them, - and have each test method call just the helpers it needs to do the setup - that it needs. + Instead use fixtures that create test objects and return them, and + inject into every method only required fixtures. - * Where appropriate, use the ``mock`` library to avoid pulling in other parts - of CKAN (especially the database), see :ref:`mock`. + * Where appropriate, use the ``monkeypatch`` `fixture + `_ to avoid + pulling in other parts of CKAN (especially the database). Independent * Each test module, class and method should be able to be run on its own. @@ -75,7 +75,7 @@ Clear You shouldn't have to figure out what a complex test method does, or go and look up a lot of code in other files to understand a test method. - * Tests should follow the canonical form for a unit test, see + * Tests should follow the canonical form for a pytest, see :ref:`test recipe`. * Write lots of small, simple test methods not a few big, complex tests. @@ -217,24 +217,6 @@ function test demonstrating the recipe: :start-after: # START-AFTER :end-before: # END-BEFORE -One common exception is when you want to use a ``for`` loop to call the -function being tested multiple times, passing it lots of different arguments -that should all produce the same return value and/or side effects. For example, -this test from :py:mod:`ckan.tests.logic.action.test_update`: - -.. literalinclude:: /../ckan/tests/logic/action/test_update.py - :start-after: # START-FOR-LOOP-EXAMPLE - :end-before: # END-FOR-LOOP-EXAMPLE - -The behavior of :py:func:`~ckan.logic.action.update.user_update` is the same -for every invalid value. -We do want to test :py:func:`~ckan.logic.action.update.user_update` with lots -of different invalid names, but we obviously don't want to write a dozen -separate test methods that are all the same apart from the value used for the -invalid user name. We don't really want to define a helper method and a dozen -test methods that call it either. So we use a simple loop. Technically this -test calls the function being tested more than once, but there's only one line -of code that calls it. How detailed should tests be? @@ -284,6 +266,14 @@ Test helper functions: :mod:`ckan.tests.helpers` :members: +.. _fixtures: + +Pytest fixtures +--------------- + +.. automodule:: ckan.tests.pytest_ckan.fixtures + :members: + .. _mock: Mocking: the ``mock`` library diff --git a/doc/maintaining/multilingual.rst b/doc/maintaining/multilingual.rst index b736378361e..c58325ea4a9 100644 --- a/doc/maintaining/multilingual.rst +++ b/doc/maintaining/multilingual.rst @@ -34,6 +34,6 @@ If you have a source installation of CKAN you can test the multilingual extensio :: - nosetests --ckan ckanext/multilingual/tests + pytest --ckan-ini=test-core.ini ckanext/multilingual/tests See :doc:`/contributing/test` for more information.