test_cacheprovider.py

from __future__ import absolute_import, division, print_function
import sys
import py
import _pytest
import pytest
import os
import shutil

pytest_plugins = "pytester",


class TestNewAPI(object):

    def test_config_cache_makedir(self, testdir):
        testdir.makeini("[pytest]")
        config = testdir.parseconfigure()
        with pytest.raises(ValueError):
            config.cache.makedir("key/name")

        p = config.cache.makedir("name")
        assert p.check()

    def test_config_cache_dataerror(self, testdir):
        testdir.makeini("[pytest]")
        config = testdir.parseconfigure()
        cache = config.cache
        pytest.raises(TypeError, lambda: cache.set("key/name", cache))
        config.cache.set("key/name", 0)
        config.cache._getvaluepath("key/name").write("123invalid")
        val = config.cache.get("key/name", -2)
        assert val == -2

    def test_cache_writefail_cachfile_silent(self, testdir):
        testdir.makeini("[pytest]")
        testdir.tmpdir.join(".pytest_cache").write("gone wrong")
        config = testdir.parseconfigure()
        cache = config.cache
        cache.set("test/broken", [])

    @pytest.mark.skipif(sys.platform.startswith("win"), reason="no chmod on windows")
    def test_cache_writefail_permissions(self, testdir):
        testdir.makeini("[pytest]")
        testdir.tmpdir.ensure_dir(".pytest_cache").chmod(0)
        config = testdir.parseconfigure()
        cache = config.cache
        cache.set("test/broken", [])

    @pytest.mark.skipif(sys.platform.startswith("win"), reason="no chmod on windows")
    def test_cache_failure_warns(self, testdir):
        testdir.tmpdir.ensure_dir(".pytest_cache").chmod(0)
        testdir.makepyfile(
            """
            def test_error():
                raise Exception

        """
        )
        result = testdir.runpytest("-rw")
        assert result.ret == 1
        result.stdout.fnmatch_lines(["*could not create cache path*", "*2 warnings*"])

    def test_config_cache(self, testdir):
        testdir.makeconftest(
            """
            def pytest_configure(config):
                # see that we get cache information early on
                assert hasattr(config, "cache")
        """
        )
        testdir.makepyfile(
            """
            def test_session(pytestconfig):
                assert hasattr(pytestconfig, "cache")
        """
        )
        result = testdir.runpytest()
        assert result.ret == 0
        result.stdout.fnmatch_lines(["*1 passed*"])

    def test_cachefuncarg(self, testdir):
        testdir.makepyfile(
            """
            import pytest
            def test_cachefuncarg(cache):
                val = cache.get("some/thing", None)
                assert val is None
                cache.set("some/thing", [1])
                pytest.raises(TypeError, lambda: cache.get("some/thing"))
                val = cache.get("some/thing", [])
                assert val == [1]
        """
        )
        result = testdir.runpytest()
        assert result.ret == 0
        result.stdout.fnmatch_lines(["*1 passed*"])

    def test_custom_rel_cache_dir(self, testdir):
        rel_cache_dir = os.path.join("custom_cache_dir", "subdir")
        testdir.makeini(
            """
            [pytest]
            cache_dir = {cache_dir}
        """.format(
                cache_dir=rel_cache_dir
            )
        )
        testdir.makepyfile(test_errored="def test_error():\n    assert False")
        testdir.runpytest()
        assert testdir.tmpdir.join(rel_cache_dir).isdir()

    def test_custom_abs_cache_dir(self, testdir, tmpdir_factory):
        tmp = str(tmpdir_factory.mktemp("tmp"))
        abs_cache_dir = os.path.join(tmp, "custom_cache_dir")
        testdir.makeini(
            """
            [pytest]
            cache_dir = {cache_dir}
        """.format(
                cache_dir=abs_cache_dir
            )
        )
        testdir.makepyfile(test_errored="def test_error():\n    assert False")
        testdir.runpytest()
        assert py.path.local(abs_cache_dir).isdir()

    def test_custom_cache_dir_with_env_var(self, testdir, monkeypatch):
        monkeypatch.setenv("env_var", "custom_cache_dir")
        testdir.makeini(
            """
            [pytest]
            cache_dir = {cache_dir}
        """.format(
                cache_dir="$env_var"
            )
        )
        testdir.makepyfile(test_errored="def test_error():\n    assert False")
        testdir.runpytest()
        assert testdir.tmpdir.join("custom_cache_dir").isdir()


def test_cache_reportheader(testdir):
    testdir.makepyfile(
        """
        def test_hello():
            pass
    """
    )
    result = testdir.runpytest("-v")
    result.stdout.fnmatch_lines(["cachedir: .pytest_cache"])


def test_cache_show(testdir):
    result = testdir.runpytest("--cache-show")
    assert result.ret == 0
    result.stdout.fnmatch_lines(["*cache is empty*"])
    testdir.makeconftest(
        """
        def pytest_configure(config):
            config.cache.set("my/name", [1,2,3])
            config.cache.set("other/some", {1:2})
            dp = config.cache.makedir("mydb")
            dp.ensure("hello")
            dp.ensure("world")
    """
    )
    result = testdir.runpytest()
    assert result.ret == 5  # no tests executed
    result = testdir.runpytest("--cache-show")
    result.stdout.fnmatch_lines_random(
        [
            "*cachedir:*",
            "-*cache values*-",
            "*my/name contains:",
            "  [1, 2, 3]",
            "*other/some contains*",
            "  {*1*: 2}",
            "-*cache directories*-",
            "*mydb/hello*length 0*",
            "*mydb/world*length 0*",
        ]
    )


class TestLastFailed(object):

    def test_lastfailed_usecase(self, testdir, monkeypatch):
        monkeypatch.setenv("PYTHONDONTWRITEBYTECODE", 1)
        p = testdir.makepyfile(
            """
            def test_1():
                assert 0
            def test_2():
                assert 0
            def test_3():
                assert 1
        """
        )
        result = testdir.runpytest()
        result.stdout.fnmatch_lines(["*2 failed*"])
        p.write(
            _pytest._code.Source(
                """
            def test_1():
                assert 1

            def test_2():
                assert 1

            def test_3():
                assert 0
        """
            )
        )
        result = testdir.runpytest("--lf")
        result.stdout.fnmatch_lines(["*2 passed*1 desel*"])
        result = testdir.runpytest("--lf")
        result.stdout.fnmatch_lines(["*1 failed*2 passed*"])
        result = testdir.runpytest("--lf", "--cache-clear")
        result.stdout.fnmatch_lines(["*1 failed*2 passed*"])

        # Run this again to make sure clear-cache is robust
        if os.path.isdir(".pytest_cache"):
            shutil.rmtree(".pytest_cache")
        result = testdir.runpytest("--lf", "--cache-clear")
        result.stdout.fnmatch_lines(["*1 failed*2 passed*"])

    def test_failedfirst_order(self, testdir):
        testdir.tmpdir.join("test_a.py").write(
            _pytest._code.Source(
                """
            def test_always_passes():
                assert 1
        """
            )
        )
        testdir.tmpdir.join("test_b.py").write(
            _pytest._code.Source(
                """
            def test_always_fails():
                assert 0
        """
            )
        )
        result = testdir.runpytest()
        # Test order will be collection order; alphabetical
        result.stdout.fnmatch_lines(["test_a.py*", "test_b.py*"])
        result = testdir.runpytest("--ff")
        # Test order will be failing tests firs
        result.stdout.fnmatch_lines(["test_b.py*", "test_a.py*"])

    def test_lastfailed_failedfirst_order(self, testdir):
        testdir.makepyfile(
            **{
                "test_a.py": """
                def test_always_passes():
                    assert 1
            """,
                "test_b.py": """
                def test_always_fails():
                    assert 0
            """,
            }
        )
        result = testdir.runpytest()
        # Test order will be collection order; alphabetical
        result.stdout.fnmatch_lines(["test_a.py*", "test_b.py*"])
        result = testdir.runpytest("--lf", "--ff")
        # Test order will be failing tests firs
        result.stdout.fnmatch_lines(["test_b.py*"])
        assert "test_a.py" not in result.stdout.str()

    def test_lastfailed_difference_invocations(self, testdir, monkeypatch):
        monkeypatch.setenv("PYTHONDONTWRITEBYTECODE", 1)
        testdir.makepyfile(
            test_a="""
            def test_a1():
                assert 0
            def test_a2():
                assert 1
        """,
            test_b="""
            def test_b1():
                assert 0
        """,
        )
        p = testdir.tmpdir.join("test_a.py")
        p2 = testdir.tmpdir.join("test_b.py")

        result = testdir.runpytest()
        result.stdout.fnmatch_lines(["*2 failed*"])
        result = testdir.runpytest("--lf", p2)
        result.stdout.fnmatch_lines(["*1 failed*"])
        p2.write(
            _pytest._code.Source(
                """
            def test_b1():
                assert 1
        """
            )
        )
        result = testdir.runpytest("--lf", p2)
        result.stdout.fnmatch_lines(["*1 passed*"])
        result = testdir.runpytest("--lf", p)
        result.stdout.fnmatch_lines(["*1 failed*1 desel*"])

    def test_lastfailed_usecase_splice(self, testdir, monkeypatch):
        monkeypatch.setenv("PYTHONDONTWRITEBYTECODE", 1)
        testdir.makepyfile(
            """
            def test_1():
                assert 0
        """
        )
        p2 = testdir.tmpdir.join("test_something.py")
        p2.write(
            _pytest._code.Source(
                """
            def test_2():
                assert 0
        """
            )
        )
        result = testdir.runpytest()
        result.stdout.fnmatch_lines(["*2 failed*"])
        result = testdir.runpytest("--lf", p2)
        result.stdout.fnmatch_lines(["*1 failed*"])
        result = testdir.runpytest("--lf")
        result.stdout.fnmatch_lines(["*2 failed*"])

    def test_lastfailed_xpass(self, testdir):
        testdir.inline_runsource(
            """
            import pytest
            @pytest.mark.xfail
            def test_hello():
                assert 1
        """
        )
        config = testdir.parseconfigure()
        lastfailed = config.cache.get("cache/lastfailed", -1)
        assert lastfailed == -1

    def test_non_serializable_parametrize(self, testdir):
        """Test that failed parametrized tests with unmarshable parameters
        don't break pytest-cache.
        """
        testdir.makepyfile(
            r"""
            import pytest

            @pytest.mark.parametrize('val', [
                b'\xac\x10\x02G',
            ])
            def test_fail(val):
                assert False
        """
        )
        result = testdir.runpytest()
        result.stdout.fnmatch_lines("*1 failed in*")

    def test_terminal_report_lastfailed(self, testdir):
        test_a = testdir.makepyfile(
            test_a="""
            def test_a1():
                pass
            def test_a2():
                pass
        """
        )
        test_b = testdir.makepyfile(
            test_b="""
            def test_b1():
                assert 0
            def test_b2():
                assert 0
        """
        )
        result = testdir.runpytest()
        result.stdout.fnmatch_lines(["collected 4 items", "*2 failed, 2 passed in*"])

        result = testdir.runpytest("--lf")
        result.stdout.fnmatch_lines(
            [
                "collected 4 items / 2 deselected",
                "run-last-failure: rerun previous 2 failures",
                "*2 failed, 2 deselected in*",
            ]
        )

        result = testdir.runpytest(test_a, "--lf")
        result.stdout.fnmatch_lines(
            [
                "collected 2 items",
                "run-last-failure: run all (no recorded failures)",
                "*2 passed in*",
            ]
        )

        result = testdir.runpytest(test_b, "--lf")
        result.stdout.fnmatch_lines(
            [
                "collected 2 items",
                "run-last-failure: rerun previous 2 failures",
                "*2 failed in*",
            ]
        )

        result = testdir.runpytest("test_b.py::test_b1", "--lf")
        result.stdout.fnmatch_lines(
            [
                "collected 1 item",
                "run-last-failure: rerun previous 1 failure",
                "*1 failed in*",
            ]
        )

    def test_terminal_report_failedfirst(self, testdir):
        testdir.makepyfile(
            test_a="""
            def test_a1():
                assert 0
            def test_a2():
                pass
        """
        )
        result = testdir.runpytest()
        result.stdout.fnmatch_lines(["collected 2 items", "*1 failed, 1 passed in*"])

        result = testdir.runpytest("--ff")
        result.stdout.fnmatch_lines(
            [
                "collected 2 items",
                "run-last-failure: rerun previous 1 failure first",
                "*1 failed, 1 passed in*",
            ]
        )

    def test_lastfailed_collectfailure(self, testdir, monkeypatch):

        testdir.makepyfile(
            test_maybe="""
            import os
            env = os.environ
            if '1' == env['FAILIMPORT']:
                raise ImportError('fail')
            def test_hello():
                assert '0' == env['FAILTEST']
        """
        )

        def rlf(fail_import, fail_run):
            monkeypatch.setenv("FAILIMPORT", fail_import)
            monkeypatch.setenv("FAILTEST", fail_run)

            testdir.runpytest("-q")
            config = testdir.parseconfigure()
            lastfailed = config.cache.get("cache/lastfailed", -1)
            return lastfailed

        lastfailed = rlf(fail_import=0, fail_run=0)
        assert lastfailed == -1

        lastfailed = rlf(fail_import=1, fail_run=0)
        assert list(lastfailed) == ["test_maybe.py"]

        lastfailed = rlf(fail_import=0, fail_run=1)
        assert list(lastfailed) == ["test_maybe.py::test_hello"]

    def test_lastfailed_failure_subset(self, testdir, monkeypatch):

        testdir.makepyfile(
            test_maybe="""
            import os
            env = os.environ
            if '1' == env['FAILIMPORT']:
                raise ImportError('fail')
            def test_hello():
                assert '0' == env['FAILTEST']
        """
        )

        testdir.makepyfile(
            test_maybe2="""
            import os
            env = os.environ
            if '1' == env['FAILIMPORT']:
                raise ImportError('fail')
            def test_hello():
                assert '0' == env['FAILTEST']

            def test_pass():
                pass
        """
        )

        def rlf(fail_import, fail_run, args=()):
            monkeypatch.setenv("FAILIMPORT", fail_import)
            monkeypatch.setenv("FAILTEST", fail_run)

            result = testdir.runpytest("-q", "--lf", *args)
            config = testdir.parseconfigure()
            lastfailed = config.cache.get("cache/lastfailed", -1)
            return result, lastfailed

        result, lastfailed = rlf(fail_import=0, fail_run=0)
        assert lastfailed == -1
        result.stdout.fnmatch_lines(["*3 passed*"])

        result, lastfailed = rlf(fail_import=1, fail_run=0)
        assert sorted(list(lastfailed)) == ["test_maybe.py", "test_maybe2.py"]

        result, lastfailed = rlf(fail_import=0, fail_run=0, args=("test_maybe2.py",))
        assert list(lastfailed) == ["test_maybe.py"]

        # edge case of test selection - even if we remember failures
        # from other tests we still need to run all tests if no test
        # matches the failures
        result, lastfailed = rlf(fail_import=0, fail_run=0, args=("test_maybe2.py",))
        assert list(lastfailed) == ["test_maybe.py"]
        result.stdout.fnmatch_lines(["*2 passed*"])

    def test_lastfailed_creates_cache_when_needed(self, testdir):
        # Issue #1342
        testdir.makepyfile(test_empty="")
        testdir.runpytest("-q", "--lf")
        assert not os.path.exists(".pytest_cache/v/cache/lastfailed")

        testdir.makepyfile(test_successful="def test_success():\n    assert True")
        testdir.runpytest("-q", "--lf")
        assert not os.path.exists(".pytest_cache/v/cache/lastfailed")

        testdir.makepyfile(test_errored="def test_error():\n    assert False")
        testdir.runpytest("-q", "--lf")
        assert os.path.exists(".pytest_cache/v/cache/lastfailed")

    def test_xfail_not_considered_failure(self, testdir):
        testdir.makepyfile(
            """
            import pytest
            @pytest.mark.xfail
            def test():
                assert 0
        """
        )
        result = testdir.runpytest()
        result.stdout.fnmatch_lines("*1 xfailed*")
        assert self.get_cached_last_failed(testdir) == []

    def test_xfail_strict_considered_failure(self, testdir):
        testdir.makepyfile(
            """
            import pytest
            @pytest.mark.xfail(strict=True)
            def test():
                pass
        """
        )
        result = testdir.runpytest()
        result.stdout.fnmatch_lines("*1 failed*")
        assert (
            self.get_cached_last_failed(testdir)
            == ["test_xfail_strict_considered_failure.py::test"]
        )

    @pytest.mark.parametrize("mark", ["mark.xfail", "mark.skip"])
    def test_failed_changed_to_xfail_or_skip(self, testdir, mark):
        testdir.makepyfile(
            """
            import pytest
            def test():
                assert 0
        """
        )
        result = testdir.runpytest()
        assert (
            self.get_cached_last_failed(testdir)
            == ["test_failed_changed_to_xfail_or_skip.py::test"]
        )
        assert result.ret == 1

        testdir.makepyfile(
            """
            import pytest
            @pytest.{mark}
            def test():
                assert 0
        """.format(
                mark=mark
            )
        )
        result = testdir.runpytest()
        assert result.ret == 0
        assert self.get_cached_last_failed(testdir) == []
        assert result.ret == 0

    def get_cached_last_failed(self, testdir):
        config = testdir.parseconfigure()
        return sorted(config.cache.get("cache/lastfailed", {}))

    def test_cache_cumulative(self, testdir):
        """
        Test workflow where user fixes errors gradually file by file using --lf.
        """
        # 1. initial run
        test_bar = testdir.makepyfile(
            test_bar="""
            def test_bar_1():
                pass
            def test_bar_2():
                assert 0
        """
        )
        test_foo = testdir.makepyfile(
            test_foo="""
            def test_foo_3():
                pass
            def test_foo_4():
                assert 0
        """
        )
        testdir.runpytest()
        assert (
            self.get_cached_last_failed(testdir)
            == ["test_bar.py::test_bar_2", "test_foo.py::test_foo_4"]
        )

        # 2. fix test_bar_2, run only test_bar.py
        testdir.makepyfile(
            test_bar="""
            def test_bar_1():
                pass
            def test_bar_2():
                pass
        """
        )
        result = testdir.runpytest(test_bar)
        result.stdout.fnmatch_lines("*2 passed*")
        # ensure cache does not forget that test_foo_4 failed once before
        assert self.get_cached_last_failed(testdir) == ["test_foo.py::test_foo_4"]

        result = testdir.runpytest("--last-failed")
        result.stdout.fnmatch_lines("*1 failed, 3 deselected*")
        assert self.get_cached_last_failed(testdir) == ["test_foo.py::test_foo_4"]

        # 3. fix test_foo_4, run only test_foo.py
        test_foo = testdir.makepyfile(
            test_foo="""
            def test_foo_3():
                pass
            def test_foo_4():
                pass
        """
        )
        result = testdir.runpytest(test_foo, "--last-failed")
        result.stdout.fnmatch_lines("*1 passed, 1 deselected*")
        assert self.get_cached_last_failed(testdir) == []

        result = testdir.runpytest("--last-failed")
        result.stdout.fnmatch_lines("*4 passed*")
        assert self.get_cached_last_failed(testdir) == []

    def test_lastfailed_no_failures_behavior_all_passed(self, testdir):
        testdir.makepyfile(
            """
            def test_1():
                assert True
            def test_2():
                assert True
        """
        )
        result = testdir.runpytest()
        result.stdout.fnmatch_lines(["*2 passed*"])
        result = testdir.runpytest("--lf")
        result.stdout.fnmatch_lines(["*2 passed*"])
        result = testdir.runpytest("--lf", "--lfnf", "all")
        result.stdout.fnmatch_lines(["*2 passed*"])
        result = testdir.runpytest("--lf", "--lfnf", "none")
        result.stdout.fnmatch_lines(["*2 desel*"])

    def test_lastfailed_no_failures_behavior_empty_cache(self, testdir):
        testdir.makepyfile(
            """
            def test_1():
                assert True
            def test_2():
                assert False
        """
        )
        result = testdir.runpytest("--lf", "--cache-clear")
        result.stdout.fnmatch_lines(["*1 failed*1 passed*"])
        result = testdir.runpytest("--lf", "--cache-clear", "--lfnf", "all")
        result.stdout.fnmatch_lines(["*1 failed*1 passed*"])
        result = testdir.runpytest("--lf", "--cache-clear", "--lfnf", "none")
        result.stdout.fnmatch_lines(["*2 desel*"])


class TestNewFirst(object):

    def test_newfirst_usecase(self, testdir):
        testdir.makepyfile(
            **{
                "test_1/test_1.py": """
                def test_1(): assert 1
                def test_2(): assert 1
                def test_3(): assert 1
            """,
                "test_2/test_2.py": """
                def test_1(): assert 1
                def test_2(): assert 1
                def test_3(): assert 1
            """,
            }
        )

        testdir.tmpdir.join("test_1/test_1.py").setmtime(1)

        result = testdir.runpytest("-v")
        result.stdout.fnmatch_lines(
            [
                "*test_1/test_1.py::test_1 PASSED*",
                "*test_1/test_1.py::test_2 PASSED*",
                "*test_1/test_1.py::test_3 PASSED*",
                "*test_2/test_2.py::test_1 PASSED*",
                "*test_2/test_2.py::test_2 PASSED*",
                "*test_2/test_2.py::test_3 PASSED*",
            ]
        )

        result = testdir.runpytest("-v", "--nf")

        result.stdout.fnmatch_lines(
            [
                "*test_2/test_2.py::test_1 PASSED*",
                "*test_2/test_2.py::test_2 PASSED*",
                "*test_2/test_2.py::test_3 PASSED*",
                "*test_1/test_1.py::test_1 PASSED*",
                "*test_1/test_1.py::test_2 PASSED*",
                "*test_1/test_1.py::test_3 PASSED*",
            ]
        )

        testdir.tmpdir.join("test_1/test_1.py").write(
            "def test_1(): assert 1\n"
            "def test_2(): assert 1\n"
            "def test_3(): assert 1\n"
            "def test_4(): assert 1\n"
        )
        testdir.tmpdir.join("test_1/test_1.py").setmtime(1)

        result = testdir.runpytest("-v", "--nf")

        result.stdout.fnmatch_lines(
            [
                "*test_1/test_1.py::test_4 PASSED*",
                "*test_2/test_2.py::test_1 PASSED*",
                "*test_2/test_2.py::test_2 PASSED*",
                "*test_2/test_2.py::test_3 PASSED*",
                "*test_1/test_1.py::test_1 PASSED*",
                "*test_1/test_1.py::test_2 PASSED*",
                "*test_1/test_1.py::test_3 PASSED*",
            ]
        )

    def test_newfirst_parametrize(self, testdir):
        testdir.makepyfile(
            **{
                "test_1/test_1.py": """
                import pytest
                @pytest.mark.parametrize('num', [1, 2])
                def test_1(num): assert num
            """,
                "test_2/test_2.py": """
                import pytest
                @pytest.mark.parametrize('num', [1, 2])
                def test_1(num): assert num
            """,
            }
        )

        testdir.tmpdir.join("test_1/test_1.py").setmtime(1)

        result = testdir.runpytest("-v")
        result.stdout.fnmatch_lines(
            [
                "*test_1/test_1.py::test_1[1*",
                "*test_1/test_1.py::test_1[2*",
                "*test_2/test_2.py::test_1[1*",
                "*test_2/test_2.py::test_1[2*",
            ]
        )

        result = testdir.runpytest("-v", "--nf")

        result.stdout.fnmatch_lines(
            [
                "*test_2/test_2.py::test_1[1*",
                "*test_2/test_2.py::test_1[2*",
                "*test_1/test_1.py::test_1[1*",
                "*test_1/test_1.py::test_1[2*",
            ]
        )

        testdir.tmpdir.join("test_1/test_1.py").write(
            "import pytest\n"
            "@pytest.mark.parametrize('num', [1, 2, 3])\n"
            "def test_1(num): assert num\n"
        )
        testdir.tmpdir.join("test_1/test_1.py").setmtime(1)

        result = testdir.runpytest("-v", "--nf")

        result.stdout.fnmatch_lines(
            [
                "*test_1/test_1.py::test_1[3*",
                "*test_2/test_2.py::test_1[1*",
                "*test_2/test_2.py::test_1[2*",
                "*test_1/test_1.py::test_1[1*",
                "*test_1/test_1.py::test_1[2*",
            ]
        )