Do not rely on the version number to know if the tests are present and fixing test matrix

.travis.yml

@@ -26,12 +26,13 @@ matrix:
     - { stage: test, python: 3.6, env: TOXENV=readme }
     - { stage: build_and_package_sanity, python: 3.6, env: SANITY_CHECK=1 }
 
+# This is a hack to go around missing test_functional in pip packages for pylint 2.4 and should be removed when relying
+# on pylint >2.5 (along with a cleanup of pylint_django/tests/test_func.py)
 before_install:
-  - git clone --depth 1 https://github.com/PyCQA/pylint.git ~/pylint
+  - git clone --depth 1 https://github.com/PyCQA/pylint.git --branch 2.4 --single-branch ~/pylint
 
 install:
   - pip install tox-travis
-  - pip install -e .[for_tests]
 script:
   - |
         if [ -z "$SANITY_CHECK" ]; then

pylint_django/tests/input/func_noerror_foreign_key_package.py

@@ -7,7 +7,7 @@
 
 
 class Book(models.Model):
-    author = models.ForeignKey(to='input.Author', on_delete=models.CASCADE)
+    author = models.ForeignKey(to='pylint_django.tests.input.Author', on_delete=models.CASCADE)
 
     def get_author_name(self):
         return self.author.id

pylint_django/tests/test_func.py

@@ -5,15 +5,43 @@
 
 import pylint
 
-if pylint.__version__ >= '2.4':
-    # after version 2.4 pylint stopped shipping the test directory
-    # as part of the package so we check it out locally for testing
-    sys.path.append(os.path.join(os.getenv('HOME', '/home/travis'), 'pylint', 'tests'))
-else:
-    # because there's no __init__ file in pylint/test/
-    sys.path.append(os.path.join(os.path.dirname(pylint.__file__), 'test'))
-
-import test_functional  # noqa: E402
+# PYLINT_TEST_FUNCTIONAL_PATH is can be used to force where to pull the classes used for functional testing
+# Just make sure you use the exact same version as the one pylint version installed or just add that to PYTHONPATH
+pylint_test_func_path = os.getenv('PYLINT_TEST_FUNCTIONAL_PATH', '')
+if pylint_test_func_path != '':
+    sys.path.append(pylint_test_func_path)
+
+# test_functional has been moved to pylint.testutils as part of the pytlint 2.5 release
+# so we try first to load the basic cases and then look in more exotic places
+try:
+    # pylint <= 2.4 case
+    # TODO: remove when the minimum supported version of pylint is 2.5.
+    from test_functional import FunctionalTestFile, LintModuleTest  # noqa: E402
+except (ImportError, AttributeError):
+    try:
+        from pylint.testutils import FunctionalTestFile, LintModuleTest
+    except (ImportError, AttributeError):
+        # test in other more exotic directories
+        if os.path.isdir(os.path.join(os.path.dirname(pylint.__file__), 'test')):
+            # pre pylint 2.4, pylint was putting files in pylint/test
+            # TODO: remove when the minimum supported version of pylint is 2.4.
+            sys.path.append(os.path.join(os.path.dirname(pylint.__file__), 'test'))
+        elif os.path.isdir(os.path.join(os.path.dirname(pylint.__file__), '..', 'tests')):
+            # after version 2.4 pylint moved the test to a 'tests' folder at the root of the github tree
+            # to stopped shipping the tests along with the pip package
+            # but some distro re-add tests in the packages so only do that when not done at all
+            # TODO: remove when the minimum supported version of pylint is 2.5.
+            sys.path.append(os.path.join(os.path.dirname(pylint.__file__), '..', 'tests'))
+        else:
+            # This is a transitional hack specific to pylint 2.4 on travis and should be irrelevant anywhere else
+            # TODO: remove when the minimum supported version of pylint is 2.5.
+            sys.path.append(os.path.join(os.getenv('HOME', '/home/travis'), 'pylint', 'tests'))
+        try:
+            # TODO: remove when the minimum supported version of pylint is 2.5.
+            sys.path.append(os.path.join(os.path.dirname(pylint.__file__), 'test'))
+            from test_functional import FunctionalTestFile, LintModuleTest  # noqa: E402
+        except (ImportError, AttributeError):
+            from pylint.testutils import FunctionalTestFile, LintModuleTest
 
 # alter sys.path again because the tests now live as a subdirectory
 # of pylint_django
@@ -22,7 +50,7 @@
 sys.path.append(os.path.join(os.path.dirname(__file__), 'input'))
 
 
-class PylintDjangoLintModuleTest(test_functional.LintModuleTest):
+class PylintDjangoLintModuleTest(LintModuleTest):
     """
         Only used so that we can load this plugin into the linter!
     """
@@ -53,7 +81,7 @@ def _file_name(test):
     suite = []
     for fname in os.listdir(input_dir):
         if fname != '__init__.py' and fname.endswith('.py'):
-            suite.append(test_functional.FunctionalTestFile(input_dir, fname))
+            suite.append(FunctionalTestFile(input_dir, fname))
 
     # when testing the db_performance plugin we need to sort by input file name
     # because the plugin reports the errors in close() which appends them to the

tox.ini

@@ -24,18 +24,30 @@ commands =
     clean: find . -type d -name __pycache__ -delete
     clean: rm -rf build/ .cache/ dist/ .eggs/ pylint_django.egg-info/ .tox/
 deps =
+    coverage
+    factory-boy
+    pytest
     django_is_installed: Django
+    django_is_installed: pylint-plugin-utils
+    django_is_installed: pylint
     flake8: flake8
     pylint: pylint
+    pylint: pylint-plugin-utils
     pylint: Django
+    pylint: django_tables2
     readme: twine
     django111: Django>=1.11,<2.0
     django20: Django>=2.0,<2.1
     django21: Django>=2.1,<2.2
     django22: Django>=2.2,<3.0
+    django{111,20,21,22}: pylint
+    django{111,20,21,22}: pylint-plugin-utils
+    django{111,20,21,22}: django_tables2
     django-master: Django
+    django-master: django_tables2
     django-master: git+https://github.com/pycqa/astroid@master
     django-master: git+https://github.com/pycqa/pylint@master
+    django-master: git+https://github.com/PyCQA/pylint-plugin-utils@master
 setenv =
     PIP_DISABLE_PIP_VERSION_CHECK = 1
     PYTHONPATH = .