Move to importlib-metadata for performance improvements (#1324)

Author: asottile

docs/changelog/1324.feature.rst

@@ -0,0 +1 @@
+Replace ``pkg_resources`` with ``importlib_metadata`` for speed - by :user:`asottile`.

setup.cfg

@@ -38,8 +38,9 @@ classifiers =
 packages = find:
 python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
 install_requires =
-    setuptools  >= 30.0.0
-    pluggy >= 0.3.0, <1
+    importlib-metadata
+    packaging
+    pluggy >= 0.12.0, <1
     py >= 1.4.17, <2
     six >= 1.0.0, <2
     virtualenv >= 14.0.0

src/tox/config/__init__.py

@@ -15,13 +15,16 @@
 from subprocess import list2cmdline
 from threading import Thread
 
-import pkg_resources
+import importlib_metadata
 import pluggy
 import py
 import toml
+from packaging import requirements
+from packaging.utils import canonicalize_name
 
 import tox
 from tox.constants import INFO
+from tox.exception import MissingDependency
 from tox.interpreters import Interpreters, NoInterpreterInfo
 from tox.reporter import (
     REPORTER_TIMESTAMP_ON_ENV,
@@ -192,10 +195,10 @@ def _cut_off_dep_comment(name):
     @classmethod
     def _is_same_dep(cls, dep1, dep2):
         """Definitions are the same if they refer to the same package, even if versions differ."""
-        dep1_name = pkg_resources.Requirement.parse(dep1).project_name
+        dep1_name = canonicalize_name(requirements.Requirement(dep1).name)
         try:
-            dep2_name = pkg_resources.Requirement.parse(dep2).project_name
-        except pkg_resources.RequirementParseError:
+            dep2_name = canonicalize_name(requirements.Requirement(dep2).name)
+        except requirements.InvalidRequirement:
             # we couldn't parse a version, probably a URL
             return False
         return dep1_name == dep2_name
@@ -1133,17 +1136,20 @@ def ensure_requires_satisfied(config, requires, min_version):
         for require in requires + [min_version]:
             # noinspection PyBroadException
             try:
-                package = pkg_resources.Requirement.parse(require)
-                if package.project_name not in exists:
+                package = requirements.Requirement(require)
+                package_name = canonicalize_name(package.name)
+                if package_name not in exists:
                     deps.append(DepConfig(require, None))
-                    exists.add(package.project_name)
-                    pkg_resources.get_distribution(package)
-            except pkg_resources.RequirementParseError as exception:
+                    exists.add(package_name)
+                    dist = importlib_metadata.distribution(package_name)
+                    if not package.specifier.contains(dist.version, prereleases=True):
+                        raise MissingDependency(package)
+            except requirements.InvalidRequirement as exception:
                 failed_to_parse = True
                 error("failed to parse {!r}".format(exception))
             except Exception as exception:
                 verbosity1("could not satisfy requires {!r}".format(exception))
-                missing_requirements.append(str(pkg_resources.Requirement(require)))
+                missing_requirements.append(str(requirements.Requirement(require)))
         if failed_to_parse:
             raise tox.exception.BadRequirement()
         config.run_provision = bool(len(missing_requirements))

src/tox/package/builder/isolated.py

@@ -3,8 +3,9 @@
 import json
 from collections import namedtuple
 
-import pkg_resources
 import six
+from packaging.requirements import Requirement
+from packaging.utils import canonicalize_name
 
 from tox import reporter
 from tox.config import DepConfig, get_py_project_toml
@@ -31,11 +32,11 @@ def build(config, session):
 
     build_requires = get_build_requires(build_info, package_venv, config.setupdir)
     # we need to filter out requirements already specified in pyproject.toml or user deps
-    base_build_deps = {pkg_resources.Requirement(r.name).key for r in package_venv.envconfig.deps}
+    base_build_deps = {canonicalize_name(r.name) for r in package_venv.envconfig.deps}
     build_requires_dep = [
         DepConfig(r, None)
         for r in build_requires
-        if pkg_resources.Requirement(r).key not in base_build_deps
+        if canonicalize_name(Requirement(r).name) not in base_build_deps
     ]
     if build_requires_dep:
         with package_venv.new_action("build_requires", package_venv.envconfig.envdir) as action:

src/tox/package/local.py

@@ -1,7 +1,7 @@
 import os
 import re
 
-import pkg_resources
+import packaging.version
 import py
 
 import tox
@@ -58,7 +58,6 @@ def get_version_from_filename(basename):
         return None
     version = m.group(1)
     try:
-
-        return pkg_resources.packaging.version.Version(version)
-    except pkg_resources.packaging.version.InvalidVersion:
+        return packaging.version.Version(version)
+    except packaging.version.InvalidVersion:
         return None

src/tox/session/commands/show_config.py

@@ -1,6 +1,9 @@
 import sys
 from collections import OrderedDict
 
+import importlib_metadata
+from packaging.requirements import Requirement
+from packaging.utils import canonicalize_name
 from six import StringIO
 from six.moves import configparser
 
@@ -57,17 +60,21 @@ def tox_info(config, parser):
 
 
 def version_info(parser):
-    import pkg_resources
-
     versions = OrderedDict()
-    visited = set()
     to_visit = {"tox"}
     while to_visit:
         current = to_visit.pop()
-        visited.add(current)
-        current_dist = pkg_resources.get_distribution(current)
-        to_visit.update(i.name for i in current_dist.requires() if i.name not in visited)
-        versions[current] = current_dist.version
+        current_dist = importlib_metadata.distribution(current)
+        current_name = canonicalize_name(current_dist.metadata["name"])
+        versions[current_name] = current_dist.version
+        if current_dist.requires is not None:
+            for require in current_dist.requires:
+                pkg = Requirement(require)
+                pkg_name = canonicalize_name(pkg.name)
+                if (
+                    pkg.marker is None or pkg.marker.evaluate({"extra": ""})
+                ) and pkg_name not in versions:
+                    to_visit.add(pkg_name)
     set_section(parser, "tox:versions", versions)
 
 

src/tox/venv.py

@@ -7,7 +7,6 @@
 from itertools import chain
 
 import py
-from pkg_resources import to_filename
 
 import tox
 from tox import reporter
@@ -324,7 +323,7 @@ def _needs_reinstall(self, setupdir, action):
             sys_path = json.loads(out)
         except ValueError:
             sys_path = []
-        egg_info_fname = ".".join((to_filename(name), "egg-info"))
+        egg_info_fname = ".".join((name.replace("-", "_"), "egg-info"))
         for d in reversed(sys_path):
             egg_info = py.path.local(d).join(egg_info_fname)
             if egg_info.check():

tests/unit/session/test_provision.py

@@ -95,7 +95,7 @@ def test_provision_bad_requires(newconfig, capsys, monkeypatch):
         """,
         )
     out, err = capsys.readouterr()
-    assert "ERROR: failed to parse RequirementParseError" in out
+    assert "ERROR: failed to parse InvalidRequirement" in out
     assert not err
 
 

tox.ini

@@ -139,7 +139,7 @@ include_trailing_comma = True
 force_grid_wrap = 0
 line_length = 99
 known_first_party = tox,tests
-known_third_party = apiclient,docutils,filelock,flaky,freezegun,git,httplib2,oauth2client,packaging,pathlib2,pkg_resources,pluggy,py,pytest,setuptools,six,sphinx,toml
+known_third_party = apiclient,docutils,filelock,flaky,freezegun,git,httplib2,importlib_metadata,oauth2client,packaging,pathlib2,pluggy,py,pytest,setuptools,six,sphinx,toml
 
 [testenv:release]
 description = do a release, required posarg of the version number