Support PEP-621

.github/workflows/check.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     steps:
@@ -39,8 +39,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10"]
-        os: [ubuntu-18.04]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3

.python-version

@@ -1 +1 @@
-3.11
+3.7

README.md

@@ -13,16 +13,16 @@ Install creosote in separate virtual environment (using e.g. [pipx](https://gith
 pipx install creosote
 ```
 
-Scan virtual environment for unused packages:
+Scan virtual environment for unused packages ([PEP-621](https://peps.python.org/pep-0621/) example below, but [Poetry](https://python-poetry.org/) and `requirements.txt` files are also supported):
 
 ```bash
-creosote --deps-file pyproject.toml --venv .venv --paths src
+creosote --venv .venv --paths src --deps-file pyproject.toml --sections project.dependencies
 ```
 
-Example output:
+Example output (using Poetry dependency definition):
 
 ```bash
-$ creosote
+$ creosote --venv .venv --paths src --deps-file pyproject.toml --sections tool.poetry.dependencies
 Parsing src/creosote/formatters.py
 Parsing src/creosote/models.py
 Parsing src/creosote/resolvers.py
@@ -45,9 +45,10 @@ creosote --help
 
 Some data is required as input:
 
-- A list of package names (fetched from e.g. `pyproject.toml`, `requirements_*.txt|.in`).
-- The path to the virtual environment.
-- The path to one or more Python files (or a folder containing such files).
+- The path to the virtual environment (`--venv`).
+- The path to one or more Python files, or a folder containing such files (`--paths`).
+- A list of package names, fetched from e.g. `pyproject.toml`, `requirements_*.txt|.in` (`--deps-file`).
+- One or more toml sections to parse, e.g. `project.dependencies` (`--sections`).
 
 The creosote tool will first scan the given python file(s) for all its imports. Then it fetches all package names (from the dependencies spec file). Finally, all imports are associated with their corresponding package name (requires the virtual environment for resolving). If a package does not have any imports associated, it will be considered to be unused.
 
@@ -72,9 +73,25 @@ Yes, kind of. There is no way to tell which part of `requirements.txt` specifies
 
 If you are using [pip-tools](https://github.com/jazzband/pip-tools), you can provide a `*.in` file.
 
-### Can I scan for pyproject's dev-dependencies?
+### Can I scan for PEP-621 dependencies?
 
-Yes! For `pyproject.toml`, just provide the `--dev` argument.
+Yes! Just provide `-sections project.dependencies`.
+
+### Can I scan for PEP-621 optional dependencies?
+
+Yes! Just provide `-sections project.optional-dependencies.<GROUP>` where `<GROUP>` is your dependency group name, e.g. `-s project.optional-dependencies.lint`.
+
+### Can I scan for Poetry's dev-dependencies?
+
+Yes! Just provide `-sections tool.poetry.dev-dependencies`.
+
+### Can I scan for Poetry's dependency groups?
+
+Yes! Just provide `-sections tool.poetry.group.<GROUP>.dependencies` where `<GROUP>` is your dependency group, e.g. `-s tool.poetry.group.lint.dependencies`.
+
+### Can I scan for multiple toml sections?
+
+Yes! Just provide each section after the `--sections` parameter, e.g. `--sections project.optional-dependencies.test project.optional-dependencies.lint`.
 
 ### Can I use this as a GitHub Action?
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "creosote"
-version = "1.0.3"
+version = "2.0.0"
 description = 'Identify unused dependencies and avoid a bloated virtual environment.'
 readme = "README.md"
 requires-python = ">=3.7"
@@ -26,6 +26,7 @@ classifiers = [
 ]
 dependencies = [
     "distlib>=0.3.4,<0.4",
+    "dotty-dict>=1.3.1,<1.4",
     "loguru>=0.6.0,<0.7",
     "toml>=0.10.2,<0.11",
 ]
@@ -104,6 +105,7 @@ python_version = "3.10"
 [[tool.mypy.overrides]]
 module = [
     "distlib.*",
+    "dotty_dict.*",
 ]
 ignore_missing_imports = true
 

src/creosote/cli.py

@@ -51,21 +51,21 @@ def parse_args(args):
         dest="deps_file",
         metavar="PATH",
         default="pyproject.toml",
-        help="path to the pyproject.toml, *.txt or *.in dependencies file",
+        help="path to the pyproject.toml, requirements[.txt|.in] file",
     )
 
     parser.add_argument(
-        "--dev",
-        dest="dev",
-        action="store_true",
-        help="scan dev dependencies instead of prod dependencies",
+        "-s",
+        "--sections",
+        dest="sections",
+        metavar="SECTION",
+        nargs="*",
+        default=["project.dependencies"],
+        help="pyproject.toml section(s) to scan for dependencies",
     )
 
     parsed_args = parser.parse_args(args)
 
-    if "pyproject.toml" not in parsed_args.deps_file and parsed_args.dev:
-        raise Exception("Option --dev must be used with pyproject.toml")
-
     return parsed_args
 
 
@@ -81,7 +81,7 @@ def main(args_=None):
 
     logger.info(f"Parsing {args.deps_file} for packages")
     deps_reader = parsers.PackageReader()
-    deps_reader.read(args.deps_file, args.dev)
+    deps_reader.read(args.deps_file, args.sections)
 
     logger.info("Resolving...")
     deps_resolver = resolvers.DepsResolver(

src/creosote/parsers.py

@@ -1,8 +1,10 @@
 import ast
 import pathlib
+import re
 from functools import lru_cache
 
 import toml
+from dotty_dict import dotty
 from loguru import logger
 
 from creosote.models import Import, Package
@@ -12,20 +14,52 @@ class PackageReader:
     def __init__(self):
         self.packages = None
 
-    def _pyproject(self, deps_file: str, dev: bool):
+    def _pyproject_pep621(self, section_contents: dict):
+        if not isinstance(section_contents, list):
+            raise TypeError("Unexpected dependency format, list expected.")
+
+        section_deps = []
+        for dep in section_contents:
+            parsed_dep = self.dependency_without_version_constraint(dep)
+            section_deps.append(parsed_dep)
+        return section_deps
+
+    def _pyproject_poetry(self, section_contents: dict):
+        if not isinstance(section_contents, dict):
+            raise TypeError("Unexpected dependency format, dict expected.")
+        return section_contents.keys()
+
+    def _pyproject(self, deps_file: str, sections: list):
         """Return dependencies from pyproject.toml."""
         with open(deps_file, "r") as infile:
             contents = toml.loads(infile.read())
 
-        try:
-            if dev:
-                deps = contents["tool"]["poetry"]["dev-dependencies"]
+        dotty_contents = dotty(contents)
+        deps = []
+
+        for section in sections:
+            try:
+                section_contents = dotty_contents[section]
+            except KeyError as err:
+                raise KeyError(f"Could not find toml section {section}.") from err
+            section_deps = []
+
+            if section.startswith("project"):
+                section_deps = self._pyproject_pep621(section_contents)
+            elif section.startswith("tool.poetry"):
+                section_deps = self._pyproject_poetry(section_contents)
             else:
-                deps = contents["tool"]["poetry"]["dependencies"]
-        except KeyError as e:
-            raise Exception("Could not find expected toml property.") from e
+                raise TypeError("Unsupported dependency format.")
 
-        return sorted(deps.keys())
+            if not section_deps:
+                logger.warning(f"No dependencies found in section {section}")
+            else:
+                logger.info(
+                    f"Dependencies found in {section}: {', '.join(section_deps)}"
+                )
+                deps.extend(section_deps)
+
+        return sorted(deps)
 
     def _requirements(self, deps_file: str):
         """Return dependencies from requirements.txt-format file."""
@@ -35,10 +69,17 @@ def _requirements(self, deps_file: str):
 
         for line in contents:
             if not line.startswith(" "):
-                deps.append(line[: line.find("=")])
+                dep = self.dependency_without_version_constraint(line)
+                deps.append(dep)
 
         return sorted(deps)
 
+    def dependency_without_version_constraint(self, dependency_string: str):
+        match = re.match(r"([\w\-\_]*)[>=|==|>=]*", dependency_string)
+        if match and match.groups():
+            dep = match.groups()[0]
+            return dep
+
     @lru_cache(maxsize=None)  # noqa: B019
     def ignore_packages(self):
         return ["python"]
@@ -50,12 +91,14 @@ def packages_sans_ignored(self, deps):
                 packages.append(Package(name=dep))
         return packages
 
-    def read(self, deps_file: str, dev: bool):
+    def read(self, deps_file: str, sections: list):
         if not pathlib.Path(deps_file).exists():
             raise Exception(f"File {deps_file} does not exist")
 
         if "pyproject.toml" in deps_file:
-            self.packages = self.packages_sans_ignored(self._pyproject(deps_file, dev))
+            self.packages = self.packages_sans_ignored(
+                self._pyproject(deps_file, sections)
+            )
         elif deps_file.endswith(".txt") or deps_file.endswith(".in"):
             self.packages = self.packages_sans_ignored(self._requirements(deps_file))
         else:

src/creosote/resolvers.py

@@ -63,7 +63,7 @@ def package_to_module(self, package: Package):
 
     def associate_imports_with_package(self, package: Package, name: str):
         for imp in self.imports.copy():
-            if not imp.module and name in imp.name:
+            if not imp.module and name in imp.name:  # noqa: SIM114
                 # import <imp.name>
                 package.associated_imports.append(imp)
             elif imp.name and name in imp.module:

tests/conftest.py

@@ -4,15 +4,15 @@
 
 
 @pytest.fixture
-def with_packages(capsys, request):
+def with_poetry_packages(capsys, request):
     with capsys.disabled():
         subprocess.run(
-            ["poetry", "add"] + request.param,
+            ["poetry", "add", *request.param],
             stdout=subprocess.DEVNULL,
         )
     yield
     with capsys.disabled():
         subprocess.run(
-            ["poetry", "remove"] + request.param,
+            ["poetry", "remove", *request.param],
             stdout=subprocess.DEVNULL,
         )

tests/test_cli.py

@@ -3,20 +3,42 @@
 from creosote import cli
 
 
-@pytest.mark.parametrize("with_packages", [["PyYAML", "protobuf"]], indirect=True)
-def test_files_as_path(capsys, with_packages):
-
+@pytest.mark.parametrize(
+    "with_poetry_packages", [["PyYAML", "protobuf"]], indirect=True
+)
+def test_files_as_path_poetry(capsys, with_poetry_packages):
     cli.main(
         [
             "-p",
             "src/creosote/formatters.py",
             "src/creosote/models.py",
             "-f",
             "porcelain",
+            "-s",
+            "tool.poetry.dependencies",
         ]
     )
 
     captured = capsys.readouterr()
     expected_log = "distlib\nprotobuf\npyyaml\ntoml\n"
 
     assert captured.out == expected_log
+
+
+def test_files_as_path_pep621(capsys):
+    cli.main(
+        [
+            "-p",
+            "src/creosote/formatters.py",
+            "src/creosote/models.py",
+            "-f",
+            "porcelain",
+            "-s",
+            "project.dependencies",
+        ]
+    )
+
+    captured = capsys.readouterr()
+    expected_log = "distlib\ndotty-dict\ntoml\n"
+
+    assert captured.out == expected_log

tests/test_formats.py

@@ -3,10 +3,11 @@
 from creosote import cli
 
 
-@pytest.mark.parametrize("with_packages", [["PyYAML", "protobuf"]], indirect=True)
-def test_format_porcelain(capsys, with_packages):
-
-    cli.main(["-f", "porcelain"])
+@pytest.mark.parametrize(
+    "with_poetry_packages", [["PyYAML", "protobuf"]], indirect=True
+)
+def test_format_porcelain(capsys, with_poetry_packages):
+    cli.main(["-f", "porcelain", "-s", "tool.poetry.dependencies"])
 
     captured = capsys.readouterr()
     expected_log = "protobuf\npyyaml\n"