From c9fcdf8bc997bc417689542f1d7177bd6ab2012d Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Sat, 30 Mar 2024 12:27:34 +0100 Subject: [PATCH 1/3] feat(visitor): skip type checking blocks --- docs/usage.md | 24 ++++++++++++++++ src/visitor.rs | 46 ++++++++++++++++++++++++++++-- tests/data/some_imports.py | 11 ++++++- tests/unit/imports/test_extract.py | 34 ++++++++++++---------- 4 files changed, 97 insertions(+), 18 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 263edc12..25ba1265 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -38,6 +38,30 @@ To determine the project's dependencies, _deptry_ will scan the directory it is _deptry_ can be configured to look for `pip` requirements files with other names or in other directories. See [Requirements files](#requirements-files) and [Requirements files dev](#requirements-files-dev). +## Imports extraction + +_deptry_ will search for imports in Python files (`*.py`, and `*.ipynb` unless [`--ignore-notebooks`](#ignore-notebooks) +is set) that are not part of excluded files. + +Imports will be extracted regardless of where they are made in a file (top-level, functions, class methods, guarded by +conditions, ...). + +The only exception is imports that are guarded +by [`TYPE_CHECKING`](https://docs.python.org/3/library/typing.html#typing.TYPE_CHECKING) when +using `from __future__ import annotations`, in accordance with [PEP 563](https://peps.python.org/pep-0563/). In this +specific case, _deptry_ will not extract those imports, as they are not considered as problematic. For instance: + +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # This import will not be extracted, as it is guarded by `TYPE_CHECKING`, and `from __future__ import annotations` + # is used. This makes the import not evaluated during runtime, and meant to only be evaluated by type checkers. + import mypy_boto3_s3 +``` + ## Excluding files and directories To determine issues with imported modules and dependencies, _deptry_ will scan the working directory and its subdirectories recursively for `.py` and `.ipynb` files, so it can diff --git a/src/visitor.rs b/src/visitor.rs index c3c0e429..3c73bf2e 100644 --- a/src/visitor.rs +++ b/src/visitor.rs @@ -1,17 +1,19 @@ use ruff_python_ast::visitor::{walk_stmt, Visitor}; -use ruff_python_ast::{self, Stmt}; +use ruff_python_ast::{self, Expr, ExprAttribute, ExprName, Stmt, StmtIf, StmtImportFrom}; use ruff_text_size::TextRange; use std::collections::HashMap; #[derive(Debug, Clone)] pub struct ImportVisitor { imports: HashMap>, + has_future_annotations: bool, } impl ImportVisitor { pub fn new() -> Self { Self { imports: HashMap::new(), + has_future_annotations: false, } } @@ -35,7 +37,10 @@ impl<'a> Visitor<'a> for ImportVisitor { Stmt::ImportFrom(import_from_stmt) => { if let Some(module) = &import_from_stmt.module { if import_from_stmt.level == Some(0) { - // Assuming Int::new(0) is comparable with 0 + if is_future_annotations_import(module.as_str(), import_from_stmt) { + self.has_future_annotations = true; + } + self.imports .entry(get_top_level_module_name(module.as_str())) .or_default() @@ -43,6 +48,9 @@ impl<'a> Visitor<'a> for ImportVisitor { } } } + Stmt::If(if_stmt) if is_typing_only_block(self.has_future_annotations, if_stmt) => { + // Avoid parsing imports that are only evaluated by type checkers. + } _ => walk_stmt(self, stmt), // Delegate other statements to walk_stmt } } @@ -57,3 +65,37 @@ fn get_top_level_module_name(module_name: &str) -> String { .unwrap_or(module_name) .to_owned() } + +/// Checks if the import is a `from __future__ import annotations` one. +fn is_future_annotations_import(module: &str, import_from_stmt: &StmtImportFrom) -> bool { + return module == "__future__" + && import_from_stmt + .names + .iter() + .any(|alias| alias.name.as_str() == "annotations"); +} + +/// Checks if we are in a block that will only be evaluated by type checkers, in accordance with +/// . If no `__future__.annotations` import is made, a block using `TYPE_CHECKING` +/// will be evaluated at runtime, so we should not consider that this is a typing only block in that case. +fn is_typing_only_block(has_future_annotations: bool, if_stmt: &StmtIf) -> bool { + if has_future_annotations { + match &if_stmt.test.as_ref() { + Expr::Attribute(ExprAttribute { value, attr, .. }) => { + if let Expr::Name(ExprName { id, .. }) = value.as_ref() { + if id.as_str() == "typing" && attr.as_str() == "TYPE_CHECKING" { + return true; + } + } + } + Expr::Name(ExprName { id, .. }) => { + if id == "TYPE_CHECKING" { + return true; + } + } + _ => (), + } + } + + false +} diff --git a/tests/data/some_imports.py b/tests/data/some_imports.py index 43f270b0..ecf191f9 100644 --- a/tests/data/some_imports.py +++ b/tests/data/some_imports.py @@ -1,6 +1,9 @@ +from __future__ import annotations + +import typing from os import chdir, walk from pathlib import Path -from typing import List +from typing import List, TYPE_CHECKING import numpy as np import pandas @@ -20,6 +23,12 @@ import barfoo as bf from randomizer import random +if TYPE_CHECKING: + import mypy_boto3_s3 + +if typing.TYPE_CHECKING: + import mypy_boto3_sagemaker + try: import click except: diff --git a/tests/unit/imports/test_extract.py b/tests/unit/imports/test_extract.py index 06888a45..5e9dc73d 100644 --- a/tests/unit/imports/test_extract.py +++ b/tests/unit/imports/test_extract.py @@ -21,23 +21,27 @@ def test_import_parser_py() -> None: some_imports_path = Path("tests/data/some_imports.py") assert get_imported_modules_from_list_of_files([some_imports_path]) == { - "barfoo": [Location(some_imports_path, 20, 8)], - "baz": [Location(some_imports_path, 16, 5)], - "click": [Location(some_imports_path, 24, 12)], - "foobar": [Location(some_imports_path, 18, 12)], - "httpx": [Location(some_imports_path, 14, 12)], - "module_in_class": [Location(some_imports_path, 35, 16)], - "module_in_func": [Location(some_imports_path, 30, 12)], - "not_click": [Location(some_imports_path, 26, 12)], + "__future__": [Location(some_imports_path, 1, 1)], + "barfoo": [Location(some_imports_path, 23, 8)], + "baz": [Location(some_imports_path, 19, 5)], + "click": [Location(some_imports_path, 33, 12)], + "foobar": [Location(some_imports_path, 21, 12)], + "httpx": [Location(some_imports_path, 17, 12)], + "module_in_class": [Location(some_imports_path, 44, 16)], + "module_in_func": [Location(some_imports_path, 39, 12)], + "not_click": [Location(some_imports_path, 35, 12)], "numpy": [ - Location(some_imports_path, 5, 8), - Location(some_imports_path, 7, 1), + Location(some_imports_path, 8, 8), + Location(some_imports_path, 10, 1), + ], + "os": [Location(some_imports_path, 4, 1)], + "pandas": [Location(some_imports_path, 9, 8)], + "pathlib": [Location(some_imports_path, 5, 1)], + "randomizer": [Location(some_imports_path, 24, 1)], + "typing": [ + Location(some_imports_path, 3, 8), + Location(some_imports_path, 6, 1), ], - "os": [Location(some_imports_path, 1, 1)], - "pandas": [Location(some_imports_path, 6, 8)], - "pathlib": [Location(some_imports_path, 2, 1)], - "randomizer": [Location(some_imports_path, 21, 1)], - "typing": [Location(some_imports_path, 3, 1)], } From f9332f5897fb8528a7e995ec3e60ac9e6036568a Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Sun, 31 Mar 2024 22:59:42 +0200 Subject: [PATCH 2/3] Update docs/usage.md Co-authored-by: Florian Maas --- docs/usage.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 25ba1265..bddaaba8 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -57,8 +57,8 @@ from __future__ import annotations from typing import TYPE_CHECKING if TYPE_CHECKING: - # This import will not be extracted, as it is guarded by `TYPE_CHECKING`, and `from __future__ import annotations` - # is used. This makes the import not evaluated during runtime, and meant to only be evaluated by type checkers. + # This import will not be extracted as it is guarded by `TYPE_CHECKING` and `from __future__ import annotations` + # is used. This means the import should only be evaluated by type checkers, and should not be evaluated during runtime. import mypy_boto3_s3 ``` From 5d4ad37ea3b33270aa8d4e9816b64661c9661735 Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Sun, 31 Mar 2024 22:59:49 +0200 Subject: [PATCH 3/3] Update docs/usage.md Co-authored-by: Florian Maas --- docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index bddaaba8..4d0e1971 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -49,7 +49,7 @@ conditions, ...). The only exception is imports that are guarded by [`TYPE_CHECKING`](https://docs.python.org/3/library/typing.html#typing.TYPE_CHECKING) when using `from __future__ import annotations`, in accordance with [PEP 563](https://peps.python.org/pep-0563/). In this -specific case, _deptry_ will not extract those imports, as they are not considered as problematic. For instance: +specific case, _deptry_ will not extract those imports, as they are not considered problematic. For instance: ```python from __future__ import annotations