Introduce a ruff_python_semantic crate (#3865)

Cargo.toml

@@ -24,6 +24,7 @@ is-macro = { version = "0.2.2" }
 itertools = { version = "0.10.5" }
 libcst = { git = "https://github.com/charliermarsh/LibCST", rev = "80e4c1399f95e5beb532fdd1e209ad2dbb470438" }
 log = { version = "0.4.17" }
+nohash-hasher = { version = "0.2.0" }
 once_cell = { version = "1.17.1" }
 path-absolutize = { version = "3.0.14" }
 proc-macro2 = { version = "1.0.51" }
@@ -40,6 +41,7 @@ serde = { version = "1.0.152", features = ["derive"] }
 serde_json = { version = "1.0.93", features = ["preserve_order"] }
 shellexpand = { version = "3.0.0" }
 similar = { version = "2.2.1" }
+smallvec = { version = "1.10.0" }
 strum = { version = "0.24.1", features = ["strum_macros"] }
 strum_macros = { version = "0.24.3" }
 syn = { version = "1.0.109" }

crates/ruff/Cargo.toml

@@ -18,6 +18,7 @@ ruff_cache = { path = "../ruff_cache" }
 ruff_diagnostics = { path = "../ruff_diagnostics", features = ["serde"] }
 ruff_macros = { path = "../ruff_macros" }
 ruff_python_ast = { path = "../ruff_python_ast" }
+ruff_python_semantic = { path = "../ruff_python_semantic" }
 ruff_python_stdlib = { path = "../ruff_python_stdlib" }
 ruff_rustpython = { path = "../ruff_rustpython" }
 
@@ -37,7 +38,7 @@ itertools = { workspace = true }
 libcst = { workspace = true }
 log = { workspace = true }
 natord = { version = "1.0.9" }
-nohash-hasher = { version = "0.2.0" }
+nohash-hasher = { workspace = true }
 num-bigint = { version = "0.4.3" }
 num-traits = { version = "0.2.15" }
 once_cell = { workspace = true }
@@ -57,7 +58,7 @@ semver = { version = "1.0.16" }
 serde = { workspace = true }
 serde_json = { workspace = true }
 shellexpand = { workspace = true }
-smallvec = { version = "1.10.0" }
+smallvec = { workspace = true }
 strum = { workspace = true }
 strum_macros = { workspace = true }
 textwrap = { workspace = true }

crates/ruff/src/autofix/actions.rs

@@ -8,12 +8,12 @@ use rustpython_parser::ast::{ExcepthandlerKind, Expr, Keyword, Location, Stmt, S
 use rustpython_parser::{lexer, Mode, Tok};
 
 use ruff_diagnostics::Edit;
-use ruff_python_ast::context::Context;
 use ruff_python_ast::helpers;
 use ruff_python_ast::helpers::to_absolute;
 use ruff_python_ast::imports::{AnyImport, Import};
 use ruff_python_ast::newlines::NewlineWithTrailingNewline;
 use ruff_python_ast::source_code::{Indexer, Locator, Stylist};
+use ruff_python_semantic::context::Context;
 
 use crate::cst::helpers::compose_module_path;
 use crate::cst::matchers::match_module;

crates/ruff/src/checkers/ast/deferred.rs

@@ -1,9 +1,9 @@
-use ruff_python_ast::scope::ScopeStack;
+use ruff_python_semantic::scope::ScopeStack;
 use rustpython_parser::ast::{Expr, Stmt};
 
 use ruff_python_ast::types::Range;
 use ruff_python_ast::types::RefEquality;
-use ruff_python_ast::visibility::{Visibility, VisibleScope};
+use ruff_python_semantic::analyze::visibility::{Visibility, VisibleScope};
 
 use crate::checkers::ast::AnnotationContext;
 use crate::docstrings::definition::Definition;

crates/ruff/src/checkers/ast/mod.rs

@@ -14,22 +14,22 @@ use rustpython_parser::ast::{
 
 use ruff_diagnostics::Diagnostic;
 use ruff_python_ast::all::{extract_all_names, AllNamesFlags};
-use ruff_python_ast::binding::{
+use ruff_python_ast::helpers::{extract_handled_exceptions, to_module_path};
+use ruff_python_ast::source_code::{Indexer, Locator, Stylist};
+use ruff_python_ast::types::{Node, Range, RefEquality};
+use ruff_python_ast::typing::parse_type_annotation;
+use ruff_python_ast::visitor::{walk_excepthandler, walk_pattern, Visitor};
+use ruff_python_ast::{branch_detection, cast, helpers, str, visitor};
+use ruff_python_semantic::analyze;
+use ruff_python_semantic::analyze::typing::{Callable, SubscriptKind};
+use ruff_python_semantic::binding::{
     Binding, BindingId, BindingKind, Exceptions, ExecutionContext, Export, FromImportation,
     Importation, StarImportation, SubmoduleImportation,
 };
-use ruff_python_ast::context::Context;
-use ruff_python_ast::helpers::{extract_handled_exceptions, to_module_path};
-use ruff_python_ast::scope::{
+use ruff_python_semantic::context::Context;
+use ruff_python_semantic::scope::{
     ClassDef, FunctionDef, Lambda, Scope, ScopeId, ScopeKind, ScopeStack,
 };
-use ruff_python_ast::source_code::{Indexer, Locator, Stylist};
-use ruff_python_ast::types::{Node, Range, RefEquality};
-use ruff_python_ast::typing::{
-    match_annotated_subscript, parse_type_annotation, Callable, SubscriptKind,
-};
-use ruff_python_ast::visitor::{walk_excepthandler, walk_pattern, Visitor};
-use ruff_python_ast::{branch_detection, cast, helpers, str, typing, visibility, visitor};
 use ruff_python_stdlib::builtins::{BUILTINS, MAGIC_GLOBALS};
 use ruff_python_stdlib::path::is_python_stub_file;
 
@@ -2248,7 +2248,7 @@ where
                                 || (self.settings.target_version >= PythonVersion::Py37
                                     && self.ctx.annotations_future_enabled
                                     && self.ctx.in_annotation))
-                            && typing::is_pep585_builtin(expr, &self.ctx)
+                            && analyze::typing::is_pep585_builtin(expr, &self.ctx)
                         {
                             pyupgrade::rules::use_pep585_annotation(self, expr);
                         }
@@ -2291,7 +2291,7 @@ where
                         || (self.settings.target_version >= PythonVersion::Py37
                             && self.ctx.annotations_future_enabled
                             && self.ctx.in_annotation))
-                    && typing::is_pep585_builtin(expr, &self.ctx)
+                    && analyze::typing::is_pep585_builtin(expr, &self.ctx)
                 {
                     pyupgrade::rules::use_pep585_annotation(self, expr);
                 }
@@ -3632,7 +3632,7 @@ where
                     self.ctx.in_subscript = true;
                     visitor::walk_expr(self, expr);
                 } else {
-                    match match_annotated_subscript(
+                    match analyze::typing::match_annotated_subscript(
                         value,
                         &self.ctx,
                         self.settings.typing_modules.iter().map(String::as_str),
@@ -4051,7 +4051,7 @@ impl<'a> Checker<'a> {
                         && binding.redefines(existing)
                         && (!self.settings.dummy_variable_rgx.is_match(name) || existing_is_import)
                         && !(existing.kind.is_function_definition()
-                            && visibility::is_overload(
+                            && analyze::visibility::is_overload(
                                 &self.ctx,
                                 cast::decorator_list(existing.source.as_ref().unwrap()),
                             ))

crates/ruff/src/docstrings/definition.rs

@@ -1,7 +1,8 @@
-use ruff_python_ast::visibility::{
+use rustpython_parser::ast::{Expr, Stmt};
+
+use ruff_python_semantic::analyze::visibility::{
     class_visibility, function_visibility, method_visibility, Modifier, Visibility, VisibleScope,
 };
-use rustpython_parser::ast::{Expr, Stmt};
 
 #[derive(Debug, Clone)]
 pub enum DefinitionKind<'a> {

crates/ruff/src/docstrings/extraction.rs

@@ -2,7 +2,7 @@
 
 use rustpython_parser::ast::{Constant, Expr, ExprKind, Stmt, StmtKind};
 
-use ruff_python_ast::visibility::{Modifier, VisibleScope};
+use ruff_python_semantic::analyze::visibility;
 
 use crate::docstrings::definition::{Definition, DefinitionKind, Documentable};
 
@@ -28,53 +28,53 @@ pub fn docstring_from(suite: &[Stmt]) -> Option<&Expr> {
 
 /// Extract a `Definition` from the AST node defined by a `Stmt`.
 pub fn extract<'a>(
-    scope: VisibleScope,
+    scope: visibility::VisibleScope,
     stmt: &'a Stmt,
     body: &'a [Stmt],
     kind: Documentable,
 ) -> Definition<'a> {
     let expr = docstring_from(body);
     match kind {
         Documentable::Function => match scope {
-            VisibleScope {
-                modifier: Modifier::Module,
+            visibility::VisibleScope {
+                modifier: visibility::Modifier::Module,
                 ..
             } => Definition {
                 kind: DefinitionKind::Function(stmt),
                 docstring: expr,
             },
-            VisibleScope {
-                modifier: Modifier::Class,
+            visibility::VisibleScope {
+                modifier: visibility::Modifier::Class,
                 ..
             } => Definition {
                 kind: DefinitionKind::Method(stmt),
                 docstring: expr,
             },
-            VisibleScope {
-                modifier: Modifier::Function,
+            visibility::VisibleScope {
+                modifier: visibility::Modifier::Function,
                 ..
             } => Definition {
                 kind: DefinitionKind::NestedFunction(stmt),
                 docstring: expr,
             },
         },
         Documentable::Class => match scope {
-            VisibleScope {
-                modifier: Modifier::Module,
+            visibility::VisibleScope {
+                modifier: visibility::Modifier::Module,
                 ..
             } => Definition {
                 kind: DefinitionKind::Class(stmt),
                 docstring: expr,
             },
-            VisibleScope {
-                modifier: Modifier::Class,
+            visibility::VisibleScope {
+                modifier: visibility::Modifier::Class,
                 ..
             } => Definition {
                 kind: DefinitionKind::NestedClass(stmt),
                 docstring: expr,
             },
-            VisibleScope {
-                modifier: Modifier::Function,
+            visibility::VisibleScope {
+                modifier: visibility::Modifier::Function,
                 ..
             } => Definition {
                 kind: DefinitionKind::NestedClass(stmt),

crates/ruff/src/rules/flake8_annotations/helpers.rs

@@ -1,7 +1,7 @@
 use rustpython_parser::ast::{Arguments, Expr, Stmt, StmtKind};
 
 use ruff_python_ast::cast;
-use ruff_python_ast::visibility;
+use ruff_python_semantic::analyze::visibility;
 
 use crate::checkers::ast::Checker;
 use crate::docstrings::definition::{Definition, DefinitionKind};

crates/ruff/src/rules/flake8_annotations/rules.rs

@@ -4,10 +4,10 @@ use ruff_diagnostics::{AlwaysAutofixableViolation, Diagnostic, Violation};
 use ruff_macros::{derive_message_formats, violation};
 use ruff_python_ast::helpers::ReturnStatementVisitor;
 use ruff_python_ast::types::Range;
-use ruff_python_ast::visibility::Visibility;
-use ruff_python_ast::visibility::{self};
 use ruff_python_ast::visitor::Visitor;
 use ruff_python_ast::{cast, helpers};
+use ruff_python_semantic::analyze::visibility;
+use ruff_python_semantic::analyze::visibility::Visibility;
 use ruff_python_stdlib::typing::SIMPLE_MAGIC_RETURN_TYPES;
 
 use crate::checkers::ast::Checker;

crates/ruff/src/rules/flake8_blind_except/rules.rs

@@ -3,8 +3,8 @@ use rustpython_parser::ast::{Expr, ExprKind, Stmt, StmtKind};
 use ruff_diagnostics::{Diagnostic, Violation};
 use ruff_macros::{derive_message_formats, violation};
 use ruff_python_ast::helpers::{find_keyword, is_const_true};
-use ruff_python_ast::logging;
 use ruff_python_ast::types::Range;
+use ruff_python_semantic::analyze::logging;
 
 use crate::checkers::ast::Checker;
 

crates/ruff/src/rules/flake8_bugbear/rules/abstract_base_class.rs

@@ -3,7 +3,7 @@ use rustpython_parser::ast::{Constant, Expr, ExprKind, Keyword, Stmt, StmtKind};
 use ruff_diagnostics::{Diagnostic, Violation};
 use ruff_macros::{derive_message_formats, violation};
 use ruff_python_ast::types::Range;
-use ruff_python_ast::visibility::{is_abstract, is_overload};
+use ruff_python_semantic::analyze::visibility::{is_abstract, is_overload};
 
 use crate::checkers::ast::Checker;
 use crate::registry::Rule;

crates/ruff/src/rules/flake8_bugbear/rules/cached_instance_method.rs

@@ -2,8 +2,8 @@ use rustpython_parser::ast::{Expr, ExprKind};
 
 use ruff_diagnostics::{Diagnostic, Violation};
 use ruff_macros::{derive_message_formats, violation};
-use ruff_python_ast::scope::ScopeKind;
 use ruff_python_ast::types::Range;
+use ruff_python_semantic::scope::ScopeKind;
 
 use crate::checkers::ast::Checker;
 

crates/ruff/src/rules/flake8_django/rules/helpers.rs

@@ -1,6 +1,6 @@
 use rustpython_parser::ast::Expr;
 
-use ruff_python_ast::context::Context;
+use ruff_python_semantic::context::Context;
 
 /// Return `true` if a Python class appears to be a Django model, based on its base classes.
 pub fn is_model(context: &Context, base: &Expr) -> bool {

crates/ruff/src/rules/flake8_logging_format/rules.rs

@@ -2,8 +2,9 @@ use rustpython_parser::ast::{Constant, Expr, ExprKind, Keyword, Location, Operat
 
 use ruff_diagnostics::{Diagnostic, Edit};
 use ruff_python_ast::helpers::{find_keyword, SimpleCallArgs};
-use ruff_python_ast::logging;
 use ruff_python_ast::types::Range;
+use ruff_python_semantic::analyze::logging;
+use ruff_python_stdlib::logging::LoggingLevel;
 
 use crate::checkers::ast::Checker;
 use crate::registry::{AsRule, Rule};
@@ -128,7 +129,7 @@ fn check_log_record_attr_clash(checker: &mut Checker, extra: &Keyword) {
 #[derive(Copy, Clone)]
 enum LoggingCallType {
     /// Logging call with a level method, e.g., `logging.info`.
-    LevelCall(logging::LoggingLevel),
+    LevelCall(LoggingLevel),
     /// Logging call with an integer level as an argument, e.g., `logger.log(level, ...)`.
     LogCall,
 }
@@ -138,7 +139,7 @@ impl LoggingCallType {
         if attr == "log" {
             Some(LoggingCallType::LogCall)
         } else {
-            logging::LoggingLevel::from_attribute(attr).map(LoggingCallType::LevelCall)
+            LoggingLevel::from_attribute(attr).map(LoggingCallType::LevelCall)
         }
     }
 }
@@ -173,7 +174,7 @@ pub fn logging_call(checker: &mut Checker, func: &Expr, args: &[Expr], keywords:
             if checker.settings.rules.enabled(Rule::LoggingWarn)
                 && matches!(
                     logging_call_type,
-                    LoggingCallType::LevelCall(logging::LoggingLevel::Warn)
+                    LoggingCallType::LevelCall(LoggingLevel::Warn)
                 )
             {
                 let mut diagnostic = Diagnostic::new(LoggingWarn, level_call_range);
@@ -228,14 +229,14 @@ pub fn logging_call(checker: &mut Checker, func: &Expr, args: &[Expr], keywords:
 
                     if let LoggingCallType::LevelCall(logging_level) = logging_call_type {
                         match logging_level {
-                            logging::LoggingLevel::Error => {
+                            LoggingLevel::Error => {
                                 if checker.settings.rules.enabled(Rule::LoggingExcInfo) {
                                     checker
                                         .diagnostics
                                         .push(Diagnostic::new(LoggingExcInfo, level_call_range));
                                 }
                             }
-                            logging::LoggingLevel::Exception => {
+                            LoggingLevel::Exception => {
                                 if checker
                                     .settings
                                     .rules

crates/ruff/src/rules/flake8_self/rules/private_member_access.rs

@@ -3,8 +3,8 @@ use rustpython_parser::ast::{Expr, ExprKind};
 use ruff_diagnostics::{Diagnostic, Violation};
 use ruff_macros::{derive_message_formats, violation};
 use ruff_python_ast::call_path::collect_call_path;
-use ruff_python_ast::scope::ScopeKind;
 use ruff_python_ast::types::Range;
+use ruff_python_semantic::scope::ScopeKind;
 
 use crate::checkers::ast::Checker;