Update to python-parser 0.3.0

crates/driver/Cargo.toml

@@ -18,8 +18,8 @@ thiserror = "1.0.58"
 lazy_static = "1.4.0"
 ignore = "0.4.22"
 globset = "0.4.14"
-rustpython-parser = { git = "https://github.com/RustPython/RustPython.git", rev = "b9ed63ed326e4ab9c97d808271ddc1d7ca05fda7"}
-rustpython-ast = { git = "https://github.com/RustPython/RustPython.git", rev = "b9ed63ed326e4ab9c97d808271ddc1d7ca05fda7", features = ["unparse"]}
+rustpython-parser = { git = "https://github.com/bazeltools/rustpython-parser.git", rev = "6f98c334d5ed709e6aa1a03ec1e20bd37859b867" }
+rustpython-ast = { git = "https://github.com/bazeltools/rustpython-parser.git", rev = "6f98c334d5ed709e6aa1a03ec1e20bd37859b867", features = ["unparse"] }
 pretty_env_logger = "0.5.0"
 log = "0.4.21"
 

crates/driver/src/print_build.rs

@@ -11,7 +11,7 @@ use crate::{
     to_directory, Opt, PrintBuildArgs,
 };
 use anyhow::{anyhow, Context, Result};
-use ast::{Located, StmtKind};
+use ast::{Expr, Stmt};
 use bzl_gen_build_python_utilities::{ast_builder, PythonProgram};
 use bzl_gen_build_shared_types::{
     build_config::{SourceConfig, TargetNameStrategy, WriteMode},
@@ -42,8 +42,8 @@ struct TargetEntry {
 }
 
 impl TargetEntry {
-    pub fn emit_build_function_call(&self) -> Result<Located<StmtKind>> {
-        let mut kw_args: Vec<(Arc<String>, Located<ast::ExprKind>)> = Default::default();
+    pub fn emit_build_function_call(&self) -> Result<Stmt> {
+        let mut kw_args: Vec<(Arc<String>, Expr)> = Default::default();
 
         kw_args.push((
             Arc::new("name".to_string()),
@@ -99,10 +99,10 @@ enum SrcType {
     List(Vec<String>),
 }
 impl SrcType {
-    pub fn to_statement(&self) -> Located<ast::ExprKind> {
+    pub fn to_statement(&self) -> Expr {
         match self {
             SrcType::Glob { include, exclude } => {
-                let mut kw_args: Vec<(Arc<String>, Located<ast::ExprKind>)> = Default::default();
+                let mut kw_args: Vec<(Arc<String>, Expr)> = Default::default();
 
                 kw_args.push((
                     Arc::new("include".to_string()),
@@ -150,7 +150,7 @@ struct TargetEntries {
 
 impl TargetEntries {
     // Helper
-    fn load_statement(from: Arc<String>, methods: Vec<Arc<String>>) -> Located<StmtKind> {
+    fn load_statement(from: Arc<String>, methods: Vec<Arc<String>>) -> Stmt {
         let mut fn_args = Vec::default();
         fn_args.push(ast_builder::with_constant_str(from.as_ref().to_owned()));
         fn_args.extend(
@@ -179,7 +179,7 @@ impl TargetEntries {
     }
 
     pub fn to_ast(&self) -> Result<PythonProgram> {
-        let mut program: Vec<Located<StmtKind>> = Vec::default();
+        let mut program: Vec<Stmt> = Vec::default();
         let mut all_load_statements: HashMap<Arc<String>, Vec<Arc<String>>> = HashMap::default();
 
         for entry in self.entries.iter() {

crates/python_extractor/Cargo.toml

@@ -16,8 +16,8 @@ sha2 = { version = "0.10.8", features = ["asm"] }
 thiserror = "1.0.58"
 lazy_static = "1.4.0"
 ignore = "0.4.22"
-rustpython-parser = { git = "https://github.com/RustPython/RustPython.git", rev = "b9ed63ed326e4ab9c97d808271ddc1d7ca05fda7"}
-rustpython-ast = { git = "https://github.com/RustPython/RustPython.git", rev = "b9ed63ed326e4ab9c97d808271ddc1d7ca05fda7", features = ["unparse"]}
+rustpython-parser = { git = "https://github.com/bazeltools/rustpython-parser.git", rev = "6f98c334d5ed709e6aa1a03ec1e20bd37859b867" }
+rustpython-ast = { git = "https://github.com/bazeltools/rustpython-parser.git", rev = "6f98c334d5ed709e6aa1a03ec1e20bd37859b867", features = ["unparse"] }
 pretty_env_logger = "0.5.0"
 log = "0.4.21"
 encoding_rs = "0.8.33"

crates/python_extractor/src/extract_py_imports.rs

@@ -1,5 +1,6 @@
+use ast::Stmt;
 use bzl_gen_build_python_utilities::PythonProgram;
-use rustpython_ast::{Located, StmtKind};
+use rustpython_parser::ast;
 
 pub fn extract(program: &PythonProgram) -> Vec<String> {
     let mut buf = Vec::default();
@@ -29,75 +30,72 @@ pub fn extract(program: &PythonProgram) -> Vec<String> {
     buf
 }
 
-fn extract_from_body(body: &Vec<Located<StmtKind>>, buf: &mut Vec<String>) {
+fn extract_from_body(body: &Vec<Stmt>, buf: &mut Vec<String>) {
     for element in body.iter() {
-        let element = &element.node;
         match element {
-            StmtKind::FunctionDef { body, .. } => extract_from_body(&body, buf),
-            StmtKind::AsyncFunctionDef { body, .. } => extract_from_body(&body, buf),
-            StmtKind::ClassDef {
-                name: _,
-                bases: _,
-                keywords: _,
-                body,
-                decorator_list: _,
-            } => extract_from_body(&body, buf),
-
-            StmtKind::For { body, orelse, .. } => {
+            Stmt::FunctionDef(ast::StmtFunctionDef { body, .. }) => extract_from_body(&body, buf),
+            Stmt::AsyncFunctionDef(ast::StmtAsyncFunctionDef { body, .. }) => {
+                extract_from_body(&body, buf)
+            }
+            Stmt::ClassDef(ast::StmtClassDef { body, .. }) => extract_from_body(&body, buf),
+            Stmt::For(ast::StmtFor { body, orelse, .. }) => {
                 extract_from_body(&body, buf);
                 extract_from_body(&orelse, buf);
             }
-            StmtKind::AsyncFor { body, orelse, .. } => {
+            Stmt::AsyncFor(ast::StmtAsyncFor { body, orelse, .. }) => {
                 extract_from_body(&body, buf);
                 extract_from_body(&orelse, buf);
             }
-            StmtKind::While { body, orelse, .. } => {
+            Stmt::While(ast::StmtWhile { body, orelse, .. }) => {
                 extract_from_body(&body, buf);
                 extract_from_body(&orelse, buf);
             }
-            StmtKind::If { body, orelse, .. } => {
+            Stmt::If(ast::StmtIf { body, orelse, .. }) => {
                 extract_from_body(&body, buf);
                 extract_from_body(&orelse, buf);
             }
-            StmtKind::With { body, .. } => extract_from_body(&body, buf),
-            StmtKind::AsyncWith { body, .. } => extract_from_body(&body, buf),
-            StmtKind::Match { cases, .. } => {
+            Stmt::With(ast::StmtWith { body, .. }) => extract_from_body(&body, buf),
+            Stmt::AsyncWith(ast::StmtAsyncWith { body, .. }) => extract_from_body(&body, buf),
+            Stmt::Match(ast::StmtMatch { cases, .. }) => {
                 for case in cases.iter() {
                     extract_from_body(&case.body, buf);
                 }
             }
-            StmtKind::Try {
+            Stmt::Try(ast::StmtTry {
+                range: _,
                 body,
                 handlers,
                 orelse,
                 finalbody,
-            } => {
+            }) => {
                 for handler in handlers.iter() {
-                    match &handler.node {
-                        rustpython_ast::ExcepthandlerKind::ExceptHandler { body, .. } => {
+                    match &handler.as_except_handler() {
+                        Some(ast::ExceptHandlerExceptHandler { body, .. }) => {
                             extract_from_body(&body, buf);
                         }
+                        None => {}
                     }
                 }
                 extract_from_body(&body, buf);
                 extract_from_body(&orelse, buf);
                 extract_from_body(&finalbody, buf);
             }
-            StmtKind::Import { names } => {
+            Stmt::Import(ast::StmtImport { names, .. }) => {
                 for nme in names.iter() {
-                    buf.push(nme.node.name.clone());
+                    buf.push(nme.name.to_string().clone());
                 }
             }
-            StmtKind::ImportFrom {
+            Stmt::ImportFrom(ast::StmtImportFrom {
                 module,
                 names,
                 level: _,
-            } => {
+                ..
+            }) => {
                 for nme in names.iter() {
                     if let Some(module) = module.as_ref() {
-                        buf.push(format!("{}.{}", module, nme.node.name));
+                        buf.push(format!("{}.{}", module, nme.name));
                     } else {
-                        buf.push(nme.node.name.clone());
+                        buf.push(nme.name.to_string().clone());
                     }
                 }
             }

crates/python_utilities/Cargo.toml

@@ -7,5 +7,5 @@ edition = "2021"
 
 [dependencies]
 anyhow = "1.0.81"
-rustpython-parser = { git = "https://github.com/RustPython/RustPython.git", rev = "b9ed63ed326e4ab9c97d808271ddc1d7ca05fda7"}
-rustpython-ast = { git = "https://github.com/RustPython/RustPython.git", rev = "b9ed63ed326e4ab9c97d808271ddc1d7ca05fda7", features = ["unparse"]}
+rustpython-parser = { git = "https://github.com/bazeltools/rustpython-parser.git", rev = "6f98c334d5ed709e6aa1a03ec1e20bd37859b867" }
+rustpython-ast = { git = "https://github.com/bazeltools/rustpython-parser.git", rev = "6f98c334d5ed709e6aa1a03ec1e20bd37859b867", features = ["unparse"] }

crates/python_utilities/src/ast_builder.rs

@@ -1,59 +1,57 @@
-use ast::{Expr, Located, Location, StmtKind};
+use ast::{text_size::TextRange, Expr, Stmt, TextSize};
 use rustpython_parser::ast;
 use std::sync::Arc;
 
-pub fn with_location<T>(t: T) -> Located<T> {
-    let location = Location::new(1, 0);
-    Located::new(location, location, t)
+fn empty_range() -> TextRange {
+    TextRange::empty(TextSize::new(0))
 }
-pub fn with_constant_str(s: String) -> Located<ast::ExprKind> {
-    with_location({
-        ast::ExprKind::Constant {
-            value: ast::Constant::Str(s),
-            kind: None,
-        }
+
+pub fn with_constant_str(s: String) -> Expr {
+    Expr::Constant(ast::ExprConstant {
+        range: empty_range(),
+        value: ast::Constant::Str(s),
+        kind: None,
     })
 }
 
-pub fn as_py_list<U>(elements: Vec<Expr<U>>) -> Located<ast::ExprKind<U>> {
-    with_location({
-        ast::ExprKind::List {
-            elts: elements,
-            ctx: ast::ExprContext::Load,
-        }
+pub fn as_py_list(elements: Vec<Expr>) -> Expr {
+    Expr::List(ast::ExprList {
+        range: empty_range(),
+        elts: elements,
+        ctx: ast::ExprContext::Load,
     })
 }
 
-pub fn as_stmt_expr(u: Located<ast::ExprKind>) -> Located<StmtKind> {
-    with_location(StmtKind::Expr { value: Box::new(u) })
+pub fn as_stmt_expr(u: Expr) -> Stmt {
+    Stmt::Expr(ast::StmtExpr {
+        range: empty_range(),
+        value: Box::new(u),
+    })
 }
 
 pub fn gen_py_function_call(
     name: Arc<String>,
-    args: Vec<Located<ast::ExprKind>>,
-    kw_args: Vec<(Arc<String>, Located<ast::ExprKind>)>,
-) -> Located<ast::ExprKind> {
-    let location = Location::new(1, 0);
+    args: Vec<Expr>,
+    kw_args: Vec<(Arc<String>, Expr)>,
+) -> Expr {
+    let location = empty_range();
     let mut kws = Vec::default();
     for (k, v) in kw_args {
-        kws.push(with_location(ast::KeywordData {
-            arg: Some(k.as_ref().to_owned()),
-            value: Box::new(v),
-        }))
+        kws.push(ast::Keyword {
+            range: location,
+            arg: Some(ast::Identifier::new(k.to_string())),
+            value: v,
+        })
     }
 
-    with_location({
-        ast::ExprKind::Call {
-            func: Box::new(Expr::new(
-                location,
-                location,
-                ast::ExprKind::Name {
-                    id: name.as_ref().to_owned(),
-                    ctx: ast::ExprContext::Load,
-                },
-            )),
-            args,
-            keywords: kws,
-        }
+    Expr::Call(ast::ExprCall {
+        range: location,
+        func: Box::new(Expr::Name(ast::ExprName {
+            range: location,
+            id: ast::Identifier::new(name.to_string()),
+            ctx: ast::ExprContext::Load,
+        })),
+        args,
+        keywords: kws,
     })
 }