refactor: ast node locations in the parser for the resolver. (#440)

kcl-lang · Mar 8, 2023 · 82c2be0 · 82c2be0
1 parent 6674a71
commit 82c2be0
Show file tree

Hide file tree

Showing 26 changed files with 65 additions and 53 deletions.
diff --git a/kclvm/ast/src/ast.rs b/kclvm/ast/src/ast.rs
@@ -38,7 +38,7 @@
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 
-use compiler_base_span::Loc;
+use compiler_base_span::{Loc, Span};
 
 use super::token;
 use crate::node_ref;
@@ -125,6 +125,14 @@ impl<T> Node<T> {
     }
 }
 
+/// Spanned<T> is the span information that all AST nodes need to contain.
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct Spanned<T> {
+    pub node: T,
+    #[serde(skip)]
+    pub span: Span,
+}
+
 impl TryInto<Node<Identifier>> for Node<Expr> {
     type Error = &'static str;
 
@@ -224,7 +232,7 @@ impl Module {
         let mut stmts = Vec::new();
         for stmt in &self.body {
             if let Stmt::Schema(schema_stmt) = &stmt.node {
-                stmts.push(node_ref!(schema_stmt.clone()));
+                stmts.push(node_ref!(schema_stmt.clone(), stmt.pos()));
             }
         }
         return stmts;

diff --git a/kclvm/ast/src/lib.rs b/kclvm/ast/src/lib.rs
@@ -1,5 +1,4 @@
 // Copyright 2021 The KCL Authors. All rights reserved.
-use crate::ast::*;
 
 pub mod ast;
 pub mod config;
@@ -44,19 +43,3 @@ macro_rules! stmt_as {
         }
     };
 }
-
-/// Construct an AssignStmt node with assign_value as value
-pub fn build_assign_node(attr_name: &str, assign_value: NodeRef<Expr>) -> NodeRef<Stmt> {
-    let iden = node_ref!(Identifier {
-        names: vec![attr_name.to_string()],
-        pkgpath: String::new(),
-        ctx: ExprContext::Store
-    });
-
-    node_ref!(Stmt::Assign(AssignStmt {
-        value: assign_value,
-        targets: vec![iden],
-        type_annotation: None,
-        ty: None
-    }))
-}
diff --git a/kclvm/ast/src/tests.rs b/kclvm/ast/src/tests.rs
@@ -1,5 +1,22 @@
+use crate::node_ref;
 use crate::walker::MutSelfMutWalker;
-use crate::{ast, build_assign_node, node_ref, Identifier, Module, Node, NodeRef, SchemaStmt};
+use crate::{ast, ast::*};
+
+/// Construct an AssignStmt node with assign_value as value
+fn build_assign_node(attr_name: &str, assign_value: NodeRef<Expr>) -> NodeRef<Stmt> {
+    let iden = node_ref!(Identifier {
+        names: vec![attr_name.to_string()],
+        pkgpath: String::new(),
+        ctx: ExprContext::Store
+    });
+
+    node_ref!(Stmt::Assign(AssignStmt {
+        value: assign_value,
+        targets: vec![iden],
+        type_annotation: None,
+        ty: None
+    }))
+}
 
 fn get_dummy_assign_ast() -> ast::Node<ast::AssignStmt> {
     let filename = "main.k";

diff --git a/kclvm/parser/src/parser/stmt.rs b/kclvm/parser/src/parser/stmt.rs
@@ -125,7 +125,7 @@ impl<'a> Parser<'_> {
         &mut self,
         open_tok: TokenKind,
         close_tok: TokenKind,
-    ) -> Vec<Box<Node<Stmt>>> {
+    ) -> Vec<NodeRef<Stmt>> {
         let mut stmt_list = Vec::new();
 
         self.bump_token(open_tok);
@@ -259,6 +259,7 @@ impl<'a> Parser<'_> {
             }
         }
 
+        let stmt_end_token = self.prev_token;
         self.skip_newlines();
 
         if let Some(value) = value_or_target {
@@ -287,7 +288,7 @@ impl<'a> Parser<'_> {
                     type_annotation,
                     ty,
                 }),
-                self.token_span_pos(token, self.prev_token)
+                self.token_span_pos(token, stmt_end_token)
             ))
         } else {
             if targets.len() == 1 && type_annotation.is_some() && is_in_schema_stmt {
@@ -303,7 +304,7 @@ impl<'a> Parser<'_> {
                             is_optional: false,
                             decorators: Vec::new(),
                         }),
-                        self.token_span_pos(token, self.prev_token)
+                        self.token_span_pos(token, stmt_end_token)
                     ));
                 }
             }
@@ -1413,7 +1414,6 @@ impl<'a> Parser<'_> {
             // bump to the first token
             parser.bump();
 
-            let _token = parser.token;
             let expr = parser.parse_expr();
 
             let mut formatted_value = FormattedValue {
@@ -1457,7 +1457,9 @@ impl<'a> Parser<'_> {
 
                     let s1_expr = parse_expr(self, s1, start_pos + new_byte_pos(lo as u32));
 
-                    joined_value.values.push(s0_expr);
+                    if !s0.is_empty() {
+                        joined_value.values.push(s0_expr);
+                    }
                     joined_value.values.push(s1_expr);
 
                     off = hi;

diff --git a/kclvm/parser/src/parser/tests.rs b/kclvm/parser/src/parser/tests.rs
@@ -673,7 +673,7 @@ fn lambda_expr_3() {
     _a
 }"####,
         expect![[r#"
-        Node { node: Lambda(LambdaExpr { args: None, return_type_str: None, body: [Node { node: If(IfStmt { body: [Node { node: Assign(AssignStmt { targets: [Node { node: Identifier { names: ["_a"], pkgpath: "", ctx: Store }, filename: "", line: 3, column: 8, end_line: 3, end_column: 10 }], value: Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(1) }), filename: "", line: 3, column: 13, end_line: 3, end_column: 14 }, type_annotation: None, ty: None }), filename: "", line: 3, column: 8, end_line: 4, end_column: 0 }], cond: Node { node: NameConstantLit(NameConstantLit { value: True }), filename: "", line: 2, column: 7, end_line: 2, end_column: 11 }, orelse: [Node { node: Assign(AssignStmt { targets: [Node { node: Identifier { names: ["_a"], pkgpath: "", ctx: Store }, filename: "", line: 5, column: 8, end_line: 5, end_column: 10 }], value: Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(2) }), filename: "", line: 5, column: 13, end_line: 5, end_column: 14 }, type_annotation: None, ty: None }), filename: "", line: 5, column: 8, end_line: 6, end_column: 0 }] }), filename: "", line: 2, column: 4, end_line: 6, end_column: 4 }, Node { node: Expr(ExprStmt { exprs: [Node { node: Identifier(Identifier { names: ["_a"], pkgpath: "", ctx: Load }), filename: "", line: 6, column: 4, end_line: 6, end_column: 6 }] }), filename: "", line: 6, column: 4, end_line: 6, end_column: 6 }], return_ty: None }), filename: "", line: 1, column: 0, end_line: 7, end_column: 1 }
+        Node { node: Lambda(LambdaExpr { args: None, return_type_str: None, body: [Node { node: If(IfStmt { body: [Node { node: Assign(AssignStmt { targets: [Node { node: Identifier { names: ["_a"], pkgpath: "", ctx: Store }, filename: "", line: 3, column: 8, end_line: 3, end_column: 10 }], value: Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(1) }), filename: "", line: 3, column: 13, end_line: 3, end_column: 14 }, type_annotation: None, ty: None }), filename: "", line: 3, column: 8, end_line: 3, end_column: 14 }], cond: Node { node: NameConstantLit(NameConstantLit { value: True }), filename: "", line: 2, column: 7, end_line: 2, end_column: 11 }, orelse: [Node { node: Assign(AssignStmt { targets: [Node { node: Identifier { names: ["_a"], pkgpath: "", ctx: Store }, filename: "", line: 5, column: 8, end_line: 5, end_column: 10 }], value: Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(2) }), filename: "", line: 5, column: 13, end_line: 5, end_column: 14 }, type_annotation: None, ty: None }), filename: "", line: 5, column: 8, end_line: 5, end_column: 14 }] }), filename: "", line: 2, column: 4, end_line: 6, end_column: 4 }, Node { node: Expr(ExprStmt { exprs: [Node { node: Identifier(Identifier { names: ["_a"], pkgpath: "", ctx: Load }), filename: "", line: 6, column: 4, end_line: 6, end_column: 6 }] }), filename: "", line: 6, column: 4, end_line: 6, end_column: 6 }], return_ty: None }), filename: "", line: 1, column: 0, end_line: 7, end_column: 1 }
         "#]],
     );
 }
@@ -1057,13 +1057,18 @@ schema TestBool:
 
 #[test]
 fn test_parse_joined_string() {
-    // todo: fix joined_string
-    // check_type_stmt(
-    //     r####"a='${123+200}'"####,
-    //     expect![[r#"
-    //     sss
-    //     "#]],
-    // );
+    check_parsing_expr(
+        r####"'${123+200}'"####,
+        expect![[r#"
+        Node { node: JoinedString(JoinedString { is_long_string: false, values: [Node { node: FormattedValue(FormattedValue { is_long_string: false, value: Node { node: Binary(BinaryExpr { left: Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(123) }), filename: "", line: 1, column: 3, end_line: 1, end_column: 6 }, op: Bin(Add), right: Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(200) }), filename: "", line: 1, column: 7, end_line: 1, end_column: 10 } }), filename: "", line: 1, column: 3, end_line: 1, end_column: 10 }, format_spec: None }), filename: "", line: 1, column: 1, end_line: 1, end_column: 1 }], raw_value: "'${123+200}'" }), filename: "", line: 1, column: 0, end_line: 1, end_column: 12 }
+        "#]],
+    );
+    check_parsing_expr(
+        r####"'abc${a+1}cde'"####,
+        expect![[r#"
+        Node { node: JoinedString(JoinedString { is_long_string: false, values: [Node { node: StringLit(StringLit { is_long_string: false, raw_value: "abc", value: "abc" }), filename: "", line: 1, column: 1, end_line: 1, end_column: 1 }, Node { node: FormattedValue(FormattedValue { is_long_string: false, value: Node { node: Binary(BinaryExpr { left: Node { node: Identifier(Identifier { names: ["a"], pkgpath: "", ctx: Load }), filename: "", line: 1, column: 6, end_line: 1, end_column: 7 }, op: Bin(Add), right: Node { node: NumberLit(NumberLit { binary_suffix: None, value: Int(1) }), filename: "", line: 1, column: 8, end_line: 1, end_column: 9 } }), filename: "", line: 1, column: 6, end_line: 1, end_column: 9 }, format_spec: None }), filename: "", line: 1, column: 1, end_line: 1, end_column: 1 }, Node { node: StringLit(StringLit { is_long_string: false, raw_value: "cde", value: "cde" }), filename: "", line: 1, column: 1, end_line: 1, end_column: 1 }], raw_value: "'abc${a+1}cde'" }), filename: "", line: 1, column: 0, end_line: 1, end_column: 14 }
+        "#]],
+    );
 }
 
 #[test]