Skip to content

Commit

Permalink
fix: wrong long str interpolation location with special chars (#1409)
Browse files Browse the repository at this point in the history 
Signed-off-by: peefy <xpf6677@163.com>
  • Loading branch information
@Peefy
Peefy committed Jun 12, 2024
1 parent 9f15e62 commit 33629fd
Show file tree
Hide file tree
Showing 6 changed files with 219 additions and 38 deletions.
53 changes: 33 additions & 20 deletions kclvm/parser/src/parser/stmt.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1548,11 +1548,7 @@ impl<'a> Parser<'a> {
return None;
}

let start_pos = if s.is_long_string {
pos + new_byte_pos(3)
} else {
pos + new_byte_pos(1)
};
let quote_space = if s.is_long_string { 3 } else { 1 };

let mut joined_value = JoinedString {
is_long_string: s.is_long_string,
Expand Down Expand Up @@ -1632,31 +1628,49 @@ impl<'a> Parser<'a> {
)
}

// Here we use double pointers of data and raw_data,
// where data is used to obtain string literal data
// and raw_data is used to obtain string interpolation
// data to ensure that their respective positional
// information is correct.

let data = s.value.as_str();
let mut off: usize = 0;
let raw_data = s.raw_value.as_str();
let raw_data = &s.raw_value.as_str()[..raw_data.len() - quote_space];
let mut data_off = 0;
let mut raw_off: usize = quote_space;
loop {
if let Some(i) = data[off..].find("${") {
if let Some(j) = data[off + i..].find('}') {
let lo: usize = off + i;
let hi: usize = off + i + j + 1;
if let (Some(i), Some(data_i)) =
(raw_data[raw_off..].find("${"), data[data_off..].find("${"))
{
if let (Some(j), Some(data_j)) = (
raw_data[raw_off + i..].find('}'),
data[data_off + i..].find('}'),
) {
let lo: usize = raw_off + i;
let hi: usize = raw_off + i + j + 1;

let data_lo: usize = data_off + data_i;
let data_hi: usize = data_off + data_i + data_j + 1;

let s0 = &data[off..lo];
let s1 = &data[lo..hi];
let s0 = &data[data_off..data_lo];
let s1 = &raw_data[lo..hi];

let s0_expr = node_ref!(Expr::StringLit(StringLit {
is_long_string: false,
raw_value: s0.to_string(),
value: s0.to_string().replace("$$", "$"),
}));

let s1_expr = parse_expr(self, s1, start_pos + new_byte_pos(lo as u32));
let s1_expr = parse_expr(self, s1, pos + new_byte_pos(lo as u32));

if !s0.is_empty() {
joined_value.values.push(s0_expr);
}
joined_value.values.push(s1_expr);

off = hi;
data_off = data_hi;
raw_off = hi;
continue;
} else {
self.sess.struct_message_error(
Expand All @@ -1667,23 +1681,22 @@ impl<'a> Parser<'a> {
.values
.push(node_ref!(Expr::StringLit(StringLit {
is_long_string: false,
raw_value: data[off..].to_string(),
value: data[off..].to_string(),
raw_value: data[data_off..].to_string(),
value: data[data_off..].to_string(),
})));
break;
}
} else {
if off >= s.value.as_str().len() {
if raw_off >= raw_data.len() || data_off >= data.len() {
break;
}

// todo: fix pos
joined_value
.values
.push(node_ref!(Expr::StringLit(StringLit {
is_long_string: false,
raw_value: data[off..].to_string(),
value: data[off..].to_string().replace("$$", "$"),
raw_value: data[data_off..].to_string(),
value: data[data_off..].to_string().replace("$$", "$"),
})));
break;
}
Expand Down
6 changes: 6 additions & 0 deletions kclvm/parser/src/tests/error_recovery.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,12 @@ parse_expr_snapshot! { joined_string_recovery_5, r#"'${a + 1 = }'"# }
parse_expr_snapshot! { joined_string_recovery_6, r#"'${a: json}'"# }
parse_expr_snapshot! { joined_string_recovery_7, r#"'\n${a: #json}'"# }
parse_expr_snapshot! { joined_string_recovery_8, r#"'a\nb${a: #json}\n'"# }
parse_expr_snapshot! { joined_string_recovery_9, r#"'''\
${CC}
'''"# }
parse_expr_snapshot! { joined_string_recovery_10, r#""""
${CC}
""""# }
parse_expr_snapshot! { lambda_recovery_0, r#"lambda"# }
parse_expr_snapshot! { lambda_recovery_1, r#"lambda {"# }
parse_expr_snapshot! { lambda_recovery_2, r#"lambda {}"# }
Expand Down
83 changes: 83 additions & 0 deletions ...r/src/tests/snapshots/kclvm_parser__tests__error_recovery__joined_string_recovery_10.snap
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
---
source: parser/src/tests/error_recovery.rs
expression: "crate::tests::parsing_expr_string(r#\"\"\"\"\n ${CC}\n\"\"\"\"#)"
---
Node {
node: JoinedString(
JoinedString {
is_long_string: true,
values: [
Node {
node: StringLit(
StringLit {
is_long_string: false,
raw_value: "\n ",
value: "\n ",
},
),
filename: "",
line: 1,
column: 1,
end_line: 1,
end_column: 1,
},
Node {
node: FormattedValue(
FormattedValue {
is_long_string: false,
value: Node {
node: Identifier(
Identifier {
names: [
Node {
node: "CC",
filename: "",
line: 2,
column: 4,
end_line: 2,
end_column: 6,
},
],
pkgpath: "",
ctx: Load,
},
),
filename: "",
line: 2,
column: 4,
end_line: 2,
end_column: 6,
},
format_spec: None,
},
),
filename: "",
line: 2,
column: 4,
end_line: 2,
end_column: 6,
},
Node {
node: StringLit(
StringLit {
is_long_string: false,
raw_value: "\n",
value: "\n",
},
),
filename: "",
line: 1,
column: 1,
end_line: 1,
end_column: 1,
},
],
raw_value: "\"\"\"\n ${CC}\n\"\"\"",
},
),
filename: "",
line: 1,
column: 0,
end_line: 3,
end_column: 3,
}
14 changes: 6 additions & 8 deletions ...er/src/tests/snapshots/kclvm_parser__tests__error_recovery__joined_string_recovery_7.snap
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
---
source: parser/src/tests/error_recovery.rs
assertion_line: 158
expression: "crate::tests::parsing_expr_string(r#\"'\\n${a: #json}'\"#)"
---
Node {
Expand Down Expand Up @@ -34,9 +33,9 @@ Node {
node: "a",
filename: "",
line: 1,
column: 4,
column: 5,
end_line: 1,
end_column: 5,
end_column: 6,
},
],
pkgpath: "",
Expand All @@ -45,9 +44,9 @@ Node {
),
filename: "",
line: 1,
column: 4,
column: 5,
end_line: 1,
end_column: 5,
end_column: 6,
},
format_spec: Some(
"#json",
Expand All @@ -56,9 +55,9 @@ Node {
),
filename: "",
line: 1,
column: 4,
column: 5,
end_line: 1,
end_column: 12,
end_column: 13,
},
],
raw_value: "'\\n${a: #json}'",
Expand All @@ -70,4 +69,3 @@ Node {
end_line: 1,
end_column: 15,
}

18 changes: 8 additions & 10 deletions ...er/src/tests/snapshots/kclvm_parser__tests__error_recovery__joined_string_recovery_8.snap
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
---
source: parser/src/tests/error_recovery.rs
assertion_line: 159
expression: "crate::tests::parsing_expr_string(r#\"'a\\nb${a: #json}\\n'\"#)"
---
Node {
Expand Down Expand Up @@ -34,9 +33,9 @@ Node {
node: "a",
filename: "",
line: 1,
column: 6,
column: 7,
end_line: 1,
end_column: 7,
end_column: 8,
},
],
pkgpath: "",
Expand All @@ -45,9 +44,9 @@ Node {
),
filename: "",
line: 1,
column: 6,
column: 7,
end_line: 1,
end_column: 7,
end_column: 8,
},
format_spec: Some(
"#json",
Expand All @@ -56,16 +55,16 @@ Node {
),
filename: "",
line: 1,
column: 6,
column: 7,
end_line: 1,
end_column: 14,
end_column: 15,
},
Node {
node: StringLit(
StringLit {
is_long_string: false,
raw_value: "\n",
value: "\n",
raw_value: "}\n",
value: "}\n",
},
),
filename: "",
Expand All @@ -84,4 +83,3 @@ Node {
end_line: 1,
end_column: 19,
}

83 changes: 83 additions & 0 deletions ...er/src/tests/snapshots/kclvm_parser__tests__error_recovery__joined_string_recovery_9.snap
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
---
source: parser/src/tests/error_recovery.rs
expression: "crate::tests::parsing_expr_string(r#\"'''\\\n ${CC}\n'''\"#)"
---
Node {
node: JoinedString(
JoinedString {
is_long_string: true,
values: [
Node {
node: StringLit(
StringLit {
is_long_string: false,
raw_value: " ",
value: " ",
},
),
filename: "",
line: 1,
column: 1,
end_line: 1,
end_column: 1,
},
Node {
node: FormattedValue(
FormattedValue {
is_long_string: false,
value: Node {
node: Identifier(
Identifier {
names: [
Node {
node: "CC",
filename: "",
line: 2,
column: 4,
end_line: 2,
end_column: 6,
},
],
pkgpath: "",
ctx: Load,
},
),
filename: "",
line: 2,
column: 4,
end_line: 2,
end_column: 6,
},
format_spec: None,
},
),
filename: "",
line: 2,
column: 4,
end_line: 2,
end_column: 6,
},
Node {
node: StringLit(
StringLit {
is_long_string: false,
raw_value: "C}\n",
value: "C}\n",
},
),
filename: "",
line: 1,
column: 1,
end_line: 1,
end_column: 1,
},
],
raw_value: "'''\\\n ${CC}\n'''",
},
),
filename: "",
line: 1,
column: 0,
end_line: 3,
end_column: 3,
}

0 comments on commit 33629fd

Please sign in to comment.