fix: align TokenView text semantics - #167
Merged
Merged
Conversation
Contributor
|
Caution The consumer version of Gemini Code Assist on GitHub has been sunset. All code review activity has officially ceased. |
📝 WalkthroughWalkthroughChangesThe PR changes Token text API migration
Estimated code review effort: 3 (Moderate) | ~20 minutes Possibly related PRs
🚥 Pre-merge checks | ✅ 4 | ❌ 1❌ Failed checks (1 warning)
✅ Passed checks (4 passed)
✨ Finishing Touches📝 Generate docstrings
🧪 Generate unit tests (beta)
Thanks for using CodeRabbit! It's free for OSS, and your support helps us grow. If you like it, consider giving us a shout-out. Comment |
Contributor
Author
|
@codex review |
tinovyatkin
marked this pull request as ready for review
July 23, 2026 07:56
Copy/Paste DetectionFound 71 duplication(s) across 14 changed Rust file(s) (threshold: 100 tokens). Show duplicationsFound a 66 line (387 tokens) duplication in the following files:
use javascript_parser_base::JavaScriptParserBase;
fn dump_tree<S: AsRef<str>>(
out: &mut dyn Write,
tree: Node<'_>,
rule_names: &[S],
depth: usize,
) -> io::Result<()> {
let pad = " ".repeat(depth);
match tree.kind() {
NodeKind::Rule => {
let rule = tree.as_rule().expect("rule node kind checked");
let name = rule_names
.get(rule.rule_index())
.map_or("<?>", AsRef::as_ref);
writeln!(
out,
"{pad}Rule({name}, children={})",
rule.child_count()
)?;
for child in rule.children() {
dump_tree(out, child, rule_names, depth + 1)?;
}
}
NodeKind::Terminal => writeln!(
out,
"{pad}Term({:?})",
tree.as_terminal().expect("terminal node kind checked").text()
)?,
NodeKind::Error => writeln!(
out,
"{pad}Err({:?})",
tree.as_error().expect("error node kind checked").text()
)?,
}
Ok(())
}
fn main() -> ExitCode {
let mut args = env::args().skip(1);
let mut input: Option<PathBuf> = None;
let mut tokens_only = false;
while let Some(arg) = args.next() {
match arg.as_str() {
"--input" => input = args.next().map(PathBuf::from),
"--tokens" => tokens_only = true,
other => {
eprintln!("unknown argument: {other}");
return ExitCode::from(2);
}
}
}
let Some(input) = input else {
eprintln!("missing --input <path>");
return ExitCode::from(2);
};
let source = match fs::read_to_string(&input) {
Ok(source) => source,
Err(error) => {
eprintln!("failed to read {}: {error}", input.display());
return ExitCode::FAILURE;
}
};
if tokens_only {
let lexer = JavaScriptLexer::with_typed_hooks(
```rust
---
Found a 54 line (320 tokens) duplication in the following files:
* Starting at line 606 of src/atn/lexer.rs
* Starting at line 677 of src/atn/lexer.rs
```rust
atn: &LexerAtn,
hooks: &mut H,
mut generated_action: A,
mut generated_predicate: P,
unknown_policy: UnknownSemanticPolicy,
mut accept_adjuster: E,
) -> Result<TokenId, TokenStoreError>
where
I: CharStream,
H: SemanticHooks,
A: FnMut(&mut BaseLexer<I>, LexerCustomAction) -> bool,
P: FnMut(&BaseLexer<I>, LexerPredicate) -> Option<bool>,
E: FnMut(&mut BaseLexer<I>, i32, usize),
{
let hooks = RefCell::new(hooks);
let token = next_token_with_hooks_impl(
lexer,
sink,
atn,
&mut |lexer, action| {
if !generated_action(lexer, action)
&& !dispatch_lexer_action_hook(&hooks, lexer, action)
&& unknown_policy == UnknownSemanticPolicy::Error
&& let (Ok(rule), Ok(index)) = (
usize::try_from(action.rule_index()),
usize::try_from(action.action_index()),
)
{
lexer.record_semantic_error(true, rule, index);
}
},
&mut |lexer, predicate| {
generated_predicate(lexer, predicate)
.or_else(|| dispatch_lexer_predicate_hook(&hooks, lexer, predicate))
.unwrap_or_else(|| match unknown_policy {
UnknownSemanticPolicy::AssumeTrue => true,
UnknownSemanticPolicy::AssumeFalse => false,
UnknownSemanticPolicy::Error => {
lexer.record_semantic_error(
false,
predicate.rule_index(),
predicate.pred_index(),
);
false
}
})
},
&mut |lexer| dispatch_lexer_before_token_hook(&hooks, lexer),
&mut accept_adjuster,
&mut |lexer, accept_position| {
dispatch_lexer_after_accept_hook(&hooks, lexer, accept_position);
},
LexerMatchStrategy {
compiled: None,Found a 44 line (265 tokens) duplication in the following files:
impl JavaScriptParserBase {
fn raw_token<S>(ctx: &mut ParserSemCtx<'_, S>, index: usize) -> Option<(i32, i32, String)>
where
S: TokenSource,
{
ctx.token_at(index).map(|token| {
(
token.channel(),
token.token_type(),
token.text_or_empty().to_owned(),
)
})
}
fn has_line_terminator_ahead<S>(ctx: &mut ParserSemCtx<'_, S>) -> bool
where
S: TokenSource,
{
let current = ctx.input_index();
let Some(previous) = current.checked_sub(1) else {
return false;
};
let Some((channel, mut token_type, mut text)) = Self::raw_token(ctx, previous) else {
return false;
};
if channel != HIDDEN_CHANNEL {
return false;
}
if token_type == LINE_TERMINATOR {
return true;
}
if token_type == WHITE_SPACES {
let Some(before_whitespace) = previous.checked_sub(1) else {
return false;
};
let Some((_, next_type, next_text)) = Self::raw_token(ctx, before_whitespace) else {
return false;
};
token_type = next_type;
text = next_text;
}
token_type == LINE_TERMINATOR
|| (token_type == MULTI_LINE_COMMENT && (text.contains('\r') || text.contains('\n')))
}
```rust
---
Found a 44 line (264 tokens) duplication in the following files:
* Starting at line 13437 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 14436 of src/bin_support/grammar/generated/antlr_v4_parser.rs
```rust
let __child = self.parse_generated_rule_38_dispatch(0, false).map_err(GeneratedRuleError::into_error);
self.base.discard_invoking_state(__invoking_marker);
let __child = __child?;
self.base.add_parse_child(&mut __ctx, __child);
}
2 => {
break;
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
}
}
2 => {
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_38_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {Found a 40 line (257 tokens) duplication in the following files:
let __child = self.parse_generated_rule_47_dispatch(0, false).map_err(GeneratedRuleError::into_error);
self.base.discard_invoking_state(__invoking_marker);
let __child = __child?;
self.base.add_parse_child(&mut __ctx, __child);
}
2 => {
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_47_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {
```rust
---
Found a 39 line (254 tokens) duplication in the following files:
* Starting at line 13690 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 15499 of src/bin_support/grammar/generated/antlr_v4_parser.rs
```rust
let __match = self.base.match_token_recovering(63, 416, atn())?;
__consumed_eof |= __match.consumed_eof();
for __child in __match.into_child_iter() { self.base.add_parse_child(&mut __ctx, __child); }
}
2 => {
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_39_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {Found a 37 line (250 tokens) duplication in the following files:
let __match = self.base.match_token_recovering(11, 505, atn())?;
__consumed_eof |= __match.consumed_eof();
for __child in __match.into_child_iter() { self.base.add_parse_child(&mut __ctx, __child); }
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_49_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {
```rust
---
Found a 38 line (248 tokens) duplication in the following files:
* Starting at line 15837 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 16584 of src/bin_support/grammar/generated/antlr_v4_parser.rs
```rust
let __invoking_marker = self.base.push_invoking_state(549isize);
let __child = self.parse_generated_rule_63_dispatch(0, false).map_err(GeneratedRuleError::into_error);
self.base.discard_invoking_state(__invoking_marker);
let __child = __child?;
self.base.add_parse_child(&mut __ctx, __child);
}
2 => {
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_56_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {Found a 36 line (244 tokens) duplication in the following files:
let __invoking_marker = self.base.push_invoking_state(539isize);
let __child = self.parse_generated_rule_55_dispatch(0, false).map_err(GeneratedRuleError::into_error);
self.base.discard_invoking_state(__invoking_marker);
let __child = __child?;
self.base.add_parse_child(&mut __ctx, __child);
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_54_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {
```rust
---
Found a 39 line (243 tokens) duplication in the following files:
* Starting at line 11328 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 11422 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 12373 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 12577 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 12771 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 13165 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 13848 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 14253 of src/bin_support/grammar/generated/antlr_v4_parser.rs
```rust
let __child = self.parse_generated_rule_33_dispatch(0, false).map_err(GeneratedRuleError::into_error);
self.base.discard_invoking_state(__invoking_marker);
let __child = __child?;
self.base.add_parse_child(&mut __ctx, __child);
}
2 => {
break;
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
}
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_17_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {Found a 37 line (240 tokens) duplication in the following files:
let __child = self.parse_generated_rule_22_dispatch(0, false).map_err(GeneratedRuleError::into_error);
self.base.discard_invoking_state(__invoking_marker);
let __child = __child?;
self.base.add_parse_child(&mut __ctx, __child);
}
2 => {
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_21_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {
```rust
---
Found a 35 line (236 tokens) duplication in the following files:
* Starting at line 10015 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 10551 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 11527 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 12222 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 13957 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 15124 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 15631 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 15748 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 15940 of src/bin_support/grammar/generated/antlr_v4_parser.rs
```rust
let __child = self.parse_generated_rule_12_dispatch(0, false).map_err(GeneratedRuleError::into_error);
self.base.discard_invoking_state(__invoking_marker);
let __child = __child?;
self.base.add_parse_child(&mut __ctx, __child);
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_4_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {Found a 36 line (235 tokens) duplication in the following files:
let __child = self.parse_generated_rule_33_dispatch(0, false).map_err(GeneratedRuleError::into_error);
self.base.discard_invoking_state(__invoking_marker);
let __child = __child?;
self.base.add_parse_child(&mut __ctx, __child);
}
2 => {
break;
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
}
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
```rust
---
Found a 34 line (233 tokens) duplication in the following files:
* Starting at line 9891 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 10342 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 11099 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 14057 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 14156 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 16252 of src/bin_support/grammar/generated/antlr_v4_parser.rs
```rust
let __match = self.base.match_token_recovering(43, 160, atn())?;
__consumed_eof |= __match.consumed_eof();
for __child in __match.into_child_iter() { self.base.add_parse_child(&mut __ctx, __child); }
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_3_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {Found a 33 line (227 tokens) duplication in the following files:
let __invoking_marker = self.base.push_invoking_state(244isize);
let __child = self.parse_generated_rule_14_dispatch(0, false).map_err(GeneratedRuleError::into_error);
self.base.discard_invoking_state(__invoking_marker);
let __child = __child?;
self.base.add_parse_child(&mut __ctx, __child);
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_13_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {
```rust
---
Found a 33 line (227 tokens) duplication in the following files:
* Starting at line 12274 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 12429 of src/bin_support/grammar/generated/antlr_v4_parser.rs
```rust
let __invoking_marker = self.base.push_invoking_state(329isize);
let __child = self.parse_generated_rule_15_dispatch(0, false).map_err(GeneratedRuleError::into_error);
self.base.discard_invoking_state(__invoking_marker);
let __child = __child?;
self.base.add_parse_child(&mut __ctx, __child);
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_25_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {Found a 32 line (219 tokens) duplication in the following files:
let __child = self.parse_generated_rule_6_dispatch(0, false).map_err(GeneratedRuleError::into_error);
self.base.discard_invoking_state(__invoking_marker);
let __child = __child?;
self.base.add_parse_child(&mut __ctx, __child);
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_6_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {
```rust
---
Found a 31 line (216 tokens) duplication in the following files:
* Starting at line 9730 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 9785 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 10122 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 10453 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 10650 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 10745 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 11147 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 11231 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 12630 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 13024 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 13751 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 14859 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 16052 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 16452 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 16642 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 16941 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 17146 of src/bin_support/grammar/generated/antlr_v4_parser.rs
```rust
let __match = self.base.match_token_recovering(-1, 149, atn())?;
__consumed_eof |= __match.consumed_eof();
for __child in __match.into_child_iter() { self.base.add_parse_child(&mut __ctx, __child); }
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_1_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {Found a 36 line (216 tokens) duplication in the following files:
}
2 => {
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_39_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {
```rust
---
Found a 34 line (212 tokens) duplication in the following files:
* Starting at line 13695 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 15016 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 15504 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 16830 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 17098 of src/bin_support/grammar/generated/antlr_v4_parser.rs
```rust
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_39_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {Found a 34 line (212 tokens) duplication in the following files:
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_47_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {
```rust
---
Found a 34 line (200 tokens) duplication in the following files:
* Starting at line 10889 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 13306 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 13447 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 14446 of src/bin_support/grammar/generated/antlr_v4_parser.rs
```rust
for __child in __match.into_child_iter() { self.base.add_parse_child(&mut __ctx, __child); }
}
2 => {
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_12_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {Found a 33 line (199 tokens) duplication in the following files:
}
2 => {
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_12_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {
```rust
---
Found a 32 line (196 tokens) duplication in the following files:
* Starting at line 9893 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 13697 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 14059 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 14158 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 14750 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 15018 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 15506 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 16254 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 16832 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 17100 of src/bin_support/grammar/generated/antlr_v4_parser.rs
```rust
for __child in __match.into_child_iter() { self.base.add_parse_child(&mut __ctx, __child); }
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_3_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {Found a 32 line (196 tokens) duplication in the following files:
for __child in __match.into_child_iter() { self.base.add_parse_child(&mut __ctx, __child); }
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_7_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {
```rust
---
Found a 32 line (196 tokens) duplication in the following files:
* Starting at line 11101 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 13697 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 14750 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 15018 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 15506 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 16832 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 17100 of src/bin_support/grammar/generated/antlr_v4_parser.rs
```rust
for __child in __match.into_child_iter() { self.base.add_parse_child(&mut __ctx, __child); }
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_14_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {Found a 31 line (195 tokens) duplication in the following files:
}
_ => return Err(self.base.no_viable_alternative_error(__decision_start)),
}
Ok(())
})();
match __result {
Ok(()) => {
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
Err(__error) => {
if let Some(__error) = __sync_error {
if allow_fallback {
self.base.exit_rule();
self.base.restore_generated_diagnostics(__generated_diagnostic_marker);
self.base.record_generated_syntax_error();
return Err(GeneratedRuleError::Fatal(__error));
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
return Ok(__tree);
}
self.base.recover_generated_rule(&mut __ctx, atn(), __error);
let __tree = self.base.finish_rule(__ctx, __consumed_eof);
Ok(__tree)
}
}
}
#[allow(dead_code)]
fn parse_generated_rule_3_dispatch(&mut self, precedence: i32, allow_fallback: bool) -> Result<antlr4_runtime::ParseTree, GeneratedRuleError> {
```rust
---
Found a 22 line (179 tokens) duplication in the following files:
* Starting at line 783 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 913 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 1035 of src/bin_support/grammar/generated/antlr_v4_parser.rs
* Starting at line 1155 of src/bin_support/grammar/generate
_(report truncated; full output in workflow logs)_ |
|
Claude finished @tinovyatkin's task in 15m 10s —— View job Code Review: Align TokenView text semantics
Verdict: LGTM — no blocking issues found. This is a clean, well-scoped API correction. Four independent review passes (2× bug-hunting, 2× CLAUDE.md compliance) plus my own checks all converged on no bugs and no CLAUDE.md violations. The core change is exactly right: What I verified (the real risk areas)
Non-blocking observations (for awareness, no change required)
Nice touch adding |
Contributor
Author
|
@coderabbitai review |
✅ Action performedReview finished.
|
📊 Source Code Metrics (this PR vs
|
| File | Cyclomatic | Cognitive | Functions | LLOC | MI |
|---|---|---|---|---|---|
| src/token.rs | 178 (main: 175) 🔴 | 44 ⚪ | 91 (main: 88) 🔴 | 189 (main: 182) 🔴 | 0 ⚪ |
| src/bin_support/grammar/lexer_adaptor.rs | 39 ⚪ | 24 ⚪ | 13 ⚪ | 43 ⚪ | 17.92 (main: 17.93) 🔴 |
Generated by mehen v1.6.0 — the code quality watcher.
![coderabbitai[bot]](https://avatars.githubusercontent.com/in/347564?s=60&v=4){kind=small}
|
Codex Review: Didn't find any major issues. Keep it up! Reviewed commit: ℹ️ About Codex in GitHubYour team has set up Codex to review pull requests in this repo. Reviews are triggered when you
If Codex has suggestions, it will comment; otherwise it will react with 👍. Codex can also answer questions or update the PR. Try commenting "@codex address that feedback". |
tinovyatkin
changed the title
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Closes #122.
Summary
TokenView::text()returnOption<&str>, matching theTokentrait for concrete and generic receiversTokenView::text_or_empty()for display, CST, generated semantic, and parity paths that intentionally collapse missing textNoneversus empty-string distinction with a regression testRoot cause
Rust method resolution preferred the inherent
TokenView::text() -> &stroverToken::text() -> Option<&str>. The same method call therefore changed type and semantics based only on whether the receiver was concrete or generic, and the inherent method silently collapsed missing text to"".Impact
This is a breaking pre-1.0 API correction. Concrete
TokenView::text()calls now handleOption<&str>exactly like genericTokencalls. Callers that deliberately want the prior lossy behavior can usetext_or_empty().Validation
cargo test --locked --all-targets --all-features(266 + 3 + 595 + 22passed)cargo clippy --locked --all-targets --all-features -- -D warningsRUSTDOCFLAGS='-D warnings' cargo doc --locked --all-features --no-depstools/grammar-frontend/update-stage0.sh --update(Stage 1 and Stage 2 byte-identical; frontend corpus checks passed)cargo run --release --quiet --bin antlr4-runtime-testsuite(357 passed, 0 failed, 0 skipped)9/9parse trees match)6/6token streams and parse trees match)5/5token streams and parse trees match)Summary by CodeRabbit
New Features
Bug Fixes
Tests