Skip to content

Optimized out unused states and production rules #27

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ehwan merged 11 commits into from
Aug 23, 2025
Merged

Optimized out unused states and production rules #27

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
8daefde
convert builder::state into intermediate states
ehwan Aug 22, 2025
e7d64bb
Merge branch 'main' into reduce_optimize
ehwan Aug 22, 2025
8fa329d
remove IntermediateState ToIndex generic
ehwan Aug 22, 2025
75b709e
add ShiftTarget
ehwan Aug 22, 2025
d32eceb
fix grammar holds IntermediateState
ehwan Aug 22, 2025
597951f
make nonterm_shift_map also take ShiftTarget
ehwan Aug 22, 2025
e47bd45
single-token-reduce states optimization
ehwan Aug 22, 2025
2e0c94b
reduce_action returns boolean that if the last pushed data was not empty
ehwan Aug 23, 2025
dd79298
do not emit reduce_action for optimized out production rules
ehwan Aug 23, 2025
c7560bd
Merge branch 'main' into reduce_optimize
ehwan Aug 23, 2025
0ca34cd
Update rusty_lr_parser/src/grammar.rs
ehwan Aug 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions rusty_lr_core/src/builder/state.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ where
Term: Ord,
{
fn from(mut state: crate::builder::State<crate::TerminalSymbol<Term>, NonTerm>) -> Self {
use crate::parser::state::ShiftTarget;

let error_shift = state
.shift_goto_map_term
.remove(&crate::TerminalSymbol::Error);
Expand All @@ -62,11 +64,31 @@ where
shift_goto_map_term: state
.shift_goto_map_term
.into_iter()
.map(|(term, state_index)| (term.into_term().unwrap(), state_index))
.map(|(term, state_index)| {
(
term.into_term().unwrap(),
ShiftTarget {
state: state_index.into(),
push: true,
},
)
})
.collect(),
error_shift,
eof_shift,
shift_goto_map_nonterm: state.shift_goto_map_nonterm.into_iter().collect(),
shift_goto_map_nonterm: state
.shift_goto_map_nonterm
.into_iter()
.map(|(nonterm, state_index)| {
(
nonterm,
ShiftTarget {
state: state_index.into(),
push: true,
},
)
})
.collect(),
reduce_map: state
.reduce_map
.into_iter()
Expand Down
5 changes: 3 additions & 2 deletions rusty_lr_core/src/parser/data_stack.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ pub trait DataStack: Sized + Default {
fn split_off(&mut self, at: usize) -> Self;
fn append(&mut self, other: &mut Self);

/// performs a reduce action with the given rule index
/// Performs a reduce action with the given rule index.
/// Returns false if the empty tag was pushed by this reduce action, true otherwise.
fn reduce_action(
// the child tokens for the reduction
// the caller (usually from generated code) must pops all of the tokens used for this reduce_action
Expand All @@ -54,5 +55,5 @@ pub trait DataStack: Sized + Default {
userdata: &mut Self::UserData,
// location of this non-terminal, e.g. `@$`
location0: &mut Self::Location,
) -> Result<(), Self::ReduceActionError>;
) -> Result<bool, Self::ReduceActionError>;
}
46 changes: 31 additions & 15 deletions rusty_lr_core/src/parser/deterministic/context.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ impl<Data: DataStack, StateIndex: Index + Copy> Context<Data, StateIndex> {
let mut popped_states = states.drain(states.len() - len..).collect::<Vec<_>>();
let last_state = states.last().unwrap().into_usize();
if let Some(next_state) = parser.get_states()[last_state].shift_goto_nonterm(&nonterm) {
states.push(StateIndex::from_usize_unchecked(next_state));
states.push(StateIndex::from_usize_unchecked(next_state.state));
self.expected_token_impl(parser, states, terms, nonterms);
states.pop();
}
Expand Down Expand Up @@ -275,6 +275,8 @@ impl<Data: DataStack, StateIndex: Index + Copy> Context<Data, StateIndex> {
}

// try shift given term again
// to check if the given terminal should be merged with `error` token
// or it can be shift right after the error token
if let Some(next_state) = parser.get_states()
[self.state_stack.last().unwrap().into_usize()]
.shift_goto_class(class)
Expand All @@ -283,12 +285,17 @@ impl<Data: DataStack, StateIndex: Index + Copy> Context<Data, StateIndex> {
self.tree_stack
.push(crate::tree::Tree::new_terminal(term.clone()));

// shift with `error` token
self.data_stack.push_terminal(term.into_term().unwrap());
// shift after `error` token
if next_state.push {
self.data_stack.push_terminal(term.into_term().unwrap());
} else {
self.data_stack.push_empty();
}

self.location_stack.push(location.unwrap());
self.precedence_stack.push(shift_prec);
self.state_stack
.push(StateIndex::from_usize_unchecked(next_state));
.push(StateIndex::from_usize_unchecked(next_state.state));
} else {
// merge term with previous error

Expand Down Expand Up @@ -400,7 +407,7 @@ impl<Data: DataStack, StateIndex: Index + Copy> Context<Data, StateIndex> {
Data::Location::new(self.location_stack.iter().rev(), tokens_len);

// call reduce action
match Data::reduce_action(
let non_empty_pushed = match Data::reduce_action(
&mut self.data_stack,
&mut self.location_stack,
reduce_rule,
Expand All @@ -409,7 +416,7 @@ impl<Data: DataStack, StateIndex: Index + Copy> Context<Data, StateIndex> {
userdata,
&mut new_location,
) {
Ok(_) => {}
Ok(ret) => ret,
Err(err) => {
return Err(ParseError::ReduceAction(term, location, err));
}
Expand Down Expand Up @@ -438,7 +445,11 @@ impl<Data: DataStack, StateIndex: Index + Copy> Context<Data, StateIndex> {
.shift_goto_nonterm(&rule.name)
{
self.state_stack
.push(StateIndex::from_usize_unchecked(next_state_id));
.push(StateIndex::from_usize_unchecked(next_state_id.state));
if !next_state_id.push && non_empty_pushed {
self.data_stack.pop();
self.data_stack.push_empty();
}
} else {
unreachable!("shift nonterm failed");
}
Expand All @@ -450,20 +461,25 @@ impl<Data: DataStack, StateIndex: Index + Copy> Context<Data, StateIndex> {
// shift with terminal
if let Some(next_state_id) = shift_to {
self.state_stack
.push(StateIndex::from_usize_unchecked(next_state_id));
.push(StateIndex::from_usize_unchecked(next_state_id.state));

#[cfg(feature = "tree")]
self.tree_stack
.push(crate::tree::Tree::new_terminal(term.clone()));

match term {
TerminalSymbol::Term(term) => {
self.data_stack.push_terminal(term);
}
TerminalSymbol::Error | TerminalSymbol::Eof => {
self.data_stack.push_empty();
if next_state_id.push {
match term {
TerminalSymbol::Term(term) => {
self.data_stack.push_terminal(term);
}
TerminalSymbol::Error | TerminalSymbol::Eof => {
self.data_stack.push_empty();
}
}
} else {
self.data_stack.push_empty();
}

// location is `Some` if it is not `Eof`
if let Some(location) = location {
self.location_stack.push(location);
Expand Down Expand Up @@ -618,7 +634,7 @@ impl<Data: DataStack, StateIndex: Index + Copy> Context<Data, StateIndex> {
[state_stack.last().unwrap().into_usize()]
.shift_goto_nonterm(&rule.name)
{
state_stack.push(StateIndex::from_usize_unchecked(next_state_id));
state_stack.push(StateIndex::from_usize_unchecked(next_state_id.state));
} else {
unreachable!("shift nonterm failed");
}
Expand Down
53 changes: 35 additions & 18 deletions rusty_lr_core/src/parser/nondeterministic/context.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -453,12 +453,16 @@ impl<Data: DataStack, StateIndex: Index + Copy> Context<Data, StateIndex> {
userdata,
&mut new_location,
) {
Ok(_) => {
Ok(non_empty_pushed) => {
if let Some(nonterm_shift_state) =
parser.get_states()[state].shift_goto_nonterm(&rule.name)
{
node.state_stack
.push(StateIndex::from_usize_unchecked(nonterm_shift_state));
.push(StateIndex::from_usize_unchecked(nonterm_shift_state.state));
if !nonterm_shift_state.push && non_empty_pushed {
node.data_stack.pop();
node.data_stack.push_empty();
}
node.location_stack.push(new_location);
node.precedence_stack.push(precedence);
#[cfg(feature = "tree")]
Expand Down Expand Up @@ -1051,7 +1055,7 @@ impl<Data: DataStack, StateIndex: Index + Copy> Context<Data, StateIndex> {
let node_ = self.node_mut(node);
node_
.state_stack
.push(StateIndex::from_usize_unchecked(shift));
.push(StateIndex::from_usize_unchecked(shift.state));
node_.precedence_stack.push(shift_prec);
if let Some(location) = &location {
node_.location_stack.push(location.clone());
Expand All @@ -1061,13 +1065,17 @@ impl<Data: DataStack, StateIndex: Index + Copy> Context<Data, StateIndex> {
.tree_stack
.push(crate::tree::Tree::new_terminal(term.clone()));

match term {
TerminalSymbol::Term(term) => {
node_.data_stack.push_terminal(term);
}
TerminalSymbol::Eof | TerminalSymbol::Error => {
node_.data_stack.push_empty();
if shift.push {
match term {
TerminalSymbol::Term(term) => {
node_.data_stack.push_terminal(term);
}
TerminalSymbol::Eof | TerminalSymbol::Error => {
node_.data_stack.push_empty();
}
}
} else {
node_.data_stack.push_empty();
}

self.next_nodes.push(node);
Expand All @@ -1083,7 +1091,7 @@ impl<Data: DataStack, StateIndex: Index + Copy> Context<Data, StateIndex> {
let node_ = self.node_mut(node);
node_
.state_stack
.push(StateIndex::from_usize_unchecked(shift));
.push(StateIndex::from_usize_unchecked(shift.state));
node_.precedence_stack.push(shift_prec);
if let Some(location) = location {
node_.location_stack.push(location);
Expand All @@ -1093,13 +1101,17 @@ impl<Data: DataStack, StateIndex: Index + Copy> Context<Data, StateIndex> {
.tree_stack
.push(crate::tree::Tree::new_terminal(term.clone()));

match term {
TerminalSymbol::Term(term) => {
node_.data_stack.push_terminal(term);
}
TerminalSymbol::Eof | TerminalSymbol::Error => {
node_.data_stack.push_empty();
if shift.push {
match term {
TerminalSymbol::Term(term) => {
node_.data_stack.push_terminal(term);
}
TerminalSymbol::Eof | TerminalSymbol::Error => {
node_.data_stack.push_empty();
}
}
} else {
node_.data_stack.push_empty();
}

self.next_nodes.push(node);
Expand Down Expand Up @@ -1243,14 +1255,19 @@ impl<Data: DataStack, StateIndex: Index + Copy> Context<Data, StateIndex> {
// and b is fed, shift error and b
let node = self.node_mut(error_node);
node.state_stack
.push(StateIndex::from_usize_unchecked(next_state));
.push(StateIndex::from_usize_unchecked(next_state.state));
node.precedence_stack.push(shift_prec);
node.location_stack.push(location.clone());
#[cfg(feature = "tree")]
node.tree_stack.push(crate::tree::Tree::new_terminal(
TerminalSymbol::Term(term.clone()),
));
node.data_stack.push_terminal(term.clone());

if next_state.push {
node.data_stack.push_terminal(term.clone());
} else {
node.data_stack.push_empty();
}

self.current_nodes.push(error_node);
} else {
Expand Down
Loading