Fix missed opportunities to merge parse states

cli/src/generate/build_tables/build_parse_table.rs

@@ -1,4 +1,4 @@
-use super::item::{ParseItem, ParseItemSet, TokenSet};
+use super::item::{ParseItem, ParseItemSet, ParseItemSetCore, TokenSet};
 use super::item_set_builder::ParseItemSetBuilder;
 use crate::error::{Error, Result};
 use crate::generate::grammars::{
@@ -13,10 +13,8 @@ use crate::generate::tables::{
 use core::ops::Range;
 use hashbrown::hash_map::Entry;
 use hashbrown::{HashMap, HashSet};
-use std::collections::hash_map::DefaultHasher;
 use std::collections::{BTreeMap, VecDeque};
 use std::fmt::Write;
-use std::hash::Hasher;
 use std::u32;
 
 #[derive(Clone)]
@@ -40,6 +38,7 @@ struct ParseTableBuilder<'a> {
     syntax_grammar: &'a SyntaxGrammar,
     lexical_grammar: &'a LexicalGrammar,
     variable_info: &'a Vec<VariableInfo>,
+    core_ids_by_core: HashMap<ParseItemSetCore<'a>, usize>,
     state_ids_by_item_set: HashMap<ParseItemSet<'a>, ParseStateId>,
     parse_state_info_by_id: Vec<ParseStateInfo<'a>>,
     parse_state_queue: VecDeque<ParseStateQueueEntry>,
@@ -94,13 +93,19 @@ impl<'a> ParseTableBuilder<'a> {
         preceding_auxiliary_symbols: &AuxiliarySymbolSequence,
         item_set: ParseItemSet<'a>,
     ) -> ParseStateId {
-        let mut hasher = DefaultHasher::new();
-        item_set.hash_unfinished_items(&mut hasher);
-        let unfinished_item_signature = hasher.finish();
-
         match self.state_ids_by_item_set.entry(item_set) {
             Entry::Occupied(o) => *o.get(),
             Entry::Vacant(v) => {
+                let core = v.key().core();
+                let core_count = self.core_ids_by_core.len();
+                let core_id = match self.core_ids_by_core.entry(core) {
+                    Entry::Occupied(e) => *e.get(),
+                    Entry::Vacant(e) => {
+                        e.insert(core_count);
+                        core_count
+                    }
+                };
+
                 let state_id = self.parse_table.states.len();
                 self.parse_state_info_by_id
                     .push((preceding_symbols.clone(), v.key().clone()));
@@ -111,7 +116,7 @@ impl<'a> ParseTableBuilder<'a> {
                     external_lex_state_id: 0,
                     terminal_entries: HashMap::new(),
                     nonterminal_entries: HashMap::new(),
-                    unfinished_item_signature,
+                    core_id,
                 });
                 self.parse_state_queue.push_back(ParseStateQueueEntry {
                     state_id,
@@ -759,6 +764,7 @@ pub(crate) fn build_parse_table<'a>(
         item_set_builder,
         variable_info,
         state_ids_by_item_set: HashMap::new(),
+        core_ids_by_core: HashMap::new(),
         parse_state_info_by_id: Vec::new(),
         parse_state_queue: VecDeque::new(),
         parse_table: ParseTable {

cli/src/generate/build_tables/item.rs

@@ -1,6 +1,4 @@
-use crate::generate::grammars::{
-    LexicalGrammar, Production, ProductionStep, SyntaxGrammar,
-};
+use crate::generate::grammars::{LexicalGrammar, Production, ProductionStep, SyntaxGrammar};
 use crate::generate::rules::Associativity;
 use crate::generate::rules::{Symbol, SymbolType};
 use lazy_static::lazy_static;
@@ -50,6 +48,11 @@ pub(crate) struct ParseItemSet<'a> {
     pub entries: Vec<(ParseItem<'a>, TokenSet)>,
 }
 
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(crate) struct ParseItemSetCore<'a> {
+    pub entries: Vec<ParseItem<'a>>,
+}
+
 pub(crate) struct ParseItemDisplay<'a>(
     pub &'a ParseItem<'a>,
     pub &'a SyntaxGrammar,
@@ -268,20 +271,8 @@ impl<'a> ParseItemSet<'a> {
         }
     }
 
-    pub fn hash_unfinished_items(&self, h: &mut impl Hasher) {
-        let mut previous_variable_index = u32::MAX;
-        let mut previous_step_index = u32::MAX;
-        for (item, _) in self.entries.iter() {
-            if item.step().is_some()
-                && (item.variable_index != previous_variable_index
-                    || item.step_index != previous_step_index)
-            {
-                h.write_u32(item.variable_index);
-                h.write_u32(item.step_index);
-                previous_variable_index = item.variable_index;
-                previous_step_index = item.step_index;
-            }
-        }
+    pub fn core(&self) -> ParseItemSetCore<'a> {
+        ParseItemSetCore { entries: self.entries.iter().map(|e| e.0).collect() }
     }
 }
 
@@ -513,3 +504,12 @@ impl<'a> Hash for ParseItemSet<'a> {
         }
     }
 }
+
+impl<'a> Hash for ParseItemSetCore<'a> {
+    fn hash<H: Hasher>(&self, hasher: &mut H) {
+        hasher.write_usize(self.entries.len());
+        for item in &self.entries {
+            item.hash(hasher);
+        }
+    }
+}