Add the BucketIndexer trait and implement for HashIndexer

benches/subword.rs

@@ -2,7 +2,7 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use rand::distributions::Alphanumeric;
 use rand::{thread_rng, Rng};
 
-use finalfusion::subword::{FinalfusionHashIndexer, Indexer, SubwordIndices};
+use finalfusion::subword::{BucketIndexer, FinalfusionHashIndexer, Indexer, SubwordIndices};
 
 const MIN_N: usize = 3;
 const MAX_N: usize = 6;

src/subword.rs

@@ -18,6 +18,15 @@ pub trait Indexer {
     fn index_ngram(&self, ngram: &StrWithCharLen) -> u64;
 }
 
+/// N-Gram indexer with bucketing.
+pub trait BucketIndexer: Indexer {
+    /// Create a new indexer.
+    ///
+    /// The buckets argument is the number of buckets or the
+    /// bucket exponent (depending on the implementation).
+    fn new(buckets: usize) -> Self;
+}
+
 /// Indexer using a hash function.
 ///
 /// This indexer first hashes a given n-gram and then maps the
@@ -29,11 +38,14 @@ pub struct HashIndexer<H> {
     _phantom: PhantomData<H>,
 }
 
-impl<H> HashIndexer<H> {
+impl<H> BucketIndexer for HashIndexer<H>
+where
+    H: Default + Hasher,
+{
     /// Construct a `HashIndexer`.
     ///
     /// The largest possible bucket exponent is 64.
-    pub fn new(buckets_exp: usize) -> Self {
+    fn new(buckets_exp: usize) -> Self {
         assert!(
             buckets_exp <= 64,
             "The largest possible buckets exponent is 64."
@@ -301,7 +313,7 @@ mod tests {
     use maplit::hashmap;
     use std::collections::HashMap;
 
-    use super::{FinalfusionHashIndexer, NGrams, NGramsIndices, SubwordIndices};
+    use super::{BucketIndexer, FinalfusionHashIndexer, NGrams, NGramsIndices, SubwordIndices};
 
     #[test]
     fn ngrams_test() {

src/vocab.rs

@@ -8,7 +8,7 @@ use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
 
 use crate::io::private::{ChunkIdentifier, ReadChunk, WriteChunk};
 use crate::io::{Error, ErrorKind, Result};
-use crate::subword::{FinalfusionHashIndexer, SubwordIndices};
+use crate::subword::{BucketIndexer, FinalfusionHashIndexer, SubwordIndices};
 
 #[derive(Clone, Debug, Eq, PartialEq)]
 /// Index of a vocabulary word.