Fix indicphone skipping all tokenization because of one bad token.

knadh · Oct 24, 2023 · de554cf · de554cf
1 parent 3190001
commit de554cf
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/tokenizers/indicphone/indicphone.go b/tokenizers/indicphone/indicphone.go
@@ -47,7 +47,7 @@ func (ip *IndicPhone) ToTokens(s string, lang string) ([]string, error) {
 		}
 
 		if key0 == "" {
-			return nil, nil
+			continue
 		}
 
 		tokens = append(tokens,