Merge pull request #32 from knadh/tokenize-import

knadh · Oct 24, 2023 · 84619de · 84619de
2 parents b0b8b3b + de554cf
commit 84619de
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 1 deletion.
diff --git a/internal/importer/importer.go b/internal/importer/importer.go
@@ -187,6 +187,17 @@ func (im *Importer) readEntry(r []string) (entry, error) {
 		e.Initial = strings.ToUpper(string(e.Content[0]))
 	}
 
+	// If the Postgres tokenizer is not set, and there are no tokens supplied,
+	// see if the language has a custom one and use it.
+	if lang.Tokenizer != nil && e.TSVectorLang == "" && e.TSVectorTokens == "" {
+		tks, err := lang.Tokenizer.ToTokens(e.Content, lang.ID)
+		if err != nil {
+			return e, fmt.Errorf("error tokenizing content (word) at column 1: %v", err)
+		}
+
+		e.TSVectorTokens = strings.Join(tks, " ")
+	}
+
 	defTypeStr := cleanString(r[9])
 	if typ == typeDef {
 		defTypes := splitString(defTypeStr)

diff --git a/tokenizers/indicphone/indicphone.go b/tokenizers/indicphone/indicphone.go
@@ -47,7 +47,7 @@ func (ip *IndicPhone) ToTokens(s string, lang string) ([]string, error) {
 		}
 
 		if key0 == "" {
-			return nil, nil
+			continue
 		}
 
 		tokens = append(tokens,