removes explicit textconfig as arg

sadit · Mar 30, 2023 · b93aa89 · b93aa89 · sadit · Mar 30, 2023
1 parent d581f8d
commit b93aa89
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 8 deletions.
diff --git a/Project.toml b/Project.toml
@@ -1,7 +1,7 @@
 name = "TextSearch"
 uuid = "7f6f6c8a-3b03-11e9-223d-e7d88259bd6c"
 authors = ["Eric S. Tellez <donsadit@gmail.com>"]
-version = "0.16.2"
+version = "0.16.1"
 
 [deps]
 CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"

diff --git a/src/voc.jl b/src/voc.jl
@@ -61,14 +61,14 @@ Computes a vocabulary from a corpus using the TextConfig `textconfig`.
 """
 function Vocabulary(textconfig::TextConfig, corpus::AbstractVector; minbatch=0)
     voc = Vocabulary(textconfig, length(corpus))
-    tokenize_and_append!(voc, textconfig, corpus; minbatch)
+    tokenize_and_append!(voc, corpus; minbatch)
     voc
 end
 
-function locked_tokenize_and_push(voc, textconfig, doc, buff, l)
+function locked_tokenize_and_push(voc, doc, buff, l)
     empty!(buff)
 
-    for token in tokenize(borrowtokenizedtext, textconfig, doc, buff)
+    for token in tokenize(borrowtokenizedtext, voc.textconfig, doc, buff)
         id = 0
         lock(l)
         try
@@ -90,11 +90,11 @@ function locked_tokenize_and_push(voc, textconfig, doc, buff, l)
 end
 
 """
-    tokenize_and_append!(voc::Vocabulary, textconfig::TextConfig, corpus; minbatch=0)
+    tokenize_and_append!(voc::Vocabulary, corpus; minbatch=0)
 
 Parse each document in the given corpus and appends each token to the vocabulary.
 """
-function tokenize_and_append!(voc::Vocabulary, textconfig::TextConfig, corpus; minbatch=0)
+function tokenize_and_append!(voc::Vocabulary, corpus; minbatch=0)
     l = Threads.SpinLock()
     n = length(corpus)
     minbatch = getminbatch(minbatch, n)
@@ -108,10 +108,10 @@ function tokenize_and_append!(voc::Vocabulary, textconfig::TextConfig, corpus; m
         try
             if doc isa AbstractVector
                 for text in doc
-                    locked_tokenize_and_push(voc, textconfig, text, buff, l)
+                    locked_tokenize_and_push(voc, text, buff, l)
                 end
             else # if doc isa AbstractString
-                locked_tokenize_and_push(voc, textconfig, doc, buff, l)
+                locked_tokenize_and_push(voc, doc, buff, l)
             end
         finally
             put!(TEXT_SEARCH_CACHES, buff)