Merge pull request #125 from aquatiko/fix-punctuation

Fixed strip_punctuation regex
JuliaText · Jan 21, 2019 · f33d1d0 · f33d1d0
2 parents 18aa868 + 76d922d
commit f33d1d0
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 1 deletion.
diff --git a/src/preprocessing.jl b/src/preprocessing.jl
@@ -340,7 +340,7 @@ function _build_regex_patterns(lang, flags::UInt32, patterns::Set{T}, words::Set
     if (flags & strip_non_letters) > 0
         push!(patterns, "[^a-zA-Z\\s]")
     else
-        ((flags & strip_punctuation) > 0) && push!(patterns, "[,;:.!?()-\\\\]+")
+        ((flags & strip_punctuation) > 0) && push!(patterns, "[-.,:;,!?'\"\\[\\]\\(\\)\\{\\}|\\`#\$%@^&*_+<>]+")
         ((flags & strip_numbers) > 0) && push!(patterns, "\\d+")
     end
     if (flags & strip_articles) > 0

diff --git a/test/preprocessing.jl b/test/preprocessing.jl
@@ -90,4 +90,15 @@
     crps = Corpus(StringDocument.(sample_texts))
     @test isempty(setdiff(frequent_terms(crps),["string","is"]))
     @test isempty(setdiff(sparse_terms(crps,0.3),["!"]))
+
+    #Tests strip_punctuation regex conditions
+    str = Document("These punctuations should be removed [-.,:;,!?'\"[](){}|\`#\$%@^&*_+<>")
+    answer = Document("These punctuations should be removed  ")
+    prepare!(str, strip_punctuation)
+    @test isequal(str.text, answer.text)
+
+    str = Document("Intel(tm) Core i5-3300k, is a geat CPU! ")
+    answer = Document("Intel tm  Core i5 3300k  is a geat CPU  ")   #tests old implementation   
+    prepare!(str, strip_punctuation)
+    @test isequal(str.text, answer.text)
 end