feat(clean): add clean_text function

sfu-db · Apr 21, 2021 · 55d3ae9 · 55d3ae9
1 parent 654ce0d
commit 55d3ae9
Show file tree

Hide file tree

Showing 5 changed files with 1,245 additions and 2 deletions.
diff --git a/dataprep/assets/english_stopwords.py b/dataprep/assets/english_stopwords.py
@@ -1,4 +1,4 @@
-english_stopwords = [
+english_stopwords = {
     "i",
     "me",
     "my",
@@ -178,4 +178,4 @@
     "won't",
     "wouldn",
     "wouldn't",
-]
+}
diff --git a/dataprep/clean/__init__.py b/dataprep/clean/__init__.py
@@ -27,6 +27,8 @@
 
 from .clean_df import clean_df
 
+from .clean_text import clean_text, default_text_pipeline
+
 
 __all__ = [
     "clean_lat_long",
@@ -50,4 +52,6 @@
     "clean_currency",
     "validate_currency",
     "clean_df",
+    "clean_text",
+    "default_text_pipeline",
 ]