-
Notifications
You must be signed in to change notification settings - Fork 94
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
13 changed files
with
264 additions
and
59 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,4 +3,4 @@ argcomplete | |
phonenumbers | ||
pandas | ||
sklearn | ||
typing_extensions | ||
typing_extensions |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
spacy-nightly[transformers]>=3.0.0rc1; python_version >= '3.6' | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
import warnings | ||
from typing import Generator, Iterable, Optional, Sequence | ||
|
||
try: | ||
import spacy | ||
from wasabi import msg | ||
except ImportError as e: | ||
if getattr(e, 'name', None) == 'spacy': | ||
warnings.warn("Could not find module 'spacy'. If you want to use extras," | ||
" make sure you install scrubadub with 'pip install scrubadub[spacy]'") | ||
|
||
|
||
from .base import Detector | ||
from ..filth import NamedEntityFilth, Filth, NameFilth, OrganizationFilth | ||
from ..utils import CanonicalStringSet | ||
|
||
|
||
class NamedEntityDetector(Detector): | ||
"""Use spacy's named entity recognition to clean named entities. | ||
List specific entities to include passing ``named_entities``, e.g. | ||
(PERSON) | ||
""" | ||
filth_cls_map = { | ||
'PERSON': NameFilth, | ||
'ORG': OrganizationFilth | ||
} | ||
name = 'named_entity' | ||
|
||
disallowed_nouns = CanonicalStringSet(["skype"]) | ||
|
||
def __init__(self, named_entities: Iterable[str] = {'PERSON'}, | ||
model: str = "en_core_web_trf", **kwargs): | ||
# Spacy NER are all upper cased | ||
self.named_entities = {entity.upper() for entity in named_entities} | ||
if model not in spacy.info()['pipelines']: | ||
msg.info("Downloading spacy model {}".format(model)) | ||
spacy.cli.download(model) | ||
|
||
self.nlp = spacy.load(model) | ||
# Only enable necessary pipes | ||
self.nlp.select_pipes(enable=["transformer", "tagger", "parser", "ner"]) | ||
super(NamedEntityDetector, self).__init__(**kwargs) | ||
|
||
def iter_filth_documents(self, doc_names: Sequence[Optional[str]], | ||
doc_list: Sequence[str]) -> Generator[Filth, None, None]: | ||
for doc_name, doc in zip(doc_names, self.nlp.pipe(doc_list)): | ||
for ent in doc.ents: | ||
if ent.label_ in self.named_entities: | ||
# If there is no standard 'filth', returns a NamedEntity filth | ||
filth_cls = self.filth_cls_map.get(ent.label_, NamedEntityFilth) | ||
yield filth_cls(beg=ent.start_char, | ||
end=ent.end_char, | ||
text=ent.text, | ||
document_name=(str(doc_name) if doc_name else None), # None if no doc_name provided | ||
detector_name=self.name, | ||
label=ent.label_) | ||
|
||
def iter_filth(self, text: str, document_name: Optional[str] = None) -> Generator[Filth, None, None]: | ||
yield from self.iter_filth_documents([document_name], [text]) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
from .base import Filth | ||
|
||
|
||
class NamedEntityFilth(Filth): | ||
""" | ||
Default filth type, for named entities (e.g. the ones in https://nightly.spacy.io/models/en#en_core_web_lg-labels), | ||
except the ones represented in any other filth. | ||
""" | ||
type = 'named_entity' | ||
|
||
def __init__(self, *args, label: str, **kwargs): | ||
super(NamedEntityFilth, self).__init__(*args, **kwargs) | ||
self.label = label.lower() | ||
self.replacement_string = "{}_{}".format(self.type, self.label) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.