Skip to content

Commit

Permalink
Allow chunking model to reuse a spacy model instead of reloading
Browse files Browse the repository at this point in the history
thus, reinclude the chunking model test in the global spacy model test
  • Loading branch information
ophelielacroix committed Oct 15, 2020
1 parent 2d93b87 commit d7b80a8
Show file tree
Hide file tree
Showing 3 changed files with 12 additions and 32 deletions.
11 changes: 7 additions & 4 deletions danlp/models/spacy_models.py
Expand Up @@ -37,8 +37,8 @@ def load_spacy_model(cache_dir=DEFAULT_CACHE_DIR, verbose=False, textcat=None, v
return nlp


def load_spacy_chunking_model(cache_dir=DEFAULT_CACHE_DIR, verbose=False):
return SpacyChunking(cache_dir=cache_dir, verbose=verbose)
def load_spacy_chunking_model(spacy_model=None,cache_dir=DEFAULT_CACHE_DIR, verbose=False):
return SpacyChunking(model=spacy_model, cache_dir=cache_dir, verbose=verbose)



Expand All @@ -47,9 +47,12 @@ class SpacyChunking:
Spacy Chunking Model
"""

def __init__(self, cache_dir=DEFAULT_CACHE_DIR, verbose=False):
def __init__(self, model=None, cache_dir=DEFAULT_CACHE_DIR, verbose=False):

self.model = load_spacy_model(cache_dir=cache_dir, verbose=verbose)
if model == None:
self.model = load_spacy_model(cache_dir=cache_dir, verbose=verbose)
else:
self.model = model

def predict(self, text: Union[str, List[str]], bio=True):
"""
Expand Down
25 changes: 0 additions & 25 deletions tests/test_spacy_chunking_model.py

This file was deleted.

8 changes: 5 additions & 3 deletions tests/test_spacy_model.py
Expand Up @@ -3,7 +3,7 @@
import spacy
import operator
from danlp.download import download_model, DEFAULT_CACHE_DIR, _unzip_process_func
from danlp.models import load_spacy_model
from danlp.models import load_spacy_model, load_spacy_chunking_model
import os

class TestSpacyModel(unittest.TestCase):
Expand All @@ -17,16 +17,18 @@ def test_download(self):
self.assertListEqual(info['pipeline'], ['tagger', 'parser', 'ner'])
self.assertEqual(info['lang'], 'da')


def test_predictions(self):

nlp = load_spacy_model()
some_text = "Jeg gik en tur med Lars"
doc = nlp(some_text)
self.assertTrue(doc.is_parsed)
self.assertTrue(doc.is_nered)
self.assertTrue(doc.is_tagged)

chunker = load_spacy_chunking_model(spacy_model=nlp)
chunks = chunker.predict(some_text)
self.assertEqual(len(chunks), len(doc))


class TestSpacySentimentModel(unittest.TestCase):
def test_download(self):
Expand Down

0 comments on commit d7b80a8

Please sign in to comment.