debug

annotation/AlpacaTag/server/models.py

@@ -5,8 +5,22 @@
 from django.contrib.auth.models import User
 from django.contrib.staticfiles.storage import staticfiles_storage
 from .utils import get_key_choices
+
 import spacy
-nlp = spacy.load('en_core_web_sm')
+from spacy.tokens import Doc
+
+class WhitespaceTokenizer(object):
+    def __init__(self, vocab):
+        self.vocab = vocab
+
+    def __call__(self, text):
+        words = text.split(' ')
+        # All tokens 'own' a subsequent space character in this tokenizer
+        spaces = [True] * len(words)
+        return Doc(self.vocab, words=words, spaces=spaces)
+
+nlp = spacy.load("en_core_web_sm")
+nlp.tokenizer = WhitespaceTokenizer(nlp.vocab)
 
 
 class Project(models.Model):

annotation/AlpacaTag/server/views.py

@@ -20,6 +20,7 @@
 
 import spacy
 from spacy.tokens import Doc
+
 class WhitespaceTokenizer(object):
     def __init__(self, vocab):
         self.vocab = vocab