-
Notifications
You must be signed in to change notification settings - Fork 0
/
populate_annotations.py
64 lines (46 loc) · 1.72 KB
/
populate_annotations.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
import os
os.environ.setdefault('DJANGO_SETTINGS_MODULE','esports_ner.settings')
import django
django.setup()
import json
from train_crf import train
#from train import ingest_json_document, NLP
CUR_DIR = os.path.dirname(os.path.abspath(__file__))
DATA_DIR = os.path.join(CUR_DIR, "CRF", "data")
from crf_predict.models import Doc, Ent, TestDoc
def populate_test(doc, text, title, game):
# populate Doc table
d = TestDoc.objects.get_or_create(title=title, text=text, game=game)[0]
d.save()
def populate_train(doc, text, title, game):
# populate Doc table
d = Doc.objects.get_or_create(title=title, text=text, game=game)[0]
d.save()
#populate Ent table
for ent in doc.ents:
e = Ent.objects.get_or_create(doc=d, label=ent.label_, text=ent.text,
start=ent.start, end=ent.end)[0]
e.save()
if __name__=="__main__":
with open(os.path.join(DATA_DIR, "corpus_train.jsonl")) as json_file:
id = 0
for review in json_file:
doc_json = json.loads(review)
doc = train.ingest_json_document(doc_json, train.NLP)
text = doc_json['text']
title = text[0:30] + "..."
game = "unk"
populate_train(doc, text, title, game)
print(id)
id += 1
with open(os.path.join(DATA_DIR, "corpus_dev.jsonl")) as json_file:
id = 0
for review in json_file:
doc_json = json.loads(review)
doc = train.ingest_json_document(doc_json, train.NLP)
text = doc_json['text']
title = text[0:30] + "..."
game = "unk"
populate_test(doc, text, title, game)
print(id)
id += 1