forked from chetannaik/question_answering_system
-
Notifications
You must be signed in to change notification settings - Fork 0
/
utils.py
executable file
·65 lines (46 loc) · 1.57 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
import numpy as np
from nltk.corpus import wordnet as wn
from nltk.stem.wordnet import WordNetLemmatizer
import config
__author__ = 'chetannaik'
FILTER_WORDS = []
def get_lemma(entry):
wnl = WordNetLemmatizer()
return str(wnl.lemmatize(entry.strip().lower()))
def get_filter_words():
process_synsets = wn.synsets('process')
global FILTER_WORDS
for p_s in process_synsets:
FILTER_WORDS.extend(p_s.lemma_names())
FILTER_WORDS = map(lambda x: str(x), set(FILTER_WORDS))
def has_filter_keyword(word_list):
for word in word_list:
if get_lemma(word) in FILTER_WORDS:
return True
return False
def remove_filter_words(input_string):
return_list = []
input_list = map(lambda x: x.strip(), input_string.split("|"))
for word in input_list:
lemmatized_word = get_lemma(word)
words = [lemmatized_word]
words.extend(lemmatized_word.split())
words = set(words)
if not words & set(FILTER_WORDS):
return_list.append(word)
return " | ".join(return_list)
def filter_score_for_logging(score):
if not np.isnan(float(score)):
return str(score)
else:
return str(np.nan)
def generate_experiment_scores(experiment):
roles = config.ROLES[experiment]
config.SCORES.extend(map(lambda x: x + '_SCORE', roles))
def main():
get_filter_words()
input_string = "physical process | changes"
print "Input String : {}".format(input_string)
print "Return String: {}".format(remove_filter_words(input_string))
if __name__ == '__main__':
main()