-
Notifications
You must be signed in to change notification settings - Fork 6
/
ngram_score.py
30 lines (27 loc) · 1000 Bytes
/
ngram_score.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
'''
Allows scoring of text using n-gram probabilities
17/07/12
'''
from math import log10
class ngram_score(object):
def __init__(self,ngramfile,sep=' '):
''' load a file containing ngrams and counts, calculate log probabilities '''
self.ngrams = {}
f = open(ngramfile,"r")
for line in f:
key,count = line.split(sep)
self.ngrams[key] = int(count)
self.L = len(key)
self.N = sum(iter(self.ngrams.values()))
#calculate log probabilities
for key in self.ngrams.keys():
self.ngrams[key] = log10(float(self.ngrams[key])/self.N)
self.floor = log10(0.01/self.N)
def score(self,text):
''' compute the score of text '''
score = 0
ngrams = self.ngrams.__getitem__
for i in range(len(text)-self.L+1):
if text[i:i+self.L] in self.ngrams: score += ngrams(text[i:i+self.L])
else: score += self.floor
return score