-
Notifications
You must be signed in to change notification settings - Fork 0
/
TermWeighting.py
76 lines (51 loc) · 1.77 KB
/
TermWeighting.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
import math
def binaryTermWeighting(terms, documents):
binaryWeight = []
for document in documents:
documentWeight = []
for term in terms:
if term in document:
documentWeight.append(1)
else:
documentWeight.append(0)
binaryWeight.append(documentWeight)
return binaryWeight
def rawTermWeighting(terms, documents):
rawWeight = []
for document in documents:
documentWeight = []
for term in terms:
documentWeight.append(document.count(term))
rawWeight.append(documentWeight)
return rawWeight
def logTermWeighting(terms, documents):
logWeight = []
for document in documents:
documentWeight = []
for term in terms:
count = document.count(term)
if count > 0:
documentWeight.append(1 + math.log10(count))
else:
documentWeight.append(0)
logWeight.append(documentWeight)
return logWeight
def documentFrequency(terms, documents):
df = []
for term in terms:
dfWeight = 0
for document in documents:
if term in document:
dfWeight += 1
df.append(dfWeight)
return df
def inverseDocumentFrequency(dfs, documents):
return [math.log10(len(documents) / df) for df in dfs]
def tf_idf(termFrequencies, inverseDocumentFrequencies):
tf_idf = []
for documentTermFrequencies in termFrequencies:
row_tf_idf = []
for i in range(0, len(inverseDocumentFrequencies)):
row_tf_idf.append(documentTermFrequencies[i]*inverseDocumentFrequencies[i])
tf_idf.append(row_tf_idf)
return tf_idf