-
Notifications
You must be signed in to change notification settings - Fork 0
/
wordcountReducer_StdDev.py
154 lines (132 loc) · 4.32 KB
/
wordcountReducer_StdDev.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
#!/usr/bin/env python
from operator import itemgetter
import sys
import math
current_word = None
current_count = 0
word = None
wordcount = {}
count_list = []
wordcountByFile = {}
def addToDictionary(current_word, current_count):
wordkey = current_word.split("_")[0]
if wordcount.get(wordkey) == None:
count_list = []
count_list.append(current_count)
wordcount[wordkey] = count_list
else:
wordcount.get(wordkey).append(current_count)
def addToDictByFile(current_word, current_count):
if wordcountByFile.get(current_word) == None:
wordcountByFile[current_word] = current_count
else:
count_val = wordcountByFile.get(current_word)
wordcountByFile[current_word] = count_val + 1
def addToWordCount(current_word, current_count):
if wordcount.get(current_word) == None:
wordcount[current_word] = current_count
else:
count_val = wordcount.get(current_word)
wordcount[current_word] = count_val + 1
def calculateMinMaxAvg():
average = -1
minimum = -1
maximum = -1
for word in wordcount:
#print wordcount.get(word)
minimum = min(wordcount.get(word))
maximum = max(wordcount.get(word))
average = sum(wordcount.get(word))/229.0
print '%s\t%s\t%s\t%s' % (word, minimum, maximum, average)
def calculateAverage():
average = -1
for word in wordcount:
average = sum(wordcount.get(word))/5.0
return average
def calculateStdDev():
stdDev = -1
average = -1
for word in wordcount:
partialSquare = -1
sumOfSquare = -1
average = sum(wordcount.get(word))/5.0
countlist = wordcount.get(word)
for count in countlist:
partialSquare = (count-average)**2
sumOfSquare += partialSquare
stdDev = math.sqrt(sumOfSquare/5.0)
print '%s\t%s\t%s' % (word, average, stdDev)
#def slicedict(d, s):
# return {k:v for k,v in d.iteritems() if k.startswith(s)}
def slicedict(sourcedict, string):
#print sourcedict
#print string
newdict = {}
for key in sourcedict.keys():
if key.startswith(string):
newdict[key] = sourcedict[key]
return newdict
def calculateStdDev2():
#stdDev = 0
average = 0
for word in wordcount:
#print word
average = wordcount.get(word)/5.0
partialSquare = 0
sumOfSquare = 0
stdDev = 0
tempdict = slicedict(wordcountByFile, word+"_")
#print tempdict
for tempkey in tempdict:
partialSquare = (int(tempdict.get(tempkey))-average)*(int(tempdict.get(tempkey))-average)
#print partialSquare
sumOfSquare += partialSquare
stdDev = math.sqrt(sumOfSquare/5.0)
print '%s\t%s\t%s' % (word, average, stdDev)
# input comes from STDIN
for line in sys.stdin:
# remove leading and trailing whitespace
line = line.strip()
# parse the input we got from mapper.py
word, count = line.split('\t', 1)
dict_word = ""
dic_count = ""
# convert count (currently a string) to int
try:
dict_word = word + "_" + count.split("_")[1]
dic_count = count.split("_")[0]
dic_count = int(dic_count)
except ValueError:
# count was not a number, so silently
# ignore/discard this line
continue
addToDictByFile(dict_word, dic_count)
addToWordCount(word, dic_count)
calculateStdDev2()
'''
# convert count (currently a string) to int
try:
word = word + count.split("_")[1]
count = count.split("_")[0]
count = int(count)
except ValueError:
# count was not a number, so silently
# ignore/discard this line
continue
# this IF-switch only works because Hadoop sorts map output
# by key (here: word) before it is passed to the reducer
if current_word == word:
current_count += count
else:
if current_word:
# write result to STDOUT
#print '%s\t%s' % (current_word, current_count)
addToDictionary(current_word, current_count)
current_count = count
current_word = word
# do not forget to output the last word if needed!
if current_word == word:
#print '%s\t%s' % (current_word, current_count)
addToDictionary(current_word, current_count)
'''
#calculateStdDev(calculateAverage())