Closes #21

klbostee · Apr 29, 2009 · 3069d0e · 3069d0e
1 parent 81cae43
commit 3069d0e
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 7 deletions.
diff --git a/dumbo/lib.py b/dumbo/lib.py
@@ -63,13 +63,13 @@ def combiner(key_, values):
 
 def statsreducer(key, values):
     columns = iizip(*values)
-    s0 = sum(columns.next())
-    s1 = sum(columns.next())
-    s2 = sum(columns.next())
+    s0 = sum(columns.next()) # n
+    s1 = sum(columns.next()) # sum(x)
+    s2 = sum(columns.next()) # sum(x**2)
     minimum = min(columns.next())
     maximum = max(columns.next())
     mean = float(s1) / s0
-    std = sqrt(s0 * s2 - s1**2) / s0
+    std = sqrt((s2-s1**2/s0)/(s0-1)) # sample standard deviation
     yield (key, (s0, mean, std, minimum, maximum))
 
 

diff --git a/tests/testlib.py b/tests/testlib.py
@@ -4,11 +4,12 @@
 class TestLib(unittest.TestCase):
 
     def teststats(self):
-        input = [('testkey',i) for i in xrange(10)]
+        input = [('testkey',i) for i in xrange(3)]
         input = core.itermapred(input, lib.identitymapper, lib.statscombiner)
         output = dict(core.itermapred(input, lib.identitymapper, lib.statsreducer))
-        self.assertEqual(output['testkey'][0], 10)
-        self.assertEqual(output['testkey'][1], 4.5)
+        self.assertEqual(output['testkey'][0], 3) # n
+        self.assertEqual(output['testkey'][1], 1) # mean
+        self.assertEqual(output['testkey'][2], 1.0) # std 
 
 if __name__ == "__main__":
     suite = unittest.TestLoader().loadTestsFromTestCase(TestMapReduce)