skip fetching records when many spams included

shingetsu/cache.py

@@ -591,6 +591,7 @@ def check_data(self, res, stamp=None, id=None, begin=None, end=None):
         '''Check a data and add it cache.'''
         flag_got = False
         flag_spam = False
+        spam_count = 0
         count = 0
         for i in res:
             count += 1
@@ -610,6 +611,7 @@ def check_data(self, res, stamp=None, id=None, begin=None, end=None):
                     self.add_data(r, False)
                     r.remove()
                     flag_spam = True
+                    spam_count += 1
                 else:
                     self.add_data(r)
             else:
@@ -622,6 +624,9 @@ def check_data(self, res, stamp=None, id=None, begin=None, end=None):
                 sys.stderr.write("Warning: %s%s: broken record.\n" %
                                  (self.datfile, str_stamp))
             r.free()
+            if spam_count > config.accept_spam_count:
+                sys.stderr.write("Warning: %s: too many spams. skip fetching it.\n" % self.datfile)
+                break
         return count, flag_got, flag_spam
 
     def get_data(self, stamp=0, id="", node=None):
@@ -664,7 +669,7 @@ def get_with_range(self, node=None):
             head = node.talk('/head/%s/%d-' % (self.datfile, begin))
             res = RecordGetter(self.datfile, node, head)
 
-        count = self.check_data(res, begin=begin, end=now)
+        count, flag_got, flag_spam = self.check_data(res, begin=begin, end=now)
         if count:
             self.sync_status()
             if oldcount == 0:

shingetsu/config.py

@@ -157,6 +157,7 @@ def _get_version():
 nodes = 5                   # Nodes keeping in node list
 share_nodes = 5             # Nodes having the file
 search_depth = 30           # Search node size
+accept_spam_count = 10
 tiedfile_cache_size = 30
 
 broadcast = "../tool/broadcast.py"  # Broadcast script path