Merge pull request #33 from ddworken/master

Added parameter to configure the bloom filter size
DanMcInerney · Dec 18, 2019 · f6e65c0 · f6e65c0
2 parents 06ad0aa + a1617d0
commit f6e65c0
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 27 deletions.
diff --git a/xsscrapy/bloom.py b/xsscrapy/bloom.py
diff --git a/xsscrapy/bloomfilters.py b/xsscrapy/bloomfilters.py
@@ -1,13 +1,14 @@
 from pybloom import BloomFilter
 from scrapy.utils.job import job_dir
 from scrapy.dupefilters import BaseDupeFilter
+from settings import bloomfilterSize
 
 class BloomURLDupeFilter(BaseDupeFilter):
     """Request Fingerprint duplicates filter"""
 
     def __init__(self, path=None):
         self.file = None
-        self.fingerprints = BloomFilter(3000000, 0.0001)
+        self.fingerprints = BloomFilter(bloomfilterSize*10, 0.0001)
 
     @classmethod
     def from_settings(cls, settings):

diff --git a/xsscrapy/middlewares.py b/xsscrapy/middlewares.py
@@ -3,12 +3,13 @@
 from pybloom import BloomFilter
 import random
 import re
+from settings import bloomfilterSize
 
 # Filter out duplicate requests with Bloom filters since they're much easier on memory
 #URLS_FORMS_HEADERS = BloomFilter(3000000, 0.00001)
-URLS_SEEN = BloomFilter(300000, .0001)
-FORMS_SEEN = BloomFilter(300000, .0001)
-HEADERS_SEEN = BloomFilter(300000, .0001)
+URLS_SEEN = BloomFilter(bloomfilterSize, .0001)
+FORMS_SEEN = BloomFilter(bloomfilterSize, .0001)
+HEADERS_SEEN = BloomFilter(bloomfilterSize, .0001)
 USER_AGENT_LIST = ['Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36',
                    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36',
                    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14',

diff --git a/xsscrapy/settings.py b/xsscrapy/settings.py
@@ -38,3 +38,5 @@
 
 CONCURRENT_REQUESTS = 30
 
+# If you get bloom filter problems, increase this number
+bloomfilterSize = 300000