make dupefilter support create from spider

rmax · Jan 5, 2017 · 384996d · acefei · Nov 21, 2017 · rmax
1 parent 500039b
commit 384996d
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 9 deletions.
diff --git a/src/scrapy_redis/dupefilter.py b/src/scrapy_redis/dupefilter.py
@@ -114,6 +114,15 @@ def request_fingerprint(self, request):
         """
         return request_fingerprint(request)
 
+    @classmethod
+    def from_spider(cls, spider):
+        settings = spider.settings
+        server = get_redis_from_settings(settings)
+        dupefilter_key = settings.get("SCHEDULER_DUPEFILTER_KEY", defaults.SCHEDULER_DUPEFILTER_KEY)
+        key = dupefilter_key % {'spider': spider.name}
+        debug = settings.getbool('DUPEFILTER_DEBUG')
+        return cls(server, key=key, debug=debug)
+
     def close(self, reason=''):
         """Delete data on close. Called by Scrapy's scheduler.
 

diff --git a/src/scrapy_redis/scheduler.py b/src/scrapy_redis/scheduler.py
@@ -134,15 +134,7 @@ def open(self, spider):
             raise ValueError("Failed to instantiate queue class '%s': %s",
                              self.queue_cls, e)
 
-        try:
-            self.df = load_object(self.dupefilter_cls)(
-                server=self.server,
-                key=self.dupefilter_key % {'spider': spider.name},
-                debug=spider.settings.getbool('DUPEFILTER_DEBUG'),
-            )
-        except TypeError as e:
-            raise ValueError("Failed to instantiate dupefilter class '%s': %s",
-                             self.dupefilter_cls, e)
+        self.df = load_object(self.dupefilter_cls).from_spider(spider)
 
         if self.flush_on_start:
             self.flush()