from cluster

jadbin · Oct 25, 2017 · ad62b18 · ad62b18
1 parent a27da6c
commit ad62b18
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 4 deletions.
diff --git a/xpaw/cluster.py b/xpaw/cluster.py
@@ -29,11 +29,11 @@ def __init__(self, proj_dir=None, config=None):
         self.dupefilter = self._new_object_from_cluster(self.config.get("dupefilter_cls"), self)
         self.downloader = Downloader(timeout=self.config.getfloat("downloader_timeout"),
                                      loop=self.loop)
-        self.spider = load_object(self.config["spider"])(self.config)
+        self.spider = self._new_object_from_cluster(self.config.get("spider"), self)
         log.debug("Spider: {}".format(".".join((type(self.spider).__module__,
                                                 type(self.spider).__name__))))
-        self.downloadermw = DownloaderMiddlewareManager.from_cluster(self.config)
-        self.spidermw = SpiderMiddlewareManager.from_cluster(self.config)
+        self.downloadermw = DownloaderMiddlewareManager.from_cluster(self)
+        self.spidermw = SpiderMiddlewareManager.from_cluster(self)
         self._last_request = None
         self._job_futures = None
         self._job_futures_done = set()

diff --git a/xpaw/spider.py b/xpaw/spider.py
@@ -8,9 +8,15 @@
 
 
 class Spider:
-    def __init__(self, config):
+    def __init__(self, config=None, **kwargs):
         self.config = config
 
+    @classmethod
+    def from_cluster(cls, cluster):
+        spider = cls(config=cluster.config)
+        spider.cluster = cluster
+        return spider
+
     @property
     def logger(self):
         return log