Merge pull request #2 from shubham76/updated-one

Removed the cache and duplication problem.
shubham76 · Feb 3, 2017 · d14f4c2 · d14f4c2
2 parents 0e4b7df + f54471b
commit d14f4c2
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/crik/crik/settings.py b/crik/crik/settings.py
@@ -79,6 +79,7 @@
 # Enable and configure HTTP caching (disabled by default)
 # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
 #HTTPCACHE_ENABLED=True
+DUPEFILTER_DEBUG = True
 #HTTPCACHE_EXPIRATION_SECS=0
 #HTTPCACHE_DIR='httpcache'
 #HTTPCACHE_IGNORE_HTTP_CODES=[]

diff --git a/crik/crik/spiders/scores.py b/crik/crik/spiders/scores.py
@@ -116,7 +116,7 @@ def parse(self, response):
             next_page='http://www.espncricinfo.com/ci/engine/match/index.html'
             if next_page is not None:
                 next_page = response.urljoin(next_page)
-                yield scrapy.Request(next_page, callback=self.parse)
+                yield scrapy.Request(next_page, callback=self.parse,dont_filter=True)
             exit=True