Fixing queen b's scraping to ignore non-keto category pages

mtlynch · May 7, 2018 · 162329c · 162329c
1 parent 824270b
commit 162329c
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/ketohub/spiders.py b/ketohub/spiders.py
@@ -258,7 +258,7 @@ class QueenBs(spiders.CrawlSpider):
         spiders.Rule(
             linkextractors.LinkExtractor(
                 allow=r'http://queenbsincredibleedibles.com/.*/$',
-                deny=r'ive-fallen-in-love-with-keto'),
+                deny=r'(category\/)|(ive-fallen-in-love-with-keto)'),
             callback=callback_handler.process_callback,
             follow=False)
     ]