Fixing lowcarbyum scraper (#65)

mtlynch · Feb 14, 2019 · 576b74f · 576b74f
1 parent 1c51fdc
commit 576b74f
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/ketohub/spiders.py b/ketohub/spiders.py
@@ -283,7 +283,9 @@ class LowCarbYum(spiders.CrawlSpider):
         # e.g. https://lowcarbyum.com/category/desserts/
         spiders.Rule(
             linkextractors.LinkExtractor(
-                allow=r'https://lowcarbyum.com/category/\w+(-\w+)*/$')),
+                allow=r'https://lowcarbyum.com/category/',
+                deny=r'https://lowcarbyum.com/category/((reviews)|(articles))')
+        ),
         # Extract links for recipes.
         spiders.Rule(
             linkextractors.LinkExtractor(