Merge pull request #21 from redapple/py3-support

Changes for new DMOZ layout + Python3 compatibility

dirbot/pipelines.py

@@ -10,7 +10,7 @@ class FilterWordsPipeline(object):
 
     def process_item(self, item, spider):
         for word in self.words_to_filter:
-            if word in unicode(item['description']).lower():
+            if word in item['description'].lower():
                 raise DropItem("Contains forbidden word: %s" % word)
         else:
             return item

dirbot/spiders/dmoz.py

@@ -20,15 +20,17 @@ def parse(self, response):
         @url http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/
         @scrapes name
         """
-        sel = Selector(response)
-        sites = sel.xpath('//ul[@class="directory-url"]/li')
+        sites = response.css('#site-list-content > div.site-item > div.title-and-desc')
         items = []
 
         for site in sites:
             item = Website()
-            item['name'] = site.xpath('a/text()').extract()
-            item['url'] = site.xpath('a/@href').extract()
-            item['description'] = site.xpath('text()').re('-\s[^\n]*\\r')
+            item['name'] = site.css(
+                'a > div.site-title::text').extract_first().strip()
+            item['url'] = site.xpath(
+                'a/@href').extract_first().strip()
+            item['description'] = site.css(
+                'div.site-descr::text').extract_first().strip()
             items.append(item)
 
         return items