fixed spiders to extract the title field

mjhea0 · Apr 20, 2016 · a9b544e · a9b544e
1 parent 97b8a4a
commit a9b544e
Show file tree

Hide file tree

Showing 3 changed files with 1,895 additions and 6 deletions.
diff --git a/basicspider/craigslist_sample/spiders/test.py b/basicspider/craigslist_sample/spiders/test.py
@@ -12,9 +12,9 @@ def parse(self, response):
         hxs = HtmlXPathSelector(response)
         titles = hxs.xpath("//span[@class='pl']")
         items = []
-        for titles in titles:
+        for title in titles:
             item = CraigslistSampleItem()
-            item["title"] = titles.select("a/text()").extract()
-            item["link"] = titles.select("a/@href").extract()
+            item["title"] = title.select('a/span[@id="titletextonly"]/text()').extract()
+            item["link"] = title.select("a/@href").extract()
             items.append(item)
         return items
diff --git a/crawlspider/craigslist_sample/spiders/test2.py b/crawlspider/craigslist_sample/spiders/test2.py
@@ -17,9 +17,9 @@ def parse_items(self, response):
         hxs = HtmlXPathSelector(response)
         titles = hxs.xpath('//span[@class="pl"]')
         items = []
-        for titles in titles:
+        for title in titles:
             item = CraigslistSampleItem()
-            item["title"] = titles.xpath("a/text()").extract()
-            item["link"] = titles.xpath("a/@href").extract()
+            item["title"] = title.select('a/span[@id="titletextonly"]/text()').extract()
+            item["link"] = title.xpath("a/@href").extract()
             items.append(item)
         return(items)