Added more rules for ebay.com, keep working host specific rules.

goose/cleaners.py

@@ -25,9 +25,10 @@
 from urlparse import urlsplit
 from goose.text import innerTrim
 from configuration import Configuration
+from host_utils import HostUtils
 
 KNOWN_HOST_REMOVE_SELECTORS = {
-    'www.ebay.com': ['#desc_div', '[class *= "drpdwn"]']
+    'www.ebay.com': '#desc_div, [class *= "drpdwn"], .dropdownmenu, #PaginationAndExpansionsContainer, #ConstraintCaptionContainer, .noImage div, .yesImage div, .yesImage img[src *= "://ir"], .yesVideo, [class ^= addCaption], .removeModalLayer',
 }
 
 class OutputFormatterCleaner(clean.Cleaner):
@@ -221,10 +222,9 @@ def clean_bad_tags(self, doc):
         return doc
 
     def remove_host_specific_nodes(self, doc):
-        remove_selectors = KNOWN_HOST_REMOVE_SELECTORS[self.article.domain]
-        for selector in remove_selectors:
-          nodes = self.parser.css_select(doc, selector)
-          for node in nodes:
+        remove_selectors = HostUtils.host_selectors(KNOWN_HOST_REMOVE_SELECTORS, self.article.domain)
+        nodes = self.parser.css_select(doc, remove_selectors)
+        for node in nodes:
             self.parser.remove(node)
 
         return doc

goose/extractors.py

@@ -26,6 +26,7 @@
 from goose.utils import StringSplitter
 from goose.utils import StringReplacement
 from goose.utils import ReplaceSequence
+from host_utils import HostUtils
 
 MOTLEY_REPLACEMENT = StringReplacement("�", "")
 ESCAPED_FRAGMENT_REPLACEMENT = StringReplacement(u"#!", u"?_escaped_fragment_=")
@@ -59,7 +60,7 @@
 ]
 
 KNOWN_HOST_CONTENT_TAGS = {
-    'www.ebay.com': ['#vi-desc-maincntr']
+    'www.ebay.com': '.vi-price, noscript [itemprop="image"], #vi-desc-maincntr, #Results, [itemprop="articleBody"]',
 }
 
 
@@ -344,10 +345,8 @@ def calculate_best_node(self):
 
     def get_top_host_node_from_known_tags(self):
         if self.article.domain in KNOWN_HOST_CONTENT_TAGS:
-            selectors = KNOWN_HOST_CONTENT_TAGS[self.article.domain]
-            content_tags = []
-            for selector in selectors:
-                content_tags += self.parser.css_select(self.article.doc, selector)
+            selectors = HostUtils.host_selectors(KNOWN_HOST_CONTENT_TAGS, self.article.domain)
+            content_tags = self.parser.css_select(self.article.doc, selectors)
 
             return self.parser.combine_nodes(content_tags)
 
@@ -605,6 +604,12 @@ def post_cleanup(self):
                                 old_attribute_name = 'src',
                                 new_attribute_name = 'data-src')
 
+        # fixing ebay images
+        self.replace_attributes(node,
+                                tag_name = 'img',
+                                old_attribute_name = 'src',
+                                new_attribute_name = 'imgurl')
+
         self.build_tag_paths(node, 'img', 'src')
         self.build_tag_paths(node, 'a', 'href')
         allowed_tags = ['p', 'img', 'ul', 'ol', 'h2', 'h3', 'h4', 'h5', 'h6',

goose/host_utils.py

@@ -0,0 +1,8 @@
+class HostUtils(object):
+
+    @classmethod
+    def host_selectors(self, all_selectors, host):
+        selectors = all_selectors[host]
+        if type(selectors) is dict:
+            selectors = all_selectors[selectors['reference']]
+        return selectors

goose/parsers.py

@@ -247,9 +247,9 @@ def decode_html(self, html_string):
     def combine_nodes(self, nodes):
         if len(nodes):
             if len(nodes) > 1:
-                root = self.parser.createElement('div')
+                root = self.createElement('div')
                 for node in nodes:
-                    self.parser.appendChild(root, node)
+                    self.appendChild(root, node)
                 return root
             else:
                 return nodes[0]