Add default schema to discovered schemaless links

mediacloud · May 7, 2019 · 989ae5a · 989ae5a
1 parent fda3e87
commit 989ae5a
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/feed_seeker/feed_seeker.py b/feed_seeker/feed_seeker.py
@@ -316,6 +316,9 @@ def find_internal_links(self):
         possible_links = []
         for link_node in self.soup.find_all('a', href=True):
             link = link_node.get('href')
+            # Sometimes links without schemas are discovered -- this applies a default "http" schema to the discovered link
+            if link.startswith('//'):
+                link = 'http:{}'.format(link)
             parsed_link = urlparse(link)
             if not parsed_link.hostname:
                 parsed_link = parsed_link._replace(netloc=parsed_url.hostname,