[MRG+1] #1487 add_scheme_if_missing for scrapy shell command

scrapy/commands/shell.py

@@ -9,6 +9,7 @@
 from scrapy.commands import ScrapyCommand
 from scrapy.shell import Shell
 from scrapy.http import Request
+from scrapy.utils.url import add_http_if_no_scheme
 from scrapy.utils.spider import spidercls_for_request, DefaultSpider
 
 
@@ -41,6 +42,8 @@ def update_vars(self, vars):
 
     def run(self, args, opts):
         url = args[0] if args else None
+        if url:
+            url = add_http_if_no_scheme(url)
         spider_loader = self.crawler_process.spider_loader
 
         spidercls = DefaultSpider

scrapy/utils/url.py

@@ -110,3 +110,14 @@ def escape_ajax(url):
     if not frag.startswith('!'):
         return url
     return add_or_replace_parameter(defrag, '_escaped_fragment_', frag[1:])
+
+
+def add_http_if_no_scheme(url):
+    """Add http as the default scheme if it is missing from the url."""
+    if url.startswith('//'):
+        url = 'http:' + url
+        return url
+    parser = parse_url(url)
+    if not parser.scheme or not parser.netloc:
+        url = 'http://' + url
+    return url

tests/test_utils_url.py

@@ -4,7 +4,7 @@
 import six
 from scrapy.spiders import Spider
 from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
-                              canonicalize_url)
+                              canonicalize_url, add_http_if_no_scheme)
 
 __doctests__ = ['scrapy.utils.url']
 
@@ -187,5 +187,112 @@ def test_quoted_slash_and_question_sign(self):
                          "http://foo.com/AC%2FDC/")
 
 
+class AddHttpIfNoScheme(unittest.TestCase):
+
+    def test_add_scheme(self):
+        self.assertEqual(add_http_if_no_scheme('www.example.com'),
+                                               'http://www.example.com')
+
+    def test_without_subdomain(self):
+        self.assertEqual(add_http_if_no_scheme('example.com'),
+                                               'http://example.com')
+
+    def test_path(self):
+        self.assertEqual(add_http_if_no_scheme('www.example.com/some/page.html'),
+                                               'http://www.example.com/some/page.html')
+
+    def test_port(self):
+        self.assertEqual(add_http_if_no_scheme('www.example.com:80'),
+                                               'http://www.example.com:80')
+
+    def test_fragment(self):
+        self.assertEqual(add_http_if_no_scheme('www.example.com/some/page#frag'),
+                                               'http://www.example.com/some/page#frag')
+
+    def test_query(self):
+        self.assertEqual(add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
+                                               'http://www.example.com/do?a=1&b=2&c=3')
+
+    def test_username_password(self):
+        self.assertEqual(add_http_if_no_scheme('username:password@www.example.com'),
+                                               'http://username:password@www.example.com')
+
+    def test_complete_url(self):
+        self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
+                                               'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+
+    def test_preserve_http(self):
+        self.assertEqual(add_http_if_no_scheme('http://www.example.com'),
+                                               'http://www.example.com')
+
+    def test_preserve_http_without_subdomain(self):
+        self.assertEqual(add_http_if_no_scheme('http://example.com'),
+                                               'http://example.com')
+
+    def test_preserve_http_path(self):
+        self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page.html'),
+                                               'http://www.example.com/some/page.html')
+
+    def test_preserve_http_port(self):
+        self.assertEqual(add_http_if_no_scheme('http://www.example.com:80'),
+                                               'http://www.example.com:80')
+
+    def test_preserve_http_fragment(self):
+        self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page#frag'),
+                                               'http://www.example.com/some/page#frag')
+
+    def test_preserve_http_query(self):
+        self.assertEqual(add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
+                                               'http://www.example.com/do?a=1&b=2&c=3')
+
+    def test_preserve_http_username_password(self):
+        self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com'),
+                                               'http://username:password@www.example.com')
+
+    def test_preserve_http_complete_url(self):
+        self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
+                                               'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+
+    def test_protocol_relative(self):
+        self.assertEqual(add_http_if_no_scheme('//www.example.com'),
+                                               'http://www.example.com')
+
+    def test_protocol_relative_without_subdomain(self):
+        self.assertEqual(add_http_if_no_scheme('//example.com'),
+                                               'http://example.com')
+
+    def test_protocol_relative_path(self):
+        self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page.html'),
+                                               'http://www.example.com/some/page.html')
+
+    def test_protocol_relative_port(self):
+        self.assertEqual(add_http_if_no_scheme('//www.example.com:80'),
+                                               'http://www.example.com:80')
+
+    def test_protocol_relative_fragment(self):
+        self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page#frag'),
+                                               'http://www.example.com/some/page#frag')
+
+    def test_protocol_relative_query(self):
+        self.assertEqual(add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),
+                                               'http://www.example.com/do?a=1&b=2&c=3')
+
+    def test_protocol_relative_username_password(self):
+        self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com'),
+                                               'http://username:password@www.example.com')
+
+    def test_protocol_relative_complete_url(self):
+        self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
+                                               'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
+
+    def test_preserve_https(self):
+        self.assertEqual(add_http_if_no_scheme('https://www.example.com'),
+                                               'https://www.example.com')
+
+    def test_preserve_ftp(self):
+        self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'),
+                                               'ftp://www.example.com')
+
+
 if __name__ == "__main__":
     unittest.main()