Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Altered the behavior of URLField to avoid a potential DOS vector, and…

… to avoid potential leakage of local filesystem data. A security announcement will be made shortly.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@16760 bcc190cf-cafb-0310-a4f2-bffc1f526a37
  • Loading branch information...
commit 5f287f75f2277ba821dcf5c444ab12d8eff6cce3 1 parent 33076af
Russell Keith-Magee authored September 10, 2011
57  django/core/validators.py
... ...
@@ -1,3 +1,4 @@
  1
+import platform
1 2
 import re
2 3
 import urllib2
3 4
 import urlparse
@@ -41,10 +42,6 @@ def __call__(self, value):
41 42
         if not self.regex.search(smart_unicode(value)):
42 43
             raise ValidationError(self.message, code=self.code)
43 44
 
44  
-class HeadRequest(urllib2.Request):
45  
-    def get_method(self):
46  
-        return "HEAD"
47  
-
48 45
 class URLValidator(RegexValidator):
49 46
     regex = re.compile(
50 47
         r'^(?:http|ftp)s?://' # http:// or https://
@@ -54,7 +51,8 @@ class URLValidator(RegexValidator):
54 51
         r'(?::\d+)?' # optional port
55 52
         r'(?:/?|[/?]\S+)$', re.IGNORECASE)
56 53
 
57  
-    def __init__(self, verify_exists=False, validator_user_agent=URL_VALIDATOR_USER_AGENT):
  54
+    def __init__(self, verify_exists=False, 
  55
+                 validator_user_agent=URL_VALIDATOR_USER_AGENT):
58 56
         super(URLValidator, self).__init__()
59 57
         self.verify_exists = verify_exists
60 58
         self.user_agent = validator_user_agent
@@ -79,6 +77,13 @@ def __call__(self, value):
79 77
             url = value
80 78
 
81 79
         if self.verify_exists:
  80
+            import warnings
  81
+            warnings.warn(
  82
+                "The URLField verify_exists argument has intractable security "
  83
+                "and performance issues. Accordingly, it has been deprecated.",
  84
+                DeprecationWarning
  85
+                )
  86
+
82 87
             headers = {
83 88
                 "Accept": "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5",
84 89
                 "Accept-Language": "en-us,en;q=0.5",
@@ -90,21 +95,37 @@ def __call__(self, value):
90 95
             broken_error = ValidationError(
91 96
                 _(u'This URL appears to be a broken link.'), code='invalid_link')
92 97
             try:
93  
-                req = HeadRequest(url, None, headers)
94  
-                u = urllib2.urlopen(req)
  98
+                req = urllib2.Request(url, None, headers)
  99
+                req.get_method = lambda: 'HEAD'
  100
+                #Create an opener that does not support local file access
  101
+                opener = urllib2.OpenerDirector()
  102
+                
  103
+                #Don't follow redirects, but don't treat them as errors either
  104
+                error_nop = lambda *args, **kwargs: True
  105
+                http_error_processor = urllib2.HTTPErrorProcessor()
  106
+                http_error_processor.http_error_301 = error_nop
  107
+                http_error_processor.http_error_302 = error_nop
  108
+                http_error_processor.http_error_307 = error_nop
  109
+
  110
+                handlers = [urllib2.UnknownHandler(),
  111
+                            urllib2.HTTPHandler(),
  112
+                            urllib2.HTTPDefaultErrorHandler(),
  113
+                            urllib2.FTPHandler(),
  114
+                            http_error_processor]
  115
+                try:
  116
+                    import ssl
  117
+                    handlers.append(urllib2.HTTPSHandler())
  118
+                except:
  119
+                    #Python isn't compiled with SSL support
  120
+                    pass
  121
+                map(opener.add_handler, handlers)
  122
+                opener.http_error_301 = lambda: True
  123
+                if platform.python_version_tuple() >= (2, 6):
  124
+                    opener.open(req, timeout=10)
  125
+                else:
  126
+                    opener.open(req)
95 127
             except ValueError:
96 128
                 raise ValidationError(_(u'Enter a valid URL.'), code='invalid')
97  
-            except urllib2.HTTPError, e:
98  
-                if e.code in (405, 501):
99  
-                    # Try a GET request (HEAD refused)
100  
-                    # See also: http://www.w3.org/Protocols/rfc2616/rfc2616.html
101  
-                    try:
102  
-                        req = urllib2.Request(url, None, headers)
103  
-                        u = urllib2.urlopen(req)
104  
-                    except:
105  
-                        raise broken_error
106  
-                else:
107  
-                    raise broken_error
108 129
             except: # urllib2.URLError, httplib.InvalidURL, etc.
109 130
                 raise broken_error
110 131
 
2  django/db/models/fields/__init__.py
@@ -1178,7 +1178,7 @@ def formfield(self, **kwargs):
1178 1178
 class URLField(CharField):
1179 1179
     description = _("URL")
1180 1180
 
1181  
-    def __init__(self, verbose_name=None, name=None, verify_exists=True, **kwargs):
  1181
+    def __init__(self, verbose_name=None, name=None, verify_exists=False, **kwargs):
1182 1182
         kwargs['max_length'] = kwargs.get('max_length', 200)
1183 1183
         CharField.__init__(self, verbose_name, name, **kwargs)
1184 1184
         self.validators.append(validators.URLValidator(verify_exists=verify_exists))
5  docs/internals/deprecation.txt
@@ -115,6 +115,11 @@ their deprecation, as per the :ref:`deprecation policy
115 115
       beyond that of a simple ``TextField`` since the removal of oldforms.
116 116
       All uses of ``XMLField`` can be replaced with ``TextField``.
117 117
 
  118
+    * ``django.db.models.fields.URLField.verify_exists`` has been
  119
+      deprecated due to intractable security and performance
  120
+      issues. Validation behavior has been removed in 1.4, and the
  121
+      argument will be removed in 1.5.
  122
+
118 123
 1.5
119 124
 ---
120 125
 
5  docs/ref/forms/fields.txt
@@ -799,6 +799,11 @@ Takes the following optional arguments:
799 799
     If ``True``, the validator will attempt to load the given URL, raising
800 800
     ``ValidationError`` if the page gives a 404. Defaults to ``False``.
801 801
 
  802
+.. deprecated:: 1.3.1
  803
+
  804
+   ``verify_exists`` was deprecated for security reasons and will be
  805
+   removed in 1.4. This deprecation also removes ``validator_user_agent``.
  806
+
802 807
 .. attribute:: URLField.validator_user_agent
803 808
 
804 809
     String used as the user-agent used when checking for a URL's existence.
13  docs/ref/models/fields.txt
@@ -872,14 +872,21 @@ shortcuts.
872 872
 ``URLField``
873 873
 ------------
874 874
 
875  
-.. class:: URLField([verify_exists=True, max_length=200, **options])
  875
+.. class:: URLField([verify_exists=False, max_length=200, **options])
876 876
 
877 877
 A :class:`CharField` for a URL. Has one extra optional argument:
878 878
 
  879
+.. deprecated:: 1.3.1 
  880
+
  881
+   ``verify_exists`` is deprecated for security reasons as of 1.3.1
  882
+   and will be removed in 1.4. Prior to 1.3.1, the default value was
  883
+   ``True``.
  884
+
879 885
 .. attribute:: URLField.verify_exists
880 886
 
881  
-    If ``True`` (the default), the URL given will be checked for existence
882  
-    (i.e., the URL actually loads and doesn't give a 404 response).
  887
+    If ``True``, the URL given will be checked for existence (i.e.,
  888
+    the URL actually loads and doesn't give a 404 response) using a
  889
+    ``HEAD`` request. Redirects are allowed, but will not be followed.
883 890
 
884 891
     Note that when you're using the single-threaded development server,
885 892
     validating a URL being served by the same server will hang. This should not
6  docs/ref/settings.txt
@@ -2012,8 +2012,10 @@ URL_VALIDATOR_USER_AGENT
2012 2012
 
2013 2013
 Default: ``Django/<version> (http://www.djangoproject.com/)``
2014 2014
 
2015  
-The string to use as the ``User-Agent`` header when checking to see if URLs
2016  
-exist (see the ``verify_exists`` option on :class:`~django.db.models.URLField`).
  2015
+The string to use as the ``User-Agent`` header when checking to see if
  2016
+URLs exist (see the ``verify_exists`` option on
  2017
+:class:`~django.db.models.URLField`). This setting was deprecated in
  2018
+1.3.1 along with ``verify_exists`` and will be removed in 1.4.
2017 2019
 
2018 2020
 .. setting:: USE_ETAGS
2019 2021
 
8  docs/releases/1.4.txt
@@ -519,6 +519,14 @@ This was an alias to ``django.template.loader`` since 2005, it has been removed
519 519
 without emitting a warning due to the length of the deprecation. If your code
520 520
 still referenced this please use ``django.template.loader`` instead.
521 521
 
  522
+``django.db.models.fields.URLField.verify_exists``
  523
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  524
+
  525
+This functionality has been removed due to intractable performance and
  526
+security issues. Any existing usage of ``verify_exists`` should be
  527
+removed.
  528
+
  529
+
522 530
 .. _deprecated-features-1.4:
523 531
 
524 532
 Features deprecated in 1.4
4  tests/modeltests/validation/__init__.py
... ...
@@ -1,8 +1,8 @@
1  
-from django.utils import unittest
  1
+from django.test import TestCase
2 2
 
3 3
 from django.core.exceptions import ValidationError
4 4
 
5  
-class ValidationTestCase(unittest.TestCase):
  5
+class ValidationTestCase(TestCase):
6 6
     def assertFailsValidation(self, clean, failed_fields):
7 7
         self.assertRaises(ValidationError, clean)
8 8
         try:
1  tests/modeltests/validation/models.py
@@ -14,6 +14,7 @@ class ModelToValidate(models.Model):
14 14
     parent = models.ForeignKey('self', blank=True, null=True, limit_choices_to={'number': 10})
15 15
     email = models.EmailField(blank=True)
16 16
     url = models.URLField(blank=True)
  17
+    url_verify = models.URLField(blank=True, verify_exists=True)
17 18
     f_with_custom_validator = models.IntegerField(blank=True, null=True, validators=[validate_answer_to_universe])
18 19
 
19 20
     def clean(self):
33  tests/modeltests/validation/tests.py
... ...
@@ -1,3 +1,5 @@
  1
+import warnings
  2
+
1 3
 from django import forms
2 4
 from django.test import TestCase
3 5
 from django.core.exceptions import NON_FIELD_ERRORS
@@ -14,6 +16,14 @@
14 16
 
15 17
 class BaseModelValidationTests(ValidationTestCase):
16 18
 
  19
+    def setUp(self):
  20
+        self.save_warnings_state()
  21
+        warnings.filterwarnings('ignore', category=DeprecationWarning,
  22
+                                module='django.core.validators')
  23
+
  24
+    def tearDown(self):
  25
+        self.restore_warnings_state()
  26
+
17 27
     def test_missing_required_field_raises_error(self):
18 28
         mtv = ModelToValidate(f_with_custom_validator=42)
19 29
         self.assertFailsValidation(mtv.full_clean, ['name', 'number'])
@@ -54,25 +64,22 @@ def test_wrong_url_value_raises_error(self):
54 64
         mtv = ModelToValidate(number=10, name='Some Name', url='not a url')
55 65
         self.assertFieldFailsValidationWithMessage(mtv.full_clean, 'url', [u'Enter a valid value.'])
56 66
 
  67
+    #The tests below which use url_verify are deprecated
57 68
     def test_correct_url_but_nonexisting_gives_404(self):
58  
-        mtv = ModelToValidate(number=10, name='Some Name', url='http://google.com/we-love-microsoft.html')
59  
-        self.assertFieldFailsValidationWithMessage(mtv.full_clean, 'url', [u'This URL appears to be a broken link.'])
  69
+        mtv = ModelToValidate(number=10, name='Some Name', url_verify='http://qa-dev.w3.org/link-testsuite/http.php?code=404')
  70
+        self.assertFieldFailsValidationWithMessage(mtv.full_clean, 'url_verify', [u'This URL appears to be a broken link.'])
60 71
 
61 72
     def test_correct_url_value_passes(self):
62  
-        mtv = ModelToValidate(number=10, name='Some Name', url='http://www.example.com/')
  73
+        mtv = ModelToValidate(number=10, name='Some Name', url_verify='http://www.google.com/')
63 74
         self.assertEqual(None, mtv.full_clean()) # This will fail if there's no Internet connection
64 75
 
65  
-    def test_correct_https_url_but_nonexisting(self):
66  
-        mtv = ModelToValidate(number=10, name='Some Name', url='https://www.example.com/')
67  
-        self.assertFieldFailsValidationWithMessage(mtv.full_clean, 'url', [u'This URL appears to be a broken link.'])
68  
-
69  
-    def test_correct_ftp_url_but_nonexisting(self):
70  
-        mtv = ModelToValidate(number=10, name='Some Name', url='ftp://ftp.google.com/we-love-microsoft.html')
71  
-        self.assertFieldFailsValidationWithMessage(mtv.full_clean, 'url', [u'This URL appears to be a broken link.'])
  76
+    def test_correct_url_with_redirect(self):
  77
+        mtv = ModelToValidate(number=10, name='Some Name', url_verify='http://qa-dev.w3.org/link-testsuite/http.php?code=301') #example.com is a redirect to iana.org now
  78
+        self.assertEqual(None, mtv.full_clean()) # This will fail if there's no Internet connection
72 79
 
73  
-    def test_correct_ftps_url_but_nonexisting(self):
74  
-        mtv = ModelToValidate(number=10, name='Some Name', url='ftps://ftp.google.com/we-love-microsoft.html')
75  
-        self.assertFieldFailsValidationWithMessage(mtv.full_clean, 'url', [u'This URL appears to be a broken link.'])
  80
+    def test_correct_https_url_but_nonexisting(self):
  81
+        mtv = ModelToValidate(number=10, name='Some Name', url_verify='https://www.example.com/')
  82
+        self.assertFieldFailsValidationWithMessage(mtv.full_clean, 'url_verify', [u'This URL appears to be a broken link.'])
76 83
 
77 84
     def test_text_greater_that_charfields_max_length_raises_erros(self):
78 85
         mtv = ModelToValidate(number=10, name='Some Name'*100)
16  tests/regressiontests/forms/tests/fields.py
@@ -28,6 +28,7 @@
28 28
 import re
29 29
 import os
30 30
 import urllib2
  31
+import warnings
31 32
 from decimal import Decimal
32 33
 from functools import wraps
33 34
 
@@ -71,6 +72,14 @@ def urlopen(req):
71 72
 
72 73
 class FieldsTests(SimpleTestCase):
73 74
 
  75
+    def setUp(self):
  76
+        self.save_warnings_state()
  77
+        warnings.filterwarnings('ignore', category=DeprecationWarning,
  78
+                                module='django.core.validators')
  79
+
  80
+    def tearDown(self):
  81
+        self.restore_warnings_state()
  82
+
74 83
     def test_field_sets_widget_is_required(self):
75 84
         self.assertTrue(Field(required=True).widget.is_required)
76 85
         self.assertFalse(Field(required=False).widget.is_required)
@@ -622,7 +631,7 @@ def test_urlfield_3(self):
622 631
             f.clean('http://www.broken.djangoproject.com') # bad domain
623 632
         except ValidationError, e:
624 633
             self.assertEqual("[u'This URL appears to be a broken link.']", str(e))
625  
-        self.assertRaises(ValidationError, f.clean, 'http://google.com/we-love-microsoft.html') # good domain, bad page
  634
+        self.assertRaises(ValidationError, f.clean, 'http://qa-dev.w3.org/link-testsuite/http.php?code=400') # good domain, bad page
626 635
         try:
627 636
             f.clean('http://google.com/we-love-microsoft.html') # good domain, bad page
628 637
         except ValidationError, e:
@@ -681,11 +690,10 @@ def test_urlfield_9(self):
681 690
         except ValidationError, e:
682 691
             self.assertEqual("[u'This URL appears to be a broken link.']", str(e))
683 692
 
684  
-    @verify_exists_urls(('http://xn--tr-xka.djangoproject.com/',))
685 693
     def test_urlfield_10(self):
686  
-        # UTF-8 char in path
  694
+        # UTF-8 in the domain. 
687 695
         f = URLField(verify_exists=True)
688  
-        url = u'http://t\xfcr.djangoproject.com/'
  696
+        url = u'http://\u03b5\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac.idn.icann.org/\u0391\u03c1\u03c7\u03b9\u03ba\u03ae_\u03c3\u03b5\u03bb\u03af\u03b4\u03b1'
689 697
         self.assertEqual(url, f.clean(url))
690 698
 
691 699
     def test_urlfield_not_string(self):

0 notes on commit 5f287f7

Please sign in to comment.
Something went wrong with that request. Please try again.